def main():
opts = docopt(__doc__, version='0.1')
opts = cleanOptions(opts)
# Use utils.Struct to convert the dict into an object for compatibility with old optparse code.
options = Struct(**opts)
runKerasTensorflowClassifierMultiprocess(options)
def main():
opts = docopt(__doc__, version='0.1')
opts = cleanOptions(opts)
# Use utils.Struct to convert the dict into an object for compatibility with old optparse code.
options = Struct(**opts)
getATLASTrainingSetCutouts(options)
def main():
opts = docopt(__doc__, version='0.1')
opts = cleanOptions(opts)
# Use utils.Struct to convert the dict into an object for compatibility with old optparse code.
options = Struct(**opts)
print(options.csvfile)
plotResults(options.csvfile, options.outputFile, options=options)
def main(argv=None):
opts = docopt(__doc__, version='0.1')
opts = cleanOptions(opts)
# Use utils.Struct to convert the dict into an object for compatibility with old optparse code.
options = Struct(**opts)
alert_schema = combine_schemas(options.schema)
if options.schemalessMessage:
m = read_avro_data_from_file(options.messageFile, alert_schema)
printMessage(m)
else:
data = read_avro_data_bulk(options.messageFile)
if data:
for m in data:
printMessage(m)
return 0
def main():
opts = docopt(__doc__, version='0.1')
opts = cleanOptions(opts)
options = Struct(**opts)
doPlots(options)
def main():
opts = docopt(__doc__, version='0.1')
opts = cleanOptions(opts)
options = Struct(**opts)
getPS1TrainingSetCutouts(options)
def main(argv=None):
opts = docopt(__doc__, version='0.1')
opts = cleanOptions(opts)
# Use utils.Struct to convert the dict into an object for compatibility with old optparse code.
options = Struct(**opts)
json_path = options.data
schema_files = options.schema
cutoutsci_path = options.cutoutSci
cutouttemp_path = options.cutoutTemp
cutoutdiff_path = options.cutoutDiff
mjdThreshold = float(options.mjdThreshold)
alert_schema = combine_schemas(schema_files)
import yaml
with open(options.configfile) as yaml_file:
config = yaml.load(yaml_file)
username = config['databases']['local']['username']
password = config['databases']['local']['password']
database = config['databases']['local']['database']
hostname = config['databases']['local']['hostname']
# Now that we have all the data, we need to construct a properly formed alert - FOR EACH ROW.
# NOTE - To get this to work, feed it junk test.json data.
# The alerts can be written to a file - but don't forget, we are using the schemaless writer
# which means that the schemaless reader MUST be used to read the data!
alert = None
# If we just have some json data, process it. Otherwise read from the database.
if json_path:
with open(json_path) as file_text:
json_data = json.load(file_text)
avro_bytes = write_avro_data(json_data, alert_schema)
# Load science stamp if included
if cutoutsci_path is not None:
cutoutTemplate = load_stamp(cutoutsci_path)
json_data['cutoutScience'] = cutoutTemplate
# Load template stamp if included
if cutouttemp_path is not None:
cutoutTemplate = load_stamp(cutouttemp_path)
json_data['cutoutTemplate'] = cutoutTemplate
# Load difference stamp if included
if cutoutdiff_path is not None:
cutoutDifference = load_stamp(cutoutdiff_path)
json_data['cutoutDifference'] = cutoutDifference
if options.writeFile:
with open('/tmp/alert.avro', 'wb') as f:
# NOTE - This code writes a schemaless message. To read it we need to pass the schema
# to the reader. How we pass this message to Kafka is the next problem to be
# resolved.
avro_bytes.seek(0)
data = avro_bytes.read()
f.write(data)
#m = read_avro_data_from_file('alert.avro', alert_schema)
m = read_avro_data(avro_bytes, alert_schema)
if options.readMessage:
if m:
# Print message text to screen
message_text = {
k: m[k]
for k in m if k not in
['cutoutScience', 'cutoutDifference', 'cutoutTemplate']
}
print(message_text)
# Collect stamps as files written to local directory 'output' and check hashes match expected
if m.get('cutoutScience') is not None:
stamp_temp_out = write_stamp_file(
message.get('cutoutScience'), 'output')
print('Science stamp ok:',
check_md5(args.cutoutSci, stamp_temp_out))
if m.get('cutoutTemplate') is not None:
stamp_temp_out = write_stamp_file(
message.get('cutoutTemplate'), 'output')
print('Template stamp ok:',
check_md5(args.cutoutTemp, stamp_temp_out))
if m.get('cutoutDifference') is not None:
stamp_diff_out = write_stamp_file(
message.get('cutoutDifference'), 'output')
print("size in bytes of json text: %d" %
sys.getsizeof(message_text))
raw_bytes = avro_bytes.getvalue()
print("size in bytes of avro message: %d" %
sys.getsizeof(raw_bytes))
print("size in bytes of json text: %d" %
sys.getsizeof(message_text))
raw_bytes = avro_bytes.getvalue()
print("size in bytes of avro message: %d" %
sys.getsizeof(raw_bytes))
return 0
conn = dbConnect(hostname, username, password, database)
if not conn:
print("Cannot connect to the database")
return 1
# Connect to the database and read out the ATLAS detections.
records = getATLASIngestedDetections(conn, mjdThreshold)
conn.close()
alerts = []
if options.writeFile and options.bulkMessage:
for row in records:
alert = {
'alertId': row['db_det_id'],
'atlas_object_id': row['atlas_object_id'],
'candidate': row
}
alerts.append(alert)
write_avro_data_to_file_with_schema('/tmp/alerts_bulk.avro',
alert_schema, alerts)
return
for row in records:
alert = {
'alertId': row['db_det_id'],
'atlas_object_id': row['atlas_object_id'],
'candidate': row
}
avro_bytes = write_avro_data(alert, alert_schema)
if options.readMessage:
#m = read_avro_data_from_file('alert.avro', alert_schema)
m = read_avro_data(avro_bytes, alert_schema)
if m:
# Print message text to screen
message_text = {
k: m[k]
for k in m if k not in
['cutoutScience', 'cutoutDifference', 'cutoutTemplate']
}
print(message_text)
# Collect stamps as files written to local directory 'output' and check hashes match expected
if m.get('cutoutScience') is not None:
stamp_temp_out = write_stamp_file(
message.get('cutoutScience'), 'output')
print('Science stamp ok:',
check_md5(args.cutoutSci, stamp_temp_out))
if m.get('cutoutTemplate') is not None:
stamp_temp_out = write_stamp_file(
message.get('cutoutTemplate'), 'output')
print('Template stamp ok:',
check_md5(args.cutoutTemp, stamp_temp_out))
if m.get('cutoutDifference') is not None:
stamp_diff_out = write_stamp_file(
message.get('cutoutDifference'), 'output')
print("size in bytes of json text: %d" %
sys.getsizeof(message_text))
raw_bytes = avro_bytes.getvalue()
print("size in bytes of avro message: %d" %
sys.getsizeof(raw_bytes))
if options.writeFile:
if not options.bulkMessage:
f = open('/tmp/alert_%s.avro' % row['db_det_id'], 'wb')
avro_bytes.seek(0)
data = avro_bytes.read()
f.write(data)
if not options.bulkMessage:
f.close()
return 0
