def test_crossmatching(self):
# SETUP ALL DATABASE CONNECTIONS
from sherlock import database
db = database(log=log, settings=settings)
dbConns, dbVersions = db.connect()
transientsDbConn = dbConns["transients"]
cataloguesDbConn = dbConns["catalogues"]
pmDbConn = dbConns["marshall"]
from sherlock.commonutils import get_crossmatch_catalogues_column_map
colMaps = get_crossmatch_catalogues_column_map(log=log,
dbConn=cataloguesDbConn)
from sherlock import transient_classifier
classifier = transient_classifier(log=log, settings=settings)
transientsMetadataList = classifier._get_transient_metadata_from_database_list(
)
crossmatches = classifier._crossmatch_transients_against_catalogues(
colMaps=colMaps, transientsMetadataList=transientsMetadataList)
classifications, crossmatches = classifier._rank_classifications(
colMaps=colMaps, crossmatches=crossmatches)
classifier._update_transient_database(
classifications=classifications,
transientsMetadataList=transientsMetadataList,
colMaps=colMaps,
crossmatches=crossmatches)
def test_get_transient_metadata_from_database_list(self):
from sherlock import transient_classifier
classifier = transient_classifier(log=log,
settings=settings,
updateNed=False)
transientsMetadataList = classifier._get_transient_metadata_from_database_list(
)
def test_get_transient_metadata_from_database_list(self):
from sherlock import transient_classifier
classifier = transient_classifier(log=log, settings=settings)
transientsMetadataList = classifier._get_transient_metadata_from_database_list(
)
classifier._update_ned_stream(
transientsMetadataList=transientsMetadataList)
def test_transient_classifier_function(self):
from sherlock import transient_classifier
this = transient_classifier(log=log,
settings=settings,
update=True,
updateNed=False,
oneRun=True)
this.classify()
def test_transient_update_classified_annotations_function(self):
from sherlock import transient_classifier
this = transient_classifier(log=log,
settings=settings,
update=True,
fast=True)
# this.update_peak_magnitudes()
this.update_classification_annotations_and_summaries()
def test_full_classifier(self):
from sherlock import transient_classifier
classifier = transient_classifier(log=log,
settings=settings,
verbose=2,
update=True,
updateNed=False,
updatePeakMags=True)
classifier.classify()
def test_transient_classifier_single_source_function(self):
from sherlock import transient_classifier
this = transient_classifier(log=log,
settings=settings,
ra="08:57:57.19",
dec="+43:25:44.1",
name="PS17gx",
verbose=0)
classifications, crossmatches = this.classify()
def test_transient_classifier_function_exception(self):
from sherlock import transient_classifier
try:
this = transient_classifier(log=log,
settings=settings,
fakeKey="break the code")
this.get()
assert False
except Exception as e:
assert True
print(str(e))
def classify(name, ra, dec, lite=False):
with open(conf['sherlock_settings'], "r") as f:
sherlock_settings = yaml.safe_load(f)
classifier = transient_classifier(log=app.logger,
settings=sherlock_settings,
ra=ra,
dec=dec,
name=name,
verbose=1,
updateNed=False,
lite=lite)
classifications, crossmatches = classifier.classify()
return classifications, crossmatches
def test_classification_annotations(self):
from sherlock import database
db = database(log=log, settings=settings)
dbConns, dbVersions = db.connect()
transientsDbConn = dbConns["transients"]
cataloguesDbConn = dbConns["catalogues"]
from sherlock.commonutils import get_crossmatch_catalogues_column_map
colMaps = get_crossmatch_catalogues_column_map(log=log,
dbConn=cataloguesDbConn)
from sherlock import transient_classifier
classifier = transient_classifier(log=log, settings=settings)
classifier.classification_annotations()
def classify(conf, log, alerts):
"send a batch of alerts to sherlock and add the responses to the alerts, return the number of alerts classified"
#global alerts
log.debug('called classify with config: ' + str(conf))
# read Sherlock settings file
sherlock_settings = {}
try:
with open(conf['sherlock_settings'], "r") as f:
sherlock_settings = yaml.safe_load(f)
except IOError as e:
log.error(e)
# look up objects in cache
annotations = {}
if conf['cache_db']:
names = []
for alert in alerts:
name = alert.get('objectId', alert.get('candid'))
names.append(name)
query = "SELECT * FROM cache WHERE name IN ('{}');".format("','".join(names))
url = urlparse(conf['cache_db'])
connection = pymysql.connect(
host=url.hostname,
user=url.username,
password=url.password,
db=url.path.lstrip('/'),
charset='utf8mb4',
cursorclass=pymysql.cursors.DictCursor)
try:
with connection.cursor() as cursor:
cursor.execute(query)
for result in cursor.fetchall():
try:
match = json.loads(result.get('crossmatch'))
annotations[result['name']] = {
'classification': result['class']
}
for key,value in match.items():
annotations[result['name']][key] = value
log.debug("Got crossmatch from cache:\n" + json.dumps(match, indent=2))
except ValueError:
log.info("Ignoring cache entry with malformed or missing crossmatch: {}".format(result['name']))
continue
except TypeError:
log.debug("Got TypeError reading cache. Entry probably present, but incomplete or malformed. Ignoring.")
finally:
connection.close()
if len(annotations)>0:
log.info("got {:d} annotations from cache".format(len(annotations)))
# make lists of names, ra, dec
names = []
ra = []
dec = []
for alert in alerts:
name = alert.get('objectId', alert.get('candid'))
# ignore SS alerts
ssnamenr = alert['candidate'].get('ssnamenr', "null")
if ssnamenr != "null":
log.debug("Skipping classification for solar system alert {}".format(name))
continue
if not name in annotations:
if not name in names:
names.append(name)
ra.append(alert['candidate']['ra'])
dec.append(alert['candidate']['dec'])
# set up sherlock
classifier = transient_classifier(
log=log,
settings=sherlock_settings,
ra=ra,
dec=dec,
name=names,
verbose=0,
updateNed=False,
lite=True
)
# run sherlock
cm_by_name = {}
if len(names) > 0:
log.log(logging.INFO_, "running Sherlock classifier on {:d} objects".format(len(names)))
classifications, crossmatches = classifier.classify()
log.log(logging.INFO_, "got {:d} classifications".format(len(classifications)))
log.log(logging.INFO_, "got {:d} crossmatches".format(len(crossmatches)))
# process classfications
for name in names:
if name in classifications:
annotations[name] = { 'classification': classifications[name][0] }
if len(classifications[name]) > 1:
annotations[name]['description'] = classifications[name][1]
# process crossmatches
for cm in crossmatches:
name = cm['transient_object_id']
if name in cm_by_name:
cm_by_name[name].append(cm)
else:
cm_by_name[name] = [cm]
for name in names:
if name in cm_by_name:
cm = cm_by_name[name]
if len(cm) > 0:
match = cm[0]
log.debug("got crossmatch:\n {}".format(json.dumps(match, indent=2)))
for key, value in match.items():
if key != 'rank':
annotations[name][key] = value
else:
log.log(logging.INFO_, "not running Sherlock as no remaining alerts to process")
# update cache database
if conf['cache_db'] and len(names)>0:
connection = pymysql.connect(
host=url.hostname,
user=url.username,
password=url.password,
db=url.path.lstrip('/'),
charset='utf8mb4',
cursorclass=pymysql.cursors.DictCursor)
values = []
crossmatches = []
for name in names:
classification = annotations[name]['classification']
cm = cm_by_name.get(name, [])
crossmatch = "{}".format(json.dumps(cm[0])) if len(cm) > 0 else "NULL"
values.append("\n ('{}','{}',%s)".format(name, classification))
crossmatches.append(crossmatch)
# Syntax for ON DUPLICATE KEY appears to differ between MySQL and MariaDB :(
##query = "INSERT INTO cache VALUES {} AS new ON DUPLICATE KEY UPDATE class=new.class, crossmatch=new.crossmatch".format(",".join(values))
query = "INSERT INTO cache VALUES {} ON DUPLICATE KEY UPDATE class=VALUES(class), crossmatch=VALUES(crossmatch)".format(",".join(values))
log.info("update cache: {}".format(query))
try:
with connection.cursor() as cursor:
# make deprecation warning non-fatal
with warnings.catch_warnings():
warnings.simplefilter('default')
cursor.execute(query, crossmatches)
finally:
connection.commit()
connection.close()
# add the annotations to the alerts
n = 0
for alert in alerts:
name = alert.get('objectId', alert.get('candid'))
if name in annotations:
annotations[name]['annotator'] = "https://github.com/thespacedoctor/sherlock"
annotations[name]['additional_output'] = "http://lasair.lsst.ac.uk/api/sherlock/object/" + name
# placeholders until sherlock returns these
#annotations[name]['summary'] = 'Placeholder'
if 'annotations' not in alert:
alert['annotations'] = {}
alert['annotations']['sherlock'] = []
alert['annotations']['sherlock'].append(annotations[name])
n += 1
return n