def updateLasairClassification(self):
now = datetime.now()
current_time = now.strftime("%d/%m/%Y, %H:%M:%S")
print("---------------------------------------")
print("{0}: UPDATING Lasair".format(current_time))
self.setLocalDB()
self.setAlerceDB()
lastdate = self.getlastLasairObject()
lasair = LasairArchive()
#lastdate[0]=58266
#lasair.getAllData(lastdate[0], page=0)
lasair.getAllData(lastdate[0], page=0)
def getLastLasair(request):
lasair = LasairArchive()
lastItems = lasair.getLastDetections(days_ago=4)
if request.GET.__contains__('jobid'):
jobid = request.GET['jobid']
else:
status = {"error": "job id is required"}
newItems = lastItems.fillna('', axis=1)
# lastItems.replace(np.nan, '', regex=True)
dic_result = newItems.to_dict('records')
# for index,row in enumerate(dic_result):
# for key in row.keys():
# if math.isnan(row[key]):
# row[key]="nan"
return JsonResponse(dic_result, safe=False)
def getClassification(ztfid):
print("getting classification for:",ztfid)
alercearchive = AlerceArchive()
lasairarchive = LasairArchive()
lightCurve = lasairarchive.getObjectInfo(ztfid)
lasair_classification = ""
alerce_classification = ""
if "objectData" in lightCurve:
if "classification" in lightCurve["objectData"]:
lasair_classification = lightCurve["objectData"]["classification"]
else:
lightCurve_lasair = alercearchive.getLightCurve(ztfid)
stats = alercearchive.getStats(ztfid)
lasair_classification = "not in lasair"
lightCurve = {"candidates":lightCurve_lasair["result"]["detections"],"crossmatches":[],"objectId":ztfid,"objectData":{"ramean":stats["result"]["stats"]["meanra"],"decmean":stats["result"]["stats"]["meandec"]}}
alerce_classification = alercearchive.getProbabilities(ztfid)
best_late = 0
best_late_key = ""
for late_key in alerce_classification["result"]["probabilities"]["late_classifier"]:
#print(type(alerce_classification["result"]["probabilities"]["late_classifier"][late_key]), late_key)
if type(alerce_classification["result"]["probabilities"]["late_classifier"][late_key]) is float and late_key != "classifier_version":
if best_late < alerce_classification["result"]["probabilities"]["late_classifier"][late_key]:
best_late = alerce_classification["result"]["probabilities"]["late_classifier"][late_key]
best_late_key = str(late_key)[0:str(late_key).find("_")].upper()
best_early = 0
best_early_key = ""
for early_key in alerce_classification["result"]["probabilities"]["early_classifier"]:
#print(type(alerce_classification["result"]["probabilities"]["early_classifier"][early_key]), early_key)
if type(alerce_classification["result"]["probabilities"]["early_classifier"][
early_key]) is float and early_key != "classifier_version":
if best_early < alerce_classification["result"]["probabilities"]["early_classifier"][early_key]:
best_early = alerce_classification["result"]["probabilities"]["early_classifier"][early_key]
best_early_key = str(early_key)[0:str(early_key).find("_")].upper()
if "probabilities" in alerce_classification["result"]:
alerce_classification = alerce_classification["result"]["probabilities"]
return {"light_curve":lightCurve,"lasair_clas":lasair_classification,"alerce_clas":alerce_classification,"alerce_early_class":best_early_key,"alerce_late_class":best_late_key}
def scoreCandidates(collection,filter={}):
lasairarchive = LasairArchive()
db = MongodbManager()
config = Config()
dbconfig = config.getDatabase("mongodb")
db.setDatabase(dbconfig["dbname"])
db.setCollection(collection)
data = db.getData(filter=filter)
for indx, row in enumerate(data):
print("ID score",row["id"])
def searchCadidates(self, days=15):
alerce = AlerceArchive()
alerceGoodCandidates = alerce.getCandidates(days)
lasairbroker = LasairArchive()
lasairGoodCandidates = lasairbroker.getLastDetections(days)
# alerceGoodCandidates = [r["result"][target] for target in r["result"]]
alerceTable = QTable.from_pandas(alerceGoodCandidates)
lasairTable = QTable.from_pandas(lasairGoodCandidates)
print("candidates found lasair{0} alerce{1} ".format(
len(lasairGoodCandidates), len(alerceGoodCandidates)))
meanaler_val = {
"ramean":
np.nan_to_num(
np.concatenate((alerceGoodCandidates["meanra"].values,
lasairGoodCandidates["meanra"].values),
axis=0)),
"decmean":
np.nan_to_num(
np.concatenate((alerceGoodCandidates["meandec"].values,
lasairGoodCandidates["meandec"].values),
axis=0)),
"maggmax":
np.nan_to_num(
np.concatenate((alerceGoodCandidates["max_magap_g"].values,
lasairGoodCandidates["maggmax"].values),
axis=0)),
"maggmin":
np.nan_to_num(
np.concatenate((alerceGoodCandidates["min_magap_g"].values,
lasairGoodCandidates["maggmin"].values),
axis=0)),
"magrmax":
np.nan_to_num(
np.concatenate((alerceGoodCandidates["max_magap_r"].values,
lasairGoodCandidates["magrmax"].values),
axis=0)),
"magrmin":
np.nan_to_num(
np.concatenate((alerceGoodCandidates["min_magap_r"].values,
lasairGoodCandidates["magrmin"].values),
axis=0)),
"id":
np.concatenate((alerceGoodCandidates["oid"].values,
lasairGoodCandidates["oid"].values),
axis=0)
}
bigdata = pd.DataFrame(meanaler_val)
bigdata_drop = bigdata.drop_duplicates(subset="id", keep=False)
bigtable = bigdata
if bigdata_drop.size > 0:
bigtable = bigdata_drop
table_candidates = QTable(QTable.from_pandas(bigtable), masked=False)
return table_candidates, alerceGoodCandidates, lasairGoodCandidates
def checkLastDetections(**kwargs):
# try:
allrecords=0
collection=current_collection
days_ago=15
if "collection" in kwargs.keys() and kwargs["collection"]!="":
collection = kwargs["collection"]
if "days_ago" in kwargs.keys() and kwargs["days_ago"]!="":
days_ago=kwargs["days_ago"]
if "IDpipeLine" in kwargs.keys() and kwargs["IDpipeLine"]!="":
updatePipeline(kwargs["IDpipeLine"],"checkLastDetections",STATE_RUNNING)
logger.info("checkLastDetections:: getting the last ZTF detections from brokers...")
lasairarchive = LasairArchive()
#coneecto to DATABASE
db = MongodbManager()
config=Config()
dbconfig=config.getDatabase("mongodb")
db.setDatabase(dbconfig["dbname"])
db.setCollection(collection)
#Get last candidates and update previews detection and light curves
bestCandidates = BestCandidates()
table_candidates, alerceDF, lasairDF = bestCandidates.searchCadidates(days_ago)
#check if the new candidates is already in DB
#get all zft id in and array to validate if exist into ddatabase and filter by
listcandidates=table_candidates["id"]
filter={"oid":{"$in":listcandidates.data.tolist()}}
projection={"oid":1 ,"lastmjd":1 ,"last_update":1}
current_data = db.getData(filter=filter, projection=projection)
for remove_data in current_data:
oid=remove_data["oid"]
print("get info for ",oid)
table_candidates.remove_rows(table_candidates["id"] == oid)
#get desi photoz
dataarchive = SussexArchive()
desi_targetsvo, desi_targetstable = dataarchive.getDesiPhotoZfromTable(table_candidates)
alerceTable = QTable.from_pandas(alerceDF)
lasairTable = QTable.from_pandas(lasairDF)
alerceTable.rename_column("oid","id")
lasairTable.rename_column("oid", "id")
alerceTable["id"] = alerceTable["id"].astype(str)
lasairTable["id"] = lasairTable["id"].astype(str)
desi_targetstable["id"] =desi_targetstable["id"].astype(str)
desi_targetstable["desidec"].mask = False
desi_targetstable["desira"].mask = False
#calc separation desi source
ra_ref = desi_targetstable["ramean"].tolist()
dec_ref = desi_targetstable["decmean"].tolist()
cref = SkyCoord(ra_ref, dec_ref, frame='icrs', unit='deg')
ra_desi = desi_targetstable["desira"].tolist()
dec_desi = desi_targetstable["desidec"].tolist()
c1 = SkyCoord(ra_desi, dec_desi, frame='icrs', unit='deg')
desi_distance = cref.separation(c1).arcsec
desi_targetstable["separation"] = desi_distance
#merge all table in one json to save in mongo
desi_targetstable = Table(desi_targetstable, masked=False)
alerceTable = Table(alerceTable, masked=False)
lasairTable = Table(lasairTable, masked=False)
alerceTable["broker"] = "alerce"
lasairTable["broker"] = "lasair"
update_alerce_table = join(alerceTable, lasairTable, join_type='outer', keys='id')
merge_table = join(update_alerce_table, desi_targetstable, join_type='outer', keys='id')
merge_table["desiid"] = merge_table["desiid"].astype(str)
merge_table["field"] = merge_table["field"].astype(str)
lastItems= merge_table.to_pandas()
newItems = lastItems.fillna('', axis=1)
dic_result = newItems.to_dict('records')
newCandidates=0
logger.info("checkLastDetections:: Ingested {0} candidates".format(str(len(dic_result))))
allrecords=len(dic_result)
for index,row in enumerate(dic_result):
id=row["id"]
print("saving candidate",id)
row["comments"]={}
row["snh_score"] = 0.0
if row["broker_1"] != "":
#alerce
#row["pclassearly"]=row["pclassearly_1"]
if row["broker_2"]!="":
row["broker"]=row["broker_1"]+"/"+row["broker_2"]
else:
row["broker"] = row["broker_1"]
row["meanra"]=row["meanra_1"]
row["meandec"]=row["meandec_1"]
row["lastmjd"]=row["lastmjd_1"]
else:
#lasair
#row["pclassearly"] = row["pclassearly_2"]
row["broker"] = row["broker_2"]
row["meanra"] = row["meanra_2"]
row["meandec"] = row["meandec_2"]
row["lastmjd"] = row["lastmjd_2"]
try:
#remove duplicate fields
#del row["pclassearly_1"]
#del row["pclassearly_2"]
del row["broker_1"]
del row["broker_2"]
del row["meanra_1"]
del row["meandec_1"]
del row["meanra_2"]
del row["meandec_2"]
del row["lastmjd_2"]
del row["lastmjd_1"]
except KeyError as er:
print("key error",er,id)
# check if already exist this candidate, if exist update light curve and run check list to alerts
currentdata = db.getData(filter={"id": id}, projection={"nobs": 1, "last_update": 1, "id": 1})
now = datetime.now().timestamp()
rowupdated={}
if len(currentdata) > 0:
currentdata = currentdata[0]
days_from_update = ((now - float(currentdata["last_update"])) / 3600) / 24
if days_from_update < 0.6:
print("last detections is the same, not getting enough to services update classify",id)
logger.info("checkLastDetections:: {0} last detections is the same, not getting enough to services update classify".format(id))
continue
classification = getClassification(id)
#peak = lasairarchive.getPeakLightCurve(classification["light_curve"]["candidates"])
rowupdated["ra"] = row["meanra"]
rowupdated["dec"] = row["meandec"]
rowupdated["lasair_clas"]=classification["lasair_clas"]
rowupdated["alerce_clas"]=classification["alerce_clas"]
rowupdated["alerce_early_class"] = classification["alerce_early_class"]
rowupdated["alerce_late_class"] = classification["alerce_late_class"]
rowupdated["crossmatch"]={"lasair":classification["light_curve"]["crossmatches"],"check":False}
rowupdated["lightcurve"] = classification["light_curve"]["candidates"]
rowupdated["report"] = row
rowupdated["broker"] = row["broker"]
rowupdated["nobs"] = row["nobs"]
rowupdated["lastmjd"] = row["lastmjd"]
rowupdated["sigmara"] = row["sigmara"]
rowupdated["sigmadec"] = row["sigmadec"]
rowupdated["last_magpsf_g"] = row["last_magpsf_g"]
rowupdated["last_magpsf_r"] = row["last_magpsf_r"]
rowupdated["first_magpsf_g"] = row["first_magpsf_g"]
rowupdated["first_magpsf_r"] = row["first_magpsf_r"]
rowupdated["sigma_magpsf_g"] = row["sigma_magpsf_g"]
rowupdated["sigma_magpsf_r"] = row["sigma_magpsf_r"]
rowupdated["max_magpsf_g"] = row["max_magpsf_g"]
rowupdated["max_magpsf_r"] = row["max_magpsf_r"]
rowupdated["id"] = row["id"]
#check if already exist this candidate, if exist update light curve and run check list to alerts
currentdata=db.getData(filter={"id":id},projection={"nobs":1,"last_update":1,"id":1})
now = datetime.now().timestamp()
if len(currentdata)>0 :
#update current data
try:
if currentdata[0]["nobs"] < rowupdated["nobs"]:
peak = lasairarchive.getPeakLightCurve(classification["light_curve"]["candidates"])
rowupdated["lightpeak"] = peak
update_query={"last_update":now,"lightcurve":rowupdated["lightcurve"],"lightpeak":peak,"lasair_clas":rowupdated["lasair_clas"],"alerce_clas":rowupdated["alerce_clas"],"nobs":rowupdated["nobs"],"state":"updated"}
update_id = db.update(filter={"id":id}, query={"$set":update_query})
print("updated source",id,update_id.raw_result)
else:
print("last detections is the same, not getting enough to services update classify",id)
except Exception as err:
print("Error updated",id,currentdata[0]["nobs"],rowupdated["nobs"])
logger.error("checkLastDetections:: {0} Error updated..".format(str(id)))
else:
peak = lasairarchive.getPeakLightCurve(classification["light_curve"]["candidates"])
rowupdated["lightpeak"] = peak
#insert new candidate
print("save new candidate")
rowupdated["state"]="new"
rowupdated["last_update"] = now
db.saveData(rowupdated)
logger.info("checkLastDetections:: {0} Saved candidate with {1} observations".format(id,rowupdated["nobs"]))
newCandidates+=1
logger.info("checkLastDetections:: {0} candidates stored..".format(str(len(dic_result))))
logger.info("checkLastDetections:: alerce table detections {0}".format(str(len(alerceTable))))
logger.info("checkLastDetections:: lasair table detections {0}".format(str(len(lasairTable))))
logger.info("checkLastDetections:: desi detections {0}".format(str(len(desi_targetstable))))
logger.info("checkLastDetections:: new Candidates {0}".format(str(newCandidates)))
db.saveData(data={"date":now,"newcandidates":newCandidates,"allrecords":allrecords,"alerce_records":len(alerceTable),"lasair_records":len(lasairTable),"desi_matchs":len(desi_targetstable),"process":"lastdetections"},collection="tasks")
if "IDpipeLine" in kwargs.keys() and kwargs["IDpipeLine"]!="":
updatePipeline(kwargs["IDpipeLine"],"checkLastDetections",STATE_COMPLETED)
def getPeaks(**kwargs):
collection = current_collection
filter = {"lightcurve": {"$exists": True}}
projection = {}
if "collection" in kwargs.keys() and kwargs["collection"] != "":
collection = kwargs["collection"]
if "filter" in kwargs.keys() and kwargs["filter"] != "":
filter = kwargs["filter"]
if "projection" in kwargs.keys() and kwargs["projection"] != "":
projection = kwargs["projection"]
if "IDpipeLine" in kwargs.keys() and kwargs["IDpipeLine"] != "":
updatePipeline(kwargs["IDpipeLine"], "getPeaks", STATE_RUNNING)
lasairarchive = LasairArchive()
db = MongodbManager()
config = Config()
dbconfig = config.getDatabase("mongodb")
db.setDatabase(dbconfig["dbname"])
db.setCollection(collection)
data = db.getData(filter=filter)
for indx,row in enumerate(data):
print("try to get peak",row["id"])
if len(row["lightcurve"])>0:
peak = lasairarchive.getPeakLightCurve(row["lightcurve"])
query = {}
query["best_photoz_gabmag"] = 999
query["best_photoz_rabmag"] = 999
query["best_specz_gabmag"] = 999
query["best_specz_rabmag"] = 999
if "Redshift" in row or ("redshift" in row and len(row["redshift"].keys())>0):
redshift = []
if "Redshift" in row:
z=row["Redshift"]
redshifts_archives=["tns"]
else:
redshifts_archives=row["redshift"].keys()
for z_key in redshifts_archives:
if z_key == "sncosmos":
if "best"in row["redshift"]["sncosmos"] and "redshift" in row["redshift"]["sncosmos"]["best"]:
z = row["redshift"]["sncosmos"]["best"]["redshift"]
else:
continue
else:
z=row["redshift"][z_key]
if "g" in peak["stats"].keys():
if "magab" not in peak["stats"]["g"]:
peak["stats"]["g"]["magab"]={}
peak["stats"]["g"]["magab"][z_key] = Convertion.aparentToAbsoluteMagnitud(peak["stats"]["g"]["y"],z=z).tolist()
if "r" in peak["stats"].keys():
if "magab" not in peak["stats"]["r"]:
peak["stats"]["r"]["magab"]={}
peak["stats"]["r"]["magab"][z_key] = Convertion.aparentToAbsoluteMagnitud(peak["stats"]["r"]["y"],
z=z).tolist()
if "g" in peak["lightcurve"].keys():
if "magab" not in peak["lightcurve"]["g"]:
peak["lightcurve"]["g"]["magab"]={}
peak["lightcurve"]["g"]["magab"][z_key] = Convertion.aparentToAbsoluteMagnitud(
peak["lightcurve"]["g"]["mag"], z=z).tolist()
if "r" in peak["lightcurve"].keys():
if "magab" not in peak["lightcurve"]["r"]:
peak["lightcurve"]["r"]["magab"]={}
peak["lightcurve"]["r"]["magab"][z_key] = Convertion.aparentToAbsoluteMagnitud(
peak["lightcurve"]["r"]["mag"], z=z).tolist()
peaks=[]
gmag=False
rmag = False
if "g" in peak["stats"].keys():
peak_g = peak["stats"]["g"]["peakmag"]
peaks.append(peak_g)
gmag = True
if "r" in peak["stats"].keys():
peak_r = peak["stats"]["r"]["peakmag"]
peaks.append(peak_r)
rmag=True
if "best_photo_z" in row.keys() and len(row["best_photo_z"])>0:
photoz = row["best_photo_z"]["photo_z"]
best_photomagab=Convertion.aparentToAbsoluteMagnitud(peaks, z=photoz).tolist()
if gmag:
query["best_photoz_gabmag"]=best_photomagab[0]
if rmag:
idxphotorbest = 1 if gmag else 0
query["best_photoz_rabmag"]=best_photomagab[idxphotorbest]
if "best_spec_z" in row.keys() and len(row["best_spec_z"])>0:
specz = row["best_spec_z"]["spec_z"]
best_specmagab=Convertion.aparentToAbsoluteMagnitud(peaks, z=specz).tolist()
if gmag:
query["best_specz_gabmag"]=best_specmagab[0]
if rmag:
idxspecbest = 1 if gmag else 0
query["best_specz_rabmag"]=best_specmagab[idxspecbest]
query["lightpeak"]= peak
if "g" in peak["status"].keys():
query["g_state"]= peak["status"]["g"]
if "r" in peak["status"].keys():
query["r_state"]= peak["status"]["r"]
update=db.update(filter={"id":row["id"]},query={"$set":query})
print("update peak ",row["id"],update)
if "IDpipeLine" in kwargs.keys() and kwargs["IDpipeLine"] != "":
updatePipeline(kwargs["IDpipeLine"], "getPeaks", STATE_COMPLETED)