def createPipeline(tasks):
db = MongodbManager()
config = Config()
dbconfig = config.getDatabase("mongodb")
db.setDatabase(dbconfig["dbname"])
id = db.saveData(tasks,collection="pipelines")
logger.info("createPipeline:: created PIPELINE {0} ".format(str(id)))
return id
def checkLastDetections(**kwargs):
# try:
allrecords=0
collection=current_collection
days_ago=15
if "collection" in kwargs.keys() and kwargs["collection"]!="":
collection = kwargs["collection"]
if "days_ago" in kwargs.keys() and kwargs["days_ago"]!="":
days_ago=kwargs["days_ago"]
if "IDpipeLine" in kwargs.keys() and kwargs["IDpipeLine"]!="":
updatePipeline(kwargs["IDpipeLine"],"checkLastDetections",STATE_RUNNING)
logger.info("checkLastDetections:: getting the last ZTF detections from brokers...")
lasairarchive = LasairArchive()
#coneecto to DATABASE
db = MongodbManager()
config=Config()
dbconfig=config.getDatabase("mongodb")
db.setDatabase(dbconfig["dbname"])
db.setCollection(collection)
#Get last candidates and update previews detection and light curves
bestCandidates = BestCandidates()
table_candidates, alerceDF, lasairDF = bestCandidates.searchCadidates(days_ago)
#check if the new candidates is already in DB
#get all zft id in and array to validate if exist into ddatabase and filter by
listcandidates=table_candidates["id"]
filter={"oid":{"$in":listcandidates.data.tolist()}}
projection={"oid":1 ,"lastmjd":1 ,"last_update":1}
current_data = db.getData(filter=filter, projection=projection)
for remove_data in current_data:
oid=remove_data["oid"]
print("get info for ",oid)
table_candidates.remove_rows(table_candidates["id"] == oid)
#get desi photoz
dataarchive = SussexArchive()
desi_targetsvo, desi_targetstable = dataarchive.getDesiPhotoZfromTable(table_candidates)
alerceTable = QTable.from_pandas(alerceDF)
lasairTable = QTable.from_pandas(lasairDF)
alerceTable.rename_column("oid","id")
lasairTable.rename_column("oid", "id")
alerceTable["id"] = alerceTable["id"].astype(str)
lasairTable["id"] = lasairTable["id"].astype(str)
desi_targetstable["id"] =desi_targetstable["id"].astype(str)
desi_targetstable["desidec"].mask = False
desi_targetstable["desira"].mask = False
#calc separation desi source
ra_ref = desi_targetstable["ramean"].tolist()
dec_ref = desi_targetstable["decmean"].tolist()
cref = SkyCoord(ra_ref, dec_ref, frame='icrs', unit='deg')
ra_desi = desi_targetstable["desira"].tolist()
dec_desi = desi_targetstable["desidec"].tolist()
c1 = SkyCoord(ra_desi, dec_desi, frame='icrs', unit='deg')
desi_distance = cref.separation(c1).arcsec
desi_targetstable["separation"] = desi_distance
#merge all table in one json to save in mongo
desi_targetstable = Table(desi_targetstable, masked=False)
alerceTable = Table(alerceTable, masked=False)
lasairTable = Table(lasairTable, masked=False)
alerceTable["broker"] = "alerce"
lasairTable["broker"] = "lasair"
update_alerce_table = join(alerceTable, lasairTable, join_type='outer', keys='id')
merge_table = join(update_alerce_table, desi_targetstable, join_type='outer', keys='id')
merge_table["desiid"] = merge_table["desiid"].astype(str)
merge_table["field"] = merge_table["field"].astype(str)
lastItems= merge_table.to_pandas()
newItems = lastItems.fillna('', axis=1)
dic_result = newItems.to_dict('records')
newCandidates=0
logger.info("checkLastDetections:: Ingested {0} candidates".format(str(len(dic_result))))
allrecords=len(dic_result)
for index,row in enumerate(dic_result):
id=row["id"]
print("saving candidate",id)
row["comments"]={}
row["snh_score"] = 0.0
if row["broker_1"] != "":
#alerce
#row["pclassearly"]=row["pclassearly_1"]
if row["broker_2"]!="":
row["broker"]=row["broker_1"]+"/"+row["broker_2"]
else:
row["broker"] = row["broker_1"]
row["meanra"]=row["meanra_1"]
row["meandec"]=row["meandec_1"]
row["lastmjd"]=row["lastmjd_1"]
else:
#lasair
#row["pclassearly"] = row["pclassearly_2"]
row["broker"] = row["broker_2"]
row["meanra"] = row["meanra_2"]
row["meandec"] = row["meandec_2"]
row["lastmjd"] = row["lastmjd_2"]
try:
#remove duplicate fields
#del row["pclassearly_1"]
#del row["pclassearly_2"]
del row["broker_1"]
del row["broker_2"]
del row["meanra_1"]
del row["meandec_1"]
del row["meanra_2"]
del row["meandec_2"]
del row["lastmjd_2"]
del row["lastmjd_1"]
except KeyError as er:
print("key error",er,id)
# check if already exist this candidate, if exist update light curve and run check list to alerts
currentdata = db.getData(filter={"id": id}, projection={"nobs": 1, "last_update": 1, "id": 1})
now = datetime.now().timestamp()
rowupdated={}
if len(currentdata) > 0:
currentdata = currentdata[0]
days_from_update = ((now - float(currentdata["last_update"])) / 3600) / 24
if days_from_update < 0.6:
print("last detections is the same, not getting enough to services update classify",id)
logger.info("checkLastDetections:: {0} last detections is the same, not getting enough to services update classify".format(id))
continue
classification = getClassification(id)
#peak = lasairarchive.getPeakLightCurve(classification["light_curve"]["candidates"])
rowupdated["ra"] = row["meanra"]
rowupdated["dec"] = row["meandec"]
rowupdated["lasair_clas"]=classification["lasair_clas"]
rowupdated["alerce_clas"]=classification["alerce_clas"]
rowupdated["alerce_early_class"] = classification["alerce_early_class"]
rowupdated["alerce_late_class"] = classification["alerce_late_class"]
rowupdated["crossmatch"]={"lasair":classification["light_curve"]["crossmatches"],"check":False}
rowupdated["lightcurve"] = classification["light_curve"]["candidates"]
rowupdated["report"] = row
rowupdated["broker"] = row["broker"]
rowupdated["nobs"] = row["nobs"]
rowupdated["lastmjd"] = row["lastmjd"]
rowupdated["sigmara"] = row["sigmara"]
rowupdated["sigmadec"] = row["sigmadec"]
rowupdated["last_magpsf_g"] = row["last_magpsf_g"]
rowupdated["last_magpsf_r"] = row["last_magpsf_r"]
rowupdated["first_magpsf_g"] = row["first_magpsf_g"]
rowupdated["first_magpsf_r"] = row["first_magpsf_r"]
rowupdated["sigma_magpsf_g"] = row["sigma_magpsf_g"]
rowupdated["sigma_magpsf_r"] = row["sigma_magpsf_r"]
rowupdated["max_magpsf_g"] = row["max_magpsf_g"]
rowupdated["max_magpsf_r"] = row["max_magpsf_r"]
rowupdated["id"] = row["id"]
#check if already exist this candidate, if exist update light curve and run check list to alerts
currentdata=db.getData(filter={"id":id},projection={"nobs":1,"last_update":1,"id":1})
now = datetime.now().timestamp()
if len(currentdata)>0 :
#update current data
try:
if currentdata[0]["nobs"] < rowupdated["nobs"]:
peak = lasairarchive.getPeakLightCurve(classification["light_curve"]["candidates"])
rowupdated["lightpeak"] = peak
update_query={"last_update":now,"lightcurve":rowupdated["lightcurve"],"lightpeak":peak,"lasair_clas":rowupdated["lasair_clas"],"alerce_clas":rowupdated["alerce_clas"],"nobs":rowupdated["nobs"],"state":"updated"}
update_id = db.update(filter={"id":id}, query={"$set":update_query})
print("updated source",id,update_id.raw_result)
else:
print("last detections is the same, not getting enough to services update classify",id)
except Exception as err:
print("Error updated",id,currentdata[0]["nobs"],rowupdated["nobs"])
logger.error("checkLastDetections:: {0} Error updated..".format(str(id)))
else:
peak = lasairarchive.getPeakLightCurve(classification["light_curve"]["candidates"])
rowupdated["lightpeak"] = peak
#insert new candidate
print("save new candidate")
rowupdated["state"]="new"
rowupdated["last_update"] = now
db.saveData(rowupdated)
logger.info("checkLastDetections:: {0} Saved candidate with {1} observations".format(id,rowupdated["nobs"]))
newCandidates+=1
logger.info("checkLastDetections:: {0} candidates stored..".format(str(len(dic_result))))
logger.info("checkLastDetections:: alerce table detections {0}".format(str(len(alerceTable))))
logger.info("checkLastDetections:: lasair table detections {0}".format(str(len(lasairTable))))
logger.info("checkLastDetections:: desi detections {0}".format(str(len(desi_targetstable))))
logger.info("checkLastDetections:: new Candidates {0}".format(str(newCandidates)))
db.saveData(data={"date":now,"newcandidates":newCandidates,"allrecords":allrecords,"alerce_records":len(alerceTable),"lasair_records":len(lasairTable),"desi_matchs":len(desi_targetstable),"process":"lastdetections"},collection="tasks")
if "IDpipeLine" in kwargs.keys() and kwargs["IDpipeLine"]!="":
updatePipeline(kwargs["IDpipeLine"],"checkLastDetections",STATE_COMPLETED)