def updateLasairClassification(self): now = datetime.now() current_time = now.strftime("%d/%m/%Y, %H:%M:%S") print("---------------------------------------") print("{0}: UPDATING Lasair".format(current_time)) self.setLocalDB() self.setAlerceDB() lastdate = self.getlastLasairObject() lasair = LasairArchive() #lastdate[0]=58266 #lasair.getAllData(lastdate[0], page=0) lasair.getAllData(lastdate[0], page=0)
def getLastLasair(request): lasair = LasairArchive() lastItems = lasair.getLastDetections(days_ago=4) if request.GET.__contains__('jobid'): jobid = request.GET['jobid'] else: status = {"error": "job id is required"} newItems = lastItems.fillna('', axis=1) # lastItems.replace(np.nan, '', regex=True) dic_result = newItems.to_dict('records') # for index,row in enumerate(dic_result): # for key in row.keys(): # if math.isnan(row[key]): # row[key]="nan" return JsonResponse(dic_result, safe=False)
def getClassification(ztfid): print("getting classification for:",ztfid) alercearchive = AlerceArchive() lasairarchive = LasairArchive() lightCurve = lasairarchive.getObjectInfo(ztfid) lasair_classification = "" alerce_classification = "" if "objectData" in lightCurve: if "classification" in lightCurve["objectData"]: lasair_classification = lightCurve["objectData"]["classification"] else: lightCurve_lasair = alercearchive.getLightCurve(ztfid) stats = alercearchive.getStats(ztfid) lasair_classification = "not in lasair" lightCurve = {"candidates":lightCurve_lasair["result"]["detections"],"crossmatches":[],"objectId":ztfid,"objectData":{"ramean":stats["result"]["stats"]["meanra"],"decmean":stats["result"]["stats"]["meandec"]}} alerce_classification = alercearchive.getProbabilities(ztfid) best_late = 0 best_late_key = "" for late_key in alerce_classification["result"]["probabilities"]["late_classifier"]: #print(type(alerce_classification["result"]["probabilities"]["late_classifier"][late_key]), late_key) if type(alerce_classification["result"]["probabilities"]["late_classifier"][late_key]) is float and late_key != "classifier_version": if best_late < alerce_classification["result"]["probabilities"]["late_classifier"][late_key]: best_late = alerce_classification["result"]["probabilities"]["late_classifier"][late_key] best_late_key = str(late_key)[0:str(late_key).find("_")].upper() best_early = 0 best_early_key = "" for early_key in alerce_classification["result"]["probabilities"]["early_classifier"]: #print(type(alerce_classification["result"]["probabilities"]["early_classifier"][early_key]), early_key) if type(alerce_classification["result"]["probabilities"]["early_classifier"][ early_key]) is float and early_key != "classifier_version": if best_early < alerce_classification["result"]["probabilities"]["early_classifier"][early_key]: best_early = alerce_classification["result"]["probabilities"]["early_classifier"][early_key] best_early_key = str(early_key)[0:str(early_key).find("_")].upper() if "probabilities" in alerce_classification["result"]: alerce_classification = alerce_classification["result"]["probabilities"] return {"light_curve":lightCurve,"lasair_clas":lasair_classification,"alerce_clas":alerce_classification,"alerce_early_class":best_early_key,"alerce_late_class":best_late_key}
def scoreCandidates(collection,filter={}): lasairarchive = LasairArchive() db = MongodbManager() config = Config() dbconfig = config.getDatabase("mongodb") db.setDatabase(dbconfig["dbname"]) db.setCollection(collection) data = db.getData(filter=filter) for indx, row in enumerate(data): print("ID score",row["id"])
def searchCadidates(self, days=15): alerce = AlerceArchive() alerceGoodCandidates = alerce.getCandidates(days) lasairbroker = LasairArchive() lasairGoodCandidates = lasairbroker.getLastDetections(days) # alerceGoodCandidates = [r["result"][target] for target in r["result"]] alerceTable = QTable.from_pandas(alerceGoodCandidates) lasairTable = QTable.from_pandas(lasairGoodCandidates) print("candidates found lasair{0} alerce{1} ".format( len(lasairGoodCandidates), len(alerceGoodCandidates))) meanaler_val = { "ramean": np.nan_to_num( np.concatenate((alerceGoodCandidates["meanra"].values, lasairGoodCandidates["meanra"].values), axis=0)), "decmean": np.nan_to_num( np.concatenate((alerceGoodCandidates["meandec"].values, lasairGoodCandidates["meandec"].values), axis=0)), "maggmax": np.nan_to_num( np.concatenate((alerceGoodCandidates["max_magap_g"].values, lasairGoodCandidates["maggmax"].values), axis=0)), "maggmin": np.nan_to_num( np.concatenate((alerceGoodCandidates["min_magap_g"].values, lasairGoodCandidates["maggmin"].values), axis=0)), "magrmax": np.nan_to_num( np.concatenate((alerceGoodCandidates["max_magap_r"].values, lasairGoodCandidates["magrmax"].values), axis=0)), "magrmin": np.nan_to_num( np.concatenate((alerceGoodCandidates["min_magap_r"].values, lasairGoodCandidates["magrmin"].values), axis=0)), "id": np.concatenate((alerceGoodCandidates["oid"].values, lasairGoodCandidates["oid"].values), axis=0) } bigdata = pd.DataFrame(meanaler_val) bigdata_drop = bigdata.drop_duplicates(subset="id", keep=False) bigtable = bigdata if bigdata_drop.size > 0: bigtable = bigdata_drop table_candidates = QTable(QTable.from_pandas(bigtable), masked=False) return table_candidates, alerceGoodCandidates, lasairGoodCandidates
def checkLastDetections(**kwargs): # try: allrecords=0 collection=current_collection days_ago=15 if "collection" in kwargs.keys() and kwargs["collection"]!="": collection = kwargs["collection"] if "days_ago" in kwargs.keys() and kwargs["days_ago"]!="": days_ago=kwargs["days_ago"] if "IDpipeLine" in kwargs.keys() and kwargs["IDpipeLine"]!="": updatePipeline(kwargs["IDpipeLine"],"checkLastDetections",STATE_RUNNING) logger.info("checkLastDetections:: getting the last ZTF detections from brokers...") lasairarchive = LasairArchive() #coneecto to DATABASE db = MongodbManager() config=Config() dbconfig=config.getDatabase("mongodb") db.setDatabase(dbconfig["dbname"]) db.setCollection(collection) #Get last candidates and update previews detection and light curves bestCandidates = BestCandidates() table_candidates, alerceDF, lasairDF = bestCandidates.searchCadidates(days_ago) #check if the new candidates is already in DB #get all zft id in and array to validate if exist into ddatabase and filter by listcandidates=table_candidates["id"] filter={"oid":{"$in":listcandidates.data.tolist()}} projection={"oid":1 ,"lastmjd":1 ,"last_update":1} current_data = db.getData(filter=filter, projection=projection) for remove_data in current_data: oid=remove_data["oid"] print("get info for ",oid) table_candidates.remove_rows(table_candidates["id"] == oid) #get desi photoz dataarchive = SussexArchive() desi_targetsvo, desi_targetstable = dataarchive.getDesiPhotoZfromTable(table_candidates) alerceTable = QTable.from_pandas(alerceDF) lasairTable = QTable.from_pandas(lasairDF) alerceTable.rename_column("oid","id") lasairTable.rename_column("oid", "id") alerceTable["id"] = alerceTable["id"].astype(str) lasairTable["id"] = lasairTable["id"].astype(str) desi_targetstable["id"] =desi_targetstable["id"].astype(str) desi_targetstable["desidec"].mask = False desi_targetstable["desira"].mask = False #calc separation desi source ra_ref = desi_targetstable["ramean"].tolist() dec_ref = desi_targetstable["decmean"].tolist() cref = SkyCoord(ra_ref, dec_ref, frame='icrs', unit='deg') ra_desi = desi_targetstable["desira"].tolist() dec_desi = desi_targetstable["desidec"].tolist() c1 = SkyCoord(ra_desi, dec_desi, frame='icrs', unit='deg') desi_distance = cref.separation(c1).arcsec desi_targetstable["separation"] = desi_distance #merge all table in one json to save in mongo desi_targetstable = Table(desi_targetstable, masked=False) alerceTable = Table(alerceTable, masked=False) lasairTable = Table(lasairTable, masked=False) alerceTable["broker"] = "alerce" lasairTable["broker"] = "lasair" update_alerce_table = join(alerceTable, lasairTable, join_type='outer', keys='id') merge_table = join(update_alerce_table, desi_targetstable, join_type='outer', keys='id') merge_table["desiid"] = merge_table["desiid"].astype(str) merge_table["field"] = merge_table["field"].astype(str) lastItems= merge_table.to_pandas() newItems = lastItems.fillna('', axis=1) dic_result = newItems.to_dict('records') newCandidates=0 logger.info("checkLastDetections:: Ingested {0} candidates".format(str(len(dic_result)))) allrecords=len(dic_result) for index,row in enumerate(dic_result): id=row["id"] print("saving candidate",id) row["comments"]={} row["snh_score"] = 0.0 if row["broker_1"] != "": #alerce #row["pclassearly"]=row["pclassearly_1"] if row["broker_2"]!="": row["broker"]=row["broker_1"]+"/"+row["broker_2"] else: row["broker"] = row["broker_1"] row["meanra"]=row["meanra_1"] row["meandec"]=row["meandec_1"] row["lastmjd"]=row["lastmjd_1"] else: #lasair #row["pclassearly"] = row["pclassearly_2"] row["broker"] = row["broker_2"] row["meanra"] = row["meanra_2"] row["meandec"] = row["meandec_2"] row["lastmjd"] = row["lastmjd_2"] try: #remove duplicate fields #del row["pclassearly_1"] #del row["pclassearly_2"] del row["broker_1"] del row["broker_2"] del row["meanra_1"] del row["meandec_1"] del row["meanra_2"] del row["meandec_2"] del row["lastmjd_2"] del row["lastmjd_1"] except KeyError as er: print("key error",er,id) # check if already exist this candidate, if exist update light curve and run check list to alerts currentdata = db.getData(filter={"id": id}, projection={"nobs": 1, "last_update": 1, "id": 1}) now = datetime.now().timestamp() rowupdated={} if len(currentdata) > 0: currentdata = currentdata[0] days_from_update = ((now - float(currentdata["last_update"])) / 3600) / 24 if days_from_update < 0.6: print("last detections is the same, not getting enough to services update classify",id) logger.info("checkLastDetections:: {0} last detections is the same, not getting enough to services update classify".format(id)) continue classification = getClassification(id) #peak = lasairarchive.getPeakLightCurve(classification["light_curve"]["candidates"]) rowupdated["ra"] = row["meanra"] rowupdated["dec"] = row["meandec"] rowupdated["lasair_clas"]=classification["lasair_clas"] rowupdated["alerce_clas"]=classification["alerce_clas"] rowupdated["alerce_early_class"] = classification["alerce_early_class"] rowupdated["alerce_late_class"] = classification["alerce_late_class"] rowupdated["crossmatch"]={"lasair":classification["light_curve"]["crossmatches"],"check":False} rowupdated["lightcurve"] = classification["light_curve"]["candidates"] rowupdated["report"] = row rowupdated["broker"] = row["broker"] rowupdated["nobs"] = row["nobs"] rowupdated["lastmjd"] = row["lastmjd"] rowupdated["sigmara"] = row["sigmara"] rowupdated["sigmadec"] = row["sigmadec"] rowupdated["last_magpsf_g"] = row["last_magpsf_g"] rowupdated["last_magpsf_r"] = row["last_magpsf_r"] rowupdated["first_magpsf_g"] = row["first_magpsf_g"] rowupdated["first_magpsf_r"] = row["first_magpsf_r"] rowupdated["sigma_magpsf_g"] = row["sigma_magpsf_g"] rowupdated["sigma_magpsf_r"] = row["sigma_magpsf_r"] rowupdated["max_magpsf_g"] = row["max_magpsf_g"] rowupdated["max_magpsf_r"] = row["max_magpsf_r"] rowupdated["id"] = row["id"] #check if already exist this candidate, if exist update light curve and run check list to alerts currentdata=db.getData(filter={"id":id},projection={"nobs":1,"last_update":1,"id":1}) now = datetime.now().timestamp() if len(currentdata)>0 : #update current data try: if currentdata[0]["nobs"] < rowupdated["nobs"]: peak = lasairarchive.getPeakLightCurve(classification["light_curve"]["candidates"]) rowupdated["lightpeak"] = peak update_query={"last_update":now,"lightcurve":rowupdated["lightcurve"],"lightpeak":peak,"lasair_clas":rowupdated["lasair_clas"],"alerce_clas":rowupdated["alerce_clas"],"nobs":rowupdated["nobs"],"state":"updated"} update_id = db.update(filter={"id":id}, query={"$set":update_query}) print("updated source",id,update_id.raw_result) else: print("last detections is the same, not getting enough to services update classify",id) except Exception as err: print("Error updated",id,currentdata[0]["nobs"],rowupdated["nobs"]) logger.error("checkLastDetections:: {0} Error updated..".format(str(id))) else: peak = lasairarchive.getPeakLightCurve(classification["light_curve"]["candidates"]) rowupdated["lightpeak"] = peak #insert new candidate print("save new candidate") rowupdated["state"]="new" rowupdated["last_update"] = now db.saveData(rowupdated) logger.info("checkLastDetections:: {0} Saved candidate with {1} observations".format(id,rowupdated["nobs"])) newCandidates+=1 logger.info("checkLastDetections:: {0} candidates stored..".format(str(len(dic_result)))) logger.info("checkLastDetections:: alerce table detections {0}".format(str(len(alerceTable)))) logger.info("checkLastDetections:: lasair table detections {0}".format(str(len(lasairTable)))) logger.info("checkLastDetections:: desi detections {0}".format(str(len(desi_targetstable)))) logger.info("checkLastDetections:: new Candidates {0}".format(str(newCandidates))) db.saveData(data={"date":now,"newcandidates":newCandidates,"allrecords":allrecords,"alerce_records":len(alerceTable),"lasair_records":len(lasairTable),"desi_matchs":len(desi_targetstable),"process":"lastdetections"},collection="tasks") if "IDpipeLine" in kwargs.keys() and kwargs["IDpipeLine"]!="": updatePipeline(kwargs["IDpipeLine"],"checkLastDetections",STATE_COMPLETED)
def getPeaks(**kwargs): collection = current_collection filter = {"lightcurve": {"$exists": True}} projection = {} if "collection" in kwargs.keys() and kwargs["collection"] != "": collection = kwargs["collection"] if "filter" in kwargs.keys() and kwargs["filter"] != "": filter = kwargs["filter"] if "projection" in kwargs.keys() and kwargs["projection"] != "": projection = kwargs["projection"] if "IDpipeLine" in kwargs.keys() and kwargs["IDpipeLine"] != "": updatePipeline(kwargs["IDpipeLine"], "getPeaks", STATE_RUNNING) lasairarchive = LasairArchive() db = MongodbManager() config = Config() dbconfig = config.getDatabase("mongodb") db.setDatabase(dbconfig["dbname"]) db.setCollection(collection) data = db.getData(filter=filter) for indx,row in enumerate(data): print("try to get peak",row["id"]) if len(row["lightcurve"])>0: peak = lasairarchive.getPeakLightCurve(row["lightcurve"]) query = {} query["best_photoz_gabmag"] = 999 query["best_photoz_rabmag"] = 999 query["best_specz_gabmag"] = 999 query["best_specz_rabmag"] = 999 if "Redshift" in row or ("redshift" in row and len(row["redshift"].keys())>0): redshift = [] if "Redshift" in row: z=row["Redshift"] redshifts_archives=["tns"] else: redshifts_archives=row["redshift"].keys() for z_key in redshifts_archives: if z_key == "sncosmos": if "best"in row["redshift"]["sncosmos"] and "redshift" in row["redshift"]["sncosmos"]["best"]: z = row["redshift"]["sncosmos"]["best"]["redshift"] else: continue else: z=row["redshift"][z_key] if "g" in peak["stats"].keys(): if "magab" not in peak["stats"]["g"]: peak["stats"]["g"]["magab"]={} peak["stats"]["g"]["magab"][z_key] = Convertion.aparentToAbsoluteMagnitud(peak["stats"]["g"]["y"],z=z).tolist() if "r" in peak["stats"].keys(): if "magab" not in peak["stats"]["r"]: peak["stats"]["r"]["magab"]={} peak["stats"]["r"]["magab"][z_key] = Convertion.aparentToAbsoluteMagnitud(peak["stats"]["r"]["y"], z=z).tolist() if "g" in peak["lightcurve"].keys(): if "magab" not in peak["lightcurve"]["g"]: peak["lightcurve"]["g"]["magab"]={} peak["lightcurve"]["g"]["magab"][z_key] = Convertion.aparentToAbsoluteMagnitud( peak["lightcurve"]["g"]["mag"], z=z).tolist() if "r" in peak["lightcurve"].keys(): if "magab" not in peak["lightcurve"]["r"]: peak["lightcurve"]["r"]["magab"]={} peak["lightcurve"]["r"]["magab"][z_key] = Convertion.aparentToAbsoluteMagnitud( peak["lightcurve"]["r"]["mag"], z=z).tolist() peaks=[] gmag=False rmag = False if "g" in peak["stats"].keys(): peak_g = peak["stats"]["g"]["peakmag"] peaks.append(peak_g) gmag = True if "r" in peak["stats"].keys(): peak_r = peak["stats"]["r"]["peakmag"] peaks.append(peak_r) rmag=True if "best_photo_z" in row.keys() and len(row["best_photo_z"])>0: photoz = row["best_photo_z"]["photo_z"] best_photomagab=Convertion.aparentToAbsoluteMagnitud(peaks, z=photoz).tolist() if gmag: query["best_photoz_gabmag"]=best_photomagab[0] if rmag: idxphotorbest = 1 if gmag else 0 query["best_photoz_rabmag"]=best_photomagab[idxphotorbest] if "best_spec_z" in row.keys() and len(row["best_spec_z"])>0: specz = row["best_spec_z"]["spec_z"] best_specmagab=Convertion.aparentToAbsoluteMagnitud(peaks, z=specz).tolist() if gmag: query["best_specz_gabmag"]=best_specmagab[0] if rmag: idxspecbest = 1 if gmag else 0 query["best_specz_rabmag"]=best_specmagab[idxspecbest] query["lightpeak"]= peak if "g" in peak["status"].keys(): query["g_state"]= peak["status"]["g"] if "r" in peak["status"].keys(): query["r_state"]= peak["status"]["r"] update=db.update(filter={"id":row["id"]},query={"$set":query}) print("update peak ",row["id"],update) if "IDpipeLine" in kwargs.keys() and kwargs["IDpipeLine"] != "": updatePipeline(kwargs["IDpipeLine"], "getPeaks", STATE_COMPLETED)