def read_lasair_json(object_name='ZTF18acsovsw'):
"""
Read light curve from lasair website API based on object name.
Parameters
----------
object_name : str
The LASAIR object name. E.g. object_name='ZTF18acsovsw'
"""
print(object_name)
if isinstance(object_name, tuple):
object_name, z_in = object_name
else:
z_in = None
url = 'https://lasair.roe.ac.uk/object/{}/json/'.format(object_name)
data = read_json(url)
objid = data['objectId']
ra = data['objectData']['ramean']
dec = data['objectData']['decmean']
# lasair_classification = data['objectData']['classification']
tns_info = data['objectData']['annotation']
photoZ = None
for cross_match in data['crossmatches']:
# print(cross_match)
photoZ = cross_match['photoZ']
separation_arcsec = cross_match['separationArcsec']
catalogue_object_type = cross_match['catalogue_object_type']
if z_in is not None and not np.isnan(z_in):
redshift = z_in
else:
if photoZ is None: # TODO: Get correct redshift
try:
if "z=" in tns_info:
photoZ = tns_info.split('z=')[1]
redshift = float(photoZ.replace(')', '').split()[0])
# print("PHOTOZZZZZZZZZZZZ", redshift, tns_info)
elif "Z=" in tns_info:
photoZ = tns_info.split('Z=')[1]
redshift = float(photoZ.split()[0])
# print("PHOTOZZZZZZZZZZZZ", redshift, tns_info)
else:
# return
print("TRYING ARBITRARY GUESS REDSHIFT = 0.1")
redshift = None
except Exception as e:
# return
redshift = None
print(e)
else:
redshift = photoZ
print("Redshift is {}".format(redshift))
if redshift is not None:
objid += "_z={}".format(round(redshift, 2))
# Get extinction TODO: Maybe add this to RAPID code
coo = coord.SkyCoord(ra * u.deg, dec * u.deg, frame='icrs')
dust = IrsaDust.get_query_table(coo, section='ebv')
mwebv = dust['ext SandF mean'][0]
print("MWEBV")
print(mwebv)
mjd = []
passband = []
mag = []
magerr = []
photflag = []
zeropoint = []
dc_mag = []
dc_magerr = []
magnr, sigmagnr, isdiffpos = [], [], []
for cand in data['candidates']:
mjd.append(cand['mjd'])
passband.append(cand['fid'])
mag.append(cand['magpsf'])
if 'sigmapsf' in cand:
magerr.append(cand['sigmapsf'])
photflag.append(4096)
# if cand['magzpsci'] == 0:
# print("NO ZEROPOINT")
# zeropoint.append(26.2) # TODO: Tell LASAIR their zeropoints are wrong
# else:
# zeropoint.append(cand['magzpsci']) #26.2) #
if cand['magzpsci'] == 0:
print(object_name, zeropoint)
raise Exception
return
zeropoint.append(cand['magzpsci'])
dc_mag.append(cand['dc_mag'])
dc_magerr.append(cand['dc_sigmag'])
magnr.append(cand['magnr'])
sigmagnr.append(cand['sigmagnr'])
isdiffpos.append(cand['isdiffpos'])
else:
magerr.append(
np.nan
) #0.01 * cand['magpsf']) #magerr.append(None) #magerr.append(0.1 * cand['magpsf']) #
photflag.append(0)
zeropoint.append(np.nan) #26.2)
dc_mag.append(np.nan)
dc_magerr.append(np.nan)
magnr.append(np.nan)
sigmagnr.append(np.nan)
isdiffpos.append(None)
mjd, passband, mag, magerr, photflag, zeropoint, dc_mag, dc_magerr, magnr, sigmagnr, isdiffpos = convert_lists_to_arrays(
mjd, passband, mag, magerr, photflag, zeropoint, dc_mag, dc_magerr,
magnr, sigmagnr, isdiffpos)
# deleteindexes = np.where(magerr == None) # [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29] #
# mjd, passband, mag, magerr, photflag, zeropoint, dc_mag, dc_magerr, magnr, sigmagnr, isdiffpos = delete_indexes(deleteindexes, mjd, passband, mag, magerr, photflag, zeropoint, dc_mag, dc_magerr, magnr, sigmagnr, isdiffpos)
deleteindexes = np.where(
(photflag == 0) & (mjd > min(mjd[photflag > 0]))
) # delete where nondetections after first detection
mjd, passband, mag, magerr, photflag, zeropoint, dc_mag, dc_magerr, magnr, sigmagnr, isdiffpos = delete_indexes(
deleteindexes, mjd, passband, mag, magerr, photflag, zeropoint, dc_mag,
dc_magerr, magnr, sigmagnr, isdiffpos)
deleteindexes = np.where(
(mag <
(np.median(mag[photflag == 0]) - 0.5 * np.std(mag[photflag == 0]))) &
(photflag == 0)) # Remove non detection outliers
mjd, passband, mag, magerr, photflag, zeropoint, dc_mag, dc_magerr, magnr, sigmagnr, isdiffpos = delete_indexes(
deleteindexes, mjd, passband, mag, magerr, photflag, zeropoint, dc_mag,
dc_magerr, magnr, sigmagnr, isdiffpos)
return mjd, passband, mag, magerr, photflag, zeropoint, ra, dec, objid, redshift, mwebv, dc_mag, dc_magerr, magnr, sigmagnr, isdiffpos
def get_real_ztf_training_data(class_name,
data_dir='data/real_ZTF_data_from_osc',
save_dir='data/saved_light_curves/',
pbs=('g', 'r'),
known_redshift=True,
nprocesses=1,
redo=False,
calculate_t0=True):
"""
Get data from saved real ZTF data with names and types from the Open Supernova Catalog
"""
save_lc_filepath = os.path.join(save_dir,
f"lc_classnum_{class_name}.pickle")
if os.path.exists(save_lc_filepath) and not redo:
with open(save_lc_filepath, "rb") as fp: # Unpickling
light_curves = pickle.load(fp)
else:
light_curves = {}
data_filepath = os.path.join(
data_dir, f"ZTF_data_{class_name}_osc-6-May-2020.pickle")
with open(data_filepath, "rb") as fp:
mjds, passbands, mags, magerrs, photflags, zeropoints, dc_mags, dc_magerrs, magnrs, \
sigmagnrs, isdiffposs, ras, decs, objids, redshifts, mwebvs = pickle.load(fp)
for i, objid in enumerate(objids):
if known_redshift and (redshifts[i] is None
or np.isnan(redshifts[i])):
print(
f"Skipping {objid} because redshift is unknown and known_redshift model is selected"
)
continue
flux = 10.**(-0.4 * (mags[i] - zeropoints[i]))
fluxerr = np.abs(flux * magerrs[i] * (np.log(10.) / 2.5))
passbands[i] = np.where(
(passbands[i] == 1) | (passbands[i] == '1'), 'g', passbands[i])
passbands[i] = np.where(
(passbands[i] == 2) | (passbands[i] == '2'), 'r', passbands[i])
mjd_first_detection = min(mjds[i][photflags[i] == 4096])
photflags[i][np.where(mjds[i] == mjd_first_detection)] = 6144
deleteindexes = np.where((
(passbands[i] == 3) | (passbands[i] == '3')) | (
(mjds[i] > mjd_first_detection) & (photflags[i] == 0))
| (np.isnan(flux)))
if deleteindexes[0].size > 0:
print("Deleting indexes {} at mjd {} and passband {}".format(
deleteindexes, mjds[i][deleteindexes],
passbands[i][deleteindexes]))
mjd, passband, flux, fluxerr, zeropoint, photflag = delete_indexes(
deleteindexes, mjds[i], passbands[i], flux, fluxerr,
zeropoints[i], photflags[i])
peakmjd = mjd[np.argmax(flux)]
inputlightcurve = InputLightCurve(mjd,
flux,
fluxerr,
passband,
photflag,
ras[i],
decs[i],
objid,
redshifts[i],
mwebvs[i],
known_redshift=known_redshift,
training_set_parameters={
'class_number': class_name,
'peakmjd': peakmjd
},
calculate_t0=calculate_t0)
light_curves[objid] = inputlightcurve.preprocess_light_curve()
with open(save_lc_filepath, "wb") as fp:
pickle.dump(light_curves, fp)
return light_curves
def classify_lasair_light_curves(
object_names=('ZTF18acsovsw', ), savename='', sntype=''):
light_curve_list = []
mjds, passbands, mags, magerrs, zeropoints, photflags = [], [], [], [], [], []
dc_mags, dc_magerrs, magnrs, sigmagnrs, isdiffposs = [], [], [], [], []
obj_names = []
ras, decs, objids, redshifts, mwebvs = [], [], [], [], []
peakmags_g, peakmags_r = [], []
for object_name in object_names:
try:
mjd, passband, mag, magerr, photflag, zeropoint, ra, dec, objid, redshift, mwebv, dc_mag, dc_magerr, magnr, sigmagnr, isdiffpos = read_lasair_json(
object_name)
sortidx = np.argsort(mjd)
mjds.append(mjd[sortidx])
passbands.append(passband[sortidx])
mags.append(mag[sortidx])
magerrs.append(magerr[sortidx])
zeropoints.append(zeropoint[sortidx])
photflags.append(photflag[sortidx])
dc_mags.append(dc_mag[sortidx])
dc_magerrs.append(dc_magerr[sortidx])
magnrs.append(magnr[sortidx])
sigmagnrs.append(sigmagnr[sortidx])
isdiffposs.append(isdiffpos[sortidx])
obj_names.append(object_name)
ras.append(ra)
decs.append(dec)
objids.append(f"{sntype}_{objid}")
redshifts.append(redshift)
mwebvs.append(mwebv)
peakmags_g.append(min(mag[passband == 1]))
peakmags_r.append(min(mag[passband == 2]))
except Exception as e:
print(e)
continue
flux = 10.**(-0.4 * (mag - zeropoint))
fluxerr = np.abs(flux * magerr * (np.log(10.) / 2.5))
passband = np.where((passband == 1) | (passband == '1'), 'g', passband)
passband = np.where((passband == 2) | (passband == '2'), 'r', passband)
# Set photflag detections when S/N > 5
photflag2 = np.zeros(len(flux))
photflag2[flux / fluxerr > 5] = 4096
photflag2[np.where(mjd == min(mjd[photflag2 == 4096]))] = 6144
mjd_first_detection = min(mjd[photflag == 4096])
photflag[np.where(mjd == mjd_first_detection)] = 6144
deleteindexes = np.where(((passband == 3) | (passband == '3'))
| (mjd > mjd_first_detection)
& (photflag == 0))
if deleteindexes[0].size > 0:
print("Deleting indexes {} at mjd {} and passband {}".format(
deleteindexes, mjd[deleteindexes], passband[deleteindexes]))
mjd, passband, flux, fluxerr, zeropoint, photflag = delete_indexes(
deleteindexes, mjd, passband, flux, fluxerr, zeropoint, photflag)
light_curve_list += [(mjd, flux, fluxerr, passband, photflag, ra, dec,
objid, redshift, mwebv)]
with open(savename, 'wb') as f:
pickle.dump([
mjds, passbands, mags, magerrs, photflags, zeropoints, dc_mags,
dc_magerrs, magnrs, sigmagnrs, isdiffposs, ras, decs, objids,
redshifts, mwebvs
], f)
# np.savez('save_real_ZTF_unprocessed_data_snia_osc_12nov2019.npz', mjds=mjds, passbands=passbands, mags=mags, magerrs=magerrs, photflags=photflags, zeropoints=zeropoints, ras=ras, decs=decs, objids=objids, redshifts=redshifts, mwebvs=mwebvs)# , peakflux_g=peakfluxes_g, peakflux_r=peakfluxes_r)
print("finished")
def classify_lasair_light_curves(
object_names=('ZTF18acsovsw', ), plot=True, figdir='.'):
light_curve_list = []
peakfluxes_g, peakfluxes_r = [], []
mjds, passbands, mags, magerrs, zeropoints, photflags = [], [], [], [], [], []
obj_names = []
ras, decs, objids, redshifts, mwebvs = [], [], [], [], []
peakmags_g, peakmags_r = [], []
for object_name in object_names:
try:
mjd, passband, mag, magerr, photflag, zeropoint, ra, dec, objid, redshift, mwebv = read_lasair_json(
object_name)
sortidx = np.argsort(mjd)
mjds.append(mjd[sortidx])
passbands.append(passband[sortidx])
mags.append(mag[sortidx])
magerrs.append(magerr[sortidx])
zeropoints.append(zeropoint[sortidx])
photflags.append(photflag[sortidx])
obj_names.append(object_name)
ras.append(ra)
decs.append(dec)
objids.append(objid)
redshifts.append(redshift)
mwebvs.append(mwebv)
peakmags_g.append(min(mag[passband == 1]))
peakmags_r.append(min(mag[passband == 2]))
except Exception as e:
print(e)
continue
flux = 10.**(-0.4 * (mag - zeropoint))
fluxerr = np.abs(flux * magerr * (np.log(10.) / 2.5))
passband = np.where((passband == 1) | (passband == '1'), 'g', passband)
passband = np.where((passband == 2) | (passband == '2'), 'r', passband)
# Set photflag detections when S/N > 5
photflag2 = np.zeros(len(flux))
photflag2[flux / fluxerr > 5] = 4096
photflag2[np.where(mjd == min(mjd[photflag2 == 4096]))] = 6144
mjd_first_detection = min(mjd[photflag == 4096])
photflag[np.where(mjd == mjd_first_detection)] = 6144
deleteindexes = np.where(((passband == 3) | (passband == '3'))
| (mjd > mjd_first_detection)
& (photflag == 0))
if deleteindexes[0].size > 0:
print("Deleting indexes {} at mjd {} and passband {}".format(
deleteindexes, mjd[deleteindexes], passband[deleteindexes]))
mjd, passband, flux, fluxerr, zeropoint, photflag = delete_indexes(
deleteindexes, mjd, passband, flux, fluxerr, zeropoint, photflag)
light_curve_list += [(mjd, flux, fluxerr, passband, photflag, ra, dec,
objid, redshift, mwebv)]
try:
dummy = max(flux[passband == 'g'])
dummy = max(flux[passband == 'r'])
except Exception as e:
print(e)
continue
peakfluxes_g.append(max(flux[passband == 'g']))
peakfluxes_r.append(max(flux[passband == 'r']))
# import sys
# import pickle
# with open('save_real_ZTF_unprocessed_data_snia_osc_12nov2019.npz', 'wb') as f:
# pickle.dump([mjds, passbands, mags, magerrs, photflags, zeropoints, ras, decs, objids, redshifts, mwebvs], f)
# # np.savez('save_real_ZTF_unprocessed_data_snia_osc_12nov2019.npz', mjds=mjds, passbands=passbands, mags=mags, magerrs=magerrs, photflags=photflags, zeropoints=zeropoints, ras=ras, decs=decs, objids=objids, redshifts=redshifts, mwebvs=mwebvs)# , peakflux_g=peakfluxes_g, peakflux_r=peakfluxes_r)
# print("finished")
# # # sys.exit(0)
# # with open('save_real_ZTF_unprocessed_data_snia_osc_12nov2019.npz', 'rb') as f:
# # a = pickle.load(f)
classification = Classify(known_redshift=True, bcut=False, zcut=None)
predictions, time_steps = classification.get_predictions(
light_curve_list, return_predictions_at_obstime=False)
print(predictions)
if plot:
# try:
classification.plot_light_curves_and_classifications(
step=True,
use_interp_flux=False,
figdir=figdir,
plot_matrix_input=True)
# except Exception as e:
# print(e)
# classification.plot_light_curves_and_classifications(step=False, use_interp_flux=True)
# classification.plot_light_curves_and_classifications(step=False, use_interp_flux=False)
# classification.plot_classification_animation_step()
# classification.plot_classification_animation()
return classification.orig_lc, classification.timesX, classification.y_predict