def read_catalog(catalog_file, kow=None):
amaj, amin, phi = None, None, None
default_err = 3.0
if ".dat" in catalog_file:
lines = [line.rstrip('\n') for line in open(catalog_file)]
names, ras, decs, errs = [], [], [], []
if ("fermi" in catalog_file) or ("chandra" in catalog_file):
amaj, amin, phi = [], [], []
for line in lines:
lineSplit = list(filter(None, line.split(" ")))
if ("blue" in catalog_file) or ("uvex" in catalog_file) or (
"xraybinary" in catalog_file
) or ("wd_rd" in catalog_file) or ("wd_bd" in catalog_file) or (
"cyclotron"
in catalog_file) or ("elm_wd" in catalog_file) or (
"amcvn"
in catalog_file) or ("sdb_dm" in catalog_file) or (
"wdb_noneclipsing"
in catalog_file) or ("sinusoidal" in catalog_file):
ra_hex, dec_hex = convert_to_hex(
float(lineSplit[0]) * 24 / 360.0,
delimiter=''), convert_to_hex(float(lineSplit[1]),
delimiter='')
if dec_hex[0] == "-":
objname = "ZTFJ%s%s" % (ra_hex[:4], dec_hex[:5])
else:
objname = "ZTFJ%s%s" % (ra_hex[:4], dec_hex[:4])
names.append(objname)
ras.append(float(lineSplit[0]))
decs.append(float(lineSplit[1]))
errs.append(default_err)
elif ("vlss" in catalog_file):
names.append(lineSplit[0])
ras.append(float(lineSplit[1]))
decs.append(float(lineSplit[2]))
err = np.sqrt(float(lineSplit[3])**2 +
float(lineSplit[4])**2) * 3600.0
errs.append(err)
elif ("fermi" in catalog_file):
names.append(lineSplit[0])
ras.append(float(lineSplit[1]))
decs.append(float(lineSplit[2]))
err = np.sqrt(float(lineSplit[3])**2 +
float(lineSplit[4])**2) * 3600.0
errs.append(err)
amaj.append(float(lineSplit[3]))
amin.append(float(lineSplit[4]))
phi.append(float(lineSplit[5]))
elif ("chandra" in catalog_file):
names.append(lineSplit[0])
ras.append(float(lineSplit[1]))
decs.append(float(lineSplit[2]))
err = np.sqrt(float(lineSplit[3])**2 +
float(lineSplit[4])**2) / 3600.0
errs.append(err)
amaj.append(float(lineSplit[3]) / 3600.0)
amin.append(float(lineSplit[4]) / 3600.0)
phi.append(float(lineSplit[5]))
elif ("swift" in catalog_file) or ("xmm" in catalog_file):
names.append(lineSplit[0])
ras.append(float(lineSplit[1]))
decs.append(float(lineSplit[2]))
err = float(lineSplit[3])
errs.append(err)
else:
names.append(lineSplit[0])
ras.append(float(lineSplit[1]))
decs.append(float(lineSplit[2]))
errs.append(default_err)
names = np.array(names)
ras, decs, errs = np.array(ras), np.array(decs), np.array(errs)
if ("fermi" in catalog_file) or ("chandra" in catalog_file):
amaj, amin, phi = np.array(amaj), np.array(amin), np.array(phi)
elif ".hdf5" in catalog_file:
with h5py.File(catalog_file, 'r') as f:
ras, decs = f['ra'][:], f['dec'][:]
errs = default_err * np.ones(ras.shape)
names = []
for ra, dec in zip(ras, decs):
ra_hex, dec_hex = convert_to_hex(
ra * 24 / 360.0, delimiter=''), convert_to_hex(dec,
delimiter='')
if dec_hex[0] == "-":
objname = "ZTFJ%s%s" % (ra_hex[:4], dec_hex[:5])
else:
objname = "ZTFJ%s%s" % (ra_hex[:4], dec_hex[:4])
names.append(objname)
names = np.array(names)
elif ".h5" in catalog_file:
df = pd.read_hdf(catalog_file)
objids = np.array(df.index).astype(int)
ras, decs = [], []
for ii, objid in enumerate(objids):
if np.mod(ii, 1000) == 0:
print("%d/%d" % (ii, len(objids)))
#if ii > 1000: continue
objid, features = get_kowalski_features_objids(
[objid], kow, featuresetname='all')
period = features["period"]
ras.append(features["ra"].values[0])
decs.append(features["dec"].values[0])
ras, decs = np.array(ras), np.array(decs)
errs = default_err * np.ones(ras.shape)
names = []
for ra, dec in zip(ras, decs):
ra_hex, dec_hex = convert_to_hex(
ra * 24 / 360.0, delimiter=''), convert_to_hex(dec,
delimiter='')
if dec_hex[0] == "-":
objname = "ZTFJ%s%s" % (ra_hex[:4], dec_hex[:5])
else:
objname = "ZTFJ%s%s" % (ra_hex[:4], dec_hex[:4])
names.append(objname)
names = np.array(names)
elif ".csv" in catalog_file:
df = pd.read_csv(catalog_file)
if "eROSITA" in catalog_file:
names = df["Name"].to_numpy()
ras = df["RA"].to_numpy()
decs = df["DEC"].to_numpy()
else:
df = df[df["Duplicate_flag"] < 2]
df = df[df["Quality_flag"] == 0]
df = df[df["Total_flux"] >= 1]
df = df[df["Total_flux"] / df["E_Total_flux"] >= 5]
names = df["Component_name"].to_numpy()
ras = df["RA"].to_numpy()
decs = df["DEC"].to_numpy()
errs = default_err * np.ones(ras.shape)
return names, ras, decs, errs, amaj, amin, phi
lc["mag"] = np.random.uniform(low=18, high=19, size=(nsample,))
lc["magerr"] = np.random.uniform(low=0.01, high=0.1, size=(nsample,))
lc["fid"] = fid*np.ones(lc["hjd"].shape)
lc["ra"] = ra*np.ones(lc["hjd"].shape)
lc["dec"] = dec*np.ones(lc["hjd"].shape)
lc["absmag"] = [np.nan, np.nan, np.nan]
lc["bp_rp"] = np.nan
lc["parallax"] = np.nan
lightcurves_all[fid] = lc
lightcurves.append([lc["hjd"], lc["mag"], lc["magerr"]])
absmags.append(lc["absmag"])
bp_rps.append(lc["bp_rp"])
else:
objid, features = get_kowalski_features_objids([index], kow_features,
dbname="ZTF_source_features_20191101", featuresetname='limited')
try:
period = features.period.values[0]
ra, dec = features.ra.values[0], features.dec.values[0]
except:
ra, dec = row["RA"], row["Dec"]
if "p" in row:
period = row["p"]
else:
period = -1
amp = -1
lightcurves = get_kowalski(ra, dec, kow_lcs, min_epochs=20, radius=2.0)
if len(lightcurves.keys()) == 0:
continue
if opts.source_type == "quadrant":
if opts.query_type == "skiplimit":
ids, features = get_kowalski_features(kow,
num_batches=Ncatalog,
nb=Ncatindex,
featuresetname=modeltype,
dbname=dbname)
elif opts.query_type == "ids":
objids = np.load(opts.ids_file)
nlightcurves = len(objids)
objids_split = np.array_split(objids, Ncatalog)
objids = objids_split[Ncatindex]
print(modeltype)
ids, features = get_kowalski_features_objids(
objids, kow, featuresetname=modeltype, dbname=dbname)
elif opts.source_type == "catalog":
amaj, amin, phi = None, None, None
if not opts.default_err is None:
default_err = opts.default_err
else:
default_err = 5.0
if ".dat" in catalog_file:
lines = [line.rstrip('\n') for line in open(catalog_file)]
ras, decs, errs = [], [], []
if ("fermi" in catalog_file):
amaj, amin, phi = [], [], []
for line in lines:
lineSplit = list(filter(None, line.split(" ")))
# Parse command line
opts = parse_commandline()
objid = opts.objid
kow = []
nquery = 10
cnt = 0
while cnt < nquery:
try:
kow = Kowalski(username=opts.user, password=opts.pwd)
break
except:
time.sleep(5)
cnt = cnt + 1
if cnt == nquery:
raise Exception('Kowalski connection failed...')
tmp, features = get_kowalski_features_objids([objid], kow)
if len(features) == 0:
exit(0)
period = features.period.values[0]
amp = features.f1_amp.values[0]
lightcurves, coordinates, filters, ids, absmags, bp_rps, names, baseline = get_kowalski_objids(
[objid], kow)
ra, dec = coordinates[0][0], coordinates[0][1]
print(ra, dec)
print(period)
cat = Table.read(catalogPath, format='fits')
idx1 = np.where(cat["prob"]<=0.1)[0]
idx2 = np.where(cat["prob"]>=0.9)[0]
nsamples = 1000
objids1 = cat["objid"][idx1]
objids2 = cat["objid"][idx2]
idx1 = np.random.choice(len(objids1), size=nsamples)
idx2 = np.random.choice(len(objids2), size=nsamples)
objids1 = objids1[idx1]
objids2 = objids2[idx2]
objids1_tmp, features1 = get_kowalski_features_objids(objids1, kow)
objids1_tmp, features2 = get_kowalski_features_objids(objids2, kow)
feature_set11 = ['median', 'wmean', 'chi2red', 'roms', 'wstd', 'norm_peak_to_peak_amp',
'norm_excess_var', 'median_abs_dev', 'iqr', 'f60', 'f70', 'f80', 'f90',
'skew', 'smallkurt', 'inv_vonneumannratio', 'welch_i', 'stetson_j',
'stetson_k', 'ad', 'sw']
keys = ["wstd", 'norm_peak_to_peak_amp', 'norm_excess_var', 'skew', 'inv_vonneumannratio', 'welch_i', 'stetson_j', 'stetson_k', 'ad', 'sw']
labels = [r"$m_{\mathrm{var}}$","Norm. Peak to Peak Amp.","Norm. Excess Var.","Skew","Inv. Von Neumann","Welch/Stetson I","Stetson J","Stetson K","Anderson-Darling","Shapiro-Wilk"]
color2 = 'coral'
color1 = 'cornflowerblue'
#labels = []
magerr = ['AllWISE__w1sigmpro', 'AllWISE__w2sigmpro',
'AllWISE__w3sigmpro', 'AllWISE__w4sigmpro',
'PS1_DR1__gMeanPSFMagErr',
'PS1_DR1__rMeanPSFMagErr',
'PS1_DR1__iMeanPSFMagErr',
'PS1_DR1__zMeanPSFMagErr',
'PS1_DR1__yMeanPSFMagErr']
parallax = ['Gaia_DR2__parallax', 'Gaia_DR2__parallax_error']
catalogFile = os.path.join(outputDir,"catalog.h5")
if not os.path.isfile(catalogFile):
objids = np.array(df.index)
#objids = objids[:100]
objids, features = get_kowalski_features_objids(objids, kow,
featuresetname='all')
snr = features["Gaia_DR2__parallax"]/features["Gaia_DR2__parallax_error"]
idx = np.where((snr >= 3) & ~np.isnan(snr))[0]
objids = np.array(objids.iloc[idx])
features = features.iloc[idx]
ra = np.array(features['ra']),
dec = np.array(features['dec'])
coord = SkyCoord(ra=ra*u.degree, dec=dec*u.degree, frame='icrs')
galcoords = np.vstack([coord.galactic.l.deg, coord.galactic.b.deg]).T
mag = np.array(features[mag])
magerr = np.array(features[magerr])
parallax = np.array(features[parallax])
if modelName not in df_merged:
continue
idx = np.where(df_merged[modelName] >= 0.9)[0]
print("Model %s: %.5f%%" %
(modelName, 100 * len(idx) / len(df_merged[modelName])))
if opts.doPlots:
plotDir = os.path.join(baseplotDir, modelName)
if not os.path.isdir(plotDir):
os.makedirs(plotDir)
cnt = 0
for index, row in df_merged[modelName].iloc[idx].iteritems():
if cnt > 10: continue
objid, features = get_kowalski_features_objids([index], kow)
if len(features) == 0: continue
period = features.period.values[0]
amp = features.f1_amp.values[0]
lightcurves, coordinates, filters, ids, absmags, bp_rps, names, baseline = get_kowalski_objids(
[index], kow)
if len(lightcurves) == 0: continue
hjd, magnitude, err = lightcurves[0]
absmag, bp_rp = absmags[0], bp_rps[0]
phases = np.mod(hjd, 2 * period) / (2 * period)