def tailored_simulation_javelin(self,lc,band,z,name,periodogram,lightcurve):
useful_funcs.create_dir(self.lc_dir+"combined")
useful_funcs.create_dir(self.lc_dir+"combined/"+name)
filename = self.make_combined_lc_for_javelin(lc,name,band)
javdata = get_data(filename,names=["Continuum"])
cont = Cont_Model(javdata)
cont.do_mcmc(nwalkers=500, nburn=100, nchain=500,fchain="test.dat")
cont.show_hist(figout="javelin_%s" % band, figext=".png")
def __init__(self):
self.query_SDSS = query.query_SDSS()
self.dir_filter = "filters/" # create filter info dir
useful_funcs.create_dir(self.dir_filter)
self.dir_spec = "spectra/" # create dir for spectra
useful_funcs.create_dir(self.dir_spec)
self.band_list = ["g", "r", "i", "z"]
self.filter_info = {"SDSS": {}, "DES": {}, "LCO": {}, "CRTS": {}}
self.get_DES_SDSS_bandpass()
self.get_LCO_bandpass()
self.get_CRTS_bandpass()
def analyze_lightcurve(self,quasar):
""" the main analyzing function """
name = quasar["name"]
print ("-- Analyzing quasar %s --" % name)
useful_funcs.create_dir(self.output_dir+name)
real_dir = self.output_dir+name+"/real"
useful_funcs.create_dir(real_dir)
for band in self.band_list:
# avoid missing band
if not self.check_lightcurves_exist(name,band):
print ("SDSS and DES lightcurves not found in "+band+" band !!")
else:
# read DES survey
survey = self.surveys[0]
data = self.read_lightcurve(name,survey,band)
#data = self.add_flux_fluxerr_into_data(data)
time, signal, error = data["mjd_obs"],data["mag_psf"],\
data["mag_err_psf"]
lc = qso_drw(time, signal, error, quasar["z"], preprocess=False)
# add other surveys
for survey in self.surveys[1:]:
data = self.read_lightcurve(name,survey,band)
#data = self.add_flux_fluxerr_into_data(data)
if len(data) > 0 :
time,signal,error = data["mjd_obs"],data["mag_psf"],\
data["mag_err_psf"]
lc.add_lc(time, signal, error, preprocess=False)
np.save("%s/lc_%s.npy" % (real_dir,band),\
[lc.time,lc.signal,lc.error])
# periodogram
if not len(lc.time)>3:
print ("Not enough data points !!")
else:
# run Lomb-Scargle for target
period, psd = lc.ls_astroML()
# save error estimated from boostrping
psd_error = self.error_boostraping(lc,band)
np.save("%s/psd_%s.npy" % (real_dir,band),\
[period, psd,psd_error])
# tailored simulation
self.tailored_simulation(lc,band,quasar["z"],name)
def __init__(self):
self.lc_dir = "lightcurves/"
useful_funcs.create_dir(self.lc_dir)
self.query = query.query_DES() # to get DES lightcurves
self.query_S82 = query.Stripe82() # get SDSS S82 lightcurves
self.query_SDSS = query.query_SDSS() # get SDSS spectra
self.query_ZTF = query.query_ZTF() # get ZTF lightcurves
self.save_dir = self.lc_dir + "DES/"
useful_funcs.create_dir(self.save_dir)
self.quasar_catalog_dir = "catalog/"
self.quasar_catalog = "DR14+DR7+OzDES+Milliq_S1S2.txt"
self.lc_info_file = "lc_info.csv"
self.log = "log_lightcurves"
self.band_list = ["g", "r", "i", "z"]
self.save_line = ""
def make_pdf_for_strong_candidates(self):
""" making the latex file and figure dir for stong cadidates """
quasars = pd.read_csv(self.stat_dir+"strong_candidates.csv")
header = "\documentclass[12pt]{article}\n"+\
"\\usepackage{graphicx}\n"+\
"\\usepackage[export]{adjustbox}\n"+\
"\\usepackage[margin=0.5in]{geometry}\n"+\
"\\begin{document}\n"
section = "\\section*{%s}\n"
body1 = "\\begin{figure}[!bp]\n"+\
"\\begin{minipage}[b]{0.48\\textwidth}\n"+\
"\\includegraphics[width=\\textwidth]"+\
"{%s/periodogram.png}\n"+\
"\\end{minipage}\n"+\
"\\hfill\n"+\
"\\begin{minipage}[b]{0.48\\textwidth}\n"+\
"\\includegraphics[width=\\textwidth,right]"+\
"{%s/lightcurve.png}\n"+\
"\\end{minipage}\n"+\
"\\includegraphics[width=0.8\\textwidth]"+\
"{%s/ACF.png}\n"+\
"\\end{figure}\n"
clearpage = "\\clearpage\n"
footer = "\\end{document}\n"
self.pdf_dir = self.stat_dir+"pdf/"
useful_funcs.create_dir(self.pdf_dir)
f = open(self.pdf_dir+"main.tex","w")
f.write(header)
for i in [4,3,2]:
quasars_times = quasars[quasars["yes"]==i]
f.write(section % (str(i)+"bands, $>$99.7\%"))
for index, row in quasars_times.iterrows():
name = row["name"]
useful_funcs.create_dir(self.pdf_dir+name)
os.system("cp "+self.output_dir+name+"/periodogram.png "+\
self.pdf_dir+name)
os.system("cp "+self.output_dir+name+"/lightcurve.png "+\
self.pdf_dir+name)
os.system("cp "+self.output_dir+name+"/ACF.png "+\
self.pdf_dir+name)
f.write(body1 % (name,name,name))
f.write(clearpage)
f.write(footer)
f.close()
def __init__(self):
self.lc_dir = "lightcurves/"
self.lc_info_file = "lc_clean.csv"
#self.lc_info_file = "lc_remain.csv"
self.output_dir = "best_candidates/"
useful_funcs.create_dir(self.output_dir)
self.stat_dir = "statistics/"
useful_funcs.create_dir(self.stat_dir)
self.carma_dir = "carma/"
self.random_state = np.random.RandomState(0)
self.band_list = ["g", "r", "i", "z"]
self.surveys = ["DES", "SDSS_corr"]
self.surveys_add = ["ZTF","PS"]
self.lc_names = ["mjd_obs", "mag_psf", "mag_err_psf"]
self.period_lowerlim = 500
self.test = False
self.large_mock = False
def _process_and_save(self,time,signal,error,band,mag_diff=None,\
outlier=False,bin_data=False,record=True):
# process the lightcurve (outlier rejection or binning) and save to
# the lightcurve directory
# remove outlier
if outlier:
mask = self._get_mask_sigma_clip_moving_avg(signal)
time = time[mask]
signal = signal[mask]
error = error[mask]
# bin the data at the same date
if bin_data:
time, signal, error = self._bin_data(time, signal, error)
# save lightcurves
np.savetxt("%s/%s.csv" % (self.save_dir , band),\
np.array([time,signal,error]).T,\
fmt="%f,%f,%f",comments="",\
header="mjd_obs,mag_psf,mag_err_psf")
if record:
self.save_line = self.save_line + "," + str(len(signal))
if mag_diff is not None: # For SDSS only
corr_dir = self.save_dir.replace("SDSS", "SDSS_corr")
corr_dir = corr_dir.replace("LCO", "LCO_corr")
corr_dir = corr_dir.replace("PS", "PS_corr")
useful_funcs.create_dir(corr_dir)
signal = signal + mag_diff[band][0]
error = np.sqrt(error**2 + mag_diff[band][1]**2)
np.savetxt("%s/%s.csv" % (corr_dir , band),\
np.array([time,signal,error]).T,\
fmt="%f,%f,%f",comments="",\
header="mjd_obs,mag_psf,mag_err_psf")
def analyze_lightcurve_orig(self,quasar):
""" the main analyzing function """
#periodogram = plot(2,2)
#multi_periodogram = plot(1,1)
#periodogram_carma = plot(2,2)
#lightcurve = plot(4,1,figsize=(8,8),sharex=True)
name = quasar["name"]
print ("-- Analyzing quasar "+name+ "--")
useful_funcs.create_dir(self.output_dir+name)
#useful_funcs.create_dir(self.output_dir+name+"/"+self.carma_dir)
lightcurves_total = []
for band in self.band_list:
if not self.check_lightcurves_exist(name,band):
print ("SDSS and DES lightcurves not found in "+band+" band !!")
else:
time_all,signal_all,error_all = [],[],[]
# read main survey
survey = self.surveys[0]
data = self.read_lightcurve(name,survey,band)
data = self.add_flux_fluxerr_into_data(data)
lc = qso_drw(data["mjd_obs"],data["mag_psf"],\
data["mag_err_psf"], quasar["z"],\
preprocess=False)
time, signal, error = lc.get_lc()
#lightcurve.plot_light_curve(time,signal,error,survey,band)
# add other surveys
for survey in self.surveys[1:]:
data = self.read_lightcurve(name,survey,band)
data = self.add_flux_fluxerr_into_data(data)
lc2 = qso_drw(data["mjd_obs"],data["mag_psf"],\
data["mag_err_psf"], quasar["z"],\
preprocess=False)
time, signal, error = lc2.get_lc()
if len(signal) > 0 :
lightcurve.plot_light_curve(time,signal,error,\
survey,band)
lc.add_lc(time, signal, error,preprocess=False)
# periodogram
if not len(lc.time)>3:
print ("Not enough data points !!")
else:
period, psd = lc.ls_astroML()
periodogram.plot_periodogram(period, psd,band)
multi_periodogram.plot_multi_periodogram(period,psd,band)
#periodogram_carma.plot_periodogram(period,psd,band)
error_list = self.error_boostraping(lc,periodogram,band)
#error_list = self.error_boostraping(lc,periodogram_carma,band)
period_max,amp_max = self._check_period_max_amp(period, psd)
if period_max is None:
print ("Can not find the peak value !!")
else:
periodogram.plot_peak_period(period_max,band)
amp, amp_err, xn,yn = self._fitting(lc.time,lc.signal,lc.error,period_max)
lightcurve.plot_fit_curve(xn,yn,band)
lightcurves_total.append(np.array([lc.time,lc.signal,\
lc.error,[band]*len(lc.time)]))
self.tailored_simulation(lc,band,quasar["z"],name,\
periodogram,lightcurve)
#self.do_carma_process(lc,band,name,periodogram_carma,lightcurve)
self.save_period_amp(period, psd,error_list,\
self.output_dir+name+\
"/periodogram_"+band+".csv")
lightcurves_total = np.concatenate(lightcurves_total, axis=1)
np.savetxt(self.output_dir+name+"/lightcurve_total.csv",\
lightcurves_total, delimiter=",", \
comments="",header="time,signal,error,band")
self.do_multi_band_periodogram(lightcurves_total,\
multi_periodogram)
lightcurve.savefig(self.output_dir+name,"/lightcurve.png",name)
periodogram.savefig(self.output_dir+name,"/periodogram.png",name)
multi_periodogram.savefig(self.output_dir+name,"/periodogram_multi.png",name)
def tailored_simulation(self,lc,band,z,name):#,periodogram,lightcurve):
""" running the DRW model fitted to each quasar """
mock_dir = self.output_dir+name+"/mock"
useful_funcs.create_dir(mock_dir)
#psd_mock_all = []
# parameters
if self.test :
nwalkers,burnin,Nsteps,draw_times = 100,5,100,50
elif self.large_mock :
nwalkers,burnin,Nsteps,draw_times = 500,200,1000,50000
#nwalkers,burnin,Nsteps,draw_times = 100,25,100,500
else:
nwalkers = 500
burnin = 150
Nsteps = 500
draw_times = 5000
# fitting DRW model
samples = lc.fit_drw_emcee(nwalkers=nwalkers, burnin=burnin,\
Nstep=Nsteps,random_state=self.random_state)
# plot walkers
walkers = plot(3,1,figsize=(10,7), sharex=True)
walkers.plot_walkers(np.exp(samples))
walkers.savefig(mock_dir,"/walker_"+band+".png",band+" band")
parameters_list = samples[:, burnin:, :].reshape((-1, 3))
# remove top 2.5% and bottome 2.5%
parameters_list_good,good_index = self.clean_parameters_list(parameters_list)
# plot posterior
theta = [parameters_list_good[:,0],parameters_list_good[:,1],\
parameters_list_good[:,2]]
likelihood = []
for theta in parameters_list_good:
likelihood.append(lnlike(theta,lc.time,lc.signal,lc.error,z))
from plot import plot_posterior
plot_posterior(np.exp(parameters_list_good),likelihood,\
band,mock_dir+"/post_%s.png" % band)
np.save("%s/drw_%s.npy" % (mock_dir,band),parameters_list_good)
# make the mock light curves from DRW parameters
mock_lcs,mock_psds = [],[]
index = self.random_state.choice(len(parameters_list_good),\
size=draw_times,replace=False)
#for i in range(draw_times):
for i in index:
tau,c,b = np.exp(parameters_list_good[i])
#mock_time,mock_signal = lc.generate_mock_lightcurve(tau,c,lc.time,\
# lc.signal,z,random_state=self.random_state)
mock_time,mock_signal = lc.get_BPL_lc(tau,lc.time,lc.signal)
mock_lcs.append(mock_signal)
#mock_lcs.append(np.array([mock_time,mock_signal]).T)
#self.save_lightcurve(mock_time,mock_signal,lc.error,\
# "%s/lc_%s.csv" % (mock_dir,i))
#lightcurve.plot_mock_curve(mock_time,mock_signal_correct,band)
period_mock, psd_mock = lc.periodogram(mock_time,mock_signal)
mock_psds.append(psd_mock)
#self.save_period_amp(period_mock,psd_mock,None,\
# "%s/psd_%s.csv" % (mock_dir,i))
#mock_psds.append(np.array([period_mock,psd_mock]).T)
#periodogram.plot_mock_periodogram(period_mock, psd_mock,band)
np.save("%s/lcs_%s.npy" % (mock_dir,band),mock_lcs)
np.save("%s/psds_%s.npy" % (mock_dir,band),mock_psds)
"""
def set_output_dir(self, output_dir):
# set the output dir (e.g. DES/, SDSS/)
self.save_dir = output_dir
useful_funcs.create_dir(self.save_dir)