if config.mltype == "splml":
if config.forcen:
ml_param = config.nmlspl
string_ML = "nmlspl"
else:
ml_param = config.mlknotsteps
string_ML = "knml"
elif config.mltype == "polyml":
ml_param = config.degree
string_ML = "deg"
else:
raise RuntimeError('I dont know your microlensing type. Choose "polyml" or "spml".')
regdiffparamskw = ut.read_preselected_regdiffparamskw(config.config_directory + 'preset_regdiff.txt')[0]
result_dir = config.lens_directory + config.combkw[0,0]
regdiffresult_dir = result_dir + 'sims_%s_opt_regdiff%s_t%s/' % (config.simset_copy, regdiffparamskw, str(int(config.tsrand)))
spl1result_dir = result_dir + 'sims_%s_opt_spl1t%s/' % (config.simset_copy, str(int(tsrand)))
group_list = [pycs3.tdcomb.comb.getresults(
pycs3.tdcomb.comb.CScontainer('Regression difference',
result_file_delays=regdiffresult_dir + 'sims_%s_opt_regdiff%s_t%s_delays.pkl' % (config.simset_copy, regdiffparamskw, str(int(config.tsrand))),
result_file_errorbars=regdiffresult_dir + 'sims_%s_opt_regdiff%s_t%s_errorbars.pkl' % (config.simset_mock, regdiffparamskw, str(int(config.tsrand))),
colour='red')),
pycs3.tdcomb.comb.getresults(
pycs3.tdcomb.comb.CScontainer('Free-knot splines',
result_file_delays=spl1result_dir + 'sims_%s_opt_spl1t%s_delays.pkl' % (config.simset_copy, str(int(tsrand))),
result_file_errorbars=spl1result_dir + 'sims_%s_opt_spl1t%s_errorbars.pkl' % (config.simset_mock, str(int(tsrand))),
colour='blue'))]
if optfctkw == "spl1":
optfct = spl1
splparamskw = ["ks%i" %knotstep[i] for i in range(len(knotstep))]
if optfctkw == "regdiff": # not used, small haxx to be able to execute 2 to check and 3 using the spl1 drawing
optfct = regdiff
splparamskw = "ks%i" % knotstep
if simoptfctkw == "spl1":
simoptfct = spl1
if simoptfctkw == "regdiff":
simoptfct = regdiff
if use_preselected_regdiff :
regdiffparamskw = ut.read_preselected_regdiffparamskw(preselection_file)
else :
regdiffparamskw = ut.generate_regdiffparamskw(pointdensity,covkernel, pow, amp)
if mltype == "splml" :
if forcen == False :
combkw = [["%s_ks%i_%s_knml_%i" %(optfctkw, knotstep[i], mlname, mlknotsteps[j]) for j in range(len(mlknotsteps))]for i in range(len(knotstep))]
else :
combkw = [["%s_ks%i_%s_nmlspl_%i" % (optfctkw, knotstep[i], mlname, nmlspl[j]) for j in range(len(nmlspl))]for i in range(len(knotstep))]
elif mltype == "polyml":
combkw = [["%s_ks%i_%s_deg_%i" % (optfctkw, knotstep[i], mlname, degree[j]) for j in range(len(degree))] for i in range(len(knotstep))]
else :
raise RuntimeError('I dont know your microlensing type. Choose "polyml" or "spml".')
combkw = np.asarray(combkw)
def main(lensname, dataname, work_dir='./'):
sys.path.append(work_dir + "config/")
config = importlib.import_module("config_" + lensname + "_" + dataname)
marginalisation_plot_dir = config.figure_directory + 'marginalisation_plots/'
regdiff_dir = os.path.join(config.lens_directory, "regdiff_outputs/")
regdiff_copie_dir = os.path.join(regdiff_dir, "copies/")
if not os.path.isdir(marginalisation_plot_dir):
os.mkdir(marginalisation_plot_dir)
indiv_marg_dir = marginalisation_plot_dir + config.name_marg_regdiff + '/'
if not os.path.isdir(indiv_marg_dir):
os.mkdir(indiv_marg_dir)
marginalisation_dir = config.lens_directory + config.name_marg_regdiff + '/'
if not os.path.isdir(marginalisation_dir):
os.mkdir(marginalisation_dir)
if config.testmode:
nbins = 500
else:
nbins = 5000
colors = [
"royalblue", "crimson", "seagreen", "darkorchid", "darkorange",
'indianred', 'purple', 'brown', 'black', 'violet', 'dodgerblue',
'palevioletred', 'olive', 'brown', 'salmon', 'chocolate', 'indigo',
'steelblue', 'cyan', 'gold', 'lightcoral'
]
f = open(
marginalisation_dir + 'report_%s_sigma%2.1f.txt' %
(config.name_marg_regdiff, config.sigmathresh), 'w')
path_list = []
name_list = []
if config.mltype == "splml":
if config.forcen:
ml_param = config.nmlspl
string_ML = "nmlspl"
else:
ml_param = config.mlknotsteps
string_ML = "knml"
elif config.mltype == "polyml":
ml_param = config.degree
string_ML = "deg"
else:
raise RuntimeError(
'I dont know your microlensing type. Choose "polyml" or "spml".')
kw_list = ut.read_preselected_regdiffparamskw(config.preselection_file)
kw_dic = ut.get_keyword_regdiff_from_file(config.preselection_file)
for s, set in enumerate(kw_dic):
if 'name' not in list(set.keys()):
set['name'] = 'Set %i' % s
for paramskw, dickw in zip(kw_list, kw_dic):
for n, noise in enumerate(config.tweakml_name_marg_regdiff):
count = 0
color_id = 0
group_list = []
medians_list = []
errors_up_list = []
errors_down_list = []
simset_mock_ava = [
"mocks_n%it%i_%s" %
(int(config.nsim * config.nsimpkls), config.truetsr, twk)
for twk in config.tweakml_name_marg_regdiff
]
opt = 'regdiff'
if not config.use_preselected_regdiff:
raise RuntimeError(
"Turn the use_preselected_regdiff to True and set your preselection_file before using this script."
)
for a, kn in enumerate(config.knotstep_marg_regdiff):
for b, ml in enumerate(config.mlknotsteps_marg_regdiff):
regdiff_mocks_dir = os.path.join(
regdiff_dir, "mocks_knt%i_mlknt%i/" % (kn, ml))
result_file_delay = regdiff_copie_dir + 'sims_%s_opt_regdiff%s' % (config.simset_copy, paramskw) \
+ 't%i_delays.pkl' % int(config.tsrand)
result_file_errorbars = regdiff_mocks_dir \
+ 'sims_%s_opt_regdiff%s' % (simset_mock_ava[n], paramskw) + \
't%i_errorbars.pkl' % int(config.tsrand)
if not os.path.isfile(
result_file_delay) or not os.path.isfile(
result_file_errorbars):
print('Error I cannot find the files %s or %s. ' \
'Did you run the 3c and 4a?' % (result_file_delay, result_file_errorbars))
f.write('Error I cannot find the files %s or %s. \n' %
(result_file_delay, result_file_errorbars))
continue
group_list.append(
pycs3.tdcomb.comb.getresults(
pycs3.tdcomb.comb.CScontainer(
data=dataname,
knots=kn,
ml=ml,
name="knstp %i mlknstp %i" % (kn, ml),
drawopt=config.optfctkw,
runopt=opt,
ncopy=config.ncopy * config.ncopypkls,
nmocks=config.nsim * config.nsimpkls,
truetsr=config.truetsr,
colour=colors[color_id],
result_file_delays=result_file_delay,
result_file_errorbars=result_file_errorbars)))
medians_list.append(group_list[-1].medians)
errors_up_list.append(group_list[-1].errors_up)
errors_down_list.append(group_list[-1].errors_down)
if np.isnan(medians_list[-1]).any() or np.isnan(
errors_up_list[-1]).any() or np.isnan(
errors_down_list[-1]).any():
print(
"There is some Nan value in %s, for noise %s, kn %i, %s%i"
% (dickw['name'], noise, kn, string_ML, ml))
print(
"I could erase this entry and continue the marginalisation. "
)
ut.proquest(True)
medians_list = medians_list[:-1]
errors_down_list = errors_down_list[:-1]
errors_up_list = errors_down_list[:-1]
group_list = group_list[:-1]
continue
color_id += 1
count += 1
if color_id >= len(colors):
print(
"Warning : I don't have enough colors in my list, I'll restart from the beginning."
)
color_id = 0 # reset the color form the beginning
f.write('Set %i, knotstep : %2.2f, %s : %2.2f \n' %
(count, kn, string_ML, ml))
f.write(
'covkernel : %s, point density: %2.2f, pow : %2.2f, errscale:%2.2f \n'
% (dickw["covkernel"], dickw["pointdensity"],
dickw["pow"], dickw["errscale"]))
f.write('Tweak ml name : %s \n' % noise)
f.write(
'------------------------------------------------ \n')
# build the bin list :
medians_list = np.asarray(medians_list)
errors_down_list = np.asarray(errors_down_list)
errors_up_list = np.asarray(errors_up_list)
binslist = []
for i, lab in enumerate(config.delay_labels):
bins = np.linspace(
min(medians_list[:, i]) - 10 * min(errors_down_list[:, i]),
max(medians_list[:, i]) + 10 * max(errors_up_list[:, i]),
nbins)
binslist.append(bins)
color_id = 0
for g, group in enumerate(group_list):
group.binslist = binslist
group.plotcolor = colors[color_id]
group.linearize(testmode=config.testmode)
group.objects = config.lcs_label
color_id += 1
if color_id >= len(colors):
print(
"Warning : I don't have enough colors in my list, I'll restart from the beginning."
)
color_id = 0 # reset the color form the beginning
combined = copy.deepcopy(
pycs3.tdcomb.comb.combine_estimates(group_list,
sigmathresh=1000.0,
testmode=config.testmode))
combined.linearize(testmode=config.testmode)
combined.name = 'Most precise'
print(
"%s : Taking the best of all spline parameters for regdiff parameters set %s"
% (config.name_marg_regdiff, dickw['name']))
combined.niceprint()
# plot the results :
text = [(0.85, 0.90, r"$\mathrm{" + config.full_lensname + "}$" +
"\n" + r"$\mathrm{PyCS\ estimates}$", {
"fontsize": 26,
"horizontalalignment": "center"
})]
radius = (np.max(errors_up_list) +
np.max(errors_down_list)) / 2.0 * 3.5
ncurve = len(config.lcs_label)
if ncurve > 2:
auto_radius = True
figsize = (17, 13)
else:
auto_radius = False
figsize = (15, 10)
if config.display:
pycs3.tdcomb.plot.delayplot(group_list + [combined],
rplot=radius,
refgroup=combined,
text=text,
hidedetails=True,
showbias=False,
showran=False,
tick_step_auto=True,
autoobj=config.lcs_label,
showlegend=True,
figsize=figsize,
horizontaldisplay=False,
legendfromrefgroup=False,
auto_radius=auto_radius)
pycs3.tdcomb.plot.delayplot(group_list + [combined],
rplot=radius,
refgroup=combined,
text=text,
hidedetails=True,
tick_step_auto=True,
autoobj=config.lcs_label,
showbias=False,
showran=False,
showlegend=True,
figsize=figsize,
horizontaldisplay=False,
auto_radius=auto_radius,
legendfromrefgroup=False,
filename=indiv_marg_dir +
config.name_marg_regdiff +
"_%s_%s.png" % (dickw['name'], noise))
pkl.dump(
group_list,
open(
marginalisation_dir + config.name_marg_regdiff + "_%s_%s" %
(dickw['name'], noise) + '_groups.pkl', 'wb'))
pkl.dump(
combined,
open(
marginalisation_dir + config.name_marg_regdiff + "_%s_%s" %
(dickw['name'], noise) + '_combined.pkl', 'wb'))
path_list.append(marginalisation_dir + config.name_marg_regdiff +
"_%s_%s" % (dickw['name'], noise) +
'_combined.pkl')
name_list.append('%s, Noise : %s ' % (dickw['name'], noise))
# ------ MAKE THE FINAL REGDIFF ESTIMATE ------#
final_groups, final_combined = pycs3.tdcomb.comb.group_estimate(
path_list,
name_list=name_list,
colors=colors,
sigma_thresh=config.sigmathresh,
new_name_marg=config.name_marg_regdiff,
testmode=config.testmode,
object_name=config.lcs_label)
radius_f = (final_combined.errors_down[0] +
final_combined.errors_up[0]) / 2.0 * 2.5
text = [(0.85, 0.90, r"$\mathrm{" + config.full_lensname + "}$" + "\n" +
r"$\mathrm{PyCS\ estimates}$", {
"fontsize": 26,
"horizontalalignment": "center"
})]
if config.display:
pycs3.tdcomb.plot.delayplot(final_groups + [final_combined],
rplot=radius_f,
refgroup=final_combined,
tick_step_auto=True,
text=text,
hidedetails=True,
showbias=False,
showran=False,
auto_radius=auto_radius,
autoobj=config.lcs_label,
showlegend=True,
figsize=(15, 10),
horizontaldisplay=False,
legendfromrefgroup=False)
pycs3.tdcomb.plot.delayplot(final_groups + [final_combined],
rplot=radius_f,
refgroup=final_combined,
text=text,
hidedetails=True,
showbias=False,
showran=False,
showlegend=True,
figsize=(15, 10),
horizontaldisplay=False,
autoobj=config.lcs_label,
legendfromrefgroup=False,
auto_radius=auto_radius,
tick_step_auto=True,
filename=indiv_marg_dir +
config.name_marg_regdiff + "_sigma_%2.2f.png" %
(config.sigmathresh))
pkl.dump(
final_groups,
open(
marginalisation_dir + config.name_marg_regdiff +
"_sigma_%2.2f" % config.sigmathresh + '_groups.pkl', 'wb'))
pkl.dump(
final_combined,
open(
marginalisation_dir + config.name_marg_regdiff +
"_sigma_%2.2f" % config.sigmathresh + '_combined.pkl', 'wb'))
def main(lensname, dataname, work_dir='./'):
sys.path.append(work_dir + "config/")
config = importlib.import_module("config_" + lensname + "_" + dataname)
figure_directory = config.figure_directory + "spline_and_residuals_plots/"
if not os.path.isdir(figure_directory):
os.mkdir(figure_directory)
for i, lc in enumerate(config.lcs_label):
print("I will apply a initial shift of : %2.4f days for %s" %
(config.timeshifts[i], config.lcs_label[i]))
# Do the optimisation with the splines
if config.mltype == "splml":
if config.forcen:
ml_param = config.nmlspl
string_ML = "nmlspl"
else:
ml_param = config.mlknotsteps
string_ML = "knml"
elif config.mltype == "polyml":
ml_param = config.degree
string_ML = "deg"
else:
raise RuntimeError(
"I don't know your microlensing type. Choose 'polyml' or 'spml'.")
chi2 = np.zeros((len(config.knotstep), len(ml_param)))
dof = np.zeros((len(config.knotstep), len(ml_param)))
for i, kn in enumerate(config.knotstep):
for j, ml in enumerate(ml_param):
print(("ML param", j, ml))
lcs = pycs3.gen.util.readpickle(config.data)
if config.magshift is None:
magsft = [-np.median(lc.getmags()) for lc in lcs]
else:
magsft = config.magshift
pycs3.gen.lc_func.applyshifts(
lcs, config.timeshifts, magsft
) #remove median and set the time shift to the initial guess
if ml != 0:
config.attachml(lcs, ml) # add microlensing
spline = config.spl1(lcs, kn=kn)
pycs3.gen.mrg.colourise(lcs)
rls = pycs3.gen.stat.subtract(lcs, spline)
chi2[i, j] = pycs3.gen.stat.compute_chi2(rls, kn, ml)
dof[i, j] = pycs3.gen.stat.compute_dof_spline(rls, kn, ml)
if config.display:
pycs3.gen.lc_func.display(lcs, [spline],
showlegend=True,
showdelays=True,
filename="screen")
pycs3.gen.stat.plotresiduals([rls])
else:
pycs3.gen.lc_func.display(lcs, [spline],
showlegend=True,
showdelays=True,
filename=figure_directory +
"spline_fit_ks%i_%s%i.png" %
(kn, string_ML, ml))
pycs3.gen.stat.plotresiduals([rls],
filename=figure_directory +
"residual_fit_ks%i_%s%i.png" %
(kn, string_ML, ml))
# and write data, again
if not os.path.isdir(config.lens_directory + config.combkw[i, j]):
os.mkdir(config.lens_directory + config.combkw[i, j])
pycs3.gen.util.writepickle(
(lcs, spline),
config.lens_directory + '%s/initopt_%s_ks%i_%s%i.pkl' %
(config.combkw[i, j], dataname, kn, string_ML, ml))
#--- REGDIFF ---#
# DO the optimisation with regdiff as well, just to have an idea, this the first point of the grid !
lcs = pycs3.gen.util.readpickle(config.data)
pycs3.gen.mrg.colourise(lcs)
pycs3.gen.lc_func.applyshifts(lcs, config.timeshifts,
[-np.median(lc.getmags()) for lc in lcs])
for ind, l in enumerate(lcs):
l.shiftmag(ind * 0.1)
if config.use_preselected_regdiff:
kwargs_optimiser_simoptfct = ut.get_keyword_regdiff_from_file(
config.preselection_file)
regdiff_param_kw = ut.read_preselected_regdiffparamskw(
config.preselection_file)
else:
kwargs_optimiser_simoptfct = ut.get_keyword_regdiff(
config.pointdensity, config.covkernel, config.pow, config.amp,
config.scale, config.errscale)
regdiff_param_kw = ut.generate_regdiffparamskw(config.pointdensity,
config.covkernel,
config.pow, config.amp,
config.scale,
config.errscale)
for i, k in enumerate(kwargs_optimiser_simoptfct):
myrslcs = [
pycs3.regdiff.rslc.factory(l,
pd=k['pointdensity'],
covkernel=k['covkernel'],
pow=k['pow'],
errscale=k['errscale']) for l in lcs
]
if config.display:
pycs3.gen.lc_func.display(lcs, myrslcs)
pycs3.gen.lc_func.display(lcs,
myrslcs,
showdelays=True,
filename=figure_directory +
"regdiff_fit%s.png" % regdiff_param_kw[i])
for ind, l in enumerate(lcs):
l.shiftmag(-ind * 0.1)
config.regdiff(lcs, **kwargs_optimiser_simoptfct[i])
if config.display:
pycs3.gen.lc_func.display(lcs, showlegend=False, showdelays=True)
pycs3.gen.lc_func.display(
lcs,
showlegend=False,
showdelays=True,
filename=figure_directory +
"regdiff_optimized_fit%s.png" % regdiff_param_kw[i])
if not os.path.isdir(config.lens_directory + 'regdiff_fitting'):
os.mkdir(config.lens_directory + 'regdiff_fitting')
pycs3.gen.util.writepickle(
lcs, config.lens_directory +
'regdiff_fitting/initopt_regdiff%s.pkl' % regdiff_param_kw[i])
# Write the report :
print("Report will be writen in " + config.lens_directory +
'report/report_fitting.txt')
f = open(config.lens_directory + 'report/report_fitting.txt', 'w')
f.write('Measured time shift after fitting the splines : \n')
f.write('------------------------------------------------\n')
for i, kn in enumerate(config.knotstep):
f.write('knotstep : %i' % kn + '\n')
f.write('\n')
for j, ml in enumerate(ml_param):
lcs, spline = pycs3.gen.util.readpickle(
config.lens_directory + '%s/initopt_%s_ks%i_%s%i.pkl' %
(config.combkw[i, j], dataname, kn, string_ML, ml),
verbose=False)
delay_pair, delay_name = ut.getdelays(lcs)
f.write('Micro-lensing %s = %i' % (string_ML, ml) +
" Delays are " + str(delay_pair) + " for pairs " +
str(delay_name) + '. Chi2 Red : %2.5f ' % chi2[i, j] +
' DoF : %i \n' % dof[i, j])
f.write('\n')
f.write('------------------------------------------------\n')
f.write('Measured time shift after fitting with regdiff : \n')
f.write('\n')
for i, k in enumerate(kwargs_optimiser_simoptfct):
lcs = pycs3.gen.util.readpickle(
config.lens_directory +
'regdiff_fitting/initopt_regdiff%s.pkl' % regdiff_param_kw[i],
verbose=False)
delay_pair, delay_name = ut.getdelays(lcs)
f.write('Regdiff : ' + " Delays are " + str(delay_pair) +
" for pairs " + str(delay_name) + '\n')
f.write('------------------------------------------------\n')
starting_point = []
for i in range(len(config.timeshifts)):
for j in range(len(config.timeshifts)):
if i >= j:
continue
else:
starting_point.append(config.timeshifts[j] -
config.timeshifts[i])
f.write('Starting point used : ' + str(starting_point) + " for pairs " +
str(delay_name) + '\n')
f.close()