def variability_indices_distributions(field_id=100018, overwrite=False):
field = pdb.Field(field_id, "R")
indices = ["eta", "j", "delta_chi_squared", "sigma_mu", "k"]
number_of_microlensing_light_curves = 1000
number_of_microlensing_simulations_per_light_curve = 100
min_number_of_good_observations = 100
# Convenience variables for filenames
file_base = "field{:06d}_Nperccd{}_Nevents{}".format(field.id, number_of_microlensing_light_curves, number_of_microlensing_simulations_per_light_curve) + ".{ext}"
pickle_filename = os.path.join("data", "var_indices", file_base.format(ext="pickle"))
plot_filename = os.path.join("plots", "var_indices", file_base.format(ext="pdf"))
if not os.path.exists(os.path.dirname(pickle_filename)):
os.mkdir(os.path.dirname(pickle_filename))
if not os.path.exists(os.path.dirname(plot_filename)):
os.mkdir(os.path.dirname(plot_filename))
if os.path.exists(pickle_filename) and overwrite:
logger.debug("Data file exists, but you want to overwrite it!")
os.remove(pickle_filename)
logger.debug("Data file deleted...")
# If the cache pickle file doesn't exist, generate the data
if not os.path.exists(pickle_filename):
logger.info("Data file {} not found. Generating data...".format(pickle_filename))
# Initialize my PDB statistic dictionary
# I use a dictionary here because after doing some sub-selection the index arrays may
# have difference lengths.
pdb_statistics = dict()
for index in indices:
pdb_statistics[index] = np.array([])
for ccd in field.ccds.values():
print "Starting with CCD {}".format(ccd.id)
chip = ccd.read()
pdb_statistics_array = []
logger.info("Starting microlensing event simulations")
# Keep track of how many light curves we've used, break after we reach the specified number
light_curve_count = 0
for source in chip.sources.where("(ngoodobs > {})".format(min_number_of_good_observations)):
source_id = source["matchedSourceID"]
light_curve = ccd.light_curve(source_id, barebones=True, clean=True)
if len(light_curve.mjd) < min_number_of_good_observations:
continue
# Add the pre-simulation statistics to an array
lc_var_indices = pa.compute_variability_indices(light_curve, indices, return_tuple=True)
pdb_statistics_array.append(lc_var_indices)
one_light_curve_statistics = vi.simulate_events_compute_indices(light_curve, events_per_light_curve=number_of_microlensing_simulations_per_light_curve, indices=indices)
try:
simulated_microlensing_statistics = np.hstack((simulated_microlensing_statistics, one_light_curve_statistics))
except NameError:
simulated_microlensing_statistics = one_light_curve_statistics
light_curve_count += 1
if light_curve_count >= number_of_microlensing_light_curves:
break
pdb_statistics_array = np.array(pdb_statistics_array, dtype=[(index,float) for index in indices])
try:
all_pdb_statistics_array = np.hstack((all_pdb_statistics_array, pdb_statistics_array))
except NameError:
all_pdb_statistics_array = pdb_statistics_array
ccd.close()
f = open(pickle_filename, "w")
pickle.dump((all_pdb_statistics_array, simulated_microlensing_statistics), f)
f.close()
f = open(pickle_filename, "r")
all_pdb_statistics_array, simulated_microlensing_statistics = pickle.load(f)
f.close()
selection_criteria = {
"eta" : 0.16167735855516213,
"delta_chi_squared" : 1.162994709319348,
"j" : 1.601729135628142
}
index_pairs = [("eta", "delta_chi_squared"), ("eta", "j"), ("delta_chi_squared", "j")]
nbins = 100
for x_index, y_index in index_pairs:
fig, axes = plt.subplots(1, 2, sharey=True, figsize=(15,7.5))
# Variable data
x = simulated_microlensing_statistics[x_index]
y = simulated_microlensing_statistics[y_index]
pos_x = x[(x > 0) & (y > 0)]
pos_y = y[(x > 0) & (y > 0)]
xbins_pos = np.logspace(np.log10(pos_x.min()), np.log10(pos_x.max()), nbins)
ybins_pos = np.logspace(np.log10(pos_y.min()), np.log10(pos_y.max()), nbins)
#print pos_x, pos_y, xbins_pos, ybins_pos
H_pos, xedges_pos, yedges_pos = np.histogram2d(pos_x, pos_y, bins=[xbins_pos, ybins_pos])
# Non-variable data
x = all_pdb_statistics_array[x_index]
y = all_pdb_statistics_array[y_index]
pos_x = x[(x > 0) & (y > 0)]
pos_y = y[(x > 0) & (y > 0)]
H_pos_boring, xedges_pos, yedges_pos = np.histogram2d(pos_x, pos_y, bins=[xedges_pos, yedges_pos])
ax1 = axes[1]
#ax1.imshow(np.log10(H), interpolation="none", cmap=cm.gist_heat)
ax1.pcolormesh(xedges_pos, yedges_pos, np.where(H_pos > 0, np.log10(H_pos), 0.).T, cmap=cm.Blues)
ax1.set_xscale("log")
ax1.set_yscale("log")
ax1.set_xlim(xedges_pos[0], xedges_pos[-1])
ax1.set_ylim(yedges_pos[0], yedges_pos[-1])
ax1.set_xlabel(pu.index_to_label(x_index), fontsize=28)
ax1.axhline(10.**selection_criteria[y_index], color='r', linestyle='--')
ax1.axvline(10.**selection_criteria[x_index], color='r', linestyle='--')
if x_index == "eta":
ax1.fill_between([xedges_pos[0], 10.**selection_criteria[x_index]], 10.**selection_criteria[y_index], yedges_pos[-1], facecolor='red', alpha=0.1)
elif x_index == "delta_chi_squared":
ax1.fill_between([10.**selection_criteria[x_index], xedges_pos[-1]], 10.**selection_criteria[y_index], yedges_pos[-1], facecolor='red', alpha=0.1)
ax2 = axes[0]
ax2.pcolormesh(xedges_pos, yedges_pos, np.where(H_pos_boring > 0, np.log10(H_pos_boring), 0.).T, cmap=cm.Blues)
ax2.set_xscale("log")
ax2.set_yscale("log")
ax2.set_xlim(xedges_pos[0], xedges_pos[-1])
ax2.set_ylim(yedges_pos[0], yedges_pos[-1])
ax2.set_xlabel(pu.index_to_label(x_index), fontsize=28)
ax2.set_ylabel(pu.index_to_label(y_index), fontsize=28)
ax2.axhline(10.**selection_criteria[y_index], color='r', linestyle='--')
ax2.axvline(10.**selection_criteria[x_index], color='r', linestyle='--')
if x_index == "eta":
ax2.fill_between([xedges_pos[0], 10.**selection_criteria[x_index]], 10.**selection_criteria[y_index], yedges_pos[-1], facecolor='red', alpha=0.1)
elif x_index == "delta_chi_squared":
ax2.fill_between([10.**selection_criteria[x_index], xedges_pos[-1]], 10.**selection_criteria[y_index], yedges_pos[-1], facecolor='red', alpha=0.1)
for ax in fig.axes:
for ticklabel in ax.get_xticklabels()+ax.get_yticklabels():
ticklabel.set_fontsize(18)
fig.savefig(os.path.join(pg.plots_path, "paper_figures", "{}_vs_{}.pdf".format(x_index, y_index)), bbox_inches="tight")
def maximum_outlier_indices_plot(field_id):
""" Same as above function, but uses variability indices computed by the photometric database instead of my own """
indices = ["eta", "delta_chi_squared", "j", "k", "sigma_mu", "con"]
min_max = {"eta" : "min",
"delta_chi_squared" : "max",
"j" : "max",
"k" : "min",
"sigma_mu": "max",
"con" : "max"}
pdb_indices = ["bestVonNeumannRatio", "bestChiSQ", "bestStetsonJ", "bestStetsonK", ["bestMagRMS", "bestMedianMag"], "bestCon"]
field = pdb.Field(field_id, "R")
fig, axes = plt.subplots(len(indices), 1, sharex=True, figsize=(15,20))
for ii, (index, pdb_index) in enumerate(zip(indices, pdb_indices)):
all_outlier_sources = []
all_ccds = []
for ccd in field.ccds.values():
chip = ccd.read()
if index == "sigma_mu":
sources = chip.sources.readWhere("(nbestobs > 100) & ({} != 0)".format(pdb_index[0]))
else:
sources = chip.sources.readWhere("(nbestobs > 100) & ({} != 0)".format(pdb_index))
if len(sources) == 0:
logger.debug("Skipping CCD {}".format(ccd.id))
continue
if index == "sigma_mu":
sigma_mus = sources[pdb_index[0]] / sources[pdb_index[1]]
if min_max[index] == "min":
w, = np.where(sigma_mus == sigma_mus.min())
best_outlier_source = sources[w][sources[w]["ngoodobs"].argmax()]
elif min_max[index] == "max":
w, = np.where(sigma_mus == sigma_mus.max())
best_outlier_source = sources[w][sources[w]["ngoodobs"].argmax()]
else:
if min_max[index] == "min":
w, = np.where(sources[pdb_index] == sources[pdb_index].min())
best_outlier_source = sources[w][sources[w]["ngoodobs"].argmax()]
elif min_max[index] == "max":
w, = np.where(sources[pdb_index] == sources[pdb_index].max())
best_outlier_source = sources[w][sources[w]["ngoodobs"].argmax()]
all_outlier_sources.append(best_outlier_source)
all_ccds.append(ccd)
all_outlier_sources = np.array(all_outlier_sources, dtype=sources.dtype)
if index == "sigma_mu":
clean = False
idx_vals = all_outlier_sources[pdb_index[0]] / all_outlier_sources[pdb_index[1]]
else:
clean = True
idx_vals = all_outlier_sources[pdb_index]
while True:
if min_max[index] == "min":
best_outlier_source = all_outlier_sources[idx_vals.argmin()]
best_outlier_lightcurve = all_ccds[idx_vals.argmin()].light_curve(best_outlier_source["matchedSourceID"], clean=clean, barebones=True)
elif min_max[index] == "max":
best_outlier_source = all_outlier_sources[idx_vals.argmax()]
best_outlier_lightcurve = all_ccds[idx_vals.argmax()].light_curve(best_outlier_source["matchedSourceID"], clean=clean, barebones=True)
sig,md = np.std(best_outlier_lightcurve.mag), np.median(best_outlier_lightcurve.mag)
ix = (best_outlier_lightcurve.mag < (md+8*sig)) & (best_outlier_lightcurve.mag > (md-8*sig))
best_outlier_lightcurve = PTFLightCurve(best_outlier_lightcurve.mjd[ix],
best_outlier_lightcurve.mag[ix],
best_outlier_lightcurve.error[ix])
try:
best_outlier_lightcurve.plot(axes[ii], ms=4)
break
except ValueError:
if min_max[index] == "min":
idx_vals[idx_vals.argmin()] = 1E8
elif min_max[index] == "max":
idx_vals[idx_vals.argmax()] = -1E8
#axes[ii].set_xlim(best_outlier_lightcurve.mjd.min()-2, best_outlier_lightcurve.mjd.max()+2)
#axes[ii, 1].set_xlim(55350, 55600)
axes[ii].set_ylabel("$R$ [mag]", fontsize=26)
yticks = np.linspace(min(axes[ii].get_yticks()), max(axes[ii].get_yticks()), 5)[1:-1]
axes[ii].set_yticks([round(yt,2) for yt in yticks])
for ticklabel in axes[ii].get_yticklabels():
ticklabel.set_fontsize(22)
bbox_props = dict(boxstyle="round,pad=0.3", fc="white", ec="k", lw=1)
axes[ii].text(55975, yticks[2], "{0}({1})".format(min_max[index], pu.index_to_label(index)),
fontsize=30,
bbox=bbox_props)
for ax in fig.axes[:-1]:
ax.xaxis.set_visible(False)
#ax.yaxis.set_visible(False)
#fig.axes[-1].yaxis.set_visible(False)
for ticklabel in fig.axes[-1].get_xticklabels():
ticklabel.set_fontsize(22)
xticks = fig.axes[-1].get_xticks()
axes[-1].set_xticklabels(["{0:d}".format(int(x-min(xticks))) for x in xticks])
axes[-1].set_xlabel("time [days]", fontsize=26)
fig.subplots_adjust(hspace=0.0, top=0.95, bottom=0.08)
fig.savefig(os.path.join(pg.plots_path, "paper_figures", "max_outlier_light_curves{0}.pdf".format(field_id)))#, bbox_inches="tight")