def get_inst_combos(db_session, inst_list, time_min, time_max):
"""
function to create instrument based lists of combo queries for image pre-processing
@param db_session: database session to query image combos from
@param inst_list: list of instruments
@param time_min: minimum query time for image combos
@param time_max: maximum query time for image combos
@return: inst list of lbc combo queries, inst list of iit combo queries
"""
start = time.time()
print("Querying Combo IDs from the database. This only needs to be done once.")
# query for combo ids within date range
lbc_combo_query = [None] * len(inst_list)
iit_combo_query = [None] * len(inst_list)
for inst_index, instrument in enumerate(inst_list):
lbc_combo = db_funcs.query_inst_combo(db_session, time_min - datetime.timedelta(days=180),
time_max + datetime.timedelta(days=180),
meth_name='LBCC', instrument=instrument)
iit_combo = db_funcs.query_inst_combo(db_session, time_min - datetime.timedelta(days=180),
time_max + datetime.timedelta(days=180), meth_name='IIT',
instrument=instrument)
lbc_combo_query[inst_index] = lbc_combo
iit_combo_query[inst_index] = iit_combo
print("Combo IDs have been queried for", instrument)
end = time.time()
print("Combo IDs have been queried from the database in", end - start, "seconds.")
return lbc_combo_query, iit_combo_query
v_cmap = cm.get_cmap('viridis')
n_mu = len(sample_mu)
color_dist = np.linspace(0., 1., n_mu)
linestyles = ['dashed']
marker_types = ['None']
meth_name = 'LBCC'
for inst_index, instrument in enumerate(inst_list):
print("Generating plots for " + instrument + ".")
# query theoretic parameters
theoretic_query = np.zeros((len(moving_avg_centers), 6))
theoretic_query2 = np.empty((len(moving_avg_centers), 6))
theoretic_query2.fill(np.nan)
# query correct image combos
combo_query = query_inst_combo(db_session, query_time_min, query_time_max,
meth_name, instrument)
plot_beta = np.zeros((sample_mu.__len__(), moving_avg_centers.__len__()))
plot_beta2 = np.zeros((sample_mu.__len__(), moving_avg_centers.__len__()))
plot_y = np.zeros((sample_mu.__len__(), moving_avg_centers.__len__()))
plot_y2 = np.zeros((sample_mu.__len__(), moving_avg_centers.__len__()))
for date_index, center_date in enumerate(moving_avg_centers):
# query for variable value
temp_vals = query_var_val(db_session,
meth_name,
date_obs=np.datetime64(center_date).astype(
datetime.datetime),
inst_combo_query=combo_query)
theoretic_query[date_index, :] = temp_vals
if (np.isnan(temp_vals).all()
or center_date < combo_query.date_mean.min()
or center_date > combo_query.date_mean.max()):
#### QUERY IMAGES ####
query_instrument = [
instrument,
]
rot_images = db_funcs.query_euv_images_rot(db_session,
rot_min=rot_min,
rot_max=rot_max,
instrument=query_instrument)
image_pd = rot_images.sort_values(by=['cr_rot'])
# get time minimum and maximum for instrument
inst_time_min = rot_images.date_obs.min()
inst_time_max = rot_images.date_obs.max()
# query correct image combos
lbc_meth_name = "LBCC"
combo_query_lbc = db_funcs.query_inst_combo(db_session, inst_time_min,
inst_time_max, lbc_meth_name,
instrument)
iit_meth_name = "IIT"
combo_query_iit = db_funcs.query_inst_combo(db_session, inst_time_min,
inst_time_max, iit_meth_name,
instrument)
# query correct image combos
combo_query_lbc = db_funcs.query_inst_combo(db_session,
hist_query_time_min,
hist_query_time_max,
meth_name="LBCC",
instrument=instrument)
# query correct image combos
combo_query_iit = db_funcs.query_inst_combo(db_session,
hist_query_time_min,
hist_query_time_max,
defs_columns = []
for column in db_class.Var_Defs.__table__.columns:
defs_columns.append(column.key)
df_cols = set().union(meth_columns, defs_columns, ("var_val", ))
methods_template = pd.DataFrame(data=None, columns=df_cols)
# generate a list of methods dataframes
methods_list = [methods_template] * query_pd.__len__()
# --- 2. Apply pre-processing corrections ------------------------------------------
for inst_index, instrument in enumerate(inst_list):
print("Starting corrections for", instrument, "images.\n")
# query correct image combos
lbc_combo_query = db_funcs.query_inst_combo(
db_session,
query_time_min - datetime.timedelta(days=7),
query_time_max + datetime.timedelta(days=7),
meth_name='LBCC Theoretic',
instrument=instrument)
iit_combo_query = db_funcs.query_inst_combo(
db_session,
query_time_min - datetime.timedelta(days=7),
query_time_max + datetime.timedelta(days=7),
meth_name='IIT',
instrument=instrument)
# create dataframe for instrument
hist_inst = query_pd['instrument']
instrument_pd = query_pd[hist_inst == instrument]
for index, row in instrument_pd.iterrows():
print("Processing image number", row.data_id,
# method information
meth_name = "LBCC"
##### QUERY IMAGES ######
for inst_index, instrument in enumerate(inst_list):
query_instrument = [
instrument,
]
image_pd = query_euv_images(db_session=db_session,
time_min=lbc_query_time_min,
time_max=lbc_query_time_max,
instrument=query_instrument)
# query correct image combos
combo_query = query_inst_combo(
db_session, lbc_query_time_min - datetime.timedelta(weeks=2),
lbc_query_time_max + datetime.timedelta(weeks=2), meth_name,
instrument)
###### GET LOS IMAGES COORDINATES (DATA) #####
# apply LBC
for index in range(n_images_plot):
row = image_pd.iloc[index]
print("Processing image number", row.data_id, ".")
if row.fname_hdf == "":
print("Warning: Image # " + str(row.data_id) +
" does not have an associated hdf file. Skipping")
continue
hdf_path = os.path.join(hdf_data_dir, row.fname_hdf)
original_los = psi_d_types.read_los_image(hdf_path)
original_los.get_coordinates(R0=R0)
theoretic_query = query_var_val(
db_class.EUV_Images.instrument == instrument).all()
# extract datetime from weird sqlalchemy construct
min_date = min_date_query[0][0]
# set lower trim max date
lower_trim_max = min_date + trim_window
# determine minimum image date
max_date_query = db_session.query(func.max(db_class.EUV_Images.date_obs)).filter(
db_class.EUV_Images.instrument == instrument).all()
# extract datetime from weird sqlalchemy construct
max_date = max_date_query[0][0]
# set lower trim max date
upper_trim_min = max_date - trim_window
# delete LBCCs before lower_trim_max ------------
del_combos = query_inst_combo(db_session, query_time_min=datetime.datetime(2000, 1, 1, 0, 0, 0),
query_time_max=lower_trim_max, meth_name=meth_name, instrument=instrument)
if del_combos.shape[0] > 0:
print("Trimming early LBCC values for ", instrument)
# First delete the variable values
nvals = db_session.query(db_class.Var_Vals).filter(
db_class.Var_Vals.combo_id.in_(del_combos.combo_id.to_list())
).delete(synchronize_session=False)
# Then delete from image_combo_assoc
nassoc = db_session.query(db_class.Data_Combo_Assoc).filter(
db_class.Data_Combo_Assoc.combo_id.in_(del_combos.combo_id.to_list())
).delete(synchronize_session=False)
# Last delete from image_combos
ncombos = db_session.query(db_class.Data_Combos).filter(
db_class.Data_Combos.combo_id.in_(del_combos.combo_id.to_list())
).delete(synchronize_session=False)
# for instrument in inst_list:
instrument = inst_list[1]
print("Calculation for instrument:", instrument)
# query EUV images
query_instrument = [
instrument,
]
image_pd = db_funcs.query_euv_images(db_session=db_session,
time_min=lbc_query_time_min,
time_max=lbc_query_time_max,
instrument=query_instrument)
# query correct image combos
combo_query = db_funcs.query_inst_combo(db_session,
lbc_query_time_min,
lbc_query_time_max,
meth_name="LBCC",
instrument=instrument)
# apply LBC
# for index, row in image_pd.iterrows():
index = 0
row = image_pd.iloc[index]
print("Calculating IIT histogram at time:", row.date_obs)
original_los, lbcc_image, mu_indices, use_indices, theoretic_query = lbcc_funcs.apply_lbc(
db_session,
hdf_data_dir,
combo_query,
image_row=row,
n_intensity_bins=n_intensity_bins,
R0=R0)
def plot_iit_histograms(db_session,
hdf_data_dir,
hist_query_time_min,
hist_query_time_max,
inst_list,
ref_inst,
n_intensity_bins=200,
lat_band=[-np.pi / 2.4, np.pi / 2.4],
R0=1.01,
log10=True):
# start time
start_time = time.time()
#### GET REFERENCE INFO FOR LATER USE ####
# get index number of reference instrument
ref_index = inst_list.index(ref_inst)
# query euv images to get carrington rotation range
ref_instrument = [
ref_inst,
]
euv_images = db_funcs.query_euv_images(db_session,
time_min=hist_query_time_min,
time_max=hist_query_time_max,
instrument=ref_instrument)
# get min and max carrington rotation
rot_max = euv_images.cr_rot.max()
rot_min = euv_images.cr_rot.min()
# method information
meth_name = "IIT"
method_id = db_funcs.get_method_id(db_session,
meth_name,
meth_desc=None,
var_names=None,
var_descs=None,
create=False)
# query for IIT histograms
pd_lbc_hist = db_funcs.query_hist(db_session=db_session,
meth_id=method_id[1],
n_intensity_bins=n_intensity_bins,
lat_band=lat_band,
time_min=hist_query_time_min,
time_max=hist_query_time_max)
pd_lbc_hist_srt = pd_lbc_hist.sort_values(by=['image_id'])
# convert the binary types back to arrays
mu_bin_edges, intensity_bin_edges, full_lbc_hist = psi_d_types.binary_to_hist(
pd_lbc_hist_srt, n_mu_bins=None, n_intensity_bins=n_intensity_bins)
# create corrected/original histograms
original_hist_list = np.full(full_lbc_hist.shape, 0, dtype=np.int64)
corrected_hist_list = np.full(full_lbc_hist.shape, 0, dtype=np.int64)
for inst_index, instrument in enumerate(inst_list):
print("Applying corrections for", instrument)
#### QUERY IMAGES ####
query_instrument = [
instrument,
]
rot_images = db_funcs.query_euv_images_rot(db_session,
rot_min=rot_min,
rot_max=rot_max,
instrument=query_instrument)
image_pd = rot_images.sort_values(by=['cr_rot'])
# get time minimum and maximum for instrument
inst_time_min = rot_images.date_obs.min()
inst_time_max = rot_images.date_obs.max()
# query correct image combos
lbc_meth_name = "LBCC"
combo_query_lbc = db_funcs.query_inst_combo(db_session, inst_time_min,
inst_time_max,
lbc_meth_name, instrument)
iit_meth_name = "IIT"
combo_query_iit = db_funcs.query_inst_combo(db_session, inst_time_min,
inst_time_max,
iit_meth_name, instrument)
# query correct image combos
combo_query_lbc = db_funcs.query_inst_combo(db_session,
hist_query_time_min,
hist_query_time_max,
meth_name="LBCC",
instrument=instrument)
# query correct image combos
combo_query_iit = db_funcs.query_inst_combo(db_session,
hist_query_time_min,
hist_query_time_max,
meth_name="IIT",
instrument=instrument)
for index, row in image_pd.iterrows():
# apply LBC
original_los, lbcc_image, mu_indices, use_indices, theoretic_query = lbcc_funcs.apply_lbc(
db_session,
hdf_data_dir,
combo_query_lbc,
image_row=row,
n_intensity_bins=n_intensity_bins,
R0=R0)
#### ORIGINAL LOS DATA ####
# calculate IIT histogram from original data
original_los_hist = psi_d_types.LosImage.iit_hist(
original_los, intensity_bin_edges, lat_band, log10)
# add 1D histogram to array
original_hist_list[:, index] = original_los_hist
#### CORRECTED DATA ####
# apply IIT correction
lbcc_image, iit_image, use_indices, alpha, x = iit_funcs.apply_iit(
db_session,
combo_query_iit,
lbcc_image,
use_indices,
original_los,
R0=R0)
#### CREATE CORRECTED IIT HISTOGRAM #####
# calculate IIT histogram from LBC
hist_iit = psi_d_types.IITImage.iit_hist(iit_image, lat_band,
log10)
# create IIT histogram datatype
corrected_hist = psi_d_types.create_iit_hist(
iit_image, method_id[1], lat_band, hist_iit)
corrected_hist_list[:, index] = corrected_hist.hist
# plotting definitions
color_list = ['red', 'blue', 'black']
linestyle_list = ['solid', 'dashed', 'dashdot']
#### CREATE NEW HISTOGRAM ####
for inst_index, instrument in enumerate(inst_list):
print("Plotting Histograms for", instrument)
#### GET INDICES TO USE ####
# get index of instrument in histogram dataframe
hist_inst = pd_lbc_hist_srt['instrument']
pd_inst_index = hist_inst[hist_inst == instrument].index
#### ORIGINAL HISTOGRAM #####
# define histogram
original_hist = original_hist_list[:, pd_inst_index].sum(axis=1)
# normalize histogram
row_sums = original_hist.sum(axis=0, keepdims=True)
norm_original_hist = original_hist / row_sums
# plot original
Plotting.Plot1d_Hist(norm_original_hist,
instrument,
inst_index,
intensity_bin_edges,
color_list,
linestyle_list,
figure=100,
xlabel="Intensity (log10)",
ylabel="H(I)",
title="Histogram: Original LOS Data")
#### LBCC HISTOGRAM #####
# define histogram
lbc_hist = full_lbc_hist[:, pd_inst_index].sum(axis=1)
# normalize histogram
lbc_sums = lbc_hist.sum(axis=0, keepdims=True)
norm_lbc_hist = lbc_hist / lbc_sums
# plot lbcc
Plotting.Plot1d_Hist(norm_lbc_hist,
instrument,
inst_index,
intensity_bin_edges,
color_list,
linestyle_list,
figure=200,
xlabel="Intensity (log10)",
ylabel="H(I)",
title="Histogram: Post LBCC")
#### CORRECTED HISTOGRAM ####
# define histogram
corrected_hist = corrected_hist_list[:, pd_inst_index].sum(axis=1)
# normalize histogram
iit_sums = corrected_hist.sum(axis=0, keepdims=True)
norm_corrected_hist = corrected_hist / iit_sums
# plot corrected
Plotting.Plot1d_Hist(norm_corrected_hist,
instrument,
inst_index,
intensity_bin_edges,
color_list,
linestyle_list,
figure=300,
xlabel="Intensity (log10)",
ylabel="H(I)",
title="Histogram: Post IIT")
# end time
end_time = time.time()
print("ITT has been applied and original/resulting histograms plotted.")
print("Total elapsed time to apply correction and plot histograms: " +
str(round(end_time - start_time, 3)) + " seconds.")
return None
def apply_iit_correction(db_session,
hdf_data_dir,
iit_query_time_min,
iit_query_time_max,
inst_list,
ref_inst,
n_intensity_bins=200,
R0=1.01,
n_images_plot=1,
plot=False):
# start time
start_time = time.time()
#### GET REFERENCE INFO FOR LATER USE ####
# query euv images to get carrington rotation range
ref_instrument = [
ref_inst,
]
euv_images = db_funcs.query_euv_images(db_session,
time_min=iit_query_time_min,
time_max=iit_query_time_max,
instrument=ref_instrument)
# get min and max carrington rotation
rot_max = euv_images.cr_rot.max()
rot_min = euv_images.cr_rot.min()
for inst_index, instrument in enumerate(inst_list):
#### QUERY IMAGES ####
query_instrument = [
instrument,
]
rot_images = db_funcs.query_euv_images_rot(db_session,
rot_min=rot_min,
rot_max=rot_max,
instrument=query_instrument)
image_pd = rot_images.sort_values(by=['cr_rot'])
# get time minimum and maximum for instrument
inst_time_min = rot_images.date_obs.min()
inst_time_max = rot_images.date_obs.max()
# query correct image combos
lbc_meth_name = "LBCC"
combo_query_lbc = db_funcs.query_inst_combo(db_session, inst_time_min,
inst_time_max,
lbc_meth_name, instrument)
iit_meth_name = "IIT"
combo_query_iit = db_funcs.query_inst_combo(db_session, inst_time_min,
inst_time_max,
iit_meth_name, instrument)
# apply LBC
for index in range(n_images_plot):
row = image_pd.iloc[index]
print("Processing image number", row.data_id,
"for IIT Correction.")
#### APPLY LBC CORRECTION #####
original_los, lbcc_image, mu_indices, use_indices, theoretic_query = lbcc_funcs.apply_lbc(
db_session,
hdf_data_dir,
combo_query_lbc,
image_row=row,
n_intensity_bins=n_intensity_bins,
R0=R0)
#### APPLY IIT CORRECTION ####
lbcc_image, iit_image, use_indices, alpha, x = apply_iit(
db_session,
combo_query_iit,
lbcc_image,
use_indices,
original_los,
R0=R0)
if plot:
lbcc_data = lbcc_image.lbcc_data
corrected_iit_data = iit_image.iit_data
# plot LBC image
Plotting.PlotCorrectedImage(lbcc_data,
los_image=original_los,
nfig=100 + inst_index * 10 + index,
title="Corrected LBCC Image for " +
instrument)
# plot IIT image
Plotting.PlotCorrectedImage(corrected_iit_data,
los_image=original_los,
nfig=200 + inst_index * 10 + index,
title="Corrected IIT Image for " +
instrument)
# plot difference
Plotting.PlotCorrectedImage(corrected_iit_data - lbcc_data,
los_image=original_los,
nfig=300 + inst_index * 10 + index,
title="Difference Plot for " +
instrument)
# end time
end_time = time.time()
print("ITT has been applied and specified images plotted.")
print("Total elapsed time to apply correction and plot: " +
str(round(end_time - start_time, 3)) + " seconds.")
return None
#### QUERY IMAGES ####
query_instrument = [
instrument,
]
rot_images = db_funcs.query_euv_images_rot(db_session,
rot_min=rot_min,
rot_max=rot_max,
instrument=query_instrument)
image_pd = rot_images.sort_values(by=['cr_rot'])
# get time minimum and maximum for instrument
inst_time_min = rot_images.date_obs.min()
inst_time_max = rot_images.date_obs.max()
# query correct image combos
lbc_meth_name = "LBCC"
combo_query_lbc = db_funcs.query_inst_combo(db_session, inst_time_min,
inst_time_max, lbc_meth_name,
instrument)
iit_meth_name = "IIT"
combo_query_iit = db_funcs.query_inst_combo(db_session, inst_time_min,
inst_time_max, iit_meth_name,
instrument)
# apply LBC
for index in range(n_images_plot):
row = image_pd.iloc[index]
#### APPLY LBC CORRECTION #####
#### APPLY LBC CORRECTION #####
original_los, lbcc_image, mu_indices, use_indices, theoretic_query = lbcc_funcs.apply_lbc(
db_session,
hdf_data_dir,
combo_query_lbc,
image_row=row,