def get_inst_combos(db_session, inst_list, time_min, time_max): """ function to create instrument based lists of combo queries for image pre-processing @param db_session: database session to query image combos from @param inst_list: list of instruments @param time_min: minimum query time for image combos @param time_max: maximum query time for image combos @return: inst list of lbc combo queries, inst list of iit combo queries """ start = time.time() print("Querying Combo IDs from the database. This only needs to be done once.") # query for combo ids within date range lbc_combo_query = [None] * len(inst_list) iit_combo_query = [None] * len(inst_list) for inst_index, instrument in enumerate(inst_list): lbc_combo = db_funcs.query_inst_combo(db_session, time_min - datetime.timedelta(days=180), time_max + datetime.timedelta(days=180), meth_name='LBCC', instrument=instrument) iit_combo = db_funcs.query_inst_combo(db_session, time_min - datetime.timedelta(days=180), time_max + datetime.timedelta(days=180), meth_name='IIT', instrument=instrument) lbc_combo_query[inst_index] = lbc_combo iit_combo_query[inst_index] = iit_combo print("Combo IDs have been queried for", instrument) end = time.time() print("Combo IDs have been queried from the database in", end - start, "seconds.") return lbc_combo_query, iit_combo_query
v_cmap = cm.get_cmap('viridis') n_mu = len(sample_mu) color_dist = np.linspace(0., 1., n_mu) linestyles = ['dashed'] marker_types = ['None'] meth_name = 'LBCC' for inst_index, instrument in enumerate(inst_list): print("Generating plots for " + instrument + ".") # query theoretic parameters theoretic_query = np.zeros((len(moving_avg_centers), 6)) theoretic_query2 = np.empty((len(moving_avg_centers), 6)) theoretic_query2.fill(np.nan) # query correct image combos combo_query = query_inst_combo(db_session, query_time_min, query_time_max, meth_name, instrument) plot_beta = np.zeros((sample_mu.__len__(), moving_avg_centers.__len__())) plot_beta2 = np.zeros((sample_mu.__len__(), moving_avg_centers.__len__())) plot_y = np.zeros((sample_mu.__len__(), moving_avg_centers.__len__())) plot_y2 = np.zeros((sample_mu.__len__(), moving_avg_centers.__len__())) for date_index, center_date in enumerate(moving_avg_centers): # query for variable value temp_vals = query_var_val(db_session, meth_name, date_obs=np.datetime64(center_date).astype( datetime.datetime), inst_combo_query=combo_query) theoretic_query[date_index, :] = temp_vals if (np.isnan(temp_vals).all() or center_date < combo_query.date_mean.min() or center_date > combo_query.date_mean.max()):
#### QUERY IMAGES #### query_instrument = [ instrument, ] rot_images = db_funcs.query_euv_images_rot(db_session, rot_min=rot_min, rot_max=rot_max, instrument=query_instrument) image_pd = rot_images.sort_values(by=['cr_rot']) # get time minimum and maximum for instrument inst_time_min = rot_images.date_obs.min() inst_time_max = rot_images.date_obs.max() # query correct image combos lbc_meth_name = "LBCC" combo_query_lbc = db_funcs.query_inst_combo(db_session, inst_time_min, inst_time_max, lbc_meth_name, instrument) iit_meth_name = "IIT" combo_query_iit = db_funcs.query_inst_combo(db_session, inst_time_min, inst_time_max, iit_meth_name, instrument) # query correct image combos combo_query_lbc = db_funcs.query_inst_combo(db_session, hist_query_time_min, hist_query_time_max, meth_name="LBCC", instrument=instrument) # query correct image combos combo_query_iit = db_funcs.query_inst_combo(db_session, hist_query_time_min, hist_query_time_max,
defs_columns = [] for column in db_class.Var_Defs.__table__.columns: defs_columns.append(column.key) df_cols = set().union(meth_columns, defs_columns, ("var_val", )) methods_template = pd.DataFrame(data=None, columns=df_cols) # generate a list of methods dataframes methods_list = [methods_template] * query_pd.__len__() # --- 2. Apply pre-processing corrections ------------------------------------------ for inst_index, instrument in enumerate(inst_list): print("Starting corrections for", instrument, "images.\n") # query correct image combos lbc_combo_query = db_funcs.query_inst_combo( db_session, query_time_min - datetime.timedelta(days=7), query_time_max + datetime.timedelta(days=7), meth_name='LBCC Theoretic', instrument=instrument) iit_combo_query = db_funcs.query_inst_combo( db_session, query_time_min - datetime.timedelta(days=7), query_time_max + datetime.timedelta(days=7), meth_name='IIT', instrument=instrument) # create dataframe for instrument hist_inst = query_pd['instrument'] instrument_pd = query_pd[hist_inst == instrument] for index, row in instrument_pd.iterrows(): print("Processing image number", row.data_id,
# method information meth_name = "LBCC" ##### QUERY IMAGES ###### for inst_index, instrument in enumerate(inst_list): query_instrument = [ instrument, ] image_pd = query_euv_images(db_session=db_session, time_min=lbc_query_time_min, time_max=lbc_query_time_max, instrument=query_instrument) # query correct image combos combo_query = query_inst_combo( db_session, lbc_query_time_min - datetime.timedelta(weeks=2), lbc_query_time_max + datetime.timedelta(weeks=2), meth_name, instrument) ###### GET LOS IMAGES COORDINATES (DATA) ##### # apply LBC for index in range(n_images_plot): row = image_pd.iloc[index] print("Processing image number", row.data_id, ".") if row.fname_hdf == "": print("Warning: Image # " + str(row.data_id) + " does not have an associated hdf file. Skipping") continue hdf_path = os.path.join(hdf_data_dir, row.fname_hdf) original_los = psi_d_types.read_los_image(hdf_path) original_los.get_coordinates(R0=R0) theoretic_query = query_var_val(
db_class.EUV_Images.instrument == instrument).all() # extract datetime from weird sqlalchemy construct min_date = min_date_query[0][0] # set lower trim max date lower_trim_max = min_date + trim_window # determine minimum image date max_date_query = db_session.query(func.max(db_class.EUV_Images.date_obs)).filter( db_class.EUV_Images.instrument == instrument).all() # extract datetime from weird sqlalchemy construct max_date = max_date_query[0][0] # set lower trim max date upper_trim_min = max_date - trim_window # delete LBCCs before lower_trim_max ------------ del_combos = query_inst_combo(db_session, query_time_min=datetime.datetime(2000, 1, 1, 0, 0, 0), query_time_max=lower_trim_max, meth_name=meth_name, instrument=instrument) if del_combos.shape[0] > 0: print("Trimming early LBCC values for ", instrument) # First delete the variable values nvals = db_session.query(db_class.Var_Vals).filter( db_class.Var_Vals.combo_id.in_(del_combos.combo_id.to_list()) ).delete(synchronize_session=False) # Then delete from image_combo_assoc nassoc = db_session.query(db_class.Data_Combo_Assoc).filter( db_class.Data_Combo_Assoc.combo_id.in_(del_combos.combo_id.to_list()) ).delete(synchronize_session=False) # Last delete from image_combos ncombos = db_session.query(db_class.Data_Combos).filter( db_class.Data_Combos.combo_id.in_(del_combos.combo_id.to_list()) ).delete(synchronize_session=False)
# for instrument in inst_list: instrument = inst_list[1] print("Calculation for instrument:", instrument) # query EUV images query_instrument = [ instrument, ] image_pd = db_funcs.query_euv_images(db_session=db_session, time_min=lbc_query_time_min, time_max=lbc_query_time_max, instrument=query_instrument) # query correct image combos combo_query = db_funcs.query_inst_combo(db_session, lbc_query_time_min, lbc_query_time_max, meth_name="LBCC", instrument=instrument) # apply LBC # for index, row in image_pd.iterrows(): index = 0 row = image_pd.iloc[index] print("Calculating IIT histogram at time:", row.date_obs) original_los, lbcc_image, mu_indices, use_indices, theoretic_query = lbcc_funcs.apply_lbc( db_session, hdf_data_dir, combo_query, image_row=row, n_intensity_bins=n_intensity_bins, R0=R0)
def plot_iit_histograms(db_session, hdf_data_dir, hist_query_time_min, hist_query_time_max, inst_list, ref_inst, n_intensity_bins=200, lat_band=[-np.pi / 2.4, np.pi / 2.4], R0=1.01, log10=True): # start time start_time = time.time() #### GET REFERENCE INFO FOR LATER USE #### # get index number of reference instrument ref_index = inst_list.index(ref_inst) # query euv images to get carrington rotation range ref_instrument = [ ref_inst, ] euv_images = db_funcs.query_euv_images(db_session, time_min=hist_query_time_min, time_max=hist_query_time_max, instrument=ref_instrument) # get min and max carrington rotation rot_max = euv_images.cr_rot.max() rot_min = euv_images.cr_rot.min() # method information meth_name = "IIT" method_id = db_funcs.get_method_id(db_session, meth_name, meth_desc=None, var_names=None, var_descs=None, create=False) # query for IIT histograms pd_lbc_hist = db_funcs.query_hist(db_session=db_session, meth_id=method_id[1], n_intensity_bins=n_intensity_bins, lat_band=lat_band, time_min=hist_query_time_min, time_max=hist_query_time_max) pd_lbc_hist_srt = pd_lbc_hist.sort_values(by=['image_id']) # convert the binary types back to arrays mu_bin_edges, intensity_bin_edges, full_lbc_hist = psi_d_types.binary_to_hist( pd_lbc_hist_srt, n_mu_bins=None, n_intensity_bins=n_intensity_bins) # create corrected/original histograms original_hist_list = np.full(full_lbc_hist.shape, 0, dtype=np.int64) corrected_hist_list = np.full(full_lbc_hist.shape, 0, dtype=np.int64) for inst_index, instrument in enumerate(inst_list): print("Applying corrections for", instrument) #### QUERY IMAGES #### query_instrument = [ instrument, ] rot_images = db_funcs.query_euv_images_rot(db_session, rot_min=rot_min, rot_max=rot_max, instrument=query_instrument) image_pd = rot_images.sort_values(by=['cr_rot']) # get time minimum and maximum for instrument inst_time_min = rot_images.date_obs.min() inst_time_max = rot_images.date_obs.max() # query correct image combos lbc_meth_name = "LBCC" combo_query_lbc = db_funcs.query_inst_combo(db_session, inst_time_min, inst_time_max, lbc_meth_name, instrument) iit_meth_name = "IIT" combo_query_iit = db_funcs.query_inst_combo(db_session, inst_time_min, inst_time_max, iit_meth_name, instrument) # query correct image combos combo_query_lbc = db_funcs.query_inst_combo(db_session, hist_query_time_min, hist_query_time_max, meth_name="LBCC", instrument=instrument) # query correct image combos combo_query_iit = db_funcs.query_inst_combo(db_session, hist_query_time_min, hist_query_time_max, meth_name="IIT", instrument=instrument) for index, row in image_pd.iterrows(): # apply LBC original_los, lbcc_image, mu_indices, use_indices, theoretic_query = lbcc_funcs.apply_lbc( db_session, hdf_data_dir, combo_query_lbc, image_row=row, n_intensity_bins=n_intensity_bins, R0=R0) #### ORIGINAL LOS DATA #### # calculate IIT histogram from original data original_los_hist = psi_d_types.LosImage.iit_hist( original_los, intensity_bin_edges, lat_band, log10) # add 1D histogram to array original_hist_list[:, index] = original_los_hist #### CORRECTED DATA #### # apply IIT correction lbcc_image, iit_image, use_indices, alpha, x = iit_funcs.apply_iit( db_session, combo_query_iit, lbcc_image, use_indices, original_los, R0=R0) #### CREATE CORRECTED IIT HISTOGRAM ##### # calculate IIT histogram from LBC hist_iit = psi_d_types.IITImage.iit_hist(iit_image, lat_band, log10) # create IIT histogram datatype corrected_hist = psi_d_types.create_iit_hist( iit_image, method_id[1], lat_band, hist_iit) corrected_hist_list[:, index] = corrected_hist.hist # plotting definitions color_list = ['red', 'blue', 'black'] linestyle_list = ['solid', 'dashed', 'dashdot'] #### CREATE NEW HISTOGRAM #### for inst_index, instrument in enumerate(inst_list): print("Plotting Histograms for", instrument) #### GET INDICES TO USE #### # get index of instrument in histogram dataframe hist_inst = pd_lbc_hist_srt['instrument'] pd_inst_index = hist_inst[hist_inst == instrument].index #### ORIGINAL HISTOGRAM ##### # define histogram original_hist = original_hist_list[:, pd_inst_index].sum(axis=1) # normalize histogram row_sums = original_hist.sum(axis=0, keepdims=True) norm_original_hist = original_hist / row_sums # plot original Plotting.Plot1d_Hist(norm_original_hist, instrument, inst_index, intensity_bin_edges, color_list, linestyle_list, figure=100, xlabel="Intensity (log10)", ylabel="H(I)", title="Histogram: Original LOS Data") #### LBCC HISTOGRAM ##### # define histogram lbc_hist = full_lbc_hist[:, pd_inst_index].sum(axis=1) # normalize histogram lbc_sums = lbc_hist.sum(axis=0, keepdims=True) norm_lbc_hist = lbc_hist / lbc_sums # plot lbcc Plotting.Plot1d_Hist(norm_lbc_hist, instrument, inst_index, intensity_bin_edges, color_list, linestyle_list, figure=200, xlabel="Intensity (log10)", ylabel="H(I)", title="Histogram: Post LBCC") #### CORRECTED HISTOGRAM #### # define histogram corrected_hist = corrected_hist_list[:, pd_inst_index].sum(axis=1) # normalize histogram iit_sums = corrected_hist.sum(axis=0, keepdims=True) norm_corrected_hist = corrected_hist / iit_sums # plot corrected Plotting.Plot1d_Hist(norm_corrected_hist, instrument, inst_index, intensity_bin_edges, color_list, linestyle_list, figure=300, xlabel="Intensity (log10)", ylabel="H(I)", title="Histogram: Post IIT") # end time end_time = time.time() print("ITT has been applied and original/resulting histograms plotted.") print("Total elapsed time to apply correction and plot histograms: " + str(round(end_time - start_time, 3)) + " seconds.") return None
def apply_iit_correction(db_session, hdf_data_dir, iit_query_time_min, iit_query_time_max, inst_list, ref_inst, n_intensity_bins=200, R0=1.01, n_images_plot=1, plot=False): # start time start_time = time.time() #### GET REFERENCE INFO FOR LATER USE #### # query euv images to get carrington rotation range ref_instrument = [ ref_inst, ] euv_images = db_funcs.query_euv_images(db_session, time_min=iit_query_time_min, time_max=iit_query_time_max, instrument=ref_instrument) # get min and max carrington rotation rot_max = euv_images.cr_rot.max() rot_min = euv_images.cr_rot.min() for inst_index, instrument in enumerate(inst_list): #### QUERY IMAGES #### query_instrument = [ instrument, ] rot_images = db_funcs.query_euv_images_rot(db_session, rot_min=rot_min, rot_max=rot_max, instrument=query_instrument) image_pd = rot_images.sort_values(by=['cr_rot']) # get time minimum and maximum for instrument inst_time_min = rot_images.date_obs.min() inst_time_max = rot_images.date_obs.max() # query correct image combos lbc_meth_name = "LBCC" combo_query_lbc = db_funcs.query_inst_combo(db_session, inst_time_min, inst_time_max, lbc_meth_name, instrument) iit_meth_name = "IIT" combo_query_iit = db_funcs.query_inst_combo(db_session, inst_time_min, inst_time_max, iit_meth_name, instrument) # apply LBC for index in range(n_images_plot): row = image_pd.iloc[index] print("Processing image number", row.data_id, "for IIT Correction.") #### APPLY LBC CORRECTION ##### original_los, lbcc_image, mu_indices, use_indices, theoretic_query = lbcc_funcs.apply_lbc( db_session, hdf_data_dir, combo_query_lbc, image_row=row, n_intensity_bins=n_intensity_bins, R0=R0) #### APPLY IIT CORRECTION #### lbcc_image, iit_image, use_indices, alpha, x = apply_iit( db_session, combo_query_iit, lbcc_image, use_indices, original_los, R0=R0) if plot: lbcc_data = lbcc_image.lbcc_data corrected_iit_data = iit_image.iit_data # plot LBC image Plotting.PlotCorrectedImage(lbcc_data, los_image=original_los, nfig=100 + inst_index * 10 + index, title="Corrected LBCC Image for " + instrument) # plot IIT image Plotting.PlotCorrectedImage(corrected_iit_data, los_image=original_los, nfig=200 + inst_index * 10 + index, title="Corrected IIT Image for " + instrument) # plot difference Plotting.PlotCorrectedImage(corrected_iit_data - lbcc_data, los_image=original_los, nfig=300 + inst_index * 10 + index, title="Difference Plot for " + instrument) # end time end_time = time.time() print("ITT has been applied and specified images plotted.") print("Total elapsed time to apply correction and plot: " + str(round(end_time - start_time, 3)) + " seconds.") return None
#### QUERY IMAGES #### query_instrument = [ instrument, ] rot_images = db_funcs.query_euv_images_rot(db_session, rot_min=rot_min, rot_max=rot_max, instrument=query_instrument) image_pd = rot_images.sort_values(by=['cr_rot']) # get time minimum and maximum for instrument inst_time_min = rot_images.date_obs.min() inst_time_max = rot_images.date_obs.max() # query correct image combos lbc_meth_name = "LBCC" combo_query_lbc = db_funcs.query_inst_combo(db_session, inst_time_min, inst_time_max, lbc_meth_name, instrument) iit_meth_name = "IIT" combo_query_iit = db_funcs.query_inst_combo(db_session, inst_time_min, inst_time_max, iit_meth_name, instrument) # apply LBC for index in range(n_images_plot): row = image_pd.iloc[index] #### APPLY LBC CORRECTION ##### #### APPLY LBC CORRECTION ##### original_los, lbcc_image, mu_indices, use_indices, theoretic_query = lbcc_funcs.apply_lbc( db_session, hdf_data_dir, combo_query_lbc, image_row=row,