def save_timescale_maps(db_session, map_data_dir, euv_combined, chd_combined,
image_info_timescale, map_info_timescale, methods_list,
combined_method, chd_combined_method,
timescale_method):
start = time.time()
# create image and map info lists
data_info = [info for sublist in image_info_timescale for info in sublist]
map_info = [info for sublist in map_info_timescale for info in sublist]
# generate a record of the method and variable values used for interpolation
euv_combined.append_method_info(methods_list)
euv_combined.append_method_info(pd.DataFrame(data=combined_method))
euv_combined.append_method_info(pd.DataFrame(data=timescale_method))
chd_combined.append_method_info(methods_list)
chd_combined.append_method_info(pd.DataFrame(data=chd_combined_method))
chd_combined.append_method_info(pd.DataFrame(data=timescale_method))
# generate record of image and map info
euv_combined.append_data_info(data_info)
euv_combined.append_map_info(map_info)
chd_combined.append_data_info(data_info)
chd_combined.append_map_info(map_info)
# plot maps
# TODO: this doesn't give max and min times, better way to find minimum time.
Plotting.PlotMap(euv_combined,
nfig="EUV Map Timescale Weighted",
title="EUV Map Running Average Times\nTime Min: " +
str(euv_combined.data_info.iloc[0].date_obs) +
"\nTime Max: " +
str(euv_combined.data_info.iloc[-1].date_obs))
Plotting.PlotMap(chd_combined,
nfig="CHD Map Timescale Weighted",
title="CHD Map Running Average Times\nTime Min: " +
str(euv_combined.data_info.iloc[0].date_obs) +
"\nTime Max: " +
str(euv_combined.data_info.iloc[-1].date_obs),
map_type='CHD')
# save EUV and CHD maps to database
euv_combined.write_to_file(map_data_dir,
map_type='runavg_euv',
filename=None,
db_session=db_session)
chd_combined.write_to_file(map_data_dir,
map_type='runavg_chd',
filename=None,
db_session=db_session)
end = time.time()
print(
"Combined Timescale Running Average Maps have been plotted and saved to the database in",
end - start, "seconds.")
def save_gauss_time_maps(db_session, map_data_dir, euv_combined, chd_combined,
data_info, map_info, methods_list, combined_method):
start = time.time()
# generate a record of the method and variable values used for interpolation
euv_combined.append_method_info(methods_list)
euv_combined.append_method_info(pd.DataFrame(data=combined_method))
chd_combined.append_method_info(methods_list)
chd_combined.append_method_info(pd.DataFrame(data=combined_method))
# generate record of image and map info
euv_combined.append_data_info(data_info)
euv_combined.append_map_info(map_info)
chd_combined.append_data_info(data_info)
chd_combined.append_map_info(map_info)
# plot maps
Plotting.PlotMap(euv_combined,
nfig="EUV Map Timescale Weighted",
title="EUV Map Gaussian Timescale Weighted\nTime Min: " +
str(euv_combined.data_info.iloc[0].date_obs) +
"\nTime Max: " +
str(euv_combined.data_info.iloc[-1].date_obs))
Plotting.PlotMap(chd_combined,
nfig="CHD Map Timescale Weighted",
title="CHD Map Gaussian Timescale Weighted\nTime Min: " +
str(euv_combined.data_info.iloc[0].date_obs) +
"\nTime Max: " +
str(euv_combined.data_info.iloc[-1].date_obs),
map_type='CHD')
# save EUV and CHD maps to database
euv_combined.write_to_file(map_data_dir,
map_type='timewgt_euv',
filename=None,
db_session=db_session)
chd_combined.write_to_file(map_data_dir,
map_type='timewgt_chd',
filename=None,
db_session=db_session)
end = time.time()
print(
"Combined Gaussian Time Weighted Maps have been plotted and saved to the database in",
end - start, "seconds.")
return None
def save_mu_probability_maps(db_session, map_data_dir, euv_combined,
chd_combined, data_info, map_info, methods_list,
euv_combined_method, chd_combined_method):
start = time.time()
# generate a record of the method and variable values used for interpolation
euv_combined.append_method_info(methods_list)
euv_combined.append_method_info(pd.DataFrame(data=euv_combined_method))
chd_combined.append_method_info(methods_list)
chd_combined.append_method_info(pd.DataFrame(data=chd_combined_method))
# generate record of image and map info
euv_combined.append_data_info(data_info)
euv_combined.append_map_info(map_info)
chd_combined.append_data_info(data_info)
chd_combined.append_map_info(map_info)
# plot maps
Plotting.PlotMap(euv_combined,
nfig="EUV Map",
title="Minimum Intensity Merge EUV Map\nTime Min: " +
str(euv_combined.data_info.iloc[0].date_obs) +
"\nTime Max: " +
str(euv_combined.data_info.iloc[-1].date_obs))
Plotting.PlotMap(
chd_combined,
nfig="Mu Dependent CHD Probability Map",
title="Mu Dependent CHD Probability "
"Map\nTime Min: " + str(chd_combined.data_info.iloc[0].date_obs) +
"\nTime Max: " + str(chd_combined.data_info.iloc[-1].date_obs),
map_type='CHD')
# save EUV and CHD maps to database
euv_combined.write_to_file(map_data_dir,
map_type='synoptic_euv',
filename=None,
db_session=db_session)
chd_combined.write_to_file(map_data_dir,
map_type='mu_synoptic_chd',
filename=None,
db_session=db_session)
end = time.time()
print(
"Combined Mu-Dependent Synoptic Maps have been plotted and saved to the database in",
end - start, "seconds.")
# iterate through the rows of map_info
for row_index, row in map_info.iterrows():
print("Processing map for:", row.date_mean)
# load map (some older maps have a leading '/' that messes with os.path.join
if row.fname[0] == "/":
rel_path = row.fname[1:]
else:
rel_path = row.fname
map_path = os.path.join(map_dir, rel_path)
my_map = psi_datatype.read_psi_map(map_path)
# extract needed metrics
euv_image_data = my_map.data
chd_data = my_map.chd # note that the grid has been reduced since the coronal
# hole detection was performed, so values are floats between 0. and 1. A rounding
# operation may be required for discrete operations.
discrete_chd = np.round(my_map.chd)
phi_coords = my_map.x
sinlat_coords = my_map.y
mu_data = my_map.mu
origin_image = my_map.origin_image
# do stuff with data
# Simple plotting
EasyPlot.PlotMap(my_map, nfig=0)
EasyPlot.PlotMap(my_map, nfig=1, map_type="CHD")
# close database connection
db_session.close()
EUV_CHD_sep=False)
#### STEP SIX: APPLY CH AND AR DETECTION ####
map = np.where(synchronic_map.data == -9999, 0, synchronic_map.data)
map2 = np.log(map)
map2 = np.where(map2 == -np.inf, 0, map2)
arr = np.zeros((map_nxcoord * map_nycoord, 3))
arr[:, 0] = idx_col_flt * weight
arr[:, 1] = idx_row_flt * weight
arr[:, 2] = map2.flatten() * 2
psi_chd_map, psi_ar_map, chd_labeled, ar_labeled = ml_funcs.kmeans_detection(
synchronic_map.data, map2, arr, N_CLUSTERS, map_nycoord,
map_nxcoord, map_x, map_y)
title = 'Minimum Intensity Merge: Unsupervised Detection Map\nDate: ' + str(
center)
Plotting.PlotMap(psi_chd_map, title=title, nfig=date_ind)
Plotting.PlotMap(psi_chd_map,
map_type='Contour',
title=title,
nfig=date_ind)
Plotting.PlotMap(psi_ar_map,
map_type='Contour1',
title=title,
nfig=date_ind)
end_time = time.time()
print("Total elapsed time: ", end_time - start_time)
db_session,
mean_time_range=[
date_time - datetime.timedelta(hours=1),
date_time + datetime.timedelta(hours=1)
],
n_images=1)
if len(map_list) == 0:
continue
combined_map = combine_maps(map_list, del_mu=del_mu)
# generate a record of the method and variable values used for interpolation
new_method = {
'meth_name': ("Min-Int-Merge_1", ),
'meth_description': ["Minimum intensity merge version 1"] * 1,
'var_name': ("del_mu", ),
'var_description': ("max acceptable mu range", ),
'var_val': (del_mu, )
}
combined_map.append_method_info(pd.DataFrame(data=new_method))
combined_map.append_data_info(data_info)
combined_map.append_map_info(map_info)
EasyPlot.PlotMap(combined_map,
nfig="Combined map for: " + str(center),
title="Minimum Intensity Merge Map\nDate: " + str(center))
plt.show()
combined_map.write_to_file(map_data_dir,
map_type='synoptic',
filename=None,
db_session=db_session)
# create one data object with EUV & CHD map
for ii in range(map_list.__len__()):
# first combine chd and image into a single map object
map_list[ii].chd = chd_map_list[ii].data.astype('float16')
#### STEP FIVE: CREATE COMBINED MAPS ####
synchronic_map = midm.create_combined_maps_2(
map_list,
mu_merge_cutoff=mu_merge_cutoff,
del_mu=del_mu,
mu_cutoff=mu_cutoff,
EUV_CHD_sep=EUV_CHD_sep)
# add synchronic clustering method to final map
synchronic_map.append_method_info(cluster_method)
#### STEP SIX: PLOT SYNCHRONIC MAPS ####
synchronic_map.chd = np.where(synchronic_map.chd > 0, 1, -9999)
title = 'Minimum Intensity Merge: Supervised (CNN) Detection Map\nDate: ' + str(
center)
Plotting.PlotMap(synchronic_map,
nfig="Supervised Map",
title=title,
map_type='EUV')
Plotting.PlotMap(synchronic_map,
nfig="Supervised Map",
title=title,
map_type='Contour')
end_time = time.time()
print("Total elapsed time: ", end_time - start_time)
# --- Begin execution ----------------------
# query maps in time range mu_merge_cutoff
map_methods = ['Synch_Im_Sel', 'GridSize_sinLat', 'MIDM-Comb-mu_merge']
map_info, data_info, method_info, image_assoc = db_funs.query_euv_maps(
db_session,
mean_time_range=(query_start, query_end),
methods=map_methods,
var_val_range=map_vars)
map_path = os.path.join(map_dir, map_info.fname[0])
merge_map = psi_datatype.read_psi_map(map_path)
# Plot and save
save_to = "/Users/turtle/Dropbox/MyNACD/ron_code/test_images/new_data_merge-overlap04.png"
EasyPlot.PlotMap(merge_map, nfig=0)
plt.savefig(save_to)
map_methods = ['Synch_Im_Sel', 'GridSize_sinLat', 'MIDM-Comb-del_mu']
map_info2, data_info2, method_info2, image_assoc2 = db_funs.query_euv_maps(
db_session,
mean_time_range=(query_start, query_end),
methods=map_methods,
var_val_range=map_vars)
for ii in range(map_info2.shape[0]):
map_path = os.path.join(map_dir, map_info2.fname[ii])
del_map = psi_datatype.read_psi_map(map_path)
method_index = (method_info2.var_name == "del_mu") & \
(method_info2.map_id == map_info2.map_id[ii])
del_mu = method_info2.var_val[method_index]
def create_combined_maps(db_session,
map_data_dir,
map_list,
chd_map_list,
methods_list,
data_info,
map_info,
mu_merge_cutoff=None,
del_mu=None,
mu_cutoff=0.0):
"""
function to create combined EUV and CHD maps and save to database with associated method information
@param db_session: database session to save maps to
@param map_data_dir: directory to save map files
@param map_list: list of EUV maps
@param chd_map_list: list of CHD maps
@param methods_list: methods list
@param data_info: image info list
@param map_info: map info list
@param mu_merge_cutoff: cutoff mu value for overlap areas
@param del_mu: maximum mu threshold value
@param mu_cutoff: lower mu value
@return: combined euv map, combined chd map
"""
# start time
start = time.time()
# create combined maps
euv_maps = []
chd_maps = []
for euv_map in map_list:
if len(euv_map.data) != 0:
euv_maps.append(euv_map)
for chd_map in chd_map_list:
if len(chd_map.data) != 0:
chd_maps.append(chd_map)
if del_mu is not None:
euv_combined, chd_combined = combine_maps(euv_maps,
chd_maps,
del_mu=del_mu,
mu_cutoff=mu_cutoff)
combined_method = {
'meth_name': ("Min-Int-Merge-del_mu", "Min-Int-Merge-del_mu"),
'meth_description':
["Minimum intensity merge for synchronic map: using del mu"] * 2,
'var_name': ("mu_cutoff", "del_mu"),
'var_description':
("lower mu cutoff value", "max acceptable mu range"),
'var_val': (mu_cutoff, del_mu)
}
else:
euv_combined, chd_combined = combine_maps(
euv_maps,
chd_maps,
mu_merge_cutoff=mu_merge_cutoff,
mu_cutoff=mu_cutoff)
combined_method = {
'meth_name': ("Min-Int-Merge-mu_merge", "Min-Int-Merge-mu_merge"),
'meth_description': [
"Minimum intensity merge for synchronic map: based on Caplan et. al."
] * 2,
'var_name': ("mu_cutoff", "mu_merge_cutoff"),
'var_description':
("lower mu cutoff value", "mu cutoff value in areas of "
"overlap"),
'var_val': (mu_cutoff, mu_merge_cutoff)
}
# generate a record of the method and variable values used for interpolation
euv_combined.append_method_info(methods_list)
euv_combined.append_method_info(pd.DataFrame(data=combined_method))
euv_combined.append_data_info(data_info)
euv_combined.append_map_info(map_info)
chd_combined.append_method_info(methods_list)
chd_combined.append_method_info(pd.DataFrame(data=combined_method))
chd_combined.append_data_info(data_info)
chd_combined.append_map_info(map_info)
# plot maps
Plotting.PlotMap(euv_combined,
nfig="EUV Combined Map for: " +
str(euv_combined.data_info.date_obs[0]),
title="Minimum Intensity Merge EUV Map\nDate: " +
str(euv_combined.data_info.date_obs[0]),
map_type='EUV')
Plotting.PlotMap(chd_combined,
nfig="CHD Combined Map for: " +
str(chd_combined.data_info.date_obs[0]),
title="Minimum Intensity Merge CHD Map\nDate: " +
str(chd_combined.data_info.date_obs[0]),
map_type='CHD')
# Plotting.PlotMap(chd_combined, nfig="CHD Contour Map for: " + str(chd_combined.data_info.date_obs[0]),
# title="Minimum Intensity Merge CHD Contour Map\nDate: " + str(chd_combined.data_info.date_obs[0]),
# map_type='Contour')
# save EUV and CHD maps to database
# euv_combined.write_to_file(map_data_dir, map_type='synchronic_euv', filename=None, db_session=db_session)
# chd_combined.write_to_file(map_data_dir, map_type='synchronic_chd', filename=None, db_session=db_session)
# end time
end = time.time()
print("Combined EUV and CHD Maps created and saved to the database in",
end - start, "seconds.")
return euv_combined, chd_combined
del_y = (y_range[1] - y_range[0]) / (map_nycoord - 1)
map_nxcoord = (np.floor((x_range[1] - x_range[0]) / del_y) + 1).astype(int)
# generate map x,y grids. y grid centered on equator, x referenced from lon=0
map_y = np.linspace(y_range[0], y_range[1], map_nycoord.astype(int))
map_x = np.linspace(x_range[0], x_range[1], map_nxcoord.astype(int))
# test LosImage function interp_to_map()
test_map = test_los.interp_to_map(R0=R0,
map_x=map_x,
map_y=map_y,
image_num=selected_image.data_id)
# compare test image to test map
EasyPlot.PlotImage(test_los, nfig=9)
EasyPlot.PlotMap(test_map, nfig=10)
# # determine number of pixels in map y-grid
# map_nycoord = sum(abs(test_los.y) < R0)
# del_y = (y_range[1] - y_range[0])/(map_nycoord-1)
# map_nxcoord = (np.floor((x_range[1] - x_range[0])/del_y) + 1).astype(int)
#
# # generate map x,y grids
# map_y = np.linspace(y_range[0], y_range[1], map_nycoord.astype(int))
# map_x = np.linspace(x_range[0], x_range[1], map_nxcoord.astype(int))
#
# # initialize grid to receive interpolation with values of -1
# interp_result = np.full((map_nycoord, map_nxcoord), np.nan)
#
# # generate as row and column vectors
# # arr_x, arr_y = np.meshgrid(map_x, map_y, sparse=True)
def save_threshold_maps(db_session, map_data_dir, euv_combined, chd_combined,
data_info, map_info, methods_list, euv_combined_method,
chd_combined_method, FWHM, n_samples):
start = time.time()
# chd threshold method
chd_threshold = {
'meth_name': ("Gaussian-Varying-CHD", ) * 2,
'meth_description': ["Gaussian Varying CHD Threshold Method"] * 2,
'var_name': ("FWHM", "n_samples"),
'var_description':
("full width - half max of gaussian "
"distribution", "number of random samples used for CHD "
"thresholding"),
'var_val': (FWHM, n_samples)
}
# generate a record of the method and variable values used for interpolation
euv_combined.append_method_info(methods_list)
euv_combined.append_method_info(pd.DataFrame(data=euv_combined_method))
euv_combined.append_method_info(pd.DataFrame(data=chd_threshold))
chd_combined.append_method_info(methods_list)
chd_combined.append_method_info(pd.DataFrame(data=chd_combined_method))
chd_combined.append_method_info(pd.DataFrame(data=chd_threshold))
# generate record of image and map info
euv_combined.append_data_info(data_info)
euv_combined.append_map_info(map_info)
chd_combined.append_data_info(data_info)
chd_combined.append_map_info(map_info)
# plot maps
Plotting.PlotMap(euv_combined,
nfig="Varying Threshold EUV Map",
title="Minimum Intensity Merge CR EUV Map\nTime "
"Min: " + str(euv_combined.data_info.iloc[0].date_obs) +
"\nTime Max: " +
str(euv_combined.data_info.iloc[-1].date_obs))
Plotting.PlotMap(chd_combined,
nfig="Varying Threshold CHD Map",
title="CHD Merge Map with "
"Gaussian Varying Threshold Values\nTime "
"Min: " + str(chd_combined.data_info.iloc[0].date_obs) +
"\nTime Max: " +
str(chd_combined.data_info.iloc[-1].date_obs),
map_type='CHD')
# save EUV and CHD maps to database
euv_combined.write_to_file(map_data_dir,
map_type='varthresh_chd',
filename=None,
db_session=db_session)
chd_combined.write_to_file(map_data_dir,
map_type='varthresh_chd',
filename=None,
db_session=db_session)
end = time.time()
print(
"Combined Gaussian Varying CHD Threshold Maps have been plotted and saved to the database in",
end - start, "seconds.")
return None
'meth_description': [
"Use SciPy.RegularGridInterpolator() to linearly interpolate from an Image to a Map"
] * 1,
'var_name': ("R0", ),
'var_description': ("Solar radii", ),
'var_val': (R0, )
}
# add to the methods dataframe for this map
methods_list[ii] = methods_list[ii].append(
pd.DataFrame(data=new_method), sort=False)
# incorporate the methods dataframe into the map object
map_list[ii].append_method_info(methods_list[ii])
# simple plot
EasyPlot.PlotMap(map_list[ii], nfig=10 + ii, title="Map " + str(ii))
# as a demonstration, save these maps to file and then push to the database
map_list[ii].write_to_file(map_data_dir,
map_type='single',
filename=None,
db_session=db_session)
# --- 6. Combine Maps ---------------------------------------------------------
del_mu = 0.2
combined_map, combined_chd = combine_maps(map_list, del_mu=del_mu)
# generate a record of the method and variable values used for interpolation
new_method = {
'meth_name': ("Min-Int-Merge_1", ),
'meth_description': ["Minimum intensity merge version 1"] * 1,