def analysis_1d(merge_df, modes, m_keys, sources):
'''
This function carries out all plotting and calculations needed for a 1-d
dataset (i.e. one frequency)
Future iterations may include optional arguments to enable selection of the
plots that are preferred
'''
if modes['verbose'] >= 2:
print("Carrying out 1-frequency Analysis")
# if "spectra" in modes["plots"]:
# #plots the values for each channel
# plot_values_1f(merge_df, m_keys, modes)
#
# if "diff" in modes["plots"]:
# #plots the differences in the values
# plot_diff_values_1f(merge_df, m_keys, modes)
if "spectra" in modes["plots"]:
#plots the values for each channel
plots_1f(merge_df, m_keys, modes, "Time", sources)
if ((all(coord in merge_df for coord in ["alt", "az", "az_ew"]))
and (any(plot in modes["plots"] for plot in ["alt", "az", "ew"]))):
alt_var, az_var, az_var_ew = get_alt_az_var(merge_df, modes)
if "alt" in modes["plots"]:
plots_1f(merge_df, m_keys, modes, alt_var, sources)
if "az" in modes["plots"]:
plots_1f(merge_df, m_keys, modes, az_var, sources)
elif any(plot in modes["plots"] for plot in ["alt", "az", "ew"]):
if modes['verbose'] >= 1:
print("Warning: Horizontal coordinates selected but unavailable")
#checks to see if there are differences to analyse
if (any("diff" in col_name for col_name in merge_df.columns)):
foms = []
if "corr" in modes["plots"]:
foms.append("corr")
if "rmse" in modes["plots"]:
foms.append("rmse")
for fom in foms:
fom_results = calc_fom_1d(merge_df, m_keys, fom)
for i in range(len(m_keys)):
out_str = ("The " + str(m_keys[i]) + "-channel " +
gen_pretty_name(fom) + " is " + str(fom_results[i]))
if modes['out_dir'] == None:
print("\n" + out_str + "\n")
else:
#creates an output-friendly string for the channel
str_channel = channel_maker(m_keys, modes)
plt_file = prep_out_file(modes,
plot=fom,
dims="1d",
channel=str_channel,
out_type="txt")
out_file = open(plt_file, 'a')
out_file.write(out_str)
out_file.close()
else:
if "corr" in modes["plots"]:
if modes['verbose'] >= 1:
print(
"Warning: Correlation selected, but no differences available"
)
if "rmse" in modes["plots"]:
if modes['verbose'] >= 1:
print("Warning: RMSE selected, but no differences available")
def analysis_nd(merge_df, modes, m_keys, sources):
'''
This function carries out all plotting and calculations needed for a n-d
dataset (i.e. multiple frequencies)
Future iterations may include optional arguments to enable selection of the
plots that are preferred
'''
if modes['verbose'] >= 2:
print("Carrying out multi-frequency Analysis")
if "spectra" in modes["plots"]:
plot_spectra_nf(merge_df, m_keys, modes, sources)
if any(plot in modes["plots"] for plot in ["alt", "az", "ew"]):
if all(coord in merge_df for coord in ["alt", "az", "az_ew"]):
plot_altaz_values_nf(merge_df, m_keys, modes, sources)
else:
if modes['verbose'] >= 1:
print(
"Warning: Alt-Azimuth plotting selected, but not available!"
)
#calculates the figures of merit at each independent variable
#return values are stored as possible future outputs
ind_dfs = {}
#checks to see if there are differences to analyse
if (any("diff" in col_name for col_name in merge_df.columns)):
foms = []
if "corr" in modes["plots"]:
foms.append("corr")
if "rmse" in modes["plots"]:
foms.append("rmse")
for fom in foms:
ind_dfs = plot_fom_vs_ind(merge_df, m_keys, modes, fom)
#calculates the overall figure of merit between scope and model
fom_1 = calc_fom_1d(merge_df, m_keys, fom)
#prints that coefficient for each key and correlation
for i in range(len(m_keys)):
out_str=("The "+str(m_keys[i])+"-channel "+\
gen_pretty_name(fom)+" is "+str(fom_1[i]))
if modes['out_dir'] == None:
print(out_str)
else:
plt_file = prep_out_file(modes,
plot=fom,
dims="1d",
channel=m_keys[i],
out_type="txt")
out_file = open(plt_file, 'a')
out_file.write(out_str)
out_file.close()
else:
if "corr" in modes["plots"]:
if modes['verbose'] >= 1:
print(
"Warning: Correlation selected, but no differences available"
)
if "rmse" in modes["plots"]:
if modes['verbose'] >= 1:
print("Warning: RMSE selected, but no differences available")
str_channel = channel_maker(m_keys, modes)
if modes['out_dir'] != None:
for plot_item in ind_dfs:
#prints the correlations to a file
path_out_df = prep_out_file(modes,
plot=plot_item,
channel=str_channel,
out_type=".csv")
try:
ind_dfs[plot_item].to_csv(path_out_df)
except IOError:
if modes['verbose'] >= 1:
print("WARNING: Unable to output to file:\n\t" +
path_out_df)
return (ind_dfs)
def plot_1f(merge_df, m_keys, modes, sources, var_str):
#creates an overlaid plot of how the sources
#varies for each of the channels against var_str
title = "Plot of "
for source in sources:
title = add_key(title, sources, source)
title = title + " for "
for key in m_keys:
title = add_key(title, m_keys, key)
title = title + "-channels over " + gen_pretty_name(var_str)
freq_in = modes['freq'][0]
freq_MHz = freq_in / 1e6
title = title + "\nat a Frequency of {:7.3f} MHz".format(freq_MHz)
if modes['verbose'] >= 2:
print(title)
if modes['colour'] in ["matching", "matching_dark"] and len(m_keys) == 1:
text_colour = colour_models(m_keys[0])
elif modes['colour'] in ["dark", "matching_dark"]:
text_colour = "white"
else: # light or None
text_colour = "black"
mpl.rcParams.update({
'text.color': text_colour,
'axes.labelcolor': text_colour,
'xtick.color': text_colour,
'ytick.color': text_colour
})
mpl.rc('axes', edgecolor=text_colour)
fig, ax = plt.subplots()
if modes['dpi'] is None:
pass
else:
fig.set_dpi(modes['dpi'])
if modes['image_size'] is None:
pass # do nothing
else:
fig.set_size_inches(modes['image_size'])
if modes['colour'] in ["dark", "matching_dark"]:
ax.set_facecolor('black')
fig.patch.set_facecolor('black')
for key in m_keys:
for source in sources:
sep = get_source_separator(source)
var_y_vals = plottable(merge_df, (key + sep + source))
# sets the y axis scale to percentage if requested.
if 'percent' in modes['scale']:
var_y_vals = var_y_vals * 100
ax.plot(plottable(merge_df, var_str),
var_y_vals,
label=key + sep + source,
color=colour_models(key + sep + source))
ax.set_title(title, wrap=True)
ax.legend(frameon=False)
# plots the axis labels rotated so they're legible
ax.tick_params(labelrotation=45)
ax.set_xlabel(gen_pretty_name(var_str, units=True))
# sets the y axis scale to logarithmic if requested.
if 'log' in modes['scale']:
ax.set_yscale('log')
# sets the y axis scale to percentage if requested.
if 'percent' in modes['scale']:
ax.yaxis.set_major_formatter(mtick.PercentFormatter())
# prints or saves the plot
if modes['out_dir'] == None:
plt.show()
else:
str_sources = channel_maker(sources, modes)
plt_file = prep_out_file(modes,
plot="vals",
dims="1d",
ind_var=var_str,
channel=list_to_string(m_keys, '_'),
source=str_sources,
freq=min(merge_df.Freq),
out_type="png")
if modes['verbose'] >= 2:
print("Saving: " + plt_file)
try:
plt.savefig(plt_file,
bbox_inches='tight',
facecolor=fig.get_facecolor(),
edgecolor='none')
except ValueError:
if modes['verbose'] >= 1:
print("ERROR: Unable to save file, showing instead.")
try:
plt.show()
except:
print("Unable to show file.")
plt.close()
return (0)
def calc_fom_nd(in_df, var_str, m_keys, modes, fom="rmse"):
"""
This function calculates a figure of merit between the scope and model
values for the specified channels as they are distributed against another
column of the dataframe merge_df which is identified by var_str
in current versions, useable values for var_str are "Time" and "Freq"
in current versions, useable values for fom are "rmse" and "corr"
"""
print("probe: ", modes['colour'])
if modes['verbose'] >= 2:
print("Calculating the " + gen_pretty_name(fom) +
" between observed and model data.")
# creates empty lists for the Errors
n_foms = []
for i in range(len(m_keys)):
n_foms.append([])
# identifies allthe unique values of the variable in the column
unique_vals = in_df[var_str].unique()
unique_vals = np.sort(unique_vals)
# iterates over all unique values
for unique_val in unique_vals:
# creates a dataframe with only the elements that match the current
# unique value
unique_merge_df = in_df[in_df[var_str] ==
unique_val].copy().reset_index(drop=True)
# uses this unique value for and the 1-dimensional calc_fom_1d function
# to calculate the Figure of merit for each channel
n_fom = calc_fom_1d(unique_merge_df, m_keys, fom)
# appends these to the list
for i in range(len(m_keys)):
n_foms[i].append(n_fom[i])
# creates an overlaid plot of how the Figure of Merit between model and scope
# varies for each of the channels against var_str
if modes['verbose'] >= 2:
print("Plotting the " + gen_pretty_name(fom) +
" between model and scope for " +
channel_maker(m_keys, modes, ", ") + " against " +
gen_pretty_name(var_str))
if modes['colour'] in ["matching", "matching_dark"] and len(m_keys) == 1:
text_colour = colour_models(m_keys[0])
elif modes['colour'] in ["dark", "matching_dark"]:
text_colour = "white"
else: # light or None
text_colour = "black"
mpl.rcParams.update({
'text.color': text_colour,
'axes.labelcolor': text_colour,
'xtick.color': text_colour,
'ytick.color': text_colour
})
mpl.rc('axes', edgecolor=text_colour)
fig, ax = plt.subplots()
if modes['dpi'] is None:
pass
else:
fig.set_dpi(modes['dpi'])
if modes['image_size'] is None:
pass # do nothing
else:
fig.set_size_inches(modes['image_size'])
if modes['colour'] in ["dark", "matching_dark"]:
ax.set_facecolor('black')
fig.patch.set_facecolor('black')
graph_title = "\n".join([modes['title'],"Plot of the "+gen_pretty_name(fom)+\
" in "])
for key in m_keys:
plt.plot(plottable(unique_vals, var_str),
n_foms[m_keys.index(key)],
label=key + '_' + fom,
color=colour_models(key))
graph_title = add_key(graph_title, m_keys, key)
# calculates and adds title with frequency in MHz
graph_title = graph_title + "-channels over " + gen_pretty_name(var_str)
plt.title(graph_title, wrap=True)
# rotates the labels. This is necessary for timestamps
plt.xticks(rotation=90)
plt.legend(frameon=False)
plt.xlabel(gen_pretty_name(var_str, units=True), wrap=True)
# prints or saves the plot
if modes['out_dir'] == None:
plt.show()
else:
# creates an output-friendly string for the channel
str_channel = list_to_string(m_keys, '_')
plt_file = prep_out_file(modes,
plot=fom,
ind_var=var_str,
channel=str_channel,
out_type=modes['image_type'])
if modes['verbose'] >= 2:
print("Saving: " + plt_file)
try:
plt.savefig(plt_file,
bbox_inches='tight',
facecolor=fig.get_facecolor(),
edgecolor='none')
except ValueError:
if modes['verbose'] >= 1:
print("ERROR: Unable to save file, showing instead.")
try:
plt.show()
except:
print("Unable to show file.")
plt.close()
# returns the correlation lists if needed
return (n_foms)
def animated_plot(merge_df,
modes,
var_x,
var_ys,
var_t,
sources,
time_delay=20):
"""
Produces an animated linegraph(s) with the X, Y and T variables specified
"""
if modes['colour'] in ["matching", "matching_dark"] and len(var_ys) == 1:
text_colour = colour_models(var_ys[0])
elif modes['colour'] in ["dark", "matching_dark"]:
text_colour = "white"
else: # light or None or matching and multiple y
text_colour = "black"
mpl.rcParams.update({
'text.color': text_colour,
'axes.labelcolor': text_colour,
'xtick.color': text_colour,
'ytick.color': text_colour
})
mpl.rc('axes', edgecolor=text_colour)
fig, ax = plt.subplots()
if modes['dpi'] is None:
pass
else:
fig.set_dpi(modes['dpi'])
if modes['image_size'] is None:
pass # do nothing
else:
fig.set_size_inches(modes['image_size'])
if modes['colour'] in ["dark", "matching_dark"]:
ax.set_facecolor('black')
fig.patch.set_facecolor('black')
# hard coded for now, need to parameterise
percentile_gap = 0 # 5
multiplier = 1 # 1.5
# sets default values for max_ and min_y
max_y = np.nextafter(0, 1) # makes max and min values distinct
min_y = 0
# Plot a scatter that persists (isn't redrawn) and the initial line.
var_t_vals = np.sort(merge_df[var_t].unique())
var_t_val = var_t_vals[0]
# str_channel = list_to_string(var_ys,", ")
if var_t == "Time":
var_t_string = str(var_t_val).rstrip('0').rstrip('.')
elif var_t == "Freq":
freq_MHz = var_t_val / 1e6
var_t_string = "{:7.3f} MHz".format(freq_MHz)
else:
var_t_string = ("%.4f" % var_t_val).rstrip('0').rstrip('.')
title = "Plot of "
for source in sources:
title = add_key(title, sources, source)
title = title + " for "
for var_y in var_ys:
title = add_key(title, var_ys, var_y)
title = (title + "-channels over " + gen_pretty_name(var_x) + " at\n" +
gen_pretty_name(var_t))
if modes['verbose'] >= 2:
print("Generating an Animated " + title)
title = "\n".join([modes["title"], title + " of " + var_t_string])
ax.set_title(title, wrap=True)
var_x_vals = plottable(
merge_df.loc[merge_df[var_t] == var_t_val].reset_index(drop=True),
var_x)
lines = []
for i in range(len(var_ys)):
var_y = var_ys[i]
for source in sources:
sep = get_source_separator(source)
var_y_vals = plottable(
merge_df.loc[merge_df[var_t] == var_t_val].reset_index(
drop=True), (var_y + sep + source))
var_y_vals_all = merge_df[var_y + sep + source]
# sets the y axis scale to percentage if requested.
if 'percent' in modes['scale']:
var_y_vals = var_y_vals * 100
line, = ax.plot(var_x_vals,
var_y_vals,
color=colour_models(var_y + sep + source))
lines.append(line)
# code to set x and y limits.
local_min_y = np.percentile(var_y_vals_all,
percentile_gap) * multiplier
min_y = min(min_y, local_min_y)
# min_y = 0#min(merge_df[(var_y+"_"+source)].min(),0)
local_max_y = np.percentile(var_y_vals_all,
100 - percentile_gap) * multiplier
max_y = max(max_y, local_max_y)
# sets the y axis scale to logarithmic if requested.
if 'log' in modes['scale']:
ax.set_yscale('log')
# sets the y axis scale to percentage if requested.
if 'percent' in modes['scale']:
ax.yaxis.set_major_formatter(mtick.PercentFormatter())
if min_y > 0 and 'linear' in modes['scale']:
min_y = 0
ax.set_ylim(min_y, max_y)
ax.set_xlabel(gen_pretty_name(var_x, units=True), wrap=True)
ax.set_ylabel(channel_maker(var_ys, modes, ", ") +
" flux\n(arbitrary units)",
wrap=True)
ax.legend(frameon=False)
if modes['out_dir'] is None:
repeat_option = True
else:
repeat_option = False
# creates a global variable as animations only work with globals
if "anim" not in globals():
global anim
anim = []
else:
pass
anim.append(
FuncAnimation(fig,
update_a,
frames=range(len(var_t_vals)),
interval=time_delay,
fargs=(merge_df, modes, var_x, var_ys, var_t, sources,
lines, ax),
repeat=repeat_option))
ax.set_aspect('auto')
plt.subplots_adjust(top=0.80) # TODO: automate this so it's not fixed
if modes['out_dir'] is not None:
str_channel = channel_maker(var_ys, modes)
str_sources = list_to_string(sources, "_")
# str_channel = list_to_string(var_ys,", ")
plot_name = var_x + "_over_" + var_t
plt_file = prep_out_file(modes,
source=str_sources,
plot=plot_name,
dims="nd",
channel=str_channel,
out_type=modes['image_type'])
try:
anim[len(anim) - 1].save(plt_file,
dpi=80,
writer='pillow',
savefig_kwargs={
'facecolor': fig.get_facecolor(),
'edgecolor': 'none'
})
except ValueError:
if modes['verbose'] >= 1:
print("ERROR: Unable to save file, try showing instead.")
try:
plt.show()
except:
print("ERROR: Unable to show file.")
# plt.close() # TODO: fix this so it works
else:
plt.show() # will just loop the animation forever.