def setup_open():
from harmonisation.core.matchup.matchupToolbox.harmonisation.harmonisationVis import HarmonisationVis
matchup = open_matchup(DATASET_PATHS, SENSOR_DATA_PATH)
harmonisation_result = HarmonisationResult(OUTPUT_DIRECTORY)
harm_vis = HarmonisationVis(matchup, harmonisation_result)
try:
makedirs(PLOTS_DIRECTORY)
except OSError:
pass
return harm_vis
def run(self, verbose=True):
"""
Generates plots for dataset specified by paths defined object attributes
"""
# Initialise
dataset_dir = self.dataset_dir
sensor_data = self.sensor_data
output_dir = self.output_dir
plots_dir = self.plots_dir
################################################################################################################
# 1. Read Match-up Data and Harmonisation Result Data
################################################################################################################
print"Match-up Dataset:"
print ">", dataset_dir
print"\nSensor Data:"
print ">", sensor_data
print "\nHarmonisation Output:"
print ">", output_dir
print("\nOpening Files...")
MatchUpData = open_matchup(dataset_dir, open_uncertainty=False)
MatchUpData.setSensorData(sensor_data)
HarmResult = HarmonisationResult(output_dir, open_residuals=False)
if verbose:
print HarmResult.parameter_sensors
print HarmResult.parameter
print HarmResult.parameter_covariance_matrix
print "\n"
################################################################################################################
# 2. Make plots
################################################################################################################
print("Generating Plots...")
HarmVisOp = HarmonisationVis(MatchUpData, HarmResult)
HarmVisOp.plot_compare_calibration(plots_dir, HarmResult.parameter,
HarmResult.parameter_covariance_matrix, verbose=verbose)
print "\nPlots written to:", plots_dir
def run(self,
dataset_dir,
sensor_data,
output_dir,
pc_input=None,
save_pc=None,
gn_input=None,
save_gn=None,
software_cfg=None,
tolPC=TOLPC,
tol=TOL,
tolA=TOLA,
tolB=TOLB,
tolU=TOLU,
show=1,
return_covariance=True):
"""
Run harmonisation for a given input satellite sensor match-up data set configuration
"""
# Default to save residual data
res = True
################################################################################################################
# 1. Read Harmonisation Matchup Data
################################################################################################################
print "\nOpening data..."
HData = open_matchup(dataset_dir)
HData.setSensorData(sensor_data)
print "Complete"
print "\nData Info"
print "=========="
print "Reference Sensors - ", [
str(sensor) for sensor in HData.idx['sensors']
if sensor not in HData.idx["parameter_sensor"]
]
print "Harmonising Sensors - ", [
str(sensor) for sensor in HData.idx['sensors']
if sensor in HData.idx["parameter_sensor"]
]
print "Total Match-Ups - ", HData.idx['cNm'][-1]
print "Total Sensor State Data Values - ", HData.idx['idx'][-1]
print "Total Harmonisation Paramaters - ", len(
HData.idx['parameter_sensor'])
################################################################################################################
# 2. Perform harmonisation
################################################################################################################
print "\nBeginning Harmonisation..."
Harmonisation = HarmonisationEIV()
HarmonisationOutput = Harmonisation.run(HData,
pc_input=pc_input,
save_pc=save_pc,
gn_input=gn_input,
save_gn=save_gn,
show=show)
print "Final Solution:"
print HarmonisationOutput.parameter
print HarmonisationOutput.parameter_covariance_matrix
################################################################################################################
# 3. Write data to file
################################################################################################################
print 'Writing data to file...'
# Add metadata
startDate = ""
endDate = ""
HarmonisationOutput.additional_attributes = {
"software":
software_cfg['software'],
"software_version":
software_cfg['version'],
"software_tag":
software_cfg['tag'],
"job_id":
software_cfg["job_id"],
"matchup_dataset":
"_".join((software_cfg['matchup_dataset'], startDate, endDate))
}
# Write data
fname_output = "_".join(
("harm", software_cfg['software'], software_cfg['version'],
software_cfg['tag'], software_cfg["job_id"],
software_cfg['matchup_dataset'])) + ".nc"
HarmonisationOutput.save(pjoin(output_dir, fname_output),
save_residuals=res)
print "\nOutput Data Written To:"
print ">", output_dir
def setup_open_matchup():
matchup = open_matchup(DATASET_PATHS)
return matchup
def setup_open_matchup():
from harmonisation.core.matchup.matchupIO.MatchUpWriter import MatchUpWriter
writer = MatchUpWriter()
matchup = open_matchup(DATASET_PATHS)
return writer, matchup
def run(self):
"""
Generates plots for dataset specified by paths defined object attributes
"""
# 1. Read Match-up Data and Harmonisation Result Data
self.logger.info("Match-up Dataset: " + abspath(self.dataset_dir))
self.logger.debug("Opening Match-up Dataset...")
MatchUpData = open_matchup(self.dataset_dir, open_uncertainty=False)
MatchUpData.setSensorData(self.sensor_data)
self.logger.debug("Complete")
self.logger.info("Harmonisation Output Dataset: " + self.output_dir)
self.logger.debug("Opening Harmonisation Result Dataset...")
HarmResult = HarmonisationResult(abspath(self.output_dir),
open_residuals=False)
self.logger.debug("Complete")
# 2. Make plots
self.logger.debug("Generating Plots...")
HarmVisOp = HarmonisationVis(MatchUpData, HarmResult)
# Define plot directories
overview_plots_dir = pjoin(self.plots_dir, "overview")
diagnostic_plots_dir = pjoin(self.plots_dir, "diagnostics")
diagnostic_kres_harmonised_plots_dir = pjoin(diagnostic_plots_dir,
"kres_harmonised")
diagnostic_kres_harmonised_X_plots_dir = pjoin(
diagnostic_kres_harmonised_plots_dir, "sensor_state_variables")
diagnostic_kres_harmonised_x_plots_dir = pjoin(
diagnostic_kres_harmonised_plots_dir, "additional_variables")
diagnostic_kres_nom_harm_plots_dir = pjoin(diagnostic_plots_dir,
"kres_nom_harm")
diagnostic_kres_nom_harm_X_plots_dir = pjoin(
diagnostic_kres_nom_harm_plots_dir, "sensor_state_variables")
diagnostic_kres_nom_harm_x_plots_dir = pjoin(
diagnostic_kres_nom_harm_plots_dir, "additional_variables")
# Make plot directories
try_makedirs(overview_plots_dir)
try_makedirs(diagnostic_plots_dir)
try_makedirs(diagnostic_kres_harmonised_plots_dir)
try_makedirs(diagnostic_kres_harmonised_X_plots_dir)
try_makedirs(diagnostic_kres_harmonised_x_plots_dir)
try_makedirs(diagnostic_kres_nom_harm_plots_dir)
try_makedirs(diagnostic_kres_nom_harm_X_plots_dir)
try_makedirs(diagnostic_kres_nom_harm_x_plots_dir)
# 1. Overview plots
# a. Kres vs. time
# Determine plot limits
kres_nom = concatenate(HarmVisOp.get_kres_nom_nom_musamples())
kres = concatenate(HarmVisOp.get_kres_harm_harm_musamples())
kres_max = max(
[max(kres),
max(kres_nom),
abs(min(kres_nom)),
abs(min(kres))])
kres_ylim = [-kres_max * 1.2, kres_max * 1.2]
# Plot
HarmVisOp.plot_kres_nom_nom_scatter(
overview_plots_dir) #, ylim=kres_ylim)
HarmVisOp.plot_kres_nom_nom_scatter_monthlymean(
overview_plots_dir) #, ylim=kres_ylim)
HarmVisOp.plot_kres_harm_harm_scatter(
overview_plots_dir) #, ylim=kres_ylim)
HarmVisOp.plot_kres_harm_harm_scatter_monthlymean(
overview_plots_dir) #, ylim=kres_ylim)
HarmVisOp.plot_kres_nom_harm_scatter(overview_plots_dir)
HarmVisOp.plot_kres_nom_harm_scatter_monthlymean(overview_plots_dir)
# b. Nominal L vs. Nominal L1
HarmVisOp.plot_L1_nom_v_L2_nom_scatter(overview_plots_dir)
HarmVisOp.plot_L1_harm_v_L2_harm_scatter(overview_plots_dir)
# c. Kres vs. sensor state variables, X
HarmVisOp.plot_kres_harm_harm_X_binned_line(
diagnostic_kres_harmonised_X_plots_dir)
HarmVisOp.plot_kres_nom_harm_X_binned_line(
diagnostic_kres_nom_harm_X_plots_dir)
HarmVisOp.plot_kres_harm_harm_additional_variables_binned_line(
diagnostic_kres_harmonised_x_plots_dir)
HarmVisOp.plot_kres_nom_harm_additional_variables_binned_line(
diagnostic_kres_nom_harm_x_plots_dir)
self.logger.debug("Complete")
self.logger.info("Plots written to: " + abspath(self.plots_dir))