def setup_open(): from harmonisation.core.matchup.matchupToolbox.harmonisation.harmonisationVis import HarmonisationVis matchup = open_matchup(DATASET_PATHS, SENSOR_DATA_PATH) harmonisation_result = HarmonisationResult(OUTPUT_DIRECTORY) harm_vis = HarmonisationVis(matchup, harmonisation_result) try: makedirs(PLOTS_DIRECTORY) except OSError: pass return harm_vis
def run(self, verbose=True): """ Generates plots for dataset specified by paths defined object attributes """ # Initialise dataset_dir = self.dataset_dir sensor_data = self.sensor_data output_dir = self.output_dir plots_dir = self.plots_dir ################################################################################################################ # 1. Read Match-up Data and Harmonisation Result Data ################################################################################################################ print"Match-up Dataset:" print ">", dataset_dir print"\nSensor Data:" print ">", sensor_data print "\nHarmonisation Output:" print ">", output_dir print("\nOpening Files...") MatchUpData = open_matchup(dataset_dir, open_uncertainty=False) MatchUpData.setSensorData(sensor_data) HarmResult = HarmonisationResult(output_dir, open_residuals=False) if verbose: print HarmResult.parameter_sensors print HarmResult.parameter print HarmResult.parameter_covariance_matrix print "\n" ################################################################################################################ # 2. Make plots ################################################################################################################ print("Generating Plots...") HarmVisOp = HarmonisationVis(MatchUpData, HarmResult) HarmVisOp.plot_compare_calibration(plots_dir, HarmResult.parameter, HarmResult.parameter_covariance_matrix, verbose=verbose) print "\nPlots written to:", plots_dir
def run(self, dataset_dir, sensor_data, output_dir, pc_input=None, save_pc=None, gn_input=None, save_gn=None, software_cfg=None, tolPC=TOLPC, tol=TOL, tolA=TOLA, tolB=TOLB, tolU=TOLU, show=1, return_covariance=True): """ Run harmonisation for a given input satellite sensor match-up data set configuration """ # Default to save residual data res = True ################################################################################################################ # 1. Read Harmonisation Matchup Data ################################################################################################################ print "\nOpening data..." HData = open_matchup(dataset_dir) HData.setSensorData(sensor_data) print "Complete" print "\nData Info" print "==========" print "Reference Sensors - ", [ str(sensor) for sensor in HData.idx['sensors'] if sensor not in HData.idx["parameter_sensor"] ] print "Harmonising Sensors - ", [ str(sensor) for sensor in HData.idx['sensors'] if sensor in HData.idx["parameter_sensor"] ] print "Total Match-Ups - ", HData.idx['cNm'][-1] print "Total Sensor State Data Values - ", HData.idx['idx'][-1] print "Total Harmonisation Paramaters - ", len( HData.idx['parameter_sensor']) ################################################################################################################ # 2. Perform harmonisation ################################################################################################################ print "\nBeginning Harmonisation..." Harmonisation = HarmonisationEIV() HarmonisationOutput = Harmonisation.run(HData, pc_input=pc_input, save_pc=save_pc, gn_input=gn_input, save_gn=save_gn, show=show) print "Final Solution:" print HarmonisationOutput.parameter print HarmonisationOutput.parameter_covariance_matrix ################################################################################################################ # 3. Write data to file ################################################################################################################ print 'Writing data to file...' # Add metadata startDate = "" endDate = "" HarmonisationOutput.additional_attributes = { "software": software_cfg['software'], "software_version": software_cfg['version'], "software_tag": software_cfg['tag'], "job_id": software_cfg["job_id"], "matchup_dataset": "_".join((software_cfg['matchup_dataset'], startDate, endDate)) } # Write data fname_output = "_".join( ("harm", software_cfg['software'], software_cfg['version'], software_cfg['tag'], software_cfg["job_id"], software_cfg['matchup_dataset'])) + ".nc" HarmonisationOutput.save(pjoin(output_dir, fname_output), save_residuals=res) print "\nOutput Data Written To:" print ">", output_dir
def setup_open_matchup(): matchup = open_matchup(DATASET_PATHS) return matchup
def setup_open_matchup(): from harmonisation.core.matchup.matchupIO.MatchUpWriter import MatchUpWriter writer = MatchUpWriter() matchup = open_matchup(DATASET_PATHS) return writer, matchup
def run(self): """ Generates plots for dataset specified by paths defined object attributes """ # 1. Read Match-up Data and Harmonisation Result Data self.logger.info("Match-up Dataset: " + abspath(self.dataset_dir)) self.logger.debug("Opening Match-up Dataset...") MatchUpData = open_matchup(self.dataset_dir, open_uncertainty=False) MatchUpData.setSensorData(self.sensor_data) self.logger.debug("Complete") self.logger.info("Harmonisation Output Dataset: " + self.output_dir) self.logger.debug("Opening Harmonisation Result Dataset...") HarmResult = HarmonisationResult(abspath(self.output_dir), open_residuals=False) self.logger.debug("Complete") # 2. Make plots self.logger.debug("Generating Plots...") HarmVisOp = HarmonisationVis(MatchUpData, HarmResult) # Define plot directories overview_plots_dir = pjoin(self.plots_dir, "overview") diagnostic_plots_dir = pjoin(self.plots_dir, "diagnostics") diagnostic_kres_harmonised_plots_dir = pjoin(diagnostic_plots_dir, "kres_harmonised") diagnostic_kres_harmonised_X_plots_dir = pjoin( diagnostic_kres_harmonised_plots_dir, "sensor_state_variables") diagnostic_kres_harmonised_x_plots_dir = pjoin( diagnostic_kres_harmonised_plots_dir, "additional_variables") diagnostic_kres_nom_harm_plots_dir = pjoin(diagnostic_plots_dir, "kres_nom_harm") diagnostic_kres_nom_harm_X_plots_dir = pjoin( diagnostic_kres_nom_harm_plots_dir, "sensor_state_variables") diagnostic_kres_nom_harm_x_plots_dir = pjoin( diagnostic_kres_nom_harm_plots_dir, "additional_variables") # Make plot directories try_makedirs(overview_plots_dir) try_makedirs(diagnostic_plots_dir) try_makedirs(diagnostic_kres_harmonised_plots_dir) try_makedirs(diagnostic_kres_harmonised_X_plots_dir) try_makedirs(diagnostic_kres_harmonised_x_plots_dir) try_makedirs(diagnostic_kres_nom_harm_plots_dir) try_makedirs(diagnostic_kres_nom_harm_X_plots_dir) try_makedirs(diagnostic_kres_nom_harm_x_plots_dir) # 1. Overview plots # a. Kres vs. time # Determine plot limits kres_nom = concatenate(HarmVisOp.get_kres_nom_nom_musamples()) kres = concatenate(HarmVisOp.get_kres_harm_harm_musamples()) kres_max = max( [max(kres), max(kres_nom), abs(min(kres_nom)), abs(min(kres))]) kres_ylim = [-kres_max * 1.2, kres_max * 1.2] # Plot HarmVisOp.plot_kres_nom_nom_scatter( overview_plots_dir) #, ylim=kres_ylim) HarmVisOp.plot_kres_nom_nom_scatter_monthlymean( overview_plots_dir) #, ylim=kres_ylim) HarmVisOp.plot_kres_harm_harm_scatter( overview_plots_dir) #, ylim=kres_ylim) HarmVisOp.plot_kres_harm_harm_scatter_monthlymean( overview_plots_dir) #, ylim=kres_ylim) HarmVisOp.plot_kres_nom_harm_scatter(overview_plots_dir) HarmVisOp.plot_kres_nom_harm_scatter_monthlymean(overview_plots_dir) # b. Nominal L vs. Nominal L1 HarmVisOp.plot_L1_nom_v_L2_nom_scatter(overview_plots_dir) HarmVisOp.plot_L1_harm_v_L2_harm_scatter(overview_plots_dir) # c. Kres vs. sensor state variables, X HarmVisOp.plot_kres_harm_harm_X_binned_line( diagnostic_kres_harmonised_X_plots_dir) HarmVisOp.plot_kres_nom_harm_X_binned_line( diagnostic_kres_nom_harm_X_plots_dir) HarmVisOp.plot_kres_harm_harm_additional_variables_binned_line( diagnostic_kres_harmonised_x_plots_dir) HarmVisOp.plot_kres_nom_harm_additional_variables_binned_line( diagnostic_kres_nom_harm_x_plots_dir) self.logger.debug("Complete") self.logger.info("Plots written to: " + abspath(self.plots_dir))