def test_remove_empty_matchups(self):
data = Data(self.path + 'resources/test_single_model_fill_values.nc')
me = MatchupEngine(data)
matchups = me.find_all_matchups()
self.assertEqual(3, len(matchups))
matchups = me.remove_empty_matchups(matchups)
self.assertEqual(2, len(matchups))
def calculate_statistics(model_name, ref_name, data, config=None):
"""
Calculates the statistics for the given model and reference variables located in the data file. Calculation will be
performed according to the provided configuration.
@param model_name: the name of the model variable.
@param ref_name: the name of the reference variable.
@param data: the input data object.
@param config: the optional configuration.
@return: a dictionary of statistics.
"""
if config is None:
config = get_default_config()
is_gridded = len(data.get_reference_dimensions(ref_name)) > 1
if is_gridded:
reference_values, model_values = data.get_values(ref_name, model_name)
unit = data.unit(model_name)
return processor.calculate_statistics(model_values, reference_values, model_name, ref_name, unit, config)
me = MatchupEngine(data, config)
matchups = me.find_all_matchups()
if config.remove_empty_matchups:
matchups = me.remove_empty_matchups(matchups)
if len(matchups) == 0:
print("No matchups found; maybe allow higher maximum time delta.")
return
unit = data.unit(model_name)
return calculate_statistics_from_matchups(matchups, model_name, ref_name, data, unit, config=None)
def test_remove_empty_matchups(self):
data = Data(self.path + 'resources/test_single_model_fill_values.nc')
me = MatchupEngine(data)
matchups = me.find_all_matchups()
self.assertEqual(3, len(matchups))
matchups = me.remove_empty_matchups(matchups)
self.assertEqual(2, len(matchups))
def get_matchups(data, config=None):
"""
Returns all matchups in the given dataset.
@param data: the source data file.
@param config: the optional configuration.
@return: all matchups.
"""
me = MatchupEngine(data, config)
return me.find_all_matchups()
def get_matchups(data, config=None):
"""
Returns all matchups in the given dataset.
@param data: the source data file.
@param config: the optional configuration.
@return: all matchups.
"""
me = MatchupEngine(data, config)
return me.find_all_matchups()
def remove_matchups_with_empty_model_data(data, config, matchups):
"""
Returns all matchups from the list that have at least one valid model data entry.
@param data: the source data file.
@param config: the optional configuration.
@param matchups: the matchups to filter.
@return: the filtered matchups.
"""
me = MatchupEngine(data, config)
return me.remove_empty_matchups(matchups)
def remove_matchups_with_empty_model_data(data, config, matchups):
"""
Returns all matchups from the list that have at least one valid model data entry.
@param data: the source data file.
@param config: the optional configuration.
@param matchups: the matchups to filter.
@return: the filtered matchups.
"""
me = MatchupEngine(data, config)
return me.remove_empty_matchups(matchups)
def test_find_all_matchups(self):
me = MatchupEngine(self.data, Configuration())
all_matchups = me.find_all_matchups()
self.assertIsNotNone(all_matchups)
expected_matchup_count = 3 #reference_records
self.assertEqual(expected_matchup_count, len(all_matchups))
matchup = all_matchups[0]
self.assertAlmostEqual(55.21, matchup.reference_record.lat, 5)
self.assertAlmostEqual(5.31, matchup.reference_record.lon, 5)
self.assertEqual(1261440250, matchup.reference_record.time)
self.assertAlmostEqual(0.0012, matchup.reference_record.depth, 5)
def test_find_all_matchups(self):
me = MatchupEngine(self.data, Configuration())
all_matchups = me.find_all_matchups()
self.assertIsNotNone(all_matchups)
expected_matchup_count = 3 #reference_records
self.assertEqual(expected_matchup_count, len(all_matchups))
matchup = all_matchups[0]
self.assertAlmostEqual(55.21, matchup.reference_record.lat, 5)
self.assertAlmostEqual(5.31, matchup.reference_record.lon, 5)
self.assertEqual(1261440250, matchup.reference_record.time)
self.assertAlmostEqual(0.0012, matchup.reference_record.depth, 5)
def test_find_matchups_single(self):
reference_record = ReferenceRecord(0, 55.20123, 6.30048, 1261447205, 0.0020015)
config = Configuration(time_delta=10, depth_delta=0.0001)
me = MatchupEngine(self.data, config)
matchups = me.find_matchups(reference_record)
self.assertEqual(1, len(matchups))
matchup = matchups[0]
self.assertIsNotNone(matchup)
self.assertAlmostEqual(55.20123, matchup.reference_record.lat)
self.assertAlmostEqual(6.30048, matchup.reference_record.lon)
self.assertAlmostEqual(1261447205, matchup.reference_record.time)
self.assertAlmostEqual(0.0020015, matchup.reference_record.depth)
def test_find_matchups_single(self):
reference_record = ReferenceRecord(0, 55.20123, 6.30048, 1261447205,
0.0020015)
config = Configuration(time_delta=10, depth_delta=0.0001)
me = MatchupEngine(self.data, config)
matchups = me.find_matchups(reference_record)
self.assertEqual(1, len(matchups))
matchup = matchups[0]
self.assertIsNotNone(matchup)
self.assertAlmostEqual(55.20123, matchup.reference_record.lat)
self.assertAlmostEqual(6.30048, matchup.reference_record.lon)
self.assertAlmostEqual(1261447205, matchup.reference_record.time)
self.assertAlmostEqual(0.0020015, matchup.reference_record.depth)
def test_find_matchup(self):
reference_record = ReferenceRecord(0, 55.3, 5.5, 1261440252, 0.0012)
me = MatchupEngine(self.data, Configuration())
matchups = me.find_matchups(reference_record)
self.assertEqual(1, len(matchups))
matchup = matchups[0]
self.assertIsNotNone(matchup)
self.assertEqual(55.3, matchup.reference_record.lat)
self.assertEqual(5.5, matchup.reference_record.lon)
self.assertEqual(1261440252, matchup.reference_record.time)
self.assertEqual(0.0012, matchup.reference_record.depth)
self.assertAlmostEqual(0.1111, matchup.get_model_value('chl', self.data), 5)
self.assertAlmostEqual(1.1111, matchup.get_model_value('sst', self.data), 5)
self.assertAlmostEqual(0.1, matchup.get_ref_value('chl_ref', self.data), 5)
def test_find_matchups_single_no_depth(self):
data = Data(self.path + 'resources/test_without_depth.nc')
config = Configuration(time_delta=10)
me = MatchupEngine(data, config)
reference_record = ReferenceRecord(0, 55.20123, 6.30048, 1261447205, None)
matchups = me.find_matchups(reference_record)
self.assertEqual(1, len(matchups))
matchup = matchups[0]
self.assertIsNotNone(matchup)
self.assertAlmostEqual(55.20123, matchup.reference_record.lat)
self.assertAlmostEqual(6.30048, matchup.reference_record.lon)
self.assertAlmostEqual(1261447205, matchup.reference_record.time)
self.assertIsNone(matchup.reference_record.depth)
def test_find_matchups_single_no_depth(self):
data = Data(self.path + 'resources/test_without_depth.nc')
config = Configuration(time_delta=10)
me = MatchupEngine(data, config)
reference_record = ReferenceRecord(0, 55.20123, 6.30048, 1261447205,
None)
matchups = me.find_matchups(reference_record)
self.assertEqual(1, len(matchups))
matchup = matchups[0]
self.assertIsNotNone(matchup)
self.assertAlmostEqual(55.20123, matchup.reference_record.lat)
self.assertAlmostEqual(6.30048, matchup.reference_record.lon)
self.assertAlmostEqual(1261447205, matchup.reference_record.time)
self.assertIsNone(matchup.reference_record.depth)
def test_find_matchup(self):
reference_record = ReferenceRecord(0, 55.3, 5.5, 1261440252, 0.0012)
me = MatchupEngine(self.data, Configuration())
matchups = me.find_matchups(reference_record)
self.assertEqual(1, len(matchups))
matchup = matchups[0]
self.assertIsNotNone(matchup)
self.assertEqual(55.3, matchup.reference_record.lat)
self.assertEqual(5.5, matchup.reference_record.lon)
self.assertEqual(1261440252, matchup.reference_record.time)
self.assertEqual(0.0012, matchup.reference_record.depth)
self.assertAlmostEqual(0.1111,
matchup.get_model_value('chl', self.data), 5)
self.assertAlmostEqual(1.1111,
matchup.get_model_value('sst', self.data), 5)
self.assertAlmostEqual(0.1,
matchup.get_ref_value('chl_ref', self.data), 5)
def calculate_statistics(model_name, ref_name, data, config=None):
"""
Calculates the statistics for the given model and reference variables located in the data file. Calculation will be
performed according to the provided configuration.
@param model_name: the name of the model variable.
@param ref_name: the name of the reference variable.
@param data: the input data object.
@param config: the optional configuration.
@return: a dictionary of statistics.
"""
if config is None:
config = get_default_config()
is_gridded = len(data.get_reference_dimensions(ref_name)) > 1
if is_gridded:
reference_values, model_values = data.get_values(ref_name, model_name)
unit = data.unit(model_name)
return processor.calculate_statistics(model_values, reference_values,
model_name, ref_name, unit,
config)
me = MatchupEngine(data, config)
matchups = me.find_all_matchups()
if config.remove_empty_matchups:
matchups = me.remove_empty_matchups(matchups)
if len(matchups) == 0:
print("No matchups found; maybe allow higher maximum time delta.")
return
unit = data.unit(model_name)
return calculate_statistics_from_matchups(matchups,
model_name,
ref_name,
data,
unit,
config=None)
def test_find_pixel_positions_huge_max_delta(self):
me = MatchupEngine(self.data, Configuration())
pixel_position = me.find_matchup_position(55.8, 6.0)
np.assert_array_almost_equal((1, 0, 5.8, 55.2), pixel_position)
def test_find_matchups_with_gridded_reference_variable(self):
data = Data(self.path + 'resources/ogs_test_smaller.nc')
me = MatchupEngine(data, Configuration())
matchups = me.find_all_matchups()
self.assertIsNotNone(matchups)
self.assertEqual(6560, len(matchups))
def test_find_time_positions_small_delta(self):
me = MatchupEngine(self.data, Configuration(time_delta=6))
time_positions = me.find_matchup_times(1261447205)[0]
np.assert_array_almost_equal((1, 1261447200), time_positions)
def test_find_time_positions_huge_delta(self):
me = MatchupEngine(self.data, Configuration(time_delta=100000))
time_position = me.find_matchup_times(1261440250)[0]
np.assert_array_almost_equal((0, 1261440000), time_position)
def test_find_time_positions_small_delta(self):
me = MatchupEngine(self.data, Configuration(time_delta=6))
time_positions = me.find_matchup_times(1261447205)[0]
np.assert_array_almost_equal((1, 1261447200), time_positions)
def test_find_matchups_with_gridded_reference_variable(self):
data = Data(self.path + 'resources/ogs_test_smaller.nc')
me = MatchupEngine(data, Configuration())
matchups = me.find_all_matchups()
self.assertIsNotNone(matchups)
self.assertEqual(6560, len(matchups))
def test_find_pixel_positions_huge_max_delta(self):
me = MatchupEngine(self.data, Configuration())
pixel_position = me.find_matchup_position(55.8, 6.0)
np.assert_array_almost_equal((1, 0, 5.8, 55.2), pixel_position)
def test_exclude_reference_records_with_out_of_bounds_lats(self):
reference_record = ReferenceRecord(0, 54.1, 5.5, 1261440252, 0.0012)
me = MatchupEngine(self.data, Configuration())
matchups = me.find_matchups(reference_record)
self.assertEqual(0, len(matchups))
def test_find_time_positions_huge_delta(self):
me = MatchupEngine(self.data, Configuration(time_delta=100000))
time_position = me.find_matchup_times(1261440250)[0]
np.assert_array_almost_equal((0, 1261440000), time_position)
def test_exclude_reference_records_with_out_of_bounds_lats(self):
reference_record = ReferenceRecord(0, 54.1, 5.5, 1261440252, 0.0012)
me = MatchupEngine(self.data, Configuration())
matchups = me.find_matchups(reference_record)
self.assertEqual(0, len(matchups))
def setUp(self):
path = os.path.dirname(os.path.realpath(__file__)) + '/../'
self.data = Data(path + 'resources/test.nc')
self.config = Configuration(time_delta=86400, depth_delta=12, ddof=0, alpha=1, beta=1)
self.me = MatchupEngine(self.data, self.config)
def main():
parsed_args = parse_arguments(sys.argv[1:])
config = Configuration(properties_file_name=parsed_args.config, target_dir=parsed_args.output_dir,
target_prefix=parsed_args.prefix)
file_handler = setup_logging(config)
if parsed_args.reference_file is not None:
data = Data(parsed_args.path, parsed_args.reference_file, config.max_cache_size)
else:
data = Data(parsed_args.path, max_cache_size=config.max_cache_size)
output = Output(config=config)
matchups = None
if data.has_one_dim_ref_var():
me = MatchupEngine(data, config)
matchups = me.find_all_matchups()
if not matchups:
logging.warning('No matchups found. System will exit.')
exit(0)
if config.remove_empty_matchups:
matchups = me.remove_empty_matchups(matchups)
if not os.name == 'nt':
logging.debug('Memory after matchups have been found: %s' % resource.getrusage(resource.RUSAGE_SELF).ru_maxrss)
matchup_count = 0 if matchups is None else len(matchups)
collected_statistics = {}
density_plot_files = []
target_files = []
density_plots = {}
for (model_name, ref_name) in parsed_args.variable_mappings:
unit = data.unit(model_name)
is_gridded = len(data.get_reference_dimensions(ref_name)) > 1
if is_gridded:
reference_values, model_values = data.get_values(ref_name, model_name)
matchup_count += ma.count(reference_values)
else:
reference_values, model_values = utils.extract_values(matchups, data, ref_name, model_name)
reference_values, model_values = utils.harmonise(reference_values, model_values)
logging.debug('Compressing ref-variable %s' % ref_name)
reference_values = reference_values.compressed()
logging.debug('Compressing model-variable %s' % model_name)
model_values = model_values.compressed()
logging.info('Calculating statistics for \'%s\' with \'%s\'' % (model_name, ref_name))
stats = processor.calculate_statistics(model_values, reference_values, model_name, ref_name, unit, config)
collected_statistics[(model_name, ref_name)] = stats
if config.write_density_plots:
axis_min = min(stats['min'], stats['ref_min'])
axis_max = max(stats['p90'], stats['ref_p90'])
logging.info('Creating density plot for \'%s\' and \'%s\'' % (model_name, ref_name))
density_plots[model_name + ref_name] = output.density_plot(model_name, ref_name, model_values,
reference_values, config.density_plot_log_scaled,
None, axis_min, axis_max, data.unit(model_name))
if not os.name == 'nt':
logging.debug(
'Memory after statistics have been computed: %s' % resource.getrusage(resource.RUSAGE_SELF).ru_maxrss)
if config.write_csv:
csv_target_file = '%s/%sstatistics.csv' % (parsed_args.output_dir, config.target_prefix)
target_files.append(csv_target_file)
output.csv(data, parsed_args.variable_mappings, collected_statistics, matchup_count, matchups=matchups, source_file=parsed_args.path, target_file=csv_target_file)
logging.info('CSV output written to \'%s\'' % csv_target_file)
if matchups is not None:
matchup_filename = '%s_matchups.csv' % os.path.splitext(csv_target_file)[0]
logging.info('Matchups written to \'%s\'' % matchup_filename)
target_files.append(matchup_filename)
taylor_target_files = []
if config.write_taylor_diagrams:
taylor_target_file = '%s/%staylor.png' % (parsed_args.output_dir, config.target_prefix)
written_taylor_diagrams, d = output.taylor(list(collected_statistics.values()), taylor_target_file)
del d
if written_taylor_diagrams:
for written_taylor_diagram in written_taylor_diagrams:
logging.info('Taylor diagram written to \'%s\'' % written_taylor_diagram)
target_files.append(written_taylor_diagram)
taylor_target_files.append(written_taylor_diagram)
if config.write_density_plots:
for (model_name, ref_name) in parsed_args.variable_mappings:
density_target = '%s/density-%s-%s.png' % (parsed_args.output_dir, model_name, ref_name)
density_plot_files.append(density_target)
target_files.append(density_target)
output.write_density_plot(density_plots[model_name + ref_name], density_target)
logging.info('Density plot written to \'%s\'' % density_target)
target_diagram_file = None
if config.write_target_diagram:
target_diagram_file = '%s/%starget.png' % (parsed_args.output_dir, config.target_prefix)
output.target_diagram(list(collected_statistics.values()), target_diagram_file)
logging.info('Target diagram written to \'%s\'' % target_diagram_file)
target_files.append(target_diagram_file)
if config.write_xhtml:
xml_target_file = '%s/%sreport.xml' % (parsed_args.output_dir, config.target_prefix)
path = str(os.path.dirname(os.path.realpath(__file__))) + '/../resources/'
xsl = path + 'analysis-summary.xsl'
css = path + 'styleset.css'
xsl_target = '%s/%s' % (parsed_args.output_dir, os.path.basename(xsl))
css_target = '%s/%s' % (parsed_args.output_dir, os.path.basename(css))
output.xhtml(list(collected_statistics.values()), matchup_count, matchups, data, xml_target_file, taylor_target_files,
target_diagram_file, density_plot_files)
logging.info('XHTML report written to \'%s\'' % xml_target_file)
shutil.copy(xsl, parsed_args.output_dir)
logging.info('XHTML support file written to \'%s/%s\'' % (parsed_args.output_dir, 'analysis-summary.xsl'))
shutil.copy(css, parsed_args.output_dir)
logging.info('XHTML support file written to \'%s/%s\'' % (parsed_args.output_dir, 'styleset.xsl'))
target_files.append(xml_target_file)
target_files.append(xsl_target)
target_files.append(css_target)
if config.zip:
create_zip(target_files, config, file_handler, parsed_args)
logging.info('End of process')
