def test_max_bounds_for_target_diagram(self): c = Configuration() self.assertIsNone(c.target_diagram_bounds) c = Configuration(target_diagram_bounds=[-2.0, 5.0, -3.0, None]) self.assertEqual(-2.0, c.target_diagram_bounds[0]) self.assertEqual(5.0, c.target_diagram_bounds[1]) self.assertEqual(-3.0, c.target_diagram_bounds[2]) self.assertIsNone(c.target_diagram_bounds[3])
def create_config(filename): """ Returns a configuration according to the properties file. @param filename: the path to the properties file used to load configuration from. @return: an object of type 'Configuration'. """ return Configuration(properties_file_name=filename)
def test_find_all_matchups(self): me = MatchupEngine(self.data, Configuration()) all_matchups = me.find_all_matchups() self.assertIsNotNone(all_matchups) expected_matchup_count = 3 #reference_records self.assertEqual(expected_matchup_count, len(all_matchups)) matchup = all_matchups[0] self.assertAlmostEqual(55.21, matchup.reference_record.lat, 5) self.assertAlmostEqual(5.31, matchup.reference_record.lon, 5) self.assertEqual(1261440250, matchup.reference_record.time) self.assertAlmostEqual(0.0012, matchup.reference_record.depth, 5)
def test_find_matchups_single(self): reference_record = ReferenceRecord(0, 55.20123, 6.30048, 1261447205, 0.0020015) config = Configuration(time_delta=10, depth_delta=0.0001) me = MatchupEngine(self.data, config) matchups = me.find_matchups(reference_record) self.assertEqual(1, len(matchups)) matchup = matchups[0] self.assertIsNotNone(matchup) self.assertAlmostEqual(55.20123, matchup.reference_record.lat) self.assertAlmostEqual(6.30048, matchup.reference_record.lon) self.assertAlmostEqual(1261447205, matchup.reference_record.time) self.assertAlmostEqual(0.0020015, matchup.reference_record.depth)
def test_find_matchups_single_no_depth(self): data = Data(self.path + 'resources/test_without_depth.nc') config = Configuration(time_delta=10) me = MatchupEngine(data, config) reference_record = ReferenceRecord(0, 55.20123, 6.30048, 1261447205, None) matchups = me.find_matchups(reference_record) self.assertEqual(1, len(matchups)) matchup = matchups[0] self.assertIsNotNone(matchup) self.assertAlmostEqual(55.20123, matchup.reference_record.lat) self.assertAlmostEqual(6.30048, matchup.reference_record.lon) self.assertAlmostEqual(1261447205, matchup.reference_record.time) self.assertIsNone(matchup.reference_record.depth)
def test_find_matchup(self): reference_record = ReferenceRecord(0, 55.3, 5.5, 1261440252, 0.0012) me = MatchupEngine(self.data, Configuration()) matchups = me.find_matchups(reference_record) self.assertEqual(1, len(matchups)) matchup = matchups[0] self.assertIsNotNone(matchup) self.assertEqual(55.3, matchup.reference_record.lat) self.assertEqual(5.5, matchup.reference_record.lon) self.assertEqual(1261440252, matchup.reference_record.time) self.assertEqual(0.0012, matchup.reference_record.depth) self.assertAlmostEqual(0.1111, matchup.get_model_value('chl', self.data), 5) self.assertAlmostEqual(1.1111, matchup.get_model_value('sst', self.data), 5) self.assertAlmostEqual(0.1, matchup.get_ref_value('chl_ref', self.data), 5)
def test_initialisation_by_file(self): test_config = os.path.dirname(os.path.realpath(__file__)) + "/../resources/test.properties" c = Configuration(alpha=5, ddof=2, log_level='INFO', properties_file_name=test_config) self.assertEqual(5, c.alpha) self.assertEqual(0.5, c.beta) self.assertEqual(2, c.ddof) self.assertEqual(3, c.depth_delta) self.assertEqual(200, c.time_delta) self.assertEqual(logging.INFO, c.log_level) # does not appear in test.properties, so it is taken from default file self.assertEqual(False, c.zip) # does not appear in test.properties, so it is taken from default file self.assertEqual(True, c.show_legends) self.assertEqual(False, c.show_negative_corrcoeff) # does not appear in test.properties, so it is taken from default file self.assertEqual(os.getcwd(), c.target_dir) self.assertEqual('benchmark_', c.target_prefix) self.assertEqual(True, c.write_taylor_diagrams) self.assertEqual(None, c.log_file) self.assertEqual(False, c.write_csv) self.assertEqual(False, c.write_xhtml)
def test_split_up_criteria(self): c = Configuration(split_diagrams='nu') self.assertTrue(c.split_on_unit) self.assertTrue(c.split_on_name) c = Configuration(split_diagrams='') self.assertFalse(c.split_on_unit) self.assertFalse(c.split_on_name) c = Configuration() self.assertTrue(c.split_on_unit) self.assertFalse(c.split_on_name) self.assertRaises(ValueError, lambda: Configuration(split_diagrams='cowabunga!')) self.assertRaises(ValueError, lambda: Configuration(split_diagrams='cow')) self.assertRaises(ValueError, lambda: Configuration(split_diagrams='p')) self.assertRaises(ValueError, lambda: Configuration(split_diagrams='ssuv'))
def test_initialisation(self): c = Configuration(alpha=5, ddof=2, show_legends=False, write_taylor_diagrams=False) self.assertEqual(5, c.alpha) self.assertEqual(1, c.beta) self.assertEqual(2, c.ddof) self.assertEqual(12, c.depth_delta) self.assertEqual(86400, c.time_delta) self.assertEqual(logging.INFO, c.log_level) self.assertEqual(False, c.zip) self.assertEqual(False, c.show_negative_corrcoeff) self.assertEqual(False, c.show_legends) self.assertEqual(os.getcwd(), c.target_dir) self.assertEqual('benchmark_', c.target_prefix) self.assertEqual('\t', c.separator) self.assertEqual(False, c.write_taylor_diagrams) self.assertEqual(None, c.log_file) self.assertEqual(True, c.write_csv) self.assertEqual(True, c.write_xhtml) self.assertEqual(True, c.write_density_plots)
def test_max_cache_size(self): c = Configuration() self.assertEqual(1024, c.max_cache_size) c = Configuration(max_cache_size=15) self.assertEqual(15, c.max_cache_size)
def test_find_time_positions_small_delta(self): me = MatchupEngine(self.data, Configuration(time_delta=6)) time_positions = me.find_matchup_times(1261447205)[0] np.assert_array_almost_equal((1, 1261447200), time_positions)
def test_find_time_positions_huge_delta(self): me = MatchupEngine(self.data, Configuration(time_delta=100000)) time_position = me.find_matchup_times(1261440250)[0] np.assert_array_almost_equal((0, 1261440000), time_position)
def test_find_pixel_positions_huge_max_delta(self): me = MatchupEngine(self.data, Configuration()) pixel_position = me.find_matchup_position(55.8, 6.0) np.assert_array_almost_equal((1, 0, 5.8, 55.2), pixel_position)
def test_find_matchups_with_gridded_reference_variable(self): data = Data(self.path + 'resources/ogs_test_smaller.nc') me = MatchupEngine(data, Configuration()) matchups = me.find_all_matchups() self.assertIsNotNone(matchups) self.assertEqual(6560, len(matchups))
def test_exclude_reference_records_with_out_of_bounds_lats(self): reference_record = ReferenceRecord(0, 54.1, 5.5, 1261440252, 0.0012) me = MatchupEngine(self.data, Configuration()) matchups = me.find_matchups(reference_record) self.assertEqual(0, len(matchups))
def test_utilise_stddev_difference(self): c = Configuration() self.assertTrue(c.utilise_stddev_difference) c = Configuration(utilise_stddev_difference=False) self.assertFalse(c.utilise_stddev_difference)
def setUp(self): path = os.path.dirname(os.path.realpath(__file__)) + '/../' self.data = Data(path + 'resources/test.nc') self.config = Configuration(time_delta=86400, depth_delta=12, ddof=0, alpha=1, beta=1) self.me = MatchupEngine(self.data, self.config)
def test_use_absolute_standard_deviation(self): c = Configuration() self.assertFalse(c.use_absolute_standard_deviation) c = Configuration(use_absolute_standard_deviation=True) self.assertTrue(c.use_absolute_standard_deviation)
def test_normalise_target_diagram(self): c = Configuration() self.assertTrue(c.normalise_target_diagram) c = Configuration(normalise_target_diagram=False) self.assertFalse(c.normalise_target_diagram)
def test_write_target_diagram(self): c = Configuration() self.assertTrue(c.write_target_diagram) c = Configuration(write_target_diagram=False) self.assertFalse(c.write_target_diagram)
def test_log_file(self): c = Configuration(log_file='somewhere_to_log_into') self.assertEqual('somewhere_to_log_into', c.log_file)
def test_disabled_log_level(self): c = Configuration(log_level='DISABLED') self.assertEqual(100, c.log_level)
def test_wrong_log_level(self): self.assertRaises(RuntimeError, lambda: Configuration(log_level='LOG_EVERYTHING_PLEASE'))
def test_remove_empty_matchups(self): c = Configuration() self.assertTrue(c.remove_empty_matchups) c = Configuration(remove_empty_matchups=False) self.assertFalse(c.remove_empty_matchups)
def main(): parsed_args = parse_arguments(sys.argv[1:]) config = Configuration(properties_file_name=parsed_args.config, target_dir=parsed_args.output_dir, target_prefix=parsed_args.prefix) file_handler = setup_logging(config) if parsed_args.reference_file is not None: data = Data(parsed_args.path, parsed_args.reference_file, config.max_cache_size) else: data = Data(parsed_args.path, max_cache_size=config.max_cache_size) output = Output(config=config) matchups = None if data.has_one_dim_ref_var(): me = MatchupEngine(data, config) matchups = me.find_all_matchups() if not matchups: logging.warning('No matchups found. System will exit.') exit(0) if config.remove_empty_matchups: matchups = me.remove_empty_matchups(matchups) if not os.name == 'nt': logging.debug('Memory after matchups have been found: %s' % resource.getrusage(resource.RUSAGE_SELF).ru_maxrss) matchup_count = 0 if matchups is None else len(matchups) collected_statistics = {} density_plot_files = [] target_files = [] density_plots = {} for (model_name, ref_name) in parsed_args.variable_mappings: unit = data.unit(model_name) is_gridded = len(data.get_reference_dimensions(ref_name)) > 1 if is_gridded: reference_values, model_values = data.get_values(ref_name, model_name) matchup_count += ma.count(reference_values) else: reference_values, model_values = utils.extract_values(matchups, data, ref_name, model_name) reference_values, model_values = utils.harmonise(reference_values, model_values) logging.debug('Compressing ref-variable %s' % ref_name) reference_values = reference_values.compressed() logging.debug('Compressing model-variable %s' % model_name) model_values = model_values.compressed() logging.info('Calculating statistics for \'%s\' with \'%s\'' % (model_name, ref_name)) stats = processor.calculate_statistics(model_values, reference_values, model_name, ref_name, unit, config) collected_statistics[(model_name, ref_name)] = stats if config.write_density_plots: axis_min = min(stats['min'], stats['ref_min']) axis_max = max(stats['p90'], stats['ref_p90']) logging.info('Creating density plot for \'%s\' and \'%s\'' % (model_name, ref_name)) density_plots[model_name + ref_name] = output.density_plot(model_name, ref_name, model_values, reference_values, config.density_plot_log_scaled, None, axis_min, axis_max, data.unit(model_name)) if not os.name == 'nt': logging.debug( 'Memory after statistics have been computed: %s' % resource.getrusage(resource.RUSAGE_SELF).ru_maxrss) if config.write_csv: csv_target_file = '%s/%sstatistics.csv' % (parsed_args.output_dir, config.target_prefix) target_files.append(csv_target_file) output.csv(data, parsed_args.variable_mappings, collected_statistics, matchup_count, matchups=matchups, source_file=parsed_args.path, target_file=csv_target_file) logging.info('CSV output written to \'%s\'' % csv_target_file) if matchups is not None: matchup_filename = '%s_matchups.csv' % os.path.splitext(csv_target_file)[0] logging.info('Matchups written to \'%s\'' % matchup_filename) target_files.append(matchup_filename) taylor_target_files = [] if config.write_taylor_diagrams: taylor_target_file = '%s/%staylor.png' % (parsed_args.output_dir, config.target_prefix) written_taylor_diagrams, d = output.taylor(list(collected_statistics.values()), taylor_target_file) del d if written_taylor_diagrams: for written_taylor_diagram in written_taylor_diagrams: logging.info('Taylor diagram written to \'%s\'' % written_taylor_diagram) target_files.append(written_taylor_diagram) taylor_target_files.append(written_taylor_diagram) if config.write_density_plots: for (model_name, ref_name) in parsed_args.variable_mappings: density_target = '%s/density-%s-%s.png' % (parsed_args.output_dir, model_name, ref_name) density_plot_files.append(density_target) target_files.append(density_target) output.write_density_plot(density_plots[model_name + ref_name], density_target) logging.info('Density plot written to \'%s\'' % density_target) target_diagram_file = None if config.write_target_diagram: target_diagram_file = '%s/%starget.png' % (parsed_args.output_dir, config.target_prefix) output.target_diagram(list(collected_statistics.values()), target_diagram_file) logging.info('Target diagram written to \'%s\'' % target_diagram_file) target_files.append(target_diagram_file) if config.write_xhtml: xml_target_file = '%s/%sreport.xml' % (parsed_args.output_dir, config.target_prefix) path = str(os.path.dirname(os.path.realpath(__file__))) + '/../resources/' xsl = path + 'analysis-summary.xsl' css = path + 'styleset.css' xsl_target = '%s/%s' % (parsed_args.output_dir, os.path.basename(xsl)) css_target = '%s/%s' % (parsed_args.output_dir, os.path.basename(css)) output.xhtml(list(collected_statistics.values()), matchup_count, matchups, data, xml_target_file, taylor_target_files, target_diagram_file, density_plot_files) logging.info('XHTML report written to \'%s\'' % xml_target_file) shutil.copy(xsl, parsed_args.output_dir) logging.info('XHTML support file written to \'%s/%s\'' % (parsed_args.output_dir, 'analysis-summary.xsl')) shutil.copy(css, parsed_args.output_dir) logging.info('XHTML support file written to \'%s/%s\'' % (parsed_args.output_dir, 'styleset.xsl')) target_files.append(xml_target_file) target_files.append(xsl_target) target_files.append(css_target) if config.zip: create_zip(target_files, config, file_handler, parsed_args) logging.info('End of process')
def test_density_plot_bin_count(self): c = Configuration() self.assertFalse(c.density_plot_log_scaled) c = Configuration(density_plot_log_scaled=True) self.assertTrue(c.density_plot_log_scaled)