def run_subcathmenttest_by_dtsec(self, dt_sec, step_end, step_start, report_steps='1..9999', wateruse_on=False): modules_to_unset = self.modules_to_unset if not wateruse_on else [] modules_to_set = self.modules_to_unset if wateruse_on else [] # long run entire domain path_out_domain = mk_path_out('data/LF_ETRS89_UseCase/out/longrun_domain{}'.format(dt_sec)) settings_longrun = self.setoptions(self.settings_files['cold'], opts_to_unset=modules_to_unset, opts_to_set=modules_to_set, vars_to_set={'StepStart': step_start, 'StepEnd': step_end, 'PathOut': path_out_domain, 'ReportSteps': report_steps, 'DtSec': dt_sec}) # ** execute lisfloodexe(settings_longrun) # long run entire on subdomain path_out_subdomain = mk_path_out('data/LF_ETRS89_UseCase/out/longrun_subdomain{}'.format(dt_sec)) settings_longrun_subdomain = self.setoptions(self.settings_files['cold'], opts_to_unset=modules_to_unset, vars_to_set={'StepStart': step_start, 'StepEnd': step_end, 'PathOut': path_out_subdomain, 'ReportSteps': report_steps, 'MaskMap': '$(PathRoot)/maps/subcatchment_mask.map', 'DtSec': dt_sec}) # ** execute lisfloodexe(settings_longrun_subdomain) # ****** compare ******* # compare using the last maskmap (subcatchment_mask.map) settings = LisSettings.instance() nc_comparator = NetCDFComparator(settings.maskpath, array_equal=True) nc_comparator.compare_dirs(path_out_subdomain, path_out_domain)
def test_dates_steps_6h(self): settings_a = self.setoptions(self.settings_files['full'], opts_to_set=['repStateMaps', 'repEndMaps', 'repDischargeMaps', 'repSnowMaps', 'repLZMaps', 'repUZMaps'], vars_to_set={'StepStart': '30/07/2016 06:00', 'StepEnd': '31/07/2016 18:00', 'DtSec': '21600', 'PathOut': '$(PathRoot)/out/1'} ) mk_path_out('data/LF_ETRS89_UseCase/out/1') lisfloodexe(settings_a) settings_b = self.setoptions(self.settings_files['full'], opts_to_set=['repStateMaps', 'repEndMaps', 'repDischargeMaps', 'repSnowMaps', 'repLZMaps', 'repUZMaps'], vars_to_set={'StepStart': 24225, 'StepEnd': 24231, 'DtSec': '21600', 'PathOut': '$(PathRoot)/out/2'}) mk_path_out('data/LF_ETRS89_UseCase/out/2') lisfloodexe(settings_b) assert settings_a.step_start_int == 24225 assert settings_a.step_end_int == 24231 assert settings_a.step_start == settings_a.step_start_dt.strftime('%d/%m/%Y %H:%M') assert settings_a.step_end == settings_a.step_end_dt.strftime('%d/%m/%Y %H:%M') assert settings_b.step_start_dt == datetime.datetime(2016, 7, 30, 6, 0) assert settings_b.step_end_dt == datetime.datetime(2016, 7, 31, 18, 0) comparator = NetCDFComparator(settings_a.maskpath, array_equal=True) out_a = settings_a.output_dir out_b = settings_b.output_dir comparator.compare_dirs(out_a, out_b)
def test_netcdfcomp_dirs(self): comp = NetCDFComparator('tests/data/area.nc', for_testing=True) comp.compare_dirs('tests/data/folder_a/', 'tests/data/folder_b/', skip_missing=True) with pytest.raises(AssertionError) as excinfo: comp.compare_dirs('tests/data/folder_a/', 'tests/data/folder_b/', skip_missing=False) assert 'is missing in tests/data/folder_b' in str(excinfo.value)
def main(cliargs): parser = ParserHelpOnError( description='Compare netCDF outputs: {}'.format(version)) parser.add_args() args = parser.parse_args(cliargs) dataset_a = args.dataset_a.as_posix() dataset_b = args.dataset_b.as_posix() maskfile = args.maskarea.as_posix() array_equal = args.array_equal skip_missing = args.skip_missing atol = args.atol rtol = args.rtol max_diff_perc = args.max_diff_percentage max_large_diff_perc = args.max_largediff_percentage save_diff_files = args.save_diffs comparator = NetCDFComparator(maskfile, array_equal=array_equal, for_testing=False, atol=atol, rtol=rtol, max_perc_diff=max_diff_perc, max_perc_large_diff=max_large_diff_perc, save_diff_files=save_diff_files) logger.info('\n\nComparing %s and %s\n\n ', dataset_a, dataset_b) errors = comparator.compare_dirs(dataset_a, dataset_b, skip_missing=skip_missing) for i, e in enumerate(errors): logger.error('%d - %s', i, e)
def run_warmstart_by_dtsec(self, dt_sec, step_end, step_start, report_steps='1..9999'): modules_to_unset = [ 'simulateReservoirs', 'repsimulateReservoirs', 'wateruse', 'useWaterDemandAveYear', 'wateruseRegion', ] check_every = 13 # steps # init path_out_init = mk_path_out( 'data/LF_ETRS89_UseCase/out/init{}'.format(dt_sec)) settings_prerun = self.setoptions(self.settings_files['prerun'], opts_to_unset=modules_to_unset, vars_to_set={ 'DtSec': dt_sec, 'PathOut': path_out_init, 'StepStart': step_start, 'ReportSteps': report_steps, 'StepEnd': step_end }) step_end_dt = settings_prerun.step_end_dt # ** execute lisfloodexe(settings_prerun) # long run lzavin_path = settings_prerun.binding['LZAvInflowMap'] avgdis_path = settings_prerun.binding['AvgDis'] path_out_reference = mk_path_out( 'data/LF_ETRS89_UseCase/out/longrun_reference{}'.format(dt_sec)) settings_longrun = self.setoptions(self.settings_files['cold'], opts_to_unset=modules_to_unset, vars_to_set={ 'StepStart': step_start, 'StepEnd': step_end, 'LZAvInflowMap': lzavin_path, 'PathOut': path_out_reference, 'AvgDis': avgdis_path, 'ReportSteps': report_steps, 'DtSec': dt_sec }) # ** execute lisfloodexe(settings_longrun) # warm run (1. Cold start) run_number = 1 cold_start_step_end = step_start path_out = mk_path_out('data/LF_ETRS89_UseCase/out/run{}_{}'.format( dt_sec, run_number)) settings_coldstart = self.setoptions(self.settings_files['cold'], opts_to_unset=modules_to_unset, vars_to_set={ 'StepStart': step_start, 'StepEnd': cold_start_step_end, 'LZAvInflowMap': lzavin_path, 'PathOut': path_out, 'AvgDis': avgdis_path, 'ReportSteps': report_steps, 'DtSec': dt_sec }) # ** execute lisfloodexe(settings_coldstart) # warm run (2. single step warm start/stop with initial conditions from previous run) prev_settings = settings_coldstart warm_step_start = prev_settings.step_end_dt + timedelta(seconds=dt_sec) warm_step_end = warm_step_start timestep_init = prev_settings.step_end_dt.strftime('%d/%m/%Y %H:%M') nc_comparator = NetCDFComparator(settings_prerun.maskpath) tss_comparator = TSSComparator(array_equal=True) while warm_step_start <= step_end_dt: run_number += 1 path_init = prev_settings.output_dir path_out = mk_path_out( 'data/LF_ETRS89_UseCase/out/run{}_{}'.format( dt_sec, run_number)) settings_warmstart = self.setoptions( self.settings_files['warm'], opts_to_unset=modules_to_unset, vars_to_set={ 'StepStart': warm_step_start.strftime('%d/%m/%Y %H:%M'), 'StepEnd': warm_step_end.strftime('%d/%m/%Y %H:%M'), 'LZAvInflowMap': lzavin_path, 'PathOut': path_out, 'PathInit': path_init, 'timestepInit': timestep_init, 'AvgDis': avgdis_path, 'ReportSteps': report_steps, 'DtSec': dt_sec }) # ** execute lisfloodexe(settings_warmstart) # checking values at current timestep (using datetime) if not (run_number % check_every): # ****** compare ******* # compare every 13 timesteps to speed up test timestep_dt = settings_warmstart.step_end_dt # NetCDFComparator takes datetime.datetime as timestep timestep = settings_warmstart.step_end_int nc_comparator.compare_dirs(path_out, path_out_reference, timestep=timestep_dt) tss_comparator.compare_dirs(path_out, path_out_reference, timestep=timestep) # setup for next warm start/stop prev_settings = settings_warmstart warm_step_start = prev_settings.step_end_dt + timedelta( seconds=dt_sec) warm_step_end = warm_step_start timestep_init = prev_settings.step_end_dt.strftime( '%d/%m/%Y %H:%M')