def test_ti_calculate(input_dict, expected_time_info): time_info = time_util.ti_calculate(input_dict) for key, value in expected_time_info.items(): assert (time_info[key] == value) time_info2 = time_util.ti_calculate(time_info) for key, value in expected_time_info.items(): assert (time_info[key] == value) assert (time_info2[key] == value)
def test_find_input_files(metplus_config, offsets, offset_to_find): pb = pb2nc_wrapper(metplus_config) # for valid 20190201_12, offsets 3 and 5, create files to find # in the fake input directory based on input template input_dict = {'valid': datetime.datetime(2019, 2, 1, 12)} fake_input_dir = os.path.join(pb.config.getdir('OUTPUT_BASE'), 'pbin') if not os.path.exists(fake_input_dir): os.makedirs(fake_input_dir) pb.c_dict['OBS_INPUT_DIR'] = fake_input_dir for offset in [3, 5]: input_dict['offset'] = int(offset * 3600) time_info = time_util.ti_calculate(input_dict) create_file = do_string_sub(pb.c_dict['OBS_INPUT_TEMPLATE'], **time_info) create_fullpath = os.path.join(fake_input_dir, create_file) open(create_fullpath, 'a').close() # unset offset in time dictionary so it will be computed del input_dict['offset'] # set offset list pb.c_dict['OFFSETS'] = offsets # look for input files based on offset list result = pb.find_input_files(input_dict) # check if correct offset file was found, if None expected, check against None if offset_to_find is None: assert result is None else: assert result['offset_hours'] == offset_to_find
def test_run_gen_vx_mask_twice(metplus_config): input_dict = {'valid': datetime.datetime.strptime("201802010000",'%Y%m%d%H%M'), 'lead': 0} time_info = time_util.ti_calculate(input_dict) wrap = gen_vx_mask_wrapper(metplus_config) wrap.c_dict['INPUT_TEMPLATE'] = '{valid?fmt=%Y%m%d%H}_ZENITH' wrap.c_dict['MASK_INPUT_TEMPLATES'] = ['LAT', 'LON'] wrap.c_dict['OUTPUT_DIR'] = os.path.join(wrap.config.getdir('OUTPUT_BASE'), 'GenVxMask_test') wrap.c_dict['OUTPUT_TEMPLATE'] = '{valid?fmt=%Y%m%d%H}_ZENITH_LAT_LON_MASK.nc' cmd_args = ["-type lat -thresh 'ge30&&le50'", "-type lon -thresh 'le-70&&ge-130' -intersection -name lat_lon_mask"] wrap.c_dict['COMMAND_OPTIONS'] = cmd_args wrap.run_at_time_all(time_info) expected_cmds = [f"{wrap.app_path} 2018020100_ZENITH LAT {wrap.config.getdir('OUTPUT_BASE')}/stage/gen_vx_mask/temp_0.nc {cmd_args[0]} -v 2", f"{wrap.app_path} {wrap.config.getdir('OUTPUT_BASE')}/stage/gen_vx_mask/temp_0.nc LON {wrap.config.getdir('OUTPUT_BASE')}/GenVxMask_test/2018020100_ZENITH_LAT_LON_MASK.nc {cmd_args[1]} -v 2"] test_passed = True if len(wrap.all_commands) != len(expected_cmds): print("Number of commands run is not the same as expected") assert(False) for (cmd, _), expected_cmd in zip(wrap.all_commands, expected_cmds): print(f" ACTUAL:{cmd}") print(f"EXPECTED:{expected_cmd}") if cmd != expected_cmd: test_passed = False assert(test_passed)
def test_setup_add_method(): rl = "OBS" pcw = pcp_combine_wrapper(rl) task_info = {} task_info['valid'] = datetime.datetime.strptime("2016090418", '%Y%m%d%H') time_info = time_util.ti_calculate(task_info) var_info = {} var_info['fcst_name'] = "APCP" var_info['obs_name'] = "ACPCP" var_info['fcst_extra'] = "" var_info['obs_extra'] = "" var_info['fcst_level'] = "A06" var_info['obs_level'] = "A06" input_dir = pcw.config.getdir( 'METPLUS_BASE') + "/internal_tests/data/accum" output_dir = pcw.config.getdir( 'OUTPUT_BASE') + "/internal_tests/data/fakeout" pcw.setup_add_method(time_info, var_info, rl) in_files = pcw.infiles out_file = pcw.get_output_path() if len(in_files) == 6 and \ input_dir+"/20160904/file.2016090418.01h" in in_files and \ input_dir+"/20160904/file.2016090417.01h" in in_files and \ input_dir+"/20160904/file.2016090416.01h" in in_files and \ input_dir+"/20160904/file.2016090415.01h" in in_files and \ input_dir+"/20160904/file.2016090414.01h" in in_files and \ input_dir+"/20160904/file.2016090413.01h" in in_files and \ out_file == output_dir+"/20160904/outfile.2016090418_A06h": assert True else: assert False
def test_find_obs_offset(metplus_config, offsets, expected_file, offset_seconds): config = metplus_config() pcw = CommandBuilder(config) v = {} v['obs_level'] = "6" task_info = {} task_info['valid'] = datetime.datetime.strptime("2020020112", '%Y%m%d%H') task_info['lead'] = 0 time_info = time_util.ti_calculate(task_info) pcw.c_dict['OFFSETS'] = offsets pcw.c_dict['OBS_INPUT_DIR'] = pcw.config.getdir( 'METPLUS_BASE') + "/internal_tests/data/obs" pcw.c_dict[ 'OBS_INPUT_TEMPLATE'] = "{da_init?fmt=%2H}z.prepbufr.tm{offset?fmt=%2H}.{da_init?fmt=%Y%m%d}" obs_file, time_info = pcw.find_obs_offset(time_info, v) print(f"OBSFILE: {obs_file}") print(f"EXPECTED FILE: {expected_file}") if expected_file is None: assert not obs_file else: assert (os.path.basename(obs_file) == expected_file and time_info['offset'] == offset_seconds)
def test_get_accumulation_1_to_6(): data_src = "OBS" pcw = pcp_combine_wrapper(data_src) input_dir = pcw.config.getdir( 'METPLUS_BASE') + "/internal_tests/data/accum" task_info = {} task_info['valid'] = datetime.datetime.strptime("2016090418", '%Y%m%d%H') time_info = time_util.ti_calculate(task_info) accum = 6 file_template = "{valid?fmt=%Y%m%d}/file.{valid?fmt=%Y%m%d%H}.{level?fmt=%HH}h" pcw.input_dir = input_dir pcw.build_input_accum_list(data_src, time_info) pcw.get_accumulation(time_info, accum, data_src) in_files = pcw.infiles if len(in_files) == 6 and \ input_dir+"/20160904/file.2016090418.01h" in in_files and \ input_dir+"/20160904/file.2016090417.01h" in in_files and \ input_dir+"/20160904/file.2016090416.01h" in in_files and \ input_dir+"/20160904/file.2016090415.01h" in in_files and \ input_dir+"/20160904/file.2016090414.01h" in in_files and \ input_dir+"/20160904/file.2016090413.01h" in in_files: assert True else: assert False
def test_get_field_info_list(conf_dict, expected_field_info_list): config = metplus_config() logger = logging.getLogger("dummy") data_type = 'OBS' for key, value in conf_dict.items(): config.set('config', key, value) input_dict = { 'valid': datetime.datetime.strptime("201802010000", '%Y%m%d%H%M'), 'lead': 0 } time_info = time_util.ti_calculate(input_dict) var_list = util.parse_var_list(config, time_info, data_type=data_type) rdp = RegridDataPlaneWrapper(config, logger) field_info_list = rdp.get_field_info_list(var_list, data_type, time_info) print(f"FIELD INFO LIST: {field_info_list}") print(f"EXPECTED FIELD INFO LIST: {expected_field_info_list}") is_good = True if len(field_info_list) != len(expected_field_info_list): assert (False) for actual_field, expected_field in zip(field_info_list, expected_field_info_list): for key, value in expected_field.items(): if actual_field[key] != value: print(f"{actual_field[key]} not equal to {value}") is_good = False
def test_run_rdp_once_per_field(): data_type = 'FCST' input_dict = { 'valid': datetime.datetime.strptime("201802010000", '%Y%m%d%H%M'), 'lead': 0 } time_info = time_util.ti_calculate(input_dict) var_list = [ { 'index': '1', 'fcst_name': 'FNAME1', 'fcst_level': 'A06' }, { 'index': '2', 'fcst_name': 'FNAME2', 'fcst_level': 'A03', 'fcst_output_name': 'OUTNAME2' }, ] wrap = rdp_wrapper() wrap.c_dict['ONCE_PER_FIELD'] = True wrap.c_dict[ 'FCST_OUTPUT_TEMPLATE'] = '{valid?fmt=%Y%m%d%H}_accum{level?fmt=%2H}.nc' wrap.c_dict['FCST_INPUT_TEMPLATE'] = '{valid?fmt=%Y%m%d%H}_ZENITH' wrap.c_dict['METHOD'] = 'BUDGET' wrap.c_dict['WIDTH'] = 2 wrap.c_dict['VERIFICATION_GRID'] = 'VERIF_GRID' wrap.c_dict['FCST_OUTPUT_DIR'] = os.path.join( wrap.config.getdir('OUTPUT_BASE'), 'RDP_test') wrap.run_at_time_once(time_info, var_list, data_type) expected_cmds = [ f"{wrap.app_path} -v 2 -method BUDGET -width 2 -field 'name=\"FNAME1\"; " "level=\"A06\";' -name FNAME1 2018020100_ZENITH \"VERIF_GRID\" " f"{wrap.config.getdir('OUTPUT_BASE')}/RDP_test/2018020100_accum06.nc", f"{wrap.app_path} -v 2 -method BUDGET -width 2 -field 'name=\"FNAME2\"; " "level=\"A03\";' -name OUTNAME2 2018020100_ZENITH \"VERIF_GRID\" " f"{wrap.config.getdir('OUTPUT_BASE')}/RDP_test/2018020100_accum03.nc", ] test_passed = True if len(wrap.all_commands) != len(expected_cmds): print("Number of commands run is not the same as expected") assert (False) for cmd, expected_cmd in zip(wrap.all_commands, expected_cmds): print(f" ACTUAL:{cmd}") print(f"EXPECTED:{expected_cmd}") if cmd != expected_cmd: test_passed = False assert (test_passed)
def test_find_data_not_a_path(metplus_config, data_type): config = metplus_config() pcw = CommandBuilder(config) task_info = {} task_info['valid'] = datetime.datetime.strptime("201802010000", '%Y%m%d%H%M') task_info['lead'] = 0 time_info = time_util.ti_calculate(task_info) pcw.c_dict[f'{data_type}FILE_WINDOW_BEGIN'] = 0 pcw.c_dict[f'{data_type}FILE_WINDOW_END'] = 0 pcw.c_dict[f'{data_type}INPUT_DIR'] = '' pcw.c_dict[f'{data_type}INPUT_TEMPLATE'] = 'G003' obs_file = pcw.find_data(time_info, var_info=None, data_type=data_type) assert obs_file == 'G003'
def test_find_obs_no_dated(metplus_config): config = metplus_config() pcw = CommandBuilder(config) v = {} v['obs_level'] = "6" task_info = {} task_info['valid'] = datetime.datetime.strptime("201802010000", '%Y%m%d%H%M') task_info['lead'] = 0 time_info = time_util.ti_calculate(task_info) pcw.c_dict['OBS_FILE_WINDOW_BEGIN'] = -3600 pcw.c_dict['OBS_FILE_WINDOW_END'] = 3600 pcw.c_dict['OBS_INPUT_DIR'] = pcw.config.getdir( 'METPLUS_BASE') + "/internal_tests/data/obs" pcw.c_dict['OBS_INPUT_TEMPLATE'] = "{valid?fmt=%Y%m%d}_{valid?fmt=%H%M}" obs_file = pcw.find_obs(time_info, v) assert obs_file == pcw.c_dict['OBS_INPUT_DIR'] + '/20180201_0045'
def test_setup_subtract_method(): rl = "FCST" pcw = pcp_combine_wrapper(rl) task_info = {} task_info['valid'] = datetime.datetime.strptime("201609050000", '%Y%m%d%H%M') task_info['lead_hours'] = 9 time_info = time_util.ti_calculate(task_info) var_info = {} var_info['fcst_name'] = "APCP" var_info['obs_name'] = "ACPCP" var_info['fcst_extra'] = "" var_info['obs_extra'] = "" var_info['fcst_level'] = "A06" var_info['obs_level'] = "A06" pcw.setup_subtract_method(time_info, var_info, rl) in_files = pcw.infiles out_file = pcw.get_output_path() assert (len(in_files) == 2)
def test_run_gen_vx_mask_once(metplus_config): input_dict = {'valid': datetime.datetime.strptime("201802010000",'%Y%m%d%H%M'), 'lead': 0} time_info = time_util.ti_calculate(input_dict) wrap = gen_vx_mask_wrapper(metplus_config) wrap.c_dict['INPUT_TEMPLATE'] = '{valid?fmt=%Y%m%d%H}_ZENITH' wrap.c_dict['MASK_INPUT_TEMPLATES'] = ['LAT'] wrap.c_dict['OUTPUT_DIR'] = os.path.join(wrap.config.getdir('OUTPUT_BASE'), 'GenVxMask_test') wrap.c_dict['OUTPUT_TEMPLATE'] = '{valid?fmt=%Y%m%d%H}_ZENITH_LAT_MASK.nc' wrap.c_dict['COMMAND_OPTIONS'] = ["-type lat -thresh 'ge30&&le50'"] # wrap.c_dict['MASK_INPUT_TEMPLATES'] = ['LAT', 'LON'] # wrap.c_dict['COMMAND_OPTIONS'] = ["-type lat -thresh 'ge30&&le50'", "-type lon -thresh 'le-70&&ge-130' -intersection"] wrap.run_at_time_all(time_info) expected_cmd = f"{wrap.app_path} 2018020100_ZENITH LAT {wrap.config.getdir('OUTPUT_BASE')}/GenVxMask_test/2018020100_ZENITH_LAT_MASK.nc -type lat -thresh 'ge30&&le50' -v 2" for cmd, _ in wrap.all_commands: print(f"COMMAND:{cmd}") print("EXPECTED:{expected_cmd}") assert(cmd == expected_cmd)
def test_sub_var_list(metplus_config, input_dict, expected_list): config = metplus_config() config.set('config', 'FCST_VAR1_NAME', 'FNAME_{init?fmt=%Y}') config.set('config', 'FCST_VAR1_LEVELS', 'Z{init?fmt=%H}, Z{valid?fmt=%H}') config.set('config', 'OBS_VAR1_NAME', 'ONAME_{init?fmt=%Y}') config.set('config', 'OBS_VAR1_LEVELS', 'L{init?fmt=%H}, L{valid?fmt=%H}') time_info = time_util.ti_calculate(input_dict) actual_temp = parse_var_list(config) pp = pprint.PrettyPrinter() print(f'Actual var list (before sub):') pp.pprint(actual_temp) actual_list = util.sub_var_list(actual_temp, time_info) print(f'Actual var list (after sub):') pp.pprint(actual_list) assert (len(actual_list) == len(expected_list)) for actual, expected in zip(actual_list, expected_list): for key, value in expected.items(): assert (actual.get(key) == value)
def test_setup_sum_method(): rl = "OBS" pcw = pcp_combine_wrapper(rl) task_info = {} task_info['valid'] = datetime.datetime.strptime("2016090418", '%Y%m%d%H') task_info['lead'] = 0 time_info = time_util.ti_calculate(task_info) var_info = {} var_info['fcst_name'] = "APCP" var_info['obs_name'] = "ACPCP" var_info['fcst_extra'] = "" var_info['obs_extra'] = "" var_info['fcst_level'] = "A06" var_info['obs_level'] = "A06" input_dir = pcw.config.getdir( 'METPLUS_BASE') + "/internal_tests/data/accum" output_dir = pcw.config.getdir( 'OUTPUT_BASE') + "/internal_tests/data/fakeout" pcw.setup_sum_method(time_info, var_info, rl) in_files = pcw.infiles out_file = pcw.get_output_path() assert (out_file == output_dir + "/20160904/outfile.2016090418_A06h")
def test_get_accumulation_6_to_6(): data_src = "FCST" pcw = pcp_combine_wrapper(data_src) input_dir = pcw.config.getdir( 'METPLUS_BASE') + "/internal_tests/data/accum" task_info = {} task_info['valid'] = datetime.datetime.strptime("2016090418", '%Y%m%d%H') time_info = time_util.ti_calculate(task_info) accum = 6 pcw.c_dict[ 'FCST_INPUT_TEMPLATE'] = "{valid?fmt=%Y%m%d}/file.{valid?fmt=%Y%m%d%H}.{level?fmt=%HH}h" pcw.input_dir = input_dir pcw.build_input_accum_list(data_src, time_info) pcw.get_accumulation(time_info, accum, data_src) in_files = pcw.infiles if len(in_files ) == 1 and input_dir + "/20160904/file.2016090418.06h" in in_files: assert True else: assert False
def test_get_skip_time(run_time, skip_times, expected_result): time_info = time_util.ti_calculate({'valid': run_time}) assert (util.skip_time(time_info, skip_times) == expected_result)
def test_set_command_line_arguments(): test_passed = True wrap = p2g_wrapper() input_dict = { 'valid': datetime.datetime.strptime("202003050000", '%Y%m%d%H%M'), 'lead': 0 } time_info = time_util.ti_calculate(input_dict) wrap.c_dict['REGRID_METHOD'] = 'UW_MEAN' expected_args = [ '-method UW_MEAN', ] wrap.set_command_line_arguments(time_info) if wrap.args != expected_args: test_passed = False print("Test 0 failed") print(f"ARGS: {wrap.args}") print(f"EXP: {expected_args}") wrap.args.clear() wrap.c_dict['GAUSSIAN_DX'] = 2 expected_args = [ '-method UW_MEAN', '-gaussian_dx 2', ] wrap.set_command_line_arguments(time_info) if wrap.args != expected_args: test_passed = False print("Test 1 failed") print(f"ARGS: {wrap.args}") print(f"EXP: {expected_args}") wrap.args.clear() wrap.c_dict['PROB_CAT_THRESH'] = 1 expected_args = [ '-method UW_MEAN', '-gaussian_dx 2', '-prob_cat_thresh 1', ] wrap.set_command_line_arguments(time_info) if wrap.args != expected_args: test_passed = False print("Test 2 failed") print(f"ARGS: {wrap.args}") print(f"EXP: {expected_args}") wrap.args.clear() wrap.c_dict['GAUSSIAN_RADIUS'] = 3 expected_args = [ '-method UW_MEAN', '-gaussian_dx 2', '-gaussian_radius 3', '-prob_cat_thresh 1', ] wrap.set_command_line_arguments(time_info) if wrap.args != expected_args: test_passed = False print("Test 3 failed") print(f"ARGS: {wrap.args}") print(f"EXP: {expected_args}") wrap.args.clear() wrap.c_dict['VLD_THRESH'] = .5 expected_args = [ '-method UW_MEAN', '-gaussian_dx 2', '-gaussian_radius 3', '-prob_cat_thresh 1', '-vld_thresh 0.5', ] wrap.set_command_line_arguments(time_info) if wrap.args != expected_args: test_passed = False print("Test 4 failed") print(f"ARGS: {wrap.args}") print(f"EXP: {expected_args}") wrap.args.clear() assert (test_passed)