def test_ti_calculate(input_dict, expected_time_info):
time_info = time_util.ti_calculate(input_dict)
for key, value in expected_time_info.items():
assert (time_info[key] == value)
time_info2 = time_util.ti_calculate(time_info)
for key, value in expected_time_info.items():
assert (time_info[key] == value)
assert (time_info2[key] == value)
def test_find_input_files(metplus_config, offsets, offset_to_find):
pb = pb2nc_wrapper(metplus_config)
# for valid 20190201_12, offsets 3 and 5, create files to find
# in the fake input directory based on input template
input_dict = {'valid': datetime.datetime(2019, 2, 1, 12)}
fake_input_dir = os.path.join(pb.config.getdir('OUTPUT_BASE'), 'pbin')
if not os.path.exists(fake_input_dir):
os.makedirs(fake_input_dir)
pb.c_dict['OBS_INPUT_DIR'] = fake_input_dir
for offset in [3, 5]:
input_dict['offset'] = int(offset * 3600)
time_info = time_util.ti_calculate(input_dict)
create_file = do_string_sub(pb.c_dict['OBS_INPUT_TEMPLATE'],
**time_info)
create_fullpath = os.path.join(fake_input_dir, create_file)
open(create_fullpath, 'a').close()
# unset offset in time dictionary so it will be computed
del input_dict['offset']
# set offset list
pb.c_dict['OFFSETS'] = offsets
# look for input files based on offset list
result = pb.find_input_files(input_dict)
# check if correct offset file was found, if None expected, check against None
if offset_to_find is None:
assert result is None
else:
assert result['offset_hours'] == offset_to_find
def test_run_gen_vx_mask_twice(metplus_config):
input_dict = {'valid': datetime.datetime.strptime("201802010000",'%Y%m%d%H%M'),
'lead': 0}
time_info = time_util.ti_calculate(input_dict)
wrap = gen_vx_mask_wrapper(metplus_config)
wrap.c_dict['INPUT_TEMPLATE'] = '{valid?fmt=%Y%m%d%H}_ZENITH'
wrap.c_dict['MASK_INPUT_TEMPLATES'] = ['LAT', 'LON']
wrap.c_dict['OUTPUT_DIR'] = os.path.join(wrap.config.getdir('OUTPUT_BASE'),
'GenVxMask_test')
wrap.c_dict['OUTPUT_TEMPLATE'] = '{valid?fmt=%Y%m%d%H}_ZENITH_LAT_LON_MASK.nc'
cmd_args = ["-type lat -thresh 'ge30&&le50'", "-type lon -thresh 'le-70&&ge-130' -intersection -name lat_lon_mask"]
wrap.c_dict['COMMAND_OPTIONS'] = cmd_args
wrap.run_at_time_all(time_info)
expected_cmds = [f"{wrap.app_path} 2018020100_ZENITH LAT {wrap.config.getdir('OUTPUT_BASE')}/stage/gen_vx_mask/temp_0.nc {cmd_args[0]} -v 2",
f"{wrap.app_path} {wrap.config.getdir('OUTPUT_BASE')}/stage/gen_vx_mask/temp_0.nc LON {wrap.config.getdir('OUTPUT_BASE')}/GenVxMask_test/2018020100_ZENITH_LAT_LON_MASK.nc {cmd_args[1]} -v 2"]
test_passed = True
if len(wrap.all_commands) != len(expected_cmds):
print("Number of commands run is not the same as expected")
assert(False)
for (cmd, _), expected_cmd in zip(wrap.all_commands, expected_cmds):
print(f" ACTUAL:{cmd}")
print(f"EXPECTED:{expected_cmd}")
if cmd != expected_cmd:
test_passed = False
assert(test_passed)
def test_setup_add_method():
rl = "OBS"
pcw = pcp_combine_wrapper(rl)
task_info = {}
task_info['valid'] = datetime.datetime.strptime("2016090418", '%Y%m%d%H')
time_info = time_util.ti_calculate(task_info)
var_info = {}
var_info['fcst_name'] = "APCP"
var_info['obs_name'] = "ACPCP"
var_info['fcst_extra'] = ""
var_info['obs_extra'] = ""
var_info['fcst_level'] = "A06"
var_info['obs_level'] = "A06"
input_dir = pcw.config.getdir(
'METPLUS_BASE') + "/internal_tests/data/accum"
output_dir = pcw.config.getdir(
'OUTPUT_BASE') + "/internal_tests/data/fakeout"
pcw.setup_add_method(time_info, var_info, rl)
in_files = pcw.infiles
out_file = pcw.get_output_path()
if len(in_files) == 6 and \
input_dir+"/20160904/file.2016090418.01h" in in_files and \
input_dir+"/20160904/file.2016090417.01h" in in_files and \
input_dir+"/20160904/file.2016090416.01h" in in_files and \
input_dir+"/20160904/file.2016090415.01h" in in_files and \
input_dir+"/20160904/file.2016090414.01h" in in_files and \
input_dir+"/20160904/file.2016090413.01h" in in_files and \
out_file == output_dir+"/20160904/outfile.2016090418_A06h":
assert True
else:
assert False
def test_find_obs_offset(metplus_config, offsets, expected_file,
offset_seconds):
config = metplus_config()
pcw = CommandBuilder(config)
v = {}
v['obs_level'] = "6"
task_info = {}
task_info['valid'] = datetime.datetime.strptime("2020020112", '%Y%m%d%H')
task_info['lead'] = 0
time_info = time_util.ti_calculate(task_info)
pcw.c_dict['OFFSETS'] = offsets
pcw.c_dict['OBS_INPUT_DIR'] = pcw.config.getdir(
'METPLUS_BASE') + "/internal_tests/data/obs"
pcw.c_dict[
'OBS_INPUT_TEMPLATE'] = "{da_init?fmt=%2H}z.prepbufr.tm{offset?fmt=%2H}.{da_init?fmt=%Y%m%d}"
obs_file, time_info = pcw.find_obs_offset(time_info, v)
print(f"OBSFILE: {obs_file}")
print(f"EXPECTED FILE: {expected_file}")
if expected_file is None:
assert not obs_file
else:
assert (os.path.basename(obs_file) == expected_file
and time_info['offset'] == offset_seconds)
def test_get_accumulation_1_to_6():
data_src = "OBS"
pcw = pcp_combine_wrapper(data_src)
input_dir = pcw.config.getdir(
'METPLUS_BASE') + "/internal_tests/data/accum"
task_info = {}
task_info['valid'] = datetime.datetime.strptime("2016090418", '%Y%m%d%H')
time_info = time_util.ti_calculate(task_info)
accum = 6
file_template = "{valid?fmt=%Y%m%d}/file.{valid?fmt=%Y%m%d%H}.{level?fmt=%HH}h"
pcw.input_dir = input_dir
pcw.build_input_accum_list(data_src, time_info)
pcw.get_accumulation(time_info, accum, data_src)
in_files = pcw.infiles
if len(in_files) == 6 and \
input_dir+"/20160904/file.2016090418.01h" in in_files and \
input_dir+"/20160904/file.2016090417.01h" in in_files and \
input_dir+"/20160904/file.2016090416.01h" in in_files and \
input_dir+"/20160904/file.2016090415.01h" in in_files and \
input_dir+"/20160904/file.2016090414.01h" in in_files and \
input_dir+"/20160904/file.2016090413.01h" in in_files:
assert True
else:
assert False
def test_get_field_info_list(conf_dict, expected_field_info_list):
config = metplus_config()
logger = logging.getLogger("dummy")
data_type = 'OBS'
for key, value in conf_dict.items():
config.set('config', key, value)
input_dict = {
'valid': datetime.datetime.strptime("201802010000", '%Y%m%d%H%M'),
'lead': 0
}
time_info = time_util.ti_calculate(input_dict)
var_list = util.parse_var_list(config, time_info, data_type=data_type)
rdp = RegridDataPlaneWrapper(config, logger)
field_info_list = rdp.get_field_info_list(var_list, data_type, time_info)
print(f"FIELD INFO LIST: {field_info_list}")
print(f"EXPECTED FIELD INFO LIST: {expected_field_info_list}")
is_good = True
if len(field_info_list) != len(expected_field_info_list):
assert (False)
for actual_field, expected_field in zip(field_info_list,
expected_field_info_list):
for key, value in expected_field.items():
if actual_field[key] != value:
print(f"{actual_field[key]} not equal to {value}")
is_good = False
def test_run_rdp_once_per_field():
data_type = 'FCST'
input_dict = {
'valid': datetime.datetime.strptime("201802010000", '%Y%m%d%H%M'),
'lead': 0
}
time_info = time_util.ti_calculate(input_dict)
var_list = [
{
'index': '1',
'fcst_name': 'FNAME1',
'fcst_level': 'A06'
},
{
'index': '2',
'fcst_name': 'FNAME2',
'fcst_level': 'A03',
'fcst_output_name': 'OUTNAME2'
},
]
wrap = rdp_wrapper()
wrap.c_dict['ONCE_PER_FIELD'] = True
wrap.c_dict[
'FCST_OUTPUT_TEMPLATE'] = '{valid?fmt=%Y%m%d%H}_accum{level?fmt=%2H}.nc'
wrap.c_dict['FCST_INPUT_TEMPLATE'] = '{valid?fmt=%Y%m%d%H}_ZENITH'
wrap.c_dict['METHOD'] = 'BUDGET'
wrap.c_dict['WIDTH'] = 2
wrap.c_dict['VERIFICATION_GRID'] = 'VERIF_GRID'
wrap.c_dict['FCST_OUTPUT_DIR'] = os.path.join(
wrap.config.getdir('OUTPUT_BASE'), 'RDP_test')
wrap.run_at_time_once(time_info, var_list, data_type)
expected_cmds = [
f"{wrap.app_path} -v 2 -method BUDGET -width 2 -field 'name=\"FNAME1\"; "
"level=\"A06\";' -name FNAME1 2018020100_ZENITH \"VERIF_GRID\" "
f"{wrap.config.getdir('OUTPUT_BASE')}/RDP_test/2018020100_accum06.nc",
f"{wrap.app_path} -v 2 -method BUDGET -width 2 -field 'name=\"FNAME2\"; "
"level=\"A03\";' -name OUTNAME2 2018020100_ZENITH \"VERIF_GRID\" "
f"{wrap.config.getdir('OUTPUT_BASE')}/RDP_test/2018020100_accum03.nc",
]
test_passed = True
if len(wrap.all_commands) != len(expected_cmds):
print("Number of commands run is not the same as expected")
assert (False)
for cmd, expected_cmd in zip(wrap.all_commands, expected_cmds):
print(f" ACTUAL:{cmd}")
print(f"EXPECTED:{expected_cmd}")
if cmd != expected_cmd:
test_passed = False
assert (test_passed)
def test_find_data_not_a_path(metplus_config, data_type):
config = metplus_config()
pcw = CommandBuilder(config)
task_info = {}
task_info['valid'] = datetime.datetime.strptime("201802010000",
'%Y%m%d%H%M')
task_info['lead'] = 0
time_info = time_util.ti_calculate(task_info)
pcw.c_dict[f'{data_type}FILE_WINDOW_BEGIN'] = 0
pcw.c_dict[f'{data_type}FILE_WINDOW_END'] = 0
pcw.c_dict[f'{data_type}INPUT_DIR'] = ''
pcw.c_dict[f'{data_type}INPUT_TEMPLATE'] = 'G003'
obs_file = pcw.find_data(time_info, var_info=None, data_type=data_type)
assert obs_file == 'G003'
def test_find_obs_no_dated(metplus_config):
config = metplus_config()
pcw = CommandBuilder(config)
v = {}
v['obs_level'] = "6"
task_info = {}
task_info['valid'] = datetime.datetime.strptime("201802010000",
'%Y%m%d%H%M')
task_info['lead'] = 0
time_info = time_util.ti_calculate(task_info)
pcw.c_dict['OBS_FILE_WINDOW_BEGIN'] = -3600
pcw.c_dict['OBS_FILE_WINDOW_END'] = 3600
pcw.c_dict['OBS_INPUT_DIR'] = pcw.config.getdir(
'METPLUS_BASE') + "/internal_tests/data/obs"
pcw.c_dict['OBS_INPUT_TEMPLATE'] = "{valid?fmt=%Y%m%d}_{valid?fmt=%H%M}"
obs_file = pcw.find_obs(time_info, v)
assert obs_file == pcw.c_dict['OBS_INPUT_DIR'] + '/20180201_0045'
def test_setup_subtract_method():
rl = "FCST"
pcw = pcp_combine_wrapper(rl)
task_info = {}
task_info['valid'] = datetime.datetime.strptime("201609050000",
'%Y%m%d%H%M')
task_info['lead_hours'] = 9
time_info = time_util.ti_calculate(task_info)
var_info = {}
var_info['fcst_name'] = "APCP"
var_info['obs_name'] = "ACPCP"
var_info['fcst_extra'] = ""
var_info['obs_extra'] = ""
var_info['fcst_level'] = "A06"
var_info['obs_level'] = "A06"
pcw.setup_subtract_method(time_info, var_info, rl)
in_files = pcw.infiles
out_file = pcw.get_output_path()
assert (len(in_files) == 2)
def test_run_gen_vx_mask_once(metplus_config):
input_dict = {'valid': datetime.datetime.strptime("201802010000",'%Y%m%d%H%M'),
'lead': 0}
time_info = time_util.ti_calculate(input_dict)
wrap = gen_vx_mask_wrapper(metplus_config)
wrap.c_dict['INPUT_TEMPLATE'] = '{valid?fmt=%Y%m%d%H}_ZENITH'
wrap.c_dict['MASK_INPUT_TEMPLATES'] = ['LAT']
wrap.c_dict['OUTPUT_DIR'] = os.path.join(wrap.config.getdir('OUTPUT_BASE'),
'GenVxMask_test')
wrap.c_dict['OUTPUT_TEMPLATE'] = '{valid?fmt=%Y%m%d%H}_ZENITH_LAT_MASK.nc'
wrap.c_dict['COMMAND_OPTIONS'] = ["-type lat -thresh 'ge30&&le50'"]
# wrap.c_dict['MASK_INPUT_TEMPLATES'] = ['LAT', 'LON']
# wrap.c_dict['COMMAND_OPTIONS'] = ["-type lat -thresh 'ge30&&le50'", "-type lon -thresh 'le-70&&ge-130' -intersection"]
wrap.run_at_time_all(time_info)
expected_cmd = f"{wrap.app_path} 2018020100_ZENITH LAT {wrap.config.getdir('OUTPUT_BASE')}/GenVxMask_test/2018020100_ZENITH_LAT_MASK.nc -type lat -thresh 'ge30&&le50' -v 2"
for cmd, _ in wrap.all_commands:
print(f"COMMAND:{cmd}")
print("EXPECTED:{expected_cmd}")
assert(cmd == expected_cmd)
def test_sub_var_list(metplus_config, input_dict, expected_list):
config = metplus_config()
config.set('config', 'FCST_VAR1_NAME', 'FNAME_{init?fmt=%Y}')
config.set('config', 'FCST_VAR1_LEVELS', 'Z{init?fmt=%H}, Z{valid?fmt=%H}')
config.set('config', 'OBS_VAR1_NAME', 'ONAME_{init?fmt=%Y}')
config.set('config', 'OBS_VAR1_LEVELS', 'L{init?fmt=%H}, L{valid?fmt=%H}')
time_info = time_util.ti_calculate(input_dict)
actual_temp = parse_var_list(config)
pp = pprint.PrettyPrinter()
print(f'Actual var list (before sub):')
pp.pprint(actual_temp)
actual_list = util.sub_var_list(actual_temp, time_info)
print(f'Actual var list (after sub):')
pp.pprint(actual_list)
assert (len(actual_list) == len(expected_list))
for actual, expected in zip(actual_list, expected_list):
for key, value in expected.items():
assert (actual.get(key) == value)
def test_setup_sum_method():
rl = "OBS"
pcw = pcp_combine_wrapper(rl)
task_info = {}
task_info['valid'] = datetime.datetime.strptime("2016090418", '%Y%m%d%H')
task_info['lead'] = 0
time_info = time_util.ti_calculate(task_info)
var_info = {}
var_info['fcst_name'] = "APCP"
var_info['obs_name'] = "ACPCP"
var_info['fcst_extra'] = ""
var_info['obs_extra'] = ""
var_info['fcst_level'] = "A06"
var_info['obs_level'] = "A06"
input_dir = pcw.config.getdir(
'METPLUS_BASE') + "/internal_tests/data/accum"
output_dir = pcw.config.getdir(
'OUTPUT_BASE') + "/internal_tests/data/fakeout"
pcw.setup_sum_method(time_info, var_info, rl)
in_files = pcw.infiles
out_file = pcw.get_output_path()
assert (out_file == output_dir + "/20160904/outfile.2016090418_A06h")
def test_get_accumulation_6_to_6():
data_src = "FCST"
pcw = pcp_combine_wrapper(data_src)
input_dir = pcw.config.getdir(
'METPLUS_BASE') + "/internal_tests/data/accum"
task_info = {}
task_info['valid'] = datetime.datetime.strptime("2016090418", '%Y%m%d%H')
time_info = time_util.ti_calculate(task_info)
accum = 6
pcw.c_dict[
'FCST_INPUT_TEMPLATE'] = "{valid?fmt=%Y%m%d}/file.{valid?fmt=%Y%m%d%H}.{level?fmt=%HH}h"
pcw.input_dir = input_dir
pcw.build_input_accum_list(data_src, time_info)
pcw.get_accumulation(time_info, accum, data_src)
in_files = pcw.infiles
if len(in_files
) == 1 and input_dir + "/20160904/file.2016090418.06h" in in_files:
assert True
else:
assert False
def test_get_skip_time(run_time, skip_times, expected_result):
time_info = time_util.ti_calculate({'valid': run_time})
assert (util.skip_time(time_info, skip_times) == expected_result)
def test_set_command_line_arguments():
test_passed = True
wrap = p2g_wrapper()
input_dict = {
'valid': datetime.datetime.strptime("202003050000", '%Y%m%d%H%M'),
'lead': 0
}
time_info = time_util.ti_calculate(input_dict)
wrap.c_dict['REGRID_METHOD'] = 'UW_MEAN'
expected_args = [
'-method UW_MEAN',
]
wrap.set_command_line_arguments(time_info)
if wrap.args != expected_args:
test_passed = False
print("Test 0 failed")
print(f"ARGS: {wrap.args}")
print(f"EXP: {expected_args}")
wrap.args.clear()
wrap.c_dict['GAUSSIAN_DX'] = 2
expected_args = [
'-method UW_MEAN',
'-gaussian_dx 2',
]
wrap.set_command_line_arguments(time_info)
if wrap.args != expected_args:
test_passed = False
print("Test 1 failed")
print(f"ARGS: {wrap.args}")
print(f"EXP: {expected_args}")
wrap.args.clear()
wrap.c_dict['PROB_CAT_THRESH'] = 1
expected_args = [
'-method UW_MEAN',
'-gaussian_dx 2',
'-prob_cat_thresh 1',
]
wrap.set_command_line_arguments(time_info)
if wrap.args != expected_args:
test_passed = False
print("Test 2 failed")
print(f"ARGS: {wrap.args}")
print(f"EXP: {expected_args}")
wrap.args.clear()
wrap.c_dict['GAUSSIAN_RADIUS'] = 3
expected_args = [
'-method UW_MEAN',
'-gaussian_dx 2',
'-gaussian_radius 3',
'-prob_cat_thresh 1',
]
wrap.set_command_line_arguments(time_info)
if wrap.args != expected_args:
test_passed = False
print("Test 3 failed")
print(f"ARGS: {wrap.args}")
print(f"EXP: {expected_args}")
wrap.args.clear()
wrap.c_dict['VLD_THRESH'] = .5
expected_args = [
'-method UW_MEAN',
'-gaussian_dx 2',
'-gaussian_radius 3',
'-prob_cat_thresh 1',
'-vld_thresh 0.5',
]
wrap.set_command_line_arguments(time_info)
if wrap.args != expected_args:
test_passed = False
print("Test 4 failed")
print(f"ARGS: {wrap.args}")
print(f"EXP: {expected_args}")
wrap.args.clear()
assert (test_passed)
