def grid_sample_data(grid_spec, dirname=None): """ grid_spec are all command line arguments except for: -o dir_name input data filenames generates (yields) a sequence of filenames for each created file. """ resulting_date_path = ('2018', 'Jul', '02') with tempfile.TemporaryDirectory() as tmpdir: tmpdirname = os.path.join(tmpdir, dirname) cmd_args = ["-o", tmpdirname] + grid_spec + samples parser = create_parser() args = parser.parse_args(cmd_args) from multiprocessing import freeze_support freeze_support() gridder, glm_filenames, start_time, end_time, grid_kwargs = grid_setup( args) gridder(glm_filenames, start_time, end_time, **grid_kwargs) # gridder(glm_filenames, start_time, end_time, **grid_kwargs) for entry in os.scandir(os.path.join(tmpdirname, *resulting_date_path)): yield entry
def grid_sample_data(grid_spec, output_sizes): """ grid_spec are all command line arguments except for: -o dir_name input data filenames output_sizes is a dictionary mapping from output filename to the expected size of the file, which may vary by up to 20% """ resulting_date_path = ('2018', 'Jul', '02') with tempfile.TemporaryDirectory() as tmpdirname: cmd_args = ["-o", tmpdirname] + grid_spec + samples parser = create_parser() args = parser.parse_args(cmd_args) from multiprocessing import freeze_support freeze_support() gridder, glm_filenames, start_time, end_time, grid_kwargs = grid_setup( args) gridder(glm_filenames, start_time, end_time, **grid_kwargs) print("Output file sizes") for entry in os.scandir(os.path.join(tmpdirname, *resulting_date_path)): target = output_sizes[entry.name] actual = entry.stat().st_size percent = 20 assert np.abs(target - actual) < int(target * percent / 100)
def main(args): to_process = download(args.raw_dir) grid_spec = ["--fixed_grid", "--split_events", "--goes_position", "east", "--goes_sector", "conus", "--dx=2.0", "--dy=2.0", # "--ctr_lat=33.5", "--ctr_lon=-101.5", ] cmd_args = ["-o", args.grid_dir] + grid_spec + to_process # need to copy make_glm_grids to this driectory as part of driver script from make_GLM_grids import create_parser as create_grid_parser from make_GLM_grids import grid_setup grid_parser = create_grid_parser() grid_args = grid_parser.parse_args(cmd_args) from multiprocessing import freeze_support freeze_support() gridder, glm_filenames, start_time, end_time, grid_kwargs = grid_setup(grid_args) the_grid = gridder(glm_filenames, start_time, end_time, **grid_kwargs) print(the_grid)
def grid_sample_data(grid_spec, output_sizes, dirname=None): """ grid_spec are all command line arguments except for: -o dir_name input data filenames output_sizes is a dictionary mapping from output filename to the expected size of the file, which may vary by up to 20% """ resulting_date_path = ('2018', 'Jul', '02') with tempfile.TemporaryDirectory() as tmpdir: tmpdirname = os.path.join(tmpdir, dirname) cmd_args = ["-o", tmpdirname] + grid_spec + samples parser = create_parser() args = parser.parse_args(cmd_args) from multiprocessing import freeze_support freeze_support() gridder, glm_filenames, start_time, end_time, grid_kwargs = grid_setup( args) gridder(glm_filenames, start_time, end_time, **grid_kwargs) # print("Output file sizes") for entry in os.scandir(os.path.join(tmpdirname, *resulting_date_path)): # File size should be close to what we expect, with some platform # differences due to OS, compression, etc. target = output_sizes[entry.name] actual = entry.stat().st_size percent = 1 assert np.abs(target - actual) < int(target * percent / 100) # Now compare the contents directly valid_file = os.path.join(sample_path, dirname, *resulting_date_path, entry.name) valid = xr.open_dataset(valid_file) check = xr.open_dataset(entry.path) xr.testing.assert_allclose(valid, check)
def grid_sample_data(grid_spec, output_sizes, dirname=None, save=None): """ grid_spec are all command line arguments except for: -o dir_name input data filenames output_sizes is a dictionary mapping from output filename to the expected size of the file, which may vary by up to 20% """ resulting_date_path = ('2018', 'Jul', '02') with tempfile.TemporaryDirectory() as tmpdir: tmpdirname = os.path.join(tmpdir, dirname) cmd_args = ["-o", tmpdirname] + grid_spec + samples parser = create_parser() args = parser.parse_args(cmd_args) from multiprocessing import freeze_support freeze_support() gridder, glm_filenames, start_time, end_time, grid_kwargs = grid_setup(args) output_return = gridder(glm_filenames, start_time, end_time, **grid_kwargs) print(output_return) if save: from shutil import copytree copytree(tmpdirname, save) # print("Output file sizes") for entry in os.scandir(os.path.join(tmpdirname, *resulting_date_path)): # File size should be close to what we expect, with some platform # differences due to OS, compression, etc. test_file = entry.name is_unified_type = ('OR_GLM-L2-GLMC-M3_G16_s20181830433000' in test_file) if is_unified_type: valid_file_key = 'OR_GLM-L2-GLMC-M3_G16_s20181830433000_e20181830434000_c20182551446.nc' else: valid_file_key = test_file # target = output_sizes[valid_file_key] # actual = entry.stat().st_size # percent = 1 # assert np.abs(target-actual) < int(target*percent/100) # Now compare the contents directly valid_file = os.path.join(sample_path, dirname, *resulting_date_path, valid_file_key) valid = xr.open_dataset(valid_file) check = xr.open_dataset(entry.path) xr.testing.assert_allclose(valid, check) if (('total_energy' in output_sizes) & ('total_energy' in test_file)): np.testing.assert_allclose(check.total_energy.sum().data, output_sizes['total_energy'], rtol=1e-6) # An abundance of caution: validate the valid data, too! np.testing.assert_allclose(valid.total_energy.sum().data, output_sizes['total_energy'], rtol=1e-6)