def grid_sample_data(grid_spec, dirname=None):
"""
grid_spec are all command line arguments except for:
-o dir_name
input data filenames
generates (yields) a sequence of filenames for each created file.
"""
resulting_date_path = ('2018', 'Jul', '02')
with tempfile.TemporaryDirectory() as tmpdir:
tmpdirname = os.path.join(tmpdir, dirname)
cmd_args = ["-o", tmpdirname] + grid_spec + samples
parser = create_parser()
args = parser.parse_args(cmd_args)
from multiprocessing import freeze_support
freeze_support()
gridder, glm_filenames, start_time, end_time, grid_kwargs = grid_setup(
args)
gridder(glm_filenames, start_time, end_time, **grid_kwargs)
# gridder(glm_filenames, start_time, end_time, **grid_kwargs)
for entry in os.scandir(os.path.join(tmpdirname,
*resulting_date_path)):
yield entry
def grid_sample_data(grid_spec, output_sizes):
"""
grid_spec are all command line arguments except for:
-o dir_name
input data filenames
output_sizes is a dictionary mapping from output filename to the expected
size of the file, which may vary by up to 20%
"""
resulting_date_path = ('2018', 'Jul', '02')
with tempfile.TemporaryDirectory() as tmpdirname:
cmd_args = ["-o", tmpdirname] + grid_spec + samples
parser = create_parser()
args = parser.parse_args(cmd_args)
from multiprocessing import freeze_support
freeze_support()
gridder, glm_filenames, start_time, end_time, grid_kwargs = grid_setup(
args)
gridder(glm_filenames, start_time, end_time, **grid_kwargs)
print("Output file sizes")
for entry in os.scandir(os.path.join(tmpdirname,
*resulting_date_path)):
target = output_sizes[entry.name]
actual = entry.stat().st_size
percent = 20
assert np.abs(target - actual) < int(target * percent / 100)
def grid_sample_data(grid_spec, output_sizes, dirname=None):
"""
grid_spec are all command line arguments except for:
-o dir_name
input data filenames
output_sizes is a dictionary mapping from output filename to the expected
size of the file, which may vary by up to 20%
"""
resulting_date_path = ('2018', 'Jul', '02')
with tempfile.TemporaryDirectory() as tmpdir:
tmpdirname = os.path.join(tmpdir, dirname)
cmd_args = ["-o", tmpdirname] + grid_spec + samples
parser = create_parser()
args = parser.parse_args(cmd_args)
from multiprocessing import freeze_support
freeze_support()
gridder, glm_filenames, start_time, end_time, grid_kwargs = grid_setup(
args)
gridder(glm_filenames, start_time, end_time, **grid_kwargs)
# print("Output file sizes")
for entry in os.scandir(os.path.join(tmpdirname,
*resulting_date_path)):
# File size should be close to what we expect, with some platform
# differences due to OS, compression, etc.
target = output_sizes[entry.name]
actual = entry.stat().st_size
percent = 1
assert np.abs(target - actual) < int(target * percent / 100)
# Now compare the contents directly
valid_file = os.path.join(sample_path, dirname,
*resulting_date_path, entry.name)
valid = xr.open_dataset(valid_file)
check = xr.open_dataset(entry.path)
xr.testing.assert_allclose(valid, check)
def grid_sample_data(grid_spec, output_sizes, dirname=None, save=None):
"""
grid_spec are all command line arguments except for:
-o dir_name
input data filenames
output_sizes is a dictionary mapping from output filename to the expected
size of the file, which may vary by up to 20%
"""
resulting_date_path = ('2018', 'Jul', '02')
with tempfile.TemporaryDirectory() as tmpdir:
tmpdirname = os.path.join(tmpdir, dirname)
cmd_args = ["-o", tmpdirname] + grid_spec + samples
parser = create_parser()
args = parser.parse_args(cmd_args)
from multiprocessing import freeze_support
freeze_support()
gridder, glm_filenames, start_time, end_time, grid_kwargs = grid_setup(args)
output_return = gridder(glm_filenames, start_time, end_time, **grid_kwargs)
print(output_return)
if save:
from shutil import copytree
copytree(tmpdirname, save)
# print("Output file sizes")
for entry in os.scandir(os.path.join(tmpdirname, *resulting_date_path)):
# File size should be close to what we expect, with some platform
# differences due to OS, compression, etc.
test_file = entry.name
is_unified_type = ('OR_GLM-L2-GLMC-M3_G16_s20181830433000' in test_file)
if is_unified_type:
valid_file_key = 'OR_GLM-L2-GLMC-M3_G16_s20181830433000_e20181830434000_c20182551446.nc'
else:
valid_file_key = test_file
# target = output_sizes[valid_file_key]
# actual = entry.stat().st_size
# percent = 1
# assert np.abs(target-actual) < int(target*percent/100)
# Now compare the contents directly
valid_file = os.path.join(sample_path, dirname,
*resulting_date_path, valid_file_key)
valid = xr.open_dataset(valid_file)
check = xr.open_dataset(entry.path)
xr.testing.assert_allclose(valid, check)
if (('total_energy' in output_sizes) &
('total_energy' in test_file)):
np.testing.assert_allclose(check.total_energy.sum().data,
output_sizes['total_energy'],
rtol=1e-6)
# An abundance of caution: validate the valid data, too!
np.testing.assert_allclose(valid.total_energy.sum().data,
output_sizes['total_energy'],
rtol=1e-6)