def test_parser_write_mkdir_failed():
"""Test that failing at filesystem operations raises an error"""
variables = {var: 1.0 for var in ["u", "v", "w", "time"]}
pack_length = 1
# Use the /TEST directory to cause a permission problem
dest = Path("/") / "TEST" / "MSU_Test{group}_{date:%H-%M-%S-%f}.npz"
parser = Parser(regex=b"",
group=Group(),
pack_length=pack_length,
dest=dest)
with pytest.raises(ParseError) as exc_info:
parser.write(variables)
assert isinstance(exc_info.value.args[0], OSError)
def test_parser_write_group_by(tmp_path):
"""Ensure that group_by files are written properly"""
data = [
dict(level=1, rh=1.23, temp=14.85, time=time.time()),
dict(level=2, rh=2.23, temp=11.85, time=time.time()),
dict(level=1, rh=1.35, temp=14.97, time=time.time()),
dict(level=2, rh=2.35, temp=11.97, time=time.time()),
]
levels = {d["level"] for d in data}
saved_vars = {key for d in data for key in d.keys() if key != "level"}
pack_length = 2
# Use microseconds and a unique file identifier
dest = tmp_path / "data" / "MSU_Test{group}_{date:%H-%M-%S-%f}.npz"
microsecond = 0.000001
# Two complete files expected as output:
buffers = {level: defaultdict(list) for level in levels}
parser = Parser(
regex=b"",
group=Group(by="level", dtype="float"),
pack_length=pack_length,
dest=dest,
)
for variables in data:
parser.write(variables)
for var, value in variables.items():
level = variables["level"]
buffers[level][var].append(value)
# Make sure that the whole iteration is at least 1 microsecond long
time.sleep(microsecond)
files = sorted([str(p) for p in tmp_path.glob("**/*") if p.is_file()])
assert len(files) == len(levels)
for level in levels:
file = [f for f in files if f"Test{level}" in f][0]
with np.load(file) as data:
for var in saved_vars:
expected = np.array(buffers[level][var])
assert np.array_equal(data[var], expected)
assert data[var].dtype == expected.dtype
def test_parser_write_inconsistent_vars(tmp_path):
"""Check that supplying a wrong set of variables triggers an exception"""
variables = {var: 1.0 for var in ["u", "v", "w", "time"]}
pack_length = 2
dest = tmp_path / "data" / "MSU_Test{group}_{date:%H-%M-%S-%f}.npz"
parser = Parser(regex=b"",
group=Group(),
pack_length=pack_length,
dest=dest)
with pytest.raises(ParseError) as exc_info:
parser.write(variables)
# Remove one of the variables, which should cause an error
del variables["u"]
parser.write(variables)
assert isinstance(exc_info.value.args[0], AssertionError)
def test_parser_write_ok(tmp_path):
"""Ensure that files are written properly"""
all_vars = ["u", "v", "w", "temp", "time"]
pack_length = 2
# Use microseconds and a unique file identifier
dest = tmp_path / "data" / "MSU_Test{group}_{date:%H-%M-%S-%f}.npz"
microsecond = 0.000001
# Two complete files expected as output:
n_iter = 2
buffers = [defaultdict(list) for _ in range(n_iter)]
parser = Parser(regex=b"",
group=Group(),
pack_length=pack_length,
dest=dest)
for i in range(n_iter):
for _ in range(pack_length):
variables = {var: random.uniform(-10, 10) for var in all_vars}
parser.write(variables)
# Form an expected representation of the data
for var, value in variables.items():
buffers[i][var].append(value)
# Make sure that the whole iteration is at least 1 microsecond long
time.sleep(microsecond)
files = sorted([str(p) for p in tmp_path.glob("**/*") if p.is_file()])
assert len(files) == n_iter
for i, file in enumerate(files):
with np.load(file) as data:
for var in all_vars:
expected = np.array(buffers[i][var])
assert np.array_equal(data[var], expected)
assert data[var].dtype == expected.dtype