def test_FileStream() -> None:
res = messages.FileStream(TEST_DATA)
leader = res[0]
assert len(leader) > 100
assert sum(1 for _ in res.items()) == leader["count"]
# __file__ is not a GRIB, but contains the "GRIB" string, so it is a very tricky corner case
res = messages.FileStream(str(__file__))
with pytest.raises(eccodes.UnsupportedEditionError):
res[0]
def test_FileStream():
res = messages.FileStream(TEST_DATA)
leader = res.first()
assert len(leader) > 100
assert sum(1 for _ in res) == leader['count']
assert len(res.index(['paramId'])) == 1
# __file__ is not a GRIB, but contains the "GRIB" string, so it is a very tricky corner case
res = messages.FileStream(str(__file__))
with pytest.raises(EOFError):
res.first()
def test_FileStream() -> None:
res = messages.FileStream(TEST_DATA)
leader = res.first()
assert len(leader) > 100
assert sum(1 for _ in res.items()) == leader["count"]
assert len(res.index(["paramId"])) == 1
# __file__ is not a GRIB, but contains the "GRIB" string, so it is a very tricky corner case
res = messages.FileStream(str(__file__))
with pytest.raises(EOFError):
res.first()
res = messages.FileStream(str(__file__), errors="ignore")
with pytest.raises(EOFError):
res.first()
def test_build_data_var_components_no_encode():
index = messages.FileStream(path=TEST_DATA).index(dataset.INDEX_KEYS).subindex(paramId=130)
dims, data_var, coord_vars = dataset.build_variable_components(index=index)
assert dims == {"number": 10, "dataDate": 2, "dataTime": 2, "level": 2, "values": 7320}
assert data_var.data.shape == (10, 2, 2, 2, 7320)
# equivalent to not np.isnan without importing numpy
assert data_var.data[:, :, :, :, :].mean() > 0.0
def test_FileIndex_from_indexpath_or_filestream(tmpdir):
grib_file = tmpdir.join('file.grib')
with open(TEST_DATA, 'rb') as file:
grib_file.write_binary(file.read())
# create index file
res = messages.FileIndex.from_indexpath_or_filestream(
messages.FileStream(str(grib_file)),
['paramId'],
)
assert isinstance(res, messages.FileIndex)
# read index file
res = messages.FileIndex.from_indexpath_or_filestream(
messages.FileStream(str(grib_file)),
['paramId'],
)
assert isinstance(res, messages.FileIndex)
# do not read nor create the index file
res = messages.FileIndex.from_indexpath_or_filestream(
messages.FileStream(str(grib_file)),
['paramId'],
indexpath='',
)
assert isinstance(res, messages.FileIndex)
# can't create nor read index file
res = messages.FileIndex.from_indexpath_or_filestream(
messages.FileStream(str(grib_file)),
['paramId'],
indexpath=str(tmpdir.join('non-existent-folder').join('non-existent-file')),
)
assert isinstance(res, messages.FileIndex)
# trigger mtime check
grib_file.remove()
with open(TEST_DATA, 'rb') as file:
grib_file.write_binary(file.read())
res = messages.FileIndex.from_indexpath_or_filestream(
messages.FileStream(str(grib_file)),
['paramId'],
)
assert isinstance(res, messages.FileIndex)
def test_FileIndex_errors() -> None:
computed_keys = {"error_key": (lambda m: bool(1 / 0), lambda m, v: None)} # pragma: no branch
stream = messages.FileStream(TEST_DATA)
res = messages.FileIndex.from_fieldset(stream, ["paramId", "error_key"], computed_keys)
assert res["paramId"] == [129, 130]
assert len(res) == 2
assert list(res) == ["paramId", "error_key"]
assert res["error_key"] == ["undef"]
def test_build_dataset_components_time_dims() -> None:
index_keys = sorted(dataset.INDEX_KEYS + ["time", "step"])
stream = messages.FileStream(TEST_DATA_UKMO, "warn")
index = dataset.open_fileindex(stream, messages.DEFAULT_INDEXPATH,
index_keys)
dims = dataset.build_dataset_components(index, read_keys=[])[0]
assert dims == {
"latitude": 6,
"longitude": 11,
"number": 28,
"step": 20,
"time": 8,
}
time_dims = ["indexing_time", "verifying_time"]
index_keys = sorted(dataset.INDEX_KEYS + time_dims)
stream = messages.FileStream(TEST_DATA_UKMO, "warn")
index = dataset.open_fileindex(stream, messages.DEFAULT_INDEXPATH,
index_keys)
dims, *_ = dataset.build_dataset_components(index,
read_keys=[],
time_dims=time_dims)
assert dims == {
"number": 28,
"indexing_time": 2,
"verifying_time": 4,
"latitude": 6,
"longitude": 11,
}
time_dims = ["indexing_time", "step"]
index_keys = sorted(dataset.INDEX_KEYS + time_dims)
stream = messages.FileStream(TEST_DATA_UKMO, "warn")
index = dataset.open_fileindex(stream, messages.DEFAULT_INDEXPATH,
index_keys)
dims, *_ = dataset.build_dataset_components(index,
read_keys=[],
time_dims=time_dims)
assert dims == {
"number": 28,
"indexing_time": 2,
"step": 20,
"latitude": 6,
"longitude": 11
}
def test_FileIndex_errors():
class MyMessage(messages.ComputedKeysMessage):
computed_keys = {'error_key': lambda m: 1 / 0}
stream = messages.FileStream(TEST_DATA, message_class=MyMessage)
res = messages.FileIndex.from_filestream(stream, ['paramId', 'error_key'])
assert res['paramId'] == [129, 130]
assert len(res) == 2
assert list(res) == ['paramId', 'error_key']
assert res['error_key'] == ['undef']
def test_FileIndex_errors():
class MyMessage(messages.ComputedKeysMessage):
computed_keys = {"error_key": lambda m: 1 / 0}
stream = messages.FileStream(TEST_DATA, message_class=MyMessage)
res = messages.FileIndex.from_filestream(stream, ["paramId", "error_key"])
assert res["paramId"] == [129, 130]
assert len(res) == 2
assert list(res) == ["paramId", "error_key"]
assert res["error_key"] == ["undef"]
def test_FileIndex_from_indexpath_or_filestream(tmpdir: py.path.local) -> None:
grib_file = tmpdir.join("file.grib")
with open(TEST_DATA, "rb") as file:
grib_file.write_binary(file.read())
# create index file
res = messages.FileIndex.from_indexpath_or_filestream(
messages.FileStream(str(grib_file)), ["paramId"]
)
assert isinstance(res, messages.FileIndex)
# read index file
res = messages.FileIndex.from_indexpath_or_filestream(
messages.FileStream(str(grib_file)), ["paramId"]
)
assert isinstance(res, messages.FileIndex)
# do not read nor create the index file
res = messages.FileIndex.from_indexpath_or_filestream(
messages.FileStream(str(grib_file)), ["paramId"], indexpath=""
)
assert isinstance(res, messages.FileIndex)
# can't create nor read index file
res = messages.FileIndex.from_indexpath_or_filestream(
messages.FileStream(str(grib_file)),
["paramId"],
indexpath=str(tmpdir.join("non-existent-folder").join("non-existent-file")),
)
assert isinstance(res, messages.FileIndex)
# trigger mtime check
grib_file.remove()
with open(TEST_DATA, "rb") as file:
grib_file.write_binary(file.read())
res = messages.FileIndex.from_indexpath_or_filestream(
messages.FileStream(str(grib_file)), ["paramId"]
)
assert isinstance(res, messages.FileIndex)
def test_FileIndex_errors() -> None:
class MyMessage(messages.ComputedKeysMessage):
computed_keys = {
"error_key": (lambda m: bool(1 / 0), lambda m, v: None)
} # pragma: no branch
stream = messages.FileStream(TEST_DATA, message_class=MyMessage)
res = messages.FileIndex.from_filestream(stream, ["paramId", "error_key"])
assert res["paramId"] == [129, 130]
assert len(res) == 2
assert list(res) == ["paramId", "error_key"]
assert res["error_key"] == ["undef"]
def test_build_data_var_components_encode_cf_geography():
stream = messages.FileStream(path=TEST_DATA, message_class=cfmessage.CfMessage)
index = stream.index(dataset.ALL_KEYS).subindex(paramId=130)
dims, data_var, coord_vars = dataset.build_variable_components(
index=index, encode_cf='geography',
)
assert dims == {
'number': 10, 'dataDate': 2, 'dataTime': 2,
'level': 2, 'latitude': 61, 'longitude': 120,
}
assert data_var.data.shape == (10, 2, 2, 2, 61, 120)
# equivalent to not np.isnan without importing numpy
assert data_var.data[:, :, :, :, :, :].mean() > 0.
def test_FileIndex() -> None:
res = messages.FileIndex.from_fieldset(messages.FileStream(TEST_DATA), ["paramId"])
assert res["paramId"] == [129, 130]
assert len(res) == 1
assert list(res) == ["paramId"]
assert res.first()
with pytest.raises(ValueError):
res.getone("paramId")
with pytest.raises(KeyError):
res["non-existent-key"]
subres = res.subindex(paramId=130)
assert subres.get("paramId") == [130]
assert subres.getone("paramId") == 130
assert len(subres) == 1
def test_FileIndex():
res = messages.FileIndex.from_filestream(messages.FileStream(TEST_DATA), ['paramId'])
assert res['paramId'] == [129, 130]
assert len(res) == 1
assert list(res) == ['paramId']
assert res.first()
with pytest.raises(ValueError):
res.getone('paramId')
with pytest.raises(KeyError):
res['non-existent-key']
subres = res.subindex(paramId=130)
assert subres.get('paramId') == [130]
assert subres.getone('paramId') == 130
assert len(subres) == 1
def test_build_data_var_components_encode_cf_geography() -> None:
stream = messages.FileStream(path=TEST_DATA)
index_keys = sorted(dataset.INDEX_KEYS + ["time", "step"])
index = messages.FieldsetIndex.from_fieldset(
stream, index_keys, cfmessage.COMPUTED_KEYS).subindex(paramId=130)
dims, data_var, coord_vars = dataset.build_variable_components(
index=index, encode_cf="geography")
assert dims == {
"number": 10,
"dataDate": 2,
"dataTime": 2,
"level": 2,
"latitude": 61,
"longitude": 120,
}
assert data_var.data.shape == (10, 2, 2, 2, 61, 120)
# equivalent to not np.isnan without importing numpy
assert data_var.data[:, :, :, :, :, :].mean() > 0.0
def test_FileStream():
res = messages.FileStream(TEST_DATA)
leader = res.first()
assert len(leader) > 100
assert sum(1 for _ in res) == leader['count']
assert len(res.index(['paramId'])) == 1
