def test_FileStream() -> None: res = messages.FileStream(TEST_DATA) leader = res[0] assert len(leader) > 100 assert sum(1 for _ in res.items()) == leader["count"] # __file__ is not a GRIB, but contains the "GRIB" string, so it is a very tricky corner case res = messages.FileStream(str(__file__)) with pytest.raises(eccodes.UnsupportedEditionError): res[0]
def test_FileStream(): res = messages.FileStream(TEST_DATA) leader = res.first() assert len(leader) > 100 assert sum(1 for _ in res) == leader['count'] assert len(res.index(['paramId'])) == 1 # __file__ is not a GRIB, but contains the "GRIB" string, so it is a very tricky corner case res = messages.FileStream(str(__file__)) with pytest.raises(EOFError): res.first()
def test_FileStream() -> None: res = messages.FileStream(TEST_DATA) leader = res.first() assert len(leader) > 100 assert sum(1 for _ in res.items()) == leader["count"] assert len(res.index(["paramId"])) == 1 # __file__ is not a GRIB, but contains the "GRIB" string, so it is a very tricky corner case res = messages.FileStream(str(__file__)) with pytest.raises(EOFError): res.first() res = messages.FileStream(str(__file__), errors="ignore") with pytest.raises(EOFError): res.first()
def test_build_data_var_components_no_encode(): index = messages.FileStream(path=TEST_DATA).index(dataset.INDEX_KEYS).subindex(paramId=130) dims, data_var, coord_vars = dataset.build_variable_components(index=index) assert dims == {"number": 10, "dataDate": 2, "dataTime": 2, "level": 2, "values": 7320} assert data_var.data.shape == (10, 2, 2, 2, 7320) # equivalent to not np.isnan without importing numpy assert data_var.data[:, :, :, :, :].mean() > 0.0
def test_FileIndex_from_indexpath_or_filestream(tmpdir): grib_file = tmpdir.join('file.grib') with open(TEST_DATA, 'rb') as file: grib_file.write_binary(file.read()) # create index file res = messages.FileIndex.from_indexpath_or_filestream( messages.FileStream(str(grib_file)), ['paramId'], ) assert isinstance(res, messages.FileIndex) # read index file res = messages.FileIndex.from_indexpath_or_filestream( messages.FileStream(str(grib_file)), ['paramId'], ) assert isinstance(res, messages.FileIndex) # do not read nor create the index file res = messages.FileIndex.from_indexpath_or_filestream( messages.FileStream(str(grib_file)), ['paramId'], indexpath='', ) assert isinstance(res, messages.FileIndex) # can't create nor read index file res = messages.FileIndex.from_indexpath_or_filestream( messages.FileStream(str(grib_file)), ['paramId'], indexpath=str(tmpdir.join('non-existent-folder').join('non-existent-file')), ) assert isinstance(res, messages.FileIndex) # trigger mtime check grib_file.remove() with open(TEST_DATA, 'rb') as file: grib_file.write_binary(file.read()) res = messages.FileIndex.from_indexpath_or_filestream( messages.FileStream(str(grib_file)), ['paramId'], ) assert isinstance(res, messages.FileIndex)
def test_FileIndex_errors() -> None: computed_keys = {"error_key": (lambda m: bool(1 / 0), lambda m, v: None)} # pragma: no branch stream = messages.FileStream(TEST_DATA) res = messages.FileIndex.from_fieldset(stream, ["paramId", "error_key"], computed_keys) assert res["paramId"] == [129, 130] assert len(res) == 2 assert list(res) == ["paramId", "error_key"] assert res["error_key"] == ["undef"]
def test_build_dataset_components_time_dims() -> None: index_keys = sorted(dataset.INDEX_KEYS + ["time", "step"]) stream = messages.FileStream(TEST_DATA_UKMO, "warn") index = dataset.open_fileindex(stream, messages.DEFAULT_INDEXPATH, index_keys) dims = dataset.build_dataset_components(index, read_keys=[])[0] assert dims == { "latitude": 6, "longitude": 11, "number": 28, "step": 20, "time": 8, } time_dims = ["indexing_time", "verifying_time"] index_keys = sorted(dataset.INDEX_KEYS + time_dims) stream = messages.FileStream(TEST_DATA_UKMO, "warn") index = dataset.open_fileindex(stream, messages.DEFAULT_INDEXPATH, index_keys) dims, *_ = dataset.build_dataset_components(index, read_keys=[], time_dims=time_dims) assert dims == { "number": 28, "indexing_time": 2, "verifying_time": 4, "latitude": 6, "longitude": 11, } time_dims = ["indexing_time", "step"] index_keys = sorted(dataset.INDEX_KEYS + time_dims) stream = messages.FileStream(TEST_DATA_UKMO, "warn") index = dataset.open_fileindex(stream, messages.DEFAULT_INDEXPATH, index_keys) dims, *_ = dataset.build_dataset_components(index, read_keys=[], time_dims=time_dims) assert dims == { "number": 28, "indexing_time": 2, "step": 20, "latitude": 6, "longitude": 11 }
def test_FileIndex_errors(): class MyMessage(messages.ComputedKeysMessage): computed_keys = {'error_key': lambda m: 1 / 0} stream = messages.FileStream(TEST_DATA, message_class=MyMessage) res = messages.FileIndex.from_filestream(stream, ['paramId', 'error_key']) assert res['paramId'] == [129, 130] assert len(res) == 2 assert list(res) == ['paramId', 'error_key'] assert res['error_key'] == ['undef']
def test_FileIndex_errors(): class MyMessage(messages.ComputedKeysMessage): computed_keys = {"error_key": lambda m: 1 / 0} stream = messages.FileStream(TEST_DATA, message_class=MyMessage) res = messages.FileIndex.from_filestream(stream, ["paramId", "error_key"]) assert res["paramId"] == [129, 130] assert len(res) == 2 assert list(res) == ["paramId", "error_key"] assert res["error_key"] == ["undef"]
def test_FileIndex_from_indexpath_or_filestream(tmpdir: py.path.local) -> None: grib_file = tmpdir.join("file.grib") with open(TEST_DATA, "rb") as file: grib_file.write_binary(file.read()) # create index file res = messages.FileIndex.from_indexpath_or_filestream( messages.FileStream(str(grib_file)), ["paramId"] ) assert isinstance(res, messages.FileIndex) # read index file res = messages.FileIndex.from_indexpath_or_filestream( messages.FileStream(str(grib_file)), ["paramId"] ) assert isinstance(res, messages.FileIndex) # do not read nor create the index file res = messages.FileIndex.from_indexpath_or_filestream( messages.FileStream(str(grib_file)), ["paramId"], indexpath="" ) assert isinstance(res, messages.FileIndex) # can't create nor read index file res = messages.FileIndex.from_indexpath_or_filestream( messages.FileStream(str(grib_file)), ["paramId"], indexpath=str(tmpdir.join("non-existent-folder").join("non-existent-file")), ) assert isinstance(res, messages.FileIndex) # trigger mtime check grib_file.remove() with open(TEST_DATA, "rb") as file: grib_file.write_binary(file.read()) res = messages.FileIndex.from_indexpath_or_filestream( messages.FileStream(str(grib_file)), ["paramId"] ) assert isinstance(res, messages.FileIndex)
def test_FileIndex_errors() -> None: class MyMessage(messages.ComputedKeysMessage): computed_keys = { "error_key": (lambda m: bool(1 / 0), lambda m, v: None) } # pragma: no branch stream = messages.FileStream(TEST_DATA, message_class=MyMessage) res = messages.FileIndex.from_filestream(stream, ["paramId", "error_key"]) assert res["paramId"] == [129, 130] assert len(res) == 2 assert list(res) == ["paramId", "error_key"] assert res["error_key"] == ["undef"]
def test_build_data_var_components_encode_cf_geography(): stream = messages.FileStream(path=TEST_DATA, message_class=cfmessage.CfMessage) index = stream.index(dataset.ALL_KEYS).subindex(paramId=130) dims, data_var, coord_vars = dataset.build_variable_components( index=index, encode_cf='geography', ) assert dims == { 'number': 10, 'dataDate': 2, 'dataTime': 2, 'level': 2, 'latitude': 61, 'longitude': 120, } assert data_var.data.shape == (10, 2, 2, 2, 61, 120) # equivalent to not np.isnan without importing numpy assert data_var.data[:, :, :, :, :, :].mean() > 0.
def test_FileIndex() -> None: res = messages.FileIndex.from_fieldset(messages.FileStream(TEST_DATA), ["paramId"]) assert res["paramId"] == [129, 130] assert len(res) == 1 assert list(res) == ["paramId"] assert res.first() with pytest.raises(ValueError): res.getone("paramId") with pytest.raises(KeyError): res["non-existent-key"] subres = res.subindex(paramId=130) assert subres.get("paramId") == [130] assert subres.getone("paramId") == 130 assert len(subres) == 1
def test_FileIndex(): res = messages.FileIndex.from_filestream(messages.FileStream(TEST_DATA), ['paramId']) assert res['paramId'] == [129, 130] assert len(res) == 1 assert list(res) == ['paramId'] assert res.first() with pytest.raises(ValueError): res.getone('paramId') with pytest.raises(KeyError): res['non-existent-key'] subres = res.subindex(paramId=130) assert subres.get('paramId') == [130] assert subres.getone('paramId') == 130 assert len(subres) == 1
def test_build_data_var_components_encode_cf_geography() -> None: stream = messages.FileStream(path=TEST_DATA) index_keys = sorted(dataset.INDEX_KEYS + ["time", "step"]) index = messages.FieldsetIndex.from_fieldset( stream, index_keys, cfmessage.COMPUTED_KEYS).subindex(paramId=130) dims, data_var, coord_vars = dataset.build_variable_components( index=index, encode_cf="geography") assert dims == { "number": 10, "dataDate": 2, "dataTime": 2, "level": 2, "latitude": 61, "longitude": 120, } assert data_var.data.shape == (10, 2, 2, 2, 61, 120) # equivalent to not np.isnan without importing numpy assert data_var.data[:, :, :, :, :, :].mean() > 0.0
def test_FileStream(): res = messages.FileStream(TEST_DATA) leader = res.first() assert len(leader) > 100 assert sum(1 for _ in res) == leader['count'] assert len(res.index(['paramId'])) == 1