Python open_dataset Examples

Example #1

File: test_50_datamodels.py Project: raybellwaves/cfgrib

def test_ecmwf():
    ds = xarray_store.open_dataset(TEST_DATA1)

    res = cfcoords.translate_coords(ds, coord_model=datamodels.ECMWF)

    assert set(
        res.dims) == {"latitude", "level", "longitude", "number", "time"}
    assert set(res.coords) == {
        "latitude",
        "level",
        "longitude",
        "number",
        "step",
        "time",
        "valid_time",
    }

    ds = xarray_store.open_dataset(TEST_DATA2)

    res = cfcoords.translate_coords(ds, coord_model=datamodels.ECMWF)

    assert set(res.dims) == {"x", "y"}
    assert set(res.coords) == {
        "heightAboveGround",
        "latitude",
        "longitude",
        "step",
        "time",
        "valid_time",
    }

Example #2

def test_ecmwf():
    ds = xarray_store.open_dataset(TEST_DATA1)

    res = cfcoords.translate_coords(ds, coord_model=datamodels.ECMWF)

    assert set(
        res.dims) == {'latitude', 'level', 'longitude', 'number', 'time'}
    assert set(res.coords) == {
        'latitude',
        'level',
        'longitude',
        'number',
        'step',
        'time',
        'valid_time',
    }

    ds = xarray_store.open_dataset(TEST_DATA2)

    res = cfcoords.translate_coords(ds, coord_model=datamodels.ECMWF)

    assert set(res.dims) == {'x', 'y'}
    assert set(res.coords) == {
        'heightAboveGround',
        'latitude',
        'longitude',
        'step',
        'time',
        'valid_time',
    }

Example #3

File: test_50_datamodels.py Project: raybellwaves/cfgrib

def test_cds():
    ds = xarray_store.open_dataset(TEST_DATA1)

    res = cfcoords.translate_coords(ds, coord_model=datamodels.CDS)

    assert set(res.dims) == {
        "forecast_reference_time", "lat", "lon", "plev", "realization"
    }
    assert set(res.coords) == {
        "forecast_reference_time",
        "lat",
        "leadtime",
        "lon",
        "plev",
        "realization",
        "time",
    }

    ds = xarray_store.open_dataset(TEST_DATA2)

    res = cfcoords.translate_coords(ds, coord_model=datamodels.CDS)

    assert set(res.dims) == {"x", "y"}
    assert set(res.coords) == {
        "forecast_reference_time",
        "heightAboveGround",
        "lat",
        "leadtime",
        "lon",
        "time",
    }

Example #4

def test_cds():
    ds = xarray_store.open_dataset(TEST_DATA1)

    res = cfcoords.translate_coords(ds, coord_model=datamodels.CDS)

    assert set(res.dims) == {'time', 'lat', 'lon', 'plev', 'realization'}
    assert set(res.coords) == {
        'forecast_reference_time',
        'lat',
        'leadtime',
        'lon',
        'plev',
        'realization',
        'time',
    }

    ds = xarray_store.open_dataset(TEST_DATA2)

    res = cfcoords.translate_coords(ds, coord_model=datamodels.CDS)

    assert set(res.dims) == {'x', 'y'}
    assert set(res.coords) == {
        'forecast_reference_time',
        'heightAboveGround',
        'lat',
        'leadtime',
        'lon',
        'time',
    }

Example #5

def test_open_dataset_fail(grib_name):
    grib_path = os.path.join(SAMPLE_DATA_FOLDER, grib_name + ".grib")

    with pytest.raises(ValueError):
        xarray_store.open_dataset(grib_path,
                                  cache=False,
                                  backend_kwargs={"errors": "raise"})

Example #6

def test_open_dataset_corrupted():
    res = xarray_store.open_dataset(TEST_CORRUPTED)

    assert res.attrs['GRIB_edition'] == 1
    assert len(res.data_vars) == 1

    with pytest.raises(eccodes.EcCodesError):
        xarray_store.open_dataset(TEST_CORRUPTED, errors='strict')

Example #7

File: test_40_xarray_store.py Project: xavierabellan/cfgrib

def test_open_dataset_corrupted():
    res = xarray_store.open_dataset(TEST_CORRUPTED)

    assert res.attrs['GRIB_edition'] == 1
    assert len(res.data_vars) == 1

    with pytest.raises(Exception):
        xarray_store.open_dataset(TEST_CORRUPTED, backend_kwargs={'grib_errors': 'raise'})

Example #8

def test_open_dataset_corrupted() -> None:
    res = xarray_store.open_dataset(TEST_CORRUPTED)

    assert res.attrs["GRIB_edition"] == 1
    assert len(res.data_vars) == 1

    with pytest.raises(Exception):
        xarray_store.open_dataset(TEST_CORRUPTED,
                                  backend_kwargs={"grib_errors": "raise"})

Example #9

File: test_50_sample_data.py Project: masabhathini/cfgrib

def test_to_grib_ecmwf(grib_name, tmpdir):
    grib_path = os.path.join(SAMPLE_DATA_FOLDER, grib_name + '.grib')
    out_path = str(tmpdir.join(grib_name + '.grib'))

    res = xarray_store.open_dataset(grib_path, cache=False)

    xarray_store.to_grib(res, out_path)
    reread = xarray_store.open_dataset(out_path, cache=False)
    assert res.equals(reread)

Example #10

File: test_50_sample_data.py Project: xavierabellan/cfgrib

def test_canonical_dataset_to_grib(grib_name, tmpdir):
    grib_path = os.path.join(SAMPLE_DATA_FOLDER, grib_name + '.grib')
    out_path = str(tmpdir.join(grib_name + '.grib'))

    res = xarray_store.open_dataset(grib_path)

    xarray_to_grib.canonical_dataset_to_grib(res, out_path)
    reread = xarray_store.open_dataset(out_path)
    assert res.equals(reread)

Example #11

def test_canonical_dataset_to_grib(grib_name, tmpdir):
    grib_path = os.path.join(SAMPLE_DATA_FOLDER, grib_name + ".grib")
    out_path = str(tmpdir.join(grib_name + ".grib"))

    res = xarray_store.open_dataset(grib_path)

    with pytest.warns(FutureWarning):
        xarray_to_grib.canonical_dataset_to_grib(res, out_path)
    reread = xarray_store.open_dataset(out_path)
    assert res.equals(reread)

Example #12

def test_all(cache):
    da = xarray_store.open_dataset(TEST_DATA, cache=cache).data_vars['t']
    va = da.values[:]

    assert va.shape == (10, 4, 2, 61, 120)

    assert da.mean() == va.mean()

Example #13

def test_getitem_int(cache: bool) -> None:
    da = xarray_store.open_dataset(TEST_DATA, cache=cache).data_vars["t"]
    va = da.values[:]

    assert da.isel(isobaricInhPa=1).values.shape == va[:, :, 1].shape
    assert da.isel(isobaricInhPa=1).mean() == va[:, :, 1].mean()
    assert da.sel(isobaricInhPa=500).mean() == va[:, :, 1].mean()

Example #14

def test_getitem_int(cache):
    da = xarray_store.open_dataset(TEST_DATA, cache=cache).data_vars['t']
    va = da.values[:]

    assert da.isel(air_pressure=1).values.shape == va[:, :, 1].shape
    assert da.isel(air_pressure=1).mean() == va[:, :, 1].mean()
    assert da.sel(air_pressure=500).mean() == va[:, :, 1].mean()

Example #15

def test_open_dataset():
    res = xarray_store.open_dataset(TEST_DATA)

    assert res.attrs['GRIB_edition'] == 1

    var = res['t']
    assert var.attrs['GRIB_gridType'] == 'regular_ll'
    assert var.attrs['units'] == 'K'
    assert var.dims == \
        ('number', 'time', 'isobaricInhPa', 'latitude', 'longitude')

    assert var.mean() > 0.

    with pytest.warns(FutureWarning):
        xarray_store.open_dataset(
            TEST_DATA, filter_by_keys={'typeOfLevel': 'isobaricInhPa'})

Example #16

File: test_40_xarray_store.py Project: xavierabellan/cfgrib

def test_open_dataset_encode_cf_vertical():
    backend_kwargs = {'encode_cf': ('vertical',)}
    res = xarray_store.open_dataset(TEST_DATA, backend_kwargs=backend_kwargs)

    var = res['t']
    assert var.dims == ('number', 'dataDate', 'dataTime', 'isobaricInhPa', 'values')

    assert var.mean() > 0.0

Example #17

def test_getitem_list(cache):
    da = xarray_store.open_dataset(TEST_DATA, cache=cache).data_vars['t']
    va = da.values[:]

    assert da.isel(number=[2, 3, 4, 5]).mean() == va[[2, 3, 4, 5]].mean()
    assert da.isel(number=[4, 3, 2, 5]).mean() == va[[4, 3, 2, 5]].mean()
    assert da.sel(number=[2, 3, 4, 5]).mean() == va[[2, 3, 4, 5]].mean()
    assert da.sel(number=[4, 3, 2, 5]).mean() == va[[4, 3, 2, 5]].mean()

Example #18

File: test_50_xarray_getitem.py Project: jlandmann/cfgrib

def test_getitem_latlon(cache):
    da = xarray_store.open_dataset(TEST_DATA, cache=cache).data_vars['t']
    va = da.values[:]

    assert da.isel(latitude=slice(0, 3),
                   longitude=slice(0, 33)).mean() == va[..., :3, :33].mean()
    assert da.sel(latitude=slice(90, 0),
                  longitude=slice(0, 90)).mean() == va[..., :31, :31].mean()

Example #19

def test_open_dataset():
    res = xarray_store.open_dataset(TEST_DATA)

    assert res.attrs['GRIB_edition'] == 1

    var = res['t']
    assert var.attrs['GRIB_gridType'] == 'regular_ll'
    assert var.attrs['units'] == 'K'
    assert var.dims == \
        ('number', 'time', 'isobaricInhPa', 'latitude', 'longitude')

    assert var.mean() > 0.

    with pytest.raises(ValueError):
        xarray_store.open_dataset(TEST_DATA, engine='netcdf4')

    res = xarray_store.open_dataset(TEST_IGNORE,
                                    backend_kwargs={'errors': 'warn'})
    assert 'isobaricInhPa' in res.dims

    res = xarray_store.open_dataset(TEST_IGNORE,
                                    backend_kwargs={'errors': 'ignore'})
    assert 'isobaricInhPa' in res.dims

    with pytest.raises(ValueError):
        xarray_store.open_dataset(TEST_IGNORE,
                                  backend_kwargs={'errors': 'raise'})

Example #20

def test_open_dataset() -> None:
    res = xarray_store.open_dataset(TEST_DATA)

    assert res.attrs["GRIB_edition"] == 1

    var = res["t"]
    assert var.attrs["GRIB_gridType"] == "regular_ll"
    assert var.attrs["units"] == "K"
    assert var.dims == ("number", "time", "isobaricInhPa", "latitude",
                        "longitude")

    assert var.mean() > 0.0

    with pytest.raises(ValueError):
        xarray_store.open_dataset(TEST_DATA, engine="any-other-engine")

    res = xarray_store.open_dataset(TEST_IGNORE,
                                    backend_kwargs={"errors": "warn"})
    assert "isobaricInhPa" in res.dims

    res = xarray_store.open_dataset(TEST_IGNORE,
                                    backend_kwargs={"errors": "ignore"})
    assert "isobaricInhPa" in res.dims

    with pytest.raises(ValueError):
        xarray_store.open_dataset(TEST_IGNORE,
                                  backend_kwargs={"errors": "raise"})

Example #21

def test_open_dataset_encode_vertical():
    res = xarray_store.open_dataset(TEST_DATA,
                                    flavour_name='eccodes',
                                    encode_vertical=True)

    var = res['t']
    assert var.dims == ('number', 'dataDate', 'dataTime', 'air_pressure', 'i')

    assert var.mean() > 0.

Example #22

def test_getitem_slice(cache):
    da = xarray_store.open_dataset(TEST_DATA, cache=cache).data_vars['t']
    va = da.values[:]

    assert da.isel(number=slice(2, 6)).mean() == va[2:6].mean()
    assert da.isel(number=slice(2, 6, 2)).mean() == va[2:6:2].mean()
    # NOTE: label based indexing in xarray is inclusive of both the start and stop bounds.
    assert da.sel(number=slice(2, 6)).mean() == va[2:7].mean()
    assert da.sel(number=slice(2, 6, 2)).mean() == va[2:7:2].mean()

Example #23

def test_open_dataset_encode_cf_vertical() -> None:
    backend_kwargs = {"encode_cf": ("vertical", )}
    res = xarray_store.open_dataset(TEST_DATA, backend_kwargs=backend_kwargs)

    var = res["t"]
    assert var.dims == ("number", "dataDate", "dataTime", "isobaricInhPa",
                        "values")

    assert var.mean() > 0.0

Example #24

def test_open_dataset_encode_cf_time() -> None:
    backend_kwargs = {"encode_cf": ("time", )}
    res = xarray_store.open_dataset(TEST_DATA, backend_kwargs=backend_kwargs)

    assert res.attrs["GRIB_edition"] == 1
    assert res["t"].attrs["GRIB_gridType"] == "regular_ll"
    assert res["t"].attrs["GRIB_units"] == "K"
    assert res["t"].dims == ("number", "time", "level", "values")

    assert res["t"].mean() > 0.0

Example #25

def test_open_dataset_encode_cf_time():
    backend_kwargs = {'encode_cf': ('time', )}
    res = xarray_store.open_dataset(TEST_DATA, backend_kwargs=backend_kwargs)

    assert res.attrs['GRIB_edition'] == 1
    assert res['t'].attrs['GRIB_gridType'] == 'regular_ll'
    assert res['t'].attrs['GRIB_units'] == 'K'
    assert res['t'].dims == ('number', 'time', 'level', 'values')

    assert res['t'].mean() > 0.

Example #26

File: test_40_xarray_store.py Project: xavierabellan/cfgrib

def test_open_dataset_eccodes():
    res = xarray_store.open_dataset(TEST_DATA)

    assert res.attrs['GRIB_edition'] == 1

    var = res['t']
    assert var.attrs['GRIB_gridType'] == 'regular_ll'
    assert var.attrs['units'] == 'K'
    assert var.dims == ('number', 'time', 'isobaricInhPa', 'latitude', 'longitude')

    assert var.mean() > 0.0

Example #27

def test_open_dataset_encode_time():
    res = xarray_store.open_dataset(TEST_DATA,
                                    flavour_name='eccodes',
                                    encode_time=True)

    assert res.attrs['GRIB_edition'] == 1
    assert res['t'].attrs['GRIB_gridType'] == 'regular_ll'
    assert res['t'].attrs['units'] == 'K'
    assert res['t'].dims == ('number', 'time', 'topLevel', 'i')

    assert res['t'].mean() > 0.

Example #28

 def to_dataset(self):
     # soft dependency on cfgrib
     try:
         from cfgrib import xarray_store
     except ImportError:
         print(
             "Package cfgrib/xarray_store not found. Try running 'pip install cfgrib'."
         )
         raise
     dataset = xarray_store.open_dataset(self.url())
     return dataset

Example #29

def test_open_dataset_extra_coords(grib_name: str, ndims: T.Any) -> None:
    grib_path = os.path.join(SAMPLE_DATA_FOLDER, grib_name + ".grib")
    res = xarray_store.open_dataset(
        grib_path,
        backend_kwargs={"extra_coords": {
            "experimentVersionNumber": "time"
        }},
        cache=False,
    )
    assert "experimentVersionNumber" in res.coords
    assert len(res["experimentVersionNumber"].dims) == ndims

Example #30

def test_open_dataset_cds():
    res = xarray_store.open_dataset(TEST_DATA, flavour_name='cds')

    assert res.attrs['GRIB_edition'] == 1

    var = res['t']
    assert var.attrs['GRIB_gridType'] == 'regular_ll'
    assert var.attrs['units'] == 'K'
    assert var.dims == ('realization', 'forecast_reference_time', 'plev',
                        'lat', 'lon')

    assert var.mean() > 0.