def oca_read_all_nc_modis(filename):
"""Read geolocation, angles info, ctth, and cma."""
oca_nc = h5netcdf.File(filename, 'r')
my_dir = os.path.dirname(filename)
my_file = os.path.basename(filename).replace('OCA', 'CLM')
oca_nc_clm = h5netcdf.File(os.path.join(my_dir, my_file))
logger.info("Opening file %s", filename)
logger.info("Reading longitude, latitude and time ...")
cloudproducts = read_oca_geoobj_modis(oca_nc, filename)
logger.info("Reading angles ...")
cloudproducts.imager_angles = read_oca_angobj_modis(oca_nc)
logger.info("Reading cloud pressure ...")
# , angle_obj)
cma = read_oca_cmask_modis(oca_nc_clm)
ctype, cma, ctth = read_oca_ctype_cmask_ctth_modis(oca_nc)
cloudproducts.cma = cma
cloudproducts.ctth = ctth
cloudproducts.ctype = ctype
cloudproducts.aux = read_oca_secondlayer_etc_info_modis(oca_nc)
logger.info("Not reading cloud microphysical properties")
# cloudproducts.cpp = read_oca_cpp(oca_nc)
logger.info("Not reading surface temperature")
logger.info("Not reading channel data")
cloudproducts.longitude[
cloudproducts.longitude >
180] = cloudproducts.longitude[cloudproducts.longitude > 180] - 360
return cloudproducts
def test_reopen_file_different_dimension_sizes(tmp_local_netcdf):
# regression test for https://github.com/shoyer/h5netcdf/issues/55
with h5netcdf.File(tmp_local_netcdf, mode='w') as f:
f.create_variable('/one/foo', data=[1], dimensions=('x', ))
with h5netcdf.File(tmp_local_netcdf, mode='a') as f:
f.create_variable('/two/foo', data=[1, 2], dimensions=('x', ))
with netCDF4.Dataset(tmp_local_netcdf, mode='r') as f:
assert f.groups['one'].variables['foo'][...].shape == (1, )
def test_reopen_file_different_dimension_sizes(tmp_local_netcdf):
# regression test for https://github.com/h5netcdf/h5netcdf/issues/55
with h5netcdf.File(tmp_local_netcdf, "w") as f:
f.create_variable("/one/foo", data=[1], dimensions=("x",))
with h5netcdf.File(tmp_local_netcdf, "a") as f:
f.create_variable("/two/foo", data=[1, 2], dimensions=("x",))
with netCDF4.Dataset(tmp_local_netcdf, "r") as f:
assert f.groups["one"].variables["foo"][...].shape == (1,)
def test_invalid_then_valid_no_ncproperties(tmp_netcdf):
with h5netcdf.File(tmp_netcdf, invalid_netcdf=True):
pass
with h5netcdf.File(tmp_netcdf):
pass
with h5py.File(tmp_netcdf) as f:
# still not a valid netcdf file
assert '_NCProperties' not in f.attrs
def test_reading_unused_unlimited_dimension(tmp_local_or_remote_netcdf):
"""Test reading a file with unused dimension of unlimited size"""
with h5netcdf.File(tmp_local_or_remote_netcdf, 'w') as f:
f.dimensions = {'x': None}
f.resize_dimension('x', 5)
assert f.dimensions == {'x': None}
f = h5netcdf.File(tmp_local_or_remote_netcdf, 'r')
def test_invalid_then_valid_no_ncproperties(tmp_local_or_remote_netcdf):
with h5netcdf.File(tmp_local_or_remote_netcdf, "w", invalid_netcdf=True):
pass
with h5netcdf.File(tmp_local_or_remote_netcdf, "a"):
pass
h5 = get_hdf5_module(tmp_local_or_remote_netcdf)
with h5.File(tmp_local_or_remote_netcdf, "r") as f:
# still not a valid netcdf file
assert "_NCProperties" not in f.attrs
def __init__(self, filename, filename_info, filetype_info):
super(NCSLSTR1B, self).__init__(filename, filename_info, filetype_info)
self.nc = h5netcdf.File(filename, 'r')
self.channel = filename_info['dataset_name']
cal_file = os.path.join(os.path.dirname(filename), 'viscal.nc')
self.cal = h5netcdf.File(cal_file, 'r')
# TODO: get metadata from the manifest file (xfdumanifest.xml)
self.platform_name = PLATFORM_NAMES[filename_info['mission_id']]
self.sensor = 'slstr'
def test_nc4_non_coord(tmp_local_netcdf):
with h5netcdf.File(tmp_local_netcdf, "w") as f:
f.dimensions = {"x": None, "y": 2}
f.create_variable("test", dimensions=("x",), dtype=np.int64)
f.create_variable("y", dimensions=("x",), dtype=np.int64)
with h5netcdf.File(tmp_local_netcdf, "r") as f:
assert f.dimensions == {"x": None, "y": 2}
assert list(f.variables) == ["y", "test"]
assert list(f._h5group.keys()) == ["_nc4_non_coord_y", "test", "x", "y"]
def test_hierarchical_access_auto_create(tmp_local_or_remote_netcdf):
ds = h5netcdf.File(tmp_local_or_remote_netcdf, "w")
ds.create_variable("/foo/bar", data=1)
g = ds.create_group("foo/baz")
g.create_variable("/foo/hello", data=2)
assert set(ds) == set(["foo"])
assert set(ds["foo"]) == set(["bar", "baz", "hello"])
ds.close()
ds = h5netcdf.File(tmp_local_or_remote_netcdf, "r")
assert set(ds) == set(["foo"])
assert set(ds["foo"]) == set(["bar", "baz", "hello"])
ds.close()
def test_hierarchical_access_auto_create(tmp_netcdf):
ds = h5netcdf.File(tmp_netcdf, 'w')
ds.create_variable('/foo/bar', data=1)
g = ds.create_group('foo/baz')
g.create_variable('/foo/hello', data=2)
assert set(ds) == set(['foo'])
assert set(ds['foo']) == set(['bar', 'baz', 'hello'])
ds.close()
ds = h5netcdf.File(tmp_netcdf, 'r')
assert set(ds) == set(['foo'])
assert set(ds['foo']) == set(['bar', 'baz', 'hello'])
ds.close()
def test_attrs_api(tmp_netcdf):
with h5netcdf.File(tmp_netcdf) as ds:
ds.attrs['conventions'] = 'CF'
ds.dimensions['x'] = 1
v = ds.create_variable('x', ('x', ), 'i4')
v.attrs.update({'units': 'meters', 'foo': 'bar'})
assert ds._closed
with h5netcdf.File(tmp_netcdf) as ds:
assert len(ds.attrs) == 1
assert dict(ds.attrs) == {'conventions': 'CF'}
assert list(ds.attrs) == ['conventions']
assert dict(ds['x'].attrs) == {'units': 'meters', 'foo': 'bar'}
assert len(ds['x'].attrs) == 2
assert sorted(ds['x'].attrs) == ['foo', 'units']
def test_attrs_api(tmp_local_or_remote_netcdf):
with h5netcdf.File(tmp_local_or_remote_netcdf, "w") as ds:
ds.attrs["conventions"] = "CF"
ds.attrs["empty_string"] = h5py.Empty(dtype=np.dtype("|S1"))
ds.dimensions["x"] = 1
v = ds.create_variable("x", ("x",), "i4")
v.attrs.update({"units": "meters", "foo": "bar"})
assert ds._closed
with h5netcdf.File(tmp_local_or_remote_netcdf, "r") as ds:
assert len(ds.attrs) == 2
assert dict(ds.attrs) == {"conventions": "CF", "empty_string": b""}
assert list(ds.attrs) == ["conventions", "empty_string"]
assert dict(ds["x"].attrs) == {"units": "meters", "foo": "bar"}
assert len(ds["x"].attrs) == 2
assert sorted(ds["x"].attrs) == ["foo", "units"]
def __init__(self, filename, filename_info, filetype_info):
super(NCSLSTRAngles, self).__init__(filename, filename_info,
filetype_info)
self.nc = None
# TODO: get metadata from the manifest file (xfdumanifest.xml)
self.platform_name = PLATFORM_NAMES[filename_info['mission_id']]
self.sensor = 'slstr'
self._start_time = filename_info['start_time']
self._end_time = filename_info['end_time']
cart_file = os.path.join(os.path.dirname(self.filename),
'cartesian_i{}.nc'.format(self.view))
self.cart = h5netcdf.File(cart_file, 'r')
cartx_file = os.path.join(os.path.dirname(self.filename),
'cartesian_tx.nc')
self.cartx = h5netcdf.File(cartx_file, 'r')
def test_reading_unlimited_dimensions_created_with_c_api(tmp_netcdf):
with netCDF4.Dataset(tmp_netcdf, "w") as f:
f.createDimension('x', None)
f.createDimension('y', 3)
f.createDimension('z', None)
dummy1 = f.createVariable('dummy1', float, ('x', 'y'))
f.createVariable('dummy2', float, ('y', 'x', 'x'))
g = f.createGroup('test')
g.createVariable('dummy3', float, ('y', 'y'))
g.createVariable('dummy4', float, ('z', 'z'))
# Assign something to trigger a resize.
dummy1[:] = [[1, 2, 3], [4, 5, 6]]
with h5netcdf.File(tmp_netcdf) as f:
assert f.dimensions['x'] is None
assert f.dimensions['y'] == 3
assert f.dimensions['z'] is None
# This is parsed correctly due to h5netcdf's init trickery.
assert f._current_dim_sizes['x'] == 2
assert f._current_dim_sizes['y'] == 3
assert f._current_dim_sizes['z'] == 0
# But the actual data-set and arrays are not correct.
assert f['dummy1'].shape == (2, 3)
# XXX: This array has some data with dimension x - netcdf does not
# appear to keep dimensions consistent.
assert f['dummy2'].shape == (3, 0, 0)
f.groups['test']['dummy3'].shape == (3, 3)
f.groups['test']['dummy4'].shape == (0, 0)
def test_c_api_can_read_unlimited_dimensions(tmp_netcdf):
with h5netcdf.File(tmp_netcdf) as f:
# Three dimensions, only one is limited.
f.dimensions['x'] = None
f.dimensions['y'] = 3
f.dimensions['z'] = None
f.create_variable('dummy1', dimensions=('x', 'y'), dtype=np.int64)
f.create_variable('dummy2', dimensions=('y', 'x', 'x'), dtype=np.int64)
g = f.create_group('test')
g.create_variable('dummy3', dimensions=('y', 'y'), dtype=np.int64)
g.create_variable('dummy4', dimensions=('z', 'z'), dtype=np.int64)
f.resize_dimension('x', 2)
with netCDF4.Dataset(tmp_netcdf) as f:
assert f.dimensions['x'].size == 2
assert f.dimensions['x'].isunlimited() is True
assert f.dimensions['y'].size == 3
assert f.dimensions['y'].isunlimited() is False
assert f.dimensions['z'].size == 0
assert f.dimensions['z'].isunlimited() is True
assert f.variables['dummy1'].shape == (2, 3)
assert f.variables['dummy2'].shape == (3, 2, 2)
g = f.groups["test"]
assert g.variables['dummy3'].shape == (3, 3)
assert g.variables['dummy4'].shape == (0, 0)
def run(
add_grid_to: Path, latitude_definitions: Path, longitude_definitions: Path
) -> None:
with h5netcdf.File(add_grid_to, "a") as f:
try:
xc_var = f.create_variable(
"xc",
dimensions=("y", "x"),
data=parse_longitude_definitions(longitude_definitions).flatten(),
)
xc_var.attrs["long_name"] = "longitude of grid cell center"
xc_var.attrs["units"] = "degrees_east"
xc_var.attrs["bounds"] = "xv"
except ValueError:
xc_var = f["xc"]
xc_var[...] = parse_longitude_definitions(longitude_definitions)
try:
yc_var = f.create_variable(
"yc",
dimensions=("y", "x"),
data=parse_latitude_definitions(latitude_definitions).flatten(),
)
yc_var.attrs["long_name"] = "latitude of grid cell center"
yc_var.attrs["units"] = "degrees_north"
yc_var.attrs["bounds"] = "yv"
except ValueError:
yc_var = f["yc"]
yc_var[...] = parse_latitude_definitions(latitude_definitions)
rain_var = f["rain"]
rain_var.attrs["coordinates"] = "yc xc"
def set_initial_conditions(self, vs):
# initial conditions for T and S
temp_data = self._read_forcing(vs, 'temperature')[:, :, ::-1]
vs.temp[2:-2, 2:-2, :, :2] = temp_data[:, :, :, np.newaxis] * \
vs.maskT[2:-2, 2:-2, :, np.newaxis]
salt_data = self._read_forcing(vs, 'salinity')[:, :, ::-1]
vs.salt[2:-2, 2:-2, :, :2] = salt_data[..., np.newaxis] * vs.maskT[2:-2, 2:-2, :, np.newaxis]
# use Trenberth wind stress from MITgcm instead of ECMWF (also contained in ecmwf_4deg.cdf)
vs.taux[2:-2, 2:-2, :] = self._read_forcing(vs, 'tau_x')
vs.tauy[2:-2, 2:-2, :] = self._read_forcing(vs, 'tau_y')
# heat flux
with h5netcdf.File(DATA_FILES['ecmwf'], 'r') as ecmwf_data:
qnec_var = ecmwf_data.variables['Q3']
vs.qnec[2:-2, 2:-2, :] = np.array(qnec_var, dtype=str(qnec_var.dtype)).transpose()
vs.qnec[vs.qnec <= -1e10] = 0.0
q = self._read_forcing(vs, 'q_net')
vs.qnet[2:-2, 2:-2, :] = -q
vs.qnet[vs.qnet <= -1e10] = 0.0
fxa = np.sum(vs.qnet[2:-2, 2:-2, :] * vs.area_t[2:-2, 2:-2, np.newaxis]) \
/ 12 / np.sum(vs.area_t[2:-2, 2:-2])
print(' removing an annual mean heat flux imbalance of %e W/m^2' % fxa)
vs.qnet[...] = (vs.qnet - fxa) * vs.maskT[:, :, -1, np.newaxis]
# SST and SSS
vs.sst_clim[2:-2, 2:-2, :] = self._read_forcing(vs, 'sst')
vs.sss_clim[2:-2, 2:-2, :] = self._read_forcing(vs, 'sss')
if vs.enable_idemix:
vs.forc_iw_bottom[2:-2, 2:-2] = self._read_forcing(vs, 'tidal_energy') / vs.rho_0
vs.forc_iw_surface[2:-2, 2:-2] = self._read_forcing(vs, 'wind_energy') / vs.rho_0 * 0.2
def save(self, path=None, out_format=None, kind=None):
name = self._get_name(kind)
if path is not None:
root, ext = os.path.splitext(path)
if ext in [".h5", ".nc"]:
path_file = path
else:
path_file = os.path.join(path, name)
else:
path_file = name
if out_format == "uvmat":
with h5netcdf.File(path_file, "w") as f:
self._save_as_uvmat(f)
else:
with h5py.File(path_file, "w") as f:
f.attrs["class_name"] = "MultipassPIVResults"
f.attrs["module_name"] = "fluidimage.data_objects.piv"
f.attrs["nb_passes"] = len(self.passes)
f.attrs["fluidimage_version"] = fluidimage_version
f.attrs["fluidimage_hg_rev"] = hg_rev
for i, r in enumerate(self.passes):
r._save_in_hdf5_object(f, tag="piv{}".format(i))
return path_file
def test_repr(tmp_netcdf):
write_h5netcdf(tmp_netcdf)
f = h5netcdf.File(tmp_netcdf, 'r')
assert 'h5netcdf.File' in repr(f)
assert 'subgroup' in repr(f)
assert 'foo' in repr(f)
assert 'other_attr' in repr(f)
assert 'h5netcdf.attrs.Attributes' in repr(f.attrs)
assert 'global' in repr(f.attrs)
d = f.dimensions
assert 'h5netcdf.Dimensions' in repr(d)
assert 'x=4' in repr(d)
g = f['subgroup']
assert 'h5netcdf.Group' in repr(g)
assert 'subvar' in repr(g)
v = f['foo']
assert 'h5netcdf.Variable' in repr(v)
assert 'float' in repr(v)
assert 'units' in repr(v)
f.dimensions['temp'] = None
assert 'temp: Unlimited (current: 0)' in repr(f)
f.resize_dimension('temp', 5)
assert 'temp: Unlimited (current: 5)' in repr(f)
f.close()
assert 'Closed' in repr(f)
assert 'Closed' in repr(d)
assert 'Closed' in repr(g)
assert 'Closed' in repr(v)
def test_detach_scale(tmp_local_netcdf):
with h5netcdf.File(tmp_local_netcdf, "w") as ds:
ds.dimensions["x"] = 2
ds.dimensions["y"] = 2
with h5netcdf.File(tmp_local_netcdf, "a") as ds:
ds.create_variable("test", dimensions=("x",), dtype=np.int64)
# this forces detach and re-creation
ds.create_variable("x", dimensions=("y",), dtype=np.int64)
with h5netcdf.File(tmp_local_netcdf, "r") as ds:
refs = ds._h5group["x"].attrs.get("REFERENCE_LIST", False)
assert len(refs) == 1
for (ref, dim), name in zip(refs, ["/test"]):
assert dim == 0
assert ds._root._h5file[ref].name == name
def test_encoding_kwarg(self):
from satpy import Scene
import xarray as xr
import tempfile
scn = Scene()
start_time = datetime(2018, 5, 30, 10, 0)
end_time = datetime(2018, 5, 30, 10, 15)
scn['test-array'] = xr.DataArray([1, 2, 3],
attrs=dict(start_time=start_time,
end_time=end_time))
try:
handle, filename = tempfile.mkstemp()
os.close(handle)
encoding = {
'test-array': {
'dtype': 'int8',
'scale_factor': 0.1,
'add_offset': 0.0,
'_FillValue': 3
}
}
scn.save_datasets(filename=filename,
encoding=encoding,
writer='cf')
import h5netcdf as nc4
with nc4.File(filename) as f:
self.assertTrue(all(f['test-array'][:] == [10, 20, 30]))
self.assertTrue(f['test-array'].attrs['scale_factor'] == 0.1)
self.assertTrue(f['test-array'].attrs['_FillValue'] == 3)
# check that dtype behave as int8
self.assertTrue(np.iinfo(f['test-array'][:].dtype).max == 127)
finally:
os.remove(filename)
def test_bounds_missing_time_info(self):
from satpy import Scene
import xarray as xr
import tempfile
scn = Scene()
start_timeA = datetime(2018, 5, 30, 10, 0)
end_timeA = datetime(2018, 5, 30, 10, 15)
test_arrayA = np.array([[1, 2], [3, 4]]).reshape(2, 2, 1)
test_arrayB = np.array([[1, 2], [3, 5]]).reshape(2, 2, 1)
scn['test-arrayA'] = xr.DataArray(
test_arrayA,
dims=['x', 'y', 'time'],
coords={'time': [np.datetime64('2018-05-30T10:05:00')]},
attrs=dict(start_time=start_timeA, end_time=end_timeA))
scn['test-arrayB'] = xr.DataArray(
test_arrayB,
dims=['x', 'y', 'time'],
coords={'time': [np.datetime64('2018-05-30T10:05:00')]})
try:
handle, filename = tempfile.mkstemp()
os.close(handle)
scn.save_datasets(filename=filename, writer='cf')
import h5netcdf as nc4
with nc4.File(filename) as f:
self.assertTrue(
all(f['time_bnds'][:] == np.array([-300., 600.])))
finally:
os.remove(filename)
def is_datasource_for(cls, **kwargs):
"""
Determines if header corresponds to a GOES lightcurve
`~sunpy.timeseries.TimeSeries`.
"""
if "source" in kwargs.keys():
return kwargs["source"].lower().startswith(cls._source)
if "meta" in kwargs.keys():
return kwargs["meta"].get("TELESCOP", "").startswith("GOES")
if "filepath" in kwargs.keys():
try:
if sunpy.io.detect_filetype(kwargs["filepath"]) == "hdf5":
with h5netcdf.File(kwargs["filepath"],
mode="r",
**cls._netcdf_read_kw) as f:
summary = f.attrs["summary"]
if not isinstance(summary, str):
# h5netcdf<0.14
summary = summary.astype(str)
return "XRS" in summary
except Exception as e:
log.debug(
f'Reading {kwargs["filepath"]} failed with the following exception:\n{e}'
)
return False
def _segy3d_ncdf(
segyfile,
ncfile,
segyio_kwargs,
n0,
ns,
head_df,
il_head_loc,
xl_head_loc,
vert_domain="TWT",
silent=False,
):
"""Helper function to load 3d data into a netcdf4 file. This function should
stream data to disk avoiding excessive memory usage.
"""
if vert_domain == "TWT":
dims = DimensionKeyField.threed_twt.value
elif vert_domain == "DEPTH":
dims = DimensionKeyField.threed_depth.value
else:
raise ValueError(f"Unknown vert_domain: {vert_domain}")
with segyio.open(segyfile, "r",
**segyio_kwargs) as segyf, h5netcdf.File(ncfile,
"a") as seisnc:
segyf.mmap()
# create data variable
seisnc_data = seisnc.create_variable(VariableKeyField.data.value, dims,
float)
seisnc.flush()
# work out fast and slow dir
if head_df[xl_head_loc].diff().min() < 0:
contig_dir = il_head_loc
broken_dir = xl_head_loc
slicer = lambda x, y: slice(x, y, ...)
else:
contig_dir = xl_head_loc
broken_dir = il_head_loc
slicer = lambda x, y: slice(y, x, ...)
print(f"Fast direction is {broken_dir}")
pb = tqdm(total=segyf.tracecount,
desc="Converting SEGY",
disable=silent,
**TQDM_ARGS)
for contig, grp in head_df.groupby(contig_dir):
for trc, val in grp.iterrows():
seisnc_data[val.il_index,
val.xl_index, :] = segyf.trace[trc][n0:ns + 1]
pb.update()
pb.close()
return open_seisnc(ncfile)
def test_c_api_can_read_unlimited_dimensions(tmp_local_netcdf):
with h5netcdf.File(tmp_local_netcdf, "w") as f:
# Three dimensions, only one is limited.
f.dimensions["x"] = None
f.dimensions["y"] = 3
f.dimensions["z"] = None
f.create_variable("dummy1", dimensions=("x", "y"), dtype=np.int64)
f.create_variable("dummy2", dimensions=("y", "x", "x"), dtype=np.int64)
g = f.create_group("test")
g.create_variable("dummy3", dimensions=("y", "y"), dtype=np.int64)
g.create_variable("dummy4", dimensions=("z", "z"), dtype=np.int64)
f.resize_dimension("x", 2)
with netCDF4.Dataset(tmp_local_netcdf, "r") as f:
assert f.dimensions["x"].size == 2
assert f.dimensions["x"].isunlimited() is True
assert f.dimensions["y"].size == 3
assert f.dimensions["y"].isunlimited() is False
assert f.dimensions["z"].size == 0
assert f.dimensions["z"].isunlimited() is True
assert f.variables["dummy1"].shape == (2, 3)
assert f.variables["dummy2"].shape == (3, 2, 2)
g = f.groups["test"]
assert g.variables["dummy3"].shape == (3, 3)
assert g.variables["dummy4"].shape == (0, 0)
def test_reading_unlimited_dimensions_created_with_c_api(tmp_local_netcdf):
with netCDF4.Dataset(tmp_local_netcdf, "w") as f:
f.createDimension("x", None)
f.createDimension("y", 3)
f.createDimension("z", None)
dummy1 = f.createVariable("dummy1", float, ("x", "y"))
f.createVariable("dummy2", float, ("y", "x", "x"))
g = f.createGroup("test")
g.createVariable("dummy3", float, ("y", "y"))
g.createVariable("dummy4", float, ("z", "z"))
# Assign something to trigger a resize.
dummy1[:] = [[1, 2, 3], [4, 5, 6]]
with h5netcdf.File(tmp_local_netcdf, "r") as f:
assert f.dimensions["x"] is None
assert f.dimensions["y"] == 3
assert f.dimensions["z"] is None
# This is parsed correctly due to h5netcdf's init trickery.
assert f._current_dim_sizes["x"] == 2
assert f._current_dim_sizes["y"] == 3
assert f._current_dim_sizes["z"] == 0
# But the actual data-set and arrays are not correct.
assert f["dummy1"].shape == (2, 3)
# XXX: This array has some data with dimension x - netcdf does not
# appear to keep dimensions consistent.
assert f["dummy2"].shape == (3, 0, 0)
f.groups["test"]["dummy3"].shape == (3, 3)
f.groups["test"]["dummy4"].shape == (0, 0)
def test_repr(tmp_local_or_remote_netcdf):
write_h5netcdf(tmp_local_or_remote_netcdf)
f = h5netcdf.File(tmp_local_or_remote_netcdf, "r")
assert "h5netcdf.File" in repr(f)
assert "subgroup" in repr(f)
assert "foo" in repr(f)
assert "other_attr" in repr(f)
assert "h5netcdf.attrs.Attributes" in repr(f.attrs)
assert "global" in repr(f.attrs)
d = f.dimensions
assert "h5netcdf.Dimensions" in repr(d)
assert "x=4" in repr(d)
g = f["subgroup"]
assert "h5netcdf.Group" in repr(g)
assert "subvar" in repr(g)
v = f["foo"]
assert "h5netcdf.Variable" in repr(v)
assert "float" in repr(v)
assert "units" in repr(v)
f.dimensions["temp"] = None
assert "temp: Unlimited (current: 0)" in repr(f)
f.resize_dimension("temp", 5)
assert "temp: Unlimited (current: 5)" in repr(f)
f.close()
assert "Closed" in repr(f)
assert "Closed" in repr(d)
assert "Closed" in repr(g)
assert "Closed" in repr(v)
def test_save_array(self):
from satpy import Scene
import xarray as xr
import tempfile
scn = Scene()
start_time = datetime(2018, 5, 30, 10, 0)
end_time = datetime(2018, 5, 30, 10, 15)
scn['test-array'] = xr.DataArray([1, 2, 3],
attrs=dict(
start_time=start_time,
end_time=end_time,
prerequisites=[DatasetID('hej')]))
try:
handle, filename = tempfile.mkstemp()
os.close(handle)
scn.save_datasets(filename=filename, writer='cf')
import h5netcdf as nc4
with nc4.File(filename) as f:
self.assertTrue(all(f['test-array'][:] == [1, 2, 3]))
expected_prereq = (
"DatasetID(name='hej', wavelength=None, "
"resolution=None, polarization=None, "
"calibration=None, level=None, modifiers=())")
self.assertEqual(f['test-array'].attrs['prerequisites'][0],
np.string_(expected_prereq))
finally:
os.remove(filename)
def threaded_io(vs, filepath, mode):
"""
If using IO threads, start a new thread to write the netCDF data to disk.
"""
import h5netcdf
if vs.use_io_threads:
_wait_for_disk(vs, filepath)
_io_locks[filepath].clear()
kwargs = {}
if runtime_state.proc_num > 1:
kwargs.update(
driver='mpio',
comm=rs.mpi_comm
)
nc_dataset = h5netcdf.File(filepath, mode, **kwargs)
try:
yield nc_dataset
finally:
if vs.use_io_threads:
threading.Thread(target=_write_to_disk, args=(vs, nc_dataset, filepath)).start()
else:
_write_to_disk(vs, nc_dataset, filepath)
def test_reading_special_datatype_created_with_c_api(tmp_local_netcdf):
"""Test reading a file with unsupported Datatype"""
with netCDF4.Dataset(tmp_local_netcdf, "w") as f:
complex128 = np.dtype([("real", np.float64), ("imag", np.float64)])
f.createCompoundType(complex128, "complex128")
with h5netcdf.File(tmp_local_netcdf, "r") as f:
pass
