def test_func_avail(testname, func, expected, msg, kwds={}):
for backend in iobackend._AVAILABLE_:
iobackend.set_backend(backend)
npt.assert_equal(iobackend.get_backend(), backend,
'Cannot set backend {}'.format(backend))
test_func(testname, func, expected,
msg='{}: {}'.format(backend, msg), kwds=kwds)
def test_nc4_NCFile_init_append(self):
iobackend.set_backend('netCDF4')
ncf = iobackend.NCFile(self.ncfaname, mode='a')
actual = type(ncf)
ncf.close()
expected = iobackend.NCFile
print_test_msg('NCFile.__init__()', actual=actual, expected=expected)
self.assertEqual(actual, expected,
'NCFile not created with correct type')
def test_nc4_NCFile_create_variable_ndim(self):
iobackend.set_backend('netCDF4')
ncf = iobackend.NCFile(self.ncfaname, mode='a')
v2 = ncf.create_variable('v2', np.dtype('f'), ('t', 'x'))
v2[:] = self.v2
ncf.close()
ncfr = Nio.open_file(self.ncfaname)
actual = ncfr.variables['v2'][:]
expected = self.v2
ncfr.close()
print_test_msg('NCFile.create_variable()',
actual=actual, expected=expected)
npt.assert_array_equal(
actual, expected, 'NCFile 2d-variable incorrect')
def test_nc4_NCFile_setncattr(self):
iobackend.set_backend('netCDF4')
ncf = iobackend.NCFile(self.ncfaname, mode='a')
for a, v in self.fattrs2.iteritems():
ncf.setncattr(a, v)
ncf.close()
ncfr = Nio.open_file(self.ncfaname)
actual = ncfr.attributes
expected = self.ncattrs
expected.update(self.fattrs2)
ncfr.close()
print_test_msg('NCFile.setncattr()', actual=actual, expected=expected)
for a, v in expected.iteritems():
self.assertTrue(
a in actual, 'NCFile attribute {0!r} not found'.format(a))
self.assertEqual(
actual[a], v, 'NCFile attribute {0!r} incorrect'.format(a))
def test_nc4_NCVariable_setncattr(self):
iobackend.set_backend('netCDF4')
ncf = iobackend.NCFile(self.ncfaname, mode='a')
v = ncf.variables['v']
for attr, value in self.vattrs2.iteritems():
v.setncattr(attr, value)
ncf.close()
ncfr = Nio.open_file(self.ncfaname)
actual = ncfr.variables['v'].attributes
expected = self.vattrs
expected.update(self.vattrs2)
ncfr.close()
print_test_msg('NCVariable.setncattr()',
actual=actual, expected=expected)
for attr, value in expected.iteritems():
self.assertTrue(
attr in actual, 'Variable attribute {0!r} not found'.format(attr))
self.assertEqual(
actual[attr], value, 'Variable attribute {0!r} incorrect'.format(attr))
def test_nc4_NCFile_variable_append(self):
iobackend.set_backend('netCDF4')
ncf = iobackend.NCFile(self.ncfaname, mode='a')
nt = self.ncdims['t']
t = ncf.variables['t']
t[nt:] = self.t2
v = ncf.variables['v']
v[nt:, :] = self.v2
ncf.close()
ncfr = Nio.open_file(self.ncfaname)
actual = ncfr.variables['t'][:]
expected = np.concatenate((self.t, self.t2))
print_test_msg('NCVariable append', actual=actual, expected=expected)
npt.assert_array_equal(actual, expected, 'NCFile t-variable incorrect')
actual = ncfr.variables['v'][:]
expected = np.concatenate((self.v, self.v2))
print_test_msg('NCVariable append', actual=actual, expected=expected)
npt.assert_array_equal(
actual, expected, 'NCFile 2d-variable incorrect')
ncfr.close()
def test_NCFile_create_variable_chunksizes(self):
def func(variable=''):
ncf = iobackend.NCFile(self.ncfwname, mode='w')
for d in self.vdims[variable]:
if d == 't':
ncf.create_dimension(d)
else:
ncf.create_dimension(d, self.ncdims[d])
ncf.create_variable(variable, self.vdtype[variable], self.vdims[variable],
chunksizes=self.chunks[variable])
ncf.close()
ncfr = iobackend.NCFile(self.ncfwname)
actual = ncfr.variables[variable].chunk_sizes
ncfr.close()
remove(self.ncfwname)
return actual
iobackend.set_backend('netCDF4')
npt.assert_equal(iobackend.get_backend(), 'netCDF4',
'Cannot set backend {}'.format(netCDF4))
for v in self.ncvars:
expected = self.chunks[v] if self.chunks[v] else 'contiguous'
test_func(test_name(), func, expected,
msg='{}: {}'.format('netCDF4', 'NCFile variables incorrect'), kwds={'variable': v})
def generate_data(backend='netCDF4'):
"""
Generate dataset for testing purposes
"""
iobackend.set_backend(backend)
# Test Data Generation
for i in xrange(len(slices) + 1):
# Open the file for writing
fname = slices[i] if i < len(slices) else 'metafile.nc'
fobj = iobackend.NCFile(fname, mode='w')
# Write attributes to file
for name, value in fattrs.iteritems():
fobj.setncattr(name, value)
# Create the dimensions in the file
fobj.create_dimension('lat', nlat)
fobj.create_dimension('lon', nlon)
fobj.create_dimension('time', None)
fobj.create_dimension('strlen', nchar)
# Create the coordinate variables & add attributes
lat = fobj.create_variable('lat', 'f', ('lat',))
lon = fobj.create_variable('lon', 'f', ('lon',))
time = fobj.create_variable('time', 'f', ('time',))
# Set the coordinate variable attributes
lat.setncattr('long_name', 'latitude')
lat.setncattr('units', 'degrees_north')
lon.setncattr('long_name', 'longitude')
lon.setncattr('units', 'degrees_east')
time.setncattr('long_name', 'time')
time.setncattr('units', 'days since 01-01-0001')
time.setncattr('calendar', 'noleap')
# Set the values of the coordinate variables
lat[:] = np.linspace(-90, 90, nlat, dtype=np.float32)
lon[:] = np.linspace(-180, 180, nlon, endpoint=False, dtype=np.float32)
time[:] = np.arange(i * ntime, (i + 1) * ntime, dtype=np.float32)
# Create the scalar variables
for n in xrange(len(scalars)):
vname = scalars[n]
v = fobj.create_variable(vname, 'd', tuple())
v.setncattr('long_name', 'scalar{0}'.format(n))
v.setncattr('units', '[{0}]'.format(vname))
v.assign_value(np.float64(n * 10))
# Create the time-invariant metadata variables
all_timvars = timvars + ([] if i < len(slices) else xtimvars)
for n in xrange(len(all_timvars)):
vname = all_timvars[n]
v = fobj.create_variable(vname, 'd', ('lat', 'lon'))
v.setncattr('long_name', 'time-invariant metadata {0}'.format(n))
v.setncattr('units', '[{0}]'.format(vname))
v[:] = np.ones((nlat, nlon), dtype=np.float64) * n
# Create the time-variant character variables
for n in xrange(len(chvars)):
vname = chvars[n]
v = fobj.create_variable(vname, 'c', ('time', 'strlen'))
v.setncattr('long_name', 'character array {0}'.format(n))
vdata = [str((n + 1) * m) * (m + 1) for m in xrange(ntime)]
v[:] = np.array(vdata, dtype='S{}'.format(
nchar)).view('S1').reshape(ntime, nchar)
# Create the time-variant metadata variables
for n in xrange(len(tvmvars)):
vname = tvmvars[n]
v = fobj.create_variable(vname, 'd', ('time', 'lat', 'lon'))
v.setncattr('long_name', 'time-variant metadata {0}'.format(n))
v.setncattr('units', '[{0}]'.format(vname))
v[:] = np.ones((ntime, nlat, nlon), dtype=np.float64) * n
# Create the time-series variables
for n in xrange(len(tsvars)):
vname = tsvars[n]
v = fobj.create_variable(
vname, 'd', ('time', 'lat', 'lon'), fill_value=1e36)
v.setncattr('long_name', 'time-series variable {0}'.format(n))
v.setncattr('units', '[{0}]'.format(vname))
v.setncattr('missing_value', 1e36)
vdata = np.ones((ntime, nlat, nlon), dtype=np.float64) * n
vmask = np.random.choice(
[True, False], ntime * nlat * nlon).reshape(ntime, nlat, nlon)
v[:] = np.ma.MaskedArray(vdata, mask=vmask)
def test_NCFile_variables(self, backend):
iobackend.set_backend(backend)
ncf = iobackend.NCFile(self.ncfrname)
for v in ncf.variables:
assert isinstance(ncf.variables[v], iobackend.NCVariable)
ncf.close()
def test_NCFile_ncattrs(self, backend):
iobackend.set_backend(backend)
ncf = iobackend.NCFile(self.ncfrname)
assert ncf.ncattrs == list(self.ncattrs.keys())
ncf.close()
def test_NCFile_unlimited(self, backend):
iobackend.set_backend(backend)
ncf = iobackend.NCFile(self.ncfrname)
assert ncf.unlimited('t') is True
assert ncf.unlimited('x') is False
ncf.close()
def test_NCFile_dimensions(self, backend):
iobackend.set_backend(backend)
ncf = iobackend.NCFile(self.ncfrname)
assert ncf.dimensions == self.ncdims
ncf.close()
def test_NCFile_init(self, backend):
iobackend.set_backend(backend)
ncf = iobackend.NCFile(self.ncfrname)
assert isinstance(ncf, iobackend.NCFile)
ncf.close()
def generate_data(backend='netCDF4'):
"""
Generate dataset for testing purposes
"""
iobackend.set_backend(backend)
# Test Data Generation
for i in range(len(config.slices) + 1):
# Open the file for writing
fname = config.slices[i] if i < len(config.slices) else 'metafile.nc'
fobj = iobackend.NCFile(fname, mode='w')
# Write attributes to file
for name, value in config.fattrs.items():
fobj.setncattr(name, value)
# Create the dimensions in the file
fobj.create_dimension('lat', config.nlat)
fobj.create_dimension('lon', config.nlon)
fobj.create_dimension('time', None)
fobj.create_dimension('strlen', config.nchar)
# Create the coordinate variables & add attributes
lat = fobj.create_variable('lat', 'f', ('lat', ))
lon = fobj.create_variable('lon', 'f', ('lon', ))
time = fobj.create_variable('time', 'f', ('time', ))
# Set the coordinate variable attributes
lat.setncattr('long_name', 'latitude')
lat.setncattr('units', 'degrees_north')
lon.setncattr('long_name', 'longitude')
lon.setncattr('units', 'degrees_east')
time.setncattr('long_name', 'time')
time.setncattr('units', 'days since 01-01-0001')
time.setncattr('calendar', 'noleap')
# Set the values of the coordinate variables
lat[:] = np.linspace(-90, 90, config.nlat, dtype=np.float32)
lon[:] = np.linspace(-180,
180,
config.nlon,
endpoint=False,
dtype=np.float32)
time[:] = np.arange(i * config.ntime, (i + 1) * config.ntime,
dtype=np.float32)
# Create the scalar variables
for n in range(len(config.scalars)):
vname = config.scalars[n]
v = fobj.create_variable(vname, 'd', tuple())
v.setncattr('long_name', 'scalar{0}'.format(n))
v.setncattr('units', '[{0}]'.format(vname))
v.assign_value(np.float64(n * 10))
# Create the time-invariant metadata variables
all_timvars = config.timvars + ([] if i < len(config.slices) else
config.xtimvars)
for n in range(len(all_timvars)):
vname = all_timvars[n]
v = fobj.create_variable(vname, 'd', ('lat', 'lon'))
v.setncattr('long_name', 'time-invariant metadata {0}'.format(n))
v.setncattr('units', '[{0}]'.format(vname))
v[:] = np.ones((config.nlat, config.nlon), dtype=np.float64) * n
# Create the time-variant character variables
for n in range(len(config.chvars)):
vname = config.chvars[n]
v = fobj.create_variable(vname, 'c', ('time', 'strlen'))
v.setncattr('long_name', 'character array {0}'.format(n))
vdata = [str((n + 1) * m) * (m + 1) for m in range(config.ntime)]
v[:] = (np.array(vdata, dtype='S{}'.format(
config.nchar)).view('S1').reshape(config.ntime, config.nchar))
# Create the time-variant metadata variables
for n in range(len(config.tvmvars)):
vname = config.tvmvars[n]
v = fobj.create_variable(vname, 'd', ('time', 'lat', 'lon'))
v.setncattr('long_name', 'time-variant metadata {0}'.format(n))
v.setncattr('units', '[{0}]'.format(vname))
v[:] = np.ones(
(config.ntime, config.nlat, config.nlon), dtype=np.float64) * n
# Create the time-series variables
for n in range(len(config.tsvars)):
vname = config.tsvars[n]
v = fobj.create_variable(vname,
'd', ('time', 'lat', 'lon'),
fill_value=1e36)
v.setncattr('long_name', 'time-series variable {0}'.format(n))
v.setncattr('units', '[{0}]'.format(vname))
v.setncattr('missing_value', 1e36)
vdata = np.ones(
(config.ntime, config.nlat, config.nlon), dtype=np.float64) * n
vmask = np.random.choice([True, False], config.ntime *
config.nlat * config.nlon).reshape(
config.ntime, config.nlat,
config.nlon)
v[:] = np.ma.MaskedArray(vdata, mask=vmask)