def test_write_variable_collection(self):
if MPI_RANK == 0:
path_in = self.get_temporary_file_path('foo.nc')
path_out = self.get_temporary_file_path('foo_out.nc')
with self.nc_scope(path_in, 'w') as ds:
ds.createDimension('seven', 7)
var = ds.createVariable('var_seven',
float,
dimensions=('seven', ))
var[:] = np.arange(7, dtype=float) + 10
var.foo = 'bar'
else:
path_in, path_out = [None] * 2
path_in = MPI_COMM.bcast(path_in)
path_out = MPI_COMM.bcast(path_out)
rd = RequestDataset(path_in)
rd.metadata['dimensions']['seven']['dist'] = True
driver = DriverNetcdf(rd)
vc = driver.get_variable_collection()
with vm.scoped_by_emptyable('write', vc):
if not vm.is_null:
vc.write(path_out)
if MPI_RANK == 0:
self.assertNcEqual(path_in, path_out)
def test_get_dump_report(self):
# Test with nested groups.
path = self.get_temporary_file_path('foo.nc')
with self.nc_scope(path, 'w') as ds:
ds.convention = 'CF Free-For-All 0.99'
ds.createDimension('dim_root', 5)
var1_root = ds.createVariable('var1_root', float, dimensions=('dim_root',))
var1_root.who_knew = 'I did not.'
group1 = ds.createGroup('group1')
group1.createDimension('dim_group1', 7)
var1_group1 = group1.createVariable('var1_group1', int, dimensions=('dim_group1',))
var1_group1.whatever = 'End of the line!'
group1_group1 = group1.createGroup('group1_group1')
group1_group1.createDimension('dim_group1_group1', 10)
var1_group1_group1 = group1_group1.createVariable('bitter_end', float, dimensions=('dim_group1_group1',))
var1_group1_group1.foo = 70
rd = RequestDataset(path)
driver = DriverNetcdf(rd)
lines = driver.get_dump_report()
desired = ['OCGIS Driver Key: netcdf {', 'dimensions:', ' dim_root = 5 ;', 'variables:',
' float64 var1_root(dim_root) ;', ' var1_root:who_knew = "I did not." ;', '',
'// global attributes:', ' :convention = "CF Free-For-All 0.99" ;', '', 'group: group1 {',
' dimensions:', ' dim_group1 = 7 ;', ' variables:', ' int64 var1_group1(dim_group1) ;',
' var1_group1:whatever = "End of the line!" ;', '', ' group: group1_group1 {',
' dimensions:', ' dim_group1_group1 = 10 ;', ' variables:',
' float64 bitter_end(dim_group1_group1) ;', ' bitter_end:foo = 70 ;',
' } // group: group1_group1', ' } // group: group1', '}']
self.assertEqual(lines, desired)
def test_create_dist(self):
def _create_dimensions_(ds, k):
if k.dim_count > 0:
ds.createDimension('one', 1)
if k.dim_count == 2:
ds.createDimension('two', 2)
kwds = dict(dim_count=[0, 1, 2], nested=[False, True])
for k in self.iter_product_keywords(kwds):
path = self.get_temporary_file_path('{}.nc'.format(k.dim_count))
with self.nc_scope(path, 'w') as ds:
_create_dimensions_(ds, k)
if k.nested:
group1 = ds.createGroup('nest1')
_create_dimensions_(group1, k)
group2 = group1.createGroup('nest2')
_create_dimensions_(group2, k)
group3 = group2.createGroup('nest1')
_create_dimensions_(group3, k)
group3a = group2.createGroup('nest3')
_create_dimensions_(group3a, k)
group3.createDimension('outlier', 4)
rd = RequestDataset(uri=path)
driver = DriverNetcdf(rd)
actual = driver.create_dist().mapping
# All dimensions are not distributed.
for keyseq in iter_all_group_keys(actual[MPI_RANK]):
group = get_group(actual[MPI_RANK], keyseq)
for dim in list(group['dimensions'].values()):
self.assertFalse(dim.dist)
if k.dim_count == 0 and k.nested:
desired = {None: {'variables': {}, 'dimensions': {}, 'groups': {
u'nest1': {'variables': {}, 'dimensions': {}, 'groups': {
u'nest2': {'variables': {}, 'dimensions': {},
'groups': {u'nest3': {'variables': {}, 'dimensions': {}, 'groups': {}},
u'nest1': {'variables': {}, 'dimensions': {
u'outlier': Dimension(name='outlier', size=4, size_current=4,
dist=False, is_empty=False, src_idx='auto')},
'groups': {}}}}}}}}}
self.assertEqual(actual[MPI_RANK], desired)
if k.dim_count == 2 and k.nested:
self.assertIsNotNone(driver.metadata_source['groups']['nest1']['groups']['nest2'])
two_dimensions = [Dimension(name='one', size=1, size_current=1),
Dimension(name='two', size=2, size_current=2)]
nest1 = {'dimensions': two_dimensions, 'groups': {}}
template = deepcopy(nest1)
nest1['groups']['nest2'] = deepcopy(template)
nest1['groups']['nest2']['groups']['nest1'] = deepcopy(template)
nest1['groups']['nest2']['groups']['nest3'] = deepcopy(template)
nest1['groups']['nest2']['groups']['nest1']['dimensions'].append(Dimension('outlier', 4))
desired = {None: {'dimensions': two_dimensions, 'groups': {'nest1': nest1}}}
groups_actual = list(iter_all_group_keys((actual[MPI_RANK])))
groups_desired = list(iter_all_group_keys(desired))
self.assertEqual(groups_actual, groups_desired)
def test_write_variable_collection_dataset_variable_kwargs(self):
"""Test writing while overloading things like the dataset data model."""
path_in = self.get_temporary_file_path('foo.nc')
path_out = self.get_temporary_file_path('foo_out.nc')
with self.nc_scope(path_in, 'w', format='NETCDF3_CLASSIC') as ds:
ds.createDimension('seven', 7)
var = ds.createVariable('var_seven', np.float32, dimensions=('seven',))
var[:] = np.arange(7, dtype=np.float32) + 10
var.foo = 'bar'
rd = RequestDataset(path_in)
driver = DriverNetcdf(rd)
vc = driver.create_raw_field()
vc.write(path_out, dataset_kwargs={'format': 'NETCDF3_CLASSIC'}, variable_kwargs={'zlib': True})
self.assertNcEqual(path_in, path_out, ignore_attributes={'var_seven': ['_FillValue']})
def test_get_write_modes(self, m_vm):
# Test asynchronous write mode is chosen.
m_vm.size = 2
env.USE_NETCDF4_MPI = True
kwds = {KeywordArgument.DATASET_KWARGS: {'parallel': True}}
actual = DriverNetcdf._get_write_modes_(m_vm, **kwds)
desired = [MPIWriteMode.ASYNCHRONOUS]
self.assertEqual(actual, desired)
def test_eq(self):
rd = self.test_data.get_rd('cancm4_tas')
d = DriverNetcdf(rd)
d2 = deepcopy(d)
self.assertEqual(d, deepcopy(d))
d2.key = 'bad'
self.assertNotEqual(d, d2)
def test_system_driver_kwargs(self, m_RequestDataset, m_MFDataset, m_Dataset):
m_RequestDataset.driver_kwargs = {'clobber': False}
m_RequestDataset.opened = None
uri = 'a/path/foo.nc'
m_RequestDataset.uri = uri
driver = DriverNetcdf(m_RequestDataset)
driver.inquire_opened_state = mock.Mock(return_value=False)
with driver_scope(driver) as _:
m_Dataset.assert_called_once_with(uri, mode='r', clobber=False)
m_RequestDataset.driver_kwargs = {'aggdim': 'not_time'}
m_RequestDataset.opened = None
uri = ['a/path/foo1.nc', 'a/path/foo2.nc']
m_RequestDataset.uri = uri
with driver_scope(driver) as _:
m_MFDataset.assert_called_once_with(uri, aggdim='not_time')
def test_get_dump_report(self):
# Test with nested groups.
path = self.get_temporary_file_path('foo.nc')
with self.nc_scope(path, 'w') as ds:
ds.convention = 'CF Free-For-All 0.99'
ds.createDimension('dim_root', 5)
var1_root = ds.createVariable('var1_root',
float,
dimensions=('dim_root', ))
var1_root.who_knew = 'I did not.'
group1 = ds.createGroup('group1')
group1.createDimension('dim_group1', 7)
var1_group1 = group1.createVariable('var1_group1',
int,
dimensions=('dim_group1', ))
var1_group1.whatever = 'End of the line!'
group1_group1 = group1.createGroup('group1_group1')
group1_group1.createDimension('dim_group1_group1', 10)
var1_group1_group1 = group1_group1.createVariable(
'bitter_end', float, dimensions=('dim_group1_group1', ))
var1_group1_group1.foo = 70
rd = RequestDataset(path)
driver = DriverNetcdf(rd)
lines = driver.get_dump_report()
desired = [
'OCGIS Driver Key: netcdf {', 'dimensions:', ' dim_root = 5 ;',
'variables:', ' float64 var1_root(dim_root) ;',
' var1_root:who_knew = "I did not." ;', '',
'// global attributes:',
' :convention = "CF Free-For-All 0.99" ;', '',
'group: group1 {', ' dimensions:', ' dim_group1 = 7 ;',
' variables:', ' int64 var1_group1(dim_group1) ;',
' var1_group1:whatever = "End of the line!" ;', '',
' group: group1_group1 {', ' dimensions:',
' dim_group1_group1 = 10 ;', ' variables:',
' float64 bitter_end(dim_group1_group1) ;',
' bitter_end:foo = 70 ;', ' } // group: group1_group1',
' } // group: group1', '}'
]
self.assertEqual(lines, desired)
def test_write_variable_collection_dataset_variable_kwargs(self):
"""Test writing while overloading things like the dataset data model."""
path_in = self.get_temporary_file_path('foo.nc')
path_out = self.get_temporary_file_path('foo_out.nc')
with self.nc_scope(path_in, 'w', format='NETCDF3_CLASSIC') as ds:
ds.createDimension('seven', 7)
var = ds.createVariable('var_seven',
np.float32,
dimensions=('seven', ))
var[:] = np.arange(7, dtype=np.float32) + 10
var.foo = 'bar'
rd = RequestDataset(path_in)
driver = DriverNetcdf(rd)
vc = driver.get_variable_collection()
vc.write(path_out,
dataset_kwargs={'format': 'NETCDF3_CLASSIC'},
variable_kwargs={'zlib': True})
self.assertNcEqual(path_in,
path_out,
ignore_attributes={'var_seven': ['_FillValue']})
def test_write_variable_collection(self):
if MPI_RANK == 0:
path_in = self.get_temporary_file_path('foo.nc')
path_out = self.get_temporary_file_path('foo_out.nc')
with self.nc_scope(path_in, 'w') as ds:
ds.createDimension('seven', 7)
var = ds.createVariable('var_seven', float, dimensions=('seven',))
var[:] = np.arange(7, dtype=float) + 10
var.foo = 'bar'
else:
path_in, path_out = [None] * 2
path_in = vm.bcast(path_in)
path_out = vm.bcast(path_out)
rd = RequestDataset(path_in)
rd.metadata['dimensions']['seven']['dist'] = True
driver = DriverNetcdf(rd)
vc = driver.create_raw_field()
with vm.scoped_by_emptyable('write', vc):
if not vm.is_null:
vc.write(path_out)
if MPI_RANK == 0:
self.assertNcEqual(path_in, path_out)
def test_create_field(self):
# test updating of regrid source flag
rd = self.test_data.get_rd('cancm4_tas')
driver = DriverNetcdf(rd)
field = driver.create_field()
self.assertTrue(field.regrid_source)
rd.regrid_source = False
driver = DriverNetcdf(rd)
field = driver.create_field()
self.assertFalse(field.regrid_source)
# test flag with an assigned coordinate system
rd = self.test_data.get_rd('cancm4_tas')
driver = DriverNetcdf(rd)
field = driver.create_field()
self.assertFalse(field._has_assigned_coordinate_system)
rd = self.test_data.get_rd('cancm4_tas', kwds={'crs': WGS84()})
self.assertTrue(rd._has_assigned_coordinate_system)
driver = DriverNetcdf(rd)
field = driver.create_field()
self.assertTrue(field._has_assigned_coordinate_system)
def test_open_parallel_netcdf4_python(self, m_vm, m_Dataset):
m_vm.size = 2
env.USE_NETCDF4_MPI = True
_ = DriverNetcdf._open_('a/path', mode='w', vm=m_vm)
m_Dataset.assert_called_with('a/path', mode='w', parallel=True, comm=m_vm.comm)
def test_inspect(self):
rd = self.test_data.get_rd('cancm4_tas')
driver = DriverNetcdf(rd)
with self.print_scope() as ps:
driver.inspect()
self.assertTrue(len(ps.storage) > 1)
def test_inspect(self):
rd = self.test_data.get_rd('cancm4_tas')
driver = DriverNetcdf(rd)
with self.print_scope() as ps:
driver.inspect()
self.assertTrue(len(ps.storage) > 1)
def test_get_dist(self):
def _create_dimensions_(ds, k):
if k.dim_count > 0:
ds.createDimension('one', 1)
if k.dim_count == 2:
ds.createDimension('two', 2)
kwds = dict(dim_count=[0, 1, 2], nested=[False, True])
for k in self.iter_product_keywords(kwds):
path = self.get_temporary_file_path('{}.nc'.format(k.dim_count))
with self.nc_scope(path, 'w') as ds:
_create_dimensions_(ds, k)
if k.nested:
group1 = ds.createGroup('nest1')
_create_dimensions_(group1, k)
group2 = group1.createGroup('nest2')
_create_dimensions_(group2, k)
group3 = group2.createGroup('nest1')
_create_dimensions_(group3, k)
group3a = group2.createGroup('nest3')
_create_dimensions_(group3a, k)
group3.createDimension('outlier', 4)
rd = RequestDataset(uri=path)
driver = DriverNetcdf(rd)
actual = driver.get_dist().mapping
# All dimensions are not distributed.
for keyseq in iter_all_group_keys(actual[MPI_RANK]):
group = get_group(actual[MPI_RANK], keyseq)
for dim in list(group['dimensions'].values()):
self.assertFalse(dim.dist)
if k.dim_count == 0 and k.nested:
desired = {
None: {
'variables': {},
'dimensions': {},
'groups': {
'nest1': {
'variables': {},
'dimensions': {},
'groups': {
'nest2': {
'variables': {},
'dimensions': {},
'groups': {
'nest1': {
'variables': {},
'dimensions': {
'outlier':
Dimension(name='outlier',
size=4,
size_current=4,
dist=False,
src_idx='auto')
},
'groups': {}
}
}
}
}
}
}
}
}
self.assertEqual(actual[MPI_RANK], desired)
if k.dim_count == 2 and k.nested:
self.assertIsNotNone(driver.metadata_source['groups']['nest1']
['groups']['nest2'])
two_dimensions = [
Dimension(name='one', size=1, size_current=1),
Dimension(name='two', size=2, size_current=2)
]
nest1 = {'dimensions': two_dimensions, 'groups': {}}
template = deepcopy(nest1)
nest1['groups']['nest2'] = deepcopy(template)
nest1['groups']['nest2']['groups']['nest1'] = deepcopy(
template)
nest1['groups']['nest2']['groups']['nest3'] = deepcopy(
template)
nest1['groups']['nest2']['groups']['nest1'][
'dimensions'].append(Dimension('outlier', 4))
desired = {
None: {
'dimensions': two_dimensions,
'groups': {
'nest1': nest1
}
}
}
groups_actual = list(iter_all_group_keys((actual[MPI_RANK])))
groups_desired = list(iter_all_group_keys(desired))
self.assertEqual(groups_actual, groups_desired)
def test_get_field(self):
# test updating of regrid source flag
rd = self.test_data.get_rd('cancm4_tas')
driver = DriverNetcdf(rd)
field = driver.get_field()
self.assertTrue(field.regrid_source)
rd.regrid_source = False
driver = DriverNetcdf(rd)
field = driver.get_field()
self.assertFalse(field.regrid_source)
# test flag with an assigned coordinate system
rd = self.test_data.get_rd('cancm4_tas')
driver = DriverNetcdf(rd)
field = driver.get_field()
self.assertFalse(field._has_assigned_coordinate_system)
rd = self.test_data.get_rd('cancm4_tas', kwds={'crs': WGS84()})
self.assertTrue(rd._has_assigned_coordinate_system)
driver = DriverNetcdf(rd)
field = driver.get_field()
self.assertTrue(field._has_assigned_coordinate_system)