def load_identifier(self, pathname):
"""Load unique identifier from catalogue."""
ncfile = netcdfobjects.nc_open_for_load(pathname)
result = ncfile.getncattr(ATTRIBUTE_IDENTIFIER)
ncfile.close()
return result
def load_data_pathnames(self, pathname, reflist):
"""Read path names based on list of data references."""
ncfile = netcdfobjects.nc_open_for_load(pathname)
names = []
for ref in reflist:
dataset = ncfile.groups[
netcdfobjects.LISTITEM_OBJECT_SUFFIX.format(
'datasets', ref.dataset)]
subset = dataset.groups[
netcdfobjects.LISTITEM_OBJECT_SUFFIX.format(
'subsets', ref.subset)]
matches = subset.groups['matches']
# Construct pathname if the entry index is valid, otherwise no match is available in the catalogue
if ref.entry >= 0:
name = str(
chartostring(matches.variables['name'][ref.entry, :]))
path = dataset.getncattr('path')
names.append(os.path.join(path, name))
else:
names.append(None)
#except:
# raise ValueError('ERROR: {0} references data which does not exist in the catalogue ({1}, {2}, {3})'
# .format(__name__, ref.dataset, ref.subset, ref.entry))
ncfile.close()
return names
def load(self, pathname):
"""Load datasets where pathname corresponds to a NetCDF file."""
ncfile = netcdfobjects.nc_open_for_load(pathname)
catalogue = netcdfobjects.load_object(ncfile)
ncfile.close()
return catalogue
def test_load_object_with_empty_list(self):
"""Read a file that has one object and an empty list."""
# Empty file with unique temporary name
testfile = tempfile.NamedTemporaryFile(suffix='.nc')
# Build NetCDF manually
nc_testdata = Dataset(testfile.name, 'w')
nc_testdata.setncattr('Conventions', 'CF-1.6')
nc_testdata.setncattr(
'python_class',
'eumopps.netcdfobjects.test.test_netcdfobjects.TestObject')
group_a = nc_testdata.createGroup('a')
group_a.createDimension('list_count', 0)
nc_testdata.close()
# Read
nc_input = netcdfobjects.nc_open_for_load(testfile.name)
stuff = netcdfobjects.load_object(nc_input)
nc_input.close()
# Check result (should be empty dataset list and have the specified
# identifier)
self.assertIsInstance(stuff, TestObject)
self.assertTrue(isinstance(stuff.a, list))
self.assertEqual(0, len(stuff.a))
def load_operation(self, pathname, modulename):
"""Load the class associated with given module name."""
ncfile = netcdfobjects.nc_open_for_load(pathname)
group = self.findoperation(ncfile, modulename)
result = netcdfobjects.load_object(group)
ncfile.close()
return result
def test_save_simple_python_dict(self):
testfile = tempfile.NamedTemporaryFile(suffix='.nc')
nc_testdata = netcdfobjects.nc_open_for_save(testfile.name)
testdata = {
'somename': 'Bob',
'anumber': 529,
'anothernumber': -82.444,
'namelist': ['some name', 'another name', 'something else'],
'oneday': datetime(1922, 3, 3),
'oneday_in_array': [datetime(1922, 3, 3)],
'datelist': [datetime(2017, 10, 10),
datetime(1853, 6, 7)],
'myintegerlist': [-39, 3, 888, 19827],
'somemorenumbers': [89.999, -1.234, 3.33333],
}
netcdfobjects.save_simple_python_object(nc_testdata, "mytest",
testdata)
nc_testdata.close()
nc_input = netcdfobjects.nc_open_for_load(testfile.name)
self.assertEqual('CF-1.6', nc_input.Conventions)
mytest = nc_input.groups['mytest']
self.assertEqual('Bob', mytest.somename)
self.assertEqual(529, mytest.anumber)
self.assertEqual(-82.444, mytest.anothernumber)
names = [
str(item) for item in chartostring(mytest.variables['namelist'][:])
]
self.assertEqual(3, len(names))
self.assertEqual(['some name', 'another name', 'something else'],
names)
self.assertEqual(
datetime(1922, 3, 3),
num2date(mytest.variables['oneday'][:],
mytest.variables['oneday'].units))
self.assertEqual([datetime(1922, 3, 3)],
num2date(mytest.variables['oneday_in_array'][:],
mytest.variables['oneday_in_array'].units))
datelist = num2date(mytest.variables['datelist'][:],
mytest.variables['datelist'].units)
self.assertEqual(2, len(datelist))
self.assertEqual(datetime(2017, 10, 10), datelist[0])
self.assertEqual(datetime(1853, 6, 7), datelist[1])
numpy.testing.assert_equal([-39, 3, 888, 19827],
mytest.variables['myintegerlist'][:])
numpy.testing.assert_equal([89.999, -1.234, 3.33333],
mytest.variables['somemorenumbers'][:])
def test_save_load_simple_python_object(self):
# Make example data as before
testfile = tempfile.NamedTemporaryFile(suffix='.nc')
nc_testdata = netcdfobjects.nc_open_for_save(testfile.name)
testdata = TestObject(
somename='Bob',
anumber=529,
anothernumber=-82.444,
namelist=['some name', 'another name', 'something else'],
oneday=datetime(1922, 3, 3),
oneday_in_array=[datetime(1922, 3, 3)],
datelist=[datetime(2017, 10, 10),
datetime(1853, 6, 7)],
myintegerlist=[-39, 3, 888, 19827],
somemorenumbers=[89.999, -1.234, 3.33333])
netcdfobjects.save_simple_python_object(nc_testdata, "mytest",
testdata)
nc_testdata.close()
# Delete just to be sure we really load new things
del testdata
del nc_testdata
# Now load it (should construct object)
nc_input = netcdfobjects.nc_open_for_load(testfile.name)
result = netcdfobjects.load_object(nc_input.groups['mytest'])
nc_input.close()
# Should be instance of original
self.assertIsInstance(result, TestObject)
# And values should be the same
self.assertEqual('Bob', result.somename)
self.assertEqual(529, result.anumber)
self.assertEqual(-82.444, result.anothernumber)
self.assertEqual(['some name', 'another name', 'something else'],
result.namelist)
self.assertEqual(datetime(1922, 3, 3), result.oneday)
self.assertEqual([datetime(1922, 3, 3)], result.oneday_in_array)
self.assertEqual(2, len(result.datelist))
self.assertEqual(datetime(2017, 10, 10), result.datelist[0])
self.assertEqual(datetime(1853, 6, 7), result.datelist[1])
numpy.testing.assert_equal([-39, 3, 888, 19827], result.myintegerlist)
numpy.testing.assert_equal([89.999, -1.234, 3.33333],
result.somemorenumbers)
def operationcount(self, pathname, modulename, batchsize=1):
"""Quickly establish number of records in NetCDF dataset, or number of batches if batchsize is > 1."""
# Load file and get op group
ncfile = netcdfobjects.nc_open_for_load(pathname)
group = self.findoperation(ncfile, modulename)
# Retrieve count value from operation
record_count = group.getncattr('count') if group else 0
# Done with file
ncfile.close()
# Express in batch numbers if requested
batch_count = record_count / batchsize
if (record_count % batchsize) != 0:
batch_count = batch_count + 1
return batch_count
def test_save_load(self):
operation_parameters = [[], [0, 1, 1], [0, 1, 2], [], [0, 1, 3], []]
tempfile = NamedTemporaryFile(
prefix=
'eumopps.catalogue.test.test_operationparameters.TestOperationFileReference.',
suffix='.nc')
ncfile = netcdfobjects.nc_open_for_save(tempfile.name)
netcdfobjects.save_object(
ncfile,
key='testsave',
value=OperationFileReference(operation_parameters))
ncfile.close()
ncfile = netcdfobjects.nc_open_for_load(tempfile.name)
result = netcdfobjects.load_object(ncfile.groups['testsave'])
self.assertEqual(operation_parameters, result.operation_parameters)