def test_DSG_contiguous(self): if self.test_only and inspect.stack()[0][3] not in self.test_only: return f = cfdm.read(self.contiguous, verbose=False) self.assertTrue(len(f) == 2) # Select the specific humidity field q = [g for g in f if g.get_property('standard_name') == 'specific_humidity'][0] self.assertTrue(q._equals(self.a, q.data.array)) # print ('\nf\n') # for x in f: # print(x) cfdm.write(f, self.tempfilename, verbose=False) g = cfdm.read(self.tempfilename) # print ('\ng\n') # for x in g: # print(x) self.assertTrue(len(g) == len(f)) for i in range(len(f)): self.assertTrue(g[i].equals(f[i], verbose=True)) # ------------------------------------------------------------ # Test creation # ------------------------------------------------------------ # Define the ragged array values ragged_array = numpy.array([280, 282.5, 281, 279, 278, 279.5], dtype='float32') # Define the count array values count_array = [2, 4] # Create the count variable count_variable = cfdm.Count(data=cfdm.Data(count_array)) count_variable.set_property('long_name', 'number of obs for this timeseries') # Create the contiguous ragged array object array = cfdm.RaggedContiguousArray( compressed_array=cfdm.NumpyArray(ragged_array), shape=(2, 4), size=8, ndim=2, count_variable=count_variable) # Create the field construct with the domain axes and the ragged # array tas = cfdm.Field() tas.set_properties({'standard_name': 'air_temperature', 'units': 'K', 'featureType': 'timeSeries'}) # Create the domain axis constructs for the uncompressed array X = tas.set_construct(cfdm.DomainAxis(4)) Y = tas.set_construct(cfdm.DomainAxis(2)) # Set the data for the field tas.set_data(cfdm.Data(array), axes=[Y, X]) cfdm.write(tas, self.tempfilename)
def test_geometry_interior_ring(self): if self.test_only and inspect.stack()[0][3] not in self.test_only: return for geometry_file in (self.geometry_interior_ring_file, self.geometry_interior_ring_file_2): f = cfdm.read(geometry_file, verbose=False) self.assertEqual(len(f), 2, 'f = ' + repr(f)) for g in f: self.assertTrue(g.equals(g.copy(), verbose=3)) self.assertEqual(len(g.auxiliary_coordinates), 4) g = f[0] for axis in ('X', 'Y'): coord = g.construct('axis=' + axis) self.assertTrue(coord.has_node_count(), 'axis=' + axis) self.assertTrue(coord.has_part_node_count(), 'axis=' + axis) self.assertTrue(coord.has_interior_ring(), 'axis=' + axis) cfdm.write(f, tempfile, Conventions='CF-' + VN) f2 = cfdm.read(tempfile) self.assertEqual(len(f2), 2, 'f2 = ' + repr(f2)) for a, b in zip(f, f2): self.assertTrue(a.equals(b, verbose=3)) # Interior ring component c = g.construct('longitude') self.assertTrue( c.interior_ring.equals( g.construct('longitude').get_interior_ring())) self.assertEqual(c.interior_ring.data.ndim, c.data.ndim + 1) self.assertEqual(c.interior_ring.data.shape[0], c.data.shape[0]) self.assertIsInstance(g.dump(display=False), str) d = c.insert_dimension(0) self.assertEqual(d.data.shape, (1, ) + c.data.shape) self.assertEqual(d.interior_ring.data.shape, (1, ) + c.interior_ring.data.shape) e = d.squeeze(0) self.assertEqual(e.data.shape, c.data.shape) self.assertEqual(e.interior_ring.data.shape, c.interior_ring.data.shape) t = d.transpose() self.assertEqual(t.data.shape, d.data.shape[::-1], (t.data.shape, c.data.shape[::-1])) self.assertEqual(t.interior_ring.data.shape, (d.interior_ring.data.shape[-2::-1] + (d.interior_ring.data.shape[-1], ))) # Subspacing g = g[1, ...] c = g.construct('longitude') self.assertEqual(c.interior_ring.data.shape[0], 1) self.assertEqual(c.interior_ring.data.ndim, c.data.ndim + 1) self.assertEqual(c.interior_ring.data.shape[0], c.data.shape[0]) # Setting of node count properties coord = f[0].construct('axis=Y') nc = coord.get_node_count() nc.set_property('long_name', 'Node counts') cfdm.write(f, tempfile) nc.nc_set_variable('new_var_name') cfdm.write(f, tempfile) # Setting of part node count properties coord = f[0].construct('axis=X') pnc = coord.get_part_node_count() pnc.set_property('long_name', 'Part node counts') cfdm.write(f, tempfile) pnc.nc_set_variable('new_var_name') cfdm.write(f, tempfile) pnc.nc_set_dimension('new_dim_name') cfdm.write(f, tempfile)
q, t = cfdm.read('file.nc') t t2 = t.squeeze() t2 print(t2.dimension_coordinates) t3 = t2.insert_dimension(axis='domainaxis3', position=1) t3 t3.transpose([2, 0, 1]) t4 = t.transpose([0, 2, 1], constructs=True) print(q) print(q.data.mask) print(q.data.mask.array) q.data[[0, 4], :] = cfdm.masked print(q.data.mask.array) q.data.mask.any() cfdm.write(q, 'masked_q.nc') no_mask_q = cfdm.read('masked_q.nc', mask=False)[0] print(no_mask_q.data.array) masked_q = no_mask_q.apply_masking() print(masked_q.data.array) data = t.data data.shape data[:, :, 1].shape data[:, 0].shape data[..., 6:3:-1, 3:6].shape data[0, [2, 9], [4, 8]].shape data[0, :, -2].shape import numpy t.data[:, 0, 0] = -1 t.data[:, :, 1] = -2
def test_create_field_3(self): # Dimension coordinates data = numpy.arange(9.) + 20 data[-1] = 34 dim0 = cfdm.DimensionCoordinate(data=cfdm.Data(data)) dim0.set_property('standard_name', 'grid_longitude') dim0.set_property('units', 'degrees') array = dim0.data.array array = numpy.array([array-0.5, array+0.5]).transpose((1,0)) array[-2, 1] = 30 array[-1, :] = [30, 36] dim0.set_bounds(cfdm.Bounds(data=cfdm.Data(array))) dim1 = cfdm.DimensionCoordinate(data=cfdm.Data(numpy.arange(10.))) dim1.set_property('standard_name', 'grid_latitude') dim1.set_property('units', 'degrees') dim2 = cfdm.DimensionCoordinate(data=cfdm.Data([1.5]), bounds=cfdm.Bounds(data=cfdm.Data([[1, 2.]]))) dim2.set_property('standard_name' , 'atmosphere_hybrid_height_coordinate') dim2.set_property('computed_standard_name', 'altitude') dim3 = cfdm.DimensionCoordinate(data=cfdm.Data(numpy.array([15.0]))) dim3.set_property('standard_name', 'time') dim3.set_property('units', 'days since 2004-06-01') dim3.set_bounds(cfdm.Bounds(data=cfdm.Data([[0, 30.]]))) # dim3.set_geometry('climatology') # Auxiliary coordinates ak = cfdm.DomainAncillary(data=cfdm.Data([10.])) ak.set_property('units', 'm') ak.set_bounds(cfdm.Bounds(data=cfdm.Data([[5, 15.]]))) bk = cfdm.DomainAncillary(data=cfdm.Data([20.])) bk.set_bounds(cfdm.Bounds(data=cfdm.Data([[14, 26.]]))) aux2 = cfdm.AuxiliaryCoordinate( data=cfdm.Data(numpy.arange(-45, 45, dtype='int32').reshape(10, 9))) aux2.set_property('units', 'degree_N') aux2.set_property('standard_name', 'latitude') aux3 = cfdm.AuxiliaryCoordinate( data=cfdm.Data(numpy.arange(60, 150, dtype='int32').reshape(9, 10))) aux3.set_property('standard_name', 'longitude') aux3.set_property('units', 'degreeE') array = numpy.ma.array(['alpha','beta','gamma','delta','epsilon', 'zeta','eta','theta','iota','kappa'], dtype='S') array[0] = numpy.ma.masked aux4 = cfdm.AuxiliaryCoordinate(data=cfdm.Data(array)) aux4.set_property('standard_name', 'greek_letters') # Cell measures msr0 = cfdm.CellMeasure( data=cfdm.Data(1+numpy.arange(90.).reshape(9, 10)*1234)) msr0.set_measure('area') msr0.set_property('units', 'km2') # Data data = cfdm.Data(numpy.arange(90.).reshape(10, 9)) properties = {'units': 'm s-1'} f = cfdm.Field(properties=properties) f.set_property('standard_name', 'eastward_wind') axisX = f.set_construct(cfdm.DomainAxis(9)) axisY = f.set_construct(cfdm.DomainAxis(10)) axisZ = f.set_construct(cfdm.DomainAxis(1)) axisT = f.set_construct(cfdm.DomainAxis(1)) f.set_data(data, axes=[axisY, axisX]) x = f.set_construct(dim0, axes=[axisX]) y = f.set_construct(dim1, axes=[axisY]) z = f.set_construct(dim2, axes=[axisZ]) t = f.set_construct(dim3, axes=[axisT]) lat = f.set_construct(aux2, axes=[axisY, axisX]) lon = f.set_construct(aux3, axes=[axisX, axisY]) greek = f.set_construct(aux4, axes=[axisY]) ak = f.set_construct(ak, axes=[axisZ]) bk = f.set_construct(bk, axes=[axisZ]) # Coordinate references # ref0 = cfdm.CoordinateReference( # parameters={'grid_mapping_name': 'rotated_latitude_longitude', # 'grid_north_pole_latitude': 38.0, # 'grid_north_pole_longitude': 190.0, # 'earth_radius': 6371007,}, # coordinates=[x, y, lat, lon] # ) coordinate_conversion = cfdm.CoordinateConversion( parameters={'grid_mapping_name': 'rotated_latitude_longitude', 'grid_north_pole_latitude': 38.0, 'grid_north_pole_longitude': 190.0}) datum = cfdm.Datum(parameters={'earth_radius': 6371007}) ref0 = cfdm.CoordinateReference( coordinate_conversion=coordinate_conversion, datum=datum, coordinates=[x, y, lat, lon] ) f.set_construct(msr0, axes=[axisX, axisY]) f.set_construct(ref0) orog = cfdm.DomainAncillary(data=f.get_data()) orog.set_property('standard_name', 'surface_altitude') orog.set_property('units', 'm') orog = f.set_construct(orog, axes=[axisY, axisX]) datum1 = cfdm.Datum({'earth_radius' : 6371007}) coordinate_conversion1 = cfdm.CoordinateConversion( parameters={'standard_name': 'atmosphere_hybrid_height_coordinate', 'computed_standard_name': 'altitude'}, domain_ancillaries={'orog': orog, 'a' : ak, 'b' : bk}) ref1 = cfdm.CoordinateReference( datum=datum1, coordinate_conversion=coordinate_conversion1, coordinates=[z] ) ref1 = f.set_construct(ref1) # Field ancillary variables # g = f.transpose([1, 0]) g = f.copy() # g.standard_name = 'ancillary0' # g *= 0.01 anc = cfdm.FieldAncillary(data=g.get_data()) anc.standard_name = 'ancillaryA' f.set_construct(anc, axes=[axisY, axisX]) g = f[0] g = g.squeeze() # g.standard_name = 'ancillary2' # g *= 0.001 anc = cfdm.FieldAncillary(data=g.get_data()) anc.standard_name = 'ancillaryB' f.set_construct(anc, axes=[axisX]) g = f[..., 0] g = g.squeeze() # g.standard_name = 'ancillary3' # g *= 0.001 anc = cfdm.FieldAncillary(data=g.get_data()) anc.standard_name = 'ancillaryC' f.set_construct(anc, axes=[axisY]) f.set_property('flag_values', numpy.array([1, 2, 4], 'int32')) f.set_property('flag_meanings', 'a bb ccc') f.set_property('flag_masks', [2, 1, 0]) cm0 = cfdm.CellMethod(axes=[axisX], method='mean', qualifiers={'interval': [cfdm.Data(1, 'day')], 'comment' : 'ok'}) cm1 = cfdm.CellMethod(axes=[axisY], method='maximum', qualifiers={'where' : 'sea'}) cm2 = cfdm.CellMethod(axes=[axisT], method='maximum', qualifiers={'within' : 'years'}) cm3 = cfdm.CellMethod(axes=[axisT], method='minimum', qualifiers={'over' : 'years'}) f.set_construct(cm0) f.set_construct(cm1) f.set_construct(cm2) f.set_construct(cm3) if verbose: print(repr(f)) print(f) print(f.constructs) print(f.construct_data_axes()) f.dump() # sys.exit(0) cfdm.write(f, self.filename, fmt='NETCDF3_CLASSIC', verbose=verbose) g = cfdm.read(self.filename, verbose=verbose) #, squeeze=True) if verbose: # g[0].dump() # sys.exit(0) for x in g: x.print_read_report() self.assertTrue(len(g) == 1, 'Read produced too many fields: {} != 1'.format(len(g))) g = g[0].squeeze() # print g self.assertTrue(sorted(f.constructs) == sorted(g.constructs), '\n\nf (created in memory)\n{}\n\n{}\n\ng (read from disk)\n{}\n\n{}'.format( sorted(f.constructs), sorted(f.constructs.items()), sorted(g.constructs), sorted(g.constructs.items()))) self.assertTrue(f.equals(f.copy(), verbose=True), "Field f not equal to a copy of itself") self.assertTrue(g.equals(g.copy(), verbose=True), "Field g not equal to a copy of itself") # print f.dump() # print'f' # print f # print 'g' # print g # f.dump() # g.dump() if verbose: f.dump() g.dump() # sys.exit(0) self.assertTrue(g.equals(f, verbose=True), "Field not equal to itself read back in") # sys.exit(0) x = g.dump(display=False) x = f.dump(display=False) g = cfdm.read(self.filename, verbose=verbose, extra=['domain_ancillary'], warnings=warnings) if verbose: for x in g: x.print_read_report() print(g) g[0].dump()
def test_groups_geometry(self): """Test that geometries are considered in the correct groups.""" f = cfdm.example_field(6) ungrouped_file = ungrouped_file2 grouped_file = grouped_file2 cfdm.write(f, ungrouped_file) g = cfdm.read(ungrouped_file, verbose=1) self.assertEqual(len(g), 1) g = g[0] self.assertTrue(f.equals(g, verbose=3)) # ------------------------------------------------------------ # Move the field construct to the /forecast/model group # ------------------------------------------------------------ g.nc_set_variable_groups(["forecast", "model"]) cfdm.write(g, grouped_file) nc = netCDF4.Dataset(grouped_file, "r") self.assertIn( f.nc_get_variable(), nc.groups["forecast"].groups["model"].variables, ) nc.close() h = cfdm.read(grouped_file) self.assertEqual(len(h), 1, repr(h)) self.assertTrue(f.equals(h[0], verbose=3)) # ------------------------------------------------------------ # Move the geometry container to the /forecast group # ------------------------------------------------------------ g.nc_set_geometry_variable_groups(["forecast"]) cfdm.write(g, grouped_file) # Check that the variable is in the right group nc = netCDF4.Dataset(grouped_file, "r") self.assertIn(f.nc_get_geometry_variable(), nc.groups["forecast"].variables) nc.close() # Check that the field construct hasn't changed h = cfdm.read(grouped_file) self.assertEqual(len(h), 1, repr(h)) self.assertTrue(f.equals(h[0], verbose=2)) # ------------------------------------------------------------ # Move a node coordinate variable to the /forecast group # ------------------------------------------------------------ g.construct("longitude").bounds.nc_set_variable_groups(["forecast"]) cfdm.write(g, grouped_file) # Check that the variable is in the right group nc = netCDF4.Dataset(grouped_file, "r") self.assertIn( f.construct("longitude").bounds.nc_get_variable(), nc.groups["forecast"].variables, ) nc.close() # Check that the field construct hasn't changed h = cfdm.read(grouped_file) self.assertEqual(len(h), 1, repr(h)) self.assertTrue(f.equals(h[0], verbose=2)) # ------------------------------------------------------------ # Move a node count variable to the /forecast group # ------------------------------------------------------------ ncvar = g.construct("longitude").get_node_count().nc_get_variable() g.nc_set_component_variable_groups("node_count", ["forecast"]) cfdm.write(g, grouped_file) # Check that the variable is in the right group nc = netCDF4.Dataset(grouped_file, "r") self.assertIn(ncvar, nc.groups["forecast"].variables) nc.close() # Check that the field construct hasn't changed h = cfdm.read(grouped_file, verbose=1) self.assertEqual(len(h), 1, repr(h)) self.assertTrue(f.equals(h[0], verbose=2)) # ------------------------------------------------------------ # Move a part node count variable to the /forecast group # ------------------------------------------------------------ ncvar = ( g.construct("longitude").get_part_node_count().nc_get_variable()) g.nc_set_component_variable_groups("part_node_count", ["forecast"]) cfdm.write(g, grouped_file) # Check that the variable is in the right group nc = netCDF4.Dataset(grouped_file, "r") self.assertIn(ncvar, nc.groups["forecast"].variables) nc.close() # Check that the field construct hasn't changed h = cfdm.read(grouped_file) self.assertEqual(len(h), 1, repr(h)) self.assertTrue(f.equals(h[0], verbose=2)) # ------------------------------------------------------------ # Move interior ring variable to the /forecast group # ------------------------------------------------------------ g.nc_set_component_variable("interior_ring", "interior_ring") g.nc_set_component_variable_groups("interior_ring", ["forecast"]) cfdm.write(g, grouped_file) # Check that the variable is in the right group nc = netCDF4.Dataset(grouped_file, "r") self.assertIn( f.construct("longitude").get_interior_ring().nc_get_variable(), nc.groups["forecast"].variables, ) nc.close() # Check that the field construct hasn't changed h = cfdm.read(grouped_file, verbose=1) self.assertEqual(len(h), 1, repr(h)) self.assertTrue(f.equals(h[0], verbose=2))
bounds[:, 0] = data - 45/2. bounds[:, 1] = data + 45/2. bounds = numpy.around(bounds, 1) b = cfdm.Bounds() b.set_data(cfdm.Data(bounds)) lon.set_bounds(b) time = cfdm.DimensionCoordinate(properties={'standard_name': 'time', 'units': 'days since 2018-12-01'}) time.set_data(cfdm.Data([31.0])) cell_method = cfdm.CellMethod(axes=['area'], properties={'method': 'mean'}) h.set_dimension_coordinate(lat , axes=[y]) h.set_dimension_coordinate(lon , axes=[x]) h.set_dimension_coordinate(time, axes=[t]) h.set_cell_method(cell_method) print h.constructs() h.dump() print h cfdm.write([h, f], 'file.nc', fmt='NETCDF3_CLASSIC') #cfdm.write(f, 'file.nc', fmt='NETCDF3_CLASSIC')
def test_netCDF_global_unlimited(self): if self.test_only and inspect.stack()[0][3] not in self.test_only: return # ------------------------------------------------------------ # Unlimited dimensions # ------------------------------------------------------------ f = cfdm.Field() self.assertTrue(f.nc_clear_unlimited_dimensions() == set()) f = cfdm.Field() f.nc_set_unlimited_dimensions(()) f = cfdm.Field() self.assertTrue(f.nc_unlimited_dimensions() == set()) f.nc_set_unlimited_dimensions(['qwerty', 'asdf']) self.assertTrue(f.nc_unlimited_dimensions() == set(['qwerty', 'asdf'])) f.nc_set_unlimited_dimensions(['zxc']) self.assertTrue( f.nc_unlimited_dimensions() == set(['qwerty', 'asdf', 'zxc'])) self.assertTrue(f.nc_clear_unlimited_dimensions() == set( ['qwerty', 'asdf', 'zxc'])) self.assertTrue(f.nc_unlimited_dimensions() == set()) # ------------------------------------------------------------ # Global attributes # ------------------------------------------------------------ f = cfdm.Field() self.assertTrue(f.nc_clear_global_attributes() == {}) f.nc_set_global_attribute('Conventions') f.nc_set_global_attribute('project', 'X') self.assertTrue(f.nc_global_attributes() == { 'Conventions': None, 'project': 'X' }) f.nc_set_global_attribute('project') f.nc_set_global_attribute('comment', None) self.assertTrue(f.nc_global_attributes() == { 'Conventions': None, 'project': None, 'comment': None }) self.assertTrue(f.nc_clear_global_attributes() == { 'Conventions': None, 'project': None, 'comment': None }) self.assertTrue(f.nc_global_attributes() == {}) f.nc_set_global_attribute('Conventions') f.nc_set_global_attribute('project') self.assertTrue(f.nc_global_attributes() == { 'Conventions': None, 'project': None }) f = cfdm.Field() f.set_properties({'foo': 'bar', 'comment': 'variable comment'}) f.nc_set_variable('tas') d = f.set_construct(cfdm.DomainAxis(2)) f.set_data(cfdm.Data([8, 9]), axes=[d]) f2 = f.copy() f2.nc_set_variable('ua') cfdm.write([f, f2], 'tempfilename.nc', file_descriptors={ 'comment': 'global comment', 'qwerty': 'asdf' }) g = cfdm.read('tempfilename.nc', verbose=False) self.assertTrue(len(g) == 2) for x in g: self.assertTrue( x.properties() == { 'comment': 'variable comment', 'foo': 'bar', 'qwerty': 'asdf', 'Conventions': 'CF-1.7' }) self.assertTrue( x.nc_global_attributes() == { 'comment': 'global comment', 'qwerty': None, 'Conventions': None }, x.nc_global_attributes()) cfdm.write(g, 'tempfilename2.nc') h = cfdm.read('tempfilename2.nc') for x, y in zip(h, g): self.assertTrue(x.properties() == y.properties()) self.assertTrue( x.nc_global_attributes() == y.nc_global_attributes()) self.assertTrue(x.equals(y, verbose=True)) self.assertTrue(y.equals(x, verbose=True)) g[1].nc_set_global_attribute('comment', 'different comment') cfdm.write(g, 'tempfilename3.nc') h = cfdm.read('tempfilename3.nc') for x, y in zip(h, g): self.assertTrue(x.properties() == y.properties()) self.assertTrue(x.nc_global_attributes() == { 'comment': None, 'qwerty': None, 'Conventions': None }) self.assertTrue(x.equals(y, verbose=True)) self.assertTrue(y.equals(x, verbose=True))
def test_create_field_3(self): """TODO DOCS.""" # Dimension coordinates data = numpy.arange(9.0) + 20 data[-1] = 34 dim0 = cfdm.DimensionCoordinate(data=cfdm.Data(data)) dim0.set_property("standard_name", "grid_longitude") dim0.set_property("units", "degrees") array = dim0.data.array array = numpy.array([array - 0.5, array + 0.5]).transpose((1, 0)) array[-2, 1] = 30 array[-1, :] = [30, 36] dim0.set_bounds(cfdm.Bounds(data=cfdm.Data(array))) dim1 = cfdm.DimensionCoordinate(data=cfdm.Data(numpy.arange(10.0))) dim1.set_property("standard_name", "grid_latitude") dim1.set_property("units", "degrees") dim2 = cfdm.DimensionCoordinate( data=cfdm.Data([1.5]), bounds=cfdm.Bounds(data=cfdm.Data([[1, 2.0]])), ) dim2.set_property("standard_name", "atmosphere_hybrid_height_coordinate") dim2.set_property("computed_standard_name", "altitude") dim3 = cfdm.DimensionCoordinate(data=cfdm.Data(numpy.array([15.0]))) dim3.set_property("standard_name", "time") dim3.set_property("units", "days since 2004-06-01") dim3.set_bounds(cfdm.Bounds(data=cfdm.Data([[0, 30.0]]))) # dim3.set_geometry('climatology') # Auxiliary coordinates ak = cfdm.DomainAncillary(data=cfdm.Data([10.0])) ak.set_property("units", "m") ak.set_bounds(cfdm.Bounds(data=cfdm.Data([[5, 15.0]]))) bk = cfdm.DomainAncillary(data=cfdm.Data([20.0])) bk.set_bounds(cfdm.Bounds(data=cfdm.Data([[14, 26.0]]))) aux2 = cfdm.AuxiliaryCoordinate(data=cfdm.Data( numpy.arange(-45, 45, dtype="int32").reshape(10, 9))) aux2.set_property("units", "degree_N") aux2.set_property("standard_name", "latitude") aux3 = cfdm.AuxiliaryCoordinate(data=cfdm.Data( numpy.arange(60, 150, dtype="int32").reshape(9, 10))) aux3.set_property("standard_name", "longitude") aux3.set_property("units", "degreeE") array = numpy.ma.array( [ "alpha", "beta", "gamma", "delta", "epsilon", "zeta", "eta", "theta", "iota", "kappa", ], dtype="S", ) array[0] = numpy.ma.masked aux4 = cfdm.AuxiliaryCoordinate(data=cfdm.Data(array)) aux4.set_property("standard_name", "greek_letters") # Cell measures msr0 = cfdm.CellMeasure( data=cfdm.Data(1 + numpy.arange(90.0).reshape(9, 10) * 1234)) msr0.set_measure("area") msr0.set_property("units", "km2") # Data data = cfdm.Data(numpy.arange(90.0).reshape(10, 9)) properties = {"units": "m s-1"} f = cfdm.Field(properties=properties) f.set_property("standard_name", "eastward_wind") axisX = f.set_construct(cfdm.DomainAxis(9)) axisY = f.set_construct(cfdm.DomainAxis(10)) axisZ = f.set_construct(cfdm.DomainAxis(1)) axisT = f.set_construct(cfdm.DomainAxis(1)) f.set_data(data, axes=[axisY, axisX]) x = f.set_construct(dim0, axes=[axisX]) y = f.set_construct(dim1, axes=[axisY]) z = f.set_construct(dim2, axes=[axisZ]) f.set_construct(dim3, axes=[axisT]) lat = f.set_construct(aux2, axes=[axisY, axisX]) lon = f.set_construct(aux3, axes=[axisX, axisY]) f.set_construct(aux4, axes=[axisY]) ak = f.set_construct(ak, axes=[axisZ]) bk = f.set_construct(bk, axes=[axisZ]) # Coordinate references # ref0 = cfdm.CoordinateReference( # parameters={'grid_mapping_name': 'rotated_latitude_longitude', # 'grid_north_pole_latitude': 38.0, # 'grid_north_pole_longitude': 190.0, # 'earth_radius': 6371007,}, # coordinates=[x, y, lat, lon] # ) coordinate_conversion = cfdm.CoordinateConversion( parameters={ "grid_mapping_name": "rotated_latitude_longitude", "grid_north_pole_latitude": 38.0, "grid_north_pole_longitude": 190.0, }) datum = cfdm.Datum(parameters={"earth_radius": 6371007}) ref0 = cfdm.CoordinateReference( coordinate_conversion=coordinate_conversion, datum=datum, coordinates=[x, y, lat, lon], ) f.set_construct(msr0, axes=[axisX, axisY]) f.set_construct(ref0) orog = cfdm.DomainAncillary(data=f.get_data()) orog.set_property("standard_name", "surface_altitude") orog.set_property("units", "m") orog = f.set_construct(orog, axes=[axisY, axisX]) datum1 = cfdm.Datum({"earth_radius": 6371007}) coordinate_conversion1 = cfdm.CoordinateConversion( parameters={ "standard_name": "atmosphere_hybrid_height_coordinate", "computed_standard_name": "altitude", }, domain_ancillaries={ "orog": orog, "a": ak, "b": bk }, ) ref1 = cfdm.CoordinateReference( datum=datum1, coordinate_conversion=coordinate_conversion1, coordinates=[z], ) ref1 = f.set_construct(ref1) # Field ancillary variables # g = f.transpose([1, 0]) g = f.copy() # g.standard_name = 'ancillary0' # g *= 0.01 anc = cfdm.FieldAncillary(data=g.get_data()) anc.standard_name = "ancillaryA" f.set_construct(anc, axes=[axisY, axisX]) g = f[0] g = g.squeeze() # g.standard_name = 'ancillary2' # g *= 0.001 anc = cfdm.FieldAncillary(data=g.get_data()) anc.standard_name = "ancillaryB" f.set_construct(anc, axes=[axisX]) g = f[..., 0] g = g.squeeze() # g.standard_name = 'ancillary3' # g *= 0.001 anc = cfdm.FieldAncillary(data=g.get_data()) anc.standard_name = "ancillaryC" f.set_construct(anc, axes=[axisY]) f.set_property("flag_values", numpy.array([1, 2, 4], "int32")) f.set_property("flag_meanings", "a bb ccc") f.set_property("flag_masks", [2, 1, 0]) cm0 = cfdm.CellMethod( axes=[axisX], method="mean", qualifiers={ "interval": [cfdm.Data(1, "day")], "comment": "ok" }, ) cm1 = cfdm.CellMethod(axes=[axisY], method="maximum", qualifiers={"where": "sea"}) cm2 = cfdm.CellMethod(axes=[axisT], method="maximum", qualifiers={"within": "years"}) cm3 = cfdm.CellMethod(axes=[axisT], method="minimum", qualifiers={"over": "years"}) f.set_construct(cm0) f.set_construct(cm1) f.set_construct(cm2) f.set_construct(cm3) cfdm.write(f, self.filename, fmt="NETCDF3_CLASSIC", verbose=verbose) g = cfdm.read(self.filename, verbose=verbose) self.assertEqual( len(g), 1, "Read produced too many fields: {} != 1".format(len(g))) g = g[0].squeeze() self.assertEqual( sorted(f.constructs), sorted(g.constructs), "\n\nf (created in memory)\n{}\n\n{}\n\ng " "(read from disk)\n{}\n\n{}".format( sorted(f.constructs), sorted(f.constructs.items()), sorted(g.constructs), sorted(g.constructs.items()), ), ) self.assertTrue( f.equals(f.copy(), verbose=verbose), "Field f not equal to a copy of itself", ) self.assertTrue( g.equals(g.copy(), verbose=verbose), "Field g not equal to a copy of itself", ) self.assertTrue( g.equals(f, verbose=verbose), "Field not equal to itself read back in", ) x = g.dump(display=False) x = f.dump(display=False) g = cfdm.read( self.filename, verbose=verbose, extra=["domain_ancillary"], warnings=warnings, )
q, t = cfdm.read("file.nc") t t2 = t.squeeze() t2 print(t2.dimension_coordinates) t3 = t2.insert_dimension(axis="domainaxis3", position=1) t3 t3.transpose([2, 0, 1]) t4 = t.transpose([0, 2, 1], constructs=True) print(q) print(q.data.mask) print(q.data.mask.array) q.data[[0, 4], :] = cfdm.masked print(q.data.mask.array) q.data.mask.any() cfdm.write(q, "masked_q.nc") no_mask_q = cfdm.read("masked_q.nc", mask=False)[0] print(no_mask_q.data.array) masked_q = no_mask_q.apply_masking() print(masked_q.data.array) data = t.data data.shape data[:, :, 1].shape data[:, 0].shape data[..., 6:3:-1, 3:6].shape data[0, [2, 9], [4, 8]].shape data[0, :, -2].shape import numpy t.data[:, 0, 0] = -1 t.data[:, :, 1] = -2
def test_netCDF_variable_dimension(self): if self.test_only and inspect.stack()[0][3] not in self.test_only: return f = cfdm.Field() f.nc_set_variable('qwerty') self.assertTrue(f.nc_has_variable()) self.assertEqual(f.nc_get_variable(), 'qwerty') self.assertEqual(f.nc_get_variable(default=None), 'qwerty') self.assertEqual(f.nc_del_variable(), 'qwerty') self.assertFalse(f.nc_has_variable()) self.assertIsNone(f.nc_get_variable(default=None)) self.assertIsNone(f.nc_del_variable(default=None)) f.nc_set_variable('/ncvar') self.assertEqual(f.nc_get_variable(), 'ncvar') f.nc_set_variable('/ncvar/qwerty') self.assertEqual(f.nc_get_variable(), '/ncvar/qwerty') with self.assertRaises(ValueError): f.nc_set_variable(None) with self.assertRaises(ValueError): f.nc_set_variable('/') with self.assertRaises(ValueError): f.nc_set_variable('group/ncvar') with self.assertRaises(ValueError): f.nc_set_variable('group/') with self.assertRaises(ValueError): f.nc_set_variable('group/ncvar/') with self.assertRaises(ValueError): f.nc_set_variable('/group/ncvar/') d = cfdm.DomainAxis() d.nc_set_dimension('qwerty') self.assertTrue(d.nc_has_dimension()) self.assertEqual(d.nc_get_dimension(), 'qwerty') self.assertEqual(d.nc_get_dimension(default=None), 'qwerty') self.assertEqual(d.nc_del_dimension(), 'qwerty') self.assertFalse(d.nc_has_dimension()) self.assertIsNone(d.nc_get_dimension(default=None)) self.assertIsNone(d.nc_del_dimension(default=None)) d.nc_set_dimension('/ncdim') self.assertEqual(d.nc_get_dimension(), 'ncdim') d.nc_set_dimension('/ncdim/qwerty') self.assertEqual(d.nc_get_dimension(), '/ncdim/qwerty') with self.assertRaises(ValueError): d.nc_set_dimension(None) with self.assertRaises(ValueError): d.nc_set_dimension('/') with self.assertRaises(ValueError): d.nc_set_dimension('group/ncdim') with self.assertRaises(ValueError): d.nc_set_dimension('group/') with self.assertRaises(ValueError): d.nc_set_dimension('group/ncdim/') with self.assertRaises(ValueError): d.nc_set_dimension('/group/ncdim/') d = cfdm.Count() d.nc_set_sample_dimension('qwerty') self.assertTrue(d.nc_has_sample_dimension()) self.assertEqual(d.nc_get_sample_dimension(), 'qwerty') self.assertEqual(d.nc_get_sample_dimension(default=None), 'qwerty') self.assertEqual(d.nc_del_sample_dimension(), 'qwerty') self.assertFalse(d.nc_has_sample_dimension()) self.assertIsNone(d.nc_get_sample_dimension(default=None)) self.assertIsNone(d.nc_del_sample_dimension(default=None)) d.nc_set_sample_dimension('/ncdim') self.assertEqual(d.nc_get_sample_dimension(), 'ncdim') d.nc_set_sample_dimension('/ncdim/qwerty') self.assertEqual(d.nc_get_sample_dimension(), '/ncdim/qwerty') with self.assertRaises(ValueError): d.nc_set_sample_dimension(None) with self.assertRaises(ValueError): d.nc_set_sample_dimension('/') with self.assertRaises(ValueError): d.nc_set_sample_dimension('group/ncdim') with self.assertRaises(ValueError): d.nc_set_sample_dimension('group/') with self.assertRaises(ValueError): d.nc_set_sample_dimension('group/ncdim/') with self.assertRaises(ValueError): d.nc_set_sample_dimension('/group/ncdim/') # ------------------------------------------------------------ # Global attributes # ------------------------------------------------------------ # values keyword f = cfdm.Field() f.nc_set_global_attribute('Conventions', 'CF-1.8') f.nc_set_global_attribute('project') f.nc_set_global_attribute('foo') f.set_property('Conventions', 'Y') f.set_property('project', 'X') self.assertEqual(f.nc_global_attributes(values=True), {'Conventions': 'CF-1.8', 'project': 'X', 'foo': None}) f = cfdm.Field() self.assertEqual(f.nc_clear_global_attributes(), {}) f.nc_set_global_attribute('Conventions') f.nc_set_global_attribute('project', 'X') self.assertEqual(f.nc_global_attributes(), {'Conventions': None, 'project': 'X'}) f.nc_set_global_attribute('project') f.nc_set_global_attribute('comment', None) self.assertEqual(f.nc_global_attributes(), {'Conventions': None, 'project': None, 'comment': None}) self.assertEqual(f.nc_clear_global_attributes(), {'Conventions': None, 'project': None, 'comment': None}) self.assertEqual(f.nc_global_attributes(), {}) f.nc_set_global_attribute('Conventions') f.nc_set_global_attribute('project') self.assertEqual(f.nc_global_attributes(), {'Conventions': None, 'project': None}) _ = f.nc_clear_global_attributes() f.nc_set_global_attributes({}) self.assertEqual(f.nc_global_attributes(), {}) f.nc_set_global_attributes({'comment': 123}, copy=False) self.assertEqual(f.nc_global_attributes(), {'comment': 123}) f.nc_set_global_attributes({'comment': None, 'foo': 'bar'}) self.assertEqual(f.nc_global_attributes(), {'comment': None, 'foo': 'bar'}) f = cfdm.Field() f.set_properties({'foo': 'bar', 'comment': 'variable comment'}) f.nc_set_variable('tas') d = f.set_construct(cfdm.DomainAxis(2)) f.set_data(cfdm.Data([8, 9]), axes=[d]) f2 = f.copy() f2.nc_set_variable('ua') cfdm.write([f, f2], tempfile1, file_descriptors={'comment': 'global comment', 'qwerty': 'asdf'}) g = cfdm.read(tempfile1) self.assertEqual(len(g), 2) for x in g: self.assertEqual(x.properties(), {'comment': 'variable comment', 'foo': 'bar', 'qwerty': 'asdf', 'Conventions': 'CF-'+cfdm.CF()}) self.assertEqual(x.nc_global_attributes(), {'comment': 'global comment', 'qwerty': None, 'Conventions': None}) cfdm.write(g, tempfile2) h = cfdm.read(tempfile2) for x, y in zip(h, g): self.assertEqual(x.properties(), y.properties()) self.assertEqual(x.nc_global_attributes(), y.nc_global_attributes()) self.assertTrue(x.equals(y, verbose=3)) self.assertTrue(y.equals(x, verbose=3)) g[1].nc_set_global_attribute('comment', 'different comment') cfdm.write(g, tempfile3) h = cfdm.read(tempfile3) for x, y in zip(h, g): self.assertEqual(x.properties(), y.properties()) self.assertEqual(x.nc_global_attributes(), {'comment': None, 'qwerty': None, 'Conventions': None}) self.assertTrue(x.equals(y, verbose=3)) self.assertTrue(y.equals(x, verbose=3))
def test_netCDF_variable_dimension(self): """Test variable and dimension access NetCDF methods.""" f = cfdm.Field() f.nc_set_variable("qwerty") self.assertTrue(f.nc_has_variable()) self.assertEqual(f.nc_get_variable(), "qwerty") self.assertEqual(f.nc_get_variable(default=None), "qwerty") self.assertEqual(f.nc_del_variable(), "qwerty") self.assertFalse(f.nc_has_variable()) self.assertIsNone(f.nc_get_variable(default=None)) self.assertIsNone(f.nc_del_variable(default=None)) f.nc_set_variable("/ncvar") self.assertEqual(f.nc_get_variable(), "ncvar") f.nc_set_variable("/ncvar/qwerty") self.assertEqual(f.nc_get_variable(), "/ncvar/qwerty") for nc_var_name in self.nc_grouped_variable_names: with self.assertRaises(ValueError): f.nc_set_variable(nc_var_name) d = cfdm.DomainAxis() d.nc_set_dimension("qwerty") self.assertTrue(d.nc_has_dimension()) self.assertEqual(d.nc_get_dimension(), "qwerty") self.assertEqual(d.nc_get_dimension(default=None), "qwerty") self.assertEqual(d.nc_del_dimension(), "qwerty") self.assertFalse(d.nc_has_dimension()) self.assertIsNone(d.nc_get_dimension(default=None)) self.assertIsNone(d.nc_del_dimension(default=None)) d.nc_set_dimension("/ncdim") self.assertEqual(d.nc_get_dimension(), "ncdim") d.nc_set_dimension("/ncdim/qwerty") self.assertEqual(d.nc_get_dimension(), "/ncdim/qwerty") for nc_dim_name in self.nc_grouped_dimension_names: with self.assertRaises(ValueError): d.nc_set_dimension(nc_dim_name) d = cfdm.Count() d.nc_set_sample_dimension("qwerty") self.assertTrue(d.nc_has_sample_dimension()) self.assertEqual(d.nc_get_sample_dimension(), "qwerty") self.assertEqual(d.nc_get_sample_dimension(default=None), "qwerty") self.assertEqual(d.nc_del_sample_dimension(), "qwerty") self.assertFalse(d.nc_has_sample_dimension()) self.assertIsNone(d.nc_get_sample_dimension(default=None)) self.assertIsNone(d.nc_del_sample_dimension(default=None)) d.nc_set_sample_dimension("/ncdim") self.assertEqual(d.nc_get_sample_dimension(), "ncdim") d.nc_set_sample_dimension("/ncdim/qwerty") self.assertEqual(d.nc_get_sample_dimension(), "/ncdim/qwerty") for nc_dim_name in self.nc_grouped_dimension_names: with self.assertRaises(ValueError): d.nc_set_sample_dimension(nc_dim_name) # ------------------------------------------------------------ # Global attributes # ------------------------------------------------------------ # values keyword f = cfdm.Field() f.nc_set_global_attribute("Conventions", "CF-1.8") f.nc_set_global_attribute("project") f.nc_set_global_attribute("foo") f.set_property("Conventions", "Y") f.set_property("project", "X") self.assertEqual( f.nc_global_attributes(values=True), { "Conventions": "CF-1.8", "project": "X", "foo": None }, ) f = cfdm.Field() self.assertEqual(f.nc_clear_global_attributes(), {}) f.nc_set_global_attribute("Conventions") f.nc_set_global_attribute("project", "X") self.assertEqual(f.nc_global_attributes(), { "Conventions": None, "project": "X" }) f.nc_set_global_attribute("project") f.nc_set_global_attribute("comment", None) self.assertEqual( f.nc_global_attributes(), { "Conventions": None, "project": None, "comment": None }, ) self.assertEqual( f.nc_clear_global_attributes(), { "Conventions": None, "project": None, "comment": None }, ) self.assertEqual(f.nc_global_attributes(), {}) f.nc_set_global_attribute("Conventions") f.nc_set_global_attribute("project") self.assertEqual(f.nc_global_attributes(), { "Conventions": None, "project": None }) _ = f.nc_clear_global_attributes() f.nc_set_global_attributes({}) self.assertEqual(f.nc_global_attributes(), {}) f.nc_set_global_attributes({"comment": 123}, copy=False) self.assertEqual(f.nc_global_attributes(), {"comment": 123}) f.nc_set_global_attributes({"comment": None, "foo": "bar"}) self.assertEqual(f.nc_global_attributes(), { "comment": None, "foo": "bar" }) f = cfdm.Field() f.set_properties({"foo": "bar", "comment": "variable comment"}) f.nc_set_variable("tas") d = f.set_construct(cfdm.DomainAxis(2)) f.set_data(cfdm.Data([8, 9]), axes=[d]) f2 = f.copy() f2.nc_set_variable("ua") cfdm.write( [f, f2], tempfile1, file_descriptors={ "comment": "global comment", "qwerty": "asdf" }, ) g = cfdm.read(tempfile1) self.assertEqual(len(g), 2) for x in g: self.assertEqual( x.properties(), { "comment": "variable comment", "foo": "bar", "qwerty": "asdf", "Conventions": "CF-" + cfdm.CF(), }, ) self.assertEqual( x.nc_global_attributes(), { "comment": "global comment", "qwerty": None, "Conventions": None, }, ) cfdm.write(g, tempfile2) h = cfdm.read(tempfile2) for x, y in zip(h, g): self.assertEqual(x.properties(), y.properties()) self.assertEqual(x.nc_global_attributes(), y.nc_global_attributes()) self.assertTrue(x.equals(y, verbose=3)) self.assertTrue(y.equals(x, verbose=3)) g[1].nc_set_global_attribute("comment", "different comment") cfdm.write(g, tempfile3) h = cfdm.read(tempfile3) for x, y in zip(h, g): self.assertEqual(x.properties(), y.properties()) self.assertEqual( x.nc_global_attributes(), { "comment": None, "qwerty": None, "Conventions": None }, ) self.assertTrue(x.equals(y, verbose=3)) self.assertTrue(y.equals(x, verbose=3))
def test_groups(self): """Test for the general handling of hierarchical groups.""" f = cfdm.example_field(1) ungrouped_file = ungrouped_file1 grouped_file = grouped_file1 # Add a second grid mapping datum = cfdm.Datum(parameters={"earth_radius": 7000000}) conversion = cfdm.CoordinateConversion( parameters={"grid_mapping_name": "latitude_longitude"}) grid = cfdm.CoordinateReference( coordinate_conversion=conversion, datum=datum, coordinates=["auxiliarycoordinate0", "auxiliarycoordinate1"], ) f.set_construct(grid) grid0 = f.construct("grid_mapping_name:rotated_latitude_longitude") grid0.del_coordinate("auxiliarycoordinate0") grid0.del_coordinate("auxiliarycoordinate1") cfdm.write(f, ungrouped_file) g = cfdm.read(ungrouped_file, verbose=1) self.assertEqual(len(g), 1) g = g[0] self.assertTrue(f.equals(g, verbose=2)) # ------------------------------------------------------------ # Move the field construct to the /forecast/model group # ------------------------------------------------------------ g.nc_set_variable_groups(["forecast", "model"]) cfdm.write(g, grouped_file) nc = netCDF4.Dataset(grouped_file, "r") self.assertIn( f.nc_get_variable(), nc.groups["forecast"].groups["model"].variables, ) nc.close() h = cfdm.read(grouped_file, verbose=1) self.assertEqual(len(h), 1, repr(h)) self.assertTrue(f.equals(h[0], verbose=2)) # ------------------------------------------------------------ # Move constructs one by one to the /forecast group. The order # in which we do this matters! # ------------------------------------------------------------ for name in ( "longitude", # Auxiliary coordinate "latitude", # Auxiliary coordinate "long_name=Grid latitude name", # Auxiliary coordinate "measure:area", # Cell measure "surface_altitude", # Domain ancillary "air_temperature standard_error", # Field ancillary "grid_mapping_name:rotated_latitude_longitude", "time", # Dimension coordinate "grid_latitude", # Dimension coordinate ): g.construct(name).nc_set_variable_groups(["forecast"]) cfdm.write(g, grouped_file, verbose=1) # Check that the variable is in the right group nc = netCDF4.Dataset(grouped_file, "r") self.assertIn( f.construct(name).nc_get_variable(), nc.groups["forecast"].variables, ) nc.close() # Check that the field construct hasn't changed h = cfdm.read(grouped_file, verbose=1) self.assertEqual(len(h), 1, repr(h)) self.assertTrue(f.equals(h[0], verbose=2), name) # ------------------------------------------------------------ # Move bounds to the /forecast group # ------------------------------------------------------------ name = "grid_latitude" g.construct(name).bounds.nc_set_variable_groups(["forecast"]) cfdm.write(g, grouped_file) nc = netCDF4.Dataset(grouped_file, "r") self.assertIn( f.construct(name).bounds.nc_get_variable(), nc.groups["forecast"].variables, ) nc.close() h = cfdm.read(grouped_file, verbose="WARNING") self.assertEqual(len(h), 1, repr(h)) self.assertTrue(f.equals(h[0], verbose=2))
def test_groups_dimension(self): """Test the dimensions of hierarchical groups.""" f = cfdm.example_field(0) ungrouped_file = ungrouped_file4 grouped_file = grouped_file4 cfdm.write(f, ungrouped_file) g = cfdm.read(ungrouped_file, verbose=1) self.assertEqual(len(g), 1) g = g[0] self.assertTrue(f.equals(g, verbose=3)) # ------------------------------------------------------------ # Move the field construct to the /forecast/model group # ------------------------------------------------------------ g.nc_set_variable_groups(["forecast", "model"]) # ------------------------------------------------------------ # Move all data constructs to the /forecast group # ------------------------------------------------------------ for construct in g.constructs.filter_by_data().values(): construct.nc_set_variable_groups(["forecast"]) # ------------------------------------------------------------ # Move all coordinate bounds constructs to the /forecast group # ------------------------------------------------------------ for construct in g.coordinates().values(): try: construct.bounds.nc_set_variable_groups(["forecast"]) except ValueError: pass cfdm.write(g, grouped_file, verbose=1) nc = netCDF4.Dataset(grouped_file, "r") self.assertIn( f.nc_get_variable(), nc.groups["forecast"].groups["model"].variables, ) for key, construct in g.constructs.filter_by_data().items(): self.assertIn( f.constructs[key].nc_get_variable(), nc.groups["forecast"].variables, ) nc.close() h = cfdm.read(grouped_file, verbose=1) self.assertEqual(len(h), 1) h = h[0] self.assertTrue(f.equals(h, verbose=3)) # ------------------------------------------------------------ # Move all the lat dimension to the /forecast group # ------------------------------------------------------------ key = g.domain_axis_key("latitude") domain_axis = g.constructs[key] domain_axis.nc_set_dimension_groups(["forecast"]) cfdm.write(g, grouped_file, verbose=1) h = cfdm.read(grouped_file, verbose=1) self.assertEqual(len(h), 1) h = h[0] self.assertTrue(f.equals(h, verbose=3))
def test_groups_compression(self): """Test the compression of hierarchical groups.""" f = cfdm.example_field(4) ungrouped_file = ungrouped_file3 grouped_file = grouped_file3 f.compress("indexed_contiguous", inplace=True) f.data.get_count().nc_set_variable("count") f.data.get_index().nc_set_variable("index") cfdm.write(f, ungrouped_file, verbose=1) g = cfdm.read(ungrouped_file)[0] self.assertTrue(f.equals(g, verbose=2)) # ------------------------------------------------------------ # Move the field construct to the /forecast/model group # ------------------------------------------------------------ g.nc_set_variable_groups(["forecast", "model"]) # ------------------------------------------------------------ # Move the count variable to the /forecast group # ------------------------------------------------------------ g.data.get_count().nc_set_variable_groups(["forecast"]) # ------------------------------------------------------------ # Move the index variable to the /forecast group # ------------------------------------------------------------ g.data.get_index().nc_set_variable_groups(["forecast"]) # ------------------------------------------------------------ # Move the coordinates that span the element dimension to the # /forecast group # ------------------------------------------------------------ name = "altitude" g.construct(name).nc_set_variable_groups(["forecast"]) # ------------------------------------------------------------ # Move the sample dimension to the /forecast group # ------------------------------------------------------------ g.data.get_count().nc_set_sample_dimension_groups(["forecast"]) cfdm.write(g, grouped_file, verbose=1) nc = netCDF4.Dataset(grouped_file, "r") self.assertIn( f.nc_get_variable(), nc.groups["forecast"].groups["model"].variables, ) self.assertIn( f.data.get_count().nc_get_variable(), nc.groups["forecast"].variables, ) self.assertIn( f.data.get_index().nc_get_variable(), nc.groups["forecast"].variables, ) self.assertIn( f.construct("altitude").nc_get_variable(), nc.groups["forecast"].variables, ) nc.close() h = cfdm.read(grouped_file, verbose=1) self.assertEqual(len(h), 1, repr(h)) self.assertTrue(f.equals(h[0], verbose=2))
def test_create_field_2(self): # Dimension coordinates dim1 = cfdm.DimensionCoordinate(data=cfdm.Data(numpy.arange(10.))) dim1.set_property('standard_name', 'projection_y_coordinate') dim1.set_property('units', 'm') data = numpy.arange(9.) + 20 data[-1] = 34 dim0 = cfdm.DimensionCoordinate(data=cfdm.Data(data)) dim0.set_property('standard_name', 'projection_x_coordinate') dim0.set_property('units', 'm') array = dim0.data.array array = numpy.array([array-0.5, array+0.5]).transpose((1, 0)) array[-2, 1] = 30 array[-1, :] = [30, 36] dim0.set_bounds(cfdm.Bounds(data=cfdm.Data(array))) dim2 = cfdm.DimensionCoordinate( data=cfdm.Data([1.5]), bounds=cfdm.Bounds(data=cfdm.Data([[1, 2.]])) ) dim2.set_property( 'standard_name', 'atmosphere_hybrid_height_coordinate') # Auxiliary coordinates aux2 = cfdm.AuxiliaryCoordinate( data=cfdm.Data( numpy.arange(-45, 45, dtype='int32').reshape(10, 9))) aux2.set_property('units', 'degree_N') aux2.set_property('standard_name', 'latitude') aux3 = cfdm.AuxiliaryCoordinate( data=cfdm.Data(numpy.arange( 60, 150, dtype='int32').reshape(9, 10)) ) aux3.set_property('standard_name', 'longitude') aux3.set_property('units', 'degreeE') array = numpy.ma.array( ['alpha', 'beta', 'gamma', 'delta', 'epsilon', 'zeta', 'eta', 'theta', 'iota', 'kappa'], dtype='S' ) array[0] = numpy.ma.masked aux4 = cfdm.AuxiliaryCoordinate(data=cfdm.Data(array)) aux4.set_property('standard_name', 'greek_letters') # Domain ancillaries ak = cfdm.DomainAncillary(data=cfdm.Data([10.])) ak.set_property('units', 'm') ak.set_bounds(cfdm.Bounds(data=cfdm.Data([[5, 15.]]))) bk = cfdm.DomainAncillary(data=cfdm.Data([20.])) bk.set_bounds(cfdm.Bounds(data=cfdm.Data([[14, 26.]]))) # Cell measures msr0 = cfdm.CellMeasure( data=cfdm.Data(1+numpy.arange(90.).reshape(9, 10)*1234)) msr0.set_measure('area') msr0.set_property('units', 'km2') # Data data = cfdm.Data(numpy.arange(90.).reshape(10, 9)) properties = {'units': 'm s-1'} f = cfdm.Field(properties=properties) f.set_property('standard_name', 'eastward_wind') axisX = f.set_construct(cfdm.DomainAxis(9)) axisY = f.set_construct(cfdm.DomainAxis(10)) axisZ = f.set_construct(cfdm.DomainAxis(1)) f.set_data(data, axes=[axisY, axisX]) x = f.set_construct(dim0, axes=[axisX]) y = f.set_construct(dim1, axes=[axisY]) z = f.set_construct(dim2, axes=[axisZ]) lat = f.set_construct(aux2, axes=[axisY, axisX]) lon = f.set_construct(aux3, axes=[axisX, axisY]) greek = f.set_construct(aux4, axes=[axisY]) ak = f.set_construct(ak, axes=[axisZ]) bk = f.set_construct(bk, axes=[axisZ]) # Coordinate references coordinate_conversion = cfdm.CoordinateConversion( parameters={'grid_mapping_name': "transverse_mercator", 'latitude_of_projection_origin': 49.0, 'longitude_of_central_meridian': -2.0, 'scale_factor_at_central_meridian': 0.9996012717, 'false_easting': 400000.0, 'false_northing': -100000.0, 'unit': "metre"}) datum0 = cfdm.Datum(parameters={'inverse_flattening': 299.3249646, 'longitude_of_prime_meridian': 0.0, 'semi_major_axis': 6377563.396}) ref0 = cfdm.CoordinateReference( coordinates=[x, y], datum=datum0, coordinate_conversion=coordinate_conversion ) coordinate_conversion = cfdm.CoordinateConversion( parameters={'grid_mapping_name': "latitude_longitude"}) datum2 = cfdm.Datum(parameters={'longitude_of_prime_meridian': 0.0, 'semi_major_axis': 6378137.0, 'inverse_flattening': 298.257223563}) ref2 = cfdm.CoordinateReference( coordinates=[lat, lon], datum=datum2, coordinate_conversion=coordinate_conversion) f.set_construct(msr0, axes=[axisX, axisY]) f.set_construct(ref0) f.set_construct(ref2) orog = cfdm.DomainAncillary(data=f.get_data()) orog.set_property('standard_name', 'surface_altitude') orog.set_property('units', 'm') orog = f.set_construct(orog, axes=[axisY, axisX]) coordinate_conversion = cfdm.CoordinateConversion( parameters={ 'standard_name': 'atmosphere_hybrid_height_coordinate' }, domain_ancillaries={ 'orog': orog, 'a': ak, 'b': bk } ) ref1 = cfdm.CoordinateReference( coordinates=[z], datum=datum0, coordinate_conversion=coordinate_conversion ) f.set_construct(ref1) # Field ancillary variables g = f.copy() anc = cfdm.FieldAncillary(data=g.get_data()) anc.standard_name = 'ancillaryA' f.set_construct(anc, axes=[axisY, axisX]) g = f[0] g = g.squeeze() anc = cfdm.FieldAncillary(data=g.get_data()) anc.standard_name = 'ancillaryB' f.set_construct(anc, axes=[axisX]) g = f[..., 0] g = g.squeeze() anc = cfdm.FieldAncillary(data=g.get_data()) anc.standard_name = 'ancillaryC' f.set_construct(anc, axes=[axisY]) f.set_property('flag_values', numpy.array([1, 2, 4], 'int32')) f.set_property('flag_meanings', 'a bb ccc') f.set_property('flag_masks', [2, 1, 0]) cm0 = cfdm.CellMethod(axes=[axisX], method='mean', qualifiers={'interval': [cfdm.Data(1, 'day')], 'comment': 'ok'}) cm1 = cfdm.CellMethod(axes=[axisY], method='maximum', qualifiers={'where': 'sea'}) f.set_construct(cm0) f.set_construct(cm1) self.assertTrue(f.equals(f.copy(), verbose=verbose), "Field f not equal to a copy of itself") for fmt in ('NETCDF3_CLASSIC', 'NETCDF3_64BIT', 'NETCDF4', 'NETCDF4_CLASSIC'): cfdm.write(f, self.filename, fmt=fmt, verbose=verbose) g = cfdm.read(self.filename, verbose=verbose) self.assertEqual(len(g), 1, '{} != 1'.format(len(g))) g = g[0].squeeze() self.assertEqual(sorted(f.constructs), sorted(g.constructs), '\n\nf\n{}\n\n{}\n\ng\n{}\n\n{}'.format( sorted(f.constructs), sorted(f.constructs.items()), sorted(g.constructs), sorted(g.constructs.items()))) self.assertTrue(g.equals(g.copy(), verbose=verbose), "Field g not equal to a copy of itself") self.assertTrue(g.equals(f, verbose=verbose), "Field not equal to itself read back in") # --- End: for x = g.dump(display=False) x = f.dump(display=False) g = cfdm.read(self.filename, verbose=verbose, extra=['domain_ancillary'], warnings=warnings)
def test_STRING(self): """Test constructs with underlying string type arrays.""" for array in ( numpy.ma.array(list("abcdefghij"), dtype="S"), numpy.ma.array( [ "a", "b1", "c12", "d123", "e1234", "f", "g", "h", "i", "j" ], dtype="S", ), ): # Initialize the field tas = cfdm.Field( properties={ "project": "research", "standard_name": "air_temperature", "units": "K", }) # Create and set domain axes tas.set_construct(cfdm.DomainAxis(1)) tas.set_construct(cfdm.DomainAxis(1)) axis_Y = tas.set_construct(cfdm.DomainAxis(10)) axis_X = tas.set_construct(cfdm.DomainAxis(9)) # Set the field data tas.set_data( cfdm.Data(numpy.arange(90.0).reshape(10, 9)), axes=[axis_Y, axis_X], ) # Create and set the dimension coordinates dimension_coordinate_Y = cfdm.DimensionCoordinate( properties={ "standard_name": "grid_latitude", "units": "degrees", }, data=cfdm.Data(numpy.arange(10.0)), bounds=cfdm.Bounds( data=cfdm.Data(numpy.arange(20).reshape(10, 2))), ) dimension_coordinate_X = cfdm.DimensionCoordinate( properties={ "standard_name": "grid_longitude", "units": "degrees", }, data=cfdm.Data(numpy.arange(9.0)), bounds=cfdm.Bounds( data=cfdm.Data(numpy.arange(18).reshape(9, 2))), ) tas.set_construct(dimension_coordinate_Y, axes=[axis_Y]) tas.set_construct(dimension_coordinate_X, axes=[axis_X]) # Create and set the auxiliary coordinates array[0] = numpy.ma.masked aux0 = cfdm.AuxiliaryCoordinate( properties={"long_name": "Grid latitude name"}, data=cfdm.Data(array), ) tas.set_construct(aux0, axes=[axis_Y]) cfdm.write(tas, tempfile) tas1 = cfdm.read(tempfile)[0] aux1 = tas1.constructs.filter_by_identity( "long_name=Grid latitude name").value() self.assertEqual(aux0.data.shape, array.shape, aux0.data.shape) self.assertEqual(aux1.data.shape, array.shape, aux1.data.shape)
def test_geometry_interior_ring(self): """Test the management of interior ring geometries.""" for geometry_file in ( self.geometry_interior_ring_file, self.geometry_interior_ring_file_2, ): f = cfdm.read(geometry_file, verbose=False) self.assertEqual(len(f), 2, "f = " + repr(f)) for g in f: self.assertTrue(g.equals(g.copy(), verbose=3)) self.assertEqual(len(g.auxiliary_coordinates()), 4) g = f[0] for axis in ("X", "Y"): coord = g.construct("axis=" + axis) self.assertTrue(coord.has_node_count(), "axis=" + axis) self.assertTrue(coord.has_part_node_count(), "axis=" + axis) self.assertTrue(coord.has_interior_ring(), "axis=" + axis) cfdm.write(f, tempfile, Conventions="CF-" + VN) f2 = cfdm.read(tempfile) self.assertEqual(len(f2), 2, "f2 = " + repr(f2)) for a, b in zip(f, f2): self.assertTrue(a.equals(b, verbose=3)) # Interior ring component c = g.construct("longitude") self.assertTrue( c.interior_ring.equals( g.construct("longitude").get_interior_ring())) self.assertEqual(c.interior_ring.data.ndim, c.data.ndim + 1) self.assertEqual(c.interior_ring.data.shape[0], c.data.shape[0]) self.assertIsInstance(g.dump(display=False), str) d = c.insert_dimension(0) self.assertEqual(d.data.shape, (1, ) + c.data.shape) self.assertEqual(d.interior_ring.data.shape, (1, ) + c.interior_ring.data.shape) e = d.squeeze(0) self.assertEqual(e.data.shape, c.data.shape) self.assertEqual(e.interior_ring.data.shape, c.interior_ring.data.shape) t = d.transpose() self.assertEqual( t.data.shape, d.data.shape[::-1], (t.data.shape, c.data.shape[::-1]), ) self.assertEqual( t.interior_ring.data.shape, (d.interior_ring.data.shape[-2::-1] + (d.interior_ring.data.shape[-1], )), ) # Subspacing g = g[1, ...] c = g.construct("longitude") self.assertEqual(c.interior_ring.data.shape[0], 1) self.assertEqual(c.interior_ring.data.ndim, c.data.ndim + 1) self.assertEqual(c.interior_ring.data.shape[0], c.data.shape[0]) # Setting of node count properties coord = f[0].construct("axis=Y") nc = coord.get_node_count() nc.set_property("long_name", "Node counts") cfdm.write(f, tempfile) nc.nc_set_variable("new_var_name") cfdm.write(f, tempfile) # Setting of part node count properties coord = f[0].construct("axis=X") pnc = coord.get_part_node_count() pnc.set_property("long_name", "Part node counts") cfdm.write(f, tempfile) pnc.nc_set_variable("new_var_name") cfdm.write(f, tempfile) pnc.nc_set_dimension("new_dim_name") cfdm.write(f, tempfile)
def test_DSG_contiguous(self): """Test the contiguous ragged array DSG representation.""" f = self.c.copy() self.assertEqual(len(f), 2) # Select the specific humidity field q = [ g for g in f if g.get_property("standard_name") == "specific_humidity" ][0] self.assertTrue(q._equals(self.a, q.data.array)) cfdm.write(f, tempfile) g = cfdm.read(tempfile) self.assertEqual(len(g), len(f)) for i in range(len(f)): self.assertTrue(g[i].equals(f[i], verbose=3)) # ------------------------------------------------------------ # Test creation # ------------------------------------------------------------ # Define the ragged array values ragged_array = numpy.array( [280, 282.5, 281, 279, 278, 279.5], dtype="float32" ) # Define the count array values count_array = [2, 4] # Create the count variable count_variable = cfdm.Count(data=cfdm.Data(count_array)) count_variable.set_property( "long_name", "number of obs for this timeseries" ) # Create the contiguous ragged array object array = cfdm.RaggedContiguousArray( compressed_array=cfdm.NumpyArray(ragged_array), shape=(2, 4), size=8, ndim=2, count_variable=count_variable, ) # Create the field construct with the domain axes and the ragged # array tas = cfdm.Field() tas.set_properties( { "standard_name": "air_temperature", "units": "K", "featureType": "timeSeries", } ) # Create the domain axis constructs for the uncompressed array X = tas.set_construct(cfdm.DomainAxis(4)) Y = tas.set_construct(cfdm.DomainAxis(2)) # Set the data for the field tas.set_data(cfdm.Data(array), axes=[Y, X]) cfdm.write(tas, tempfile)
def test_create_field_2(self): """Test ab initio creation of a second variation of field.""" # Dimension coordinates dim1 = cfdm.DimensionCoordinate(data=cfdm.Data(numpy.arange(10.0))) dim1.set_property("standard_name", "projection_y_coordinate") dim1.set_property("units", "m") data = numpy.arange(9.0) + 20 data[-1] = 34 dim0 = cfdm.DimensionCoordinate(data=cfdm.Data(data)) dim0.set_property("standard_name", "projection_x_coordinate") dim0.set_property("units", "m") array = dim0.data.array array = numpy.array([array - 0.5, array + 0.5]).transpose((1, 0)) array[-2, 1] = 30 array[-1, :] = [30, 36] dim0.set_bounds(cfdm.Bounds(data=cfdm.Data(array))) dim2 = cfdm.DimensionCoordinate( data=cfdm.Data([1.5]), bounds=cfdm.Bounds(data=cfdm.Data([[1, 2.0]])), ) dim2.set_property("standard_name", "atmosphere_hybrid_height_coordinate") # Auxiliary coordinates aux2 = cfdm.AuxiliaryCoordinate(data=cfdm.Data( numpy.arange(-45, 45, dtype="int32").reshape(10, 9))) aux2.set_property("units", "degree_N") aux2.set_property("standard_name", "latitude") aux3 = cfdm.AuxiliaryCoordinate(data=cfdm.Data( numpy.arange(60, 150, dtype="int32").reshape(9, 10))) aux3.set_property("standard_name", "longitude") aux3.set_property("units", "degreeE") array = numpy.ma.array( [ "alpha", "beta", "gamma", "delta", "epsilon", "zeta", "eta", "theta", "iota", "kappa", ], dtype="S", ) array[0] = numpy.ma.masked aux4 = cfdm.AuxiliaryCoordinate(data=cfdm.Data(array)) aux4.set_property("standard_name", "greek_letters") # Domain ancillaries ak = cfdm.DomainAncillary(data=cfdm.Data([10.0])) ak.set_property("units", "m") ak.set_bounds(cfdm.Bounds(data=cfdm.Data([[5, 15.0]]))) bk = cfdm.DomainAncillary(data=cfdm.Data([20.0])) bk.set_bounds(cfdm.Bounds(data=cfdm.Data([[14, 26.0]]))) # Cell measures msr0 = cfdm.CellMeasure( data=cfdm.Data(1 + numpy.arange(90.0).reshape(9, 10) * 1234)) msr0.set_measure("area") msr0.set_property("units", "km2") # Data data = cfdm.Data(numpy.arange(90.0).reshape(10, 9)) properties = {"units": "m s-1"} f = cfdm.Field(properties=properties) f.set_property("standard_name", "eastward_wind") axisX = f.set_construct(cfdm.DomainAxis(9)) axisY = f.set_construct(cfdm.DomainAxis(10)) axisZ = f.set_construct(cfdm.DomainAxis(1)) f.set_data(data, axes=[axisY, axisX]) x = f.set_construct(dim0, axes=[axisX]) y = f.set_construct(dim1, axes=[axisY]) z = f.set_construct(dim2, axes=[axisZ]) lat = f.set_construct(aux2, axes=[axisY, axisX]) lon = f.set_construct(aux3, axes=[axisX, axisY]) f.set_construct(aux4, axes=[axisY]) ak = f.set_construct(ak, axes=[axisZ]) bk = f.set_construct(bk, axes=[axisZ]) # Coordinate references coordinate_conversion = cfdm.CoordinateConversion( parameters={ "grid_mapping_name": "transverse_mercator", "latitude_of_projection_origin": 49.0, "longitude_of_central_meridian": -2.0, "scale_factor_at_central_meridian": 0.9996012717, "false_easting": 400000.0, "false_northing": -100000.0, "unit": "metre", }) datum0 = cfdm.Datum( parameters={ "inverse_flattening": 299.3249646, "longitude_of_prime_meridian": 0.0, "semi_major_axis": 6377563.396, }) ref0 = cfdm.CoordinateReference( coordinates=[x, y], datum=datum0, coordinate_conversion=coordinate_conversion, ) coordinate_conversion = cfdm.CoordinateConversion( parameters={"grid_mapping_name": "latitude_longitude"}) datum2 = cfdm.Datum( parameters={ "longitude_of_prime_meridian": 0.0, "semi_major_axis": 6378137.0, "inverse_flattening": 298.257223563, }) ref2 = cfdm.CoordinateReference( coordinates=[lat, lon], datum=datum2, coordinate_conversion=coordinate_conversion, ) f.set_construct(msr0, axes=[axisX, axisY]) f.set_construct(ref0) f.set_construct(ref2) orog = cfdm.DomainAncillary(data=f.get_data()) orog.set_property("standard_name", "surface_altitude") orog.set_property("units", "m") orog = f.set_construct(orog, axes=[axisY, axisX]) coordinate_conversion = cfdm.CoordinateConversion( parameters={ "standard_name": "atmosphere_hybrid_height_coordinate" }, domain_ancillaries={ "orog": orog, "a": ak, "b": bk }, ) ref1 = cfdm.CoordinateReference( coordinates=[z], datum=datum0, coordinate_conversion=coordinate_conversion, ) f.set_construct(ref1) # Field ancillary variables g = f.copy() anc = cfdm.FieldAncillary(data=g.get_data()) anc.standard_name = "ancillaryA" f.set_construct(anc, axes=[axisY, axisX]) g = f[0] g = g.squeeze() anc = cfdm.FieldAncillary(data=g.get_data()) anc.standard_name = "ancillaryB" f.set_construct(anc, axes=[axisX]) g = f[..., 0] g = g.squeeze() anc = cfdm.FieldAncillary(data=g.get_data()) anc.standard_name = "ancillaryC" f.set_construct(anc, axes=[axisY]) f.set_property("flag_values", numpy.array([1, 2, 4], "int32")) f.set_property("flag_meanings", "a bb ccc") f.set_property("flag_masks", [2, 1, 0]) cm0 = cfdm.CellMethod( axes=[axisX], method="mean", qualifiers={ "interval": [cfdm.Data(1, "day")], "comment": "ok" }, ) cm1 = cfdm.CellMethod(axes=[axisY], method="maximum", qualifiers={"where": "sea"}) f.set_construct(cm0) f.set_construct(cm1) self.assertTrue( f.equals(f.copy(), verbose=verbose), "Field f not equal to a copy of itself", ) for fmt in ( "NETCDF3_CLASSIC", "NETCDF3_64BIT", "NETCDF4", "NETCDF4_CLASSIC", ): cfdm.write(f, self.filename, fmt=fmt, verbose=verbose) g = cfdm.read(self.filename, verbose=verbose) self.assertEqual(len(g), 1, "{} != 1".format(len(g))) g = g[0].squeeze() self.assertEqual( sorted(f.constructs), sorted(g.constructs), "\n\nf\n{}\n\n{}\n\ng\n{}\n\n{}".format( sorted(f.constructs), sorted(f.constructs.items()), sorted(g.constructs), sorted(g.constructs.items()), ), ) self.assertTrue( g.equals(g.copy(), verbose=verbose), "Field g not equal to a copy of itself", ) self.assertTrue( g.equals(f, verbose=verbose), "Field not equal to itself read back in", ) x = g.dump(display=False) x = f.dump(display=False) g = cfdm.read( self.filename, verbose=verbose, extra=["domain_ancillary"], warnings=warnings, )
def test_write_netcdf_mode(self): """Test the `mode` parameter to `write`, notably append mode.""" g = cfdm.read(self.filename) # note 'g' has one field # Test special case #1: attempt to append fields with groups # (other than 'root') which should be forbidden. Using fmt="NETCDF4" # since it is the only format where groups are allowed. # # Note: this is not the most natural test to do first, but putting # it before the rest reduces spurious seg faults for me, so... g[0].nc_set_variable_groups(["forecast", "model"]) cfdm.write(g, tmpfile, fmt="NETCDF4", mode="w") # 1. overwrite to wipe f = cfdm.read(tmpfile) with self.assertRaises(ValueError): cfdm.write(g[0], tmpfile, fmt="NETCDF4", mode="a") # Test special case #2: attempt to append fields with contradictory # featureType to the original file: g[0].nc_clear_variable_groups() g[0].nc_set_global_attribute("featureType", "profile") cfdm.write( g, tmpfile, fmt="NETCDF4", mode="w", global_attributes=("featureType", "profile"), ) # 1. overwrite to wipe h = cfdm.example_field(3) h.nc_set_global_attribute("featureType", "timeSeries") with self.assertRaises(ValueError): cfdm.write(h, tmpfile, fmt="NETCDF4", mode="a") # Now remove featureType attribute for subsquent tests: g_attrs = g[0].nc_clear_global_attributes() del g_attrs["featureType"] g[0].nc_set_global_attributes(g_attrs) # Set a non-trivial (i.e. not only 'Conventions') global attribute to # make the global attribute testing more robust: add_global_attr = ["remark", "A global comment."] original_global_attrs = g[0].nc_global_attributes() original_global_attrs[add_global_attr[0]] = None # -> None on fields g[0].nc_set_global_attribute(*add_global_attr) # First test a bad mode value: with self.assertRaises(ValueError): cfdm.write(g[0], tmpfile, mode="g") for fmt in self.netcdf_fmts: # test over all netCDF 3 and 4 formats # Other tests cover write as default mode (i.e. test with no mode # argument); here test explicit provision of 'w' as argument: cfdm.write( g, tmpfile, fmt=fmt, mode="w", global_attributes=add_global_attr, ) f = cfdm.read(tmpfile) new_length = 1 # since 1 == len(g) self.assertEqual(len(f), new_length) # Ignore as 'remark' should be 'None' on the field as tested below self.assertTrue(f[0].equals(g[0], ignore_properties=["remark"])) self.assertEqual(f[0].nc_global_attributes(), original_global_attrs) # Main aspect of this test: testing the append mode ('a'): now # append all other example fields, to check a diverse variety. for ex_field_n, ex_field in enumerate(cfdm.example_fields()): # Note: after Issue #141, this skip can be removed. if ex_field_n == 1: continue # Skip since "RuntimeError: Can't create variable in # NETCDF4_CLASSIC file from (2) (NetCDF: Attempting netcdf-4 # operation on strict nc3 netcdf-4 file)" i.e. not possible. if fmt == "NETCDF4_CLASSIC" and ex_field_n in (6, 7): continue cfdm.write(ex_field, tmpfile, fmt=fmt, mode="a") f = cfdm.read(tmpfile) new_length += 1 # there should be exactly one more field now self.assertEqual(len(f), new_length) # Can't guarantee order of fields read in after the appends, so # check that the field is *somewhere* in the read-in fieldlist self.assertTrue( any([ ex_field.equals( file_field, ignore_properties=[ "comment", "featureType", "remark", ], ) for file_field in f ])) for file_field in f: self.assertEqual( file_field.nc_global_attributes(), original_global_attrs, ) # Now do the same test, but appending all of the example fields in # one operation rather than one at a time, to check that it works. cfdm.write(g, tmpfile, fmt=fmt, mode="w") # 1. overwrite to wipe append_ex_fields = cfdm.example_fields() del append_ex_fields[1] # note: can remove after Issue #141 closed # Note: can remove this del when Issue #140 is closed: if fmt in self.netcdf3_fmts: del append_ex_fields[5] # n=6 ex_field, minus 1 for above del overall_length = len(append_ex_fields) + 1 # 1 for original 'g' cfdm.write(append_ex_fields, tmpfile, fmt=fmt, mode="a") # 2. now append f = cfdm.read(tmpfile) self.assertEqual(len(f), overall_length) # Also test the mode="r+" alias for mode="a". cfdm.write(g, tmpfile, fmt=fmt, mode="w") # 1. overwrite to wipe cfdm.write(append_ex_fields, tmpfile, fmt=fmt, mode="r+") # 2. now append f = cfdm.read(tmpfile) self.assertEqual(len(f), overall_length) # The appended fields themselves are now known to be correct, # but we also need to check that any coordinates that are # equal across different fields have been shared in the # source netCDF, rather than written in separately. # # Note that the coordinates that are shared across the set of # all example fields plus the field 'g' from the contents of # the original file (self.filename) are as follows: # # 1. Example fields n=0 and n=1 share: # <DimensionCoordinate: time(1) days since 2018-12-01 > # 2. Example fields n=0, n=2 and n=5 share: # <DimensionCoordinate: latitude(5) degrees_north> and # <DimensionCoordinate: longitude(8) degrees_east> # 3. Example fields n=2 and n=5 share: # <DimensionCoordinate: air_pressure(1) hPa> # 4. The original file field ('g') and example field n=1 share: # <AuxiliaryCoordinate: latitude(10, 9) degrees_N>, # <AuxiliaryCoordinate: longitude(9, 10) degrees_E>, # <Dimension...: atmosphere_hybrid_height_coordinate(1) >, # <DimensionCoordinate: grid_latitude(10) degrees>, # <DimensionCoordinate: grid_longitude(9) degrees> and # <DimensionCoordinate: time(1) days since 2018-12-01 > # # Therefore we check all of those coordinates for singularity, # i.e. the same underlying netCDF variables, in turn. # But first, since the order of the fields appended isn't # guaranteed, we must find the mapping of the example fields to # their position in the read-in FieldList. f = cfdm.read(tmpfile) # Element at index N gives position of example field n=N in file file_field_order = [] for ex_field in cfdm.example_fields(): position = [ f.index(file_field) for file_field in f if ex_field.equals( file_field, ignore_properties=["comment", "featureType", "remark"], ) ] if not position: position = [None] # to record skipped example fields file_field_order.append(position[0]) equal_coors = { ((0, "dimensioncoordinate2"), (1, "dimensioncoordinate3")), ((0, "dimensioncoordinate0"), (2, "dimensioncoordinate1")), ((0, "dimensioncoordinate1"), (2, "dimensioncoordinate2")), ((0, "dimensioncoordinate0"), (5, "dimensioncoordinate1")), ((0, "dimensioncoordinate1"), (5, "dimensioncoordinate2")), ((2, "dimensioncoordinate3"), (5, "dimensioncoordinate3")), } for coor_1, coor_2 in equal_coors: ex_field_1_position, c_1 = coor_1 ex_field_2_position, c_2 = coor_2 # Now map the appropriate example field to the file FieldList f_1 = file_field_order[ex_field_1_position] f_2 = file_field_order[ex_field_2_position] # None for fields skipped in test, distinguish from falsy 0 if f_1 is None or f_2 is None: continue self.assertEqual( f[f_1].constructs().filter_by_identity( c_1).value().nc_get_variable(), f[f_2].constructs().filter_by_identity( c_2).value().nc_get_variable(), ) # Note: after Issue #141, the block below should be un-commented. # # The original file field 'g' must be at the remaining position: # rem_position = list(set( # range(len(f))).difference(set(file_field_order)))[0] # # In the final cases, it is easier to remove the one differing # # coordinate to get the equal coordinates that should be shared: # original_field_coors = dict(f[rem_position].coordinates()) # ex_field_1_coors = dict(f[file_field_order[1]].coordinates()) # for orig_coor, ex_1_coor in zip( # original_field_coors.values(), ex_field_1_coors.values()): # # The 'auxiliarycoordinate2' construct differs for both, so # # skip that but otherwise the two fields have the same coors: # if orig_coor.identity == "auxiliarycoordinate2": # continue # self.assertEqual( # orig_coor.nc_get_variable(), # ex_1_coor.nc_get_variable(), # ) # Check behaviour when append identical fields, as an edge case # 1. Set up the fields and file to use to conduct this test g_new = cfdm.read(self.filename)[0] # note 'g' has one field # There is an unresolved netcdf4-python issue when reading # VLEN arrays (see github.com/Unidata/netcdf4-python/issues/261) # so, until fixed, for NETCDF4 only we must delete the VLEN array # to conduct the test without it (but on a "kitchen sink" field). if fmt == "NETCDF4": aux = g_new.constructs.filter_by_property( long_name="greek_letters") g_new.del_construct(aux.key()) g_copy = g_new.copy() cfdm.write(g_new, tmpfile, fmt=fmt, mode="w") # overwrite to wipe # 2. Conduct the test by appending the identical field g_copy cfdm.write(g_copy, tmpfile, fmt=fmt, mode="a") f = cfdm.read(tmpfile) self.assertEqual(len(f), 2) # i.e. len(f) == 2*len(g_new) == 2*1 self.assertTrue( any([ file_field.equals(g_new, ignore_properties=["remark"]) for file_field in f ])) self.assertEqual(f[0].nc_global_attributes(), g_new.nc_global_attributes())
def test_create_field(self): """Test ab initio creation of a first variation of field.""" # Dimension coordinates dim1 = cfdm.DimensionCoordinate(data=cfdm.Data(numpy.arange(10.0))) dim1.set_property("standard_name", "grid_latitude") dim1.set_property("units", "degrees") data = numpy.arange(9.0) + 20 data[-1] = 34 dim0 = cfdm.DimensionCoordinate(data=cfdm.Data(data)) dim0.set_property("standard_name", "grid_longitude") dim0.set_property("units", "degrees") dim0.nc_set_variable("x") array = dim0.data.array array = numpy.array([array - 0.5, array + 0.5]).transpose((1, 0)) array[-2, 1] = 30 array[-1, :] = [30, 36] dim0.set_bounds(cfdm.Bounds(data=cfdm.Data(array))) dim2 = cfdm.DimensionCoordinate( data=cfdm.Data([1.5]), bounds=cfdm.Bounds(data=cfdm.Data([[1, 2.0]])), ) dim2.set_property("standard_name", "atmosphere_hybrid_height_coordinate") dim2.set_property("computed_standard_name", "altitude") # Auxiliary coordinates ak = cfdm.DomainAncillary(data=cfdm.Data([10.0])) ak.set_property("units", "m") ak.id = "atmosphere_hybrid_height_coordinate_ak" ak.set_bounds(cfdm.Bounds(data=cfdm.Data([[5, 15.0]]))) bk = cfdm.DomainAncillary(data=cfdm.Data([20.0])) bk.id = "atmosphere_hybrid_height_coordinate_bk" bk.set_bounds(cfdm.Bounds(data=cfdm.Data([[14, 26.0]]))) aux2 = cfdm.AuxiliaryCoordinate(data=cfdm.Data( numpy.arange(-45, 45, dtype="int32").reshape(10, 9))) aux2.set_property("units", "degree_N") aux2.set_property("standard_name", "latitude") aux3 = cfdm.AuxiliaryCoordinate(data=cfdm.Data( numpy.arange(60, 150, dtype="int32").reshape(9, 10))) aux3.set_property("standard_name", "longitude") aux3.set_property("units", "degreeE") array = numpy.ma.array( [ "alpha", "beta", "gamma", "delta", "epsilon", "zeta", "eta", "theta", "iota", "kappa", ], dtype="S", ) array[0] = numpy.ma.masked aux4 = cfdm.AuxiliaryCoordinate(data=cfdm.Data(array)) aux4.set_property("long_name", "greek_letters") # Cell measures msr0 = cfdm.CellMeasure( data=cfdm.Data(1 + numpy.arange(90.0).reshape(9, 10) * 1234)) msr0.set_measure("area") msr0.set_property("units", "km2") msr0.nc_set_variable("areacella") # Data data = cfdm.Data(numpy.arange(90.0).reshape(10, 9)) properties = {"units": "m s-1"} f = cfdm.Field(properties=properties) f.set_property("standard_name", "eastward_wind") da = cfdm.DomainAxis(9) da.nc_set_dimension("grid_longitude") axisX = f.set_construct(da) axisY = f.set_construct(cfdm.DomainAxis(10)) axisZ = f.set_construct(cfdm.DomainAxis(1)) f.set_data(data, axes=[axisY, axisX]) x = f.set_construct(dim0, axes=axisX) y = f.set_construct(dim1, axes=axisY) z = f.set_construct(dim2, axes=[axisZ]) lat = f.set_construct(aux2, axes=[axisY, axisX]) lon = f.set_construct(aux3, axes=[axisX, axisY]) f.set_construct(aux4, axes=[axisY]) ak = f.set_construct(ak, axes=axisZ) bk = f.set_construct(bk, axes=[axisZ]) # Coordinate references coordinate_conversion = cfdm.CoordinateConversion( parameters={ "grid_mapping_name": "rotated_latitude_longitude", "grid_north_pole_latitude": 38.0, "grid_north_pole_longitude": 190.0, }) datum = cfdm.Datum(parameters={"earth_radius": 6371007}) ref0 = cfdm.CoordinateReference( coordinate_conversion=coordinate_conversion, datum=datum, coordinates=[x, y, lat, lon], ) f.set_construct(msr0, axes=[axisX, axisY]) f.set_construct(ref0) orog = cfdm.DomainAncillary(data=f.get_data()) orog.set_property("standard_name", "surface_altitude") orog.set_property("units", "m") orog = f.set_construct(orog, axes=[axisY, axisX]) coordinate_conversion = cfdm.CoordinateConversion( parameters={ "standard_name": "atmosphere_hybrid_height_coordinate", "computed_standard_name": "altitude", }, domain_ancillaries={ "orog": orog, "a": ak, "b": bk }, ) ref1 = cfdm.CoordinateReference( coordinates=[z], datum=datum, coordinate_conversion=coordinate_conversion, ) ref1 = f.set_construct(ref1) # Field ancillary variables g = f.copy() anc = cfdm.FieldAncillary(data=g.get_data()) anc.set_property("standard_name", "ancillaryA") f.set_construct(anc, axes=[axisY, axisX]) g = f[0] g = g.squeeze() anc = cfdm.FieldAncillary(data=g.get_data()) anc.set_property("long_name", "ancillaryB") f.set_construct(anc, axes=axisX) g = f[..., 0] g = g.squeeze() anc = cfdm.FieldAncillary(data=g.get_data()) anc.set_property("foo", "bar") f.set_construct(anc, axes=[axisY]) f.set_property("flag_values", numpy.array([1, 2, 4], "int32")) f.set_property("flag_meanings", "a bb ccc") f.set_property("flag_masks", [2, 1, 0]) cm0 = cfdm.CellMethod( axes=axisX, method="mean", qualifiers={ "interval": [cfdm.Data(1, "day")], "comment": "ok" }, ) cm1 = cfdm.CellMethod(axes=[axisY], method="maximum", qualifiers={"where": "sea"}) f.set_construct(cm0) f.set_construct(cm1) self.assertTrue(f.equals(f, verbose=verbose), "Field f not equal to itself") self.assertTrue( f.equals(f.copy(), verbose=verbose), "Field f not equal to a copy of itself", ) cfdm.write(f, self.filename, fmt="NETCDF3_CLASSIC", verbose=verbose) g = cfdm.read(self.filename, verbose=1) array = (g[0].constructs.filter_by_identity( "long_name=greek_letters").value().data.array) self.assertEqual(array[1], b"beta") self.assertEqual(len(g), 1) g = g[0].squeeze() self.assertEqual( sorted(f.constructs), sorted(g.constructs), "\n\nf (created in memory)\n{}\n\n{}\n\ng " "(read from disk)\n{}\n\n{}".format( sorted(f.constructs), sorted(f.constructs.items()), sorted(g.constructs), sorted(g.constructs.items()), ), ) self.assertTrue( f.equals(f, verbose=verbose), "Field f not equal to itself after having been written to disk", ) self.assertTrue( f.equals(f.copy(), verbose=verbose), "Field f not equal to a copy of itself after having been " "written to disk", ) self.assertTrue( g.equals(g.copy(), verbose=verbose), "Field g not equal to a copy of itself", ) self.assertTrue( g.equals(f, verbose=verbose), "Field (f) not equal to itself read back in (g)", ) x = g.dump(display=False) x = f.dump(display=False) g = cfdm.read( self.filename, verbose=verbose, extra="domain_ancillary", warnings=warnings, )
def test_write_filename_expansion(self): """Test the writing to a file name that requires expansions.""" f = cfdm.example_field(0) filename = os.path.join("$PWD", os.path.basename(tmpfile)) cfdm.write(f, filename)
def test_create_field(self): # Dimension coordinates dim1 = cfdm.DimensionCoordinate(data=cfdm.Data(numpy.arange(10.))) dim1.set_property('standard_name', 'grid_latitude') dim1.set_property('units', 'degrees') data = numpy.arange(9.) + 20 data[-1] = 34 dim0 = cfdm.DimensionCoordinate(data=cfdm.Data(data)) dim0.set_property('standard_name', 'grid_longitude') dim0.set_property('units', 'degrees') dim0.nc_set_variable('x') array = dim0.data.array array = numpy.array([array - 0.5, array + 0.5]).transpose((1, 0)) array[-2, 1] = 30 array[-1, :] = [30, 36] dim0.set_bounds(cfdm.Bounds(data=cfdm.Data(array))) dim2 = cfdm.DimensionCoordinate( data=cfdm.Data([1.5]), bounds=cfdm.Bounds(data=cfdm.Data([[1, 2.]]))) dim2.set_property('standard_name', 'atmosphere_hybrid_height_coordinate') dim2.set_property('computed_standard_name', 'altitude') # Auxiliary coordinates ak = cfdm.DomainAncillary(data=cfdm.Data([10.])) ak.set_property('units', 'm') ak.id = 'atmosphere_hybrid_height_coordinate_ak' ak.set_bounds(cfdm.Bounds(data=cfdm.Data([[5, 15.]]))) bk = cfdm.DomainAncillary(data=cfdm.Data([20.])) bk.id = 'atmosphere_hybrid_height_coordinate_bk' bk.set_bounds(cfdm.Bounds(data=cfdm.Data([[14, 26.]]))) aux2 = cfdm.AuxiliaryCoordinate(data=cfdm.Data( numpy.arange(-45, 45, dtype='int32').reshape(10, 9))) aux2.set_property('units', 'degree_N') aux2.set_property('standard_name', 'latitude') aux3 = cfdm.AuxiliaryCoordinate(data=cfdm.Data( numpy.arange(60, 150, dtype='int32').reshape(9, 10))) aux3.set_property('standard_name', 'longitude') aux3.set_property('units', 'degreeE') array = numpy.ma.array([ 'alpha', 'beta', 'gamma', 'delta', 'epsilon', 'zeta', 'eta', 'theta', 'iota', 'kappa' ], dtype='S') array[0] = numpy.ma.masked aux4 = cfdm.AuxiliaryCoordinate(data=cfdm.Data(array)) aux4.set_property('long_name', 'greek_letters') # Cell measures msr0 = cfdm.CellMeasure( data=cfdm.Data(1 + numpy.arange(90.).reshape(9, 10) * 1234)) msr0.set_measure('area') msr0.set_property('units', 'km2') msr0.nc_set_variable('areacella') # Data data = cfdm.Data(numpy.arange(90.).reshape(10, 9)) properties = {'units': 'm s-1'} f = cfdm.Field(properties=properties) f.set_property('standard_name', 'eastward_wind') da = cfdm.DomainAxis(9) da.nc_set_dimension('grid_longitude') axisX = f.set_construct(da) axisY = f.set_construct(cfdm.DomainAxis(10)) axisZ = f.set_construct(cfdm.DomainAxis(1)) f.set_data(data, axes=[axisY, axisX]) x = f.set_construct(dim0, axes=axisX) y = f.set_construct(dim1, axes=axisY) z = f.set_construct(dim2, axes=[axisZ]) lat = f.set_construct(aux2, axes=[axisY, axisX]) lon = f.set_construct(aux3, axes=[axisX, axisY]) greek = f.set_construct(aux4, axes=[axisY]) ak = f.set_construct(ak, axes=axisZ) bk = f.set_construct(bk, axes=[axisZ]) # Coordinate references coordinate_conversion = cfdm.CoordinateConversion( parameters={ 'grid_mapping_name': 'rotated_latitude_longitude', 'grid_north_pole_latitude': 38.0, 'grid_north_pole_longitude': 190.0 }) datum = cfdm.Datum(parameters={'earth_radius': 6371007}) ref0 = cfdm.CoordinateReference( coordinate_conversion=coordinate_conversion, datum=datum, coordinates=[x, y, lat, lon]) f.set_construct(msr0, axes=[axisX, axisY]) f.set_construct(ref0) orog = cfdm.DomainAncillary(data=f.get_data()) orog.set_property('standard_name', 'surface_altitude') orog.set_property('units', 'm') orog = f.set_construct(orog, axes=[axisY, axisX]) coordinate_conversion = cfdm.CoordinateConversion(parameters={ 'standard_name': 'atmosphere_hybrid_height_coordinate', 'computed_standard_name': 'altitude' }, domain_ancillaries={ 'orog': orog, 'a': ak, 'b': bk }) ref1 = cfdm.CoordinateReference( coordinates=[z], datum=datum, coordinate_conversion=coordinate_conversion) ref1 = f.set_construct(ref1) # Field ancillary variables g = f.copy() anc = cfdm.FieldAncillary(data=g.get_data()) anc.set_property('standard_name', 'ancillaryA') f.set_construct(anc, axes=[axisY, axisX]) g = f[0] g = g.squeeze() anc = cfdm.FieldAncillary(data=g.get_data()) anc.set_property('long_name', 'ancillaryB') f.set_construct(anc, axes=axisX) g = f[..., 0] g = g.squeeze() anc = cfdm.FieldAncillary(data=g.get_data()) anc.set_property('foo', 'bar') f.set_construct(anc, axes=[axisY]) f.set_property('flag_values', numpy.array([1, 2, 4], 'int32')) f.set_property('flag_meanings', 'a bb ccc') f.set_property('flag_masks', [2, 1, 0]) cm0 = cfdm.CellMethod(axes=axisX, method='mean', qualifiers={ 'interval': [cfdm.Data(1, 'day')], 'comment': 'ok' }) cm1 = cfdm.CellMethod(axes=[axisY], method='maximum', qualifiers={'where': 'sea'}) f.set_construct(cm0) f.set_construct(cm1) self.assertTrue(f.equals(f, verbose=verbose), "Field f not equal to itself") self.assertTrue(f.equals(f.copy(), verbose=verbose), "Field f not equal to a copy of itself") cfdm.write(f, self.filename, fmt='NETCDF3_CLASSIC', verbose=verbose) g = cfdm.read(self.filename, verbose=1) array = g[0].constructs.filter_by_identity( 'long_name=greek_letters').value().data.array self.assertEqual(array[1], b'beta', 'greek_letters = {!r}'.format(array)) self.assertEqual( len(g), 1, 'Read produced the wrong number of fields: {} != 1'.format(len(g))) g = g[0].squeeze() self.assertEqual( sorted(f.constructs), sorted(g.constructs), '\n\nf (created in memory)\n{}\n\n{}\n\ng ' '(read from disk)\n{}\n\n{}'.format(sorted(f.constructs), sorted(f.constructs.items()), sorted(g.constructs), sorted(g.constructs.items()))) self.assertTrue( f.equals(f, verbose=verbose), "Field f not equal to itself after having been written to disk") self.assertTrue( f.equals(f.copy(), verbose=verbose), "Field f not equal to a copy of itself after having been " "written to disk") self.assertTrue(g.equals(g.copy(), verbose=verbose), "Field g not equal to a copy of itself") self.assertTrue(g.equals(f, verbose=verbose), "Field (f) not equal to itself read back in (g)") x = g.dump(display=False) x = f.dump(display=False) g = cfdm.read(self.filename, verbose=verbose, extra='domain_ancillary', warnings=warnings)
def test_STRING(self): if self.test_only and inspect.stack()[0][3] not in self.test_only: return for array in (numpy.ma.array(list('abcdefghij'), dtype='S'), numpy.ma.array([ 'a', 'b1', 'c12', 'd123', 'e1234', 'f', 'g', 'h', 'i', 'j' ], dtype='S')): # Initialize the field tas = cfdm.Field( properties={ 'project': 'research', 'standard_name': 'air_temperature', 'units': 'K' }) # Create and set domain axes axis_T = tas.set_construct(cfdm.DomainAxis(1)) axis_Z = tas.set_construct(cfdm.DomainAxis(1)) axis_Y = tas.set_construct(cfdm.DomainAxis(10)) axis_X = tas.set_construct(cfdm.DomainAxis(9)) # Set the field data tas.set_data(cfdm.Data(numpy.arange(90.).reshape(10, 9)), axes=[axis_Y, axis_X]) # Create and set the dimension coordinates dimension_coordinate_Y = cfdm.DimensionCoordinate( properties={ 'standard_name': 'grid_latitude', 'units': 'degrees' }, data=cfdm.Data(numpy.arange(10.)), bounds=cfdm.Bounds( data=cfdm.Data(numpy.arange(20).reshape(10, 2)))) dimension_coordinate_X = cfdm.DimensionCoordinate( properties={ 'standard_name': 'grid_longitude', 'units': 'degrees' }, data=cfdm.Data(numpy.arange(9.)), bounds=cfdm.Bounds( data=cfdm.Data(numpy.arange(18).reshape(9, 2)))) dim_Y = tas.set_construct(dimension_coordinate_Y, axes=[axis_Y]) dim_X = tas.set_construct(dimension_coordinate_X, axes=[axis_X]) # Create and set the auxiliary coordinates array[0] = numpy.ma.masked aux0 = cfdm.AuxiliaryCoordinate( properties={'long_name': 'Grid latitude name'}, data=cfdm.Data(array)) tas.set_construct(aux0, axes=[axis_Y]) cfdm.write(tas, tempfile) tas1 = cfdm.read(tempfile)[0] aux1 = tas1.constructs.filter_by_identity( 'long_name=Grid latitude name').value() self.assertEqual(aux0.data.shape, array.shape, aux0.data.shape) self.assertEqual(aux1.data.shape, array.shape, aux1.data.shape)