def test_system_subset(self):
"""Test data may be subsetted and that coordinate transformations return the same value arrays."""
ops = OcgOperations(dataset=self.rd, output_format=constants.OutputFormatName.OCGIS, snippet=True,
geom='world_countries', select_ugid=[69])
ret = ops.execute()
# Assert some of the geometry values are masked
actual = ret.get_element().grid.get_mask()
self.assertTrue(actual.any())
# Perform the operations but change the output coordinate system. The value arrays should be equivalent
# regardless of coordinate transformation.
ops2 = OcgOperations(dataset=self.rd, output_format=constants.OutputFormatName.OCGIS, snippet=True,
geom='world_countries', select_ugid=[69], output_crs=Spherical())
ret2 = ops2.execute()
# Value arrays should be the same
ret_value = ret.get_element(variable_name='pr').get_value()
ret2_value = ret2.get_element(variable_name='pr').get_value()
self.assertNumpyAll(ret_value, ret2_value)
# Grid coordinates should not be the same.
ret_grid_value = ret.get_element().grid.get_value_stacked()
ret2_grid_value = ret2.get_element().grid.get_value_stacked()
diff = np.abs(ret_grid_value - ret2_grid_value)
select = diff > 1
self.assertTrue(select.all())
def test_get_base_request_size(self):
rd = self.test_data.get_rd('cancm4_tas')
ops = OcgOperations(dataset=rd)
size = ops.get_base_request_size()
self.assertAlmostEqual(size['total'], 116890.046875)
self.assertAsSetEqual(list(size['field']['tas'].keys()), list(rd.get().keys()))
with self.assertRaises(DefinitionValidationError):
OcgOperations(dataset=rd, regrid_destination=rd).get_base_request_size()
def test_get_meta(self):
ops = OcgOperations(dataset=self.datasets)
meta = ops.get_meta()
self.assertTrue(len(meta) > 100)
self.assertTrue('\n' in meta)
ops = OcgOperations(dataset=self.datasets, calc=[{'func': 'mean', 'name': 'my_mean'}],
calc_grouping=['month'])
meta = ops.get_meta()
self.assertTrue(len(meta) > 100)
self.assertTrue('\n' in meta)
def test_validate(self):
# snippets should be allowed for field objects
field = self.test_data.get_rd('cancm4_tas').get()
ops = OcgOperations(dataset=field, snippet=True)
self.assertTrue(ops.snippet)
# test driver validation is called appropriately
path = GeomCabinet().get_shp_path('state_boundaries')
rd = RequestDataset(path)
with self.assertRaises(DefinitionValidationError):
OcgOperations(dataset=rd, output_format='csv')
def test_calculate_operations(self):
rd = self.test_data.get_rd('cancm4_tas')
slc = [None, None, None, [0, 10], [0, 10]]
calc_icclim = [{'func': 'icclim_TG', 'name': 'TG'}]
calc_ocgis = [{'func': 'mean', 'name': 'mean'}]
_calc_grouping = [['month'], ['month', 'year']]
for cg in _calc_grouping:
ops_ocgis = OcgOperations(calc=calc_ocgis, calc_grouping=cg, slice=slc, dataset=rd)
ret_ocgis = ops_ocgis.execute()
ops_icclim = OcgOperations(calc=calc_icclim, calc_grouping=cg, slice=slc, dataset=rd)
ret_icclim = ops_icclim.execute()
desired = ret_ocgis.get_element(variable_name='mean').get_masked_value()
actual = ret_icclim.get_element(variable_name='TG').get_masked_value()
self.assertNumpyAll(desired, actual)
def test_system_dataset_as_field_from_file(self):
"""Test with dataset argument coming in as a field as opposed to a request dataset collection."""
rd = self.test_data.get_rd('cancm4_tas')
geom = 'state_boundaries'
select_ugid = [23]
field = rd.get()
ops = OcgOperations(dataset=field, snippet=True, geom=geom, select_ugid=select_ugid)
ret = ops.execute()
field_out_from_field = ret.get_element(container_ugid=23)
self.assertEqual(field_out_from_field.data_variables[0].shape, (1, 3, 3))
ops = OcgOperations(dataset=rd, snippet=True, geom=geom, select_ugid=select_ugid)
ret = ops.execute()
field_out_from_rd = ret.get_element(container_ugid=23)
self.assertNumpyAll(field_out_from_field['tas'].get_value(), field_out_from_rd['tas'].get_value())
def test_seasonal_calc_grouping(self):
"""Test seasonal calculation grouping with an ICCLIM function."""
rd = self.test_data.get_rd('cancm4_tas')
slc = [None, [0, 600], None, [0, 10], [0, 10]]
calc_icclim = [{'func': 'icclim_TG', 'name': 'TG'}]
calc_ocgis = [{'func': 'mean', 'name': 'mean'}]
cg = [[12, 1, 2], 'unique']
ops_ocgis = OcgOperations(calc=calc_ocgis, calc_grouping=cg, slice=slc, dataset=rd)
ret_ocgis = ops_ocgis.execute()
ops_icclim = OcgOperations(calc=calc_icclim, calc_grouping=cg, slice=slc, dataset=rd)
ret_icclim = ops_icclim.execute()
desired = ret_ocgis.get_element(variable_name='mean').get_masked_value()
actual = ret_icclim.get_element(variable_name='TG').get_masked_value()
self.assertNumpyAll(desired, actual)
def test_keyword_spatial_operations_bounding_box(self):
geom = [-80, 22.5, 50, 70.0]
rd = self.test_data.get_rd('subset_test_slp')
ops = OcgOperations(dataset=rd, geom=geom)
ret = ops.execute()
field = ret.get_element()
self.assertEqual(field.data_variables[0].shape, (365, 20, 144))
def test_keyword_calc_grouping_none_date_parts(self):
_cg = [None, ['day', 'month'], 'day']
for cg in _cg:
if cg is not None:
eq = tuple(cg)
else:
eq = cg
obj = definition.CalcGrouping(cg)
try:
self.assertEqual(obj.value, eq)
except AssertionError:
self.assertEqual(obj.value, ('day',))
# # only month, year, and day combinations are currently supported
rd = self.test_data.get_rd('cancm4_tas')
calcs = [None, [{'func': 'mean', 'name': 'mean'}]]
acceptable = ['day', 'month', 'year']
for calc in calcs:
for length in [1, 2, 3, 4, 5]:
for combo in itertools.combinations(['day', 'month', 'year', 'hour', 'minute'], length):
try:
OcgOperations(dataset=rd, calc=calc, calc_grouping=combo)
except DefinitionValidationError:
reraise = True
for c in combo:
if c not in acceptable:
reraise = False
if reraise:
raise
def test_get_base_request_size_time_range(self):
path = self.get_temporary_file_path('foo.nc')
field = self.get_field(ntime=60)
time_range = [field.time.get_value()[0], field.time.get_value()[1]]
field.write(path)
rd = RequestDataset(path)
ops = OcgOperations(dataset=rd)
size_no_tr = ops.get_base_request_size()
time_size_no_tr = size_no_tr['field']['foo']['time']['kb']
ops = OcgOperations(dataset=rd, time_range=time_range)
size_with_tr = ops.get_base_request_size()
time_size_with_tr = size_with_tr['field']['foo']['time']['kb']
self.assertAlmostEqual(time_size_no_tr - time_size_with_tr, 0.453125)
def test_calculation_operations_to_nc(self):
rd = self.test_data.get_rd('cancm4_tasmax_2011')
slc = [None, None, None, [0, 10], [0, 10]]
ops_ocgis = OcgOperations(calc=[{'func': 'icclim_SU', 'name': 'SU'}], calc_grouping=['month'], slice=slc,
dataset=rd, output_format='nc')
ret = ops_ocgis.execute()
with nc_scope(ret) as ds:
to_test = deepcopy(ds.__dict__)
history = to_test.pop('history')
self.assertEqual(history[111:187],
' Calculation of SU indice (monthly climatology) from 2011-1-1 to 2020-12-31.')
actual = OrderedDict([('source_data_global_attributes',
'{"institution": "CCCma (Canadian Centre for Climate Modelling and Analysis, Victoria, BC, Canada)", "institute_id": "CCCma", "experiment_id": "decadal2010", "source": "CanCM4 2010 atmosphere: CanAM4 (AGCM15i, T63L35) ocean: CanOM4 (OGCM4.0, 256x192L40) sea ice: CanSIM1 (Cavitating Fluid, T63 Gaussian Grid) land: CLASS2.7", "model_id": "CanCM4", "forcing": "GHG,Oz,SA,BC,OC,LU,Sl,Vl (GHG includes CO2,CH4,N2O,CFC11,effective CFC12)", "parent_experiment_id": "N/A", "parent_experiment_rip": "N/A", "branch_time": 0.0, "contact": "*****@*****.**", "references": "http://www.cccma.ec.gc.ca/models", "initialization_method": 1, "physics_version": 1, "tracking_id": "64384802-3f0f-4ab4-b569-697bd5430854", "branch_time_YMDH": "2011:01:01:00", "CCCma_runid": "DHFP1B_E002_I2011_M01", "CCCma_parent_runid": "DHFP1_E002", "CCCma_data_licence": "1) GRANT OF LICENCE - The Government of Canada (Environment Canada) is the \\nowner of all intellectual property rights (including copyright) that may exist in this Data \\nproduct. You (as \\"The Licensee\\") are hereby granted a non-exclusive, non-assignable, \\nnon-transferable unrestricted licence to use this data product for any purpose including \\nthe right to share these data with others and to make value-added and derivative \\nproducts from it. This licence is not a sale of any or all of the owner\'s rights.\\n2) NO WARRANTY - This Data product is provided \\"as-is\\"; it has not been designed or \\nprepared to meet the Licensee\'s particular requirements. Environment Canada makes no \\nwarranty, either express or implied, including but not limited to, warranties of \\nmerchantability and fitness for a particular purpose. In no event will Environment Canada \\nbe liable for any indirect, special, consequential or other damages attributed to the \\nLicensee\'s use of the Data product.", "product": "output", "experiment": "10- or 30-year run initialized in year 2010", "frequency": "day", "creation_date": "2012-03-28T15:32:08Z", "history": "2012-03-28T15:32:08Z CMOR rewrote data to comply with CF standards and CMIP5 requirements.", "Conventions": "CF-1.4", "project_id": "CMIP5", "table_id": "Table day (28 March 2011) f9d6cfec5981bb8be1801b35a81002f0", "title": "CanCM4 model output prepared for CMIP5 10- or 30-year run initialized in year 2010", "parent_experiment": "N/A", "modeling_realm": "atmos", "realization": 2, "cmor_version": "2.8.0"}'),
('title', 'ECA heat indice SU'), (
'references',
'ATBD of the ECA indices calculation (http://eca.knmi.nl/documents/atbd.pdf)'),
('institution', 'Climate impact portal (http://climate4impact.eu)'),
('comment', ' ')])
self.assertDictEqual(to_test, actual)
var = ds.variables['SU']
to_test = dict(var.__dict__)
to_test.pop('_FillValue', None)
self.assertEqual(to_test, {'units': 'days',
'standard_name': AbstractIcclimFunction.standard_name,
'long_name': 'Summer days (number of days where daily maximum temperature > 25 degrees)',
'grid_mapping': 'latitude_longitude'})
def test(self):
raise SkipTest('benchmarking only')
# development laptop: 8 procs: 139 seconds
ocgis.env.VERBOSE = True
uri = '/home/benkoziol/l/data/bekozi-work/lisa-rensi-nwm/nwm.t00z.analysis_assim.terrain_rt.tm00.conus.nc_georeferenced.nc'
# dimension_map = {'time': {'variable': 'time', DimensionMapKey.DIMS: ['time']},
# 'x': {'variable': 'x', DimensionMapKey.DIMS: ['x']},
# 'y': {'variable': 'y', DimensionMapKey.DIMS: ['y']},
# 'crs': {'variable': 'ProjectionCoordinateSystem'}}
dimension_map = {
'time': {
'variable': 'time'
},
'x': {
'variable': 'x'
},
'y': {
'variable': 'y'
},
'crs': {
'variable': 'ProjectionCoordinateSystem'
}
}
rd = RequestDataset(uri, dimension_map=dimension_map)
# barrier_print(rd.dist.get_dimension('x').bounds_local)
# barrier_print(rd.dist.get_dimension('y').bounds_local)
# tkk
# field = rd.get()
# None
ops = OcgOperations(dataset=rd,
geom='state_boundaries',
geom_select_uid=[16])
ret = ops.execute()
def test_bad_time_dimension(self):
"""Test not formatting the time dimension."""
for output_format in [constants.OutputFormatName.OCGIS, constants.OutputFormatName.CSV,
constants.OutputFormatName.CSV_SHAPEFILE, constants.OutputFormatName.SHAPEFILE,
constants.OutputFormatName.NETCDF]:
dataset = self.test_data.get_rd('snippet_seasonalbias')
ops = OcgOperations(dataset=dataset, output_format=output_format, format_time=False, prefix=output_format)
ret = ops.execute()
if output_format == constants.OutputFormatName.OCGIS:
actual = ret.get_element()
self.assertFalse(actual.temporal.format_time)
self.assertNumpyAll(actual.temporal.value_numtime.data,
np.array([-712208.5, -712117., -712025., -711933.5]))
self.assertNumpyAll(actual.temporal.bounds.value_numtime.data,
np.array([[-712254., -712163.], [-712163., -712071.], [-712071., -711979.],
[-711979., -711888.]]))
if output_format == constants.OutputFormatName.CSV:
with open(ret) as f:
reader = DictReader(f)
for row in reader:
self.assertTrue(all([row[k] == '' for k in ['YEAR', 'MONTH', 'DAY']]))
self.assertTrue(float(row['TIME']) < -50000)
if output_format == constants.OutputFormatName.NETCDF:
self.assertNcEqual(ret, dataset.uri, check_types=False,
ignore_attributes={'global': ['history'], 'bounds_time': ['calendar', 'units'],
'bias': ['_FillValue', 'grid_mapping', 'units'],
'latitude': ['standard_name', 'units'],
'longitude': ['standard_name', 'units']},
ignore_variables=['latitude_longitude'])
def test_system_process_geometries(self):
"""Test multiple geometries with coordinate system update."""
a = 'POLYGON((-105.21347987288135073 40.21514830508475313,-104.39928495762711691 40.21514830508475313,-104.3192002118643984 39.5677966101694949,-102.37047139830508513 39.61451271186440692,-102.12354343220337682 37.51896186440677639,-105.16009004237288593 37.51896186440677639,-105.21347987288135073 40.21514830508475313))'
b = 'POLYGON((-104.15235699152542281 39.02722457627118757,-103.71189088983049942 39.44099576271186436,-102.71750529661017026 39.28082627118644155,-102.35712394067796538 37.63908898305084705,-104.13900953389830306 37.63241525423728717,-104.15235699152542281 39.02722457627118757))'
geom = [{'geom': wkt.loads(xx), 'properties': {'UGID': ugid}} for ugid, xx in enumerate([a, b])]
grid_value = [
[[37.0, 37.0, 37.0, 37.0], [38.0, 38.0, 38.0, 38.0], [39.0, 39.0, 39.0, 39.0], [40.0, 40.0, 40.0, 40.0]],
[[-105.0, -104.0, -103.0, -102.0], [-105.0, -104.0, -103.0, -102.0], [-105.0, -104.0, -103.0, -102.0],
[-105.0, -104.0, -103.0, -102.0]]]
output_crs = CoordinateReferenceSystem(
value={'a': 6370997, 'lon_0': -100, 'y_0': 0, 'no_defs': True, 'proj': 'laea', 'x_0': 0, 'units': 'm',
'b': 6370997, 'lat_0': 45})
x = Variable('x', grid_value[1], dimensions=['lat', 'lon'])
y = Variable('y', grid_value[0], dimensions=['lat', 'lon'])
grid = Grid(x, y)
field = Field(grid=grid, crs=Spherical())
ops = OcgOperations(dataset=field, geom=geom, output_crs=output_crs)
ret = ops.execute()
expected = {0: -502052.79407259845,
1: -510391.37909706926}
for field, container in ret.iter_fields(yield_container=True):
self.assertAlmostEqual(field.grid.get_value_stacked().mean(),
expected[container.geom.ugid.get_value()[0]])
def test_system_regridding_crs(self):
"""Test with coordinate systems."""
dest_crs = WGS84()
grid_spherical = self.get_gridxy_global(resolution=10.0, wrapped=False, crs=Spherical())
self.assertEqual(grid_spherical.crs, Spherical())
coords = grid_spherical.get_value_stacked()
data_value = self.get_exact_field_value(coords[1], coords[0])
desired = data_value.copy()
data_var = Variable(name='data_src', value=data_value, dimensions=grid_spherical.dimensions)
source = Field(grid=grid_spherical, is_data=data_var, crs=grid_spherical.crs)
self.assertEqual(source.crs, Spherical())
destination = deepcopy(source)
destination.update_crs(dest_crs)
source_expanded = deepcopy(source.grid)
source_expanded.expand()
diff = np.abs(destination.y.get_value() - source_expanded.y.get_value())
self.assertAlmostEqual(diff.max(), 0.19231511439)
for output_crs in [None, WGS84()]:
ops = OcgOperations(dataset=source, regrid_destination=destination, output_crs=output_crs)
ret = ops.execute()
actual = ret.get_element(variable_name=data_var.name)
if output_crs is None:
self.assertEqual(actual.parent.crs, Spherical())
else:
self.assertEqual(actual.parent.crs, WGS84())
actual = actual.get_value()
diff = np.abs(actual - desired)
self.assertTrue(diff.max() < 1e-5)
def test_mfdataset_to_nc(self):
rd = self.test_data.get_rd('maurer_2010_pr')
ops = OcgOperations(dataset=rd, output_format='nc', calc=[{'func': 'mean', 'name': 'my_mean'}],
calc_grouping=['year'], geom='state_boundaries', select_ugid=[23])
ret = ops.execute()
field = RequestDataset(ret, 'my_mean').get()
self.assertNumpyAll(field.temporal.get_value(), np.array([18444., 18809.]))
def test_system_spatial_averaging_through_operations(self):
data_name = 'data'
with vm.scoped('write', [0]):
if not vm.is_null:
x = Variable('x', range(5), 'x', float)
y = Variable('y', range(7), 'y', float)
grid = Grid(x, y)
data_value = np.arange(x.size * y.size).reshape(grid.shape)
data = Variable(data_name, data_value, grid.dimensions, float)
data_value = data.get_value()
field = Field(grid=grid, is_data=data)
path = self.get_temporary_file_path('data.nc')
field.write(path)
else:
data_value, path = None, None
data_value = MPI_COMM.bcast(data_value)
path = MPI_COMM.bcast(path)
rd = RequestDataset(path, variable=data_name)
ops = OcgOperations(dataset=rd, aggregate=True)
ret = ops.execute()
if ret is None:
self.assertNotEqual(vm.rank, vm.root)
else:
out_field = ret.get_element()
if MPI_RANK == 0:
desired = data_value.mean()
actual = out_field.data_variables[0].get_value()[0]
self.assertEqual(actual, desired)
def test_system_multiple_netcdf_files(self):
"""Test subsetting multiple netCDF files and returning a spatial collection."""
grid = create_gridxy_global(resolution=3.0)
vars = ['ocgis_example_tasmin', 'ocgis_example_tas', 'ocgis_example_tasmax']
paths = [self.get_temporary_file_path('{}.nc'.format(ii)) for ii in vars]
geom_select_uid = [16, 23]
field_names = ['tasmin', 'tas', 'tasmax']
for ctr, (path, var) in enumerate(zip(paths, vars), start=1):
field = create_exact_field(grid.copy(), var, ntime=3)
field.data_variables[0].get_value()[:] = 10 * ctr
field.write(path)
rds = [RequestDataset(uri=uri, variable=var, field_name=field_name) for uri, var, field_name in
zip(paths, vars, field_names)]
ops = OcgOperations(dataset=rds, spatial_operation='clip', aggregate=True, geom=self.path_state_boundaries,
geom_select_uid=geom_select_uid)
ret = ops.execute()
self.assertAsSetEqual(ret.children.keys(), geom_select_uid)
for geom_uid in geom_select_uid:
actual = ret.children[geom_uid].children.keys()
self.assertAsSetEqual(actual, field_names)
for idx, field_name in enumerate(field_names):
actual = ret.get_element(container_ugid=geom_uid, field_name=field_names[idx], variable_name=vars[idx])
actual = actual.get_value()
actual = actual == (idx + 1) * 10
self.assertTrue(np.all(actual))
def test_system_dataset_identifiers_on_variables(self):
"""Test dataset identifiers make it to output variables for iteration."""
paths = []
variables = []
for suffix in [1, 2]:
path = self.get_temporary_file_path('foo{}.nc'.format(suffix))
paths.append(path)
x = Variable(name='x{}'.format(suffix), value=[2, 3], dimensions='x')
y = Variable(name='y{}'.format(suffix), value=[4, 5, 6], dimensions='y')
data_variable_name = 'data{}'.format(suffix)
variables.append(data_variable_name)
data = Variable(name=data_variable_name, value=np.arange(6).reshape(2, 3) + suffix,
dimensions=['x', 'y'])
grid = Grid(x, y)
field = Field(grid=grid, is_data=data)
field.write(path)
rds = [RequestDataset(uri=p, variable=dv) for p, dv in zip(paths, variables)]
ops = OcgOperations(dataset=rds)
rds_uids = [ds.uid for ds in ops.dataset]
self.assertEqual(rds_uids, [1, 2])
ret = ops.execute()
for field in ret.iter_fields():
self.assertFalse(field.grid.has_allocated_abstraction_geometry)
for variable in list(field.values()):
if isinstance(variable, CoordinateReferenceSystem):
continue
self.assertIsNotNone(variable._request_dataset.uid)
for row in variable.get_iter():
self.assertIsNotNone(row[HeaderName.DATASET_IDENTIFER])
def test_clip_aggregate(self):
# This geometry was hanging.
rd = self.test_data.get_rd('cancm4_tas', kwds={'time_region': {'year': [2003]}})
field = rd.get()
ops = OcgOperations(dataset=rd, geom='state_boundaries', select_ugid=[14, 16],
aggregate=False, spatial_operation='clip',
output_format=constants.OutputFormatName.CSV_SHAPEFILE)
ops.execute()
def test_keyword_calc_grouping_seasonal_with_year(self):
calc_grouping = [[1, 2, 3], 'year']
calc = [{'func': 'mean', 'name': 'mean'}]
rd = self.test_data.get_rd('cancm4_tas')
ops = OcgOperations(dataset=rd, calc=calc, calc_grouping=calc_grouping,
geom='state_boundaries', select_ugid=[25])
ret = ops.execute()
self.assertEqual(ret.get_element(variable_name='mean').shape, (10, 4, 4))
def test_keyword_calc_grouping_with_string_expression(self):
"""Test that no calculation grouping is allowed with a string expression."""
calc = 'es=tas*3'
calc_grouping = ['month']
rd = self.test_data.get_rd('cancm4_tas')
with self.assertRaises(DefinitionValidationError):
OcgOperations(dataset=rd, calc=calc, calc_grouping=calc_grouping)
def test_keyword_time_range_and_time_region_null_parms(self):
ops = OcgOperations(dataset=self.datasets_no_range)
self.assertEqual(ops.geom, None)
self.assertEqual(len(list(ops.dataset)), 3)
for ds in ops.dataset:
self.assertEqual(ds.time_range, None)
self.assertEqual(ds.level_range, None)
ops.__repr__()
def test_system_calculation_variable_naming(self):
"""Test the same data variable name may be used for the calculation."""
field = self.get_field()
calc = [{'func': 'mean', 'name': field.data_variables[0].name}]
ops = OcgOperations(dataset=field, calc=calc, calc_grouping=['month'])
ret = ops.execute()
out_field = ret.get_element()
self.assertIn(field.data_variables[0].name, out_field)
def test_keyword_geom_string(self):
ops = OcgOperations(dataset=self.datasets, geom='state_boundaries')
self.assertEqual(len(list(ops.geom)), 51)
ops.geom = None
self.assertEqual(ops.geom, None)
ops.geom = 'mi_watersheds'
self.assertEqual(len(list(ops.geom)), 60)
ops.geom = [-120, 40, -110, 50]
self.assertEqual(ops.geom[0].geom.get_value()[0].bounds, (-120.0, 40.0, -110.0, 50.0))
def test_get_base_request_size_multifile(self):
rd1 = self.test_data.get_rd('cancm4_tas')
rd2 = self.test_data.get_rd('narccap_pr_wrfg_ncep')
rds = [rd1, rd2]
ops = OcgOperations(dataset=rds)
size = ops.get_base_request_size()
actual = size['total']
desired = 1784714.6640625
self.assertAlmostEqual(actual, desired)
def test_system_field_is_untouched(self):
"""Test field is untouched if passed through operations with nothing happening."""
field = self.get_field()
gid_name = HeaderName.ID_GEOMETRY
self.assertNotIn(gid_name, field)
ops = OcgOperations(dataset=field, output_format=constants.OutputFormatName.OCGIS)
ret = ops.execute()
actual = ret.get_element()
self.assertEqual(list(field.keys()), list(actual.keys()))
def test_system_user_geometry_identifier_typed_appropriately(self):
"""Test UGID is typed appropriately according to the data model."""
ofo = {'data_model': 'NETCDF3_64BIT_OFFSET'}
grid = create_gridxy_global(resolution=3.0)
field = create_exact_field(grid, 'foo', crs=Spherical())
ops = OcgOperations(dataset=field, output_format_options=ofo, geom=[-100, 30, -90, 40], aggregate=True)
actual = ops.execute()
actual = actual[1]
self.assertEqual(actual.geom.ugid.dtype, np.int32)
def test_system_date_groups_all(self):
calc = [{'func': 'mean', 'name': 'mean'}]
rd = self.test_data.get_rd('cancm4_tasmax_2011')
ops = OcgOperations(dataset=rd, calc=calc, calc_grouping='all', geom='state_boundaries', select_ugid=[25])
ret_calc = ops.execute()
ops = OcgOperations(dataset=rd, geom='state_boundaries', select_ugid=[25])
ret_no_calc = ops.execute()
field = ret_calc.get_element(container_ugid=25, field_name='tasmax')
variable = field['mean']
parents = ret_no_calc.get_element(container_ugid=25, field_name='tasmax')
self.assertEqual(parents['tasmax'].shape, (3650, 4, 4))
self.assertEqual(variable.shape, (1, 4, 4))
desired_value = parents['tasmax'].get_masked_value()
lhs = np.ma.mean(desired_value, axis=0).reshape(1, 4, 4).astype(desired_value.dtype)
# NumPy does not update the fill value type in "astype". Set this manually.
lhs.fill_value = desired_value.fill_value
rhs = variable.get_masked_value()
self.assertNumpyAll(lhs, rhs)
def test_init(self):
with self.assertRaises(DefinitionValidationError):
OcgOperations()
ops = OcgOperations(dataset=self.datasets)
self.assertEqual(ops.regrid_destination, None)
self.assertDictEqual(ops.regrid_options, RegridOptions.default)
self.assertIsNone(ops.geom_select_uid)
self.assertIsNone(ops.geom_uid)
self.assertFalse(ops.melted)
env.MELTED = True
ops = OcgOperations(dataset=self.datasets)
self.assertEqual(ops.melted, env.MELTED)
self.assertTrue(ops.melted)
ops = OcgOperations(dataset=self.datasets, geom_select_uid=[4, 5], select_ugid=[5, 6, 7])
self.assertEqual(ops.geom_select_uid, (4, 5))
ops = OcgOperations(dataset=self.datasets, geom_uid='ID')
self.assertEqual(ops.geom_uid, 'ID')
geom = ops._get_object_('geom')
self.assertEqual(geom.geom_uid, 'ID')
s = "STATE_NAME in ('Wisconsin', 'Vermont')"
ops = OcgOperations(dataset=self.datasets, geom_select_sql_where=s, geom='state_boundaries')
self.assertEqual(ops.geom_select_sql_where, s)
self.assertEqual(len(ops.geom), 2)
