def test_iter_regridded_fields(self): """Test with equivalent input and output expectations. The shapes of the grids are equal.""" ofield = self.get_ofield() ofield.spatial.crs = Spherical() ofield2 = deepcopy(ofield) sources = [ofield, ofield2] destination_field = deepcopy(ofield) keywords = dict(use_sdim=[True, False], split=[False, True]) for k in itr_products_keywords(keywords, as_namedtuple=True): if k.use_sdim: destination = destination_field.spatial sdim = destination else: destination = destination_field sdim = destination.spatial self.assertIsNotNone(sdim.grid.row) self.assertIsNotNone(sdim.grid.col) for ctr, regridded in enumerate( iter_regridded_fields(sources, destination, with_corners="auto", value_mask=None, split=k.split) ): self.assertIsInstance(regridded, Field) self.assertNumpyAll(regridded.spatial.grid.value, sdim.grid.value) self.assertEqual(regridded.spatial.crs, sdim.crs) self.assertNumpyAll(regridded.spatial.grid.row.value, sdim.grid.row.value) self.assertNumpyAll(regridded.spatial.grid.row.bounds, sdim.grid.row.bounds) self.assertNumpyAll(regridded.spatial.grid.col.value, sdim.grid.col.value) self.assertNumpyAll(regridded.spatial.grid.col.bounds, sdim.grid.col.bounds) for variable in regridded.variables.itervalues(): self.assertGreater(variable.value.mean(), 2.0) self.assertNumpyAll(variable.value, sources[ctr].variables[variable.alias].value) self.assertFalse(np.may_share_memory(variable.value, sources[ctr].variables[variable.alias].value)) self.assertEqual(ctr, 1)
def test_get_sdim_from_esmf_grid(self): rd = ocgis.RequestDataset(**self.get_dataset()) keywords = dict( has_corners=[True, False], has_mask=[True, False], crs=[None, CoordinateReferenceSystem(epsg=4326)] ) for k in itr_products_keywords(keywords, as_namedtuple=True): field = rd.get() sdim = field.spatial if not k.has_corners: sdim.grid.row.remove_bounds() sdim.grid.col.remove_bounds() self.assertIsNone(sdim.grid.corners) egrid = get_esmf_grid_from_sdim(sdim) if not k.has_mask: egrid.mask[0][2, 2] = 0 sdim.grid.value.mask[:, 2, 2] = True if k.has_corners: sdim.grid.corners.mask[:, 2, 2] = True nsdim = get_sdim_from_esmf_grid(egrid, crs=k.crs) self.assertEqual(nsdim.crs, k.crs) self.assertNumpyAll(sdim.grid.value, nsdim.grid.value) if k.has_corners: self.assertNumpyAll(sdim.grid.corners, nsdim.grid.corners) else: self.assertIsNone(nsdim.grid.corners)
def test_regrid_field(self): """Test with equivalent input and output expectations. The shapes of the grids are equal.""" source = self.get_ofield() source.set_crs(Spherical()) destination = deepcopy(source) desired = deepcopy(source) keywords = dict(split=[False, True]) for k in itr_products_keywords(keywords, as_namedtuple=True): from ocgis.regrid.base import regrid_field regridded = regrid_field(source, destination, split=k.split) self.assertIsInstance(regridded, Field) self.assertNumpyAll(regridded.grid.get_value_stacked(), desired.grid.get_value_stacked()) self.assertEqual(regridded.crs, source.crs) for variable in regridded.data_variables: self.assertGreater(variable.get_value().mean(), 2.0) self.assertNumpyAll( variable.get_value(), source[variable.name].get_value().squeeze()) self.assertFalse( np.may_share_memory(variable.get_value(), source[variable.name].get_value()))
def test_get_sdim_from_esmf_grid(self): rd = ocgis.RequestDataset(**self.get_dataset()) keywords = dict( has_corners=[True, False], has_mask=[True, False], crs=[None, CoordinateReferenceSystem(epsg=4326)] ) for k in itr_products_keywords(keywords, as_namedtuple=True): field = rd.get() sdim = field.spatial egrid = get_esmf_grid_from_sdim(sdim) if not k.has_mask: # set the grid flag to indicate no mask is present egrid.item_done[ESMF.StaggerLoc.CENTER][0] = False # remove the mask from the grid egrid.mask[0] = [1] else: egrid.mask[0][2, 2] = 0 sdim.grid.value.mask[:, 2, 2] = True sdim.grid.corners.mask[:, 2, 2] = True if not k.has_corners: egrid.coords[ESMF.StaggerLoc.CORNER] = [np.array(0.0), np.array(0.0)] egrid.coords_done[ESMF.StaggerLoc.CORNER] = [False, False] nsdim = get_sdim_from_esmf_grid(egrid, crs=k.crs) self.assertEqual(nsdim.crs, k.crs) self.assertNumpyAll(sdim.grid.value, nsdim.grid.value) if k.has_corners: self.assertNumpyAll(sdim.grid.corners, nsdim.grid.corners) else: self.assertIsNone(nsdim.grid.corners)
def test_init(self): keywords = dict(value=[None, {'no_defs': True, 'ellps': 'WGS84', 'proj': 'longlat', 'towgs84': '0,0,0,0,0,0,0'}], epsg=[None, 4326], proj4=[None, '+proj=longlat +ellps=WGS84 +towgs84=0,0,0,0,0,0,0 +no_defs ']) prev_crs = None for k in itr_products_keywords(keywords): try: crs = CoordinateReferenceSystem(**k) except ValueError: if all([ii is None for ii in k.values()]): continue else: raise self.assertEqual(crs.proj4, '+proj=longlat +ellps=WGS84 +towgs84=0,0,0,0,0,0,0 +no_defs ') self.assertDictEqual(crs.value, {'no_defs': True, 'ellps': 'WGS84', 'proj': 'longlat', 'towgs84': '0,0,0,0,0,0,0'}) if prev_crs is not None: self.assertEqual(crs, prev_crs) prev_crs = deepcopy(crs) # test with a name parameter crs = CoordinateReferenceSystem(epsg=4326) self.assertEqual(crs.name, constants.DEFAULT_COORDINATE_SYSTEM_NAME) crs = CoordinateReferenceSystem(epsg=4326, name='foo') self.assertEqual(crs.name, 'foo') # test using the init parameter value = {'init': 'epsg:4326'} self.assertEqual(CoordinateReferenceSystem(value=value), WGS84())
def test_system_regridding_same_field(self): """Test regridding operations with same field used to regrid the source.""" rd_dest = self.test_data.get_rd('cancm4_tas') keywords = dict(regrid_destination=[rd_dest, rd_dest.get()], geom=['state_boundaries']) select_ugid = [25, 41] for ctr, k in enumerate(itr_products_keywords(keywords, as_namedtuple=True)): rd1 = self.test_data.get_rd('cancm4_tas') rd2 = self.test_data.get_rd('cancm4_tas', kwds={'field_name': 'tas2'}) ops = ocgis.OcgOperations(dataset=[rd1, rd2], geom=k.geom, regrid_destination=k.regrid_destination, time_region={'month': [1], 'year': [2002]}, select_ugid=select_ugid) subset = OperationsEngine(ops) colls = list(subset) self.assertEqual(len(colls), 4) for coll in colls: for d in coll.iter_melted(tag=TagName.DATA_VARIABLES): field = d['field'] self.assertEqual(field.crs, env.DEFAULT_COORDSYS) self.assertTrue(d['variable'].get_value().mean() > 100) self.assertTrue(np.any(field.grid.get_mask())) self.assertTrue(np.any(d['variable'].get_mask()))
def test_iter_kernel_values_asserts(self): """Test assert statements.""" k = [1, 2, 3, 4] values = [ np.array([[2, 3], [4, 5]]), np.arange(0, 13).reshape(-1, 1, 1) ] mode = ['same', 'valid', 'foo'] for kwds in itr_products_keywords( { 'k': k, 'values': values, 'mode': mode }, as_namedtuple=True): try: list(MovingWindow._iter_kernel_values_(kwds.values, kwds.k)) except AssertionError: if kwds.k == 3: if kwds.values.shape == (2, 2): continue else: raise else: continue except NotImplementedError: if kwds.mode == 'foo': continue else: raise
def test_meta_attrs(self): """Test various forms for meta_attrs in the calculation definition dictionary.""" kwds = dict( meta_attr=[None, {}, { 'something_else': 'is_here with us' }], calc=[{ 'func': 'mean', 'name': 'my_mean' }, 'foo=tas+4', { 'func': 'foo=tas+4', 'meta_attrs': { 'something': 'special' } }], add_meta_attrs_if_none=[True, False]) for k in itr_products_keywords(kwds, as_namedtuple=True): k = deepcopy(k) if not k.add_meta_attrs_if_none and k.meta_attr is None: pass else: try: k.calc.update({'meta_attrs': k.meta_attr}) # likely the string representation except AttributeError: if k.calc == 'foo=tas+4': pass else: raise calc = Calc([k.calc]) self.assertEqual( set(calc.value[0].keys()), set(['ref', 'meta_attrs', 'name', 'func', 'kwds']))
def test_init(self): dataset = self.get_dataset() uri = dataset['uri'] variable = dataset['variable'] with nc_scope(uri) as ds: nc_metadata = NcMetadata(ds) keywords = dict( uri=[None, self.get_dataset()['uri']], variable=[None, self.get_dataset()['variable']], request_dataset=[None, RequestDataset(uri=uri, variable=variable)], meta=[None, nc_metadata]) for k in itr_products_keywords(keywords, as_namedtuple=True): try: ip = Inspect(**k._asdict()) except ValueError: if k.uri is None and k.request_dataset is None and k.meta is None: continue else: raise ret = ip.__repr__() search = re.search('URI = (.*)\n', ret).groups()[0] if k.uri is None and k.meta is not None and k.request_dataset is None: self.assertEqual(search, 'None') else: self.assertTrue(os.path.exists(search))
def __iter__(self): keywords = dict(target=self.target, output_crs=self.get_output_crs(), wrap=[None, True, False]) for k in itr_products_keywords(keywords, as_namedtuple=True): kwargs = k._asdict() target = kwargs.pop('target') ss = SpatialSubsetOperation(list(target.values())[0], **kwargs) yield (ss, k)
def iter_keywords(self): rd1 = self.test_data.get_rd('cancm4_tas') rd2 = self.test_data.get_rd('cancm4_rhs') keywords = dict(target=[None, rd1, [rd1], [rd1, rd2], {'uri': rd1.uri, 'variable': rd1.variable}, rd1.get(), [rd1.get(), rd2.get()], [rd1, rd2.get()]]) for k in itr_products_keywords(keywords, as_namedtuple=True): yield k
def __iter__(self): keywords = dict(target=self.target, output_crs=self.get_output_crs(), wrap=[None, True, False]) for k in itr_products_keywords(keywords, as_namedtuple=True): kwargs = k._asdict() target = kwargs.pop('target') ss = SpatialSubsetOperation(target, **kwargs) yield (ss, k)
def test_init(self): rd1 = self.test_data.get_rd('cancm4_tas') rd2 = self.test_data.get_rd('cancm4_rhs') keywords = dict(request_datasets=[None, rd1, [rd1], [rd1, rd2], {'uri': rd1.uri, 'variable': rd1.variable}]) for k in itr_products_keywords(keywords, as_namedtuple=True): rdc = RequestDatasetCollection(request_datasets=k.request_datasets) if k.request_datasets is not None: self.assertEqual(len(rdc), len(list(get_iter(k.request_datasets, dtype=(dict, RequestDataset))))) else: self.assertEqual(len(rdc), 0)
def test_init_conform_units_to(self): """Test using the conform_units_to keyword argument.""" def _get_units_(v): try: v = Units(v) except AttributeError: pass return v value = np.array([1, 2, 3, 4, 5]) value_masked = np.ma.array(value, mask=[False, True, False, True, False]) kwds = dict(units=['celsius', None, 'mm/day'], conform_units_to=['kelvin', Units('kelvin'), None], value=[value, value_masked]) for k in itr_products_keywords(kwds, as_namedtuple=True): try: var = Variable(**k._asdict()) except NoUnitsError: # without units defined on the input array, the values may not be conformed if k.units is None: continue else: raise except ValueError: # units are not convertible if _get_units_(k.units) == _get_units_('mm/day') and _get_units_(k.conform_units_to) == _get_units_( 'kelvin'): continue else: raise if k.conform_units_to is not None: try: self.assertEqual(var.conform_units_to, Units(k.conform_units_to)) # may already be a Units object except AttributeError: self.assertEqual(var.conform_units_to, k.conform_units_to) else: self.assertIsNone(var.conform_units_to) if k.conform_units_to is not None: actual = [274.15, 275.15, 276.15, 277.15, 278.15] if isinstance(k.value, MaskedArray): mask = value_masked.mask else: mask = False actual = np.ma.array(actual, mask=mask, fill_value=var.value.fill_value) self.assertNumpyAll(actual, var.value)
def test_regridding_same_field(self): """Test regridding operations with same field used to regrid the source.""" # todo: what happens with multivariate calculations # todo: test with all masked values rd_dest = self.test_data.get_rd('cancm4_tas') keywords = dict(regrid_destination=[rd_dest, rd_dest.get().spatial, rd_dest.get()], geom=['state_boundaries']) select_ugid = [25, 41] for ctr, k in enumerate(itr_products_keywords(keywords, as_namedtuple=True)): rd1 = self.test_data.get_rd('cancm4_tas') rd2 = self.test_data.get_rd('cancm4_tas', kwds={'alias': 'tas2'}) # print ctr # if ctr != 1: continue # if the target is a spatial dimension, change the crs to spherical. otherwise, the program will attempt to # convert from wgs84 to spherical if isinstance(k.regrid_destination, SpatialDimension): # the spatial dimension's crs needs to be updated specifically otherwise it will assume the data is # wgs84 and attempt updating. the request datasets must also be assigned spherical to ensure the program # so the subsetting comes out okay. k = deepcopy(k) k.regrid_destination.crs = Spherical() rd1 = self.test_data.get_rd('cancm4_tas', kwds={'crs': Spherical()}) rd2 = self.test_data.get_rd('cancm4_tas', kwds={'alias': 'tas2', 'crs': Spherical()}) ops = ocgis.OcgOperations(dataset=[rd1, rd2], geom=k.geom, regrid_destination=k.regrid_destination, time_region={'month': [1], 'year': [2002]}, select_ugid=select_ugid) subset = SubsetOperation(ops) colls = list(subset) self.assertEqual(len(colls), 4) for coll in colls: for d in coll.get_iter_melted(): field = d['field'] if isinstance(k.regrid_destination, SpatialDimension): self.assertEqual(field.spatial.crs, Spherical()) else: self.assertEqual(field.spatial.crs, env.DEFAULT_COORDSYS) self.assertTrue(d['variable'].value.mean() > 100) self.assertTrue(np.any(field.spatial.get_mask())) self.assertTrue(np.any(d['variable'].value.mask)) for to_check in [field.spatial.grid.row.bounds, field.spatial.grid.col.bounds, field.spatial.grid.corners, field.spatial.geom.polygon.value]: self.assertIsNotNone(to_check)
def test_execute_directory(self): """Test that the output directory is removed appropriately following an operations failure.""" kwds = dict(add_auxiliary_files=[True, False]) rd = self.test_data.get_rd('cancm4_tas') ## this geometry is outside the domain and will result in an exception geom = [1000, 1000, 1100, 1100] for k in itr_products_keywords(kwds, as_namedtuple=True): ops = OcgOperations(dataset=rd, output_format='csv', add_auxiliary_files=k.add_auxiliary_files, geom=geom) try: ret = ops.execute() except ExtentError: contents = os.listdir(self._test_dir) self.assertEqual(len(contents), 0)
def test_get_ocgis_grid_from_esmf_grid(self): from ocgis.regrid.base import get_esmf_grid from ocgis.regrid.base import get_ocgis_grid_from_esmf_grid rd = RequestDataset(**self.get_dataset()) keywords = dict(has_corners=[True, False], has_mask=[True, False], crs=[None, CoordinateReferenceSystem(epsg=4326)]) for k in itr_products_keywords(keywords, as_namedtuple=True): field = rd.get() grid = field.grid if k.has_mask: gmask = grid.get_mask(create=True) gmask[1, :] = True grid.set_mask(gmask) if not k.has_corners: grid.x.set_bounds(None) grid.y.set_bounds(None) self.assertFalse(grid.has_bounds) egrid = get_esmf_grid(grid) ogrid = get_ocgis_grid_from_esmf_grid(egrid, crs=k.crs) if k.has_mask: actual_mask = ogrid.get_mask() self.assertEqual(actual_mask.sum(), 4) self.assertTrue(actual_mask[1, :].all()) self.assertEqual(ogrid.crs, k.crs) self.assertNumpyAll(grid.get_value_stacked(), ogrid.get_value_stacked()) if k.has_corners: desired = grid.x.bounds.get_value() actual = ogrid.x.bounds.get_value() self.assertNumpyAll(actual, desired) desired = grid.y.bounds.get_value() actual = ogrid.y.bounds.get_value() self.assertNumpyAll(actual, desired) else: self.assertFalse(ogrid.has_bounds)
def test_init(self): keywords = dict( value=[None, {'no_defs': True, 'ellps': 'WGS84', 'proj': 'longlat', 'towgs84': '0,0,0,0,0,0,0'}], epsg=[None, 4326], proj4=[None, '+proj=longlat +ellps=WGS84 +towgs84=0,0,0,0,0,0,0 +no_defs ']) prev_crs = None for k in itr_products_keywords(keywords): if k['epsg'] is not None and k['value'] is None and k['proj4'] is None: epsg_only = True prev_crs = None else: epsg_only = False try: crs = CoordinateReferenceSystem(**k) except ValueError: if all([ii is None for ii in list(k.values())]): continue else: raise self.assertEqual(crs.proj4, '+proj=longlat +ellps=WGS84 +towgs84=0,0,0,0,0,0,0 +no_defs ') try: self.assertDictEqual(crs.value, {'no_defs': True, 'ellps': 'WGS84', 'proj': 'longlat', 'towgs84': '0,0,0,0,0,0,0'}) except AssertionError: self.assertDictEqual(crs.value, {'no_defs': True, 'datum': 'WGS84', 'proj': 'longlat', 'towgs84': '0,0,0,0,0,0,0'}) if prev_crs is not None: self.assertEqual(crs, prev_crs) if not epsg_only: prev_crs = deepcopy(crs) # test with a name parameter crs = CoordinateReferenceSystem(epsg=4326) self.assertEqual(crs.name, constants.OcgisConvention.Name.COORDSYS) crs = CoordinateReferenceSystem(epsg=4326, name='foo') self.assertEqual(crs.name, 'foo') # test using the init parameter value = {'init': 'epsg:4326'} self.assertEqual(CoordinateReferenceSystem(value=value), WGS84())
def test_execute_directory(self): """Test that the output directory is removed appropriately following an operations failure.""" kwds = dict(add_auxiliary_files=[True, False]) rd = self.test_data.get_rd('cancm4_tas') # this geometry is outside the domain and will result in an exception geom = [1000, 1000, 1100, 1100] for k in itr_products_keywords(kwds, as_namedtuple=True): ops = OcgOperations(dataset=rd, output_format='csv', add_auxiliary_files=k.add_auxiliary_files, geom=geom) try: ops.execute() except ExtentError: contents = os.listdir(self.current_dir_output) self.assertEqual(len(contents), 0)
def test_get_wrap_action(self): _w = WrappableCoordinateReferenceSystem possible = [_w._flag_wrapped, _w._flag_unwrapped, _w._flag_unknown, 'foo'] keywords = dict(state_src=possible, state_dst=possible) for k in itr_products_keywords(keywords, as_namedtuple=True): try: ret = _w.get_wrap_action(k.state_src, k.state_dst) except ValueError: self.assertTrue(k.state_src == 'foo' or k.state_dst == 'foo') continue if k.state_dst == _w._flag_unknown: self.assertIsNone(ret) elif k.state_src == _w._flag_unwrapped and k.state_dst == _w._flag_wrapped: self.assertEqual(ret, _w._flag_action_wrap) elif k.state_src == _w._flag_wrapped and k.state_dst == _w._flag_unwrapped: self.assertEqual(ret, _w._flag_action_unwrap) else: self.assertIsNone(ret)
def test_get_wrap_action(self): c = CoordinateReferenceSystem possible = [WrappedState.WRAPPED, WrappedState.UNWRAPPED, WrappedState.UNKNOWN] keywords = dict(state_src=possible, state_dst=possible) for k in itr_products_keywords(keywords, as_namedtuple=True): try: ret = c.get_wrap_action(k.state_src, k.state_dst) except ValueError: self.assertTrue(k.state_src == 'foo' or k.state_dst == 'foo') continue if k.state_dst == WrappedState.UNKNOWN: self.assertIsNone(ret) elif k.state_src == WrappedState.UNWRAPPED and k.state_dst == WrappedState.WRAPPED: self.assertEqual(ret, WrapAction.WRAP) elif k.state_src == WrappedState.WRAPPED and k.state_dst == WrappedState.UNWRAPPED: self.assertEqual(ret, WrapAction.UNWRAP) else: self.assertIsNone(ret)
def test_regrid_field(self): """Test with equivalent input and output expectations. The shapes of the grids are equal.""" source = self.get_ofield() source.set_crs(Spherical()) destination = deepcopy(source) desired = deepcopy(source) keywords = dict(split=[False, True], fill_value=[None, 1e20]) for k in itr_products_keywords(keywords, as_namedtuple=True): from ocgis.regrid.base import regrid_field regridded = regrid_field(source, destination, split=k.split) self.assertIsInstance(regridded, Field) self.assertNumpyAll(regridded.grid.get_value_stacked(), desired.grid.get_value_stacked()) self.assertEqual(regridded.crs, source.crs) for variable in regridded.data_variables: self.assertGreater(variable.get_value().mean(), 2.0) self.assertNumpyAll(variable.get_value(), source[variable.name].get_value()) self.assertFalse(np.may_share_memory(variable.get_value(), source[variable.name].get_value()))
def test_get_wrap_action(self): c = CoordinateReferenceSystem possible = [ WrappedState.WRAPPED, WrappedState.UNWRAPPED, WrappedState.UNKNOWN ] keywords = dict(state_src=possible, state_dst=possible) for k in itr_products_keywords(keywords, as_namedtuple=True): try: ret = c.get_wrap_action(k.state_src, k.state_dst) except ValueError: self.assertTrue(k.state_src == 'foo' or k.state_dst == 'foo') continue if k.state_dst == WrappedState.UNKNOWN: self.assertIsNone(ret) elif k.state_src == WrappedState.UNWRAPPED and k.state_dst == WrappedState.WRAPPED: self.assertEqual(ret, WrapAction.WRAP) elif k.state_src == WrappedState.WRAPPED and k.state_dst == WrappedState.UNWRAPPED: self.assertEqual(ret, WrapAction.UNWRAP) else: self.assertIsNone(ret)
def test_get_intersects_irregular_polygon(self): irregular = wkt.loads( 'POLYGON((-100.106049 38.211305,-99.286894 38.251591,-99.286894 38.258306,-99.286894 38.258306,-99.260036 39.252035,-98.769886 39.252035,-98.722885 37.734583,-100.092620 37.714440,-100.106049 38.211305))') keywords = dict(b=[True, False], with_corners=[True, False]) for k in itr_products_keywords(keywords, as_namedtuple=True): field = self.get_field(with_value=True) if k.with_corners: field.spatial.grid.corners ret = field.get_intersects(irregular, use_spatial_index=k.b) self.assertEqual(ret.shape, (2, 31, 2, 2, 2)) self.assertNumpyAll(ret.variables['tmax'].value.mask[0, 2, 1, :, :], np.array([[True, False], [False, False]])) self.assertEqual(ret.spatial.uid.data[ret.spatial.get_mask()][0], 5) if k.with_corners: self.assertNumpyAll(ret.spatial.grid.corners.mask, np.array([ [[[True, True, True, True], [False, False, False, False]], [[False, False, False, False], [False, False, False, False]]], [[[True, True, True, True], [False, False, False, False]], [[False, False, False, False], [False, False, False, False]]]])) else: self.assertIsNone(ret.spatial.grid._corners)
def test_iter_kernel_values_asserts(self): """Test assert statements.""" k = [1, 2, 3, 4] values = [np.array([[2, 3], [4, 5]]), np.arange(0, 13).reshape(-1, 1, 1)] mode = ['same', 'valid', 'foo'] for kwds in itr_products_keywords({'k': k, 'values': values, 'mode': mode}, as_namedtuple=True): try: list(MovingWindow._iter_kernel_values_(kwds.values, kwds.k)) except AssertionError: if kwds.k == 3: if kwds.values.shape == (2, 2): continue else: raise else: continue except NotImplementedError: if kwds.mode == 'foo': continue else: raise
def test_meta_attrs(self): """Test various forms for meta_attrs in the calculation definition dictionary.""" kwds = dict( meta_attr=[None, {}, {'something_else': 'is_here with us'}], calc=[{'func': 'mean', 'name': 'my_mean'}, 'foo=tas+4', {'func': 'foo=tas+4', 'meta_attrs': {'something': 'special'}}], add_meta_attrs_if_none=[True, False] ) for k in itr_products_keywords(kwds, as_namedtuple=True): k = deepcopy(k) if not k.add_meta_attrs_if_none and k.meta_attr is None: pass else: try: k.calc.update({'meta_attrs': k.meta_attr}) # likely the string representation except AttributeError: if k.calc == 'foo=tas+4': pass else: raise calc = Calc([k.calc]) self.assertEqual(set(calc.value[0].keys()), set(['ref', 'meta_attrs', 'name', 'func', 'kwds']))
def test_init(self): dataset = self.get_dataset() uri = dataset["uri"] variable = dataset["variable"] keywords = dict( uri=[None, self.get_dataset()["uri"]], variable=[None, self.get_dataset()["variable"]], request_dataset=[None, RequestDataset(uri=uri, variable=variable)], ) for k in itr_products_keywords(keywords, as_namedtuple=True): try: ip = Inspect(**k._asdict()) except ValueError: if k.uri is None and k.request_dataset is None: continue else: raise ret = ip.__str__() search = re.search("URI = (.*)\n", ret).groups()[0] if k.uri is None and k.request_dataset is None: self.assertEqual(search, "None") else: self.assertTrue(os.path.exists(search))
def test_regridding_update_crs(self): """Test with different CRS values than spherical on input data.""" # test regridding lambert conformal to 0 to 360 grid geom = 'state_boundaries' select_ugid = [25] keywords = dict(assign_source_crs=[False, True], assign_destination_crs=[False, True], destination_type=['rd', 'field']) for k in itr_products_keywords(keywords, as_namedtuple=True): if k.assign_source_crs: # assign the coordinate system changes some regridding behavior. in this case the value read from file # is the same as the assigned value. rd1 = self.test_data.get_rd('narccap_lambert_conformal', kwds={'crs': rd1.crs}) else: rd1 = self.test_data.get_rd('narccap_lambert_conformal') if k.assign_destination_crs: # assign the coordinate system changes some regridding behavior. in this case the value read from file # is the same as the assigned value. rd2 = self.test_data.get_rd('cancm4_tas', kwds={'crs': CFWGS84()}) else: rd2 = self.test_data.get_rd('cancm4_tas') if k.destination_type == 'rd': destination = rd2 elif k.destination_type == 'field': destination = rd2.get() elif k.destination_type == 'sdim': destination = rd2.get().spatial else: raise NotImplementedError ops = ocgis.OcgOperations(dataset=rd1, regrid_destination=destination, geom=geom, select_ugid=select_ugid, snippet=True) subset = SubsetOperation(ops) colls = list(subset) self.assertEqual(len(colls), 1) for coll in colls: for dd in coll.get_iter_melted(): field = dd['field'] self.assertEqual(field.spatial.crs, rd1.crs) # swap source and destination grids # test regridding lambert conformal to 0 to 360 grid rd1 = self.test_data.get_rd('cancm4_tas') rd2 = self.test_data.get_rd('narccap_lambert_conformal') actual = np.ma.array([[[[[0.0, 0.0, 0.0, 0.0, 289.309326171875, 288.7110290527344, 287.92108154296875, 287.1899108886719, 286.51715087890625, 285.9024658203125, 0.0, 0.0, 0.0], [0.0, 288.77825927734375, 288.62823486328125, 288.3404541015625, 287.9151611328125, 287.32000732421875, 286.633544921875, 286.0067138671875, 285.43914794921875, 284.93060302734375, 284.48077392578125, 0.0, 0.0], [288.4192199707031, 288.18804931640625, 287.8165588378906, 287.30499267578125, 286.65362548828125, 285.86676025390625, 285.28515625, 284.7640686035156, 284.30316162109375, 283.90216064453125, 283.560791015625, 0.0, 0.0], [288.19488525390625, 287.74169921875, 287.14593505859375, 286.4078063964844, 285.52752685546875, 284.5051574707031, 283.87457275390625, 283.4606628417969, 283.1078186035156, 282.8158264160156, 282.58441162109375, 0.0, 0.0], [288.023193359375, 287.4422607421875, 286.6193542480469, 285.65179443359375, 284.5396728515625, 283.2830505371094, 282.4002685546875, 282.09503173828125, 281.8517761230469, 281.6702575683594, 281.55029296875, 0.0, 0.0], [287.8075866699219, 287.2928771972656, 286.2398986816406, 285.0399475097656, 283.6930236816406, 282.19915771484375, 280.86077880859375, 280.66571044921875, 280.5335388183594, 280.4640808105469, 280.4613952636719, 280.4708251953125, 0.0], [287.591552734375, 287.296875, 286.0108337402344, 284.5754089355469, 282.99066162109375, 281.2564392089844, 279.47003173828125, 279.34307861328125, 279.3382263183594, 279.3432922363281, 279.3581848144531, 279.3829040527344, 0.0], [287.3750305175781, 287.322265625, 285.8916931152344, 284.12139892578125, 282.3462829589844, 280.566162109375, 278.7807922363281, 278.1846618652344, 278.1950988769531, 278.2154846191406, 278.24578857421875, 278.2860107421875, 0.0], [286.864013671875, 286.48724365234375, 285.2509460449219, 283.4699401855469, 281.6840515136719, 279.8930358886719, 278.0966796875, 277.01617431640625, 277.0421142578125, 277.07806396484375, 277.1240234375, 277.1799621582031, 0.0], [286.0535583496094, 285.5471496582031, 284.6158752441406, 282.8240661621094, 281.0272521972656, 279.2252197265625, 277.4177551269531, 275.8373107910156, 275.8789367675781, 275.9306945800781, 275.9925231933594, 276.0644226074219, 0.0], [285.3349609375, 284.69732666015625, 283.9648132324219, 282.183837890625, 280.3759765625, 278.56280517578125, 276.74407958984375, 274.91961669921875, 274.7053527832031, 274.77313232421875, 274.85107421875, 274.9391784667969, 275.0374450683594], [284.7100830078125, 283.93963623046875, 283.07275390625, 281.54925537109375, 279.730224609375, 277.90576171875, 276.07568359375, 274.2397155761719, 273.5210266113281, 273.6050720214844, 273.69940185546875, 273.9654235839844, 274.24139404296875], [284.1809387207031, 283.27606201171875, 282.2731018066406, 280.9204406738281, 279.090087890625, 277.25421142578125, 275.41259765625, 273.7033996582031, 272.687744140625, 272.9641418457031, 273.2394104003906, 273.5135498046875, 273.78662109375], [283.7496337890625, 282.70855712890625, 281.56787109375, 280.3042907714844, 278.54541015625, 276.8524169921875, 275.22515869140625, 273.6634826660156, 272.24554443359375, 272.5191955566406, 272.7915954589844, 273.0628662109375, 273.3330078125], [283.39312744140625, 282.1578369140625, 280.91937255859375, 279.67755126953125, 278.1316223144531, 276.5411071777344, 275.017333984375, 273.56024169921875, 272.16973876953125, 272.07550048828125, 272.3450622558594, 272.6134338378906, 272.8805847167969], [282.7581481933594, 281.516845703125, 280.27227783203125, 279.0242614746094, 277.64892578125, 276.16229248046875, 274.743408203125, 273.3922424316406, 272.1087646484375, 271.63311767578125, 271.8998107910156, 272.1651916503906, 272.4293518066406], [282.1268615722656, 280.87945556640625, 279.6286926269531, 278.3744201660156, 277.095703125, 275.7143249511719, 274.4017639160156, 273.157958984375, 271.98297119140625, 271.1920471191406, 271.4557800292969, 271.71820068359375, 271.97930908203125], [281.499267578125, 280.24566650390625, 278.9886779785156, 277.7280578613281, 276.4637145996094, 275.19561767578125, 273.9908142089844, 272.85589599609375, 271.7908630371094, 270.79583740234375, 271.0130615234375, 271.26873779296875, 271.4607238769531], [280.8753662109375, 279.6155090332031, 278.3522033691406, 277.085205078125, 275.81439208984375, 274.6044921875, 273.50897216796875, 272.4844055175781, 271.58203125, 270.6971130371094, 270.5581359863281, 270.7032165527344, 270.7939453125], [280.25518798828125, 278.989013671875, 277.71929931640625, 276.4458312988281, 275.1974182128906, 274.173828125, 273.29473876953125, 272.4113464355469, 271.5235290527344, 270.63116455078125, 270.1499938964844, 270.1932678222656, 270.1813049316406], [0.0, 278.4078063964844, 277.3578186035156, 276.3003234863281, 275.2351379394531, 274.162109375, 273.2556457519531, 272.36480712890625, 271.4695129394531, 270.569580078125, 269.8001403808594, 269.7401428222656, 269.6240234375], [0.0, 278.4853820800781, 277.42474365234375, 276.3564758300781, 275.2804260253906, 274.1964416503906, 273.2213134765625, 272.3229675292969, 271.4200744628906, 270.5124816894531, 269.6001281738281, 269.3451232910156, 269.1233825683594], [0.0, 278.5711669921875, 277.49969482421875, 276.4205017089844, 275.33343505859375, 274.23834228515625, 273.1918640136719, 272.2858581542969, 271.3752746582031, 270.4599304199219, 269.53973388671875, 269.00958251953125, 268.6806335449219], [0.0, 0.0, 277.5827941894531, 276.4925537109375, 275.3943176269531, 274.2879638671875, 273.1732482910156, 272.25360107421875, 271.335205078125, 270.4120178222656, 269.48388671875, 268.73492431640625, 0.0]]]]], mask=[[[[[True, True, True, True, True, True, True, True, True, False, False, False, True], [True, True, True, True, True, True, True, False, False, False, False, False, True], [True, True, True, True, True, True, True, False, False, False, False, False, True], [True, True, True, True, True, True, False, False, False, False, False, False, False], [True, True, True, True, True, False, False, False, False, False, False, False, False], [True, True, True, True, False, False, False, False, False, False, False, False, True], [True, True, False, False, False, False, False, False, False, False, False, False, True], [True, True, False, False, False, False, False, False, False, False, False, True, True], [True, True, False, False, False, False, False, False, False, False, True, True, True], [True, True, False, False, False, False, False, False, False, False, True, True, True], [True, False, False, False, False, False, False, False, False, True, True, True, True], [True, False, False, False, False, False, False, False, True, True, True, True, True], [True, False, False, False, False, False, False, False, True, True, True, True, True], [True, False, False, False, False, False, False, True, True, True, True, True, True], [True, False, False, False, False, False, False, True, True, True, True, True, True], [False, False, False, False, False, False, True, True, True, True, True, True, True], [False, False, False, False, False, False, True, True, True, True, True, True, True], [False, False, False, False, False, False, True, True, True, True, True, True, True], [False, False, False, False, False, False, False, True, True, True, True, True, True], [False, False, False, False, False, False, False, True, True, True, True, True, True], [False, False, False, False, False, False, False, True, True, True, True, True, True], [False, False, False, False, False, False, False, False, True, True, True, True, True], [True, False, False, False, False, False, True, True, True, True, True, True, True], [True, False, False, True, True, True, True, True, True, True, True, True, True]]]]], dtype=np.float32, fill_value=np.float32(1e20)) geom = 'state_boundaries' select_ugid = [25] ops = ocgis.OcgOperations(dataset=rd1, regrid_destination=rd2, geom=geom, select_ugid=select_ugid, snippet=True) subset = SubsetOperation(ops) colls = list(subset) self.assertEqual(len(colls), 1) for coll in colls: for dd in coll.get_iter_melted(): field = dd['field'] self.assertEqual(field.spatial.crs, rd1.crs) to_test = field.variables.first().value self.assertEqual(to_test.shape, (1, 1, 1, 24, 13)) self.assertNumpyAll(to_test, actual)
def test_write_netcdf(self): keywords = dict(file_only=[False, True], second_variable_alias=[None, 'tmin_alias'], with_realization=[False, True], remove_dimension_names=[False, True], crs=[None, Spherical()], with_level=[True, False], variable_kwargs=[{}, {'zlib': True, 'complevel': 5}, {'bad': 'keyword'}]) path = os.path.join(self.current_dir_output, 'foo.nc') for k in itr_products_keywords(keywords, as_namedtuple=True): field = self.get_field(with_value=True, with_realization=k.with_realization, crs=k.crs, with_level=k.with_level) if k.remove_dimension_names: try: field.level.name = None except AttributeError: self.assertFalse(k.with_level) field.temporal.name = None field.spatial.grid.row.name = None field.spatial.grid.col.name = None # Add another variable. value = np.random.rand(*field.shape) second_variable_name = 'tmin' second_variable_alias = k.second_variable_alias or second_variable_name variable = Variable(value=value, name=second_variable_name, alias=k.second_variable_alias) variable.attrs['open'] = 'up' field.variables.add_variable(variable, assign_new_uid=True) # Add some attributes. field.attrs['foo'] = 'some information' field.attrs['another'] = 'some more information' with nc_scope(path, 'w') as ds: try: field.write_netcdf(ds, file_only=k.file_only, **k.variable_kwargs) except TypeError: # Test variable keyword are actually passed by providing a bad keyword argument. self.assertDictEqual(k.variable_kwargs, {'bad': 'keyword'}) continue except ValueError: self.assertTrue(k.with_realization) self.assertIsNotNone(field.realization) continue with nc_scope(path) as ds: self.assertEqual(ds.another, 'some more information') try: variable_names = ['time', 'time_bounds', 'latitude', 'latitude_bounds', 'longitude', 'longitude_bounds', 'tmax', second_variable_alias] dimension_names = ['time', 'bounds', 'latitude', 'longitude'] if k.crs is not None: variable_names.append(k.crs.name) if k.with_level: variable_names += ['level', 'level_bounds'] dimension_names.append('level') self.assertEqual(set(ds.variables.keys()), set(variable_names)) self.assertEqual(set(ds.dimensions.keys()), set(dimension_names)) except AssertionError: self.assertTrue(k.remove_dimension_names) variable_names = ['time', 'time_bounds', 'yc', 'yc_bounds', 'xc', 'xc_bounds', 'tmax', second_variable_alias] dimension_names = ['time', 'bounds', 'yc', 'xc'] if k.crs is not None: variable_names.append(k.crs.name) if k.with_level: variable_names += ['level', 'level_bounds'] dimension_names.append('level') self.assertEqual(set(ds.variables.keys()), set(variable_names)) self.assertEqual(set(ds.dimensions.keys()), set(dimension_names)) nc_second_variable = ds.variables[second_variable_alias] try: for field_variable in field.variables.itervalues(): self.assertEqual(ds.variables[field_variable.alias].grid_mapping, k.crs.name) except AttributeError: self.assertIsNone(k.crs) self.assertEqual(nc_second_variable.open, 'up') try: self.assertNumpyAll(nc_second_variable[:], value.squeeze()) except AssertionError: self.assertTrue(k.file_only) self.assertTrue(nc_second_variable[:].mask.all()) self.assertEqual(ds.variables['tmax'].units, field.variables['tmax'].units) self.assertEqual(nc_second_variable.units, '') new_field = RequestDataset(path).get() self.assertEqual(new_field.variables.keys(), ['tmax', second_variable_alias]) if k.with_level: level_shape = 2 else: level_shape = 1 self.assertEqual(new_field.shape, (1, 31, level_shape, 3, 4))
def iter_product_keywords(self, keywords, as_namedtuple=True): return itr_products_keywords(keywords, as_namedtuple=as_namedtuple)
def test_write_netcdf(self): path = os.path.join(self.current_dir_output, 'foo.nc') other_bounds_name = 'bnds' keywords = dict(with_bounds=[True, False], with_attrs=[True, False], unlimited=[False, True], kwargs=[{}, {'zlib': True}], bounds_dimension_name=[None, other_bounds_name], axis=[None, 'GG'], name=[None, 'temporal'], name_bounds=[None, 'time_bounds'], name_value=[None, 'time'], format=[None, 'NETCDF4_CLASSIC']) for k in itr_products_keywords(keywords, as_namedtuple=True): if k.with_attrs: attrs = {'a': 5, 'b': np.array([5, 6])} else: attrs = None vd = VectorDimension(value=[2., 4.], attrs=attrs, name=k.name, name_bounds=k.name_bounds, name_value=k.name_value, axis=k.axis, unlimited=k.unlimited) if k.with_bounds: vd.set_extrapolated_bounds() with nc_scope(path, 'w') as ds: try: vd.write_netcdf(ds, bounds_dimension_name=k.bounds_dimension_name, **k.kwargs) except ValueError: self.assertIsNone(vd.name) continue with nc_scope(path, 'r') as ds: var = ds.variables[vd.name_value] if k.axis is None: axis_actual = '' else: axis_actual = vd.axis self.assertEqual(var.axis, axis_actual) try: self.assertIn(constants.OCGIS_BOUNDS, ds.dimensions) except AssertionError: try: self.assertFalse(k.with_bounds) except AssertionError: try: self.assertEqual(k.bounds_dimension_name, other_bounds_name) except AssertionError: self.assertIsNotNone(k.name_bounds_suffix) self.assertIsNone(k.bounds_dimension_name) self.assertIn(k.name_bounds_suffix, ds.variables[vd.name_bounds].dimensions) try: self.assertFalse(ds.dimensions[vd.name].isunlimited()) except AssertionError: self.assertTrue(k.unlimited) try: self.assertEqual(var.a, attrs['a']) self.assertNumpyAll(var.b, attrs['b']) except AttributeError: self.assertFalse(k.with_attrs) try: self.assertEqual(var.bounds, vd.name_bounds) self.assertNumpyAll(vd.bounds, ds.variables[vd.name_bounds][:]) except (AttributeError, KeyError): self.assertFalse(k.with_bounds) self.assertEqual(var._name, vd.name_value) self.assertEqual(var.dimensions, (vd.name,)) self.assertNumpyAll(vd.value, var[:])
def test_init_combinations(self): rd_orig = self.test_data.get_rd('cancm4_tas') dest_uri = os.path.join(self.current_dir_output, os.path.split(rd_orig.uri)[1]) shutil.copy2(rd_orig.uri, dest_uri) with nc_scope(dest_uri, 'a') as ds: var = ds.variables['tas'] outvar = ds.createVariable(var._name + 'max', var.dtype, var.dimensions) outvar[:] = var[:] + 3 outvar.setncatts(var.__dict__) with nc_scope(dest_uri) as ds: self.assertTrue(set(['tas', 'tasmax']).issubset(set(ds.variables.keys()))) keywords = dict( name=[None, 'foo'], uri=[None, dest_uri], variable=[None, 'tas', ['tas', 'tasmax'], 'crap'], alias=[None, 'tas', ['tas', 'tasmax'], ['tas_alias', 'tasmax_alias']], units=[None, [None, None], ['celsius', 'fahrenheit'], 'crap', [None, 'kelvin'], ['crap', 'crappy']], conform_units_to=[None, [None, None], ['celsius', 'fahrenheit'], 'crap', [None, 'kelvin'], ['crap', 'crappy'], [None, 'coulomb'], ['coulomb', 'coulomb']]) def itr_row(key, sequence): for element in sequence: yield ({key: element}) def itr_products_keywords(keywords): iterators = [itr_row(ki, vi) for ki, vi in keywords.iteritems()] for dictionaries in itertools.product(*iterators): yld = {} for dictionary in dictionaries: yld.update(dictionary) yield yld for k in itr_products_keywords(keywords): try: rd = RequestDataset(**k) self.assertEqual(rd._source_metadata, None) self.assertEqual(len(get_tuple(rd.variable)), len(get_tuple(rd.units))) if k['name'] is None: self.assertEqual(rd.name, '_'.join(get_tuple(rd.alias))) else: self.assertEqual(rd.name, 'foo') for v in rd._variable: try: self.assertTrue(v in rd.source_metadata['variables'].keys()) except VariableNotFoundError: if 'crap' in rd._variable: self.assertEqual(rd._source_metadata, None) break if k['units'] is None and len(rd._variable) == 1: self.assertEqual(rd.units, None) self.assertEqual(rd._units, None) try: field = rd.get() self.assertEqual(field.name, rd.name) self.assertEqual(set(field.variables.keys()), set(get_tuple(rd.alias))) except VariableNotFoundError: if 'crap' in rd._variable: continue else: raise except RequestValidationError: if 'coulomb' in get_tuple(k['conform_units_to']): continue else: raise except RequestValidationError as e: # uris cannot be None if k['uri'] is None: pass # variables cannot be None elif k['variable'] is None: pass # 'crap' is not a real variable name elif k['conform_units_to'] is not None and (k['conform_units_to'] == 'crap' or \ 'crap' in k['conform_units_to']): pass # conform_units_to must match units element-wise elif k['conform_units_to'] is not None and k['variable'] is not None and \ len(k['conform_units_to']) != len(k['variable']): pass # aliases must occur for each variable elif len(get_tuple(k['alias'])) != len(get_tuple(k['variable'])): pass # units must occur for each variable elif len(get_tuple(k['units'])) != len(get_tuple(k['variable'])): pass # bad unit definition # 'crap' is not a real variable name elif k['units'] is not None and (k['units'] == 'crap' or \ 'crap' in k['units']): pass # alway need a uri and variable elif k['uri'] is None: pass else: raise except: raise
def test_init(self): keywords = dict( value=[ None, { 'no_defs': True, 'ellps': 'WGS84', 'proj': 'longlat', 'towgs84': '0,0,0,0,0,0,0' } ], epsg=[None, 4326], proj4=[ None, '+proj=longlat +ellps=WGS84 +towgs84=0,0,0,0,0,0,0 +no_defs ' ]) prev_crs = None for k in itr_products_keywords(keywords): if k['epsg'] is not None and k['value'] is None and k[ 'proj4'] is None: epsg_only = True prev_crs = None else: epsg_only = False try: crs = CoordinateReferenceSystem(**k) except ValueError: if all([ii is None for ii in list(k.values())]): continue else: raise self.assertEqual( crs.proj4, '+proj=longlat +ellps=WGS84 +towgs84=0,0,0,0,0,0,0 +no_defs ') try: self.assertDictEqual( crs.value, { 'no_defs': True, 'ellps': 'WGS84', 'proj': 'longlat', 'towgs84': '0,0,0,0,0,0,0' }) except AssertionError: self.assertDictEqual( crs.value, { 'no_defs': True, 'datum': 'WGS84', 'proj': 'longlat', 'towgs84': '0,0,0,0,0,0,0' }) if prev_crs is not None: self.assertEqual(crs, prev_crs) if not epsg_only: prev_crs = deepcopy(crs) # test with a name parameter crs = CoordinateReferenceSystem(epsg=4326) self.assertEqual(crs.name, constants.OcgisConvention.Name.COORDSYS) crs = CoordinateReferenceSystem(epsg=4326, name='foo') self.assertEqual(crs.name, 'foo') # test using the init parameter value = {'init': 'epsg:4326'} self.assertEqual(CoordinateReferenceSystem(value=value), WGS84())