def create_dimension_map(self, group_metadata, **kwargs): """ Create a dimension map for a group from its metadata. :param dict group_metadata: Group metadata to use when creating the dimension map. :rtype: :class:`~ocgis.DimensionMap` """ ret = DimensionMap() ret.set_driver(self.key) return ret
def test_get_topology(self): dmap = DimensionMap() self.assertEqual(dmap.get_topology(GridAbstraction.POINT), None) dmap_poly = dmap.get_topology(GridAbstraction.POLYGON, create=True) self.assertEqual(dmap_poly, DimensionMap()) dmap_poly.set_variable(DMK.X, 'polyx') dmap_poly.set_variable(DMK.ELEMENT_NODE_CONNECTIVITY, 'element_connectivity') dmap_point = dmap.get_topology(GridAbstraction.POINT, create=True) dmap_point.set_variable(DMK.Y, 'point_y') self.assertEqual(dmap.get_topology(GridAbstraction.POINT).get_variable(DMK.Y), 'point_y')
def test_set_bounds(self): dmap = DimensionMap() dmap.set_variable(DMK.X, 'lon', bounds='lon_bounds') actual = dmap.get_bounds(DMK.X) self.assertEqual(actual, 'lon_bounds') dmap.set_bounds(DMK.X, None) actual = dmap.get_bounds(DMK.X) self.assertIsNone(actual)
def test_system_grouped_dimension_map(self): initial_data = { DMK.X: { DMK.VARIABLE: 'longitude' }, DMK.GROUPS: { 'nested': { DMK.Y: { DMK.VARIABLE: 'latitude' }, DMK.GROUPS: { 'nested_in_nested': { DMK.CRS: { DMK.VARIABLE: 'coord_sys' } } } } } } dmap = DimensionMap.from_dict(initial_data) nested = dmap.get_group('nested') self.assertEqual(nested.get_variable(DMK.Y), 'latitude') self.assertIsNone(dmap.get_variable(DMK.Y)) self.assertEqual( dmap.get_group([None, 'nested', 'nested_in_nested']).get_crs(), 'coord_sys')
def create_dimension_map(self, group_metadata): ret = {DMK.GEOM: {DMK.VARIABLE: VariableName.GEOMETRY_VARIABLE, DMK.DIMENSION: DimensionName.GEOMETRY_DIMENSION}} ret = DimensionMap.from_dict(ret) crs = self.get_crs(group_metadata) if crs is not None: ret.set_crs(crs) return ret
def test_from_old_style_dimension_map(self): old_style = { 'Y': { 'variable': u'lat', 'bounds': u'lat_bnds', 'dimension': u'lat', 'pos': 1 }, 'X': { 'variable': u'lon', 'bounds': u'lon_bnds', 'dimension': u'lon', 'pos': 2 }, 'Z': { 'variable': u'height', 'bounds': None, 'dimension': None, 'pos': None }, 'T': { 'variable': u'time', 'bounds': u'time_bnds', 'dimension': u'time', 'pos': 0 } } new_style = DimensionMap.from_old_style_dimension_map(old_style) desired = { 'level': { 'attrs': {}, 'bounds': None, 'dimension': [], 'variable': u'height' }, 'time': { 'attrs': { 'axis': 'T' }, 'bounds': u'time_bnds', 'dimension': [u'time'], 'variable': u'time' }, 'x': { 'attrs': {}, 'bounds': u'lon_bnds', 'dimension': [u'lon'], 'variable': u'lon' }, 'y': { 'attrs': {}, 'bounds': u'lat_bnds', 'dimension': [u'lat'], 'variable': u'lat' } } self.assertEqual(new_style.as_dict(), desired)
def get_dimension_map(self, group_metadata, strict=False): # Get dimension variable metadata. This involves checking for the presence of any bounds variables. variables = group_metadata['variables'] dimensions = group_metadata['dimensions'] axes = { 'realization': 'R', 'time': 'T', 'level': 'Z', 'x': 'X', 'y': 'Y' } check_bounds = list(axes.keys()) check_bounds.pop(check_bounds.index('realization')) # Get the main entry for each axis. for k, v in list(axes.items()): axes[k] = get_dimension_map_entry(v, variables, dimensions, strict=strict) # Attempt to find bounds for each entry (ignoring realizations). for k in check_bounds: if axes[k] is not None: keys = ['bounds'] if k == 'time': keys += ['climatology'] bounds_var = get_by_key_list( variables[axes[k]['variable']]['attrs'], keys) if bounds_var is not None: if bounds_var not in variables: msg = 'Bounds listed for variable "{0}" but the destination bounds variable "{1}" does not exist.'. \ format(axes[k]['variable'], bounds_var) ocgis_lh(msg, logger='nc.driver', level=logging.WARNING) bounds_var = None axes[k]['bounds'] = bounds_var # Create the template dimension map dictionary. ret = {k: v for k, v in list(axes.items()) if v is not None} # Check for coordinate system variables. This will check every variable. crs_name = get_coordinate_system_variable_name(self, group_metadata) if crs_name is not None: ret[DimensionMapKey.CRS] = {DimensionMapKey.VARIABLE: crs_name} ret = DimensionMap.from_dict(ret) # Check for a spatial mask. for varname, var in group_metadata['variables'].items(): if 'ocgis_role' in var.get('attrs', {}): if var['attrs']['ocgis_role'] == 'spatial_mask': ret.set_spatial_mask(varname, attrs=var['attrs']) return ret
def test_system_grouped_dimension_map(self): initial_data = {DMK.X: {DMK.VARIABLE: 'longitude'}, DMK.GROUPS: {'nested': {DMK.Y: {DMK.VARIABLE: 'latitude'}, DMK.GROUPS: { 'nested_in_nested': {DMK.CRS: {DMK.VARIABLE: 'coord_sys'}}}}}} dmap = DimensionMap.from_dict(initial_data) nested = dmap.get_group('nested') self.assertEqual(nested.get_variable(DMK.Y), 'latitude') self.assertIsNone(dmap.get_variable(DMK.Y)) self.assertEqual(dmap.get_group([None, 'nested', 'nested_in_nested']).get_crs(), 'coord_sys')
def create_dimension_map(self, group_metadata): ret = { DMK.GEOM: { DMK.VARIABLE: VariableName.GEOMETRY_VARIABLE, DMK.DIMENSION: DimensionName.GEOMETRY_DIMENSION } } ret = DimensionMap.from_dict(ret) crs = self.get_crs(group_metadata) if crs is not None: ret.set_crs(crs) return ret
def test(self): dmap = DimensionMap() _ = dmap.get_variable('x') _ = dmap.get_dimension('x') _ = dmap.get_attrs('x') _ = dmap.get_bounds('x') dmap.set_variable('y', 'latitude', dimension='ache', bounds='lat_bounds') _ = dmap.get_crs() dmap.set_crs('latitude_longitude') desired = {'crs': {'variable': 'latitude_longitude'}, 'x': {'attrs': {}, 'bounds': None, 'dimension': [], 'variable': None}, 'y': {'attrs': {}, 'bounds': 'lat_bounds', 'dimension': ['ache'], 'variable': 'latitude'}} self.assertDictEqual(dmap._storage, desired)
def test_get_topology(self): dmap = DimensionMap() self.assertEqual(dmap.get_topology(GridAbstraction.POINT), None) dmap_poly = dmap.get_topology(GridAbstraction.POLYGON, create=True) self.assertEqual(dmap_poly, DimensionMap()) dmap_poly.set_variable(DMK.X, 'polyx') dmap_poly.set_variable(DMK.ELEMENT_NODE_CONNECTIVITY, 'element_connectivity') dmap_point = dmap.get_topology(GridAbstraction.POINT, create=True) dmap_point.set_variable(DMK.Y, 'point_y') self.assertEqual( dmap.get_topology(GridAbstraction.POINT).get_variable(DMK.Y), 'point_y')
def test_get_data_variable_names(self): driver = self.get_drivernetcdf() dvars = driver.get_data_variable_names(driver.rd.metadata, driver.rd.dimension_map) self.assertEqual(len(dvars), 0) # Test a found variable. dimension_map = {'time': {'variable': 'the_time', DimensionMapKey.DIMENSION: ['tt', 'ttt']}, 'x': {'variable': 'xx', DimensionMapKey.DIMENSION: ['xx', 'xxx']}, 'y': {'variable': 'yy', DimensionMapKey.DIMENSION: ['yy', 'yyy']}} metadata = {'variables': {'tas': {'dimensions': ('xx', 'ttt', 'yyy')}, 'pr': {'dimensions': ('foo',)}}} dimension_map = DimensionMap.from_dict(dimension_map) dvars = driver.get_data_variable_names(metadata, dimension_map) self.assertEqual(dvars, ('tas',)) # Test request dataset uses the dimensioned variables. driver = self.get_drivernetcdf() with self.assertRaises(NoDataVariablesFound): assert driver.rd.variable
def test_get_data_variable_names(self): driver = self.get_drivernetcdf() dvars = driver.get_data_variable_names(driver.rd.metadata, driver.rd.dimension_map) self.assertEqual(len(dvars), 0) # Test a found variable. dimension_map = { 'time': { 'variable': 'the_time', DimensionMapKey.DIMENSION: ['tt', 'ttt'] }, 'x': { 'variable': 'xx', DimensionMapKey.DIMENSION: ['xx', 'xxx'] }, 'y': { 'variable': 'yy', DimensionMapKey.DIMENSION: ['yy', 'yyy'] } } metadata = { 'variables': { 'tas': { 'dimensions': ('xx', 'ttt', 'yyy') }, 'pr': { 'dimensions': ('foo', ) } } } dimension_map = DimensionMap.from_dict(dimension_map) dvars = driver.get_data_variable_names(metadata, dimension_map) self.assertEqual(dvars, ('tas', )) # Test request dataset uses the dimensioned variables. driver = self.get_drivernetcdf() with self.assertRaises(NoDataVariablesFound): assert driver.rd.variable
def test_from_old_style_dimension_map(self): old_style = {'Y': {'variable': u'lat', 'bounds': u'lat_bnds', 'dimension': u'lat', 'pos': 1}, 'X': {'variable': u'lon', 'bounds': u'lon_bnds', 'dimension': u'lon', 'pos': 2}, 'Z': {'variable': u'height', 'bounds': None, 'dimension': None, 'pos': None}, 'T': {'variable': u'time', 'bounds': u'time_bnds', 'dimension': u'time', 'pos': 0}} new_style = DimensionMap.from_old_style_dimension_map(old_style) desired = {'level': {'attrs': {}, 'bounds': None, 'dimension': [], 'variable': u'height'}, 'time': {'attrs': {'axis': 'T'}, 'bounds': u'time_bnds', 'dimension': [u'time'], 'variable': u'time'}, 'x': {'attrs': {}, 'bounds': u'lon_bnds', 'dimension': [u'lon'], 'variable': u'lon'}, 'y': {'attrs': {}, 'bounds': u'lat_bnds', 'dimension': [u'lat'], 'variable': u'lat'}} self.assertEqual(new_style.as_dict(), desired)
def test_get_group(self): dmap = DimensionMap() dmap.set_crs('who') actual = dmap.get_group(None) self.assertEqual(dmap.as_dict(), actual.as_dict()) dmap = DimensionMap() dmap.set_crs('what') with self.assertRaises(DimensionMapError): dmap.get_group(['level_1']) level_1 = DimensionMap() level_1.set_variable(DMK.LEVEL, 'level_1') dmap.set_group('level_1', level_1) desired = { 'crs': { 'variable': 'what' }, 'groups': { 'level_1': { 'level': { 'attrs': {}, 'bounds': None, 'dimension': [], 'variable': 'level_1' } } } } actual = dmap.as_dict() self.assertEqual(actual, desired) level_1_1 = DimensionMap() level_1_1.set_variable(DMK.X, 'longitude') level_1.set_group('level_1_1', level_1_1) desired = { 'crs': { 'variable': 'what' }, 'groups': { 'level_1': { 'groups': { 'level_1_1': { 'x': { 'attrs': {}, 'bounds': None, 'dimension': [], 'variable': 'longitude' } } }, 'level': { 'attrs': {}, 'bounds': None, 'dimension': [], 'variable': 'level_1' } } } } actual = dmap.as_dict() self.assertEqual(actual, desired)
def test_get_driver(self): dmap = DimensionMap() self.assertEqual(dmap.get_driver(), DriverNetcdfCF.key)
def test(self): dmap = DimensionMap() _ = dmap.get_variable('x') _ = dmap.get_dimension('x') _ = dmap.get_attrs('x') _ = dmap.get_bounds('x') dmap.set_variable('y', 'latitude', dimension='ache', bounds='lat_bounds') _ = dmap.get_crs() dmap.set_crs('latitude_longitude') desired = { 'crs': { 'variable': 'latitude_longitude' }, 'x': { 'attrs': {}, 'bounds': None, 'dimension': [], 'variable': None }, 'y': { 'attrs': {}, 'bounds': 'lat_bounds', 'dimension': ['ache'], 'variable': 'latitude' } } self.assertDictEqual(dmap._storage, desired)
def test_get_group(self): dmap = DimensionMap() dmap.set_crs('who') actual = dmap.get_group(None) self.assertEqual(dmap.as_dict(), actual.as_dict()) dmap = DimensionMap() dmap.set_crs('what') with self.assertRaises(DimensionMapError): dmap.get_group(['level_1']) level_1 = DimensionMap() level_1.set_variable(DMK.LEVEL, 'level_1') dmap.set_group('level_1', level_1) desired = {'crs': {'variable': 'what'}, 'groups': {'level_1': {'level': {'attrs': {}, 'bounds': None, 'dimension': [], 'variable': 'level_1'}}}} actual = dmap.as_dict() self.assertEqual(actual, desired) level_1_1 = DimensionMap() level_1_1.set_variable(DMK.X, 'longitude') level_1.set_group('level_1_1', level_1_1) desired = {'crs': {'variable': 'what'}, 'groups': {'level_1': {'groups': {'level_1_1': {'x': {'attrs': {}, 'bounds': None, 'dimension': [], 'variable': 'longitude'}}}, 'level': {'attrs': {}, 'bounds': None, 'dimension': [], 'variable': 'level_1'}}}} actual = dmap.as_dict() self.assertEqual(actual, desired)
def test_set_spatial_mask(self): dmap = DimensionMap() dims = Dimension('x', 3), Dimension('y', 7) mask_var = create_spatial_mask_variable('a_mask', None, dims) self.assertFalse(np.any(mask_var.get_mask())) dmap.set_spatial_mask(mask_var) self.assertEqual(dmap.get_spatial_mask(), mask_var.name) with self.assertRaises(DimensionMapError): dmap.set_variable(DMK.SPATIAL_MASK, mask_var) # Test custom variables may be used. dmap = DimensionMap() dims = Dimension('x', 3), Dimension('y', 7) mask_var = create_spatial_mask_variable('a_mask', None, dims) attrs = {'please keep me': 'no overwriting'} dmap.set_spatial_mask(mask_var, attrs=attrs) attrs = dmap.get_attrs(DMK.SPATIAL_MASK) self.assertIn('please keep me', attrs) # Test default attributes are not added. dmap = DimensionMap() dmap.set_spatial_mask('foo', default_attrs={'blue': 'world'}) prop = dmap.get_property(DMK.SPATIAL_MASK) self.assertEqual(prop['attrs'], {'blue': 'world'})
def test_set_spatial_mask(self): dmap = DimensionMap() dims = Dimension('x', 3), Dimension('y', 7) mask_var = create_grid_mask_variable('a_mask', None, dims) self.assertFalse(np.any(mask_var.get_mask())) dmap.set_spatial_mask(mask_var) self.assertEqual(dmap.get_spatial_mask(), mask_var.name) with self.assertRaises(DimensionMapError): dmap.set_variable(DMK.SPATIAL_MASK, mask_var) # Test custom variables may be used. dmap = DimensionMap() dims = Dimension('x', 3), Dimension('y', 7) mask_var = create_grid_mask_variable('a_mask', None, dims) attrs = {'please keep me': 'no overwriting'} dmap.set_spatial_mask(mask_var, attrs=attrs) attrs = dmap.get_attrs(DMK.SPATIAL_MASK) self.assertIn('please keep me', attrs)
def __init__(self, uri=None, variable=None, units=None, time_range=None, time_region=None, time_subset_func=None, level_range=None, conform_units_to=None, crs='auto', t_units=None, t_calendar=None, t_conform_units_to=None, grid_abstraction='auto', grid_is_isomorphic='auto', dimension_map=None, field_name=None, driver=None, regrid_source=True, regrid_destination=False, metadata=None, format_time=True, opened=None, uid=None, rename_variable=None, predicate=None, rotated_pole_priority=False, driver_kwargs=None, decomp_type=DecompositionType.OCGIS): self._is_init = True self._field_name = field_name self._level_range = None self._time_range = None self._time_region = None self._time_subset_func = None self._driver_kwargs = driver_kwargs self._decomp_type = decomp_type if rename_variable is not None: rename_variable = get_tuple(rename_variable) self._rename_variable = rename_variable self.rotated_pole_priority = rotated_pole_priority self.predicate = predicate if dimension_map is not None and isinstance(dimension_map, dict): dimension_map = DimensionMap.from_dict(dimension_map) self._dimension_map = dimension_map self._metadata = deepcopy(metadata) self._uri = None self.uid = uid # This is an "open" file-like object that may be passed in-place of file location parameters. self._opened = opened if opened is not None and driver is None: msg = 'If "opened" is not None, then a "driver" must be provided.' ocgis_lh(logger='request', exc=RequestValidationError('driver', msg)) # Field creation options. self.format_time = format_time self.grid_abstraction = grid_abstraction self.grid_is_isomorphic = grid_is_isomorphic # Flag used for regridding to determine if the coordinate system was assigned during initialization. self._has_assigned_coordinate_system = False if crs == 'auto' else True if uri is None: # Fields may be created from pure metadata. if metadata is not None: # The default OCGIS driver is NetCDF. if driver is None: driver = DriverKey.NETCDF_CF elif opened is None: ocgis_lh(logger='request', exc=RequestValidationError('uri', 'Cannot be None')) else: self._uri = get_uri(uri) if driver is None: klass = get_autodiscovered_driver(uri) else: klass = get_driver(driver) self._driver = klass(self) if variable is not None: variable = get_tuple(variable) self._variable = variable self.time_range = time_range self.time_region = time_region self.time_subset_func = time_subset_func self.level_range = level_range self._crs = deepcopy(crs) self.regrid_source = regrid_source self.regrid_destination = regrid_destination self.units = units self.conform_units_to = conform_units_to self._is_init = False self._validate_time_subset_() # Update metadata for time variable. tvar = self.dimension_map.get_variable(DMK.TIME) if tvar is not None: m = self.metadata['variables'][tvar] if t_units is not None: m['attrs']['units'] = t_units if t_calendar is not None: m['attrs']['calendar'] = t_calendar if t_conform_units_to is not None: from ocgis.util.units import get_units_object t_calendar = m['attrs'].get('calendar', constants.DEFAULT_TEMPORAL_CALENDAR) t_conform_units_to = get_units_object(t_conform_units_to, calendar=t_calendar) m['conform_units_to'] = t_conform_units_to
def __init__(self, uri=None, variable=None, units=None, time_range=None, time_region=None, time_subset_func=None, level_range=None, conform_units_to=None, crs='auto', t_units=None, t_calendar=None, t_conform_units_to=None, grid_abstraction='auto', grid_is_isomorphic='auto', dimension_map=None, field_name=None, driver=None, regrid_source=True, regrid_destination=False, metadata=None, format_time=True, opened=None, uid=None, rename_variable=None, predicate=None, rotated_pole_priority=False, driver_kwargs=None): self._is_init = True self._field_name = field_name self._level_range = None self._time_range = None self._time_region = None self._time_subset_func = None self._driver_kwargs = driver_kwargs if rename_variable is not None: rename_variable = get_tuple(rename_variable) self._rename_variable = rename_variable self.rotated_pole_priority = rotated_pole_priority self.predicate = predicate if dimension_map is not None and isinstance(dimension_map, dict): dimension_map = DimensionMap.from_dict(dimension_map) self._dimension_map = dimension_map self._metadata = deepcopy(metadata) self._uri = None self.uid = uid # This is an "open" file-like object that may be passed in-place of file location parameters. self._opened = opened if opened is not None and driver is None: msg = 'If "opened" is not None, then a "driver" must be provided.' ocgis_lh(logger='request', exc=RequestValidationError('driver', msg)) # Field creation options. self.format_time = format_time self.grid_abstraction = grid_abstraction self.grid_is_isomorphic = grid_is_isomorphic # Flag used for regridding to determine if the coordinate system was assigned during initialization. self._has_assigned_coordinate_system = False if crs == 'auto' else True if uri is None: # Fields may be created from pure metadata. if metadata is not None: # The default OCGIS driver is NetCDF. if driver is None: driver = DriverKey.NETCDF_CF elif opened is None: ocgis_lh(logger='request', exc=RequestValidationError('uri', 'Cannot be None')) else: self._uri = get_uri(uri) if driver is None: klass = get_autodiscovered_driver(uri) else: klass = get_driver(driver) self._driver = klass(self) if variable is not None: variable = get_tuple(variable) self._variable = variable self.time_range = time_range self.time_region = time_region self.time_subset_func = time_subset_func self.level_range = level_range self._crs = deepcopy(crs) self.regrid_source = regrid_source self.regrid_destination = regrid_destination self.units = units self.conform_units_to = conform_units_to self._is_init = False self._validate_time_subset_() # Update metadata for time variable. tvar = self.dimension_map.get_variable(DMK.TIME) if tvar is not None: m = self.metadata['variables'][tvar] if t_units is not None: m['attrs']['units'] = t_units if t_calendar is not None: m['attrs']['calendar'] = t_calendar if t_conform_units_to is not None: from ocgis.util.units import get_units_object t_calendar = m['attrs'].get( 'calendar', constants.DEFAULT_TEMPORAL_CALENDAR) t_conform_units_to = get_units_object(t_conform_units_to, calendar=t_calendar) m['conform_units_to'] = t_conform_units_to
def test_set_driver(self): dmap = DimensionMap() dmap.set_driver(DriverNetcdfUGRID) self.assertEqual(dmap.get_driver(), DriverNetcdfUGRID.key)
def create_dimension_map_raw(driver, group_metadata): ret = DimensionMap.from_metadata(driver, group_metadata) return ret
def test_set_variable(self): var = Variable(name='test', value=[1, 2], dimensions='two') dmap = DimensionMap() dmap.set_variable(DMK.X, var, dimension='not_two') actual = dmap.get_dimension(DMK.X) desired = ['not_two'] self.assertEqual(actual, desired) # Test setting with a variable and bounds. var = Variable(name='center', value=[1, 2, 3], dtype=float, dimensions='one') var.set_extrapolated_bounds('bounds_data', 'bounds') dmap = DimensionMap() dmap.set_variable(DMK.Y, var) self.assertEqual(dmap.get_bounds(DMK.Y), var.bounds.name) new_var = Variable(name='new_center', value=[1, 2, 3], dtype=float, dimensions='one') dmap.set_variable(DMK.Y, new_var) self.assertIsNone(dmap.get_bounds(DMK.Y)) # Test a dimension position argument is needed if the variable has more than one dimension. var = Variable(name='two_dims', value=np.zeros((4, 5)), dimensions=['one', 'two']) dmap = DimensionMap() with self.assertRaises(DimensionMapError): dmap.set_variable(DMK.X, var) dmap.set_variable(DMK.X, var, pos=1) self.assertEqual(dmap.get_dimension(DMK.X), ['two']) # Test a scalar dimension. var = Variable(name='scalar_dimension', dimensions=[]) dmap = DimensionMap() dmap.set_variable(DMK.LEVEL, var) self.assertEqual(dmap.get_variable(DMK.LEVEL), 'scalar_dimension') self.assertIsNone(dmap.get_bounds(DMK.LEVEL)) self.assertEqual(dmap.get_dimension(DMK.LEVEL), []) # Test dimensionless variable. var = Variable(name='two_dims', value=np.zeros((4, 5)), dimensions=['one', 'two']) dmap = DimensionMap() dmap.set_variable(DMK.X, var, dimensionless=True) self.assertEqual(dmap.get_dimension(DMK.X), [])
def get_dimension_map(self, group_metadata): """:rtype: :class:`ocgis.DimensionMap`""" return DimensionMap()
def get_dimension_map_raw(driver, group_metadata): ret = DimensionMap.from_metadata(driver, group_metadata) return ret