def get_collection_for_write_ugeom(self, crs):
pt1 = Point(1, 2)
pt2 = Point(4, 5)
props1 = [('UGID', 10), ('STATE_FIPS', '06'), ('ID', 25.0), ('STATE_NAME', 'California'), ('STATE_ABBR', 'CA')]
props2 = [('UGID', 11), ('STATE_FIPS', '08'), ('ID', 26.0), ('STATE_NAME', 'Ontario'), ('STATE_ABBR', 'CB')]
schema = {'geometry': 'Point',
'properties': OrderedDict([('UGID', 'int'),
('STATE_FIPS', 'str:2'),
('ID', 'float'),
('STATE_NAME', 'str'),
('STATE_ABBR', 'str:2')])}
def get_sdim(geom, props, crs):
record = {'geom': geom, 'properties': OrderedDict(props)}
return SpatialDimension.from_records([record], schema, crs=crs)
sdim1, sdim2 = [get_sdim(g, p, crs) for g, p in zip([pt1, pt2], [props1, props2])]
coll = SpatialCollection(crs=crs)
coll.ugeom[10] = sdim1
coll.ugeom[11] = sdim2
pdtype = [('UGID', '<i8'), ('STATE_FIPS', '|S2'), ('ID', '<f8'), ('STATE_NAME', '|S80'), ('STATE_ABBR', '|S2')]
return coll, pdtype, pt1, pt2
def test_multivariate_iteration(self):
field = self.get_field(with_value=True, month_count=1)
field.variables.add_variable(Variable(value=field.variables['tmax'].value + 5,
name='tmin', alias='tmin'))
field.temporal.name_uid = 'tid'
field.level.name_uid = 'lid'
field.spatial.geom.name_uid = 'gid'
div = Divide(field=field, parms={'arr1': 'tmin', 'arr2': 'tmax'}, alias='some_division',
dtype=np.float64)
ret = div.execute()
cfield = DerivedMultivariateField(variables=ret, realization=field.realization, temporal=field.temporal,
level=field.level,
spatial=field.spatial, meta=field.meta, uid=field.uid)
cfield.spatial.name_uid = 'gid'
sc = ShpCabinet()
meta = sc.get_meta('state_boundaries')
sp = SpatialCollection(meta=meta, key='state_boundaries', headers=constants.HEADERS_MULTI)
for row in sc.iter_geoms('state_boundaries', as_spatial_dimension=True):
sp.add_field(cfield, ugeom=row)
for ii, row in enumerate(sp.get_iter_dict(melted=True)):
if ii == 0:
self.assertDictEqual(row[1], {'lid': 1, 'ugid': 1, 'cid': 1, 'did': None, 'year': 2000,
'time': datetime.datetime(2000, 1, 1, 12, 0),
'calc_alias': 'some_division', 'value': 12.989774984574424, 'month': 1,
'gid': 1, 'calc_key': 'divide', 'tid': 1, 'level': 50, 'day': 1})
self.assertEqual(ii + 1, 2 * 31 * 2 * 3 * 4 * 51)
def test_calculation_iteration(self):
field = self.get_field(with_value=True,month_count=2)
field.variables.add_variable(Variable(value=field.variables['tmax'].value+5,
name='tmin',alias='tmin'))
field.temporal.name_uid = 'tid'
field.level.name_uid = 'lid'
field.spatial.geom.name_uid = 'gid'
grouping = ['month']
tgd = field.temporal.get_grouping(grouping)
mu = Mean(field=field,tgd=tgd,alias='my_mean')
ret = mu.execute()
kwds = copy(field.__dict__)
kwds.pop('_raw')
kwds.pop('_variables')
kwds['temporal'] = tgd
kwds['variables'] = ret
cfield = DerivedField(**kwds)
cfield.temporal.name_uid = 'tid'
cfield.temporal.name_value = 'time'
cfield.spatial.name_uid = 'gid'
sc = ShpCabinet()
meta = sc.get_meta('state_boundaries')
sp = SpatialCollection(meta=meta,key='state_boundaries',headers=constants.calc_headers)
for row in sc.iter_geoms('state_boundaries'):
sp.add_field(row['properties']['UGID'],row['geom'],cfield.variables.keys()[0],
cfield,properties=row['properties'])
for ii,row in enumerate(sp.get_iter_dict()):
if ii == 0:
self.assertEqual(row[0].bounds,(-100.5, 39.5, -99.5, 40.5))
self.assertDictEqual(row[1],{'lid': 1, 'ugid': 1, 'vid': 1, 'cid': 1, 'did': 1, 'year': 2000, 'time': datetime.datetime(2000, 1, 16, 0, 0), 'calc_alias': 'my_mean_tmax', 'value': 0.44808476666433006, 'month': 1, 'alias': 'tmax', 'variable': 'tmax', 'gid': 1, 'calc_key': 'mean', 'tid': 1, 'level': 50, 'day': 16})
self.assertEqual(len(row),2)
self.assertEqual(len(row[1]),len(constants.calc_headers))
def test_geoms(self):
pt = Point(1, 2)
sc = self.get_spatial_collection_two_geometries(pt=pt)
self.assertEqual({4: pt, 5: pt}, sc.geoms)
sc = SpatialCollection()
sc.ugeom[1] = None
self.assertEqual({1: None}, sc.geoms)
def test_properties(self):
sc = self.get_spatial_collection_two_geometries()
self.assertAsSetEqual(sc.keys(), [4, 5])
self.assertEqual(sc.properties.values()[0].dtype.names, ('ID', 'NAME'))
sc = SpatialCollection()
sc.ugeom[1] = None
self.assertEqual({1: None}, sc.properties)
def get_collection(self):
field = self.get_field(with_value=True)
sc = ShpCabinet()
meta = sc.get_meta('state_boundaries')
sp = SpatialCollection(meta=meta, key='state_boundaries')
for row in sc.iter_geoms('state_boundaries', as_spatial_dimension=True):
sp.add_field(field, ugeom=row)
return sp
def write(self):
build = True
for coll in self:
if build:
ret = SpatialCollection(meta=coll.meta,key=coll.key,crs=coll.crs,headers=coll.headers)
build = False
for k,v in coll.iteritems():
ret.add_field(k,coll.geoms[k],v.keys()[0],v.values()[0],properties=coll.properties[k])
return(ret)
def get_spatial_collection_two_geometries(self, pt=None):
pt = pt or Point(1, 2)
record1 = {'geom': pt, 'properties': {'ID': 4, 'NAME': 'hello'}}
record2 = {'geom': pt, 'properties': {'ID': 5, 'NAME': 'another'}}
sd1 = SpatialDimension.from_records([record1], uid='ID')
sd2 = SpatialDimension.from_records([record2], uid='ID')
sc = SpatialCollection()
field = self.get_field()
sc.add_field(field, ugeom=sd1)
sc.add_field(field, ugeom=sd2)
return sc
def test_constructor(self):
field = self.get_field(with_value=True)
sc = ShpCabinet()
meta = sc.get_meta('state_boundaries')
sp = SpatialCollection(meta=meta,key='state_boundaries')
for row in sc.iter_geoms('state_boundaries'):
sp.add_field(row['properties']['UGID'],row['geom'],field.variables.keys()[0],
field,properties=row['properties'])
self.assertEqual(len(sp),51)
self.assertIsInstance(sp.geoms[25],MultiPolygon)
self.assertIsInstance(sp.properties[25],dict)
self.assertEqual(sp[25]['tmax'].variables['tmax'].value.shape,(2, 31, 2, 3, 4))
def test_calculation_iteration_two_calculations(self):
field = self.get_field(with_value=True, month_count=2)
field.variables.add_variable(Variable(value=field.variables['tmax'].value + 5, name='tmin', alias='tmin'))
field.temporal.name_uid = 'tid'
field.level.name_uid = 'lid'
field.spatial.geom.name_uid = 'gid'
grouping = ['month']
tgd = field.temporal.get_grouping(grouping)
mu = Mean(field=field, tgd=tgd, alias='my_mean', dtype=np.float64, add_parents=True)
ret = mu.execute()
thresh = Threshold(field=field, vc=ret, tgd=tgd, alias='a_treshold', add_parents=True,
parms={'operation': 'gte', 'threshold': 0.5})
ret = thresh.execute()
kwds = copy(field.__dict__)
kwds.pop('_raw')
kwds.pop('_variables')
kwds.pop('_should_regrid')
kwds.pop('_has_assigned_coordinate_system')
kwds.pop('_attrs')
kwds['name'] = kwds.pop('_name')
kwds['temporal'] = tgd
kwds['variables'] = ret
cfield = DerivedField(**kwds)
cfield.temporal.name_uid = 'tid'
cfield.temporal.name_value = 'time'
cfield.spatial.name_uid = 'gid'
sc = ShpCabinet()
meta = sc.get_meta('state_boundaries')
sp = SpatialCollection(meta=meta, key='state_boundaries', headers=constants.HEADERS_CALC)
for row in sc.iter_geoms('state_boundaries', as_spatial_dimension=True):
sp.add_field(cfield, ugeom=row)
cids = set()
for ii, row in enumerate(sp.get_iter_dict(melted=True)):
cids.update([row[1]['cid']])
if ii == 0:
self.assertEqual(row[0].bounds, (-100.5, 39.5, -99.5, 40.5))
self.assertDictEqual(row[1], {'lid': 1, 'ugid': 1, 'vid': 1, 'cid': 1, 'did': 1, 'year': 2000,
'time': datetime.datetime(2000, 1, 16, 0, 0),
'calc_alias': 'my_mean_tmax', 'value': 0.44808476666433006, 'month': 1,
'alias': 'tmax', 'variable': 'tmax', 'gid': 1, 'calc_key': 'mean',
'tid': 1, 'level': 50, 'day': 16})
self.assertEqual(len(row), 2)
self.assertEqual(len(row[1]), len(constants.HEADERS_CALC))
self.assertEqual(ii + 1, 2 * 2 * 2 * 3 * 4 * 51 * 4)
self.assertEqual(len(cids), 4)
def write(self):
build = True
for coll in self:
if build:
ret = SpatialCollection(meta=coll.meta, key=coll.key, crs=coll.crs, headers=coll.headers)
build = False
for k, v in coll.iteritems():
field = v.values()[0]
if field is None:
name = v.keys()[0]
else:
name = None
ret.add_field(v.values()[0], ugeom=coll.ugeom[k], name=name)
return ret
def get_esmf_field(self, **kwargs):
"""
:keyword field: (``=None``) The field object. If ``None``, call :meth:`~ocgis.test.base.TestBase.get_field`
:type field: :class:`~ocgis.Field`
:param kwargs: Other keyword arguments to :meth:`ocgis.test.base.TestBase.get_field`.
:returns: An ESMF field object.
:rtype: :class:`ESMF.Field`
"""
from ocgis.conv.esmpy import ESMPyConverter
field = kwargs.pop("field", None) or self.get_field(**kwargs)
coll = SpatialCollection()
coll.add_field(field)
conv = ESMPyConverter([coll])
efield = conv.write()
return efield
def test_iteration(self):
field = self.get_field(with_value=True)
field.temporal.name_uid = 'tid'
field.level.name_uid = 'lid'
field.spatial.geom.name_uid = 'gid'
field.spatial.name_uid = 'gid'
sc = ShpCabinet()
meta = sc.get_meta('state_boundaries')
sp = SpatialCollection(meta=meta,key='state_boundaries')
for row in sc.iter_geoms('state_boundaries'):
sp.add_field(row['properties']['UGID'],row['geom'],field.variables.keys()[0],
field,properties=row['properties'])
for ii,row in enumerate(sp.get_iter_dict()):
if ii == 1:
self.assertEqual(row[1],{'lid': 1, 'ugid': 1, 'vid': 1, 'alias': 'tmax', 'did': 1, 'year': 2000, 'value': 0.7203244934421581, 'month': 1, 'variable': 'tmax', 'gid': 2, 'time': datetime.datetime(2000, 1, 1, 12, 0), 'tid': 1, 'level': 50, 'day': 1})
self.assertIsInstance(row[0],MultiPolygon)
self.assertEqual(len(row),2)
self.assertEqual(len(row[1]),len(constants.raw_headers))
def test_iteration_methods(self):
field = self.get_field(with_value=True)
field.temporal.name_uid = 'tid'
field.level.name_uid = 'lid'
field.spatial.geom.name_uid = 'gid'
field.spatial.name_uid = 'gid'
sc = ShpCabinet()
meta = sc.get_meta('state_boundaries')
sp = SpatialCollection(meta=meta, key='state_boundaries', headers=constants.HEADERS_RAW)
for row in sc.iter_geoms('state_boundaries', as_spatial_dimension=True):
sp.add_field(field, ugeom=row)
for ii, row in enumerate(sp.get_iter_dict(melted=True)):
if ii == 1:
self.assertDictEqual(row[1], {'lid': 1, 'ugid': 1, 'vid': 1, 'alias': 'tmax', 'did': 1, 'year': 2000,
'value': 0.7203244934421581, 'month': 1, 'variable': 'tmax', 'gid': 2,
'time': datetime.datetime(2000, 1, 1, 12, 0), 'tid': 1, 'level': 50,
'day': 1})
self.assertIsInstance(row[0], MultiPolygon)
self.assertEqual(len(row), 2)
self.assertEqual(len(row[1]), len(constants.HEADERS_RAW))
def test_crs(self):
sc = SpatialCollection()
self.assertIsNone(sc.crs)
field = self.get_field(crs=CFWGS84())
sc.add_field(field)
self.assertEqual(sc.crs, CFWGS84())
self.assertIsNone(sc._crs)
sc = SpatialCollection(crs=Spherical())
self.assertEqual(sc.crs, Spherical())
sc = SpatialCollection()
field = self.get_field()
field.spatial = None
sc.add_field(field)
self.assertIsNone(sc.crs)
def test_write(self):
records = [{'geom': Point(1, 2).buffer(1), 'properties': {'ID': 5, 'name': 'heaven'}},
{'geom': Point(7, 8).buffer(1), 'properties': {'ID': 50, 'name': 'hell'}}]
sdim1 = SpatialDimension.from_records([records[0]], uid='ID')
sdim2 = SpatialDimension.from_records([records[1]], uid='ID')
field = self.get_field(crs=WGS84())
coll1 = SpatialCollection()
coll1.add_field(field, ugeom=sdim1)
coll2 = SpatialCollection()
coll2.add_field(field, ugeom=sdim2)
colls = [coll1, coll2]
f = FakeAbstractCollectionConverter(colls, outdir=self.current_dir_output, prefix='me')
ret = f.write()
path = os.path.join(ret, 'shp', 'me_ugid.shp')
with fiona.open(path, 'r') as source:
records = list(source)
self.assertEqual(len(records), 2)
self.assertEqual([r['properties']['ID'] for r in records], [5, 50])
self.assertEqual([r['properties']['name'] for r in records], ['heaven', 'hell'])
def _process_geometries_(self,itr,field,headers,value_keys,alias):
'''
:param sequence itr: Contains geometry dictionaries to process. If there
are no geometries to process, this will be a sequence of one element with
an empty dictionary.
:param :class:`ocgis.interface.Field` field: The field object to use for
operations.
:param sequence headers: Sequence of strings to use as headers for the
creation of the collection.
:param sequence value_keys: Sequence of strings to use as headers for the
keyed output functions.
:param str alias: The request data alias currently being processed.
:yields: :class:~`ocgis.SpatialCollection`
'''
## loop over the iterator
for gd in itr:
## always work with a new geometry dictionary
gd = deepcopy(gd)
## CFRotatedPole takes special treatment. only do this if a subset
## geometry is available. this variable is needed to determine if
## backtransforms are necessary.
original_rotated_pole_crs = None
if isinstance(field.spatial.crs,CFRotatedPole):
## only transform if there is a subset geometry
if len(gd) > 0:
## store row and column dimension metadata and names before
## transforming as this information is lost w/out row and
## column dimensions on the transformations.
original_row_column_metadata = {'row':{'name':field.spatial.grid.row.name,
'meta':field.spatial.grid.row.meta},
'col':{'name':field.spatial.grid.col.name,
'meta':field.spatial.grid.col.meta}}
## reset the geometries
field.spatial._geom = None
## get the new grid dimension
field.spatial.grid = get_rotated_pole_spatial_grid_dimension(field.spatial.crs,field.spatial.grid)
## update the CRS. copy the original CRS for possible later
## transformation back to rotated pole.
original_rotated_pole_crs = deepcopy(field.spatial.crs)
field.spatial.crs = CFWGS84()
## initialize the collection object to store the subsetted data. if
## the output CRS differs from the field's CRS, adjust accordingly
## when initializing.
if self.ops.output_crs is not None and field.spatial.crs != self.ops.output_crs:
collection_crs = self.ops.output_crs
else:
collection_crs = field.spatial.crs
coll = SpatialCollection(crs=collection_crs,headers=headers,meta=gd.get('meta'),
value_keys=value_keys)
## reference variables from the geometry dictionary
geom = gd.get('geom')
## keep this around for the collection creation
coll_geom = deepcopy(geom)
crs = gd.get('crs')
## if there is a spatial abstraction, ensure it may be loaded.
if self.ops.abstraction is not None:
try:
getattr(field.spatial.geom,self.ops.abstraction)
except ImproperPolygonBoundsError:
exc = ImproperPolygonBoundsError('A "polygon" spatial abstraction is not available without the presence of bounds.')
ocgis_lh(exc=exc,logger='subset')
except Exception as e:
ocgis_lh(exc=e,logger='subset')
## if there is a snippet, return the first realization, time, and level
if self.ops.snippet:
field = field[0,0,0,:,:]
## if there is a slice, use it to subset the field.
elif self.ops.slice is not None:
field = field.__getitem__(self.ops.slice)
## see if the selection crs matches the field's crs
if crs is not None and crs != field.spatial.crs:
geom = project_shapely_geometry(geom,crs.sr,field.spatial.crs.sr)
crs = field.spatial.crs
## if the geometry is a point, we need to buffer it...
if type(geom) in [Point,MultiPoint]:
ocgis_lh(logger=self._subset_log,msg='buffering point geometry',level=logging.DEBUG)
geom = geom.buffer(self.ops.search_radius_mult*field.spatial.grid.resolution)
## update the geometry to store in the collection
coll_geom = deepcopy(geom)
## get the ugid following geometry manipulations
if 'properties' in gd and 'UGID' in gd['properties']:
ugid = gd['properties']['UGID']
else:
ugid = 1
if geom is None:
msg = 'No selection geometry. Returning all data. Assiging UGID as 1.'
else:
msg = 'Subsetting with selection geometry having UGID={0}'.format(ugid)
ocgis_lh(msg=msg,logger=self._subset_log)
## check for unique ugids. this is an issue with point subsetting
## as the buffer radius changes by dataset.
if ugid in self._ugid_unique_store and geom is not None:
## only update if the geometry is unique
if not any([__.almost_equals(geom) for __ in self._geom_unique_store]):
prev_ugid = ugid
ugid = max(self._ugid_unique_store) + 1
self._ugid_unique_store.append(ugid)
msg = 'Updating UGID {0} to {1} to maintain uniqueness.'.format(prev_ugid,ugid)
ocgis_lh(msg,self._subset_log,level=logging.WARN,alias=alias,ugid=ugid)
else:
self._geom_unique_store.append(geom)
else:
self._ugid_unique_store.append(ugid)
self._geom_unique_store.append(geom)
## try to update the properties
try:
gd['properties']['UGID'] = ugid
except KeyError:
if not isinstance(gd,dict):
raise
## unwrap the data if it is geographic and 360
if geom is not None and crs == CFWGS84():
if CFWGS84.get_is_360(field.spatial):
ocgis_lh('unwrapping selection geometry',self._subset_log,alias=alias,ugid=ugid,level=logging.DEBUG)
geom = Wrapper().unwrap(geom)
## perform the spatial operation
if geom is not None:
try:
if self.ops.spatial_operation == 'intersects':
sfield = field.get_intersects(geom, use_spatial_index=env.USE_SPATIAL_INDEX,
select_nearest=self.ops.select_nearest)
elif self.ops.spatial_operation == 'clip':
sfield = field.get_clip(geom, use_spatial_index=env.USE_SPATIAL_INDEX,
select_nearest=self.ops.select_nearest)
else:
ocgis_lh(exc=NotImplementedError(self.ops.spatial_operation))
except EmptySubsetError as e:
if self.ops.allow_empty:
ocgis_lh(alias=alias,ugid=ugid,msg='empty geometric operation but empty returns allowed',level=logging.WARN)
sfield = None
else:
msg = str(e) + ' This typically means the selection geometry falls outside the spatial domain of the target dataset.'
ocgis_lh(exc=ExtentError(message=msg),alias=alias,logger=self._subset_log)
else:
sfield = field
## if the base size is being requested, bypass the rest of the
## operations.
if self._request_base_size_only == False:
## if empty returns are allowed, there be an empty field
if sfield is not None:
## aggregate if requested
if self.ops.aggregate:
ocgis_lh('executing spatial average',self._subset_log,alias=alias,ugid=ugid)
sfield = sfield.get_spatially_aggregated(new_spatial_uid=ugid)
## wrap the returned data.
if not env.OPTIMIZE_FOR_CALC:
if CFWGS84.get_is_360(sfield.spatial):
if self.ops.output_format != 'nc' and self.ops.vector_wrap:
ocgis_lh('wrapping output geometries',self._subset_log,alias=alias,ugid=ugid,
level=logging.DEBUG)
## modifying these values in place will change the values
## in the base field. a copy is necessary.
sfield.spatial = deepcopy(sfield.spatial)
sfield.spatial.crs.wrap(sfield.spatial)
## check for all masked values
if env.OPTIMIZE_FOR_CALC is False and self.ops.file_only is False:
for variable in sfield.variables.itervalues():
ocgis_lh(msg='Fetching data for variable with alias "{0}".'.format(variable.alias),
logger=self._subset_log)
if variable.value.mask.all():
## masked data may be okay depending on other opeartional
## conditions.
if self.ops.snippet or self.ops.allow_empty or (self.ops.output_format == 'numpy' and self.ops.allow_empty):
if self.ops.snippet:
ocgis_lh('all masked data encountered but allowed for snippet',
self._subset_log,alias=alias,ugid=ugid,level=logging.WARN)
if self.ops.allow_empty:
ocgis_lh('all masked data encountered but empty returns allowed',
self._subset_log,alias=alias,ugid=ugid,level=logging.WARN)
if self.ops.output_format == 'numpy':
ocgis_lh('all masked data encountered but numpy data being returned allowed',
logger=self._subset_log,alias=alias,ugid=ugid,level=logging.WARN)
else:
## if the geometry is also masked, it is an empty spatial
## operation.
if sfield.spatial.abstraction_geometry.value.mask.all():
ocgis_lh(exc=EmptyData,logger=self._subset_log)
## if none of the other conditions are met, raise the masked data error
else:
ocgis_lh(logger=self._subset_log,exc=MaskedDataError(),alias=alias,ugid=ugid)
## transform back to rotated pole if necessary
if original_rotated_pole_crs is not None:
if self.ops.output_crs is None and not isinstance(self.ops.output_crs,CFWGS84):
# copy the spatial mask to the new spatial array
spatial_mask_before_transform = deepcopy(sfield.spatial.get_mask())
# need to load the values before proceeding. source indices will disappear.
for variable in sfield.variables.itervalues():
variable.value
# reset the geometries
sfield.spatial._geom = None
sfield.spatial.grid = get_rotated_pole_spatial_grid_dimension(
original_rotated_pole_crs,sfield.spatial.grid,inverse=True,
rc_original=original_row_column_metadata)
# update the grid mask with the previous spatial mask
sfield.spatial.grid.value.mask = spatial_mask_before_transform
## update the uid mask to match the spatial mask
sfield.spatial.uid = np.ma.array(sfield.spatial.uid,mask=spatial_mask_before_transform)
sfield.spatial.crs = original_rotated_pole_crs
## update the coordinate system of the data output
if self.ops.output_crs is not None:
## if the geometry is not None, it may need to be projected to match
## the output crs.
if geom is not None and crs != self.ops.output_crs:
geom = project_shapely_geometry(geom,crs.sr,self.ops.output_crs.sr)
coll_geom = deepcopy(geom)
## update the coordinate reference system of the spatial
## dimension.
try:
sfield.spatial.update_crs(self.ops.output_crs)
## this is likely a rotated pole origin
except RuntimeError as e:
if isinstance(sfield.spatial.crs,CFRotatedPole):
assert(isinstance(self.ops.output_crs,WGS84))
sfield.spatial._geom = None
sfield.spatial.grid = get_rotated_pole_spatial_grid_dimension(
sfield.spatial.crs,sfield.spatial.grid)
sfield.spatial.crs = self.ops.output_crs
else:
ocgis_lh(exc=e,logger=self._subset_log)
## the geometry may need to be wrapped or unwrapped depending on
## the vector wrap situation
name = alias if sfield is None else None
coll.add_field(ugid, coll_geom, sfield, properties=gd.get('properties'), name=name)
yield(coll)
def _process_subsettables_(self,rds):
'''
:param rds: Sequence of :class:~`ocgis.RequestDataset` objects.
:type rds: sequence
:yields: :class:~`ocgis.SpatialCollection`
'''
ocgis_lh(msg='entering _process_geometries_',logger=self._subset_log,level=logging.DEBUG)
## select headers
if self.ops.headers is not None:
headers = self.ops.headers
else:
if self.cengine is not None:
if self._has_multivariate_calculations:
headers = constants.multi_headers
else:
headers = constants.calc_headers
else:
headers = constants.raw_headers
## keyed output functions require appending headers regardless. there is
## only one keyed output function allowed in a request.
if self.cengine is not None:
if self.cengine._check_calculation_members_(self.cengine.funcs,AbstractKeyedOutputFunction):
value_keys = self.cengine.funcs[0]['ref'].structure_dtype['names']
headers = list(headers) + value_keys
## remove the 'value' attribute headers as this is replaced by the
## keyed output names.
try:
headers.remove('value')
## it may not be in the list because of a user overload
except ValueError:
pass
else:
value_keys = None
else:
value_keys = None
alias = '_'.join([r.name for r in rds])
ocgis_lh('processing...',self._subset_log,alias=alias,level=logging.DEBUG)
## return the field object
try:
## look for field optimizations
if self.ops.optimizations is not None and 'fields' in self.ops.optimizations:
field = [self.ops.optimizations['fields'][rd.alias] for rd in rds]
else:
field = [rd.get(format_time=self.ops.format_time,
interpolate_spatial_bounds=self.ops.interpolate_spatial_bounds) for rd in rds]
## update the spatial abstraction to match the operations value. sfield
## will be none if the operation returns empty and it is allowed to have
## empty returns.
for f in field:
f.spatial.abstraction = self.ops.abstraction
if len(field) > 1:
try:
## reset the variable uid and let the collection handle its assignment
variable_to_add = field[1].variables.first()
variable_to_add.uid = None
field[0].variables.add_variable(variable_to_add)
## reset the field names and let these be auto-generated
for f in field:
f._name = None
## this will fail for optimizations as the fields are already joined
except VariableInCollectionError:
if self.ops.optimizations is not None and 'fields' in self.ops.optimizations:
pass
else:
raise
field = field[0]
## this error is related to subsetting by time or level. spatial subsetting
## occurs below.
except EmptySubsetError as e:
if self.ops.allow_empty:
ocgis_lh(msg='time or level subset empty but empty returns allowed',
logger=self._subset_log,level=logging.WARN)
coll = SpatialCollection(headers=headers)
coll.add_field(1, None, None, name='_'.join([rd.name for rd in rds]))
try:
yield(coll)
finally:
return
else:
ocgis_lh(exc=ExtentError(message=str(e)),alias=rd.alias,logger=self._subset_log)
## set iterator based on presence of slice. slice always overrides geometry.
if self.ops.slice is not None:
itr = [{}]
else:
itr = [{}] if self.ops.geom is None else self.ops.geom
for coll in self._process_geometries_(itr,field,headers,value_keys,alias):
yield(coll)
def test_add_field(self):
sc = SpatialCollection()
field = self.get_field()
sc.add_field(field)
self.assertIsInstance(sc[1][field.name], Field)
with self.assertRaises(ValueError):
sc.add_field(field)
field2 = deepcopy(field)
field2.name = 'another'
sc.add_field(field2)
self.assertIsInstance(sc[1]['another'], Field)
sc.add_field(field, name='hiding')
self.assertIsInstance(sc[1]['hiding'], Field)
self.assertIsNone(sc.ugeom[1])
record = {'geom': Point(1, 2), 'properties': {'ID': 4, 'NAME': 'hello'}}
sd = SpatialDimension.from_records([record], uid='ID')
sc.add_field(field, ugeom=sd)
self.assertEqual(sc.keys(), [1, 4])
self.assertEqual(sc.ugeom[4].geom.point.value[0, 0], record['geom'])
sc = SpatialCollection()
sc.add_field(field)
self.assertEqual(field.uid, 1)
field.uid = 10
sc = SpatialCollection()
sc.add_field(field)
self.assertEqual(field.uid, 10)
field2 = deepcopy(field)
field2.spatial.crs = Spherical()
with self.assertRaises(ValueError):
# coordinate systems must be same
sc.add_field(field2, name='hover')
sc = SpatialCollection()
sc.add_field(None, name='food')
self.assertIsNone(sc[1]['food'])
def get_spatial_collection(self, field=None):
rd = self.test_data.get_rd('cancm4_tas')
field = field or rd.get()[:, 0, :, 0, 0]
coll = SpatialCollection()
coll.add_field(field)
return coll
def _process_geometries_(self,rds):
ocgis_lh(msg='entering _process_geometries_',logger=self._subset_log,level=logging.DEBUG)
## select headers
if self.ops.headers is not None:
headers = self.ops.headers
else:
if self.cengine is not None:
if self.cengine._check_calculation_members_(self.cengine.funcs,AbstractMultivariateFunction):
headers = constants.multi_headers
else:
headers = constants.calc_headers
else:
headers = constants.raw_headers
## keyed output functions require appending headers regardless. there is
## only one keyed output function allowed in a request.
if self.cengine is not None:
if self.cengine._check_calculation_members_(self.cengine.funcs,AbstractKeyedOutputFunction):
value_keys = self.cengine.funcs[0]['ref'].structure_dtype['names']
headers = list(headers) + value_keys
## remove the 'value' attribute headers as this is replaced by the
## keyed output names.
try:
headers.remove('value')
## it may not be in the list because of a user overload
except ValueError:
pass
else:
value_keys = None
else:
value_keys = None
alias = '_'.join([r.alias for r in rds])
ocgis_lh('processing...',self._subset_log,alias=alias)
## return the field object
try:
field = [rd.get(format_time=self.ops.format_time) for rd in rds]
if len(field) > 1:
field[0].variables.add_variable(field[1].variables.first())
field = field[0]
except EmptySubsetError as e:
if self.ops.allow_empty:
ocgis_lh(msg='time or level subset empty but empty returns allowed',
logger=self._subset_log,level=logging.WARN)
coll = SpatialCollection(headers=headers)
coll.add_field(1,None,rd.alias,None)
try:
yield(coll)
finally:
return
else:
ocgis_lh(exc=ExtentError(message=str(e)),alias=rd.alias,logger=self._subset_log)
## set iterator based on presence of slice. slice always overrides geometry.
if self.ops.slice is not None:
itr = [{}]
else:
itr = [{}] if self.ops.geom is None else self.ops.geom
## loop over the iterator
for gd in itr:
## initialize the collection object to store the subsetted data. if
## the output CRS differs from the field's CRS, adjust accordingly
## when initilizing.
if self.ops.output_crs is not None and field.spatial.crs != self.ops.output_crs:
collection_crs = self.ops.output_crs
else:
collection_crs = field.spatial.crs
coll = SpatialCollection(crs=collection_crs,headers=headers,meta=gd.get('meta'),
value_keys=value_keys)
## reference variables from the geometry dictionary
geom = gd.get('geom')
crs = gd.get('crs')
if 'properties' in gd and 'UGID' in gd['properties']:
ugid = gd['properties']['UGID']
else:
## try to get lowercase ugid in case the shapefile is not perfectly
## formed. however, if there is no geometry accept the error and
## use the default geometry identifier.
if len(gd) == 0:
ugid = 1
else:
ugid = gd['properties']['ugid']
ocgis_lh('processing',self._subset_log,level=logging.DEBUG,alias=alias,ugid=ugid)
## if there is a spatial abstraction, ensure it may be loaded.
if self.ops.abstraction is not None:
try:
getattr(field.spatial.geom,self.ops.abstraction)
except ImproperPolygonBoundsError:
exc = ImproperPolygonBoundsError('A "polygon" spatial abstraction is not available without the presence of bounds.')
ocgis_lh(exc=exc,logger='subset')
except Exception as e:
ocgis_lh(exc=e,logger='subset')
## if there is a snippet, return the first realization, time, and level
if self.ops.snippet:
field = field[0,0,0,:,:]
## if there is a slice, use it to subset the field.
elif self.ops.slice is not None:
field = field.__getitem__(self.ops.slice)
## see if the selection crs matches the field's crs
if crs is not None and crs != field.spatial.crs:
geom = project_shapely_geometry(geom,crs.sr,field.spatial.crs.sr)
crs = field.spatial.crs
## if the geometry is a point, we need to buffer it...
if type(geom) in [Point,MultiPoint]:
ocgis_lh(logger=self._subset_log,msg='buffering point geometry',level=logging.DEBUG)
geom = geom.buffer(self.ops.search_radius_mult*field.spatial.grid.resolution)
## unwrap the data if it is geographic and 360
if geom is not None and crs == CFWGS84():
if CFWGS84.get_is_360(field.spatial):
ocgis_lh('unwrapping selection geometry',self._subset_log,alias=alias,ugid=ugid)
geom = Wrapper().unwrap(geom)
## perform the spatial operation
if geom is not None:
try:
if self.ops.spatial_operation == 'intersects':
sfield = field.get_intersects(geom)
elif self.ops.spatial_operation == 'clip':
sfield = field.get_clip(geom)
else:
ocgis_lh(exc=NotImplementedError(self.ops.spatial_operation))
except EmptySubsetError as e:
if self.ops.allow_empty:
ocgis_lh(alias=alias,ugid=ugid,msg='empty geometric operation but empty returns allowed',level=logging.WARN)
sfield = None
else:
ocgis_lh(exc=ExtentError(message=str(e)),alias=alias,logger=self._subset_log)
else:
sfield = field
## if empty returns are allowed, there be an empty field
if sfield is not None:
## aggregate if requested
if self.ops.aggregate:
sfield = sfield.get_spatially_aggregated(new_spatial_uid=ugid)
## wrap the returned data.
if not env.OPTIMIZE_FOR_CALC:
if CFWGS84.get_is_360(sfield.spatial):
if self.ops.output_format != 'nc' and self.ops.vector_wrap:
ocgis_lh('wrapping output geometries',self._subset_log,alias=alias,ugid=ugid)
sfield.spatial.crs.wrap(sfield.spatial)
## check for all masked values
if env.OPTIMIZE_FOR_CALC is False and self.ops.file_only is False:
for variable in sfield.variables.itervalues():
if variable.value.mask.all():
## masked data may be okay depending on other opeartional
## conditions.
if self.ops.snippet or self.ops.allow_empty or (self.ops.output_format == 'numpy' and self.ops.allow_empty):
if self.ops.snippet:
ocgis_lh('all masked data encountered but allowed for snippet',
self._subset_log,alias=alias,ugid=ugid,level=logging.WARN)
if self.ops.allow_empty:
ocgis_lh('all masked data encountered but empty returns allowed',
self._subset_log,alias=alias,ugid=ugid,level=logging.WARN)
if self.ops.output_format == 'numpy':
ocgis_lh('all masked data encountered but numpy data being returned allowed',
logger=self._subset_log,alias=alias,ugid=ugid,level=logging.WARN)
else:
## if the geometry is also masked, it is an empty spatial
## operation.
if sfield.spatial.abstraction_geometry.value.mask.all():
ocgis_lh(exc=EmptyData,logger=self._subset_log)
## if none of the other conditions are met, raise the masked data error
else:
ocgis_lh(logger=self._subset_log,exc=MaskedDataError(),alias=alias,ugid=ugid)
## update the coordinate system of the data output
if self.ops.output_crs is not None:
## if the geometry is not None, it may need to be projected to match
## the output crs.
if geom is not None and crs != self.ops.output_crs:
geom = project_shapely_geometry(geom,crs.sr,self.ops.output_crs.sr)
sfield.spatial.update_crs(self.ops.output_crs)
## update the spatial abstraction to match the operations value. sfield
## will be none if the operation returns empty and it is allowed to have
## empty returns.
if sfield is not None:
sfield.spatial.abstraction = self.ops.abstraction
coll.add_field(ugid,geom,alias,sfield,properties=gd.get('properties'))
yield(coll)
def _process_subsettables_(self, rds):
"""
:param rds: Sequence of :class:~`ocgis.RequestDataset` objects.
:type rds: sequence
:rtype: :class:`ocgis.api.collection.AbstractCollection`
"""
ocgis_lh(msg='entering _process_geometries_', logger=self._subset_log, level=logging.DEBUG)
# select headers and any value keys for keyed output functions
value_keys = None
if self.ops.headers is not None:
headers = self.ops.headers
else:
if self.ops.melted:
if self.cengine is not None:
if self._has_multivariate_calculations:
headers = constants.HEADERS_MULTI
else:
headers = constants.HEADERS_CALC
else:
headers = constants.HEADERS_RAW
else:
headers = None
# keyed output functions require appending headers regardless. there is only one keyed output function
# allowed in a request.
if headers is not None:
if self.cengine is not None:
if self.cengine._check_calculation_members_(self.cengine.funcs, AbstractKeyedOutputFunction):
value_keys = self.cengine.funcs[0]['ref'].structure_dtype['names']
headers = list(headers) + value_keys
# remove the 'value' attribute headers as this is replaced by the keyed output names.
try:
headers.remove('value')
# it may not be in the list because of a user overload
except ValueError:
pass
alias = '_'.join([r.name for r in rds])
ocgis_lh('processing...', self._subset_log, alias=alias, level=logging.DEBUG)
# return the field object
try:
# look for field optimizations
if self.ops.optimizations is not None and 'fields' in self.ops.optimizations:
ocgis_lh('applying optimizations', self._subset_log, level=logging.DEBUG)
field = [self.ops.optimizations['fields'][rd.alias] for rd in rds]
# no field optimizations, extract the target data from the dataset collection
else:
ocgis_lh('creating field objects', self._subset_log, level=logging.DEBUG)
len_rds = len(rds)
field = [None] * len_rds
for ii in range(len_rds):
rds_element = rds[ii]
try:
field_object = rds_element.get(format_time=self.ops.format_time)
except AttributeError:
# likely a field object which does not need to be loaded from source
if not self.ops.format_time:
raise NotImplementedError
field_object = rds_element
# extrapolate the spatial bounds if requested
if self.ops.interpolate_spatial_bounds:
try:
try:
field_object.spatial.grid.row.set_extrapolated_bounds()
field_object.spatial.grid.col.set_extrapolated_bounds()
except AttributeError:
# row/col is likely none. attempt to extrapolate using the grid values
field_object.spatial.grid.set_extrapolated_corners()
except BoundsAlreadyAvailableError:
msg = 'Bounds/corners already on object. Ignoring "interpolate_spatial_bounds".'
ocgis_lh(msg=msg, logger=self._subset_log, level=logging.WARNING)
field[ii] = field_object
# update the spatial abstraction to match the operations value. sfield will be none if the operation returns
# empty and it is allowed to have empty returns.
for f in field:
f.spatial.abstraction = self.ops.abstraction
if len(field) > 1:
try:
# reset the variable uid and let the collection handle its assignment
variable_to_add = field[1].variables.first()
variable_to_add.uid = None
field[0].variables.add_variable(variable_to_add)
# reset the field names and let these be auto-generated
for f in field:
f._name = None
# this will fail for optimizations as the fields are already joined
except VariableInCollectionError:
if self.ops.optimizations is not None and 'fields' in self.ops.optimizations:
pass
else:
raise
field = field[0]
# this error is related to subsetting by time or level. spatial subsetting occurs below.
except EmptySubsetError as e:
if self.ops.allow_empty:
ocgis_lh(msg='time or level subset empty but empty returns allowed', logger=self._subset_log,
level=logging.WARN)
coll = SpatialCollection(headers=headers)
name = '_'.join([rd.name for rd in rds])
coll.add_field(None, name=name)
try:
yield coll
finally:
return
else:
ocgis_lh(exc=ExtentError(message=str(e)), alias=str([rd.name for rd in rds]), logger=self._subset_log)
# set iterator based on presence of slice. slice always overrides geometry.
if self.ops.slice is not None:
itr = [None]
else:
itr = [None] if self.ops.geom is None else self.ops.geom
for coll in self._process_geometries_(itr, field, headers, value_keys, alias):
yield (coll)
