def test_crs(self):
"""Test overloading by geometry and grid."""
field = Field()
self.assertIsNone(field.crs)
geom = GeometryVariable(name='geom', value=[Point(1, 2)], dimensions='g', crs=Spherical())
field = Field(geom=geom)
self.assertEqual(field.crs, geom.crs)
grid = self.get_gridxy_global(crs=Spherical())
field = Field(grid=grid)
self.assertEqual(field.crs, grid.crs)
grid = self.get_gridxy_global(crs=Spherical())
# Grid and field coordinate systems do not match.
with self.assertRaises(ValueError):
Field(grid=grid, crs=WGS84())
geom = GeometryVariable(name='geom', value=[Point(1, 2)], dimensions='g', crs=Spherical())
with self.assertRaises(ValueError):
Field(geom=geom, crs=WGS84())
geom = GeometryVariable(name='geom', value=[Point(1, 2)], dimensions='g')
grid = self.get_gridxy_global()
field = Field(geom=geom, grid=grid, crs=WGS84())
self.assertEqual(field.crs, WGS84())
self.assertEqual(field.geom.crs, WGS84())
self.assertEqual(field.grid.crs, WGS84())
g = self.get_gridxy_global()
f = Field(grid=g, crs=Spherical())
self.assertIn('standard_name', f.grid.x.attrs)
self.assertIn('standard_name', f.grid.y.attrs)
def test_get_buffered_geometry(self):
proj4 = '+proj=aea +lat_1=20 +lat_2=60 +lat_0=40 +lon_0=-96 +x_0=0 +y_0=0 +ellps=GRS80 +datum=NAD83 +units=m +no_defs'
buffer_crs_list = [None, CoordinateReferenceSystem(proj4=proj4)]
poly = make_poly((36, 44), (-104, -95))
for buffer_crs in buffer_crs_list:
gvar = GeometryVariable(value=poly, name='geoms', dimensions='dim', crs=WGS84())
self.assertEqual(gvar.crs, WGS84())
if buffer_crs is None:
buffer_value = 1
else:
buffer_value = 10
ret = SpatialSubsetOperation._get_buffered_geometry_(gvar, buffer_value, buffer_crs=buffer_crs)
ref = ret.get_value()[0]
if buffer_crs is None:
self.assertEqual(ref.bounds, (-105.0, 35.0, -94.0, 45.0))
else:
self.assertNumpyAllClose(np.array(ref.bounds), np.array(
(-104.00013263459613, 35.9999147913708, -94.99986736540386, 44.00008450528758)))
self.assertEqual(gvar.crs, ret.crs)
# check deepcopy
ret.get_value()[0] = make_poly((1, 2), (3, 4))
ref_buffered = ret.get_value()[0]
ref_original = gvar.get_value()[0]
with self.assertRaises(AssertionError):
self.assertNumpyAllClose(np.array(ref_buffered.bounds), np.array(ref_original.bounds))
def test_get_unioned_spatial_average_parallel(self):
if MPI_SIZE != 8:
raise SkipTest('MPI_SIZE != 8')
dist = OcgDist()
geom_count = dist.create_dimension('geom_count', size=8, dist=True)
time_count = dist.create_dimension('time', size=3)
dist.update_dimension_bounds()
if not geom_count.is_empty:
gvar = GeometryVariable(value=[Point(1.0, 1.0).buffer(MPI_RANK + 2)] * len(geom_count),
dimensions=geom_count)
value = np.zeros((len(time_count), len(geom_count)), dtype=float)
for ii in range(value.shape[0]):
value[ii] = [MPI_RANK + 1 + ii + 1] * len(geom_count)
data = Variable(name='data', value=value, dtype=float, dimensions=[time_count, geom_count])
else:
gvar = GeometryVariable(dimensions=geom_count)
data = Variable(name='data', dimensions=[time_count, geom_count], dtype=float)
gvar.parent.add_variable(data)
self.assertTrue(gvar.is_empty == data.is_empty == gvar.parent.is_empty)
with vm.scoped_by_emptyable('union', gvar):
if vm.is_null:
unioned = None
else:
unioned = gvar.get_unioned(spatial_average='data')
if unioned is not None:
self.assertIsInstance(unioned, GeometryVariable)
actual = unioned.parent[data.name]
self.assertAlmostEqual(actual.get_value().max(), 5.5466666666666677)
else:
self.assertIsNone(unioned)
def test_prepare_geometry(self):
for geometry_record in self.get_subset_geometries():
for ss, k in self:
gvar = GeometryVariable(value=geometry_record['geom'],
dimensions='dim',
crs=WGS84())
self.assertIsNotNone(gvar.crs)
prepared = ss._prepare_geometry_(gvar)
self.assertNotEqual(gvar.get_value()[0].bounds,
prepared.get_value()[0].bounds)
self.assertFalse(
np.may_share_memory(gvar.get_value(),
prepared.get_value()))
try:
self.assertEqual(prepared.crs, ss.field.crs)
except AssertionError:
# Rotated pole on the fields become spherical.
self.assertEqual(prepared.crs, CFSpherical())
self.assertIsInstance(ss.field.crs, CFRotatedPole)
# Test nebraska against an unwrapped dataset.
nebraska = GeometryVariable(value=self.nebraska['geom'],
dimensions='d',
crs=WGS84())
field = self.test_data.get_rd('cancm4_tas').get()
ss = SpatialSubsetOperation(field)
prepared = ss._prepare_geometry_(nebraska)
self.assertEqual(prepared.wrapped_state, WrappedState.UNWRAPPED)
def test_init(self):
field = self.get_ocgfield()
self.assertIsInstance(field, Field)
# Test unique identifier.
field = Field(uid=4)
self.assertEqual(field.uid, 4)
# Test with a coordinate system and geometry.
desired_crs = WGS84()
geom = GeometryVariable(name='geom',
value=[Point(1, 2)],
dimensions='geom')
field = Field(crs=desired_crs, geom=geom)
self.assertEqual(field.crs, desired_crs)
# Test dimension names are automatically added to dimension map.
g = GeometryVariable(name='geom',
value=[Point(1, 2)],
dimensions='the_geom_dim')
f = Field(geom=g)
actual = f.dimension_map.get_dimension(DimensionMapKey.GEOM)
self.assertEqual(actual, ['the_geom_dim'])
# Test dimension map does not have any entries at initialization.
actual = Field()
desired = actual.dimension_map.as_dict()
self.assertEqual(len(desired), 0)
def test_update_crs_to_cartesian(self):
"""Test a spherical to cartesian CRS update."""
bbox = box(-170., 40., 150., 80.)
original_bounds = deepcopy(bbox.bounds)
geom = GeometryVariable(name='geom', value=[bbox], dimensions='geom', crs=Spherical())
other_crs = Cartesian()
geom.update_crs(other_crs)
actual = geom.get_value()[0].bounds
desired = (-0.7544065067354889, -0.13302222155948895, -0.15038373318043535, 0.38302222155948895)
self.assertNumpyAllClose(np.array(actual), np.array(desired))
self.assertIsInstance(geom.crs, Cartesian)
other_crs = Spherical()
geom.update_crs(other_crs)
self.assertEqual(geom.crs, Spherical())
actual = geom.get_value()[0].bounds
self.assertNumpyAllClose(np.array(original_bounds), np.array(actual))
# Test data may not be wrapped.
bbox = box(0, 40, 270, 80)
geom = GeometryVariable(name='geom', value=[bbox], dimensions='geom', crs=Spherical())
other_crs = Cartesian()
with self.assertRaises(ValueError):
geom.update_crs(other_crs)
def test_init(self):
# Test empty.
gvar = GeometryVariable()
self.assertEqual(gvar.dtype, object)
gvar = self.get_geometryvariable()
self.assertIsInstance(gvar.get_masked_value(), MaskedArray)
self.assertEqual(gvar.ndim, 1)
# Test passing a "crs".
gvar = self.get_geometryvariable(crs=WGS84(),
name='my_geom',
dimensions='ngeom')
self.assertEqual(gvar.crs, WGS84())
# Test using lines.
line1 = LineString([(0, 0), (1, 1)])
line2 = LineString([(1, 1), (2, 2)])
gvar = GeometryVariable(value=[line1, line2], dimensions='two')
self.assertTrue(gvar.get_value()[1].almost_equals(line2))
self.assertEqual(gvar.geom_type, line1.geom_type)
lines = MultiLineString([line1, line2])
lines2 = [lines, lines]
for actual in [lines, lines2, lines]:
gvar2 = GeometryVariable(value=actual, dimensions='ngeom')
self.assertTrue(gvar2.get_value()[0].almost_equals(lines))
self.assertEqual(gvar2.geom_type, lines.geom_type)
self.assertTrue(gvar2.shape[0] > 0)
self.assertIsNone(gvar2.get_mask())
def test_deepcopy(self):
gvar = GeometryVariable(value=Point(1, 2), crs=Spherical(), dimensions='d')
self.assertEqual(gvar.crs, Spherical())
d = gvar.deepcopy()
d.crs = WGS84()
self.assertEqual(gvar.crs, Spherical())
self.assertFalse(np.may_share_memory(gvar.get_value(), d.get_value()))
self.assertIsNotNone(d.crs)
def test_wrap(self):
geom = box(195, -40, 225, -30)
gvar = GeometryVariable(name='geoms',
value=geom,
crs=Spherical(),
dimensions='geoms')
gvar.wrap()
self.assertEqual(gvar.get_value()[0].bounds,
(-165.0, -40.0, -135.0, -30.0))
def test_get_mask_from_intersects(self):
poly = wkt.loads(
'POLYGON((-98.26574367088608142 40.19952531645570559,-98.71764240506330168 39.54825949367089066,-99.26257911392406186 39.16281645569620906,-99.43536392405064817 38.64446202531645724,-98.78409810126584034 38.33876582278481493,-98.23916139240508016 37.71408227848101546,-97.77397151898735217 37.67420886075949937,-97.62776898734178133 38.15268987341772799,-98.39865506329114453 38.52484177215190186,-98.23916139240508016 39.33560126582278826,-97.73409810126582897 39.58813291139241386,-97.52143987341773368 40.27927215189873777,-97.52143987341773368 40.27927215189873777,-98.26574367088608142 40.19952531645570559))'
)
desired_mask = np.array([[True, True, False, True],
[True, False, True, True],
[True, True, False, True]])
dist = OcgDist()
xdim = dist.create_dimension('x', 4, dist=True)
ydim = dist.create_dimension('y', 3)
dist.create_dimension('bounds', 2)
dist.update_dimension_bounds()
if MPI_RANK == 0:
x = self.get_variable_x()
y = self.get_variable_y()
grid = Grid(x=x, y=y, abstraction='point', crs=WGS84())
pa = get_geometry_variable(grid)
else:
pa = None
pa = variable_scatter(pa, dist)
vm.create_subcomm_by_emptyable('test_get_mask_from_intersects',
pa,
is_current=True)
if vm.is_null:
self.assertTrue(pa.is_empty)
return
usi = [False]
if env.USE_SPATIAL_INDEX:
usi.append(True)
keywords = dict(use_spatial_index=usi)
for k in self.iter_product_keywords(keywords):
ret = pa.get_mask_from_intersects(
poly, use_spatial_index=k.use_spatial_index)
desired_mask_local = desired_mask[slice(*ydim.bounds_local),
slice(*xdim.bounds_local)]
if MPI_RANK > 1:
self.assertIsNone(ret)
else:
self.assertNumpyAll(desired_mask_local, ret)
# This does not test a parallel operation.
if MPI_RANK == 0:
# Test pre-masked values in geometry are okay for intersects operation.
value = [Point(1, 1), Point(2, 2), Point(3, 3)]
value = np.ma.array(value,
mask=[False, True, False],
dtype=object)
pa2 = GeometryVariable(value=value, dimensions='ngeom')
b = box(0, 0, 5, 5)
res = pa2.get_mask_from_intersects(
b, use_spatial_index=k.use_spatial_index)
self.assertNumpyAll(res, value.mask)
def test_as_shapely(self):
coords = (12, 3, 15, 4)
bbox = box(*coords)
gvar = GeometryVariable(value=bbox,
is_bbox=True,
dimensions='geom',
crs=Spherical())
geom = gvar.as_shapely()
self.assertEqual(geom.bounds, coords)
def test_deepcopy(self):
gvar = GeometryVariable(value=Point(1, 2),
crs=Spherical(),
dimensions='d')
self.assertEqual(gvar.crs, Spherical())
d = gvar.deepcopy()
d.crs = WGS84()
self.assertEqual(gvar.crs, Spherical())
self.assertFalse(np.may_share_memory(gvar.get_value(), d.get_value()))
self.assertIsNotNone(d.crs)
def test_prepare(self):
coords = (-10.0, 40.0, 50.0, 50.0)
bbox = box(*coords)
gvar = GeometryVariable(value=bbox, dimensions='geom', crs=Spherical(), is_bbox=True,
wrapped_state=WrappedState.UNWRAPPED)
for _ in range(3):
prepared = gvar.prepare()
self.assertNotEqual(id(prepared), id(gvar))
actual = []
desired = [(-10.0, 40.0, 50.0, 50.0), (350.0, 40.0, 370.0, 50.0)]
for g in prepared.get_value().flatten()[0]:
actual.append(g.bounds)
self.assertEqual(actual, desired)
# Test updating the coordinate system.
update_crs_call_count = {WGS84: 1, Spherical: 0, None: 1}
wrapped_state = [None, WrappedState.WRAPPED, WrappedState.UNWRAPPED, WrappedState.UNKNOWN]
keywords = dict(archetype_crs=update_crs_call_count.keys(), wrapped_state=wrapped_state)
for k in self.iter_product_keywords(keywords):
dgvar = deepcopy(gvar)
dgvar.update_crs = mock.Mock()
dgvar.deepcopy = mock.Mock(return_value=dgvar)
dgvar.copy = mock.Mock()
archetype = mock.create_autospec(GeometryVariable, spec_set=True)
archetype.wrapped_state = k.wrapped_state
try:
archetype.crs = k.archetype_crs()
archetype.crs.wrap_or_unwrap = mock.Mock()
except TypeError:
archetype.crs = None
_ = dgvar.prepare(archetype=archetype)
desired_count = update_crs_call_count[k.archetype_crs]
self.assertEqual(dgvar.update_crs.call_count, desired_count)
if desired_count > 0:
dgvar.update_crs.assert_called_once_with(archetype.crs)
if k.archetype_crs is not None and k.wrapped_state not in (WrappedState.UNKNOWN, None):
archetype.crs.wrap_or_unwrap.assert_called_once_with(archetype.wrapped_state, dgvar)
else:
if k.archetype_crs is not None:
archetype.crs.wrap_or_unwrap.assert_not_called()
dgvar.deepcopy.assert_called_once()
dgvar.copy.assert_not_called()
# Test identical object is returned if nothing happens.
gvar = GeometryVariable()
gvar.deepcopy = mock.Mock(spec=GeometryVariable.deepcopy)
gvar.copy = mock.Mock(spec=GeometryVariable.copy)
actual = gvar.prepare()
self.assertNotEqual(id(actual), id(gvar))
gvar.deepcopy.assert_not_called()
gvar.copy.assert_called_once()
# Test exception for more than one geometry.
gvar = mock.create_autospec(GeometryVariable, spec_set=True)
gvar.size = 2
with self.assertRaises(RequestableFeature):
GeometryVariable.prepare(gvar)
def test_get_intersection_state_boundaries(self):
path_shp = self.path_state_boundaries
geoms = []
with fiona.open(path_shp) as source:
for record in source:
geom = shape(record['geometry'])
geoms.append(geom)
gvar = GeometryVariable(value=geoms, dimensions='ngeom')
gvar_sub = gvar.get_unioned()
if gvar_sub is not None:
subset = gvar_sub.get_value().flatten()[0]
else:
subset = None
subset = MPI_COMM.bcast(subset)
resolution = 2.0
keywords = dict(with_bounds=[False])
for k in self.iter_product_keywords(keywords):
grid = self.get_gridxy_global(resolution=resolution,
with_bounds=k.with_bounds)
res = grid.get_intersection(subset)
if not res.is_empty:
self.assertTrue(res.get_mask().any())
else:
self.assertIsInstance(res, GeometryVariable)
if k.with_bounds:
area = res.area
if area is None:
area = 0.0
else:
area = area.sum()
areas = MPI_COMM.gather(area)
if MPI_RANK == 0:
area_global = sum(areas)
self.assertAlmostEqual(area_global, 1096.0819224080542)
else:
mask = res.get_mask()
if mask is None:
masked = 0
else:
masked = mask.sum()
masked = MPI_COMM.gather(masked)
if MPI_RANK == 0:
total_masked = sum(masked)
self.assertEqual(total_masked, 858)
def test_crs(self):
crs = WGS84()
gvar = GeometryVariable(crs=crs, name='var')
self.assertEqual(gvar.crs, crs)
self.assertIn(crs.name, gvar.parent)
gvar.crs = None
self.assertIsNone(gvar.crs)
self.assertEqual(len(gvar.parent), 1)
self.assertNotIn(crs.name, gvar.parent)
# Test coordinate system is maintained.
gvar = GeometryVariable(crs=crs, name='var')
vc = VariableCollection(variables=gvar)
self.assertIn(crs.name, vc)
def run_do_remove_self_intersects(self, fixture, debug=False):
poly = Polygon(fixture)
if debug:
gvar = GeometryVariable.from_shapely(poly)
gvar.write_vector('/tmp/vector.shp')
new_poly = do_remove_self_intersects_multi(poly)
self.assertEqual(
np.array(poly.exterior.coords).shape[0] - 1,
np.array(new_poly.exterior.coords).shape[0])
self.assertTrue(new_poly.is_valid)
if debug:
GeometryVariable.from_shapely(new_poly).write_vector(
'/tmp/new_vector.shp')
def test_write_variable_collection(self):
# Attempt to write without a geometry variable.
v = Variable('a', value=[1, 2], dimensions='bb')
field = Field(variables=v)
path = self.get_temporary_file_path('out.shp')
with self.assertRaises(ValueError):
field.write(path, driver=DriverVector)
# Test writing a field with two-dimensional geometry storage.
value = [Point(1, 2), Point(3, 4), Point(5, 6), Point(6, 7), Point(8, 9), Point(10, 11)]
gvar = GeometryVariable(value=value, name='points', dimensions='ngeoms')
gvar.reshape([Dimension('lat', 2), Dimension('lon', 3)])
var1 = Variable(name='dummy', value=[6, 7, 8], dimensions=['a'])
var2 = Variable(name='some_lats', value=[41, 41], dimensions=['lat'])
var3 = Variable(name='some_lons', value=[0, 90, 280], dimensions=['lon'])
var4 = Variable(name='data', value=np.random.rand(4, 3, 2), dimensions=['time', 'lon', 'lat'])
field = Field(variables=[var1, var2, var3, var4], geom=gvar, is_data=['data'])
path = self.get_temporary_file_path('2d.shp')
field.write(path, iter_kwargs={'followers': ['some_lats', 'some_lons']}, driver=DriverVector)
read = RequestDataset(uri=path).get()
self.assertTrue(len(read) > 2)
self.assertEqual(list(read.keys()),
['data', 'some_lats', 'some_lons', constants.VariableName.GEOMETRY_VARIABLE])
# Test writing a subset of the variables.
path = self.get_temporary_file_path('limited.shp')
value = [Point(1, 2), Point(3, 4), Point(5, 6)]
gvar = GeometryVariable(value=value, name='points', dimensions='points')
var1 = Variable('keep', value=[1, 2, 3], dimensions='points')
var2 = Variable('remove', value=[4, 5, 6], dimensions='points')
field = Field(variables=[var1, var2], geom=gvar, is_data=[var1])
field.write(path, variable_names=['keep'], driver=DriverVector)
read = RequestDataset(uri=path).get()
self.assertNotIn('remove', read)
# Test using append.
path = self.get_temporary_file_path('limited.shp')
value = [Point(1, 2), Point(3, 4), Point(5, 6)]
gvar = GeometryVariable(value=value, name='points', dimensions='points')
var1 = Variable('keep', value=[1, 2, 3], dimensions='points')
var2 = Variable('remove', value=[4, 5, 6], dimensions='points')
field = Field(variables=[var1, var2], geom=gvar, is_data=[var1, var2])
for idx in range(3):
sub = field[{'points': idx}]
if idx == 0:
write_mode = MPIWriteMode.WRITE
else:
write_mode = MPIWriteMode.APPEND
sub.write(path, write_mode=write_mode, driver=DriverVector)
self.assertOGRFileLength(path, idx + 1)
def test_crs(self):
crs = WGS84()
gvar = GeometryVariable(crs=crs, name='var')
self.assertEqual(gvar.crs, crs)
self.assertIn(crs.name, gvar.parent)
gvar.crs = None
self.assertIsNone(gvar.crs)
self.assertEqual(len(gvar.parent), 1)
self.assertNotIn(crs.name, gvar.parent)
# Test coordinate system is maintained.
gvar = GeometryVariable(crs=crs, name='var')
vc = VariableCollection(variables=gvar)
self.assertIn(crs.name, vc)
def test_get_mask_from_intersects(self):
poly = wkt.loads(
'POLYGON((-98.26574367088608142 40.19952531645570559,-98.71764240506330168 39.54825949367089066,-99.26257911392406186 39.16281645569620906,-99.43536392405064817 38.64446202531645724,-98.78409810126584034 38.33876582278481493,-98.23916139240508016 37.71408227848101546,-97.77397151898735217 37.67420886075949937,-97.62776898734178133 38.15268987341772799,-98.39865506329114453 38.52484177215190186,-98.23916139240508016 39.33560126582278826,-97.73409810126582897 39.58813291139241386,-97.52143987341773368 40.27927215189873777,-97.52143987341773368 40.27927215189873777,-98.26574367088608142 40.19952531645570559))')
desired_mask = np.array([[True, True, False, True],
[True, False, True, True],
[True, True, False, True]])
dist = OcgDist()
xdim = dist.create_dimension('x', 4, dist=True)
ydim = dist.create_dimension('y', 3)
dist.create_dimension('bounds', 2)
dist.update_dimension_bounds()
if MPI_RANK == 0:
x = self.get_variable_x()
y = self.get_variable_y()
grid = Grid(x=x, y=y, abstraction='point', crs=WGS84())
pa = get_geometry_variable(grid)
else:
pa = None
pa = variable_scatter(pa, dist)
vm.create_subcomm_by_emptyable('test_get_mask_from_intersects', pa, is_current=True)
if vm.is_null:
self.assertTrue(pa.is_empty)
return
usi = [False]
if env.USE_SPATIAL_INDEX:
usi.append(True)
keywords = dict(use_spatial_index=usi)
for k in self.iter_product_keywords(keywords):
ret = pa.get_mask_from_intersects(poly, use_spatial_index=k.use_spatial_index)
desired_mask_local = desired_mask[slice(*ydim.bounds_local), slice(*xdim.bounds_local)]
if MPI_RANK > 1:
self.assertIsNone(ret)
else:
self.assertNumpyAll(desired_mask_local, ret)
# This does not test a parallel operation.
if MPI_RANK == 0:
# Test pre-masked values in geometry are okay for intersects operation.
value = [Point(1, 1), Point(2, 2), Point(3, 3)]
value = np.ma.array(value, mask=[False, True, False], dtype=object)
pa2 = GeometryVariable(value=value, dimensions='ngeom')
b = box(0, 0, 5, 5)
res = pa2.get_mask_from_intersects(b, use_spatial_index=k.use_spatial_index)
self.assertNumpyAll(res, value.mask)
def test_create_ugid_global(self):
ompi = OcgDist()
m = ompi.create_dimension('m', 4)
n = ompi.create_dimension('n', 70, dist=True)
ompi.update_dimension_bounds()
gvar = GeometryVariable(name='geom', dimensions=(m, n))
ugid = gvar.create_ugid_global('gid')
if not gvar.is_empty:
self.assertEqual(gvar.dist, ugid.dist)
gathered = variable_gather(ugid)
if MPI_RANK == 0:
actual = gathered.get_value()
self.assertEqual(actual.size, len(set(actual.flatten().tolist())))
def test_convert_to_self_intersecting(self):
poly = Polygon(self.fixture_self_intersecting_polygon_coords)
gvar = GeometryVariable.from_shapely(poly)
without_si = gvar.convert_to(remove_self_intersects=True)
gvar2 = without_si.convert_to()
desired = do_remove_self_intersects_multi(poly)
self.assertPolygonSimilar(gvar2.v()[0], desired)
def test_set_geom(self):
f = Field()
self.assertIsNone(f.crs)
g = GeometryVariable(value=[Point(1, 2)], dimensions='geom', crs=Spherical())
f.set_geom(g)
def test_update_crs(self):
# Test copying allows the CRS to be updated on the copy w/out changing the source CRS.
desired = Spherical()
gvar = GeometryVariable(value=[Point(1, 2)], name='geom', dimensions='geom')
field = Field(crs=desired, geom=gvar)
cfield = field.copy()
self.assertEqual(cfield.crs, desired)
new_crs = CoordinateReferenceSystem(name='i_am_new', epsg=4326)
cfield.update_crs(new_crs)
self.assertEqual(field.crs, desired)
# Test geometry crs update is called.
mfield = Mock(Field)
mfield.is_empty = False
mfield.dimension_map = Mock(DimensionMap)
mfield.grid = Mock(Grid)
mfield.geom = Mock(GeometryVariable)
mcrs = Mock(Spherical)
Field.update_crs(mfield, mcrs)
mfield.grid.update_crs.assert_called_once_with(mcrs, from_crs=mfield.crs)
mfield.geom.update_crs.assert_called_once_with(mcrs, from_crs=mfield.crs)
from_crs = Mock(WGS84)
mfield.grid.update_crs.reset_mock()
mfield.geom.update_crs.reset_mock()
Field.update_crs(mfield, mcrs, from_crs=from_crs)
mfield.grid.update_crs.assert_called_once_with(mcrs, from_crs=from_crs)
mfield.geom.update_crs.assert_called_once_with(mcrs, from_crs=from_crs)
def test_system_with_time_data(self):
"""Test writing data with a time dimension."""
path = self.get_temporary_file_path('what.shp')
t = TemporalVariable(value=[1.5, 2.5], name='time', dimensions='time')
geom = GeometryVariable(value=[Point(1, 2), Point(3, 4)],
name='geom',
dimensions='time')
field = Field(variables=[t, geom],
dimension_map={
'time': {
'variable': 'time'
},
'geom': {
'variable': 'geom'
}
})
field.write(path,
iter_kwargs={'variable': 'time'},
driver=DriverVector)
rd = RequestDataset(uri=path)
field2 = rd.get()
# netcdftime worthlessness
poss = [['0001-01-02 12:00:00', '0001-01-03 12:00:00'],
['1-01-02 12:00:00', '1-01-03 12:00:00']]
actual = field2['TIME'].get_value().tolist()
res = [p == actual for p in poss]
self.assertTrue(any(res))
def test_create_ugid_global(self):
ompi = OcgDist()
m = ompi.create_dimension('m', 4)
n = ompi.create_dimension('n', 70, dist=True)
ompi.update_dimension_bounds()
gvar = GeometryVariable(name='geom', dimensions=(m, n))
ugid = gvar.create_ugid_global('gid')
if not gvar.is_empty:
self.assertEqual(gvar.dist, ugid.dist)
gathered = variable_gather(ugid)
if MPI_RANK == 0:
actual = gathered.get_value()
self.assertEqual(actual.size, len(set(actual.flatten().tolist())))
def test_properties(self):
# Test an empty properties dictionary is created if there are no data variables on the container/parent field.
gvar = GeometryVariable(name='empty', value=[Point(1, 2)], ugid=20, dimensions='ngeom')
info = Variable(name='info')
gvar.parent.add_variable(info)
sc = SpatialCollection()
sc.add_field(Field(name='what'), gvar.parent)
self.assertEqual(len(sc.properties), 1)
def geoms(self):
geoms = GeometryVariable(
name='geoms',
value=[Point(100.972, 41.941),
Point(102.898, 40.978)],
dimensions='ngeom',
crs=self.crs)
return geoms
def test_convert_to_geometry_coordinates_multipolygon(self):
p1 = 'Polygon ((-116.94238466549290933 52.12861711455555991, -82.00526805089285176 61.59075286434307372, -59.92695130138864101 31.0207758265680269, -107.72286778108455962 22.0438778075388484, -122.76523743459291893 37.08624746104720771, -116.94238466549290933 52.12861711455555991))'
p2 = 'Polygon ((-63.08099655131782413 21.31602121140134898, -42.70101185946779765 9.42769680782217279, -65.99242293586783603 9.912934538580501, -63.08099655131782413 21.31602121140134898))'
p1 = wkt.loads(p1)
p2 = wkt.loads(p2)
mp1 = MultiPolygon([p1, p2])
mp2 = mp1.buffer(0.1)
geoms = [mp1, mp2]
gvar = GeometryVariable(name='gc', value=geoms, dimensions='gd')
# Test the element node connectivity arrays.
results = []
for pack in [False, True]:
gc = gvar.convert_to(pack=pack)
self.assertEqual(gc.dimension_map.get_driver(),
DriverKey.NETCDF_UGRID)
self.assertTrue(gc.has_multi)
self.assertIn(OcgisConvention.Name.MULTI_BREAK_VALUE,
gc.cindex.attrs)
# Test multi-break values are part of the element node connectivity arrays.
actual = gc.cindex.get_value()
for idx, ii in enumerate(actual.flat):
self.assertGreater(
np.sum(ii == OcgisConvention.Value.MULTI_BREAK_VALUE), 0)
results.append(actual)
self.assertIsNotNone(gc.x.get_value())
self.assertIsNotNone(gc.y.get_value())
maxes = []
for ii in actual.flat:
maxes.append(ii.max())
actual_max = max(maxes)
for c in [gc.x, gc.y]:
self.assertEqual(c.size - 1, actual_max)
geoms = list(gc.iter_geometries())
for ctr, g in enumerate(geoms):
self.assertIsInstance(g[1], BaseMultipartGeometry)
self.assertEqual(ctr, 1)
self.assertPolygonSimilar(geoms[0][1], mp1)
self.assertPolygonSimilar(geoms[1][1], mp2)
for idx in range(len(results[0])):
self.assertNumpyAll(results[0][idx], results[1][idx])
def test_unwrap(self):
geom = box(195, -40, 225, -30)
gvar = GeometryVariable(name='geoms', value=geom, crs=Spherical(), dimensions='geoms')
gvar.wrap()
self.assertEqual(gvar.get_value()[0].bounds, (-165.0, -40.0, -135.0, -30.0))
gvar.unwrap()
self.assertEqual(gvar.get_value()[0].bounds, (195.0, -40.0, 225.0, -30.0))
def test_convert_to_geometry_coordinates_multipolygon_node_threshold(self):
mp = wkt.loads(strings.S7)
desired_count = len(get_split_polygon_by_node_threshold(mp, 10))
self.assertGreater(desired_count, len(mp))
gvar = GeometryVariable.from_shapely(mp)
gc = gvar.convert_to(node_threshold=10)
actual_count = gc.cindex.get_value()[0]
actual_count = np.sum(actual_count == OcgisConvention.Value.MULTI_BREAK_VALUE) + 1
self.assertEqual(actual_count, desired_count)
def test_write_variable_collection_different_data_types(self):
"""Test multiple data types are handled by the shapefile write when melted is True."""
v_int = Variable(name='an_int', value=[1, 2, 3], dtype=int, dimensions='three')
v_flt = Variable(name='a_float', value=[10., 20., 30.], dtype=float, dimensions='three')
g = GeometryVariable(name='points', value=[Point(1, 2), Point(3, 4), Point(5, 6)], dimensions='three')
field = Field(is_data=[v_int, v_flt], geom=g)
self.assertEqual(len(field.data_variables), 2)
path = self.get_temporary_file_path('foo.shp')
field.write(path, driver='vector', iter_kwargs=dict(melted=True))
def test_convert_to_geometry_coordinates_multipolygon_node_threshold(self):
mp = wkt.loads(strings.S7)
desired_count = len(get_split_polygon_by_node_threshold(mp, 10))
self.assertGreater(desired_count, len(mp))
gvar = GeometryVariable.from_shapely(mp)
gc = gvar.convert_to(node_threshold=10)
actual_count = gc.cindex.get_value()[0]
actual_count = np.sum(
actual_count == OcgisConvention.Value.MULTI_BREAK_VALUE) + 1
self.assertEqual(actual_count, desired_count)
def test_geom_type_global(self):
if MPI_SIZE != 3:
raise SkipTest('MPI_SIZE != 3')
geoms = [Point(1, 2), MultiPoint([Point(3, 4)]), Point(5, 6)]
geom = GeometryVariable(name='geom',
value=[geoms[MPI_RANK]],
dimensions='geom')
self.assertEqual(geom.geom_type, geoms[MPI_RANK].geom_type)
self.assertEqual(geom.geom_type_global, geoms[1].geom_type)
def test_convert_to_geometry_coordinates_multipolygon(self):
p1 = 'Polygon ((-116.94238466549290933 52.12861711455555991, -82.00526805089285176 61.59075286434307372, -59.92695130138864101 31.0207758265680269, -107.72286778108455962 22.0438778075388484, -122.76523743459291893 37.08624746104720771, -116.94238466549290933 52.12861711455555991))'
p2 = 'Polygon ((-63.08099655131782413 21.31602121140134898, -42.70101185946779765 9.42769680782217279, -65.99242293586783603 9.912934538580501, -63.08099655131782413 21.31602121140134898))'
p1 = wkt.loads(p1)
p2 = wkt.loads(p2)
mp1 = MultiPolygon([p1, p2])
mp2 = mp1.buffer(0.1)
geoms = [mp1, mp2]
gvar = GeometryVariable(name='gc', value=geoms, dimensions='gd')
# Test the element node connectivity arrays.
results = []
for pack in [False, True]:
gc = gvar.convert_to(pack=pack)
self.assertEqual(gc.dimension_map.get_driver(), DriverKey.NETCDF_UGRID)
self.assertTrue(gc.has_multi)
self.assertIn(OcgisConvention.Name.MULTI_BREAK_VALUE, gc.cindex.attrs)
# Test multi-break values are part of the element node connectivity arrays.
actual = gc.cindex.get_value()
for idx, ii in enumerate(actual.flat):
self.assertGreater(np.sum(ii == OcgisConvention.Value.MULTI_BREAK_VALUE), 0)
results.append(actual)
self.assertIsNotNone(gc.x.get_value())
self.assertIsNotNone(gc.y.get_value())
maxes = []
for ii in actual.flat:
maxes.append(ii.max())
actual_max = max(maxes)
for c in [gc.x, gc.y]:
self.assertEqual(c.size - 1, actual_max)
geoms = list(gc.iter_geometries())
for ctr, g in enumerate(geoms):
self.assertIsInstance(g[1], BaseMultipartGeometry)
self.assertEqual(ctr, 1)
self.assertPolygonSimilar(geoms[0][1], mp1)
self.assertPolygonSimilar(geoms[1][1], mp2)
for idx in range(len(results[0])):
self.assertNumpyAll(results[0][idx], results[1][idx])
def test_get_unioned_spatial_average_parallel(self):
if MPI_SIZE != 8:
raise SkipTest('MPI_SIZE != 8')
dist = OcgDist()
geom_count = dist.create_dimension('geom_count', size=8, dist=True)
time_count = dist.create_dimension('time', size=3)
dist.update_dimension_bounds()
if not geom_count.is_empty:
gvar = GeometryVariable(
value=[Point(1.0, 1.0).buffer(MPI_RANK + 2)] * len(geom_count),
dimensions=geom_count)
value = np.zeros((len(time_count), len(geom_count)), dtype=float)
for ii in range(value.shape[0]):
value[ii] = [MPI_RANK + 1 + ii + 1] * len(geom_count)
data = Variable(name='data',
value=value,
dtype=float,
dimensions=[time_count, geom_count])
else:
gvar = GeometryVariable(dimensions=geom_count)
data = Variable(name='data',
dimensions=[time_count, geom_count],
dtype=float)
gvar.parent.add_variable(data)
self.assertTrue(gvar.is_empty == data.is_empty == gvar.parent.is_empty)
with vm.scoped_by_emptyable('union', gvar):
if vm.is_null:
unioned = None
else:
unioned = gvar.get_unioned(spatial_average='data')
if unioned is not None:
self.assertIsInstance(unioned, GeometryVariable)
actual = unioned.parent[data.name]
self.assertAlmostEqual(actual.get_value().max(),
5.5466666666666677)
else:
self.assertIsNone(unioned)
def get_variable_collection(self, **kwargs):
parent = VariableCollection(**kwargs)
for n, v in list(self.metadata_source['variables'].items()):
SourcedVariable(name=n, request_dataset=self.rd, parent=parent)
GeometryVariable(name=DimensionName.GEOMETRY_DIMENSION,
request_dataset=self.rd,
parent=parent)
crs = self.get_crs(self.metadata_source)
if crs is not None:
parent.add_variable(crs)
return parent
def test_get_spatial_subset_circular_geometries(self):
"""Test circular geometries. They were causing wrapping errors."""
geoms = TestGeom.get_geometry_dictionaries()
rd = self.test_data.get_rd('cancm4_tas')
buffered = [element['geom'].buffer(rd.get().grid.resolution * 2) for element in geoms]
for buff in buffered:
ss = SpatialSubsetOperation(rd.get(), wrap=False)
gvar = GeometryVariable(value=buff, name='geom', dimensions='dim', crs=WGS84())
ret = ss.get_spatial_subset('intersects', gvar)
self.assertTrue(np.all(ret.grid.x.get_value() >= 0))
def test_system_conform_units(self):
"""Test conforming units on data read from shapefile."""
path = self.get_temporary_file_path('temps.shp')
gvar = GeometryVariable(value=[Point(1, 2), Point(3, 4)], dimensions='g', name='geom')
var = Variable(name='temp', value=[10., 20.], dimensions='g')
field = Field(variables=[gvar, var], geom=gvar, is_data=var)
field.write(path, driver=DriverVector)
field = RequestDataset(path, units='celsius', variable='temp', conform_units_to='fahrenheit').get()
self.assertNumpyAllClose(field['temp'].get_value(), np.array([50., 68.]))
def test_convert_to_geometry_coordinates_points(self):
pt1 = Point(1, 2, 3)
pt2 = Point(3, 4, 4)
pt3 = Point(5, 6, 5)
pt4 = Point(7, 8, 6)
crs = WGS84()
value = [pt1, pt2, pt3, pt4]
mask = [True, False, True, False]
gvar = GeometryVariable(value=value, dimensions='ngeom', crs=crs, mask=mask)
gvar_mask = gvar.get_mask().tolist()
self.assertEqual(mask, gvar_mask)
actual = gvar.convert_to()
self.assertEqual(actual.get_mask().tolist(), gvar_mask)
for actual_geom, desired_geom in zip(actual.get_geometry_iterable(use_mask=False), value):
self.assertEqual(actual_geom[1], desired_geom)
self.assertEqual(actual.crs, crs)
def get_geometryvariable_with_parent():
vpa = np.array([None, None, None])
vpa[:] = [Point(1, 2), Point(3, 4), Point(5, 6)]
value = np.arange(0, 30).reshape(10, 3)
tas = Variable(name='tas', value=value, dimensions=['time', 'ngeom'])
backref = Field(variables=[tas])
pa = GeometryVariable(value=vpa,
parent=backref,
name='point',
dimensions='ngeom')
backref[pa.name] = pa
return pa
def test_write_variable_collection(self):
# Attempt to write without a geometry variable.
v = Variable('a', value=[1, 2], dimensions='bb')
field = Field(variables=v)
path = self.get_temporary_file_path('out.shp')
with self.assertRaises(ValueError):
field.write(path, driver=DriverVector)
# Test writing a field with two-dimensional geometry storage.
value = [Point(1, 2), Point(3, 4), Point(5, 6), Point(6, 7), Point(8, 9), Point(10, 11)]
gvar = GeometryVariable(value=value, name='points', dimensions='ngeoms')
gvar.reshape([Dimension('lat', 2), Dimension('lon', 3)])
var1 = Variable(name='dummy', value=[6, 7, 8], dimensions=['a'])
var2 = Variable(name='some_lats', value=[41, 41], dimensions=['lat'])
var3 = Variable(name='some_lons', value=[0, 90, 280], dimensions=['lon'])
var4 = Variable(name='data', value=np.random.rand(4, 3, 2), dimensions=['time', 'lon', 'lat'])
field = Field(variables=[var1, var2, var3, var4], geom=gvar, is_data=['data'])
path = self.get_temporary_file_path('2d.shp')
field.write(path, iter_kwargs={'followers': ['some_lats', 'some_lons']}, driver=DriverVector)
read = RequestDataset(uri=path).get()
self.assertTrue(len(read) > 2)
self.assertEqual(list(read.keys()),
['data', 'some_lats', 'some_lons', constants.DimensionName.GEOMETRY_DIMENSION])
# Test writing a subset of the variables.
path = self.get_temporary_file_path('limited.shp')
value = [Point(1, 2), Point(3, 4), Point(5, 6)]
gvar = GeometryVariable(value=value, name='points', dimensions='points')
var1 = Variable('keep', value=[1, 2, 3], dimensions='points')
var2 = Variable('remove', value=[4, 5, 6], dimensions='points')
field = Field(variables=[var1, var2], geom=gvar, is_data=[var1])
field.write(path, variable_names=['keep'], driver=DriverVector)
read = RequestDataset(uri=path).get()
self.assertNotIn('remove', read)
# Test using append.
path = self.get_temporary_file_path('limited.shp')
value = [Point(1, 2), Point(3, 4), Point(5, 6)]
gvar = GeometryVariable(value=value, name='points', dimensions='points')
var1 = Variable('keep', value=[1, 2, 3], dimensions='points')
var2 = Variable('remove', value=[4, 5, 6], dimensions='points')
field = Field(variables=[var1, var2], geom=gvar, is_data=[var1, var2])
for idx in range(3):
sub = field[{'points': idx}]
if idx == 0:
write_mode = MPIWriteMode.WRITE
else:
write_mode = MPIWriteMode.APPEND
sub.write(path, write_mode=write_mode, driver=DriverVector)
self.assertOGRFileLength(path, idx + 1)
def test_update_crs(self):
# Test copying allows the CRS to be updated on the copy w/out changing the source CRS.
desired = Spherical()
gvar = GeometryVariable(value=[Point(1, 2)],
name='geom',
dimensions='geom')
field = Field(crs=desired, geom=gvar)
cfield = field.copy()
self.assertEqual(cfield.crs, desired)
new_crs = CoordinateReferenceSystem(name='i_am_new', epsg=4326)
cfield.update_crs(new_crs)
self.assertEqual(field.crs, desired)
def test_convert_to_geometry_coordinates_polygon_interior(self):
ph = self.fixture_polygon_with_hole
gvar = GeometryVariable.from_shapely(ph)
desired_count = len(GeometrySplitter(ph).split())
keywords = dict(split_interiors=[True, False])
for k in self.iter_product_keywords(keywords):
try:
gc = gvar.convert_to(split_interiors=k.split_interiors)
except ValueError:
self.assertFalse(k.split_interiors)
continue
actual_count = gc.cindex.get_value()[0]
actual_count = np.sum(actual_count == OcgisConvention.Value.MULTI_BREAK_VALUE) + 1
self.assertEqual(actual_count, desired_count)
def test_init(self):
# Test empty.
gvar = GeometryVariable()
self.assertEqual(gvar.dtype, object)
gvar = self.get_geometryvariable()
self.assertIsInstance(gvar.get_masked_value(), MaskedArray)
self.assertEqual(gvar.ndim, 1)
# The geometry variable should set itself as the representative geometry on its parent field if that parent does
# not have a representative geometry set.
self.assertIsNotNone(gvar.parent.geom)
# Test with a parent that already has a geometry.
field = Field()
field.set_geom(GeometryVariable(name='empty'))
gvar = self.get_geometryvariable(parent=field)
self.assertEqual(field.geom.name, 'empty')
self.assertIn(gvar.name, field)
# Test passing a "crs".
gvar = self.get_geometryvariable(crs=WGS84(), name='my_geom', dimensions='ngeom')
self.assertEqual(gvar.crs, WGS84())
# Test using lines.
line1 = LineString([(0, 0), (1, 1)])
line2 = LineString([(1, 1), (2, 2)])
gvar = GeometryVariable(value=[line1, line2], dimensions='two')
self.assertTrue(gvar.get_value()[1].almost_equals(line2))
self.assertEqual(gvar.geom_type, line1.geom_type)
lines = MultiLineString([line1, line2])
lines2 = [lines, lines]
for actual in [lines, lines2, lines]:
gvar2 = GeometryVariable(value=actual, dimensions='ngeom')
self.assertTrue(gvar2.get_value()[0].almost_equals(lines))
self.assertEqual(gvar2.geom_type, lines.geom_type)
self.assertTrue(gvar2.shape[0] > 0)
self.assertIsNone(gvar2.get_mask())
def get_spatial_subset(self, operation, geom, use_spatial_index=env.USE_SPATIAL_INDEX, buffer_value=None,
buffer_crs=None, geom_crs=None, select_nearest=False, optimized_bbox_subset=False):
"""
Perform a spatial subset operation on ``target``.
:param str operation: Either ``'intersects'`` or ``'clip'``.
:param geom: The input geometry object to use for subsetting of ``target``.
:type geom: :class:`shapely.geometry.base.BaseGeometry` | :class:`ocgis.GeometryVariable`
:param bool use_spatial_index: If ``True``, use an ``rtree`` spatial index.
:param bool select_nearest: If ``True``, select the geometry nearest ``polygon`` using
:meth:`shapely.geometry.base.BaseGeometry.distance`.
:rtype: Same as ``target``. If ``target`` is a :class:`ocgis.RequestDataset`,
then a :class:`ocgis.interface.base.field.Field` will be returned.
:param float buffer_value: The buffer radius to use in units of the coordinate system of ``subset_sdim``.
:param buffer_crs: If provided, then ``buffer_value`` are not in units of the coordinate system of
``subset_sdim`` but in units of ``buffer_crs``.
:param geom_crs: The coordinate reference system for the subset geometry.
:type geom_crs: :class:`ocgis.crs.CRS`
:param bool select_nearest: If ``True``, following the spatial subset operation, select the nearest geometry
in the subset data to ``geom``. Centroid-based distance is used.
:type buffer_crs: :class:`ocgis.interface.base.crs.CoordinateReferenceSystem`
:param bool optimized_bbox_subset: If ``True``, only do a bounding box subset and do not perform more complext
GIS subset operations such as constructing a spatial index.
:raises: ValueError
"""
if not isinstance(geom, GeometryVariable):
geom = GeometryVariable(value=geom, name='geom', dimensions='one', crs=geom_crs)
if geom.get_value().flatten().shape != (1,):
msg = 'Only one subset geometry allowed. The shape of the geometry variable is {}.'.format(geom.shape)
raise ValueError(msg)
if optimized_bbox_subset:
if self.field.grid is None:
msg = 'Subset operation must be performed on a grid when "optimized_bbox_subset=True".'
raise ValueError(msg)
if operation != 'intersects':
msg = 'Only "intersects" spatial operations when "optimized_bbox_subset=True".'
raise ValueError(msg)
# Buffer the subset if a buffer value is provided.
if buffer_value is not None:
geom = self._get_buffered_geometry_(geom, buffer_value, buffer_crs=buffer_crs)
prepared = self._prepare_geometry_(geom)
base_geometry = prepared.get_value().flatten()[0]
# Prepare the target field.
self._prepare_target_()
# execute the spatial operation
if operation == 'intersects':
if self.field.grid is None:
ret = self.field.geom.get_intersects(base_geometry, use_spatial_index=use_spatial_index,
cascade=True).parent
else:
ret = self.field.grid.get_intersects(base_geometry, cascade=True,
optimized_bbox_subset=optimized_bbox_subset).parent
elif operation in ('clip', 'intersection'):
if self.field.grid is None:
ret = self.field.geom.get_intersection(base_geometry, use_spatial_index=use_spatial_index,
cascade=True).parent
else:
ret = self.field.grid.get_intersection(base_geometry, cascade=True)
# An intersection with a grid returns a geometry variable. Set this on the field.
ret.parent.set_geom(ret)
ret = ret.parent
else:
msg = 'The spatial operation "{0}" is not supported.'.format(operation)
raise ValueError(msg)
with vm.scoped_by_emptyable('return finalize', ret):
if not vm.is_null:
# Select the nearest geometry if requested.
if select_nearest:
ret.set_abstraction_geom()
ret = ret.geom.get_nearest(base_geometry).parent
# check for rotated pole and convert back to default CRS
if self._original_rotated_pole_state is not None and self.output_crs == 'input':
ret.update_crs(self._original_rotated_pole_state)
# wrap the data...
if self._get_should_wrap_(ret):
ret.wrap()
# convert the coordinate system if requested...
if self.should_update_crs:
ret.update_crs(self.output_crs)
return ret
def from_records(cls, records, schema=None, crs=UNINITIALIZED, uid=None, union=False, data_model=None):
"""
Create a :class:`~ocgis.Field` from Fiona-like records.
:param records: A sequence of records returned from an Fiona file object.
:type records: `sequence` of :class:`dict`
:param schema: A Fiona-like schema dictionary. If ``None`` and any records properties are ``None``, then this
must be provided.
:type schema: dict
>>> schema = {'geometry': 'Point', 'properties': {'UGID': 'int', 'NAME', 'str:4'}}
:param crs: If :attr:`ocgis.constants.UNINITIALIZED`, default to :attr:`ocgis.env.DEFAULT_COORDSYS`.
:type crs: :class:`dict` | :class:`~ocgis.variable.crs.AbstractCoordinateReferenceSystem`
:param str uid: If provided, use this attribute name as the unique identifier. Otherwise search for
:attr:`env.DEFAULT_GEOM_UID` and, if not present, construct a 1-based identifier with this name.
:param bool union: If ``True``, union the geometries from records yielding a single geometry with a unique
identifier value of ``1``.
:param str data_model: See :meth:`~ocgis.driver.nc.create_typed_variable_from_data_model`.
:returns: Field object constructed from records.
:rtype: :class:`~ocgis.Field`
"""
if uid is None:
uid = env.DEFAULT_GEOM_UID
if isinstance(crs, dict):
crs = CoordinateReferenceSystem(value=crs)
elif crs == UNINITIALIZED:
crs = env.DEFAULT_COORDSYS
if union:
deque_geoms = None
deque_uid = [1]
else:
# Holds geometry objects.
deque_geoms = deque()
# Holds unique identifiers.
deque_uid = deque()
build = True
for ctr, record in enumerate(records, start=1):
# Get the geometry from a keyword present on the input dictionary or construct from the coordinates
# sequence.
try:
current_geom = record['geom']
except KeyError:
current_geom = shape(record['geometry'])
if union:
if build:
deque_geoms = current_geom
else:
deque_geoms = deque_geoms.union(current_geom)
else:
deque_geoms.append(current_geom)
# Set up the properties array
if build:
build = False
if uid in record['properties']:
has_uid = True
else:
has_uid = False
# The geometry unique identifier may be present as a property. Otherwise the enumeration counter is used for
# the identifier.
if not union:
if has_uid:
to_append = int(record['properties'][uid])
else:
to_append = ctr
deque_uid.append(to_append)
# If we are unioning, the target geometry is not yet a sequence.
if union:
deque_geoms = [deque_geoms]
# Dimension for the outgoing field.
if union:
size = 1
else:
size = ctr
dim = Dimension(name=DimensionName.GEOMETRY_DIMENSION, size=size)
# Set default geometry type if no schema is provided.
if schema is None:
geom_type = 'auto'
else:
geom_type = schema['geometry']
geom = GeometryVariable(value=deque_geoms, geom_type=geom_type, dimensions=dim)
uid = create_typed_variable_from_data_model('int', data_model=data_model, name=uid, value=deque_uid,
dimensions=dim)
geom.set_ugid(uid)
field = Field(geom=geom, crs=crs)
# All records from a unioned geometry are not relevant.
if not union:
from ocgis.driver.vector import get_dtype_from_fiona_type, get_fiona_type_from_pydata
if schema is None:
has_schema = False
else:
has_schema = True
for idx, record in enumerate(records):
if idx == 0 and not has_schema:
schema = {'properties': OrderedDict()}
for k, v in list(record['properties'].items()):
schema['properties'][k] = get_fiona_type_from_pydata(v)
if idx == 0:
for k, v in list(schema['properties'].items()):
if k == uid.name:
continue
dtype = get_dtype_from_fiona_type(v, data_model=data_model)
var = Variable(name=k, dtype=dtype, dimensions=dim)
if v.startswith('str:'):
var.set_string_max_length_global(value=int(v.split(':')[1]))
field.add_variable(var)
for k, v in list(record['properties'].items()):
if k == uid.name:
continue
if v is None:
# Mask the value if it is None. NULLs are allowed in OGR Vector files, but they do not translate
# well to Python. Strings are generally okay but floats/ints case problems.
field[k].get_mask(create=True)[idx] = v
else:
# Set the associated field value.
field[k].get_value()[idx] = v
data_variables = [uid.name]
if not union:
data_variables += [k for k in list(schema['properties'].keys()) if k != uid.name]
field.append_to_tags(TagName.DATA_VARIABLES, data_variables, create=True)
return field
def iter_src_grid_subsets(self, yield_dst=False, yield_idx=None):
"""
Yield source grid subset using the extent of its associated destination grid subset.
:param bool yield_dst: If ``True``, yield the destination subset as well as the source grid subset.
:param int yield_idx: If a zero-based integer, only yield for this chunk index and skip everything else.
:rtype: tuple(:class:`ocgis.spatial.grid.AbstractGrid`, `slice-like`)
"""
if yield_dst:
yield_slice = True
else:
yield_slice = False
buffer_value = self.buffer_value
dst_grid_wrapped_state = self.dst_grid.wrapped_state
dst_grid_crs = self.dst_grid.crs
# Use a destination grid iterator if provided.
if self.iter_dst is not None:
iter_dst = self.iter_dst(self, yield_slice=yield_slice, yield_idx=yield_idx)
else:
iter_dst = self.iter_dst_grid_subsets(yield_slice=yield_slice, yield_idx=yield_idx)
# Loop over each destination grid subset.
ocgis_lh(logger='grid_chunker', msg='starting "for yld in iter_dst"', level=logging.DEBUG)
for iter_dst_ctr, yld in enumerate(iter_dst, start=1):
ocgis_lh(msg=["iter_dst_ctr", iter_dst_ctr], level=logging.DEBUG)
if yield_slice:
dst_grid_subset, dst_slice = yld
else:
dst_grid_subset = yld
# All masked destinations are very problematic for ESMF
with vm.scoped_by_emptyable('global mask', dst_grid_subset):
if not vm.is_null:
if dst_grid_subset.has_mask_global:
if dst_grid_subset.has_mask and dst_grid_subset.has_masked_values:
all_masked = dst_grid_subset.get_mask().all()
else:
all_masked = False
all_masked_gather = vm.gather(all_masked)
if vm.rank == 0:
if all(all_masked_gather):
exc = ValueError("Destination subset all masked")
try:
raise exc
finally:
vm.abort(exc=exc)
dst_box = None
with vm.scoped_by_emptyable('extent_global', dst_grid_subset):
if not vm.is_null:
# Use the extent of the polygon for determining the bounding box. This ensures conservative
# regridding will be fully mapped.
if isinstance(dst_grid_subset, AbstractGeometryCoordinates):
target_grid = dst_grid_subset.parent.grid
else:
target_grid = dst_grid_subset
# Try to reduce the coordinates in the case of unstructured grid data.
if hasattr(target_grid, 'reduce_global') and Topology.POLYGON in target_grid.abstractions_available:
ocgis_lh(logger='grid_chunker', msg='starting reduce_global for dst_grid_subset',
level=logging.DEBUG)
target_grid = target_grid.reduce_global()
ocgis_lh(logger='grid_chunker', msg='finished reduce_global for dst_grid_subset',
level=logging.DEBUG)
extent_global = target_grid.parent.attrs.get('extent_global')
if extent_global is None:
with grid_abstraction_scope(target_grid, Topology.POLYGON):
extent_global = target_grid.extent_global
if self.check_contains:
dst_box = box(*target_grid.extent_global)
sub_box = box(*extent_global)
if buffer_value is not None:
# Use the envelope! A buffer returns "fancy" borders. We just want to expand the bounding box.
sub_box = sub_box.buffer(buffer_value).envelope
ocgis_lh(msg=str(sub_box.bounds), level=logging.DEBUG, logger='grid_chunker')
else:
sub_box, dst_box = [None, None]
live_ranks = vm.get_live_ranks_from_object(dst_grid_subset)
sub_box = vm.bcast(sub_box, root=live_ranks[0])
if self.check_contains:
dst_box = vm.bcast(dst_box, root=live_ranks[0])
sub_box = GeometryVariable.from_shapely(sub_box, is_bbox=True, wrapped_state=dst_grid_wrapped_state,
crs=dst_grid_crs)
ocgis_lh(logger='grid_chunker', msg='starting "self.src_grid.get_intersects"', level=logging.DEBUG)
src_grid_subset, src_grid_slice = self.src_grid.get_intersects(sub_box, keep_touches=False, cascade=False,
optimized_bbox_subset=self.optimized_bbox_subset,
return_slice=True)
ocgis_lh(logger='grid_chunker', msg='finished "self.src_grid.get_intersects"', level=logging.DEBUG)
# Reload the data using a new source index distribution.
if hasattr(src_grid_subset, 'reduce_global') and src_grid_subset.cindex is not None:
# Only redistribute if we have one live rank.
if self.redistribute and len(vm.get_live_ranks_from_object(src_grid_subset)) > 0:
ocgis_lh(logger='grid_chunker', msg='starting redistribute', level=logging.DEBUG)
topology = src_grid_subset.abstractions_available[Topology.POLYGON]
cindex = topology.cindex
redist_dimname = self.src_grid.abstractions_available[Topology.POLYGON].element_dim.name
if src_grid_subset.is_empty:
redist_dim = None
else:
redist_dim = topology.element_dim
redistribute_by_src_idx(cindex, redist_dimname, redist_dim)
ocgis_lh(logger='grid_chunker', msg='finished redistribute', level=logging.DEBUG)
with vm.scoped_by_emptyable('src_grid_subset', src_grid_subset):
if not vm.is_null:
if not self.allow_masked:
gmask = src_grid_subset.get_mask()
if gmask is not None and gmask.any():
raise ValueError('Masked values in source grid subset.')
if self.check_contains:
src_box = box(*src_grid_subset.extent_global)
if not does_contain(src_box, dst_box):
raise ValueError('Contains check failed.')
# Try to reduce the coordinates in the case of unstructured grid data.
if hasattr(src_grid_subset, 'reduce_global') and src_grid_subset.cindex is not None:
ocgis_lh(logger='grid_chunker', msg='starting reduce_global', level=logging.DEBUG)
src_grid_subset = src_grid_subset.reduce_global()
ocgis_lh(logger='grid_chunker', msg='finished reduce_global', level=logging.DEBUG)
else:
pass
# src_grid_subset = VariableCollection(is_empty=True)
if src_grid_subset.is_empty:
src_grid_slice = None
else:
src_grid_slice = {src_grid_subset.dimensions[ii].name: src_grid_slice[ii] for ii in
range(src_grid_subset.ndim)}
if yield_dst:
yld = (src_grid_subset, src_grid_slice, dst_grid_subset, dst_slice)
else:
yld = src_grid_subset, src_grid_slice
yield yld
def test_as_shapely(self):
coords = (12, 3, 15, 4)
bbox = box(*coords)
gvar = GeometryVariable(value=bbox, is_bbox=True, dimensions='geom', crs=Spherical())
geom = gvar.as_shapely()
self.assertEqual(geom.bounds, coords)
