def test_init(self):
g = GeomCabinetIterator(key='state_boundaries')
records = list(g)
self.assertEqual(len(records), 51)
# Test overloading the unique identifier.
path = self.get_shapefile_path_no_default_unique_identifier()
sci = GeomCabinetIterator(path=path, uid='ID')
self.assertIsNone(sci.select_sql_where)
self.assertEqual(sci.uid, 'ID')
for _ in range(2):
for record in sci:
self.assertNotIn('UGID', record['properties'])
self.assertEqual(type(record['properties']['ID']), int)
# Test slice is not allowed with other select statements.
with self.assertRaises(ValueError):
list(
GeomCabinetIterator(key='state_boundaries',
select_uid=[1],
slc=slice(0, 3)))
# Test SQL statement is stored.
s = 'STATE_NAME = "Wisconsin"'
sci = GeomCabinetIterator(key='state_boundaries', select_sql_where=s)
self.assertEqual(sci.select_sql_where, s)
def test_iter(self):
# Test with a select statement.
sci = GeomCabinetIterator(key='state_boundaries', select_sql_where="STATE_NAME in ('Wisconsin', 'Vermont')")
for row in sci:
self.assertIn(row['properties']['STATE_NAME'], ("Wisconsin", "Vermont"))
# Test with a slice object.
g = GeomCabinetIterator(key='state_boundaries', slc=slice(3, 7))
self.assertEqual(len(list(g)), 7 - 3)
def test_len(self):
path = GeomCabinet().get_shp_path('state_boundaries')
sci = GeomCabinetIterator(path=path)
self.assertEqual(len(sci), 51)
sci = GeomCabinetIterator(path=path, select_uid=[16, 19])
self.assertEqual(len(sci), 2)
sci = GeomCabinetIterator(key='state_boundaries', select_sql_where="STATE_NAME = 'Vermont'")
self.assertEqual(len(sci), 1)
sci = GeomCabinetIterator(key='state_boundaries', slc=[4, 10, 20])
self.assertEqual(len(sci), 3)
def test_from_records(self):
gci = GeomCabinetIterator(path=self.path_state_boundaries)
actual = Field.from_records(gci, data_model='NETCDF3_CLASSIC')
desired = {'UGID': np.int32,
'ID': np.int32}
for v in desired.keys():
self.assertEqual(actual[v].get_value().dtype, desired[v])
def test_iteration_by_path(self):
# test that a shapefile may be retrieved by passing a full path to the file
path = GeomCabinet().get_shp_path('state_boundaries')
ocgis.env.DIR_GEOMCABINET = None
sci = GeomCabinetIterator(path=path)
self.assertEqual(len(list(sci)), 51)
for geom in sci:
self.assertIn(type(geom['geom']), (Polygon, MultiPolygon))
def _open_(uri, mode='r', **kwargs):
if mode == 'r':
from ocgis import GeomCabinetIterator
return GeomCabinetIterator(path=uri, **kwargs)
elif mode in ('a', 'w'):
ret = fiona.open(uri, mode=mode, **kwargs)
else:
raise ValueError('Mode not supported: "{}"'.format(mode))
return ret
def test_select_ugids_absent_raises_exception(self):
path = os.path.join(self.path_bin, 'shp', 'state_boundaries', 'state_boundaries.shp')
sci = GeomCabinetIterator(path=path, select_uid=[999])
with self.assertRaises(ValueError):
list(sci)
ops = ocgis.OcgOperations(dataset=self.test_data.get_rd('cancm4_tas'),
geom=path,
select_ugid=[9999])
with self.assertRaises(ValueError):
ops.execute()
def test_number_in_shapefile_name(self):
"""Test number in shapefile name."""
sc = GeomCabinet()
path = sc.get_shp_path('state_boundaries')
out_path = os.path.join(self.current_dir_output, '51_states.shp')
with fiona.open(path) as source:
with fiona.open(out_path, mode='w', driver='ESRI Shapefile', schema=source.meta['schema'],
crs=source.meta['crs']) as sink:
for record in source:
sink.write(record)
ret = list(GeomCabinetIterator(select_uid=[23], path=out_path))
self.assertEqual(len(ret), 1)
def parse_string(self, value):
if isinstance(value, six.string_types):
if os.path.isdir(value):
exc = DefinitionValidationError(
self, 'The provided path is a directory.')
ocgis_lh(exc=exc, logger='definition')
elements = value.split('|')
try:
elements = [float(e) for e in elements]
# switch geometry creation based on length. length of 2 is a point otherwise a bounding box
if len(elements) == 2:
geom = Point(elements[0], elements[1])
else:
minx, miny, maxx, maxy = elements
geom = Polygon(
((minx, miny), (minx, maxy), (maxx, maxy), (maxx, miny)))
if not geom.is_valid:
raise DefinitionValidationError(
self, 'Parsed geometry is not valid.')
ret = [{'geom': geom, 'properties': {'ugid': 1}}]
self._bounds = elements
# try the value as a key or path
except ValueError:
# if the path exists, then assume it is a path to a shapefile, otherwise assume it is a key
kwds = {}
if os.path.exists(value):
kwds['path'] = value
else:
kwds['key'] = value
# this is saved for later use by the openclimategis metadata output as the input value is inherently
# transformed
self._shp_key = value
# get the select_ugid test value
try:
test_value = self.select_ugid.value
# it may not have been passed as a parameter object
except AttributeError:
test_value = self.select_ugid
if test_value is None:
select_ugid = None
else:
select_ugid = test_value
kwds['select_uid'] = select_ugid
kwds['select_sql_where'] = self.geom_select_sql_where
kwds['uid'] = self.geom_uid
kwds[KeywordArgument.UNION] = self.union
kwds[KeywordArgument.DATA_MODEL] = self.data_model
ret = GeomCabinetIterator(**kwds)
return ret
def _open_(uri, mode='r', **kwargs):
kwargs = kwargs.copy()
if mode == 'r':
from ocgis import GeomCabinetIterator
# The feature class keyword is driver specific.
if 'feature_class' in kwargs:
driver_kwargs = {'feature_class': kwargs.pop('feature_class')}
kwargs[KeywordArgument.DRIVER_KWARGS] = driver_kwargs
return GeomCabinetIterator(path=uri, **kwargs)
elif mode in ('a', 'w'):
ret = fiona.open(uri, mode=mode, **kwargs)
else:
raise ValueError('Mode not supported: "{}"'.format(mode))
return ret
def test_as_field(self):
"""Test iteration returned as field objects."""
select_ugid = [16, 17, 51]
desired_data_variables = ('UGID', 'STATE_FIPS', 'ID', 'STATE_NAME', 'STATE_ABBR')
sci = GeomCabinetIterator(key='state_boundaries', select_uid=select_ugid, as_field=True)
for _ in range(2):
ugids = []
for field in sci:
self.assertIsInstance(field, Field)
self.assertEqual(field.geom.ugid.shape, (1,))
self.assertEqual(get_variable_names(field.data_variables), desired_data_variables)
self.assertEqual(field.crs, WGS84())
ugids.append(field.geom.ugid.get_value()[0])
def test_system_merge_geometries_across_shapefiles(self):
geoms_to_union = []
state_names = ('Nebraska', 'South Dakota', 'North Dakota')
gci = GeomCabinetIterator(path=self.path_state_boundaries)
for row in gci:
if row['properties']['STATE_NAME'] in state_names:
geoms_to_union.append(row['geom'])
self.assertEqual(len(geoms_to_union), 3)
unioned = cascaded_union(geoms_to_union)
grid = create_gridxy_global()
field = create_exact_field(grid, 'data', crs=WGS84())
original_shape = field.grid.shape
ops = OcgOperations(dataset=field, geom=unioned)
ret = ops.execute()
actual_shape = ret.get_element().grid.shape
self.assertNotEqual(actual_shape, original_shape)
def test_keyword_geom(self):
geom = make_poly((37.762, 38.222), (-102.281, -101.754))
g = definition.Geom(geom)
self.assertEqual(type(g.value), tuple)
self.assertEqual(g.value[0].geom.get_value()[0].bounds, (-102.281, 37.762, -101.754, 38.222))
g = definition.Geom(None)
self.assertEqual(g.value, None)
self.assertEqual(str(g), 'geom=None')
g = definition.Geom('mi_watersheds')
self.assertEqual(str(g), 'geom="mi_watersheds"')
geoms = GeomCabinetIterator('mi_watersheds')
g = definition.Geom(geoms)
self.assertEqual(len(list(g.value)), 60)
self.assertEqual(g._shp_key, 'mi_watersheds')
def test_keyword_regrid_destination_to_shp_vector_wrap(self):
"""Test writing to shapefile with different vector wrap options."""
rd1 = self.test_data.get_rd('cancm4_tas')
rd2 = self.test_data.get_rd('cancm4_tas')
for vector_wrap in [True, False]:
ops = OcgOperations(dataset=rd1, regrid_destination=rd2, output_format='shp', snippet=True,
geom='state_boundaries', select_ugid=[25], vector_wrap=vector_wrap,
prefix=str(vector_wrap), melted=True)
ret = ops.execute()
sci = GeomCabinetIterator(path=ret)
geoms = [element['geom'] for element in sci]
for geom in geoms:
if vector_wrap:
self.assertLess(geom.bounds[0], 0)
else:
self.assertGreater(geom.bounds[0], 0)
def test_from_records(self):
gci = GeomCabinetIterator(path=self.path_state_boundaries)
actual = Field.from_records(gci, data_model='NETCDF3_CLASSIC')
desired = {'UGID': np.int32, 'ID': np.int32}
for v in desired.keys():
self.assertEqual(actual[v].get_value().dtype, desired[v])
# Test strictness with list/tuple
def _run_():
env.SUPPRESS_WARNINGS = False
ocgis_lh.configure(to_stream=True)
records = [{
'geom': Point(1, 2),
'properties': {
'a_list': [1, 2, 3]
}
}]
actual = Field.from_records(records)
self.assertNotIn("a_list", actual.keys())
env.SUPPRESS_WARNINGS = True
self.assertWarns(OcgWarning, _run_)
def test_add_shapefile_unique_identifier(self):
in_path = os.path.join(self.current_dir_output, 'foo_in.shp')
# create a shapefile without a ugid and another integer attribute
data = [{
'geom': Point(1, 2),
'fid': 6
}, {
'geom': Point(2, 3),
'fid': 60
}]
crs = Spherical()
driver = 'ESRI Shapefile'
schema = {'properties': {'fid': 'int'}, 'geometry': 'Point'}
with fiona.open(in_path,
'w',
driver=driver,
crs=crs.value,
schema=schema) as source:
for xx in data:
record = {
'properties': {
'fid': xx['fid']
},
'geometry': mapping(xx['geom'])
}
source.write(record)
out_path = os.path.join(self.current_dir_output, 'foo_out.shp')
add_shapefile_unique_identifier(in_path, out_path)
sci = GeomCabinetIterator(path=out_path)
records = list(sci)
self.assertAsSetEqual([1, 2], [
xx['properties'][OCGIS_UNIQUE_GEOMETRY_IDENTIFIER]
for xx in records
])
self.assertAsSetEqual([6, 60],
[xx['properties']['fid'] for xx in records])
actual_crs = create_crs(records[0]['meta']['crs'])
self.assertEqual(actual_crs, crs)
# test it works for the current working directory
cwd = os.getcwd()
os.chdir(self.current_dir_output)
try:
add_shapefile_unique_identifier(in_path, 'foo3.shp')
self.assertTrue(
os.path.exists(
os.path.join(self.current_dir_output, 'foo3.shp')))
finally:
os.chdir(cwd)
# test using a template attribute
out_path = os.path.join(self.current_dir_output, 'template.shp')
add_shapefile_unique_identifier(in_path, out_path, template='fid')
sci = GeomCabinetIterator(path=out_path)
records = list(sci)
self.assertAsSetEqual([6, 60], [
xx['properties'][OCGIS_UNIQUE_GEOMETRY_IDENTIFIER]
for xx in records
])
# test with a different name attribute
out_path = os.path.join(self.current_dir_output, 'name.shp')
add_shapefile_unique_identifier(in_path,
out_path,
template='fid',
name='new_id')
with fiona.open(out_path) as sci:
records = list(sci)
self.assertAsSetEqual([6, 60],
[xx['properties']['new_id'] for xx in records])
def nebraska(self):
path = os.path.join(self.path_bin, 'shp', 'state_boundaries', 'state_boundaries.shp')
sci = GeomCabinetIterator(path=path, select_uid=[16])
geom = list(sci)
return geom
def test_key_used_before_path(self):
# the key always takes preference over the path
sci = GeomCabinetIterator(key='state_boundaries',
path='/foo/foo/foo/foo/foo')
self.assertEqual(len(list(sci)), 51)
def test_iteration_by_path_with_bad_path(self):
# if the path does not exist on the filesystem, then an exception should be raised
ocgis.env.DIR_GEOMCABINET = None
sci = GeomCabinetIterator(path='/foo/foo/foo/foo/foo')
with self.assertRaises(RuntimeError):
list(sci)
def test_iteration_no_geoms(self):
sci = GeomCabinetIterator(key='state_boundaries', load_geoms=False)
for geom in sci:
self.assertNotIn('geom', geom)