def test_calculation_iteration(self):
field = self.get_field(with_value=True,month_count=2)
field.variables.add_variable(Variable(value=field.variables['tmax'].value+5,
name='tmin',alias='tmin'))
field.temporal.name_uid = 'tid'
field.level.name_uid = 'lid'
field.spatial.geom.name_uid = 'gid'
grouping = ['month']
tgd = field.temporal.get_grouping(grouping)
mu = Mean(field=field,tgd=tgd,alias='my_mean')
ret = mu.execute()
kwds = copy(field.__dict__)
kwds.pop('_raw')
kwds.pop('_variables')
kwds['temporal'] = tgd
kwds['variables'] = ret
cfield = DerivedField(**kwds)
cfield.temporal.name_uid = 'tid'
cfield.temporal.name_value = 'time'
cfield.spatial.name_uid = 'gid'
sc = ShpCabinet()
meta = sc.get_meta('state_boundaries')
sp = SpatialCollection(meta=meta,key='state_boundaries',headers=constants.calc_headers)
for row in sc.iter_geoms('state_boundaries'):
sp.add_field(row['properties']['UGID'],row['geom'],cfield.variables.keys()[0],
cfield,properties=row['properties'])
for ii,row in enumerate(sp.get_iter_dict()):
if ii == 0:
self.assertEqual(row[0].bounds,(-100.5, 39.5, -99.5, 40.5))
self.assertDictEqual(row[1],{'lid': 1, 'ugid': 1, 'vid': 1, 'cid': 1, 'did': 1, 'year': 2000, 'time': datetime.datetime(2000, 1, 16, 0, 0), 'calc_alias': 'my_mean_tmax', 'value': 0.44808476666433006, 'month': 1, 'alias': 'tmax', 'variable': 'tmax', 'gid': 1, 'calc_key': 'mean', 'tid': 1, 'level': 50, 'day': 16})
self.assertEqual(len(row),2)
self.assertEqual(len(row[1]),len(constants.calc_headers))
def get_shp(request,key=None):
query = helpers.parse_qs(request.META['QUERY_STRING'])
select_ugid = SelectUgid()
select_ugid.parse_query(query)
prefix = Prefix()
prefix.parse_query(query)
unwrap = Unwrap()
unwrap.parse_query(query)
pm = PrimeMeridian()
pm.parse_query(query)
sc = ShpCabinet()
geom_dict = sc.get_geom_dict(key,attr_filter=select_ugid.value)
## unwrap coordinates if requested
if unwrap.value:
unwrap_geoms(geom_dict,pm.value)
dir_path = get_temp_path(nest=True,only_dir=True,wd=env.DIR_OUTPUT)
if prefix.value is None:
out_name = key
else:
out_name = prefix.value
filename = '{0}.shp'.format(out_name)
path = os.path.join(dir_path,filename)
path = sc.write(geom_dict,path)
path = os.path.split(path)[0]
resp = helpers._zip_response_(path,filename=filename.replace('shp','zip'))
return(resp)
def test_get_keys(self, dir_shpcabinet=None):
ocgis.env.DIR_SHPCABINET = dir_shpcabinet or ocgis.env.DIR_SHPCABINET
sc = ShpCabinet()
ret = sc.keys()
target_keys = ['state_boundaries', 'world_countries']
self.assertEqual(len(set(target_keys).intersection(set(ret))),
len(target_keys))
def _write_(self):
gid_file = OrderedDict()
build = True
is_aggregated = self.ops.aggregate
with open(self.path,'w') as f:
ocgis_lh(msg='opened csv file: {0}'.format(self.path),level=logging.DEBUG,
logger='conv.csv+')
writer = csv.writer(f,dialect=OcgDialect)
for coll in self:
ocgis_lh('writing collection','conv.csv+',level=logging.DEBUG)
if build:
ocgis_lh('starting build','conv.csv+',level=logging.DEBUG)
headers = coll.get_headers(upper=True)
if env.WRITE_TO_REFERENCE_PROJECTION:
projection = env.REFERENCE_PROJECTION
else:
projection = coll._archetype.spatial.projection
writer.writerow(headers)
build = False
ocgis_lh(msg='build finished'.format(self.path),level=logging.DEBUG,
logger='conv.csv+')
for geom,row,geom_ids in coll.get_iter(with_geometry_ids=True):
if not is_aggregated:
ugid = geom_ids['ugid']
did = geom_ids['did']
gid = geom_ids['gid']
if ugid not in gid_file:
gid_file[ugid] = OrderedDict()
if did not in gid_file[ugid]:
gid_file[ugid][did] = OrderedDict()
gid_file[ugid][did][gid] = geom
writer.writerow(row)
ocgis_lh('finished writing collection','conv.csv+',level=logging.DEBUG)
if is_aggregated is True:
ocgis_lh('creating a UGID-GID shapefile is not necessary for aggregated data. use UGID shapefile.',
'conv.csv+',
logging.WARN)
else:
ocgis_lh('writing UGID-GID shapefile','conv.csv+',logging.DEBUG)
sc = ShpCabinet()
shp_dir = os.path.join(self.outdir,'shp')
try:
os.mkdir(shp_dir)
## catch if the directory exists
except OSError:
if os.path.exists(shp_dir):
pass
else:
raise
shp_path = os.path.join(shp_dir,self.prefix+'_gid.shp')
def iter_gid_file():
for ugid,did_gid in gid_file.iteritems():
for did,gid_geom in did_gid.iteritems():
for gid,geom in gid_geom.iteritems():
yield({'geom':geom,'DID':did,
'UGID':ugid,'GID':gid})
sc.write(iter_gid_file(),shp_path,sr=projection.sr)
def write(self,path):
geoms = []
uid = self.spatial.uid
for ii,geom in iter_array(self.spatial.geom,return_value=True):
geoms.append({'geom':geom,'ugid':uid[ii]})
sc = ShpCabinet()
sc.write(geoms,path,sr=self.spatial.projection.sr)
def write(self, path):
geoms = []
uid = self.spatial.uid
for ii, geom in iter_array(self.spatial.geom, return_value=True):
geoms.append({'geom': geom, 'ugid': uid[ii]})
sc = ShpCabinet()
sc.write(geoms, path, sr=self.spatial.projection.sr)
def test_shp(self):
sc = ShpCabinet()
keys = sc.keys()
for key in keys:
url = '/shp/{0}'.format(key)
resp = self.c.get(url)
self.assertTrue(len(resp.content) > 100)
def test_shp(self):
sc = ShpCabinet()
keys = sc.keys()
for key in keys:
url = '/shp/{0}'.format(key)
resp = self.c.get(url)
self.assertTrue(len(resp.content) > 100)
def test_multivariate_iteration(self):
field = self.get_field(with_value=True, month_count=1)
field.variables.add_variable(Variable(value=field.variables['tmax'].value + 5,
name='tmin', alias='tmin'))
field.temporal.name_uid = 'tid'
field.level.name_uid = 'lid'
field.spatial.geom.name_uid = 'gid'
div = Divide(field=field, parms={'arr1': 'tmin', 'arr2': 'tmax'}, alias='some_division',
dtype=np.float64)
ret = div.execute()
cfield = DerivedMultivariateField(variables=ret, realization=field.realization, temporal=field.temporal,
level=field.level,
spatial=field.spatial, meta=field.meta, uid=field.uid)
cfield.spatial.name_uid = 'gid'
sc = ShpCabinet()
meta = sc.get_meta('state_boundaries')
sp = SpatialCollection(meta=meta, key='state_boundaries', headers=constants.HEADERS_MULTI)
for row in sc.iter_geoms('state_boundaries', as_spatial_dimension=True):
sp.add_field(cfield, ugeom=row)
for ii, row in enumerate(sp.get_iter_dict(melted=True)):
if ii == 0:
self.assertDictEqual(row[1], {'lid': 1, 'ugid': 1, 'cid': 1, 'did': None, 'year': 2000,
'time': datetime.datetime(2000, 1, 1, 12, 0),
'calc_alias': 'some_division', 'value': 12.989774984574424, 'month': 1,
'gid': 1, 'calc_key': 'divide', 'tid': 1, 'level': 50, 'day': 1})
self.assertEqual(ii + 1, 2 * 31 * 2 * 3 * 4 * 51)
def get_shp(request, key=None):
query = helpers.parse_qs(request.META['QUERY_STRING'])
select_ugid = SelectUgid()
select_ugid.parse_query(query)
prefix = Prefix()
prefix.parse_query(query)
unwrap = Unwrap()
unwrap.parse_query(query)
pm = PrimeMeridian()
pm.parse_query(query)
sc = ShpCabinet()
geom_dict = sc.get_geom_dict(key, attr_filter=select_ugid.value)
## unwrap coordinates if requested
if unwrap.value:
unwrap_geoms(geom_dict, pm.value)
dir_path = get_temp_path(nest=True, only_dir=True, wd=env.DIR_OUTPUT)
if prefix.value is None:
out_name = key
else:
out_name = prefix.value
filename = '{0}.shp'.format(out_name)
path = os.path.join(dir_path, filename)
path = sc.write(geom_dict, path)
path = os.path.split(path)[0]
resp = helpers._zip_response_(path,
filename=filename.replace('shp', 'zip'))
return (resp)
def test_iter_all(self):
raise(SkipTest('dev - long'))
sc = ShpCabinet()
for key in sc.keys():
print key
for geom in sc.iter_geoms(key):
self.assertTrue(True)
def test_sql_subset(self):
sc = ShpCabinet()
path = sc.get_shp_path('state_boundaries')
ds = ogr.Open(path)
ret = ds.ExecuteSQL('select * from state_boundaries where state_name = "New Jersey"')
ret.ResetReading()
self.assertEqual(len(ret),1)
def test_sql_subset(self):
sc = ShpCabinet()
path = sc.get_shp_path('state_boundaries')
ds = ogr.Open(path)
ret = ds.ExecuteSQL(
'select * from state_boundaries where state_name = "New Jersey"')
ret.ResetReading()
self.assertEqual(len(ret), 1)
def get_collection(self):
field = self.get_field(with_value=True)
sc = ShpCabinet()
meta = sc.get_meta('state_boundaries')
sp = SpatialCollection(meta=meta, key='state_boundaries')
for row in sc.iter_geoms('state_boundaries', as_spatial_dimension=True):
sp.add_field(field, ugeom=row)
return sp
def test_unwrap(self):
sc = ShpCabinet()
_key = ['state_boundaries','world_countries']
for key in _key:
geoms = sc.get_geoms(key,unwrap=True)
for geom in geoms:
x = geom['geom'].centroid.x
self.assertTrue(x > 0)
def test_iter_geoms_no_load_geoms(self):
sc = ShpCabinet()
it = sc.iter_geoms('state_boundaries',load_geoms=False)
geoms = list(it)
self.assertEqual(len(geoms),51)
self.assertEqual(geoms[12]['properties']['STATE_NAME'],'New Hampshire')
for geom in geoms:
self.assertNotIn('geom',geom)
def test_iter_geoms(self):
sc = ShpCabinet()
it = sc.iter_geoms('state_boundaries')
geoms = list(it)
self.assertEqual(len(geoms),51)
self.assertEqual(geoms[12]['properties']['STATE_NAME'],'New Hampshire')
for geom in geoms:
self.assertIn(type(geom['geom']),(Polygon,MultiPolygon))
def test_unwrap_pm(self):
_pm = [-4.0,-10.0,-20.0,5.0]
sc = ShpCabinet()
for pm in _pm:
geoms = sc.get_geoms('world_countries',unwrap=True,pm=pm)
path = '/tmp/shp{0}.shp'.format(time.time())
sc.write(geoms,path)
for geom in geoms:
bounds = geom['geom'].bounds
self.assertTrue(bounds[0] >= pm)
def write(self,path):
geoms = []
uid = self.spatial.uid
attrs = self.spatial.attrs
for ii,geom in iter_array(self.spatial.geom,return_value=True):
dct = {'geom':geom,'UGID':uid[ii]}
for k,v in attrs.iteritems():
dct[k] = v[ii]
geoms.append(dct)
sc = ShpCabinet()
sc.write(geoms,path,sr=self.spatial.projection.sr)
def write(self, path):
geoms = []
uid = self.spatial.uid
attrs = self.spatial.attrs
for ii, geom in iter_array(self.spatial.geom, return_value=True):
dct = {'geom': geom, 'UGID': uid[ii]}
for k, v in attrs.iteritems():
dct[k] = v[ii]
geoms.append(dct)
sc = ShpCabinet()
sc.write(geoms, path, sr=self.spatial.projection.sr)
def test_shapefiles_not_in_folders(self):
for dirpath,dirnames,filenames in os.walk(ocgis.env.DIR_SHPCABINET):
for filename in filenames:
if filename.startswith('state_boundaries') or filename.startswith('world_countries'):
dst = os.path.join(self._test_dir,filename)
src = os.path.join(dirpath,filename)
shutil.copy2(src,dst)
self.test_get_keys(dir_shpcabinet=self._test_dir)
sc = ShpCabinet(path=self._test_dir)
path = sc.get_shp_path('world_countries')
self.assertEqual(path,os.path.join(self._test_dir,'world_countries.shp'))
def test_constructor(self):
field = self.get_field(with_value=True)
sc = ShpCabinet()
meta = sc.get_meta('state_boundaries')
sp = SpatialCollection(meta=meta,key='state_boundaries')
for row in sc.iter_geoms('state_boundaries'):
sp.add_field(row['properties']['UGID'],row['geom'],field.variables.keys()[0],
field,properties=row['properties'])
self.assertEqual(len(sp),51)
self.assertIsInstance(sp.geoms[25],MultiPolygon)
self.assertIsInstance(sp.properties[25],dict)
self.assertEqual(sp[25]['tmax'].variables['tmax'].value.shape,(2, 31, 2, 3, 4))
def test_number_in_shapefile_name(self):
"""Test number in shapefile name."""
sc = ShpCabinet()
path = sc.get_shp_path('state_boundaries')
out_path = os.path.join(self.current_dir_output, '51_states.shp')
with fiona.open(path) as source:
with fiona.open(out_path, mode='w', driver='ESRI Shapefile', schema=source.meta['schema'],
crs=source.meta['crs']) as sink:
for record in source:
sink.write(record)
ret = list(ShpCabinetIterator(select_uid=[23], path=out_path))
self.assertEqual(len(ret), 1)
def _format_all_elements_(self,value):
try:
minx,miny,maxx,maxy = value
geom = Polygon(((minx,miny),
(minx,maxy),
(maxx,maxy),
(maxx,miny)))
assert(geom.is_valid)
ret = [{'id':1,'geom':geom}]
except ValueError:
sc = ShpCabinet()
ret = sc.get_geom_dict(value[0])
return(ret)
def test_shp_process(self):
copy_path = os.path.join(self._test_dir,'test_shp_process')
shutil.copytree(self._test_path,copy_path)
shp_path = os.path.join(copy_path,'wc_4326.shp')
out_folder = tempfile.mkdtemp(dir=self._test_dir)
sp = ShpProcess(shp_path,out_folder)
sp.process(key='world_countries',ugid=None)
sc = ShpCabinet(path=out_folder)
select_ugid = [33,126,199]
geoms = list(sc.iter_geoms('world_countries',select_ugid=select_ugid))
self.assertEqual(len(geoms),3)
names = [item['properties']['NAME'] for item in geoms]
self.assertEqual(set(names),set(['Canada','Mexico','United States']))
def test_iter_geoms(self):
sc = ShpCabinet()
it = sc.iter_geoms('state_boundaries')
geoms = list(it)
self.assertEqual(len(geoms), 51)
self.assertEqual(geoms[12]['properties']['STATE_NAME'], 'New Hampshire')
for geom in geoms:
self.assertIn(type(geom['geom']), (Polygon, MultiPolygon))
# test with a shapefile not having a unique identifier
env.DEFAULT_GEOM_UID = 'ggidd'
new = self.get_shapefile_path_with_no_ugid()
sc = ShpCabinet()
target = list(sc.iter_geoms(path=new))
self.assertEqual(len(target), 11)
self.assertEqual(target[0]['properties'][env.DEFAULT_GEOM_UID], 1)
self.assertEqual(target[3]['properties'][env.DEFAULT_GEOM_UID], 4)
target = list(sc.iter_geoms(path=new, uid='ID'))
self.assertNotIn(env.DEFAULT_GEOM_UID, target[9]['properties'])
self.assertEqual(int, type(target[7]['properties']['ID']))
target = list(sc.iter_geoms(path=new, uid='ID', as_spatial_dimension=True))
ref = target[4]
self.assertEqual(ref.uid[0, 0], 10)
self.assertNotIn(env.DEFAULT_GEOM_UID, ref.properties.dtype.names)
# test with a different geometry unique identifier
path = self.get_shapefile_path_with_no_ugid()
geom_select_uid = [12, 15]
geom_uid = 'ID'
sc = ShpCabinet()
records = list(sc.iter_geoms(path=path, uid=geom_uid, select_uid=geom_select_uid))
self.assertEqual(len(records), 2)
self.assertEqual([r['properties']['ID'] for r in records], geom_select_uid)
def test_process_name(self):
copy_path = os.path.join(self.current_dir_output, 'test_shp_process')
sc = ShpCabinet()
test_path = os.path.split(sc.get_shp_path('wc_4326'))[0]
shutil.copytree(test_path, copy_path)
shp_path = os.path.join(copy_path, 'wc_4326.shp')
out_folder = tempfile.mkdtemp(dir=self.current_dir_output)
sp = ShpProcess(shp_path, out_folder)
sp.process(key='world_countries', ugid=None, name='new_id')
path = os.path.join(out_folder, 'world_countries.shp')
with fiona.open(path, 'r') as sci:
uids = [record['properties']['new_id'] for record in sci]
self.assertEqual(uids, range(1, 212))
def test_calculation_iteration_two_calculations(self):
field = self.get_field(with_value=True, month_count=2)
field.variables.add_variable(Variable(value=field.variables['tmax'].value + 5, name='tmin', alias='tmin'))
field.temporal.name_uid = 'tid'
field.level.name_uid = 'lid'
field.spatial.geom.name_uid = 'gid'
grouping = ['month']
tgd = field.temporal.get_grouping(grouping)
mu = Mean(field=field, tgd=tgd, alias='my_mean', dtype=np.float64, add_parents=True)
ret = mu.execute()
thresh = Threshold(field=field, vc=ret, tgd=tgd, alias='a_treshold', add_parents=True,
parms={'operation': 'gte', 'threshold': 0.5})
ret = thresh.execute()
kwds = copy(field.__dict__)
kwds.pop('_raw')
kwds.pop('_variables')
kwds.pop('_should_regrid')
kwds.pop('_has_assigned_coordinate_system')
kwds.pop('_attrs')
kwds['name'] = kwds.pop('_name')
kwds['temporal'] = tgd
kwds['variables'] = ret
cfield = DerivedField(**kwds)
cfield.temporal.name_uid = 'tid'
cfield.temporal.name_value = 'time'
cfield.spatial.name_uid = 'gid'
sc = ShpCabinet()
meta = sc.get_meta('state_boundaries')
sp = SpatialCollection(meta=meta, key='state_boundaries', headers=constants.HEADERS_CALC)
for row in sc.iter_geoms('state_boundaries', as_spatial_dimension=True):
sp.add_field(cfield, ugeom=row)
cids = set()
for ii, row in enumerate(sp.get_iter_dict(melted=True)):
cids.update([row[1]['cid']])
if ii == 0:
self.assertEqual(row[0].bounds, (-100.5, 39.5, -99.5, 40.5))
self.assertDictEqual(row[1], {'lid': 1, 'ugid': 1, 'vid': 1, 'cid': 1, 'did': 1, 'year': 2000,
'time': datetime.datetime(2000, 1, 16, 0, 0),
'calc_alias': 'my_mean_tmax', 'value': 0.44808476666433006, 'month': 1,
'alias': 'tmax', 'variable': 'tmax', 'gid': 1, 'calc_key': 'mean',
'tid': 1, 'level': 50, 'day': 16})
self.assertEqual(len(row), 2)
self.assertEqual(len(row[1]), len(constants.HEADERS_CALC))
self.assertEqual(ii + 1, 2 * 2 * 2 * 3 * 4 * 51 * 4)
self.assertEqual(len(cids), 4)
def _format_string_element_(self, value):
elements = value.split('|')
try:
elements = [float(e) for e in elements]
minx, miny, maxx, maxy = elements
geom = Polygon(
((minx, miny), (minx, maxy), (maxx, maxy), (maxx, miny)))
self._assert_(geom.is_valid)
ret = [{'ugid': 1, 'geom': geom}]
self._bounds = elements
except ValueError:
from ocgis.util.shp_cabinet import ShpCabinet
sc = ShpCabinet()
ret = sc.get_geom_dict(value)
return (ret)
def test_shp_process(self):
test_path = '/home/local/WX/ben.koziol/Dropbox/nesii/project/ocg/bin/test_data/test_shp_process'
copy_path = os.path.join(self._test_dir,'test_shp_process')
shutil.copytree(test_path,copy_path)
shp_path = os.path.join(copy_path,'wc_4326.shp')
sp = ShpProcess(shp_path)
out_folder = tempfile.mkdtemp(dir=self._test_dir)
sp.process(out_folder,'world_countries',ugid=None)
sc = ShpCabinet(path=out_folder)
select_ugid = [33,126,199]
geoms = list(sc.iter_geoms('world_countries',select_ugid=select_ugid))
self.assertEqual(len(geoms),3)
names = [item['properties']['NAME'] for item in geoms]
self.assertEqual(set(names),set(['Canada','Mexico','United States']))
def test_shapefiles_not_in_folders(self):
for dirpath, dirnames, filenames in os.walk(ocgis.env.DIR_SHPCABINET):
for filename in filenames:
if filename.startswith(
'state_boundaries') or filename.startswith(
'world_countries'):
dst = os.path.join(self._test_dir, filename)
src = os.path.join(dirpath, filename)
shutil.copy2(src, dst)
self.test_get_keys(dir_shpcabinet=self._test_dir)
sc = ShpCabinet(path=self._test_dir)
path = sc.get_shp_path('world_countries')
self.assertEqual(path,
os.path.join(self._test_dir, 'world_countries.shp'))
def _run_(s, func):
try:
ds = ogr.Open(path)
obj = ShpCabinet._get_features_object_(ds, select_sql_where=s)
func(obj)
finally:
ds.Destroy()
def test_shp_process(self):
test_path = '/home/local/WX/ben.koziol/Dropbox/nesii/project/ocg/bin/test_data/test_shp_process'
copy_path = os.path.join(self._test_dir, 'test_shp_process')
shutil.copytree(test_path, copy_path)
shp_path = os.path.join(copy_path, 'wc.shp')
sp = ShpProcess(shp_path)
out_folder = tempfile.mkdtemp(dir=self._test_dir)
sp.process(out_folder, 'world_countries', ugid=None)
sc = ShpCabinet(path=out_folder)
select_ugid = [33, 126, 199]
geoms = sc.get_geoms('world_countries', select_ugid=select_ugid)
self.assertEqual(len(geoms), 3)
names = [item['NAME'] for item in geoms]
self.assertEqual(set(names), set(['Canada', 'Mexico',
'United States']))
def test_geom(self):
geom = make_poly((37.762, 38.222), (-102.281, -101.754))
g = Geom(geom)
self.assertEqual(type(g.value), GeometryDataset)
g.value = None
self.assertEqual(None, g.value)
g = Geom(None)
self.assertEqual(g.value, None)
self.assertEqual(str(g), 'geom=None')
g = Geom('-120|40|-110|50')
self.assertEqual(g.value.spatial.geom[0].bounds,
(-120.0, 40.0, -110.0, 50.0))
self.assertEqual(str(g), 'geom=-120.0|40.0|-110.0|50.0')
g = Geom('mi_watersheds')
self.assertEqual(str(g), 'geom=mi_watersheds')
geoms = ShpCabinet().get_geoms('mi_watersheds')
g = Geom('mi_watersheds')
self.assertEqual(len(g.value), len(geoms))
su = SelectUgid([1, 2, 3])
g = Geom('mi_watersheds', select_ugid=su)
self.assertEqual(len(g.value), 3)
geoms = GeometryDataset(uid=[1, 2], geom=[geom, geom])
g = Geom(geoms)
bbox = [-120, 40, -110, 50]
g = Geom(bbox)
self.assertEqual(g.value.spatial.geom[0].bounds,
tuple(map(float, bbox)))
def _format_string_element_(self,value):
elements = value.split('|')
try:
elements = [float(e) for e in elements]
minx,miny,maxx,maxy = elements
geom = Polygon(((minx,miny),
(minx,maxy),
(maxx,maxy),
(maxx,miny)))
self._assert_(geom.is_valid)
ret = [{'ugid':1,'geom':geom}]
self._bounds = elements
except ValueError:
from ocgis.util.shp_cabinet import ShpCabinet
sc = ShpCabinet()
ret = sc.get_geom_dict(value)
return(ret)
def test_none_path(self):
try:
ocgis.env.DIR_SHPCABINET = None
with self.assertRaises(ValueError):
ShpCabinet().get_geoms('state_boundaries')
finally:
ocgis.env.reset()
def test_iteration(self):
field = self.get_field(with_value=True)
field.temporal.name_uid = 'tid'
field.level.name_uid = 'lid'
field.spatial.geom.name_uid = 'gid'
field.spatial.name_uid = 'gid'
sc = ShpCabinet()
meta = sc.get_meta('state_boundaries')
sp = SpatialCollection(meta=meta,key='state_boundaries')
for row in sc.iter_geoms('state_boundaries'):
sp.add_field(row['properties']['UGID'],row['geom'],field.variables.keys()[0],
field,properties=row['properties'])
for ii,row in enumerate(sp.get_iter_dict()):
if ii == 1:
self.assertEqual(row[1],{'lid': 1, 'ugid': 1, 'vid': 1, 'alias': 'tmax', 'did': 1, 'year': 2000, 'value': 0.7203244934421581, 'month': 1, 'variable': 'tmax', 'gid': 2, 'time': datetime.datetime(2000, 1, 1, 12, 0), 'tid': 1, 'level': 50, 'day': 1})
self.assertIsInstance(row[0],MultiPolygon)
self.assertEqual(len(row),2)
self.assertEqual(len(row[1]),len(constants.raw_headers))
def test_attribute_flags(self):
attr_flags = ['none','all']
for attr_flag in attr_flags:
## make temporary directory
path = get_temp_path(only_dir=True,nest=True)
try:
## the shapefile key to use
key = 'state_boundaries'
## get path to shapefile and copy directory to temporary location
sc = ShpCabinet()
origin = os.path.split(sc.get_shp_path(key))[0]
shutil.copytree(origin,os.path.join(path,key))
## remove original config file
os.remove(os.path.join(path,key,key+'.cfg'))
## write config file
config = ConfigParser.ConfigParser()
config.add_section('mapping')
config.set('mapping','ugid','none')
config.set('mapping','attributes',attr_flag)
with open(os.path.join(path,key,key+'.cfg'),'w') as f:
config.write(f)
## load data
sc = ShpCabinet(path)
geoms = sc.get_geoms(key)
for geom in geoms:
if attr_flag == 'none':
self.assertEqual(set(['ugid','geom']),set(geom.keys()))
else:
self.assertEqual(set(['ugid', 'state_name', 'state_fips', 'geom', 'state_abbr', 'id']),
set(geom.keys()))
finally:
shutil.rmtree(path)
def test_iteration_methods(self):
field = self.get_field(with_value=True)
field.temporal.name_uid = 'tid'
field.level.name_uid = 'lid'
field.spatial.geom.name_uid = 'gid'
field.spatial.name_uid = 'gid'
sc = ShpCabinet()
meta = sc.get_meta('state_boundaries')
sp = SpatialCollection(meta=meta, key='state_boundaries', headers=constants.HEADERS_RAW)
for row in sc.iter_geoms('state_boundaries', as_spatial_dimension=True):
sp.add_field(field, ugeom=row)
for ii, row in enumerate(sp.get_iter_dict(melted=True)):
if ii == 1:
self.assertDictEqual(row[1], {'lid': 1, 'ugid': 1, 'vid': 1, 'alias': 'tmax', 'did': 1, 'year': 2000,
'value': 0.7203244934421581, 'month': 1, 'variable': 'tmax', 'gid': 2,
'time': datetime.datetime(2000, 1, 1, 12, 0), 'tid': 1, 'level': 50,
'day': 1})
self.assertIsInstance(row[0], MultiPolygon)
self.assertEqual(len(row), 2)
self.assertEqual(len(row[1]), len(constants.HEADERS_RAW))
def _write_(self):
gid_file = []
build = True
with open(self.path,'w') as f:
writer = csv.writer(f,dialect=OcgDialect)
for coll in self:
if build:
headers = coll.get_headers(upper=True)
if env.WRITE_TO_REFERENCE_PROJECTION:
projection = constants.reference_projection
else:
projection = coll._archetype.spatial.projection
ugid_idx = headers.index('UGID')
gid_idx = headers.index('GID')
did_idx = headers.index('DID')
writer.writerow(headers)
build = False
for geom,row in coll.get_iter():
gid_file.append({'geom':geom,'did':row[did_idx],
'ugid':row[ugid_idx],'gid':row[gid_idx]})
writer.writerow(row)
if self.ops.aggregate is True and self.ops.abstraction == 'point':
if env.VERBOSE:
print('creating a UGID-GID shapefile is not necessary for aggregated point data. use UGID shapefile.')
else:
sc = ShpCabinet()
shp_dir = os.path.join(self.outdir,'shp')
try:
os.mkdir(shp_dir)
## catch if the directory exists
except OSError:
if os.path.exists(shp_dir):
pass
else:
raise
shp_path = os.path.join(shp_dir,self.prefix+'_gid.shp')
sc.write(gid_file,shp_path,sr=projection.sr)
def test_get_features_object(self):
# test with a shapefile not having the default unique geometry identifier
path = self.get_shapefile_path_with_no_ugid()
keywords = dict(uid=[None, 'ID'],
select_uid=[None, [8, 11, 13]],
select_sql_where=[None, 'STATE_NAME = "Wisconsin"'])
for k in self.iter_product_keywords(keywords):
ds = ogr.Open(path)
try:
try:
obj = ShpCabinet._get_features_object_(ds, uid=k.uid, select_uid=k.select_uid,
select_sql_where=k.select_sql_where)
except RuntimeError:
self.assertIsNone(k.uid)
self.assertIsNotNone(k.select_uid)
continue
if k.select_sql_where is not None:
length = 1
elif k.select_uid is not None:
length = 3
else:
length = 11
self.assertEqual(len(obj), length)
self.assertIsInstance(obj, Layer)
finally:
ds.Destroy()
# test on a shapefile having the default unique geometry identifier
path = ShpCabinet().get_shp_path('state_boundaries')
ds = ogr.Open(path)
try:
obj = ShpCabinet._get_features_object_(ds, select_uid=[8, 11, 13])
self.assertEqual(len(obj), 3)
finally:
ds.Destroy()
def test_load_all(self):
sc = ShpCabinet()
for key in sc.keys():
geoms = sc.get_geoms(key)
self.assertTrue(len(geoms) > 2)
def test_get_keys(self):
sc = ShpCabinet()
ret = sc.keys()
self.assertTrue(len(ret) >= 1)
def test_bad_path(self):
bp = '/a/bad/location'
with self.assertRaises(ValueError):
ShpCabinet(bp).get_geoms('state_boundaries')
def _sc(self):
if self.__sc is None:
self.__sc = ShpCabinet()
return (self.__sc)
def test_load_all(self):
raise (SkipTest('too long for no benefit'))
sc = ShpCabinet()
for key in sc.keys():
geoms = sc.get_geoms(key)
self.assertTrue(len(geoms) > 2)
def test_huc_loading(self):
raise (SkipTest('tests loading time for HUC data'))
sc = ShpCabinet()
geoms = sc.get_geoms('WBDHU12_June2013', select_ugid=[2221, 5000])
self.assertEqual(len(geoms), 2)
self.assertEqual(geoms[0]['UGID'], 2221)
def _write_(self):
gid_file = OrderedDict()
build = True
is_aggregated = self.ops.aggregate
with open(self.path, 'w') as f:
ocgis_lh(msg='opened csv file: {0}'.format(self.path),
level=logging.DEBUG,
logger='conv.csv+')
writer = csv.writer(f, dialect=OcgDialect)
for coll in self:
ocgis_lh('writing collection',
'conv.csv+',
level=logging.DEBUG)
if build:
ocgis_lh('starting build',
'conv.csv+',
level=logging.DEBUG)
headers = coll.get_headers(upper=True)
if env.WRITE_TO_REFERENCE_PROJECTION:
projection = env.REFERENCE_PROJECTION
else:
projection = coll._archetype.spatial.projection
writer.writerow(headers)
build = False
ocgis_lh(msg='build finished'.format(self.path),
level=logging.DEBUG,
logger='conv.csv+')
for geom, row, geom_ids in coll.get_iter(
with_geometry_ids=True):
if not is_aggregated:
ugid = geom_ids['ugid']
did = geom_ids['did']
gid = geom_ids['gid']
if ugid not in gid_file:
gid_file[ugid] = OrderedDict()
if did not in gid_file[ugid]:
gid_file[ugid][did] = OrderedDict()
gid_file[ugid][did][gid] = geom
writer.writerow(row)
ocgis_lh('finished writing collection',
'conv.csv+',
level=logging.DEBUG)
if is_aggregated is True:
ocgis_lh(
'creating a UGID-GID shapefile is not necessary for aggregated data. use UGID shapefile.',
'conv.csv+', logging.WARN)
else:
ocgis_lh('writing UGID-GID shapefile', 'conv.csv+', logging.DEBUG)
sc = ShpCabinet()
shp_dir = os.path.join(self.outdir, 'shp')
try:
os.mkdir(shp_dir)
## catch if the directory exists
except OSError:
if os.path.exists(shp_dir):
pass
else:
raise
shp_path = os.path.join(shp_dir, self.prefix + '_gid.shp')
def iter_gid_file():
for ugid, did_gid in gid_file.iteritems():
for did, gid_geom in did_gid.iteritems():
for gid, geom in gid_geom.iteritems():
yield ({
'geom': geom,
'DID': did,
'UGID': ugid,
'GID': gid
})
sc.write(iter_gid_file(), shp_path, sr=projection.sr)
def california(self):
sc = ShpCabinet()
ret = sc.get_geom_dict('state_boundaries', {'ugid': [25]})
return (ret)
def world_countries(self):
sc = ShpCabinet()
ret = sc.get_geom_dict('world_countries')
return (ret)
def alaska(self):
sc = ShpCabinet()
ret = sc.get_geom_dict('state_boundaries', {'ugid': [51]})
return (ret)
def state_boundaries(self):
sc = ShpCabinet()
ret = sc.get_geom_dict('state_boundaries')
return (ret)