def test_attribute_flags(self):
attr_flags = ['none','all']
for attr_flag in attr_flags:
## make temporary directory
path = get_temp_path(only_dir=True,nest=True)
try:
## the shapefile key to use
key = 'state_boundaries'
## get path to shapefile and copy directory to temporary location
sc = ShpCabinet()
origin = os.path.split(sc.get_shp_path(key))[0]
shutil.copytree(origin,os.path.join(path,key))
## remove original config file
os.remove(os.path.join(path,key,key+'.cfg'))
## write config file
config = ConfigParser.ConfigParser()
config.add_section('mapping')
config.set('mapping','ugid','none')
config.set('mapping','attributes',attr_flag)
with open(os.path.join(path,key,key+'.cfg'),'w') as f:
config.write(f)
## load data
sc = ShpCabinet(path)
geoms = sc.get_geoms(key)
for geom in geoms:
if attr_flag == 'none':
self.assertEqual(set(['ugid','geom']),set(geom.keys()))
else:
self.assertEqual(set(['ugid', 'state_name', 'state_fips', 'geom', 'state_abbr', 'id']),
set(geom.keys()))
finally:
shutil.rmtree(path)
def test_shp(self):
sc = ShpCabinet()
keys = sc.keys()
for key in keys:
url = '/shp/{0}'.format(key)
resp = self.c.get(url)
self.assertTrue(len(resp.content) > 100)
def test_geom(self):
geom = make_poly((37.762, 38.222), (-102.281, -101.754))
g = Geom(geom)
self.assertEqual(type(g.value), GeometryDataset)
g.value = None
self.assertEqual(None, g.value)
g = Geom(None)
self.assertEqual(g.value, None)
self.assertEqual(str(g), 'geom=None')
g = Geom('-120|40|-110|50')
self.assertEqual(g.value.spatial.geom[0].bounds,
(-120.0, 40.0, -110.0, 50.0))
self.assertEqual(str(g), 'geom=-120.0|40.0|-110.0|50.0')
g = Geom('mi_watersheds')
self.assertEqual(str(g), 'geom=mi_watersheds')
geoms = ShpCabinet().get_geoms('mi_watersheds')
g = Geom('mi_watersheds')
self.assertEqual(len(g.value), len(geoms))
su = SelectUgid([1, 2, 3])
g = Geom('mi_watersheds', select_ugid=su)
self.assertEqual(len(g.value), 3)
geoms = GeometryDataset(uid=[1, 2], geom=[geom, geom])
g = Geom(geoms)
bbox = [-120, 40, -110, 50]
g = Geom(bbox)
self.assertEqual(g.value.spatial.geom[0].bounds,
tuple(map(float, bbox)))
def test_get_keys(self, dir_shpcabinet=None):
ocgis.env.DIR_SHPCABINET = dir_shpcabinet or ocgis.env.DIR_SHPCABINET
sc = ShpCabinet()
ret = sc.keys()
target_keys = ['state_boundaries', 'world_countries']
self.assertEqual(len(set(target_keys).intersection(set(ret))),
len(target_keys))
def write(self, path):
geoms = []
uid = self.spatial.uid
for ii, geom in iter_array(self.spatial.geom, return_value=True):
geoms.append({'geom': geom, 'ugid': uid[ii]})
sc = ShpCabinet()
sc.write(geoms, path, sr=self.spatial.projection.sr)
def get_shp(request, key=None):
query = helpers.parse_qs(request.META['QUERY_STRING'])
select_ugid = SelectUgid()
select_ugid.parse_query(query)
prefix = Prefix()
prefix.parse_query(query)
unwrap = Unwrap()
unwrap.parse_query(query)
pm = PrimeMeridian()
pm.parse_query(query)
sc = ShpCabinet()
geom_dict = sc.get_geom_dict(key, attr_filter=select_ugid.value)
## unwrap coordinates if requested
if unwrap.value:
unwrap_geoms(geom_dict, pm.value)
dir_path = get_temp_path(nest=True, only_dir=True, wd=env.DIR_OUTPUT)
if prefix.value is None:
out_name = key
else:
out_name = prefix.value
filename = '{0}.shp'.format(out_name)
path = os.path.join(dir_path, filename)
path = sc.write(geom_dict, path)
path = os.path.split(path)[0]
resp = helpers._zip_response_(path,
filename=filename.replace('shp', 'zip'))
return (resp)
def test_sql_subset(self):
sc = ShpCabinet()
path = sc.get_shp_path('state_boundaries')
ds = ogr.Open(path)
ret = ds.ExecuteSQL(
'select * from state_boundaries where state_name = "New Jersey"')
ret.ResetReading()
self.assertEqual(len(ret), 1)
def test_none_path(self):
try:
ocgis.env.DIR_SHPCABINET = None
with self.assertRaises(ValueError):
ShpCabinet().get_geoms('state_boundaries')
finally:
ocgis.env.reset()
def write(self, path):
geoms = []
uid = self.spatial.uid
attrs = self.spatial.attrs
for ii, geom in iter_array(self.spatial.geom, return_value=True):
dct = {'geom': geom, 'UGID': uid[ii]}
for k, v in attrs.iteritems():
dct[k] = v[ii]
geoms.append(dct)
sc = ShpCabinet()
sc.write(geoms, path, sr=self.spatial.projection.sr)
def _format_all_elements_(self,value):
try:
minx,miny,maxx,maxy = value
geom = Polygon(((minx,miny),
(minx,maxy),
(maxx,maxy),
(maxx,miny)))
assert(geom.is_valid)
ret = [{'id':1,'geom':geom}]
except ValueError:
sc = ShpCabinet()
ret = sc.get_geom_dict(value[0])
return(ret)
def _format_string_element_(self, value):
elements = value.split('|')
try:
elements = [float(e) for e in elements]
minx, miny, maxx, maxy = elements
geom = Polygon(
((minx, miny), (minx, maxy), (maxx, maxy), (maxx, miny)))
self._assert_(geom.is_valid)
ret = [{'ugid': 1, 'geom': geom}]
self._bounds = elements
except ValueError:
from ocgis.util.shp_cabinet import ShpCabinet
sc = ShpCabinet()
ret = sc.get_geom_dict(value)
return (ret)
def test_shapefiles_not_in_folders(self):
for dirpath, dirnames, filenames in os.walk(ocgis.env.DIR_SHPCABINET):
for filename in filenames:
if filename.startswith(
'state_boundaries') or filename.startswith(
'world_countries'):
dst = os.path.join(self._test_dir, filename)
src = os.path.join(dirpath, filename)
shutil.copy2(src, dst)
self.test_get_keys(dir_shpcabinet=self._test_dir)
sc = ShpCabinet(path=self._test_dir)
path = sc.get_shp_path('world_countries')
self.assertEqual(path,
os.path.join(self._test_dir, 'world_countries.shp'))
def test_shp_process(self):
test_path = '/home/local/WX/ben.koziol/Dropbox/nesii/project/ocg/bin/test_data/test_shp_process'
copy_path = os.path.join(self._test_dir, 'test_shp_process')
shutil.copytree(test_path, copy_path)
shp_path = os.path.join(copy_path, 'wc.shp')
sp = ShpProcess(shp_path)
out_folder = tempfile.mkdtemp(dir=self._test_dir)
sp.process(out_folder, 'world_countries', ugid=None)
sc = ShpCabinet(path=out_folder)
select_ugid = [33, 126, 199]
geoms = sc.get_geoms('world_countries', select_ugid=select_ugid)
self.assertEqual(len(geoms), 3)
names = [item['NAME'] for item in geoms]
self.assertEqual(set(names), set(['Canada', 'Mexico',
'United States']))
def _write_(self):
gid_file = []
build = True
with open(self.path,'w') as f:
writer = csv.writer(f,dialect=OcgDialect)
for coll in self:
if build:
headers = coll.get_headers(upper=True)
if env.WRITE_TO_REFERENCE_PROJECTION:
projection = constants.reference_projection
else:
projection = coll._archetype.spatial.projection
ugid_idx = headers.index('UGID')
gid_idx = headers.index('GID')
did_idx = headers.index('DID')
writer.writerow(headers)
build = False
for geom,row in coll.get_iter():
gid_file.append({'geom':geom,'did':row[did_idx],
'ugid':row[ugid_idx],'gid':row[gid_idx]})
writer.writerow(row)
if self.ops.aggregate is True and self.ops.abstraction == 'point':
if env.VERBOSE:
print('creating a UGID-GID shapefile is not necessary for aggregated point data. use UGID shapefile.')
else:
sc = ShpCabinet()
shp_dir = os.path.join(self.outdir,'shp')
try:
os.mkdir(shp_dir)
## catch if the directory exists
except OSError:
if os.path.exists(shp_dir):
pass
else:
raise
shp_path = os.path.join(shp_dir,self.prefix+'_gid.shp')
sc.write(gid_file,shp_path,sr=projection.sr)
def test_bad_path(self):
bp = '/a/bad/location'
with self.assertRaises(ValueError):
ShpCabinet(bp).get_geoms('state_boundaries')
def _write_(self):
gid_file = OrderedDict()
build = True
is_aggregated = self.ops.aggregate
with open(self.path, 'w') as f:
ocgis_lh(msg='opened csv file: {0}'.format(self.path),
level=logging.DEBUG,
logger='conv.csv+')
writer = csv.writer(f, dialect=OcgDialect)
for coll in self:
ocgis_lh('writing collection',
'conv.csv+',
level=logging.DEBUG)
if build:
ocgis_lh('starting build',
'conv.csv+',
level=logging.DEBUG)
headers = coll.get_headers(upper=True)
if env.WRITE_TO_REFERENCE_PROJECTION:
projection = env.REFERENCE_PROJECTION
else:
projection = coll._archetype.spatial.projection
writer.writerow(headers)
build = False
ocgis_lh(msg='build finished'.format(self.path),
level=logging.DEBUG,
logger='conv.csv+')
for geom, row, geom_ids in coll.get_iter(
with_geometry_ids=True):
if not is_aggregated:
ugid = geom_ids['ugid']
did = geom_ids['did']
gid = geom_ids['gid']
if ugid not in gid_file:
gid_file[ugid] = OrderedDict()
if did not in gid_file[ugid]:
gid_file[ugid][did] = OrderedDict()
gid_file[ugid][did][gid] = geom
writer.writerow(row)
ocgis_lh('finished writing collection',
'conv.csv+',
level=logging.DEBUG)
if is_aggregated is True:
ocgis_lh(
'creating a UGID-GID shapefile is not necessary for aggregated data. use UGID shapefile.',
'conv.csv+', logging.WARN)
else:
ocgis_lh('writing UGID-GID shapefile', 'conv.csv+', logging.DEBUG)
sc = ShpCabinet()
shp_dir = os.path.join(self.outdir, 'shp')
try:
os.mkdir(shp_dir)
## catch if the directory exists
except OSError:
if os.path.exists(shp_dir):
pass
else:
raise
shp_path = os.path.join(shp_dir, self.prefix + '_gid.shp')
def iter_gid_file():
for ugid, did_gid in gid_file.iteritems():
for did, gid_geom in did_gid.iteritems():
for gid, geom in gid_geom.iteritems():
yield ({
'geom': geom,
'DID': did,
'UGID': ugid,
'GID': gid
})
sc.write(iter_gid_file(), shp_path, sr=projection.sr)
def test_load_all(self):
sc = ShpCabinet()
for key in sc.keys():
geoms = sc.get_geoms(key)
self.assertTrue(len(geoms) > 2)
def test_huc_loading(self):
raise (SkipTest('tests loading time for HUC data'))
sc = ShpCabinet()
geoms = sc.get_geoms('WBDHU12_June2013', select_ugid=[2221, 5000])
self.assertEqual(len(geoms), 2)
self.assertEqual(geoms[0]['UGID'], 2221)
def test_load_all(self):
raise (SkipTest('too long for no benefit'))
sc = ShpCabinet()
for key in sc.keys():
geoms = sc.get_geoms(key)
self.assertTrue(len(geoms) > 2)
def test_get_keys(self):
sc = ShpCabinet()
ret = sc.keys()
self.assertTrue(len(ret) >= 1)
def alaska(self):
sc = ShpCabinet()
ret = sc.get_geom_dict('state_boundaries', {'ugid': [51]})
return (ret)
def state_boundaries(self):
sc = ShpCabinet()
ret = sc.get_geom_dict('state_boundaries')
return (ret)
def _sc(self):
if self.__sc is None:
self.__sc = ShpCabinet()
return (self.__sc)
def california(self):
sc = ShpCabinet()
ret = sc.get_geom_dict('state_boundaries', {'ugid': [25]})
return (ret)
def world_countries(self):
sc = ShpCabinet()
ret = sc.get_geom_dict('world_countries')
return (ret)
