def test_attribute_flags(self): attr_flags = ['none','all'] for attr_flag in attr_flags: ## make temporary directory path = get_temp_path(only_dir=True,nest=True) try: ## the shapefile key to use key = 'state_boundaries' ## get path to shapefile and copy directory to temporary location sc = ShpCabinet() origin = os.path.split(sc.get_shp_path(key))[0] shutil.copytree(origin,os.path.join(path,key)) ## remove original config file os.remove(os.path.join(path,key,key+'.cfg')) ## write config file config = ConfigParser.ConfigParser() config.add_section('mapping') config.set('mapping','ugid','none') config.set('mapping','attributes',attr_flag) with open(os.path.join(path,key,key+'.cfg'),'w') as f: config.write(f) ## load data sc = ShpCabinet(path) geoms = sc.get_geoms(key) for geom in geoms: if attr_flag == 'none': self.assertEqual(set(['ugid','geom']),set(geom.keys())) else: self.assertEqual(set(['ugid', 'state_name', 'state_fips', 'geom', 'state_abbr', 'id']), set(geom.keys())) finally: shutil.rmtree(path)
def test_shp(self): sc = ShpCabinet() keys = sc.keys() for key in keys: url = '/shp/{0}'.format(key) resp = self.c.get(url) self.assertTrue(len(resp.content) > 100)
def test_geom(self): geom = make_poly((37.762, 38.222), (-102.281, -101.754)) g = Geom(geom) self.assertEqual(type(g.value), GeometryDataset) g.value = None self.assertEqual(None, g.value) g = Geom(None) self.assertEqual(g.value, None) self.assertEqual(str(g), 'geom=None') g = Geom('-120|40|-110|50') self.assertEqual(g.value.spatial.geom[0].bounds, (-120.0, 40.0, -110.0, 50.0)) self.assertEqual(str(g), 'geom=-120.0|40.0|-110.0|50.0') g = Geom('mi_watersheds') self.assertEqual(str(g), 'geom=mi_watersheds') geoms = ShpCabinet().get_geoms('mi_watersheds') g = Geom('mi_watersheds') self.assertEqual(len(g.value), len(geoms)) su = SelectUgid([1, 2, 3]) g = Geom('mi_watersheds', select_ugid=su) self.assertEqual(len(g.value), 3) geoms = GeometryDataset(uid=[1, 2], geom=[geom, geom]) g = Geom(geoms) bbox = [-120, 40, -110, 50] g = Geom(bbox) self.assertEqual(g.value.spatial.geom[0].bounds, tuple(map(float, bbox)))
def test_get_keys(self, dir_shpcabinet=None): ocgis.env.DIR_SHPCABINET = dir_shpcabinet or ocgis.env.DIR_SHPCABINET sc = ShpCabinet() ret = sc.keys() target_keys = ['state_boundaries', 'world_countries'] self.assertEqual(len(set(target_keys).intersection(set(ret))), len(target_keys))
def write(self, path): geoms = [] uid = self.spatial.uid for ii, geom in iter_array(self.spatial.geom, return_value=True): geoms.append({'geom': geom, 'ugid': uid[ii]}) sc = ShpCabinet() sc.write(geoms, path, sr=self.spatial.projection.sr)
def get_shp(request, key=None): query = helpers.parse_qs(request.META['QUERY_STRING']) select_ugid = SelectUgid() select_ugid.parse_query(query) prefix = Prefix() prefix.parse_query(query) unwrap = Unwrap() unwrap.parse_query(query) pm = PrimeMeridian() pm.parse_query(query) sc = ShpCabinet() geom_dict = sc.get_geom_dict(key, attr_filter=select_ugid.value) ## unwrap coordinates if requested if unwrap.value: unwrap_geoms(geom_dict, pm.value) dir_path = get_temp_path(nest=True, only_dir=True, wd=env.DIR_OUTPUT) if prefix.value is None: out_name = key else: out_name = prefix.value filename = '{0}.shp'.format(out_name) path = os.path.join(dir_path, filename) path = sc.write(geom_dict, path) path = os.path.split(path)[0] resp = helpers._zip_response_(path, filename=filename.replace('shp', 'zip')) return (resp)
def test_sql_subset(self): sc = ShpCabinet() path = sc.get_shp_path('state_boundaries') ds = ogr.Open(path) ret = ds.ExecuteSQL( 'select * from state_boundaries where state_name = "New Jersey"') ret.ResetReading() self.assertEqual(len(ret), 1)
def test_none_path(self): try: ocgis.env.DIR_SHPCABINET = None with self.assertRaises(ValueError): ShpCabinet().get_geoms('state_boundaries') finally: ocgis.env.reset()
def write(self, path): geoms = [] uid = self.spatial.uid attrs = self.spatial.attrs for ii, geom in iter_array(self.spatial.geom, return_value=True): dct = {'geom': geom, 'UGID': uid[ii]} for k, v in attrs.iteritems(): dct[k] = v[ii] geoms.append(dct) sc = ShpCabinet() sc.write(geoms, path, sr=self.spatial.projection.sr)
def _format_all_elements_(self,value): try: minx,miny,maxx,maxy = value geom = Polygon(((minx,miny), (minx,maxy), (maxx,maxy), (maxx,miny))) assert(geom.is_valid) ret = [{'id':1,'geom':geom}] except ValueError: sc = ShpCabinet() ret = sc.get_geom_dict(value[0]) return(ret)
def _format_string_element_(self, value): elements = value.split('|') try: elements = [float(e) for e in elements] minx, miny, maxx, maxy = elements geom = Polygon( ((minx, miny), (minx, maxy), (maxx, maxy), (maxx, miny))) self._assert_(geom.is_valid) ret = [{'ugid': 1, 'geom': geom}] self._bounds = elements except ValueError: from ocgis.util.shp_cabinet import ShpCabinet sc = ShpCabinet() ret = sc.get_geom_dict(value) return (ret)
def test_shapefiles_not_in_folders(self): for dirpath, dirnames, filenames in os.walk(ocgis.env.DIR_SHPCABINET): for filename in filenames: if filename.startswith( 'state_boundaries') or filename.startswith( 'world_countries'): dst = os.path.join(self._test_dir, filename) src = os.path.join(dirpath, filename) shutil.copy2(src, dst) self.test_get_keys(dir_shpcabinet=self._test_dir) sc = ShpCabinet(path=self._test_dir) path = sc.get_shp_path('world_countries') self.assertEqual(path, os.path.join(self._test_dir, 'world_countries.shp'))
def test_shp_process(self): test_path = '/home/local/WX/ben.koziol/Dropbox/nesii/project/ocg/bin/test_data/test_shp_process' copy_path = os.path.join(self._test_dir, 'test_shp_process') shutil.copytree(test_path, copy_path) shp_path = os.path.join(copy_path, 'wc.shp') sp = ShpProcess(shp_path) out_folder = tempfile.mkdtemp(dir=self._test_dir) sp.process(out_folder, 'world_countries', ugid=None) sc = ShpCabinet(path=out_folder) select_ugid = [33, 126, 199] geoms = sc.get_geoms('world_countries', select_ugid=select_ugid) self.assertEqual(len(geoms), 3) names = [item['NAME'] for item in geoms] self.assertEqual(set(names), set(['Canada', 'Mexico', 'United States']))
def _write_(self): gid_file = [] build = True with open(self.path,'w') as f: writer = csv.writer(f,dialect=OcgDialect) for coll in self: if build: headers = coll.get_headers(upper=True) if env.WRITE_TO_REFERENCE_PROJECTION: projection = constants.reference_projection else: projection = coll._archetype.spatial.projection ugid_idx = headers.index('UGID') gid_idx = headers.index('GID') did_idx = headers.index('DID') writer.writerow(headers) build = False for geom,row in coll.get_iter(): gid_file.append({'geom':geom,'did':row[did_idx], 'ugid':row[ugid_idx],'gid':row[gid_idx]}) writer.writerow(row) if self.ops.aggregate is True and self.ops.abstraction == 'point': if env.VERBOSE: print('creating a UGID-GID shapefile is not necessary for aggregated point data. use UGID shapefile.') else: sc = ShpCabinet() shp_dir = os.path.join(self.outdir,'shp') try: os.mkdir(shp_dir) ## catch if the directory exists except OSError: if os.path.exists(shp_dir): pass else: raise shp_path = os.path.join(shp_dir,self.prefix+'_gid.shp') sc.write(gid_file,shp_path,sr=projection.sr)
def test_bad_path(self): bp = '/a/bad/location' with self.assertRaises(ValueError): ShpCabinet(bp).get_geoms('state_boundaries')
def _write_(self): gid_file = OrderedDict() build = True is_aggregated = self.ops.aggregate with open(self.path, 'w') as f: ocgis_lh(msg='opened csv file: {0}'.format(self.path), level=logging.DEBUG, logger='conv.csv+') writer = csv.writer(f, dialect=OcgDialect) for coll in self: ocgis_lh('writing collection', 'conv.csv+', level=logging.DEBUG) if build: ocgis_lh('starting build', 'conv.csv+', level=logging.DEBUG) headers = coll.get_headers(upper=True) if env.WRITE_TO_REFERENCE_PROJECTION: projection = env.REFERENCE_PROJECTION else: projection = coll._archetype.spatial.projection writer.writerow(headers) build = False ocgis_lh(msg='build finished'.format(self.path), level=logging.DEBUG, logger='conv.csv+') for geom, row, geom_ids in coll.get_iter( with_geometry_ids=True): if not is_aggregated: ugid = geom_ids['ugid'] did = geom_ids['did'] gid = geom_ids['gid'] if ugid not in gid_file: gid_file[ugid] = OrderedDict() if did not in gid_file[ugid]: gid_file[ugid][did] = OrderedDict() gid_file[ugid][did][gid] = geom writer.writerow(row) ocgis_lh('finished writing collection', 'conv.csv+', level=logging.DEBUG) if is_aggregated is True: ocgis_lh( 'creating a UGID-GID shapefile is not necessary for aggregated data. use UGID shapefile.', 'conv.csv+', logging.WARN) else: ocgis_lh('writing UGID-GID shapefile', 'conv.csv+', logging.DEBUG) sc = ShpCabinet() shp_dir = os.path.join(self.outdir, 'shp') try: os.mkdir(shp_dir) ## catch if the directory exists except OSError: if os.path.exists(shp_dir): pass else: raise shp_path = os.path.join(shp_dir, self.prefix + '_gid.shp') def iter_gid_file(): for ugid, did_gid in gid_file.iteritems(): for did, gid_geom in did_gid.iteritems(): for gid, geom in gid_geom.iteritems(): yield ({ 'geom': geom, 'DID': did, 'UGID': ugid, 'GID': gid }) sc.write(iter_gid_file(), shp_path, sr=projection.sr)
def test_load_all(self): sc = ShpCabinet() for key in sc.keys(): geoms = sc.get_geoms(key) self.assertTrue(len(geoms) > 2)
def test_huc_loading(self): raise (SkipTest('tests loading time for HUC data')) sc = ShpCabinet() geoms = sc.get_geoms('WBDHU12_June2013', select_ugid=[2221, 5000]) self.assertEqual(len(geoms), 2) self.assertEqual(geoms[0]['UGID'], 2221)
def test_load_all(self): raise (SkipTest('too long for no benefit')) sc = ShpCabinet() for key in sc.keys(): geoms = sc.get_geoms(key) self.assertTrue(len(geoms) > 2)
def test_get_keys(self): sc = ShpCabinet() ret = sc.keys() self.assertTrue(len(ret) >= 1)
def alaska(self): sc = ShpCabinet() ret = sc.get_geom_dict('state_boundaries', {'ugid': [51]}) return (ret)
def state_boundaries(self): sc = ShpCabinet() ret = sc.get_geom_dict('state_boundaries') return (ret)
def _sc(self): if self.__sc is None: self.__sc = ShpCabinet() return (self.__sc)
def california(self): sc = ShpCabinet() ret = sc.get_geom_dict('state_boundaries', {'ugid': [25]}) return (ret)
def world_countries(self): sc = ShpCabinet() ret = sc.get_geom_dict('world_countries') return (ret)