def test_write_multipolygon_shapefile(self): hndl, shapefile = tempfile.mkstemp(suffix=".shp") schema = { 'geometry': 'MultiPolygon', 'properties': { 'INT': 'int', 'FLOAT': 'float', 'STRING': 'str', } } records = [] for i in range(1, 4): coords = [[dg.generate_polygon_coords(x0=i, y0=i)]] records.append({ 'id': i, 'geometry': { 'type': 'MultiPolygon', 'coordinates': coords }, 'properties': { 'INT': i, 'FLOAT': float(i), 'STRING': str(i), } }) writer = shapefile_util.get_shapefile_writer( shapefile=shapefile, crs='EPSG:4326', schema=schema, ) for record in records: writer.write(record) writer.close() print "shp: ", shapefile
def generate_shp_section(data_dir, section): section_data_dir = os.path.join(data_dir, section['id']) os.makedirs(section_data_dir) shpfile = os.path.join(section_data_dir, "%s.shp" % section['id']) w = shapefile_util.get_shapefile_writer( shapefile=shpfile, crs='EPSG:4326', schema=section['schema'], ) for record in section['records']: w.write(record) w.close()
def generate_shapefile(clz, shpfile=None, crs='EPSG:4326', schema=None, records=None): if not shpfile: hndl, shpfile = tempfile.msktemp(suffix=".shp") w = shapefile_util.get_shapefile_writer( shapefile=shpfile, crs=crs, schema=schema ) for record in records: w.write(record) w.close() return shpfile
def test_write_multipolygon_shapefile(self): hndl, shapefile = tempfile.mkstemp(suffix=".shp") schema = {"geometry": "MultiPolygon", "properties": {"INT": "int", "FLOAT": "float", "STRING": "str"}} records = [] for i in range(1, 4): coords = [[dg.generate_polygon_coords(x0=i, y0=i)]] records.append( { "id": i, "geometry": {"type": "MultiPolygon", "coordinates": coords}, "properties": {"INT": i, "FLOAT": float(i), "STRING": str(i)}, } ) writer = shapefile_util.get_shapefile_writer(shapefile=shapefile, crs="EPSG:4326", schema=schema) for record in records: writer.write(record) writer.close() print "shp: ", shapefile
def generate_map_layer(layer_id="layer", layer_dir=None): if not layer_dir: layer_dir = tempfile.mkdtemp(prefix="layer.") shpfile = os.path.join(layer_dir, "%s.shp" % layer_id) writer = shapefile_util.get_shapefile_writer( shapefile=shpfile, driver='ESRI Shapefile', crs='EPSG:4326', schema={ 'geometry': 'MultiPolygon', 'properties': { 'INT_ATTR': 'int', 'STR_ATTR': 'str', } }, ) for j in range(3): coords = [[j, j], [j, j + 1], [j + 1, j + 1], [j + 1, j], [j, j]] record = { 'id': j, 'geometry': { 'type': 'MultiPolygon', 'coordinates': [[coords]] }, 'properties': { 'INT_ATTR': j, 'STR_ATTR': "str_%s" % layer_id } } writer.write(record) writer.close() # Write Mapfile. mapfile = os.path.join(layer_dir, "%s.map" % layer_id) with open(mapfile, "w") as f: f.write(get_mapfile(layer_id)) # Write config file. config_file = os.path.join(layer_dir, "config.json") config = {"mapfile": "%s.map" % layer_id} with open(config_file, 'w') as f: f.write(json.dumps(config)) return layer_dir
def generate_map_layer(layer_id="layer", layer_dir=None): if not layer_dir: layer_dir = tempfile.mkdtemp(prefix="layer.") shpfile = os.path.join(layer_dir, "%s.shp" % layer_id) writer = shapefile_util.get_shapefile_writer( shapefile=shpfile, driver='ESRI Shapefile', crs='EPSG:4326', schema={ 'geometry': 'MultiPolygon', 'properties': { 'INT_ATTR': 'int', 'STR_ATTR': 'str', } }, ) for j in range(3): coords = [[j, j], [j,j+1], [j+1, j+1], [j+1,j], [j,j]] record = { 'id': j, 'geometry': { 'type': 'MultiPolygon', 'coordinates': [[coords]] }, 'properties': { 'INT_ATTR': j, 'STR_ATTR': "str_%s" % layer_id } } writer.write(record) writer.close() # Write Mapfile. mapfile = os.path.join(layer_dir, "%s.map" % layer_id) with open(mapfile, "w") as f: f.write(get_mapfile(layer_id)) # Write config file. config_file= os.path.join(layer_dir, "config.json") config = {"mapfile": "%s.map" % layer_id} with open(config_file, 'w') as f: f.write(json.dumps(config)) return layer_dir
def create_substrates_layer(self): logger = self.get_logger_for_section( section_id='substrates_layer', base_msg="Creating substrates layer...") # Read substrates. substrates = [] substrates_path = os.path.join( self.source_dir, 'substrates.csv') with open(substrates_path, 'rb') as f: for row in csv.DictReader(f): css_color = row.get('color', '#000000') hex_color = css_color.lstrip('#') row['rgb'] = [int(hex_color[i:i+2],16) for i in range(0, 6, 2)] substrates.append(row) # Make substrate shapefile from habitats, # with EPSG:3857 as the CRS. layer_dir = os.path.join(self.layers_dir, 'substrates') os.makedirs(layer_dir) habs_shapefile = os.path.join( self.source_dir, 'habitats', 'habitats.shp') logger.info("Reading shapes from habitats file...") reader = shapefile_util.get_shapefile_reader(habs_shapefile) source_mbr = reader.get_mbr() source_crs = reader.get_crs() source_schema = reader.get_schema() write_msg = "Writing shapes..." logger.info(write_msg) reader = shapefile_util.get_shapefile_reader(habs_shapefile) substrate_shapefile = os.path.join(layer_dir, 'substrates.shp') writer = shapefile_util.get_shapefile_writer( shapefile=substrate_shapefile, crs='EPSG:3857', schema=source_schema, ) skipped = 0 counter = 0 log_interval = 1e3 for record in reader.records(): counter += 1 if (counter % log_interval) == 0: prog_msg = "%s %d of %d (%.1f)" % ( write_msg, counter, reader.size, 100.0 * counter/reader.size) logger.info(prog_msg) shp = gis_util.geojson_to_shape(record['geometry']) proj_shp = gis_util.reproject_shape(shp, source_crs, 'EPSG:3857') record['geometry'] = json.loads(gis_util.shape_to_geojson(proj_shp)) bad_rec = False ps = record['properties'] if type(ps['ENERGY']) is not unicode: bad_rec = True if type(ps['Z']) is not float: bad_rec = True if type(ps['SUBSTRATE']) is not unicode: bad_rec = True if record['geometry']['type'] != source_schema['geometry']: bad_rec = True if not bad_rec: writer.write(record) else: skipped += 1 writer.close() reader.close() if skipped: self.logger.info("%s Skipped %s records due to formatting" % (write_msg, skipped)) # Transform bounds to EPSG:3857. mbr_diag = gis_util.wkt_to_shape( "LINESTRING (%s %s, %s %s)" % source_mbr) projected_diag = gis_util.reproject_shape( mbr_diag, source_crs, 'EPSG:3857') projected_mbr = gis_util.get_shape_mbr(projected_diag) # Write layer client config. client_path = os.path.join(layer_dir, 'client.json') info_template = self.template_env.get_template( 'georefine/substrates_info.html') info_html = info_template.render(substrates=substrates) client_config = { 'id': 'substrates', 'label': 'Substrates', 'source': 'georefine_wms', 'layer_type': "WMS", 'params': { 'srs':'EPSG:3857', 'transparent': True, 'layers': 'substrates', }, 'info': info_html, 'properties': { 'maxExtent': projected_mbr, 'projection': 'EPSG:3857', 'serverResolutions': [4891.96981024998, 2445.98490512499, 1222.99245256249, 611.49622628138, 305.748113140558], 'tileSize': {'w': 512, 'h': 512}, 'visibility': False }, 'zIndex': 30, } with open(client_path, 'wb') as f: json.dump(client_config, f) # Write mapfile. mapfile_path = os.path.join(layer_dir, 'substrates.map') with open(mapfile_path, 'wb') as f: mapfile_template = self.template_env.get_template( 'georefine/substrates.map') f.write(mapfile_template.render( substrates=substrates, mbr=projected_mbr )) # Write wms config. wms_config_path = os.path.join(layer_dir, 'wms.json') wms_config = { 'mapfile': 'substrates.map' } with open(wms_config_path, 'wb') as f: json.dump(wms_config, f)
def test_shapefile_ingestor(self): Base = declarative_base() class TestClass(Base): __tablename__ = 'testclass' id = Column(Integer, primary_key=True) attr1 = Column(Integer) attr2 = Column(String) geom = GeometryColumn(MultiPolygon(2)) GeometryDDL(TestClass.__table__) schema = { 'sources': { 'TestClass': TestClass } } Base.metadata.create_all(self.connection) dao = ORM_DAO(session=self.session, schema=schema) shapedir = tempfile.mkdtemp() shapefile = os.path.join(shapedir, "test.shp") schema = { 'geometry': 'MultiPolygon', 'properties': { 'S_ATTR1': 'int', 'S_ATTR2': 'str', } } records = [] for i in range(5): coords = [[dg.generate_polygon_coords(x=i, y=i)]] records.append({ 'id': i, 'geometry': { 'type': 'MultiPolygon', 'coordinates': coords }, 'properties': { 'S_ATTR1': i, 'S_ATTR2': str(i), } }) writer = shapefile_util.get_shapefile_writer( shapefile=shapefile, crs='EPSG:4326', schema=schema, ) for record in records: writer.write(record) writer.close() mappings = [ { 'source': 'S_ATTR1', 'target': 'attr1', 'processor': lambda value: int(value) * 10 }, { 'source': 'S_ATTR2', 'target': 'attr2', }, { 'source': '__shape', 'target': 'geom', 'processor': gis_util.shape_to_wkt, } ] Ingestor( reader=ShapefileReader(shp_file=shapefile), processors=[ ClassMapper(clazz=TestClass, mappings=mappings), DAOWriter(dao=dao, commit=False), ] ).ingest() results = dao.query({ 'SELECT': ['__TestClass'] }).all() for r in results: print r.attr1, r.attr2, dao.session.scalar(r.geom.wkt)
def test_shapefile_ingestor(self): Base = declarative_base() class TestClass(Base): __tablename__ = 'testclass' id = Column(Integer, primary_key=True) attr1 = Column(Integer) attr2 = Column(String) geom = GeometryColumn(MultiPolygon(2)) GeometryDDL(TestClass.__table__) schema = {'sources': {'TestClass': TestClass}} Base.metadata.create_all(self.connection) dao = ORM_DAO(session=self.session, schema=schema) shapedir = tempfile.mkdtemp() shapefile = os.path.join(shapedir, "test.shp") schema = { 'geometry': 'MultiPolygon', 'properties': { 'S_ATTR1': 'int', 'S_ATTR2': 'str', } } records = [] for i in range(5): coords = [[dg.generate_polygon_coords(x=i, y=i)]] records.append({ 'id': i, 'geometry': { 'type': 'MultiPolygon', 'coordinates': coords }, 'properties': { 'S_ATTR1': i, 'S_ATTR2': str(i), } }) writer = shapefile_util.get_shapefile_writer( shapefile=shapefile, crs='EPSG:4326', schema=schema, ) for record in records: writer.write(record) writer.close() mappings = [{ 'source': 'S_ATTR1', 'target': 'attr1', 'processor': lambda value: int(value) * 10 }, { 'source': 'S_ATTR2', 'target': 'attr2', }, { 'source': '__shape', 'target': 'geom', 'processor': gis_util.shape_to_wkt, }] Ingestor(reader=ShapefileReader(shp_file=shapefile), processors=[ ClassMapper(clazz=TestClass, mappings=mappings), DAOWriter(dao=dao, commit=False), ]).ingest() results = dao.query({'SELECT': ['__TestClass']}).all() for r in results: print r.attr1, r.attr2, dao.session.scalar(r.geom.wkt)