Example #1
0
 def test_write_multipolygon_shapefile(self):
     hndl, shapefile = tempfile.mkstemp(suffix=".shp")
     schema = {
         'geometry': 'MultiPolygon',
         'properties': {
             'INT': 'int',
             'FLOAT': 'float',
             'STRING': 'str',
         }
     }
     records = []
     for i in range(1, 4):
         coords = [[dg.generate_polygon_coords(x0=i, y0=i)]]
         records.append({
             'id': i,
             'geometry': {
                 'type': 'MultiPolygon',
                 'coordinates': coords
             },
             'properties': {
                 'INT': i,
                 'FLOAT': float(i),
                 'STRING': str(i),
             }
         })
     writer = shapefile_util.get_shapefile_writer(
         shapefile=shapefile,
         crs='EPSG:4326',
         schema=schema,
     )
     for record in records:
         writer.write(record)
     writer.close()
     print "shp: ", shapefile
Example #2
0
def generate_shp_section(data_dir, section):
    section_data_dir = os.path.join(data_dir, section['id'])
    os.makedirs(section_data_dir)
    shpfile = os.path.join(section_data_dir, "%s.shp" % section['id'])
    w = shapefile_util.get_shapefile_writer(
        shapefile=shpfile, 
        crs='EPSG:4326',
        schema=section['schema'],
    )
    for record in section['records']:
        w.write(record)
    w.close()
Example #3
0
def generate_shp_section(data_dir, section):
    section_data_dir = os.path.join(data_dir, section['id'])
    os.makedirs(section_data_dir)
    shpfile = os.path.join(section_data_dir, "%s.shp" % section['id'])
    w = shapefile_util.get_shapefile_writer(
        shapefile=shpfile,
        crs='EPSG:4326',
        schema=section['schema'],
    )
    for record in section['records']:
        w.write(record)
    w.close()
 def generate_shapefile(clz, shpfile=None, crs='EPSG:4326', schema=None, 
                        records=None):
     if not shpfile:
         hndl, shpfile = tempfile.msktemp(suffix=".shp")
     w = shapefile_util.get_shapefile_writer(
         shapefile=shpfile, 
         crs=crs,
         schema=schema
     )
     for record in records:
         w.write(record)
     w.close()
     return shpfile
Example #5
0
 def test_write_multipolygon_shapefile(self):
     hndl, shapefile = tempfile.mkstemp(suffix=".shp")
     schema = {"geometry": "MultiPolygon", "properties": {"INT": "int", "FLOAT": "float", "STRING": "str"}}
     records = []
     for i in range(1, 4):
         coords = [[dg.generate_polygon_coords(x0=i, y0=i)]]
         records.append(
             {
                 "id": i,
                 "geometry": {"type": "MultiPolygon", "coordinates": coords},
                 "properties": {"INT": i, "FLOAT": float(i), "STRING": str(i)},
             }
         )
     writer = shapefile_util.get_shapefile_writer(shapefile=shapefile, crs="EPSG:4326", schema=schema)
     for record in records:
         writer.write(record)
     writer.close()
     print "shp: ", shapefile
Example #6
0
def generate_map_layer(layer_id="layer", layer_dir=None):
    if not layer_dir:
        layer_dir = tempfile.mkdtemp(prefix="layer.")
    shpfile = os.path.join(layer_dir, "%s.shp" % layer_id)
    writer = shapefile_util.get_shapefile_writer(
        shapefile=shpfile,
        driver='ESRI Shapefile',
        crs='EPSG:4326',
        schema={
            'geometry': 'MultiPolygon',
            'properties': {
                'INT_ATTR': 'int',
                'STR_ATTR': 'str',
            }
        },
    )
    for j in range(3):
        coords = [[j, j], [j, j + 1], [j + 1, j + 1], [j + 1, j], [j, j]]
        record = {
            'id': j,
            'geometry': {
                'type': 'MultiPolygon',
                'coordinates': [[coords]]
            },
            'properties': {
                'INT_ATTR': j,
                'STR_ATTR': "str_%s" % layer_id
            }
        }
        writer.write(record)
    writer.close()

    # Write Mapfile.
    mapfile = os.path.join(layer_dir, "%s.map" % layer_id)
    with open(mapfile, "w") as f:
        f.write(get_mapfile(layer_id))

    # Write config file.
    config_file = os.path.join(layer_dir, "config.json")
    config = {"mapfile": "%s.map" % layer_id}
    with open(config_file, 'w') as f:
        f.write(json.dumps(config))

    return layer_dir
Example #7
0
def generate_map_layer(layer_id="layer", layer_dir=None):
    if not layer_dir:
        layer_dir = tempfile.mkdtemp(prefix="layer.")
    shpfile = os.path.join(layer_dir, "%s.shp" % layer_id)
    writer = shapefile_util.get_shapefile_writer(
        shapefile=shpfile, driver='ESRI Shapefile',
        crs='EPSG:4326',
        schema={
            'geometry': 'MultiPolygon',
            'properties': {
                'INT_ATTR': 'int',
                'STR_ATTR': 'str',
            }
        },
    )
    for j in range(3):
        coords = [[j, j], [j,j+1], [j+1, j+1], [j+1,j], [j,j]]
        record = {
            'id': j,
            'geometry': {
                'type': 'MultiPolygon',
                'coordinates': [[coords]]
            },
            'properties': {
                'INT_ATTR': j,
                'STR_ATTR': "str_%s" % layer_id
            }
        }
        writer.write(record)
    writer.close()

    # Write Mapfile.
    mapfile = os.path.join(layer_dir, "%s.map" % layer_id)
    with open(mapfile, "w") as f:
        f.write(get_mapfile(layer_id))

    # Write config file.
    config_file= os.path.join(layer_dir, "config.json")
    config = {"mapfile": "%s.map" % layer_id}
    with open(config_file, 'w') as f:
        f.write(json.dumps(config))

    return layer_dir
Example #8
0
    def create_substrates_layer(self):
        logger = self.get_logger_for_section(
            section_id='substrates_layer', 
            base_msg="Creating substrates layer...")
        # Read substrates.
        substrates = []
        substrates_path = os.path.join(
            self.source_dir, 'substrates.csv')
        with open(substrates_path, 'rb') as f:
            for row in csv.DictReader(f):
                css_color = row.get('color', '#000000')
                hex_color = css_color.lstrip('#')
                row['rgb'] = [int(hex_color[i:i+2],16) for i in range(0, 6, 2)]
                substrates.append(row)

        # Make substrate shapefile from habitats, 
        # with EPSG:3857 as the CRS.
        layer_dir = os.path.join(self.layers_dir, 'substrates')
        os.makedirs(layer_dir)
        habs_shapefile = os.path.join(
            self.source_dir, 'habitats', 'habitats.shp')
        logger.info("Reading shapes from habitats file...")
        reader = shapefile_util.get_shapefile_reader(habs_shapefile)
        source_mbr = reader.get_mbr()
        source_crs = reader.get_crs()
        source_schema = reader.get_schema()
        
        write_msg = "Writing shapes..."
        logger.info(write_msg)
        reader = shapefile_util.get_shapefile_reader(habs_shapefile)
        substrate_shapefile = os.path.join(layer_dir, 'substrates.shp')
        writer = shapefile_util.get_shapefile_writer(
            shapefile=substrate_shapefile, 
            crs='EPSG:3857',
            schema=source_schema,
        )
        skipped = 0
        counter = 0
        log_interval = 1e3
        for record in reader.records():
            counter += 1
            if (counter % log_interval) == 0:
                prog_msg = "%s %d of %d (%.1f)" % (
                    write_msg, counter, reader.size, 
                    100.0 * counter/reader.size)
                logger.info(prog_msg)
            shp = gis_util.geojson_to_shape(record['geometry'])
            proj_shp = gis_util.reproject_shape(shp, source_crs, 'EPSG:3857')
            record['geometry'] = json.loads(gis_util.shape_to_geojson(proj_shp))
            bad_rec = False
            ps = record['properties']
            if type(ps['ENERGY']) is not unicode:
                bad_rec = True
            if type(ps['Z']) is not float:
                bad_rec = True
            if type(ps['SUBSTRATE']) is not unicode:
                bad_rec = True
            if record['geometry']['type'] != source_schema['geometry']:
                bad_rec = True

            if not bad_rec:
                writer.write(record)
            else:
                skipped += 1

        writer.close()
        reader.close()
        if skipped:
            self.logger.info("%s Skipped %s records due to formatting" %
                             (write_msg, skipped))

        # Transform bounds to EPSG:3857.
        mbr_diag = gis_util.wkt_to_shape(
            "LINESTRING (%s %s, %s %s)" % source_mbr)
        projected_diag = gis_util.reproject_shape(
            mbr_diag, source_crs, 'EPSG:3857')
        projected_mbr = gis_util.get_shape_mbr(projected_diag)

        # Write layer client config.
        client_path = os.path.join(layer_dir, 'client.json')
        info_template = self.template_env.get_template(
            'georefine/substrates_info.html')
        info_html = info_template.render(substrates=substrates)
        client_config = {
            'id': 'substrates',
            'label': 'Substrates',
            'source': 'georefine_wms',
            'layer_type': "WMS",
            'params': {
                'srs':'EPSG:3857',
                'transparent': True,
                'layers': 'substrates',
            },
            'info': info_html,
            'properties': {
                'maxExtent': projected_mbr,
                'projection': 'EPSG:3857',
                'serverResolutions': [4891.96981024998, 2445.98490512499, 1222.99245256249, 611.49622628138, 305.748113140558],
                'tileSize': {'w': 512, 'h': 512},
                'visibility': False
            },
            'zIndex': 30,
        }
        with open(client_path, 'wb') as f:
            json.dump(client_config, f)

        # Write mapfile.
        mapfile_path = os.path.join(layer_dir, 'substrates.map')
        with open(mapfile_path, 'wb') as f:
            mapfile_template = self.template_env.get_template(
                'georefine/substrates.map')
            f.write(mapfile_template.render(
                substrates=substrates,
                mbr=projected_mbr
            ))

        # Write wms config.
        wms_config_path = os.path.join(layer_dir, 'wms.json')
        wms_config = {
            'mapfile': 'substrates.map'
        }
        with open(wms_config_path, 'wb') as f:
            json.dump(wms_config, f)
    def test_shapefile_ingestor(self):
        Base = declarative_base()

        class TestClass(Base):
            __tablename__ = 'testclass'
            id = Column(Integer, primary_key=True)
            attr1 = Column(Integer) 
            attr2 = Column(String)
            geom = GeometryColumn(MultiPolygon(2))
        GeometryDDL(TestClass.__table__)
        schema = {
            'sources': {
                'TestClass': TestClass
            }
        }

        Base.metadata.create_all(self.connection)

        dao = ORM_DAO(session=self.session, schema=schema)

        shapedir = tempfile.mkdtemp()
        shapefile = os.path.join(shapedir, "test.shp")
        schema = {
            'geometry': 'MultiPolygon',
            'properties': {
                'S_ATTR1': 'int',
                'S_ATTR2': 'str',
            }
        }
        records = []
        for i in range(5):
            coords = [[dg.generate_polygon_coords(x=i, y=i)]]
            records.append({
                'id': i,
                'geometry': {
                    'type': 'MultiPolygon',
                    'coordinates': coords
                },
                'properties': {
                    'S_ATTR1': i,
                    'S_ATTR2': str(i),
                }
            })
        writer = shapefile_util.get_shapefile_writer(
            shapefile=shapefile,
            crs='EPSG:4326',
            schema=schema,
        )
        for record in records:
            writer.write(record)
        writer.close()

        mappings = [
            {
                'source': 'S_ATTR1', 
                'target': 'attr1',
                'processor': lambda value: int(value) * 10
            },
            {
                'source': 'S_ATTR2', 
                'target': 'attr2',
            },
            {
                'source': '__shape',
                'target': 'geom',
                'processor': gis_util.shape_to_wkt,
            }
        ]

        Ingestor(
            reader=ShapefileReader(shp_file=shapefile),
            processors=[
                ClassMapper(clazz=TestClass, mappings=mappings),
                DAOWriter(dao=dao, commit=False),
            ]
        ).ingest()
        results = dao.query({
            'SELECT': ['__TestClass']
        }).all()

        for r in results:
            print r.attr1, r.attr2, dao.session.scalar(r.geom.wkt)
Example #10
0
    def test_shapefile_ingestor(self):
        Base = declarative_base()

        class TestClass(Base):
            __tablename__ = 'testclass'
            id = Column(Integer, primary_key=True)
            attr1 = Column(Integer)
            attr2 = Column(String)
            geom = GeometryColumn(MultiPolygon(2))

        GeometryDDL(TestClass.__table__)
        schema = {'sources': {'TestClass': TestClass}}

        Base.metadata.create_all(self.connection)

        dao = ORM_DAO(session=self.session, schema=schema)

        shapedir = tempfile.mkdtemp()
        shapefile = os.path.join(shapedir, "test.shp")
        schema = {
            'geometry': 'MultiPolygon',
            'properties': {
                'S_ATTR1': 'int',
                'S_ATTR2': 'str',
            }
        }
        records = []
        for i in range(5):
            coords = [[dg.generate_polygon_coords(x=i, y=i)]]
            records.append({
                'id': i,
                'geometry': {
                    'type': 'MultiPolygon',
                    'coordinates': coords
                },
                'properties': {
                    'S_ATTR1': i,
                    'S_ATTR2': str(i),
                }
            })
        writer = shapefile_util.get_shapefile_writer(
            shapefile=shapefile,
            crs='EPSG:4326',
            schema=schema,
        )
        for record in records:
            writer.write(record)
        writer.close()

        mappings = [{
            'source': 'S_ATTR1',
            'target': 'attr1',
            'processor': lambda value: int(value) * 10
        }, {
            'source': 'S_ATTR2',
            'target': 'attr2',
        }, {
            'source': '__shape',
            'target': 'geom',
            'processor': gis_util.shape_to_wkt,
        }]

        Ingestor(reader=ShapefileReader(shp_file=shapefile),
                 processors=[
                     ClassMapper(clazz=TestClass, mappings=mappings),
                     DAOWriter(dao=dao, commit=False),
                 ]).ingest()
        results = dao.query({'SELECT': ['__TestClass']}).all()

        for r in results:
            print r.attr1, r.attr2, dao.session.scalar(r.geom.wkt)