예제 #1
0
def test_remove_layer(tmpdir):
    filename = str(tmpdir.join("a_filename.gpkg"))
    create_sample_data(filename, "GPKG", layer="layer1")
    create_sample_data(filename, "GPKG", layer="layer2")
    create_sample_data(filename, "GPKG", layer="layer3")
    create_sample_data(filename, "GPKG", layer="layer4")
    assert fiona.listlayers(filename) == [
        "layer1", "layer2", "layer3", "layer4"
    ]

    # remove by index
    fiona.remove(filename, layer=2)
    assert fiona.listlayers(filename) == ["layer1", "layer2", "layer4"]

    # remove by name
    fiona.remove(filename, layer="layer2")
    assert fiona.listlayers(filename) == ["layer1", "layer4"]

    # remove by negative index
    fiona.remove(filename, layer=-1)
    assert fiona.listlayers(filename) == ["layer1"]

    # invalid layer name
    with pytest.raises(ValueError):
        fiona.remove(filename, layer="invalid_layer_name")

    # invalid layer index
    with pytest.raises(DatasetDeleteError):
        fiona.remove(filename, layer=999)
예제 #2
0
def test_remove_layer(tmpdir):
    filename = str(tmpdir.join("a_filename.gpkg"))
    create_sample_data(filename, "GPKG", layer="layer1")
    create_sample_data(filename, "GPKG", layer="layer2")
    create_sample_data(filename, "GPKG", layer="layer3")
    create_sample_data(filename, "GPKG", layer="layer4")
    assert fiona.listlayers(filename) == ["layer1", "layer2", "layer3", "layer4"]
    
    # remove by index
    fiona.remove(filename, layer=2)
    assert fiona.listlayers(filename) == ["layer1", "layer2", "layer4"]
    
    # remove by name
    fiona.remove(filename, layer="layer2")
    assert fiona.listlayers(filename) == ["layer1", "layer4"]
    
    # remove by negative index
    fiona.remove(filename, layer=-1)
    assert fiona.listlayers(filename) == ["layer1"]
    
    # invalid layer name
    with pytest.raises(ValueError):
        fiona.remove(filename, layer="invalid_layer_name")
    
    # invalid layer index
    with pytest.raises(DatasetDeleteError):
        fiona.remove(filename, layer=999)
예제 #3
0
def test_remove_collection(tmpdir):
    outdir = str(tmpdir.mkdir('test_remove_collection'))
    filename_shp = os.path.join(outdir, 'test.shp')
    
    create_sample_data(filename_shp, driver='ESRI Shapefile')
    collection = fiona.open(filename_shp, 'r')
    fiona.remove(collection)
    assert(not os.path.exists(filename_shp))
예제 #4
0
def test_remove_collection(tmpdir):
    outdir = str(tmpdir.mkdir('test_remove_collection'))
    filename_shp = os.path.join(outdir, 'test.shp')

    create_sample_data(filename_shp, driver='ESRI Shapefile')
    collection = fiona.open(filename_shp, 'r')
    fiona.remove(collection)
    assert (not os.path.exists(filename_shp))
예제 #5
0
def test_remove_collection(tmpdir=None):
    if tmpdir is None:
        tmpdir = tempfile.mkdtemp()
    filename_shp = os.path.join(tmpdir, 'test.shp')

    create_sample_data(filename_shp, driver='ESRI Shapefile')
    collection = fiona.open(filename_shp, 'r')
    fiona.remove(collection)
    assert (not os.path.exists(filename_shp))
예제 #6
0
def test_remove_collection(tmpdir=None):
    if tmpdir is None:
        tmpdir = tempfile.mkdtemp()
    filename_shp = os.path.join(tmpdir, 'test.shp')
    
    create_sample_data(filename_shp, driver='ESRI Shapefile')
    collection = fiona.open(filename_shp, 'r')
    fiona.remove(collection)
    assert(not os.path.exists(filename_shp))
예제 #7
0
def test_remove(tmpdir):
    outdir = str(tmpdir.mkdir('test_remove'))
    filename_shp = os.path.join(outdir, 'test.shp')
    
    create_sample_data(filename_shp, driver='ESRI Shapefile')
    fiona.remove(filename_shp, driver='ESRI Shapefile')
    assert(not os.path.exists(filename_shp))
    
    with pytest.raises(RuntimeError):
        fiona.remove(filename_shp, driver='ESRI Shapefile')
예제 #8
0
def test_remove_path_without_driver(tmpdir):
    outdir = str(tmpdir.mkdir('test_remove_path_without_driver'))
    filename_shp = os.path.join(outdir, 'test.shp')

    create_sample_data(filename_shp, driver='ESRI Shapefile')

    with pytest.raises(Exception):
        fiona.remove(filename_shp)

    assert(os.path.exists(filename_shp))
예제 #9
0
def test_remove_driver(tmpdir):
    outdir = str(tmpdir.mkdir('test_remove_driver'))
    filename_shp = os.path.join(outdir, 'test.shp')
    filename_json = os.path.join(outdir, 'test.json')

    create_sample_data(filename_shp, driver='ESRI Shapefile')
    create_sample_data(filename_json, driver='GeoJSON')
    fiona.remove(filename_json, driver='GeoJSON')
    assert (not os.path.exists(filename_json))
    assert (os.path.exists(filename_shp))
예제 #10
0
def test_remove(tmpdir):
    outdir = str(tmpdir.mkdir('test_remove'))
    filename_shp = os.path.join(outdir, 'test.shp')

    create_sample_data(filename_shp, driver='ESRI Shapefile')
    fiona.remove(filename_shp, driver='ESRI Shapefile')
    assert (not os.path.exists(filename_shp))

    with pytest.raises(RuntimeError):
        fiona.remove(filename_shp, driver='ESRI Shapefile')
예제 #11
0
def test_remove_driver(tmpdir):
    outdir = str(tmpdir.mkdir('test_remove_driver'))
    filename_shp = os.path.join(outdir, 'test.shp')
    filename_json = os.path.join(outdir, 'test.json')
        
    create_sample_data(filename_shp, driver='ESRI Shapefile')
    create_sample_data(filename_json, driver='GeoJSON')
    fiona.remove(filename_json, driver='GeoJSON')
    assert(not os.path.exists(filename_json))
    assert(os.path.exists(filename_shp))
예제 #12
0
def test_remove_path_without_driver(tmpdir):
    outdir = str(tmpdir.mkdir('test_remove_path_without_driver'))
    filename_shp = os.path.join(outdir, 'test.shp')

    create_sample_data(filename_shp, driver='ESRI Shapefile')

    with pytest.raises(Exception):
        fiona.remove(filename_shp)

    assert (os.path.exists(filename_shp))
예제 #13
0
def test_remove(tmpdir=None):
    if tmpdir is None:
        tmpdir = tempfile.mkdtemp()
    filename_shp = os.path.join(tmpdir, 'test.shp')
    
    create_sample_data(filename_shp, driver='ESRI Shapefile')
    fiona.remove(filename_shp, driver='ESRI Shapefile')
    assert(not os.path.exists(filename_shp))
    
    with pytest.raises(RuntimeError):
        fiona.remove(filename_shp, driver='ESRI Shapefile')
예제 #14
0
def test_remove(tmpdir=None):
    if tmpdir is None:
        tmpdir = tempfile.mkdtemp()
    filename_shp = os.path.join(tmpdir, 'test.shp')

    create_sample_data(filename_shp, driver='ESRI Shapefile')
    fiona.remove(filename_shp, driver='ESRI Shapefile')
    assert (not os.path.exists(filename_shp))

    with pytest.raises(RuntimeError):
        fiona.remove(filename_shp, driver='ESRI Shapefile')
예제 #15
0
def test_remove_path_without_driver(tmpdir=None):
    if tmpdir is None:
        tmpdir = tempfile.mkdtemp()
    filename_shp = os.path.join(tmpdir, 'test.shp')

    create_sample_data(filename_shp, driver='ESRI Shapefile')

    with pytest.raises(Exception):
        fiona.remove(filename_shp)

    assert(os.path.exists(filename_shp))
예제 #16
0
def test_remove_driver(tmpdir=None):
    if tmpdir is None:
        tmpdir = tempfile.mkdtemp()
    filename_shp = os.path.join(tmpdir, 'test.shp')
    filename_json = os.path.join(tmpdir, 'test.json')
        
    create_sample_data(filename_shp, driver='ESRI Shapefile')
    create_sample_data(filename_json, driver='GeoJSON')
    fiona.remove(filename_json, driver='GeoJSON')
    assert(not os.path.exists(filename_json))
    assert(os.path.exists(filename_shp))
예제 #17
0
def test_remove_path_without_driver(tmpdir=None):
    if tmpdir is None:
        tmpdir = tempfile.mkdtemp()
    filename_shp = os.path.join(tmpdir, 'test.shp')

    create_sample_data(filename_shp, driver='ESRI Shapefile')

    with pytest.raises(Exception):
        fiona.remove(filename_shp)

    assert (os.path.exists(filename_shp))
예제 #18
0
def test_remove_driver(tmpdir=None):
    if tmpdir is None:
        tmpdir = tempfile.mkdtemp()
    filename_shp = os.path.join(tmpdir, 'test.shp')
    filename_json = os.path.join(tmpdir, 'test.json')

    create_sample_data(filename_shp, driver='ESRI Shapefile')
    create_sample_data(filename_json, driver='GeoJSON')
    fiona.remove(filename_json, driver='GeoJSON')
    assert (not os.path.exists(filename_json))
    assert (os.path.exists(filename_shp))
예제 #19
0
def test_remove_layer_geojson(tmpdir):
    """Removal of layers is not supported by GeoJSON driver

    The reason for failure is slightly different between GDAL 2.2+ and < 2.2.
    With < 2.2 the datasource will fail to open in write mode (OSError), while
    with 2.2+ the datasource will open but the removal operation will fail (not
    supported).
    """
    filename = str(tmpdir.join("a_filename.geojson"))
    create_sample_data(filename, "GeoJSON")
    with pytest.raises((RuntimeError, OSError)):
        fiona.remove(filename, layer=0)
    assert os.path.exists(filename)
예제 #20
0
def test_remove_layer_geojson(tmpdir):
    """Removal of layers is not supported by GeoJSON driver
    
    The reason for failure is slightly different between GDAL 2.2+ and < 2.2.
    With < 2.2 the datasource will fail to open in write mode (IOError), while
    with 2.2+ the datasource will open but the removal operation will fail (not
    supported).
    """
    filename = str(tmpdir.join("a_filename.geojson"))
    create_sample_data(filename, "GeoJSON")
    with pytest.raises((RuntimeError, IOError)):
        fiona.remove(filename, layer=0)
    assert os.path.exists(filename)
예제 #21
0
    def save_fiona_objects(self, table_name, objs, mode="w", crs=None):
        if not mode in ["w", "a"]:
            raise NotImplementedError("supported modes are ['w', 'a']")

        if mode == "w":
            if crs is None:
                crs = self.get_crs(table_name)
            fiona.remove(str(self._gpkg_path), layer=table_name)

        if not self._table_exists(table_name):
            self.create_table(table_name, crs)

        schema = self.get_schema(table_name)

        for obj in objs:
            if "id" in obj["properties"]:
                del obj["properties"]["id"]

        ### Don't use fiona's writerecords because it doesn't keep fid. ###
        ds = gdal.OpenEx(str(self._gpkg_path), gdal.OF_UPDATE | gdal.OF_VECTOR)
        layer = ds.GetLayerByName(table_name)

        # Add id if not exist
        for i, obj in enumerate(objs):
            if not "id" in obj:
                obj["id"] = i + 1

                # Get max id and increment in append mode
                if mode == "a":
                    gdf = self.get_all_features_as_gdf(table_name)
                    max_id = 0 if gdf.empty else max(gdf.id)
                    obj["id"] += max_id

        for obj in objs:
            feature = ogr.Feature(layer.GetLayerDefn())
            feature.SetFID(obj["id"])
            if obj["geometry"]:
                geometry = ogr.CreateGeometryFromWkt(
                    shape(obj["geometry"]).wkt)
                feature.SetGeometry(geometry)

            for prop in schema["properties"]:
                feature.SetField(prop, obj["properties"][prop])

            ret = layer.CreateFeature(feature)

            if ret != 0:
                raise RuntimeError("failed to create feature")

        self._reload_table(table_name)
예제 #22
0
파일: rm.py 프로젝트: Toblerity/Fiona
def rm(ctx, input, layer, yes):
    if layer is None:
        kind = "datasource"
    else:
        kind = "layer"

    if not yes:
        click.confirm("The {} will be removed. Are you sure?".format(kind), abort=True)

    try:
        fiona.remove(input, layer=layer)
    except Exception:
        logger.exception("Failed to remove {}.".format(kind))
        raise click.Abort()
예제 #23
0
def rm(ctx, input, layer, yes):
    if layer is None:
        kind = "datasource"
    else:
        kind = "layer"

    if not yes:
        click.confirm("The {} will be removed. Are you sure?".format(kind),
                      abort=True)

    try:
        fiona.remove(input, layer=layer)
    except Exception:
        logger.exception("Failed to remove {}.".format(kind))
        raise click.Abort()
예제 #24
0
def test_remove(tmpdir, kind, driver, specify_driver):
    """Test various dataset removal operations"""
    extension = {"ESRI Shapefile": "shp", "GeoJSON": "json"}[driver]
    filename = "delete_me.{extension}".format(extension=extension)
    output_filename = str(tmpdir.join(filename))

    create_sample_data(output_filename, driver=driver)
    if kind == "collection":
        to_delete = fiona.open(output_filename, "r")
    else:
        to_delete = output_filename

    assert os.path.exists(output_filename)
    if specify_driver:
        fiona.remove(to_delete, driver=driver)
    else:
        fiona.remove(to_delete)
    assert not os.path.exists(output_filename)
예제 #25
0
def test_remove(tmpdir, kind, driver, specify_driver):
    """Test various dataset removal operations"""
    extension = {"ESRI Shapefile": "shp", "GeoJSON": "json"}[driver]
    filename = "delete_me.{extension}".format(extension=extension)
    output_filename = str(tmpdir.join(filename))
    
    create_sample_data(output_filename, driver=driver)
    if kind == "collection":
        to_delete = fiona.open(output_filename, "r")
    else:
        to_delete = output_filename
    
    assert os.path.exists(output_filename)
    if specify_driver:
        fiona.remove(to_delete, driver=driver)
    else:
        fiona.remove(to_delete)
    assert not os.path.exists(output_filename)
예제 #26
0
    def __init__(self, out_path=None, crs=None, fieldname=None, driver=None):
        self._append = "a" in fiona.supported_drivers[driver]
        logger.debug("initialize %s writer with append %s", driver,
                     self._append)
        self.path = out_path
        self.driver = driver
        self.fieldname = fieldname
        self.new_entries = 0
        schema = deepcopy(spatial_schema)
        schema["properties"][fieldname] = "str:254"

        if self._append:
            if os.path.isfile(self.path):
                logger.debug("read existing entries")
                with fiona.open(self.path, "r") as src:
                    self._existing = {
                        f["properties"]["tile_id"]: f
                        for f in src
                    }
                self.file_obj = fiona.open(self.path, "a")
            else:
                self.file_obj = fiona.open(self.path,
                                           "w",
                                           driver=self.driver,
                                           crs=crs,
                                           schema=schema)
                self._existing = {}
        else:
            if os.path.isfile(self.path):
                logger.debug("read existing entries")
                with fiona.open(self.path, "r") as src:
                    self._existing = {
                        f["properties"]["tile_id"]: f
                        for f in src
                    }
                fiona.remove(self.path, driver=driver)
            else:
                self._existing = {}
            self.file_obj = fiona.open(self.path,
                                       "w",
                                       driver=self.driver,
                                       crs=crs,
                                       schema=schema)
            self.file_obj.writerecords(self._existing.values())
예제 #27
0
	def init_handler(self, df=None):
		import fiona
		schema = self._get_schema(df)

		layername = None
		if '.gpkg:' in self.target:
			try:
				self.filename, layername = self.target.split(':')
			except ValueError as e:
				raise argh.CommandError('File name should be name.gpkg or name.gpkg:layer_name. Got "%s" instead.' % self.target)
		else:
			self.filename = self.target
			layername = os.path.splitext(os.path.basename(self.target))[0]

		# instead of self._cleanup_target(), delete fiona layer
		if os.path.exists(self.filename) and layername in fiona.listlayers(self.filename):
			fiona.remove(self.filename, self.fiona_driver, layername)

		crs = df.crs if df is not None else None
		self._handler = fiona.open(self.filename, 'w', crs=crs, driver=self.fiona_driver, schema=schema, layer=layername)
예제 #28
0
 def __init__(self, out_path=None, crs=None, fieldname=None, driver=None):
     logger.debug("initialize %s writer", driver)
     self.path = out_path
     self.driver = driver
     if os.path.isfile(self.path):
         with fiona.open(self.path) as src:
             self.existing = {f["properties"]["tile_id"]: f for f in src}
         fiona.remove(self.path, driver=driver)
     else:
         self.existing = {}
     self.new_entries = 0
     self.fieldname = fieldname
     schema = deepcopy(spatial_schema)
     schema["properties"][fieldname] = "str:254"
     self.file_obj = fiona.open(self.path,
                                "w",
                                driver=self.driver,
                                crs=crs,
                                schema=schema)
     self.file_obj.writerecords(self.existing.values())
예제 #29
0
def createStreetSegment(inshp, outshp, mini_dist):
    '''
    This function will segment streets into different street segments
    Required modules: Fiona and Shapely
    
    parameters:
        inshp: the input linear shapefile, must be in WGS84 projection, ESPG: 4326
        output: the result street segment feature class
        mini_dist: the minimum distance between two created point
    
    First version May 3rd, 2018
    last modified by Xiaojiang Li, MIT Senseable City Lab
    
    '''

    import fiona
    import os, os.path
    from shapely.geometry import shape, mapping, Point, LineString
    from shapely.ops import transform
    from functools import partial
    import pyproj
    from fiona.crs import from_epsg

    count = 0
    s = {
        'trunk_link', 'tertiary', 'motorway', 'motorway_link', 'steps', None,
        ' ', 'pedestrian', 'primary', 'primary_link', 'footway',
        'tertiary_link', 'trunk', 'secondary', 'secondary_link',
        'tertiary_link', 'bridleway', 'service'
    }

    # the temporaray file of the cleaned data
    root = os.path.dirname(inshp)
    basename = 'clean_' + os.path.basename(inshp)
    temp_cleanedStreetmap = os.path.join(root, basename)

    # if the tempfile exist then delete it
    if os.path.exists(temp_cleanedStreetmap):
        fiona.remove(temp_cleanedStreetmap, 'ESRI Shapefile')

    # clean the original street maps by removing highways, if it the street map not from Open street data, users'd better to clean the data themselve
    with fiona.open(inshp) as source, fiona.open(temp_cleanedStreetmap,
                                                 'w',
                                                 driver=source.driver,
                                                 crs=source.crs,
                                                 schema=source.schema) as dest:

        for feat in source:
            try:
                i = feat['properties']['highway']  # for the OSM street data
                if i in s:
                    continue
            except:
                # if the street map is not osm, do nothing. You'd better to clean the street map, if you don't want to map the GVI for highways
                key = dest.schema['properties'].keys(
                )[0]  # get the field of the input shapefile and duplicate the input feature
                i = feat['properties'][key]
                if i in s:
                    continue

            dest.write(feat)

    schema = {
        'geometry': 'LineString',
        'properties': {
            'id': 'int'
        },
    }

    # Create street segment along the streets
    with fiona.drivers():
        #with fiona.open(outshp, 'w', 'ESRI Shapefile', crs=source.crs, schema) as output:
        with fiona.open(outshp,
                        'w',
                        crs=from_epsg(4326),
                        driver='ESRI Shapefile',
                        schema=schema) as output:
            for line in fiona.open(temp_cleanedStreetmap):

                # define projection and the inverse projection
                project = partial(pyproj.transform,
                                  pyproj.Proj(init='EPSG:4326'),
                                  pyproj.Proj(init='EPSG:3857')
                                  )  #3857 is psudo WGS84 the unit is meter
                project2 = partial(pyproj.transform,
                                   pyproj.Proj(init='EPSG:3857'),
                                   pyproj.Proj(init='EPSG:4326'))

                # deal with MultiLineString and LineString
                featureType = line['geometry']['type']
                dist = mini_dist

                # for the LineString
                if featureType == "LineString":
                    first = shape(line['geometry'])
                    length = first.length

                    # convert the projection of wgs84 from degree to meters
                    line2 = transform(project, first)

                    # cut the line feature using the distance of dist
                    st_seg = cutLine_series_pnt(line2, dist)

                    # loop all the street segments and then save them as a new shapefile
                    for seg in st_seg:
                        # do inverse projection, and the new feature has WGS84
                        line3 = transform(project2, seg)
                        output.write({
                            'geometry': mapping(line3),
                            'properties': {
                                'id': 1
                            }
                        })

                else:
                    continue

    print("Process Complete")

    # delete the temprary cleaned shapefile
    fiona.remove(temp_cleanedStreetmap, 'ESRI Shapefile')
예제 #30
0
def write(df, fname):
	"""
	Writes [Geo]DataFrame to file or database. Format (driver) is detected by the fname parameter.

	'*.csv' => csv (geometry is transformed to WKT)
	'*.geojson' => GeoJSON
	'*.gpkg[:<layer_name>]' => GeoPackage (GPKG)
	'postgresql://' => Postgresql table (postgresql://[user[:password]@]hostname[:port]/<db_name>#<table_name or query>)
	"""
	from . import utils
	if isinstance(df, gpd.GeoDataFrame) and len(df) > 0:
		df['geometry'] = utils.fix_multitypes(df['geometry'])

	match_gpkg = re.match(r'^(?P<filename>.*/(?P<file_own_name>.*)\.(?P<extension>gpkg))(?:\:(?P<layer_name>[a-z0-9_]+))?$', fname)
	match_geojson = re.match(r'^(?P<filename>.*/(?P<file_own_name>.*)\.(?P<extension>gpkg))$', fname)
	match_postgres = re.match(r'^postgresql\://', fname)
	match_csv = fname.endswith('.csv')

	if match_postgres:
		engine, table_name = _connect_postgres(fname)
		df = df[list(df)]

		with engine.begin() as connection:
			if 'geometry' in df:
				df['geometry'] = df['geometry'].apply(wkb.dumps).apply(bytes.hex)

			pd.io.sql.execute('DROP TABLE IF EXISTS %s' % table_name, connection)
			df.to_sql(table_name, connection, chunksize=1000)
			if 'geometry' in df:
				if not df.crs and -181 < df['geometry'].extents[0] < 181:
					crs_num = 4326
				elif not df.crs:
					crs_num = 3857
				else:
					crs_num = re.match(r'.*\:(\d+)', df.crs['init'])[1]

				pd.io.sql.execute("""
					ALTER TABLE %s
					ALTER COLUMN "geometry" TYPE Geometry""" % table_name, connection)
				pd.io.sql.execute("""
					UPDATE %s SET "geometry"=st_setsrid(geometry, %s)
					""" % (table_name, crs_num), connection)


	elif isinstance(df, gpd.GeoDataFrame):
		if match_csv:
			if os.path.exists(fname):
				os.unlink(fname)
			df = pd.DataFrame(df)
			df.to_csv(fname, index=False)

		elif match_gpkg:
			filename = match_gpkg['filename']
			layer_name = match_gpkg['layer_name'] or match_gpkg['file_own_name']
			
			if os.path.exists(filename):
				if layer_name in listlayers(filename):
					remove(filename, 'GPKG', layer_name)

			df.to_file(filename, driver='GPKG', encoding='utf-8', layer=layer_name)
		elif match_geojson:
			if os.path.exists(filename):
				os.unlink(filename)

			df.to_file(filename, driver='GeoJSON', encoding='utf-8')

	elif isinstance(df, pd.DataFrame):
		if fname.endswith('.json'):
			df.to_json(fname)
		else:
			df.to_csv(fname, index=False)
예제 #31
0
def test_remove_nonexistent(tmpdir):
    """Attempting to remove a file that does not exist results in an IOError"""
    filename = str(tmpdir.join("does_not_exist.shp"))
    assert not os.path.exists(filename)
    with pytest.raises(IOError):
        fiona.remove(filename)
예제 #32
0
def createPoints(inshp, outshp, mini_dist):
    '''
    This function will parse throigh the street network of provided city and
    clean all highways and create points every mini_dist meters (or as specified) along
    the linestrings
    Required modules: Fiona and Shapely

    parameters:
        inshp: the input linear shapefile, must be in WGS84 projection, ESPG: 4326
        output: the result point feature class
        mini_dist: the minimum distance between two created point

    modified by May 2ed, 2018, consider the linestring and multi-linear string
    last modified by Xiaojiang Li, MIT Senseable City Lab
    
    '''

    import fiona
    import os, os.path
    from shapely.geometry import shape, mapping
    from shapely.ops import transform
    from functools import partial
    import pyproj
    from fiona.crs import from_epsg
    import sys

    count = 0
    # s = {'trunk_link','tertiary','motorway','motorway_link','steps', None, ' ','pedestrian','primary', 'primary_link','footway','tertiary_link', 'trunk','secondary','secondary_link','tertiary_link','bridleway','service'}
    # s = {'trunk_link','tertiary','motorway','motorway_link','steps', ' ','pedestrian','primary', 'primary_link','footway','tertiary_link', 'trunk','secondary','secondary_link','tertiary_link','bridleway','service'}
    s = {}

    # the temporaray file of the cleaned data
    root = os.path.dirname(inshp)
    basename = 'clean_' + os.path.basename(inshp)
    temp_cleanedStreetmap = os.path.join(root, basename)

    # if the tempfile exist then delete it
    if os.path.exists(temp_cleanedStreetmap):
        fiona.remove(temp_cleanedStreetmap, 'ESRI Shapefile')
        print('removed the existed tempfile')

    # clean the original street maps by removing highways, if it the street map not from Open street data, users'd better to clean the data themselve
    with fiona.open(inshp) as source, fiona.open(temp_cleanedStreetmap,
                                                 'w',
                                                 driver=source.driver,
                                                 crs=source.crs,
                                                 schema=source.schema) as dest:
        for feat in source:
            try:
                i = feat['properties']['highway']  # for the OSM street data
                # i = feat['properties']['fclass'] # for the OSM tokyo street data
                if i in s:
                    continue
            except:
                # if the street map is not osm, do nothing. You'd better to clean the street map, if you don't want to map the GVI for highways
                key = list(dest.schema['properties'].keys())[0]
                # key = dest.schema['properties'].keys()[0] # get the field of the input shapefile and duplicate the input feature
                i = feat['properties'][key]
                if i in s:
                    continue

            # print feat
            dest.write(feat)

    schema = {
        'geometry': 'Point',
        'properties': {
            'id': 'int'
        },
    }

    # Create point along the streets
    with fiona.Env():
        with fiona.open(outshp,
                        'w',
                        crs=from_epsg(4326),
                        driver='ESRI Shapefile',
                        schema=schema) as output:
            for line in fiona.open(temp_cleanedStreetmap):
                line_geom = line['geometry']
                featureType = line_geom['type']

                try:
                    if featureType == 'LineString':
                        line_geom_degree = shape(line_geom)

                        # convert degree to meter, in order to split by distance in meter
                        project = partial(
                            pyproj.transform, pyproj.Proj(init='EPSG:4326'),
                            pyproj.Proj(init='EPSG:3857')
                        )  #3857 is psudo WGS84 the unit is meter
                        line_geom_meter = transform(project, line_geom_degree)

                        for distance in range(0, int(line_geom_meter.length),
                                              mini_dist):
                            point = line_geom_meter.interpolate(distance)

                            # convert the local projection back the the WGS84 and write to the output shp
                            project2 = partial(pyproj.transform,
                                               pyproj.Proj(init='EPSG:3857'),
                                               pyproj.Proj(init='EPSG:4326'))
                            point_deg = transform(project2, point)
                            output.write({
                                'geometry': mapping(point_deg),
                                'properties': {
                                    'id': 1
                                }
                            })

                    # for the MultiLineString, seperate these lines, then partition those lines
                    elif featureType == "MultiLineString":
                        multiline_geom = shape(line['geometry'])
                        print('This is a multiline')
                        for singleLine in multiline_geom:
                            length = singleLine.length

                            # partion each single line in the multiline
                            project = partial(
                                pyproj.transform,
                                pyproj.Proj(init='EPSG:4326'),
                                pyproj.Proj(init='EPSG:3857')
                            )  #3857 is psudo WGS84 the unit is meter
                            line2 = transform(project, singleLine)
                            linestr = list(line2.coords)
                            dist = mini_dist  #set

                            for distance in range(0, int(line2.length), dist):
                                point = line2.interpolate(distance)
                                project2 = partial(
                                    pyproj.transform,
                                    pyproj.Proj(init='EPSG:3857'),
                                    pyproj.Proj(init='EPSG:4326'))
                                point = transform(project2, point)
                                output.write({
                                    'geometry': mapping(point),
                                    'properties': {
                                        'id': 1
                                    }
                                })

                    else:
                        print('Else--------')
                        continue

                except:
                    print("You should make sure the input shapefile is WGS84")
                    print(sys.exc_info())
                    return

    print("Process Complete")

    # delete the temprary cleaned shapefile
    fiona.remove(temp_cleanedStreetmap, 'ESRI Shapefile')
예제 #33
0
def create_points(inshp, outshp, mini_dist):
    """
    This function will parse through the street network of provided city and
    clean all highways and create points every mini_dist meters (or as specified) along
    the linestrings
    Required modules: Fiona and Shapely

    parameters:
        inshp: the input linear shapefile, must be in WGS84 projection, ESPG: 4326
        output: the result point feature class
        mini_dist: the minimum distance between two created point

    last modified by Xiaojiang Li, MIT Senseable City Lab
    """

    import fiona
    import os.path
    from shapely.geometry import shape, mapping
    from shapely.ops import transform
    from pyproj import Transformer
    from fiona.crs import from_epsg

    s = {
        'trunk_link', 'tertiary', 'motorway', 'motorway_link', 'steps', None,
        ' ', 'pedestrian', 'primary', 'primary_link', 'footway',
        'tertiary_link', 'trunk', 'secondary', 'secondary_link',
        'tertiary_link', 'bridleway', 'service'
    }

    # the temporary file of the cleaned data
    root_dir = os.path.dirname(inshp)
    basename = 'clean_' + os.path.basename(inshp)
    temp_cleaned_streetmap = os.path.join(root_dir, basename)

    # if the tempfile exist then delete it
    if os.path.exists(temp_cleaned_streetmap):
        fiona.remove(temp_cleaned_streetmap, 'ESRI Shapefile')

    # clean the original street maps by removing highways, if it the street map not from Open street data, users'd
    # better to clean the data themselves
    with fiona.open(inshp) as source, fiona.open(temp_cleaned_streetmap,
                                                 'w',
                                                 driver=source.driver,
                                                 crs=source.crs,
                                                 schema=source.schema) as dest:

        for feat in source:
            try:
                i = feat['properties']['highway']  # for the OSM street data
                if i in s:
                    continue
            except:
                # if the street map is not osm, do nothing. You'd better to clean the street map, if you don't want
                # to map the GVI for highways
                # get the field of the input shapefile and duplicate the input feature
                key = list(dest.schema['properties'].keys())[0]
                i = feat['properties'][key]
                if i in s:
                    continue

            dest.write(feat)

    schema = {
        'geometry': 'Point',
        'properties': {
            'id': 'int'
        },
    }

    # Create points along the streets
    with fiona.Env():
        # with fiona.open(outshp, 'w', 'ESRI Shapefile', crs=source.crs, schema) as output:
        with fiona.open(outshp,
                        'w',
                        crs=from_epsg(4326),
                        driver='ESRI Shapefile',
                        schema=schema) as output:
            transformation1 = Transformer.from_crs('EPSG:4326',
                                                   'EPSG:3857',
                                                   always_xy=True).transform
            transformation2 = Transformer.from_crs('EPSG:3857',
                                                   'EPSG:4326',
                                                   always_xy=True).transform
            for line in fiona.open(temp_cleaned_streetmap):
                first = shape(line['geometry'])

                try:
                    # convert degree to meter, in order to split by distance in meter
                    # EPSG:3857 is pseudo WGS84 the unit is meter
                    line2 = transform(transformation1, first)
                    # print(line2.coords)
                    # linestr = list(line2.coords)
                    dist = mini_dist  # set
                    for distance in range(0, int(line2.length), dist):
                        point = line2.interpolate(distance)

                        # convert the local projection back to the WGS84 and write to the output shp
                        point = transform(transformation2, point)
                        output.write({
                            'geometry': mapping(point),
                            'properties': {
                                'id': 1
                            }
                        })
                except:
                    print("You should make sure the input shapefile is WGS84")
                    raise

    print("Process Complete")

    # delete the temporary cleaned shapefile
    fiona.remove(temp_cleaned_streetmap, 'ESRI Shapefile')
예제 #34
0
def test_remove_layer_shapefile(tmpdir):
    """Removal of layer in shapefile actually deletes the datasource"""
    filename = str(tmpdir.join("a_filename.shp"))
    create_sample_data(filename, "ESRI Shapefile")
    fiona.remove(filename, layer=0)
    assert not os.path.exists(filename)
예제 #35
0
def test_remove_layer_shapefile(tmpdir):
    """Removal of layer in shapefile actually deletes the datasource"""
    filename = str(tmpdir.join("a_filename.shp"))
    create_sample_data(filename, "ESRI Shapefile")
    fiona.remove(filename, layer=0)
    assert not os.path.exists(filename)
예제 #36
0
def mask_to_shp(src, out_shp, mask_value=0, driver='ESRI Shapefile'):
    """ 将掩膜数据矢量化(输出为shapefile格式).

    Note:
        * 输出shapefile字段及属性写死在代码;
        * 最好只用于掩膜值为特定值的数据,如果将每一个像素矢量化会非常慢;

    Args:
        src (rasterio dataset): 输入的包含掩膜值的栅格数据集.
        out_shp (str): 输出shapefile文件.
        mask_value (float): 掩膜值,默认0.
        driver (str): 输出矢量类型, 默认 'ESRI Shapefile'.

    """
    if src.count != 1:
        return

    if os.path.isfile(out_shp):
        fiona.remove(out_shp, driver=driver)

    import mask
    # 分窗口读取,用于占用内存比较大数组
    windows = mask.window_list(10000, src.width, src.height)

    # 直接用于PS处理后每一层的蒙版导出结果
    for idx_window, window in enumerate(windows):
        print("process {} of total {}\n".format(idx_window + 1, len(windows)))

        # read data of current window
        mask_data = src.read(1, window=window)

        # transfrom of current window
        transform = rasterio.windows.transform(window, src.transform)

        # get features of the same value from an array
        none_zero = features.shapes(mask_data,
                                    mask=mask_data == mask_value,
                                    connectivity=4,
                                    transform=transform)
        mask_data = None

        # get record
        results = ({
            'properties': {
                'date': '20181007'
            },
            'geometry': s
        } for i, (s, v) in enumerate(none_zero))

        # shapefile schema
        source_schema = {
            'geometry': 'Polygon',
            'properties': {
                'date': 'str:20'
            }
        }

        # write to shapefile using fiona
        if results is not None:
            for r in results:
                # TODO merge result features by order
                try:
                    with fiona.open(out_shp,
                                    'a',
                                    driver=driver,
                                    crs=from_epsg(src.crs.to_epsg()),
                                    schema=source_schema) as c:
                        c.write(r)
                except OSError:
                    with fiona.open(out_shp,
                                    'w',
                                    driver=driver,
                                    crs=from_epsg(src.crs.to_epsg()),
                                    schema=source_schema) as c:
                        c.write(r)
예제 #37
0
def test_remove_nonexistent(tmpdir):
    """Attempting to remove a file that does not exist results in an OSError"""
    filename = str(tmpdir.join("does_not_exist.shp"))
    assert not os.path.exists(filename)
    with pytest.raises(OSError):
        fiona.remove(filename)
예제 #38
0
def createPoints(inshp, outshp, mini_dist):
    '''
    This function will parse throigh the street network of provided city and
    clean all highways and create points every mini_dist meters (or as specified) along
    the linestrings
    Required modules: Fiona and Shapely
    parameters:
        inshp: the input linear shapefile, must be in WGS84 projection, ESPG: 4326
        output: the result point feature class
        mini_dist: the minimum distance between two created point
    
    '''

    import fiona
    import os, os.path
    from shapely.geometry import shape, mapping
    from shapely.ops import transform
    from functools import partial
    import pyproj
    from fiona.crs import from_epsg

    count = 0

    #name the road types NOT to be used.
    s = {
        'trunk_link', 'motorway', 'motorway_link', 'steps', None, 'pedestrian',
        'trunk', 'bridleway', 'service'
    }

    # the temporaray file of the cleaned data
    root = os.path.dirname(inshp)
    basename = 'clean_' + os.path.basename(inshp)
    temp_cleanedStreetmap = os.path.join(root, basename)

    # if the tempfile exist then delete it
    if os.path.exists(temp_cleanedStreetmap):
        fiona.remove(temp_cleanedStreetmap, 'ESRI Shapefile')

    # clean the original street maps by removing highways, if it the street map not from Open street data, users'd better to clean the data themselve
    with fiona.open(inshp) as source, fiona.open(temp_cleanedStreetmap,
                                                 'w',
                                                 driver=source.driver,
                                                 crs=source.crs,
                                                 schema=source.schema) as dest:

        for feat in source:
            try:
                i = feat['properties']['fclass']  # for the OSM street data
                if i in s:
                    continue
            except:
                # if the street map is not osm, do nothing. You'd better to clean the street map, if you don't want to map the GVI for highways
                # key = dest.schema['properties'].keys()[0] # get the field of the input shapefile and duplicate the input feature
                # i = feat['properties'][key]
                # if i in s:
                continue

            dest.write(feat)

    schema = {
        'geometry': 'Point',
        'properties': {
            'id': 'int'
        },
    }

    # Create pointS along the streets
    with fiona.drivers():
        #with fiona.open(outshp, 'w', 'ESRI Shapefile', crs=source.crs, schema) as output:
        with fiona.open(outshp,
                        'w',
                        crs=from_epsg(4326),
                        driver='ESRI Shapefile',
                        schema=schema) as output:
            for line in fiona.open(temp_cleanedStreetmap):
                first = shape(line['geometry'])
                if first.geom_type != 'LineString': continue

                length = first.length

                try:
                    # convert degree to meter, in order to split by distance in meter
                    project = partial(pyproj.transform,
                                      pyproj.Proj(init='EPSG:4326'),
                                      pyproj.Proj(init='EPSG:3067')
                                      )  #3857 is psudo WGS84 the unit is meter

                    line2 = transform(project, first)
                    linestr = list(line2.coords)
                    dist = mini_dist  #set
                    for distance in range(0, int(line2.length), dist):
                        point = line2.interpolate(distance)

                        # convert the local projection back the the WGS84 and write to the output shp
                        project2 = partial(pyproj.transform,
                                           pyproj.Proj(init='EPSG:3067'),
                                           pyproj.Proj(init='EPSG:4326'))
                        point = transform(project2, point)
                        output.write({
                            'geometry': mapping(point),
                            'properties': {
                                'id': 1
                            }
                        })
                except Exception as e:
                    print(e)
                    print("You should make sure the input shapefile is WGS84")
                    return

    print("Process Complete")
예제 #39
0
def createPoints(inshp, outshp, mini_dist):
    '''
    This function will parse throigh the street network of provided city and
    clean all highways and create points every mini_dist meters (or as specified) along
    the linestrings
    Required modules: Fiona and Shapely

    parameters:
        inshp: the input linear shapefile, must be in WGS84 projection, ESPG: 4326
        output: the result point feature class
        mini_dist: the minimum distance between two created point

    last modified by Xiaojiang Li, MIT Senseable City Lab
    
    '''

    import fiona
    import os, os.path
    from shapely.geometry import shape, mapping
    from shapely.ops import transform
    from functools import partial
    import pyproj
    from fiona.crs import from_epsg

    count = 0
    s = {
        'trunk_link', 'tertiary', 'motorway', 'motorway_link', 'steps', None,
        ' ', 'pedestrian', 'primary', 'primary_link', 'footway',
        'tertiary_link', 'trunk', 'secondary', 'secondary_link',
        'tertiary_link', 'bridleway', 'service'
    }

    # the temporaray file of the cleaned data
    root = os.path.dirname(inshp)
    basename = 'clean_' + os.path.basename(inshp)
    temp_cleanedStreetmap = os.path.join(root, basename)

    # if the tempfile exist then delete it
    if os.path.exists(temp_cleanedStreetmap):
        fiona.remove(temp_cleanedStreetmap, 'ESRI Shapefile')
        #driver = ogr.GetDriverByName("ESRI Shapefile")
        #driver.DeleteDataSource(temp_cleanedStreetmap)

    # clean the original street maps
    with fiona.open(inshp) as source, fiona.open(temp_cleanedStreetmap,
                                                 'w',
                                                 driver=source.driver,
                                                 crs=source.crs,
                                                 schema=source.schema) as dest:
        for feat in source:
            try:
                i = feat['properties']['highway']  # for the OSM street data
                if i in s:
                    continue
            except:
                i = feat['properties'][
                    'Restrictio']  # for cambridge center line data
                if i in s:
                    continue

            dest.write(feat)

    schema = {
        'geometry': 'Point',
        'properties': {
            'id': 'int'
        },
    }

    with fiona.drivers():
        #with fiona.open(outshp, 'w', 'ESRI Shapefile', crs=source.crs, schema) as output:
        with fiona.open(outshp,
                        'w',
                        crs=from_epsg(4326),
                        driver='ESRI Shapefile',
                        schema=schema) as output:
            for line in fiona.open(temp_cleanedStreetmap):
                first = shape(line['geometry'])
                length = first.length

                # convert degree to meter, in order to split by distance in meter
                project = partial(pyproj.transform,
                                  pyproj.Proj(init='EPSG:4326'),
                                  pyproj.Proj(init='EPSG:3857')
                                  )  #3857 is psudo WGS84 the unit is meter

                line2 = transform(project, first)
                linestr = list(line2.coords)
                dist = mini_dist  #set
                for distance in range(0, int(line2.length), 20):
                    point = line2.interpolate(distance)

                    # convert the local projection back the the WGS84 and write to the output shp
                    project2 = partial(pyproj.transform,
                                       pyproj.Proj(init='EPSG:3857'),
                                       pyproj.Proj(init='EPSG:4326'))
                    point = transform(project2, point)
                    output.write({
                        'geometry': mapping(point),
                        'properties': {
                            'id': 1
                        }
                    })

    print("Process Complete")

    # delete the temprary cleaned shapefile
    fiona.remove(temp_cleanedStreetmap, 'ESRI Shapefile')