def test_remove_layer(tmpdir):
filename = str(tmpdir.join("a_filename.gpkg"))
create_sample_data(filename, "GPKG", layer="layer1")
create_sample_data(filename, "GPKG", layer="layer2")
create_sample_data(filename, "GPKG", layer="layer3")
create_sample_data(filename, "GPKG", layer="layer4")
assert fiona.listlayers(filename) == [
"layer1", "layer2", "layer3", "layer4"
]
# remove by index
fiona.remove(filename, layer=2)
assert fiona.listlayers(filename) == ["layer1", "layer2", "layer4"]
# remove by name
fiona.remove(filename, layer="layer2")
assert fiona.listlayers(filename) == ["layer1", "layer4"]
# remove by negative index
fiona.remove(filename, layer=-1)
assert fiona.listlayers(filename) == ["layer1"]
# invalid layer name
with pytest.raises(ValueError):
fiona.remove(filename, layer="invalid_layer_name")
# invalid layer index
with pytest.raises(DatasetDeleteError):
fiona.remove(filename, layer=999)
def test_remove_layer(tmpdir):
filename = str(tmpdir.join("a_filename.gpkg"))
create_sample_data(filename, "GPKG", layer="layer1")
create_sample_data(filename, "GPKG", layer="layer2")
create_sample_data(filename, "GPKG", layer="layer3")
create_sample_data(filename, "GPKG", layer="layer4")
assert fiona.listlayers(filename) == ["layer1", "layer2", "layer3", "layer4"]
# remove by index
fiona.remove(filename, layer=2)
assert fiona.listlayers(filename) == ["layer1", "layer2", "layer4"]
# remove by name
fiona.remove(filename, layer="layer2")
assert fiona.listlayers(filename) == ["layer1", "layer4"]
# remove by negative index
fiona.remove(filename, layer=-1)
assert fiona.listlayers(filename) == ["layer1"]
# invalid layer name
with pytest.raises(ValueError):
fiona.remove(filename, layer="invalid_layer_name")
# invalid layer index
with pytest.raises(DatasetDeleteError):
fiona.remove(filename, layer=999)
def test_remove_collection(tmpdir):
outdir = str(tmpdir.mkdir('test_remove_collection'))
filename_shp = os.path.join(outdir, 'test.shp')
create_sample_data(filename_shp, driver='ESRI Shapefile')
collection = fiona.open(filename_shp, 'r')
fiona.remove(collection)
assert(not os.path.exists(filename_shp))
def test_remove_collection(tmpdir):
outdir = str(tmpdir.mkdir('test_remove_collection'))
filename_shp = os.path.join(outdir, 'test.shp')
create_sample_data(filename_shp, driver='ESRI Shapefile')
collection = fiona.open(filename_shp, 'r')
fiona.remove(collection)
assert (not os.path.exists(filename_shp))
def test_remove_collection(tmpdir=None):
if tmpdir is None:
tmpdir = tempfile.mkdtemp()
filename_shp = os.path.join(tmpdir, 'test.shp')
create_sample_data(filename_shp, driver='ESRI Shapefile')
collection = fiona.open(filename_shp, 'r')
fiona.remove(collection)
assert (not os.path.exists(filename_shp))
def test_remove_collection(tmpdir=None):
if tmpdir is None:
tmpdir = tempfile.mkdtemp()
filename_shp = os.path.join(tmpdir, 'test.shp')
create_sample_data(filename_shp, driver='ESRI Shapefile')
collection = fiona.open(filename_shp, 'r')
fiona.remove(collection)
assert(not os.path.exists(filename_shp))
def test_remove(tmpdir):
outdir = str(tmpdir.mkdir('test_remove'))
filename_shp = os.path.join(outdir, 'test.shp')
create_sample_data(filename_shp, driver='ESRI Shapefile')
fiona.remove(filename_shp, driver='ESRI Shapefile')
assert(not os.path.exists(filename_shp))
with pytest.raises(RuntimeError):
fiona.remove(filename_shp, driver='ESRI Shapefile')
def test_remove_path_without_driver(tmpdir):
outdir = str(tmpdir.mkdir('test_remove_path_without_driver'))
filename_shp = os.path.join(outdir, 'test.shp')
create_sample_data(filename_shp, driver='ESRI Shapefile')
with pytest.raises(Exception):
fiona.remove(filename_shp)
assert(os.path.exists(filename_shp))
def test_remove_driver(tmpdir):
outdir = str(tmpdir.mkdir('test_remove_driver'))
filename_shp = os.path.join(outdir, 'test.shp')
filename_json = os.path.join(outdir, 'test.json')
create_sample_data(filename_shp, driver='ESRI Shapefile')
create_sample_data(filename_json, driver='GeoJSON')
fiona.remove(filename_json, driver='GeoJSON')
assert (not os.path.exists(filename_json))
assert (os.path.exists(filename_shp))
def test_remove(tmpdir):
outdir = str(tmpdir.mkdir('test_remove'))
filename_shp = os.path.join(outdir, 'test.shp')
create_sample_data(filename_shp, driver='ESRI Shapefile')
fiona.remove(filename_shp, driver='ESRI Shapefile')
assert (not os.path.exists(filename_shp))
with pytest.raises(RuntimeError):
fiona.remove(filename_shp, driver='ESRI Shapefile')
def test_remove_driver(tmpdir):
outdir = str(tmpdir.mkdir('test_remove_driver'))
filename_shp = os.path.join(outdir, 'test.shp')
filename_json = os.path.join(outdir, 'test.json')
create_sample_data(filename_shp, driver='ESRI Shapefile')
create_sample_data(filename_json, driver='GeoJSON')
fiona.remove(filename_json, driver='GeoJSON')
assert(not os.path.exists(filename_json))
assert(os.path.exists(filename_shp))
def test_remove_path_without_driver(tmpdir):
outdir = str(tmpdir.mkdir('test_remove_path_without_driver'))
filename_shp = os.path.join(outdir, 'test.shp')
create_sample_data(filename_shp, driver='ESRI Shapefile')
with pytest.raises(Exception):
fiona.remove(filename_shp)
assert (os.path.exists(filename_shp))
def test_remove(tmpdir=None):
if tmpdir is None:
tmpdir = tempfile.mkdtemp()
filename_shp = os.path.join(tmpdir, 'test.shp')
create_sample_data(filename_shp, driver='ESRI Shapefile')
fiona.remove(filename_shp, driver='ESRI Shapefile')
assert(not os.path.exists(filename_shp))
with pytest.raises(RuntimeError):
fiona.remove(filename_shp, driver='ESRI Shapefile')
def test_remove(tmpdir=None):
if tmpdir is None:
tmpdir = tempfile.mkdtemp()
filename_shp = os.path.join(tmpdir, 'test.shp')
create_sample_data(filename_shp, driver='ESRI Shapefile')
fiona.remove(filename_shp, driver='ESRI Shapefile')
assert (not os.path.exists(filename_shp))
with pytest.raises(RuntimeError):
fiona.remove(filename_shp, driver='ESRI Shapefile')
def test_remove_path_without_driver(tmpdir=None):
if tmpdir is None:
tmpdir = tempfile.mkdtemp()
filename_shp = os.path.join(tmpdir, 'test.shp')
create_sample_data(filename_shp, driver='ESRI Shapefile')
with pytest.raises(Exception):
fiona.remove(filename_shp)
assert(os.path.exists(filename_shp))
def test_remove_driver(tmpdir=None):
if tmpdir is None:
tmpdir = tempfile.mkdtemp()
filename_shp = os.path.join(tmpdir, 'test.shp')
filename_json = os.path.join(tmpdir, 'test.json')
create_sample_data(filename_shp, driver='ESRI Shapefile')
create_sample_data(filename_json, driver='GeoJSON')
fiona.remove(filename_json, driver='GeoJSON')
assert(not os.path.exists(filename_json))
assert(os.path.exists(filename_shp))
def test_remove_path_without_driver(tmpdir=None):
if tmpdir is None:
tmpdir = tempfile.mkdtemp()
filename_shp = os.path.join(tmpdir, 'test.shp')
create_sample_data(filename_shp, driver='ESRI Shapefile')
with pytest.raises(Exception):
fiona.remove(filename_shp)
assert (os.path.exists(filename_shp))
def test_remove_driver(tmpdir=None):
if tmpdir is None:
tmpdir = tempfile.mkdtemp()
filename_shp = os.path.join(tmpdir, 'test.shp')
filename_json = os.path.join(tmpdir, 'test.json')
create_sample_data(filename_shp, driver='ESRI Shapefile')
create_sample_data(filename_json, driver='GeoJSON')
fiona.remove(filename_json, driver='GeoJSON')
assert (not os.path.exists(filename_json))
assert (os.path.exists(filename_shp))
def test_remove_layer_geojson(tmpdir):
"""Removal of layers is not supported by GeoJSON driver
The reason for failure is slightly different between GDAL 2.2+ and < 2.2.
With < 2.2 the datasource will fail to open in write mode (OSError), while
with 2.2+ the datasource will open but the removal operation will fail (not
supported).
"""
filename = str(tmpdir.join("a_filename.geojson"))
create_sample_data(filename, "GeoJSON")
with pytest.raises((RuntimeError, OSError)):
fiona.remove(filename, layer=0)
assert os.path.exists(filename)
def test_remove_layer_geojson(tmpdir):
"""Removal of layers is not supported by GeoJSON driver
The reason for failure is slightly different between GDAL 2.2+ and < 2.2.
With < 2.2 the datasource will fail to open in write mode (IOError), while
with 2.2+ the datasource will open but the removal operation will fail (not
supported).
"""
filename = str(tmpdir.join("a_filename.geojson"))
create_sample_data(filename, "GeoJSON")
with pytest.raises((RuntimeError, IOError)):
fiona.remove(filename, layer=0)
assert os.path.exists(filename)
def save_fiona_objects(self, table_name, objs, mode="w", crs=None):
if not mode in ["w", "a"]:
raise NotImplementedError("supported modes are ['w', 'a']")
if mode == "w":
if crs is None:
crs = self.get_crs(table_name)
fiona.remove(str(self._gpkg_path), layer=table_name)
if not self._table_exists(table_name):
self.create_table(table_name, crs)
schema = self.get_schema(table_name)
for obj in objs:
if "id" in obj["properties"]:
del obj["properties"]["id"]
### Don't use fiona's writerecords because it doesn't keep fid. ###
ds = gdal.OpenEx(str(self._gpkg_path), gdal.OF_UPDATE | gdal.OF_VECTOR)
layer = ds.GetLayerByName(table_name)
# Add id if not exist
for i, obj in enumerate(objs):
if not "id" in obj:
obj["id"] = i + 1
# Get max id and increment in append mode
if mode == "a":
gdf = self.get_all_features_as_gdf(table_name)
max_id = 0 if gdf.empty else max(gdf.id)
obj["id"] += max_id
for obj in objs:
feature = ogr.Feature(layer.GetLayerDefn())
feature.SetFID(obj["id"])
if obj["geometry"]:
geometry = ogr.CreateGeometryFromWkt(
shape(obj["geometry"]).wkt)
feature.SetGeometry(geometry)
for prop in schema["properties"]:
feature.SetField(prop, obj["properties"][prop])
ret = layer.CreateFeature(feature)
if ret != 0:
raise RuntimeError("failed to create feature")
self._reload_table(table_name)
def rm(ctx, input, layer, yes):
if layer is None:
kind = "datasource"
else:
kind = "layer"
if not yes:
click.confirm("The {} will be removed. Are you sure?".format(kind), abort=True)
try:
fiona.remove(input, layer=layer)
except Exception:
logger.exception("Failed to remove {}.".format(kind))
raise click.Abort()
def rm(ctx, input, layer, yes):
if layer is None:
kind = "datasource"
else:
kind = "layer"
if not yes:
click.confirm("The {} will be removed. Are you sure?".format(kind),
abort=True)
try:
fiona.remove(input, layer=layer)
except Exception:
logger.exception("Failed to remove {}.".format(kind))
raise click.Abort()
def test_remove(tmpdir, kind, driver, specify_driver):
"""Test various dataset removal operations"""
extension = {"ESRI Shapefile": "shp", "GeoJSON": "json"}[driver]
filename = "delete_me.{extension}".format(extension=extension)
output_filename = str(tmpdir.join(filename))
create_sample_data(output_filename, driver=driver)
if kind == "collection":
to_delete = fiona.open(output_filename, "r")
else:
to_delete = output_filename
assert os.path.exists(output_filename)
if specify_driver:
fiona.remove(to_delete, driver=driver)
else:
fiona.remove(to_delete)
assert not os.path.exists(output_filename)
def test_remove(tmpdir, kind, driver, specify_driver):
"""Test various dataset removal operations"""
extension = {"ESRI Shapefile": "shp", "GeoJSON": "json"}[driver]
filename = "delete_me.{extension}".format(extension=extension)
output_filename = str(tmpdir.join(filename))
create_sample_data(output_filename, driver=driver)
if kind == "collection":
to_delete = fiona.open(output_filename, "r")
else:
to_delete = output_filename
assert os.path.exists(output_filename)
if specify_driver:
fiona.remove(to_delete, driver=driver)
else:
fiona.remove(to_delete)
assert not os.path.exists(output_filename)
def __init__(self, out_path=None, crs=None, fieldname=None, driver=None):
self._append = "a" in fiona.supported_drivers[driver]
logger.debug("initialize %s writer with append %s", driver,
self._append)
self.path = out_path
self.driver = driver
self.fieldname = fieldname
self.new_entries = 0
schema = deepcopy(spatial_schema)
schema["properties"][fieldname] = "str:254"
if self._append:
if os.path.isfile(self.path):
logger.debug("read existing entries")
with fiona.open(self.path, "r") as src:
self._existing = {
f["properties"]["tile_id"]: f
for f in src
}
self.file_obj = fiona.open(self.path, "a")
else:
self.file_obj = fiona.open(self.path,
"w",
driver=self.driver,
crs=crs,
schema=schema)
self._existing = {}
else:
if os.path.isfile(self.path):
logger.debug("read existing entries")
with fiona.open(self.path, "r") as src:
self._existing = {
f["properties"]["tile_id"]: f
for f in src
}
fiona.remove(self.path, driver=driver)
else:
self._existing = {}
self.file_obj = fiona.open(self.path,
"w",
driver=self.driver,
crs=crs,
schema=schema)
self.file_obj.writerecords(self._existing.values())
def init_handler(self, df=None):
import fiona
schema = self._get_schema(df)
layername = None
if '.gpkg:' in self.target:
try:
self.filename, layername = self.target.split(':')
except ValueError as e:
raise argh.CommandError('File name should be name.gpkg or name.gpkg:layer_name. Got "%s" instead.' % self.target)
else:
self.filename = self.target
layername = os.path.splitext(os.path.basename(self.target))[0]
# instead of self._cleanup_target(), delete fiona layer
if os.path.exists(self.filename) and layername in fiona.listlayers(self.filename):
fiona.remove(self.filename, self.fiona_driver, layername)
crs = df.crs if df is not None else None
self._handler = fiona.open(self.filename, 'w', crs=crs, driver=self.fiona_driver, schema=schema, layer=layername)
def __init__(self, out_path=None, crs=None, fieldname=None, driver=None):
logger.debug("initialize %s writer", driver)
self.path = out_path
self.driver = driver
if os.path.isfile(self.path):
with fiona.open(self.path) as src:
self.existing = {f["properties"]["tile_id"]: f for f in src}
fiona.remove(self.path, driver=driver)
else:
self.existing = {}
self.new_entries = 0
self.fieldname = fieldname
schema = deepcopy(spatial_schema)
schema["properties"][fieldname] = "str:254"
self.file_obj = fiona.open(self.path,
"w",
driver=self.driver,
crs=crs,
schema=schema)
self.file_obj.writerecords(self.existing.values())
def createStreetSegment(inshp, outshp, mini_dist):
'''
This function will segment streets into different street segments
Required modules: Fiona and Shapely
parameters:
inshp: the input linear shapefile, must be in WGS84 projection, ESPG: 4326
output: the result street segment feature class
mini_dist: the minimum distance between two created point
First version May 3rd, 2018
last modified by Xiaojiang Li, MIT Senseable City Lab
'''
import fiona
import os, os.path
from shapely.geometry import shape, mapping, Point, LineString
from shapely.ops import transform
from functools import partial
import pyproj
from fiona.crs import from_epsg
count = 0
s = {
'trunk_link', 'tertiary', 'motorway', 'motorway_link', 'steps', None,
' ', 'pedestrian', 'primary', 'primary_link', 'footway',
'tertiary_link', 'trunk', 'secondary', 'secondary_link',
'tertiary_link', 'bridleway', 'service'
}
# the temporaray file of the cleaned data
root = os.path.dirname(inshp)
basename = 'clean_' + os.path.basename(inshp)
temp_cleanedStreetmap = os.path.join(root, basename)
# if the tempfile exist then delete it
if os.path.exists(temp_cleanedStreetmap):
fiona.remove(temp_cleanedStreetmap, 'ESRI Shapefile')
# clean the original street maps by removing highways, if it the street map not from Open street data, users'd better to clean the data themselve
with fiona.open(inshp) as source, fiona.open(temp_cleanedStreetmap,
'w',
driver=source.driver,
crs=source.crs,
schema=source.schema) as dest:
for feat in source:
try:
i = feat['properties']['highway'] # for the OSM street data
if i in s:
continue
except:
# if the street map is not osm, do nothing. You'd better to clean the street map, if you don't want to map the GVI for highways
key = dest.schema['properties'].keys(
)[0] # get the field of the input shapefile and duplicate the input feature
i = feat['properties'][key]
if i in s:
continue
dest.write(feat)
schema = {
'geometry': 'LineString',
'properties': {
'id': 'int'
},
}
# Create street segment along the streets
with fiona.drivers():
#with fiona.open(outshp, 'w', 'ESRI Shapefile', crs=source.crs, schema) as output:
with fiona.open(outshp,
'w',
crs=from_epsg(4326),
driver='ESRI Shapefile',
schema=schema) as output:
for line in fiona.open(temp_cleanedStreetmap):
# define projection and the inverse projection
project = partial(pyproj.transform,
pyproj.Proj(init='EPSG:4326'),
pyproj.Proj(init='EPSG:3857')
) #3857 is psudo WGS84 the unit is meter
project2 = partial(pyproj.transform,
pyproj.Proj(init='EPSG:3857'),
pyproj.Proj(init='EPSG:4326'))
# deal with MultiLineString and LineString
featureType = line['geometry']['type']
dist = mini_dist
# for the LineString
if featureType == "LineString":
first = shape(line['geometry'])
length = first.length
# convert the projection of wgs84 from degree to meters
line2 = transform(project, first)
# cut the line feature using the distance of dist
st_seg = cutLine_series_pnt(line2, dist)
# loop all the street segments and then save them as a new shapefile
for seg in st_seg:
# do inverse projection, and the new feature has WGS84
line3 = transform(project2, seg)
output.write({
'geometry': mapping(line3),
'properties': {
'id': 1
}
})
else:
continue
print("Process Complete")
# delete the temprary cleaned shapefile
fiona.remove(temp_cleanedStreetmap, 'ESRI Shapefile')
def write(df, fname):
"""
Writes [Geo]DataFrame to file or database. Format (driver) is detected by the fname parameter.
'*.csv' => csv (geometry is transformed to WKT)
'*.geojson' => GeoJSON
'*.gpkg[:<layer_name>]' => GeoPackage (GPKG)
'postgresql://' => Postgresql table (postgresql://[user[:password]@]hostname[:port]/<db_name>#<table_name or query>)
"""
from . import utils
if isinstance(df, gpd.GeoDataFrame) and len(df) > 0:
df['geometry'] = utils.fix_multitypes(df['geometry'])
match_gpkg = re.match(r'^(?P<filename>.*/(?P<file_own_name>.*)\.(?P<extension>gpkg))(?:\:(?P<layer_name>[a-z0-9_]+))?$', fname)
match_geojson = re.match(r'^(?P<filename>.*/(?P<file_own_name>.*)\.(?P<extension>gpkg))$', fname)
match_postgres = re.match(r'^postgresql\://', fname)
match_csv = fname.endswith('.csv')
if match_postgres:
engine, table_name = _connect_postgres(fname)
df = df[list(df)]
with engine.begin() as connection:
if 'geometry' in df:
df['geometry'] = df['geometry'].apply(wkb.dumps).apply(bytes.hex)
pd.io.sql.execute('DROP TABLE IF EXISTS %s' % table_name, connection)
df.to_sql(table_name, connection, chunksize=1000)
if 'geometry' in df:
if not df.crs and -181 < df['geometry'].extents[0] < 181:
crs_num = 4326
elif not df.crs:
crs_num = 3857
else:
crs_num = re.match(r'.*\:(\d+)', df.crs['init'])[1]
pd.io.sql.execute("""
ALTER TABLE %s
ALTER COLUMN "geometry" TYPE Geometry""" % table_name, connection)
pd.io.sql.execute("""
UPDATE %s SET "geometry"=st_setsrid(geometry, %s)
""" % (table_name, crs_num), connection)
elif isinstance(df, gpd.GeoDataFrame):
if match_csv:
if os.path.exists(fname):
os.unlink(fname)
df = pd.DataFrame(df)
df.to_csv(fname, index=False)
elif match_gpkg:
filename = match_gpkg['filename']
layer_name = match_gpkg['layer_name'] or match_gpkg['file_own_name']
if os.path.exists(filename):
if layer_name in listlayers(filename):
remove(filename, 'GPKG', layer_name)
df.to_file(filename, driver='GPKG', encoding='utf-8', layer=layer_name)
elif match_geojson:
if os.path.exists(filename):
os.unlink(filename)
df.to_file(filename, driver='GeoJSON', encoding='utf-8')
elif isinstance(df, pd.DataFrame):
if fname.endswith('.json'):
df.to_json(fname)
else:
df.to_csv(fname, index=False)
def test_remove_nonexistent(tmpdir):
"""Attempting to remove a file that does not exist results in an IOError"""
filename = str(tmpdir.join("does_not_exist.shp"))
assert not os.path.exists(filename)
with pytest.raises(IOError):
fiona.remove(filename)
def createPoints(inshp, outshp, mini_dist):
'''
This function will parse throigh the street network of provided city and
clean all highways and create points every mini_dist meters (or as specified) along
the linestrings
Required modules: Fiona and Shapely
parameters:
inshp: the input linear shapefile, must be in WGS84 projection, ESPG: 4326
output: the result point feature class
mini_dist: the minimum distance between two created point
modified by May 2ed, 2018, consider the linestring and multi-linear string
last modified by Xiaojiang Li, MIT Senseable City Lab
'''
import fiona
import os, os.path
from shapely.geometry import shape, mapping
from shapely.ops import transform
from functools import partial
import pyproj
from fiona.crs import from_epsg
import sys
count = 0
# s = {'trunk_link','tertiary','motorway','motorway_link','steps', None, ' ','pedestrian','primary', 'primary_link','footway','tertiary_link', 'trunk','secondary','secondary_link','tertiary_link','bridleway','service'}
# s = {'trunk_link','tertiary','motorway','motorway_link','steps', ' ','pedestrian','primary', 'primary_link','footway','tertiary_link', 'trunk','secondary','secondary_link','tertiary_link','bridleway','service'}
s = {}
# the temporaray file of the cleaned data
root = os.path.dirname(inshp)
basename = 'clean_' + os.path.basename(inshp)
temp_cleanedStreetmap = os.path.join(root, basename)
# if the tempfile exist then delete it
if os.path.exists(temp_cleanedStreetmap):
fiona.remove(temp_cleanedStreetmap, 'ESRI Shapefile')
print('removed the existed tempfile')
# clean the original street maps by removing highways, if it the street map not from Open street data, users'd better to clean the data themselve
with fiona.open(inshp) as source, fiona.open(temp_cleanedStreetmap,
'w',
driver=source.driver,
crs=source.crs,
schema=source.schema) as dest:
for feat in source:
try:
i = feat['properties']['highway'] # for the OSM street data
# i = feat['properties']['fclass'] # for the OSM tokyo street data
if i in s:
continue
except:
# if the street map is not osm, do nothing. You'd better to clean the street map, if you don't want to map the GVI for highways
key = list(dest.schema['properties'].keys())[0]
# key = dest.schema['properties'].keys()[0] # get the field of the input shapefile and duplicate the input feature
i = feat['properties'][key]
if i in s:
continue
# print feat
dest.write(feat)
schema = {
'geometry': 'Point',
'properties': {
'id': 'int'
},
}
# Create point along the streets
with fiona.Env():
with fiona.open(outshp,
'w',
crs=from_epsg(4326),
driver='ESRI Shapefile',
schema=schema) as output:
for line in fiona.open(temp_cleanedStreetmap):
line_geom = line['geometry']
featureType = line_geom['type']
try:
if featureType == 'LineString':
line_geom_degree = shape(line_geom)
# convert degree to meter, in order to split by distance in meter
project = partial(
pyproj.transform, pyproj.Proj(init='EPSG:4326'),
pyproj.Proj(init='EPSG:3857')
) #3857 is psudo WGS84 the unit is meter
line_geom_meter = transform(project, line_geom_degree)
for distance in range(0, int(line_geom_meter.length),
mini_dist):
point = line_geom_meter.interpolate(distance)
# convert the local projection back the the WGS84 and write to the output shp
project2 = partial(pyproj.transform,
pyproj.Proj(init='EPSG:3857'),
pyproj.Proj(init='EPSG:4326'))
point_deg = transform(project2, point)
output.write({
'geometry': mapping(point_deg),
'properties': {
'id': 1
}
})
# for the MultiLineString, seperate these lines, then partition those lines
elif featureType == "MultiLineString":
multiline_geom = shape(line['geometry'])
print('This is a multiline')
for singleLine in multiline_geom:
length = singleLine.length
# partion each single line in the multiline
project = partial(
pyproj.transform,
pyproj.Proj(init='EPSG:4326'),
pyproj.Proj(init='EPSG:3857')
) #3857 is psudo WGS84 the unit is meter
line2 = transform(project, singleLine)
linestr = list(line2.coords)
dist = mini_dist #set
for distance in range(0, int(line2.length), dist):
point = line2.interpolate(distance)
project2 = partial(
pyproj.transform,
pyproj.Proj(init='EPSG:3857'),
pyproj.Proj(init='EPSG:4326'))
point = transform(project2, point)
output.write({
'geometry': mapping(point),
'properties': {
'id': 1
}
})
else:
print('Else--------')
continue
except:
print("You should make sure the input shapefile is WGS84")
print(sys.exc_info())
return
print("Process Complete")
# delete the temprary cleaned shapefile
fiona.remove(temp_cleanedStreetmap, 'ESRI Shapefile')
def create_points(inshp, outshp, mini_dist):
"""
This function will parse through the street network of provided city and
clean all highways and create points every mini_dist meters (or as specified) along
the linestrings
Required modules: Fiona and Shapely
parameters:
inshp: the input linear shapefile, must be in WGS84 projection, ESPG: 4326
output: the result point feature class
mini_dist: the minimum distance between two created point
last modified by Xiaojiang Li, MIT Senseable City Lab
"""
import fiona
import os.path
from shapely.geometry import shape, mapping
from shapely.ops import transform
from pyproj import Transformer
from fiona.crs import from_epsg
s = {
'trunk_link', 'tertiary', 'motorway', 'motorway_link', 'steps', None,
' ', 'pedestrian', 'primary', 'primary_link', 'footway',
'tertiary_link', 'trunk', 'secondary', 'secondary_link',
'tertiary_link', 'bridleway', 'service'
}
# the temporary file of the cleaned data
root_dir = os.path.dirname(inshp)
basename = 'clean_' + os.path.basename(inshp)
temp_cleaned_streetmap = os.path.join(root_dir, basename)
# if the tempfile exist then delete it
if os.path.exists(temp_cleaned_streetmap):
fiona.remove(temp_cleaned_streetmap, 'ESRI Shapefile')
# clean the original street maps by removing highways, if it the street map not from Open street data, users'd
# better to clean the data themselves
with fiona.open(inshp) as source, fiona.open(temp_cleaned_streetmap,
'w',
driver=source.driver,
crs=source.crs,
schema=source.schema) as dest:
for feat in source:
try:
i = feat['properties']['highway'] # for the OSM street data
if i in s:
continue
except:
# if the street map is not osm, do nothing. You'd better to clean the street map, if you don't want
# to map the GVI for highways
# get the field of the input shapefile and duplicate the input feature
key = list(dest.schema['properties'].keys())[0]
i = feat['properties'][key]
if i in s:
continue
dest.write(feat)
schema = {
'geometry': 'Point',
'properties': {
'id': 'int'
},
}
# Create points along the streets
with fiona.Env():
# with fiona.open(outshp, 'w', 'ESRI Shapefile', crs=source.crs, schema) as output:
with fiona.open(outshp,
'w',
crs=from_epsg(4326),
driver='ESRI Shapefile',
schema=schema) as output:
transformation1 = Transformer.from_crs('EPSG:4326',
'EPSG:3857',
always_xy=True).transform
transformation2 = Transformer.from_crs('EPSG:3857',
'EPSG:4326',
always_xy=True).transform
for line in fiona.open(temp_cleaned_streetmap):
first = shape(line['geometry'])
try:
# convert degree to meter, in order to split by distance in meter
# EPSG:3857 is pseudo WGS84 the unit is meter
line2 = transform(transformation1, first)
# print(line2.coords)
# linestr = list(line2.coords)
dist = mini_dist # set
for distance in range(0, int(line2.length), dist):
point = line2.interpolate(distance)
# convert the local projection back to the WGS84 and write to the output shp
point = transform(transformation2, point)
output.write({
'geometry': mapping(point),
'properties': {
'id': 1
}
})
except:
print("You should make sure the input shapefile is WGS84")
raise
print("Process Complete")
# delete the temporary cleaned shapefile
fiona.remove(temp_cleaned_streetmap, 'ESRI Shapefile')
def test_remove_layer_shapefile(tmpdir):
"""Removal of layer in shapefile actually deletes the datasource"""
filename = str(tmpdir.join("a_filename.shp"))
create_sample_data(filename, "ESRI Shapefile")
fiona.remove(filename, layer=0)
assert not os.path.exists(filename)
def test_remove_layer_shapefile(tmpdir):
"""Removal of layer in shapefile actually deletes the datasource"""
filename = str(tmpdir.join("a_filename.shp"))
create_sample_data(filename, "ESRI Shapefile")
fiona.remove(filename, layer=0)
assert not os.path.exists(filename)
def mask_to_shp(src, out_shp, mask_value=0, driver='ESRI Shapefile'):
""" 将掩膜数据矢量化（输出为shapefile格式）.
Note:
* 输出shapefile字段及属性写死在代码；
* 最好只用于掩膜值为特定值的数据，如果将每一个像素矢量化会非常慢;
Args:
src (rasterio dataset): 输入的包含掩膜值的栅格数据集.
out_shp (str): 输出shapefile文件.
mask_value (float): 掩膜值，默认0.
driver (str): 输出矢量类型, 默认 'ESRI Shapefile'.
"""
if src.count != 1:
return
if os.path.isfile(out_shp):
fiona.remove(out_shp, driver=driver)
import mask
# 分窗口读取，用于占用内存比较大数组
windows = mask.window_list(10000, src.width, src.height)
# 直接用于PS处理后每一层的蒙版导出结果
for idx_window, window in enumerate(windows):
print("process {} of total {}\n".format(idx_window + 1, len(windows)))
# read data of current window
mask_data = src.read(1, window=window)
# transfrom of current window
transform = rasterio.windows.transform(window, src.transform)
# get features of the same value from an array
none_zero = features.shapes(mask_data,
mask=mask_data == mask_value,
connectivity=4,
transform=transform)
mask_data = None
# get record
results = ({
'properties': {
'date': '20181007'
},
'geometry': s
} for i, (s, v) in enumerate(none_zero))
# shapefile schema
source_schema = {
'geometry': 'Polygon',
'properties': {
'date': 'str:20'
}
}
# write to shapefile using fiona
if results is not None:
for r in results:
# TODO merge result features by order
try:
with fiona.open(out_shp,
'a',
driver=driver,
crs=from_epsg(src.crs.to_epsg()),
schema=source_schema) as c:
c.write(r)
except OSError:
with fiona.open(out_shp,
'w',
driver=driver,
crs=from_epsg(src.crs.to_epsg()),
schema=source_schema) as c:
c.write(r)
def test_remove_nonexistent(tmpdir):
"""Attempting to remove a file that does not exist results in an OSError"""
filename = str(tmpdir.join("does_not_exist.shp"))
assert not os.path.exists(filename)
with pytest.raises(OSError):
fiona.remove(filename)
def createPoints(inshp, outshp, mini_dist):
'''
This function will parse throigh the street network of provided city and
clean all highways and create points every mini_dist meters (or as specified) along
the linestrings
Required modules: Fiona and Shapely
parameters:
inshp: the input linear shapefile, must be in WGS84 projection, ESPG: 4326
output: the result point feature class
mini_dist: the minimum distance between two created point
'''
import fiona
import os, os.path
from shapely.geometry import shape, mapping
from shapely.ops import transform
from functools import partial
import pyproj
from fiona.crs import from_epsg
count = 0
#name the road types NOT to be used.
s = {
'trunk_link', 'motorway', 'motorway_link', 'steps', None, 'pedestrian',
'trunk', 'bridleway', 'service'
}
# the temporaray file of the cleaned data
root = os.path.dirname(inshp)
basename = 'clean_' + os.path.basename(inshp)
temp_cleanedStreetmap = os.path.join(root, basename)
# if the tempfile exist then delete it
if os.path.exists(temp_cleanedStreetmap):
fiona.remove(temp_cleanedStreetmap, 'ESRI Shapefile')
# clean the original street maps by removing highways, if it the street map not from Open street data, users'd better to clean the data themselve
with fiona.open(inshp) as source, fiona.open(temp_cleanedStreetmap,
'w',
driver=source.driver,
crs=source.crs,
schema=source.schema) as dest:
for feat in source:
try:
i = feat['properties']['fclass'] # for the OSM street data
if i in s:
continue
except:
# if the street map is not osm, do nothing. You'd better to clean the street map, if you don't want to map the GVI for highways
# key = dest.schema['properties'].keys()[0] # get the field of the input shapefile and duplicate the input feature
# i = feat['properties'][key]
# if i in s:
continue
dest.write(feat)
schema = {
'geometry': 'Point',
'properties': {
'id': 'int'
},
}
# Create pointS along the streets
with fiona.drivers():
#with fiona.open(outshp, 'w', 'ESRI Shapefile', crs=source.crs, schema) as output:
with fiona.open(outshp,
'w',
crs=from_epsg(4326),
driver='ESRI Shapefile',
schema=schema) as output:
for line in fiona.open(temp_cleanedStreetmap):
first = shape(line['geometry'])
if first.geom_type != 'LineString': continue
length = first.length
try:
# convert degree to meter, in order to split by distance in meter
project = partial(pyproj.transform,
pyproj.Proj(init='EPSG:4326'),
pyproj.Proj(init='EPSG:3067')
) #3857 is psudo WGS84 the unit is meter
line2 = transform(project, first)
linestr = list(line2.coords)
dist = mini_dist #set
for distance in range(0, int(line2.length), dist):
point = line2.interpolate(distance)
# convert the local projection back the the WGS84 and write to the output shp
project2 = partial(pyproj.transform,
pyproj.Proj(init='EPSG:3067'),
pyproj.Proj(init='EPSG:4326'))
point = transform(project2, point)
output.write({
'geometry': mapping(point),
'properties': {
'id': 1
}
})
except Exception as e:
print(e)
print("You should make sure the input shapefile is WGS84")
return
print("Process Complete")
def createPoints(inshp, outshp, mini_dist):
'''
This function will parse throigh the street network of provided city and
clean all highways and create points every mini_dist meters (or as specified) along
the linestrings
Required modules: Fiona and Shapely
parameters:
inshp: the input linear shapefile, must be in WGS84 projection, ESPG: 4326
output: the result point feature class
mini_dist: the minimum distance between two created point
last modified by Xiaojiang Li, MIT Senseable City Lab
'''
import fiona
import os, os.path
from shapely.geometry import shape, mapping
from shapely.ops import transform
from functools import partial
import pyproj
from fiona.crs import from_epsg
count = 0
s = {
'trunk_link', 'tertiary', 'motorway', 'motorway_link', 'steps', None,
' ', 'pedestrian', 'primary', 'primary_link', 'footway',
'tertiary_link', 'trunk', 'secondary', 'secondary_link',
'tertiary_link', 'bridleway', 'service'
}
# the temporaray file of the cleaned data
root = os.path.dirname(inshp)
basename = 'clean_' + os.path.basename(inshp)
temp_cleanedStreetmap = os.path.join(root, basename)
# if the tempfile exist then delete it
if os.path.exists(temp_cleanedStreetmap):
fiona.remove(temp_cleanedStreetmap, 'ESRI Shapefile')
#driver = ogr.GetDriverByName("ESRI Shapefile")
#driver.DeleteDataSource(temp_cleanedStreetmap)
# clean the original street maps
with fiona.open(inshp) as source, fiona.open(temp_cleanedStreetmap,
'w',
driver=source.driver,
crs=source.crs,
schema=source.schema) as dest:
for feat in source:
try:
i = feat['properties']['highway'] # for the OSM street data
if i in s:
continue
except:
i = feat['properties'][
'Restrictio'] # for cambridge center line data
if i in s:
continue
dest.write(feat)
schema = {
'geometry': 'Point',
'properties': {
'id': 'int'
},
}
with fiona.drivers():
#with fiona.open(outshp, 'w', 'ESRI Shapefile', crs=source.crs, schema) as output:
with fiona.open(outshp,
'w',
crs=from_epsg(4326),
driver='ESRI Shapefile',
schema=schema) as output:
for line in fiona.open(temp_cleanedStreetmap):
first = shape(line['geometry'])
length = first.length
# convert degree to meter, in order to split by distance in meter
project = partial(pyproj.transform,
pyproj.Proj(init='EPSG:4326'),
pyproj.Proj(init='EPSG:3857')
) #3857 is psudo WGS84 the unit is meter
line2 = transform(project, first)
linestr = list(line2.coords)
dist = mini_dist #set
for distance in range(0, int(line2.length), 20):
point = line2.interpolate(distance)
# convert the local projection back the the WGS84 and write to the output shp
project2 = partial(pyproj.transform,
pyproj.Proj(init='EPSG:3857'),
pyproj.Proj(init='EPSG:4326'))
point = transform(project2, point)
output.write({
'geometry': mapping(point),
'properties': {
'id': 1
}
})
print("Process Complete")
# delete the temprary cleaned shapefile
fiona.remove(temp_cleanedStreetmap, 'ESRI Shapefile')
