def process_dataset(self, workdir, db, url, extractor):
'''
Extract territories from a given file for a given level
with a given extractor function.
'''
loaded = 0
filename = join(workdir, basename(url))
layer = extractor.layer
info('processing {0}', basename(filename))
if filename.endswith('.zip'):
loader = self.load_shp_zip
elif filename.endswith('.geojson'):
loader = self.load_geojzon
with loader(filename, layer=layer) as collection:
if layer:
info('Extracting {0} elements from {1} => {2} ({3} {4})',
len(collection), basename(filename), layer, collection.driver,
to_string(collection.crs))
else:
info('Extracting {0} elements from {1} => ({2} {3})',
len(collection), basename(filename), collection.driver,
to_string(collection.crs))
for polygon in collection:
try:
zone = extractor(db, polygon)
if not zone:
continue
zone['keys'] = dict(
(k, v)
for k, v in zone.get('keys', {}).items()
if v is not None)
geom = shape(polygon['geometry'])
if extractor.simplify:
geom = geom.simplify(extractor.simplify)
if geom.geom_type == 'Polygon':
geom = MultiPolygon([geom])
elif geom.geom_type != 'MultiPolygon':
warning('Unsupported geometry type "{0}" for "{1}"',
geom.geom_type, zone['name'])
continue
zone.update(geom=geom.__geo_interface__)
zone_id = zone.get('_id')
if not zone_id:
zone_id = ':'.join((self.id, zone['code']))
zone.update(_id=zone_id, level=self.id)
db.find_one_and_replace(
{'_id': zone_id}, zone, upsert=True)
loaded += 1
except Exception:
error('Error extracting polygon {0}',
polygon['properties'])
error(traceback.format_exc())
info('Loaded {0} zones for level {1} from file {2}',
loaded, self.id, filename)
return loaded
def _read_extent_shapefile(self, shpfile, buffer=0):
import fiona
from fiona.crs import to_string, from_epsg
print('reading extent from {}...'.format(shpfile))
shp = fiona.open(shpfile)
g = shape(shp.next()['geometry'])
if to_string(from_epsg(coord_datums_epsg[self.datum])) != to_string(shp.crs):
print('reprojecting extent from {} to {}'.format(to_string(shp.crs), self.proj4))
return project(g, to_string(shp.crs), self.proj4)
else:
return g
def _read_extent_shapefile(self, shpfile, buffer=0):
import fiona
from fiona.crs import to_string, from_epsg
print('reading extent from {}...'.format(shpfile))
shp = fiona.open(shpfile)
g = shape(shp.next()['geometry'])
if to_string(from_epsg(coord_datums_epsg[self.datum])) != to_string(shp.crs):
print('reprojecting extent from {} to {}'.format(to_string(shp.crs), self.proj4))
return project(g, to_string(shp.crs), self.proj4)
else:
return g
def test_to_string_zeroval():
# Make a string with some 0 values (e.g. esri:102017)
val = {'proj': 'laea', 'lat_0': 90, 'lon_0': 0, 'x_0': 0, 'y_0': 0,
'ellps': 'WGS84', 'datum': 'WGS84', 'units': 'm', 'no_defs': True}
assert crs.to_string(val) == (
"+datum=WGS84 +ellps=WGS84 +lat_0=90 +lon_0=0 +no_defs +proj=laea "
"+units=m +x_0=0 +y_0=0")
def get_mbb(layerDict,crs):
for name in layerDict:
if 'logic' in layerDict[name] and layerDict[name]['logic'] == 'boundary':
with fiona.open(layerDict[name]['file']) as s:
x1, y1, x2, y2 = s.bounds
if to_string(crs) != to_string(s.crs):
proj1=pyproj.Proj(s.crs,preserve_units=True)
proj2=pyproj.Proj(crs,preserve_units=True)
minx, miny = pyproj.transform(proj1,proj2,x1,y1)
maxx, maxy = pyproj.transform(proj1,proj2,x2,y2)
return (minx,miny,maxx,maxy)
else:
return (x1,y1,x2,y2)
else:
continue
raise NameError("No Layer of Type Boundary was provided")
def test_to_string():
# Make a string from a mapping with a few bogus items
val = {
'proj': 'longlat', 'ellps': 'WGS84', 'datum': 'WGS84',
'no_defs': True, 'foo': True, 'axis': False, 'belgium': [1,2] }
assert crs.to_string(
val) == "+datum=WGS84 +ellps=WGS84 +no_defs +proj=longlat"
def test_to_string_zeroval():
# Make a string with some 0 values (e.g. esri:102017)
val = {'proj': 'laea', 'lat_0': 90, 'lon_0': 0, 'x_0': 0, 'y_0': 0,
'ellps': 'WGS84', 'datum': 'WGS84', 'units': 'm', 'no_defs': True}
assert crs.to_string(val) == (
"+datum=WGS84 +ellps=WGS84 +lat_0=90 +lon_0=0 +no_defs +proj=laea "
"+units=m +x_0=0 +y_0=0")
def test_to_string_utm():
# Make a string from a mapping with a few bogus items
val = {
'proj': 'utm', 'ellps': 'WGS84', 'zone': 13,
'no_defs': True, 'foo': True, 'axis': False, 'belgium': [1, 2]}
assert crs.to_string(
val) == "+ellps=WGS84 +no_defs +proj=utm +zone=13"
def test_to_string():
# Make a string from a mapping with a few bogus items
val = {
'proj': 'longlat', 'ellps': 'WGS84', 'datum': 'WGS84',
'no_defs': True, 'foo': True, 'axis': False, 'belgium': [1, 2]}
assert crs.to_string(
val) == "+datum=WGS84 +ellps=WGS84 +no_defs +proj=longlat"
def test_to_string_utm():
# Make a string from a mapping with a few bogus items
val = {
'proj': 'utm', 'ellps': 'WGS84', 'zone': 13,
'no_defs': True, 'foo': True, 'axis': False, 'belgium': [1,2] }
assert crs.to_string(
val) == "+ellps=WGS84 +no_defs +proj=utm +zone=13"
def add(request):
if request.method == 'POST':
form = LayerForm(request.POST, request.FILES)
if form.is_valid():
l = form.save()
col = form.get_collection()
srs = to_string(form.layer_crs())
shape_path = "%s/uploads/shapefile/%s/%s.shp" % (
settings.MEDIA_ROOT, request.tenant.schema_name, l.pk)
if not os.path.exists(os.path.dirname(shape_path)):
os.makedirs(os.path.dirname(shape_path))
with fiona.collection(shape_path,
"w",
schema=col.schema,
crs=col.crs,
driver="ESRI Shapefile") as out:
for f in col:
out.write(f)
django_rq.enqueue(process_shapefile, request.tenant.schema_name,
l.pk, srs)
messages.success(request, "Layer added.")
return redirect('layers:index')
else:
messages.error(request,
"The layer could not be saved due to errors.")
else:
form = LayerForm()
return render(request, 'layers/add.html', {'form': form})
def __init__(self,crs=None,prjs=None,epsg=None):
if crs is None:
if prjs is not None:
crs = from_string(prjs)
elif epsg is not None:
sr = SpatialReference()
sr.ImportFromEPSG(epsg)
crs = from_string(sr.ExportToProj4())
else:
raise(NotImplementedError)
else:
## remove unicode and change to python types
for k,v in crs.iteritems():
if type(v) == unicode:
crs[k] = str(v)
else:
try:
crs[k] = v.tolist()
except AttributeError:
continue
sr = SpatialReference()
sr.ImportFromProj4(to_string(crs))
self.value = from_string(sr.ExportToProj4())
try:
assert(self.value != {})
except AssertionError:
ocgis_lh(logger='crs',exc=ValueError('Empty CRS: The conversion to PROJ4 may have failed. The CRS value is: {0}'.format(crs)))
def process_dataset(self, workdir, db, url, extractor):
'''Extract territories from a given file for a given level with a given extractor function'''
loaded = 0
filename = join(workdir, basename(url))
with fiona.open('/', vfs='zip://{0}'.format(filename), encoding='utf8') as collection:
info('Extracting {0} elements from {1} ({2} {3})'.format(
len(collection), basename(filename), collection.driver, to_string(collection.crs)
))
for polygon in collection:
try:
zone = extractor(polygon)
if not zone:
continue
zone['keys'] = dict((k, v) for k, v in zone.get('keys', {}).items() if v is not None)
geom = shape(polygon['geometry'])
if extractor.simplify:
geom = geom.simplify(extractor.simplify)
if geom.geom_type == 'Polygon':
geom = MultiPolygon([geom])
elif geom.geom_type != 'MultiPolygon':
warning('Unsupported geometry type "{0}" for "{1}"'.format(geom.geom_type, zone['name']))
continue
zoneid = '/'.join((self.id, zone['code']))
zone.update(_id=zoneid, level=self.id, geom=geom.__geo_interface__)
db.find_one_and_replace({'_id': zoneid}, zone, upsert=True)
loaded += 1
except Exception as e:
error('Error extracting polygon {0}: {1}', polygon['properties'], str(e))
info('Loaded {0} zones for level {1} from file {2}'.format(loaded, self.id, filename))
return loaded
def get_bbox(feature, dest_crs):
"""Get bounding box for a Polygon feature.
Parameters
----------
feature : str (shapefile path), shapely Polygon or GeoJSON
dest_crs : proj str
Desired output coordinate system (shapefiles only)
"""
if isinstance(feature, str):
with fiona.open(feature) as src:
l, b, r, t = src.bounds
bbox_src_crs = box(*src.bounds)
shpcrs = crs(proj_str=to_string(src.crs))
if dest_crs is not None and shpcrs != dest_crs:
bbox_dest_crs = project(bbox_src_crs, shpcrs.proj_str,
dest_crs.proj_str)
l, b, r, t = bbox_dest_crs.bounds
# x, y = project([(l, b),
# (r, t)], shpcrs.proj_str, dest_crs.proj_str)
# filter = (x[0], y[0], x[1], y[1])
# else:
filter = (l, b, r, t)
elif isinstance(feature, Polygon):
filter = feature.bounds
elif isinstance(feature, dict):
try:
filter = shape(feature).bounds
except Exception as ex:
print(ex)
print("Supplied dictionary doesn't appear to be valid GeoJSON.")
return filter
def extract_shapefile(filename, level, extractor):
'''Extract territories from a given file for a given level with a given extractor function'''
imported = 0
with fiona.open('/', vfs='zip://{0}'.format(filename), encoding='utf8') as collection:
print 'Extracting {0} elements from {1} ({2} {3})'.format(
len(collection), basename(filename), collection.driver, to_string(collection.crs)
)
for polygon in collection:
try:
code, name, keys = extractor(polygon)
geom = shape(polygon['geometry'])
if geom.geom_type == 'Polygon':
geom = MultiPolygon([geom])
elif geom.geom_type != 'MultiPolygon':
print 'Unsupported geometry type "{0}" for "{1}"'.format(geom.geom_type, name)
continue
territory, _ = Territory.objects.get_or_create(level=level, code=code, defaults={
'geom': geom.__geo_interface__,
'name': name,
'keys': keys,
})
territory.geom = geom.__geo_interface__
territory.name = name
territory.keys = keys
territory.save()
imported += 1
except Exception as e:
print 'Error extracting polygon {0}: {1}'.format(polygon['properties'], e)
print 'Imported {0} territories for level {1} from file {2}'.format(imported, level, filename)
return imported
def handle(self, **options):
input_file = options['input_file']
year = options['year']
scheme = options['scheme']
field = options['field']
name = options['name']
# Create ValidClassification objects list
# Push it to database
# Read file and Optionally reproject the features to longlat
with fiona.open(input_file) as src:
p = Proj(src.crs)
if p.is_latlong(
): # Here we assume that geographic coordinates are automatically 4326 (not quite true)
fc = list(src)
else:
crs_str = to_string(src.crs)
fc = [
feature_transform(x,
crs_out='+proj=longlat',
crs_in=crs_str) for x in src
]
# Write features to ValidObject table
def valid_obj_builder(x):
"""Build individual ValidObjects
"""
geom = GEOSGeometry(json.dumps(x['geometry']))
obj = ValidObject(the_geom=geom)
return obj
obj_list = [valid_obj_builder(x) for x in fc]
ValidObject.objects.bulk_create(obj_list)
# Get list of unique tags
unique_numeric_codes = list(set([x['properties'][field] for x in fc]))
# Update Tag table using get or create
def make_tag_tuple(x):
obj, _ = Tag.objects.get_or_create(numeric_code=x, scheme=scheme)
return (x, obj)
tag_dict = dict([make_tag_tuple(x) for x in unique_numeric_codes])
# Build validClassification object list (valid_tag, valid_object, valid_set)
def valid_class_obj_builder(x):
"""x is a tuple (ValidObject, feature)"""
tag = tag_dict[x[1]['properties'][field]]
obj = ValidClassification(valid_tag=tag,
valid_object=x[0],
valid_set=name,
interpretation_year=year)
return obj
valid_class_obj_list = [
valid_class_obj_builder(x) for x in zip(obj_list, fc)
]
ValidClassification.objects.bulk_create(valid_class_obj_list)
def get_mbb(layerDict, crs):
for name in layerDict:
if 'logic' in layerDict[name] and layerDict[name][
'logic'] == 'boundary':
with fiona.open(layerDict[name]['file']) as s:
x1, y1, x2, y2 = s.bounds
if to_string(crs) != to_string(s.crs):
proj1 = pyproj.Proj(s.crs, preserve_units=True)
proj2 = pyproj.Proj(crs, preserve_units=True)
minx, miny = pyproj.transform(proj1, proj2, x1, y1)
maxx, maxy = pyproj.transform(proj1, proj2, x2, y2)
return (minx, miny, maxx, maxy)
else:
return (x1, y1, x2, y2)
else:
continue
raise NameError("No Layer of Type Boundary was provided")
def _set_proj4(self):
# establish a proj4 string for basemap projection regardless of input
if self.epsg is not None:
self.proj4 = to_string(from_epsg(self.epsg))
if self.projection_shapefile is not None:
self._read_shapefile_projection(self.projection_shapefile)
self.proj4 = get_proj4(self.projection_shapefile)
def test_to_string_unicode():
# See issue #83.
val = crs.to_string({
'units': 'm',
'no_defs': True,
'datum': 'NAD83',
'proj': 'utm',
'zone': 16})
assert 'NAD83' in val
def test_to_string_unicode():
# See issue #83.
val = crs.to_string({
u'units': u'm',
u'no_defs': True,
u'datum': u'NAD83',
u'proj': u'utm',
u'zone': 16})
assert 'NAD83' in val
def ras2vec(raster_file, output_path):
# Create a generic polygon schema for the output vector file
i = 0
feat_schema = {
'geometry': 'Polygon',
'properties': OrderedDict([('value', 'int')])
}
class_value_domain = set()
out_features = []
print(" - Processing raster file: {}".format(raster_file))
with rasterio.open(raster_file, 'r') as src:
raster = src.read(1)
mask = raster != 0
# Vectorize the polygons
polygons = features.shapes(raster, mask, transform=src.transform)
# Create shapely polygon featyres
for polygon in polygons:
feature = {
'geometry': {
'type': 'Polygon',
'coordinates': None
},
'properties': OrderedDict([('value', 0)])
}
feature['geometry']['coordinates'] = polygon[0]['coordinates']
value = int(polygon[1]) # Pixel value of the class (layer)
class_value_domain.add(value)
feature['properties']['value'] = value
i += 1
out_features.append(feature)
print(" - Writing output vector file: {}".format(output_path))
num_layers = list(class_value_domain) # Number of unique pixel value
for num_layer in num_layers:
polygons = [
feature for feature in out_features
if feature['properties']['value'] == num_layer
]
layer_name = 'vector_' + str(num_layer).rjust(3, '0')
print(" - Writing layer: {}".format(layer_name))
with fiona.open(output_path,
'w',
crs=to_string(src.crs),
layer=layer_name,
schema=feat_schema,
driver='GPKG') as dest:
for polygon in polygons:
dest.write(polygon)
print("")
print("Number of features written: {}".format(i))
def _read_shapefile_projection(self, shapefile):
"""sets the proj4 string from a shapefile"""
crs = fiona.open(shapefile).crs
init = crs.get('init', '')
if 'epsg' in init:
self.epsg = int(init.strip('epsg:'))
if self.epsg is not None:
self.proj4 = getproj4(self.epsg)
else:
self.proj4 = to_string(crs)
def _read_shapefile_projection(self, shapefile):
"""sets the proj4 string from a shapefile"""
crs = fiona.open(shapefile).crs
init = crs.get('init', '')
if 'epsg' in init:
self.epsg = int(init.strip('epsg:'))
if self.epsg is not None:
self.proj4 = getproj4(self.epsg)
else:
self.proj4 = to_string(crs)
def proj42wkt(file_path):
c = fiona.open(file_path)
"res/geo/subjects/42-loire/42-.dbf"
prj4 = to_string(c.crs)
sr = osr.SpatialReference()
sr.ImportFromProj4(prj4)
wkt = sr.ExportToWkt()
return wkt
def __init__(self,
value=None,
proj4=None,
epsg=None,
name=OcgisConvention.Name.COORDSYS):
self.name = name
# Allows operations on data variables to look through an empty dimension list. Alleviates instance checking.
self.dimensions = tuple()
self.dimension_names = tuple()
self.has_bounds = False
self._epsg = epsg
# Some basic overloaded for WGS84.
if epsg == 4326:
value = WGS84().value
# Add a special check for init keys in value dictionary.
if value is not None:
if 'init' in value and list(value.values())[0].startswith('epsg'):
epsg = int(list(value.values())[0].split(':')[1])
value = None
if value is None:
if proj4 is not None:
value = from_string(proj4)
elif epsg is not None:
sr = SpatialReference()
sr.ImportFromEPSG(epsg)
value = from_string(sr.ExportToProj4())
else:
msg = 'A value dictionary, PROJ.4 string, or EPSG code is required.'
raise ValueError(msg)
else:
# Remove unicode to avoid strange issues with proj and fiona.
for k, v in value.items():
if isinstance(v, six.string_types):
value[k] = str(v)
else:
try:
value[k] = v.tolist()
# this may be a numpy arr that needs conversion
except AttributeError:
continue
sr = SpatialReference()
sr.ImportFromProj4(to_string(value))
self.value = from_string(sr.ExportToProj4())
try:
assert self.value != {}
except AssertionError:
msg = 'Empty CRS: The conversion to PROJ.4 may have failed. The CRS value is: {0}'.format(
value)
raise ValueError(msg)
def test_to_string():
# Make a string from a mapping with a few bogus items
val = {
"proj": "longlat",
"ellps": "WGS84",
"datum": "WGS84",
"no_defs": True,
"foo": True,
"axis": False,
"belgium": [1, 2],
}
assert crs.to_string(val) == "+datum=WGS84 +ellps=WGS84 +no_defs +proj=longlat"
def process_dataset(self, workdir, db, url, extractor):
'''Extract territories from a given file for a given level with a given extractor function'''
loaded = 0
filename = join(workdir, basename(url))
# Identify the shapefile to avoid multiple file error on GDAL 2
with ZipFile(filename) as z:
candidates = [n for n in z.namelist() if n.endswith('.shp')]
if len(candidates) != 1:
raise ValueError(
'Unable to find a unique shpaefile into {0}'.format(
filename))
shp = candidates[0]
with fiona.open('/{0}'.format(shp),
vfs='zip://{0}'.format(filename),
encoding='utf8') as collection:
info('Extracting {0} elements from {1} ({2} {3})'.format(
len(collection), basename(filename), collection.driver,
to_string(collection.crs)))
for polygon in collection:
try:
zone = extractor(polygon)
if not zone:
continue
zone['keys'] = dict(
(k, v) for k, v in zone.get('keys', {}).items()
if v is not None)
geom = shape(polygon['geometry'])
if extractor.simplify:
geom = geom.simplify(extractor.simplify)
if geom.geom_type == 'Polygon':
geom = MultiPolygon([geom])
elif geom.geom_type != 'MultiPolygon':
warning(
'Unsupported geometry type "{0}" for "{1}"'.format(
geom.geom_type, zone['name']))
continue
zoneid = '/'.join((self.id, zone['code']))
zone.update(_id=zoneid,
level=self.id,
geom=geom.__geo_interface__)
db.find_one_and_replace({'_id': zoneid}, zone, upsert=True)
loaded += 1
except Exception as e:
error('Error extracting polygon {0}: {1}',
polygon['properties'], str(e))
info('Loaded {0} zones for level {1} from file {2}'.format(
loaded, self.id, filename))
return loaded
def __init__(self, value=None, proj4=None, epsg=None, name=OcgisConvention.Name.COORDSYS):
self.name = name
# Allows operations on data variables to look through an empty dimension list. Alleviates instance checking.
self.dimensions = tuple()
self.dimension_names = tuple()
self.has_bounds = False
self._epsg = epsg
# Some basic overloaded for WGS84.
if epsg == 4326:
value = WGS84().value
# Add a special check for init keys in value dictionary.
if value is not None:
if 'init' in value and list(value.values())[0].startswith('epsg'):
epsg = int(list(value.values())[0].split(':')[1])
value = None
if value is None:
if proj4 is not None:
value = from_string(proj4)
elif epsg is not None:
sr = SpatialReference()
sr.ImportFromEPSG(epsg)
value = from_string(sr.ExportToProj4())
else:
msg = 'A value dictionary, PROJ.4 string, or EPSG code is required.'
raise ValueError(msg)
else:
# Remove unicode to avoid strange issues with proj and fiona.
for k, v in value.items():
if isinstance(v, six.string_types):
value[k] = str(v)
else:
try:
value[k] = v.tolist()
# this may be a numpy arr that needs conversion
except AttributeError:
continue
sr = SpatialReference()
sr.ImportFromProj4(to_string(value))
self.value = from_string(sr.ExportToProj4())
try:
assert self.value != {}
except AssertionError:
msg = 'Empty CRS: The conversion to PROJ.4 may have failed. The CRS value is: {0}'.format(value)
raise ValueError(msg)
def get_authority(file_path):
'''
Get the authority used by a raster i.e. EPSG:4326
:param file_path: path to the file
:return: return the SRID of the raster projection
'''
with fiona.open(file_path) as c:
log.info(c)
# print c.crs
# print to_string(c.crs)
# print get_epsg_code_from_proj4(c.crs)
if 'init' in c.crs:
return c.crs['init']
elif 'proj' in c.crs:
return "EPSG:" + str(get_epsg_code_from_proj4(to_string(c.crs)))
return None
def crs_to_srs(crs):
"""
Recover our own WKT definition of projections from a pyproj / fiona CRS
Args:
crs (pyproj.CRS or dict): what is returned from GeoDataFrame().crs
Returns:
string: a "EPSG:xxx" or WKT representation of the crs.
"""
if crs is None:
return
elif isinstance(crs, dict): # geopandas < 0.7
proj4_str = fiona.crs.to_string(crs)
return get_epsg_or_wkt(proj4_str)
else: # geopandas >= 0.7
return crs.to_string()
def extract_shapefile(filename, level, extractor):
'''Extract territories from a given file for a given level with a given extractor function'''
imported = 0
with fiona.open('/', vfs='zip://{0}'.format(filename),
encoding='utf8') as collection:
print 'Extracting {0} elements from {1} ({2} {3})'.format(
len(collection), basename(filename), collection.driver,
to_string(collection.crs))
for polygon in collection:
try:
code, name, keys = extractor(polygon)
geom = shape(polygon['geometry'])
if geom.geom_type == 'Polygon':
geom = MultiPolygon([geom])
elif geom.geom_type != 'MultiPolygon':
print 'Unsupported geometry type "{0}" for "{1}"'.format(
geom.geom_type, name)
continue
territory, _ = Territory.objects.get_or_create(
level=level,
code=code,
defaults={
'geom': geom.__geo_interface__,
'name': name,
'keys': keys,
})
territory.geom = geom.__geo_interface__
territory.name = name
territory.keys = keys
territory.save()
imported += 1
except Exception as e:
print 'Error extracting polygon {0}: {1}'.format(
polygon['properties'], e)
print 'Imported {0} territories for level {1} from file {2}'.format(
imported, level, filename)
return imported
def __init__(self, value=None, proj4=None, epsg=None, name=constants.DEFAULT_COORDINATE_SYSTEM_NAME):
self.name = name
# Add a special check for init keys in value dictionary.
if value is not None:
if 'init' in value and value.values()[0].startswith('epsg'):
epsg = int(value.values()[0].split(':')[1])
value = None
if value is None:
if proj4 is not None:
value = from_string(proj4)
elif epsg is not None:
sr = SpatialReference()
sr.ImportFromEPSG(epsg)
value = from_string(sr.ExportToProj4())
else:
msg = 'A value dictionary, PROJ.4 string, or EPSG code is required.'
raise ValueError(msg)
else:
# Remove unicode to avoid strange issues with proj and fiona.
for k, v in value.iteritems():
if type(v) == unicode:
value[k] = str(v)
else:
try:
value[k] = v.tolist()
# this may be a numpy arr that needs conversion
except AttributeError:
continue
sr = SpatialReference()
sr.ImportFromProj4(to_string(value))
self.value = from_string(sr.ExportToProj4())
try:
assert self.value != {}
except AssertionError:
msg = 'Empty CRS: The conversion to PROJ.4 may have failed. The CRS value is: {0}'.format(value)
raise ValueError(msg)
def _load_from_fiona(self, f):
"""Load from fiona object. Return FeatureCollection.
:param f: fiona object
"""
a = {}
try:
a['crs'] = to_string(f.crs)
except:
pass
for k, v in self.attributes.items():
if util.is_callable(v):
a[k] = v(f)
else:
a[k] = v
a['features'] = []
for s in f:
s['properties'] = util.rec_apply(util.reinterpret_decode,
s['properties'],
lambda x: isinstance(x, unicode))
a['features'].append(Feature(s))
return FeatureCollection(a)
def create_land_use_rasters(land_use_folder, raster_output_folder, crs = None):
for ags_district in os.listdir(land_use_folder):
folder_abs = os.path.join(land_use_folder, ags_district)
if os.path.isdir(folder_abs):
#find siedlung shapefile name
#TODO: Take first shapefile in list
shapefile = [f for f in os.listdir(folder_abs) if f.endswith('.shp')][0]
#if there is a name filter, filter shapefiles.
#for each land use shapefile, tabulate intersections for each zone in that shapefile
full_sp_path = os.path.join(folder_abs, shapefile)
layer_name = os.path.splitext(shapefile)[0]
with fiona.open(full_sp_path, 'r') as vector_f:
assert crs or vector_f.crs, "a CRS must be specified either in in the shapefile or as an argument"
if not crs:
crs = vector_f.crs
(minx, miny, maxx, maxy) = map(lambda a:int(a - a % 50), vector_f.bounds)
cmd = ["gdal_rasterize",
"-a",
"lu_code",
"-a_srs",
to_string(crs),
"-te",
str(minx - 100), str(miny - 100), str(maxx+100), str(maxy+100),
"-tr",
"10.0", #10m resolution
"10.0",
"-ot", "Int16",
"-l",
layer_name,
full_sp_path,
os.path.join(raster_output_folder, ags_district + "_" + layer_name + '.tif')]
print(cmd)
subprocess.check_call(cmd)
def reproject_point(lat, lon, shp_path):
# get projections in proj4 string format
with fiona.open(shp_path + ".shp") as source:
tocrs_proj4 = to_string(source.crs)
fromcrs_proj4 = fiona.crs.to_string(from_epsg(4326))
# wgs84_prj = os.path.join(get_project_dir(), "shp", "4326.prj")
# fromcrs = pycrs.loader.from_file(wgs84_prj)
# fromcrs_proj4 = fromcrs.to_proj4()
# tocrs = pycrs.loader.from_file(shp_path + ".prj")
# tocrs_proj4 = tocrs.to_proj4()
point_wgs84 = Point(lat, lon)
project = partial(
pyproj.transform,
pyproj.Proj(fromcrs_proj4),
pyproj.Proj(tocrs_proj4))
return transform(project, point_wgs84)
def shapeify(self, weekday_rst_path):
rst = rasterio.open(weekday_rst_path)
array = rst.read()
array = array[0]
array = array.astype(int)
print('get_shapes_from_raster')
shape_list = self.get_shapes_from_raster(array, None, daymaps=False)
fname = weekday_rst_path.replace('.tiff', '.shp')
schema = {'properties': [('class', 'int')], 'geometry': 'Polygon'}
# crs, transform, area_shape = area2transform_baws300_sweref99tm()
crs, transform, area_shape = area2transform_baws1000_sweref99tm()
with fiona.open(fname,
'w',
driver='ESRI Shapefile',
crs=to_string(crs),
schema=schema) as dst:
dst.writerecords(shape_list)
print('shapeify completed!')
def bbox_copyraster(in_file, out_file, bbox, dst_crs=None):
"""Rather than re-invent rio clip, rio warp just call them directly
"""
with rasterio.open(in_file) as source:
bbox = transform_bbox(bbox, from_epsg(4326), out_crs=source.meta['crs'])
bbox = [str(b) for b in bbox]
if not dst_crs:
clip_file = out_file
else:
clip_file = os.path.join(tempfile.gettempdir(), "rio_temp.tif")
command = ['rio', 'clip', in_file, clip_file,
'--bounds', '"'+" ".join(bbox)+'"']
subprocess.call(" ".join(command), shell=True)
if dst_crs:
# convert crs to string and wrap in quotes
if type(dst_crs) == dict:
dst_crs = to_string(dst_crs)
dst_crs = '"'+dst_crs+'"'
command = ['rio', 'warp', clip_file, out_file,
'--dst-crs', dst_crs,
'--force-overwrite']
subprocess.call(" ".join(command), shell=True)
os.unlink(clip_file)
def importData(file, characterEncoding, format, user, folder):
cursor = connection.cursor()
start_time = time.time()
#manage zipfile
fd,fname = tempfile.mkstemp(suffix=fileExt_dic[format])
os.close(fd)
f = open(fname, "wb")
for chunk in file.chunks():
f.write(chunk)
f.close()
if not zipfile.is_zipfile(fname):
os.remove(fname)
return "Not a valid zip archive.", None
zip = zipfile.ZipFile(fname)
hasSuffix = {}
required_suffixes = suffixes_dic[format]
for suffix in required_suffixes:
hasSuffix[suffix] = False
for info in zip.infolist():
extension = os.path.splitext(info.filename)[1].lower()
if extension in required_suffixes:
hasSuffix[extension] = True
for suffix in required_suffixes:
if not hasSuffix[suffix]:
zip.close()
os.remove(fname)
return "Archive missing required %s file." % suffix, None
zip = zipfile.ZipFile(fname)
shapefileName = None
dirname = tempfile.mkdtemp()
for info in zip.infolist():
if info.filename.lower().endswith(filenameExt_dic[format]):
shapefileName = info.filename
dstFile = os.path.join(dirname, info.filename)
f = open(dstFile, "wb")
f.write(zip.read(info.filename))
f.close()
zip.close()
#verify if shapefile is valid
try:
srcPath = os.path.join(dirname,shapefileName)
srcLayers = fiona.listlayers(srcPath)
shapefileOK = True
except:
traceback.print_exc()
shapefileOK = False
if not shapefileOK:
os.remove(fname)
shutil.rmtree(dirname)
return "Not a valid vector file.", None
#add shapefile object to database
try:
for i in srcLayers:
with fiona.open(srcPath) as c:
srcSpatialRef = to_string(c.crs)
print srcSpatialRef
project = CoordTransform(SpatialReference(srcSpatialRef),SpatialReference(3857))
geometryType = c.schema['geometry']
shapefile = Shapefile.objects.create(filename=c.name, parent=folder, srs_wkt=srcSpatialRef, geom_type=geometryType, encoding=characterEncoding, created_by=user)
#define shapefile's attributes
for keys, values in c.schema['properties'].iteritems():
dict = {}
dict['name'] = keys
props = re.split('\W+', values)
dict['type'] = utils.fionaTypeToInt(props[0])
try:
dict['width'] = int(props[1])
except IndexError:
dict['width'] = 0
if dict['type'] == 2:
try:
dict['precision'] = int(props[2])
except IndexError:
dict['precision'] = 15
else:
dict['precision'] = 0
attr = Attribute.objects.create(shapefile=shapefile, **dict)
#store shapefile's features
for srcFeature in c:
try:
wkt = dumps(srcFeature['geometry'])
geosGeometry = GEOSGeometry(wkt)
geosGeometry.srid = SpatialReference(srcSpatialRef).srid
geosGeometry.transform(project)
except TypeError:
geosGeometry = None
geometryField = utils.calcGeometryField(geometryType)
args = {}
args['shapefile'] = shapefile
args[geometryField] = geosGeometry
args['attribute_value'] = srcFeature['properties']
args['id_relat'] = srcFeature['id']
feature = Feature.objects.create(**args)
print("Temps final: --- %s seconds ---" % str(time.time() - start_time))
return None, shapefile
except BaseException, e:
#cleaning up
os.remove(fname)
shutil.rmtree(dirname, ignore_errors=False, onerror=handleRemoveReadonly)
shapefile.delete()
return e, None
from pprint import pprint
with fiona.open('/gdata/world_borders.shp') as src:
pprint(src[1])
###############################################################################
try:
with fiona.open('/gdata/world_borders.shp') as c:
print(len(list(c)))
# assert True is False
except:
print(c.closed)
raise
###############################################################################
import fiona
c = fiona.open('/gdata/world_borders.shp')
c.driver
###############################################################################
c.crs
###############################################################################
from fiona.crs import to_string
print(to_string(c.crs))
###############################################################################
from fiona.crs import from_string
from_string("+datum=WGS84 +ellps=WGS84 +no_defs +proj=longlat")
###############################################################################
from fiona.crs import from_epsg
from_epsg(3857)
###############################################################################
len(c)
###############################################################################
c.bounds
def main():
args = arguments_parse()
direc = args.base_directory
input_filename = args.input_filename
list_name_attributes = args.list_attributes.replace(' ', '').split(',')
region = ''.join(["\"", args.region, "\""])
name = ''.join(["\"", args.name, "\""])
title = ''.join(["\"", args.title, "\""])
abstract = ''.join(["\"", args.abstract, "\""])
kw = ''.join(["\"", args.key_words, "\""])
output_filename = input_filename.split('.')[0]
output_filename += '_wgs84_fiona.shp'
layer = output_filename.split('.')[0]
input_filename = os.path.join(direc, input_filename)
output_filename = os.path.join(direc, output_filename)
with fiona.open(input_filename) as src:
fc = (feature for feature in src if not None in
[feature['properties'][att] for att in list_name_attributes]
and feature['geometry'] is not None)
src_crs = to_string(src.crs)
proj_src_crs = Proj(src_crs)
fc_schema = src.schema
if not proj_src_crs.crs.is_geographic:
reproj_normalize_and_write_large_vector(fc,
fc_schema,
list_name_attributes,
layer,
output_filename,
src_crs,
is_geographic=False)
else:
reproj_normalize_and_write_large_vector(fc, fc_schema,
list_name_attributes,
layer, output_filename,
src_crs)
name_table = layer.lower()
[result_create_table, out,
err] = create_table_via_shp2pgsql(output_filename, name_table)
print((result_create_table, out, err))
result_publish = publish_featuretype_via_docker(name_table)
print(result_publish)
result_update = update_layers_via_docker(name_table)
print(result_update)
output_filename_without_extension = output_filename.split('.')[0]
l_output_filenames = glob(output_filename_without_extension + '*',
recursive=True)
for filenames in l_output_filenames:
os.remove(filenames)
def process_dataset(self, workdir, db, url, extractor):
'''
Extract territories from a given file for a given level
with a given extractor function.
'''
loaded = 0
filename = join(workdir, self.filename_for(url, extractor))
layer = getattr(extractor, 'layer', None)
with load(filename, **extractor.kwargs) as collection:
if layer:
msg = '{0}/{1} ({2} {3})'.format(basename(filename), layer,
collection.driver,
to_string(collection.crs))
elif hasattr(collection, 'driver'):
msg = '{0} ({1} {2})'.format(basename(filename),
collection.driver,
to_string(collection.crs))
else:
msg = '{0}'.format(basename(filename))
for polygon in progress(collection, msg):
try:
zone = extractor(db, polygon)
if not zone:
continue
zone['keys'] = dict(
(k, v) for k, v in zone.get('keys', {}).items()
if v is not None)
geom = shape(polygon['geometry'])
if extractor.simplify:
geom = geom.simplify(extractor.simplify)
if geom.geom_type == 'Polygon':
geom = MultiPolygon([geom])
elif geom.geom_type != 'MultiPolygon':
warning('Unsupported geometry type "{0}" for "{1}"',
geom.geom_type, zone['name'])
continue
zone.update(geom=geom.__geo_interface__)
zone_id = zone.get('_id')
if not zone_id:
zone_id = ':'.join((self.id, zone['code']))
if 'validity' in zone and zone['validity'].get(
'start'):
start = zone['validity']['start']
zone_id = '@'.join((zone_id, start))
if not geom.is_valid:
warning('Invalid geometry for "{0}": {1}', zone_id,
explain_validity(geom))
zone.update(_id=zone_id, level=self.id)
db.find_one_and_replace({'_id': zone_id},
zone,
upsert=True)
loaded += 1
except Exception:
props = dict(polygon.get('properties', {}))
error('Error extracting polygon {0}:\n{1}', props,
traceback.format_exc())
info('Loaded {0} zones for level {1} from file {2}', loaded, self.id,
filename)
return loaded
def process_dataset(self, workdir, db, url, extractor):
'''
Extract territories from a given file for a given level
with a given extractor function.
'''
loaded = 0
filename = join(workdir, basename(url))
layer = getattr(extractor, 'layer', None)
info('processing {0}', basename(filename))
if filename.endswith('.zip'):
loader = self.load_shp_zip
elif filename.endswith('.geojson'):
loader = self.load_geojzon
with loader(filename, **extractor.kwargs) as collection:
if layer:
info('Extracting {0} elements from {1} => {2} ({3} {4})',
len(collection), basename(filename), layer, collection.driver,
to_string(collection.crs))
else:
info('Extracting {0} elements from {1} => ({2} {3})',
len(collection), basename(filename), collection.driver,
to_string(collection.crs))
for polygon in collection:
try:
zone = extractor(db, polygon)
if not zone:
continue
zone['keys'] = dict(
(k, v)
for k, v in zone.get('keys', {}).items()
if v is not None)
geom = shape(polygon['geometry'])
if extractor.simplify:
geom = geom.simplify(extractor.simplify)
if geom.geom_type == 'Polygon':
geom = MultiPolygon([geom])
elif geom.geom_type != 'MultiPolygon':
warning('Unsupported geometry type "{0}" for "{1}"',
geom.geom_type, zone['name'])
continue
zone.update(geom=geom.__geo_interface__)
zone_id = zone.get('_id')
if not zone_id:
zone_id = ':'.join((self.id, zone['code']))
if 'validity' in zone and zone['validity'].get('start'):
start = zone['validity']['start']
zone_id = '@'.join((zone_id, start))
zone.update(_id=zone_id, level=self.id)
db.find_one_and_replace(
{'_id': zone_id}, zone, upsert=True)
loaded += 1
except Exception:
error('Error extracting polygon {0}',
polygon['properties'])
error(traceback.format_exc())
info('Loaded {0} zones for level {1} from file {2}',
loaded, self.id, filename)
return loaded
def sr(self):
sr = SpatialReference()
sr.ImportFromProj4(to_string(self.value))
return sr
def get_crs(layerDict):
crsList = []
for name in layerDict:
with fiona.open(layerDict[name]['file']) as s:
crsList.append(to_string(s.crs))
return from_string(Counter(crsList).most_common()[0][0])
def test_to_string_utm():
# Make a string from a mapping with a few bogus items
val = {"proj": "utm", "ellps": "WGS84", "zone": 13, "no_defs": True, "foo": True, "axis": False, "belgium": [1, 2]}
assert crs.to_string(val) == "+ellps=WGS84 +no_defs +proj=utm +zone=13"
from pprint import pprint
with fiona.open('/gdata/GSHHS_c.shp') as src:
pprint(src[1])
###############################################################################
try:
with fiona.open('/gdata/GSHHS_c.shp') as c:
print(len(list(c)))
except:
print(c.closed)
raise
###############################################################################
c = fiona.open('/gdata/GSHHS_c.shp')
c.driver
###############################################################################
c.crs
###############################################################################
from fiona.crs import to_string
to_string(c.crs)
###############################################################################
from fiona.crs import from_string
from_string("+datum=WGS84 +ellps=WGS84 +no_defs +proj=longlat")
###############################################################################
from fiona.crs import from_epsg
from_epsg(3857)
###############################################################################
len(c)
###############################################################################
c.bounds
def test_to_string_epsg():
val = {"init": "epsg:4326", "no_defs": True}
assert crs.to_string(val) == "+init=epsg:4326 +no_defs"
import fiona
shpfile_city = "/Users/xiaomuliu/CrimeProject/SpatioTemporalModeling/GISdata/City_Boundary/City_Boundary.shp"
chi_city = fiona.open(shpfile_city)
next(chi_city)
chi_city.close()
chi_city.closed
import pprint
from fiona.crs import to_string
with fiona.open(shpfile_city) as chi_city:
pprint.pprint(chi_city[0])
print(chi_city.driver)
print(chi_city.crs)
print(to_string(chi_city.crs))
print(chi_city.bounds)
#GeoPandas extends the datatypes used by pandas to allow spatial operations on geometric types.
#Geometric operations are performed by 'shapely'.
#Geopandas further depends on 'fiona' for file access and descartes and 'matplotlib' for plotting.
import geopandas as gpd
chi_city = gpd.read_file(shpfile_city)
chi_city.head()
chi_city.plot()
shpfile_beat = "/Users/xiaomuliu/CrimeProject/SpatioTemporalModeling/GISdata/cpd_beats/cpd_beats.shp"
beat = gpd.read_file(shpfile_beat)
beat.head()
beat.plot()
def sr(self):
sr = SpatialReference()
sr.ImportFromProj4(to_string(self.value))
return(sr)
def proj_str(self):
if self._proj_str is None and self.crs is not None:
self._proj_str = to_string(self._crs)
elif self._proj_str is None and self.prjfile is not None:
self._proj_str = get_proj_str(self.prjfile)
return self._proj_str
def calculate_area_length_in_metres(in_filename, dissolve_overlap=True):
"""Calculate the total Area and Length in metres for an input polygon file.
If the input file is geographic, then the feature will be 'projected' into wgs84 utm system
Currently there is no Summarize by Column or update existing features .. implemented (see TODO).
Args:
in_filename (str): Input polygon file.
dissolve_overlap (bool): Return values after Dissolve ALL geometries.
Returns:
list[area,length]: The Total Area and Length
"""
start_time = time.time()
if not os.path.exists(in_filename):
raise IOError("Invalid path: {}".format(in_filename))
with fiona.open(in_filename, 'r') as source:
inSRS = osr.SpatialReference()
inSRS.ImportFromProj4(to_string(source.crs))
if not inSRS.IsProjected():
zone, utmSRS, wgs84SRS = pyprecag_crs.getUTMfromWGS84(
*source.bounds[:2])
# Create the Project Transformation object
project = functools.partial(pyproj.transform,
pyproj.Proj(**source.crs),
pyproj.Proj(utmSRS.ExportToProj4()))
# Store Results in a dictionary
resultsDict = collections.defaultdict(float)
# Store projected polygons
polygonsGeom = []
# loop through polygons collecting areas and lengths
for eaFeat in source:
# get the geometry of the feature
eaGeom = shape(eaFeat['geometry'])
# if this is geographic the it will be dd
resultsDict['origArea'] += eaGeom.area
resultsDict['origLength'] += eaGeom.length
# If inSRS is Geographic the reproject the feature to a projected system to get area in metres
if not inSRS.IsProjected():
project = functools.partial(
pyproj.transform, pyproj.Proj(**source.crs),
pyproj.Proj(utmSRS.ExportToProj4()))
# Get the projected geometry
eaGeom = shapely.ops.transform(project, eaGeom)
# store numbers in metres
resultsDict['Area_m'] += eaGeom.area
resultsDict['Length_m'] += eaGeom.length
# If overlaps are to be dissolved we need a projected geometry collection
if dissolve_overlap:
polygonsGeom.append(eaGeom)
if dissolve_overlap:
# Run the dissolve.
polygondis = shapely.ops.unary_union(polygonsGeom)
resultsDict['Area_m_NoOverlap'] += polygondis.area
resultsDict['Length_m_NoOverlap'] += polygondis.length
# get the final values
area = resultsDict['Area_m']
length = resultsDict['Length_m']
if dissolve_overlap:
area = resultsDict['Area_m_NoOverlap']
length = resultsDict['Length_m_NoOverlap']
LOGGER.debug('CRS: {}'.format(to_string(source.crs)))
if not inSRS.IsProjected():
LOGGER.debug(
'\t{:<25} Area (dd): {:>20.5f} Length (dd): {:>20.5f}'.format(
'No Overlap-', resultsDict['origArea'],
resultsDict['origLength']))
LOGGER.debug('Projected To CRS: {}'.format(
utmSRS.GetAttrValue("PROJCS", 0)))
LOGGER.debug(
'\t{:<25} Area (m) : {:>20.5f} Length (m) : {:>20.5f}'.format(
'Total with Overlap-', resultsDict['Area_m'],
resultsDict['Length_m']))
if dissolve_overlap:
LOGGER.debug(
'\t{:<25} Area (m) : {:>20.5f} Length (m) : {:>20.5f}'.format(
'Total with No Overlap-', resultsDict['Area_m_NoOverlap'],
resultsDict['Length_m_NoOverlap']))
LOGGER.debug('{} complete !! Duration H:M:SS - {dur}'.format(
inspect.currentframe().f_code.co_name,
dur=datetime.timedelta(seconds=time.time() - start_time)))
return area, length
def test_to_string_epsg():
val = {'init': 'epsg:4326', 'no_defs': True}
assert crs.to_string(val) == "+init=epsg:4326 +no_defs"
def clip_raster(inraster, features, outraster,
clip_feature_epsg=None, clip_feature_proj4=None,
clip_kwargs={},
**project_kwargs):
"""Clip a raster to feature extent(s).
Parameters
----------
clip_kwargs: dict
Keyword arguments to rasterio.mask
kwargs : key word arguments to project_raster
These are only used if the clip features are
in a different coordinate system, in which case
the raster will be reprojected into that coordinate
system.
"""
rasterio = import_rasterio()
from fiona.crs import to_string
from shapely.geometry import box
# compare coordinates references for raster and clip feature
# (if the clip feature is a shapefile path)
with rasterio.open(inraster) as src:
raster_crs = to_string(src.crs)
clip_crs = raster_crs # start with assumption of same coordinates
if isinstance(features, str):
with fiona.open(features) as src2:
clip_crs = to_string(src2.crs)
elif isinstance(features, list):
if clip_feature_epsg is not None:
clip_crs = '+init=epsg:{}'.format(clip_feature_epsg)
elif clip_feature_proj4 is not None:
clip_crs = clip_feature_proj4
# convert the features to geojson
geoms = _to_geojson(features)
print('input raster crs:\n{}\n\n'.format(raster_crs),
'clip feature crs:\n{}\n'.format(clip_crs))
# if the coordinate systems are not the same
# reproject the raster first before clipping
# this could be greatly sped up by first clipping the input raster prior to reprojecting
if raster_crs != clip_crs:
tmpraster = 'tmp.tif'
tmpraster2 = 'tmp2.tif'
print('Input raster and clip feature(s) are in different coordinate systems.\n'
'Input raster will be reprojected to the clip feature coordinate system.\n')
# make prelim clip of raster to speed up reprojection
xmin, xmax, ymin, ymax = _get_bounds(geoms)
longest_side = np.max([xmax-xmin, ymax-ymin])
bounds = box(xmin, ymin, xmax, ymax).buffer(longest_side*0.1)
bounds = project(bounds, clip_crs, raster_crs)
_clip_raster(inraster, [bounds], tmpraster, **clip_kwargs)
project_raster(tmpraster, tmpraster2, clip_crs, **project_kwargs)
inraster = tmpraster2
_clip_raster(inraster, geoms, outraster, **clip_kwargs)
if raster_crs != clip_crs:
for tmp in [tmpraster, tmpraster2]:
if os.path.exists(tmp):
print('removing {}...'.format(tmp))
os.remove(tmp)
print('Done.')
