def process_dataset(self, workdir, db, url, extractor): ''' Extract territories from a given file for a given level with a given extractor function. ''' loaded = 0 filename = join(workdir, basename(url)) layer = extractor.layer info('processing {0}', basename(filename)) if filename.endswith('.zip'): loader = self.load_shp_zip elif filename.endswith('.geojson'): loader = self.load_geojzon with loader(filename, layer=layer) as collection: if layer: info('Extracting {0} elements from {1} => {2} ({3} {4})', len(collection), basename(filename), layer, collection.driver, to_string(collection.crs)) else: info('Extracting {0} elements from {1} => ({2} {3})', len(collection), basename(filename), collection.driver, to_string(collection.crs)) for polygon in collection: try: zone = extractor(db, polygon) if not zone: continue zone['keys'] = dict( (k, v) for k, v in zone.get('keys', {}).items() if v is not None) geom = shape(polygon['geometry']) if extractor.simplify: geom = geom.simplify(extractor.simplify) if geom.geom_type == 'Polygon': geom = MultiPolygon([geom]) elif geom.geom_type != 'MultiPolygon': warning('Unsupported geometry type "{0}" for "{1}"', geom.geom_type, zone['name']) continue zone.update(geom=geom.__geo_interface__) zone_id = zone.get('_id') if not zone_id: zone_id = ':'.join((self.id, zone['code'])) zone.update(_id=zone_id, level=self.id) db.find_one_and_replace( {'_id': zone_id}, zone, upsert=True) loaded += 1 except Exception: error('Error extracting polygon {0}', polygon['properties']) error(traceback.format_exc()) info('Loaded {0} zones for level {1} from file {2}', loaded, self.id, filename) return loaded
def _read_extent_shapefile(self, shpfile, buffer=0): import fiona from fiona.crs import to_string, from_epsg print('reading extent from {}...'.format(shpfile)) shp = fiona.open(shpfile) g = shape(shp.next()['geometry']) if to_string(from_epsg(coord_datums_epsg[self.datum])) != to_string(shp.crs): print('reprojecting extent from {} to {}'.format(to_string(shp.crs), self.proj4)) return project(g, to_string(shp.crs), self.proj4) else: return g
def test_to_string_zeroval(): # Make a string with some 0 values (e.g. esri:102017) val = {'proj': 'laea', 'lat_0': 90, 'lon_0': 0, 'x_0': 0, 'y_0': 0, 'ellps': 'WGS84', 'datum': 'WGS84', 'units': 'm', 'no_defs': True} assert crs.to_string(val) == ( "+datum=WGS84 +ellps=WGS84 +lat_0=90 +lon_0=0 +no_defs +proj=laea " "+units=m +x_0=0 +y_0=0")
def get_mbb(layerDict,crs): for name in layerDict: if 'logic' in layerDict[name] and layerDict[name]['logic'] == 'boundary': with fiona.open(layerDict[name]['file']) as s: x1, y1, x2, y2 = s.bounds if to_string(crs) != to_string(s.crs): proj1=pyproj.Proj(s.crs,preserve_units=True) proj2=pyproj.Proj(crs,preserve_units=True) minx, miny = pyproj.transform(proj1,proj2,x1,y1) maxx, maxy = pyproj.transform(proj1,proj2,x2,y2) return (minx,miny,maxx,maxy) else: return (x1,y1,x2,y2) else: continue raise NameError("No Layer of Type Boundary was provided")
def test_to_string(): # Make a string from a mapping with a few bogus items val = { 'proj': 'longlat', 'ellps': 'WGS84', 'datum': 'WGS84', 'no_defs': True, 'foo': True, 'axis': False, 'belgium': [1,2] } assert crs.to_string( val) == "+datum=WGS84 +ellps=WGS84 +no_defs +proj=longlat"
def test_to_string_utm(): # Make a string from a mapping with a few bogus items val = { 'proj': 'utm', 'ellps': 'WGS84', 'zone': 13, 'no_defs': True, 'foo': True, 'axis': False, 'belgium': [1, 2]} assert crs.to_string( val) == "+ellps=WGS84 +no_defs +proj=utm +zone=13"
def test_to_string(): # Make a string from a mapping with a few bogus items val = { 'proj': 'longlat', 'ellps': 'WGS84', 'datum': 'WGS84', 'no_defs': True, 'foo': True, 'axis': False, 'belgium': [1, 2]} assert crs.to_string( val) == "+datum=WGS84 +ellps=WGS84 +no_defs +proj=longlat"
def test_to_string_utm(): # Make a string from a mapping with a few bogus items val = { 'proj': 'utm', 'ellps': 'WGS84', 'zone': 13, 'no_defs': True, 'foo': True, 'axis': False, 'belgium': [1,2] } assert crs.to_string( val) == "+ellps=WGS84 +no_defs +proj=utm +zone=13"
def add(request): if request.method == 'POST': form = LayerForm(request.POST, request.FILES) if form.is_valid(): l = form.save() col = form.get_collection() srs = to_string(form.layer_crs()) shape_path = "%s/uploads/shapefile/%s/%s.shp" % ( settings.MEDIA_ROOT, request.tenant.schema_name, l.pk) if not os.path.exists(os.path.dirname(shape_path)): os.makedirs(os.path.dirname(shape_path)) with fiona.collection(shape_path, "w", schema=col.schema, crs=col.crs, driver="ESRI Shapefile") as out: for f in col: out.write(f) django_rq.enqueue(process_shapefile, request.tenant.schema_name, l.pk, srs) messages.success(request, "Layer added.") return redirect('layers:index') else: messages.error(request, "The layer could not be saved due to errors.") else: form = LayerForm() return render(request, 'layers/add.html', {'form': form})
def __init__(self,crs=None,prjs=None,epsg=None): if crs is None: if prjs is not None: crs = from_string(prjs) elif epsg is not None: sr = SpatialReference() sr.ImportFromEPSG(epsg) crs = from_string(sr.ExportToProj4()) else: raise(NotImplementedError) else: ## remove unicode and change to python types for k,v in crs.iteritems(): if type(v) == unicode: crs[k] = str(v) else: try: crs[k] = v.tolist() except AttributeError: continue sr = SpatialReference() sr.ImportFromProj4(to_string(crs)) self.value = from_string(sr.ExportToProj4()) try: assert(self.value != {}) except AssertionError: ocgis_lh(logger='crs',exc=ValueError('Empty CRS: The conversion to PROJ4 may have failed. The CRS value is: {0}'.format(crs)))
def process_dataset(self, workdir, db, url, extractor): '''Extract territories from a given file for a given level with a given extractor function''' loaded = 0 filename = join(workdir, basename(url)) with fiona.open('/', vfs='zip://{0}'.format(filename), encoding='utf8') as collection: info('Extracting {0} elements from {1} ({2} {3})'.format( len(collection), basename(filename), collection.driver, to_string(collection.crs) )) for polygon in collection: try: zone = extractor(polygon) if not zone: continue zone['keys'] = dict((k, v) for k, v in zone.get('keys', {}).items() if v is not None) geom = shape(polygon['geometry']) if extractor.simplify: geom = geom.simplify(extractor.simplify) if geom.geom_type == 'Polygon': geom = MultiPolygon([geom]) elif geom.geom_type != 'MultiPolygon': warning('Unsupported geometry type "{0}" for "{1}"'.format(geom.geom_type, zone['name'])) continue zoneid = '/'.join((self.id, zone['code'])) zone.update(_id=zoneid, level=self.id, geom=geom.__geo_interface__) db.find_one_and_replace({'_id': zoneid}, zone, upsert=True) loaded += 1 except Exception as e: error('Error extracting polygon {0}: {1}', polygon['properties'], str(e)) info('Loaded {0} zones for level {1} from file {2}'.format(loaded, self.id, filename)) return loaded
def get_bbox(feature, dest_crs): """Get bounding box for a Polygon feature. Parameters ---------- feature : str (shapefile path), shapely Polygon or GeoJSON dest_crs : proj str Desired output coordinate system (shapefiles only) """ if isinstance(feature, str): with fiona.open(feature) as src: l, b, r, t = src.bounds bbox_src_crs = box(*src.bounds) shpcrs = crs(proj_str=to_string(src.crs)) if dest_crs is not None and shpcrs != dest_crs: bbox_dest_crs = project(bbox_src_crs, shpcrs.proj_str, dest_crs.proj_str) l, b, r, t = bbox_dest_crs.bounds # x, y = project([(l, b), # (r, t)], shpcrs.proj_str, dest_crs.proj_str) # filter = (x[0], y[0], x[1], y[1]) # else: filter = (l, b, r, t) elif isinstance(feature, Polygon): filter = feature.bounds elif isinstance(feature, dict): try: filter = shape(feature).bounds except Exception as ex: print(ex) print("Supplied dictionary doesn't appear to be valid GeoJSON.") return filter
def extract_shapefile(filename, level, extractor): '''Extract territories from a given file for a given level with a given extractor function''' imported = 0 with fiona.open('/', vfs='zip://{0}'.format(filename), encoding='utf8') as collection: print 'Extracting {0} elements from {1} ({2} {3})'.format( len(collection), basename(filename), collection.driver, to_string(collection.crs) ) for polygon in collection: try: code, name, keys = extractor(polygon) geom = shape(polygon['geometry']) if geom.geom_type == 'Polygon': geom = MultiPolygon([geom]) elif geom.geom_type != 'MultiPolygon': print 'Unsupported geometry type "{0}" for "{1}"'.format(geom.geom_type, name) continue territory, _ = Territory.objects.get_or_create(level=level, code=code, defaults={ 'geom': geom.__geo_interface__, 'name': name, 'keys': keys, }) territory.geom = geom.__geo_interface__ territory.name = name territory.keys = keys territory.save() imported += 1 except Exception as e: print 'Error extracting polygon {0}: {1}'.format(polygon['properties'], e) print 'Imported {0} territories for level {1} from file {2}'.format(imported, level, filename) return imported
def handle(self, **options): input_file = options['input_file'] year = options['year'] scheme = options['scheme'] field = options['field'] name = options['name'] # Create ValidClassification objects list # Push it to database # Read file and Optionally reproject the features to longlat with fiona.open(input_file) as src: p = Proj(src.crs) if p.is_latlong( ): # Here we assume that geographic coordinates are automatically 4326 (not quite true) fc = list(src) else: crs_str = to_string(src.crs) fc = [ feature_transform(x, crs_out='+proj=longlat', crs_in=crs_str) for x in src ] # Write features to ValidObject table def valid_obj_builder(x): """Build individual ValidObjects """ geom = GEOSGeometry(json.dumps(x['geometry'])) obj = ValidObject(the_geom=geom) return obj obj_list = [valid_obj_builder(x) for x in fc] ValidObject.objects.bulk_create(obj_list) # Get list of unique tags unique_numeric_codes = list(set([x['properties'][field] for x in fc])) # Update Tag table using get or create def make_tag_tuple(x): obj, _ = Tag.objects.get_or_create(numeric_code=x, scheme=scheme) return (x, obj) tag_dict = dict([make_tag_tuple(x) for x in unique_numeric_codes]) # Build validClassification object list (valid_tag, valid_object, valid_set) def valid_class_obj_builder(x): """x is a tuple (ValidObject, feature)""" tag = tag_dict[x[1]['properties'][field]] obj = ValidClassification(valid_tag=tag, valid_object=x[0], valid_set=name, interpretation_year=year) return obj valid_class_obj_list = [ valid_class_obj_builder(x) for x in zip(obj_list, fc) ] ValidClassification.objects.bulk_create(valid_class_obj_list)
def get_mbb(layerDict, crs): for name in layerDict: if 'logic' in layerDict[name] and layerDict[name][ 'logic'] == 'boundary': with fiona.open(layerDict[name]['file']) as s: x1, y1, x2, y2 = s.bounds if to_string(crs) != to_string(s.crs): proj1 = pyproj.Proj(s.crs, preserve_units=True) proj2 = pyproj.Proj(crs, preserve_units=True) minx, miny = pyproj.transform(proj1, proj2, x1, y1) maxx, maxy = pyproj.transform(proj1, proj2, x2, y2) return (minx, miny, maxx, maxy) else: return (x1, y1, x2, y2) else: continue raise NameError("No Layer of Type Boundary was provided")
def _set_proj4(self): # establish a proj4 string for basemap projection regardless of input if self.epsg is not None: self.proj4 = to_string(from_epsg(self.epsg)) if self.projection_shapefile is not None: self._read_shapefile_projection(self.projection_shapefile) self.proj4 = get_proj4(self.projection_shapefile)
def test_to_string_unicode(): # See issue #83. val = crs.to_string({ 'units': 'm', 'no_defs': True, 'datum': 'NAD83', 'proj': 'utm', 'zone': 16}) assert 'NAD83' in val
def test_to_string_unicode(): # See issue #83. val = crs.to_string({ u'units': u'm', u'no_defs': True, u'datum': u'NAD83', u'proj': u'utm', u'zone': 16}) assert 'NAD83' in val
def ras2vec(raster_file, output_path): # Create a generic polygon schema for the output vector file i = 0 feat_schema = { 'geometry': 'Polygon', 'properties': OrderedDict([('value', 'int')]) } class_value_domain = set() out_features = [] print(" - Processing raster file: {}".format(raster_file)) with rasterio.open(raster_file, 'r') as src: raster = src.read(1) mask = raster != 0 # Vectorize the polygons polygons = features.shapes(raster, mask, transform=src.transform) # Create shapely polygon featyres for polygon in polygons: feature = { 'geometry': { 'type': 'Polygon', 'coordinates': None }, 'properties': OrderedDict([('value', 0)]) } feature['geometry']['coordinates'] = polygon[0]['coordinates'] value = int(polygon[1]) # Pixel value of the class (layer) class_value_domain.add(value) feature['properties']['value'] = value i += 1 out_features.append(feature) print(" - Writing output vector file: {}".format(output_path)) num_layers = list(class_value_domain) # Number of unique pixel value for num_layer in num_layers: polygons = [ feature for feature in out_features if feature['properties']['value'] == num_layer ] layer_name = 'vector_' + str(num_layer).rjust(3, '0') print(" - Writing layer: {}".format(layer_name)) with fiona.open(output_path, 'w', crs=to_string(src.crs), layer=layer_name, schema=feat_schema, driver='GPKG') as dest: for polygon in polygons: dest.write(polygon) print("") print("Number of features written: {}".format(i))
def _read_shapefile_projection(self, shapefile): """sets the proj4 string from a shapefile""" crs = fiona.open(shapefile).crs init = crs.get('init', '') if 'epsg' in init: self.epsg = int(init.strip('epsg:')) if self.epsg is not None: self.proj4 = getproj4(self.epsg) else: self.proj4 = to_string(crs)
def proj42wkt(file_path): c = fiona.open(file_path) "res/geo/subjects/42-loire/42-.dbf" prj4 = to_string(c.crs) sr = osr.SpatialReference() sr.ImportFromProj4(prj4) wkt = sr.ExportToWkt() return wkt
def __init__(self, value=None, proj4=None, epsg=None, name=OcgisConvention.Name.COORDSYS): self.name = name # Allows operations on data variables to look through an empty dimension list. Alleviates instance checking. self.dimensions = tuple() self.dimension_names = tuple() self.has_bounds = False self._epsg = epsg # Some basic overloaded for WGS84. if epsg == 4326: value = WGS84().value # Add a special check for init keys in value dictionary. if value is not None: if 'init' in value and list(value.values())[0].startswith('epsg'): epsg = int(list(value.values())[0].split(':')[1]) value = None if value is None: if proj4 is not None: value = from_string(proj4) elif epsg is not None: sr = SpatialReference() sr.ImportFromEPSG(epsg) value = from_string(sr.ExportToProj4()) else: msg = 'A value dictionary, PROJ.4 string, or EPSG code is required.' raise ValueError(msg) else: # Remove unicode to avoid strange issues with proj and fiona. for k, v in value.items(): if isinstance(v, six.string_types): value[k] = str(v) else: try: value[k] = v.tolist() # this may be a numpy arr that needs conversion except AttributeError: continue sr = SpatialReference() sr.ImportFromProj4(to_string(value)) self.value = from_string(sr.ExportToProj4()) try: assert self.value != {} except AssertionError: msg = 'Empty CRS: The conversion to PROJ.4 may have failed. The CRS value is: {0}'.format( value) raise ValueError(msg)
def test_to_string(): # Make a string from a mapping with a few bogus items val = { "proj": "longlat", "ellps": "WGS84", "datum": "WGS84", "no_defs": True, "foo": True, "axis": False, "belgium": [1, 2], } assert crs.to_string(val) == "+datum=WGS84 +ellps=WGS84 +no_defs +proj=longlat"
def process_dataset(self, workdir, db, url, extractor): '''Extract territories from a given file for a given level with a given extractor function''' loaded = 0 filename = join(workdir, basename(url)) # Identify the shapefile to avoid multiple file error on GDAL 2 with ZipFile(filename) as z: candidates = [n for n in z.namelist() if n.endswith('.shp')] if len(candidates) != 1: raise ValueError( 'Unable to find a unique shpaefile into {0}'.format( filename)) shp = candidates[0] with fiona.open('/{0}'.format(shp), vfs='zip://{0}'.format(filename), encoding='utf8') as collection: info('Extracting {0} elements from {1} ({2} {3})'.format( len(collection), basename(filename), collection.driver, to_string(collection.crs))) for polygon in collection: try: zone = extractor(polygon) if not zone: continue zone['keys'] = dict( (k, v) for k, v in zone.get('keys', {}).items() if v is not None) geom = shape(polygon['geometry']) if extractor.simplify: geom = geom.simplify(extractor.simplify) if geom.geom_type == 'Polygon': geom = MultiPolygon([geom]) elif geom.geom_type != 'MultiPolygon': warning( 'Unsupported geometry type "{0}" for "{1}"'.format( geom.geom_type, zone['name'])) continue zoneid = '/'.join((self.id, zone['code'])) zone.update(_id=zoneid, level=self.id, geom=geom.__geo_interface__) db.find_one_and_replace({'_id': zoneid}, zone, upsert=True) loaded += 1 except Exception as e: error('Error extracting polygon {0}: {1}', polygon['properties'], str(e)) info('Loaded {0} zones for level {1} from file {2}'.format( loaded, self.id, filename)) return loaded
def __init__(self, value=None, proj4=None, epsg=None, name=OcgisConvention.Name.COORDSYS): self.name = name # Allows operations on data variables to look through an empty dimension list. Alleviates instance checking. self.dimensions = tuple() self.dimension_names = tuple() self.has_bounds = False self._epsg = epsg # Some basic overloaded for WGS84. if epsg == 4326: value = WGS84().value # Add a special check for init keys in value dictionary. if value is not None: if 'init' in value and list(value.values())[0].startswith('epsg'): epsg = int(list(value.values())[0].split(':')[1]) value = None if value is None: if proj4 is not None: value = from_string(proj4) elif epsg is not None: sr = SpatialReference() sr.ImportFromEPSG(epsg) value = from_string(sr.ExportToProj4()) else: msg = 'A value dictionary, PROJ.4 string, or EPSG code is required.' raise ValueError(msg) else: # Remove unicode to avoid strange issues with proj and fiona. for k, v in value.items(): if isinstance(v, six.string_types): value[k] = str(v) else: try: value[k] = v.tolist() # this may be a numpy arr that needs conversion except AttributeError: continue sr = SpatialReference() sr.ImportFromProj4(to_string(value)) self.value = from_string(sr.ExportToProj4()) try: assert self.value != {} except AssertionError: msg = 'Empty CRS: The conversion to PROJ.4 may have failed. The CRS value is: {0}'.format(value) raise ValueError(msg)
def get_authority(file_path): ''' Get the authority used by a raster i.e. EPSG:4326 :param file_path: path to the file :return: return the SRID of the raster projection ''' with fiona.open(file_path) as c: log.info(c) # print c.crs # print to_string(c.crs) # print get_epsg_code_from_proj4(c.crs) if 'init' in c.crs: return c.crs['init'] elif 'proj' in c.crs: return "EPSG:" + str(get_epsg_code_from_proj4(to_string(c.crs))) return None
def crs_to_srs(crs): """ Recover our own WKT definition of projections from a pyproj / fiona CRS Args: crs (pyproj.CRS or dict): what is returned from GeoDataFrame().crs Returns: string: a "EPSG:xxx" or WKT representation of the crs. """ if crs is None: return elif isinstance(crs, dict): # geopandas < 0.7 proj4_str = fiona.crs.to_string(crs) return get_epsg_or_wkt(proj4_str) else: # geopandas >= 0.7 return crs.to_string()
def extract_shapefile(filename, level, extractor): '''Extract territories from a given file for a given level with a given extractor function''' imported = 0 with fiona.open('/', vfs='zip://{0}'.format(filename), encoding='utf8') as collection: print 'Extracting {0} elements from {1} ({2} {3})'.format( len(collection), basename(filename), collection.driver, to_string(collection.crs)) for polygon in collection: try: code, name, keys = extractor(polygon) geom = shape(polygon['geometry']) if geom.geom_type == 'Polygon': geom = MultiPolygon([geom]) elif geom.geom_type != 'MultiPolygon': print 'Unsupported geometry type "{0}" for "{1}"'.format( geom.geom_type, name) continue territory, _ = Territory.objects.get_or_create( level=level, code=code, defaults={ 'geom': geom.__geo_interface__, 'name': name, 'keys': keys, }) territory.geom = geom.__geo_interface__ territory.name = name territory.keys = keys territory.save() imported += 1 except Exception as e: print 'Error extracting polygon {0}: {1}'.format( polygon['properties'], e) print 'Imported {0} territories for level {1} from file {2}'.format( imported, level, filename) return imported
def __init__(self, value=None, proj4=None, epsg=None, name=constants.DEFAULT_COORDINATE_SYSTEM_NAME): self.name = name # Add a special check for init keys in value dictionary. if value is not None: if 'init' in value and value.values()[0].startswith('epsg'): epsg = int(value.values()[0].split(':')[1]) value = None if value is None: if proj4 is not None: value = from_string(proj4) elif epsg is not None: sr = SpatialReference() sr.ImportFromEPSG(epsg) value = from_string(sr.ExportToProj4()) else: msg = 'A value dictionary, PROJ.4 string, or EPSG code is required.' raise ValueError(msg) else: # Remove unicode to avoid strange issues with proj and fiona. for k, v in value.iteritems(): if type(v) == unicode: value[k] = str(v) else: try: value[k] = v.tolist() # this may be a numpy arr that needs conversion except AttributeError: continue sr = SpatialReference() sr.ImportFromProj4(to_string(value)) self.value = from_string(sr.ExportToProj4()) try: assert self.value != {} except AssertionError: msg = 'Empty CRS: The conversion to PROJ.4 may have failed. The CRS value is: {0}'.format(value) raise ValueError(msg)
def _load_from_fiona(self, f): """Load from fiona object. Return FeatureCollection. :param f: fiona object """ a = {} try: a['crs'] = to_string(f.crs) except: pass for k, v in self.attributes.items(): if util.is_callable(v): a[k] = v(f) else: a[k] = v a['features'] = [] for s in f: s['properties'] = util.rec_apply(util.reinterpret_decode, s['properties'], lambda x: isinstance(x, unicode)) a['features'].append(Feature(s)) return FeatureCollection(a)
def create_land_use_rasters(land_use_folder, raster_output_folder, crs = None): for ags_district in os.listdir(land_use_folder): folder_abs = os.path.join(land_use_folder, ags_district) if os.path.isdir(folder_abs): #find siedlung shapefile name #TODO: Take first shapefile in list shapefile = [f for f in os.listdir(folder_abs) if f.endswith('.shp')][0] #if there is a name filter, filter shapefiles. #for each land use shapefile, tabulate intersections for each zone in that shapefile full_sp_path = os.path.join(folder_abs, shapefile) layer_name = os.path.splitext(shapefile)[0] with fiona.open(full_sp_path, 'r') as vector_f: assert crs or vector_f.crs, "a CRS must be specified either in in the shapefile or as an argument" if not crs: crs = vector_f.crs (minx, miny, maxx, maxy) = map(lambda a:int(a - a % 50), vector_f.bounds) cmd = ["gdal_rasterize", "-a", "lu_code", "-a_srs", to_string(crs), "-te", str(minx - 100), str(miny - 100), str(maxx+100), str(maxy+100), "-tr", "10.0", #10m resolution "10.0", "-ot", "Int16", "-l", layer_name, full_sp_path, os.path.join(raster_output_folder, ags_district + "_" + layer_name + '.tif')] print(cmd) subprocess.check_call(cmd)
def reproject_point(lat, lon, shp_path): # get projections in proj4 string format with fiona.open(shp_path + ".shp") as source: tocrs_proj4 = to_string(source.crs) fromcrs_proj4 = fiona.crs.to_string(from_epsg(4326)) # wgs84_prj = os.path.join(get_project_dir(), "shp", "4326.prj") # fromcrs = pycrs.loader.from_file(wgs84_prj) # fromcrs_proj4 = fromcrs.to_proj4() # tocrs = pycrs.loader.from_file(shp_path + ".prj") # tocrs_proj4 = tocrs.to_proj4() point_wgs84 = Point(lat, lon) project = partial( pyproj.transform, pyproj.Proj(fromcrs_proj4), pyproj.Proj(tocrs_proj4)) return transform(project, point_wgs84)
def shapeify(self, weekday_rst_path): rst = rasterio.open(weekday_rst_path) array = rst.read() array = array[0] array = array.astype(int) print('get_shapes_from_raster') shape_list = self.get_shapes_from_raster(array, None, daymaps=False) fname = weekday_rst_path.replace('.tiff', '.shp') schema = {'properties': [('class', 'int')], 'geometry': 'Polygon'} # crs, transform, area_shape = area2transform_baws300_sweref99tm() crs, transform, area_shape = area2transform_baws1000_sweref99tm() with fiona.open(fname, 'w', driver='ESRI Shapefile', crs=to_string(crs), schema=schema) as dst: dst.writerecords(shape_list) print('shapeify completed!')
def bbox_copyraster(in_file, out_file, bbox, dst_crs=None): """Rather than re-invent rio clip, rio warp just call them directly """ with rasterio.open(in_file) as source: bbox = transform_bbox(bbox, from_epsg(4326), out_crs=source.meta['crs']) bbox = [str(b) for b in bbox] if not dst_crs: clip_file = out_file else: clip_file = os.path.join(tempfile.gettempdir(), "rio_temp.tif") command = ['rio', 'clip', in_file, clip_file, '--bounds', '"'+" ".join(bbox)+'"'] subprocess.call(" ".join(command), shell=True) if dst_crs: # convert crs to string and wrap in quotes if type(dst_crs) == dict: dst_crs = to_string(dst_crs) dst_crs = '"'+dst_crs+'"' command = ['rio', 'warp', clip_file, out_file, '--dst-crs', dst_crs, '--force-overwrite'] subprocess.call(" ".join(command), shell=True) os.unlink(clip_file)
def importData(file, characterEncoding, format, user, folder): cursor = connection.cursor() start_time = time.time() #manage zipfile fd,fname = tempfile.mkstemp(suffix=fileExt_dic[format]) os.close(fd) f = open(fname, "wb") for chunk in file.chunks(): f.write(chunk) f.close() if not zipfile.is_zipfile(fname): os.remove(fname) return "Not a valid zip archive.", None zip = zipfile.ZipFile(fname) hasSuffix = {} required_suffixes = suffixes_dic[format] for suffix in required_suffixes: hasSuffix[suffix] = False for info in zip.infolist(): extension = os.path.splitext(info.filename)[1].lower() if extension in required_suffixes: hasSuffix[extension] = True for suffix in required_suffixes: if not hasSuffix[suffix]: zip.close() os.remove(fname) return "Archive missing required %s file." % suffix, None zip = zipfile.ZipFile(fname) shapefileName = None dirname = tempfile.mkdtemp() for info in zip.infolist(): if info.filename.lower().endswith(filenameExt_dic[format]): shapefileName = info.filename dstFile = os.path.join(dirname, info.filename) f = open(dstFile, "wb") f.write(zip.read(info.filename)) f.close() zip.close() #verify if shapefile is valid try: srcPath = os.path.join(dirname,shapefileName) srcLayers = fiona.listlayers(srcPath) shapefileOK = True except: traceback.print_exc() shapefileOK = False if not shapefileOK: os.remove(fname) shutil.rmtree(dirname) return "Not a valid vector file.", None #add shapefile object to database try: for i in srcLayers: with fiona.open(srcPath) as c: srcSpatialRef = to_string(c.crs) print srcSpatialRef project = CoordTransform(SpatialReference(srcSpatialRef),SpatialReference(3857)) geometryType = c.schema['geometry'] shapefile = Shapefile.objects.create(filename=c.name, parent=folder, srs_wkt=srcSpatialRef, geom_type=geometryType, encoding=characterEncoding, created_by=user) #define shapefile's attributes for keys, values in c.schema['properties'].iteritems(): dict = {} dict['name'] = keys props = re.split('\W+', values) dict['type'] = utils.fionaTypeToInt(props[0]) try: dict['width'] = int(props[1]) except IndexError: dict['width'] = 0 if dict['type'] == 2: try: dict['precision'] = int(props[2]) except IndexError: dict['precision'] = 15 else: dict['precision'] = 0 attr = Attribute.objects.create(shapefile=shapefile, **dict) #store shapefile's features for srcFeature in c: try: wkt = dumps(srcFeature['geometry']) geosGeometry = GEOSGeometry(wkt) geosGeometry.srid = SpatialReference(srcSpatialRef).srid geosGeometry.transform(project) except TypeError: geosGeometry = None geometryField = utils.calcGeometryField(geometryType) args = {} args['shapefile'] = shapefile args[geometryField] = geosGeometry args['attribute_value'] = srcFeature['properties'] args['id_relat'] = srcFeature['id'] feature = Feature.objects.create(**args) print("Temps final: --- %s seconds ---" % str(time.time() - start_time)) return None, shapefile except BaseException, e: #cleaning up os.remove(fname) shutil.rmtree(dirname, ignore_errors=False, onerror=handleRemoveReadonly) shapefile.delete() return e, None
from pprint import pprint with fiona.open('/gdata/world_borders.shp') as src: pprint(src[1]) ############################################################################### try: with fiona.open('/gdata/world_borders.shp') as c: print(len(list(c))) # assert True is False except: print(c.closed) raise ############################################################################### import fiona c = fiona.open('/gdata/world_borders.shp') c.driver ############################################################################### c.crs ############################################################################### from fiona.crs import to_string print(to_string(c.crs)) ############################################################################### from fiona.crs import from_string from_string("+datum=WGS84 +ellps=WGS84 +no_defs +proj=longlat") ############################################################################### from fiona.crs import from_epsg from_epsg(3857) ############################################################################### len(c) ############################################################################### c.bounds
def main(): args = arguments_parse() direc = args.base_directory input_filename = args.input_filename list_name_attributes = args.list_attributes.replace(' ', '').split(',') region = ''.join(["\"", args.region, "\""]) name = ''.join(["\"", args.name, "\""]) title = ''.join(["\"", args.title, "\""]) abstract = ''.join(["\"", args.abstract, "\""]) kw = ''.join(["\"", args.key_words, "\""]) output_filename = input_filename.split('.')[0] output_filename += '_wgs84_fiona.shp' layer = output_filename.split('.')[0] input_filename = os.path.join(direc, input_filename) output_filename = os.path.join(direc, output_filename) with fiona.open(input_filename) as src: fc = (feature for feature in src if not None in [feature['properties'][att] for att in list_name_attributes] and feature['geometry'] is not None) src_crs = to_string(src.crs) proj_src_crs = Proj(src_crs) fc_schema = src.schema if not proj_src_crs.crs.is_geographic: reproj_normalize_and_write_large_vector(fc, fc_schema, list_name_attributes, layer, output_filename, src_crs, is_geographic=False) else: reproj_normalize_and_write_large_vector(fc, fc_schema, list_name_attributes, layer, output_filename, src_crs) name_table = layer.lower() [result_create_table, out, err] = create_table_via_shp2pgsql(output_filename, name_table) print((result_create_table, out, err)) result_publish = publish_featuretype_via_docker(name_table) print(result_publish) result_update = update_layers_via_docker(name_table) print(result_update) output_filename_without_extension = output_filename.split('.')[0] l_output_filenames = glob(output_filename_without_extension + '*', recursive=True) for filenames in l_output_filenames: os.remove(filenames)
def process_dataset(self, workdir, db, url, extractor): ''' Extract territories from a given file for a given level with a given extractor function. ''' loaded = 0 filename = join(workdir, self.filename_for(url, extractor)) layer = getattr(extractor, 'layer', None) with load(filename, **extractor.kwargs) as collection: if layer: msg = '{0}/{1} ({2} {3})'.format(basename(filename), layer, collection.driver, to_string(collection.crs)) elif hasattr(collection, 'driver'): msg = '{0} ({1} {2})'.format(basename(filename), collection.driver, to_string(collection.crs)) else: msg = '{0}'.format(basename(filename)) for polygon in progress(collection, msg): try: zone = extractor(db, polygon) if not zone: continue zone['keys'] = dict( (k, v) for k, v in zone.get('keys', {}).items() if v is not None) geom = shape(polygon['geometry']) if extractor.simplify: geom = geom.simplify(extractor.simplify) if geom.geom_type == 'Polygon': geom = MultiPolygon([geom]) elif geom.geom_type != 'MultiPolygon': warning('Unsupported geometry type "{0}" for "{1}"', geom.geom_type, zone['name']) continue zone.update(geom=geom.__geo_interface__) zone_id = zone.get('_id') if not zone_id: zone_id = ':'.join((self.id, zone['code'])) if 'validity' in zone and zone['validity'].get( 'start'): start = zone['validity']['start'] zone_id = '@'.join((zone_id, start)) if not geom.is_valid: warning('Invalid geometry for "{0}": {1}', zone_id, explain_validity(geom)) zone.update(_id=zone_id, level=self.id) db.find_one_and_replace({'_id': zone_id}, zone, upsert=True) loaded += 1 except Exception: props = dict(polygon.get('properties', {})) error('Error extracting polygon {0}:\n{1}', props, traceback.format_exc()) info('Loaded {0} zones for level {1} from file {2}', loaded, self.id, filename) return loaded
def process_dataset(self, workdir, db, url, extractor): ''' Extract territories from a given file for a given level with a given extractor function. ''' loaded = 0 filename = join(workdir, basename(url)) layer = getattr(extractor, 'layer', None) info('processing {0}', basename(filename)) if filename.endswith('.zip'): loader = self.load_shp_zip elif filename.endswith('.geojson'): loader = self.load_geojzon with loader(filename, **extractor.kwargs) as collection: if layer: info('Extracting {0} elements from {1} => {2} ({3} {4})', len(collection), basename(filename), layer, collection.driver, to_string(collection.crs)) else: info('Extracting {0} elements from {1} => ({2} {3})', len(collection), basename(filename), collection.driver, to_string(collection.crs)) for polygon in collection: try: zone = extractor(db, polygon) if not zone: continue zone['keys'] = dict( (k, v) for k, v in zone.get('keys', {}).items() if v is not None) geom = shape(polygon['geometry']) if extractor.simplify: geom = geom.simplify(extractor.simplify) if geom.geom_type == 'Polygon': geom = MultiPolygon([geom]) elif geom.geom_type != 'MultiPolygon': warning('Unsupported geometry type "{0}" for "{1}"', geom.geom_type, zone['name']) continue zone.update(geom=geom.__geo_interface__) zone_id = zone.get('_id') if not zone_id: zone_id = ':'.join((self.id, zone['code'])) if 'validity' in zone and zone['validity'].get('start'): start = zone['validity']['start'] zone_id = '@'.join((zone_id, start)) zone.update(_id=zone_id, level=self.id) db.find_one_and_replace( {'_id': zone_id}, zone, upsert=True) loaded += 1 except Exception: error('Error extracting polygon {0}', polygon['properties']) error(traceback.format_exc()) info('Loaded {0} zones for level {1} from file {2}', loaded, self.id, filename) return loaded
def sr(self): sr = SpatialReference() sr.ImportFromProj4(to_string(self.value)) return sr
def get_crs(layerDict): crsList = [] for name in layerDict: with fiona.open(layerDict[name]['file']) as s: crsList.append(to_string(s.crs)) return from_string(Counter(crsList).most_common()[0][0])
def test_to_string_utm(): # Make a string from a mapping with a few bogus items val = {"proj": "utm", "ellps": "WGS84", "zone": 13, "no_defs": True, "foo": True, "axis": False, "belgium": [1, 2]} assert crs.to_string(val) == "+ellps=WGS84 +no_defs +proj=utm +zone=13"
from pprint import pprint with fiona.open('/gdata/GSHHS_c.shp') as src: pprint(src[1]) ############################################################################### try: with fiona.open('/gdata/GSHHS_c.shp') as c: print(len(list(c))) except: print(c.closed) raise ############################################################################### c = fiona.open('/gdata/GSHHS_c.shp') c.driver ############################################################################### c.crs ############################################################################### from fiona.crs import to_string to_string(c.crs) ############################################################################### from fiona.crs import from_string from_string("+datum=WGS84 +ellps=WGS84 +no_defs +proj=longlat") ############################################################################### from fiona.crs import from_epsg from_epsg(3857) ############################################################################### len(c) ############################################################################### c.bounds
def test_to_string_epsg(): val = {"init": "epsg:4326", "no_defs": True} assert crs.to_string(val) == "+init=epsg:4326 +no_defs"
import fiona shpfile_city = "/Users/xiaomuliu/CrimeProject/SpatioTemporalModeling/GISdata/City_Boundary/City_Boundary.shp" chi_city = fiona.open(shpfile_city) next(chi_city) chi_city.close() chi_city.closed import pprint from fiona.crs import to_string with fiona.open(shpfile_city) as chi_city: pprint.pprint(chi_city[0]) print(chi_city.driver) print(chi_city.crs) print(to_string(chi_city.crs)) print(chi_city.bounds) #GeoPandas extends the datatypes used by pandas to allow spatial operations on geometric types. #Geometric operations are performed by 'shapely'. #Geopandas further depends on 'fiona' for file access and descartes and 'matplotlib' for plotting. import geopandas as gpd chi_city = gpd.read_file(shpfile_city) chi_city.head() chi_city.plot() shpfile_beat = "/Users/xiaomuliu/CrimeProject/SpatioTemporalModeling/GISdata/cpd_beats/cpd_beats.shp" beat = gpd.read_file(shpfile_beat) beat.head() beat.plot()
def sr(self): sr = SpatialReference() sr.ImportFromProj4(to_string(self.value)) return(sr)
def proj_str(self): if self._proj_str is None and self.crs is not None: self._proj_str = to_string(self._crs) elif self._proj_str is None and self.prjfile is not None: self._proj_str = get_proj_str(self.prjfile) return self._proj_str
def calculate_area_length_in_metres(in_filename, dissolve_overlap=True): """Calculate the total Area and Length in metres for an input polygon file. If the input file is geographic, then the feature will be 'projected' into wgs84 utm system Currently there is no Summarize by Column or update existing features .. implemented (see TODO). Args: in_filename (str): Input polygon file. dissolve_overlap (bool): Return values after Dissolve ALL geometries. Returns: list[area,length]: The Total Area and Length """ start_time = time.time() if not os.path.exists(in_filename): raise IOError("Invalid path: {}".format(in_filename)) with fiona.open(in_filename, 'r') as source: inSRS = osr.SpatialReference() inSRS.ImportFromProj4(to_string(source.crs)) if not inSRS.IsProjected(): zone, utmSRS, wgs84SRS = pyprecag_crs.getUTMfromWGS84( *source.bounds[:2]) # Create the Project Transformation object project = functools.partial(pyproj.transform, pyproj.Proj(**source.crs), pyproj.Proj(utmSRS.ExportToProj4())) # Store Results in a dictionary resultsDict = collections.defaultdict(float) # Store projected polygons polygonsGeom = [] # loop through polygons collecting areas and lengths for eaFeat in source: # get the geometry of the feature eaGeom = shape(eaFeat['geometry']) # if this is geographic the it will be dd resultsDict['origArea'] += eaGeom.area resultsDict['origLength'] += eaGeom.length # If inSRS is Geographic the reproject the feature to a projected system to get area in metres if not inSRS.IsProjected(): project = functools.partial( pyproj.transform, pyproj.Proj(**source.crs), pyproj.Proj(utmSRS.ExportToProj4())) # Get the projected geometry eaGeom = shapely.ops.transform(project, eaGeom) # store numbers in metres resultsDict['Area_m'] += eaGeom.area resultsDict['Length_m'] += eaGeom.length # If overlaps are to be dissolved we need a projected geometry collection if dissolve_overlap: polygonsGeom.append(eaGeom) if dissolve_overlap: # Run the dissolve. polygondis = shapely.ops.unary_union(polygonsGeom) resultsDict['Area_m_NoOverlap'] += polygondis.area resultsDict['Length_m_NoOverlap'] += polygondis.length # get the final values area = resultsDict['Area_m'] length = resultsDict['Length_m'] if dissolve_overlap: area = resultsDict['Area_m_NoOverlap'] length = resultsDict['Length_m_NoOverlap'] LOGGER.debug('CRS: {}'.format(to_string(source.crs))) if not inSRS.IsProjected(): LOGGER.debug( '\t{:<25} Area (dd): {:>20.5f} Length (dd): {:>20.5f}'.format( 'No Overlap-', resultsDict['origArea'], resultsDict['origLength'])) LOGGER.debug('Projected To CRS: {}'.format( utmSRS.GetAttrValue("PROJCS", 0))) LOGGER.debug( '\t{:<25} Area (m) : {:>20.5f} Length (m) : {:>20.5f}'.format( 'Total with Overlap-', resultsDict['Area_m'], resultsDict['Length_m'])) if dissolve_overlap: LOGGER.debug( '\t{:<25} Area (m) : {:>20.5f} Length (m) : {:>20.5f}'.format( 'Total with No Overlap-', resultsDict['Area_m_NoOverlap'], resultsDict['Length_m_NoOverlap'])) LOGGER.debug('{} complete !! Duration H:M:SS - {dur}'.format( inspect.currentframe().f_code.co_name, dur=datetime.timedelta(seconds=time.time() - start_time))) return area, length
def test_to_string_epsg(): val = {'init': 'epsg:4326', 'no_defs': True} assert crs.to_string(val) == "+init=epsg:4326 +no_defs"
def clip_raster(inraster, features, outraster, clip_feature_epsg=None, clip_feature_proj4=None, clip_kwargs={}, **project_kwargs): """Clip a raster to feature extent(s). Parameters ---------- clip_kwargs: dict Keyword arguments to rasterio.mask kwargs : key word arguments to project_raster These are only used if the clip features are in a different coordinate system, in which case the raster will be reprojected into that coordinate system. """ rasterio = import_rasterio() from fiona.crs import to_string from shapely.geometry import box # compare coordinates references for raster and clip feature # (if the clip feature is a shapefile path) with rasterio.open(inraster) as src: raster_crs = to_string(src.crs) clip_crs = raster_crs # start with assumption of same coordinates if isinstance(features, str): with fiona.open(features) as src2: clip_crs = to_string(src2.crs) elif isinstance(features, list): if clip_feature_epsg is not None: clip_crs = '+init=epsg:{}'.format(clip_feature_epsg) elif clip_feature_proj4 is not None: clip_crs = clip_feature_proj4 # convert the features to geojson geoms = _to_geojson(features) print('input raster crs:\n{}\n\n'.format(raster_crs), 'clip feature crs:\n{}\n'.format(clip_crs)) # if the coordinate systems are not the same # reproject the raster first before clipping # this could be greatly sped up by first clipping the input raster prior to reprojecting if raster_crs != clip_crs: tmpraster = 'tmp.tif' tmpraster2 = 'tmp2.tif' print('Input raster and clip feature(s) are in different coordinate systems.\n' 'Input raster will be reprojected to the clip feature coordinate system.\n') # make prelim clip of raster to speed up reprojection xmin, xmax, ymin, ymax = _get_bounds(geoms) longest_side = np.max([xmax-xmin, ymax-ymin]) bounds = box(xmin, ymin, xmax, ymax).buffer(longest_side*0.1) bounds = project(bounds, clip_crs, raster_crs) _clip_raster(inraster, [bounds], tmpraster, **clip_kwargs) project_raster(tmpraster, tmpraster2, clip_crs, **project_kwargs) inraster = tmpraster2 _clip_raster(inraster, geoms, outraster, **clip_kwargs) if raster_crs != clip_crs: for tmp in [tmpraster, tmpraster2]: if os.path.exists(tmp): print('removing {}...'.format(tmp)) os.remove(tmp) print('Done.')