def sel_by_loc(shp, boundary_filter, filtered_output): """ Filter a shp using the location of a boundary_filter shp For now the boundary must have only one feature Writes the filter on a new shp """ import os from osgeo import ogr from gasp.gt.prop.ff import drv_name from gasp.gt.prop.feat import get_gtype from gasp.g.lyr.fld import copy_flds from gasp.gt.toshp import copy_feat from gasp.pyt.oss import fprop # Open main data dtSrc = ogr.GetDriverByName(drv_name(shp)).Open(shp, 0) lyr = dtSrc.GetLayer() # Get filter geom filter_shp = ogr.GetDriverByName(drv_name(boundary_filter)).Open( boundary_filter, 0) filter_lyr = filter_shp.GetLayer() c = 0 for f in filter_lyr: if c: break geom = f.GetGeometryRef() c += 1 filter_shp.Destroy() # Apply filter lyr.SetSpatialFilter(geom) # Copy filter objects to a new shape out = ogr.GetDriverByName( drv_name(filtered_output)).CreateDataSource(filtered_output) outLyr = out.CreateLayer(fprop(filtered_output, 'fn'), geom_type=get_gtype(shp, gisApi='ogr', name=None, py_cls=True)) # Copy fields copy_flds(lyr, outLyr) copy_feat(lyr, outLyr, outDefn=outLyr.GetLayerDefn(), only_geom=False, gisApi='ogrlyr')
def apndtbl_in_otherdb(db_a, db_b, tblA, tblB, mapCols, geomCol=None, srsEpsg=None): """ Append data of one table to another table in other database. """ from gasp.sql.fm import q_to_obj if geomCol and srsEpsg: df = q_to_obj(db_a, "SELECT {} FROM {}".format( ", ".join(list(mapCols.keys())), tblA ), db_api='psql', geomCol=geomCol, epsg=srsEpsg) else: df = q_to_obj(db_b, "SELECT {} FROM {}".format( ", ".join(list(mapCols.keys())), tblA ), db_api='psql', geomCol=None, epsg=None) # Change Names df.rename(columns=mapCols, inplace=True) if geomCol: for k in mapCols: if geomCol == k: geomCol = mapCols[k] break # Get Geom Type # Send data to other database if geomCol and srsEpsg: from gasp.gt.prop.feat import get_gtype gType = get_gtype(df, geomCol=geomCol, gisApi='pandas') df_to_db( db_b, df, tblB, append=True, api='psql', epsg=srsEpsg, geomType=gType, colGeom=geomCol ) else: df_to_db(db_b, df, tblB, append=True, api='psql') return tblB
def proj(inShp, outShp, outEPSG, inEPSG=None, gisApi='ogr', sql=None, db_name=None): """ Project Geodata using GIS API's Available: * ogr; * ogr2ogr; * pandas; * ogr2ogr_SQLITE; * psql; """ import os if gisApi == 'ogr': """ Using ogr Python API """ if not inEPSG: raise ValueError( 'To use ogr API, you should specify the EPSG Code of the' ' input data using inEPSG parameter' ) from osgeo import ogr from gasp.g.lyr.fld import copy_flds from gasp.gt.prop.feat import get_gtype from gasp.gt.prop.ff import drv_name from gasp.gt.prop.prj import get_sref_from_epsg, get_trans_param from gasp.pyt.oss import fprop def copyShp(out, outDefn, lyr_in, trans): for f in lyr_in: g = f.GetGeometryRef() g.Transform(trans) new = ogr.Feature(outDefn) new.SetGeometry(g) for i in range(0, outDefn.GetFieldCount()): new.SetField(outDefn.GetFieldDefn(i).GetNameRef(), f.GetField(i)) out.CreateFeature(new) new.Destroy() f.Destroy() # ####### # # Project # # ####### # transP = get_trans_param(inEPSG, outEPSG) inData = ogr.GetDriverByName( drv_name(inShp)).Open(inShp, 0) inLyr = inData.GetLayer() out = ogr.GetDriverByName( drv_name(outShp)).CreateDataSource(outShp) outlyr = out.CreateLayer( fprop(outShp, 'fn'), get_sref_from_epsg(outEPSG), geom_type=get_gtype( inShp, name=None, py_cls=True, gisApi='ogr' ) ) # Copy fields to the output copy_flds(inLyr, outlyr) # Copy/transform features from the input to the output outlyrDefn = outlyr.GetLayerDefn() copyShp(outlyr, outlyrDefn, inLyr, transP) inData.Destroy() out.Destroy() elif gisApi == 'ogr2ogr': """ Transform SRS of any OGR Compilant Data. Save the transformed data in a new file """ if not inEPSG: from gasp.gt.prop.prj import get_epsg_shp inEPSG = get_epsg_shp(inShp) if not inEPSG: raise ValueError('To use ogr2ogr, you must specify inEPSG') from gasp import exec_cmd from gasp.gt.prop.ff import drv_name cmd = ( 'ogr2ogr -f "{}" {} {}{} -s_srs EPSG:{} -t_srs EPSG:{}' ).format( drv_name(outShp), outShp, inShp, '' if not sql else ' -dialect sqlite -sql "{}"'.format(sql), str(inEPSG), str(outEPSG) ) outcmd = exec_cmd(cmd) elif gisApi == 'ogr2ogr_SQLITE': """ Transform SRS of a SQLITE DB table. Save the transformed data in a new table """ from gasp import exec_cmd if not inEPSG: raise ValueError(( 'With ogr2ogr_SQLITE, the definition of inEPSG is ' 'demandatory.' )) # TODO: Verify if database is sqlite db, tbl = inShp['DB'], inShp['TABLE'] sql = 'SELECT * FROM {}'.format(tbl) if not sql else sql outcmd = exec_cmd(( 'ogr2ogr -update -append -f "SQLite" {db} -nln "{nt}" ' '-dialect sqlite -sql "{_sql}" -s_srs EPSG:{inepsg} ' '-t_srs EPSG:{outepsg} {db}' ).format( db=db, nt=outShp, _sql=sql, inepsg=str(inEPSG), outepsg=str(outEPSG) )) elif gisApi == 'pandas': # Test if input Shp is GeoDataframe from gasp.gt.fmshp import shp_to_obj from gasp.gt.toshp import df_to_shp df = shp_to_obj(inShp) # Project df newDf = df.to_crs({'init' : 'epsg:{}'.format(str(outEPSG))}) # Save as file return df_to_shp(df, outShp) elif gisApi == 'psql': from gasp.sql.db import create_db from gasp.pyt.oss import fprop from gasp.gql.to import shp_to_psql from gasp.gt.toshp.db import dbtbl_to_shp from gasp.gql.prj import sql_proj # Create Database if not db_name: db_name = create_db(fprop( outShp, 'fn', forceLower=True), api='psql' ) else: from gasp.sql.i import db_exists isDb = db_exists(db_name) if not isDb: create_db(db_name, api='psql') # Import Data inTbl = shp_to_psql(db_name, inShp, api='shp2pgsql', encoding="LATIN1") # Transform oTbl = sql_proj( db_name, inTbl, fprop(outShp, 'fn', forceLower=True), outEPSG, geomCol='geom', newGeom='geom' ) # Export outShp = dbtbl_to_shp( db_name, oTbl, 'geom', outShp, api='psql', epsg=outEPSG ) else: raise ValueError('Sorry, API {} is not available'.format(gisApi)) return outShp
def eachfeat_to_newshp(inShp, outFolder, epsg=None, idCol=None): """ Export each feature in inShp to a new/single File """ import os; from osgeo import ogr from gasp.gt.prop.ff import drv_name from gasp.gt.prop.feat import get_gtype, lst_fld from gasp.g.lyr.fld import copy_flds from gasp.pyt.oss import fprop inDt = ogr.GetDriverByName( drv_name(inShp)).Open(inShp) lyr = inDt.GetLayer() # Get SRS for the output if not epsg: from gasp.gt.prop.prj import get_shp_sref srs = get_shp_sref(lyr) else: from gasp.gt.prop.prj import get_sref_from_epsg srs = get_sref_from_epsg(epsg) # Get fields name fields = lst_fld(lyr) # Get Geometry type geomCls = get_gtype(inShp, gisApi='ogr', name=None, py_cls=True) # Read features and create a new file for each feature RESULT_SHP = [] for feat in lyr: # Create output ff = fprop(inShp, ['fn', 'ff']) newShp = os.path.join(outFolder, "{}_{}{}".format( ff['filename'], str(feat.GetFID()) if not idCol else str(feat.GetField(idCol)), ff['fileformat'] )) newData = ogr.GetDriverByName( drv_name(newShp)).CreateDataSource(newShp) newLyr = newData.CreateLayer( fprop(newShp, 'fn'), srs, geom_type=geomCls ) # Copy fields from input to output copy_flds(lyr, newLyr) newLyrDefn = newLyr.GetLayerDefn() # Create new feature newFeat = ogr.Feature(newLyrDefn) # Copy geometry geom = feat.GetGeometryRef() newFeat.SetGeometry(geom) # Set fields attributes for fld in fields: newFeat.SetField(fld, feat.GetField(fld)) # Save feature newLyr.CreateFeature(newFeat) newFeat.Destroy() del newLyr newData.Destroy() RESULT_SHP.append(newShp) return RESULT_SHP
def make_dem(grass_workspace, data, field, output, extent_template, method="IDW", cell_size=None, mask=None): """ Create Digital Elevation Model Methods Available: * IDW; * BSPLINE; * SPLINE; * CONTOUR; """ from gasp.pyt.oss import fprop from gasp.gt.wenv.grs import run_grass from gasp.gt.prop.prj import get_epsg LOC_NAME = fprop(data, 'fn', forceLower=True)[:5] + "_loc" # Get EPSG From Raster EPSG = get_epsg(extent_template) if not EPSG: raise ValueError( 'Cannot get EPSG code of Extent Template File ({})'.format( extent_template)) # Know if data geometry are points if method == 'BSPLINE' or method == 'SPLINE': from gasp.gt.prop.feat import get_gtype data_gtype = get_gtype(data, gisApi='ogr') # Create GRASS GIS Location grass_base = run_grass(grass_workspace, location=LOC_NAME, srs=EPSG) # Start GRASS GIS Session import grass.script as grass import grass.script.setup as gsetup gsetup.init(grass_base, grass_workspace, LOC_NAME, 'PERMANENT') # Get Extent Raster ref_template = ob_ref_rst(extent_template, os.path.join(grass_workspace, LOC_NAME), cellsize=cell_size) # IMPORT GRASS GIS MODULES # from gasp.gt.torst import rst_to_grs, grs_to_rst from gasp.gt.toshp.cff import shp_to_grs from gasp.gt.wenv.grs import rst_to_region # Configure region rst_to_grs(ref_template, 'extent') rst_to_region('extent') # Convert elevation "data" to GRASS Vector elv = shp_to_grs(data, 'elevation') OUTPUT_NAME = fprop(output, 'fn', forceLower=True) if method == "BSPLINE": from gasp.gt.nop.itp import bspline # Convert to points if necessary if data_gtype != 'POINT' and data_gtype != 'MULTIPOINT': from gasp.gt.toshp.cgeo import feat_vertex_to_pnt elev_pnt = feat_vertex_to_pnt(elv, "elev_pnt", nodes=None) else: elev_pnt = elv outRst = bspline(elev_pnt, field, OUTPUT_NAME, mway='bicubic', lyrN=1, asCMD=True) elif method == "SPLINE": from gasp.gt.nop.itp import surfrst # Convert to points if necessary if data_gtype != 'POINT' and data_gtype != 'MULTIPOINT': from gasp.gt.toshp.cgeo import feat_vertex_to_pnt elev_pnt = feat_vertex_to_pnt(elv, "elev_pnt", nodes=None) else: elev_pnt = elv outRst = surfrst(elev_pnt, field, OUTPUT_NAME, lyrN=1, ascmd=True) elif method == "CONTOUR": from gasp.gt.torst import shp_to_rst from gasp.gt.nop.itp import surfcontour # Apply mask if mask if mask: from gasp.gt.torst import grs_to_mask, rst_to_grs rst_mask = rst_to_grs(mask, 'rst_mask', as_cmd=True) grs_to_mask(rst_mask) # Elevation (GRASS Vector) to Raster elevRst = shp_to_rst(elv, field, None, None, 'rst_elevation', api="pygrass") # Run Interpolator outRst = surfcontour(elevRst, OUTPUT_NAME, ascmd=True) elif method == "IDW": from gasp.gt.nop.itp import ridw from gasp.gt.nop.alg import rstcalc from gasp.gt.torst import shp_to_rst # Elevation (GRASS Vector) to Raster elevRst = shp_to_rst(elv, field, None, None, 'rst_elevation', api='pygrass') # Multiply cells values by 100 000.0 rstcalc('int(rst_elevation * 100000)', 'rst_elev_int', api='pygrass') # Run IDW to generate the new DEM ridw('rst_elev_int', 'dem_int', numberPoints=15) # DEM to Float rstcalc('dem_int / 100000.0', OUTPUT_NAME, api='pygrass') # Export DEM to a file outside GRASS Workspace grs_to_rst(OUTPUT_NAME, output) return output
def connect_lines_to_near_lines(inLines, nearLines, outLines, tollerance=1000): """ Connect all vertex in a line to the nearest vertex of the nearest line """ import os from osgeo import ogr from gasp.pyt.oss import fprop from gasp.gt.prop.ff import drv_name from gasp.gt.prop.feat import get_gtype from gasp.g.gop.prox import draw_buffer # Check Geometries inLinesGeom = get_gtype(inLines, gisApi='ogr') nearLinesGeom = get_gtype(nearLines, gisApi='ogr') if inLinesGeom != 'LINESTRING' or \ nearLinesGeom != 'LINESTRING': raise ValueError('This method supports only LINESTRINGS') # Open inLines shpLines = ogr.GetDriverByName(drv_name(inLines)).Open(inLines) # Get Layer lyrLines = shpLines.GetLayer() # Open near shpNear = ogr.GetDriverByName(drv_name(nearLines)).Open(nearLines) # Create Output outSrc = ogr.GetDriverByName(drv_name(outLines)).CreateDataSource(outLines) outLyr = outSrc.CreateLayer(fprop(outLines, 'fn'), geom_type=ogr.wkbLineString) lineDefn = outLyr.GetLayerDefn() # For each point in 'inLines', find the near point on the # the 'nearLines' layer nearPoints = {} for feat in lyrLines: FID = feat.GetFID() # Get Geometry geom = feat.GetGeometryRef() # Get points nrPnt = geom.GetPointCount() for p in range(nrPnt): x, y, z = geom.GetPoint(p) pnt = ogr.Geometry(ogr.wkbPoint) pnt.AddPoint(x, y) # Get point buffer bufPnt = draw_buffer(pnt, tollerance) # Apply a spatial filter based on the buffer # to restrict the nearLines Layer lyrNear = shpNear.GetLayer() lyrNear.SetSpatialFilter(bufPnt) # For line in the filtered 'nearLyr' # Find the closest point dist = 0 for __feat in lyrNear: __FID = __feat.GetFID() __geom = __feat.GetGeometryRef() points = __geom.GetPointCount() for _p in range(points): _x, _y, _z = __geom.GetPoint(_p) distance = ((x - _x)**2 + (y - _y)**2)**0.5 if not dist: dist = [distance, _x, _y] else: if distance < dist[0]: dist = [distance, _x, _y] # Write a new line line = ogr.Geometry(ogr.wkbLineString) line.AddPoint(x, y) line.AddPoint(dist[1], dist[2]) new_feature = ogr.Feature(lineDefn) new_feature.SetGeometry(line) outLyr.CreateFeature(new_feature) new_feature.Destroy() del lyrNear outSrc.Destroy() shpPnt.Destroy() shpNear.Destroy() return outLines
def connect_points_to_near_line(inPnt, nearLines, outLines, tollerance=1000, nearLinesWpnt=None): """ Connect all points to the nearest line in the perpendicular """ import os import numpy as np from osgeo import ogr from shapely.geometry import LineString, Point from gasp.gt.prop.ff import drv_name from gasp.gt.prop.feat import get_gtype from gasp.pyt.oss import fprop # Check Geometries inPntGeom = get_gtype(inPnt, gisApi='ogr') nearLinesGeom = get_gtype(nearLines, gisApi='ogr') if inPntGeom != 'POINT' or \ nearLinesGeom != 'LINESTRING': raise ValueError('This method supports only LINESTRINGS') # Open inLines shpPnt = ogr.GetDriverByName(drv_name(inPnt)).Open(inPnt) # Get Layer lyrPnt = shpPnt.GetLayer() # Open near shpNear = ogr.GetDriverByName(drv_name(nearLines)).Open(nearLines) # Create Output outSrc = ogr.GetDriverByName(drv_name(outLines)).CreateDataSource(outLines) outLyr = outSrc.CreateLayer(os.path.splitext( os.path.basename(outLines))[0], geom_type=ogr.wkbLineString) if nearLinesWpnt: newPointsInLines = {} lineDefn = outLyr.GetLayerDefn() # For each point in 'inLines', find the near point on the # the 'nearLines' layer for feat in lyrPnt: FID = feat.GetFID() # Get Geometry pnt = feat.GetGeometryRef() x, y = pnt.GetX(), pnt.GetY() # Get point buffer bufPnt = draw_buffer(pnt, tollerance) # Apply a spatial filter based on the buffer # to restrict the nearLines Layer lyrNear = shpNear.GetLayer() lyrNear.SetSpatialFilter(bufPnt) # For line in the filtered 'nearLyr' # Find point in the perpendicular dist = 0 for __feat in lyrNear: __FID = __feat.GetFID() __geom = __feat.GetGeometryRef() points = __geom.GetPointCount() for _p in range(points - 1): # Get line segment x1, y1, z1 = __geom.GetPoint(_p) x2, y2, z2 = __geom.GetPoint(_p + 1) # Create Shapely Geometries lnh = LineString([(x1, y1), (x2, y2)]) pnt = Point(x, y) # Get distance between point and line # Get near point of the line d = pnt.distance(lnh) npnt = lnh.interpolate(lnh.project(pnt)) if not dist: dist = [d, npnt.x, npnt.y] LINE_FID = __FID else: if d < dist[0]: dist = [d, npnt.x, npnt.y] LINE_FID = __FID # Write a new line line = ogr.Geometry(ogr.wkbLineString) line.AddPoint(x, y) line.AddPoint(dist[1], dist[2]) new_feature = ogr.Feature(lineDefn) new_feature.SetGeometry(line) outLyr.CreateFeature(new_feature) new_feature.Destroy() if nearLinesWpnt: if LINE_FID not in newPointsInLines: newPointsInLines[LINE_FID] = [Point(dist[1], dist[2])] else: newPointsInLines[LINE_FID].append(Point(dist[1], dist[2])) del lyrNear outSrc.Destroy() shpPnt.Destroy() shpNear.Destroy() if nearLinesWpnt: from gasp.g.lyr.fld import copy_flds from shapely.ops import split as lnhSplit shpNear = ogr.GetDriverByName(drv_name(nearLines)).Open(nearLines) updateLines = ogr.GetDriverByName( drv_name(nearLinesWpnt)).CreateDataSource(nearLinesWpnt) upLnhLyr = updateLines.CreateLayer(fprop(nearLinesWpnt, 'fn'), geom_type=ogr.wkbLineString) # Create shpNear Layer Again lyrNear = shpNear.GetLayer() # Copy fields copy_flds(lyrNear, upLnhLyr) # Out lyr definition upDefn = upLnhLyr.GetLayerDefn() for feat in lyrNear: LINE_FID = feat.GetFID() geom = feat.GetGeometryRef() new_feature = ogr.Feature(upDefn) if LINE_FID not in newPointsInLines: # Copy line to updateLines layer new_feature.SetGeometry(geom) else: # Copy to Shapely Line String points = geom.GetPointCount() lstPnt = [] for _p in range(points): x1, y1, z1 = geom.GetPoint(_p) lstPnt.append((x1, y1)) shplyLnh = LineString(lstPnt) # For new point: # Line split and reconstruction for pnt in newPointsInLines[LINE_FID]: try: splitted = lnhSplit(shplyLnh, pnt) except: shpTstL = ogr.GetDriverByName( "ESRI Shapefile").CreateDataSource( r'D:\gis\xyz\lnht.shp') shpL = shpTstL.CreateLayer('lnht', geom_type=ogr.wkbLineString) shpTstP = ogr.GetDriverByName( "ESRI Shapefile").CreateDataSource( r'D:\gis\xyz\pntt.shp') shpP = shpTstL.CreateLayer('pntt', geom_type=ogr.wkbPoint) defnL = shpL.GetLayerDefn() defnP = shpP.GetLayerDefn() featL = ogr.Feature(defnL) featP = ogr.Feature(defnP) geomL = ogr.Geometry(ogr.wkbLineString) for i in list(shplyLnh.coords): geomL.AddPoint(i[0], i[1]) geomP = ogr.Geometry(ogr.wkbPoint) geomP.AddPoint( list(pnt.coords)[0][0], list(pnt.coords)[0][1]) featL.SetGeometry(geomL) featP.SetGeometry(geomP) shpL.CreateFeature(featL) shpP.CreateFeature(featP) shpTstL.Destroy() shpTstP.Destroy() return pnt, shplyLnh c = 0 for l in splitted: if not c: newLnh = list(l.coords) else: newlnh += list(l.coords)[1:] c += 1 shplyLnh = LineString(newLnh) # Finally copy line to updateLines Layer gLine = ogr.Geometry(ogr.wkbLineString) for __pnt in list(shplyLnh.coords): gLine.AddPoint(__pnt[0], __pnt[1]) for i in range(0, upDefn.GetFieldCount()): new_feature.SetField( upDefn.GetFieldDefn(i).GetNameRef(), feat.GetField(i)) upLnhLyr.CreateFeature(new_feature) new_feature.Destroy() shpNear.Destroy() return outLines
def shp_to_psql(dbname, shpData, pgTable=None, api="pandas", mapCols=None, srsEpsgCode=None, encoding="UTF-8", dbset='default'): """ Send Shapefile to PostgreSQL if api is equal to "pandas" - GeoPandas API will be used; if api is equal to "shp2pgsql" - shp2pgsql tool will be used. """ import os from gasp.pyt.oss import fprop from gasp.gt.prop.prj import get_epsg_shp # If defined, srsEpsgCode must be a integer value if srsEpsgCode: if type(srsEpsgCode) != int: raise ValueError('srsEpsgCode should be a integer value') if api == "pandas": from gasp.fm import tbl_to_obj from gasp.sql.to import df_to_db from gasp.gt.prop.feat import get_gtype elif api == "shp2pgsql": from gasp import exec_cmd from gasp.sql import psql_cmd from gasp.pyt.oss import del_file else: raise ValueError( 'api value is not valid. options are: pandas and shp2pgsql') # Check if shp is folder if os.path.isdir(shpData): from gasp.pyt.oss import lst_ff shapes = lst_ff(shpData, file_format='.shp') else: from gasp.pyt import obj_to_lst shapes = obj_to_lst(shpData) epsgs = [get_epsg_shp(i) for i in shapes] if not srsEpsgCode else [srsEpsgCode] if None in epsgs: raise ValueError( ("Cannot obtain EPSG code. Use the srsEpsgCode parameter " "to specify the EPSG code of your data.")) tables = [] for _i in range(len(shapes)): # Get Table name tname = fprop(shapes[_i], 'fn', forceLower=True) if not pgTable else \ pgTable[_i] if type(pgTable) == list else pgTable if len(shapes) == 1 \ else pgTable + '_{}'.format(_i+1) # Import data if api == "pandas": # SHP to DataFrame df = tbl_to_obj(shapes[_i]) if not mapCols: df.rename(columns={x: x.lower() for x in df.columns.values}, inplace=True) else: renameD = { x : mapCols[x].lower() if x in mapCols else \ x.lower() for x in df.columns.values } df.rename(columns=renameD, inplace=True) if "geometry" in df.columns.values: geomCol = "geometry" elif "geom" in df.columns.values: geomCol = "geom" else: raise ValueError("No Geometry found in shp") # GeoDataFrame to PSQL df_to_db(dbname, df, tname, append=True, api='psql', epsg=epsgs[_i] if not srsEpsgCode else srsEpsgCode, colGeom=geomCol, geomType=get_gtype(shapes[_i], name=True, py_cls=False, gisApi='ogr')) else: sql_script = os.path.join(os.path.dirname(shapes[_i]), tname + '.sql') cmd = ('shp2pgsql -I -s {epsg} -W {enc} ' '{shp} public.{name} > {out}').format( epsg=epsgs[_i] if not srsEpsgCode else srsEpsgCode, shp=shapes[_i], name=tname, out=sql_script, enc=encoding) outcmd = exec_cmd(cmd) psql_cmd(dbname, sql_script, dbcon=dbset) del_file(sql_script) tables.append(tname) return tables[0] if len(tables) == 1 else tables