def orig_dest_to_polyline(srcPoints, srcField, destPoints, destField, outShp): """ Connect origins to destinations with a polyline which length is the minimum distance between the origin related with a specific destination. One origin should be related with one destination. These relations should be expressed in srcField and destField """ from geopandas import GeoDataFrame from shapely.geometry import LineString from gasp.gt.fmshp import shp_to_obj from gasp.gt.toshp import df_to_shp srcPnt = shp_to_obj(srcPoints) desPnt = shp_to_obj(destPoints) joinDf = srcPnt.merge(desPnt, how='inner', left_on=srcField, right_on=destField) joinDf["geometry"] = joinDf.apply( lambda x: LineString(x["geometry_x"], x["geometry_y"]), axis=1) joinDf.drop(["geometry_x", "geometry_y"], axis=1, inplace=True) a = GeoDataFrame(joinDf) df_to_shp(joinDf, outShp) return outShp
def tweets_to_shp(buffer_shp, epsg_in, outshp, keyword=None, epsg_out=4326, __encoding__='plain_str', keyAPI=None): """ Search data in Twitter and create a vectorial file with that data """ from gasp.gt.toshp import df_to_shp tweets = geotweets_location(buffer_shp, epsg_in, keyword=keyword, epsg_out=epsg_out, keyToUse=keyAPI, onlySearchAreaContained=None) try: if not tweets: return 0 except: pass df_to_shp(tweets, outshp) return outshp
def photos_to_shp(buffer_shp, epsg_in, outshp, keyword=None, epsg_out=4326, apikey=None, onlyInsideInput=True): """ Search for data in Flickr and return a Shapefile with the data. """ from gasp.gt.toshp import df_to_shp photos = photos_location(buffer_shp, epsg_in, keyword=keyword, epsg_out=epsg_out, keyToUse=apikey, onlySearchAreaContained=onlyInsideInput) try: if not photos: return 0 except: pass df_to_shp(photos, outshp) return outshp
def intersection(inShp, intersectShp, outShp, api='geopandas'): """ Intersection between ESRI Shapefile 'API's Available: * geopandas * saga; * pygrass; * grass; """ if api == 'geopandas': import geopandas from gasp.gt.fmshp import shp_to_obj from gasp.gt.toshp import df_to_shp dfShp = shp_to_obj(inShp) dfIntersect = shp_to_obj(intersectShp) res_interse = geopandas.overlay(dfShp, dfIntersect, how='intersection') df_to_shp(res_interse, outShp) elif api == 'saga': from gasp import exec_cmd cmdout = exec_cmd( ("saga_cmd shapes_polygons 14 -A {} -B {} -RESULT {} -SPLIT 1" ).format(inShp, intersectShp, outShp)) elif api == 'pygrass': from grass.pygrass.modules import Module clip = Module("v.overlay", ainput=inShp, atype="area", binput=intersectShp, btype="area", operator="and", output=outShp, overwrite=True, run_=False, quiet=True) clip() elif api == 'grass': from gasp import exec_cmd rcmd = exec_cmd( ("v.overlay ainput={} atype=area, binput={} btype=area " "operator=and output={} --overwrite --quiet").format( inShp, intersectShp, outShp)) else: raise ValueError("{} is not available!".format(api)) return outShp
def join_bgrishp_with_bgridata(bgriShp, bgriCsv, outShp, shpJoinField="BGRI11", dataJoinField="GEO_COD", joinFieldsMantain=None, newNames=None): """ Join BGRI ESRI Shapefile with the CSV with the BGRI Data """ from gasp.pyt import obj_to_lst from gasp.fm import tbl_to_obj from gasp.gt.fmshp import shp_to_obj from gasp.gt.toshp import df_to_shp # Read main_table mainDf = shp_to_obj(bgriShp) # Read join table joinDf = tbl_to_obj(bgriCsv, _delimiter=';', encoding_='utf-8') # Sanitize GEO_COD of bgriCsv joinDf[dataJoinField] = joinDf[dataJoinField].str.replace("'", "") if joinFieldsMantain: joinFieldsMantain = obj_to_lst(joinFieldsMantain) dropCols = [] for col in joinDf.columns.values: if col not in [dataJoinField] + joinFieldsMantain: dropCols.append(col) joinDf.drop(dropCols, axis=1, inplace=True) resultDf = mainDf.merge(joinDf, how='inner', left_on=shpJoinField, right_on=dataJoinField) if newNames: newNames = obj_to_lst(newNames) renDict = { joinFieldsMantain[n]: newNames[n] for n in range(len(joinFieldsMantain)) } resultDf.rename(columns=renDict, inplace=True) df_to_shp(resultDf, outShp) return outShp
def split_shp_by_attr(inShp, attr, outDir, _format='.shp'): """ Create a new shapefile for each value in a column """ import os from gasp.gt.fmshp import shp_to_obj from gasp.pyt.oss import fprop from gasp.pyt.df.fld import col_distinct from gasp.gt.toshp import df_to_shp # Sanitize format FFF = _format if _format[0] == '.' else '.' + _format # SHP TO DF dataDf = shp_to_obj(inShp) # Get values in attr uniqueAttr = col_distinct(dataDf, attr) # Export Features with the same value in attr to a new File BASENAME = fprop(inShp, 'fn', forceLower=True) SHPS_RESULT = {} i = 1 for val in uniqueAttr: df = dataDf[dataDf[attr] == val] newShp = df_to_shp( df, os.path.join(outDir, "{}_{}{}".format(BASENAME, str(i), FFF))) SHPS_RESULT[val] = newShp i += 1 return SHPS_RESULT
def pntDf_to_convex_hull(pntDf, xCol, yCol, epsg, outEpsg=None, outShp=None): """ Create a GeoDataFrame with a Convex Hull Polygon from a DataFrame with points in two columns, one with the X Values, other with the Y Values """ from scipy.spatial import ConvexHull from shapely import geometry from geopandas import GeoDataFrame hull = ConvexHull(pntDf[[xCol, yCol]]) poly = geometry.Polygon([[pntDf[xCol].iloc[idx], pntDf[yCol].iloc[idx]] for idx in hull.vertices]) convexDf = GeoDataFrame([1], columns=['cat'], crs={'init': 'epsg:' + str(epsg)}, geometry=[poly]) if outEpsg and outEpsg != epsg: from gasp.g.prj import df_prj convexDf = df_prj(convexDf, outEpsg) if outShp: from gasp.gt.toshp import df_to_shp return df_to_shp(convexDf, outShp) return convexDf
def geodf_buffer_to_shp(geoDf, dist, outfile, colgeom='geometry'): """ Execute the Buffer Function of GeoPandas and export the result to a new shp """ from gasp.gt.toshp import df_to_shp __geoDf = geoDf.copy() __geoDf["buffer_geom"] = __geoDf[colgeom].buffer(dist, resolution=16) __geoDf.drop(colgeom, axis=1, inplace=True) __geoDf.rename(columns={"buffer_geom": colgeom}, inplace=True) df_to_shp(__geoDf, outfile) return outfile
def pointXls_to_shp(xlsFile, outShp, x_col, y_col, epsg, sheet=None): """ Excel table with Point information to ESRI Shapefile """ from gasp.fm import tbl_to_obj from gasp.g.to import pnt_dfwxy_to_geodf from gasp.gt.toshp import df_to_shp # XLS TO PANDAS DATAFRAME dataDf = tbl_to_obj(xlsFile, sheet=sheet) # DATAFRAME TO GEO DATAFRAME geoDataDf = pnt_dfwxy_to_geodf(dataDf, x_col, y_col, epsg) # GEODATAFRAME TO ESRI SHAPEFILE df_to_shp(geoDataDf, outShp) return outShp
def rst_to_pnt(in_rst, out_pnt): """ Raster to Point Feature Class """ from gasp.gt.toshp import df_to_shp api = 'pandas' if api == 'pandas': from gasp.g.fmrst import rst_to_geodf gdf = rst_to_geodf(in_rst) df_to_shp(gdf, out_pnt) else: raise ValueError('API {} is not available'.format(api)) return out_pnt
def buffer_ext(inShp, meterTolerance, outShp, inEpsg=None): """ For all geometries, calculate the boundary given by the sum between the feature extent and the Tolerance variable """ from gasp.fm import tbl_to_obj from gasp.gt.toshp import df_to_shp from gasp.g.gop.prox import df_buffer_extent from gasp.gt.prop.prj import get_epsg_shp inDf = tbl_to_obj(inShp) epsg = get_epsg_shp(inShp) if not inEpsg else inEpsg result = df_buffer_extent(inDf, epsg, meterTolerance) return df_to_shp(result, outShp)
def same_attr_to_shp(inShps, interestCol, outFolder, basename="data_", resultDict=None): """ For several SHPS with the same field, this program will list all values in such field and will create a new shp for all values with the respective geometry regardeless the origin shp. """ import os from gasp.pyt import obj_to_lst from gasp.gt.fmshp import shp_to_obj from gasp.pyt.df.to import merge_df from gasp.gt.toshp import df_to_shp EXT = os.path.splitext(inShps[0])[1] shpDfs = [shp_to_obj(shp) for shp in inShps] DF = merge_df(shpDfs, ignIndex=True) #DF.dropna(axis=0, how='any', inplace=True) uniqueVal = DF[interestCol].unique() nShps = [] if not resultDict else {} for val in uniqueVal: ndf = DF[DF[interestCol] == val] KEY = str(val).split('.')[0] if '.' in str(val) else str(val) nshp = df_to_shp( ndf, os.path.join(outFolder, '{}{}{}'.format(basename, KEY, EXT))) if not resultDict: nShps.append(nshp) else: nShps[KEY] = nshp return nShps
def shply_break_lines_on_points(lineShp, pointShp, lineIdInPntShp, splitedShp): """ Break lines on points location The points should be contained by the lines; The points table should have a column with the id of the line that contains the point. lineIDInPntShp is a reference to the FID of lineShp """ from shapely.ops import split from shapely.geometry import Point, LineString from gasp.gt.fmshp import shp_to_obj from gasp.pyt.df.mng import col_list_val_to_row from gasp.gt.prop.prj import get_epsg_shp from gasp.gt.toshp import df_to_shp from gasp.pyt.df.to import dict_to_df srs_code = get_epsg_shp(lineShp) # Sanitize line geometry def fix_line(line, point): buff = point.buffer(0.0001) splitLine = split(line, buff) nline = LineString( list(splitLine[0].coords) + list(point.coords) + list(splitLine[-1].coords) ) return nline pnts = shp_to_obj(shp_to_obj) lines = shp_to_obj(shp_to_obj, output='dict') # Split Rows def split_geom(row): # Get related line rel_line = lines[row[lineIdInPntShp]] if type(rel_line["GEOM"]) != list: line_geom = fix_line(rel_line["GEOM"], row.geometry) split_lines = split(line_geom, row.geometry) lines[row[lineIdInPntShp]]["GEOM"] = [l for l in split_lines] else: for i in range(len(rel_line["GEOM"])): if rel_line["GEOM"][i].distance(row.geometry) < 1e-8: line_geom = fix_line(rel_line["GEOM"][i], row.geometry) split_lines = split(line_geom, row.geometry) split_lines = [l for l in split_lines] lines[row[lineIdInPntShp]]["GEOM"][i] = split_lines[0] lines[row[lineIdInPntShp]]["GEOM"] += split_lines[1:] break else: continue return row pnts = pnts.apply(lambda x: split_geom(x), axis=1) # Result to Dataframe linesDf = dict_to_df(lines) # Where GEOM is a List, create a new row for each element in list linesDf = col_list_val_to_row( linesDf, "GEOM", geomCol="GEOM", epsg=srs_code ) # Save result return df_to_shp(linesDf, splitedShp)
def dissolve(inShp, outShp, fld, statistics=None, geomMultiPart=True, api='ogr', inputIsLines=None): """ Dissolve Geometries API's Available: * qgis; * saga; * ogr; * pygrass; * grass; """ if api == 'qgis': import processing processing.runalg("qgis:dissolve", inShp, False, fld, outShp) elif api == 'saga': """ Dissolve vectorial data by field This algorithm doesn't allow self intersections """ from gasp import exec_cmd if not inputIsLines: cmd = ( 'saga_cmd shapes_polygons 5 -POLYGONS {in_poly} -FIELDS {fld} ' '-DISSOLVED {out_shp}' ).format( in_poly=inShp, fld=fld, out_shp=outShp ) else: cmd = ( 'saga_cmd shapes_lines 5 -LINES {} -FIELD_1 {} -DISSOLVED {} ' '-ALL 0' ).format(inShp, fld, outShp) outcmd = exec_cmd(cmd) elif api == 'ogr': """ Dissolve with OGR and sqlite sql field_statitics used to preserve numeric fields aggregating their values using some statistics field_statistics = {fld_name: SUM, fld_name: AVG} TODO: DISSOLVE WITHOUT FIELD """ import os; from gasp import exec_cmd from gasp.pyt.oss import fprop if not statistics: cmd = ( 'ogr2ogr {o} {i} -dialect sqlite -sql ' '"SELECT ST_Union(geometry), {f} ' 'FROM {t} GROUP BY {f};"' ).format(o=outShp, i=inShp, f=fld, t=fprop(shp, 'fn')) else: cmd = ( 'ogr2ogr {o} {i} -dialect sqlite -sql ' '"SELECT ST_Union(geometry), {f}, {stat} ' 'FROM {t} GROUP BY {f};"' ).format( o=outShp, i=inShp, f=fld, t=fprop(shp, 'fn'), stat=','.join([ '{s}({f}) AS {f}'.format( f=str(fld), s=statistics[fld] ) for fld in statistics] ) ) # Execute command outcmd = exec_cmd(cmd) elif api == 'pygrass': from grass.pygrass.modules import Module diss = Module( "v.dissolve", input=inShp, column=fld, output=outShp, overwrite=True, run_=False, quiet=True ) diss() elif api == 'grass': from gasp import exec_cmd outCmd = exec_cmd(( "v.dissolve input={}{} output={} " "--overwrite --quiet" ).format(inShp, " column={}".format(fld) if fld else "", outShp)) elif api == 'pandas': from gasp.gt.fmshp import shp_to_obj from gasp.gt.toshp import df_to_shp from gasp.g.gop.genze import df_dissolve gdf = shp_to_obj(inShp) ndf = df_dissolve(gdf, fld) outshp = df_to_shp(ndf, outShp) else: raise ValueError('The api {} is not available'.format(api)) return outShp
def lst_prod_by_cell_and_year(shp, id_col, year, outshp, platform="Sentinel-2", processingl='Level-2A', epsg=32629): """ Get a list of images: * one for each grid in shp; * one for each month in one year - the choosen image will be the one with lesser area occupied by clouds; total_images = grid_number * number_months_year """ from gasp.gt.fmshp import shp_to_obj from gasp.pyt.df.to import merge_df from gasp.gt.toshp import df_to_shp from gasp.g.to import df_to_geodf months = { '01': '31', '02': '28', '03': '31', '04': '30', '05': '31', '06': '30', '07': '31', '08': '31', '09': '30', '10': '31', '11': '30', '12': '31' } # Open SHP grid = shp_to_obj(shp, srs_to=4326) def get_grid_id(row): row['cellid'] = row.title.split('_')[5][1:] return row # Search for images dfs = [] for idx, cell in grid.iterrows(): for k in months: start = "{}{}01".format(str(year), k) end = "{}{}{}".format(str(year), k, months[k]) if year == 2018 and processingl == 'Level-2A': if k == '01' or k == '02': plevel = 'Level-2Ap' else: plevel = processingl else: plevel = processingl prod = lst_prod(cell.geometry.wkt, start, end, platname=platform, procLevel=plevel) if not prod.shape[0]: continue # Get area prod = prod.to_crs({'init': 'epsg:{}'.format(str(epsg))}) prod['areav'] = prod.geometry.area / 1000000 # We want only images with more than 70% of data prod = prod[prod.areav >= 7000] # ID Cell ID prod = prod.apply(lambda x: get_grid_id(x), axis=1) # Filter Cell ID prod = prod[prod.cellid == cell[id_col]] # Sort by cloud cover and date prod = prod.sort_values(['cloudcoverpercentage', 'ingestiondate'], ascending=[True, True]) # Get only the image with less cloud cover prod = prod.head(1) dfs.append(prod) fdf = merge_df(dfs) fdf = df_to_geodf(fdf, 'geometry', epsg) df_to_shp(fdf, outshp) return outshp
def lst_prod(shpExtent, start_time, end_time, outShp=None, platname="Sentinel-2", procLevel="Level-2A", max_cloud_cover=None): """ List Sentinel Products for one specific area platformname: * Sentinel-1 * Sentinel-2 * Sentinel-3 processinglevel: * Level-1A * Level-1B * Level-1C * Level-2A ... """ import os from sentinelsat import SentinelAPI, read_geojson, geojson_to_wkt from datetime import date from gasp.gt.fmshp import shp_to_obj from gasp.pyt.oss import fprop from gasp.gt.toshp import df_to_shp from gasp.cons.sentinel import con_datahub # Get Search Area if os.path.isfile(shpExtent): if fprop(shpExtent, 'ff') == '.json': boundary = geojson_to_wkt(shpExtent) else: boundary = shp_to_obj(shpExtent, output='array', fields=None, geom_as_wkt=True, srs_to=4326)[0]["GEOM"] else: # Assuming we have an WKT boundary = shpExtent # Create API instance user, password = con_datahub() api = SentinelAPI(user, password, URL_COPERNICUS) # Search for products products = api.query( boundary, date=(start_time, end_time), platformname=platname, cloudcoverpercentage=(0, 100 if not max_cloud_cover else max_cloud_cover), processinglevel=procLevel) df_prod = api.to_geodataframe(products) if not df_prod.shape[0]: return df_prod df_prod['ingestiondate'] = df_prod.ingestiondate.astype(str) df_prod['beginposition'] = df_prod.beginposition.astype(str) df_prod['endposition'] = df_prod.endposition.astype(str) # Export results to Shapefile if outShp: return df_to_shp(df_prod, outShp) else: return df_prod
def osm_countries(out_shp): """ Get the boundary representing areas with OSM data in every OSM PBF country file in GeoFabrik """ import os import codecs import pandas as pd import geopandas as gpd from shapely.geometry import MultiPolygon, Polygon from gasp.cons.osm import osm_files from gasp.to.web import get_file from gasp.gt.toshp import df_to_shp from gasp.pyt.oss import del_file url = 'http://download.geofabrik.de/{}/{}.poly' url_russia = 'http://download.geofabrik.de/russia.poly' countries_boundaries = [] for c in osm_files: for _c in osm_files[c]: # Get poly file ff = get_file( url.format(c, _c) if c != 'russia' else url_russia, os.path.join( os.path.dirname(out_shp), "{}_{}.poly".format(c, _c.replace('/', '_')) if c != 'russia' else "russia.poly")) # Get Polygon Coordinates in_ring = False coords = [] with open(ff, 'r') as txt: i = 0 for l in txt.readlines(): if i == 0: # first line is junk i += 1 continue elif i == 1: # second line is the first polygon ring. coords.append([[], []]) ring = coords[-1][0] in_ring = True i += 1 elif in_ring and l.strip() == 'END': # we are at the end of a ring, perhaps with more to come. in_ring = False elif in_ring: # we are in a ring and picking up new coordinates. pnt = l.strip().split(' ') ring.append((float(pnt[0]), float(pnt[1]))) elif not in_ring and l.strip() == 'END': # we are at the end of the whole polygon. break elif not in_ring and l.startswith('!'): # we are at the start of a polygon part hole. coords[-1][1].append([]) ring = coords[-1][1][-1] in_ring = True elif not in_ring: # we are at the start of a polygon part. coords.append([[], []]) ring = coords[-1][0] in_ring = True polygon = MultiPolygon(coords) countries_boundaries.append([c, _c, polygon]) del_file(ff) countries_boundaries = gpd.GeoDataFrame(pd.DataFrame( countries_boundaries, columns=['continent', 'country', 'geometry']), crs={'init': 'epsg:4326'}, geometry='geometry') return df_to_shp(countries_boundaries, out_shp)
def count_rows_by_entity_and_shpJoin(dbn, PG_TABLE, PG_ENTITY, PG_PIVOT_COL, SHP_TABLE, SHP_ENTITY, RESULT_SHP, WHERE=None): """ Select and GROUP BY attrs generating a table as: ENTITY | ATTR_N | COUNT 1 | 12ECIR | X 2 | 12ECIR | X 1 | 15ECIR | X 2 | 15ECIR | X Then convert this table to the following ENTITY | 12ECIR | 15ECIR 1 | X | X 2 | X | X The last table will be joined with a given Shapefile TODO: See if PGSQL crosstab works to solve this problem """ from gasp.fm import tbl_to_obj from gasp.sql.fm import q_to_obj from gasp.pyt.df.to import series_to_list from gasp.gt.toshp import df_to_shp from gasp.pyt.df.joins import combine_dfs from gasp.sql.to import q_to_ntbl from gasp.sql.tbl import del_tables # Get GROUP BYed data # Get row counting using GROUPBY with ENTITY AND PIVOT_COL q = ("SELECT {entity}, {pivc}, COUNT({entity}) AS n{entity} " "FROM {tbl} {whr}GROUP BY {entity}, {pivc}").format( entity=PG_ENTITY, tbl=PG_TABLE, pivc=PG_PIVOT_COL, whr="" if not WHERE else "WHERE {} ".format(WHERE)) selData = q_to_ntbl(dbn, "seldata", q, api='psql') # Get columns of the output table pivotCols = q_to_obj(dbn, "SELECT {piv} FROM {tb} GROUP BY {piv}".format( tb=selData, piv=PG_PIVOT_COL), db_api='psql') pivotCols = series_to_list(pivotCols[PG_PIVOT_COL]) # Get data for each new column - new column data in one dataframe pre_pivot = [ q_to_obj( dbn, "SELECT {entity}, n{entity} FROM {t} WHERE {c}='{pivcol}'".format( entity=PG_ENTITY, t=selData, c=PG_PIVOT_COL, pivcol=col), db_api='psql') for col in pivotCols ] # In pre_pivot DF, give the correct name to the n{entity} column for i in range(len(pre_pivot)): pre_pivot[i].rename(columns={"n{}".format(PG_ENTITY): pivotCols[i]}, inplace=True) # Join all dataframes into one pivot_df = pre_pivot[0] pivot_df = combine_dfs(pivot_df, pre_pivot[1:], PG_ENTITY) # Join pivot_df to the Given ESRI Shapefile shpDf = tbl_to_obj(SHP_TABLE) shpDf = shpDf.merge(pivot_df, how='outer', left_on=SHP_ENTITY, right_on=PG_ENTITY) df_to_shp(shpDf, RESULT_SHP) del_tables(dbn, selData) return RESULT_SHP
df = df[~df.b_refid.isnull()] if fn == 'ovl_union': df['areav'] = df.geometry.area df = pd.DataFrame({ 'areav': df.groupby(['a_FID'])['areav'].agg('sum') }).reset_index() fish_df = fish_df.merge(df, how='left', left_on='fid', right_on='a_FID') if fn != 'ovl_union': fish_df[fn] = fish_df.areav * 100 / fish_df.area else: fish_df['overlay'] = fish_df.areav * 100 / fish_df.area fish_df.drop(['areav', 'a_FID'], axis=1, inplace=True) # Save file df_to_shp(fish_df, os.path.join(results, os.path.basename(fishp))) # Write List of Fishnet from gasp.to import obj_to_tbl obj_to_tbl(df_fnet, os.path.join(results, 'fishnet_list.xlsx'))
def shps_to_shp(shps, outShp, api="ogr2ogr", fformat='.shp', dbname=None): """ Get all features in several Shapefiles and save them in one file api options: * ogr2ogr; * psql; * pandas; * psql; """ import os if type(shps) != list: # Check if is dir if os.path.isdir(shps): from gasp.pyt.oss import lst_ff # List shps in dir shps = lst_ff(shps, file_format=fformat) else: raise ValueError(( 'shps should be a list with paths for Feature Classes or a path to ' 'folder with Feature Classes')) if api == "ogr2ogr": from gasp import exec_cmd from gasp.gt.prop.ff import drv_name out_drv = drv_name(outShp) # Create output and copy some features of one layer (first in shps) cmdout = exec_cmd('ogr2ogr -f "{}" {} {}'.format( out_drv, outShp, shps[0])) # Append remaining layers lcmd = [ exec_cmd('ogr2ogr -f "{}" -update -append {} {}'.format( out_drv, outShp, shps[i])) for i in range(1, len(shps)) ] elif api == 'pandas': """ Merge SHP using pandas """ from gasp.gt.fmshp import shp_to_obj from gasp.gt.toshp import df_to_shp if type(shps) != list: raise ValueError( 'shps should be a list with paths for Feature Classes') dfs = [shp_to_obj(shp) for shp in shps] result = dfs[0] for df in dfs[1:]: result = result.append(df, ignore_index=True, sort=True) df_to_shp(result, outShp) elif api == 'psql': import os from gasp.sql.tbl import tbls_to_tbl, del_tables from gasp.gql.to import shp_to_psql if not dbname: from gasp.sql.db import create_db create_db(dbname, api='psql') pg_tbls = shp_to_psql(dbname, shps, api="shp2pgsql") if os.path.isfile(outShp): from gasp.pyt.oss import fprop outbl = fprop(outShp, 'fn') else: outbl = outShp tbls_to_tbl(dbname, pg_tbls, outbl) if outbl != outShp: from gasp.gt.toshp.db import dbtbl_to_shp dbtbl_to_shp(dbname, outbl, 'geom', outShp, inDB='psql', api="pgsql2shp") del_tables(dbname, pg_tbls) elif api == 'grass': from gasp import exec_cmd rcmd = exec_cmd( ("v.patch input={} output={} --overwrite --quiet").format( ",".join(shps), outShp)) else: raise ValueError("{} API is not available") return outShp
def dbtbl_to_shp(db, tbl, geom_col, outShp, where=None, inDB='psql', notTable=None, filterByReg=None, outShpIsGRASS=None, tableIsQuery=None, api='psql', epsg=None): """ Database Table to Feature Class file idDB Options: * psql * sqlite api Options: * psql * sqlite * pgsql2shp if outShpIsGRASS if true, the method assumes that outShp is a GRASS Vector. That implies that a GRASS Session was been started already. """ from gasp.gt.toshp import df_to_shp if outShpIsGRASS: from gasp import exec_cmd from gasp.cons.psql import con_psql db_con = con_psql() whr = "" if not where else " where=\"{}\"".format(where) cmd_str = ( "v.in.ogr input=\"PG:host={} dbname={} user={} password={} " "port={}\" output={} layer={} geometry={}{}{}{} -o --overwrite --quiet" ).format( db_con["HOST"], db, db_con["USER"], db_con["PASSWORD"], db_con["PORT"], outShp, tbl, geom_col, whr, " -t" if notTable else "", " -r" if filterByReg else "" ) if inDB == 'psql' else ( "v.in.ogr -o input={} layer={} output={}{}{}{}" ).format(db, tbl, outShp, whr, " -t" if notTable else "", " -r" if filterByReg else "" ) if inDB == 'sqlite' else None rcmd = exec_cmd(cmd_str) else: if api == 'pgsql2shp': from gasp import exec_cmd from gasp.cons.psql import con_psql db_con = con_psql() outcmd = exec_cmd(( 'pgsql2shp -f {out} -h {hst} -u {usr} -p {pt} -P {pas}{geom} ' '{bd} {t}' ).format( hst=db_con['HOST'], usr=db_con["USER"], pt=db_con["PORT"], pas=db_con['PASSWORD'], bd=db, out=outShp, t=tbl if not tableIsQuery else '"{}"'.format(tbl), geom="" if not geom_col else " -g {}".format(geom_col) )) elif api == 'psql' or api == 'sqlite': from gasp.sql.fm import q_to_obj q = "SELECT * FROM {}".format(tbl) if not tableIsQuery else tbl df = q_to_obj(db, q, db_api=api, geomCol=geom_col, epsg=epsg) outsh = df_to_shp(df, outShp) else: raise ValueError(( 'api value must be \'psql\', \'sqlite\' or \'pgsql2shp\'')) return outShp
def proj(inShp, outShp, outEPSG, inEPSG=None, gisApi='ogr', sql=None, db_name=None): """ Project Geodata using GIS API's Available: * ogr; * ogr2ogr; * pandas; * ogr2ogr_SQLITE; * psql; """ import os if gisApi == 'ogr': """ Using ogr Python API """ if not inEPSG: raise ValueError( 'To use ogr API, you should specify the EPSG Code of the' ' input data using inEPSG parameter' ) from osgeo import ogr from gasp.g.lyr.fld import copy_flds from gasp.gt.prop.feat import get_gtype from gasp.gt.prop.ff import drv_name from gasp.gt.prop.prj import get_sref_from_epsg, get_trans_param from gasp.pyt.oss import fprop def copyShp(out, outDefn, lyr_in, trans): for f in lyr_in: g = f.GetGeometryRef() g.Transform(trans) new = ogr.Feature(outDefn) new.SetGeometry(g) for i in range(0, outDefn.GetFieldCount()): new.SetField(outDefn.GetFieldDefn(i).GetNameRef(), f.GetField(i)) out.CreateFeature(new) new.Destroy() f.Destroy() # ####### # # Project # # ####### # transP = get_trans_param(inEPSG, outEPSG) inData = ogr.GetDriverByName( drv_name(inShp)).Open(inShp, 0) inLyr = inData.GetLayer() out = ogr.GetDriverByName( drv_name(outShp)).CreateDataSource(outShp) outlyr = out.CreateLayer( fprop(outShp, 'fn'), get_sref_from_epsg(outEPSG), geom_type=get_gtype( inShp, name=None, py_cls=True, gisApi='ogr' ) ) # Copy fields to the output copy_flds(inLyr, outlyr) # Copy/transform features from the input to the output outlyrDefn = outlyr.GetLayerDefn() copyShp(outlyr, outlyrDefn, inLyr, transP) inData.Destroy() out.Destroy() elif gisApi == 'ogr2ogr': """ Transform SRS of any OGR Compilant Data. Save the transformed data in a new file """ if not inEPSG: from gasp.gt.prop.prj import get_epsg_shp inEPSG = get_epsg_shp(inShp) if not inEPSG: raise ValueError('To use ogr2ogr, you must specify inEPSG') from gasp import exec_cmd from gasp.gt.prop.ff import drv_name cmd = ( 'ogr2ogr -f "{}" {} {}{} -s_srs EPSG:{} -t_srs EPSG:{}' ).format( drv_name(outShp), outShp, inShp, '' if not sql else ' -dialect sqlite -sql "{}"'.format(sql), str(inEPSG), str(outEPSG) ) outcmd = exec_cmd(cmd) elif gisApi == 'ogr2ogr_SQLITE': """ Transform SRS of a SQLITE DB table. Save the transformed data in a new table """ from gasp import exec_cmd if not inEPSG: raise ValueError(( 'With ogr2ogr_SQLITE, the definition of inEPSG is ' 'demandatory.' )) # TODO: Verify if database is sqlite db, tbl = inShp['DB'], inShp['TABLE'] sql = 'SELECT * FROM {}'.format(tbl) if not sql else sql outcmd = exec_cmd(( 'ogr2ogr -update -append -f "SQLite" {db} -nln "{nt}" ' '-dialect sqlite -sql "{_sql}" -s_srs EPSG:{inepsg} ' '-t_srs EPSG:{outepsg} {db}' ).format( db=db, nt=outShp, _sql=sql, inepsg=str(inEPSG), outepsg=str(outEPSG) )) elif gisApi == 'pandas': # Test if input Shp is GeoDataframe from gasp.gt.fmshp import shp_to_obj from gasp.gt.toshp import df_to_shp df = shp_to_obj(inShp) # Project df newDf = df.to_crs({'init' : 'epsg:{}'.format(str(outEPSG))}) # Save as file return df_to_shp(df, outShp) elif gisApi == 'psql': from gasp.sql.db import create_db from gasp.pyt.oss import fprop from gasp.gql.to import shp_to_psql from gasp.gt.toshp.db import dbtbl_to_shp from gasp.gql.prj import sql_proj # Create Database if not db_name: db_name = create_db(fprop( outShp, 'fn', forceLower=True), api='psql' ) else: from gasp.sql.i import db_exists isDb = db_exists(db_name) if not isDb: create_db(db_name, api='psql') # Import Data inTbl = shp_to_psql(db_name, inShp, api='shp2pgsql', encoding="LATIN1") # Transform oTbl = sql_proj( db_name, inTbl, fprop(outShp, 'fn', forceLower=True), outEPSG, geomCol='geom', newGeom='geom' ) # Export outShp = dbtbl_to_shp( db_name, oTbl, 'geom', outShp, api='psql', epsg=outEPSG ) else: raise ValueError('Sorry, API {} is not available'.format(gisApi)) return outShp
def joinLines_by_spatial_rel_raster(mainLines, mainId, joinLines, joinCol, outfile, epsg): """ Join Attributes based on a spatial overlap. An raster based approach """ import os import pandas from geopandas import GeoDataFrame from gasp.gt.fmshp import shp_to_obj from gasp.gt.toshp import df_to_shp from gasp.gt.toshp.coord import shpext_to_boundshp from gasp.gt.torst import shp_to_rst from gasp.g.to import df_to_geodf from gasp.gt.wenv.grs import run_grass from gasp.pyt.df.joins import join_dfs from gasp.pyt.df.agg import df_groupBy from gasp.pyt.oss import fprop, mkdir workspace = mkdir(os.path.join(os.path.dirname(mainLines, 'tmp_dt'))) # Create boundary file boundary = shpext_to_boundshp(mainLines, os.path.join(workspace, "bound.shp"), epsg) boundRst = shp_to_rst(boundary, None, 5, -99, os.path.join(workspace, "rst_base.tif"), epsg=epsg, api='gdal') # Start GRASS GIS Session gbase = run_grass(workspace, location="grs_loc", srs=boundRst) import grass.script as grass import grass.script.setup as gsetup gsetup.init(gbase, workspace, "grs_loc", "PERMANENT") from gasp.gt.nop.local import combine from gasp.gt.prop.rst import get_rst_report_data from gasp.gt.toshp.cff import shp_to_grs, grs_to_shp from gasp.gt.torst import shp_to_rst # Add data to GRASS GIS mainVector = shp_to_grs(mainLines, fprop(mainLines, 'fn', forceLower=True)) joinVector = shp_to_grs(joinLines, fprop(joinLines, 'fn', forceLower=True)) mainRst = shp_to_rst(mainVector, mainId, None, None, "rst_" + mainVector, api='pygrass') joinRst = shp_to_rst(joinVector, joinCol, None, None, "rst_" + joinVector, api='pygrass') combRst = combine(mainRst, joinRst, "combine_rst", api="pygrass") combine_data = get_rst_report_data(combRst, UNITS="c") combDf = pandas.DataFrame(combine_data, columns=["comb_cat", "rst_1", "rst_2", "ncells"]) combDf = combDf[combDf["rst_2"] != '0'] combDf["ncells"] = combDf["ncells"].astype(int) gbdata = df_groupBy(combDf, ["rst_1"], "MAX", "ncells") fTable = join_dfs(gbdata, combDf, ["rst_1", "ncells"], ["rst_1", "ncells"]) fTable["rst_2"] = fTable["rst_2"].astype(int) fTable = df_groupBy(fTable, ["rst_1", "ncells"], STAT='MIN', STAT_FIELD="rst_2") mainLinesCat = grs_to_shp(mainVector, os.path.join(workspace, mainVector + '.shp'), 'line') mainLinesDf = shp_to_obj(mainLinesCat) resultDf = join_dfs(mainLinesDf, fTable, "cat", "rst_1", onlyCombinations=None) resultDf.rename(columns={"rst_2": joinCol}, inplace=True) resultDf = df_to_geodf(resultDf, "geometry", epsg) df_to_shp(resultDf, outfile) return outfile
def join_attr_by_distance(mainTable, joinTable, workGrass, epsg_code, output): """ Find nearest feature and join attributes of the nearest feature to the mainTable Uses GRASS GIS to find near lines. """ import os from gasp.gt.wenv.grs import run_grass from gasp.gt.fmshp import shp_to_obj from gasp.g.to import df_to_geodf from gasp.gt.toshp import df_to_shp from gasp.pyt.oss import fprop # Create GRASS GIS Location grassBase = run_grass(workGrass, location='join_loc', srs=epsg_code) import grass.script as grass import grass.script.setup as gsetup gsetup.init(grassBase, workGrass, 'join_loc', 'PERMANENT') # Import some GRASS GIS tools from gasp.gt.prox import grs_near as near from gasp.gt.tbl.attr import geomattr_to_db from gasp.gt.toshp.cff import shp_to_grs, grs_to_shp # Import data into GRASS GIS grsMain = shp_to_grs(mainTable, fprop(mainTable, 'fn', forceLower=True)) grsJoin = shp_to_grs(joinTable, fprop(joinTable, 'fn', forceLower=True)) # Get distance from each feature of mainTable to the nearest feature # of the join table near(grsMain, grsJoin, nearCatCol="tocat", nearDistCol="todistance") # Export data from GRASS GIS ogrMain = grs_to_shp(grsMain, os.path.join(workGrass, 'join_loc', grsMain + '_grs.shp'), None, asMultiPart=True) ogrJoin = grs_to_shp(grsJoin, os.path.join(workGrass, 'join_loc', grsJoin + '_grs.shp'), None, asMultiPart=True) dfMain = shp_to_obj(ogrMain) dfJoin = shp_to_obj(ogrJoin) dfResult = dfMain.merge(dfJoin, how='inner', left_on='tocat', right_on='cat') dfResult.drop(["geometry_y", "cat_y"], axis=1, inplace=True) dfResult.rename(columns={"cat_x": "cat_grass"}, inplace=True) dfResult["tocat"] = dfResult["tocat"] - 1 dfResult["cat_grass"] = dfResult["cat_grass"] - 1 dfResult = df_to_geodf(dfResult, "geometry_x", epsg_code) df_to_shp(dfResult, output) return output
def snap_points_to_near_line(lineShp, pointShp, epsg, workGrass, outPoints, location='overlap_pnts', api='grass', movesShp=None): """ Move points to overlap near line API's Available: * grass; * saga. """ if api == 'grass': """ Uses GRASS GIS to find near lines. """ import os import numpy from geopandas import GeoDataFrame from gasp.pyt.oss import fprop from gasp.gt.wenv.grs import run_grass from gasp.gt.fmshp import shp_to_obj from gasp.gt.toshp import df_to_shp # Create GRASS GIS Location grassBase = run_grass(workGrass, location=location, srs=epsg) import grass.script as grass import grass.script.setup as gsetup gsetup.init(grassBase, workGrass, location, 'PERMANENT') # Import some GRASS GIS tools from gasp.gt.prox import grs_near as near from gasp.gt.tbl.attr import geomattr_to_db from gasp.gt.toshp.cff import shp_to_grs, grs_to_shp # Import data into GRASS GIS grsLines = shp_to_grs(lineShp, fprop(lineShp, 'fn', forceLower=True)) grsPoint = shp_to_grs(pointShp, fprop(pointShp, 'fn', forceLower=True)) # Get distance from points to near line near(grsPoint, grsLines, nearCatCol="tocat", nearDistCol="todistance") # Get coord of start/end points of polylines geomattr_to_db(grsLines, ['sta_pnt_x', 'sta_pnt_y'], 'start', 'line') geomattr_to_db(grsLines, ['end_pnt_x', 'end_pnt_y'], 'end', 'line') # Export data from GRASS GIS ogrPoint = grs_to_shp( grsPoint, os.path.join(workGrass, grsPoint + '.shp', 'point', asMultiPart=True)) ogrLine = grs_to_shp( grsLines, os.path.join(workGrass, grsLines + '.shp', 'point', asMultiPart=True)) # Points to GeoDataFrame pntDf = shp_to_obj(ogrPoint) # Lines to GeoDataFrame lnhDf = shp_to_obj(ogrLine) # Erase unecessary fields pntDf.drop(["todistance"], axis=1, inplace=True) lnhDf.drop([ c for c in lnhDf.columns.values if c != 'geometry' and c != 'cat' and c != 'sta_pnt_x' and c != 'sta_pnt_y' and c != 'end_pnt_x' and c != 'end_pnt_y' ], axis=1, inplace=True) # Join Geometries - Table with Point Geometry and Geometry of the # nearest line resultDf = pntDf.merge(lnhDf, how='inner', left_on='tocat', right_on='cat') # Move points resultDf['geometry'] = [ geoms[0].interpolate(geoms[0].project(geoms[1])) for geoms in zip(resultDf.geometry_y, resultDf.geometry_x) ] resultDf.drop(["geometry_x", "geometry_y", "cat_x", "cat_y"], axis=1, inplace=True) resultDf = GeoDataFrame(resultDf, crs={"init": 'epsg:{}'.format(epsg)}, geometry="geometry") # Check if points are equal to any start/end points resultDf["x"] = resultDf.geometry.x resultDf["y"] = resultDf.geometry.y resultDf["check"] = numpy.where( (resultDf["x"] == resultDf["sta_pnt_x"]) & (resultDf["y"] == resultDf["sta_pnt_y"]), 1, 0) resultDf["check"] = numpy.where( (resultDf["x"] == resultDf["end_pnt_x"]) & (resultDf["y"] == resultDf["end_pnt_y"]), 1, 0) # To file df_to_shp(resultDf, outPoints) elif api == 'saga': """ Snap Points to Lines using SAGA GIS """ from gasp import exec_cmd cmd = ("saga_cmd shapes_points 19 -INPUT {pnt} -SNAP {lnh} " "-OUTPUT {out}{mv}").format( pnt=pointShp, lnh=lineShp, out=outPoints, mv="" if not movesShp else " -MOVES {}".format(movesShp)) outcmd = exec_cmd(cmd) else: raise ValueError("{} is not available!".format(api)) return outPoints
def obj_to_shp(dd, geomkey, srs, outshp): from gasp.g.to import df_to_geodf as obj_to_geodf geodf = obj_to_geodf(dd, geomkey, srs) return df_to_shp(geodf, outshp)
def fishnet(top_left, bottom_right, outfishnet, x, y, epsg=None, xy_row_col=None): """ Produce fishnet from extent - Use pandas to do the job """ from math import ceil import numpy as np import pandas as pd from shapely import wkt import geopandas as gp from gasp.pyt.df.mng import dfcolstorows from gasp.gt.toshp import df_to_shp x_min, y_max = top_left x_max, y_min = bottom_right if xy_row_col: # X, Y are row and col number # Find cellsize nrow = x ncol = y width = ceil((x_max - x_min) / nrow) height = ceil((y_max - y_min) / ncol) else: # X, Y are cellsize # Find N Row and Col width = x height = y nrow = ceil((y_max - y_min) / height) ncol = ceil((x_max - x_min) / width) # Create array with right shape num = np.full((nrow, ncol), 1, dtype=np.ubyte) # Array to DataFrame numdf = pd.DataFrame(num) numdf['idx'] = numdf.index fishtbl = dfcolstorows(numdf, 'col', 'val', colFid='idx') # Add polygon vertices fishtbl['x_min'] = x_min + (width * fishtbl.col) fishtbl['x_max'] = fishtbl.x_min + width fishtbl['y_max'] = y_max - (height * fishtbl.idx) fishtbl['y_min'] = fishtbl.y_max - height fishtbl['x_min'] = fishtbl.x_min.astype(str) fishtbl['x_max'] = fishtbl.x_max.astype(str) fishtbl['y_max'] = fishtbl.y_max.astype(str) fishtbl['y_min'] = fishtbl.y_min.astype(str) # Create Polygon WKT fishtbl['wkt'] = 'POLYGON ((' + fishtbl.x_min + ' ' + fishtbl.y_max + ', ' + \ fishtbl.x_min + ' ' + fishtbl.y_min + ", " + \ fishtbl.x_max + ' ' + fishtbl.y_min + ", " + \ fishtbl.x_max + ' ' + fishtbl.y_max + ", " + \ fishtbl.x_min + ' ' + fishtbl.y_max + '))' # Create Polygon from WKT fishtbl["geom"] = fishtbl.wkt.apply(wkt.loads) # Delete unecessary cols fishtbl.drop( ["val", "idx", "col", "x_min", "x_max", "y_min", "y_max", "wkt"], axis=1, inplace=True) # DataFrame to GeoDataFrame fishtbl = gp.GeoDataFrame(fishtbl, geometry="geom", crs={'init': 'epsg:{}'.format(str(epsg))}) # GeoDataFrame to File return df_to_shp(fishtbl, outfishnet)