def matrix_od(origins, destinations, networkShp, speedLimitCol, onewayCol, grsWorkspace, grsLocation, outputShp): """ Produce matrix OD using GRASS GIS """ from gasp.oss import get_filename from gasp.session import run_grass # Open an GRASS GIS Session gbase = run_grass(grsWorkspace, grassBIN="grass74", location=grsLocation, srs=networkShp) import grass.script as grass import grass.script.setup as gsetup gsetup.init(gbase, grsWorkspace, grsLocation, 'PERMANENT') # Import GRASS GIS Module from gasp.to.shp.grs import shp_to_grs # Add Data to GRASS GIS rdvMain = shp_to_grs(networkShp, get_filename(networkShp, forceLower=True)) """Get matrix distance:""" MATRIX_OD = prod_matrix(origins, destinations, rdvMain, speedLimitCol, onewayCol) return grass_converter(MATRIX_OD, outputShp, geom_type="line", lyrN=3)
def bash_matrix_od(origins, destinationShp, network, costCol, oneway, grsWork, output): """ Produce matrix OD using GRASS GIS - BASH MODE """ from gasp.session import run_grass from gasp.oss import get_filename from gasp.oss.ops import create_folder from gasp.mng.split import splitShp_by_range from gasp.mng.gen import merge_feat # SPLIT ORIGINS IN PARTS originsFld = create_folder(os.path.join(grsWork, 'origins_parts')) originsList = splitShp_by_range(origins, 100, originsFld) # Open an GRASS GIS Session gbase = run_grass(grsWork, grassBIN="grass76", location=grsLoc, srs=network) import grass.script as grass import grass.script.setup as gsetup RESULTS = [] R_FOLDER = create_folder(os.path.join(grsWork, 'res_parts')) for e in range(len(originsList)): gsetup.init(gbase, grsWork, "grs_loc_{}".format(e), 'PERMANENT') from gasp.to.shp.grs import shp_to_grs, grs_to_shp # Add Data to GRASS GIS rdvMain = shp_to_grs(network, get_filename(network, forceLower=True)) # Produce Matrix result_part = prod_matrix(originsList[e], destinationShp, rdvMain, costCol, oneway) # Export Result shp = grs_to_shp(result_part, os.path.join(R_FOLDER, result_part + '.shp'), geom_type="line", lyrN=3) RESULTS.append(shp) merge_feat(RESULTS, output, api='pandas') return output
def acumulated_cost(cst_surface, dest_pnt, cst_dist): """ Uses a cost surface to estimate the time between each cell and the close destination """ from gasp.spanlst.algebra import rstcalc from gasp.spanlst.dist import rcost from gasp.to.rst import rst_to_grs, grs_to_rst from gasp.to.shp.grs import shp_to_grs # Add Cost Surface to GRASS GIS rst_to_grs(cst_surface, 'cst_surf') # Add Destination To GRASS shp_to_grs(dest_pnt, 'destination') # Execute r.cost rcost('cst_surf', 'destination', 'cst_dist') # Convert to minutes rstcalc('cst_dist / 60.0', 'CstDistMin', api="grass") # Export result grs_to_rst('CstDistMin', cst_dist) return cst_dist
def __prod_mtxod(O, D, THRD): result_part = prod_matrix(O, D, rdvMain, costCol, oneway, thrdId=THRD, asCmd=True) shp = shp_to_grs(result_part, os.path.join(R_FOLDER, result_part + '.shp'), geom_type="line", lyrN=3, asCMD=True) RESULTS.append(shp)
def prod_matrix(origins, destinations, networkGrs, speedLimitCol, onewayCol, thrdId="1", asCmd=None): """ Get Matrix Distance: """ from gasp.to.shp.grs import shp_to_grs from gasp.cpu.grs.mng import category from gasp.anls.exct import sel_by_attr from gasp.mng.grstbl import add_field, add_table, update_table from gasp.mob.grstbx import network_from_arcs from gasp.mob.grstbx import add_pnts_to_network from gasp.mob.grstbx import run_allpairs from gasp.cpu.grs.mng.feat import geomattr_to_db from gasp.cpu.grs.mng.feat import copy_insame_vector from gasp.mng.gen import merge_feat from gasp.prop.feat import feat_count # Merge Origins and Destinations into the same Feature Class ORIGINS_NFEAT = feat_count(origins, gisApi='pandas') DESTINATIONS_NFEAT = feat_count(destinations, gisApi='pandas') ORIGINS_DESTINATIONS = merge_feat([origins, destinations], os.path.join( os.path.dirname(origins), "points_od_{}.shp".format(thrdId)), api='pandas') pointsGrs = shp_to_grs(ORIGINS_DESTINATIONS, "points_od_{}".format(thrdId), asCMD=asCmd) # Connect Points to Network newNetwork = add_pnts_to_network(networkGrs, pointsGrs, "rdv_points_{}".format(thrdId), asCMD=asCmd) # Sanitize Network Table and Cost Columns newNetwork = category(newNetwork, "rdv_points_time_{}".format(thrdId), "add", LyrN="3", geomType="line", asCMD=asCmd) add_table(newNetwork, ("cat integer,kph double precision,length double precision," "ft_minutes double precision," "tf_minutes double precision,oneway text"), lyrN=3, asCMD=asCmd) copy_insame_vector(newNetwork, "kph", speedLimitCol, 3, geomType="line", asCMD=asCmd) copy_insame_vector(newNetwork, "oneway", onewayCol, 3, geomType="line", asCMD=asCmd) geomattr_to_db(newNetwork, "length", "length", "line", createCol=False, unit="meters", lyrN=3, ascmd=asCmd) update_table(newNetwork, "kph", "3.6", "kph IS NULL", lyrN=3, ascmd=asCmd) update_table(newNetwork, "kph", "3.6", "oneway = 'N'", lyrN=3, ascmd=asCmd) update_table(newNetwork, "ft_minutes", "(length * 60) / (kph * 1000.0)", "ft_minutes IS NULL", lyrN=3, ascmd=asCmd) update_table(newNetwork, "tf_minutes", "(length * 60) / (kph * 1000.0)", "tf_minutes IS NULL", lyrN=3, ascmd=asCmd) # Exagerate Oneway's update_table(newNetwork, "ft_minutes", "1000", "oneway = 'TF'", lyrN=3, ascmd=asCmd) update_table(newNetwork, "tf_minutes", "1000", "oneway = 'FT'", lyrN=3, ascmd=asCmd) # Produce matrix matrix = run_allpairs(newNetwork, "ft_minutes", "tf_minutes", 'result_{}'.format(thrdId), arcLyr=3, nodeLyr=2, asCMD=asCmd) # Exclude unwanted OD Pairs q = "({}) AND ({})".format( " OR ".join( ["from_cat={}".format(str(i + 1)) for i in range(ORIGINS_NFEAT)]), " OR ".join([ "to_cat={}".format(str(ORIGINS_NFEAT + i + 1)) for i in range(DESTINATIONS_NFEAT) ])) matrix_sel = sel_by_attr(matrix, q, "sel_{}".format(matrix), geomType="line", lyrN=3, asCMD=asCmd) add_field(matrix_sel, "from_fid", "INTEGER", lyrN=3, asCMD=asCmd) add_field(matrix_sel, "to_fid", "INTEGER", lyrN=3, asCMD=asCmd) update_table(matrix_sel, "from_fid", "from_cat - 1", "from_fid IS NULL", lyrN=3, ascmd=asCmd) update_table(matrix_sel, "to_fid", "to_cat - {} - 1".format(str(ORIGINS_NFEAT)), "to_fid IS NULL", lyrN=3, ascmd=asCmd) return matrix_sel
def thrd_matrix_od(origins, destinationShp, network, costCol, oneway, grsWork, grsLoc, output): """ Produce matrix OD using GRASS GIS - Thread MODE PROBLEM: * O programa baralha-se todo porque ha muitas sessoes do grass a serem executadas. E preciso verificar se e possivel segregar as varias sessoes do grass """ from threading import Thread from gasp.session import run_grass from gasp.oss import get_filename from gasp.oss.ops import create_folder from gasp.mng.split import splitShp_by_range from gasp.mng.gen import merge_feat # SPLIT ORIGINS IN PARTS originsFld = create_folder(os.path.join(grsWork, 'origins_parts')) originsList = splitShp_by_range(origins, 100, originsFld) gbase = run_grass(grsWork, grassBIN="grass74", location=grsLoc, srs=network) import grass.script as grass import grass.script.setup as gsetup gsetup.init(gbase, grsWork, grsLoc, 'PERMANENT') from gasp.to.shp.grs import shp_to_grs from gasp.to.shp.grs import grs_to_shp # Add Data to GRASS GIS rdvMain = shp_to_grs(network, get_filename(network, forceLower=True), asCMD=True) RESULTS = [] R_FOLDER = create_folder(os.path.join(grsWork, 'res_parts')) def __prod_mtxod(O, D, THRD): result_part = prod_matrix(O, D, rdvMain, costCol, oneway, thrdId=THRD, asCmd=True) shp = shp_to_grs(result_part, os.path.join(R_FOLDER, result_part + '.shp'), geom_type="line", lyrN=3, asCMD=True) RESULTS.append(shp) thrds = [] for i in range(len(originsList)): thrds.append( Thread(name='tk-{}'.format(str(i)), target=__prod_mtxod, args=(originsList[i], destinationShp, str(i)))) for t in thrds: t.start() for t in thrds: t.join() merge_feat(RESULTS, output, api='pandas') return output
def joinLines_by_spatial_rel_raster(mainLines, mainId, joinLines, joinCol, outfile, epsg): """ Join Attributes based on a spatial overlap. An raster based approach """ import os; import pandas from geopandas import GeoDataFrame from gasp.to.geom import regulardf_to_geodf from gasp.session import run_grass from gasp.oss import get_filename from gasp.oss.ops import create_folder from gasp.mng.ext import shpextent_to_boundary from gasp.mng.joins import join_dfs from gasp.mng.df import df_groupBy from gasp.to.rst import shp_to_raster from gasp.fm import tbl_to_obj from gasp.to.shp import df_to_shp workspace = create_folder(os.path.join( os.path.dirname(mainLines, 'tmp_dt') )) # Create boundary file boundary = shpextent_to_boundary( mainLines, os.path.join(workspace, "bound.shp"), epsg ) boundRst = shp_to_raster(boundary, None, 5, -99, os.path.join( workspace, "rst_base.tif"), epsg=epsg, api='gdal') # Start GRASS GIS Session gbase = run_grass(workspace, location="grs_loc", srs=boundRst) import grass.script as grass import grass.script.setup as gsetup gsetup.init(gbase, workspace, "grs_loc", "PERMANENT") from gasp.spanlst.local import combine from gasp.cpu.grs.spanlst import get_rst_report_data from gasp.to.shp.grs import shp_to_grs, grs_to_shp from gasp.to.rst import shp_to_raster # Add data to GRASS GIS mainVector = shp_to_grs( mainLines, get_filename(mainLines, forceLower=True)) joinVector = shp_to_grs( joinLines, get_filename(joinLines, forceLower=True)) mainRst = shp_to_raster( mainVector, mainId, None, None, "rst_" + mainVector, api='pygrass' ); joinRst = shp_to_raster( joinVector, joinCol, None, None, "rst_" + joinVector, api='pygrass' ) combRst = combine(mainRst, joinRst, "combine_rst", api="pygrass") combine_data = get_rst_report_data(combRst, UNITS="c") combDf = pandas.DataFrame(combine_data, columns=[ "comb_cat", "rst_1", "rst_2", "ncells" ]) combDf = combDf[combDf["rst_2"] != '0'] combDf["ncells"] = combDf["ncells"].astype(int) gbdata = df_groupBy(combDf, ["rst_1"], "MAX", "ncells") fTable = join_dfs(gbdata, combDf, ["rst_1", "ncells"], ["rst_1", "ncells"]) fTable["rst_2"] = fTable["rst_2"].astype(int) fTable = df_groupBy( fTable, ["rst_1", "ncells"], STAT='MIN', STAT_FIELD="rst_2" ) mainLinesCat = grs_to_shp( mainVector, os.path.join(workspace, mainVector + '.shp'), 'line') mainLinesDf = tbl_to_obj(mainLinesCat) resultDf = join_dfs( mainLinesDf, fTable, "cat", "rst_1", onlyCombinations=None ) resultDf.rename(columns={"rst_2" : joinCol}, inplace=True) resultDf = regulardf_to_geodf(resultDf, "geometry", epsg) df_to_shp(resultDf, outfile) return outfile
def basic_buffer(osmLink, lineTable, dataFolder, apidb='SQLITE'): """ Data from Lines table to Polygons using a basic buffering stratagie """ import datetime from gasp.fm.sql import query_to_df if apidb == 'POSTGIS': from gasp.sql.anls.prox import st_buffer else: from gasp.sql.anls.prox import splite_buffer as st_buffer from gasp.to.rst import shp_to_raster from gasp.to.shp.grs import shp_to_grs time_a = datetime.datetime.now().replace(microsecond=0) lulcCls = query_to_df( osmLink, ("SELECT basic_buffer FROM {} WHERE basic_buffer IS NOT NULL " "GROUP BY basic_buffer").format(lineTable), db_api='psql' if apidb == 'POSTGIS' else 'sqlite').basic_buffer.tolist() time_b = datetime.datetime.now().replace(microsecond=0) timeGasto = {0: ('check_cls', time_b - time_a)} clsRst = {} tk = 1 for cls in lulcCls: # Run BUFFER Tool time_x = datetime.datetime.now().replace(microsecond=0) bb_file = st_buffer(osmLink, lineTable, "bf_basic_buffer", "geometry", os.path.join( dataFolder, 'bb_rule5_{}.shp'.format(str(int(cls)))), whrClause="basic_buffer={}".format(str(int(cls))), outTblIsFile=True, dissolve="ALL", cols_select="basic_buffer") time_y = datetime.datetime.now().replace(microsecond=0) # Data TO GRASS grsVect = shp_to_grs(bb_file, "bb_{}".format(int(cls)), asCMD=True, filterByReg=True) time_z = datetime.datetime.now().replace(microsecond=0) # Data to Raster rstVect = shp_to_raster(grsVect, int(cls), None, None, "rbb_{}".format(int(cls)), api="grass") time_w = datetime.datetime.now().replace(microsecond=0) clsRst[int(cls)] = rstVect timeGasto[tk] = ('do_buffer_{}'.format(cls), time_y - time_x) timeGasto[tk + 1] = ('import_{}'.format(cls), time_z - time_y) timeGasto[tk + 2] = ('torst_{}'.format(cls), time_w - time_z) tk += 3 return clsRst, timeGasto
def vector_based(osmdata, nomenclature, refRaster, lulcShp, overwrite=None, dataStore=None, RoadsAPI='SQLITE'): """ Convert OSM Data into Land Use/Land Cover Information An vector based approach. TODO: Add a detailed description. RoadsAPI Options: * SQLITE * POSTGIS """ # ************************************************************************ # # Python Modules from Reference Packages # # ************************************************************************ # import datetime import os import json # ************************************************************************ # # GASP dependencies # # ************************************************************************ # from gasp.oss.ops import create_folder from gasp.prop.rst import get_epsg_raster from gasp.session import run_grass if RoadsAPI == 'POSTGIS': from gasp.sql.mng.db import create_db from gasp.osm2lulc.utils import osm_to_pgsql else: from gasp.osm2lulc.utils import osm_to_sqdb from gasp.osm2lulc.utils import osm_project, add_lulc_to_osmfeat from gasp.mng.gen import merge_feat from gasp.osm2lulc.mod1 import grs_vector if RoadsAPI == 'SQLITE' or RoadsAPI == 'POSTGIS': from gasp.osm2lulc.mod2 import roads_sqdb else: from gasp.osm2lulc.mod2 import grs_vec_roads from gasp.osm2lulc.m3_4 import grs_vect_selbyarea from gasp.osm2lulc.mod5 import grs_vect_bbuffer from gasp.osm2lulc.mod6 import vector_assign_pntags_to_build # ************************************************************************ # # Global Settings # # ************************************************************************ # if not os.path.exists(os.path.dirname(lulcShp)): raise ValueError('{} does not exist!'.format(os.path.dirname(lulcShp))) # Get Parameters to connect to PostgreSQL conPGSQL = json.load( open( os.path.join(os.path.dirname(os.path.abspath(__file__)), 'con-postgresql.json'), 'r')) if RoadsAPI == 'POSTGIS' else None # Get EPSG of Reference Raster epsg = get_epsg_raster(refRaster) if not epsg: raise ValueError('Cannot get epsg code of ref raster') time_a = datetime.datetime.now().replace(microsecond=0) from gasp.osm2lulc.var import osmTableData, PRIORITIES workspace = os.path.join(os.path.dirname(lulcShp), 'osmtolulc') if not dataStore else dataStore # Check if workspace exists if os.path.exists(workspace): if overwrite: create_folder(workspace) else: raise ValueError('Path {} already exists'.format(workspace)) else: create_folder(workspace) __priorities = PRIORITIES[nomenclature] time_b = datetime.datetime.now().replace(microsecond=0) if RoadsAPI != 'POSTGIS': # ******************************************************************** # # Convert OSM file to SQLITE DB # # ******************************************************************** # osm_db = osm_to_sqdb(osmdata, os.path.join(workspace, 'osm.sqlite')) else: # Convert OSM file to POSTGRESQL DB # conPGSQL["DATABASE"] = create_db(conPGSQL, os.path.splitext( os.path.basename(osmdata))[0], overwrite=True) osm_db = osm_to_pgsql(osmdata, conPGSQL) time_c = datetime.datetime.now().replace(microsecond=0) # ************************************************************************ # # Add Lulc Classes to OSM_FEATURES by rule # # ************************************************************************ # add_lulc_to_osmfeat(osm_db if RoadsAPI != 'POSTGIS' else conPGSQL, osmTableData, nomenclature, api='SQLITE' if RoadsAPI != 'POSTGIS' else RoadsAPI) time_d = datetime.datetime.now().replace(microsecond=0) # ************************************************************************ # # Transform SRS of OSM Data # # ************************************************************************ # osmTableData = osm_project( osm_db if RoadsAPI != 'POSTGIS' else conPGSQL, epsg, api='SQLITE' if RoadsAPI != 'POSTGIS' else RoadsAPI) time_e = datetime.datetime.now().replace(microsecond=0) # ************************************************************************ # # Start a GRASS GIS Session # # ************************************************************************ # grass_base = run_grass(workspace, grassBIN='grass76', location='grloc', srs=epsg) #import grass.script as grass import grass.script.setup as gsetup gsetup.init(grass_base, workspace, 'grloc', 'PERMANENT') # ************************************************************************ # # IMPORT SOME GASP MODULES FOR GRASS GIS # # ************************************************************************ # from gasp.anls.ovlay import erase from gasp.prop.grs import rst_to_region from gasp.mng.genze import dissolve from gasp.mng.grstbl import add_and_update, reset_table from gasp.to.shp.grs import shp_to_grs, grs_to_shp from gasp.to.rst import rst_to_grs # ************************************************************************ # # SET GRASS GIS LOCATION EXTENT # # ************************************************************************ # extRst = rst_to_grs(refRaster, 'extent_raster') rst_to_region(extRst) time_f = datetime.datetime.now().replace(microsecond=0) # ************************************************************************ # # MapResults # # ************************************************************************ # osmShps = [] # ************************************************************************ # # 1 - Selection Rule # # ************************************************************************ # ruleOneShp, timeCheck1 = grs_vector( osm_db if RoadsAPI != 'POSTGIS' else conPGSQL, osmTableData['polygons'], apidb=RoadsAPI) osmShps.append(ruleOneShp) time_g = datetime.datetime.now().replace(microsecond=0) # ************************************************************************ # # 2 - Get Information About Roads Location # # ************************************************************************ # ruleRowShp, timeCheck2 = roads_sqdb( osm_db if RoadsAPI == 'SQLITE' else conPGSQL, osmTableData['lines'], osmTableData['polygons'], apidb=RoadsAPI ) if RoadsAPI == 'SQLITE' or RoadsAPI == 'POSTGIS' else grs_vec_roads( osm_db, osmTableData['lines'], osmTableData['polygons']) osmShps.append(ruleRowShp) time_h = datetime.datetime.now().replace(microsecond=0) # ************************************************************************ # # 3 - Area Upper than # # ************************************************************************ # ruleThreeShp, timeCheck3 = grs_vect_selbyarea( osm_db if RoadsAPI != 'POSTGIS' else conPGSQL, osmTableData['polygons'], UPPER=True, apidb=RoadsAPI) osmShps.append(ruleThreeShp) time_l = datetime.datetime.now().replace(microsecond=0) # ************************************************************************ # # 4 - Area Lower than # # ************************************************************************ # ruleFourShp, timeCheck4 = grs_vect_selbyarea( osm_db if RoadsAPI != 'POSTGIS' else conPGSQL, osmTableData['polygons'], UPPER=False, apidb=RoadsAPI) osmShps.append(ruleFourShp) time_j = datetime.datetime.now().replace(microsecond=0) # ************************************************************************ # # 5 - Get data from lines table (railway | waterway) # # ************************************************************************ # ruleFiveShp, timeCheck5 = grs_vect_bbuffer( osm_db if RoadsAPI != 'POSTGIS' else conPGSQL, osmTableData["lines"], api_db=RoadsAPI) osmShps.append(ruleFiveShp) time_m = datetime.datetime.now().replace(microsecond=0) # ************************************************************************ # # 7 - Assign untagged Buildings to tags # # ************************************************************************ # if nomenclature != "GLOBE_LAND_30": ruleSeven11, ruleSeven12, timeCheck7 = vector_assign_pntags_to_build( osm_db if RoadsAPI != 'POSTGIS' else conPGSQL, osmTableData['points'], osmTableData['polygons'], apidb=RoadsAPI) if ruleSeven11: osmShps.append(ruleSeven11) if ruleSeven12: osmShps.append(ruleSeven12) time_n = datetime.datetime.now().replace(microsecond=0) else: timeCheck7 = None time_n = datetime.datetime.now().replace(microsecond=0) # ************************************************************************ # # Produce LULC Map # # ************************************************************************ # """ Get Shps with all geometries related with one class - One Shape for Classe """ from gasp.mng.gen import same_attr_to_shp _osmShps = [] for i in range(len(osmShps)): if not osmShps[i]: continue _osmShps.append( grs_to_shp(osmShps[i], os.path.join(workspace, osmShps[i] + '.shp'), 'auto', lyrN=1, asCMD=True, asMultiPart=None)) _osmShps = same_attr_to_shp(_osmShps, "cat", workspace, "osm_", resultDict=True) del osmShps time_o = datetime.datetime.now().replace(microsecond=0) """ Merge all Classes into one feature class using a priority rule """ osmShps = {} for cls in _osmShps: if cls == '1': osmShps[1221] = shp_to_grs(_osmShps[cls], "osm_1221", asCMD=True) else: osmShps[int(cls)] = shp_to_grs(_osmShps[cls], "osm_" + cls, asCMD=True) # Erase overlapping areas by priority import copy osmNameRef = copy.deepcopy(osmShps) for e in range(len(__priorities)): if e + 1 == len(__priorities): break if __priorities[e] not in osmShps: continue else: for i in range(e + 1, len(__priorities)): if __priorities[i] not in osmShps: continue else: osmShps[__priorities[i]] = erase( osmShps[__priorities[i]], osmShps[__priorities[e]], "{}_{}".format(osmNameRef[__priorities[i]], e), notTbl=True, api='pygrass') time_p = datetime.datetime.now().replace(microsecond=0) # Export all classes lst_merge = [] for cls in osmShps: if cls == __priorities[0]: reset_table(osmShps[cls], {'cls': 'varchar(5)'}, {'cls': str(cls)}) else: add_and_update(osmShps[cls], {'cls': 'varchar(5)'}, {'cls': str(cls)}) ds = dissolve(osmShps[cls], 'dl_{}'.format(str(cls)), 'cls', api="grass") lst_merge.append( grs_to_shp(ds, os.path.join(workspace, "lulc_{}.shp".format(str(cls))), 'auto', lyrN=1, asCMD=True, asMultiPart=None)) time_q = datetime.datetime.now().replace(microsecond=0) merge_feat(lst_merge, lulcShp, api='pandas') time_r = datetime.datetime.now().replace(microsecond=0) return lulcShp, { 0: ('set_settings', time_b - time_a), 1: ('osm_to_sqdb', time_c - time_b), 2: ('cls_in_sqdb', time_d - time_c), 3: ('proj_data', time_e - time_d), 4: ('set_grass', time_f - time_e), 5: ('rule_1', time_g - time_f, timeCheck1), 6: ('rule_2', time_h - time_g, timeCheck2), 7: ('rule_3', time_l - time_h, timeCheck3), 8: ('rule_4', time_j - time_l, timeCheck4), 9: ('rule_5', time_m - time_j, timeCheck5), 10: None if not timeCheck7 else ('rule_7', time_n - time_m, timeCheck7), 11: ('disj_cls', time_o - time_n), 12: ('priority_rule', time_p - time_o), 13: ('export_cls', time_q - time_p), 14: ('merge_cls', time_r - time_q) }
def cost_surface(dem, lulc, cls_lulc, prod_lulc, roads, kph, barr, grass_location, output, grass_path=None): """ Tool for make a cost surface based on the roads, slope, land use and physical barriers. ach cell has a value that represents the resistance to the movement. """ import os from gasp.oss.ops import create_folder from gasp.os import os_name from gasp.session import run_grass from gasp.prop.rst import get_cellsize from gasp.prop.rst import rst_distinct from .constants import lulc_weight from .constants import get_slope_categories """ Auxiliar Methods """ def edit_lulc(shp, fld_cls, new_cls): FT_TF_GRASS(shp, 'lulc', 'None') add_field('lulc', 'leg', 'INT') for key in new_cls.keys(): l = new_cls[key]['cls'] sql = " OR ".join([ "{campo}='{value}'".format(campo=fld_cls, value=i) for i in l ]) update_table('lulc', 'leg', int(key), sql) return {'shp': 'lulc', 'fld': 'leg'} def combine_to_cost(rst_combined, lst_rst, work, slope_weight, rdv_cos_weight, cellsize, mode_movement): # The tool r.report doesn't work properly, for that we need some aditional information l = [] for i in lst_rst: FT_TF_GRASS(i, os.path.join(work, i + '.tif'), 'None') values = rst_distinct(os.path.join(work, i + '.tif'), gisApi='gdal') l.append(min(values)) # ****** # Now, we can procede normaly txt_file = os.path.join(work, 'text_combine.txt') raster_report(rst_combined, txt_file) open_txt = open(txt_file, 'r') c = 0 dic_combine = {} for line in open_txt.readlines(): try: if c == 4: dic_combine[0] = [str(l[0]), str(l[1])] elif c >= 5: pl = line.split('|') cat = pl[2].split('; ') cat1 = cat[0].split(' ') cat2 = cat[1].split(' ') dic_combine[int(pl[1])] = [cat1[1], cat2[1]] c += 1 except: break cst_dic = {} for key in dic_combine.keys(): cls_slope = int(dic_combine[key][0]) cos_vias = int(dic_combine[key][1]) if cos_vias >= 6: weight4slope = slope_weight[cls_slope]['rdv'] if mode_movement == 'pedestrian': weight4other = (3600.0 * cellsize) / (5.0 * 1000.0) else: weight4other = (3600.0 * cellsize) / (cos_vias * 1000.0) else: weight4slope = slope_weight[cls_slope]['cos'] weight4other = rdv_cos_weight[cos_vias]['weight'] cst_dic[key] = (weight4slope * weight4other) * 10000000.0 return cst_dic def Rules4CstSurface(dic, work): txt = open(os.path.join(work, 'cst_surface.txt'), 'w') for key in dic.keys(): txt.write('{cat} = {cst}\n'.format(cat=str(key), cst=str(dic[key]))) txt.close() return os.path.join(work, 'cst_surface.txt') """ Prepare GRASS GIS Environment """ workspace = os.path.dirname(grass_location) location = os.path.basename(grass_location) # Start GRASS GIS Engine grass_base = run_grass(workspace, location, dem, win_path=grass_path) import grass.script as grass import grass.script.setup as gsetup gsetup.init(grass_base, workspace, location, 'PERMANENT') # Import GRASS GIS Modules from gasp.cpu.grs import grass_converter from gasp.spanlst.surf import slope from gasp.spanlst.rcls import reclassify from gasp.spanlst.rcls import interval_rules from gasp.spanlst.rcls import category_rules from gasp.spanlst.rcls import grass_set_null from gasp.mng.grstbl import add_field, update_table from gasp.anls.ovlay import union from gasp.to.rst import rst_to_grs, grs_to_rst from gasp.to.rst import shp_to_raster from gasp.to.shp.grs import shp_to_grs from gasp.cpu.grs.spanlst import mosaic_raster from gasp.spanlst.local import combine from gasp.spanlst.algebra import rstcalc from gasp.cpu.grs.spanlst import raster_report """Global variables""" # Workspace for temporary files wTmp = create_folder(os.path.join(workspace, 'tmp')) # Cellsize cellsize = float(get_cellsize(dem), gisApi='gdal') # Land Use Land Cover weights lulcWeight = lulc_weight(prod_lulc, cellsize) # Slope classes and weights slope_cls = get_slope_categories() """Make Cost Surface""" # Generate slope raster rst_to_grs(dem, 'dem') slope('dem', 'rst_slope', api="pygrass") # Reclassify Slope rulesSlope = interval_rules(slope_cls, os.path.join(wTmp, 'slope.txt')) reclassify('rst_slope', 'recls_slope', rulesSlope) # LULC - Dissolve, union with barriers and conversion to raster lulc_shp = edit_lulc(lulc, cls_lulc, lulc_weight) shp_to_grs(barr, 'barriers') union(lulc_shp['shp'], 'barriers', 'barrcos', api_gis="grass") update_table('barrcos', 'a_' + lulc_shp['fld'], 99, 'b_cat=1') shp_to_raster('barrcos', 'a_' + lulc_shp['fld'], None, None, 'rst_barrcos', api='pygrass') # Reclassify this raster - convert the values 99 to NULL or NODATA grass_set_null('rst_barrcos', 99) # Add the roads layer to the GRASS GIS shp_to_grs(roads, 'rdv') if kph == 'pedestrian': add_field('rdv', 'foot', 'INT') update_table('rdv', 'foot', 50, 'foot IS NULL') shp_to_raster('rdv', 'foot', None, None, 'rst_rdv', api='pygrass') else: shp_to_raster('rdv', kph, None, None, 'rst_rdv', api='pygrass') # Merge LULC/BARR and Roads mosaic_raster('rst_rdv', 'rst_barrcos', 'rdv_barrcos') # Combine LULC/BARR/ROADS with Slope combine('recls_slope', 'rdv_barrcos', 'rst_combine', api="pygrass") """ Estimating cost for every combination at rst_combine The order of the rasters on the following list has to be the same of GRASS Combine""" cst = combine_to_cost('rst_combine', ['recls_slope', 'rdv_barrcos'], wTmp, slope_cls, lulc_weight, cell_size, kph) # Reclassify combined rst rulesSurface = category_rules(cst, os.path.join('r_surface.txt')) reclassify('rst_combine', 'cst_tmp', rulesSurface) rstcalc('cst_tmp / 10000000.0', 'cst_surface', api='pygrass') grs_to_rst('cst_surface', output)
def check_shape_diff(SHAPES_TO_COMPARE, OUT_FOLDER, REPORT, conPARAM, DB, SRS_CODE, GIS_SOFTWARE="GRASS", GRASS_REGION_TEMPLATE=None): """ Script to check differences between pairs of Feature Classes Suponha que temos diversas Feature Classes (FC) e que cada uma delas possui um determinado atributo; imagine também que, considerando todos os pares possíveis entre estas FC, se pretende comparar as diferenças na distribuição dos valores desse atributo em cada par. * Dependências: - ArcGIS; - GRASS; - PostgreSQL; - PostGIS. * GIS_SOFTWARE Options: - ARCGIS; - GRASS. """ import datetime import os import pandas from gasp.fm.sql import query_to_df from gasp.sql.mng.tbl import tbls_to_tbl from gasp.sql.mng.geom import fix_geom, check_geomtype_in_table from gasp.sql.mng.geom import select_main_geom_type from gasp.sql.mng.qw import ntbl_by_query from gasp.prop.ff import check_isRaster from gasp.oss import get_filename from gasp.sql.mng.db import create_db from gasp.to.sql import shp_to_psql, df_to_db from gasp.to.shp import rst_to_polyg from gasp.to.shp import shp_to_shp, psql_to_shp from gasp.to import db_to_tbl # Check if folder exists, if not create it if not os.path.exists(OUT_FOLDER): from gasp.oss.ops import create_folder create_folder(OUT_FOLDER, overwrite=None) else: raise ValueError('{} already exists!'.format(OUT_FOLDER)) # Start GRASS GIS Session if GIS_SOFTWARE == GRASS if GIS_SOFTWARE == "GRASS": if not GRASS_REGION_TEMPLATE: raise ValueError( 'To use GRASS GIS you need to specify GRASS_REGION_TEMPLATE') from gasp.session import run_grass gbase = run_grass(OUT_FOLDER, grassBIN='grass76', location='shpdif', srs=GRASS_REGION_TEMPLATE) import grass.script as grass import grass.script.setup as gsetup gsetup.init(gbase, OUT_FOLDER, 'shpdif', 'PERMANENT') from gasp.mng.grstbl import rename_col from gasp.to.shp.grs import shp_to_grs, grs_to_shp from gasp.to.rst import rst_to_grs from gasp.mng.fld import rename_column # Convert to SHAPE if file is Raster # Import to GRASS GIS if GIS SOFTWARE == GRASS i = 0 _SHP_TO_COMPARE = {} for s in SHAPES_TO_COMPARE: isRaster = check_isRaster(s) if isRaster: if GIS_SOFTWARE == "ARCGIS": d = rst_to_polyg(s, os.path.join(os.path.dirname(s), get_filename(s) + '.shp'), gisApi='arcpy') _SHP_TO_COMPARE[d] = "gridcode" elif GIS_SOFTWARE == "GRASS": # To GRASS rstName = get_filename(s) inRst = rst_to_grs(s, "rst_" + rstName, as_cmd=True) # To Raster d = rst_to_polyg(inRst, rstName, rstColumn="lulc_{}".format(i), gisApi="grasscmd") # Export Shapefile shp = grs_to_shp(d, os.path.join(OUT_FOLDER, d + '.shp'), "area") _SHP_TO_COMPARE[shp] = "lulc_{}".format(i) else: if GIS_SOFTWARE == "ARCGIS": _SHP_TO_COMPARE[s] = SHAPES_TO_COMPARE[s] elif GIS_SOFTWARE == "GRASS": # To GRASS grsV = shp_to_grs(s, get_filename(s), asCMD=True) # Change name of column with comparing value rename_col(grsV, SHAPES_TO_COMPARE[s], "lulc_{}".format(i), as_cmd=True) # Export shp = grs_to_shp(grsV, os.path.join(OUT_FOLDER, grsV + '_rn.shp'), "area") _SHP_TO_COMPARE[shp] = "lulc_{}".format(i) i += 1 SHAPES_TO_COMPARE = _SHP_TO_COMPARE if GIS_SOFTWARE == "ARCGIS": from gasp.cpu.arcg.mng.fld import calc_fld from gasp.cpu.arcg.mng.wspace import create_geodb from gasp.mng.gen import copy_feat # Sanitize data and Add new field __SHAPES_TO_COMPARE = {} i = 0 # Create GeoDatabase geodb = create_geodb(OUT_FOLDER, 'geo_sanitize') """ Sanitize Data """ for k in SHAPES_TO_COMPARE: # Send data to GeoDatabase only to sanitize newFc = shp_to_shp(k, os.path.join(geodb, get_filename(k)), gisApi='arcpy') # Create a copy to change newShp = copy_feat(newFc, os.path.join(OUT_FOLDER, os.path.basename(k)), gisApi='arcpy') # Sanitize field name with interest data NEW_FLD = "lulc_{}".format(i) calc_fld(newShp, NEW_FLD, "[{}]".format(SHAPES_TO_COMPARE[k]), isNewField={ "TYPE": "INTEGER", "LENGTH": 5, "PRECISION": "" }) __SHAPES_TO_COMPARE[newShp] = NEW_FLD i += 1 else: __SHAPES_TO_COMPARE = SHAPES_TO_COMPARE # Create database conPARAM["DATABASE"] = create_db(conPARAM, DB) """ Union SHAPEs """ UNION_SHAPE = {} FIX_GEOM = {} def fix_geometry(shp): # Send data to PostgreSQL nt = shp_to_psql(conPARAM, shp, SRS_CODE, api='shp2pgsql') # Fix data corr_tbl = fix_geom(conPARAM, nt, "geom", "corr_{}".format(nt), colsSelect=['gid', __SHAPES_TO_COMPARE[shp]]) # Check if we have multiple geometries geomN = check_geomtype_in_table(conPARAM, corr_tbl) if geomN > 1: corr_tbl = select_main_geom_type(conPARAM, corr_tbl, "corr2_{}".format(nt)) # Export data again newShp = psql_to_shp(conPARAM, corr_tbl, os.path.join(OUT_FOLDER, corr_tbl + '.shp'), api='pgsql2shp', geom_col='geom') return newShp SHPS = __SHAPES_TO_COMPARE.keys() for i in range(len(SHPS)): for e in range(i + 1, len(SHPS)): if GIS_SOFTWARE == 'ARCGIS': # Try the union thing unShp = union(SHPS[i], SHPS[e], os.path.join(OUT_FOLDER, "un_{}_{}.shp".format(i, e)), api_gis="arcpy") # See if the union went all right if not os.path.exists(unShp): # Union went not well # See if geometry was already fixed if SHPS[i] not in FIX_GEOM: # Fix SHPS[i] geometry FIX_GEOM[SHPS[i]] = fix_geometry(SHPS[i]) if SHPS[e] not in FIX_GEOM: FIX_GEOM[SHPS[e]] = fix_geometry(SHPS[e]) # Run Union again unShp = union(FIX_GEOM[SHPS[i]], FIX_GEOM[SHPS[e]], os.path.join(OUT_FOLDER, "un_{}_{}_f.shp".format(i, e)), api_gis="arcpy") elif GIS_SOFTWARE == "GRASS": # Optimized Union print "Union between {} and {}".format(SHPS[i], SHPS[e]) time_a = datetime.datetime.now().replace(microsecond=0) __unShp = optimized_union_anls( SHPS[i], SHPS[e], os.path.join(OUT_FOLDER, "un_{}_{}.shp".format(i, e)), GRASS_REGION_TEMPLATE, SRS_CODE, os.path.join(OUT_FOLDER, "work_{}_{}".format(i, e)), multiProcess=True) time_b = datetime.datetime.now().replace(microsecond=0) print time_b - time_a # Rename cols unShp = rename_column( __unShp, { "a_" + __SHAPES_TO_COMPARE[SHPS[i]]: __SHAPES_TO_COMPARE[SHPS[i]], "b_" + __SHAPES_TO_COMPARE[SHPS[e]]: __SHAPES_TO_COMPARE[SHPS[e]] }, os.path.join(OUT_FOLDER, "un_{}_{}_rn.shp".format(i, e))) UNION_SHAPE[(SHPS[i], SHPS[e])] = unShp # Send data one more time to postgresql SYNTH_TBL = {} for uShp in UNION_SHAPE: # Send data to PostgreSQL union_tbl = shp_to_psql(conPARAM, UNION_SHAPE[uShp], SRS_CODE, api='shp2pgsql') # Produce table with % of area equal in both maps areaMapTbl = ntbl_by_query( conPARAM, "{}_syn".format(union_tbl), ("SELECT CAST('{lulc_1}' AS text) AS lulc_1, " "CAST('{lulc_2}' AS text) AS lulc_2, " "round(" "CAST(SUM(g_area) / 1000000 AS numeric), 4" ") AS agree_area, round(" "CAST((SUM(g_area) / MIN(total_area)) * 100 AS numeric), 4" ") AS agree_percentage, " "round(" "CAST(MIN(total_area) / 1000000 AS numeric), 4" ") AS total_area FROM (" "SELECT {map1_cls}, {map2_cls}, ST_Area(geom) AS g_area, " "CASE " "WHEN {map1_cls} = {map2_cls} " "THEN 1 ELSE 0 " "END AS isthesame, total_area FROM {tbl}, (" "SELECT SUM(ST_Area(geom)) AS total_area FROM {tbl}" ") AS foo2" ") AS foo WHERE isthesame = 1 " "GROUP BY isthesame").format( lulc_1=get_filename(uShp[0]), lulc_2=get_filename(uShp[1]), map1_cls=__SHAPES_TO_COMPARE[uShp[0]], map2_cls=__SHAPES_TO_COMPARE[uShp[1]], tbl=union_tbl), api='psql') # Produce confusion matrix for the pair in comparison lulcCls = query_to_df( conPARAM, ("SELECT fcol FROM (" "SELECT CAST({map1_cls} AS text) AS fcol FROM {tbl} " "GROUP BY {map1_cls} " "UNION ALL SELECT CAST({map2_cls} AS text) FROM {tbl} " "GROUP BY {map2_cls}" ") AS foo GROUP BY fcol ORDER BY fcol").format( tbl=union_tbl, map1_cls=__SHAPES_TO_COMPARE[uShp[0]], map2_cls=__SHAPES_TO_COMPARE[uShp[1]]), db_api='psql').fcol.tolist() matrixTbl = ntbl_by_query( conPARAM, "{}_matrix".format(union_tbl), ("SELECT * FROM crosstab('" "SELECT CASE " "WHEN foo.{map1_cls} IS NOT NULL " "THEN foo.{map1_cls} ELSE jtbl.flyr " "END AS lulc1_cls, CASE " "WHEN foo.{map2_cls} IS NOT NULL " "THEN foo.{map2_cls} ELSE jtbl.slyr " "END AS lulc2_cls, CASE " "WHEN foo.garea IS NOT NULL " "THEN round(CAST(foo.garea / 1000000 AS numeric)" ", 3) ELSE 0 " "END AS garea FROM (" "SELECT CAST({map1_cls} AS text) AS {map1_cls}, " "CAST({map2_cls} AS text) AS {map2_cls}, " "SUM(ST_Area(geom)) AS garea " "FROM {tbl} GROUP BY {map1_cls}, {map2_cls}" ") AS foo FULL JOIN (" "SELECT f.flyr, s.slyr FROM (" "SELECT CAST({map1_cls} AS text) AS flyr " "FROM {tbl} GROUP BY {map1_cls}" ") AS f, (" "SELECT CAST({map2_cls} AS text) AS slyr " "FROM {tbl} GROUP BY {map2_cls}" ") AS s" ") AS jtbl " "ON foo.{map1_cls} = jtbl.flyr AND " "foo.{map2_cls} = jtbl.slyr " "ORDER BY 1,2" "') AS ct(" "lulc_cls text, {crossCols}" ")").format(crossCols=", ".join( ["cls_{} numeric".format(c) for c in lulcCls]), tbl=union_tbl, map1_cls=__SHAPES_TO_COMPARE[uShp[0]], map2_cls=__SHAPES_TO_COMPARE[uShp[1]]), api='psql') SYNTH_TBL[uShp] = {"TOTAL": areaMapTbl, "MATRIX": matrixTbl} # UNION ALL TOTAL TABLES total_table = tbls_to_tbl(conPARAM, [SYNTH_TBL[k]["TOTAL"] for k in SYNTH_TBL], 'total_table') # Create table with % of agreement between each pair of maps mapsNames = query_to_df( conPARAM, ("SELECT lulc FROM (" "SELECT lulc_1 AS lulc FROM {tbl} GROUP BY lulc_1 " "UNION ALL " "SELECT lulc_2 AS lulc FROM {tbl} GROUP BY lulc_2" ") AS lu GROUP BY lulc ORDER BY lulc").format(tbl=total_table), db_api='psql').lulc.tolist() FLDS_TO_PIVOT = ["agree_percentage", "total_area"] Q = ("SELECT * FROM crosstab('" "SELECT CASE " "WHEN foo.lulc_1 IS NOT NULL THEN foo.lulc_1 ELSE jtbl.tmp1 " "END AS lulc_1, CASE " "WHEN foo.lulc_2 IS NOT NULL THEN foo.lulc_2 ELSE jtbl.tmp2 " "END AS lulc_2, CASE " "WHEN foo.{valCol} IS NOT NULL THEN foo.{valCol} ELSE 0 " "END AS agree_percentage FROM (" "SELECT lulc_1, lulc_2, {valCol} FROM {tbl} UNION ALL " "SELECT lulc_1, lulc_2, {valCol} FROM (" "SELECT lulc_1 AS lulc_2, lulc_2 AS lulc_1, {valCol} " "FROM {tbl}" ") AS tst" ") AS foo FULL JOIN (" "SELECT lulc_1 AS tmp1, lulc_2 AS tmp2 FROM (" "SELECT lulc_1 AS lulc_1 FROM {tbl} GROUP BY lulc_1 " "UNION ALL " "SELECT lulc_2 AS lulc_1 FROM {tbl} GROUP BY lulc_2" ") AS tst_1, (" "SELECT lulc_1 AS lulc_2 FROM {tbl} GROUP BY lulc_1 " "UNION ALL " "SELECT lulc_2 AS lulc_2 FROM {tbl} GROUP BY lulc_2" ") AS tst_2 WHERE lulc_1 = lulc_2 GROUP BY lulc_1, lulc_2" ") AS jtbl ON foo.lulc_1 = jtbl.tmp1 AND foo.lulc_2 = jtbl.tmp2 " "ORDER BY lulc_1, lulc_2" "') AS ct(" "lulc_map text, {crossCols}" ")") TOTAL_AGREE_TABLE = None TOTAL_AREA_TABLE = None for f in FLDS_TO_PIVOT: if not TOTAL_AGREE_TABLE: TOTAL_AGREE_TABLE = ntbl_by_query( conPARAM, "agreement_table", Q.format(tbl=total_table, valCol=f, crossCols=", ".join([ "{} numeric".format(map_) for map_ in mapsNames ])), api='psql') else: TOTAL_AREA_TABLE = ntbl_by_query( conPARAM, "area_table", Q.format(tbl=total_table, valCol=f, crossCols=", ".join([ "{} numeric".format(map_) for map_ in mapsNames ])), api='psql') # Union Mapping UNION_MAPPING = pandas.DataFrame( [[ get_filename(k[0]), get_filename(k[1]), get_filename(UNION_SHAPE[k]) ] for k in UNION_SHAPE], columns=['shp_a', 'shp_b', 'union_shp' ]) if GIS_SOFTWARE == "ARCGIS" else pandas.DataFrame( [[k[0], k[1], get_filename(UNION_SHAPE[k])] for k in UNION_SHAPE], columns=['shp_a', 'shp_b', 'union_shp']) UNION_MAPPING = df_to_db(conPARAM, UNION_MAPPING, 'union_map', api='psql') # Export Results TABLES = [UNION_MAPPING, TOTAL_AGREE_TABLE, TOTAL_AREA_TABLE ] + [SYNTH_TBL[x]["MATRIX"] for x in SYNTH_TBL] SHEETS = ["union_map", "agreement_percentage", "area_with_data_km"] + [ "{}_{}".format(get_filename(x[0])[:15], get_filename(x[1])[:15]) for x in SYNTH_TBL ] db_to_xls(conPARAM, ["SELECT * FROM {}".format(x) for x in TABLES], REPORT, sheetsNames=SHEETS, dbAPI='psql') return REPORT
def optimized_union_anls(lyr_a, lyr_b, outShp, ref_boundary, epsg, workspace=None, multiProcess=None): """ Optimized Union Analysis Goal: optimize v.overlay performance for Union operations """ import os from gasp.oss import get_filename from gasp.mng.sample import create_fishnet from gasp.mng.feat import eachfeat_to_newshp from gasp.mng.gen import merge_feat from gasp.session import run_grass from gasp.anls.exct import split_shp_by_attr if workspace: if not os.path.exists(workspace): from gasp.oss.ops import create_folder create_folder(workspace, overwrite=True) else: from gasp.oss.ops import create_folder workspace = create_folder( os.path.join(os.path.dirname(outShp), "union_work")) # Create Fishnet gridShp = create_fishnet(ref_boundary, os.path.join(workspace, 'ref_grid.shp'), rowN=4, colN=4) # Split Fishnet in several files cellsShp = eachfeat_to_newshp(gridShp, workspace, epsg=epsg) if not multiProcess: # INIT GRASS GIS Session grsbase = run_grass(workspace, location="grs_loc", srs=ref_boundary) import grass.script.setup as gsetup gsetup.init(grsbase, workspace, "grs_loc", 'PERMANENT') # Add data to GRASS GIS from gasp.to.shp.grs import shp_to_grs cellsShp = [ shp_to_grs(shp, get_filename(shp), asCMD=True) for shp in cellsShp ] LYR_A = shp_to_grs(lyr_a, get_filename(lyr_a), asCMD=True) LYR_B = shp_to_grs(lyr_b, get_filename(lyr_b), asCMD=True) # Clip Layers A and B for each CELL in fishnet LYRS_A = [ clip(LYR_A, cellsShp[x], LYR_A + "_" + str(x), api_gis="grass_cmd") for x in range(len(cellsShp)) ] LYRS_B = [ clip(LYR_B, cellsShp[x], LYR_B + "_" + str(x), api_gis="grass_cmd") for x in range(len(cellsShp)) ] # Union SHPS UNION_SHP = [ union(LYRS_A[i], LYRS_B[i], "un_{}".format(i), api_gis="grass_cmd") for i in range(len(cellsShp)) ] # Export Data from gasp.to.shp.grs import grs_to_shp _UNION_SHP = [ grs_to_shp(shp, os.path.join(workspace, shp + ".shp"), "area") for shp in UNION_SHP ] else: def clip_and_union(la, lb, cell, work, ref, proc, output): # Start GRASS GIS Session grsbase = run_grass(work, location="proc_" + str(proc), srs=ref) import grass.script.setup as gsetup gsetup.init(grsbase, work, "proc_" + str(proc), 'PERMANENT') # Import GRASS GIS modules from gasp.to.shp.grs import shp_to_grs from gasp.to.shp.grs import grs_to_shp # Add data to GRASS a = shp_to_grs(la, get_filename(la), asCMD=True) b = shp_to_grs(lb, get_filename(lb), asCMD=True) c = shp_to_grs(cell, get_filename(cell), asCMD=True) # Clip a_clip = clip(a, c, "{}_clip".format(a), api_gis="grass_cmd") b_clip = clip(b, c, "{}_clip".format(b), api_gis="grass_cmd") # Union u_shp = union(a_clip, b_clip, "un_{}".format(c), api_gis="grass_cmd") # Export o = grs_to_shp(u_shp, output, "area") import multiprocessing thrds = [ multiprocessing.Process( target=clip_and_union, name="th-{}".format(i), args=(lyr_a, lyr_b, cellsShp[i], os.path.join(workspace, "th_{}".format(i)), ref_boundary, i, os.path.join(workspace, "uniao_{}.shp".format(i)))) for i in range(len(cellsShp)) ] for t in thrds: t.start() for t in thrds: t.join() _UNION_SHP = [ os.path.join(workspace, "uniao_{}.shp".format(i)) for i in range(len(cellsShp)) ] # Merge all union into the same layer MERGED_SHP = merge_feat(_UNION_SHP, outShp, api="ogr2ogr") return outShp
def distance_between_catpoints(srcShp, facilitiesShp, networkShp, speedLimitCol, onewayCol, grsWorkspace, grsLocation, outputShp): """ Path bet points TODO: Work with files with cat """ import os from gasp.oss import get_filename from gasp.session import run_grass from gasp.mng.gen import merge_feat from gasp.prop.feat import feat_count # Merge Source points and Facilities into the same Feature Class SRC_NFEAT = feat_count(srcShp, gisApi='pandas') FACILITY_NFEAT = feat_count(facilitiesShp, gisApi='pandas') POINTS = merge_feat([srcShp, facilitiesShp], os.path.join(os.path.dirname(outputShp), "points_net.shp"), api='pandas') # Open an GRASS GIS Session gbase = run_grass(grsWorkspace, grassBIN="grass76", location=grsLocation, srs=networkShp) import grass.script as grass import grass.script.setup as gsetup gsetup.init(gbase, grsWorkspace, grsLocation, 'PERMANENT') # Import GRASS GIS Module from gasp.to.shp.grs import shp_to_grs from gasp.to.shp.grs import grs_to_shp from gasp.cpu.grs.mng import category from gasp.mng.grstbl import add_field, add_table, update_table from gasp.mob.grstbx import network_from_arcs from gasp.mob.grstbx import add_pnts_to_network from gasp.mob.grstbx import netpath from gasp.cpu.grs.mng.feat import geomattr_to_db from gasp.cpu.grs.mng.feat import copy_insame_vector # Add Data to GRASS GIS rdvMain = shp_to_grs(networkShp, get_filename(networkShp, forceLower=True)) pntShp = shp_to_grs(POINTS, "points_net") """Get closest facility layer:""" # Connect Points to Network newNetwork = add_pnts_to_network(rdvMain, pntShp, "rdv_points") # Sanitize Network Table and Cost Columns newNetwork = category(newNetwork, "rdv_points_time", "add", LyrN="3", geomType="line") add_table(newNetwork, ("cat integer,kph double precision,length double precision," "ft_minutes double precision," "tf_minutes double precision,oneway text"), lyrN=3) copy_insame_vector(newNetwork, "kph", speedLimitCol, 3, geomType="line") copy_insame_vector(newNetwork, "oneway", onewayCol, 3, geomType="line") geomattr_to_db(newNetwork, "length", "length", "line", createCol=False, unit="meters", lyrN=3) update_table(newNetwork, "kph", "3.6", "kph IS NULL", lyrN=3) update_table(newNetwork, "ft_minutes", "(length * 60) / (kph * 1000.0)", "ft_minutes IS NULL", lyrN=3) update_table(newNetwork, "tf_minutes", "(length * 60) / (kph * 1000.0)", "tf_minutes IS NULL", lyrN=3) # Exagerate Oneway's update_table(newNetwork, "ft_minutes", "1000", "oneway = 'TF'", lyrN=3) update_table(newNetwork, "tf_minutes", "1000", "oneway = 'FT'", lyrN=3) # Produce result result = netpath(newNetwork, "ft_minutes", "tf_minutes", get_filename(outputShp), arcLyr=3, nodeLyr=2) return grs_to_shp(result, outputShp, geomType="line", lyrN=3)
def snap_points_to_near_line(lineShp, pointShp, epsg, workGrass, outPoints, location='overlap_pnts', api='grass', movesShp=None): """ Move points to overlap near line API's Available: * grass; * saga. """ if api == 'grass': """ Uses GRASS GIS to find near lines. """ import os; import numpy from geopandas import GeoDataFrame from gasp.oss import get_filename from gasp.session import run_grass from gasp.fm import shp_to_df from gasp.to.shp import df_to_shp # Create GRASS GIS Location grassBase = run_grass(workGrass, location=location, srs=epsg) import grass.script as grass import grass.script.setup as gsetup gsetup.init(grassBase, workGrass, location, 'PERMANENT') # Import some GRASS GIS tools from gasp.anls.prox import grs_near as near from gasp.cpu.grs.mng.feat import geomattr_to_db from gasp.to.shp.grs import shp_to_grs, grs_to_shp # Import data into GRASS GIS grsLines = shp_to_grs( lineShp, get_filename(lineShp, forceLower=True) ) grsPoint = shp_to_grs( pointShp, get_filename(pointShp, forceLower=True) ) # Get distance from points to near line near(grsPoint, grsLines, nearCatCol="tocat", nearDistCol="todistance") # Get coord of start/end points of polylines geomattr_to_db(grsLines, ['sta_pnt_x', 'sta_pnt_y'], 'start', 'line') geomattr_to_db(grsLines, ['end_pnt_x', 'end_pnt_y'], 'end', 'line') # Export data from GRASS GIS ogrPoint = grs_to_shp(grsPoint, os.path.join( workGrass, grsPoint + '.shp', 'point', asMultiPart=True )) ogrLine = grs_to_shp(grsLines, os.path.join( workGrass, grsLines + '.shp', 'point', asMultiPart=True )) # Points to GeoDataFrame pntDf = tbl_to_obj(ogrPoint) # Lines to GeoDataFrame lnhDf = tbl_to_obj(ogrLine) # Erase unecessary fields pntDf.drop(["todistance"], axis=1, inplace=True) lnhDf.drop([c for c in lnhDf.columns.values if c != 'geometry' and c != 'cat' and c != 'sta_pnt_x' and c != 'sta_pnt_y' and c != 'end_pnt_x' and c != 'end_pnt_y'], axis=1, inplace=True) # Join Geometries - Table with Point Geometry and Geometry of the # nearest line resultDf = pntDf.merge( lnhDf, how='inner', left_on='tocat', right_on='cat') # Move points resultDf['geometry'] = [geoms[0].interpolate( geoms[0].project(geoms[1]) ) for geoms in zip(resultDf.geometry_y, resultDf.geometry_x)] resultDf.drop(["geometry_x", "geometry_y", "cat_x", "cat_y"], axis=1, inplace=True) resultDf = GeoDataFrame( resultDf, crs={"init" : 'epsg:{}'.format(epsg)}, geometry="geometry" ) # Check if points are equal to any start/end points resultDf["x"] = resultDf.geometry.x resultDf["y"] = resultDf.geometry.y resultDf["check"] = numpy.where( (resultDf["x"] == resultDf["sta_pnt_x"]) & (resultDf["y"] == resultDf["sta_pnt_y"]), 1, 0 ) resultDf["check"] = numpy.where( (resultDf["x"] == resultDf["end_pnt_x"]) & (resultDf["y"] == resultDf["end_pnt_y"]), 1, 0 ) # To file df_to_shp(resultDf, outPoints) elif api == 'saga': """ Snap Points to Lines using SAGA GIS """ from gasp import exec_cmd cmd = ( "saga_cmd shapes_points 19 -INPUT {pnt} -SNAP {lnh} " "-OUTPUT {out}{mv}" ).format( pnt=pointShp, lnh=lineShp, out=outPoints, mv="" if not movesShp else " -MOVES {}".format(movesShp) ) outcmd = exec_cmd(cmd) else: raise ValueError("{} is not available!".format(api)) return outPoints
def join_attr_by_distance(mainTable, joinTable, workGrass, epsg_code, output): """ Find nearest feature and join attributes of the nearest feature to the mainTable Uses GRASS GIS to find near lines. """ import os from gasp.session import run_grass from gasp.fm import tbl_to_obj from gasp.to.geom import regulardf_to_geodf from gasp.to.shp import df_to_shp from gasp.oss import get_filename # Create GRASS GIS Location grassBase = run_grass(workGrass, location='join_loc', srs=epsg_code) import grass.script as grass import grass.script.setup as gsetup gsetup.init(grassBase, workGrass, 'join_loc', 'PERMANENT') # Import some GRASS GIS tools from gasp.anls.prox import grs_near as near from gasp.cpu.grs.mng.feat import geomattr_to_db from gasp.to.shp.grs import shp_to_grs, grs_to_shp # Import data into GRASS GIS grsMain = shp_to_grs(mainTable, get_filename( mainTable, forceLower=True) ); grsJoin = shp_to_grs(joinTable, get_filename( joinTable, forceLower=True) ) # Get distance from each feature of mainTable to the nearest feature # of the join table near(grsMain, grsJoin, nearCatCol="tocat", nearDistCol="todistance") # Export data from GRASS GIS ogrMain = grs_to_shp(grsMain, os.path.join( workGrass, 'join_loc', grsMain + '_grs.shp'), None, asMultiPart=True ); ogrJoin = grs_to_shp(grsJoin, os.path.join( workGrass, 'join_loc', grsJoin + '_grs.shp'), None, asMultiPart=True) dfMain = tbl_to_obj(ogrMain) dfJoin = tbl_to_obj(ogrJoin) dfResult = dfMain.merge(dfJoin, how='inner', left_on='tocat', right_on='cat') dfResult.drop(["geometry_y", "cat_y"], axis=1, inplace=True) dfResult.rename(columns={"cat_x" : "cat_grass"}, inplace=True) dfResult["tocat"] = dfResult["tocat"] - 1 dfResult["cat_grass"] = dfResult["cat_grass"] - 1 dfResult = regulardf_to_geodf(dfResult, "geometry_x", epsg_code) df_to_shp(dfResult, output) return output
def v_break_at_points(workspace, loc, lineShp, pntShp, conParam, srs, out_correct, out_tocorrect): """ Break lines at points - Based on GRASS GIS v.edit Use PostGIS to sanitize the result TODO: Confirm utility """ import os from gasp.session import run_grass from gasp.sql.mng.db import create_db from gasp.to.sql import shp_to_psql from gasp.to.shp import psql_to_shp from gasp.sql.mng.qw import ntbl_by_query from gasp.oss import get_filename tmpFiles = os.path.join(workspace, loc) gbase = run_grass(workspace, location=loc, srs=srs) import grass.script as grass import grass.script.setup as gsetup gsetup.init(gbase, workspace, loc, 'PERMANENT') from gasp.to.shp.grs import shp_to_grs, grs_to_shp grsLine = shp_to_grs( lineShp, get_filename(lineShp, forceLower=True) ) vedit_break(grsLine, pntShp, geomType='line') LINES = grass_converter( grsLine, os.path.join(tmpFiles, grsLine + '_v1.shp'), 'line') # Sanitize output of v.edit.break using PostGIS create_db(conParam, conParam["DB"], overwrite=True) conParam["DATABASE"] = conParam["DB"] LINES_TABLE = shp_to_psql( conParam, LINES, srs, pgTable=get_filename(LINES, forceLower=True), api="shp2pgsql" ) # Delete old/original lines and stay only with the breaked one Q = ( "SELECT {t}.*, foo.cat_count FROM {t} INNER JOIN (" "SELECT cat, COUNT(cat) AS cat_count, " "MAX(ST_Length(geom)) AS max_len " "FROM {t} GROUP BY cat" ") AS foo ON {t}.cat = foo.cat " "WHERE foo.cat_count = 1 OR foo.cat_count = 2 OR (" "foo.cat_count = 3 AND ST_Length({t}.geom) <= foo.max_len)" ).format(t=LINES_TABLE) CORR_LINES = ntbl_by_query( conParam, "{}_corrected".format(LINES_TABLE), Q, api='psql' ) # TODO: Delete Rows that have exactly the same geometry # Highlight problems that the user must solve case by case Q = ( "SELECT {t}.*, foo.cat_count FROM {t} INNER JOIN (" "SELECT cat, COUNT(cat) AS cat_count FROM {t} GROUP BY cat" ") AS foo ON {t}.cat = foo.cat " "WHERE foo.cat_count > 3" ).format(t=LINES_TABLE) ERROR_LINES = ntbl_by_query( conParam, "{}_not_corr".format(LINES_TABLE), Q, api='psql' ) psql_to_shp( conParam, CORR_LINES, out_correct, api="pgsql2shp", geom_col="geom" ) psql_to_shp( conParam, ERROR_LINES, out_tocorrect, api="pgsql2shp", geom_col="geom" )
def make_DEM(grass_workspace, data, field, output, extent_template, method="IDW"): """ Create Digital Elevation Model Methods Available: * IDW; * BSPLINE; * SPLINE; * CONTOUR """ from gasp.oss import get_filename from gasp.session import run_grass from gasp.prop.rst import get_epsg_raster LOC_NAME = get_filename(data, forceLower=True)[:5] + "_loc" # Get EPSG From Raster EPSG = get_epsg_raster(extent_template) # Create GRASS GIS Location grass_base = run_grass(grass_workspace, location=LOC_NAME, srs=EPSG) # Start GRASS GIS Session import grass.script as grass import grass.script.setup as gsetup gsetup.init(grass_base, grass_workspace, LOC_NAME, 'PERMANENT') # IMPORT GRASS GIS MODULES # from gasp.to.rst import rst_to_grs, grs_to_rst from gasp.to.shp.grs import shp_to_grs from gasp.prop.grs import rst_to_region # Configure region rst_to_grs(extent_template, 'extent') rst_to_region('extent') # Convert elevation "data" to GRASS Vector elv = shp_to_grs(data, 'elevation') OUTPUT_NAME = get_filename(output, forceLower=True) if method == "BSPLINE": # Convert to points from gasp.cpu.grs.mng.feat import feat_vertex_to_pnt from gasp.spanlst.interp import bspline elev_pnt = feat_vertex_to_pnt(elv, "elev_pnt", nodes=None) outRst = bspline(elev_pnt, field, OUTPUT_NAME, lyrN=1, asCMD=True) elif method == "SPLINE": # Convert to points from gasp.cpu.grs.mng.feat import feat_vertex_to_pnt from gasp.spanlst.interp import surfrst elev_pnt = feat_vertex_to_pnt(elv, "elev_pnt", nodes=None) outRst = surfrst(elev_pnt, field, OUTPUT_NAME, lyrN=1, ascmd=True) elif method == "CONTOUR": from gasp.to.rst import shp_to_raster from gasp.spanlst.interp import surfcontour # Elevation (GRASS Vector) to Raster elevRst = shp_to_raster(elv, field, None, None, 'rst_elevation', api="pygrass") # Run Interpolator outRst = surfcontour(elevRst, OUTPUT_NAME, ascmd=True) elif method == "IDW": from gasp.spanlst.interp import ridw from gasp.spanlst.algebra import rstcalc from gasp.to.rst import shp_to_raster # Elevation (GRASS Vector) to Raster elevRst = shp_to_raster(elv, field, None, None, 'rst_elevation', api='pygrass') # Multiply cells values by 100 000.0 rstcalc('int(rst_elevation * 100000)', 'rst_elev_int', api='pygrass') # Run IDW to generate the new DEM ridw('rst_elev_int', 'dem_int', numberPoints=15) # DEM to Float rstcalc('dem_int / 100000.0', OUTPUT_NAME, api='pygrass') # Export DEM to a file outside GRASS Workspace grs_to_rst(OUTPUT_NAME, output) return output