def acumulated_cost(cst_surface, dest_pnt, cst_dist): """ Uses a cost surface to estimate the time between each cell and the close destination """ from gasp.spanlst.algebra import rstcalc from gasp.spanlst.dist import rcost from gasp.to.rst import rst_to_grs, grs_to_rst from gasp.to.shp.grs import shp_to_grs # Add Cost Surface to GRASS GIS rst_to_grs(cst_surface, 'cst_surf') # Add Destination To GRASS shp_to_grs(dest_pnt, 'destination') # Execute r.cost rcost('cst_surf', 'destination', 'cst_dist') # Convert to minutes rstcalc('cst_dist / 60.0', 'CstDistMin', api="grass") # Export result grs_to_rst('CstDistMin', cst_dist) return cst_dist
def raster_based(osmdata, nomenclature, refRaster, lulcRst, overwrite=None, dataStore=None, roadsAPI='SQLITE'): """ Convert OSM Data into Land Use/Land Cover Information An raster based approach. TODO: Add detailed description """ # ************************************************************************ # # Python Modules from Reference Packages # # ************************************************************************ # import datetime import os import pandas import json # ************************************************************************ # # Gasp dependencies # # ************************************************************************ # from gasp.oss.ops import create_folder from gasp.prop.rst import get_epsg_raster from gasp.session import run_grass if roadsAPI == 'POSTGIS': from gasp.sql.mng.db import create_db from gasp.osm2lulc.utils import osm_to_pgsql from gasp.osm2lulc.mod2 import roads_sqdb else: from gasp.osm2lulc.utils import osm_to_sqdb from gasp.osm2lulc.mod2 import grs_rst_roads from gasp.osm2lulc.utils import osm_project, add_lulc_to_osmfeat from gasp.osm2lulc.mod1 import grs_rst from gasp.osm2lulc.m3_4 import rst_area from gasp.osm2lulc.mod5 import basic_buffer from gasp.osm2lulc.mod6 import rst_pnt_to_build # ************************************************************************ # # Global Settings # # ************************************************************************ # if not os.path.exists(os.path.dirname(lulcRst)): raise ValueError('{} does not exist!'.format(os.path.dirname(lulcRst))) # Get EPSG of Reference Raster epsg = get_epsg_raster(refRaster) if not epsg: raise ValueError('Cannot get epsg code of ref raster') # Get Parameters to connect to PostgreSQL conPGSQL = json.load( open( os.path.join(os.path.dirname(os.path.abspath(__file__)), 'con-postgresql.json'), 'r')) if roadsAPI == 'POSTGIS' else None time_a = datetime.datetime.now().replace(microsecond=0) from gasp.osm2lulc.var import PRIORITIES, osmTableData workspace = os.path.join(os.path.dirname(lulcRst), 'osmtolulc') if not dataStore else dataStore # Check if workspace exists if os.path.exists(workspace): if overwrite: create_folder(workspace) else: raise ValueError('Path {} already exists'.format(workspace)) else: create_folder(workspace) __priorites = PRIORITIES[nomenclature] time_b = datetime.datetime.now().replace(microsecond=0) # ************************************************************************ # # Convert OSM file to SQLITE DB or to POSTGIS DB # # ************************************************************************ # if roadsAPI == 'POSTGIS': conPGSQL["DATABASE"] = create_db(conPGSQL, os.path.splitext( os.path.basename(osmdata))[0], overwrite=True) osm_db = osm_to_pgsql(osmdata, conPGSQL) else: osm_db = osm_to_sqdb(osmdata, os.path.join(workspace, 'osm.sqlite')) time_c = datetime.datetime.now().replace(microsecond=0) # ************************************************************************ # # Add Lulc Classes to OSM_FEATURES by rule # # ************************************************************************ # add_lulc_to_osmfeat(conPGSQL if roadsAPI == 'POSTGIS' else osm_db, osmTableData, nomenclature, api=roadsAPI) time_d = datetime.datetime.now().replace(microsecond=0) # ************************************************************************ # # Transform SRS of OSM Data # # ************************************************************************ # osmTableData = osm_project(conPGSQL if roadsAPI == 'POSTGIS' else osm_db, epsg, api=roadsAPI) time_e = datetime.datetime.now().replace(microsecond=0) # ************************************************************************ # # Start a GRASS GIS Session # # ************************************************************************ # grass_base = run_grass(workspace, grassBIN='grass76', location='grloc', srs=epsg) import grass.script as grass import grass.script.setup as gsetup gsetup.init(grass_base, workspace, 'grloc', 'PERMANENT') # ************************************************************************ # # IMPORT SOME GASP MODULES FOR GRASS GIS # # ************************************************************************ # from gasp.to.rst import rst_to_grs, grs_to_rst from gasp.cpu.grs.spanlst import mosaic_raster from gasp.prop.grs import rst_to_region # ************************************************************************ # # SET GRASS GIS LOCATION EXTENT # # ************************************************************************ # extRst = rst_to_grs(refRaster, 'extent_raster') rst_to_region(extRst) time_f = datetime.datetime.now().replace(microsecond=0) # ************************************************************************ # # MapResults # mergeOut = {} # ************************************************************************ # # ************************************************************************ # # 1 - Selection Rule # # ************************************************************************ # """ selOut = { cls_code : rst_name, ... } """ selOut, timeCheck1 = grs_rst(conPGSQL if roadsAPI == 'POSTGIS' else osm_db, osmTableData['polygons'], api=roadsAPI) for cls in selOut: mergeOut[cls] = [selOut[cls]] time_g = datetime.datetime.now().replace(microsecond=0) # ************************************************************************ # # 2 - Get Information About Roads Location # # ************************************************************************ # """ roads = { cls_code : rst_name, ... } """ if roadsAPI != 'POSTGIS': roads, timeCheck2 = grs_rst_roads( osm_db, osmTableData['lines'], osmTableData['polygons'], workspace, 1221 if nomenclature != "GLOBE_LAND_30" else 801) else: roadCls = 1221 if nomenclature != "GLOBE_LAND_30" else 801 roads, timeCheck2 = roads_sqdb(conPGSQL, osmTableData['lines'], osmTableData['polygons'], apidb='POSTGIS', asRst=roadCls) roads = {roadCls: roads} for cls in roads: if cls not in mergeOut: mergeOut[cls] = [roads[cls]] else: mergeOut[cls].append(roads[cls]) time_h = datetime.datetime.now().replace(microsecond=0) # ************************************************************************ # # 3 - Area Upper than # # ************************************************************************ # """ auOut = { cls_code : rst_name, ... } """ auOut, timeCheck3 = rst_area(conPGSQL if roadsAPI == 'POSTGIS' else osm_db, osmTableData['polygons'], UPPER=True, api=roadsAPI) for cls in auOut: if cls not in mergeOut: mergeOut[cls] = [auOut[cls]] else: mergeOut[cls].append(auOut[cls]) time_l = datetime.datetime.now().replace(microsecond=0) # ************************************************************************ # # 4 - Area Lower than # # ************************************************************************ # """ alOut = { cls_code : rst_name, ... } """ alOut, timeCheck4 = rst_area(conPGSQL if roadsAPI == 'POSTGIS' else osm_db, osmTableData['polygons'], UPPER=None, api=roadsAPI) for cls in alOut: if cls not in mergeOut: mergeOut[cls] = [alOut[cls]] else: mergeOut[cls].append(alOut[cls]) time_j = datetime.datetime.now().replace(microsecond=0) # ************************************************************************ # # 5 - Get data from lines table (railway | waterway) # # ************************************************************************ # """ bfOut = { cls_code : rst_name, ... } """ bfOut, timeCheck5 = basic_buffer( conPGSQL if roadsAPI == 'POSTGIS' else osm_db, osmTableData['lines'], workspace, apidb=roadsAPI) for cls in bfOut: if cls not in mergeOut: mergeOut[cls] = [bfOut[cls]] else: mergeOut[cls].append(bfOut[cls]) time_m = datetime.datetime.now().replace(microsecond=0) # ************************************************************************ # # 7 - Assign untagged Buildings to tags # # ************************************************************************ # if nomenclature != "GLOBE_LAND_30": buildsOut, timeCheck7 = rst_pnt_to_build( conPGSQL if roadsAPI == 'POSTGIS' else osm_db, osmTableData['points'], osmTableData['polygons'], api_db=roadsAPI) for cls in buildsOut: if cls not in mergeOut: mergeOut[cls] = buildsOut[cls] else: mergeOut[cls] += buildsOut[cls] time_n = datetime.datetime.now().replace(microsecond=0) else: timeCheck7 = None time_n = datetime.datetime.now().replace(microsecond=0) # ************************************************************************ # # Produce LULC Map # # ************************************************************************ # """ Merge all results for one cls into one raster mergeOut = { cls_code : [rst_name, rst_name, ...], ... } into mergeOut = { cls_code : patched_raster, ... } """ for cls in mergeOut: if len(mergeOut[cls]) == 1: mergeOut[cls] = mergeOut[cls][0] else: mergeOut[cls] = mosaic_raster(mergeOut[cls], 'mosaic_{}'.format(str(cls)), asCmd=True) time_o = datetime.datetime.now().replace(microsecond=0) """ Merge all Class Raster using a priority rule """ __priorities = PRIORITIES[nomenclature] lst_rst = [] for cls in __priorities: if cls not in mergeOut: continue else: lst_rst.append(mergeOut[cls]) outGrs = mosaic_raster(lst_rst, os.path.splitext(os.path.basename(lulcRst))[0], asCmd=True) time_p = datetime.datetime.now().replace(microsecond=0) grs_to_rst(outGrs, lulcRst, as_cmd=True) time_q = datetime.datetime.now().replace(microsecond=0) return lulcRst, { 0: ('set_settings', time_b - time_a), 1: ('osm_to_sqdb', time_c - time_b), 2: ('cls_in_sqdb', time_d - time_c), 3: ('proj_data', time_e - time_d), 4: ('set_grass', time_f - time_e), 5: ('rule_1', time_g - time_f, timeCheck1), 6: ('rule_2', time_h - time_g, timeCheck2), 7: ('rule_3', time_l - time_h, timeCheck3), 8: ('rule_4', time_j - time_l, timeCheck4), 9: ('rule_5', time_m - time_j, timeCheck5), 10: None if not timeCheck7 else ('rule_7', time_n - time_m, timeCheck7), 11: ('merge_rst', time_o - time_n), 12: ('priority_rule', time_p - time_o), 13: ('export_rst', time_q - time_p) }
def vector_based(osmdata, nomenclature, refRaster, lulcShp, overwrite=None, dataStore=None, RoadsAPI='SQLITE'): """ Convert OSM Data into Land Use/Land Cover Information An vector based approach. TODO: Add a detailed description. RoadsAPI Options: * SQLITE * POSTGIS """ # ************************************************************************ # # Python Modules from Reference Packages # # ************************************************************************ # import datetime import os import json # ************************************************************************ # # GASP dependencies # # ************************************************************************ # from gasp.oss.ops import create_folder from gasp.prop.rst import get_epsg_raster from gasp.session import run_grass if RoadsAPI == 'POSTGIS': from gasp.sql.mng.db import create_db from gasp.osm2lulc.utils import osm_to_pgsql else: from gasp.osm2lulc.utils import osm_to_sqdb from gasp.osm2lulc.utils import osm_project, add_lulc_to_osmfeat from gasp.mng.gen import merge_feat from gasp.osm2lulc.mod1 import grs_vector if RoadsAPI == 'SQLITE' or RoadsAPI == 'POSTGIS': from gasp.osm2lulc.mod2 import roads_sqdb else: from gasp.osm2lulc.mod2 import grs_vec_roads from gasp.osm2lulc.m3_4 import grs_vect_selbyarea from gasp.osm2lulc.mod5 import grs_vect_bbuffer from gasp.osm2lulc.mod6 import vector_assign_pntags_to_build # ************************************************************************ # # Global Settings # # ************************************************************************ # if not os.path.exists(os.path.dirname(lulcShp)): raise ValueError('{} does not exist!'.format(os.path.dirname(lulcShp))) # Get Parameters to connect to PostgreSQL conPGSQL = json.load( open( os.path.join(os.path.dirname(os.path.abspath(__file__)), 'con-postgresql.json'), 'r')) if RoadsAPI == 'POSTGIS' else None # Get EPSG of Reference Raster epsg = get_epsg_raster(refRaster) if not epsg: raise ValueError('Cannot get epsg code of ref raster') time_a = datetime.datetime.now().replace(microsecond=0) from gasp.osm2lulc.var import osmTableData, PRIORITIES workspace = os.path.join(os.path.dirname(lulcShp), 'osmtolulc') if not dataStore else dataStore # Check if workspace exists if os.path.exists(workspace): if overwrite: create_folder(workspace) else: raise ValueError('Path {} already exists'.format(workspace)) else: create_folder(workspace) __priorities = PRIORITIES[nomenclature] time_b = datetime.datetime.now().replace(microsecond=0) if RoadsAPI != 'POSTGIS': # ******************************************************************** # # Convert OSM file to SQLITE DB # # ******************************************************************** # osm_db = osm_to_sqdb(osmdata, os.path.join(workspace, 'osm.sqlite')) else: # Convert OSM file to POSTGRESQL DB # conPGSQL["DATABASE"] = create_db(conPGSQL, os.path.splitext( os.path.basename(osmdata))[0], overwrite=True) osm_db = osm_to_pgsql(osmdata, conPGSQL) time_c = datetime.datetime.now().replace(microsecond=0) # ************************************************************************ # # Add Lulc Classes to OSM_FEATURES by rule # # ************************************************************************ # add_lulc_to_osmfeat(osm_db if RoadsAPI != 'POSTGIS' else conPGSQL, osmTableData, nomenclature, api='SQLITE' if RoadsAPI != 'POSTGIS' else RoadsAPI) time_d = datetime.datetime.now().replace(microsecond=0) # ************************************************************************ # # Transform SRS of OSM Data # # ************************************************************************ # osmTableData = osm_project( osm_db if RoadsAPI != 'POSTGIS' else conPGSQL, epsg, api='SQLITE' if RoadsAPI != 'POSTGIS' else RoadsAPI) time_e = datetime.datetime.now().replace(microsecond=0) # ************************************************************************ # # Start a GRASS GIS Session # # ************************************************************************ # grass_base = run_grass(workspace, grassBIN='grass76', location='grloc', srs=epsg) #import grass.script as grass import grass.script.setup as gsetup gsetup.init(grass_base, workspace, 'grloc', 'PERMANENT') # ************************************************************************ # # IMPORT SOME GASP MODULES FOR GRASS GIS # # ************************************************************************ # from gasp.anls.ovlay import erase from gasp.prop.grs import rst_to_region from gasp.mng.genze import dissolve from gasp.mng.grstbl import add_and_update, reset_table from gasp.to.shp.grs import shp_to_grs, grs_to_shp from gasp.to.rst import rst_to_grs # ************************************************************************ # # SET GRASS GIS LOCATION EXTENT # # ************************************************************************ # extRst = rst_to_grs(refRaster, 'extent_raster') rst_to_region(extRst) time_f = datetime.datetime.now().replace(microsecond=0) # ************************************************************************ # # MapResults # # ************************************************************************ # osmShps = [] # ************************************************************************ # # 1 - Selection Rule # # ************************************************************************ # ruleOneShp, timeCheck1 = grs_vector( osm_db if RoadsAPI != 'POSTGIS' else conPGSQL, osmTableData['polygons'], apidb=RoadsAPI) osmShps.append(ruleOneShp) time_g = datetime.datetime.now().replace(microsecond=0) # ************************************************************************ # # 2 - Get Information About Roads Location # # ************************************************************************ # ruleRowShp, timeCheck2 = roads_sqdb( osm_db if RoadsAPI == 'SQLITE' else conPGSQL, osmTableData['lines'], osmTableData['polygons'], apidb=RoadsAPI ) if RoadsAPI == 'SQLITE' or RoadsAPI == 'POSTGIS' else grs_vec_roads( osm_db, osmTableData['lines'], osmTableData['polygons']) osmShps.append(ruleRowShp) time_h = datetime.datetime.now().replace(microsecond=0) # ************************************************************************ # # 3 - Area Upper than # # ************************************************************************ # ruleThreeShp, timeCheck3 = grs_vect_selbyarea( osm_db if RoadsAPI != 'POSTGIS' else conPGSQL, osmTableData['polygons'], UPPER=True, apidb=RoadsAPI) osmShps.append(ruleThreeShp) time_l = datetime.datetime.now().replace(microsecond=0) # ************************************************************************ # # 4 - Area Lower than # # ************************************************************************ # ruleFourShp, timeCheck4 = grs_vect_selbyarea( osm_db if RoadsAPI != 'POSTGIS' else conPGSQL, osmTableData['polygons'], UPPER=False, apidb=RoadsAPI) osmShps.append(ruleFourShp) time_j = datetime.datetime.now().replace(microsecond=0) # ************************************************************************ # # 5 - Get data from lines table (railway | waterway) # # ************************************************************************ # ruleFiveShp, timeCheck5 = grs_vect_bbuffer( osm_db if RoadsAPI != 'POSTGIS' else conPGSQL, osmTableData["lines"], api_db=RoadsAPI) osmShps.append(ruleFiveShp) time_m = datetime.datetime.now().replace(microsecond=0) # ************************************************************************ # # 7 - Assign untagged Buildings to tags # # ************************************************************************ # if nomenclature != "GLOBE_LAND_30": ruleSeven11, ruleSeven12, timeCheck7 = vector_assign_pntags_to_build( osm_db if RoadsAPI != 'POSTGIS' else conPGSQL, osmTableData['points'], osmTableData['polygons'], apidb=RoadsAPI) if ruleSeven11: osmShps.append(ruleSeven11) if ruleSeven12: osmShps.append(ruleSeven12) time_n = datetime.datetime.now().replace(microsecond=0) else: timeCheck7 = None time_n = datetime.datetime.now().replace(microsecond=0) # ************************************************************************ # # Produce LULC Map # # ************************************************************************ # """ Get Shps with all geometries related with one class - One Shape for Classe """ from gasp.mng.gen import same_attr_to_shp _osmShps = [] for i in range(len(osmShps)): if not osmShps[i]: continue _osmShps.append( grs_to_shp(osmShps[i], os.path.join(workspace, osmShps[i] + '.shp'), 'auto', lyrN=1, asCMD=True, asMultiPart=None)) _osmShps = same_attr_to_shp(_osmShps, "cat", workspace, "osm_", resultDict=True) del osmShps time_o = datetime.datetime.now().replace(microsecond=0) """ Merge all Classes into one feature class using a priority rule """ osmShps = {} for cls in _osmShps: if cls == '1': osmShps[1221] = shp_to_grs(_osmShps[cls], "osm_1221", asCMD=True) else: osmShps[int(cls)] = shp_to_grs(_osmShps[cls], "osm_" + cls, asCMD=True) # Erase overlapping areas by priority import copy osmNameRef = copy.deepcopy(osmShps) for e in range(len(__priorities)): if e + 1 == len(__priorities): break if __priorities[e] not in osmShps: continue else: for i in range(e + 1, len(__priorities)): if __priorities[i] not in osmShps: continue else: osmShps[__priorities[i]] = erase( osmShps[__priorities[i]], osmShps[__priorities[e]], "{}_{}".format(osmNameRef[__priorities[i]], e), notTbl=True, api='pygrass') time_p = datetime.datetime.now().replace(microsecond=0) # Export all classes lst_merge = [] for cls in osmShps: if cls == __priorities[0]: reset_table(osmShps[cls], {'cls': 'varchar(5)'}, {'cls': str(cls)}) else: add_and_update(osmShps[cls], {'cls': 'varchar(5)'}, {'cls': str(cls)}) ds = dissolve(osmShps[cls], 'dl_{}'.format(str(cls)), 'cls', api="grass") lst_merge.append( grs_to_shp(ds, os.path.join(workspace, "lulc_{}.shp".format(str(cls))), 'auto', lyrN=1, asCMD=True, asMultiPart=None)) time_q = datetime.datetime.now().replace(microsecond=0) merge_feat(lst_merge, lulcShp, api='pandas') time_r = datetime.datetime.now().replace(microsecond=0) return lulcShp, { 0: ('set_settings', time_b - time_a), 1: ('osm_to_sqdb', time_c - time_b), 2: ('cls_in_sqdb', time_d - time_c), 3: ('proj_data', time_e - time_d), 4: ('set_grass', time_f - time_e), 5: ('rule_1', time_g - time_f, timeCheck1), 6: ('rule_2', time_h - time_g, timeCheck2), 7: ('rule_3', time_l - time_h, timeCheck3), 8: ('rule_4', time_j - time_l, timeCheck4), 9: ('rule_5', time_m - time_j, timeCheck5), 10: None if not timeCheck7 else ('rule_7', time_n - time_m, timeCheck7), 11: ('disj_cls', time_o - time_n), 12: ('priority_rule', time_p - time_o), 13: ('export_cls', time_q - time_p), 14: ('merge_cls', time_r - time_q) }
def cost_surface(dem, lulc, cls_lulc, prod_lulc, roads, kph, barr, grass_location, output, grass_path=None): """ Tool for make a cost surface based on the roads, slope, land use and physical barriers. ach cell has a value that represents the resistance to the movement. """ import os from gasp.oss.ops import create_folder from gasp.os import os_name from gasp.session import run_grass from gasp.prop.rst import get_cellsize from gasp.prop.rst import rst_distinct from .constants import lulc_weight from .constants import get_slope_categories """ Auxiliar Methods """ def edit_lulc(shp, fld_cls, new_cls): FT_TF_GRASS(shp, 'lulc', 'None') add_field('lulc', 'leg', 'INT') for key in new_cls.keys(): l = new_cls[key]['cls'] sql = " OR ".join([ "{campo}='{value}'".format(campo=fld_cls, value=i) for i in l ]) update_table('lulc', 'leg', int(key), sql) return {'shp': 'lulc', 'fld': 'leg'} def combine_to_cost(rst_combined, lst_rst, work, slope_weight, rdv_cos_weight, cellsize, mode_movement): # The tool r.report doesn't work properly, for that we need some aditional information l = [] for i in lst_rst: FT_TF_GRASS(i, os.path.join(work, i + '.tif'), 'None') values = rst_distinct(os.path.join(work, i + '.tif'), gisApi='gdal') l.append(min(values)) # ****** # Now, we can procede normaly txt_file = os.path.join(work, 'text_combine.txt') raster_report(rst_combined, txt_file) open_txt = open(txt_file, 'r') c = 0 dic_combine = {} for line in open_txt.readlines(): try: if c == 4: dic_combine[0] = [str(l[0]), str(l[1])] elif c >= 5: pl = line.split('|') cat = pl[2].split('; ') cat1 = cat[0].split(' ') cat2 = cat[1].split(' ') dic_combine[int(pl[1])] = [cat1[1], cat2[1]] c += 1 except: break cst_dic = {} for key in dic_combine.keys(): cls_slope = int(dic_combine[key][0]) cos_vias = int(dic_combine[key][1]) if cos_vias >= 6: weight4slope = slope_weight[cls_slope]['rdv'] if mode_movement == 'pedestrian': weight4other = (3600.0 * cellsize) / (5.0 * 1000.0) else: weight4other = (3600.0 * cellsize) / (cos_vias * 1000.0) else: weight4slope = slope_weight[cls_slope]['cos'] weight4other = rdv_cos_weight[cos_vias]['weight'] cst_dic[key] = (weight4slope * weight4other) * 10000000.0 return cst_dic def Rules4CstSurface(dic, work): txt = open(os.path.join(work, 'cst_surface.txt'), 'w') for key in dic.keys(): txt.write('{cat} = {cst}\n'.format(cat=str(key), cst=str(dic[key]))) txt.close() return os.path.join(work, 'cst_surface.txt') """ Prepare GRASS GIS Environment """ workspace = os.path.dirname(grass_location) location = os.path.basename(grass_location) # Start GRASS GIS Engine grass_base = run_grass(workspace, location, dem, win_path=grass_path) import grass.script as grass import grass.script.setup as gsetup gsetup.init(grass_base, workspace, location, 'PERMANENT') # Import GRASS GIS Modules from gasp.cpu.grs import grass_converter from gasp.spanlst.surf import slope from gasp.spanlst.rcls import reclassify from gasp.spanlst.rcls import interval_rules from gasp.spanlst.rcls import category_rules from gasp.spanlst.rcls import grass_set_null from gasp.mng.grstbl import add_field, update_table from gasp.anls.ovlay import union from gasp.to.rst import rst_to_grs, grs_to_rst from gasp.to.rst import shp_to_raster from gasp.to.shp.grs import shp_to_grs from gasp.cpu.grs.spanlst import mosaic_raster from gasp.spanlst.local import combine from gasp.spanlst.algebra import rstcalc from gasp.cpu.grs.spanlst import raster_report """Global variables""" # Workspace for temporary files wTmp = create_folder(os.path.join(workspace, 'tmp')) # Cellsize cellsize = float(get_cellsize(dem), gisApi='gdal') # Land Use Land Cover weights lulcWeight = lulc_weight(prod_lulc, cellsize) # Slope classes and weights slope_cls = get_slope_categories() """Make Cost Surface""" # Generate slope raster rst_to_grs(dem, 'dem') slope('dem', 'rst_slope', api="pygrass") # Reclassify Slope rulesSlope = interval_rules(slope_cls, os.path.join(wTmp, 'slope.txt')) reclassify('rst_slope', 'recls_slope', rulesSlope) # LULC - Dissolve, union with barriers and conversion to raster lulc_shp = edit_lulc(lulc, cls_lulc, lulc_weight) shp_to_grs(barr, 'barriers') union(lulc_shp['shp'], 'barriers', 'barrcos', api_gis="grass") update_table('barrcos', 'a_' + lulc_shp['fld'], 99, 'b_cat=1') shp_to_raster('barrcos', 'a_' + lulc_shp['fld'], None, None, 'rst_barrcos', api='pygrass') # Reclassify this raster - convert the values 99 to NULL or NODATA grass_set_null('rst_barrcos', 99) # Add the roads layer to the GRASS GIS shp_to_grs(roads, 'rdv') if kph == 'pedestrian': add_field('rdv', 'foot', 'INT') update_table('rdv', 'foot', 50, 'foot IS NULL') shp_to_raster('rdv', 'foot', None, None, 'rst_rdv', api='pygrass') else: shp_to_raster('rdv', kph, None, None, 'rst_rdv', api='pygrass') # Merge LULC/BARR and Roads mosaic_raster('rst_rdv', 'rst_barrcos', 'rdv_barrcos') # Combine LULC/BARR/ROADS with Slope combine('recls_slope', 'rdv_barrcos', 'rst_combine', api="pygrass") """ Estimating cost for every combination at rst_combine The order of the rasters on the following list has to be the same of GRASS Combine""" cst = combine_to_cost('rst_combine', ['recls_slope', 'rdv_barrcos'], wTmp, slope_cls, lulc_weight, cell_size, kph) # Reclassify combined rst rulesSurface = category_rules(cst, os.path.join('r_surface.txt')) reclassify('rst_combine', 'cst_tmp', rulesSurface) rstcalc('cst_tmp / 10000000.0', 'cst_surface', api='pygrass') grs_to_rst('cst_surface', output)
def check_shape_diff(SHAPES_TO_COMPARE, OUT_FOLDER, REPORT, conPARAM, DB, SRS_CODE, GIS_SOFTWARE="GRASS", GRASS_REGION_TEMPLATE=None): """ Script to check differences between pairs of Feature Classes Suponha que temos diversas Feature Classes (FC) e que cada uma delas possui um determinado atributo; imagine também que, considerando todos os pares possíveis entre estas FC, se pretende comparar as diferenças na distribuição dos valores desse atributo em cada par. * Dependências: - ArcGIS; - GRASS; - PostgreSQL; - PostGIS. * GIS_SOFTWARE Options: - ARCGIS; - GRASS. """ import datetime import os import pandas from gasp.fm.sql import query_to_df from gasp.sql.mng.tbl import tbls_to_tbl from gasp.sql.mng.geom import fix_geom, check_geomtype_in_table from gasp.sql.mng.geom import select_main_geom_type from gasp.sql.mng.qw import ntbl_by_query from gasp.prop.ff import check_isRaster from gasp.oss import get_filename from gasp.sql.mng.db import create_db from gasp.to.sql import shp_to_psql, df_to_db from gasp.to.shp import rst_to_polyg from gasp.to.shp import shp_to_shp, psql_to_shp from gasp.to import db_to_tbl # Check if folder exists, if not create it if not os.path.exists(OUT_FOLDER): from gasp.oss.ops import create_folder create_folder(OUT_FOLDER, overwrite=None) else: raise ValueError('{} already exists!'.format(OUT_FOLDER)) # Start GRASS GIS Session if GIS_SOFTWARE == GRASS if GIS_SOFTWARE == "GRASS": if not GRASS_REGION_TEMPLATE: raise ValueError( 'To use GRASS GIS you need to specify GRASS_REGION_TEMPLATE') from gasp.session import run_grass gbase = run_grass(OUT_FOLDER, grassBIN='grass76', location='shpdif', srs=GRASS_REGION_TEMPLATE) import grass.script as grass import grass.script.setup as gsetup gsetup.init(gbase, OUT_FOLDER, 'shpdif', 'PERMANENT') from gasp.mng.grstbl import rename_col from gasp.to.shp.grs import shp_to_grs, grs_to_shp from gasp.to.rst import rst_to_grs from gasp.mng.fld import rename_column # Convert to SHAPE if file is Raster # Import to GRASS GIS if GIS SOFTWARE == GRASS i = 0 _SHP_TO_COMPARE = {} for s in SHAPES_TO_COMPARE: isRaster = check_isRaster(s) if isRaster: if GIS_SOFTWARE == "ARCGIS": d = rst_to_polyg(s, os.path.join(os.path.dirname(s), get_filename(s) + '.shp'), gisApi='arcpy') _SHP_TO_COMPARE[d] = "gridcode" elif GIS_SOFTWARE == "GRASS": # To GRASS rstName = get_filename(s) inRst = rst_to_grs(s, "rst_" + rstName, as_cmd=True) # To Raster d = rst_to_polyg(inRst, rstName, rstColumn="lulc_{}".format(i), gisApi="grasscmd") # Export Shapefile shp = grs_to_shp(d, os.path.join(OUT_FOLDER, d + '.shp'), "area") _SHP_TO_COMPARE[shp] = "lulc_{}".format(i) else: if GIS_SOFTWARE == "ARCGIS": _SHP_TO_COMPARE[s] = SHAPES_TO_COMPARE[s] elif GIS_SOFTWARE == "GRASS": # To GRASS grsV = shp_to_grs(s, get_filename(s), asCMD=True) # Change name of column with comparing value rename_col(grsV, SHAPES_TO_COMPARE[s], "lulc_{}".format(i), as_cmd=True) # Export shp = grs_to_shp(grsV, os.path.join(OUT_FOLDER, grsV + '_rn.shp'), "area") _SHP_TO_COMPARE[shp] = "lulc_{}".format(i) i += 1 SHAPES_TO_COMPARE = _SHP_TO_COMPARE if GIS_SOFTWARE == "ARCGIS": from gasp.cpu.arcg.mng.fld import calc_fld from gasp.cpu.arcg.mng.wspace import create_geodb from gasp.mng.gen import copy_feat # Sanitize data and Add new field __SHAPES_TO_COMPARE = {} i = 0 # Create GeoDatabase geodb = create_geodb(OUT_FOLDER, 'geo_sanitize') """ Sanitize Data """ for k in SHAPES_TO_COMPARE: # Send data to GeoDatabase only to sanitize newFc = shp_to_shp(k, os.path.join(geodb, get_filename(k)), gisApi='arcpy') # Create a copy to change newShp = copy_feat(newFc, os.path.join(OUT_FOLDER, os.path.basename(k)), gisApi='arcpy') # Sanitize field name with interest data NEW_FLD = "lulc_{}".format(i) calc_fld(newShp, NEW_FLD, "[{}]".format(SHAPES_TO_COMPARE[k]), isNewField={ "TYPE": "INTEGER", "LENGTH": 5, "PRECISION": "" }) __SHAPES_TO_COMPARE[newShp] = NEW_FLD i += 1 else: __SHAPES_TO_COMPARE = SHAPES_TO_COMPARE # Create database conPARAM["DATABASE"] = create_db(conPARAM, DB) """ Union SHAPEs """ UNION_SHAPE = {} FIX_GEOM = {} def fix_geometry(shp): # Send data to PostgreSQL nt = shp_to_psql(conPARAM, shp, SRS_CODE, api='shp2pgsql') # Fix data corr_tbl = fix_geom(conPARAM, nt, "geom", "corr_{}".format(nt), colsSelect=['gid', __SHAPES_TO_COMPARE[shp]]) # Check if we have multiple geometries geomN = check_geomtype_in_table(conPARAM, corr_tbl) if geomN > 1: corr_tbl = select_main_geom_type(conPARAM, corr_tbl, "corr2_{}".format(nt)) # Export data again newShp = psql_to_shp(conPARAM, corr_tbl, os.path.join(OUT_FOLDER, corr_tbl + '.shp'), api='pgsql2shp', geom_col='geom') return newShp SHPS = __SHAPES_TO_COMPARE.keys() for i in range(len(SHPS)): for e in range(i + 1, len(SHPS)): if GIS_SOFTWARE == 'ARCGIS': # Try the union thing unShp = union(SHPS[i], SHPS[e], os.path.join(OUT_FOLDER, "un_{}_{}.shp".format(i, e)), api_gis="arcpy") # See if the union went all right if not os.path.exists(unShp): # Union went not well # See if geometry was already fixed if SHPS[i] not in FIX_GEOM: # Fix SHPS[i] geometry FIX_GEOM[SHPS[i]] = fix_geometry(SHPS[i]) if SHPS[e] not in FIX_GEOM: FIX_GEOM[SHPS[e]] = fix_geometry(SHPS[e]) # Run Union again unShp = union(FIX_GEOM[SHPS[i]], FIX_GEOM[SHPS[e]], os.path.join(OUT_FOLDER, "un_{}_{}_f.shp".format(i, e)), api_gis="arcpy") elif GIS_SOFTWARE == "GRASS": # Optimized Union print "Union between {} and {}".format(SHPS[i], SHPS[e]) time_a = datetime.datetime.now().replace(microsecond=0) __unShp = optimized_union_anls( SHPS[i], SHPS[e], os.path.join(OUT_FOLDER, "un_{}_{}.shp".format(i, e)), GRASS_REGION_TEMPLATE, SRS_CODE, os.path.join(OUT_FOLDER, "work_{}_{}".format(i, e)), multiProcess=True) time_b = datetime.datetime.now().replace(microsecond=0) print time_b - time_a # Rename cols unShp = rename_column( __unShp, { "a_" + __SHAPES_TO_COMPARE[SHPS[i]]: __SHAPES_TO_COMPARE[SHPS[i]], "b_" + __SHAPES_TO_COMPARE[SHPS[e]]: __SHAPES_TO_COMPARE[SHPS[e]] }, os.path.join(OUT_FOLDER, "un_{}_{}_rn.shp".format(i, e))) UNION_SHAPE[(SHPS[i], SHPS[e])] = unShp # Send data one more time to postgresql SYNTH_TBL = {} for uShp in UNION_SHAPE: # Send data to PostgreSQL union_tbl = shp_to_psql(conPARAM, UNION_SHAPE[uShp], SRS_CODE, api='shp2pgsql') # Produce table with % of area equal in both maps areaMapTbl = ntbl_by_query( conPARAM, "{}_syn".format(union_tbl), ("SELECT CAST('{lulc_1}' AS text) AS lulc_1, " "CAST('{lulc_2}' AS text) AS lulc_2, " "round(" "CAST(SUM(g_area) / 1000000 AS numeric), 4" ") AS agree_area, round(" "CAST((SUM(g_area) / MIN(total_area)) * 100 AS numeric), 4" ") AS agree_percentage, " "round(" "CAST(MIN(total_area) / 1000000 AS numeric), 4" ") AS total_area FROM (" "SELECT {map1_cls}, {map2_cls}, ST_Area(geom) AS g_area, " "CASE " "WHEN {map1_cls} = {map2_cls} " "THEN 1 ELSE 0 " "END AS isthesame, total_area FROM {tbl}, (" "SELECT SUM(ST_Area(geom)) AS total_area FROM {tbl}" ") AS foo2" ") AS foo WHERE isthesame = 1 " "GROUP BY isthesame").format( lulc_1=get_filename(uShp[0]), lulc_2=get_filename(uShp[1]), map1_cls=__SHAPES_TO_COMPARE[uShp[0]], map2_cls=__SHAPES_TO_COMPARE[uShp[1]], tbl=union_tbl), api='psql') # Produce confusion matrix for the pair in comparison lulcCls = query_to_df( conPARAM, ("SELECT fcol FROM (" "SELECT CAST({map1_cls} AS text) AS fcol FROM {tbl} " "GROUP BY {map1_cls} " "UNION ALL SELECT CAST({map2_cls} AS text) FROM {tbl} " "GROUP BY {map2_cls}" ") AS foo GROUP BY fcol ORDER BY fcol").format( tbl=union_tbl, map1_cls=__SHAPES_TO_COMPARE[uShp[0]], map2_cls=__SHAPES_TO_COMPARE[uShp[1]]), db_api='psql').fcol.tolist() matrixTbl = ntbl_by_query( conPARAM, "{}_matrix".format(union_tbl), ("SELECT * FROM crosstab('" "SELECT CASE " "WHEN foo.{map1_cls} IS NOT NULL " "THEN foo.{map1_cls} ELSE jtbl.flyr " "END AS lulc1_cls, CASE " "WHEN foo.{map2_cls} IS NOT NULL " "THEN foo.{map2_cls} ELSE jtbl.slyr " "END AS lulc2_cls, CASE " "WHEN foo.garea IS NOT NULL " "THEN round(CAST(foo.garea / 1000000 AS numeric)" ", 3) ELSE 0 " "END AS garea FROM (" "SELECT CAST({map1_cls} AS text) AS {map1_cls}, " "CAST({map2_cls} AS text) AS {map2_cls}, " "SUM(ST_Area(geom)) AS garea " "FROM {tbl} GROUP BY {map1_cls}, {map2_cls}" ") AS foo FULL JOIN (" "SELECT f.flyr, s.slyr FROM (" "SELECT CAST({map1_cls} AS text) AS flyr " "FROM {tbl} GROUP BY {map1_cls}" ") AS f, (" "SELECT CAST({map2_cls} AS text) AS slyr " "FROM {tbl} GROUP BY {map2_cls}" ") AS s" ") AS jtbl " "ON foo.{map1_cls} = jtbl.flyr AND " "foo.{map2_cls} = jtbl.slyr " "ORDER BY 1,2" "') AS ct(" "lulc_cls text, {crossCols}" ")").format(crossCols=", ".join( ["cls_{} numeric".format(c) for c in lulcCls]), tbl=union_tbl, map1_cls=__SHAPES_TO_COMPARE[uShp[0]], map2_cls=__SHAPES_TO_COMPARE[uShp[1]]), api='psql') SYNTH_TBL[uShp] = {"TOTAL": areaMapTbl, "MATRIX": matrixTbl} # UNION ALL TOTAL TABLES total_table = tbls_to_tbl(conPARAM, [SYNTH_TBL[k]["TOTAL"] for k in SYNTH_TBL], 'total_table') # Create table with % of agreement between each pair of maps mapsNames = query_to_df( conPARAM, ("SELECT lulc FROM (" "SELECT lulc_1 AS lulc FROM {tbl} GROUP BY lulc_1 " "UNION ALL " "SELECT lulc_2 AS lulc FROM {tbl} GROUP BY lulc_2" ") AS lu GROUP BY lulc ORDER BY lulc").format(tbl=total_table), db_api='psql').lulc.tolist() FLDS_TO_PIVOT = ["agree_percentage", "total_area"] Q = ("SELECT * FROM crosstab('" "SELECT CASE " "WHEN foo.lulc_1 IS NOT NULL THEN foo.lulc_1 ELSE jtbl.tmp1 " "END AS lulc_1, CASE " "WHEN foo.lulc_2 IS NOT NULL THEN foo.lulc_2 ELSE jtbl.tmp2 " "END AS lulc_2, CASE " "WHEN foo.{valCol} IS NOT NULL THEN foo.{valCol} ELSE 0 " "END AS agree_percentage FROM (" "SELECT lulc_1, lulc_2, {valCol} FROM {tbl} UNION ALL " "SELECT lulc_1, lulc_2, {valCol} FROM (" "SELECT lulc_1 AS lulc_2, lulc_2 AS lulc_1, {valCol} " "FROM {tbl}" ") AS tst" ") AS foo FULL JOIN (" "SELECT lulc_1 AS tmp1, lulc_2 AS tmp2 FROM (" "SELECT lulc_1 AS lulc_1 FROM {tbl} GROUP BY lulc_1 " "UNION ALL " "SELECT lulc_2 AS lulc_1 FROM {tbl} GROUP BY lulc_2" ") AS tst_1, (" "SELECT lulc_1 AS lulc_2 FROM {tbl} GROUP BY lulc_1 " "UNION ALL " "SELECT lulc_2 AS lulc_2 FROM {tbl} GROUP BY lulc_2" ") AS tst_2 WHERE lulc_1 = lulc_2 GROUP BY lulc_1, lulc_2" ") AS jtbl ON foo.lulc_1 = jtbl.tmp1 AND foo.lulc_2 = jtbl.tmp2 " "ORDER BY lulc_1, lulc_2" "') AS ct(" "lulc_map text, {crossCols}" ")") TOTAL_AGREE_TABLE = None TOTAL_AREA_TABLE = None for f in FLDS_TO_PIVOT: if not TOTAL_AGREE_TABLE: TOTAL_AGREE_TABLE = ntbl_by_query( conPARAM, "agreement_table", Q.format(tbl=total_table, valCol=f, crossCols=", ".join([ "{} numeric".format(map_) for map_ in mapsNames ])), api='psql') else: TOTAL_AREA_TABLE = ntbl_by_query( conPARAM, "area_table", Q.format(tbl=total_table, valCol=f, crossCols=", ".join([ "{} numeric".format(map_) for map_ in mapsNames ])), api='psql') # Union Mapping UNION_MAPPING = pandas.DataFrame( [[ get_filename(k[0]), get_filename(k[1]), get_filename(UNION_SHAPE[k]) ] for k in UNION_SHAPE], columns=['shp_a', 'shp_b', 'union_shp' ]) if GIS_SOFTWARE == "ARCGIS" else pandas.DataFrame( [[k[0], k[1], get_filename(UNION_SHAPE[k])] for k in UNION_SHAPE], columns=['shp_a', 'shp_b', 'union_shp']) UNION_MAPPING = df_to_db(conPARAM, UNION_MAPPING, 'union_map', api='psql') # Export Results TABLES = [UNION_MAPPING, TOTAL_AGREE_TABLE, TOTAL_AREA_TABLE ] + [SYNTH_TBL[x]["MATRIX"] for x in SYNTH_TBL] SHEETS = ["union_map", "agreement_percentage", "area_with_data_km"] + [ "{}_{}".format(get_filename(x[0])[:15], get_filename(x[1])[:15]) for x in SYNTH_TBL ] db_to_xls(conPARAM, ["SELECT * FROM {}".format(x) for x in TABLES], REPORT, sheetsNames=SHEETS, dbAPI='psql') return REPORT
def make_DEM(grass_workspace, data, field, output, extent_template, method="IDW"): """ Create Digital Elevation Model Methods Available: * IDW; * BSPLINE; * SPLINE; * CONTOUR """ from gasp.oss import get_filename from gasp.session import run_grass from gasp.prop.rst import get_epsg_raster LOC_NAME = get_filename(data, forceLower=True)[:5] + "_loc" # Get EPSG From Raster EPSG = get_epsg_raster(extent_template) # Create GRASS GIS Location grass_base = run_grass(grass_workspace, location=LOC_NAME, srs=EPSG) # Start GRASS GIS Session import grass.script as grass import grass.script.setup as gsetup gsetup.init(grass_base, grass_workspace, LOC_NAME, 'PERMANENT') # IMPORT GRASS GIS MODULES # from gasp.to.rst import rst_to_grs, grs_to_rst from gasp.to.shp.grs import shp_to_grs from gasp.prop.grs import rst_to_region # Configure region rst_to_grs(extent_template, 'extent') rst_to_region('extent') # Convert elevation "data" to GRASS Vector elv = shp_to_grs(data, 'elevation') OUTPUT_NAME = get_filename(output, forceLower=True) if method == "BSPLINE": # Convert to points from gasp.cpu.grs.mng.feat import feat_vertex_to_pnt from gasp.spanlst.interp import bspline elev_pnt = feat_vertex_to_pnt(elv, "elev_pnt", nodes=None) outRst = bspline(elev_pnt, field, OUTPUT_NAME, lyrN=1, asCMD=True) elif method == "SPLINE": # Convert to points from gasp.cpu.grs.mng.feat import feat_vertex_to_pnt from gasp.spanlst.interp import surfrst elev_pnt = feat_vertex_to_pnt(elv, "elev_pnt", nodes=None) outRst = surfrst(elev_pnt, field, OUTPUT_NAME, lyrN=1, ascmd=True) elif method == "CONTOUR": from gasp.to.rst import shp_to_raster from gasp.spanlst.interp import surfcontour # Elevation (GRASS Vector) to Raster elevRst = shp_to_raster(elv, field, None, None, 'rst_elevation', api="pygrass") # Run Interpolator outRst = surfcontour(elevRst, OUTPUT_NAME, ascmd=True) elif method == "IDW": from gasp.spanlst.interp import ridw from gasp.spanlst.algebra import rstcalc from gasp.to.rst import shp_to_raster # Elevation (GRASS Vector) to Raster elevRst = shp_to_raster(elv, field, None, None, 'rst_elevation', api='pygrass') # Multiply cells values by 100 000.0 rstcalc('int(rst_elevation * 100000)', 'rst_elev_int', api='pygrass') # Run IDW to generate the new DEM ridw('rst_elev_int', 'dem_int', numberPoints=15) # DEM to Float rstcalc('dem_int / 100000.0', OUTPUT_NAME, api='pygrass') # Export DEM to a file outside GRASS Workspace grs_to_rst(OUTPUT_NAME, output) return output