def grs_vect_selbyarea(osmdb, polyTbl, UPPER=True, apidb='SQLITE'): """ Select features with area upper than. A field with threshold is needed in the database. """ import datetime from glass.g.gp.gen import dissolve from glass.g.tbl.grs import add_table from glass.ete.osm2lulc import GEOM_AREA from glass.ng.prop.sql import row_num as cnt_row from glass.g.it.shp import dbtbl_to_shp as db_to_shp OPERATOR = ">" if UPPER else "<" DIRECTION = "upper" if UPPER else "lower" WHR = "{ga} {op} t_area_{r} and area_{r} IS NOT NULL".format(op=OPERATOR, r=DIRECTION, ga=GEOM_AREA) # Check if we have interest data time_a = datetime.datetime.now().replace(microsecond=0) N = cnt_row(osmdb, polyTbl, where=WHR, api='psql' if apidb == 'POSTGIS' else 'sqlite') time_b = datetime.datetime.now().replace(microsecond=0) if not N: return None, {0: ('count_rows', time_b - time_a)} # Data to GRASS GIS grsVect = db_to_shp(osmdb, polyTbl, "geometry", "area_{}".format(DIRECTION), where=WHR, inDB='psql' if apidb == 'POSTGIS' else 'sqlite', filterByReg=True, outShpIsGRASS=True) time_c = datetime.datetime.now().replace(microsecond=0) dissVect = dissolve(grsVect, "diss_area_{}".format(DIRECTION), "area_{}".format(DIRECTION), api="grass") add_table(dissVect, None, lyrN=1, asCMD=True) time_d = datetime.datetime.now().replace(microsecond=0) return dissVect, { 0: ('count_rows', time_b - time_a), 1: ('import', time_c - time_b), 2: ('dissolve', time_d - time_c) }
def grs_vect_bbuffer(osmdata, lineTbl, api_db='SQLITE'): """ Basic Buffer strategie """ import datetime from glass.g.gp.prox.bfing import _buffer from glass.g.gp.gen import dissolve from glass.g.tbl.grs import add_table from glass.ng.prop.sql import row_num as cnt_row from glass.g.it.shp import dbtbl_to_shp as db_to_shp WHR = "basic_buffer IS NOT NULL" # Check if we have data time_a = datetime.datetime.now().replace(microsecond=0) N = cnt_row(osmdata, lineTbl, where=WHR, api='psql' if api_db == 'POSTGIS' else 'sqlite') time_b = datetime.datetime.now().replace(microsecond=0) if not N: return None, {0: ('count_rows_roads', time_b - time_a)} grsVect = db_to_shp(osmdata, lineTbl, "geometry", "bb_lnh", where=WHR, filterByReg=True, inDB='psql' if api_db == 'POSTGIS' else 'sqlite', outShpIsGRASS=True) time_c = datetime.datetime.now().replace(microsecond=0) grsBuf = _buffer(grsVect, "bf_basic_buffer", "bb_poly", api="grass", geom_type="line") time_d = datetime.datetime.now().replace(microsecond=0) grsDiss = dissolve(grsBuf, "bb_diss", "basic_buffer", api="grass") add_table(grsDiss, None, lyrN=1, asCMD=True) time_e = datetime.datetime.now().replace(microsecond=0) return grsDiss, { 0: ('count_rows', time_b - time_a), 1: ('import', time_c - time_b), 2: ('buffer', time_d - time_c), 3: ('dissolve', time_e - time_d) }
def grs_vector(db, polyTable, apidb='SQLITE'): """ Simple Selection using GRASS GIS """ import datetime from glass.g.gp.gen import dissolve from glass.g.tbl.grs import add_table from glass.ng.prop.sql import row_num as cont_row from glass.g.it.shp import dbtbl_to_shp as db_to_grs WHR = "selection IS NOT NULL" # Check if we have interest data time_a = datetime.datetime.now().replace(microsecond=0) N = cont_row(db, polyTable, where=WHR, api='psql' if apidb == 'POSTGIS' else 'sqlite') time_b = datetime.datetime.now().replace(microsecond=0) if not N: return None, {0: ('count_rows', time_b - time_a)} # Data to GRASS GIS grsVect = db_to_grs(db, polyTable, "geometry", "sel_rule", where=WHR, filterByReg=True, inDB='psql' if apidb == 'POSTGIS' else 'sqlite', outShpIsGRASS=True) time_c = datetime.datetime.now().replace(microsecond=0) dissVect = dissolve(grsVect, "diss_sel_rule", "selection", api="grass") add_table(dissVect, None, lyrN=1, asCMD=True) time_d = datetime.datetime.now().replace(microsecond=0) return dissVect, { 0: ('count_rows', time_b - time_a), 1: ('import', time_c - time_b), 2: ('dissolve', time_d - time_c) }
def grs_vec_roads(osmdb, lineTbl, polyTbl): """ Select Roads for GRASS GIS """ import datetime from glass.ng.prop.sql import row_num from glass.g.it.shp import dbtbl_to_shp from glass.g.gp.prox.bfing import _buffer from glass.g.gp.gen import dissolve from glass.g.tbl.grs import add_table # Roads to GRASS GIS time_a = datetime.datetime.now().replace(microsecond=0) NR = row_num(osmdb, lineTbl, where="roads IS NOT NULL", api='sqlite') time_b = datetime.datetime.now().replace(microsecond=0) if not NR: return None, {0: ('count_rows_roads', time_b - time_a)} roadsVect = dbtbl_to_shp(osmdb, lineTbl, "geometry", "all_roads", where="roads IS NOT NULL", inDB='sqlite', outShpIsGRASS=True) time_c = datetime.datetime.now().replace(microsecond=0) # Buildings to GRASS GIS NB = row_num(osmdb, polyTbl, where="building IS NOT NULL", api='sqlite') time_d = datetime.datetime.now().replace(microsecond=0) if NB: from glass.g.gp.prox import grs_near as near from glass.g.tbl.grs import update_table builds = dbtbl_to_shp(osmdb, polyTbl, "geometry", "all_builds", where="building IS NOT NULL", filterByReg=True, inDB='sqlite', outShpIsGRASS=True) time_e = datetime.datetime.now().replace(microsecond=0) near(roadsVect, builds, nearDistCol="todist", maxDist=12, as_cmd=True) time_f = datetime.datetime.now().replace(microsecond=0) update_table(roadsVect, "bf_roads", "round(todist,0)", "\"todist > 0\"", lyrN=1, ascmd=True) time_g = datetime.datetime.now().replace(microsecond=0) else: time_e = None time_f = None time_g = None # Run Buffer tool roadsBf = _buffer(roadsVect, "bf_roads", "bf_roads", api='grass', geom_type="line") time_h = datetime.datetime.now().replace(microsecond=0) # Dissolve Roads roadsDiss = dissolve(roadsBf, "diss_roads", "roads", api="grass") add_table(roadsDiss, None, lyrN=1, asCMD=True) time_i = datetime.datetime.now().replace(microsecond=0) return roadsDiss, { 0: ('count_rows_roads', time_b - time_a), 1: ('import_roads', time_c - time_b), 2: ('count_rows_build', time_d - time_c), 3: None if not time_e else ('import_builds', time_e - time_d), 4: None if not time_f else ('near_analysis', time_f - time_e), 5: None if not time_g else ('update_buffer_tbl', time_g - time_f), 6: ('buffer_roads', time_h - time_g if time_g else time_h - time_d), 7: ('diss_roads', time_i - time_h) }
def vector_based(osmdata, nomenclature, refRaster, lulcShp, overwrite=None, dataStore=None, RoadsAPI='POSTGIS'): """ Convert OSM Data into Land Use/Land Cover Information An vector based approach. TODO: Add a detailed description. RoadsAPI Options: * GRASS * SQLITE * POSTGIS """ # ************************************************************************ # # Python Modules from Reference Packages # # ************************************************************************ # import datetime; import os; import copy # ************************************************************************ # # glass dependencies # # ************************************************************************ # from glass.pys.oss import fprop, mkdir from glass.g.wenv.grs import run_grass if RoadsAPI == 'POSTGIS': from glass.ng.sql.db import create_db from glass.g.it.db import osm_to_psql from glass.ng.sql.db import drop_db from glass.ng.sql.bkup import dump_db else: from glass.g.it.osm import osm_to_sqdb from glass.ete.osm2lulc.utils import osm_project, add_lulc_to_osmfeat, get_ref_raster from glass.g.dp.mge import shps_to_shp from glass.ete.osm2lulc.mod1 import grs_vector if RoadsAPI == 'SQLITE' or RoadsAPI == 'POSTGIS': from glass.ete.osm2lulc.mod2 import roads_sqdb else: from glass.ete.osm2lulc.mod2 import grs_vec_roads from glass.ete.osm2lulc.m3_4 import grs_vect_selbyarea from glass.ete.osm2lulc.mod5 import grs_vect_bbuffer from glass.ete.osm2lulc.mod6 import vector_assign_pntags_to_build from glass.g.dp.mge import same_attr_to_shp from glass.g.prj import def_prj # ************************************************************************ # # Global Settings # # ************************************************************************ # # Check if input parameters exists! if not os.path.exists(os.path.dirname(lulcShp)): raise ValueError('{} does not exist!'.format(os.path.dirname(lulcShp))) if not os.path.exists(osmdata): raise ValueError('File with OSM DATA ({}) does not exist!'.format(osmdata)) if not os.path.exists(refRaster): raise ValueError('File with reference area ({}) does not exist!'.format(refRaster)) # Check if Nomenclature is valid nomenclature = "URBAN_ATLAS" if nomenclature != "URBAN_ATLAS" and \ nomenclature != "CORINE_LAND_COVER" and \ nomenclature == "GLOBE_LAND_30" else nomenclature time_a = datetime.datetime.now().replace(microsecond=0) # Create workspace for temporary files workspace = os.path.join(os.path.dirname( lulcShp), 'osmtolulc') if not dataStore else dataStore # Check if workspace exists if os.path.exists(workspace): if overwrite: mkdir(workspace) else: raise ValueError('Path {} already exists'.format(workspace)) else: mkdir(workspace) # Get Reference Raster refRaster, epsg = get_ref_raster(refRaster, workspace, cellsize=10) from glass.ete.osm2lulc import osmTableData, PRIORITIES, LEGEND __priorities = PRIORITIES[nomenclature] __legend = LEGEND[nomenclature] time_b = datetime.datetime.now().replace(microsecond=0) if RoadsAPI != 'POSTGIS': # ******************************************************************** # # Convert OSM file to SQLITE DB # # ******************************************************************** # osm_db = osm_to_sqdb(osmdata, os.path.join(workspace, 'osm.sqlite')) else: # Convert OSM file to POSTGRESQL DB # osm_db = create_db(fprop( osmdata, 'fn', forceLower=True), overwrite=True) osm_db = osm_to_psql(osmdata, osm_db) time_c = datetime.datetime.now().replace(microsecond=0) # ************************************************************************ # # Add Lulc Classes to OSM_FEATURES by rule # # ************************************************************************ # add_lulc_to_osmfeat( osm_db, osmTableData, nomenclature, api='SQLITE' if RoadsAPI != 'POSTGIS' else RoadsAPI ) time_d = datetime.datetime.now().replace(microsecond=0) # ************************************************************************ # # Transform SRS of OSM Data # # ************************************************************************ # osmTableData = osm_project( osm_db, epsg, api='SQLITE' if RoadsAPI != 'POSTGIS' else RoadsAPI, isGlobeLand=None if nomenclature != 'GLOBE_LAND_30' else True ) time_e = datetime.datetime.now().replace(microsecond=0) # ************************************************************************ # # Start a GRASS GIS Session # # ************************************************************************ # grass_base = run_grass(workspace, grassBIN='grass78', location='grloc', srs=epsg) #import grass.script as grass import grass.script.setup as gsetup gsetup.init(grass_base, workspace, 'grloc', 'PERMANENT') # ************************************************************************ # # IMPORT SOME glass MODULES FOR GRASS GIS # # ************************************************************************ # from glass.g.gp.ovl import erase from glass.g.wenv.grs import rst_to_region from glass.g.gp.gen import dissolve from glass.g.tbl.grs import add_and_update, reset_table, update_table from glass.g.tbl.col import add_fields from glass.g.it.shp import shp_to_grs, grs_to_shp from glass.g.it.rst import rst_to_grs # ************************************************************************ # # SET GRASS GIS LOCATION EXTENT # # ************************************************************************ # extRst = rst_to_grs(refRaster, 'extent_raster') rst_to_region(extRst) time_f = datetime.datetime.now().replace(microsecond=0) # ************************************************************************ # # MapResults # # ************************************************************************ # osmShps = [] # ************************************************************************ # # 1 - Selection Rule # # ************************************************************************ # ruleOneShp, timeCheck1 = grs_vector( osm_db, osmTableData['polygons'], apidb=RoadsAPI ) osmShps.append(ruleOneShp) time_g = datetime.datetime.now().replace(microsecond=0) # ************************************************************************ # # 2 - Get Information About Roads Location # # ************************************************************************ # ruleRowShp, timeCheck2 = roads_sqdb( osm_db, osmTableData['lines'], osmTableData['polygons'], apidb=RoadsAPI ) if RoadsAPI == 'SQLITE' or RoadsAPI == 'POSTGIS' else grs_vec_roads( osm_db, osmTableData['lines'], osmTableData['polygons']) osmShps.append(ruleRowShp) time_h = datetime.datetime.now().replace(microsecond=0) # ************************************************************************ # # 3 - Area Upper than # # ************************************************************************ # if nomenclature != "GLOBE_LAND_30": ruleThreeShp, timeCheck3 = grs_vect_selbyarea( osm_db, osmTableData['polygons'], UPPER=True, apidb=RoadsAPI ) osmShps.append(ruleThreeShp) time_l = datetime.datetime.now().replace(microsecond=0) else: timeCheck3 = None time_l = None # ************************************************************************ # # 4 - Area Lower than # # ************************************************************************ # if nomenclature != "GLOBE_LAND_30": ruleFourShp, timeCheck4 = grs_vect_selbyarea( osm_db, osmTableData['polygons'], UPPER=False, apidb=RoadsAPI ) osmShps.append(ruleFourShp) time_j = datetime.datetime.now().replace(microsecond=0) else: timeCheck4 = None time_j = None # ************************************************************************ # # 5 - Get data from lines table (railway | waterway) # # ************************************************************************ # ruleFiveShp, timeCheck5 = grs_vect_bbuffer( osm_db, osmTableData["lines"], api_db=RoadsAPI ) osmShps.append(ruleFiveShp) time_m = datetime.datetime.now().replace(microsecond=0) # ************************************************************************ # # 7 - Assign untagged Buildings to tags # # ************************************************************************ # if nomenclature != "GLOBE_LAND_30": ruleSeven11, ruleSeven12, timeCheck7 = vector_assign_pntags_to_build( osm_db, osmTableData['points'], osmTableData['polygons'], apidb=RoadsAPI ) if ruleSeven11: osmShps.append(ruleSeven11) if ruleSeven12: osmShps.append(ruleSeven12) time_n = datetime.datetime.now().replace(microsecond=0) else: timeCheck7 = None time_n = datetime.datetime.now().replace(microsecond=0) # ************************************************************************ # # Produce LULC Map # # ************************************************************************ # """ Get Shps with all geometries related with one class - One Shape for Classe """ _osmShps = [] for i in range(len(osmShps)): if not osmShps[i]: continue _osmShps.append(grs_to_shp( osmShps[i], os.path.join(workspace, osmShps[i] + '.shp'), 'auto', lyrN=1, asCMD=True, asMultiPart=None )) for shp in _osmShps: def_prj(os.path.splitext(shp)[0] + '.prj', epsg=epsg, api='epsgio') _osmShps = same_attr_to_shp( _osmShps, "cat", workspace, "osm_", resultDict=True ) del osmShps time_o = datetime.datetime.now().replace(microsecond=0) """ Merge all Classes into one feature class using a priority rule """ osmShps = {} for cls in _osmShps: if cls == '1': osmShps[1221] = shp_to_grs(_osmShps[cls], "osm_1221", asCMD=True) else: osmShps[int(cls)] = shp_to_grs(_osmShps[cls], "osm_" + cls, asCMD=True) # Erase overlapping areas by priority osmNameRef = copy.deepcopy(osmShps) for e in range(len(__priorities)): if e + 1 == len(__priorities): break if __priorities[e] not in osmShps: continue else: for i in range(e+1, len(__priorities)): if __priorities[i] not in osmShps: continue else: osmShps[__priorities[i]] = erase( osmShps[__priorities[i]], osmShps[__priorities[e]], "{}_{}".format(osmNameRef[__priorities[i]], e), notTbl=True, api='pygrass' ) time_p = datetime.datetime.now().replace(microsecond=0) # Export all classes lst_merge = [] a = None for i in range(len(__priorities)): if __priorities[i] not in osmShps: continue if not a: reset_table( osmShps[__priorities[i]], {'cls' : 'varchar(5)', 'leg' : 'varchar(75)'}, {'cls' : str(__priorities[i]), 'leg' : str(__legend[__priorities[i]])} ) a = 1 else: add_and_update( osmShps[__priorities[i]], {'cls' : 'varchar(5)'}, {'cls' : str(__priorities[i])} ) ds = dissolve( osmShps[__priorities[i]], 'dl_{}'.format(str(__priorities[i])), 'cls', api="grass" ) add_fields(ds, {'leg': 'varchar(75)'}, api="grass") update_table(ds, 'leg', str(__legend[__priorities[i]]), 'leg is null') lst_merge.append(grs_to_shp( ds, os.path.join( workspace, "lulc_{}.shp".format(str(__priorities[i])) ), 'auto', lyrN=1, asCMD=True, asMultiPart=None )) time_q = datetime.datetime.now().replace(microsecond=0) if fprop(lulcShp, 'ff') != '.shp': lulcShp = os.path.join( os.path.dirname(lulcShp), fprop(lulcShp, 'fn') + '.shp') shps_to_shp(lst_merge, lulcShp, api='pandas') # Check if prj of lulcShp exists and create it if necessary prj_ff = os.path.splitext(lulcShp)[0] + '.prj' if not os.path.exists(prj_ff): def_prj(prj_ff, epsg=epsg, api='epsgio') time_r = datetime.datetime.now().replace(microsecond=0) # Dump Database if PostGIS was used # Drop Database if PostGIS was used if RoadsAPI == 'POSTGIS': dump_db(osm_db, os.path.join( workspace, osm_db + '.sql' ), api='psql') drop_db(osm_db) return lulcShp, { 0 : ('set_settings', time_b - time_a), 1 : ('osm_to_sqdb', time_c - time_b), 2 : ('cls_in_sqdb', time_d - time_c), 3 : ('proj_data', time_e - time_d), 4 : ('set_grass', time_f - time_e), 5 : ('rule_1', time_g - time_f, timeCheck1), 6 : ('rule_2', time_h - time_g, timeCheck2), 7 : None if not timeCheck3 else ('rule_3', time_l - time_h, timeCheck3), 8 : None if not timeCheck4 else ('rule_4', time_j - time_l, timeCheck4), 9 : ('rule_5', time_m - time_j if timeCheck4 else time_m - time_h, timeCheck5), 10 : None if not timeCheck7 else ('rule_7', time_n - time_m, timeCheck7), 11 : ('disj_cls', time_o - time_n), 12 : ('priority_rule', time_p - time_o), 13 : ('export_cls', time_q - time_p), 14 : ('merge_cls', time_r - time_q) }
def lulc_by_cell(tid, boundary, lulc_shps, fishnet, result, workspace): from glass.g.wenv.grs import run_grass from glass.g.dp.torst import shp_to_rst bname = fprop(boundary, 'fn') # Boundary to Raster ref_rst = shp_to_rst(boundary, None, 10, 0, os.path.join(workspace, f'rst_{bname}.tif')) # Create GRASS GIS Session loc_name = 'loc_' + bname gbase = run_grass(workspace, location=loc_name, srs=ref_rst) import grass.script.setup as gsetup gsetup.init(gbase, workspace, loc_name, 'PERMANENT') # GRASS GIS Modules from glass.g.it.shp import shp_to_grs, grs_to_shp from glass.g.gp.ovl import grsintersection from glass.g.tbl.attr import geomattr_to_db from glass.g.prop.feat import feat_count # Send Fishnet to GRASS GIS fnet = shp_to_grs(fishnet, fprop(fishnet, 'fn'), asCMD=True) # Processing ulst = [] l_lulc_grs = [] for shp in lulc_shps: iname = fprop(shp, 'fn') # LULC Class to GRASS GIS lulc_grs = shp_to_grs(shp, iname, filterByReg=True, asCMD=True) if not feat_count( lulc_grs, gisApi='grass', work=workspace, loc=loc_name): continue # Intersect Fishnet | LULC CLass union_grs = grsintersection(fnet, lulc_grs, iname + '_i', cmd=True) # Get Areas geomattr_to_db(union_grs, "areav", "area", "boundary", unit='meters') # Export Table funion = grs_to_shp(union_grs, os.path.join(result, iname + '.shp'), 'area') ulst.append(funion) l_lulc_grs.append(lulc_grs) # Intersect between all LULC SHPS ist_shp = [] if len(l_lulc_grs) > 1: for i in range(len(l_lulc_grs)): for e in range(i + 1, len(l_lulc_grs)): ishp = grsintersection(l_lulc_grs[i], l_lulc_grs[e], 'lulcint_' + str(i) + '_' + str(e), cmd=True) if not feat_count( ishp, gisApi='grass', work=workspace, loc=loc_name): continue else: ist_shp.append(ishp) if len(ist_shp): from glass.g.gp.gen import dissolve from glass.g.tbl.grs import reset_table if len(ist_shp) > 1: from glass.g.dp.mge import shps_to_shp # Export shapes _ist_shp = [ grs_to_shp(s, os.path.join(workspace, loc_name, s + '.shp'), 'area') for s in ist_shp ] # Merge Intersections merge_shp = shps_to_shp(_ist_shp, os.path.join(workspace, loc_name, 'merge_shp.shp'), api='pandas') # Import GRASS merge_shp = shp_to_grs(merge_shp, 'merge_shp') else: merge_shp = ist_shp[0] # Dissolve Shape reset_table(merge_shp, {'refid': 'varchar(2)'}, {'refid': '1'}) overlay_areas = dissolve(merge_shp, 'overlay_areas', 'refid', api='grass') # Union Fishnet | Overlay's union_ovl = grsintersection(fnet, overlay_areas, 'ovl_union', cmd=True) funion_ovl = grs_to_shp(union_ovl, os.path.join(result, union_ovl + '.shp'), 'area') ulst.append(funion_ovl) # Export Tables return ulst