def arcg_area(polyTbl, lulcNomenclature, UPPER=True): """ Select Features with area upper than """ from gasp.osm2lulc.utils import osm_features_by_rule from gasp.cpu.arcg.anls.exct import select_by_attr from gasp.cpu.arcg.mng.fld import add_geom_attr from gasp.cpu.arcg.mng.gen import delete from gasp.mng.genze import dissolve from gasp.prop.feat import feat_count KEY_COL = DB_SCHEMA["OSM_FEATURES"]["OSM_KEY"] VALUE_COL = DB_SCHEMA["OSM_FEATURES"]["OSM_VALUE"] LULC_COL = DB_SCHEMA[lulcNomenclature]["CLS_FK"] RULE_COL = DB_SCHEMA[lulcNomenclature]["RULES_FIELDS"]["AREA"] # Get OSM Features to be selected for this rule __serv = 'area_upper' if UPPER else 'area_lower' osmToSelect = osm_features_by_rule(lulcNomenclature, __serv) operator = " > " if UPPER else " < " osmToSelect[VALUE_COL] = "(" + osmToSelect[KEY_COL] + "='" + \ osmToSelect[VALUE_COL] + "' AND shp_area" + operator + \ osmToSelect[RULE_COL].astype(str) + ")" lulcCls = osmToSelect[LULC_COL].unique() clsVect = {} WORK = os.path.dirname(polyTbl) for cls in lulcCls: # Select and Export filterDf = osmToSelect[osmToSelect[LULC_COL] == cls] fShp = select_by_attr( polyTbl, str(filterDf[VALUE_COL].str.cat(sep=" OR ")), os.path.join(WORK, "{}_{}".format(__serv, str(cls)))) if not feat_count(fShp, gisApi='arcpy'): continue # Dissolve dissShp = dissolve(fShp, os.path.join(WORK, "{}_d_{}".format(__serv, str(cls))), "OBJECTID", geomMultiPart=None, api='arcpy') if not feat_count(dissShp, gisApi='arcpy'): continue clsVect[int(cls)] = dissShp delete(fShp) return clsVect
def grs_vect_selbyarea(osmcon, polyTbl, UPPER=True, apidb='SQLITE'): """ Select features with area upper than. A field with threshold is needed in the database. """ import datetime from gasp.mng.genze import dissolve from gasp.mng.grstbl import add_table from gasp.osm2lulc.var import GEOM_AREA from gasp.sql.mng.tbl import row_num as cnt_row if apidb != 'POSTGIS': from gasp.to.shp.grs import sqlite_to_shp as db_to_shp else: from gasp.to.shp.grs import psql_to_grs as db_to_shp OPERATOR = ">" if UPPER else "<" DIRECTION = "upper" if UPPER else "lower" WHR = "{ga} {op} t_area_{r} and area_{r} IS NOT NULL".format(op=OPERATOR, r=DIRECTION, ga=GEOM_AREA) # Check if we have interest data time_a = datetime.datetime.now().replace(microsecond=0) N = cnt_row(osmcon, polyTbl, where=WHR, api='psql' if apidb == 'POSTGIS' else 'sqlite') time_b = datetime.datetime.now().replace(microsecond=0) if not N: return None, {0: ('count_rows', time_b - time_a)} # Data to GRASS GIS grsVect = db_to_shp(osmcon, polyTbl, "area_{}".format(DIRECTION), where=WHR, filterByReg=True) time_c = datetime.datetime.now().replace(microsecond=0) dissVect = dissolve(grsVect, "diss_area_{}".format(DIRECTION), "area_{}".format(DIRECTION), api="grass") add_table(dissVect, None, lyrN=1, asCMD=True) time_d = datetime.datetime.now().replace(microsecond=0) return dissVect, { 0: ('count_rows', time_b - time_a), 1: ('import', time_c - time_b), 2: ('dissolve', time_d - time_c) }
def grs_vect_bbuffer(osmdata, lineTbl, api_db='SQLITE'): """ Basic Buffer strategie """ import datetime from gasp.anls.prox.bf import _buffer from gasp.mng.genze import dissolve from gasp.mng.grstbl import add_table from gasp.sql.mng.tbl import row_num as cnt_row if api_db != 'POSTGIS': from gasp.to.shp.grs import sqlite_to_shp as db_to_shp else: from gasp.to.shp.grs import psql_to_grs as db_to_shp WHR = "basic_buffer IS NOT NULL" # Check if we have data time_a = datetime.datetime.now().replace(microsecond=0) N = cnt_row(osmdata, lineTbl, where=WHR, api='psql' if api_db == 'POSTGIS' else 'sqlite') time_b = datetime.datetime.now().replace(microsecond=0) if not N: return None, {0: ('count_rows_roads', time_b - time_a)} grsVect = db_to_shp(osmdata, lineTbl, "bb_lnh", where=WHR, filterByReg=True) time_c = datetime.datetime.now().replace(microsecond=0) grsBuf = _buffer(grsVect, "bf_basic_buffer", "bb_poly", api="grass", geom_type="line") time_d = datetime.datetime.now().replace(microsecond=0) grsDiss = dissolve(grsBuf, "bb_diss", "basic_buffer", api="grass") add_table(grsDiss, None, lyrN=1, asCMD=True) time_e = datetime.datetime.now().replace(microsecond=0) return grsDiss, { 0: ('count_rows', time_b - time_a), 1: ('import', time_c - time_b), 2: ('buffer', time_d - time_c), 3: ('dissolve', time_e - time_d) }
def grs_vector(dbcon, polyTable, apidb='SQLITE'): """ Simple Selection using GRASS GIS """ import datetime from gasp.mng.genze import dissolve from gasp.mng.grstbl import add_table from gasp.sql.mng.tbl import row_num as cont_row if apidb != 'POSTGIS': from gasp.to.shp.grs import sqlite_to_shp as db_to_grs else: from gasp.to.shp.grs import psql_to_grs as db_to_grs WHR = "selection IS NOT NULL" # Check if we have interest data time_a = datetime.datetime.now().replace(microsecond=0) N = cont_row(dbcon, polyTable, where=WHR, api='psql' if apidb == 'POSTGIS' else 'sqlite') time_b = datetime.datetime.now().replace(microsecond=0) if not N: return None, {0: ('count_rows', time_b - time_a)} # Data to GRASS GIS grsVect = db_to_grs(dbcon, polyTable, "sel_rule", where=WHR, filterByReg=True) time_c = datetime.datetime.now().replace(microsecond=0) dissVect = dissolve(grsVect, "diss_sel_rule", field="selection", api="grass") add_table(dissVect, None, lyrN=1, asCMD=True) time_d = datetime.datetime.now().replace(microsecond=0) return dissVect, { 0: ('count_rows', time_b - time_a), 1: ('import', time_c - time_b), 2: ('dissolve', time_d - time_c) }
def arcg_selection(db, polTbl, fld): """ Select, Dissolve and Reproject using ArcGIS """ import datetime import os from gasp.mng.genze import dissolve from gasp.fm.sql import query_to_df from gasp.anls.exct import sel_by_attr # Get LULC Classes time_a = datetime.datetime.now().replace(microsecond=0) lulcCls = query_to_df( db, ("SELECT selection FROM {} " "WHERE selection IS NOT NULL GROUP BY selection").format(polTbl), db_api='sqlite').selection.tolist() time_b = datetime.datetime.now().replace(microsecond=0) timeGasto = {0: ('check_cls', time_b - time_a)} # Extract Shps from DB clsShp = {} tk = 1 SQL = "SELECT selection, geometry FROM {} WHERE selection={}" for cls in lulcCls: time_x = datetime.datetime.now().replace(microsecond=0) shp = sel_by_attr(db, SQL.format(polTbl, str(cls)), os.path.join(fld, 'rule1_{}.shp'.format(cls)), api_gis='ogr') time_y = datetime.datetime.now().replace(microsecond=0) dShp = dissolve(shp, os.path.join(fld, "rul1_d_{}.shp".format(str(cls))), "FID", geomMultiPart=True) time_z = datetime.datetime.now().replace(microsecond=0) clsShp[int(cls)] = dShp timeGasto[tk] = ("export_{}".format(cls), time_y - time_x) timeGasto[tk + 1] = ("dissolve_{}".format(cls), time_z - time_y) tk += 2 return clsShp, timeGasto
def area_by_population(polygons, inhabitants, field_inhabitants, work, area_field='area_pop'): """ Feature area (polygons) by feature inhabitant (inhabitants) """ from gasp.cpu.arcg.lyr import feat_lyr from gasp.cpu.arcg.mng.fld import add_field from gasp.mng.genze import dissolve from gasp.cpu.arcg.anls.ovlay import intersect from gasp.cpu.arcg.anls.exct import select_by_attr # ArcGIS environment arcpy.env.overwriteOutput = True arcpy.env.workspace = work inhabitant_lyr = feat_lyr(inhabitants) add_field(inhabitant_lyr, area_field, "FLOAT", "") cursor = arcpy.UpdateCursor(inhabitant_lyr) lnh = cursor.next() while lnh: """TODO: Use intersection only once""" f = int(lnh.getValue("FID")) poly_extracted = "poly_{fid}.shp".format(fid=str(f)) select_by_attr(inhabitant_lyr, "FID = {fid}".format(fid=str(f)), poly_extracted) intersected = "int_{fid}.shp".format(fid=str(f)) intersect([polygons, poly_extracted], intersected) dissolved = dissolve(intersected, "diss_{f_}.shp".format(f_=str(f)), "FID", api='arcpy') cs = arcpy.SearchCursor(dissolved) l = cs.next() geom = arcpy.Describe(dissolved).shapeFieldName while l: area = float(l.getValue(geom).area) l = cs.next() pop = int(lnh.getValue(field_inhabitants)) try: indicator = area / pop except: indicator = 0.0 / pop lnh.setValue(area_field) cursor.updateRow(lnh) lnh = cursor.next()
def vector_based(osmdata, nomenclature, refRaster, lulcShp, overwrite=None, dataStore=None, RoadsAPI='SQLITE'): """ Convert OSM Data into Land Use/Land Cover Information An vector based approach. TODO: Add a detailed description. RoadsAPI Options: * SQLITE * POSTGIS """ # ************************************************************************ # # Python Modules from Reference Packages # # ************************************************************************ # import datetime import os import json # ************************************************************************ # # GASP dependencies # # ************************************************************************ # from gasp.oss.ops import create_folder from gasp.prop.rst import get_epsg_raster from gasp.session import run_grass if RoadsAPI == 'POSTGIS': from gasp.sql.mng.db import create_db from gasp.osm2lulc.utils import osm_to_pgsql else: from gasp.osm2lulc.utils import osm_to_sqdb from gasp.osm2lulc.utils import osm_project, add_lulc_to_osmfeat from gasp.mng.gen import merge_feat from gasp.osm2lulc.mod1 import grs_vector if RoadsAPI == 'SQLITE' or RoadsAPI == 'POSTGIS': from gasp.osm2lulc.mod2 import roads_sqdb else: from gasp.osm2lulc.mod2 import grs_vec_roads from gasp.osm2lulc.m3_4 import grs_vect_selbyarea from gasp.osm2lulc.mod5 import grs_vect_bbuffer from gasp.osm2lulc.mod6 import vector_assign_pntags_to_build # ************************************************************************ # # Global Settings # # ************************************************************************ # if not os.path.exists(os.path.dirname(lulcShp)): raise ValueError('{} does not exist!'.format(os.path.dirname(lulcShp))) # Get Parameters to connect to PostgreSQL conPGSQL = json.load( open( os.path.join(os.path.dirname(os.path.abspath(__file__)), 'con-postgresql.json'), 'r')) if RoadsAPI == 'POSTGIS' else None # Get EPSG of Reference Raster epsg = get_epsg_raster(refRaster) if not epsg: raise ValueError('Cannot get epsg code of ref raster') time_a = datetime.datetime.now().replace(microsecond=0) from gasp.osm2lulc.var import osmTableData, PRIORITIES workspace = os.path.join(os.path.dirname(lulcShp), 'osmtolulc') if not dataStore else dataStore # Check if workspace exists if os.path.exists(workspace): if overwrite: create_folder(workspace) else: raise ValueError('Path {} already exists'.format(workspace)) else: create_folder(workspace) __priorities = PRIORITIES[nomenclature] time_b = datetime.datetime.now().replace(microsecond=0) if RoadsAPI != 'POSTGIS': # ******************************************************************** # # Convert OSM file to SQLITE DB # # ******************************************************************** # osm_db = osm_to_sqdb(osmdata, os.path.join(workspace, 'osm.sqlite')) else: # Convert OSM file to POSTGRESQL DB # conPGSQL["DATABASE"] = create_db(conPGSQL, os.path.splitext( os.path.basename(osmdata))[0], overwrite=True) osm_db = osm_to_pgsql(osmdata, conPGSQL) time_c = datetime.datetime.now().replace(microsecond=0) # ************************************************************************ # # Add Lulc Classes to OSM_FEATURES by rule # # ************************************************************************ # add_lulc_to_osmfeat(osm_db if RoadsAPI != 'POSTGIS' else conPGSQL, osmTableData, nomenclature, api='SQLITE' if RoadsAPI != 'POSTGIS' else RoadsAPI) time_d = datetime.datetime.now().replace(microsecond=0) # ************************************************************************ # # Transform SRS of OSM Data # # ************************************************************************ # osmTableData = osm_project( osm_db if RoadsAPI != 'POSTGIS' else conPGSQL, epsg, api='SQLITE' if RoadsAPI != 'POSTGIS' else RoadsAPI) time_e = datetime.datetime.now().replace(microsecond=0) # ************************************************************************ # # Start a GRASS GIS Session # # ************************************************************************ # grass_base = run_grass(workspace, grassBIN='grass76', location='grloc', srs=epsg) #import grass.script as grass import grass.script.setup as gsetup gsetup.init(grass_base, workspace, 'grloc', 'PERMANENT') # ************************************************************************ # # IMPORT SOME GASP MODULES FOR GRASS GIS # # ************************************************************************ # from gasp.anls.ovlay import erase from gasp.prop.grs import rst_to_region from gasp.mng.genze import dissolve from gasp.mng.grstbl import add_and_update, reset_table from gasp.to.shp.grs import shp_to_grs, grs_to_shp from gasp.to.rst import rst_to_grs # ************************************************************************ # # SET GRASS GIS LOCATION EXTENT # # ************************************************************************ # extRst = rst_to_grs(refRaster, 'extent_raster') rst_to_region(extRst) time_f = datetime.datetime.now().replace(microsecond=0) # ************************************************************************ # # MapResults # # ************************************************************************ # osmShps = [] # ************************************************************************ # # 1 - Selection Rule # # ************************************************************************ # ruleOneShp, timeCheck1 = grs_vector( osm_db if RoadsAPI != 'POSTGIS' else conPGSQL, osmTableData['polygons'], apidb=RoadsAPI) osmShps.append(ruleOneShp) time_g = datetime.datetime.now().replace(microsecond=0) # ************************************************************************ # # 2 - Get Information About Roads Location # # ************************************************************************ # ruleRowShp, timeCheck2 = roads_sqdb( osm_db if RoadsAPI == 'SQLITE' else conPGSQL, osmTableData['lines'], osmTableData['polygons'], apidb=RoadsAPI ) if RoadsAPI == 'SQLITE' or RoadsAPI == 'POSTGIS' else grs_vec_roads( osm_db, osmTableData['lines'], osmTableData['polygons']) osmShps.append(ruleRowShp) time_h = datetime.datetime.now().replace(microsecond=0) # ************************************************************************ # # 3 - Area Upper than # # ************************************************************************ # ruleThreeShp, timeCheck3 = grs_vect_selbyarea( osm_db if RoadsAPI != 'POSTGIS' else conPGSQL, osmTableData['polygons'], UPPER=True, apidb=RoadsAPI) osmShps.append(ruleThreeShp) time_l = datetime.datetime.now().replace(microsecond=0) # ************************************************************************ # # 4 - Area Lower than # # ************************************************************************ # ruleFourShp, timeCheck4 = grs_vect_selbyarea( osm_db if RoadsAPI != 'POSTGIS' else conPGSQL, osmTableData['polygons'], UPPER=False, apidb=RoadsAPI) osmShps.append(ruleFourShp) time_j = datetime.datetime.now().replace(microsecond=0) # ************************************************************************ # # 5 - Get data from lines table (railway | waterway) # # ************************************************************************ # ruleFiveShp, timeCheck5 = grs_vect_bbuffer( osm_db if RoadsAPI != 'POSTGIS' else conPGSQL, osmTableData["lines"], api_db=RoadsAPI) osmShps.append(ruleFiveShp) time_m = datetime.datetime.now().replace(microsecond=0) # ************************************************************************ # # 7 - Assign untagged Buildings to tags # # ************************************************************************ # if nomenclature != "GLOBE_LAND_30": ruleSeven11, ruleSeven12, timeCheck7 = vector_assign_pntags_to_build( osm_db if RoadsAPI != 'POSTGIS' else conPGSQL, osmTableData['points'], osmTableData['polygons'], apidb=RoadsAPI) if ruleSeven11: osmShps.append(ruleSeven11) if ruleSeven12: osmShps.append(ruleSeven12) time_n = datetime.datetime.now().replace(microsecond=0) else: timeCheck7 = None time_n = datetime.datetime.now().replace(microsecond=0) # ************************************************************************ # # Produce LULC Map # # ************************************************************************ # """ Get Shps with all geometries related with one class - One Shape for Classe """ from gasp.mng.gen import same_attr_to_shp _osmShps = [] for i in range(len(osmShps)): if not osmShps[i]: continue _osmShps.append( grs_to_shp(osmShps[i], os.path.join(workspace, osmShps[i] + '.shp'), 'auto', lyrN=1, asCMD=True, asMultiPart=None)) _osmShps = same_attr_to_shp(_osmShps, "cat", workspace, "osm_", resultDict=True) del osmShps time_o = datetime.datetime.now().replace(microsecond=0) """ Merge all Classes into one feature class using a priority rule """ osmShps = {} for cls in _osmShps: if cls == '1': osmShps[1221] = shp_to_grs(_osmShps[cls], "osm_1221", asCMD=True) else: osmShps[int(cls)] = shp_to_grs(_osmShps[cls], "osm_" + cls, asCMD=True) # Erase overlapping areas by priority import copy osmNameRef = copy.deepcopy(osmShps) for e in range(len(__priorities)): if e + 1 == len(__priorities): break if __priorities[e] not in osmShps: continue else: for i in range(e + 1, len(__priorities)): if __priorities[i] not in osmShps: continue else: osmShps[__priorities[i]] = erase( osmShps[__priorities[i]], osmShps[__priorities[e]], "{}_{}".format(osmNameRef[__priorities[i]], e), notTbl=True, api='pygrass') time_p = datetime.datetime.now().replace(microsecond=0) # Export all classes lst_merge = [] for cls in osmShps: if cls == __priorities[0]: reset_table(osmShps[cls], {'cls': 'varchar(5)'}, {'cls': str(cls)}) else: add_and_update(osmShps[cls], {'cls': 'varchar(5)'}, {'cls': str(cls)}) ds = dissolve(osmShps[cls], 'dl_{}'.format(str(cls)), 'cls', api="grass") lst_merge.append( grs_to_shp(ds, os.path.join(workspace, "lulc_{}.shp".format(str(cls))), 'auto', lyrN=1, asCMD=True, asMultiPart=None)) time_q = datetime.datetime.now().replace(microsecond=0) merge_feat(lst_merge, lulcShp, api='pandas') time_r = datetime.datetime.now().replace(microsecond=0) return lulcShp, { 0: ('set_settings', time_b - time_a), 1: ('osm_to_sqdb', time_c - time_b), 2: ('cls_in_sqdb', time_d - time_c), 3: ('proj_data', time_e - time_d), 4: ('set_grass', time_f - time_e), 5: ('rule_1', time_g - time_f, timeCheck1), 6: ('rule_2', time_h - time_g, timeCheck2), 7: ('rule_3', time_l - time_h, timeCheck3), 8: ('rule_4', time_j - time_l, timeCheck4), 9: ('rule_5', time_m - time_j, timeCheck5), 10: None if not timeCheck7 else ('rule_7', time_n - time_m, timeCheck7), 11: ('disj_cls', time_o - time_n), 12: ('priority_rule', time_p - time_o), 13: ('export_cls', time_q - time_p), 14: ('merge_cls', time_r - time_q) }
def arcg_mean_time_WByPop(netDt, rdv, infraestruturas, unidades, conjuntos, popf, w, output, oneway=None): """ Tempo medio ponderado pela populacao residente a infra-estrutura mais proxima (min) * netDt = Path to Network Dataset * infraestruturas = Points of destiny * unidades = BGRI; Freg; Concelhos * conjuntos = Freg; Concelhos; NUT - field * popf = Field with the population of the statistic unity * w = Workspace * output = Path to store the final output * rdv = Name of feature class with the streets network """ import arcpy import os from gasp.cpu.arcg.lyr import feat_lyr from gasp.cpu.arcg.mng.feat import feat_to_pnt from gasp.cpu.arcg.mng.fld import add_field from gasp.cpu.arcg.mng.fld import calc_fld from gasp.cpu.arcg.mng.joins import join_table from gasp.mng.genze import dissolve from gasp.mng.gen import copy_feat from gasp.mob.arctbx.closest import closest_facility def get_freg_denominator(shp, groups, population, fld_time="Total_Minu"): cursor = arcpy.SearchCursor(shp) groups_sum = {} for lnh in cursor: group = lnh.getValue(groups) nrInd = float(lnh.getValue(population)) time = float(lnh.getValue(fld_time)) if group not in groups_sum.keys(): groups_sum[group] = time * nrInd else: groups_sum[group] += time * nrInd del cursor, lnh return groups_sum arcpy.env.overwriteOutput = True arcpy.env.workspace = w # Start Procedure # # Create copy of statitic unities to preserve the original data copy_unities = copy_feat(unidades, os.path.join(w, os.path.basename(unidades)), gisApi='arcpy') # Generate centroids of the statistic unities - unidades lyr_unidades = feat_lyr(copy_unities) pnt_unidades = feat_to_pnt(lyr_unidades, 'pnt_unidades.shp') # Network Processing - Distance between CENTROID and Destiny points closest_facility(netDt, rdv, infraestruturas, pnt_unidades, os.path.join(w, "cls_table.dbf"), oneway_restriction=oneway) add_field("cls_table.dbf", 'j', "SHORT", "6") calc_fld("cls_table.dbf", 'j', "[IncidentID]-1") join_table(lyr_unidades, "FID", "cls_table.dbf", "j", "Total_Minu") # Calculo dos somatorios por freguesia (conjunto) groups = get_freg_denominator(lyr_unidades, conjuntos, popf) add_field(lyr_unidades, "tm", "FLOAT", "10", "3") cs = arcpy.UpdateCursor(lyr_unidades) linha = cs.next() while linha: group = linha.getValue(conjuntos) t = float(linha.getValue("Total_Minu")) p = int(linha.getValue(popf)) total = groups[group] indi = ((t * p) / total) * t linha.setValue("tm", indi) cs.updateRow(linha) linha = cs.next() return dissolve(lyr_unidades, output, conjuntos, statistics="tm SUM", api="arcpy")
def gdl_mean_time_wByPop(unities, unities_groups, population_field, destinations, output, workspace=None, unities_epsg=4326, destinations_epsg=4326): """ Tempo medio ponderado pela populacao residente a infra-estrutura mais proxima # TODO: Migrate to Pandas """ import os from osgeo import ogr from gasp.prop.ff import drv_name from gasp.fm import points_to_list from gasp.mng.feat import feat_to_pnt from gasp.mng.prj import project_geom from gasp.mng.fld import add_fields from gasp.mng.genze import dissolve from gasp.web.glg.direct import get_time_pnt_destinations workspace = workspace if workspace else \ os.path.dirname(output) # Unities to centroid pnt_unities = feat_to_pnt( unities, os.path.join(workspace, 'pnt_' + os.path.basename(unities))) # List destinations lst_destinies = points_to_list(destinations, listVal="dict", inEpsg=destinations_epsg, outEpsg=4326) # Calculate indicator polyUnit = ogr.GetDriverByName(drv_name(unities)).Open(unities, 1) polyLyr = polyUnit.GetLayer() polyLyr = add_fields(polyLyr, {'meantime': ogr.OFTReal}) pntUnit = ogr.GetDriverByName(drv_name(pnt_unities)).Open(pnt_unities, 0) pntLyr = pntUnit.GetLayer() polyFeat = polyLyr.GetNextFeature() distUnities = {} groups = {} for pntFeat in pntLyr: geom = pntFeat.GetGeometryRef() if unities_epsg == 4326: originGeom = geom else: originGeom = project_geom(geom, unities_epsg, 4326, api='ogr') _id, duration, distance = get_time_pnt_destinations( originGeom, lst_destinies) __min = duration['value'] / 60.0 pop = polyFeat.GetField(population_field) group = polyFeat.GetField(unities_groups) distUnities[polyFeat.GetFID()] = (__min, __min * pop) if group not in groups: groups[group] = __min * pop else: groups[group] += __min * pop polyFeat = polyLyr.GetNextFeature() del polyLyr polyUnit.Destroy() polyUnit = ogr.GetDriverByName(drv_name(unities)).Open(unities, 1) polyLyr = polyUnit.GetLayer() for feat in polyLyr: unitId = feat.GetFID() groupId = feat.GetField(unities_groups) indicator = (distUnities[unitId][1] / groups[groupId]) * distUnities[unitId][0] feat.SetField('meantime', indicator) polyLyr.SetFeature(feat) del polyLyr, pntLyr polyUnit.Destroy() pntUnit.Destroy() dissolve(unities, output, unities_groups, statistics={'meantime': 'SUM'}, api='ogr')
def population_within_point_buffer(netDataset, rdvName, pointShp, populationShp, popField, bufferDist, epsg, output, workspace=None, bufferIsTimeMinutes=None, useOneway=None): """ Assign to points the population within a certain distance (metric or time) * Creates a Service Area Polygon for each point in pointShp; * Intersect the Service Area Polygons with the populationShp; * Count the number of persons within each Service Area Polygon (this number will be weighted by the area % of the statistic unit intersected with the Service Area Polygon). """ import arcpy import os from geopandas import GeoDataFrame from gasp.cpu.arcg.lyr import feat_lyr from gasp.cpu.arcg.anls.ovlay import intersect from gasp.mng.gen import copy_feat from gasp.cpu.arcg.mng.fld import add_geom_attr from gasp.cpu.arcg.mng.fld import add_field from gasp.cpu.arcg.mng.fld import calc_fld from gasp.mng.genze import dissolve from gasp.mob.arctbx.svarea import service_area_use_meters from gasp.mob.arctbx.svarea import service_area_polygon from gasp.fm import tbl_to_obj from gasp.to.shp import df_to_shp workspace = os.path.dirname(pointShp) if not workspace else workspace if not os.path.exists(workspace): from gasp.oss.ops import create_folder workspace = create_folder(workspace, overwrite=False) # Copy population layer populationShp = copy_feat( populationShp, os.path.join(workspace, 'cop_{}'.format(os.path.basename(populationShp))), gisApi='arcpy') # Create layer pntLyr = feat_lyr(pointShp) popLyr = feat_lyr(populationShp) # Create Service Area if not bufferIsTimeMinutes: servArea = service_area_use_meters( netDataset, rdvName, bufferDist, pointShp, os.path.join(workspace, 'servare_{}'.format(os.path.basename(pointShp))), OVERLAP=False, ONEWAY=useOneway) else: servArea = service_area_polygon( netDataset, rdvName, bufferDist, pointShp, os.path.join(workspace, "servare_{}".format(os.path.basename(pointShp))), ONEWAY_RESTRICTION=useOneway, OVERLAP=None) servAreaLyr = feat_lyr(servArea) # Add Column with Polygons area to Feature Class population add_geom_attr(popLyr, "total", geom_attr="AREA") # Intersect buffer and Population Feature Class intSrc = intersect([servAreaLyr, popLyr], os.path.join(workspace, "int_servarea_pop.shp")) intLyr = feat_lyr(intSrc) # Get area of intersected statistical unities with population add_geom_attr(intLyr, "partarea", geom_attr="AREA") # Get population weighted by area intersected calc_fld(intLyr, "population", "((([partarea] * 100) / [total]) * [{}]) / 100".format(popField), { "TYPE": "DOUBLE", "LENGTH": "10", "PRECISION": "3" }) # Dissolve service area by Facility ID diss = dissolve(intLyr, os.path.join(workspace, 'diss_servpop.shp'), "FacilityID", statistics="population SUM") # Get original Point FID from FacilityID calc_fld(diss, "pnt_fid", "[FacilityID] - 1", { "TYPE": "INTEGER", "LENGTH": "5", "PRECISION": None }) dfPnt = tbl_to_obj(pointShp) dfDiss = tbl_to_obj(diss) dfDiss.rename(columns={"SUM_popula": "n_pessoas"}, inplace=True) resultDf = dfPnt.merge(dfDiss, how='inner', left_index=True, right_on="pnt_fid") resultDf.drop('geometry_y', axis=1, inplace=True) resultDf = GeoDataFrame(resultDf, crs={'init': 'epsg:{}'.format(epsg)}, geometry='geometry_x') df_to_shp(resultDf, output) return output
def pop_less_dist_x2(net_dataset, rdv_name, locations, interval, unities, fld_groups, fld_pop, w, output, useOneway=None): """ Network processing - executar service area de modo a conhecer as areas a menos de x minutos de qualquer coisa """ import arcpy import numpy import os import pandas from gasp.cpu.arcg.lyr import feat_lyr from gasp.mng.genze import dissolve from gasp.cpu.arcg.anls.ovlay import intersect from gasp.cpu.arcg.mng.fld import calc_fld from gasp.mob.arctbx.svarea import service_area_polygon from gasp.fm import tbl_to_obj from gasp.oss import get_filename from gasp.to.shp import df_to_shp from gasp.cpu.arcg.mng.fld import del_field if arcpy.CheckExtension("Network") == "Available": arcpy.CheckOutExtension("Network") # Procedure # # Generate Service Area svArea = service_area_polygon(net_dataset, rdv_name, interval, locations, os.path.join(w, "servarea.shp"), ONEWAY_RESTRICTION=useOneway) # Dissolve Service Area svArea = dissolve(svArea, os.path.join(w, 'svarea_diss.shp'), "FID", api="arcpy") # Intersect unities with Service Area lyr_unities = feat_lyr(unities) unities_servarea = intersect([lyr_unities, svArea], os.path.join(w, "unidades_mx.shp")) # In the original Unities SHP, create a col with the population # only for the unities intersected with service area intersectDf = tbl_to_obj(unities_servarea) unities_less_than = intersectDf[fld_pop].unique() unities_less_than = pandas.DataFrame(unities_less_than, columns=['cod_']) popDf = tbl_to_obj(unities) popDf = popDf.merge(unities_less_than, how='outer', left_on=fld_pop, right_on="cod_") popDf["less_than"] = popDf.cod_.fillna(value='0') popDf["less_than"] = numpy.where(popDf["less_than"] != '0', '1', '0') popDf["population"] = numpy.where(popDf["less_than"] == '1', popDf[fld_pop], 0) popDf["original"] = popDf[fld_pop] newUnities = df_to_shp(popDf, os.path.join(w, 'unities_pop.shp')) # Dissolve and Get result result = dissolve(newUnities, output, fld_groups, statistics="original SUM;population SUM", api="arcpy") calc_fld(result, "pop_{}".format(interval), "[SUM_popula]", { "TYPE": "INTEGER", "LENGTH": "10", "PRECISION": "" }) calc_fld(result, fld_pop, "[SUM_origin]", { "TYPE": "INTEGER", "LENGTH": "10", "PRECISION": "" }) calc_fld(result, "pop_{}_p".format(interval), "([pop_{}] / [{}]) *100".format(interval, fld_pop), { "TYPE": "DOUBLE", "LENGTH": "6", "PRECISION": "2" }) del_field(result, "SUM_popula") del_field(result, "SUM_origin") return result
def pop_less_dist_x(net_dataset, rdv_name, junctions_name, locations, interval, unities, fld_groups, fld_pop, w, output): """ Network processing - executar service area de modo a conhecer as areas a menos de x minutos de qualquer coisa """ import arcpy import os from gasp.cpu.arcg.lyr import feat_lyr from gasp.mng.genze import dissolve from gasp.cpu.arcg.anls.ovlay import intersect from gasp.cpu.arcg.mng.fld import add_field from gasp.mob.arctbx.svarea import service_area_polygon def GetUnitiesIntersected(shpintersected, shpUnities): # AND IF SHPUNITIES HAS LESS THAN 6 CHARACTERS if len(os.path.basename(shpUnities)) > 6: fld_tag = os.path.basename(shpUnities)[:6] else: fld_tag = os.path.basename(shpUnities) c = arcpy.SearchCursor(shpintersected) l = c.next() u = [] while l: fid_entity = int(l.getValue("FID_{name}".format(name=fld_tag))) if fid_entity not in u: u.append(fid_entity) l = c.next() return l def WritePopLessXMin(shp, fld_pop, lst_intersected): add_field(shp, "poxX", "SHORT", "8") cursor = arcpy.UpdateCursor(shp) linha = cursor.next() while linha: bgri = int(linha.getValue("FID")) if bgri in lst_intersected: p = int(linha.getValue(fld_pop)) linha.setValue("popX", p) cursor.UpdateRow(linha) linha = cursor.next() return "popX" arcpy.env.overwriteOutput = True arcpy.env.workspace = w # Procedure # # Generate Service Area ServiceArea = service_area_polygon(net_dataset, rdv_name, junctions_name, interval, locations, "servarea.shp") # Intersect unities with Service Area lyr_unities = feat_lyr(unities) unities_servarea = intersect([lyr_unities, ServiceArea], "unidades_mx.shp") # Get the FID of the unities that intersects with the service area id_unities = GetUnitiesIntersected(unities_servarea, unities) # Update original shape with the population a menos de x minutes fld_pop_less_x = WritePopLessXMin(lyr_unities, fld_pop, id_unities) groups = dissolve( lyr_unities, output, fld_groups, "{pop} SUM;{popx} SUM".format(pop=fld_pop, popx=fld_pop_less_x)) # Estimate population percent if len(fld_pop) > 6: fld_pop_tag = fld_pop[:6] else: fld_pop_tag = fld_pop add_field(shp, "lessX", "FLOAT", "8", "3") cursor = arcpy.UpdateCursor(output) linha = cursor.next() while linha: p = float(linha.getValue("SUM_{pop}".format(pop=fld_pop_tag))) pt = float(linha.getValue("SUM_{p}".format(p=fld_pop_less_x))) per = (p / pt) * 100.0 linha.setValue("lessX", per) cursor.updateRow(linha) linha = cursor.next() return output
def mean_time_by_influence_area(netDt, rdv, infraestruturas, fld_infraestruturas, unidades, id_unidade, conjuntos, popf, influence_areas_unities, w, output, oneway=True): """ Tempo medio ponderado pela populacao residente a infra-estrutura mais proxima (min), por area de influencia * netDt - Path to Network Dataset * infraestruturas - Points of destiny * fld_infraestruturas - Field on destiny points to relate with influence area * unidades - BGRI; Freg; Concelhos * conjuntos - Freg; Concelhos; NUT - field * popf - Field with the population of the statistic unity * influence_areas_unities - Field on statistic unities layer to relate with influence area * w = Workspace * output = Path to store the final output * rdv - Name of feature class with the streets network * junctions - Name of feature class with the junctions """ import arcpy import os from gasp.cpu.arcg.lyr import feat_lyr from gasp.cpu.arcg.mng.feat import feat_to_pnt from gasp.cpu.arcg.mng.gen import merge from gasp.mng.gen import copy_feat from gasp.mng.genze import dissolve from gasp.cpu.arcg.mng.fld import add_field from gasp.cpu.arcg.mng.fld import calc_fld from gasp.cpu.arcg.mng.fld import field_statistics from gasp.cpu.arcg.mng.fld import type_fields from gasp.cpu.arcg.mng.joins import join_table from gasp.cpu.arcg.anls.exct import select_by_attr from gasp.cpu.arcg.netanlst.closest import closest_facility """if arcpy.CheckExtension("Network") == "Available": arcpy.CheckOutExtension("Network") else: raise ValueError('Network analyst extension is not avaiable')""" def ListGroupArea(lyr, fld_ia, fld_grp): d = {} cs = arcpy.SearchCursor(lyr) for lnh in cs: id_group = lnh.getValue(fld_grp) id_ia = lnh.getValue(fld_ia) if id_group not in d.keys(): d[id_group] = [id_ia] else: if id_ia not in d[id_group]: d[id_group].append(id_ia) return d arcpy.env.overwriteOutput = True arcpy.env.workspace = w # Procedure # copy_unities = copy_feat(unidades, os.path.join(w, os.path.basename(unidades)), gisApi='arcpy') # Generate centroids of the statistic unities - unidades lyr_unidades = feat_lyr(copy_unities) pnt_unidades = feat_to_pnt(lyr_unidades, 'pnt_unidades.shp', pnt_position="INSIDE") # List all groups of unities (conjuntos) group_areas = ListGroupArea(lyr_unidades, influence_areas_unities, conjuntos) # Create Layers lyr_pnt_unidades = feat_lyr(pnt_unidades) lyr_pnt_facilities = feat_lyr(infraestruturas) result_list = [] fld_type_unities = type_fields(lyr_pnt_unidades, field=conjuntos) SELECT_UNITIES = '{fld}=\'{c}\'' if str(fld_type_unities) == 'String' \ else '{fld}={c}' fld_type_facilities = type_fields(lyr_pnt_facilities, field=fld_infraestruturas) SELECT_FACILITIES = '{fld}=\'{obj}\'' if str(fld_type_facilities) == 'String' \ else '{fld}={obj}' for group in group_areas.keys(): # Select centroids of interest interest_centroids = select_by_attr( lyr_pnt_unidades, SELECT_UNITIES.format(c=str(group), fld=conjuntos), 'pnt_{c}.shp'.format(c=str(group))) # Select facilities of interest expression = ' OR '.join([ SELECT_FACILITIES.format(fld=fld_infraestruturas, obj=str(group_areas[group][i])) for i in range(len(group_areas[group])) ]) interest_facilities = select_by_attr( lyr_pnt_facilities, expression, 'facilities_{c}.shp'.format(c=str(group))) # Run closest facilitie - Distance between selected CENTROID and selected facilities cls_fac_table = os.path.join(w, "clsf_{c}.dbf".format(c=str(group))) closest_facility(netDt, rdv, interest_facilities, interest_centroids, cls_fac_table, oneway_restriction=oneway) add_field(cls_fac_table, 'j', "SHORT", "6") calc_fld(cls_fac_table, 'j', "[IncidentID]-1") join_table(interest_centroids, "FID", cls_fac_table, "j", "Total_Minu") # Calculate sum of time x population add_field(interest_centroids, 'sum', "DOUBLE", "10", "3") calc_fld(interest_centroids, 'sum', "[{pop}]*[Total_Minu]".format(pop=popf)) denominador = field_statistics(interest_centroids, 'sum', 'SUM') add_field(interest_centroids, 'tm', "DOUBLE", "10", "3") calc_fld( interest_centroids, 'tm', "([sum]/{sumatorio})*[Total_Minu]".format( sumatorio=str(denominador))) result_list.append(interest_centroids) merge_shp = merge(result_list, "merge_centroids.shp") join_table(lyr_unidades, id_unidade, "merge_centroids.shp", id_unidade, "tm") return dissolve(lyr_unidades, output, conjuntos, statistics="tm SUM", api='arcpy')
def vector_assign_pntags_to_build(osmdb, pntTable, polyTable, apidb='SQLITE'): """ Replace buildings with tag yes using the info in the Points Layer Only used for URBAN ATLAS and CORINE LAND COVER """ import datetime from gasp.sql.mng.tbl import row_num as cnt_row if apidb != "POSTGIS": from gasp.to.shp.grs import sqlite_to_shp as db_to_shp else: from gasp.to.shp.grs import psql_to_grs as db_to_shp from gasp.sql.anls.ovlay import sgbd_get_feat_within from gasp.sql.anls.ovlay import sgbd_get_feat_not_within from gasp.mng.genze import dissolve from gasp.mng.grstbl import add_table time_a = datetime.datetime.now().replace(microsecond=0) new_build = sgbd_get_feat_within( osmdb, ("(SELECT buildings AS pnt_build, geometry AS pnt_geom " "FROM {} WHERE buildings IS NOT NULL)").format(pntTable), "pnt_geom", ("(SELECT buildings AS poly_build, geometry AS poly_geom " "FROM {} WHERE buildings IS NOT NULL)").format(polyTable), "poly_geom", "new_buildings", inTblCols="pnt_build AS cls", withinCols="poly_geom AS geometry", outTblIsFile=None, apiToUse="OGR_SPATIALITE" if apidb != "POSTGIS" else apidb) time_b = datetime.datetime.now().replace(microsecond=0) yes_build = sgbd_get_feat_not_within( osmdb, ("(SELECT buildings AS poly_build, geometry AS poly_geom " "FROM {} WHERE buildings IS NOT NULL)").format(polyTable), "poly_geom", ("(SELECT buildings AS pnt_build, geometry AS pnt_geom " "FROM {} WHERE buildings IS NOT NULL)").format(pntTable), "pnt_geom", "yes_builds", inTblCols="poly_geom AS geometry, 11 AS cls", outTblIsFile=None, apiToUse="OGR_SPATIALITE" if apidb != "POSTGIS" else apidb) time_c = datetime.datetime.now().replace(microsecond=0) N12 = cnt_row(osmdb, new_build, api='psql' if apidb == 'POSTGIS' else 'sqlite') time_d = datetime.datetime.now().replace(microsecond=0) N11 = cnt_row(osmdb, yes_build, api='psql' if apidb == 'POSTGIS' else 'sqlite') time_e = datetime.datetime.now().replace(microsecond=0) if N11: # Add data into grasss grsBuild11 = db_to_shp(osmdb, yes_build, "yes_builds", filterByReg=True) time_f = datetime.datetime.now().replace(microsecond=0) # Dissolve dissVect = dissolve(grsBuild11, "dss_{}".format(grsBuild11), 'cls', api="grass") add_table(dissVect, None, lyrN=1, asCMD=True) time_g = datetime.datetime.now().replace(microsecond=0) else: dissVect = None time_f = None time_g = None if N12: # Add data into GRASS GIS grsBuild12 = db_to_shp(osmdb, new_build, "pnt_build", filterByReg=True) time_h = datetime.datetime.now().replace(microsecond=0) # Dissolve dissVect12 = dissolve(grsBuild12, "dss_{}".format(grsBuild12), 'cls', api="grass") add_table(dissVect12, None, lyrN=1, asCMD=True) time_i = datetime.datetime.now().replace(microsecond=0) else: dissVect12 = None time_h = None time_i = None return dissVect, dissVect12, { 0: ('intersect', time_b - time_a), 1: ('disjoint', time_c - time_b), 2: ('count_b12', time_d - time_c), 3: ('count_b11', time_e - time_d), 4: None if not time_f else ('import_b11', time_f - time_e), 5: None if not time_g else ('dissolve_b11', time_g - time_f), 6: None if not time_h else ('import_b12', time_h - time_g if time_g else time_h - time_e), 7: None if not time_i else ('dissolve_b12', time_i - time_h) }
def grs_vec_roads(osmdb, lineTbl, polyTbl): """ Select Roads for GRASS GIS """ import datetime from gasp.sql.mng.tbl import row_num from gasp.to.shp.grs import sqlite_to_shp from gasp.anls.prox.bf import _buffer from gasp.mng.genze import dissolve from gasp.mng.grstbl import add_table # Roads to GRASS GIS time_a = datetime.datetime.now().replace(microsecond=0) NR = row_num(osmdb, lineTbl, where="roads IS NOT NULL", api='sqlite') time_b = datetime.datetime.now().replace(microsecond=0) if not NR: return None, {0: ('count_rows_roads', time_b - time_a)} roadsVect = sqlite_to_shp(osmdb, lineTbl, "all_roads", where="roads IS NOT NULL") time_c = datetime.datetime.now().replace(microsecond=0) # Buildings to GRASS GIS NB = row_num(osmdb, polyTbl, where="building IS NOT NULL", api='sqlite') time_d = datetime.datetime.now().replace(microsecond=0) if NB: from gasp.anls.prox import grs_near as near from gasp.mng.grstbl import update_table builds = sqlite_to_shp(osmdb, polyTbl, "all_builds", where="building IS NOT NULL", filterByReg=True) time_e = datetime.datetime.now().replace(microsecond=0) near(roadsVect, builds, nearDistCol="todist", maxDist=12, as_cmd=True) time_f = datetime.datetime.now().replace(microsecond=0) update_table(roadsVect, "bf_roads", "round(todist,0)", "\"todist > 0\"", lyrN=1, ascmd=True) time_g = datetime.datetime.now().replace(microsecond=0) else: time_e = None time_f = None time_g = None # Run Buffer tool roadsBf = _buffer(roadsVect, "bf_roads", "bf_roads", api='grass', geom_type="line") time_h = datetime.datetime.now().replace(microsecond=0) # Dissolve Roads roadsDiss = dissolve(roadsBf, "diss_roads", field="roads", api="grass") add_table(roadsDiss, None, lyrN=1, asCMD=True) time_i = datetime.datetime.now().replace(microsecond=0) return roadsDiss, { 0: ('count_rows_roads', time_b - time_a), 1: ('import_roads', time_c - time_b), 2: ('count_rows_build', time_d - time_c), 3: None if not time_e else ('import_builds', time_e - time_d), 4: None if not time_f else ('near_analysis', time_f - time_e), 5: None if not time_g else ('update_buffer_tbl', time_g - time_f), 6: ('buffer_roads', time_h - time_g if time_g else time_h - time_d), 7: ('diss_roads', time_i - time_h) }