def arcg_area(polyTbl, lulcNomenclature, UPPER=True):
"""
Select Features with area upper than
"""
from gasp.osm2lulc.utils import osm_features_by_rule
from gasp.cpu.arcg.anls.exct import select_by_attr
from gasp.cpu.arcg.mng.fld import add_geom_attr
from gasp.cpu.arcg.mng.gen import delete
from gasp.mng.genze import dissolve
from gasp.prop.feat import feat_count
KEY_COL = DB_SCHEMA["OSM_FEATURES"]["OSM_KEY"]
VALUE_COL = DB_SCHEMA["OSM_FEATURES"]["OSM_VALUE"]
LULC_COL = DB_SCHEMA[lulcNomenclature]["CLS_FK"]
RULE_COL = DB_SCHEMA[lulcNomenclature]["RULES_FIELDS"]["AREA"]
# Get OSM Features to be selected for this rule
__serv = 'area_upper' if UPPER else 'area_lower'
osmToSelect = osm_features_by_rule(lulcNomenclature, __serv)
operator = " > " if UPPER else " < "
osmToSelect[VALUE_COL] = "(" + osmToSelect[KEY_COL] + "='" + \
osmToSelect[VALUE_COL] + "' AND shp_area" + operator + \
osmToSelect[RULE_COL].astype(str) + ")"
lulcCls = osmToSelect[LULC_COL].unique()
clsVect = {}
WORK = os.path.dirname(polyTbl)
for cls in lulcCls:
# Select and Export
filterDf = osmToSelect[osmToSelect[LULC_COL] == cls]
fShp = select_by_attr(
polyTbl, str(filterDf[VALUE_COL].str.cat(sep=" OR ")),
os.path.join(WORK, "{}_{}".format(__serv, str(cls))))
if not feat_count(fShp, gisApi='arcpy'): continue
# Dissolve
dissShp = dissolve(fShp,
os.path.join(WORK,
"{}_d_{}".format(__serv, str(cls))),
"OBJECTID",
geomMultiPart=None,
api='arcpy')
if not feat_count(dissShp, gisApi='arcpy'): continue
clsVect[int(cls)] = dissShp
delete(fShp)
return clsVect
def clip_each_feature(rst, shp, feature_id, work, out_basename):
"""
Clip a raster dataset for each feature in a feature class
"""
import arcpy
import os
from gasp.cpu.arcg.lyr import feat_lyr
from gasp.cpu.arcg.lyr import rst_lyr
from gasp.cpu.arcg.anls.exct import select_by_attr
from gasp.oss.ops import create_folder
# ########### #
# Environment #
# ########### #
arcpy.env.overwriteOutput = True
arcpy.env.workspace = work
# ###### #
# Do it! #
# ###### #
# Open feature class
lyr_shp = feat_lyr(shp)
lyr_rst = rst_lyr(rst)
# Create folder for some temporary files
wTmp = create_folder(os.path.join(work, 'tmp'))
# Get id's field type
fields = arcpy.ListFields(lyr_shp)
for f in fields:
if str(f.name) == str(feature_id):
fld_type = f.type
break
expression = '{fld}=\'{_id}\'' if str(fld_type) == 'String' else \
'{fld}={_id}'
del fields, f
# Run the clip tool for each feature in the shp input
c = arcpy.SearchCursor(lyr_shp)
l = c.next()
while l:
fid = str(l.getValue(feature_id))
selection = select_by_attr(
lyr_shp, expression.format(fld=feature_id, _id=fid),
os.path.join(wTmp, 'each_{}.shp'.format(fid)))
clip_rst = clip_raster(lyr_rst, selection,
'{b}_{_id}.tif'.format(b=out_basename, _id=fid))
l = c.next()
def mean_rst_by_polygon(polygons, raster, work, resultShp):
"""
Mean of all cells intersect with the input polygon features
"""
import arcpy
import os
from gasp.cpu.arcg.lyr import feat_lyr, rst_lyr
from gasp.prop.rst import rst_stats
from gasp.cpu.arcg.mng.fld import add_field
from gasp.mng.gen import copy_feat
from gasp.cpu.arcg.anls.exct import select_by_attr
from gasp.cpu.arcg.mng.rst.proc import clip_raster
# ########### #
# Environment #
# ########### #
arcpy.env.overwriteOutput = True
arcpy.env.workspace = work
# ###### #
# Do it! #
# ###### #
# Copy the Input
poly_copy = copy_feat(polygons, resultShp, gisApi='arcpy')
# Create Layers
lyrShp = feat_lyr(poly_copy)
lyrRst = rst_lyr(raster)
# Create field for register calculated statistics
if len(os.path.basename(raster)) <= 10:
fld_name = os.path.basename(raster)
else:
fld_name = os.path.basename(raster)[:10]
add_field(lyrShp, fld_name, "DOUBLE", "20", "3")
# Calculate mean
c = arcpy.UpdateCursor(lyrShp)
l = c.next()
while l:
fid = str(l.getValue("FID"))
selection = select_by_attr(lyrShp, "FID={c}".format(c=fid),
"poly_{c}.shp".format(c=fid))
sel_rst = clip_raster(lyrRst, selection,
"clip_rst_{c}.img".format(c=fid))
mean = rst_stats(sel_rst, api='arcpy')["MEAN"]
l.setValue(fld_name, mean)
c.updateRow(l)
l = c.next()
def clip_several_each_feature(rst_folder,
shp,
feature_id,
work,
template=None,
rst_file_format='.tif'):
"""
Clip a folder of rasters by each feature in a feature class
The rasters clipped for a feature will be in an individual folder
"""
import arcpy
import os
from gasp.cpu.arcg.lyr import feat_lyr
from gasp.cpu.arcg.lyr import rst_lyr
from gasp.cpu.arcg.anls.exct import select_by_attr
from gasp.cpu.arcg.mng.fld import type_fields
from gasp.oss.ops import create_folder
from gasp.oss import list_files
# ########### #
# Environment #
# ########### #
arcpy.env.overwriteOutput = True
arcpy.env.workspace = work
# ###### #
# Do it! #
# ###### #
# Open feature class
lyr_shp = feat_lyr(shp)
# Create folder for some temporary files
wTmp = create_folder(os.path.join(work, 'tmp'))
# Split feature class in parts
c = arcpy.SearchCursor(lyr_shp)
l = c.next()
features = {}
# Get id's field type
fld_type = type_fields(lyr_shp, field=feature_id)
expression = '{fld}=\'{_id}\'' if str(fld_type) == 'String' else \
'{fld}={_id}'
del fields, f
while l:
fid = str(l.getValue(feature_id))
selection = select_by_attr(
lyr_shp, expression.format(fld=feature_id, _id=fid),
os.path.join(wTmp, 'each_{}.shp'.format(fid)))
f_lyr = feat_lyr(selection)
features[fid] = f_lyr
l = c.next()
rasters = list_files(rst_folder, file_format='.tif')
for raster in rasters:
r_lyr = rst_lyr(raster)
for feat in features:
clip_rst = clip_raster(
r_lyr, features[feat],
os.path.join(work,
os.path.splitext(os.path.basename(feat))[0],
os.path.basename(raster)), template)
def area_by_population(polygons,
inhabitants,
field_inhabitants,
work,
area_field='area_pop'):
"""
Feature area (polygons) by feature inhabitant (inhabitants)
"""
from gasp.cpu.arcg.lyr import feat_lyr
from gasp.cpu.arcg.mng.fld import add_field
from gasp.mng.genze import dissolve
from gasp.cpu.arcg.anls.ovlay import intersect
from gasp.cpu.arcg.anls.exct import select_by_attr
# ArcGIS environment
arcpy.env.overwriteOutput = True
arcpy.env.workspace = work
inhabitant_lyr = feat_lyr(inhabitants)
add_field(inhabitant_lyr, area_field, "FLOAT", "")
cursor = arcpy.UpdateCursor(inhabitant_lyr)
lnh = cursor.next()
while lnh:
"""TODO: Use intersection only once"""
f = int(lnh.getValue("FID"))
poly_extracted = "poly_{fid}.shp".format(fid=str(f))
select_by_attr(inhabitant_lyr, "FID = {fid}".format(fid=str(f)),
poly_extracted)
intersected = "int_{fid}.shp".format(fid=str(f))
intersect([polygons, poly_extracted], intersected)
dissolved = dissolve(intersected,
"diss_{f_}.shp".format(f_=str(f)),
"FID",
api='arcpy')
cs = arcpy.SearchCursor(dissolved)
l = cs.next()
geom = arcpy.Describe(dissolved).shapeFieldName
while l:
area = float(l.getValue(geom).area)
l = cs.next()
pop = int(lnh.getValue(field_inhabitants))
try:
indicator = area / pop
except:
indicator = 0.0 / pop
lnh.setValue(area_field)
cursor.updateRow(lnh)
lnh = cursor.next()
def count_geom_by_polygon(shp_geom,
polygons,
w,
count_geom_field='COUNT',
population_field=None):
"""
Intersect a given geometry with all polygons in a feature class and
count the number of geometries inside each polygon.
The user can also give a population field. The method will return the number
of geometries by person * 1000.
E.g. Count the number of points (health care centers, sports) by
statistical unit;
E.g. Count the number of points by inhabitants in each statistical unit.
"""
from gasp.cpu.arcg.lyr import feat_lyr
from gasp.cpu.arcg.mng.fld import add_field
from gasp.cpu.arcg.anls.ovlay import intersect
from gasp.cpu.arcg.anls.exct import select_by_attr
# ArcGIS environment
arcpy.env.workspace = w
arcpy.env.overwriteOutput = True
polygon_lyr = feat_lyr(polygons)
add_field(polygon_lyr, count_geom_field, "SHORT", "8")
if population_field:
geom_pop = count_geom_field + population_field
if len(geom_pop) >= 10:
geom_pop = geom_pop[:10]
add_field(polygon_lyr, geom_pop, "FLOAT", "")
# Update polygon layer
cursor = arcpy.UpdateCursor(polygon_lyr)
line = cursor.next()
while line:
"""TODO: Use select by attributes and not select analysis"""
fid = int(line.getValue("FID"))
poly_extracted = "poly_{f}.shp".format(f=str(fid))
select_by_attr(polygon_lyr, "FID = {f}".format(f=str(fid)),
poly_extracted)
intersected = "pnt_{f}.shp".format(f=str(fid))
intersect([shp_geom, poly_extracted], intersected)
cs = arcpy.SearchCursor(intersected)
nr_pnt = 0
for i in cs:
nr_pnt += 1
if population_field:
population = int(line.getValue(population_field))
pnt_by_pop = (nr_pnt / float(pnt_by_pop)) * 1000.0
line.setValue(geom_pop, pnt_by_pop)
line.setValue(count_geom_field, nr_pnt)
cursor.updateRow(line)
line = cursor.next()
def mean_time_in_povoated_areas(network,
rdv_name,
stat_units,
popFld,
destinations,
output,
workspace,
ONEWAY=True,
GRID_REF_CELLSIZE=10):
"""
Receive statistical units and some destinations. Estimates the mean distance
to that destinations for each statistical unit.
The mean for each statistical will be calculated using a point grid:
-> Statistical unit to grid point;
-> Distance from grid point to destination;
-> Mean of these distances.
This method will only do the math for areas (statistic units)
with population.
"""
import os
import arcpy
from gasp.cpu.arcg.lyr import feat_lyr
from gasp.cpu.arcg.anls.exct import select_by_attr
from gasp.cpu.arcg.mng.fld import field_statistics
from gasp.cpu.arcg.mng.fld import add_field
from gasp.cpu.arcg.mng.gen import merge
from gasp.mng.gen import copy_feat
from gasp.mob.arctbx.closest import closest_facility
from gasp.to.shp.arcg import rst_to_pnt
from gasp.to.rst import shp_to_raster
if arcpy.CheckExtension("Network") == "Available":
arcpy.CheckOutExtension("Network")
else:
raise ValueError('Network analyst extension is not avaiable')
arcpy.env.overwriteOutput = True
WORK = workspace
# Add field
stat_units = copy_feat(stat_units,
os.path.join(WORK, os.path.basename(stat_units)),
gisApi='arcpy')
add_field(stat_units, "TIME", "DOUBLE", "10", precision="3")
# Split stat_units into two layers
# One with population
# One with no population
withPop = select_by_attr(stat_units, '{}>0'.format(popFld),
os.path.join(WORK, 'with_pop.shp'))
noPop = select_by_attr(stat_units, '{}=0'.format(popFld),
os.path.join(WORK, 'no_pop.shp'))
# For each statistic unit with population
withLyr = feat_lyr(withPop)
cursor = arcpy.UpdateCursor(withLyr)
FID = 0
for feature in cursor:
# Create a new file
unity = select_by_attr(
withLyr, 'FID = {}'.format(str(FID)),
os.path.join(WORK, 'unit_{}.shp'.format(str(FID))))
# Convert to raster
rst_unity = shp_to_raster(unity,
"FID",
GRID_REF_CELLSIZE,
None,
os.path.join(WORK,
'unit_{}.tif'.format(str(FID))),
api='arcpy')
# Convert to point
pnt_unity = rst_to_pnt(
rst_unity, os.path.join(WORK, 'pnt_un_{}.shp'.format(str(FID))))
# Execute closest facilitie
CLOSEST_TABLE = os.path.join(WORK, 'cls_fac_{}.dbf'.format(str(FID)))
closest_facility(network,
rdv_name,
destinations,
pnt_unity,
CLOSEST_TABLE,
oneway_restriction=ONEWAY)
# Get Mean
MEAN_TIME = field_statistics(CLOSEST_TABLE, 'Total_Minu', 'MEAN')[0]
# Record Mean
feature.setValue("TIME", MEAN_TIME)
cursor.updateRow(feature)
FID += 1
merge([withPop, noPop], output)
return output
def mean_time_by_influence_area(netDt,
rdv,
infraestruturas,
fld_infraestruturas,
unidades,
id_unidade,
conjuntos,
popf,
influence_areas_unities,
w,
output,
oneway=True):
"""
Tempo medio ponderado pela populacao residente a infra-estrutura mais
proxima (min), por area de influencia
* netDt - Path to Network Dataset
* infraestruturas - Points of destiny
* fld_infraestruturas - Field on destiny points to relate with influence area
* unidades - BGRI; Freg; Concelhos
* conjuntos - Freg; Concelhos; NUT - field
* popf - Field with the population of the statistic unity
* influence_areas_unities - Field on statistic unities layer to relate
with influence area
* w = Workspace
* output = Path to store the final output
* rdv - Name of feature class with the streets network
* junctions - Name of feature class with the junctions
"""
import arcpy
import os
from gasp.cpu.arcg.lyr import feat_lyr
from gasp.cpu.arcg.mng.feat import feat_to_pnt
from gasp.cpu.arcg.mng.gen import merge
from gasp.mng.gen import copy_feat
from gasp.mng.genze import dissolve
from gasp.cpu.arcg.mng.fld import add_field
from gasp.cpu.arcg.mng.fld import calc_fld
from gasp.cpu.arcg.mng.fld import field_statistics
from gasp.cpu.arcg.mng.fld import type_fields
from gasp.cpu.arcg.mng.joins import join_table
from gasp.cpu.arcg.anls.exct import select_by_attr
from gasp.cpu.arcg.netanlst.closest import closest_facility
"""if arcpy.CheckExtension("Network") == "Available":
arcpy.CheckOutExtension("Network")
else:
raise ValueError('Network analyst extension is not avaiable')"""
def ListGroupArea(lyr, fld_ia, fld_grp):
d = {}
cs = arcpy.SearchCursor(lyr)
for lnh in cs:
id_group = lnh.getValue(fld_grp)
id_ia = lnh.getValue(fld_ia)
if id_group not in d.keys():
d[id_group] = [id_ia]
else:
if id_ia not in d[id_group]:
d[id_group].append(id_ia)
return d
arcpy.env.overwriteOutput = True
arcpy.env.workspace = w
# Procedure #
copy_unities = copy_feat(unidades,
os.path.join(w, os.path.basename(unidades)),
gisApi='arcpy')
# Generate centroids of the statistic unities - unidades
lyr_unidades = feat_lyr(copy_unities)
pnt_unidades = feat_to_pnt(lyr_unidades,
'pnt_unidades.shp',
pnt_position="INSIDE")
# List all groups of unities (conjuntos)
group_areas = ListGroupArea(lyr_unidades, influence_areas_unities,
conjuntos)
# Create Layers
lyr_pnt_unidades = feat_lyr(pnt_unidades)
lyr_pnt_facilities = feat_lyr(infraestruturas)
result_list = []
fld_type_unities = type_fields(lyr_pnt_unidades, field=conjuntos)
SELECT_UNITIES = '{fld}=\'{c}\'' if str(fld_type_unities) == 'String' \
else '{fld}={c}'
fld_type_facilities = type_fields(lyr_pnt_facilities,
field=fld_infraestruturas)
SELECT_FACILITIES = '{fld}=\'{obj}\'' if str(fld_type_facilities) == 'String' \
else '{fld}={obj}'
for group in group_areas.keys():
# Select centroids of interest
interest_centroids = select_by_attr(
lyr_pnt_unidades, SELECT_UNITIES.format(c=str(group),
fld=conjuntos),
'pnt_{c}.shp'.format(c=str(group)))
# Select facilities of interest
expression = ' OR '.join([
SELECT_FACILITIES.format(fld=fld_infraestruturas,
obj=str(group_areas[group][i]))
for i in range(len(group_areas[group]))
])
interest_facilities = select_by_attr(
lyr_pnt_facilities, expression,
'facilities_{c}.shp'.format(c=str(group)))
# Run closest facilitie - Distance between selected CENTROID and selected facilities
cls_fac_table = os.path.join(w, "clsf_{c}.dbf".format(c=str(group)))
closest_facility(netDt,
rdv,
interest_facilities,
interest_centroids,
cls_fac_table,
oneway_restriction=oneway)
add_field(cls_fac_table, 'j', "SHORT", "6")
calc_fld(cls_fac_table, 'j', "[IncidentID]-1")
join_table(interest_centroids, "FID", cls_fac_table, "j", "Total_Minu")
# Calculate sum of time x population
add_field(interest_centroids, 'sum', "DOUBLE", "10", "3")
calc_fld(interest_centroids, 'sum',
"[{pop}]*[Total_Minu]".format(pop=popf))
denominador = field_statistics(interest_centroids, 'sum', 'SUM')
add_field(interest_centroids, 'tm', "DOUBLE", "10", "3")
calc_fld(
interest_centroids, 'tm',
"([sum]/{sumatorio})*[Total_Minu]".format(
sumatorio=str(denominador)))
result_list.append(interest_centroids)
merge_shp = merge(result_list, "merge_centroids.shp")
join_table(lyr_unidades, id_unidade, "merge_centroids.shp", id_unidade,
"tm")
return dissolve(lyr_unidades,
output,
conjuntos,
statistics="tm SUM",
api='arcpy')
def arcg_buffering(lineTbl, nomenclature):
"""
Create a beautiful buffer from osm line features
"""
from gasp.osm2lulc.utils import osm_features_by_rule
from gasp.cpu.arcg.anls.exct import select_by_attr
from gasp.anls.prox.bf import _buffer
from gasp.cpu.arcg.mng.gen import delete
from gasp.prop.feat import feat_count
KEY_COL = DB_SCHEMA["OSM_FEATURES"]["OSM_KEY"]
VALUE_COL = DB_SCHEMA["OSM_FEATURES"]["OSM_VALUE"]
LULC_COL = DB_SCHEMA[nomenclature]["CLS_FK"]
BF_COL = DB_SCHEMA[nomenclature]["RULES_FIELDS"]["BUFFER"]
# Get OSM Features to be selected for this rule
osmToSelect = osm_features_by_rule(nomenclature, 'basic_buffer')
osmToSelect[VALUE_COL] = osmToSelect[KEY_COL] + "='" + \
osmToSelect[VALUE_COL] + "'"
# Get LULC Classes to be selected
lulcCls = osmToSelect[LULC_COL].unique()
resCls = {}
WORK = os.path.dirname(lineTbl)
for cls in lulcCls:
filterDf = osmToSelect[osmToSelect[LULC_COL] == cls]
# Get distances for this cls
distances = filterDf[BF_COL].unique()
for dist in distances:
__filter = filterDf[filterDf[BF_COL] == dist]
fShp = select_by_attr(
lineTbl, str(__filter[VALUE_COL].str.cat(sep=" OR ")),
os.path.join(WORK, "buf_{}_{}".format(str(cls),
str(int(dist)))))
if not feat_count(fShp, gisApi='arcpy'): continue
bfShp = _buffer(fShp,
str(dist).replace('.', ','),
os.path.join(
WORK, "bufbf_{}_{}".format(
str(cls),
str(dist).replace(".", "_"))),
api='arcpy',
dissolve="ALL")
if cls not in resCls:
resCls[int(cls)] = [bfShp]
else:
resCls[int(cls)].append(bfShp)
delete(fShp)
return resCls