def gdb_to_shp(workspace, outworkspace):
"""
Export all feature classes in a Geodatabase to a single file.
Do this for all Geodatabases in workspace.
"""
import os
from gasp.mng.gen import copy_feat
# List GeoDatabases
lst_gdb = arcpy.ListWorkspaces(workspace_type="FileGDB")
dic_gdb = {}
for gdb in lst_gdb:
arcpy.env.workspace = gdb
dic_gdb[gdb] = {}
dts = arcpy.ListDatasets()
for dt in dts:
dic_gdb[gdb][dt] = arcpy.ListFeatureClasses(feature_dataset=dt)
for gdb in dic_gdb:
for dt in dic_gdb[gdb]:
for fc in dic_gdb[gdb][dt]:
copy_feat(
os.path.join(gdb, dt, fc),
os.path.join(outworkspace, "{}_{}.shp".format(
os.path.splitext(os.path.basename(gdb))[0],
fc
)), gisApi='arcpy'
)
def ogr_select_by_location(shp, boundary_filter, filtered_output):
"""
Filter a shp using the location of a boundary_filter shp
For now the boundary must have only one feature
Writes the filter on a new shp
"""
import os
from osgeo import ogr
from gasp.prop.ff import drv_name
from gasp.prop.feat import get_geom_type
from gasp.mng.gen import copy_feat
from gasp.mng.fld import ogr_copy_fields
# Open main data
dtSrc = ogr.GetDriverByName(drv_name(shp)).Open(shp, 0)
lyr = dtSrc.GetLayer()
# Get filter geom
filter_shp = ogr.GetDriverByName(drv_name(boundary_filter)).Open(
boundary_filter, 0)
filter_lyr = filter_shp.GetLayer()
c = 0
for f in filter_lyr:
if c:
break
geom = f.GetGeometryRef()
c += 1
filter_shp.Destroy()
# Apply filter
lyr.SetSpatialFilter(geom)
# Copy filter objects to a new shape
out = ogr.GetDriverByName(
drv_name(filtered_output)).CreateDataSource(filtered_output)
outLyr = out.CreateLayer(os.path.splitext(
os.path.basename(filtered_output))[0],
geom_type=get_geom_type(shp,
gisApi='ogr',
name=None,
py_cls=True))
# Copy fields
ogr_copy_fields(lyr, outLyr)
copy_feat(lyr,
outLyr,
outDefn=outLyr.GetLayerDefn(),
only_geom=False,
gisApi='ogrlyr')
def round_table_values(table, decimal_by_col_file, outputFile):
"""
Round all column values using the number of decimal places written
in a excel file
COL_NAME | nr_decimal_places
COL_1 | 1
COL_2 | 0
... | ...
COL_N | 2
"""
from gasp.cpu.arcg.lyr import feat_lyr
from gasp.mng.gen import copy_feat
from gasp.cpu.arcg.mng.fld import type_fields
from gasp.fm import tbl_to_obj
arcpy.env.overwriteOutput = True
# Get number of decimal places for the values of each column
places_by_col = tbl_to_obj(decimal_by_col_file,
sheet=0,
output='dict',
useFirstColAsIndex=True)
PLACES_FIELD = places_by_col[places_by_col.keys()[0]].keys()[0]
# Copy table
outTable = copy_feat(table, outputFile, gisApi='arcpy')
# Edit copy putting the correct decimal places in each column
lyr = feat_lyr(outTable)
# List table fields
fields = type_fields(lyr)
cursor = arcpy.UpdateCursor(lyr)
for lnh in cursor:
for col in fields:
if col in places_by_col:
if fields[col] == 'Integer' or fields[col] == 'Double' \
or fields[col] == 'SmallInteger':
value = lnh.getValue(col)
lnh.setValue(
col, round(value,
int(places_by_col[col][PLACES_FIELD])))
else:
continue
else:
print "{} not in {}".format(col, decimal_by_col_file)
cursor.updateRow(lnh)
del lyr
return outputFile
def mean_rst_by_polygon(polygons, raster, work, resultShp):
"""
Mean of all cells intersect with the input polygon features
"""
import arcpy
import os
from gasp.cpu.arcg.lyr import feat_lyr, rst_lyr
from gasp.prop.rst import rst_stats
from gasp.cpu.arcg.mng.fld import add_field
from gasp.mng.gen import copy_feat
from gasp.cpu.arcg.anls.exct import select_by_attr
from gasp.cpu.arcg.mng.rst.proc import clip_raster
# ########### #
# Environment #
# ########### #
arcpy.env.overwriteOutput = True
arcpy.env.workspace = work
# ###### #
# Do it! #
# ###### #
# Copy the Input
poly_copy = copy_feat(polygons, resultShp, gisApi='arcpy')
# Create Layers
lyrShp = feat_lyr(poly_copy)
lyrRst = rst_lyr(raster)
# Create field for register calculated statistics
if len(os.path.basename(raster)) <= 10:
fld_name = os.path.basename(raster)
else:
fld_name = os.path.basename(raster)[:10]
add_field(lyrShp, fld_name, "DOUBLE", "20", "3")
# Calculate mean
c = arcpy.UpdateCursor(lyrShp)
l = c.next()
while l:
fid = str(l.getValue("FID"))
selection = select_by_attr(lyrShp, "FID={c}".format(c=fid),
"poly_{c}.shp".format(c=fid))
sel_rst = clip_raster(lyrRst, selection,
"clip_rst_{c}.img".format(c=fid))
mean = rst_stats(sel_rst, api='arcpy')["MEAN"]
l.setValue(fld_name, mean)
c.updateRow(l)
l = c.next()
def arcg_mean_time_WByPop(netDt,
rdv,
infraestruturas,
unidades,
conjuntos,
popf,
w,
output,
oneway=None):
"""
Tempo medio ponderado pela populacao residente a infra-estrutura mais
proxima (min)
* netDt = Path to Network Dataset
* infraestruturas = Points of destiny
* unidades = BGRI; Freg; Concelhos
* conjuntos = Freg; Concelhos; NUT - field
* popf = Field with the population of the statistic unity
* w = Workspace
* output = Path to store the final output
* rdv = Name of feature class with the streets network
"""
import arcpy
import os
from gasp.cpu.arcg.lyr import feat_lyr
from gasp.cpu.arcg.mng.feat import feat_to_pnt
from gasp.cpu.arcg.mng.fld import add_field
from gasp.cpu.arcg.mng.fld import calc_fld
from gasp.cpu.arcg.mng.joins import join_table
from gasp.mng.genze import dissolve
from gasp.mng.gen import copy_feat
from gasp.mob.arctbx.closest import closest_facility
def get_freg_denominator(shp, groups, population, fld_time="Total_Minu"):
cursor = arcpy.SearchCursor(shp)
groups_sum = {}
for lnh in cursor:
group = lnh.getValue(groups)
nrInd = float(lnh.getValue(population))
time = float(lnh.getValue(fld_time))
if group not in groups_sum.keys():
groups_sum[group] = time * nrInd
else:
groups_sum[group] += time * nrInd
del cursor, lnh
return groups_sum
arcpy.env.overwriteOutput = True
arcpy.env.workspace = w
# Start Procedure #
# Create copy of statitic unities to preserve the original data
copy_unities = copy_feat(unidades,
os.path.join(w, os.path.basename(unidades)),
gisApi='arcpy')
# Generate centroids of the statistic unities - unidades
lyr_unidades = feat_lyr(copy_unities)
pnt_unidades = feat_to_pnt(lyr_unidades, 'pnt_unidades.shp')
# Network Processing - Distance between CENTROID and Destiny points
closest_facility(netDt,
rdv,
infraestruturas,
pnt_unidades,
os.path.join(w, "cls_table.dbf"),
oneway_restriction=oneway)
add_field("cls_table.dbf", 'j', "SHORT", "6")
calc_fld("cls_table.dbf", 'j', "[IncidentID]-1")
join_table(lyr_unidades, "FID", "cls_table.dbf", "j", "Total_Minu")
# Calculo dos somatorios por freguesia (conjunto)
groups = get_freg_denominator(lyr_unidades, conjuntos, popf)
add_field(lyr_unidades, "tm", "FLOAT", "10", "3")
cs = arcpy.UpdateCursor(lyr_unidades)
linha = cs.next()
while linha:
group = linha.getValue(conjuntos)
t = float(linha.getValue("Total_Minu"))
p = int(linha.getValue(popf))
total = groups[group]
indi = ((t * p) / total) * t
linha.setValue("tm", indi)
cs.updateRow(linha)
linha = cs.next()
return dissolve(lyr_unidades,
output,
conjuntos,
statistics="tm SUM",
api="arcpy")
def population_within_point_buffer(netDataset,
rdvName,
pointShp,
populationShp,
popField,
bufferDist,
epsg,
output,
workspace=None,
bufferIsTimeMinutes=None,
useOneway=None):
"""
Assign to points the population within a certain distance (metric or time)
* Creates a Service Area Polygon for each point in pointShp;
* Intersect the Service Area Polygons with the populationShp;
* Count the number of persons within each Service Area Polygon
(this number will be weighted by the area % of the statistic unit
intersected with the Service Area Polygon).
"""
import arcpy
import os
from geopandas import GeoDataFrame
from gasp.cpu.arcg.lyr import feat_lyr
from gasp.cpu.arcg.anls.ovlay import intersect
from gasp.mng.gen import copy_feat
from gasp.cpu.arcg.mng.fld import add_geom_attr
from gasp.cpu.arcg.mng.fld import add_field
from gasp.cpu.arcg.mng.fld import calc_fld
from gasp.mng.genze import dissolve
from gasp.mob.arctbx.svarea import service_area_use_meters
from gasp.mob.arctbx.svarea import service_area_polygon
from gasp.fm import tbl_to_obj
from gasp.to.shp import df_to_shp
workspace = os.path.dirname(pointShp) if not workspace else workspace
if not os.path.exists(workspace):
from gasp.oss.ops import create_folder
workspace = create_folder(workspace, overwrite=False)
# Copy population layer
populationShp = copy_feat(
populationShp,
os.path.join(workspace,
'cop_{}'.format(os.path.basename(populationShp))),
gisApi='arcpy')
# Create layer
pntLyr = feat_lyr(pointShp)
popLyr = feat_lyr(populationShp)
# Create Service Area
if not bufferIsTimeMinutes:
servArea = service_area_use_meters(
netDataset,
rdvName,
bufferDist,
pointShp,
os.path.join(workspace,
'servare_{}'.format(os.path.basename(pointShp))),
OVERLAP=False,
ONEWAY=useOneway)
else:
servArea = service_area_polygon(
netDataset,
rdvName,
bufferDist,
pointShp,
os.path.join(workspace,
"servare_{}".format(os.path.basename(pointShp))),
ONEWAY_RESTRICTION=useOneway,
OVERLAP=None)
servAreaLyr = feat_lyr(servArea)
# Add Column with Polygons area to Feature Class population
add_geom_attr(popLyr, "total", geom_attr="AREA")
# Intersect buffer and Population Feature Class
intSrc = intersect([servAreaLyr, popLyr],
os.path.join(workspace, "int_servarea_pop.shp"))
intLyr = feat_lyr(intSrc)
# Get area of intersected statistical unities with population
add_geom_attr(intLyr, "partarea", geom_attr="AREA")
# Get population weighted by area intersected
calc_fld(intLyr, "population",
"((([partarea] * 100) / [total]) * [{}]) / 100".format(popField),
{
"TYPE": "DOUBLE",
"LENGTH": "10",
"PRECISION": "3"
})
# Dissolve service area by Facility ID
diss = dissolve(intLyr,
os.path.join(workspace, 'diss_servpop.shp'),
"FacilityID",
statistics="population SUM")
# Get original Point FID from FacilityID
calc_fld(diss, "pnt_fid", "[FacilityID] - 1", {
"TYPE": "INTEGER",
"LENGTH": "5",
"PRECISION": None
})
dfPnt = tbl_to_obj(pointShp)
dfDiss = tbl_to_obj(diss)
dfDiss.rename(columns={"SUM_popula": "n_pessoas"}, inplace=True)
resultDf = dfPnt.merge(dfDiss,
how='inner',
left_index=True,
right_on="pnt_fid")
resultDf.drop('geometry_y', axis=1, inplace=True)
resultDf = GeoDataFrame(resultDf,
crs={'init': 'epsg:{}'.format(epsg)},
geometry='geometry_x')
df_to_shp(resultDf, output)
return output
def mean_time_in_povoated_areas(network,
rdv_name,
stat_units,
popFld,
destinations,
output,
workspace,
ONEWAY=True,
GRID_REF_CELLSIZE=10):
"""
Receive statistical units and some destinations. Estimates the mean distance
to that destinations for each statistical unit.
The mean for each statistical will be calculated using a point grid:
-> Statistical unit to grid point;
-> Distance from grid point to destination;
-> Mean of these distances.
This method will only do the math for areas (statistic units)
with population.
"""
import os
import arcpy
from gasp.cpu.arcg.lyr import feat_lyr
from gasp.cpu.arcg.anls.exct import select_by_attr
from gasp.cpu.arcg.mng.fld import field_statistics
from gasp.cpu.arcg.mng.fld import add_field
from gasp.cpu.arcg.mng.gen import merge
from gasp.mng.gen import copy_feat
from gasp.mob.arctbx.closest import closest_facility
from gasp.to.shp.arcg import rst_to_pnt
from gasp.to.rst import shp_to_raster
if arcpy.CheckExtension("Network") == "Available":
arcpy.CheckOutExtension("Network")
else:
raise ValueError('Network analyst extension is not avaiable')
arcpy.env.overwriteOutput = True
WORK = workspace
# Add field
stat_units = copy_feat(stat_units,
os.path.join(WORK, os.path.basename(stat_units)),
gisApi='arcpy')
add_field(stat_units, "TIME", "DOUBLE", "10", precision="3")
# Split stat_units into two layers
# One with population
# One with no population
withPop = select_by_attr(stat_units, '{}>0'.format(popFld),
os.path.join(WORK, 'with_pop.shp'))
noPop = select_by_attr(stat_units, '{}=0'.format(popFld),
os.path.join(WORK, 'no_pop.shp'))
# For each statistic unit with population
withLyr = feat_lyr(withPop)
cursor = arcpy.UpdateCursor(withLyr)
FID = 0
for feature in cursor:
# Create a new file
unity = select_by_attr(
withLyr, 'FID = {}'.format(str(FID)),
os.path.join(WORK, 'unit_{}.shp'.format(str(FID))))
# Convert to raster
rst_unity = shp_to_raster(unity,
"FID",
GRID_REF_CELLSIZE,
None,
os.path.join(WORK,
'unit_{}.tif'.format(str(FID))),
api='arcpy')
# Convert to point
pnt_unity = rst_to_pnt(
rst_unity, os.path.join(WORK, 'pnt_un_{}.shp'.format(str(FID))))
# Execute closest facilitie
CLOSEST_TABLE = os.path.join(WORK, 'cls_fac_{}.dbf'.format(str(FID)))
closest_facility(network,
rdv_name,
destinations,
pnt_unity,
CLOSEST_TABLE,
oneway_restriction=ONEWAY)
# Get Mean
MEAN_TIME = field_statistics(CLOSEST_TABLE, 'Total_Minu', 'MEAN')[0]
# Record Mean
feature.setValue("TIME", MEAN_TIME)
cursor.updateRow(feature)
FID += 1
merge([withPop, noPop], output)
return output
def mean_time_by_influence_area(netDt,
rdv,
infraestruturas,
fld_infraestruturas,
unidades,
id_unidade,
conjuntos,
popf,
influence_areas_unities,
w,
output,
oneway=True):
"""
Tempo medio ponderado pela populacao residente a infra-estrutura mais
proxima (min), por area de influencia
* netDt - Path to Network Dataset
* infraestruturas - Points of destiny
* fld_infraestruturas - Field on destiny points to relate with influence area
* unidades - BGRI; Freg; Concelhos
* conjuntos - Freg; Concelhos; NUT - field
* popf - Field with the population of the statistic unity
* influence_areas_unities - Field on statistic unities layer to relate
with influence area
* w = Workspace
* output = Path to store the final output
* rdv - Name of feature class with the streets network
* junctions - Name of feature class with the junctions
"""
import arcpy
import os
from gasp.cpu.arcg.lyr import feat_lyr
from gasp.cpu.arcg.mng.feat import feat_to_pnt
from gasp.cpu.arcg.mng.gen import merge
from gasp.mng.gen import copy_feat
from gasp.mng.genze import dissolve
from gasp.cpu.arcg.mng.fld import add_field
from gasp.cpu.arcg.mng.fld import calc_fld
from gasp.cpu.arcg.mng.fld import field_statistics
from gasp.cpu.arcg.mng.fld import type_fields
from gasp.cpu.arcg.mng.joins import join_table
from gasp.cpu.arcg.anls.exct import select_by_attr
from gasp.cpu.arcg.netanlst.closest import closest_facility
"""if arcpy.CheckExtension("Network") == "Available":
arcpy.CheckOutExtension("Network")
else:
raise ValueError('Network analyst extension is not avaiable')"""
def ListGroupArea(lyr, fld_ia, fld_grp):
d = {}
cs = arcpy.SearchCursor(lyr)
for lnh in cs:
id_group = lnh.getValue(fld_grp)
id_ia = lnh.getValue(fld_ia)
if id_group not in d.keys():
d[id_group] = [id_ia]
else:
if id_ia not in d[id_group]:
d[id_group].append(id_ia)
return d
arcpy.env.overwriteOutput = True
arcpy.env.workspace = w
# Procedure #
copy_unities = copy_feat(unidades,
os.path.join(w, os.path.basename(unidades)),
gisApi='arcpy')
# Generate centroids of the statistic unities - unidades
lyr_unidades = feat_lyr(copy_unities)
pnt_unidades = feat_to_pnt(lyr_unidades,
'pnt_unidades.shp',
pnt_position="INSIDE")
# List all groups of unities (conjuntos)
group_areas = ListGroupArea(lyr_unidades, influence_areas_unities,
conjuntos)
# Create Layers
lyr_pnt_unidades = feat_lyr(pnt_unidades)
lyr_pnt_facilities = feat_lyr(infraestruturas)
result_list = []
fld_type_unities = type_fields(lyr_pnt_unidades, field=conjuntos)
SELECT_UNITIES = '{fld}=\'{c}\'' if str(fld_type_unities) == 'String' \
else '{fld}={c}'
fld_type_facilities = type_fields(lyr_pnt_facilities,
field=fld_infraestruturas)
SELECT_FACILITIES = '{fld}=\'{obj}\'' if str(fld_type_facilities) == 'String' \
else '{fld}={obj}'
for group in group_areas.keys():
# Select centroids of interest
interest_centroids = select_by_attr(
lyr_pnt_unidades, SELECT_UNITIES.format(c=str(group),
fld=conjuntos),
'pnt_{c}.shp'.format(c=str(group)))
# Select facilities of interest
expression = ' OR '.join([
SELECT_FACILITIES.format(fld=fld_infraestruturas,
obj=str(group_areas[group][i]))
for i in range(len(group_areas[group]))
])
interest_facilities = select_by_attr(
lyr_pnt_facilities, expression,
'facilities_{c}.shp'.format(c=str(group)))
# Run closest facilitie - Distance between selected CENTROID and selected facilities
cls_fac_table = os.path.join(w, "clsf_{c}.dbf".format(c=str(group)))
closest_facility(netDt,
rdv,
interest_facilities,
interest_centroids,
cls_fac_table,
oneway_restriction=oneway)
add_field(cls_fac_table, 'j', "SHORT", "6")
calc_fld(cls_fac_table, 'j', "[IncidentID]-1")
join_table(interest_centroids, "FID", cls_fac_table, "j", "Total_Minu")
# Calculate sum of time x population
add_field(interest_centroids, 'sum', "DOUBLE", "10", "3")
calc_fld(interest_centroids, 'sum',
"[{pop}]*[Total_Minu]".format(pop=popf))
denominador = field_statistics(interest_centroids, 'sum', 'SUM')
add_field(interest_centroids, 'tm', "DOUBLE", "10", "3")
calc_fld(
interest_centroids, 'tm',
"([sum]/{sumatorio})*[Total_Minu]".format(
sumatorio=str(denominador)))
result_list.append(interest_centroids)
merge_shp = merge(result_list, "merge_centroids.shp")
join_table(lyr_unidades, id_unidade, "merge_centroids.shp", id_unidade,
"tm")
return dissolve(lyr_unidades,
output,
conjuntos,
statistics="tm SUM",
api='arcpy')
def check_shape_diff(SHAPES_TO_COMPARE,
OUT_FOLDER,
REPORT,
conPARAM,
DB,
SRS_CODE,
GIS_SOFTWARE="GRASS",
GRASS_REGION_TEMPLATE=None):
"""
Script to check differences between pairs of Feature Classes
Suponha que temos diversas Feature Classes (FC) e que cada uma delas
possui um determinado atributo; imagine também que,
considerando todos os pares possíveis entre estas FC,
se pretende comparar as diferenças na distribuição dos valores
desse atributo em cada par.
* Dependências:
- ArcGIS;
- GRASS;
- PostgreSQL;
- PostGIS.
* GIS_SOFTWARE Options:
- ARCGIS;
- GRASS.
"""
import datetime
import os
import pandas
from gasp.fm.sql import query_to_df
from gasp.sql.mng.tbl import tbls_to_tbl
from gasp.sql.mng.geom import fix_geom, check_geomtype_in_table
from gasp.sql.mng.geom import select_main_geom_type
from gasp.sql.mng.qw import ntbl_by_query
from gasp.prop.ff import check_isRaster
from gasp.oss import get_filename
from gasp.sql.mng.db import create_db
from gasp.to.sql import shp_to_psql, df_to_db
from gasp.to.shp import rst_to_polyg
from gasp.to.shp import shp_to_shp, psql_to_shp
from gasp.to import db_to_tbl
# Check if folder exists, if not create it
if not os.path.exists(OUT_FOLDER):
from gasp.oss.ops import create_folder
create_folder(OUT_FOLDER, overwrite=None)
else:
raise ValueError('{} already exists!'.format(OUT_FOLDER))
# Start GRASS GIS Session if GIS_SOFTWARE == GRASS
if GIS_SOFTWARE == "GRASS":
if not GRASS_REGION_TEMPLATE:
raise ValueError(
'To use GRASS GIS you need to specify GRASS_REGION_TEMPLATE')
from gasp.session import run_grass
gbase = run_grass(OUT_FOLDER,
grassBIN='grass76',
location='shpdif',
srs=GRASS_REGION_TEMPLATE)
import grass.script as grass
import grass.script.setup as gsetup
gsetup.init(gbase, OUT_FOLDER, 'shpdif', 'PERMANENT')
from gasp.mng.grstbl import rename_col
from gasp.to.shp.grs import shp_to_grs, grs_to_shp
from gasp.to.rst import rst_to_grs
from gasp.mng.fld import rename_column
# Convert to SHAPE if file is Raster
# Import to GRASS GIS if GIS SOFTWARE == GRASS
i = 0
_SHP_TO_COMPARE = {}
for s in SHAPES_TO_COMPARE:
isRaster = check_isRaster(s)
if isRaster:
if GIS_SOFTWARE == "ARCGIS":
d = rst_to_polyg(s,
os.path.join(os.path.dirname(s),
get_filename(s) + '.shp'),
gisApi='arcpy')
_SHP_TO_COMPARE[d] = "gridcode"
elif GIS_SOFTWARE == "GRASS":
# To GRASS
rstName = get_filename(s)
inRst = rst_to_grs(s, "rst_" + rstName, as_cmd=True)
# To Raster
d = rst_to_polyg(inRst,
rstName,
rstColumn="lulc_{}".format(i),
gisApi="grasscmd")
# Export Shapefile
shp = grs_to_shp(d, os.path.join(OUT_FOLDER, d + '.shp'),
"area")
_SHP_TO_COMPARE[shp] = "lulc_{}".format(i)
else:
if GIS_SOFTWARE == "ARCGIS":
_SHP_TO_COMPARE[s] = SHAPES_TO_COMPARE[s]
elif GIS_SOFTWARE == "GRASS":
# To GRASS
grsV = shp_to_grs(s, get_filename(s), asCMD=True)
# Change name of column with comparing value
rename_col(grsV,
SHAPES_TO_COMPARE[s],
"lulc_{}".format(i),
as_cmd=True)
# Export
shp = grs_to_shp(grsV,
os.path.join(OUT_FOLDER, grsV + '_rn.shp'),
"area")
_SHP_TO_COMPARE[shp] = "lulc_{}".format(i)
i += 1
SHAPES_TO_COMPARE = _SHP_TO_COMPARE
if GIS_SOFTWARE == "ARCGIS":
from gasp.cpu.arcg.mng.fld import calc_fld
from gasp.cpu.arcg.mng.wspace import create_geodb
from gasp.mng.gen import copy_feat
# Sanitize data and Add new field
__SHAPES_TO_COMPARE = {}
i = 0
# Create GeoDatabase
geodb = create_geodb(OUT_FOLDER, 'geo_sanitize')
""" Sanitize Data """
for k in SHAPES_TO_COMPARE:
# Send data to GeoDatabase only to sanitize
newFc = shp_to_shp(k,
os.path.join(geodb, get_filename(k)),
gisApi='arcpy')
# Create a copy to change
newShp = copy_feat(newFc,
os.path.join(OUT_FOLDER, os.path.basename(k)),
gisApi='arcpy')
# Sanitize field name with interest data
NEW_FLD = "lulc_{}".format(i)
calc_fld(newShp,
NEW_FLD,
"[{}]".format(SHAPES_TO_COMPARE[k]),
isNewField={
"TYPE": "INTEGER",
"LENGTH": 5,
"PRECISION": ""
})
__SHAPES_TO_COMPARE[newShp] = NEW_FLD
i += 1
else:
__SHAPES_TO_COMPARE = SHAPES_TO_COMPARE
# Create database
conPARAM["DATABASE"] = create_db(conPARAM, DB)
""" Union SHAPEs """
UNION_SHAPE = {}
FIX_GEOM = {}
def fix_geometry(shp):
# Send data to PostgreSQL
nt = shp_to_psql(conPARAM, shp, SRS_CODE, api='shp2pgsql')
# Fix data
corr_tbl = fix_geom(conPARAM,
nt,
"geom",
"corr_{}".format(nt),
colsSelect=['gid', __SHAPES_TO_COMPARE[shp]])
# Check if we have multiple geometries
geomN = check_geomtype_in_table(conPARAM, corr_tbl)
if geomN > 1:
corr_tbl = select_main_geom_type(conPARAM, corr_tbl,
"corr2_{}".format(nt))
# Export data again
newShp = psql_to_shp(conPARAM,
corr_tbl,
os.path.join(OUT_FOLDER, corr_tbl + '.shp'),
api='pgsql2shp',
geom_col='geom')
return newShp
SHPS = __SHAPES_TO_COMPARE.keys()
for i in range(len(SHPS)):
for e in range(i + 1, len(SHPS)):
if GIS_SOFTWARE == 'ARCGIS':
# Try the union thing
unShp = union(SHPS[i],
SHPS[e],
os.path.join(OUT_FOLDER,
"un_{}_{}.shp".format(i, e)),
api_gis="arcpy")
# See if the union went all right
if not os.path.exists(unShp):
# Union went not well
# See if geometry was already fixed
if SHPS[i] not in FIX_GEOM:
# Fix SHPS[i] geometry
FIX_GEOM[SHPS[i]] = fix_geometry(SHPS[i])
if SHPS[e] not in FIX_GEOM:
FIX_GEOM[SHPS[e]] = fix_geometry(SHPS[e])
# Run Union again
unShp = union(FIX_GEOM[SHPS[i]],
FIX_GEOM[SHPS[e]],
os.path.join(OUT_FOLDER,
"un_{}_{}_f.shp".format(i, e)),
api_gis="arcpy")
elif GIS_SOFTWARE == "GRASS":
# Optimized Union
print "Union between {} and {}".format(SHPS[i], SHPS[e])
time_a = datetime.datetime.now().replace(microsecond=0)
__unShp = optimized_union_anls(
SHPS[i],
SHPS[e],
os.path.join(OUT_FOLDER, "un_{}_{}.shp".format(i, e)),
GRASS_REGION_TEMPLATE,
SRS_CODE,
os.path.join(OUT_FOLDER, "work_{}_{}".format(i, e)),
multiProcess=True)
time_b = datetime.datetime.now().replace(microsecond=0)
print time_b - time_a
# Rename cols
unShp = rename_column(
__unShp, {
"a_" + __SHAPES_TO_COMPARE[SHPS[i]]:
__SHAPES_TO_COMPARE[SHPS[i]],
"b_" + __SHAPES_TO_COMPARE[SHPS[e]]:
__SHAPES_TO_COMPARE[SHPS[e]]
}, os.path.join(OUT_FOLDER, "un_{}_{}_rn.shp".format(i,
e)))
UNION_SHAPE[(SHPS[i], SHPS[e])] = unShp
# Send data one more time to postgresql
SYNTH_TBL = {}
for uShp in UNION_SHAPE:
# Send data to PostgreSQL
union_tbl = shp_to_psql(conPARAM,
UNION_SHAPE[uShp],
SRS_CODE,
api='shp2pgsql')
# Produce table with % of area equal in both maps
areaMapTbl = ntbl_by_query(
conPARAM,
"{}_syn".format(union_tbl),
("SELECT CAST('{lulc_1}' AS text) AS lulc_1, "
"CAST('{lulc_2}' AS text) AS lulc_2, "
"round("
"CAST(SUM(g_area) / 1000000 AS numeric), 4"
") AS agree_area, round("
"CAST((SUM(g_area) / MIN(total_area)) * 100 AS numeric), 4"
") AS agree_percentage, "
"round("
"CAST(MIN(total_area) / 1000000 AS numeric), 4"
") AS total_area FROM ("
"SELECT {map1_cls}, {map2_cls}, ST_Area(geom) AS g_area, "
"CASE "
"WHEN {map1_cls} = {map2_cls} "
"THEN 1 ELSE 0 "
"END AS isthesame, total_area FROM {tbl}, ("
"SELECT SUM(ST_Area(geom)) AS total_area FROM {tbl}"
") AS foo2"
") AS foo WHERE isthesame = 1 "
"GROUP BY isthesame").format(
lulc_1=get_filename(uShp[0]),
lulc_2=get_filename(uShp[1]),
map1_cls=__SHAPES_TO_COMPARE[uShp[0]],
map2_cls=__SHAPES_TO_COMPARE[uShp[1]],
tbl=union_tbl),
api='psql')
# Produce confusion matrix for the pair in comparison
lulcCls = query_to_df(
conPARAM,
("SELECT fcol FROM ("
"SELECT CAST({map1_cls} AS text) AS fcol FROM {tbl} "
"GROUP BY {map1_cls} "
"UNION ALL SELECT CAST({map2_cls} AS text) FROM {tbl} "
"GROUP BY {map2_cls}"
") AS foo GROUP BY fcol ORDER BY fcol").format(
tbl=union_tbl,
map1_cls=__SHAPES_TO_COMPARE[uShp[0]],
map2_cls=__SHAPES_TO_COMPARE[uShp[1]]),
db_api='psql').fcol.tolist()
matrixTbl = ntbl_by_query(
conPARAM,
"{}_matrix".format(union_tbl),
("SELECT * FROM crosstab('"
"SELECT CASE "
"WHEN foo.{map1_cls} IS NOT NULL "
"THEN foo.{map1_cls} ELSE jtbl.flyr "
"END AS lulc1_cls, CASE "
"WHEN foo.{map2_cls} IS NOT NULL "
"THEN foo.{map2_cls} ELSE jtbl.slyr "
"END AS lulc2_cls, CASE "
"WHEN foo.garea IS NOT NULL "
"THEN round(CAST(foo.garea / 1000000 AS numeric)"
", 3) ELSE 0 "
"END AS garea FROM ("
"SELECT CAST({map1_cls} AS text) AS {map1_cls}, "
"CAST({map2_cls} AS text) AS {map2_cls}, "
"SUM(ST_Area(geom)) AS garea "
"FROM {tbl} GROUP BY {map1_cls}, {map2_cls}"
") AS foo FULL JOIN ("
"SELECT f.flyr, s.slyr FROM ("
"SELECT CAST({map1_cls} AS text) AS flyr "
"FROM {tbl} GROUP BY {map1_cls}"
") AS f, ("
"SELECT CAST({map2_cls} AS text) AS slyr "
"FROM {tbl} GROUP BY {map2_cls}"
") AS s"
") AS jtbl "
"ON foo.{map1_cls} = jtbl.flyr AND "
"foo.{map2_cls} = jtbl.slyr "
"ORDER BY 1,2"
"') AS ct("
"lulc_cls text, {crossCols}"
")").format(crossCols=", ".join(
["cls_{} numeric".format(c) for c in lulcCls]),
tbl=union_tbl,
map1_cls=__SHAPES_TO_COMPARE[uShp[0]],
map2_cls=__SHAPES_TO_COMPARE[uShp[1]]),
api='psql')
SYNTH_TBL[uShp] = {"TOTAL": areaMapTbl, "MATRIX": matrixTbl}
# UNION ALL TOTAL TABLES
total_table = tbls_to_tbl(conPARAM,
[SYNTH_TBL[k]["TOTAL"] for k in SYNTH_TBL],
'total_table')
# Create table with % of agreement between each pair of maps
mapsNames = query_to_df(
conPARAM,
("SELECT lulc FROM ("
"SELECT lulc_1 AS lulc FROM {tbl} GROUP BY lulc_1 "
"UNION ALL "
"SELECT lulc_2 AS lulc FROM {tbl} GROUP BY lulc_2"
") AS lu GROUP BY lulc ORDER BY lulc").format(tbl=total_table),
db_api='psql').lulc.tolist()
FLDS_TO_PIVOT = ["agree_percentage", "total_area"]
Q = ("SELECT * FROM crosstab('"
"SELECT CASE "
"WHEN foo.lulc_1 IS NOT NULL THEN foo.lulc_1 ELSE jtbl.tmp1 "
"END AS lulc_1, CASE "
"WHEN foo.lulc_2 IS NOT NULL THEN foo.lulc_2 ELSE jtbl.tmp2 "
"END AS lulc_2, CASE "
"WHEN foo.{valCol} IS NOT NULL THEN foo.{valCol} ELSE 0 "
"END AS agree_percentage FROM ("
"SELECT lulc_1, lulc_2, {valCol} FROM {tbl} UNION ALL "
"SELECT lulc_1, lulc_2, {valCol} FROM ("
"SELECT lulc_1 AS lulc_2, lulc_2 AS lulc_1, {valCol} "
"FROM {tbl}"
") AS tst"
") AS foo FULL JOIN ("
"SELECT lulc_1 AS tmp1, lulc_2 AS tmp2 FROM ("
"SELECT lulc_1 AS lulc_1 FROM {tbl} GROUP BY lulc_1 "
"UNION ALL "
"SELECT lulc_2 AS lulc_1 FROM {tbl} GROUP BY lulc_2"
") AS tst_1, ("
"SELECT lulc_1 AS lulc_2 FROM {tbl} GROUP BY lulc_1 "
"UNION ALL "
"SELECT lulc_2 AS lulc_2 FROM {tbl} GROUP BY lulc_2"
") AS tst_2 WHERE lulc_1 = lulc_2 GROUP BY lulc_1, lulc_2"
") AS jtbl ON foo.lulc_1 = jtbl.tmp1 AND foo.lulc_2 = jtbl.tmp2 "
"ORDER BY lulc_1, lulc_2"
"') AS ct("
"lulc_map text, {crossCols}"
")")
TOTAL_AGREE_TABLE = None
TOTAL_AREA_TABLE = None
for f in FLDS_TO_PIVOT:
if not TOTAL_AGREE_TABLE:
TOTAL_AGREE_TABLE = ntbl_by_query(
conPARAM,
"agreement_table",
Q.format(tbl=total_table,
valCol=f,
crossCols=", ".join([
"{} numeric".format(map_) for map_ in mapsNames
])),
api='psql')
else:
TOTAL_AREA_TABLE = ntbl_by_query(
conPARAM,
"area_table",
Q.format(tbl=total_table,
valCol=f,
crossCols=", ".join([
"{} numeric".format(map_) for map_ in mapsNames
])),
api='psql')
# Union Mapping
UNION_MAPPING = pandas.DataFrame(
[[
get_filename(k[0]),
get_filename(k[1]),
get_filename(UNION_SHAPE[k])
] for k in UNION_SHAPE],
columns=['shp_a', 'shp_b', 'union_shp'
]) if GIS_SOFTWARE == "ARCGIS" else pandas.DataFrame(
[[k[0], k[1], get_filename(UNION_SHAPE[k])]
for k in UNION_SHAPE],
columns=['shp_a', 'shp_b', 'union_shp'])
UNION_MAPPING = df_to_db(conPARAM, UNION_MAPPING, 'union_map', api='psql')
# Export Results
TABLES = [UNION_MAPPING, TOTAL_AGREE_TABLE, TOTAL_AREA_TABLE
] + [SYNTH_TBL[x]["MATRIX"] for x in SYNTH_TBL]
SHEETS = ["union_map", "agreement_percentage", "area_with_data_km"] + [
"{}_{}".format(get_filename(x[0])[:15],
get_filename(x[1])[:15]) for x in SYNTH_TBL
]
db_to_xls(conPARAM, ["SELECT * FROM {}".format(x) for x in TABLES],
REPORT,
sheetsNames=SHEETS,
dbAPI='psql')
return REPORT
def service_area_use_meters(net,
rdv,
distance,
loc,
out,
OVERLAP=True,
ONEWAY=None):
"""
Execute service area tool using metric distances
"""
from gasp.mng.gen import copy_feat
if arcpy.CheckExtension("Network") == "Available":
arcpy.CheckOutExtension("Network")
else:
raise ValueError('Network analyst extension is not avaiable')
network_name = str(os.path.basename(net))
JUNCTIONS = network_name + '_Junctions'
oneway = "" if not ONEWAY else "Oneway"
INTERVALS = str(distance) if type(distance) == int or \
type(distance) == float else distance if \
type(distance) == str or type(distance) == unicode \
else ' '.join([str(int(x)) for x in distance]) if \
type(distance) == list else None
if not INTERVALS: raise ValueError(('distance format is not valid'))
arcpy.MakeServiceAreaLayer_na(
in_network_dataset=net,
out_network_analysis_layer="servArea",
impedance_attribute="Length",
travel_from_to="TRAVEL_FROM",
default_break_values=INTERVALS,
polygon_type="DETAILED_POLYS",
merge="NO_MERGE" if OVERLAP else "NO_OVERLAP",
nesting_type="RINGS",
line_type="NO_LINES",
overlap="OVERLAP" if OVERLAP else "NON_OVERLAP",
split="NO_SPLIT",
excluded_source_name="",
accumulate_attribute_name="",
UTurn_policy="NO_UTURNS",
restriction_attribute_name=oneway,
polygon_trim="TRIM_POLYS",
poly_trim_value="250 Meters",
lines_source_fields="NO_LINES_SOURCE_FIELDS",
hierarchy="NO_HIERARCHY",
time_of_day="")
# Add locations to the service area layer
arcpy.AddLocations_na(
"servArea", "Facilities", loc, "", "5000 Meters", "",
"{_rdv} SHAPE;{j} NONE".format(_rdv=str(rdv), j=str(JUNCTIONS)),
"MATCH_TO_CLOSEST", "APPEND", "NO_SNAP", "5 Meters", "INCLUDE",
"{_rdv} #;{j} #".format(_rdv=str(rdv), j=str(JUNCTIONS)))
# Solve
arcpy.Solve_na("servArea", "SKIP", "TERMINATE", "")
# Export to a shapefile
save_servArea = copy_feat("servArea\\Polygons", out, gisApi='arcpy')
return save_servArea