def arcg_area(polyTbl, lulcNomenclature, UPPER=True):
"""
Select Features with area upper than
"""
from gasp.osm2lulc.utils import osm_features_by_rule
from gasp.cpu.arcg.anls.exct import select_by_attr
from gasp.cpu.arcg.mng.fld import add_geom_attr
from gasp.cpu.arcg.mng.gen import delete
from gasp.mng.genze import dissolve
from gasp.prop.feat import feat_count
KEY_COL = DB_SCHEMA["OSM_FEATURES"]["OSM_KEY"]
VALUE_COL = DB_SCHEMA["OSM_FEATURES"]["OSM_VALUE"]
LULC_COL = DB_SCHEMA[lulcNomenclature]["CLS_FK"]
RULE_COL = DB_SCHEMA[lulcNomenclature]["RULES_FIELDS"]["AREA"]
# Get OSM Features to be selected for this rule
__serv = 'area_upper' if UPPER else 'area_lower'
osmToSelect = osm_features_by_rule(lulcNomenclature, __serv)
operator = " > " if UPPER else " < "
osmToSelect[VALUE_COL] = "(" + osmToSelect[KEY_COL] + "='" + \
osmToSelect[VALUE_COL] + "' AND shp_area" + operator + \
osmToSelect[RULE_COL].astype(str) + ")"
lulcCls = osmToSelect[LULC_COL].unique()
clsVect = {}
WORK = os.path.dirname(polyTbl)
for cls in lulcCls:
# Select and Export
filterDf = osmToSelect[osmToSelect[LULC_COL] == cls]
fShp = select_by_attr(
polyTbl, str(filterDf[VALUE_COL].str.cat(sep=" OR ")),
os.path.join(WORK, "{}_{}".format(__serv, str(cls))))
if not feat_count(fShp, gisApi='arcpy'): continue
# Dissolve
dissShp = dissolve(fShp,
os.path.join(WORK,
"{}_d_{}".format(__serv, str(cls))),
"OBJECTID",
geomMultiPart=None,
api='arcpy')
if not feat_count(dissShp, gisApi='arcpy'): continue
clsVect[int(cls)] = dissShp
delete(fShp)
return clsVect
def download_by_boundary(input_boundary,
output_osm,
epsg,
area_threshold=None,
GetUrl=True):
"""
Download data from OSM using a bounding box
"""
import os
from decimal import Decimal
from gasp.prop.feat import feat_count
from gasp.prop.ext import get_extent
from gasp.web import get_file
# Check number of features
number_feat = feat_count(input_boundary)
if number_feat != 1:
raise ValueError(
('Your boundary has more than one feature. '
'Only feature classes with one feature are allowed.'))
# Check boundary area
if area_threshold:
from gasp.cpu.gdl import area_to_dic
d_area = area_to_dic(input_boundary)
if d_area[0] > area_threshold:
raise ValueError('{} has more than than {} square meters'.format(
os.path.basename(input_boundary), str(area_threshold)))
# Get boundary extent
left, right, bottom, top = get_extent(input_boundary, gisApi='ogr')
if epsg != 4326:
from gasp.to.geom import create_point
from gasp.mng.prj import project_geom
bottom_left = project_geom(create_point(left, bottom, api='ogr'),
epsg,
4326,
api='ogr')
top_right = project_geom(create_point(right, top, api='ogr'),
epsg,
4326,
api='ogr')
left, bottom = bottom_left.GetX(), bottom_left.GetY()
right, top = top_right.GetX(), top_right.GetY()
bbox_str = ','.join([str(left), str(bottom), str(right), str(top)])
url = "http://overpass-api.de/api/map?bbox={box}".format(box=bbox_str)
if GetUrl:
return url
osm_file = get_file(url, output_osm)
return output_osm
def del_empty_files(folder, file_format):
"""
List all feature classes in a folder and del the files with
0 features
"""
from gasp.oss import list_files
from gasp.prop.feat import feat_count
fc = list_files(folder, file_format=file_format)
for shp in fc:
feat_number = feat_count(shp, gisApi='arcpy')
if not feat_number:
delete(shp)
def splitShp_by_range(shp, nrFeat, outFolder):
"""
Split one feature class by range
"""
import os
from gasp.oss import get_filename, get_fileformat
from gasp.prop.feat import feat_count
from gasp.mng.fld import lst_fld
from gasp.anls.exct import sel_by_attr
rowsN = feat_count(shp, gisApi='ogr')
nrShp = int(rowsN / float(nrFeat)) + 1 if nrFeat != rowsN else 1
fields = lst_fld(shp)
offset = 0
exportedShp = []
for i in range(nrShp):
outShp = sel_by_attr(
shp,
"SELECT {cols}, geometry FROM {t} ORDER BY {cols} LIMIT {l} OFFSET {o}"
.format(t=os.path.splitext(os.path.basename(shp))[0],
l=str(nrFeat),
o=str(offset),
cols=", ".join(fields)),
os.path.join(
outFolder, "{}_{}{}".format(get_filename(shp, forceLower=True),
str(i), get_fileformat(shp))),
api_gis='ogr')
exportedShp.append(outShp)
offset += nrFeat
return exportedShp
def download_by_boundary(input_boundary, output_osm, epsg, GetUrl=True):
"""
Download data from OSM using a bounding box
"""
import os
from osgeo import ogr
from gasp.web import get_file
from gasp.oss import get_fileformat, get_filename
if type(input_boundary) == dict:
if 'top' in input_boundary and 'bottom' in input_boundary \
and 'left' in input_boundary and 'right' in input_boundary:
left, right, bottom, top = (input_boundary['left'],
input_boundary['right'],
input_boundary['bottom'],
input_boundary['top'])
else:
raise ValueError(
('input_boundary is a dict but the keys are not correct. '
'Please use left, right, top and bottom as keys'))
elif type(input_boundary) == list:
if len(input_boundary) == 4:
left, right, bottom, top = input_boundary
else:
raise ValueError(
('input boundary is a list with more than 4 objects. '
'The list should be like: '
'l = [left, right, bottom, top]'))
elif type(input_boundary) == ogr.Geometry:
# Check if we have a polygon
geom_name = input_boundary.GetGeometryName()
if geom_name == 'POLYGON' or geom_name == 'MULTIPOLYGON':
# Get polygon extent
left, right, bottom, top = input_boundary.GetEnvelope()
else:
raise ValueError(('Your boundary is a non POLYGON ogr Geometry '))
else:
# Assuming input boundary is a file
#Check if file exists
if not os.path.exists(input_boundary):
raise ValueError(
("Sorry, but the file {} does not exist inside the folder {}!"
).format(os.path.basename(input_boundary),
os.path.dirname(input_boundary)))
# Check if is a raster
from gasp.prop.ff import check_isRaster
isRst = check_isRaster(input_boundary)
if isRst:
from gasp.prop.ext import rst_ext
# Get raster extent
left, right, bottom, top = rst_ext(input_boundary)
else:
from gasp.prop.feat import feat_count
from gasp.prop.ext import get_extent
# Check number of features
number_feat = feat_count(input_boundary, gisApi='ogr')
if number_feat != 1:
raise ValueError(
('Your boundary has more than one feature. '
'Only feature classes with one feature are allowed.'))
# Get boundary extent
left, right, bottom, top = get_extent(input_boundary, gisApi='ogr')
if epsg != 4326:
from gasp.to.geom import create_point
from gasp.mng.prj import project_geom
bottom_left = project_geom(create_point(left, bottom, api='ogr'),
epsg,
4326,
api='ogr')
top_right = project_geom(create_point(right, top, api='ogr'),
epsg,
4326,
api='ogr')
left, bottom = bottom_left.GetX(), bottom_left.GetY()
right, top = top_right.GetX(), top_right.GetY()
bbox_str = ','.join([str(left), str(bottom), str(right), str(top)])
url = "https://overpass-api.de/api/map?bbox={box}".format(box=bbox_str)
if GetUrl:
return url
if get_fileformat(output_osm) != '.xml':
output_osm = os.path.join(os.path.dirname(output_osm),
get_filename(output_osm) + '.xml')
osm_file = get_file(url, output_osm)
return output_osm
def conditional_dependence(movs, indp):
"""
Estimate conditional dependence between several rasters
"""
import math
from decimal import Decimal
from gasp.prop.feat import feat_count
from gasp.prop.rst import get_cellsize, count_cells
from gasp.stats.rst import frequencies
def foundPredT(dic):
PredT = 0.0
for value in dic.keys():
count = dic[value]
PredT += (float(value) * count)
return PredT
def foundTStd(dic, c):
TVar = 0.0
for value in dic.keys():
count = dic[value]
TVar += (float(value) * count * c)**2
TStd = math.sqrt(TVar)
return TStd
def calcCondIndp(Z):
pi = math.pi
const6 = 0.2316419
b1 = 0.31938153
b2 = -0.356563782
b3 = 1.781477937
b4 = -1.821255978
b5 = 1.330274429
Z = float(Z)
X = Z
if X < 0.0: X = -Z
t = 1.0 / (const6 * X + 1.0)
pid = 2.0 * pi
XX = -X * X / 2.0
XX = math.exp(XX) / math.sqrt(pid)
PZ = (b1 * t) + (b2 * t * t) + (b3 * (t**3)) + (b4 * (t**4)) + (b5 *
(t**5))
PZ = 1.0 - (PZ * XX)
if Z < 0: PZ = 1.0 - PZ
return PZ
# Count phenomena ocorrences - total number of landslides
ExpNumTP = Decimal(feat_count(movs, gisApi='ogr'))
# Count the number of cell raster
NrCell = count_cells(indp[0])
# Get Cellsize of the raster's
cellsize = Decimal(get_cellsize(indp[0], gisApi='gdal'))
# Calculate UnitArea
area_km = Decimal(((cellsize * cellsize) * NrCell) / 1000000.0)
UnitArea = Decimal((area_km / ExpNumTP) / 40.0)
ConvFac = Decimal((cellsize**2) / 1000000.0 / UnitArea)
# Count the number of times that one class has representation in the raster; put this associations in a ditionary
sudoDic = {var: frequencies(var) for var in indp}
# Calculate Conditional Dependence
nr = 1
for rst in indp:
l_overall = {}
l_ric = {}
l_tac = {}
# Calculate PredT
PredT = foundPredT(sudoDic[rst])
PredT *= ConvFac
for _rst_ in indp:
# Calculate TStd
TStd = foundTStd(sudoDic[_rst_], ConvFac)
TS = (PredT - ExpNumTP) / TStd
n = ExpNumTP
T = PredT
P = calcCondIndp(TS) * 100.0
if P > 50.0: overallCI = 100.0 * (100.0 - P) / 50.0
else: overallCI = 100.0 * (100.0 - (50.0 + (50.0 - P))) / 50.0
l_overall[(rst, _rst_)] = "%.2f" % overallCI
ric = n / T
l_ric[(rst, _rst_)] = "%.2f" % ric
l_tac[(rst, _rst_)] = "%.2f" % P
nr += 1
return {'OVERALL': l_overall, 'RIC': l_ric, 'TAC': l_tac}
def prod_matrix(origins,
destinations,
networkGrs,
speedLimitCol,
onewayCol,
thrdId="1",
asCmd=None):
"""
Get Matrix Distance:
"""
from gasp.to.shp.grs import shp_to_grs
from gasp.cpu.grs.mng import category
from gasp.anls.exct import sel_by_attr
from gasp.mng.grstbl import add_field, add_table, update_table
from gasp.mob.grstbx import network_from_arcs
from gasp.mob.grstbx import add_pnts_to_network
from gasp.mob.grstbx import run_allpairs
from gasp.cpu.grs.mng.feat import geomattr_to_db
from gasp.cpu.grs.mng.feat import copy_insame_vector
from gasp.mng.gen import merge_feat
from gasp.prop.feat import feat_count
# Merge Origins and Destinations into the same Feature Class
ORIGINS_NFEAT = feat_count(origins, gisApi='pandas')
DESTINATIONS_NFEAT = feat_count(destinations, gisApi='pandas')
ORIGINS_DESTINATIONS = merge_feat([origins, destinations],
os.path.join(
os.path.dirname(origins),
"points_od_{}.shp".format(thrdId)),
api='pandas')
pointsGrs = shp_to_grs(ORIGINS_DESTINATIONS,
"points_od_{}".format(thrdId),
asCMD=asCmd)
# Connect Points to Network
newNetwork = add_pnts_to_network(networkGrs,
pointsGrs,
"rdv_points_{}".format(thrdId),
asCMD=asCmd)
# Sanitize Network Table and Cost Columns
newNetwork = category(newNetwork,
"rdv_points_time_{}".format(thrdId),
"add",
LyrN="3",
geomType="line",
asCMD=asCmd)
add_table(newNetwork,
("cat integer,kph double precision,length double precision,"
"ft_minutes double precision,"
"tf_minutes double precision,oneway text"),
lyrN=3,
asCMD=asCmd)
copy_insame_vector(newNetwork,
"kph",
speedLimitCol,
3,
geomType="line",
asCMD=asCmd)
copy_insame_vector(newNetwork,
"oneway",
onewayCol,
3,
geomType="line",
asCMD=asCmd)
geomattr_to_db(newNetwork,
"length",
"length",
"line",
createCol=False,
unit="meters",
lyrN=3,
ascmd=asCmd)
update_table(newNetwork, "kph", "3.6", "kph IS NULL", lyrN=3, ascmd=asCmd)
update_table(newNetwork, "kph", "3.6", "oneway = 'N'", lyrN=3, ascmd=asCmd)
update_table(newNetwork,
"ft_minutes",
"(length * 60) / (kph * 1000.0)",
"ft_minutes IS NULL",
lyrN=3,
ascmd=asCmd)
update_table(newNetwork,
"tf_minutes",
"(length * 60) / (kph * 1000.0)",
"tf_minutes IS NULL",
lyrN=3,
ascmd=asCmd)
# Exagerate Oneway's
update_table(newNetwork,
"ft_minutes",
"1000",
"oneway = 'TF'",
lyrN=3,
ascmd=asCmd)
update_table(newNetwork,
"tf_minutes",
"1000",
"oneway = 'FT'",
lyrN=3,
ascmd=asCmd)
# Produce matrix
matrix = run_allpairs(newNetwork,
"ft_minutes",
"tf_minutes",
'result_{}'.format(thrdId),
arcLyr=3,
nodeLyr=2,
asCMD=asCmd)
# Exclude unwanted OD Pairs
q = "({}) AND ({})".format(
" OR ".join(
["from_cat={}".format(str(i + 1)) for i in range(ORIGINS_NFEAT)]),
" OR ".join([
"to_cat={}".format(str(ORIGINS_NFEAT + i + 1))
for i in range(DESTINATIONS_NFEAT)
]))
matrix_sel = sel_by_attr(matrix,
q,
"sel_{}".format(matrix),
geomType="line",
lyrN=3,
asCMD=asCmd)
add_field(matrix_sel, "from_fid", "INTEGER", lyrN=3, asCMD=asCmd)
add_field(matrix_sel, "to_fid", "INTEGER", lyrN=3, asCMD=asCmd)
update_table(matrix_sel,
"from_fid",
"from_cat - 1",
"from_fid IS NULL",
lyrN=3,
ascmd=asCmd)
update_table(matrix_sel,
"to_fid",
"to_cat - {} - 1".format(str(ORIGINS_NFEAT)),
"to_fid IS NULL",
lyrN=3,
ascmd=asCmd)
return matrix_sel
def distance_between_catpoints(srcShp, facilitiesShp, networkShp,
speedLimitCol, onewayCol, grsWorkspace,
grsLocation, outputShp):
"""
Path bet points
TODO: Work with files with cat
"""
import os
from gasp.oss import get_filename
from gasp.session import run_grass
from gasp.mng.gen import merge_feat
from gasp.prop.feat import feat_count
# Merge Source points and Facilities into the same Feature Class
SRC_NFEAT = feat_count(srcShp, gisApi='pandas')
FACILITY_NFEAT = feat_count(facilitiesShp, gisApi='pandas')
POINTS = merge_feat([srcShp, facilitiesShp],
os.path.join(os.path.dirname(outputShp),
"points_net.shp"),
api='pandas')
# Open an GRASS GIS Session
gbase = run_grass(grsWorkspace,
grassBIN="grass76",
location=grsLocation,
srs=networkShp)
import grass.script as grass
import grass.script.setup as gsetup
gsetup.init(gbase, grsWorkspace, grsLocation, 'PERMANENT')
# Import GRASS GIS Module
from gasp.to.shp.grs import shp_to_grs
from gasp.to.shp.grs import grs_to_shp
from gasp.cpu.grs.mng import category
from gasp.mng.grstbl import add_field, add_table, update_table
from gasp.mob.grstbx import network_from_arcs
from gasp.mob.grstbx import add_pnts_to_network
from gasp.mob.grstbx import netpath
from gasp.cpu.grs.mng.feat import geomattr_to_db
from gasp.cpu.grs.mng.feat import copy_insame_vector
# Add Data to GRASS GIS
rdvMain = shp_to_grs(networkShp, get_filename(networkShp, forceLower=True))
pntShp = shp_to_grs(POINTS, "points_net")
"""Get closest facility layer:"""
# Connect Points to Network
newNetwork = add_pnts_to_network(rdvMain, pntShp, "rdv_points")
# Sanitize Network Table and Cost Columns
newNetwork = category(newNetwork,
"rdv_points_time",
"add",
LyrN="3",
geomType="line")
add_table(newNetwork,
("cat integer,kph double precision,length double precision,"
"ft_minutes double precision,"
"tf_minutes double precision,oneway text"),
lyrN=3)
copy_insame_vector(newNetwork, "kph", speedLimitCol, 3, geomType="line")
copy_insame_vector(newNetwork, "oneway", onewayCol, 3, geomType="line")
geomattr_to_db(newNetwork,
"length",
"length",
"line",
createCol=False,
unit="meters",
lyrN=3)
update_table(newNetwork, "kph", "3.6", "kph IS NULL", lyrN=3)
update_table(newNetwork,
"ft_minutes",
"(length * 60) / (kph * 1000.0)",
"ft_minutes IS NULL",
lyrN=3)
update_table(newNetwork,
"tf_minutes",
"(length * 60) / (kph * 1000.0)",
"tf_minutes IS NULL",
lyrN=3)
# Exagerate Oneway's
update_table(newNetwork, "ft_minutes", "1000", "oneway = 'TF'", lyrN=3)
update_table(newNetwork, "tf_minutes", "1000", "oneway = 'FT'", lyrN=3)
# Produce result
result = netpath(newNetwork,
"ft_minutes",
"tf_minutes",
get_filename(outputShp),
arcLyr=3,
nodeLyr=2)
return grs_to_shp(result, outputShp, geomType="line", lyrN=3)
def arcg_buffering(lineTbl, nomenclature):
"""
Create a beautiful buffer from osm line features
"""
from gasp.osm2lulc.utils import osm_features_by_rule
from gasp.cpu.arcg.anls.exct import select_by_attr
from gasp.anls.prox.bf import _buffer
from gasp.cpu.arcg.mng.gen import delete
from gasp.prop.feat import feat_count
KEY_COL = DB_SCHEMA["OSM_FEATURES"]["OSM_KEY"]
VALUE_COL = DB_SCHEMA["OSM_FEATURES"]["OSM_VALUE"]
LULC_COL = DB_SCHEMA[nomenclature]["CLS_FK"]
BF_COL = DB_SCHEMA[nomenclature]["RULES_FIELDS"]["BUFFER"]
# Get OSM Features to be selected for this rule
osmToSelect = osm_features_by_rule(nomenclature, 'basic_buffer')
osmToSelect[VALUE_COL] = osmToSelect[KEY_COL] + "='" + \
osmToSelect[VALUE_COL] + "'"
# Get LULC Classes to be selected
lulcCls = osmToSelect[LULC_COL].unique()
resCls = {}
WORK = os.path.dirname(lineTbl)
for cls in lulcCls:
filterDf = osmToSelect[osmToSelect[LULC_COL] == cls]
# Get distances for this cls
distances = filterDf[BF_COL].unique()
for dist in distances:
__filter = filterDf[filterDf[BF_COL] == dist]
fShp = select_by_attr(
lineTbl, str(__filter[VALUE_COL].str.cat(sep=" OR ")),
os.path.join(WORK, "buf_{}_{}".format(str(cls),
str(int(dist)))))
if not feat_count(fShp, gisApi='arcpy'): continue
bfShp = _buffer(fShp,
str(dist).replace('.', ','),
os.path.join(
WORK, "bufbf_{}_{}".format(
str(cls),
str(dist).replace(".", "_"))),
api='arcpy',
dissolve="ALL")
if cls not in resCls:
resCls[int(cls)] = [bfShp]
else:
resCls[int(cls)].append(bfShp)
delete(fShp)
return resCls