def clip_and_union(la, lb, cell, work, ref, proc, output):
# Start GRASS GIS Session
grsbase = run_grass(work, location="proc_" + str(proc), srs=ref)
import grass.script.setup as gsetup
gsetup.init(grsbase, work, "proc_" + str(proc), 'PERMANENT')
# Import GRASS GIS modules
from gasp.to.shp.grs import shp_to_grs
from gasp.to.shp.grs import grs_to_shp
# Add data to GRASS
a = shp_to_grs(la, get_filename(la), asCMD=True)
b = shp_to_grs(lb, get_filename(lb), asCMD=True)
c = shp_to_grs(cell, get_filename(cell), asCMD=True)
# Clip
a_clip = clip(a, c, "{}_clip".format(a), api_gis="grass_cmd")
b_clip = clip(b, c, "{}_clip".format(b), api_gis="grass_cmd")
# Union
u_shp = union(a_clip,
b_clip,
"un_{}".format(c),
api_gis="grass_cmd")
# Export
o = grs_to_shp(u_shp, output, "area")
def bash_matrix_od(origins, destinationShp, network, costCol, oneway, grsWork,
output):
"""
Produce matrix OD using GRASS GIS - BASH MODE
"""
from gasp.session import run_grass
from gasp.oss import get_filename
from gasp.oss.ops import create_folder
from gasp.mng.split import splitShp_by_range
from gasp.mng.gen import merge_feat
# SPLIT ORIGINS IN PARTS
originsFld = create_folder(os.path.join(grsWork, 'origins_parts'))
originsList = splitShp_by_range(origins, 100, originsFld)
# Open an GRASS GIS Session
gbase = run_grass(grsWork,
grassBIN="grass76",
location=grsLoc,
srs=network)
import grass.script as grass
import grass.script.setup as gsetup
RESULTS = []
R_FOLDER = create_folder(os.path.join(grsWork, 'res_parts'))
for e in range(len(originsList)):
gsetup.init(gbase, grsWork, "grs_loc_{}".format(e), 'PERMANENT')
from gasp.to.shp.grs import shp_to_grs, grs_to_shp
# Add Data to GRASS GIS
rdvMain = shp_to_grs(network, get_filename(network, forceLower=True))
# Produce Matrix
result_part = prod_matrix(originsList[e], destinationShp, rdvMain,
costCol, oneway)
# Export Result
shp = grs_to_shp(result_part,
os.path.join(R_FOLDER, result_part + '.shp'),
geom_type="line",
lyrN=3)
RESULTS.append(shp)
merge_feat(RESULTS, output, api='pandas')
return output
def snap_points_to_near_line(lineShp, pointShp, epsg, workGrass,
outPoints, location='overlap_pnts', api='grass',
movesShp=None):
"""
Move points to overlap near line
API's Available:
* grass;
* saga.
"""
if api == 'grass':
"""
Uses GRASS GIS to find near lines.
"""
import os; import numpy
from geopandas import GeoDataFrame
from gasp.oss import get_filename
from gasp.session import run_grass
from gasp.fm import shp_to_df
from gasp.to.shp import df_to_shp
# Create GRASS GIS Location
grassBase = run_grass(workGrass, location=location, srs=epsg)
import grass.script as grass
import grass.script.setup as gsetup
gsetup.init(grassBase, workGrass, location, 'PERMANENT')
# Import some GRASS GIS tools
from gasp.anls.prox import grs_near as near
from gasp.cpu.grs.mng.feat import geomattr_to_db
from gasp.to.shp.grs import shp_to_grs, grs_to_shp
# Import data into GRASS GIS
grsLines = shp_to_grs(
lineShp, get_filename(lineShp, forceLower=True)
)
grsPoint = shp_to_grs(
pointShp, get_filename(pointShp, forceLower=True)
)
# Get distance from points to near line
near(grsPoint, grsLines, nearCatCol="tocat", nearDistCol="todistance")
# Get coord of start/end points of polylines
geomattr_to_db(grsLines, ['sta_pnt_x', 'sta_pnt_y'], 'start', 'line')
geomattr_to_db(grsLines, ['end_pnt_x', 'end_pnt_y'], 'end', 'line')
# Export data from GRASS GIS
ogrPoint = grs_to_shp(grsPoint, os.path.join(
workGrass, grsPoint + '.shp', 'point', asMultiPart=True
))
ogrLine = grs_to_shp(grsLines, os.path.join(
workGrass, grsLines + '.shp', 'point', asMultiPart=True
))
# Points to GeoDataFrame
pntDf = tbl_to_obj(ogrPoint)
# Lines to GeoDataFrame
lnhDf = tbl_to_obj(ogrLine)
# Erase unecessary fields
pntDf.drop(["todistance"], axis=1, inplace=True)
lnhDf.drop([c for c in lnhDf.columns.values if c != 'geometry' and
c != 'cat' and c != 'sta_pnt_x' and c != 'sta_pnt_y' and
c != 'end_pnt_x' and c != 'end_pnt_y'],
axis=1, inplace=True)
# Join Geometries - Table with Point Geometry and Geometry of the
# nearest line
resultDf = pntDf.merge(
lnhDf, how='inner', left_on='tocat', right_on='cat')
# Move points
resultDf['geometry'] = [geoms[0].interpolate(
geoms[0].project(geoms[1])
) for geoms in zip(resultDf.geometry_y, resultDf.geometry_x)]
resultDf.drop(["geometry_x", "geometry_y", "cat_x", "cat_y"],
axis=1, inplace=True)
resultDf = GeoDataFrame(
resultDf, crs={"init" : 'epsg:{}'.format(epsg)}, geometry="geometry"
)
# Check if points are equal to any start/end points
resultDf["x"] = resultDf.geometry.x
resultDf["y"] = resultDf.geometry.y
resultDf["check"] = numpy.where(
(resultDf["x"] == resultDf["sta_pnt_x"]) & (resultDf["y"] == resultDf["sta_pnt_y"]),
1, 0
)
resultDf["check"] = numpy.where(
(resultDf["x"] == resultDf["end_pnt_x"]) & (resultDf["y"] == resultDf["end_pnt_y"]),
1, 0
)
# To file
df_to_shp(resultDf, outPoints)
elif api == 'saga':
"""
Snap Points to Lines using SAGA GIS
"""
from gasp import exec_cmd
cmd = (
"saga_cmd shapes_points 19 -INPUT {pnt} -SNAP {lnh} "
"-OUTPUT {out}{mv}"
).format(
pnt=pointShp, lnh=lineShp, out=outPoints,
mv="" if not movesShp else " -MOVES {}".format(movesShp)
)
outcmd = exec_cmd(cmd)
else:
raise ValueError("{} is not available!".format(api))
return outPoints
def joinLines_by_spatial_rel_raster(mainLines, mainId, joinLines,
joinCol, outfile, epsg):
"""
Join Attributes based on a spatial overlap.
An raster based approach
"""
import os; import pandas
from geopandas import GeoDataFrame
from gasp.to.geom import regulardf_to_geodf
from gasp.session import run_grass
from gasp.oss import get_filename
from gasp.oss.ops import create_folder
from gasp.mng.ext import shpextent_to_boundary
from gasp.mng.joins import join_dfs
from gasp.mng.df import df_groupBy
from gasp.to.rst import shp_to_raster
from gasp.fm import tbl_to_obj
from gasp.to.shp import df_to_shp
workspace = create_folder(os.path.join(
os.path.dirname(mainLines, 'tmp_dt')
))
# Create boundary file
boundary = shpextent_to_boundary(
mainLines, os.path.join(workspace, "bound.shp"),
epsg
)
boundRst = shp_to_raster(boundary, None, 5, -99, os.path.join(
workspace, "rst_base.tif"), epsg=epsg, api='gdal')
# Start GRASS GIS Session
gbase = run_grass(workspace, location="grs_loc", srs=boundRst)
import grass.script as grass
import grass.script.setup as gsetup
gsetup.init(gbase, workspace, "grs_loc", "PERMANENT")
from gasp.spanlst.local import combine
from gasp.cpu.grs.spanlst import get_rst_report_data
from gasp.to.shp.grs import shp_to_grs, grs_to_shp
from gasp.to.rst import shp_to_raster
# Add data to GRASS GIS
mainVector = shp_to_grs(
mainLines, get_filename(mainLines, forceLower=True))
joinVector = shp_to_grs(
joinLines, get_filename(joinLines, forceLower=True))
mainRst = shp_to_raster(
mainVector, mainId, None, None, "rst_" + mainVector, api='pygrass'
); joinRst = shp_to_raster(
joinVector, joinCol, None, None, "rst_" + joinVector, api='pygrass'
)
combRst = combine(mainRst, joinRst, "combine_rst", api="pygrass")
combine_data = get_rst_report_data(combRst, UNITS="c")
combDf = pandas.DataFrame(combine_data, columns=[
"comb_cat", "rst_1", "rst_2", "ncells"
])
combDf = combDf[combDf["rst_2"] != '0']
combDf["ncells"] = combDf["ncells"].astype(int)
gbdata = df_groupBy(combDf, ["rst_1"], "MAX", "ncells")
fTable = join_dfs(gbdata, combDf, ["rst_1", "ncells"], ["rst_1", "ncells"])
fTable["rst_2"] = fTable["rst_2"].astype(int)
fTable = df_groupBy(
fTable, ["rst_1", "ncells"],
STAT='MIN', STAT_FIELD="rst_2"
)
mainLinesCat = grs_to_shp(
mainVector, os.path.join(workspace, mainVector + '.shp'), 'line')
mainLinesDf = tbl_to_obj(mainLinesCat)
resultDf = join_dfs(
mainLinesDf, fTable, "cat", "rst_1",
onlyCombinations=None
)
resultDf.rename(columns={"rst_2" : joinCol}, inplace=True)
resultDf = regulardf_to_geodf(resultDf, "geometry", epsg)
df_to_shp(resultDf, outfile)
return outfile
def join_attr_by_distance(mainTable, joinTable, workGrass, epsg_code,
output):
"""
Find nearest feature and join attributes of the nearest feature
to the mainTable
Uses GRASS GIS to find near lines.
"""
import os
from gasp.session import run_grass
from gasp.fm import tbl_to_obj
from gasp.to.geom import regulardf_to_geodf
from gasp.to.shp import df_to_shp
from gasp.oss import get_filename
# Create GRASS GIS Location
grassBase = run_grass(workGrass, location='join_loc', srs=epsg_code)
import grass.script as grass
import grass.script.setup as gsetup
gsetup.init(grassBase, workGrass, 'join_loc', 'PERMANENT')
# Import some GRASS GIS tools
from gasp.anls.prox import grs_near as near
from gasp.cpu.grs.mng.feat import geomattr_to_db
from gasp.to.shp.grs import shp_to_grs, grs_to_shp
# Import data into GRASS GIS
grsMain = shp_to_grs(mainTable, get_filename(
mainTable, forceLower=True)
); grsJoin = shp_to_grs(joinTable, get_filename(
joinTable, forceLower=True)
)
# Get distance from each feature of mainTable to the nearest feature
# of the join table
near(grsMain, grsJoin, nearCatCol="tocat", nearDistCol="todistance")
# Export data from GRASS GIS
ogrMain = grs_to_shp(grsMain, os.path.join(
workGrass, 'join_loc', grsMain + '_grs.shp'), None, asMultiPart=True
); ogrJoin = grs_to_shp(grsJoin, os.path.join(
workGrass, 'join_loc', grsJoin + '_grs.shp'), None, asMultiPart=True)
dfMain = tbl_to_obj(ogrMain)
dfJoin = tbl_to_obj(ogrJoin)
dfResult = dfMain.merge(dfJoin, how='inner',
left_on='tocat', right_on='cat')
dfResult.drop(["geometry_y", "cat_y"], axis=1, inplace=True)
dfResult.rename(columns={"cat_x" : "cat_grass"}, inplace=True)
dfResult["tocat"] = dfResult["tocat"] - 1
dfResult["cat_grass"] = dfResult["cat_grass"] - 1
dfResult = regulardf_to_geodf(dfResult, "geometry_x", epsg_code)
df_to_shp(dfResult, output)
return output
def vector_based(osmdata,
nomenclature,
refRaster,
lulcShp,
overwrite=None,
dataStore=None,
RoadsAPI='SQLITE'):
"""
Convert OSM Data into Land Use/Land Cover Information
An vector based approach.
TODO: Add a detailed description.
RoadsAPI Options:
* SQLITE
* POSTGIS
"""
# ************************************************************************ #
# Python Modules from Reference Packages #
# ************************************************************************ #
import datetime
import os
import json
# ************************************************************************ #
# GASP dependencies #
# ************************************************************************ #
from gasp.oss.ops import create_folder
from gasp.prop.rst import get_epsg_raster
from gasp.session import run_grass
if RoadsAPI == 'POSTGIS':
from gasp.sql.mng.db import create_db
from gasp.osm2lulc.utils import osm_to_pgsql
else:
from gasp.osm2lulc.utils import osm_to_sqdb
from gasp.osm2lulc.utils import osm_project, add_lulc_to_osmfeat
from gasp.mng.gen import merge_feat
from gasp.osm2lulc.mod1 import grs_vector
if RoadsAPI == 'SQLITE' or RoadsAPI == 'POSTGIS':
from gasp.osm2lulc.mod2 import roads_sqdb
else:
from gasp.osm2lulc.mod2 import grs_vec_roads
from gasp.osm2lulc.m3_4 import grs_vect_selbyarea
from gasp.osm2lulc.mod5 import grs_vect_bbuffer
from gasp.osm2lulc.mod6 import vector_assign_pntags_to_build
# ************************************************************************ #
# Global Settings #
# ************************************************************************ #
if not os.path.exists(os.path.dirname(lulcShp)):
raise ValueError('{} does not exist!'.format(os.path.dirname(lulcShp)))
# Get Parameters to connect to PostgreSQL
conPGSQL = json.load(
open(
os.path.join(os.path.dirname(os.path.abspath(__file__)),
'con-postgresql.json'),
'r')) if RoadsAPI == 'POSTGIS' else None
# Get EPSG of Reference Raster
epsg = get_epsg_raster(refRaster)
if not epsg:
raise ValueError('Cannot get epsg code of ref raster')
time_a = datetime.datetime.now().replace(microsecond=0)
from gasp.osm2lulc.var import osmTableData, PRIORITIES
workspace = os.path.join(os.path.dirname(lulcShp),
'osmtolulc') if not dataStore else dataStore
# Check if workspace exists
if os.path.exists(workspace):
if overwrite:
create_folder(workspace)
else:
raise ValueError('Path {} already exists'.format(workspace))
else:
create_folder(workspace)
__priorities = PRIORITIES[nomenclature]
time_b = datetime.datetime.now().replace(microsecond=0)
if RoadsAPI != 'POSTGIS':
# ******************************************************************** #
# Convert OSM file to SQLITE DB #
# ******************************************************************** #
osm_db = osm_to_sqdb(osmdata, os.path.join(workspace, 'osm.sqlite'))
else:
# Convert OSM file to POSTGRESQL DB #
conPGSQL["DATABASE"] = create_db(conPGSQL,
os.path.splitext(
os.path.basename(osmdata))[0],
overwrite=True)
osm_db = osm_to_pgsql(osmdata, conPGSQL)
time_c = datetime.datetime.now().replace(microsecond=0)
# ************************************************************************ #
# Add Lulc Classes to OSM_FEATURES by rule #
# ************************************************************************ #
add_lulc_to_osmfeat(osm_db if RoadsAPI != 'POSTGIS' else conPGSQL,
osmTableData,
nomenclature,
api='SQLITE' if RoadsAPI != 'POSTGIS' else RoadsAPI)
time_d = datetime.datetime.now().replace(microsecond=0)
# ************************************************************************ #
# Transform SRS of OSM Data #
# ************************************************************************ #
osmTableData = osm_project(
osm_db if RoadsAPI != 'POSTGIS' else conPGSQL,
epsg,
api='SQLITE' if RoadsAPI != 'POSTGIS' else RoadsAPI)
time_e = datetime.datetime.now().replace(microsecond=0)
# ************************************************************************ #
# Start a GRASS GIS Session #
# ************************************************************************ #
grass_base = run_grass(workspace,
grassBIN='grass76',
location='grloc',
srs=epsg)
#import grass.script as grass
import grass.script.setup as gsetup
gsetup.init(grass_base, workspace, 'grloc', 'PERMANENT')
# ************************************************************************ #
# IMPORT SOME GASP MODULES FOR GRASS GIS #
# ************************************************************************ #
from gasp.anls.ovlay import erase
from gasp.prop.grs import rst_to_region
from gasp.mng.genze import dissolve
from gasp.mng.grstbl import add_and_update, reset_table
from gasp.to.shp.grs import shp_to_grs, grs_to_shp
from gasp.to.rst import rst_to_grs
# ************************************************************************ #
# SET GRASS GIS LOCATION EXTENT #
# ************************************************************************ #
extRst = rst_to_grs(refRaster, 'extent_raster')
rst_to_region(extRst)
time_f = datetime.datetime.now().replace(microsecond=0)
# ************************************************************************ #
# MapResults #
# ************************************************************************ #
osmShps = []
# ************************************************************************ #
# 1 - Selection Rule #
# ************************************************************************ #
ruleOneShp, timeCheck1 = grs_vector(
osm_db if RoadsAPI != 'POSTGIS' else conPGSQL,
osmTableData['polygons'],
apidb=RoadsAPI)
osmShps.append(ruleOneShp)
time_g = datetime.datetime.now().replace(microsecond=0)
# ************************************************************************ #
# 2 - Get Information About Roads Location #
# ************************************************************************ #
ruleRowShp, timeCheck2 = roads_sqdb(
osm_db if RoadsAPI == 'SQLITE' else conPGSQL,
osmTableData['lines'],
osmTableData['polygons'],
apidb=RoadsAPI
) if RoadsAPI == 'SQLITE' or RoadsAPI == 'POSTGIS' else grs_vec_roads(
osm_db, osmTableData['lines'], osmTableData['polygons'])
osmShps.append(ruleRowShp)
time_h = datetime.datetime.now().replace(microsecond=0)
# ************************************************************************ #
# 3 - Area Upper than #
# ************************************************************************ #
ruleThreeShp, timeCheck3 = grs_vect_selbyarea(
osm_db if RoadsAPI != 'POSTGIS' else conPGSQL,
osmTableData['polygons'],
UPPER=True,
apidb=RoadsAPI)
osmShps.append(ruleThreeShp)
time_l = datetime.datetime.now().replace(microsecond=0)
# ************************************************************************ #
# 4 - Area Lower than #
# ************************************************************************ #
ruleFourShp, timeCheck4 = grs_vect_selbyarea(
osm_db if RoadsAPI != 'POSTGIS' else conPGSQL,
osmTableData['polygons'],
UPPER=False,
apidb=RoadsAPI)
osmShps.append(ruleFourShp)
time_j = datetime.datetime.now().replace(microsecond=0)
# ************************************************************************ #
# 5 - Get data from lines table (railway | waterway) #
# ************************************************************************ #
ruleFiveShp, timeCheck5 = grs_vect_bbuffer(
osm_db if RoadsAPI != 'POSTGIS' else conPGSQL,
osmTableData["lines"],
api_db=RoadsAPI)
osmShps.append(ruleFiveShp)
time_m = datetime.datetime.now().replace(microsecond=0)
# ************************************************************************ #
# 7 - Assign untagged Buildings to tags #
# ************************************************************************ #
if nomenclature != "GLOBE_LAND_30":
ruleSeven11, ruleSeven12, timeCheck7 = vector_assign_pntags_to_build(
osm_db if RoadsAPI != 'POSTGIS' else conPGSQL,
osmTableData['points'],
osmTableData['polygons'],
apidb=RoadsAPI)
if ruleSeven11:
osmShps.append(ruleSeven11)
if ruleSeven12:
osmShps.append(ruleSeven12)
time_n = datetime.datetime.now().replace(microsecond=0)
else:
timeCheck7 = None
time_n = datetime.datetime.now().replace(microsecond=0)
# ************************************************************************ #
# Produce LULC Map #
# ************************************************************************ #
"""
Get Shps with all geometries related with one class - One Shape for Classe
"""
from gasp.mng.gen import same_attr_to_shp
_osmShps = []
for i in range(len(osmShps)):
if not osmShps[i]: continue
_osmShps.append(
grs_to_shp(osmShps[i],
os.path.join(workspace, osmShps[i] + '.shp'),
'auto',
lyrN=1,
asCMD=True,
asMultiPart=None))
_osmShps = same_attr_to_shp(_osmShps,
"cat",
workspace,
"osm_",
resultDict=True)
del osmShps
time_o = datetime.datetime.now().replace(microsecond=0)
"""
Merge all Classes into one feature class using a priority rule
"""
osmShps = {}
for cls in _osmShps:
if cls == '1':
osmShps[1221] = shp_to_grs(_osmShps[cls], "osm_1221", asCMD=True)
else:
osmShps[int(cls)] = shp_to_grs(_osmShps[cls],
"osm_" + cls,
asCMD=True)
# Erase overlapping areas by priority
import copy
osmNameRef = copy.deepcopy(osmShps)
for e in range(len(__priorities)):
if e + 1 == len(__priorities): break
if __priorities[e] not in osmShps:
continue
else:
for i in range(e + 1, len(__priorities)):
if __priorities[i] not in osmShps:
continue
else:
osmShps[__priorities[i]] = erase(
osmShps[__priorities[i]],
osmShps[__priorities[e]],
"{}_{}".format(osmNameRef[__priorities[i]], e),
notTbl=True,
api='pygrass')
time_p = datetime.datetime.now().replace(microsecond=0)
# Export all classes
lst_merge = []
for cls in osmShps:
if cls == __priorities[0]:
reset_table(osmShps[cls], {'cls': 'varchar(5)'}, {'cls': str(cls)})
else:
add_and_update(osmShps[cls], {'cls': 'varchar(5)'},
{'cls': str(cls)})
ds = dissolve(osmShps[cls],
'dl_{}'.format(str(cls)),
'cls',
api="grass")
lst_merge.append(
grs_to_shp(ds,
os.path.join(workspace, "lulc_{}.shp".format(str(cls))),
'auto',
lyrN=1,
asCMD=True,
asMultiPart=None))
time_q = datetime.datetime.now().replace(microsecond=0)
merge_feat(lst_merge, lulcShp, api='pandas')
time_r = datetime.datetime.now().replace(microsecond=0)
return lulcShp, {
0: ('set_settings', time_b - time_a),
1: ('osm_to_sqdb', time_c - time_b),
2: ('cls_in_sqdb', time_d - time_c),
3: ('proj_data', time_e - time_d),
4: ('set_grass', time_f - time_e),
5: ('rule_1', time_g - time_f, timeCheck1),
6: ('rule_2', time_h - time_g, timeCheck2),
7: ('rule_3', time_l - time_h, timeCheck3),
8: ('rule_4', time_j - time_l, timeCheck4),
9: ('rule_5', time_m - time_j, timeCheck5),
10: None if not timeCheck7 else
('rule_7', time_n - time_m, timeCheck7),
11: ('disj_cls', time_o - time_n),
12: ('priority_rule', time_p - time_o),
13: ('export_cls', time_q - time_p),
14: ('merge_cls', time_r - time_q)
}
def check_shape_diff(SHAPES_TO_COMPARE,
OUT_FOLDER,
REPORT,
conPARAM,
DB,
SRS_CODE,
GIS_SOFTWARE="GRASS",
GRASS_REGION_TEMPLATE=None):
"""
Script to check differences between pairs of Feature Classes
Suponha que temos diversas Feature Classes (FC) e que cada uma delas
possui um determinado atributo; imagine também que,
considerando todos os pares possíveis entre estas FC,
se pretende comparar as diferenças na distribuição dos valores
desse atributo em cada par.
* Dependências:
- ArcGIS;
- GRASS;
- PostgreSQL;
- PostGIS.
* GIS_SOFTWARE Options:
- ARCGIS;
- GRASS.
"""
import datetime
import os
import pandas
from gasp.fm.sql import query_to_df
from gasp.sql.mng.tbl import tbls_to_tbl
from gasp.sql.mng.geom import fix_geom, check_geomtype_in_table
from gasp.sql.mng.geom import select_main_geom_type
from gasp.sql.mng.qw import ntbl_by_query
from gasp.prop.ff import check_isRaster
from gasp.oss import get_filename
from gasp.sql.mng.db import create_db
from gasp.to.sql import shp_to_psql, df_to_db
from gasp.to.shp import rst_to_polyg
from gasp.to.shp import shp_to_shp, psql_to_shp
from gasp.to import db_to_tbl
# Check if folder exists, if not create it
if not os.path.exists(OUT_FOLDER):
from gasp.oss.ops import create_folder
create_folder(OUT_FOLDER, overwrite=None)
else:
raise ValueError('{} already exists!'.format(OUT_FOLDER))
# Start GRASS GIS Session if GIS_SOFTWARE == GRASS
if GIS_SOFTWARE == "GRASS":
if not GRASS_REGION_TEMPLATE:
raise ValueError(
'To use GRASS GIS you need to specify GRASS_REGION_TEMPLATE')
from gasp.session import run_grass
gbase = run_grass(OUT_FOLDER,
grassBIN='grass76',
location='shpdif',
srs=GRASS_REGION_TEMPLATE)
import grass.script as grass
import grass.script.setup as gsetup
gsetup.init(gbase, OUT_FOLDER, 'shpdif', 'PERMANENT')
from gasp.mng.grstbl import rename_col
from gasp.to.shp.grs import shp_to_grs, grs_to_shp
from gasp.to.rst import rst_to_grs
from gasp.mng.fld import rename_column
# Convert to SHAPE if file is Raster
# Import to GRASS GIS if GIS SOFTWARE == GRASS
i = 0
_SHP_TO_COMPARE = {}
for s in SHAPES_TO_COMPARE:
isRaster = check_isRaster(s)
if isRaster:
if GIS_SOFTWARE == "ARCGIS":
d = rst_to_polyg(s,
os.path.join(os.path.dirname(s),
get_filename(s) + '.shp'),
gisApi='arcpy')
_SHP_TO_COMPARE[d] = "gridcode"
elif GIS_SOFTWARE == "GRASS":
# To GRASS
rstName = get_filename(s)
inRst = rst_to_grs(s, "rst_" + rstName, as_cmd=True)
# To Raster
d = rst_to_polyg(inRst,
rstName,
rstColumn="lulc_{}".format(i),
gisApi="grasscmd")
# Export Shapefile
shp = grs_to_shp(d, os.path.join(OUT_FOLDER, d + '.shp'),
"area")
_SHP_TO_COMPARE[shp] = "lulc_{}".format(i)
else:
if GIS_SOFTWARE == "ARCGIS":
_SHP_TO_COMPARE[s] = SHAPES_TO_COMPARE[s]
elif GIS_SOFTWARE == "GRASS":
# To GRASS
grsV = shp_to_grs(s, get_filename(s), asCMD=True)
# Change name of column with comparing value
rename_col(grsV,
SHAPES_TO_COMPARE[s],
"lulc_{}".format(i),
as_cmd=True)
# Export
shp = grs_to_shp(grsV,
os.path.join(OUT_FOLDER, grsV + '_rn.shp'),
"area")
_SHP_TO_COMPARE[shp] = "lulc_{}".format(i)
i += 1
SHAPES_TO_COMPARE = _SHP_TO_COMPARE
if GIS_SOFTWARE == "ARCGIS":
from gasp.cpu.arcg.mng.fld import calc_fld
from gasp.cpu.arcg.mng.wspace import create_geodb
from gasp.mng.gen import copy_feat
# Sanitize data and Add new field
__SHAPES_TO_COMPARE = {}
i = 0
# Create GeoDatabase
geodb = create_geodb(OUT_FOLDER, 'geo_sanitize')
""" Sanitize Data """
for k in SHAPES_TO_COMPARE:
# Send data to GeoDatabase only to sanitize
newFc = shp_to_shp(k,
os.path.join(geodb, get_filename(k)),
gisApi='arcpy')
# Create a copy to change
newShp = copy_feat(newFc,
os.path.join(OUT_FOLDER, os.path.basename(k)),
gisApi='arcpy')
# Sanitize field name with interest data
NEW_FLD = "lulc_{}".format(i)
calc_fld(newShp,
NEW_FLD,
"[{}]".format(SHAPES_TO_COMPARE[k]),
isNewField={
"TYPE": "INTEGER",
"LENGTH": 5,
"PRECISION": ""
})
__SHAPES_TO_COMPARE[newShp] = NEW_FLD
i += 1
else:
__SHAPES_TO_COMPARE = SHAPES_TO_COMPARE
# Create database
conPARAM["DATABASE"] = create_db(conPARAM, DB)
""" Union SHAPEs """
UNION_SHAPE = {}
FIX_GEOM = {}
def fix_geometry(shp):
# Send data to PostgreSQL
nt = shp_to_psql(conPARAM, shp, SRS_CODE, api='shp2pgsql')
# Fix data
corr_tbl = fix_geom(conPARAM,
nt,
"geom",
"corr_{}".format(nt),
colsSelect=['gid', __SHAPES_TO_COMPARE[shp]])
# Check if we have multiple geometries
geomN = check_geomtype_in_table(conPARAM, corr_tbl)
if geomN > 1:
corr_tbl = select_main_geom_type(conPARAM, corr_tbl,
"corr2_{}".format(nt))
# Export data again
newShp = psql_to_shp(conPARAM,
corr_tbl,
os.path.join(OUT_FOLDER, corr_tbl + '.shp'),
api='pgsql2shp',
geom_col='geom')
return newShp
SHPS = __SHAPES_TO_COMPARE.keys()
for i in range(len(SHPS)):
for e in range(i + 1, len(SHPS)):
if GIS_SOFTWARE == 'ARCGIS':
# Try the union thing
unShp = union(SHPS[i],
SHPS[e],
os.path.join(OUT_FOLDER,
"un_{}_{}.shp".format(i, e)),
api_gis="arcpy")
# See if the union went all right
if not os.path.exists(unShp):
# Union went not well
# See if geometry was already fixed
if SHPS[i] not in FIX_GEOM:
# Fix SHPS[i] geometry
FIX_GEOM[SHPS[i]] = fix_geometry(SHPS[i])
if SHPS[e] not in FIX_GEOM:
FIX_GEOM[SHPS[e]] = fix_geometry(SHPS[e])
# Run Union again
unShp = union(FIX_GEOM[SHPS[i]],
FIX_GEOM[SHPS[e]],
os.path.join(OUT_FOLDER,
"un_{}_{}_f.shp".format(i, e)),
api_gis="arcpy")
elif GIS_SOFTWARE == "GRASS":
# Optimized Union
print "Union between {} and {}".format(SHPS[i], SHPS[e])
time_a = datetime.datetime.now().replace(microsecond=0)
__unShp = optimized_union_anls(
SHPS[i],
SHPS[e],
os.path.join(OUT_FOLDER, "un_{}_{}.shp".format(i, e)),
GRASS_REGION_TEMPLATE,
SRS_CODE,
os.path.join(OUT_FOLDER, "work_{}_{}".format(i, e)),
multiProcess=True)
time_b = datetime.datetime.now().replace(microsecond=0)
print time_b - time_a
# Rename cols
unShp = rename_column(
__unShp, {
"a_" + __SHAPES_TO_COMPARE[SHPS[i]]:
__SHAPES_TO_COMPARE[SHPS[i]],
"b_" + __SHAPES_TO_COMPARE[SHPS[e]]:
__SHAPES_TO_COMPARE[SHPS[e]]
}, os.path.join(OUT_FOLDER, "un_{}_{}_rn.shp".format(i,
e)))
UNION_SHAPE[(SHPS[i], SHPS[e])] = unShp
# Send data one more time to postgresql
SYNTH_TBL = {}
for uShp in UNION_SHAPE:
# Send data to PostgreSQL
union_tbl = shp_to_psql(conPARAM,
UNION_SHAPE[uShp],
SRS_CODE,
api='shp2pgsql')
# Produce table with % of area equal in both maps
areaMapTbl = ntbl_by_query(
conPARAM,
"{}_syn".format(union_tbl),
("SELECT CAST('{lulc_1}' AS text) AS lulc_1, "
"CAST('{lulc_2}' AS text) AS lulc_2, "
"round("
"CAST(SUM(g_area) / 1000000 AS numeric), 4"
") AS agree_area, round("
"CAST((SUM(g_area) / MIN(total_area)) * 100 AS numeric), 4"
") AS agree_percentage, "
"round("
"CAST(MIN(total_area) / 1000000 AS numeric), 4"
") AS total_area FROM ("
"SELECT {map1_cls}, {map2_cls}, ST_Area(geom) AS g_area, "
"CASE "
"WHEN {map1_cls} = {map2_cls} "
"THEN 1 ELSE 0 "
"END AS isthesame, total_area FROM {tbl}, ("
"SELECT SUM(ST_Area(geom)) AS total_area FROM {tbl}"
") AS foo2"
") AS foo WHERE isthesame = 1 "
"GROUP BY isthesame").format(
lulc_1=get_filename(uShp[0]),
lulc_2=get_filename(uShp[1]),
map1_cls=__SHAPES_TO_COMPARE[uShp[0]],
map2_cls=__SHAPES_TO_COMPARE[uShp[1]],
tbl=union_tbl),
api='psql')
# Produce confusion matrix for the pair in comparison
lulcCls = query_to_df(
conPARAM,
("SELECT fcol FROM ("
"SELECT CAST({map1_cls} AS text) AS fcol FROM {tbl} "
"GROUP BY {map1_cls} "
"UNION ALL SELECT CAST({map2_cls} AS text) FROM {tbl} "
"GROUP BY {map2_cls}"
") AS foo GROUP BY fcol ORDER BY fcol").format(
tbl=union_tbl,
map1_cls=__SHAPES_TO_COMPARE[uShp[0]],
map2_cls=__SHAPES_TO_COMPARE[uShp[1]]),
db_api='psql').fcol.tolist()
matrixTbl = ntbl_by_query(
conPARAM,
"{}_matrix".format(union_tbl),
("SELECT * FROM crosstab('"
"SELECT CASE "
"WHEN foo.{map1_cls} IS NOT NULL "
"THEN foo.{map1_cls} ELSE jtbl.flyr "
"END AS lulc1_cls, CASE "
"WHEN foo.{map2_cls} IS NOT NULL "
"THEN foo.{map2_cls} ELSE jtbl.slyr "
"END AS lulc2_cls, CASE "
"WHEN foo.garea IS NOT NULL "
"THEN round(CAST(foo.garea / 1000000 AS numeric)"
", 3) ELSE 0 "
"END AS garea FROM ("
"SELECT CAST({map1_cls} AS text) AS {map1_cls}, "
"CAST({map2_cls} AS text) AS {map2_cls}, "
"SUM(ST_Area(geom)) AS garea "
"FROM {tbl} GROUP BY {map1_cls}, {map2_cls}"
") AS foo FULL JOIN ("
"SELECT f.flyr, s.slyr FROM ("
"SELECT CAST({map1_cls} AS text) AS flyr "
"FROM {tbl} GROUP BY {map1_cls}"
") AS f, ("
"SELECT CAST({map2_cls} AS text) AS slyr "
"FROM {tbl} GROUP BY {map2_cls}"
") AS s"
") AS jtbl "
"ON foo.{map1_cls} = jtbl.flyr AND "
"foo.{map2_cls} = jtbl.slyr "
"ORDER BY 1,2"
"') AS ct("
"lulc_cls text, {crossCols}"
")").format(crossCols=", ".join(
["cls_{} numeric".format(c) for c in lulcCls]),
tbl=union_tbl,
map1_cls=__SHAPES_TO_COMPARE[uShp[0]],
map2_cls=__SHAPES_TO_COMPARE[uShp[1]]),
api='psql')
SYNTH_TBL[uShp] = {"TOTAL": areaMapTbl, "MATRIX": matrixTbl}
# UNION ALL TOTAL TABLES
total_table = tbls_to_tbl(conPARAM,
[SYNTH_TBL[k]["TOTAL"] for k in SYNTH_TBL],
'total_table')
# Create table with % of agreement between each pair of maps
mapsNames = query_to_df(
conPARAM,
("SELECT lulc FROM ("
"SELECT lulc_1 AS lulc FROM {tbl} GROUP BY lulc_1 "
"UNION ALL "
"SELECT lulc_2 AS lulc FROM {tbl} GROUP BY lulc_2"
") AS lu GROUP BY lulc ORDER BY lulc").format(tbl=total_table),
db_api='psql').lulc.tolist()
FLDS_TO_PIVOT = ["agree_percentage", "total_area"]
Q = ("SELECT * FROM crosstab('"
"SELECT CASE "
"WHEN foo.lulc_1 IS NOT NULL THEN foo.lulc_1 ELSE jtbl.tmp1 "
"END AS lulc_1, CASE "
"WHEN foo.lulc_2 IS NOT NULL THEN foo.lulc_2 ELSE jtbl.tmp2 "
"END AS lulc_2, CASE "
"WHEN foo.{valCol} IS NOT NULL THEN foo.{valCol} ELSE 0 "
"END AS agree_percentage FROM ("
"SELECT lulc_1, lulc_2, {valCol} FROM {tbl} UNION ALL "
"SELECT lulc_1, lulc_2, {valCol} FROM ("
"SELECT lulc_1 AS lulc_2, lulc_2 AS lulc_1, {valCol} "
"FROM {tbl}"
") AS tst"
") AS foo FULL JOIN ("
"SELECT lulc_1 AS tmp1, lulc_2 AS tmp2 FROM ("
"SELECT lulc_1 AS lulc_1 FROM {tbl} GROUP BY lulc_1 "
"UNION ALL "
"SELECT lulc_2 AS lulc_1 FROM {tbl} GROUP BY lulc_2"
") AS tst_1, ("
"SELECT lulc_1 AS lulc_2 FROM {tbl} GROUP BY lulc_1 "
"UNION ALL "
"SELECT lulc_2 AS lulc_2 FROM {tbl} GROUP BY lulc_2"
") AS tst_2 WHERE lulc_1 = lulc_2 GROUP BY lulc_1, lulc_2"
") AS jtbl ON foo.lulc_1 = jtbl.tmp1 AND foo.lulc_2 = jtbl.tmp2 "
"ORDER BY lulc_1, lulc_2"
"') AS ct("
"lulc_map text, {crossCols}"
")")
TOTAL_AGREE_TABLE = None
TOTAL_AREA_TABLE = None
for f in FLDS_TO_PIVOT:
if not TOTAL_AGREE_TABLE:
TOTAL_AGREE_TABLE = ntbl_by_query(
conPARAM,
"agreement_table",
Q.format(tbl=total_table,
valCol=f,
crossCols=", ".join([
"{} numeric".format(map_) for map_ in mapsNames
])),
api='psql')
else:
TOTAL_AREA_TABLE = ntbl_by_query(
conPARAM,
"area_table",
Q.format(tbl=total_table,
valCol=f,
crossCols=", ".join([
"{} numeric".format(map_) for map_ in mapsNames
])),
api='psql')
# Union Mapping
UNION_MAPPING = pandas.DataFrame(
[[
get_filename(k[0]),
get_filename(k[1]),
get_filename(UNION_SHAPE[k])
] for k in UNION_SHAPE],
columns=['shp_a', 'shp_b', 'union_shp'
]) if GIS_SOFTWARE == "ARCGIS" else pandas.DataFrame(
[[k[0], k[1], get_filename(UNION_SHAPE[k])]
for k in UNION_SHAPE],
columns=['shp_a', 'shp_b', 'union_shp'])
UNION_MAPPING = df_to_db(conPARAM, UNION_MAPPING, 'union_map', api='psql')
# Export Results
TABLES = [UNION_MAPPING, TOTAL_AGREE_TABLE, TOTAL_AREA_TABLE
] + [SYNTH_TBL[x]["MATRIX"] for x in SYNTH_TBL]
SHEETS = ["union_map", "agreement_percentage", "area_with_data_km"] + [
"{}_{}".format(get_filename(x[0])[:15],
get_filename(x[1])[:15]) for x in SYNTH_TBL
]
db_to_xls(conPARAM, ["SELECT * FROM {}".format(x) for x in TABLES],
REPORT,
sheetsNames=SHEETS,
dbAPI='psql')
return REPORT
def optimized_union_anls(lyr_a,
lyr_b,
outShp,
ref_boundary,
epsg,
workspace=None,
multiProcess=None):
"""
Optimized Union Analysis
Goal: optimize v.overlay performance for Union operations
"""
import os
from gasp.oss import get_filename
from gasp.mng.sample import create_fishnet
from gasp.mng.feat import eachfeat_to_newshp
from gasp.mng.gen import merge_feat
from gasp.session import run_grass
from gasp.anls.exct import split_shp_by_attr
if workspace:
if not os.path.exists(workspace):
from gasp.oss.ops import create_folder
create_folder(workspace, overwrite=True)
else:
from gasp.oss.ops import create_folder
workspace = create_folder(
os.path.join(os.path.dirname(outShp), "union_work"))
# Create Fishnet
gridShp = create_fishnet(ref_boundary,
os.path.join(workspace, 'ref_grid.shp'),
rowN=4,
colN=4)
# Split Fishnet in several files
cellsShp = eachfeat_to_newshp(gridShp, workspace, epsg=epsg)
if not multiProcess:
# INIT GRASS GIS Session
grsbase = run_grass(workspace, location="grs_loc", srs=ref_boundary)
import grass.script.setup as gsetup
gsetup.init(grsbase, workspace, "grs_loc", 'PERMANENT')
# Add data to GRASS GIS
from gasp.to.shp.grs import shp_to_grs
cellsShp = [
shp_to_grs(shp, get_filename(shp), asCMD=True) for shp in cellsShp
]
LYR_A = shp_to_grs(lyr_a, get_filename(lyr_a), asCMD=True)
LYR_B = shp_to_grs(lyr_b, get_filename(lyr_b), asCMD=True)
# Clip Layers A and B for each CELL in fishnet
LYRS_A = [
clip(LYR_A, cellsShp[x], LYR_A + "_" + str(x), api_gis="grass_cmd")
for x in range(len(cellsShp))
]
LYRS_B = [
clip(LYR_B, cellsShp[x], LYR_B + "_" + str(x), api_gis="grass_cmd")
for x in range(len(cellsShp))
]
# Union SHPS
UNION_SHP = [
union(LYRS_A[i], LYRS_B[i], "un_{}".format(i), api_gis="grass_cmd")
for i in range(len(cellsShp))
]
# Export Data
from gasp.to.shp.grs import grs_to_shp
_UNION_SHP = [
grs_to_shp(shp, os.path.join(workspace, shp + ".shp"), "area")
for shp in UNION_SHP
]
else:
def clip_and_union(la, lb, cell, work, ref, proc, output):
# Start GRASS GIS Session
grsbase = run_grass(work, location="proc_" + str(proc), srs=ref)
import grass.script.setup as gsetup
gsetup.init(grsbase, work, "proc_" + str(proc), 'PERMANENT')
# Import GRASS GIS modules
from gasp.to.shp.grs import shp_to_grs
from gasp.to.shp.grs import grs_to_shp
# Add data to GRASS
a = shp_to_grs(la, get_filename(la), asCMD=True)
b = shp_to_grs(lb, get_filename(lb), asCMD=True)
c = shp_to_grs(cell, get_filename(cell), asCMD=True)
# Clip
a_clip = clip(a, c, "{}_clip".format(a), api_gis="grass_cmd")
b_clip = clip(b, c, "{}_clip".format(b), api_gis="grass_cmd")
# Union
u_shp = union(a_clip,
b_clip,
"un_{}".format(c),
api_gis="grass_cmd")
# Export
o = grs_to_shp(u_shp, output, "area")
import multiprocessing
thrds = [
multiprocessing.Process(
target=clip_and_union,
name="th-{}".format(i),
args=(lyr_a, lyr_b, cellsShp[i],
os.path.join(workspace,
"th_{}".format(i)), ref_boundary, i,
os.path.join(workspace, "uniao_{}.shp".format(i))))
for i in range(len(cellsShp))
]
for t in thrds:
t.start()
for t in thrds:
t.join()
_UNION_SHP = [
os.path.join(workspace, "uniao_{}.shp".format(i))
for i in range(len(cellsShp))
]
# Merge all union into the same layer
MERGED_SHP = merge_feat(_UNION_SHP, outShp, api="ogr2ogr")
return outShp
def distance_between_catpoints(srcShp, facilitiesShp, networkShp,
speedLimitCol, onewayCol, grsWorkspace,
grsLocation, outputShp):
"""
Path bet points
TODO: Work with files with cat
"""
import os
from gasp.oss import get_filename
from gasp.session import run_grass
from gasp.mng.gen import merge_feat
from gasp.prop.feat import feat_count
# Merge Source points and Facilities into the same Feature Class
SRC_NFEAT = feat_count(srcShp, gisApi='pandas')
FACILITY_NFEAT = feat_count(facilitiesShp, gisApi='pandas')
POINTS = merge_feat([srcShp, facilitiesShp],
os.path.join(os.path.dirname(outputShp),
"points_net.shp"),
api='pandas')
# Open an GRASS GIS Session
gbase = run_grass(grsWorkspace,
grassBIN="grass76",
location=grsLocation,
srs=networkShp)
import grass.script as grass
import grass.script.setup as gsetup
gsetup.init(gbase, grsWorkspace, grsLocation, 'PERMANENT')
# Import GRASS GIS Module
from gasp.to.shp.grs import shp_to_grs
from gasp.to.shp.grs import grs_to_shp
from gasp.cpu.grs.mng import category
from gasp.mng.grstbl import add_field, add_table, update_table
from gasp.mob.grstbx import network_from_arcs
from gasp.mob.grstbx import add_pnts_to_network
from gasp.mob.grstbx import netpath
from gasp.cpu.grs.mng.feat import geomattr_to_db
from gasp.cpu.grs.mng.feat import copy_insame_vector
# Add Data to GRASS GIS
rdvMain = shp_to_grs(networkShp, get_filename(networkShp, forceLower=True))
pntShp = shp_to_grs(POINTS, "points_net")
"""Get closest facility layer:"""
# Connect Points to Network
newNetwork = add_pnts_to_network(rdvMain, pntShp, "rdv_points")
# Sanitize Network Table and Cost Columns
newNetwork = category(newNetwork,
"rdv_points_time",
"add",
LyrN="3",
geomType="line")
add_table(newNetwork,
("cat integer,kph double precision,length double precision,"
"ft_minutes double precision,"
"tf_minutes double precision,oneway text"),
lyrN=3)
copy_insame_vector(newNetwork, "kph", speedLimitCol, 3, geomType="line")
copy_insame_vector(newNetwork, "oneway", onewayCol, 3, geomType="line")
geomattr_to_db(newNetwork,
"length",
"length",
"line",
createCol=False,
unit="meters",
lyrN=3)
update_table(newNetwork, "kph", "3.6", "kph IS NULL", lyrN=3)
update_table(newNetwork,
"ft_minutes",
"(length * 60) / (kph * 1000.0)",
"ft_minutes IS NULL",
lyrN=3)
update_table(newNetwork,
"tf_minutes",
"(length * 60) / (kph * 1000.0)",
"tf_minutes IS NULL",
lyrN=3)
# Exagerate Oneway's
update_table(newNetwork, "ft_minutes", "1000", "oneway = 'TF'", lyrN=3)
update_table(newNetwork, "tf_minutes", "1000", "oneway = 'FT'", lyrN=3)
# Produce result
result = netpath(newNetwork,
"ft_minutes",
"tf_minutes",
get_filename(outputShp),
arcLyr=3,
nodeLyr=2)
return grs_to_shp(result, outputShp, geomType="line", lyrN=3)
