def run_viewshed_by_cpu(tid,
obs,
dem,
output,
vis_basename='vis',
maxdst=None,
obselevation=None):
# Create GRASS GIS Session
loc_name = 'loc_' + str(tid)
gbase = run_grass(output, location=loc_name, srs=dem)
# Start GRASS GIS Session
import grass.script as grass
import grass.script.setup as gsetup
gsetup.init(gbase, output, loc_name, 'PERMANENT')
from gasp.gt.torst import rst_to_grs, grs_to_rst
from gasp.gt.nop.surf import grs_viewshed
# Send DEM to GRASS GIS
grs_dem = rst_to_grs(dem, 'grs_dem', as_cmd=True)
# Produce Viewshed for each point in obs
for idx, row in obs.iterrows():
vrst = grs_viewshed(grs_dem, (row.geometry.x, row.geometry.y),
'{}_{}'.format(vis_basename, str(row[obs_id])),
max_dist=maxdst,
obs_elv=obselevation)
frst = grs_to_rst(vrst, os.path.join(output, vrst + '.tif'))
def clip_and_union(la, lb, cell, work, proc, output):
ref_rst = shpext_to_rst(cell,
os.path.join(os.path.dirname(cell),
fprop(cell, 'fn') + '.tif'),
cellsize=10)
# Start GRASS GIS Session
loc = "proc_" + str(proc)
grsbase = run_grass(work, location=loc, srs=ref_rst)
import grass.script.setup as gsetup
gsetup.init(grsbase, work, loc, 'PERMANENT')
# Import GRASS GIS modules
from gasp.gt.toshp.cff import shp_to_grs, grs_to_shp
from gasp.gt.prop.feat import feat_count
# Add data to GRASS
a = shp_to_grs(la, fprop(la, 'fn'), filterByReg=True, asCMD=True)
b = shp_to_grs(lb, fprop(lb, 'fn'), filterByReg=True, asCMD=True)
if not feat_count(a, gisApi="grass", work=work, loc=loc):
return
if not feat_count(b, gisApi="grass", work=work, loc=loc):
return
# Clip
a_clip = clip(a,
None,
"{}_clip".format(a),
api_gis="grass",
clip_by_region=True)
b_clip = clip(b,
None,
"{}_clip".format(b),
api_gis="grass",
clip_by_region=True)
# Union
u_shp = union(a_clip,
b_clip,
"un_{}".format(fprop(cell, 'fn')),
api_gis="grass")
# Export
o = grs_to_shp(u_shp, output, "area")
def multi_run(ti, df, ofolder):
loc_name = 'loc_{}'.format(str(ti))
grsbase = run_grass(ofolder, location=loc_name, srs=srs_epsg)
import grass.script.setup as gsetup
gsetup.init(grsbase, ofolder, loc_name, 'PERMANENT')
from gasp.gt.toshp.cff import shp_to_grs, grs_to_shp
from gasp.gt.gop.ovlay import union
for idx, row in df.iterrows():
# Import data into GRASS GIS
lyr_a = shp_to_grs(df.shp_a, fprop(df.shp_a, 'fn'), asCMD=True)
lyr_b = shp_to_grs(df.shp_b, fprop(df.shp_b, 'fn'), asCMD=True)
# Run Union
shpUnion = union(lyr_a, lyr_b, lyr_a[:10] + '_' + lyr_b[:10], api_gis="grass")
# Export data
result = grs_to_shp(shpUnion, os.path.join(ofolder, shpUnion + '.shp'), "area")
def join_attr_by_distance(mainTable, joinTable, workGrass, epsg_code, output):
"""
Find nearest feature and join attributes of the nearest feature
to the mainTable
Uses GRASS GIS to find near lines.
"""
import os
from gasp.gt.wenv.grs import run_grass
from gasp.gt.fmshp import shp_to_obj
from gasp.g.to import df_to_geodf
from gasp.gt.toshp import df_to_shp
from gasp.pyt.oss import fprop
# Create GRASS GIS Location
grassBase = run_grass(workGrass, location='join_loc', srs=epsg_code)
import grass.script as grass
import grass.script.setup as gsetup
gsetup.init(grassBase, workGrass, 'join_loc', 'PERMANENT')
# Import some GRASS GIS tools
from gasp.gt.prox import grs_near as near
from gasp.gt.tbl.attr import geomattr_to_db
from gasp.gt.toshp.cff import shp_to_grs, grs_to_shp
# Import data into GRASS GIS
grsMain = shp_to_grs(mainTable, fprop(mainTable, 'fn', forceLower=True))
grsJoin = shp_to_grs(joinTable, fprop(joinTable, 'fn', forceLower=True))
# Get distance from each feature of mainTable to the nearest feature
# of the join table
near(grsMain, grsJoin, nearCatCol="tocat", nearDistCol="todistance")
# Export data from GRASS GIS
ogrMain = grs_to_shp(grsMain,
os.path.join(workGrass, 'join_loc',
grsMain + '_grs.shp'),
None,
asMultiPart=True)
ogrJoin = grs_to_shp(grsJoin,
os.path.join(workGrass, 'join_loc',
grsJoin + '_grs.shp'),
None,
asMultiPart=True)
dfMain = shp_to_obj(ogrMain)
dfJoin = shp_to_obj(ogrJoin)
dfResult = dfMain.merge(dfJoin,
how='inner',
left_on='tocat',
right_on='cat')
dfResult.drop(["geometry_y", "cat_y"], axis=1, inplace=True)
dfResult.rename(columns={"cat_x": "cat_grass"}, inplace=True)
dfResult["tocat"] = dfResult["tocat"] - 1
dfResult["cat_grass"] = dfResult["cat_grass"] - 1
dfResult = df_to_geodf(dfResult, "geometry_x", epsg_code)
df_to_shp(dfResult, output)
return output
def bnds_to_mosaic(bands, outdata, ref_raster, loc=None):
"""
Satellite image To mosaic
bands = {
'bnd_2' : [path_to_file, path_to_file],
'bnd_3' : [path_to_file, path_to_file],
'bnd_4' : [path_to_file, path_to_file],
}
"""
"""
Start GRASS GIS Session
"""
import os
from gasp.pyt.oss import fprop
from gasp.gt.prop.prj import get_rst_epsg
from gasp.gt.wenv.grs import run_grass
# Get EPSG from refRaster
epsg = get_rst_epsg(ref_raster, returnIsProj=None)
LOC = loc if loc else 'gr_loc'
grass_base = run_grass(
outdata, grassBIN='grass78',
location=LOC, srs=epsg
)
import grass.script as grass
import grass.script.setup as gsetup
gsetup.init(grass_base, outdata, LOC, 'PERMANENT')
# ************************************************************************ #
# GRASS MODULES #
# ************************************************************************ #
from gasp.gt.torst import rst_to_grs, grs_to_rst
from gasp.gt.wenv.grs import rst_to_region
# ************************************************************************ #
# SET GRASS GIS LOCATION EXTENT #
# ************************************************************************ #
extRst = rst_to_grs(ref_raster, 'extent_raster')
rst_to_region(extRst)
# ************************************************************************ #
# SEND DATA TO GRASS GIS #
# ************************************************************************ #
grs_bnds = {}
for bnd in bands:
l= []
for b in bands[bnd]:
bb = rst_to_grs(b, fprop(b, 'fn'))
l.append(bb)
grs_bnds[bnd] = l
# ************************************************************************ #
# PATCH bands and export #
# ************************************************************************ #
for bnd in grs_bnds:
mosaic_band = rseries(grs_bnds[bnd], bnd, 'maximum')
grs_bnds[bnd] = grs_to_rst(mosaic_band, os.path.join(
outdata, mosaic_band + '.tif'
), as_cmd=True)
return grs_bnds
def raster_based(osmdata,
nomenclature,
refRaster,
lulcRst,
overwrite=None,
dataStore=None,
roadsAPI='POSTGIS'):
"""
Convert OSM Data into Land Use/Land Cover Information
An raster based approach.
TODO: Add detailed description
"""
# ************************************************************************ #
# Python Modules from Reference Packages #
# ************************************************************************ #
import datetime
import os
import pandas
import copy
# ************************************************************************ #
# Gasp dependencies #
# ************************************************************************ #
from gasp.pyt.oss import mkdir, fprop
from gasp.gt.prop.ff import check_isRaster
from gasp.gt.prop.prj import get_rst_epsg
from gasp.gt.wenv.grs import run_grass
if roadsAPI == 'POSTGIS':
from gasp.sql.db import create_db
from gasp.gql.to.osm import osm_to_psql
from gasp.sds.osm2lulc.mod2 import roads_sqdb
from gasp.sql.fm import dump_db
from gasp.sql.db import drop_db
else:
from gasp.gt.toshp.osm import osm_to_sqdb
from gasp.sds.osm2lulc.mod2 import grs_rst_roads
from gasp.sds.osm2lulc.utils import osm_project, add_lulc_to_osmfeat, osmlulc_rsttbl
from gasp.sds.osm2lulc.utils import get_ref_raster
from gasp.sds.osm2lulc.mod1 import grs_rst
from gasp.sds.osm2lulc.m3_4 import rst_area
from gasp.sds.osm2lulc.mod5 import basic_buffer
from gasp.sds.osm2lulc.mod6 import rst_pnt_to_build
# ************************************************************************ #
# Global Settings #
# ************************************************************************ #
# Check if input parameters exists!
if not os.path.exists(os.path.dirname(lulcRst)):
raise ValueError('{} does not exist!'.format(os.path.dirname(lulcRst)))
if not os.path.exists(osmdata):
raise ValueError(
'File with OSM DATA ({}) does not exist!'.format(osmdata))
if not os.path.exists(refRaster):
raise ValueError(
'File with reference area ({}) does not exist!'.format(refRaster))
# Check if Nomenclature is valid
nomenclature = "URBAN_ATLAS" if nomenclature != "URBAN_ATLAS" and \
nomenclature != "CORINE_LAND_COVER" and \
nomenclature == "GLOBE_LAND_30" else nomenclature
time_a = datetime.datetime.now().replace(microsecond=0)
workspace = os.path.join(os.path.dirname(lulcRst),
'osmtolulc') if not dataStore else dataStore
# Check if workspace exists
if os.path.exists(workspace):
if overwrite:
mkdir(workspace)
else:
raise ValueError('Path {} already exists'.format(workspace))
else:
mkdir(workspace)
# Get Ref Raster
refRaster, epsg = get_ref_raster(refRaster, workspace, cellsize=2)
from gasp.sds.osm2lulc import PRIORITIES, osmTableData, LEGEND
__priorites = PRIORITIES[nomenclature]
__legend = LEGEND[nomenclature]
time_b = datetime.datetime.now().replace(microsecond=0)
# ************************************************************************ #
# Convert OSM file to SQLITE DB or to POSTGIS DB #
# ************************************************************************ #
if roadsAPI == 'POSTGIS':
osm_db = create_db(fprop(osmdata, 'fn', forceLower=True),
overwrite=True)
osm_db = osm_to_psql(osmdata, osm_db)
else:
osm_db = osm_to_sqdb(osmdata, os.path.join(workspace, 'osm.sqlite'))
time_c = datetime.datetime.now().replace(microsecond=0)
# ************************************************************************ #
# Add Lulc Classes to OSM_FEATURES by rule #
# ************************************************************************ #
add_lulc_to_osmfeat(osm_db, osmTableData, nomenclature, api=roadsAPI)
time_d = datetime.datetime.now().replace(microsecond=0)
# ************************************************************************ #
# Transform SRS of OSM Data #
# ************************************************************************ #
osmTableData = osm_project(
osm_db,
epsg,
api=roadsAPI,
isGlobeLand=None if nomenclature != 'GLOBE_LAND_30' else True)
time_e = datetime.datetime.now().replace(microsecond=0)
# ************************************************************************ #
# Start a GRASS GIS Session #
# ************************************************************************ #
grass_base = run_grass(workspace,
grassBIN='grass78',
location='grloc',
srs=epsg)
import grass.script as grass
import grass.script.setup as gsetup
gsetup.init(grass_base, workspace, 'grloc', 'PERMANENT')
# ************************************************************************ #
# IMPORT SOME GASP MODULES FOR GRASS GIS #
# ************************************************************************ #
from gasp.gt.torst import rst_to_grs, grs_to_rst
from gasp.gt.nop.mos import rsts_to_mosaic
from gasp.gt.wenv.grs import rst_to_region
# ************************************************************************ #
# SET GRASS GIS LOCATION EXTENT #
# ************************************************************************ #
extRst = rst_to_grs(refRaster, 'extent_raster')
rst_to_region(extRst)
time_f = datetime.datetime.now().replace(microsecond=0)
# ************************************************************************ #
# MapResults #
mergeOut = {}
# ************************************************************************ #
# ************************************************************************ #
# 1 - Selection Rule #
# ************************************************************************ #
"""
selOut = {
cls_code : rst_name, ...
}
"""
selOut, timeCheck1 = grs_rst(osm_db,
osmTableData['polygons'],
api=roadsAPI)
for cls in selOut:
mergeOut[cls] = [selOut[cls]]
time_g = datetime.datetime.now().replace(microsecond=0)
# ************************************************************************ #
# 2 - Get Information About Roads Location #
# ************************************************************************ #
"""
roads = {
cls_code : rst_name, ...
}
"""
if roadsAPI != 'POSTGIS':
roads, timeCheck2 = grs_rst_roads(
osm_db, osmTableData['lines'], osmTableData['polygons'], workspace,
1221 if nomenclature != "GLOBE_LAND_30" else 801)
else:
roadCls = 1221 if nomenclature != "GLOBE_LAND_30" else 801
roads, timeCheck2 = roads_sqdb(osm_db,
osmTableData['lines'],
osmTableData['polygons'],
apidb='POSTGIS',
asRst=roadCls)
roads = {roadCls: roads}
for cls in roads:
if cls not in mergeOut:
mergeOut[cls] = [roads[cls]]
else:
mergeOut[cls].append(roads[cls])
time_h = datetime.datetime.now().replace(microsecond=0)
# ************************************************************************ #
# 3 - Area Upper than #
# ************************************************************************ #
"""
auOut = {
cls_code : rst_name, ...
}
"""
if nomenclature != 'GLOBE_LAND_30':
auOut, timeCheck3 = rst_area(osm_db,
osmTableData['polygons'],
UPPER=True,
api=roadsAPI)
for cls in auOut:
if cls not in mergeOut:
mergeOut[cls] = [auOut[cls]]
else:
mergeOut[cls].append(auOut[cls])
time_l = datetime.datetime.now().replace(microsecond=0)
else:
timeCheck3 = None
time_l = None
# ************************************************************************ #
# 4 - Area Lower than #
# ************************************************************************ #
"""
alOut = {
cls_code : rst_name, ...
}
"""
if nomenclature != 'GLOBE_LAND_30':
alOut, timeCheck4 = rst_area(osm_db,
osmTableData['polygons'],
UPPER=None,
api=roadsAPI)
for cls in alOut:
if cls not in mergeOut:
mergeOut[cls] = [alOut[cls]]
else:
mergeOut[cls].append(alOut[cls])
time_j = datetime.datetime.now().replace(microsecond=0)
else:
timeCheck4 = None
time_j = None
# ************************************************************************ #
# 5 - Get data from lines table (railway | waterway) #
# ************************************************************************ #
"""
bfOut = {
cls_code : rst_name, ...
}
"""
bfOut, timeCheck5 = basic_buffer(osm_db,
osmTableData['lines'],
workspace,
apidb=roadsAPI)
for cls in bfOut:
if cls not in mergeOut:
mergeOut[cls] = [bfOut[cls]]
else:
mergeOut[cls].append(bfOut[cls])
time_m = datetime.datetime.now().replace(microsecond=0)
# ************************************************************************ #
# 7 - Assign untagged Buildings to tags #
# ************************************************************************ #
if nomenclature != "GLOBE_LAND_30":
buildsOut, timeCheck7 = rst_pnt_to_build(osm_db,
osmTableData['points'],
osmTableData['polygons'],
api_db=roadsAPI)
for cls in buildsOut:
if cls not in mergeOut:
mergeOut[cls] = buildsOut[cls]
else:
mergeOut[cls] += buildsOut[cls]
time_n = datetime.datetime.now().replace(microsecond=0)
else:
timeCheck7 = None
time_n = datetime.datetime.now().replace(microsecond=0)
# ************************************************************************ #
# Produce LULC Map #
# ************************************************************************ #
"""
Merge all results for one cls into one raster
mergeOut = {
cls_code : [rst_name, rst_name, ...], ...
}
into
mergeOut = {
cls_code : patched_raster, ...
}
"""
for cls in mergeOut:
if len(mergeOut[cls]) == 1:
mergeOut[cls] = mergeOut[cls][0]
else:
mergeOut[cls] = rsts_to_mosaic(mergeOut[cls],
'mosaic_{}'.format(str(cls)),
api="grass")
time_o = datetime.datetime.now().replace(microsecond=0)
"""
Merge all Class Raster using a priority rule
"""
__priorities = PRIORITIES[nomenclature]
lst_rst = []
for cls in __priorities:
if cls not in mergeOut:
continue
else:
lst_rst.append(mergeOut[cls])
outGrs = rsts_to_mosaic(lst_rst,
os.path.splitext(os.path.basename(lulcRst))[0],
api="grass")
time_p = datetime.datetime.now().replace(microsecond=0)
# Ceck if lulc Rst has an valid format
outIsRst = check_isRaster(lulcRst)
if not outIsRst:
from gasp.pyt.oss import fprop
lulcRst = os.path.join(os.path.dirname(lulcRst),
fprop(lulcRst, 'fn') + '.tif')
grs_to_rst(outGrs, lulcRst, as_cmd=True)
osmlulc_rsttbl(
nomenclature,
os.path.join(os.path.dirname(lulcRst),
os.path.basename(lulcRst) + '.vat.dbf'))
time_q = datetime.datetime.now().replace(microsecond=0)
# Dump Database if PostGIS was used
# Drop Database if PostGIS was used
if roadsAPI == 'POSTGIS':
dump_db(osm_db, os.path.join(workspace, osm_db + '.sql'), api='psql')
drop_db(osm_db)
return lulcRst, {
0: ('set_settings', time_b - time_a),
1: ('osm_to_sqdb', time_c - time_b),
2: ('cls_in_sqdb', time_d - time_c),
3: ('proj_data', time_e - time_d),
4: ('set_grass', time_f - time_e),
5: ('rule_1', time_g - time_f, timeCheck1),
6: ('rule_2', time_h - time_g, timeCheck2),
7:
None if not timeCheck3 else ('rule_3', time_l - time_h, timeCheck3),
8:
None if not timeCheck4 else ('rule_4', time_j - time_l, timeCheck4),
9: ('rule_5', time_m - time_j if timeCheck4 else time_m - time_h,
timeCheck5),
10:
None if not timeCheck7 else ('rule_7', time_n - time_m, timeCheck7),
11: ('merge_rst', time_o - time_n),
12: ('priority_rule', time_p - time_o),
13: ('export_rst', time_q - time_p)
}
def vector_based(osmdata,
nomenclature,
refRaster,
lulcShp,
overwrite=None,
dataStore=None,
RoadsAPI='POSTGIS'):
"""
Convert OSM Data into Land Use/Land Cover Information
An vector based approach.
TODO: Add a detailed description.
RoadsAPI Options:
* GRASS
* SQLITE
* POSTGIS
"""
# ************************************************************************ #
# Python Modules from Reference Packages #
# ************************************************************************ #
import datetime
import os
import copy
# ************************************************************************ #
# GASP dependencies #
# ************************************************************************ #
from gasp.pyt.oss import fprop, mkdir
from gasp.gt.wenv.grs import run_grass
if RoadsAPI == 'POSTGIS':
from gasp.sql.db import create_db
from gasp.gql.to.osm import osm_to_psql
from gasp.sql.db import drop_db
from gasp.sql.fm import dump_db
else:
from gasp.gt.toshp.osm import osm_to_sqdb
from gasp.sds.osm2lulc.utils import osm_project, add_lulc_to_osmfeat, get_ref_raster
from gasp.gt.toshp.mtos import shps_to_shp
from gasp.sds.osm2lulc.mod1 import grs_vector
if RoadsAPI == 'SQLITE' or RoadsAPI == 'POSTGIS':
from gasp.sds.osm2lulc.mod2 import roads_sqdb
else:
from gasp.sds.osm2lulc.mod2 import grs_vec_roads
from gasp.sds.osm2lulc.m3_4 import grs_vect_selbyarea
from gasp.sds.osm2lulc.mod5 import grs_vect_bbuffer
from gasp.sds.osm2lulc.mod6 import vector_assign_pntags_to_build
from gasp.gt.toshp.mtos import same_attr_to_shp
from gasp.gt.prj import def_prj
# ************************************************************************ #
# Global Settings #
# ************************************************************************ #
# Check if input parameters exists!
if not os.path.exists(os.path.dirname(lulcShp)):
raise ValueError('{} does not exist!'.format(os.path.dirname(lulcShp)))
if not os.path.exists(osmdata):
raise ValueError(
'File with OSM DATA ({}) does not exist!'.format(osmdata))
if not os.path.exists(refRaster):
raise ValueError(
'File with reference area ({}) does not exist!'.format(refRaster))
# Check if Nomenclature is valid
nomenclature = "URBAN_ATLAS" if nomenclature != "URBAN_ATLAS" and \
nomenclature != "CORINE_LAND_COVER" and \
nomenclature == "GLOBE_LAND_30" else nomenclature
time_a = datetime.datetime.now().replace(microsecond=0)
# Create workspace for temporary files
workspace = os.path.join(os.path.dirname(lulcShp),
'osmtolulc') if not dataStore else dataStore
# Check if workspace exists
if os.path.exists(workspace):
if overwrite:
mkdir(workspace)
else:
raise ValueError('Path {} already exists'.format(workspace))
else:
mkdir(workspace)
# Get Reference Raster
refRaster, epsg = get_ref_raster(refRaster, workspace, cellsize=10)
from gasp.sds.osm2lulc import osmTableData, PRIORITIES, LEGEND
__priorities = PRIORITIES[nomenclature]
__legend = LEGEND[nomenclature]
time_b = datetime.datetime.now().replace(microsecond=0)
if RoadsAPI != 'POSTGIS':
# ******************************************************************** #
# Convert OSM file to SQLITE DB #
# ******************************************************************** #
osm_db = osm_to_sqdb(osmdata, os.path.join(workspace, 'osm.sqlite'))
else:
# Convert OSM file to POSTGRESQL DB #
osm_db = create_db(fprop(osmdata, 'fn', forceLower=True),
overwrite=True)
osm_db = osm_to_psql(osmdata, osm_db)
time_c = datetime.datetime.now().replace(microsecond=0)
# ************************************************************************ #
# Add Lulc Classes to OSM_FEATURES by rule #
# ************************************************************************ #
add_lulc_to_osmfeat(osm_db,
osmTableData,
nomenclature,
api='SQLITE' if RoadsAPI != 'POSTGIS' else RoadsAPI)
time_d = datetime.datetime.now().replace(microsecond=0)
# ************************************************************************ #
# Transform SRS of OSM Data #
# ************************************************************************ #
osmTableData = osm_project(
osm_db,
epsg,
api='SQLITE' if RoadsAPI != 'POSTGIS' else RoadsAPI,
isGlobeLand=None if nomenclature != 'GLOBE_LAND_30' else True)
time_e = datetime.datetime.now().replace(microsecond=0)
# ************************************************************************ #
# Start a GRASS GIS Session #
# ************************************************************************ #
grass_base = run_grass(workspace,
grassBIN='grass78',
location='grloc',
srs=epsg)
#import grass.script as grass
import grass.script.setup as gsetup
gsetup.init(grass_base, workspace, 'grloc', 'PERMANENT')
# ************************************************************************ #
# IMPORT SOME GASP MODULES FOR GRASS GIS #
# ************************************************************************ #
from gasp.gt.gop.ovlay import erase
from gasp.gt.wenv.grs import rst_to_region
from gasp.gt.gop.genze import dissolve
from gasp.gt.tbl.grs import add_and_update, reset_table, update_table
from gasp.gt.tbl.fld import add_fields
from gasp.gt.toshp.cff import shp_to_grs, grs_to_shp
from gasp.gt.torst import rst_to_grs
# ************************************************************************ #
# SET GRASS GIS LOCATION EXTENT #
# ************************************************************************ #
extRst = rst_to_grs(refRaster, 'extent_raster')
rst_to_region(extRst)
time_f = datetime.datetime.now().replace(microsecond=0)
# ************************************************************************ #
# MapResults #
# ************************************************************************ #
osmShps = []
# ************************************************************************ #
# 1 - Selection Rule #
# ************************************************************************ #
ruleOneShp, timeCheck1 = grs_vector(osm_db,
osmTableData['polygons'],
apidb=RoadsAPI)
osmShps.append(ruleOneShp)
time_g = datetime.datetime.now().replace(microsecond=0)
# ************************************************************************ #
# 2 - Get Information About Roads Location #
# ************************************************************************ #
ruleRowShp, timeCheck2 = roads_sqdb(
osm_db,
osmTableData['lines'],
osmTableData['polygons'],
apidb=RoadsAPI
) if RoadsAPI == 'SQLITE' or RoadsAPI == 'POSTGIS' else grs_vec_roads(
osm_db, osmTableData['lines'], osmTableData['polygons'])
osmShps.append(ruleRowShp)
time_h = datetime.datetime.now().replace(microsecond=0)
# ************************************************************************ #
# 3 - Area Upper than #
# ************************************************************************ #
if nomenclature != "GLOBE_LAND_30":
ruleThreeShp, timeCheck3 = grs_vect_selbyarea(osm_db,
osmTableData['polygons'],
UPPER=True,
apidb=RoadsAPI)
osmShps.append(ruleThreeShp)
time_l = datetime.datetime.now().replace(microsecond=0)
else:
timeCheck3 = None
time_l = None
# ************************************************************************ #
# 4 - Area Lower than #
# ************************************************************************ #
if nomenclature != "GLOBE_LAND_30":
ruleFourShp, timeCheck4 = grs_vect_selbyarea(osm_db,
osmTableData['polygons'],
UPPER=False,
apidb=RoadsAPI)
osmShps.append(ruleFourShp)
time_j = datetime.datetime.now().replace(microsecond=0)
else:
timeCheck4 = None
time_j = None
# ************************************************************************ #
# 5 - Get data from lines table (railway | waterway) #
# ************************************************************************ #
ruleFiveShp, timeCheck5 = grs_vect_bbuffer(osm_db,
osmTableData["lines"],
api_db=RoadsAPI)
osmShps.append(ruleFiveShp)
time_m = datetime.datetime.now().replace(microsecond=0)
# ************************************************************************ #
# 7 - Assign untagged Buildings to tags #
# ************************************************************************ #
if nomenclature != "GLOBE_LAND_30":
ruleSeven11, ruleSeven12, timeCheck7 = vector_assign_pntags_to_build(
osm_db,
osmTableData['points'],
osmTableData['polygons'],
apidb=RoadsAPI)
if ruleSeven11:
osmShps.append(ruleSeven11)
if ruleSeven12:
osmShps.append(ruleSeven12)
time_n = datetime.datetime.now().replace(microsecond=0)
else:
timeCheck7 = None
time_n = datetime.datetime.now().replace(microsecond=0)
# ************************************************************************ #
# Produce LULC Map #
# ************************************************************************ #
"""
Get Shps with all geometries related with one class - One Shape for Classe
"""
_osmShps = []
for i in range(len(osmShps)):
if not osmShps[i]: continue
_osmShps.append(
grs_to_shp(osmShps[i],
os.path.join(workspace, osmShps[i] + '.shp'),
'auto',
lyrN=1,
asCMD=True,
asMultiPart=None))
for shp in _osmShps:
def_prj(os.path.splitext(shp)[0] + '.prj', epsg=epsg, api='epsgio')
_osmShps = same_attr_to_shp(_osmShps,
"cat",
workspace,
"osm_",
resultDict=True)
del osmShps
time_o = datetime.datetime.now().replace(microsecond=0)
"""
Merge all Classes into one feature class using a priority rule
"""
osmShps = {}
for cls in _osmShps:
if cls == '1':
osmShps[1221] = shp_to_grs(_osmShps[cls], "osm_1221", asCMD=True)
else:
osmShps[int(cls)] = shp_to_grs(_osmShps[cls],
"osm_" + cls,
asCMD=True)
# Erase overlapping areas by priority
osmNameRef = copy.deepcopy(osmShps)
for e in range(len(__priorities)):
if e + 1 == len(__priorities): break
if __priorities[e] not in osmShps:
continue
else:
for i in range(e + 1, len(__priorities)):
if __priorities[i] not in osmShps:
continue
else:
osmShps[__priorities[i]] = erase(
osmShps[__priorities[i]],
osmShps[__priorities[e]],
"{}_{}".format(osmNameRef[__priorities[i]], e),
notTbl=True,
api='pygrass')
time_p = datetime.datetime.now().replace(microsecond=0)
# Export all classes
lst_merge = []
a = None
for i in range(len(__priorities)):
if __priorities[i] not in osmShps:
continue
if not a:
reset_table(osmShps[__priorities[i]], {
'cls': 'varchar(5)',
'leg': 'varchar(75)'
}, {
'cls': str(__priorities[i]),
'leg': str(__legend[__priorities[i]])
})
a = 1
else:
add_and_update(osmShps[__priorities[i]], {'cls': 'varchar(5)'},
{'cls': str(__priorities[i])})
ds = dissolve(osmShps[__priorities[i]],
'dl_{}'.format(str(__priorities[i])),
'cls',
api="grass")
add_fields(ds, {'leg': 'varchar(75)'}, api="grass")
update_table(ds, 'leg', str(__legend[__priorities[i]]), 'leg is null')
lst_merge.append(
grs_to_shp(ds,
os.path.join(workspace, "lulc_{}.shp".format(
str(__priorities[i]))),
'auto',
lyrN=1,
asCMD=True,
asMultiPart=None))
time_q = datetime.datetime.now().replace(microsecond=0)
if fprop(lulcShp, 'ff') != '.shp':
lulcShp = os.path.join(os.path.dirname(lulcShp),
fprop(lulcShp, 'fn') + '.shp')
shps_to_shp(lst_merge, lulcShp, api='pandas')
# Check if prj of lulcShp exists and create it if necessary
prj_ff = os.path.splitext(lulcShp)[0] + '.prj'
if not os.path.exists(prj_ff):
def_prj(prj_ff, epsg=epsg, api='epsgio')
time_r = datetime.datetime.now().replace(microsecond=0)
# Dump Database if PostGIS was used
# Drop Database if PostGIS was used
if RoadsAPI == 'POSTGIS':
dump_db(osm_db, os.path.join(workspace, osm_db + '.sql'), api='psql')
drop_db(osm_db)
return lulcShp, {
0: ('set_settings', time_b - time_a),
1: ('osm_to_sqdb', time_c - time_b),
2: ('cls_in_sqdb', time_d - time_c),
3: ('proj_data', time_e - time_d),
4: ('set_grass', time_f - time_e),
5: ('rule_1', time_g - time_f, timeCheck1),
6: ('rule_2', time_h - time_g, timeCheck2),
7:
None if not timeCheck3 else ('rule_3', time_l - time_h, timeCheck3),
8:
None if not timeCheck4 else ('rule_4', time_j - time_l, timeCheck4),
9: ('rule_5', time_m - time_j if timeCheck4 else time_m - time_h,
timeCheck5),
10:
None if not timeCheck7 else ('rule_7', time_n - time_m, timeCheck7),
11: ('disj_cls', time_o - time_n),
12: ('priority_rule', time_p - time_o),
13: ('export_cls', time_q - time_p),
14: ('merge_cls', time_r - time_q)
}
def snap_points_to_near_line(lineShp,
pointShp,
epsg,
workGrass,
outPoints,
location='overlap_pnts',
api='grass',
movesShp=None):
"""
Move points to overlap near line
API's Available:
* grass;
* saga.
"""
if api == 'grass':
"""
Uses GRASS GIS to find near lines.
"""
import os
import numpy
from geopandas import GeoDataFrame
from gasp.pyt.oss import fprop
from gasp.gt.wenv.grs import run_grass
from gasp.gt.fmshp import shp_to_obj
from gasp.gt.toshp import df_to_shp
# Create GRASS GIS Location
grassBase = run_grass(workGrass, location=location, srs=epsg)
import grass.script as grass
import grass.script.setup as gsetup
gsetup.init(grassBase, workGrass, location, 'PERMANENT')
# Import some GRASS GIS tools
from gasp.gt.prox import grs_near as near
from gasp.gt.tbl.attr import geomattr_to_db
from gasp.gt.toshp.cff import shp_to_grs, grs_to_shp
# Import data into GRASS GIS
grsLines = shp_to_grs(lineShp, fprop(lineShp, 'fn', forceLower=True))
grsPoint = shp_to_grs(pointShp, fprop(pointShp, 'fn', forceLower=True))
# Get distance from points to near line
near(grsPoint, grsLines, nearCatCol="tocat", nearDistCol="todistance")
# Get coord of start/end points of polylines
geomattr_to_db(grsLines, ['sta_pnt_x', 'sta_pnt_y'], 'start', 'line')
geomattr_to_db(grsLines, ['end_pnt_x', 'end_pnt_y'], 'end', 'line')
# Export data from GRASS GIS
ogrPoint = grs_to_shp(
grsPoint,
os.path.join(workGrass,
grsPoint + '.shp',
'point',
asMultiPart=True))
ogrLine = grs_to_shp(
grsLines,
os.path.join(workGrass,
grsLines + '.shp',
'point',
asMultiPart=True))
# Points to GeoDataFrame
pntDf = shp_to_obj(ogrPoint)
# Lines to GeoDataFrame
lnhDf = shp_to_obj(ogrLine)
# Erase unecessary fields
pntDf.drop(["todistance"], axis=1, inplace=True)
lnhDf.drop([
c for c in lnhDf.columns.values
if c != 'geometry' and c != 'cat' and c != 'sta_pnt_x'
and c != 'sta_pnt_y' and c != 'end_pnt_x' and c != 'end_pnt_y'
],
axis=1,
inplace=True)
# Join Geometries - Table with Point Geometry and Geometry of the
# nearest line
resultDf = pntDf.merge(lnhDf,
how='inner',
left_on='tocat',
right_on='cat')
# Move points
resultDf['geometry'] = [
geoms[0].interpolate(geoms[0].project(geoms[1]))
for geoms in zip(resultDf.geometry_y, resultDf.geometry_x)
]
resultDf.drop(["geometry_x", "geometry_y", "cat_x", "cat_y"],
axis=1,
inplace=True)
resultDf = GeoDataFrame(resultDf,
crs={"init": 'epsg:{}'.format(epsg)},
geometry="geometry")
# Check if points are equal to any start/end points
resultDf["x"] = resultDf.geometry.x
resultDf["y"] = resultDf.geometry.y
resultDf["check"] = numpy.where(
(resultDf["x"] == resultDf["sta_pnt_x"]) &
(resultDf["y"] == resultDf["sta_pnt_y"]), 1, 0)
resultDf["check"] = numpy.where(
(resultDf["x"] == resultDf["end_pnt_x"]) &
(resultDf["y"] == resultDf["end_pnt_y"]), 1, 0)
# To file
df_to_shp(resultDf, outPoints)
elif api == 'saga':
"""
Snap Points to Lines using SAGA GIS
"""
from gasp import exec_cmd
cmd = ("saga_cmd shapes_points 19 -INPUT {pnt} -SNAP {lnh} "
"-OUTPUT {out}{mv}").format(
pnt=pointShp,
lnh=lineShp,
out=outPoints,
mv="" if not movesShp else " -MOVES {}".format(movesShp))
outcmd = exec_cmd(cmd)
else:
raise ValueError("{} is not available!".format(api))
return outPoints
def lnh_to_polygons(inShp, outShp, api='saga', db=None):
"""
Line to Polygons
API's Available:
* saga;
* grass;
* pygrass;
* psql;
"""
if api == 'saga':
"""
http://www.saga-gis.org/saga_tool_doc/7.0.0/shapes_polygons_3.html
Converts lines to polygons. Line arcs are closed to polygons simply
by connecting the last point with the first. Optionally parts of
polylines can be merged into one polygon optionally.
"""
from gasp import exec_cmd
rcmd = exec_cmd(("saga_cmd shapes_polygons 3 -POLYGONS {} "
"LINES {} -SINGLE 1 -MERGE 1").format(outShp, inShp))
elif api == 'grass' or api == 'pygrass':
# Do it using GRASS GIS
import os
from gasp.gt.wenv.grs import run_grass
from gasp.pyt.oss import fprop
# Create GRASS GIS Session
wk = os.path.dirname(outShp)
lo = fprop(outShp, 'fn', forceLower=True)
gs = run_grass(wk, lo, srs=inShp)
import grass.script as grass
import grass.script.setup as gsetup
gsetup.init(gs, wk, lo, 'PERMANENT')
# Import Packages
from gasp.gt.toshp.cff import shp_to_grs, grs_to_shp
from gasp.gt.toshp.cgeo import line_to_polyline
from gasp.gt.toshp.cgeo import geomtype_to_geomtype
from gasp.gt.toshp.cgeo import boundary_to_areas
# Send data to GRASS GIS
lnh_shp = shp_to_grs(inShp,
fprop(inShp, 'fn', forceLower=True),
asCMD=True if api == 'grass' else None)
# Build Polylines
pol_lnh = line_to_polyline(lnh_shp,
"polylines",
asCmd=True if api == 'grass' else None)
# Polyline to boundary
bound = geomtype_to_geomtype(pol_lnh,
'bound_shp',
'line',
'boundary',
cmd=True if api == 'grass' else None)
# Boundary to Area
areas_shp = boundary_to_areas(bound,
lo,
useCMD=True if api == 'grass' else None)
# Export data
outShp = grs_to_shp(areas_shp,
outShp,
'area',
asCMD=True if api == 'grass' else None)
elif api == 'psql':
""" Do it using PostGIS """
from gasp.pyt.oss import fprop
from gasp.sql.db import create_db
from gasp.gql.to import shp_to_psql
from gasp.gt.toshp.db import dbtbl_to_shp
from gasp.gql.cnv import lnh_to_polg
from gasp.gt.prop.prj import get_epsg_shp
# Create DB
if not db:
db = create_db(fprop(inShp, 'fn', forceLower=True), api='psql')
else:
from gasp.sql.i import db_exists
isDB = db_exists(db)
if not isDB:
create_db(db, api='psql')
# Send data to DB
in_tbl = shp_to_psql(db, inShp, api="shp2pgsql")
# Get Result
result = lnh_to_polg(db, in_tbl, fprop(outShp, 'fn', forceLower=True))
# Export Result
outshp = dbtbl_to_shp(db,
result,
"geom",
outShp,
api='psql',
epsg=get_epsg_shp(inShp))
else:
raise ValueError("API {} is not available".format(api))
return outShp
def check_shape_diff(SHAPES_TO_COMPARE, OUT_FOLDER, REPORT, DB,
GRASS_REGION_TEMPLATE):
"""
Script to check differences between pairs of Feature Classes
Suponha que temos diversas Feature Classes (FC) e que cada uma delas
possui um determinado atributo; imagine tambem que,
considerando todos os pares possiveis entre estas FC,
se pretende comparar as diferencas na distribuicao dos valores
desse atributo para cada par.
* Dependencias:
- GRASS;
- PostgreSQL;
- PostGIS.
"""
import datetime
import os
import pandas
from gasp.sql.fm import q_to_obj
from gasp.to import db_to_tbl
from gasp.sql.to import df_to_db
from gasp.gt.toshp.cff import shp_to_shp
from gasp.gt.toshp.db import dbtbl_to_shp
from gasp.gt.toshp.rst import rst_to_polyg
from gasp.gql.to import shp_to_psql
from gasp.gql.tomtx import tbl_to_area_mtx
from gasp.gt.prop.ff import check_isRaster
from gasp.pyt.oss import fprop
from gasp.sql.db import create_db
from gasp.sql.tbl import tbls_to_tbl
from gasp.sql.to import q_to_ntbl
from gasp.gql.cln import fix_geom
from gasp.to import db_to_tbl
# Check if folder exists, if not create it
if not os.path.exists(OUT_FOLDER):
from gasp.pyt.oss import mkdir
mkdir(OUT_FOLDER, overwrite=None)
else:
raise ValueError('{} already exists!'.format(OUT_FOLDER))
from gasp.gt.wenv.grs import run_grass
gbase = run_grass(OUT_FOLDER,
grassBIN='grass78',
location='shpdif',
srs=GRASS_REGION_TEMPLATE)
import grass.script as grass
import grass.script.setup as gsetup
gsetup.init(gbase, OUT_FOLDER, 'shpdif', 'PERMANENT')
from gasp.gt.toshp.cff import shp_to_grs, grs_to_shp
from gasp.gt.torst import rst_to_grs
from gasp.gt.tbl.fld import rn_cols
# Convert to SHAPE if file is Raster
i = 0
_SHP_TO_COMPARE = {}
for s in SHAPES_TO_COMPARE:
isRaster = check_isRaster(s)
if isRaster:
# To GRASS
rstName = fprop(s, 'fn')
inRst = rst_to_grs(s, "rst_" + rstName, as_cmd=True)
# To Vector
d = rst_to_polyg(inRst,
rstName,
rstColumn="lulc_{}".format(i),
gisApi="grass")
# Export Shapefile
shp = grs_to_shp(d, os.path.join(OUT_FOLDER, d + '.shp'), "area")
_SHP_TO_COMPARE[shp] = "lulc_{}".format(i)
else:
# To GRASS
grsV = shp_to_grs(s, fprop(s, 'fn'), asCMD=True)
# Change name of column with comparing value
ncol = "lulc_{}".format(str(i))
rn_cols(grsV, {SHAPES_TO_COMPARE[s]: "lulc_{}".format(str(i))},
api="grass")
# Export
shp = grs_to_shp(grsV, os.path.join(OUT_FOLDER, grsV + '_rn.shp'),
"area")
_SHP_TO_COMPARE[shp] = "lulc_{}".format(str(i))
i += 1
SHAPES_TO_COMPARE = _SHP_TO_COMPARE
__SHAPES_TO_COMPARE = SHAPES_TO_COMPARE
# Create database
create_db(DB, api='psql')
""" Union SHAPEs """
UNION_SHAPE = {}
FIX_GEOM = {}
SHPS = list(__SHAPES_TO_COMPARE.keys())
for i in range(len(SHPS)):
for e in range(i + 1, len(SHPS)):
# Optimized Union
print("Union between {} and {}".format(SHPS[i], SHPS[e]))
time_a = datetime.datetime.now().replace(microsecond=0)
__unShp = optimized_union_anls(
SHPS[i],
SHPS[e],
os.path.join(OUT_FOLDER, "un_{}_{}.shp".format(i, e)),
GRASS_REGION_TEMPLATE,
os.path.join(OUT_FOLDER, "work_{}_{}".format(i, e)),
multiProcess=True)
time_b = datetime.datetime.now().replace(microsecond=0)
print(time_b - time_a)
# Rename cols
unShp = rn_cols(
__unShp, {
"a_" + __SHAPES_TO_COMPARE[SHPS[i]]:
__SHAPES_TO_COMPARE[SHPS[i]],
"b_" + __SHAPES_TO_COMPARE[SHPS[e]]:
__SHAPES_TO_COMPARE[SHPS[e]]
})
UNION_SHAPE[(SHPS[i], SHPS[e])] = unShp
# Send data to postgresql
SYNTH_TBL = {}
for uShp in UNION_SHAPE:
# Send data to PostgreSQL
union_tbl = shp_to_psql(DB, UNION_SHAPE[uShp], api='shp2pgsql')
# Produce table with % of area equal in both maps
areaMapTbl = q_to_ntbl(
DB,
"{}_syn".format(union_tbl),
("SELECT CAST('{lulc_1}' AS text) AS lulc_1, "
"CAST('{lulc_2}' AS text) AS lulc_2, "
"round("
"CAST(SUM(g_area) / 1000000 AS numeric), 4"
") AS agree_area, round("
"CAST((SUM(g_area) / MIN(total_area)) * 100 AS numeric), 4"
") AS agree_percentage, "
"round("
"CAST(MIN(total_area) / 1000000 AS numeric), 4"
") AS total_area FROM ("
"SELECT {map1_cls}, {map2_cls}, ST_Area(geom) AS g_area, "
"CASE "
"WHEN {map1_cls} = {map2_cls} "
"THEN 1 ELSE 0 "
"END AS isthesame, total_area FROM {tbl}, ("
"SELECT SUM(ST_Area(geom)) AS total_area FROM {tbl}"
") AS foo2"
") AS foo WHERE isthesame = 1 "
"GROUP BY isthesame").format(
lulc_1=fprop(uShp[0], 'fn'),
lulc_2=fprop(uShp[1], 'fn'),
map1_cls=__SHAPES_TO_COMPARE[uShp[0]],
map2_cls=__SHAPES_TO_COMPARE[uShp[1]],
tbl=union_tbl),
api='psql')
# Produce confusion matrix for the pair in comparison
matrixTbl = tbl_to_area_mtx(DB, union_tbl,
__SHAPES_TO_COMPARE[uShp[0]],
__SHAPES_TO_COMPARE[uShp[1]],
union_tbl + '_mtx')
SYNTH_TBL[uShp] = {"TOTAL": areaMapTbl, "MATRIX": matrixTbl}
# UNION ALL TOTAL TABLES
total_table = tbls_to_tbl(DB, [SYNTH_TBL[k]["TOTAL"] for k in SYNTH_TBL],
'total_table')
# Create table with % of agreement between each pair of maps
mapsNames = q_to_obj(
DB,
("SELECT lulc FROM ("
"SELECT lulc_1 AS lulc FROM {tbl} GROUP BY lulc_1 "
"UNION ALL "
"SELECT lulc_2 AS lulc FROM {tbl} GROUP BY lulc_2"
") AS lu GROUP BY lulc ORDER BY lulc").format(tbl=total_table),
db_api='psql').lulc.tolist()
FLDS_TO_PIVOT = ["agree_percentage", "total_area"]
Q = ("SELECT * FROM crosstab('"
"SELECT CASE "
"WHEN foo.lulc_1 IS NOT NULL THEN foo.lulc_1 ELSE jtbl.tmp1 "
"END AS lulc_1, CASE "
"WHEN foo.lulc_2 IS NOT NULL THEN foo.lulc_2 ELSE jtbl.tmp2 "
"END AS lulc_2, CASE "
"WHEN foo.{valCol} IS NOT NULL THEN foo.{valCol} ELSE 0 "
"END AS agree_percentage FROM ("
"SELECT lulc_1, lulc_2, {valCol} FROM {tbl} UNION ALL "
"SELECT lulc_1, lulc_2, {valCol} FROM ("
"SELECT lulc_1 AS lulc_2, lulc_2 AS lulc_1, {valCol} "
"FROM {tbl}"
") AS tst"
") AS foo FULL JOIN ("
"SELECT lulc_1 AS tmp1, lulc_2 AS tmp2 FROM ("
"SELECT lulc_1 AS lulc_1 FROM {tbl} GROUP BY lulc_1 "
"UNION ALL "
"SELECT lulc_2 AS lulc_1 FROM {tbl} GROUP BY lulc_2"
") AS tst_1, ("
"SELECT lulc_1 AS lulc_2 FROM {tbl} GROUP BY lulc_1 "
"UNION ALL "
"SELECT lulc_2 AS lulc_2 FROM {tbl} GROUP BY lulc_2"
") AS tst_2 WHERE lulc_1 = lulc_2 GROUP BY lulc_1, lulc_2"
") AS jtbl ON foo.lulc_1 = jtbl.tmp1 AND foo.lulc_2 = jtbl.tmp2 "
"ORDER BY lulc_1, lulc_2"
"') AS ct("
"lulc_map text, {crossCols}"
")")
TOTAL_AGREE_TABLE = None
TOTAL_AREA_TABLE = None
for f in FLDS_TO_PIVOT:
if not TOTAL_AGREE_TABLE:
TOTAL_AGREE_TABLE = q_to_ntbl(
DB,
"agreement_table",
Q.format(tbl=total_table,
valCol=f,
crossCols=", ".join([
"{} numeric".format(map_) for map_ in mapsNames
])),
api='psql')
else:
TOTAL_AREA_TABLE = q_to_ntbl(DB,
"area_table",
Q.format(tbl=total_table,
valCol=f,
crossCols=", ".join([
"{} numeric".format(map_)
for map_ in mapsNames
])),
api='psql')
# Union Mapping
UNION_MAPPING = pandas.DataFrame(
[[k[0], k[1], fprop(UNION_SHAPE[k], 'fn')] for k in UNION_SHAPE],
columns=['shp_a', 'shp_b', 'union_shp'])
UNION_MAPPING = df_to_db(DB, UNION_MAPPING, 'union_map', api='psql')
# Export Results
TABLES = [UNION_MAPPING, TOTAL_AGREE_TABLE, TOTAL_AREA_TABLE
] + [SYNTH_TBL[x]["MATRIX"] for x in SYNTH_TBL]
SHEETS = ["union_map", "agreement_percentage", "area_with_data_km"] + [
"{}_{}".format(fprop(x[0], 'fn')[:15],
fprop(x[1], 'fn')[:15]) for x in SYNTH_TBL
]
db_to_tbl(DB, ["SELECT * FROM {}".format(x) for x in TABLES],
REPORT,
sheetsNames=SHEETS,
dbAPI='psql')
return REPORT
def optimized_union_anls(lyr_a,
lyr_b,
outShp,
ref_boundary,
workspace=None,
multiProcess=None):
"""
Optimized Union Analysis
Goal: optimize v.overlay performance for Union operations
"""
import os
from gasp.pyt.oss import fprop, lst_ff
from gasp.pyt.oss import cpu_cores
from gasp.gt.sample import create_fishnet
from gasp.gt.wenv.grs import run_grass
from gasp.gt.toshp import eachfeat_to_newshp
from gasp.gt.toshp.mtos import shps_to_shp
from gasp.gt.attr import split_shp_by_attr
from gasp.gt.torst import shpext_to_rst
from gasp.gt.prop.ext import get_ext
if workspace:
if not os.path.exists(workspace):
from gasp.pyt.oss import mkdir
mkdir(workspace, overwrite=True)
else:
from gasp.pyt.oss import mkdir
workspace = mkdir(os.path.join(os.path.dirname(outShp), "union_work"))
# Create Fishnet
ncpu = cpu_cores()
if ncpu == 12:
nrow = 4
ncol = 3
elif ncpu == 8:
nrow = 4
ncol = 2
else:
nrow = 2
ncol = 2
ext = get_ext(ref_boundary)
width = (ext[1] - ext[0]) / ncol
height = (ext[3] - ext[2]) / nrow
gridShp = create_fishnet(ref_boundary,
os.path.join(workspace, 'ref_grid.shp'),
width,
height,
xy_row_col=None)
# Split Fishnet in several files
cellsShp = eachfeat_to_newshp(gridShp, workspace)
if not multiProcess:
# INIT GRASS GIS Session
grsbase = run_grass(workspace, location="grs_loc", srs=ref_boundary)
import grass.script.setup as gsetup
gsetup.init(grsbase, workspace, "grs_loc", 'PERMANENT')
# Add data to GRASS GIS
from gasp.gt.toshp.cff import shp_to_grs
cellsShp = [
shp_to_grs(shp, fprop(shp, 'fn'), asCMD=True) for shp in cellsShp
]
LYR_A = shp_to_grs(lyr_a, fprop(lyr_a, 'fn'), asCMD=True)
LYR_B = shp_to_grs(lyr_b, fprop(lyr_b, 'fn'), asCMD=True)
# Clip Layers A and B for each CELL in fishnet
LYRS_A = [
clip(LYR_A, cellsShp[x], LYR_A + "_" + str(x), api_gis="grass")
for x in range(len(cellsShp))
]
LYRS_B = [
clip(LYR_B, cellsShp[x], LYR_B + "_" + str(x), api_gis="grass")
for x in range(len(cellsShp))
]
# Union SHPS
UNION_SHP = [
union(LYRS_A[i], LYRS_B[i], "un_{}".format(i), api_gis="grass")
for i in range(len(cellsShp))
]
# Export Data
from gasp.gt.toshp.cff import grs_to_shp
_UNION_SHP = [
grs_to_shp(shp, os.path.join(workspace, shp + ".shp"), "area")
for shp in UNION_SHP
]
else:
def clip_and_union(la, lb, cell, work, proc, output):
ref_rst = shpext_to_rst(cell,
os.path.join(os.path.dirname(cell),
fprop(cell, 'fn') + '.tif'),
cellsize=10)
# Start GRASS GIS Session
loc = "proc_" + str(proc)
grsbase = run_grass(work, location=loc, srs=ref_rst)
import grass.script.setup as gsetup
gsetup.init(grsbase, work, loc, 'PERMANENT')
# Import GRASS GIS modules
from gasp.gt.toshp.cff import shp_to_grs, grs_to_shp
from gasp.gt.prop.feat import feat_count
# Add data to GRASS
a = shp_to_grs(la, fprop(la, 'fn'), filterByReg=True, asCMD=True)
b = shp_to_grs(lb, fprop(lb, 'fn'), filterByReg=True, asCMD=True)
if not feat_count(a, gisApi="grass", work=work, loc=loc):
return
if not feat_count(b, gisApi="grass", work=work, loc=loc):
return
# Clip
a_clip = clip(a,
None,
"{}_clip".format(a),
api_gis="grass",
clip_by_region=True)
b_clip = clip(b,
None,
"{}_clip".format(b),
api_gis="grass",
clip_by_region=True)
# Union
u_shp = union(a_clip,
b_clip,
"un_{}".format(fprop(cell, 'fn')),
api_gis="grass")
# Export
o = grs_to_shp(u_shp, output, "area")
import multiprocessing
thrds = [
multiprocessing.Process(
target=clip_and_union,
name="th-{}".format(i),
args=(lyr_a, lyr_b, cellsShp[i],
os.path.join(workspace, "th_{}".format(i)), i,
os.path.join(workspace, "uniao_{}.shp".format(i))))
for i in range(len(cellsShp))
]
for t in thrds:
t.start()
for t in thrds:
t.join()
ff_shp = lst_ff(workspace, file_format='.shp')
_UNION_SHP = []
for i in range(len(cellsShp)):
p = os.path.join(workspace, "uniao_{}.shp".format(i))
if p in ff_shp:
_UNION_SHP.append(p)
else:
continue
# Merge all union into the same layer
MERGED_SHP = shps_to_shp(_UNION_SHP, outShp, api="ogr2ogr")
return MERGED_SHP
def lulc_by_cell(tid, boundary, lulc_shps, fishnet, result, workspace):
bname = fprop(boundary, 'fn')
# Boundary to Raster
ref_rst = shp_to_rst(boundary, None, 10, 0,
os.path.join(workspace, 'rst_{}.tif'.format(bname)))
# Create GRASS GIS Session
loc_name = 'loc_' + bname
gbase = run_grass(workspace, location=loc_name, srs=ref_rst)
import grass.script as grass
import grass.script.setup as gsetup
gsetup.init(gbase, workspace, loc_name, 'PERMANENT')
# GRASS GIS Modules
from gasp.gt.toshp.cff import shp_to_grs, grs_to_shp
from gasp.gt.gop.ovlay import intersection
from gasp.gt.tbl.attr import geomattr_to_db
from gasp.gt.prop.feat import feat_count
# Send Fishnet to GRASS GIS
fnet = shp_to_grs(fishnet, fprop(fishnet, 'fn'), asCMD=True)
# Processing
ulst = []
l_lulc_grs = []
for shp in lulc_shps:
iname = fprop(shp, 'fn')
# LULC Class to GRASS GIS
lulc_grs = shp_to_grs(shp, iname, filterByReg=True, asCMD=True)
if not feat_count(
lulc_grs, gisApi='grass', work=workspace, loc=loc_name):
continue
# Intersect Fishnet | LULC CLass
union_grs = intersection(fnet, lulc_grs, iname + '_i', api="grass")
# Get Areas
geomattr_to_db(union_grs, "areav", "area", "boundary", unit='meters')
# Export Table
funion = grs_to_shp(union_grs, os.path.join(result, iname + '.shp'),
'area')
ulst.append(funion)
l_lulc_grs.append(lulc_grs)
# Intersect between all LULC SHPS
ist_shp = []
if len(l_lulc_grs) > 1:
for i in range(len(l_lulc_grs)):
for e in range(i + 1, len(l_lulc_grs)):
ishp = intersection(l_lulc_grs[i],
l_lulc_grs[e],
'lulcint_' + str(i) + '_' + str(e),
api="grass")
if not feat_count(
ishp, gisApi='grass', work=workspace, loc=loc_name):
continue
else:
ist_shp.append(ishp)
if len(ist_shp):
from gasp.gt.gop.genze import dissolve
from gasp.gt.tbl.grs import reset_table
if len(ist_shp) > 1:
from gasp.gt.toshp.mtos import shps_to_shp
# Export shapes
_ist_shp = [
grs_to_shp(s, os.path.join(workspace, loc_name,
s + '.shp'), 'area')
for s in ist_shp
]
# Merge Intersections
merge_shp = shps_to_shp(_ist_shp,
os.path.join(workspace, loc_name,
'merge_shp.shp'),
api='pandas')
# Import GRASS
merge_shp = shp_to_grs(merge_shp, 'merge_shp')
else:
merge_shp = ist_shp[0]
# Dissolve Shape
reset_table(merge_shp, {'refid': 'varchar(2)'}, {'refid': '1'})
overlay_areas = dissolve(merge_shp,
'overlay_areas',
'refid',
api='grass')
# Union Fishnet | Overlay's
union_ovl = intersection(fnet,
overlay_areas,
'ovl_union',
api="grass")
funion_ovl = grs_to_shp(union_ovl,
os.path.join(result, union_ovl + '.shp'),
'area')
ulst.append(funion_ovl)
# Export Tables
return ulst
def make_dem(grass_workspace,
data,
field,
output,
extent_template,
method="IDW",
cell_size=None,
mask=None):
"""
Create Digital Elevation Model
Methods Available:
* IDW;
* BSPLINE;
* SPLINE;
* CONTOUR;
"""
from gasp.pyt.oss import fprop
from gasp.gt.wenv.grs import run_grass
from gasp.gt.prop.prj import get_epsg
LOC_NAME = fprop(data, 'fn', forceLower=True)[:5] + "_loc"
# Get EPSG From Raster
EPSG = get_epsg(extent_template)
if not EPSG:
raise ValueError(
'Cannot get EPSG code of Extent Template File ({})'.format(
extent_template))
# Know if data geometry are points
if method == 'BSPLINE' or method == 'SPLINE':
from gasp.gt.prop.feat import get_gtype
data_gtype = get_gtype(data, gisApi='ogr')
# Create GRASS GIS Location
grass_base = run_grass(grass_workspace, location=LOC_NAME, srs=EPSG)
# Start GRASS GIS Session
import grass.script as grass
import grass.script.setup as gsetup
gsetup.init(grass_base, grass_workspace, LOC_NAME, 'PERMANENT')
# Get Extent Raster
ref_template = ob_ref_rst(extent_template,
os.path.join(grass_workspace, LOC_NAME),
cellsize=cell_size)
# IMPORT GRASS GIS MODULES #
from gasp.gt.torst import rst_to_grs, grs_to_rst
from gasp.gt.toshp.cff import shp_to_grs
from gasp.gt.wenv.grs import rst_to_region
# Configure region
rst_to_grs(ref_template, 'extent')
rst_to_region('extent')
# Convert elevation "data" to GRASS Vector
elv = shp_to_grs(data, 'elevation')
OUTPUT_NAME = fprop(output, 'fn', forceLower=True)
if method == "BSPLINE":
from gasp.gt.nop.itp import bspline
# Convert to points if necessary
if data_gtype != 'POINT' and data_gtype != 'MULTIPOINT':
from gasp.gt.toshp.cgeo import feat_vertex_to_pnt
elev_pnt = feat_vertex_to_pnt(elv, "elev_pnt", nodes=None)
else:
elev_pnt = elv
outRst = bspline(elev_pnt,
field,
OUTPUT_NAME,
mway='bicubic',
lyrN=1,
asCMD=True)
elif method == "SPLINE":
from gasp.gt.nop.itp import surfrst
# Convert to points if necessary
if data_gtype != 'POINT' and data_gtype != 'MULTIPOINT':
from gasp.gt.toshp.cgeo import feat_vertex_to_pnt
elev_pnt = feat_vertex_to_pnt(elv, "elev_pnt", nodes=None)
else:
elev_pnt = elv
outRst = surfrst(elev_pnt, field, OUTPUT_NAME, lyrN=1, ascmd=True)
elif method == "CONTOUR":
from gasp.gt.torst import shp_to_rst
from gasp.gt.nop.itp import surfcontour
# Apply mask if mask
if mask:
from gasp.gt.torst import grs_to_mask, rst_to_grs
rst_mask = rst_to_grs(mask, 'rst_mask', as_cmd=True)
grs_to_mask(rst_mask)
# Elevation (GRASS Vector) to Raster
elevRst = shp_to_rst(elv,
field,
None,
None,
'rst_elevation',
api="pygrass")
# Run Interpolator
outRst = surfcontour(elevRst, OUTPUT_NAME, ascmd=True)
elif method == "IDW":
from gasp.gt.nop.itp import ridw
from gasp.gt.nop.alg import rstcalc
from gasp.gt.torst import shp_to_rst
# Elevation (GRASS Vector) to Raster
elevRst = shp_to_rst(elv,
field,
None,
None,
'rst_elevation',
api='pygrass')
# Multiply cells values by 100 000.0
rstcalc('int(rst_elevation * 100000)', 'rst_elev_int', api='pygrass')
# Run IDW to generate the new DEM
ridw('rst_elev_int', 'dem_int', numberPoints=15)
# DEM to Float
rstcalc('dem_int / 100000.0', OUTPUT_NAME, api='pygrass')
# Export DEM to a file outside GRASS Workspace
grs_to_rst(OUTPUT_NAME, output)
return output
def run_viewshed_by_cpu(tid,
db,
obs,
dem,
srs,
vis_basename='vis',
maxdst=None,
obselevation=None):
# Create Database
new_db = create_db("{}_{}".format(db, str(tid)), api='psql')
# Points to Database
pnt_tbl = df_to_db(new_db,
obs,
'pnt_tbl',
api='psql',
epsg=srs,
geomType='Point',
colGeom='geometry')
# Create GRASS GIS Session
workspace = mkdir(
os.path.join(os.path.dirname(dem), 'work_{}'.format(str(tid))))
loc_name = 'vis_loc'
gbase = run_grass(workspace, location=loc_name, srs=dem)
# Start GRASS GIS Session
import grass.script as grass
import grass.script.setup as gsetup
gsetup.init(gbase, workspace, loc_name, 'PERMANENT')
from gasp.gt.torst import rst_to_grs, grs_to_rst
from gasp.gt.nop.surf import grs_viewshed
from gasp.gt.deldt import del_rst
# Send DEM to GRASS GIS
grs_dem = rst_to_grs(dem, 'grs_dem', as_cmd=True)
# Produce Viewshed for each point in obs
for idx, row in obs.iterrows():
# Get Viewshed raster
vrst = grs_viewshed(grs_dem, (row.geometry.x, row.geometry.y),
'{}_{}'.format(vis_basename, str(row[obs_id])),
max_dist=maxdst,
obs_elv=obselevation)
# Export Raster to File
frst = grs_to_rst(vrst, os.path.join(workspace, vrst + '.tif'))
# Raster to Array
img = gdal.Open(frst)
num = img.ReadAsArray()
# Two Dimension to One Dimension
# Reshape Array
numone = num.reshape(num.shape[0] * num.shape[1])
# Get Indexes with visibility
visnum = np.arange(numone.shape[0]).astype(np.uint32)
visnum = visnum[numone == 1]
# Get rows indexes
visrow = visnum / num.shape[0]
visrow = visrow.astype(np.uint32)
# Get cols indexes
viscol = visnum - (visrow * num.shape[1])
# Visibility indexes to Pandas DataFrame
idxnum = np.full(visrow.shape, row[obs_id])
visdf = pd.DataFrame({
'pntid': idxnum,
'rowi': visrow,
'coli': viscol
})
# Pandas DF to database
# Create Visibility table
df_to_db(new_db,
visdf,
vis_basename,
api='psql',
colGeom=None,
append=None if not idx else True)
# Delete all variables
numone = None
visnum = None
visrow = None
viscol = None
idxnum = None
visdf = None
del img
# Delete GRASS GIS File
del_rst(vrst)
# Delete TIFF File
del_file(frst)
frst = None
def joinLines_by_spatial_rel_raster(mainLines, mainId, joinLines, joinCol,
outfile, epsg):
"""
Join Attributes based on a spatial overlap.
An raster based approach
"""
import os
import pandas
from geopandas import GeoDataFrame
from gasp.gt.fmshp import shp_to_obj
from gasp.gt.toshp import df_to_shp
from gasp.gt.toshp.coord import shpext_to_boundshp
from gasp.gt.torst import shp_to_rst
from gasp.g.to import df_to_geodf
from gasp.gt.wenv.grs import run_grass
from gasp.pyt.df.joins import join_dfs
from gasp.pyt.df.agg import df_groupBy
from gasp.pyt.oss import fprop, mkdir
workspace = mkdir(os.path.join(os.path.dirname(mainLines, 'tmp_dt')))
# Create boundary file
boundary = shpext_to_boundshp(mainLines,
os.path.join(workspace, "bound.shp"), epsg)
boundRst = shp_to_rst(boundary,
None,
5,
-99,
os.path.join(workspace, "rst_base.tif"),
epsg=epsg,
api='gdal')
# Start GRASS GIS Session
gbase = run_grass(workspace, location="grs_loc", srs=boundRst)
import grass.script as grass
import grass.script.setup as gsetup
gsetup.init(gbase, workspace, "grs_loc", "PERMANENT")
from gasp.gt.nop.local import combine
from gasp.gt.prop.rst import get_rst_report_data
from gasp.gt.toshp.cff import shp_to_grs, grs_to_shp
from gasp.gt.torst import shp_to_rst
# Add data to GRASS GIS
mainVector = shp_to_grs(mainLines, fprop(mainLines, 'fn', forceLower=True))
joinVector = shp_to_grs(joinLines, fprop(joinLines, 'fn', forceLower=True))
mainRst = shp_to_rst(mainVector,
mainId,
None,
None,
"rst_" + mainVector,
api='pygrass')
joinRst = shp_to_rst(joinVector,
joinCol,
None,
None,
"rst_" + joinVector,
api='pygrass')
combRst = combine(mainRst, joinRst, "combine_rst", api="pygrass")
combine_data = get_rst_report_data(combRst, UNITS="c")
combDf = pandas.DataFrame(combine_data,
columns=["comb_cat", "rst_1", "rst_2", "ncells"])
combDf = combDf[combDf["rst_2"] != '0']
combDf["ncells"] = combDf["ncells"].astype(int)
gbdata = df_groupBy(combDf, ["rst_1"], "MAX", "ncells")
fTable = join_dfs(gbdata, combDf, ["rst_1", "ncells"], ["rst_1", "ncells"])
fTable["rst_2"] = fTable["rst_2"].astype(int)
fTable = df_groupBy(fTable, ["rst_1", "ncells"],
STAT='MIN',
STAT_FIELD="rst_2")
mainLinesCat = grs_to_shp(mainVector,
os.path.join(workspace, mainVector + '.shp'),
'line')
mainLinesDf = shp_to_obj(mainLinesCat)
resultDf = join_dfs(mainLinesDf,
fTable,
"cat",
"rst_1",
onlyCombinations=None)
resultDf.rename(columns={"rst_2": joinCol}, inplace=True)
resultDf = df_to_geodf(resultDf, "geometry", epsg)
df_to_shp(resultDf, outfile)
return outfile
def shp_diff_fm_ref(refshp, refcol, shps, out_folder, refrst, db=None):
"""
Check differences between each shp in shps and one reference shape
Dependencies:
- GRASS;
- PostgreSQL with Postgis or GeoPandas;
"""
import os
from gasp.gt.prop.ff import check_isRaster
from gasp.gt.wenv.grs import run_grass
from gasp.pyt.oss import fprop
from gasp.gt.tbl.tomtx import tbl_to_areamtx
# Check if folder exists, if not create it
if not os.path.exists(out_folder):
from gasp.pyt.oss import mkdir
mkdir(out_folder)
# Start GRASS GIS Session
gbase = run_grass(out_folder,
grassBIN='grass78',
location='shpdif',
srs=refrst)
import grass.script.setup as gsetup
gsetup.init(gbase, out_folder, 'shpdif', 'PERMANENT')
from gasp.gt.toshp.cff import shp_to_grs, grs_to_shp
from gasp.gt.torst import rst_to_grs
from gasp.gt.tbl.fld import rn_cols
from gasp.gt.toshp.rst import rst_to_polyg
# Convert to SHAPE if file is Raster
# Rename interest columns
i = 0
lstff = [refshp] + list(shps.keys())
__shps = {}
for s in lstff:
is_rst = check_isRaster(s)
if is_rst:
# To GRASS
rname = fprop(s, 'fn')
inrst = rst_to_grs(s, "rst_" + rname, as_cmd=True)
# To vector
d = rst_to_polyg(inrst,
rname,
rstColumn="lulc_{}".format(str(i)),
gisApi="grass")
else:
# To GRASS
d = shp_to_grs(s, fprop(s, 'fn'), asCMD=True)
# Change name of interest colum
rn_cols(d, {shps[s] if i else refcol: "lulc_{}".format(str(i))},
api="grass")
# Export To Shapefile
if not i:
refshp = grs_to_shp(d, os.path.join(out_folder, d + '.shp'),
'area')
refcol = "lulc_{}".format(str(i))
else:
shp = grs_to_shp(d, os.path.join(out_folder, d + '.shp'), 'area')
__shps[shp] = "lulc_{}".format(str(i))
i += 1
# Union Shapefiles
union_shape = {}
for shp in __shps:
# Optimized Union
sname = fprop(shp, 'fn')
union_shape[shp] = optimized_union_anls(
shp,
refshp,
os.path.join(out_folder, sname + '_un.shp'),
refrst,
os.path.join(out_folder, "wk_" + sname),
multiProcess=True)
# Produce confusion matrices
mtxf = tbl_to_areamtx(union_shape[shp],
"a_" + __shps[shp],
'b_' + refcol,
os.path.join(out_folder, sname + '.xlsx'),
db=db,
with_metrics=True)
return out_folder
def match_cellsize_and_clip(rstBands, refRaster, outFolder,
clipShp=None):
"""
Resample images to make them with the same resolution and clip
Good to resample Sentinel bands with more than 10 meters.
Dependencies:
* GRASS GIS;
* GDAL/OGR.
"""
import os
from gasp.gt.prop.prj import get_rst_epsg
from gasp.gt.wenv.grs import run_grass
from gasp.pyt.oss import fprop, mkdir
# Check if outfolder exists
if not os.path.exists(outFolder):
mkdir(outFolder, overwrite=None)
# Get EPSG from refRaster
epsg = get_rst_epsg(refRaster, returnIsProj=None)
"""
Start GRASS GIS Session
"""
GRS_WORKSPACE = mkdir(os.path.join(outFolder, 'grswork'))
grsb = run_grass(
GRS_WORKSPACE, grassBIN='grass78', location='resample',
srs=epsg
)
import grass.script as grass
import grass.script.setup as gsetup
gsetup.init(grsb, GRS_WORKSPACE, 'resample', 'PERMANENT')
"""
Import packages related with GRASS GIS
"""
from gasp.gt.torst import rst_to_grs, grs_to_rst
from gasp.gt.wenv.grs import rst_to_region
from gasp.gt.toshp.cff import shp_to_grs
from gasp.gt.torst import shp_to_rst, grs_to_mask
# Send Ref Raster to GRASS GIS and set region
extRst = rst_to_grs(refRaster, 'ext_rst')
rst_to_region(extRst)
# Import all bands in rstBands
grs_bands = [rst_to_grs(i, fprop(i, 'fn')) for i in rstBands]
if clipShp:
# Add clipShp to GRASS
grs_clip = shp_to_grs(clipShp, fprop(clipShp, 'fn'), asCMD=True)
# SHP to Raster
rstClip = shp_to_rst(
grs_clip, 1, None, 0, 'rst_' + grs_clip,
api='grass'
)
# Set region using
rst_to_region(rstClip)
# Set mask
grs_to_mask(rstClip)
# Export bands
return [grs_to_rst(
i, os.path.join(outFolder, i + '.tif')
) for i in grs_bands]
def v_break_at_points(workspace, loc, lineShp, pntShp, db, srs, out_correct,
out_tocorrect):
"""
Break lines at points - Based on GRASS GIS v.edit
Use PostGIS to sanitize the result
TODO: Confirm utility
Problem: GRASS GIS always uses the first line to break.
"""
import os
from gasp.gql.to import shp_to_psql
from gasp.gt.toshp.db import dbtbl_to_shp
from gasp.gt.wenv.grs import run_grass
from gasp.pyt.oss import fprop
from gasp.sql.db import create_db
from gasp.sql.to import q_to_ntbl
tmpFiles = os.path.join(workspace, loc)
gbase = run_grass(workspace, location=loc, srs=srs)
import grass.script as grass
import grass.script.setup as gsetup
gsetup.init(gbase, workspace, loc, 'PERMANENT')
from gasp.gt.toshp.cff import shp_to_grs, grs_to_shp
grsLine = shp_to_grs(
lineShp, fprop(lineShp, 'fn', forceLower=True)
)
vedit_break(grsLine, pntShp, geomType='line')
LINES = grs_to_shp(grsLine, os.path.join(
tmpFiles, grsLine + '_v1.shp'), 'line')
# Sanitize output of v.edit.break using PostGIS
create_db(db, overwrite=True, api='psql')
LINES_TABLE = shp_to_psql(
db, LINES, srsEpsgCode=srs,
pgTable=fprop(LINES, 'fn', forceLower=True), api="shp2pgsql"
)
# Delete old/original lines and stay only with the breaked one
Q = (
"SELECT {t}.*, foo.cat_count FROM {t} INNER JOIN ("
"SELECT cat, COUNT(cat) AS cat_count, "
"MAX(ST_Length(geom)) AS max_len "
"FROM {t} GROUP BY cat"
") AS foo ON {t}.cat = foo.cat "
"WHERE foo.cat_count = 1 OR foo.cat_count = 2 OR ("
"foo.cat_count = 3 AND ST_Length({t}.geom) <= foo.max_len)"
).format(t=LINES_TABLE)
CORR_LINES = q_to_ntbl(
db, "{}_corrected".format(LINES_TABLE), Q, api='psql'
)
# TODO: Delete Rows that have exactly the same geometry
# Highlight problems that the user must solve case by case
Q = (
"SELECT {t}.*, foo.cat_count FROM {t} INNER JOIN ("
"SELECT cat, COUNT(cat) AS cat_count FROM {t} GROUP BY cat"
") AS foo ON {t}.cat = foo.cat "
"WHERE foo.cat_count > 3"
).format(t=LINES_TABLE)
ERROR_LINES = q_to_ntbl(
db, "{}_not_corr".format(LINES_TABLE), Q, api='psql'
)
dbtbl_to_shp(
db, CORR_LINES, "geom", out_correct,
api="pgsql2shp"
)
dbtbl_to_shp(
db, ERROR_LINES, "geom", out_tocorrect,
api="pgsql2shp"
)