def grs_vect_selbyarea(osmdb, polyTbl, UPPER=True, apidb='SQLITE'):
"""
Select features with area upper than.
A field with threshold is needed in the database.
"""
import datetime
from glass.g.gp.gen import dissolve
from glass.g.tbl.grs import add_table
from glass.ete.osm2lulc import GEOM_AREA
from glass.ng.prop.sql import row_num as cnt_row
from glass.g.it.shp import dbtbl_to_shp as db_to_shp
OPERATOR = ">" if UPPER else "<"
DIRECTION = "upper" if UPPER else "lower"
WHR = "{ga} {op} t_area_{r} and area_{r} IS NOT NULL".format(op=OPERATOR,
r=DIRECTION,
ga=GEOM_AREA)
# Check if we have interest data
time_a = datetime.datetime.now().replace(microsecond=0)
N = cnt_row(osmdb,
polyTbl,
where=WHR,
api='psql' if apidb == 'POSTGIS' else 'sqlite')
time_b = datetime.datetime.now().replace(microsecond=0)
if not N: return None, {0: ('count_rows', time_b - time_a)}
# Data to GRASS GIS
grsVect = db_to_shp(osmdb,
polyTbl,
"geometry",
"area_{}".format(DIRECTION),
where=WHR,
inDB='psql' if apidb == 'POSTGIS' else 'sqlite',
filterByReg=True,
outShpIsGRASS=True)
time_c = datetime.datetime.now().replace(microsecond=0)
dissVect = dissolve(grsVect,
"diss_area_{}".format(DIRECTION),
"area_{}".format(DIRECTION),
api="grass")
add_table(dissVect, None, lyrN=1, asCMD=True)
time_d = datetime.datetime.now().replace(microsecond=0)
return dissVect, {
0: ('count_rows', time_b - time_a),
1: ('import', time_c - time_b),
2: ('dissolve', time_d - time_c)
}
def grs_vect_bbuffer(osmdata, lineTbl, api_db='SQLITE'):
"""
Basic Buffer strategie
"""
import datetime
from glass.g.gp.prox.bfing import _buffer
from glass.g.gp.gen import dissolve
from glass.g.tbl.grs import add_table
from glass.ng.prop.sql import row_num as cnt_row
from glass.g.it.shp import dbtbl_to_shp as db_to_shp
WHR = "basic_buffer IS NOT NULL"
# Check if we have data
time_a = datetime.datetime.now().replace(microsecond=0)
N = cnt_row(osmdata,
lineTbl,
where=WHR,
api='psql' if api_db == 'POSTGIS' else 'sqlite')
time_b = datetime.datetime.now().replace(microsecond=0)
if not N: return None, {0: ('count_rows_roads', time_b - time_a)}
grsVect = db_to_shp(osmdata,
lineTbl,
"geometry",
"bb_lnh",
where=WHR,
filterByReg=True,
inDB='psql' if api_db == 'POSTGIS' else 'sqlite',
outShpIsGRASS=True)
time_c = datetime.datetime.now().replace(microsecond=0)
grsBuf = _buffer(grsVect,
"bf_basic_buffer",
"bb_poly",
api="grass",
geom_type="line")
time_d = datetime.datetime.now().replace(microsecond=0)
grsDiss = dissolve(grsBuf, "bb_diss", "basic_buffer", api="grass")
add_table(grsDiss, None, lyrN=1, asCMD=True)
time_e = datetime.datetime.now().replace(microsecond=0)
return grsDiss, {
0: ('count_rows', time_b - time_a),
1: ('import', time_c - time_b),
2: ('buffer', time_d - time_c),
3: ('dissolve', time_e - time_d)
}
def grs_vector(db, polyTable, apidb='SQLITE'):
"""
Simple Selection using GRASS GIS
"""
import datetime
from glass.g.gp.gen import dissolve
from glass.g.tbl.grs import add_table
from glass.ng.prop.sql import row_num as cont_row
from glass.g.it.shp import dbtbl_to_shp as db_to_grs
WHR = "selection IS NOT NULL"
# Check if we have interest data
time_a = datetime.datetime.now().replace(microsecond=0)
N = cont_row(db,
polyTable,
where=WHR,
api='psql' if apidb == 'POSTGIS' else 'sqlite')
time_b = datetime.datetime.now().replace(microsecond=0)
if not N: return None, {0: ('count_rows', time_b - time_a)}
# Data to GRASS GIS
grsVect = db_to_grs(db,
polyTable,
"geometry",
"sel_rule",
where=WHR,
filterByReg=True,
inDB='psql' if apidb == 'POSTGIS' else 'sqlite',
outShpIsGRASS=True)
time_c = datetime.datetime.now().replace(microsecond=0)
dissVect = dissolve(grsVect, "diss_sel_rule", "selection", api="grass")
add_table(dissVect, None, lyrN=1, asCMD=True)
time_d = datetime.datetime.now().replace(microsecond=0)
return dissVect, {
0: ('count_rows', time_b - time_a),
1: ('import', time_c - time_b),
2: ('dissolve', time_d - time_c)
}
def grs_vec_roads(osmdb, lineTbl, polyTbl):
"""
Select Roads for GRASS GIS
"""
import datetime
from glass.ng.prop.sql import row_num
from glass.g.it.shp import dbtbl_to_shp
from glass.g.gp.prox.bfing import _buffer
from glass.g.gp.gen import dissolve
from glass.g.tbl.grs import add_table
# Roads to GRASS GIS
time_a = datetime.datetime.now().replace(microsecond=0)
NR = row_num(osmdb, lineTbl, where="roads IS NOT NULL", api='sqlite')
time_b = datetime.datetime.now().replace(microsecond=0)
if not NR: return None, {0: ('count_rows_roads', time_b - time_a)}
roadsVect = dbtbl_to_shp(osmdb,
lineTbl,
"geometry",
"all_roads",
where="roads IS NOT NULL",
inDB='sqlite',
outShpIsGRASS=True)
time_c = datetime.datetime.now().replace(microsecond=0)
# Buildings to GRASS GIS
NB = row_num(osmdb, polyTbl, where="building IS NOT NULL", api='sqlite')
time_d = datetime.datetime.now().replace(microsecond=0)
if NB:
from glass.g.gp.prox import grs_near as near
from glass.g.tbl.grs import update_table
builds = dbtbl_to_shp(osmdb,
polyTbl,
"geometry",
"all_builds",
where="building IS NOT NULL",
filterByReg=True,
inDB='sqlite',
outShpIsGRASS=True)
time_e = datetime.datetime.now().replace(microsecond=0)
near(roadsVect, builds, nearDistCol="todist", maxDist=12, as_cmd=True)
time_f = datetime.datetime.now().replace(microsecond=0)
update_table(roadsVect,
"bf_roads",
"round(todist,0)",
"\"todist > 0\"",
lyrN=1,
ascmd=True)
time_g = datetime.datetime.now().replace(microsecond=0)
else:
time_e = None
time_f = None
time_g = None
# Run Buffer tool
roadsBf = _buffer(roadsVect,
"bf_roads",
"bf_roads",
api='grass',
geom_type="line")
time_h = datetime.datetime.now().replace(microsecond=0)
# Dissolve Roads
roadsDiss = dissolve(roadsBf, "diss_roads", "roads", api="grass")
add_table(roadsDiss, None, lyrN=1, asCMD=True)
time_i = datetime.datetime.now().replace(microsecond=0)
return roadsDiss, {
0: ('count_rows_roads', time_b - time_a),
1: ('import_roads', time_c - time_b),
2: ('count_rows_build', time_d - time_c),
3: None if not time_e else ('import_builds', time_e - time_d),
4: None if not time_f else ('near_analysis', time_f - time_e),
5: None if not time_g else ('update_buffer_tbl', time_g - time_f),
6: ('buffer_roads', time_h - time_g if time_g else time_h - time_d),
7: ('diss_roads', time_i - time_h)
}
def vector_based(osmdata, nomenclature, refRaster, lulcShp,
overwrite=None, dataStore=None, RoadsAPI='POSTGIS'):
"""
Convert OSM Data into Land Use/Land Cover Information
An vector based approach.
TODO: Add a detailed description.
RoadsAPI Options:
* GRASS
* SQLITE
* POSTGIS
"""
# ************************************************************************ #
# Python Modules from Reference Packages #
# ************************************************************************ #
import datetime; import os; import copy
# ************************************************************************ #
# glass dependencies #
# ************************************************************************ #
from glass.pys.oss import fprop, mkdir
from glass.g.wenv.grs import run_grass
if RoadsAPI == 'POSTGIS':
from glass.ng.sql.db import create_db
from glass.g.it.db import osm_to_psql
from glass.ng.sql.db import drop_db
from glass.ng.sql.bkup import dump_db
else:
from glass.g.it.osm import osm_to_sqdb
from glass.ete.osm2lulc.utils import osm_project, add_lulc_to_osmfeat, get_ref_raster
from glass.g.dp.mge import shps_to_shp
from glass.ete.osm2lulc.mod1 import grs_vector
if RoadsAPI == 'SQLITE' or RoadsAPI == 'POSTGIS':
from glass.ete.osm2lulc.mod2 import roads_sqdb
else:
from glass.ete.osm2lulc.mod2 import grs_vec_roads
from glass.ete.osm2lulc.m3_4 import grs_vect_selbyarea
from glass.ete.osm2lulc.mod5 import grs_vect_bbuffer
from glass.ete.osm2lulc.mod6 import vector_assign_pntags_to_build
from glass.g.dp.mge import same_attr_to_shp
from glass.g.prj import def_prj
# ************************************************************************ #
# Global Settings #
# ************************************************************************ #
# Check if input parameters exists!
if not os.path.exists(os.path.dirname(lulcShp)):
raise ValueError('{} does not exist!'.format(os.path.dirname(lulcShp)))
if not os.path.exists(osmdata):
raise ValueError('File with OSM DATA ({}) does not exist!'.format(osmdata))
if not os.path.exists(refRaster):
raise ValueError('File with reference area ({}) does not exist!'.format(refRaster))
# Check if Nomenclature is valid
nomenclature = "URBAN_ATLAS" if nomenclature != "URBAN_ATLAS" and \
nomenclature != "CORINE_LAND_COVER" and \
nomenclature == "GLOBE_LAND_30" else nomenclature
time_a = datetime.datetime.now().replace(microsecond=0)
# Create workspace for temporary files
workspace = os.path.join(os.path.dirname(
lulcShp), 'osmtolulc') if not dataStore else dataStore
# Check if workspace exists
if os.path.exists(workspace):
if overwrite:
mkdir(workspace)
else:
raise ValueError('Path {} already exists'.format(workspace))
else:
mkdir(workspace)
# Get Reference Raster
refRaster, epsg = get_ref_raster(refRaster, workspace, cellsize=10)
from glass.ete.osm2lulc import osmTableData, PRIORITIES, LEGEND
__priorities = PRIORITIES[nomenclature]
__legend = LEGEND[nomenclature]
time_b = datetime.datetime.now().replace(microsecond=0)
if RoadsAPI != 'POSTGIS':
# ******************************************************************** #
# Convert OSM file to SQLITE DB #
# ******************************************************************** #
osm_db = osm_to_sqdb(osmdata, os.path.join(workspace, 'osm.sqlite'))
else:
# Convert OSM file to POSTGRESQL DB #
osm_db = create_db(fprop(
osmdata, 'fn', forceLower=True), overwrite=True)
osm_db = osm_to_psql(osmdata, osm_db)
time_c = datetime.datetime.now().replace(microsecond=0)
# ************************************************************************ #
# Add Lulc Classes to OSM_FEATURES by rule #
# ************************************************************************ #
add_lulc_to_osmfeat(
osm_db, osmTableData, nomenclature,
api='SQLITE' if RoadsAPI != 'POSTGIS' else RoadsAPI
)
time_d = datetime.datetime.now().replace(microsecond=0)
# ************************************************************************ #
# Transform SRS of OSM Data #
# ************************************************************************ #
osmTableData = osm_project(
osm_db, epsg,
api='SQLITE' if RoadsAPI != 'POSTGIS' else RoadsAPI,
isGlobeLand=None if nomenclature != 'GLOBE_LAND_30' else True
)
time_e = datetime.datetime.now().replace(microsecond=0)
# ************************************************************************ #
# Start a GRASS GIS Session #
# ************************************************************************ #
grass_base = run_grass(workspace, grassBIN='grass78', location='grloc', srs=epsg)
#import grass.script as grass
import grass.script.setup as gsetup
gsetup.init(grass_base, workspace, 'grloc', 'PERMANENT')
# ************************************************************************ #
# IMPORT SOME glass MODULES FOR GRASS GIS #
# ************************************************************************ #
from glass.g.gp.ovl import erase
from glass.g.wenv.grs import rst_to_region
from glass.g.gp.gen import dissolve
from glass.g.tbl.grs import add_and_update, reset_table, update_table
from glass.g.tbl.col import add_fields
from glass.g.it.shp import shp_to_grs, grs_to_shp
from glass.g.it.rst import rst_to_grs
# ************************************************************************ #
# SET GRASS GIS LOCATION EXTENT #
# ************************************************************************ #
extRst = rst_to_grs(refRaster, 'extent_raster')
rst_to_region(extRst)
time_f = datetime.datetime.now().replace(microsecond=0)
# ************************************************************************ #
# MapResults #
# ************************************************************************ #
osmShps = []
# ************************************************************************ #
# 1 - Selection Rule #
# ************************************************************************ #
ruleOneShp, timeCheck1 = grs_vector(
osm_db, osmTableData['polygons'], apidb=RoadsAPI
)
osmShps.append(ruleOneShp)
time_g = datetime.datetime.now().replace(microsecond=0)
# ************************************************************************ #
# 2 - Get Information About Roads Location #
# ************************************************************************ #
ruleRowShp, timeCheck2 = roads_sqdb(
osm_db, osmTableData['lines'], osmTableData['polygons'], apidb=RoadsAPI
) if RoadsAPI == 'SQLITE' or RoadsAPI == 'POSTGIS' else grs_vec_roads(
osm_db, osmTableData['lines'], osmTableData['polygons'])
osmShps.append(ruleRowShp)
time_h = datetime.datetime.now().replace(microsecond=0)
# ************************************************************************ #
# 3 - Area Upper than #
# ************************************************************************ #
if nomenclature != "GLOBE_LAND_30":
ruleThreeShp, timeCheck3 = grs_vect_selbyarea(
osm_db, osmTableData['polygons'], UPPER=True, apidb=RoadsAPI
)
osmShps.append(ruleThreeShp)
time_l = datetime.datetime.now().replace(microsecond=0)
else:
timeCheck3 = None
time_l = None
# ************************************************************************ #
# 4 - Area Lower than #
# ************************************************************************ #
if nomenclature != "GLOBE_LAND_30":
ruleFourShp, timeCheck4 = grs_vect_selbyarea(
osm_db, osmTableData['polygons'], UPPER=False, apidb=RoadsAPI
)
osmShps.append(ruleFourShp)
time_j = datetime.datetime.now().replace(microsecond=0)
else:
timeCheck4 = None
time_j = None
# ************************************************************************ #
# 5 - Get data from lines table (railway | waterway) #
# ************************************************************************ #
ruleFiveShp, timeCheck5 = grs_vect_bbuffer(
osm_db, osmTableData["lines"], api_db=RoadsAPI
)
osmShps.append(ruleFiveShp)
time_m = datetime.datetime.now().replace(microsecond=0)
# ************************************************************************ #
# 7 - Assign untagged Buildings to tags #
# ************************************************************************ #
if nomenclature != "GLOBE_LAND_30":
ruleSeven11, ruleSeven12, timeCheck7 = vector_assign_pntags_to_build(
osm_db, osmTableData['points'], osmTableData['polygons'],
apidb=RoadsAPI
)
if ruleSeven11:
osmShps.append(ruleSeven11)
if ruleSeven12:
osmShps.append(ruleSeven12)
time_n = datetime.datetime.now().replace(microsecond=0)
else:
timeCheck7 = None
time_n = datetime.datetime.now().replace(microsecond=0)
# ************************************************************************ #
# Produce LULC Map #
# ************************************************************************ #
"""
Get Shps with all geometries related with one class - One Shape for Classe
"""
_osmShps = []
for i in range(len(osmShps)):
if not osmShps[i]: continue
_osmShps.append(grs_to_shp(
osmShps[i], os.path.join(workspace, osmShps[i] + '.shp'),
'auto', lyrN=1, asCMD=True, asMultiPart=None
))
for shp in _osmShps:
def_prj(os.path.splitext(shp)[0] + '.prj', epsg=epsg, api='epsgio')
_osmShps = same_attr_to_shp(
_osmShps, "cat", workspace, "osm_", resultDict=True
)
del osmShps
time_o = datetime.datetime.now().replace(microsecond=0)
"""
Merge all Classes into one feature class using a priority rule
"""
osmShps = {}
for cls in _osmShps:
if cls == '1':
osmShps[1221] = shp_to_grs(_osmShps[cls], "osm_1221", asCMD=True)
else:
osmShps[int(cls)] = shp_to_grs(_osmShps[cls], "osm_" + cls,
asCMD=True)
# Erase overlapping areas by priority
osmNameRef = copy.deepcopy(osmShps)
for e in range(len(__priorities)):
if e + 1 == len(__priorities): break
if __priorities[e] not in osmShps:
continue
else:
for i in range(e+1, len(__priorities)):
if __priorities[i] not in osmShps:
continue
else:
osmShps[__priorities[i]] = erase(
osmShps[__priorities[i]], osmShps[__priorities[e]],
"{}_{}".format(osmNameRef[__priorities[i]], e),
notTbl=True, api='pygrass'
)
time_p = datetime.datetime.now().replace(microsecond=0)
# Export all classes
lst_merge = []
a = None
for i in range(len(__priorities)):
if __priorities[i] not in osmShps:
continue
if not a:
reset_table(
osmShps[__priorities[i]],
{'cls' : 'varchar(5)', 'leg' : 'varchar(75)'},
{'cls' : str(__priorities[i]), 'leg' : str(__legend[__priorities[i]])}
)
a = 1
else:
add_and_update(
osmShps[__priorities[i]],
{'cls' : 'varchar(5)'},
{'cls' : str(__priorities[i])}
)
ds = dissolve(
osmShps[__priorities[i]],
'dl_{}'.format(str(__priorities[i])), 'cls', api="grass"
)
add_fields(ds, {'leg': 'varchar(75)'}, api="grass")
update_table(ds, 'leg', str(__legend[__priorities[i]]), 'leg is null')
lst_merge.append(grs_to_shp(
ds, os.path.join(
workspace, "lulc_{}.shp".format(str(__priorities[i]))
), 'auto', lyrN=1, asCMD=True, asMultiPart=None
))
time_q = datetime.datetime.now().replace(microsecond=0)
if fprop(lulcShp, 'ff') != '.shp':
lulcShp = os.path.join(
os.path.dirname(lulcShp), fprop(lulcShp, 'fn') + '.shp')
shps_to_shp(lst_merge, lulcShp, api='pandas')
# Check if prj of lulcShp exists and create it if necessary
prj_ff = os.path.splitext(lulcShp)[0] + '.prj'
if not os.path.exists(prj_ff):
def_prj(prj_ff, epsg=epsg, api='epsgio')
time_r = datetime.datetime.now().replace(microsecond=0)
# Dump Database if PostGIS was used
# Drop Database if PostGIS was used
if RoadsAPI == 'POSTGIS':
dump_db(osm_db, os.path.join(
workspace, osm_db + '.sql'
), api='psql')
drop_db(osm_db)
return lulcShp, {
0 : ('set_settings', time_b - time_a),
1 : ('osm_to_sqdb', time_c - time_b),
2 : ('cls_in_sqdb', time_d - time_c),
3 : ('proj_data', time_e - time_d),
4 : ('set_grass', time_f - time_e),
5 : ('rule_1', time_g - time_f, timeCheck1),
6 : ('rule_2', time_h - time_g, timeCheck2),
7 : None if not timeCheck3 else ('rule_3', time_l - time_h, timeCheck3),
8 : None if not timeCheck4 else ('rule_4', time_j - time_l, timeCheck4),
9 : ('rule_5',
time_m - time_j if timeCheck4 else time_m - time_h, timeCheck5),
10 : None if not timeCheck7 else ('rule_7', time_n - time_m, timeCheck7),
11 : ('disj_cls', time_o - time_n),
12 : ('priority_rule', time_p - time_o),
13 : ('export_cls', time_q - time_p),
14 : ('merge_cls', time_r - time_q)
}
def lulc_by_cell(tid, boundary, lulc_shps, fishnet, result, workspace):
from glass.g.wenv.grs import run_grass
from glass.g.dp.torst import shp_to_rst
bname = fprop(boundary, 'fn')
# Boundary to Raster
ref_rst = shp_to_rst(boundary, None, 10, 0,
os.path.join(workspace, f'rst_{bname}.tif'))
# Create GRASS GIS Session
loc_name = 'loc_' + bname
gbase = run_grass(workspace, location=loc_name, srs=ref_rst)
import grass.script.setup as gsetup
gsetup.init(gbase, workspace, loc_name, 'PERMANENT')
# GRASS GIS Modules
from glass.g.it.shp import shp_to_grs, grs_to_shp
from glass.g.gp.ovl import grsintersection
from glass.g.tbl.attr import geomattr_to_db
from glass.g.prop.feat import feat_count
# Send Fishnet to GRASS GIS
fnet = shp_to_grs(fishnet, fprop(fishnet, 'fn'), asCMD=True)
# Processing
ulst = []
l_lulc_grs = []
for shp in lulc_shps:
iname = fprop(shp, 'fn')
# LULC Class to GRASS GIS
lulc_grs = shp_to_grs(shp, iname, filterByReg=True, asCMD=True)
if not feat_count(
lulc_grs, gisApi='grass', work=workspace, loc=loc_name):
continue
# Intersect Fishnet | LULC CLass
union_grs = grsintersection(fnet, lulc_grs, iname + '_i', cmd=True)
# Get Areas
geomattr_to_db(union_grs, "areav", "area", "boundary", unit='meters')
# Export Table
funion = grs_to_shp(union_grs, os.path.join(result, iname + '.shp'),
'area')
ulst.append(funion)
l_lulc_grs.append(lulc_grs)
# Intersect between all LULC SHPS
ist_shp = []
if len(l_lulc_grs) > 1:
for i in range(len(l_lulc_grs)):
for e in range(i + 1, len(l_lulc_grs)):
ishp = grsintersection(l_lulc_grs[i],
l_lulc_grs[e],
'lulcint_' + str(i) + '_' + str(e),
cmd=True)
if not feat_count(
ishp, gisApi='grass', work=workspace, loc=loc_name):
continue
else:
ist_shp.append(ishp)
if len(ist_shp):
from glass.g.gp.gen import dissolve
from glass.g.tbl.grs import reset_table
if len(ist_shp) > 1:
from glass.g.dp.mge import shps_to_shp
# Export shapes
_ist_shp = [
grs_to_shp(s, os.path.join(workspace, loc_name,
s + '.shp'), 'area')
for s in ist_shp
]
# Merge Intersections
merge_shp = shps_to_shp(_ist_shp,
os.path.join(workspace, loc_name,
'merge_shp.shp'),
api='pandas')
# Import GRASS
merge_shp = shp_to_grs(merge_shp, 'merge_shp')
else:
merge_shp = ist_shp[0]
# Dissolve Shape
reset_table(merge_shp, {'refid': 'varchar(2)'}, {'refid': '1'})
overlay_areas = dissolve(merge_shp,
'overlay_areas',
'refid',
api='grass')
# Union Fishnet | Overlay's
union_ovl = grsintersection(fnet,
overlay_areas,
'ovl_union',
cmd=True)
funion_ovl = grs_to_shp(union_ovl,
os.path.join(result, union_ovl + '.shp'),
'area')
ulst.append(funion_ovl)
# Export Tables
return ulst