def fields_to_tbls(inFolder, fields, tbl_format='.shp'):
"""
Add fields to several tables in a folder
"""
from gasp.pyt.oss import lst_ff
tables = lst_ff(inFolder, file_format=tbl_format)
for table in tables:
add_fields(table, fields, api='ogr')
def filename_to_col(tables, new_field, table_format='.dbf'):
"""
Update a table with the filename in a new field
"""
import os
from gasp.pyt.oss import lst_ff
from gasp.gt.tbl.fld import add_fields
if os.path.isdir(tables):
__tables = lst_ff(tables, file_format=table_format)
else:
__tables = [tables]
for table in __tables:
add_fields(table, {new_field: 'varchar(50)'})
name_tbl = os.path.splitext(os.path.basename(table))[0]
name_tbl = name_tbl.lower() if name_tbl.isupper() else name_tbl
update_cols(table, {new_field: name_tbl})
def grs_near(fromShp,
toShp,
nearCatCol='tocat',
nearDistCol="todistance",
maxDist=-1,
as_cmd=None):
"""
v.distance - Finds the nearest element in vector map 'to'
for elements in vector map 'from'.
"""
from gasp.gt.tbl.fld import add_fields
add_fields(fromShp, {
nearCatCol: 'INTEGER',
nearDistCol: 'DOUBLE PRECISION'
},
api="grass" if as_cmd else "pygrass")
if not as_cmd:
import grass.script as grass
grass.run_command("v.distance",
_from=fromShp,
to=toShp,
upload='cat,dist',
column='{},{}'.format(nearCatCol, nearDistCol),
dmax=maxDist)
else:
from gasp import exec_cmd
rcmd = exec_cmd(
("v.distance from={} to={} upload=cat,dist "
"column={},{} dmax={}").format(fromShp, toShp, nearCatCol,
nearDistCol, maxDist))
def vector_based(osmdata,
nomenclature,
refRaster,
lulcShp,
overwrite=None,
dataStore=None,
RoadsAPI='POSTGIS'):
"""
Convert OSM Data into Land Use/Land Cover Information
An vector based approach.
TODO: Add a detailed description.
RoadsAPI Options:
* GRASS
* SQLITE
* POSTGIS
"""
# ************************************************************************ #
# Python Modules from Reference Packages #
# ************************************************************************ #
import datetime
import os
import copy
# ************************************************************************ #
# GASP dependencies #
# ************************************************************************ #
from gasp.pyt.oss import fprop, mkdir
from gasp.gt.wenv.grs import run_grass
if RoadsAPI == 'POSTGIS':
from gasp.sql.db import create_db
from gasp.gql.to.osm import osm_to_psql
from gasp.sql.db import drop_db
from gasp.sql.fm import dump_db
else:
from gasp.gt.toshp.osm import osm_to_sqdb
from gasp.sds.osm2lulc.utils import osm_project, add_lulc_to_osmfeat, get_ref_raster
from gasp.gt.toshp.mtos import shps_to_shp
from gasp.sds.osm2lulc.mod1 import grs_vector
if RoadsAPI == 'SQLITE' or RoadsAPI == 'POSTGIS':
from gasp.sds.osm2lulc.mod2 import roads_sqdb
else:
from gasp.sds.osm2lulc.mod2 import grs_vec_roads
from gasp.sds.osm2lulc.m3_4 import grs_vect_selbyarea
from gasp.sds.osm2lulc.mod5 import grs_vect_bbuffer
from gasp.sds.osm2lulc.mod6 import vector_assign_pntags_to_build
from gasp.gt.toshp.mtos import same_attr_to_shp
from gasp.gt.prj import def_prj
# ************************************************************************ #
# Global Settings #
# ************************************************************************ #
# Check if input parameters exists!
if not os.path.exists(os.path.dirname(lulcShp)):
raise ValueError('{} does not exist!'.format(os.path.dirname(lulcShp)))
if not os.path.exists(osmdata):
raise ValueError(
'File with OSM DATA ({}) does not exist!'.format(osmdata))
if not os.path.exists(refRaster):
raise ValueError(
'File with reference area ({}) does not exist!'.format(refRaster))
# Check if Nomenclature is valid
nomenclature = "URBAN_ATLAS" if nomenclature != "URBAN_ATLAS" and \
nomenclature != "CORINE_LAND_COVER" and \
nomenclature == "GLOBE_LAND_30" else nomenclature
time_a = datetime.datetime.now().replace(microsecond=0)
# Create workspace for temporary files
workspace = os.path.join(os.path.dirname(lulcShp),
'osmtolulc') if not dataStore else dataStore
# Check if workspace exists
if os.path.exists(workspace):
if overwrite:
mkdir(workspace)
else:
raise ValueError('Path {} already exists'.format(workspace))
else:
mkdir(workspace)
# Get Reference Raster
refRaster, epsg = get_ref_raster(refRaster, workspace, cellsize=10)
from gasp.sds.osm2lulc import osmTableData, PRIORITIES, LEGEND
__priorities = PRIORITIES[nomenclature]
__legend = LEGEND[nomenclature]
time_b = datetime.datetime.now().replace(microsecond=0)
if RoadsAPI != 'POSTGIS':
# ******************************************************************** #
# Convert OSM file to SQLITE DB #
# ******************************************************************** #
osm_db = osm_to_sqdb(osmdata, os.path.join(workspace, 'osm.sqlite'))
else:
# Convert OSM file to POSTGRESQL DB #
osm_db = create_db(fprop(osmdata, 'fn', forceLower=True),
overwrite=True)
osm_db = osm_to_psql(osmdata, osm_db)
time_c = datetime.datetime.now().replace(microsecond=0)
# ************************************************************************ #
# Add Lulc Classes to OSM_FEATURES by rule #
# ************************************************************************ #
add_lulc_to_osmfeat(osm_db,
osmTableData,
nomenclature,
api='SQLITE' if RoadsAPI != 'POSTGIS' else RoadsAPI)
time_d = datetime.datetime.now().replace(microsecond=0)
# ************************************************************************ #
# Transform SRS of OSM Data #
# ************************************************************************ #
osmTableData = osm_project(
osm_db,
epsg,
api='SQLITE' if RoadsAPI != 'POSTGIS' else RoadsAPI,
isGlobeLand=None if nomenclature != 'GLOBE_LAND_30' else True)
time_e = datetime.datetime.now().replace(microsecond=0)
# ************************************************************************ #
# Start a GRASS GIS Session #
# ************************************************************************ #
grass_base = run_grass(workspace,
grassBIN='grass78',
location='grloc',
srs=epsg)
#import grass.script as grass
import grass.script.setup as gsetup
gsetup.init(grass_base, workspace, 'grloc', 'PERMANENT')
# ************************************************************************ #
# IMPORT SOME GASP MODULES FOR GRASS GIS #
# ************************************************************************ #
from gasp.gt.gop.ovlay import erase
from gasp.gt.wenv.grs import rst_to_region
from gasp.gt.gop.genze import dissolve
from gasp.gt.tbl.grs import add_and_update, reset_table, update_table
from gasp.gt.tbl.fld import add_fields
from gasp.gt.toshp.cff import shp_to_grs, grs_to_shp
from gasp.gt.torst import rst_to_grs
# ************************************************************************ #
# SET GRASS GIS LOCATION EXTENT #
# ************************************************************************ #
extRst = rst_to_grs(refRaster, 'extent_raster')
rst_to_region(extRst)
time_f = datetime.datetime.now().replace(microsecond=0)
# ************************************************************************ #
# MapResults #
# ************************************************************************ #
osmShps = []
# ************************************************************************ #
# 1 - Selection Rule #
# ************************************************************************ #
ruleOneShp, timeCheck1 = grs_vector(osm_db,
osmTableData['polygons'],
apidb=RoadsAPI)
osmShps.append(ruleOneShp)
time_g = datetime.datetime.now().replace(microsecond=0)
# ************************************************************************ #
# 2 - Get Information About Roads Location #
# ************************************************************************ #
ruleRowShp, timeCheck2 = roads_sqdb(
osm_db,
osmTableData['lines'],
osmTableData['polygons'],
apidb=RoadsAPI
) if RoadsAPI == 'SQLITE' or RoadsAPI == 'POSTGIS' else grs_vec_roads(
osm_db, osmTableData['lines'], osmTableData['polygons'])
osmShps.append(ruleRowShp)
time_h = datetime.datetime.now().replace(microsecond=0)
# ************************************************************************ #
# 3 - Area Upper than #
# ************************************************************************ #
if nomenclature != "GLOBE_LAND_30":
ruleThreeShp, timeCheck3 = grs_vect_selbyarea(osm_db,
osmTableData['polygons'],
UPPER=True,
apidb=RoadsAPI)
osmShps.append(ruleThreeShp)
time_l = datetime.datetime.now().replace(microsecond=0)
else:
timeCheck3 = None
time_l = None
# ************************************************************************ #
# 4 - Area Lower than #
# ************************************************************************ #
if nomenclature != "GLOBE_LAND_30":
ruleFourShp, timeCheck4 = grs_vect_selbyarea(osm_db,
osmTableData['polygons'],
UPPER=False,
apidb=RoadsAPI)
osmShps.append(ruleFourShp)
time_j = datetime.datetime.now().replace(microsecond=0)
else:
timeCheck4 = None
time_j = None
# ************************************************************************ #
# 5 - Get data from lines table (railway | waterway) #
# ************************************************************************ #
ruleFiveShp, timeCheck5 = grs_vect_bbuffer(osm_db,
osmTableData["lines"],
api_db=RoadsAPI)
osmShps.append(ruleFiveShp)
time_m = datetime.datetime.now().replace(microsecond=0)
# ************************************************************************ #
# 7 - Assign untagged Buildings to tags #
# ************************************************************************ #
if nomenclature != "GLOBE_LAND_30":
ruleSeven11, ruleSeven12, timeCheck7 = vector_assign_pntags_to_build(
osm_db,
osmTableData['points'],
osmTableData['polygons'],
apidb=RoadsAPI)
if ruleSeven11:
osmShps.append(ruleSeven11)
if ruleSeven12:
osmShps.append(ruleSeven12)
time_n = datetime.datetime.now().replace(microsecond=0)
else:
timeCheck7 = None
time_n = datetime.datetime.now().replace(microsecond=0)
# ************************************************************************ #
# Produce LULC Map #
# ************************************************************************ #
"""
Get Shps with all geometries related with one class - One Shape for Classe
"""
_osmShps = []
for i in range(len(osmShps)):
if not osmShps[i]: continue
_osmShps.append(
grs_to_shp(osmShps[i],
os.path.join(workspace, osmShps[i] + '.shp'),
'auto',
lyrN=1,
asCMD=True,
asMultiPart=None))
for shp in _osmShps:
def_prj(os.path.splitext(shp)[0] + '.prj', epsg=epsg, api='epsgio')
_osmShps = same_attr_to_shp(_osmShps,
"cat",
workspace,
"osm_",
resultDict=True)
del osmShps
time_o = datetime.datetime.now().replace(microsecond=0)
"""
Merge all Classes into one feature class using a priority rule
"""
osmShps = {}
for cls in _osmShps:
if cls == '1':
osmShps[1221] = shp_to_grs(_osmShps[cls], "osm_1221", asCMD=True)
else:
osmShps[int(cls)] = shp_to_grs(_osmShps[cls],
"osm_" + cls,
asCMD=True)
# Erase overlapping areas by priority
osmNameRef = copy.deepcopy(osmShps)
for e in range(len(__priorities)):
if e + 1 == len(__priorities): break
if __priorities[e] not in osmShps:
continue
else:
for i in range(e + 1, len(__priorities)):
if __priorities[i] not in osmShps:
continue
else:
osmShps[__priorities[i]] = erase(
osmShps[__priorities[i]],
osmShps[__priorities[e]],
"{}_{}".format(osmNameRef[__priorities[i]], e),
notTbl=True,
api='pygrass')
time_p = datetime.datetime.now().replace(microsecond=0)
# Export all classes
lst_merge = []
a = None
for i in range(len(__priorities)):
if __priorities[i] not in osmShps:
continue
if not a:
reset_table(osmShps[__priorities[i]], {
'cls': 'varchar(5)',
'leg': 'varchar(75)'
}, {
'cls': str(__priorities[i]),
'leg': str(__legend[__priorities[i]])
})
a = 1
else:
add_and_update(osmShps[__priorities[i]], {'cls': 'varchar(5)'},
{'cls': str(__priorities[i])})
ds = dissolve(osmShps[__priorities[i]],
'dl_{}'.format(str(__priorities[i])),
'cls',
api="grass")
add_fields(ds, {'leg': 'varchar(75)'}, api="grass")
update_table(ds, 'leg', str(__legend[__priorities[i]]), 'leg is null')
lst_merge.append(
grs_to_shp(ds,
os.path.join(workspace, "lulc_{}.shp".format(
str(__priorities[i]))),
'auto',
lyrN=1,
asCMD=True,
asMultiPart=None))
time_q = datetime.datetime.now().replace(microsecond=0)
if fprop(lulcShp, 'ff') != '.shp':
lulcShp = os.path.join(os.path.dirname(lulcShp),
fprop(lulcShp, 'fn') + '.shp')
shps_to_shp(lst_merge, lulcShp, api='pandas')
# Check if prj of lulcShp exists and create it if necessary
prj_ff = os.path.splitext(lulcShp)[0] + '.prj'
if not os.path.exists(prj_ff):
def_prj(prj_ff, epsg=epsg, api='epsgio')
time_r = datetime.datetime.now().replace(microsecond=0)
# Dump Database if PostGIS was used
# Drop Database if PostGIS was used
if RoadsAPI == 'POSTGIS':
dump_db(osm_db, os.path.join(workspace, osm_db + '.sql'), api='psql')
drop_db(osm_db)
return lulcShp, {
0: ('set_settings', time_b - time_a),
1: ('osm_to_sqdb', time_c - time_b),
2: ('cls_in_sqdb', time_d - time_c),
3: ('proj_data', time_e - time_d),
4: ('set_grass', time_f - time_e),
5: ('rule_1', time_g - time_f, timeCheck1),
6: ('rule_2', time_h - time_g, timeCheck2),
7:
None if not timeCheck3 else ('rule_3', time_l - time_h, timeCheck3),
8:
None if not timeCheck4 else ('rule_4', time_j - time_l, timeCheck4),
9: ('rule_5', time_m - time_j if timeCheck4 else time_m - time_h,
timeCheck5),
10:
None if not timeCheck7 else ('rule_7', time_n - time_m, timeCheck7),
11: ('disj_cls', time_o - time_n),
12: ('priority_rule', time_p - time_o),
13: ('export_cls', time_q - time_p),
14: ('merge_cls', time_r - time_q)
}
def geomattr_to_db(shp, attrCol, attr, geomType, createCol=True,
unit=None, lyrN=1, ascmd=None):
"""
v.to.db - Populates attribute values from vector features.
v.to.db loads vector map features or metrics into a database table,
or prints them (or the SQL queries used to obtain them) in a form
of a human-readable report. For uploaded/printed category values '-1'
is used for 'no category' and 'null'/'-' if category cannot be
found or multiple categories were found. For line azimuths '-1' is used
for closed lines (start equals end).
attrs options area:
* cat: insert new row for each category if doesn't exist yet
* area: area size
* compact: compactness of an area, calculated as
compactness = perimeter / (2 * sqrt(PI * area))
* fd: fractal dimension of boundary defining a polygon, calculated as
fd = 2 * (log(perimeter) / log(area))
* perimeter: perimeter length of an area
* length: line length
* count: number of features for each category
* coor: point coordinates, X,Y or X,Y,Z
* start: line/boundary starting point coordinates, X,Y or X,Y,Z
* end: line/boundary end point coordinates, X,Y or X,Y,Z
* sides: categories of areas on the left and right side of the boundary,
'query_layer' is used for area category
* query: result of a database query for all records of the geometry(or
geometries) from table specified by 'query_layer' option
* slope: slope steepness of vector line or boundary
* sinuous: line sinuousity, calculated as line length / distance between
end points
* azimuth: line azimuth, calculated as angle between North direction and
endnode direction at startnode
* bbox: bounding box of area, N,S,E,W
"""
from gasp.pyt import obj_to_lst
attrCol = obj_to_lst(attrCol)
if createCol:
from gasp.gt.tbl.fld import add_fields
for c in attrCol:
add_fields(
shp, {c : "DOUBLE PRECISION"},
api="grass" if ascmd else "pygrass"
)
if not ascmd:
from grass.pygrass.modules import Module
vtodb = Module(
"v.to.db", map=shp, type=geomType, layer=lyrN, option=attr,
columns=",".join(attrCol) if attr != 'length' else attrCol[0],
units=unit, run_=False, quiet=True
)
vtodb()
else:
from gasp import exec_cmd
rcmd = exec_cmd((
"v.to.db map={} type={} layer={} option={} "
"columns={} units={} --quiet"
).format(
shp, geomType, lyrN, attr,
",".join(attrCol) if attr != 'length' else attrCol[0],
unit
))