def ob_ref_rst(ref, folder, cellsize=None):
"""
Get Reference Raster
"""
from gasp.gt.prop.ff import check_isRaster
# Check if refRaster is really a Raster
isRst = check_isRaster(ref)
if not isRst:
from gasp.gt.prop.ff import check_isShp
if not check_isShp(ref):
raise ValueError(
('Extent Template File has an invalid file format. '
'Please give a file with one of the following extensions: '
'shp, gml, json, kml, tif or img'))
else:
# We have a shapefile
# Convert it to Raster
from gasp.gt.torst import shp_to_rst
ref_rst = shp_to_rst(ref,
None,
10 if not cellsize else cellsize,
-1,
os.path.join(folder, 'ref_raster.tif'),
api='gdal')
return ref_rst
else:
return ref
def get_epsg(inFile):
"""
Get EPSG of any GIS File
"""
from gasp.gt.prop.ff import check_isRaster, check_isShp
if check_isRaster(inFile):
return get_rst_epsg(inFile)
else:
if check_isShp(inFile):
return get_epsg_shp(inFile)
else:
return None
def get_ext(inFile, outEpsg=None):
"""
Get Extent of any GIS Data
return None if inFile is not a GIS File
"""
from gasp.gt.prop.ff import check_isRaster, check_isShp
if check_isRaster(inFile):
from gasp.gt.prop.rst import rst_ext
extent = rst_ext(inFile)
else:
if check_isShp(inFile):
from gasp.gt.prop.feat import get_ext as gext
extent = gext(inFile)
else:
return None
if outEpsg:
from gasp.gt.prop.prj import get_epsg
fileEpsg = get_epsg(inFile)
if not fileEpsg:
raise ValueError('cannot get EPSG of input file')
if fileEpsg != outEpsg:
from gasp.g.to import new_pnt
from gasp.g.prj import prj_ogrgeom
bt_left = prj_ogrgeom(new_pnt(extent[0], extent[2]), fileEpsg,
outEpsg)
top_right = prj_ogrgeom(new_pnt(extent[1], extent[3]), fileEpsg,
outEpsg)
left, bottom = bt_left.GetX(), bt_left.GetY()
right, top = top_right.GetX(), top_right.GetY()
extent = [left, right, bottom, top]
return extent
def fext_to_geof(inF, outF, ocellsize=10):
"""
Extent of a File to Raster or Shapefile
"""
from gasp.gt.prop.ext import get_ext
from gasp.gt.prop.ff import check_isRaster
from gasp.gt.prop.prj import get_epsg
# Get extent
left, right, bottom, top = get_ext(inF)
# Get EPSG of inF
EPSG = get_epsg(inF)
# Export Boundary
isRst = check_isRaster(outF)
if isRst:
from gasp.gt.torst import ext_to_rst
return ext_to_rst((left, top), (right, bottom),
outF,
cellsize=ocellsize,
epsg=EPSG,
invalidResultAsNull=None)
else:
from gasp.gt.prop.ff import check_isShp
isShp = check_isShp(outF)
if isShp:
from gasp.gt.toshp.coord import coords_to_boundshp
return coords_to_boundshp((left, top), (right, bottom), EPSG, outF)
else:
raise ValueError(
'{} is not recognized as a file with GeoData'.format(inF))
def osm2lulc(osmdata, nomenclature, refRaster, lulcRst,
overwrite=None, dataStore=None, roadsAPI='POSTGIS'):
"""
Convert OSM data into Land Use/Land Cover Information
A matrix based approach
roadsAPI Options:
* SQLITE
* POSTGIS
"""
# ************************************************************************ #
# Python Modules from Reference Packages #
# ************************************************************************ #
import os; import numpy; import datetime
from threading import Thread
from osgeo import gdal
# ************************************************************************ #
# Dependencies #
# ************************************************************************ #
from gasp.gt.fmrst import rst_to_array
from gasp.gt.prop.ff import check_isRaster
from gasp.gt.prop.rst import get_cellsize
from gasp.gt.prop.prj import get_rst_epsg
from gasp.pyt.oss import mkdir, copy_file
from gasp.pyt.oss import fprop
if roadsAPI == 'POSTGIS':
from gasp.sql.db import create_db
from gasp.gql.to.osm import osm_to_psql
from gasp.sds.osm2lulc.mod2 import pg_num_roads
from gasp.sql.fm import dump_db
from gasp.sql.db import drop_db
else:
from gasp.gt.toshp.osm import osm_to_sqdb
from gasp.sds.osm2lulc.mod2 import num_roads
from gasp.sds.osm2lulc.utils import osm_project, add_lulc_to_osmfeat
from gasp.sds.osm2lulc.utils import osmlulc_rsttbl
from gasp.sds.osm2lulc.utils import get_ref_raster
from gasp.sds.osm2lulc.mod1 import num_selection
from gasp.sds.osm2lulc.m3_4 import num_selbyarea
from gasp.sds.osm2lulc.mod5 import num_base_buffer
from gasp.sds.osm2lulc.mod6 import num_assign_builds
from gasp.gt.torst import obj_to_rst
# ************************************************************************ #
# Global Settings #
# ************************************************************************ #
# Check if input parameters exists!
if not os.path.exists(os.path.dirname(lulcRst)):
raise ValueError('{} does not exist!'.format(os.path.dirname(lulcRst)))
if not os.path.exists(osmdata):
raise ValueError('File with OSM DATA ({}) does not exist!'.format(osmdata))
if not os.path.exists(refRaster):
raise ValueError('File with reference area ({}) does not exist!'.format(refRaster))
# Check if Nomenclature is valid
nomenclature = "URBAN_ATLAS" if nomenclature != "URBAN_ATLAS" and \
nomenclature != "CORINE_LAND_COVER" and \
nomenclature == "GLOBE_LAND_30" else nomenclature
time_a = datetime.datetime.now().replace(microsecond=0)
workspace = os.path.join(os.path.dirname(
lulcRst), 'num_osmto') if not dataStore else dataStore
# Check if workspace exists:
if os.path.exists(workspace):
if overwrite:
mkdir(workspace, overwrite=True)
else:
raise ValueError('Path {} already exists'.format(workspace))
else:
mkdir(workspace, overwrite=None)
# Get Ref Raster and EPSG
refRaster, epsg = get_ref_raster(refRaster, workspace, cellsize=2)
CELLSIZE = get_cellsize(refRaster, gisApi='gdal')
from gasp.sds.osm2lulc import osmTableData, PRIORITIES
time_b = datetime.datetime.now().replace(microsecond=0)
# ************************************************************************ #
# Convert OSM file to SQLITE DB or to POSTGIS DB #
# ************************************************************************ #
if roadsAPI == 'POSTGIS':
osm_db = create_db(fprop(
osmdata, 'fn', forceLower=True), overwrite=True)
osm_db = osm_to_psql(osmdata, osm_db)
else:
osm_db = osm_to_sqdb(osmdata, os.path.join(workspace, 'osm.sqlite'))
time_c = datetime.datetime.now().replace(microsecond=0)
# ************************************************************************ #
# Add Lulc Classes to OSM_FEATURES by rule #
# ************************************************************************ #
add_lulc_to_osmfeat(osm_db, osmTableData, nomenclature, api=roadsAPI)
time_d = datetime.datetime.now().replace(microsecond=0)
# ************************************************************************ #
# Transform SRS of OSM Data #
# ************************************************************************ #
osmTableData = osm_project(
osm_db, epsg, api=roadsAPI,
isGlobeLand=None if nomenclature != "GLOBE_LAND_30" else True
)
time_e = datetime.datetime.now().replace(microsecond=0)
# ************************************************************************ #
# MapResults #
# ************************************************************************ #
mergeOut = {}
timeCheck = {}
RULES = [1, 2, 3, 4, 5, 7]
def run_rule(ruleID):
time_start = datetime.datetime.now().replace(microsecond=0)
_osmdb = copy_file(
osm_db, os.path.splitext(osm_db)[0] + '_r{}.sqlite'.format(ruleID)
) if roadsAPI == 'SQLITE' else None
# ******************************************************************** #
# 1 - Selection Rule #
# ******************************************************************** #
if ruleID == 1:
res, tm = num_selection(
_osmdb if _osmdb else osm_db, osmTableData['polygons'], workspace,
CELLSIZE, epsg, refRaster, api=roadsAPI
)
# ******************************************************************** #
# 2 - Get Information About Roads Location #
# ******************************************************************** #
elif ruleID == 2:
res, tm = num_roads(
_osmdb, nomenclature, osmTableData['lines'],
osmTableData['polygons'], workspace, CELLSIZE, epsg,
refRaster
) if _osmdb else pg_num_roads(
osm_db, nomenclature,
osmTableData['lines'], osmTableData['polygons'],
workspace, CELLSIZE, epsg, refRaster
)
# ******************************************************************** #
# 3 - Area Upper than #
# ******************************************************************** #
elif ruleID == 3:
if nomenclature != "GLOBE_LAND_30":
res, tm = num_selbyarea(
osm_db if not _osmdb else _osmdb,
osmTableData['polygons'], workspace,
CELLSIZE, epsg, refRaster, UPPER=True, api=roadsAPI
)
else:
return
# ******************************************************************** #
# 4 - Area Lower than #
# ******************************************************************** #
elif ruleID == 4:
if nomenclature != "GLOBE_LAND_30":
res, tm = num_selbyarea(
osm_db if not _osmdb else _osmdb,
osmTableData['polygons'], workspace,
CELLSIZE, epsg, refRaster, UPPER=False, api=roadsAPI
)
else:
return
# ******************************************************************** #
# 5 - Get data from lines table (railway | waterway) #
# ******************************************************************** #
elif ruleID == 5:
res, tm = num_base_buffer(
osm_db if not _osmdb else _osmdb,
osmTableData['lines'], workspace,
CELLSIZE, epsg, refRaster, api=roadsAPI
)
# ******************************************************************** #
# 7 - Assign untagged Buildings to tags #
# ******************************************************************** #
elif ruleID == 7:
if nomenclature != "GLOBE_LAND_30":
res, tm = num_assign_builds(
osm_db if not _osmdb else _osmdb,
osmTableData['points'], osmTableData['polygons'],
workspace, CELLSIZE, epsg, refRaster, apidb=roadsAPI
)
else:
return
time_end = datetime.datetime.now().replace(microsecond=0)
mergeOut[ruleID] = res
timeCheck[ruleID] = {'total': time_end - time_start, 'detailed': tm}
thrds = []
for r in RULES:
thrds.append(Thread(
name="to_{}".format(str(r)), target=run_rule,
args=(r,)
))
for t in thrds: t.start()
for t in thrds: t.join()
# Merge all results into one Raster
compileResults = {}
for rule in mergeOut:
for cls in mergeOut[rule]:
if cls not in compileResults:
if type(mergeOut[rule][cls]) == list:
compileResults[cls] = mergeOut[rule][cls]
else:
compileResults[cls] = [mergeOut[rule][cls]]
else:
if type(mergeOut[rule][cls]) == list:
compileResults[cls] += mergeOut[rule][cls]
else:
compileResults[cls].append(mergeOut[rule][cls])
time_m = datetime.datetime.now().replace(microsecond=0)
# All Rasters to Array
arrayRst = {}
for cls in compileResults:
for raster in compileResults[cls]:
if not raster:
continue
array = rst_to_array(raster)
if cls not in arrayRst:
arrayRst[cls] = [array.astype(numpy.uint8)]
else:
arrayRst[cls].append(array.astype(numpy.uint8))
time_n = datetime.datetime.now().replace(microsecond=0)
# Sum Rasters of each class
for cls in arrayRst:
if len(arrayRst[cls]) == 1:
sumArray = arrayRst[cls][0]
else:
sumArray = arrayRst[cls][0]
for i in range(1, len(arrayRst[cls])):
sumArray = sumArray + arrayRst[cls][i]
arrayRst[cls] = sumArray
time_o = datetime.datetime.now().replace(microsecond=0)
# Apply priority rule
__priorities = PRIORITIES[nomenclature + "_NUMPY"]
for lulcCls in __priorities:
__lulcCls = rstcls_map(lulcCls)
if __lulcCls not in arrayRst:
continue
else:
numpy.place(arrayRst[__lulcCls], arrayRst[__lulcCls] > 0,
lulcCls
)
for i in range(len(__priorities)):
lulc_i = rstcls_map(__priorities[i])
if lulc_i not in arrayRst:
continue
else:
for e in range(i+1, len(__priorities)):
lulc_e = rstcls_map(__priorities[e])
if lulc_e not in arrayRst:
continue
else:
numpy.place(arrayRst[lulc_e],
arrayRst[lulc_i] == __priorities[i], 0
)
time_p = datetime.datetime.now().replace(microsecond=0)
# Merge all rasters
startCls = 'None'
for i in range(len(__priorities)):
lulc_i = rstcls_map(__priorities[i])
if lulc_i in arrayRst:
resultSum = arrayRst[lulc_i]
startCls = i
break
if startCls == 'None':
return 'NoResults'
for i in range(startCls + 1, len(__priorities)):
lulc_i = rstcls_map(__priorities[i])
if lulc_i not in arrayRst:
continue
resultSum = resultSum + arrayRst[lulc_i]
# Save Result
outIsRst = check_isRaster(lulcRst)
if not outIsRst:
from gasp.pyt.oss import fprop
lulcRst = os.path.join(
os.path.dirname(lulcRst), fprop(lulcRst, 'fn') + '.tif'
)
numpy.place(resultSum, resultSum==0, 1)
obj_to_rst(resultSum, lulcRst, refRaster, noData=1)
osmlulc_rsttbl(nomenclature + "_NUMPY", os.path.join(
os.path.dirname(lulcRst), os.path.basename(lulcRst) + '.vat.dbf'
))
time_q = datetime.datetime.now().replace(microsecond=0)
# Dump Database if PostGIS was used
# Drop Database if PostGIS was used
if roadsAPI == 'POSTGIS':
dump_db(osm_db, os.path.join(workspace, osm_db + '.sql'), api='psql')
drop_db(osm_db)
return lulcRst, {
0 : ('set_settings', time_b - time_a),
1 : ('osm_to_sqdb', time_c - time_b),
2 : ('cls_in_sqdb', time_d - time_c),
3 : ('proj_data', time_e - time_d),
4 : ('rule_1', timeCheck[1]['total'], timeCheck[1]['detailed']),
5 : ('rule_2', timeCheck[2]['total'], timeCheck[2]['detailed']),
6 : None if 3 not in timeCheck else (
'rule_3', timeCheck[3]['total'], timeCheck[3]['detailed']),
7 : None if 4 not in timeCheck else (
'rule_4', timeCheck[4]['total'], timeCheck[4]['detailed']),
8 : ('rule_5', timeCheck[5]['total'], timeCheck[5]['detailed']),
9 : None if 7 not in timeCheck else (
'rule_7', timeCheck[7]['total'], timeCheck[7]['detailed']),
10 : ('rst_to_array', time_n - time_m),
11 : ('sum_cls', time_o - time_n),
12 : ('priority_rule', time_p - time_o),
13 : ('merge_rst', time_q - time_p)
}
def raster_based(osmdata,
nomenclature,
refRaster,
lulcRst,
overwrite=None,
dataStore=None,
roadsAPI='POSTGIS'):
"""
Convert OSM Data into Land Use/Land Cover Information
An raster based approach.
TODO: Add detailed description
"""
# ************************************************************************ #
# Python Modules from Reference Packages #
# ************************************************************************ #
import datetime
import os
import pandas
import copy
# ************************************************************************ #
# Gasp dependencies #
# ************************************************************************ #
from gasp.pyt.oss import mkdir, fprop
from gasp.gt.prop.ff import check_isRaster
from gasp.gt.prop.prj import get_rst_epsg
from gasp.gt.wenv.grs import run_grass
if roadsAPI == 'POSTGIS':
from gasp.sql.db import create_db
from gasp.gql.to.osm import osm_to_psql
from gasp.sds.osm2lulc.mod2 import roads_sqdb
from gasp.sql.fm import dump_db
from gasp.sql.db import drop_db
else:
from gasp.gt.toshp.osm import osm_to_sqdb
from gasp.sds.osm2lulc.mod2 import grs_rst_roads
from gasp.sds.osm2lulc.utils import osm_project, add_lulc_to_osmfeat, osmlulc_rsttbl
from gasp.sds.osm2lulc.utils import get_ref_raster
from gasp.sds.osm2lulc.mod1 import grs_rst
from gasp.sds.osm2lulc.m3_4 import rst_area
from gasp.sds.osm2lulc.mod5 import basic_buffer
from gasp.sds.osm2lulc.mod6 import rst_pnt_to_build
# ************************************************************************ #
# Global Settings #
# ************************************************************************ #
# Check if input parameters exists!
if not os.path.exists(os.path.dirname(lulcRst)):
raise ValueError('{} does not exist!'.format(os.path.dirname(lulcRst)))
if not os.path.exists(osmdata):
raise ValueError(
'File with OSM DATA ({}) does not exist!'.format(osmdata))
if not os.path.exists(refRaster):
raise ValueError(
'File with reference area ({}) does not exist!'.format(refRaster))
# Check if Nomenclature is valid
nomenclature = "URBAN_ATLAS" if nomenclature != "URBAN_ATLAS" and \
nomenclature != "CORINE_LAND_COVER" and \
nomenclature == "GLOBE_LAND_30" else nomenclature
time_a = datetime.datetime.now().replace(microsecond=0)
workspace = os.path.join(os.path.dirname(lulcRst),
'osmtolulc') if not dataStore else dataStore
# Check if workspace exists
if os.path.exists(workspace):
if overwrite:
mkdir(workspace)
else:
raise ValueError('Path {} already exists'.format(workspace))
else:
mkdir(workspace)
# Get Ref Raster
refRaster, epsg = get_ref_raster(refRaster, workspace, cellsize=2)
from gasp.sds.osm2lulc import PRIORITIES, osmTableData, LEGEND
__priorites = PRIORITIES[nomenclature]
__legend = LEGEND[nomenclature]
time_b = datetime.datetime.now().replace(microsecond=0)
# ************************************************************************ #
# Convert OSM file to SQLITE DB or to POSTGIS DB #
# ************************************************************************ #
if roadsAPI == 'POSTGIS':
osm_db = create_db(fprop(osmdata, 'fn', forceLower=True),
overwrite=True)
osm_db = osm_to_psql(osmdata, osm_db)
else:
osm_db = osm_to_sqdb(osmdata, os.path.join(workspace, 'osm.sqlite'))
time_c = datetime.datetime.now().replace(microsecond=0)
# ************************************************************************ #
# Add Lulc Classes to OSM_FEATURES by rule #
# ************************************************************************ #
add_lulc_to_osmfeat(osm_db, osmTableData, nomenclature, api=roadsAPI)
time_d = datetime.datetime.now().replace(microsecond=0)
# ************************************************************************ #
# Transform SRS of OSM Data #
# ************************************************************************ #
osmTableData = osm_project(
osm_db,
epsg,
api=roadsAPI,
isGlobeLand=None if nomenclature != 'GLOBE_LAND_30' else True)
time_e = datetime.datetime.now().replace(microsecond=0)
# ************************************************************************ #
# Start a GRASS GIS Session #
# ************************************************************************ #
grass_base = run_grass(workspace,
grassBIN='grass78',
location='grloc',
srs=epsg)
import grass.script as grass
import grass.script.setup as gsetup
gsetup.init(grass_base, workspace, 'grloc', 'PERMANENT')
# ************************************************************************ #
# IMPORT SOME GASP MODULES FOR GRASS GIS #
# ************************************************************************ #
from gasp.gt.torst import rst_to_grs, grs_to_rst
from gasp.gt.nop.mos import rsts_to_mosaic
from gasp.gt.wenv.grs import rst_to_region
# ************************************************************************ #
# SET GRASS GIS LOCATION EXTENT #
# ************************************************************************ #
extRst = rst_to_grs(refRaster, 'extent_raster')
rst_to_region(extRst)
time_f = datetime.datetime.now().replace(microsecond=0)
# ************************************************************************ #
# MapResults #
mergeOut = {}
# ************************************************************************ #
# ************************************************************************ #
# 1 - Selection Rule #
# ************************************************************************ #
"""
selOut = {
cls_code : rst_name, ...
}
"""
selOut, timeCheck1 = grs_rst(osm_db,
osmTableData['polygons'],
api=roadsAPI)
for cls in selOut:
mergeOut[cls] = [selOut[cls]]
time_g = datetime.datetime.now().replace(microsecond=0)
# ************************************************************************ #
# 2 - Get Information About Roads Location #
# ************************************************************************ #
"""
roads = {
cls_code : rst_name, ...
}
"""
if roadsAPI != 'POSTGIS':
roads, timeCheck2 = grs_rst_roads(
osm_db, osmTableData['lines'], osmTableData['polygons'], workspace,
1221 if nomenclature != "GLOBE_LAND_30" else 801)
else:
roadCls = 1221 if nomenclature != "GLOBE_LAND_30" else 801
roads, timeCheck2 = roads_sqdb(osm_db,
osmTableData['lines'],
osmTableData['polygons'],
apidb='POSTGIS',
asRst=roadCls)
roads = {roadCls: roads}
for cls in roads:
if cls not in mergeOut:
mergeOut[cls] = [roads[cls]]
else:
mergeOut[cls].append(roads[cls])
time_h = datetime.datetime.now().replace(microsecond=0)
# ************************************************************************ #
# 3 - Area Upper than #
# ************************************************************************ #
"""
auOut = {
cls_code : rst_name, ...
}
"""
if nomenclature != 'GLOBE_LAND_30':
auOut, timeCheck3 = rst_area(osm_db,
osmTableData['polygons'],
UPPER=True,
api=roadsAPI)
for cls in auOut:
if cls not in mergeOut:
mergeOut[cls] = [auOut[cls]]
else:
mergeOut[cls].append(auOut[cls])
time_l = datetime.datetime.now().replace(microsecond=0)
else:
timeCheck3 = None
time_l = None
# ************************************************************************ #
# 4 - Area Lower than #
# ************************************************************************ #
"""
alOut = {
cls_code : rst_name, ...
}
"""
if nomenclature != 'GLOBE_LAND_30':
alOut, timeCheck4 = rst_area(osm_db,
osmTableData['polygons'],
UPPER=None,
api=roadsAPI)
for cls in alOut:
if cls not in mergeOut:
mergeOut[cls] = [alOut[cls]]
else:
mergeOut[cls].append(alOut[cls])
time_j = datetime.datetime.now().replace(microsecond=0)
else:
timeCheck4 = None
time_j = None
# ************************************************************************ #
# 5 - Get data from lines table (railway | waterway) #
# ************************************************************************ #
"""
bfOut = {
cls_code : rst_name, ...
}
"""
bfOut, timeCheck5 = basic_buffer(osm_db,
osmTableData['lines'],
workspace,
apidb=roadsAPI)
for cls in bfOut:
if cls not in mergeOut:
mergeOut[cls] = [bfOut[cls]]
else:
mergeOut[cls].append(bfOut[cls])
time_m = datetime.datetime.now().replace(microsecond=0)
# ************************************************************************ #
# 7 - Assign untagged Buildings to tags #
# ************************************************************************ #
if nomenclature != "GLOBE_LAND_30":
buildsOut, timeCheck7 = rst_pnt_to_build(osm_db,
osmTableData['points'],
osmTableData['polygons'],
api_db=roadsAPI)
for cls in buildsOut:
if cls not in mergeOut:
mergeOut[cls] = buildsOut[cls]
else:
mergeOut[cls] += buildsOut[cls]
time_n = datetime.datetime.now().replace(microsecond=0)
else:
timeCheck7 = None
time_n = datetime.datetime.now().replace(microsecond=0)
# ************************************************************************ #
# Produce LULC Map #
# ************************************************************************ #
"""
Merge all results for one cls into one raster
mergeOut = {
cls_code : [rst_name, rst_name, ...], ...
}
into
mergeOut = {
cls_code : patched_raster, ...
}
"""
for cls in mergeOut:
if len(mergeOut[cls]) == 1:
mergeOut[cls] = mergeOut[cls][0]
else:
mergeOut[cls] = rsts_to_mosaic(mergeOut[cls],
'mosaic_{}'.format(str(cls)),
api="grass")
time_o = datetime.datetime.now().replace(microsecond=0)
"""
Merge all Class Raster using a priority rule
"""
__priorities = PRIORITIES[nomenclature]
lst_rst = []
for cls in __priorities:
if cls not in mergeOut:
continue
else:
lst_rst.append(mergeOut[cls])
outGrs = rsts_to_mosaic(lst_rst,
os.path.splitext(os.path.basename(lulcRst))[0],
api="grass")
time_p = datetime.datetime.now().replace(microsecond=0)
# Ceck if lulc Rst has an valid format
outIsRst = check_isRaster(lulcRst)
if not outIsRst:
from gasp.pyt.oss import fprop
lulcRst = os.path.join(os.path.dirname(lulcRst),
fprop(lulcRst, 'fn') + '.tif')
grs_to_rst(outGrs, lulcRst, as_cmd=True)
osmlulc_rsttbl(
nomenclature,
os.path.join(os.path.dirname(lulcRst),
os.path.basename(lulcRst) + '.vat.dbf'))
time_q = datetime.datetime.now().replace(microsecond=0)
# Dump Database if PostGIS was used
# Drop Database if PostGIS was used
if roadsAPI == 'POSTGIS':
dump_db(osm_db, os.path.join(workspace, osm_db + '.sql'), api='psql')
drop_db(osm_db)
return lulcRst, {
0: ('set_settings', time_b - time_a),
1: ('osm_to_sqdb', time_c - time_b),
2: ('cls_in_sqdb', time_d - time_c),
3: ('proj_data', time_e - time_d),
4: ('set_grass', time_f - time_e),
5: ('rule_1', time_g - time_f, timeCheck1),
6: ('rule_2', time_h - time_g, timeCheck2),
7:
None if not timeCheck3 else ('rule_3', time_l - time_h, timeCheck3),
8:
None if not timeCheck4 else ('rule_4', time_j - time_l, timeCheck4),
9: ('rule_5', time_m - time_j if timeCheck4 else time_m - time_h,
timeCheck5),
10:
None if not timeCheck7 else ('rule_7', time_n - time_m, timeCheck7),
11: ('merge_rst', time_o - time_n),
12: ('priority_rule', time_p - time_o),
13: ('export_rst', time_q - time_p)
}
def get_ref_raster(refBoundBox, folder, cellsize=None):
"""
Get Reference Raster
"""
import os
from gasp.gt.prop.ff import check_isRaster
# Check if refRaster is really a Raster
isRst = check_isRaster(refBoundBox)
if not isRst:
from gasp.gt.prop.ff import check_isShp
if not check_isShp(refBoundBox):
raise ValueError((
'refRaster File has an invalid file format. Please give a file '
'with one of the following extensions: '
'shp, gml, json, kml, tif or img'
))
else:
# We have a shapefile
# Check SRS and see if it is a projected SRS
from gasp.gt.prop.prj import get_epsg_shp
epsg, isProj = get_epsg_shp(refBoundBox, returnIsProj=True)
if not epsg:
raise ValueError('Cannot get epsg code from {}'.format(refBoundBox))
if not isProj:
# A conversion between SRS is needed
from gasp.gt.prj import proj
ref_shp = proj(
refBoundBox, os.path.join(folder, 'tmp_ref_shp.shp'),
outEPSG=3857, inEPSG=epsg, gisApi='ogr2ogr'
)
epsg = 3857
else:
ref_shp = refBoundBox
# Convert to Raster
from gasp.gt.torst import shp_to_rst
refRaster = shp_to_rst(
ref_shp, None, 2 if not cellsize else cellsize,
-1, os.path.join(folder, 'ref_raster.tif'), api='gdal'
)
else:
# We have a raster
from gasp.gt.prop.prj import get_rst_epsg
epsg, isProj = get_rst_epsg(refBoundBox, returnIsProj=True)
if not epsg:
raise ValueError('Cannot get epsg code from {}'.format(refBoundBox))
# Check if Raster has a SRS with projected coordinates
if not isProj:
# We need to reproject raster
from gasp.gt.prj import reprj_rst
refRaster = reprj_rst(
refBoundBox,
os.path.join(folder, 'refrst_3857.tif'),
epsg, 3857
)
epsg = 3857
else:
refRaster = refBoundBox
return refRaster, epsg
def adjust_ext_to_snap(outExt, snapRst):
"""
Adjust extent for a output raster to snap with other raster
"""
from gasp.gt.prop.ff import check_isShp, check_isRaster
from gasp.gt.prop.rst import rst_ext, get_cellsize
from gasp.g.to import new_pnt, create_polygon
# Check if outExt is a raster or not
isRst = check_isRaster(outExt)
if isRst:
shpAExt = rst_ext(outExt)
else:
isShp = check_isShp(outExt)
if isShp:
from gasp.gt.prop.feat import get_ext
shpAExt = get_ext(outExt)
else:
raise ValueError((
"outExt value should be a path to a SHP or to a Raster file"
))
# Check if snapRst is a raster
isRst = check_isRaster(snapRst)
if not isRst:
raise ValueError((
"snapRst should be a path to a raster file"
))
# Get snapRst Extent
snapRstExt = rst_ext(snapRst)
# Get cellsize
csize = get_cellsize(snapRst)
# Find extent point of outExt inside the two extents
# This will be used as pseudo origin
snapRstPnt = [
new_pnt(snapRstExt[0], snapRstExt[3]),
new_pnt(snapRstExt[1], snapRstExt[3]),
new_pnt(snapRstExt[1], snapRstExt[2]),
new_pnt(snapRstExt[0], snapRstExt[2]),
new_pnt(snapRstExt[0], snapRstExt[3]),
]
poly_snap_rst = create_polygon(snapRstPnt)
outExtPnt = {
'top_left' : new_pnt(shpAExt[0], shpAExt[3]),
'top_right' : new_pnt(shpAExt[1], shpAExt[3]),
'bottom_right' : new_pnt(shpAExt[1], shpAExt[2]),
'bottom_left' : new_pnt(shpAExt[0], shpAExt[2])
}
out_rst_pseudo = {}
for pnt in outExtPnt:
out_rst_pseudo[pnt] = outExtPnt[pnt].Intersects(poly_snap_rst)
pseudoOrigin = outExtPnt['top_left'] if out_rst_pseudo['top_left'] else \
outExtPnt['bottom_left'] if out_rst_pseudo['bottom_left'] else \
outExtPnt['top_right'] if out_rst_pseudo['top_right'] else \
outExtPnt['bottom_right'] if out_rst_pseudo['bottom_right'] else None
if not pseudoOrigin:
raise ValueError((
'Extents doesn\'t have overlapping areas'
))
pseudoOriginName = 'top_left' if out_rst_pseudo['top_left'] else \
'bottom_left' if out_rst_pseudo['bottom_left'] else \
'top_right' if out_rst_pseudo['top_right'] else \
'bottom_right' if out_rst_pseudo['bottom_right'] else None
# Get out Raster Shape
n_col = int((shpAExt[1] - shpAExt[0]) / csize)
n_row = int((shpAExt[3] - shpAExt[2]) / csize)
# Get Output Raster real origin/top left
yName, xName = pseudoOriginName.split('_')
if xName == 'left':
# Obtain left of output Raster
left_out_rst = snapRstExt[0] + (
csize * int((shpAExt[0] - snapRstExt[0]) / csize))
else:
# obtain right of output Raster
right_out_rst = snapRstExt[1] - (
csize * int((snapRstExt[1] - shpAExt[1]) / csize))
# Use right to obtain left coordinate
left_out_rst = right_out_rst - (n_col * csize)
if yName == 'top':
# Obtain top of output Raster
top_out_rst = snapRstExt[3] - (
csize * int((snapRstExt[3] - shpAExt[3]) / csize))
else:
# obtain bottom of output raster
bot_out_rst = snapRstExt[2] + (
csize * int((shpAExt[2] - snapRstExt[2]) / csize))
# use bottom to find the top of the output raster
top_out_rst = bot_out_rst + (n_row * csize)
return left_out_rst, top_out_rst, n_row, n_col, csize
def shp_diff_fm_ref(refshp, refcol, shps, out_folder, refrst, db=None):
"""
Check differences between each shp in shps and one reference shape
Dependencies:
- GRASS;
- PostgreSQL with Postgis or GeoPandas;
"""
import os
from gasp.gt.prop.ff import check_isRaster
from gasp.gt.wenv.grs import run_grass
from gasp.pyt.oss import fprop
from gasp.gt.tbl.tomtx import tbl_to_areamtx
# Check if folder exists, if not create it
if not os.path.exists(out_folder):
from gasp.pyt.oss import mkdir
mkdir(out_folder)
# Start GRASS GIS Session
gbase = run_grass(out_folder,
grassBIN='grass78',
location='shpdif',
srs=refrst)
import grass.script.setup as gsetup
gsetup.init(gbase, out_folder, 'shpdif', 'PERMANENT')
from gasp.gt.toshp.cff import shp_to_grs, grs_to_shp
from gasp.gt.torst import rst_to_grs
from gasp.gt.tbl.fld import rn_cols
from gasp.gt.toshp.rst import rst_to_polyg
# Convert to SHAPE if file is Raster
# Rename interest columns
i = 0
lstff = [refshp] + list(shps.keys())
__shps = {}
for s in lstff:
is_rst = check_isRaster(s)
if is_rst:
# To GRASS
rname = fprop(s, 'fn')
inrst = rst_to_grs(s, "rst_" + rname, as_cmd=True)
# To vector
d = rst_to_polyg(inrst,
rname,
rstColumn="lulc_{}".format(str(i)),
gisApi="grass")
else:
# To GRASS
d = shp_to_grs(s, fprop(s, 'fn'), asCMD=True)
# Change name of interest colum
rn_cols(d, {shps[s] if i else refcol: "lulc_{}".format(str(i))},
api="grass")
# Export To Shapefile
if not i:
refshp = grs_to_shp(d, os.path.join(out_folder, d + '.shp'),
'area')
refcol = "lulc_{}".format(str(i))
else:
shp = grs_to_shp(d, os.path.join(out_folder, d + '.shp'), 'area')
__shps[shp] = "lulc_{}".format(str(i))
i += 1
# Union Shapefiles
union_shape = {}
for shp in __shps:
# Optimized Union
sname = fprop(shp, 'fn')
union_shape[shp] = optimized_union_anls(
shp,
refshp,
os.path.join(out_folder, sname + '_un.shp'),
refrst,
os.path.join(out_folder, "wk_" + sname),
multiProcess=True)
# Produce confusion matrices
mtxf = tbl_to_areamtx(union_shape[shp],
"a_" + __shps[shp],
'b_' + refcol,
os.path.join(out_folder, sname + '.xlsx'),
db=db,
with_metrics=True)
return out_folder
def check_shape_diff(SHAPES_TO_COMPARE, OUT_FOLDER, REPORT, DB,
GRASS_REGION_TEMPLATE):
"""
Script to check differences between pairs of Feature Classes
Suponha que temos diversas Feature Classes (FC) e que cada uma delas
possui um determinado atributo; imagine tambem que,
considerando todos os pares possiveis entre estas FC,
se pretende comparar as diferencas na distribuicao dos valores
desse atributo para cada par.
* Dependencias:
- GRASS;
- PostgreSQL;
- PostGIS.
"""
import datetime
import os
import pandas
from gasp.sql.fm import q_to_obj
from gasp.to import db_to_tbl
from gasp.sql.to import df_to_db
from gasp.gt.toshp.cff import shp_to_shp
from gasp.gt.toshp.db import dbtbl_to_shp
from gasp.gt.toshp.rst import rst_to_polyg
from gasp.gql.to import shp_to_psql
from gasp.gql.tomtx import tbl_to_area_mtx
from gasp.gt.prop.ff import check_isRaster
from gasp.pyt.oss import fprop
from gasp.sql.db import create_db
from gasp.sql.tbl import tbls_to_tbl
from gasp.sql.to import q_to_ntbl
from gasp.gql.cln import fix_geom
from gasp.to import db_to_tbl
# Check if folder exists, if not create it
if not os.path.exists(OUT_FOLDER):
from gasp.pyt.oss import mkdir
mkdir(OUT_FOLDER, overwrite=None)
else:
raise ValueError('{} already exists!'.format(OUT_FOLDER))
from gasp.gt.wenv.grs import run_grass
gbase = run_grass(OUT_FOLDER,
grassBIN='grass78',
location='shpdif',
srs=GRASS_REGION_TEMPLATE)
import grass.script as grass
import grass.script.setup as gsetup
gsetup.init(gbase, OUT_FOLDER, 'shpdif', 'PERMANENT')
from gasp.gt.toshp.cff import shp_to_grs, grs_to_shp
from gasp.gt.torst import rst_to_grs
from gasp.gt.tbl.fld import rn_cols
# Convert to SHAPE if file is Raster
i = 0
_SHP_TO_COMPARE = {}
for s in SHAPES_TO_COMPARE:
isRaster = check_isRaster(s)
if isRaster:
# To GRASS
rstName = fprop(s, 'fn')
inRst = rst_to_grs(s, "rst_" + rstName, as_cmd=True)
# To Vector
d = rst_to_polyg(inRst,
rstName,
rstColumn="lulc_{}".format(i),
gisApi="grass")
# Export Shapefile
shp = grs_to_shp(d, os.path.join(OUT_FOLDER, d + '.shp'), "area")
_SHP_TO_COMPARE[shp] = "lulc_{}".format(i)
else:
# To GRASS
grsV = shp_to_grs(s, fprop(s, 'fn'), asCMD=True)
# Change name of column with comparing value
ncol = "lulc_{}".format(str(i))
rn_cols(grsV, {SHAPES_TO_COMPARE[s]: "lulc_{}".format(str(i))},
api="grass")
# Export
shp = grs_to_shp(grsV, os.path.join(OUT_FOLDER, grsV + '_rn.shp'),
"area")
_SHP_TO_COMPARE[shp] = "lulc_{}".format(str(i))
i += 1
SHAPES_TO_COMPARE = _SHP_TO_COMPARE
__SHAPES_TO_COMPARE = SHAPES_TO_COMPARE
# Create database
create_db(DB, api='psql')
""" Union SHAPEs """
UNION_SHAPE = {}
FIX_GEOM = {}
SHPS = list(__SHAPES_TO_COMPARE.keys())
for i in range(len(SHPS)):
for e in range(i + 1, len(SHPS)):
# Optimized Union
print("Union between {} and {}".format(SHPS[i], SHPS[e]))
time_a = datetime.datetime.now().replace(microsecond=0)
__unShp = optimized_union_anls(
SHPS[i],
SHPS[e],
os.path.join(OUT_FOLDER, "un_{}_{}.shp".format(i, e)),
GRASS_REGION_TEMPLATE,
os.path.join(OUT_FOLDER, "work_{}_{}".format(i, e)),
multiProcess=True)
time_b = datetime.datetime.now().replace(microsecond=0)
print(time_b - time_a)
# Rename cols
unShp = rn_cols(
__unShp, {
"a_" + __SHAPES_TO_COMPARE[SHPS[i]]:
__SHAPES_TO_COMPARE[SHPS[i]],
"b_" + __SHAPES_TO_COMPARE[SHPS[e]]:
__SHAPES_TO_COMPARE[SHPS[e]]
})
UNION_SHAPE[(SHPS[i], SHPS[e])] = unShp
# Send data to postgresql
SYNTH_TBL = {}
for uShp in UNION_SHAPE:
# Send data to PostgreSQL
union_tbl = shp_to_psql(DB, UNION_SHAPE[uShp], api='shp2pgsql')
# Produce table with % of area equal in both maps
areaMapTbl = q_to_ntbl(
DB,
"{}_syn".format(union_tbl),
("SELECT CAST('{lulc_1}' AS text) AS lulc_1, "
"CAST('{lulc_2}' AS text) AS lulc_2, "
"round("
"CAST(SUM(g_area) / 1000000 AS numeric), 4"
") AS agree_area, round("
"CAST((SUM(g_area) / MIN(total_area)) * 100 AS numeric), 4"
") AS agree_percentage, "
"round("
"CAST(MIN(total_area) / 1000000 AS numeric), 4"
") AS total_area FROM ("
"SELECT {map1_cls}, {map2_cls}, ST_Area(geom) AS g_area, "
"CASE "
"WHEN {map1_cls} = {map2_cls} "
"THEN 1 ELSE 0 "
"END AS isthesame, total_area FROM {tbl}, ("
"SELECT SUM(ST_Area(geom)) AS total_area FROM {tbl}"
") AS foo2"
") AS foo WHERE isthesame = 1 "
"GROUP BY isthesame").format(
lulc_1=fprop(uShp[0], 'fn'),
lulc_2=fprop(uShp[1], 'fn'),
map1_cls=__SHAPES_TO_COMPARE[uShp[0]],
map2_cls=__SHAPES_TO_COMPARE[uShp[1]],
tbl=union_tbl),
api='psql')
# Produce confusion matrix for the pair in comparison
matrixTbl = tbl_to_area_mtx(DB, union_tbl,
__SHAPES_TO_COMPARE[uShp[0]],
__SHAPES_TO_COMPARE[uShp[1]],
union_tbl + '_mtx')
SYNTH_TBL[uShp] = {"TOTAL": areaMapTbl, "MATRIX": matrixTbl}
# UNION ALL TOTAL TABLES
total_table = tbls_to_tbl(DB, [SYNTH_TBL[k]["TOTAL"] for k in SYNTH_TBL],
'total_table')
# Create table with % of agreement between each pair of maps
mapsNames = q_to_obj(
DB,
("SELECT lulc FROM ("
"SELECT lulc_1 AS lulc FROM {tbl} GROUP BY lulc_1 "
"UNION ALL "
"SELECT lulc_2 AS lulc FROM {tbl} GROUP BY lulc_2"
") AS lu GROUP BY lulc ORDER BY lulc").format(tbl=total_table),
db_api='psql').lulc.tolist()
FLDS_TO_PIVOT = ["agree_percentage", "total_area"]
Q = ("SELECT * FROM crosstab('"
"SELECT CASE "
"WHEN foo.lulc_1 IS NOT NULL THEN foo.lulc_1 ELSE jtbl.tmp1 "
"END AS lulc_1, CASE "
"WHEN foo.lulc_2 IS NOT NULL THEN foo.lulc_2 ELSE jtbl.tmp2 "
"END AS lulc_2, CASE "
"WHEN foo.{valCol} IS NOT NULL THEN foo.{valCol} ELSE 0 "
"END AS agree_percentage FROM ("
"SELECT lulc_1, lulc_2, {valCol} FROM {tbl} UNION ALL "
"SELECT lulc_1, lulc_2, {valCol} FROM ("
"SELECT lulc_1 AS lulc_2, lulc_2 AS lulc_1, {valCol} "
"FROM {tbl}"
") AS tst"
") AS foo FULL JOIN ("
"SELECT lulc_1 AS tmp1, lulc_2 AS tmp2 FROM ("
"SELECT lulc_1 AS lulc_1 FROM {tbl} GROUP BY lulc_1 "
"UNION ALL "
"SELECT lulc_2 AS lulc_1 FROM {tbl} GROUP BY lulc_2"
") AS tst_1, ("
"SELECT lulc_1 AS lulc_2 FROM {tbl} GROUP BY lulc_1 "
"UNION ALL "
"SELECT lulc_2 AS lulc_2 FROM {tbl} GROUP BY lulc_2"
") AS tst_2 WHERE lulc_1 = lulc_2 GROUP BY lulc_1, lulc_2"
") AS jtbl ON foo.lulc_1 = jtbl.tmp1 AND foo.lulc_2 = jtbl.tmp2 "
"ORDER BY lulc_1, lulc_2"
"') AS ct("
"lulc_map text, {crossCols}"
")")
TOTAL_AGREE_TABLE = None
TOTAL_AREA_TABLE = None
for f in FLDS_TO_PIVOT:
if not TOTAL_AGREE_TABLE:
TOTAL_AGREE_TABLE = q_to_ntbl(
DB,
"agreement_table",
Q.format(tbl=total_table,
valCol=f,
crossCols=", ".join([
"{} numeric".format(map_) for map_ in mapsNames
])),
api='psql')
else:
TOTAL_AREA_TABLE = q_to_ntbl(DB,
"area_table",
Q.format(tbl=total_table,
valCol=f,
crossCols=", ".join([
"{} numeric".format(map_)
for map_ in mapsNames
])),
api='psql')
# Union Mapping
UNION_MAPPING = pandas.DataFrame(
[[k[0], k[1], fprop(UNION_SHAPE[k], 'fn')] for k in UNION_SHAPE],
columns=['shp_a', 'shp_b', 'union_shp'])
UNION_MAPPING = df_to_db(DB, UNION_MAPPING, 'union_map', api='psql')
# Export Results
TABLES = [UNION_MAPPING, TOTAL_AGREE_TABLE, TOTAL_AREA_TABLE
] + [SYNTH_TBL[x]["MATRIX"] for x in SYNTH_TBL]
SHEETS = ["union_map", "agreement_percentage", "area_with_data_km"] + [
"{}_{}".format(fprop(x[0], 'fn')[:15],
fprop(x[1], 'fn')[:15]) for x in SYNTH_TBL
]
db_to_tbl(DB, ["SELECT * FROM {}".format(x) for x in TABLES],
REPORT,
sheetsNames=SHEETS,
dbAPI='psql')
return REPORT
def download_by_boundary(input_boundary, folder_out, osm_name, epsg,
GetUrl=True, db_name=None, geomCol=None,
justOneFeature=None):
"""
Download data from OSM using a bounding box
"""
import os
from osgeo import ogr
from gasp.to.web import get_file
from gasp.pyt.oss import os_name
OS_NAME = os_name()
EXTENTS = []
if db_name and geomCol:
"""
Assuming input_boundary is a PostgreSQL Table
"""
from gasp.pyt import obj_to_lst
from gasp.gql.prop import tbl_ext
for t in obj_to_lst(input_boundary):
EXTENTS.append(tbl_ext(db_name, t, geomCol))
else:
if type(input_boundary) == dict:
if 'top' in input_boundary and 'bottom' in input_boundary \
and 'left' in input_boundary and 'right' in input_boundary:
EXTENTS.append([
input_boundary['left'],input_boundary['right'],
input_boundary['bottom'], input_boundary['top']
])
else:
raise ValueError((
'input_boundary is a dict but the keys are not correct. '
'Please use left, right, top and bottom as keys'
))
elif type(input_boundary) == list:
if len(input_boundary) == 4:
EXTENTS.append(input_boundary)
else:
raise ValueError((
'input boundary is a list with more than 4 objects. '
'The list should be like: '
'l = [left, right, bottom, top]'
))
elif type(input_boundary) == ogr.Geometry:
EXTENTS.append(input_boundary.GetEnvelope())
else:
# Assuming input boundary is a file
#Check if file exists
if not os.path.exists(input_boundary):
raise ValueError((
"Sorry, but the file {} does not exist inside the folder {}!"
).format(
os.path.basename(input_boundary),
os.path.dirname(input_boundary)
))
# Check if is a raster
from gasp.gt.prop.ff import check_isRaster
isRst = check_isRaster(input_boundary)
# Get EPSG
if not epsg:
from gasp.gt.prop.prj import get_epsg
epsg = get_epsg(input_boundary)
if isRst:
from gasp.gt.prop.rst import rst_ext
# Get raster extent
EXTENTS.append(rst_ext(input_boundary))
else:
from gasp.gt.prop.ff import drv_name
# Todo: check if it is shape
# Open Dataset
inSrc = ogr.GetDriverByName(drv_name(
input_boundary)).Open(input_boundary)
lyr = inSrc.GetLayer()
i = 1
for feat in lyr:
geom = feat.GetGeometryRef()
featext = geom.GetEnvelope()
EXTENTS.append(featext)
if justOneFeature:
break
if epsg != 4326:
from gasp.g.to import new_pnt
from gasp.g.prj import prj_ogrgeom
for i in range(len(EXTENTS)):
bottom_left = prj_ogrgeom(new_pnt(
EXTENTS[i][0], EXTENTS[i][2]), epsg, 4326)
top_right = prj_ogrgeom(new_pnt(
EXTENTS[i][1], EXTENTS[i][3]), epsg, 4326)
left , bottom = bottom_left.GetX(), bottom_left.GetY()
right, top = top_right.GetX() , top_right.GetY()
EXTENTS[i] = [left, right, bottom, top]
#url = "https://overpass-api.de/api/map?bbox={}"
url = "https://lz4.overpass-api.de/api/interpreter?bbox={}"
RESULTS = []
for e in range(len(EXTENTS)):
bbox_str = ','.join([str(p) for p in EXTENTS[e]])
if GetUrl:
RESULTS.append(url.format(bbox_str))
continue
if len(EXTENTS) == 1:
outOsm = os.path.join(folder_out, osm_name + '.xml')
else:
outOsm = os.path.join(folder_out, "{}_{}.xml".format(osm_name, str(e)))
osm_file = get_file(
url.format(bbox_str), outOsm,
useWget=None if OS_NAME == 'Windows' else None
)
RESULTS.append(osm_file)
return RESULTS[0] if len(RESULTS) == 1 else RESULTS