def FilterAndExport(CLS, cnt):
time_x = datetime.datetime.now().replace(microsecond=0)
if api == 'SQLITE':
shp = sel_by_attr(
osmdb, SQL_Q.format(lc=str(CLS), tbl=polyTbl),
os.path.join(folder, 'sel_{}.shp'.format(str(CLS))),
api_gis='ogr'
)
else:
shp = sel_by_attr(
osmdb, SQL_Q.format(lc=str(CLS), tbl=polyTbl), "geometry",
os.path.join(folder, 'sel_{}.shp'.format(str(CLS))),
api='pgsql2shp', tableIsQuery=True
)
time_y = datetime.datetime.now().replace(microsecond=0)
rstCls = shp_to_rst(
shp, None, cellsize, 0,
os.path.join(folder, 'sel_{}.tif'.format(str(CLS))),
epsg=srscode, rst_template=rstTemplate, api='gdal'
)
time_z = datetime.datetime.now().replace(microsecond=0)
clsRst[int(CLS)] = rstCls
timeGasto[cnt + 1] = ('toshp_{}'.format(str(CLS)), time_y - time_x)
timeGasto[cnt + 2] = ('torst_{}'.format(str(CLS)), time_z - time_y)
def ob_ref_rst(ref, folder, cellsize=None):
"""
Get Reference Raster
"""
from gasp.gt.prop.ff import check_isRaster
# Check if refRaster is really a Raster
isRst = check_isRaster(ref)
if not isRst:
from gasp.gt.prop.ff import check_isShp
if not check_isShp(ref):
raise ValueError(
('Extent Template File has an invalid file format. '
'Please give a file with one of the following extensions: '
'shp, gml, json, kml, tif or img'))
else:
# We have a shapefile
# Convert it to Raster
from gasp.gt.torst import shp_to_rst
ref_rst = shp_to_rst(ref,
None,
10 if not cellsize else cellsize,
-1,
os.path.join(folder, 'ref_raster.tif'),
api='gdal')
return ref_rst
else:
return ref
def exportBuild():
time_ee = datetime.datetime.now().replace(microsecond=0)
NB = row_num(
osmdata, polyTbl, where="building IS NOT NULL", api='sqlite'
)
time_e = datetime.datetime.now().replace(microsecond=0)
timeGasto[3] = ('check_builds', time_e - time_ee)
if not NB:
return
bShp = sel_by_attr(
osmdata,
"SELECT geometry FROM {} WHERE building IS NOT NULL".format(
polyTbl
),
os.path.join(folder, 'road_builds.shp'),
api_gis='ogr'
)
time_f = datetime.datetime.now().replace(microsecond=0)
bRst = shp_to_rst(
bShp, None, cellsize, -1,
os.path.join(folder, 'road_builds.tif'),
epsg=srs, rst_template=rstTemplate, api='gdal'
)
time_g = datetime.datetime.now().replace(microsecond=0)
BUILDINGS.append(bRst)
timeGasto[4] = ('export_builds', time_f - time_e)
timeGasto[5] = ('builds_to_rst', time_g - time_f)
def exportAndBufferB(CLS, cnt):
# Run BUFFER Tool
time_x = datetime.datetime.now().replace(microsecond=0)
bb_file = st_buffer(osmdb,
lineTbl,
"bf_basic_buffer",
"geometry",
os.path.join(
folder,
'bb_rule5_{}.shp'.format(str(int(CLS)))),
whrClause="basic_buffer={}".format(str(int(CLS))),
outTblIsFile=True,
dissolve=None,
cols_select="basic_buffer")
time_y = datetime.datetime.now().replace(microsecond=0)
# To raster
rstCls = shp_to_rst(bb_file,
None,
cells,
0,
os.path.join(folder,
'rst_bbfr_{}.tif'.format(CLS)),
epsg=srscode,
rst_template=rtemplate,
api='gdal')
time_z = datetime.datetime.now().replace(microsecond=0)
clsRst[CLS] = rstCls
timeGasto[cnt + 1] = ('buffer_{}'.format(str(CLS)), time_y - time_x)
timeGasto[cnt + 2] = ('torst_{}'.format(str(CLS)), time_z - time_y)
def build12_torst(buildTbl):
LulcCls = q_to_obj(
osmdb,
"SELECT cls FROM {} GROUP BY cls".format(buildTbl),
db_api='psql' if apidb == 'POSTGIS' else 'sqlite').cls.tolist()
for lulc_cls in LulcCls:
time_m = dt.datetime.now().replace(microsecond=0)
# To SHP
if apidb == 'SQLITE':
shpB = sel_by_attr(osmdb,
"SELECT * FROM {} WHERE cls={}".format(
buildTbl, str(lulc_cls)),
os.path.join(
folder,
'nshp_build_{}.shp'.format(lulc_cls)),
api_gis='ogr')
else:
shpB = sel_by_attr(osmdb,
"SELECT * FROM {} WHERE cls={}".format(
buildTbl, str(lulc_cls)),
"geometry",
os.path.join(
folder,
'nshp_build_{}.shp'.format(lulc_cls)),
api='pgsql2shp',
tableIsQuery=True)
time_n = dt.datetime.now().replace(microsecond=0)
# To RST
brst = shp_to_rst(shpB,
None,
cells,
0,
os.path.join(
folder,
'nrst_build_{}.tif'.format(lulc_cls)),
srscode,
rstT,
api='gdal')
time_o = dt.datetime.now().replace(microsecond=0)
resLyr[int(lulc_cls)] = [brst]
timeGasto[int(lulc_cls)] = ('to_shp_{}'.format(str(lulc_cls)),
time_n - time_m)
timeGasto[int(lulc_cls) + 1] = ('to_rst_n_{}'.format(
str(lulc_cls)), time_o - time_n)
def grs_rst(db, polyTbl, api='SQLITE'):
"""
Simple selection, convert result to Raster
"""
import datetime
from gasp.sql.fm import q_to_obj
from gasp.gt.toshp.db import dbtbl_to_shp as db_to_grs
from gasp.gt.torst import shp_to_rst
# Get Classes
time_a = datetime.datetime.now().replace(microsecond=0)
lulcCls = q_to_obj(db, (
"SELECT selection FROM {} "
"WHERE selection IS NOT NULL "
"GROUP BY selection"
).format(polyTbl), db_api='psql' if api == 'POSTGIS' else 'sqlite').selection.tolist()
time_b = datetime.datetime.now().replace(microsecond=0)
timeGasto = {0 : ('check_cls', time_b - time_a)}
# Import data into GRASS and convert it to raster
clsRst = {}
tk = 1
for cls in lulcCls:
time_x = datetime.datetime.now().replace(microsecond=0)
grsVect = db_to_grs(
db, polyTbl, "geometry", "rule1_{}".format(str(cls)),
inDB='psql' if api == 'POSTGIS' else 'sqlite',
where="selection = {}".format(str(cls)), notTable=True,
filterByReg=True, outShpIsGRASS=True
)
time_y = datetime.datetime.now().replace(microsecond=0)
grsRst = shp_to_rst(
grsVect, int(cls), None, None, "rst_rule1_{}".format(str(cls)),
api='grass'
)
time_z = datetime.datetime.now().replace(microsecond=0)
clsRst[int(cls)] = grsRst
timeGasto[tk] = ('import_{}'.format(cls), time_y - time_x)
timeGasto[tk+1] = ('torst_{}'.format(cls), time_z - time_y)
tk += 2
return clsRst, timeGasto
def selAndExport(CLS, cnt):
time_x = datetime.datetime.now().replace(microsecond=0)
if api == "SQLITE":
shpCls = sel_by_attr(db,
SQL_Q.format(c=str(CLS),
tbl=polyTbl,
w=WHR.format(op=OPERATOR,
r=RULE_COL,
ga=GEOM_AREA,
cls_=CLS)),
os.path.join(
folder, "{}_{}.shp".format(RULE_COL,
CLS)),
api_gis='ogr')
else:
shpCls = sel_by_attr(db,
SQL_Q.format(c=str(CLS),
tbl=polyTbl,
w=WHR.format(op=OPERATOR,
r=RULE_COL,
ga=GEOM_AREA,
cls_=CLS)),
"geometry",
os.path.join(
folder,
"{}_{}.shp".format(RULE_COL, str(CLS))),
api='pgsql2shp',
tableIsQuery=True)
time_y = datetime.datetime.now().replace(microsecond=0)
rst = shp_to_rst(shpCls,
None,
cellsize,
0,
os.path.join(folder,
"{}_{}.tif".format(RULE_COL, CLS)),
epsg=srscode,
rst_template=rstTemplate,
api='gdal')
time_z = datetime.datetime.now().replace(microsecond=0)
clsRst[int(CLS)] = rst
timeGasto[cnt + 1] = ('sq_to_shp_{}'.format(str(CLS)), time_y - time_x)
timeGasto[cnt + 2] = ('shp_to_rst_{}'.format(str(CLS)),
time_z - time_y)
def toRaster(buildShp, cls):
if not os.path.exists(buildShp): return
# To Raster
time_x = dt.datetime.now().replace(microsecond=0)
rstbuild = shp_to_rst(buildShp,
None,
cells,
0,
os.path.join(folder,
'rst_build_{}.tif'.format(cls)),
srscode,
rstT,
api='gdal')
time_y = dt.datetime.now().replace(microsecond=0)
resLyr[33] = rstbuild
timeGasto[33] = ('to_rst_{}'.format(cls), time_y - time_x)
def exportAndBuffer():
time_cc = datetime.datetime.now().replace(microsecond=0)
roadFile = splite_buffer(
osmdata, lineTbl, "bf_roads", "geometry",
os.path.join(folder, 'bf_roads.gml'),
whrClause="roads IS NOT NULL",
outTblIsFile=True, dissolve=None
)
time_c = datetime.datetime.now().replace(microsecond=0)
distRst = shp_to_rst(
roadFile, None, cellsize, -1,
os.path.join(folder, 'rst_roads.tif'),
epsg=srs, rst_template=rstTemplate, api="gdal"
)
time_d = datetime.datetime.now().replace(microsecond=0)
bfShps.append(distRst)
timeGasto[1] = ('buffer_roads', time_c - time_cc)
timeGasto[2] = ('to_rst_roads', time_d - time_c)
def pg_num_roads(osmdb, nom, lnhTbl, polyTbl, folder, cellsize, srs, rstT):
"""
Select, Calculate Buffer distance using POSTGIS, make buffer of roads
and convert roads to raster
"""
import datetime; import os
from osgeo import gdal
from gasp.sql.i import row_num
from gasp.gql.prox import st_buffer
from gasp.gt.torst import shp_to_rst
# There are roads?
time_a = datetime.datetime.now().replace(microsecond=0)
NR = row_num(osmdb, lnhTbl, where="roads IS NOT NULL", api='psql')
time_b = datetime.datetime.now().replace(microsecond=0)
if not NR: return None, {0 : ('count_rows_roads', time_b - time_a)}
# There are buildings?
NB = row_num(osmdb, polyTbl, where="building IS NOT NULL", api='psql')
time_c = datetime.datetime.now().replace(microsecond=0)
if NB:
from gasp.gql.prox import st_near
from gasp.sql.q import exec_write_q
nroads = st_near(osmdb, (
"(SELECT gid, roads, bf_roads, geometry FROM {} "
"WHERE roads IS NOT NULL)"
).format(lnhTbl), "geometry", (
"(SELECT * FROM {} WHERE building IS NOT NULL)"
).format(polyTbl), "geometry", "near_roads", until_dist="12",
near_col="dist_near", intbl_pk="gid"
)
time_d = datetime.datetime.now().replace(microsecond=0)
exec_write_q(osmdb, [(
"UPDATE near_roads SET "
"bf_roads = CAST(round(CAST(dist_near AS numeric), 0) AS integer) "
"WHERE dist_near >= 1 AND dist_near <= 12"
), "CREATE INDEX near_dist_idx ON near_roads USING gist (geometry)"])
time_e = datetime.datetime.now().replace(microsecond=0)
else:
nroads = (
"(SELECT roads, bf_roads, geometry FROM {} "
"WHERE roads IS NOT NULL) AS foo"
).format(lnhTbl)
time_d = None; time_e=None
# Execute Buffer
bufferShp = st_buffer(
osmdb, nroads, "bf_roads", "geometry",
os.path.join(folder, "bf_roads.shp"),
cols_select="roads", outTblIsFile=True, dissolve=None
)
time_f = datetime.datetime.now().replace(microsecond=0)
# Convert to Raster
roadsRst = shp_to_rst(
bufferShp, None, cellsize, 0,
os.path.join(folder, "rst_roads.tif"), epsg=srs, rst_template=rstT,
api='gdal'
)
time_g = datetime.datetime.now().replace(microsecond=0)
LULC_CLS = '1221' if nom != "GLOBE_LAND_30" else '801'
return {int(LULC_CLS) : roadsRst}, {
0 : ('count_rows_roads', time_b - time_a),
1 : ('count_rows_build', time_c - time_b),
2 : None if not time_d else ('near_analysis', time_d - time_c),
3 : None if not time_e else ('update_buffer_tbl', time_e - time_d),
4 : ('buffer_roads', time_f - time_e if time_e else time_f - time_c),
5 : ('roads_to_raster', time_g - time_f)
}
def make_dem(grass_workspace,
data,
field,
output,
extent_template,
method="IDW",
cell_size=None,
mask=None):
"""
Create Digital Elevation Model
Methods Available:
* IDW;
* BSPLINE;
* SPLINE;
* CONTOUR;
"""
from gasp.pyt.oss import fprop
from gasp.gt.wenv.grs import run_grass
from gasp.gt.prop.prj import get_epsg
LOC_NAME = fprop(data, 'fn', forceLower=True)[:5] + "_loc"
# Get EPSG From Raster
EPSG = get_epsg(extent_template)
if not EPSG:
raise ValueError(
'Cannot get EPSG code of Extent Template File ({})'.format(
extent_template))
# Know if data geometry are points
if method == 'BSPLINE' or method == 'SPLINE':
from gasp.gt.prop.feat import get_gtype
data_gtype = get_gtype(data, gisApi='ogr')
# Create GRASS GIS Location
grass_base = run_grass(grass_workspace, location=LOC_NAME, srs=EPSG)
# Start GRASS GIS Session
import grass.script as grass
import grass.script.setup as gsetup
gsetup.init(grass_base, grass_workspace, LOC_NAME, 'PERMANENT')
# Get Extent Raster
ref_template = ob_ref_rst(extent_template,
os.path.join(grass_workspace, LOC_NAME),
cellsize=cell_size)
# IMPORT GRASS GIS MODULES #
from gasp.gt.torst import rst_to_grs, grs_to_rst
from gasp.gt.toshp.cff import shp_to_grs
from gasp.gt.wenv.grs import rst_to_region
# Configure region
rst_to_grs(ref_template, 'extent')
rst_to_region('extent')
# Convert elevation "data" to GRASS Vector
elv = shp_to_grs(data, 'elevation')
OUTPUT_NAME = fprop(output, 'fn', forceLower=True)
if method == "BSPLINE":
from gasp.gt.nop.itp import bspline
# Convert to points if necessary
if data_gtype != 'POINT' and data_gtype != 'MULTIPOINT':
from gasp.gt.toshp.cgeo import feat_vertex_to_pnt
elev_pnt = feat_vertex_to_pnt(elv, "elev_pnt", nodes=None)
else:
elev_pnt = elv
outRst = bspline(elev_pnt,
field,
OUTPUT_NAME,
mway='bicubic',
lyrN=1,
asCMD=True)
elif method == "SPLINE":
from gasp.gt.nop.itp import surfrst
# Convert to points if necessary
if data_gtype != 'POINT' and data_gtype != 'MULTIPOINT':
from gasp.gt.toshp.cgeo import feat_vertex_to_pnt
elev_pnt = feat_vertex_to_pnt(elv, "elev_pnt", nodes=None)
else:
elev_pnt = elv
outRst = surfrst(elev_pnt, field, OUTPUT_NAME, lyrN=1, ascmd=True)
elif method == "CONTOUR":
from gasp.gt.torst import shp_to_rst
from gasp.gt.nop.itp import surfcontour
# Apply mask if mask
if mask:
from gasp.gt.torst import grs_to_mask, rst_to_grs
rst_mask = rst_to_grs(mask, 'rst_mask', as_cmd=True)
grs_to_mask(rst_mask)
# Elevation (GRASS Vector) to Raster
elevRst = shp_to_rst(elv,
field,
None,
None,
'rst_elevation',
api="pygrass")
# Run Interpolator
outRst = surfcontour(elevRst, OUTPUT_NAME, ascmd=True)
elif method == "IDW":
from gasp.gt.nop.itp import ridw
from gasp.gt.nop.alg import rstcalc
from gasp.gt.torst import shp_to_rst
# Elevation (GRASS Vector) to Raster
elevRst = shp_to_rst(elv,
field,
None,
None,
'rst_elevation',
api='pygrass')
# Multiply cells values by 100 000.0
rstcalc('int(rst_elevation * 100000)', 'rst_elev_int', api='pygrass')
# Run IDW to generate the new DEM
ridw('rst_elev_int', 'dem_int', numberPoints=15)
# DEM to Float
rstcalc('dem_int / 100000.0', OUTPUT_NAME, api='pygrass')
# Export DEM to a file outside GRASS Workspace
grs_to_rst(OUTPUT_NAME, output)
return output
def lulc_by_cell(tid, boundary, lulc_shps, fishnet, result, workspace):
bname = fprop(boundary, 'fn')
# Boundary to Raster
ref_rst = shp_to_rst(boundary, None, 10, 0,
os.path.join(workspace, 'rst_{}.tif'.format(bname)))
# Create GRASS GIS Session
loc_name = 'loc_' + bname
gbase = run_grass(workspace, location=loc_name, srs=ref_rst)
import grass.script as grass
import grass.script.setup as gsetup
gsetup.init(gbase, workspace, loc_name, 'PERMANENT')
# GRASS GIS Modules
from gasp.gt.toshp.cff import shp_to_grs, grs_to_shp
from gasp.gt.gop.ovlay import intersection
from gasp.gt.tbl.attr import geomattr_to_db
from gasp.gt.prop.feat import feat_count
# Send Fishnet to GRASS GIS
fnet = shp_to_grs(fishnet, fprop(fishnet, 'fn'), asCMD=True)
# Processing
ulst = []
l_lulc_grs = []
for shp in lulc_shps:
iname = fprop(shp, 'fn')
# LULC Class to GRASS GIS
lulc_grs = shp_to_grs(shp, iname, filterByReg=True, asCMD=True)
if not feat_count(
lulc_grs, gisApi='grass', work=workspace, loc=loc_name):
continue
# Intersect Fishnet | LULC CLass
union_grs = intersection(fnet, lulc_grs, iname + '_i', api="grass")
# Get Areas
geomattr_to_db(union_grs, "areav", "area", "boundary", unit='meters')
# Export Table
funion = grs_to_shp(union_grs, os.path.join(result, iname + '.shp'),
'area')
ulst.append(funion)
l_lulc_grs.append(lulc_grs)
# Intersect between all LULC SHPS
ist_shp = []
if len(l_lulc_grs) > 1:
for i in range(len(l_lulc_grs)):
for e in range(i + 1, len(l_lulc_grs)):
ishp = intersection(l_lulc_grs[i],
l_lulc_grs[e],
'lulcint_' + str(i) + '_' + str(e),
api="grass")
if not feat_count(
ishp, gisApi='grass', work=workspace, loc=loc_name):
continue
else:
ist_shp.append(ishp)
if len(ist_shp):
from gasp.gt.gop.genze import dissolve
from gasp.gt.tbl.grs import reset_table
if len(ist_shp) > 1:
from gasp.gt.toshp.mtos import shps_to_shp
# Export shapes
_ist_shp = [
grs_to_shp(s, os.path.join(workspace, loc_name,
s + '.shp'), 'area')
for s in ist_shp
]
# Merge Intersections
merge_shp = shps_to_shp(_ist_shp,
os.path.join(workspace, loc_name,
'merge_shp.shp'),
api='pandas')
# Import GRASS
merge_shp = shp_to_grs(merge_shp, 'merge_shp')
else:
merge_shp = ist_shp[0]
# Dissolve Shape
reset_table(merge_shp, {'refid': 'varchar(2)'}, {'refid': '1'})
overlay_areas = dissolve(merge_shp,
'overlay_areas',
'refid',
api='grass')
# Union Fishnet | Overlay's
union_ovl = intersection(fnet,
overlay_areas,
'ovl_union',
api="grass")
funion_ovl = grs_to_shp(union_ovl,
os.path.join(result, union_ovl + '.shp'),
'area')
ulst.append(funion_ovl)
# Export Tables
return ulst
def joinLines_by_spatial_rel_raster(mainLines, mainId, joinLines, joinCol,
outfile, epsg):
"""
Join Attributes based on a spatial overlap.
An raster based approach
"""
import os
import pandas
from geopandas import GeoDataFrame
from gasp.gt.fmshp import shp_to_obj
from gasp.gt.toshp import df_to_shp
from gasp.gt.toshp.coord import shpext_to_boundshp
from gasp.gt.torst import shp_to_rst
from gasp.g.to import df_to_geodf
from gasp.gt.wenv.grs import run_grass
from gasp.pyt.df.joins import join_dfs
from gasp.pyt.df.agg import df_groupBy
from gasp.pyt.oss import fprop, mkdir
workspace = mkdir(os.path.join(os.path.dirname(mainLines, 'tmp_dt')))
# Create boundary file
boundary = shpext_to_boundshp(mainLines,
os.path.join(workspace, "bound.shp"), epsg)
boundRst = shp_to_rst(boundary,
None,
5,
-99,
os.path.join(workspace, "rst_base.tif"),
epsg=epsg,
api='gdal')
# Start GRASS GIS Session
gbase = run_grass(workspace, location="grs_loc", srs=boundRst)
import grass.script as grass
import grass.script.setup as gsetup
gsetup.init(gbase, workspace, "grs_loc", "PERMANENT")
from gasp.gt.nop.local import combine
from gasp.gt.prop.rst import get_rst_report_data
from gasp.gt.toshp.cff import shp_to_grs, grs_to_shp
from gasp.gt.torst import shp_to_rst
# Add data to GRASS GIS
mainVector = shp_to_grs(mainLines, fprop(mainLines, 'fn', forceLower=True))
joinVector = shp_to_grs(joinLines, fprop(joinLines, 'fn', forceLower=True))
mainRst = shp_to_rst(mainVector,
mainId,
None,
None,
"rst_" + mainVector,
api='pygrass')
joinRst = shp_to_rst(joinVector,
joinCol,
None,
None,
"rst_" + joinVector,
api='pygrass')
combRst = combine(mainRst, joinRst, "combine_rst", api="pygrass")
combine_data = get_rst_report_data(combRst, UNITS="c")
combDf = pandas.DataFrame(combine_data,
columns=["comb_cat", "rst_1", "rst_2", "ncells"])
combDf = combDf[combDf["rst_2"] != '0']
combDf["ncells"] = combDf["ncells"].astype(int)
gbdata = df_groupBy(combDf, ["rst_1"], "MAX", "ncells")
fTable = join_dfs(gbdata, combDf, ["rst_1", "ncells"], ["rst_1", "ncells"])
fTable["rst_2"] = fTable["rst_2"].astype(int)
fTable = df_groupBy(fTable, ["rst_1", "ncells"],
STAT='MIN',
STAT_FIELD="rst_2")
mainLinesCat = grs_to_shp(mainVector,
os.path.join(workspace, mainVector + '.shp'),
'line')
mainLinesDf = shp_to_obj(mainLinesCat)
resultDf = join_dfs(mainLinesDf,
fTable,
"cat",
"rst_1",
onlyCombinations=None)
resultDf.rename(columns={"rst_2": joinCol}, inplace=True)
resultDf = df_to_geodf(resultDf, "geometry", epsg)
df_to_shp(resultDf, outfile)
return outfile
def rst_pnt_to_build(osmdb, pntTable, polyTable, api_db='SQLITE'):
"""
Replace buildings with tag yes using the info in the Points Layer
Only used for URBAN ATLAS and CORINE LAND COVER
"""
import datetime as dt
from gasp.sql.i import row_num as cnt_row
from gasp.sql.fm import q_to_obj
from gasp.gt.toshp.db import dbtbl_to_shp as db_to_shp
from gasp.gql.ovly import feat_within, feat_not_within
from gasp.gt.torst import shp_to_rst
time_a = dt.datetime.now().replace(microsecond=0)
new_build = feat_within(
osmdb,
("(SELECT buildings AS pnt_build, geometry AS pnt_geom "
"FROM {} WHERE buildings IS NOT NULL)").format(pntTable),
"pnt_geom",
("(SELECT buildings AS poly_build, geometry AS poly_geom "
"FROM {} WHERE buildings IS NOT NULL)").format(polyTable),
"poly_geom",
"new_buildings",
inTblCols="pnt_build AS cls",
withinCols="poly_geom AS geometry",
outTblIsFile=None,
apiToUse="OGR_SPATIALITE" if api_db != "POSTGIS" else api_db,
geom_col="geometry")
time_b = dt.datetime.now().replace(microsecond=0)
yes_build = feat_not_within(
osmdb,
("(SELECT buildings AS poly_build, geometry AS poly_geom "
"FROM {} WHERE buildings IS NOT NULL)").format(polyTable),
"poly_geom",
("(SELECT buildings AS pnt_build, geometry AS pnt_geom "
"FROM {} WHERE buildings IS NOT NULL)").format(pntTable),
"pnt_geom",
"yes_builds",
inTblCols="poly_geom AS geometry, 11 AS cls",
outTblIsFile=None,
apiToUse="OGR_SPATIALITE" if api_db != "POSTGIS" else api_db,
geom_col="geometry")
time_c = dt.datetime.now().replace(microsecond=0)
resLayers = {}
N11 = cnt_row(osmdb,
yes_build,
api='psql' if api_db == 'POSTGIS' else 'sqlite')
time_d = dt.datetime.now().replace(microsecond=0)
if N11:
# Data to GRASS GIS
grsBuild11 = db_to_shp(
osmdb,
yes_build,
"geometry",
"yes_builds",
notTable=True,
filterByReg=True,
inDB='psql' if api_db == 'POSTGIS' else 'sqlite',
outShpIsGRASS=True)
time_f = dt.datetime.now().replace(microsecond=0)
# To raster
rstBuild11 = shp_to_rst(grsBuild11,
11,
None,
None,
"rst_builds11",
api="grass")
time_g = dt.datetime.now().replace(microsecond=0)
resLayers[11] = [rstBuild11]
else:
time_f = None
time_g = None
# Add data into GRASS GIS
lulcCls = q_to_obj(
osmdb,
"SELECT cls FROM {} GROUP BY cls".format(new_build),
db_api='psql' if api_db == 'POSTGIS' else 'sqlite').cls.tolist()
timeGasto = {
0: ('intersect', time_b - time_a),
1: ('disjoint', time_c - time_b),
2: ('count_b11', time_d - time_c),
3: None if not time_f else ('import_b11', time_f - time_d),
4: None if not time_g else ('torst_b11', time_g - time_f),
}
tk = 5
for cls in lulcCls:
time_x = dt.datetime.now().replace(microsecond=0)
shp = db_to_shp(osmdb,
new_build,
"geometry",
"nbuild_{}".format(str(cls)),
"cls = {}".format(cls),
notTable=True,
filterByReg=True,
outShpIsGRASS=True)
time_y = dt.datetime.now().replace(microsecond=0)
rstb = shp_to_rst(shp,
int(cls),
None,
None,
"rst_nbuild_{}".format(str(cls)),
api="grass")
time_z = dt.datetime.now().replace(microsecond=0)
if int(cls) == 11 and int(cls) in resLayers:
resLayers[int(cls)].append(rstb)
else:
resLayers[int(cls)] = [rstb]
timeGasto[tk] = ('import_bn{}'.format(cls), time_y - time_x)
timeGasto[tk + 1] = ('torst_bn{}'.format(cls), time_z - time_y)
tk += 2
return resLayers, timeGasto
def roads_sqdb(osmdb, lnhTbl, plTbl, apidb='SQLITE', asRst=None):
"""
Raods procedings using SQLITE
"""
import datetime
from gasp.sql.i import row_num as cnt_rows
from gasp.gt.toshp.db import dbtbl_to_shp as db_to_shp
if apidb=='SQLITE':
from gasp.gql.prox import splite_buffer as st_buffer
else:
from gasp.gql.prox import st_buffer
time_a = datetime.datetime.now().replace(microsecond=0)
NR = cnt_rows(osmdb, lnhTbl, where="roads IS NOT NULL",
api='psql' if apidb == 'POSTGIS' else 'sqlite'
)
time_b = datetime.datetime.now().replace(microsecond=0)
if not NR: return None, {0 : ('count_rows_roads', time_b - time_a)}
NB = cnt_rows(osmdb, plTbl, where="building IS NOT NULL",
api='psql' if apidb == 'POSTGIS' else 'sqlite'
)
time_c = datetime.datetime.now().replace(microsecond=0)
if NB:
from gasp.sql.q import exec_write_q
from gasp.gql.prox import st_near
ROADS_Q = "(SELECT{} roads, bf_roads, geometry FROM {} WHERE roads IS NOT NULL)".format(
"" if apidb == 'SQLITE' else " gid,", lnhTbl)
if apidb == 'SQLITE':
nroads = st_near(
osmdb, ROADS_Q, "geometry",
plTbl, "geometry", "near_roads",
whrNear="building IS NOT NULL", api='splite',
near_col='dist_near'
)
time_d = datetime.datetime.now().replace(microsecond=0)
# Update buffer distance field
exec_write_q(osmdb, [(
"UPDATE near_roads SET bf_roads = CAST(round(dist_near, 0) AS integer) "
"WHERE dist_near >= 1 AND dist_near <= 12"
), (
"UPDATE near_roads SET bf_roads = 1 WHERE dist_near >= 0 AND "
"dist_near < 1"
)], api='sqlite')
time_e = datetime.datetime.now().replace(microsecond=0)
else:
nroads = st_near(
osmdb, ROADS_Q, 'geometry',
"(SELECT * FROM {} WHERE building IS NOT NULL)".format(plTbl),
"geometry", "near_roads", intbl_pk="gid",
until_dist="12", near_col="dist_near"
)
time_d = datetime.datetime.now().replace(microsecond=0)
exec_write_q(osmdb, [(
"UPDATE near_roads SET "
"bf_roads = CAST(round(CAST(dist_near AS numeric), 0) AS integer) "
"WHERE dist_near >= 1 AND dist_near <= 12"
), (
"UPDATE near_roads SET bf_roads = 1 WHERE dist_near >= 0 AND "
"dist_near < 1"
), (
"CREATE INDEX near_dist_idx ON near_roads USING gist (geometry)"
)], api='psql')
time_e = datetime.datetime.now().replace(microsecond=0)
else:
nroads = (
"(SELECT roads, bf_roads, geometry "
"FROM {} WHERE roads IS NOT NULL) AS foo"
).format(lnhTbl)
time_d = None; time_e = None
# Execute Buffer
bfTbl = st_buffer(
osmdb, nroads, "bf_roads", "geometry", "bf_roads",
cols_select="roads", outTblIsFile=None, dissolve="ALL"
)
time_f = datetime.datetime.now().replace(microsecond=0)
# Send data to GRASS GIS
roadsGrs = db_to_shp(
osmdb, bfTbl, "geometry", "froads", notTable=None, filterByReg=True,
inDB="psql" if apidb == 'POSTGIS' else 'sqlite',
outShpIsGRASS=True
)
time_g = datetime.datetime.now().replace(microsecond=0)
if asRst:
from gasp.gt.torst import shp_to_rst
roadsGrs = shp_to_rst(
roadsGrs, int(asRst), None, None, "rst_roads", api="grass"
)
time_h = datetime.datetime.now().replace(microsecond=0)
else:
time_h = None
return roadsGrs, {
0 : ('count_rows_roads', time_b - time_a),
1 : ('count_rows_build', time_c - time_b),
2 : None if not time_d else ('near_analysis', time_d - time_c),
3 : None if not time_e else ('update_buffer_tbl', time_e - time_d),
4 : ('buffer_roads', time_f - time_e if time_e else time_f - time_c),
5 : ('import_roads', time_g - time_f),
6 : None if not time_h else ('roads_to_raster', time_h - time_g)
}
def rst_area(db, polygonTable, UPPER=True, api='SQLITE'):
"""
Select features with area upper than.
A field with threshold is needed in the database.
"""
import datetime
from gasp.sql.fm import q_to_obj
from gasp.gt.toshp.db import dbtbl_to_shp as db_to_grs
from gasp.gt.torst import shp_to_rst
from gasp.sds.osm2lulc import GEOM_AREA
RULE_COL = 'area_upper' if UPPER else 'area_lower'
OPERATOR = " > " if UPPER else " < "
WHR = "{ga} {op} t_{r} AND {r}={cls_}"
# Get Classes
time_a = datetime.datetime.now().replace(microsecond=0)
lulcCls = q_to_obj(
db,
("SELECT {r} FROM {tbl} WHERE {ga} {op} t_{r} GROUP BY {r}").format(
r=RULE_COL, tbl=polygonTable, ga=GEOM_AREA, op=OPERATOR),
db_api='psql' if api == 'POSTGIS' else 'sqlite')[RULE_COL].tolist()
time_b = datetime.datetime.now().replace(microsecond=0)
timeGasto = {0: ('check_cls', time_b - time_a)}
# Import data into GRASS and convert it to raster
clsRst = {}
tk = 1
for cls in lulcCls:
time_x = datetime.datetime.now().replace(microsecond=0)
grsVect = db_to_grs(db,
polygonTable,
"geometry",
"{}_{}".format(RULE_COL, cls),
inDB="psql" if api == 'POSTGIS' else 'sqlite',
where=WHR.format(op=OPERATOR,
r=RULE_COL,
ga=GEOM_AREA,
cls_=cls),
notTable=True,
filterByReg=True,
outShpIsGRASS=True)
time_y = datetime.datetime.now().replace(microsecond=0)
timeGasto[tk] = ('import_{}'.format(cls), time_y - time_x)
grsRst = shp_to_rst(grsVect,
int(cls),
None,
None,
"rst_{}".format(RULE_COL),
api='grass')
time_z = datetime.datetime.now().replace(microsecond=0)
timeGasto[tk + 1] = ('torst_{}'.format(cls), time_z - time_y)
clsRst[int(cls)] = grsRst
tk += 2
return clsRst, timeGasto
def basic_buffer(osmdb, lineTable, dataFolder, apidb='SQLITE'):
"""
Data from Lines table to Polygons using a basic buffering stratagie
"""
import datetime
from gasp.sql.fm import q_to_obj
if apidb == 'POSTGIS':
from gasp.gql.prox import st_buffer
else:
from gasp.gql.prox import splite_buffer as st_buffer
from gasp.gt.torst import shp_to_rst
from gasp.gt.toshp.cff import shp_to_grs
time_a = datetime.datetime.now().replace(microsecond=0)
lulcCls = q_to_obj(
osmdb,
("SELECT basic_buffer FROM {} WHERE basic_buffer IS NOT NULL "
"GROUP BY basic_buffer").format(lineTable),
db_api='psql'
if apidb == 'POSTGIS' else 'sqlite').basic_buffer.tolist()
time_b = datetime.datetime.now().replace(microsecond=0)
timeGasto = {0: ('check_cls', time_b - time_a)}
clsRst = {}
tk = 1
for cls in lulcCls:
# Run BUFFER Tool
time_x = datetime.datetime.now().replace(microsecond=0)
bb_file = st_buffer(osmdb,
lineTable,
"bf_basic_buffer",
"geometry",
os.path.join(
dataFolder,
'bb_rule5_{}.shp'.format(str(int(cls)))),
whrClause="basic_buffer={}".format(str(int(cls))),
outTblIsFile=True,
dissolve="ALL",
cols_select="basic_buffer")
time_y = datetime.datetime.now().replace(microsecond=0)
# Data TO GRASS
grsVect = shp_to_grs(bb_file,
"bb_{}".format(int(cls)),
asCMD=True,
filterByReg=True)
time_z = datetime.datetime.now().replace(microsecond=0)
# Data to Raster
rstVect = shp_to_rst(grsVect,
int(cls),
None,
None,
"rbb_{}".format(int(cls)),
api="grass")
time_w = datetime.datetime.now().replace(microsecond=0)
clsRst[int(cls)] = rstVect
timeGasto[tk] = ('do_buffer_{}'.format(cls), time_y - time_x)
timeGasto[tk + 1] = ('import_{}'.format(cls), time_z - time_y)
timeGasto[tk + 2] = ('torst_{}'.format(cls), time_w - time_z)
tk += 3
return clsRst, timeGasto
def grs_rst_roads(osmdb, lineTbl, polyTbl, dataFolder, LULC_CLS):
"""
Raster Roads for GRASS
"""
import os; import datetime
from gasp.gt.toshp.cff import shp_to_grs
from gasp.gt.toshp.db import dbtbl_to_shp
from gasp.gt.torst import shp_to_rst
from gasp.gql.prox import splite_buffer
from gasp.sql.i import row_num
time_a = datetime.datetime.now().replace(microsecond=0)
NR = row_num(osmdb, lineTbl, where="roads IS NOT NULL", api='sqlite')
time_b = datetime.datetime.now().replace(microsecond=0)
if not NR: return None, {0 : ('count_rows_roads', time_b - time_a)}
roadFile = splite_buffer(
osmdb, lineTbl, "bf_roads", "geometry", 'bfu_roads',
#os.path.join(dataFolder, 'bf_roads.gml'),
whrClause="roads IS NOT NULL",
outTblIsFile=None, dissolve="ALL"
)
time_c = datetime.datetime.now().replace(microsecond=0)
#roadGrs = shp_to_grs(roadFile, "bf_roads", filterByReg=True, asCMD=True)
roadGrs = dbtbl_to_shp(
osmdb, roadFile, "geom", 'bf_roads',
notTable=True, outShpIsGRASS=True, inDB='sqlite'
)
time_d = datetime.datetime.now().replace(microsecond=0)
roadRst = shp_to_rst(
roadGrs, int(LULC_CLS), None, None, "rst_roads", api="grass"
)
time_e = datetime.datetime.now().replace(microsecond=0)
# Builds to GRASS and to RASTER
NB = row_num(osmdb, polyTbl, where="building IS NOT NULL", api='sqlite')
time_f = datetime.datetime.now().replace(microsecond=0)
if NB:
from gasp.gt.nop.alg import rstcalc
from gasp.gt.nop.rcls import set_null, null_to_value
buildsShp = dbtbl_to_shp(
osmdb, polyTbl, "geom", "all_builds",
where="building IS NOT NULL",
notTable=True, outShpIsGRASS=True, inDB='sqlite'
)
time_g = datetime.datetime.now().replace(microsecond=0)
buildsRst = shp_to_rst(
buildsShp, 1, None, None, "rst_builds", api="grass"
)
time_h = datetime.datetime.now().replace(microsecond=0)
# Buildings to nodata | Nodata to 0
null_to_value(buildsRst, 0, as_cmd=True)
time_i = datetime.datetime.now().replace(microsecond=0)
set_null(buildsRst, 1, ascmd=True)
time_j = datetime.datetime.now().replace(microsecond=0)
# Do the math: roads + builds | if builds and roads at the same cell
# cell will be null in the road layer
roadsRes = rstcalc(
"{} + {}".format(roadRst, buildsRst), "cls_roads", api="grass")
time_l = datetime.datetime.now().replace(microsecond=0)
return {LULC_CLS : roadsRes}, {
0 : ('count_rows_roads', time_b - time_a),
1 : ('buffer_roads', time_c - time_b),
2 : ('import_roads', time_d - time_c),
3 : ('roads_to_rst', time_e - time_d),
4 : ('count_build', time_f - time_e),
5 : ('builds_to_grs', time_g - time_f),
6 : ('builds_to_rst', time_h - time_g),
7 : ('bnull_to_val', time_i - time_h),
8 : ('builds_to_nd', time_j - time_i),
9 : ('roads_build_mc', time_l - time_j)
}
else:
return {LULC_CLS : roadRst}, {
0 : ('count_rows_roads', time_b - time_a),
1 : ('buffer_roads', time_c - time_b),
2 : ('import_roads', time_d - time_c),
3 : ('roads_to_rst', time_e - time_d),
4 : ('count_build', time_f - time_e)
}
def match_cellsize_and_clip(rstBands, refRaster, outFolder,
clipShp=None):
"""
Resample images to make them with the same resolution and clip
Good to resample Sentinel bands with more than 10 meters.
Dependencies:
* GRASS GIS;
* GDAL/OGR.
"""
import os
from gasp.gt.prop.prj import get_rst_epsg
from gasp.gt.wenv.grs import run_grass
from gasp.pyt.oss import fprop, mkdir
# Check if outfolder exists
if not os.path.exists(outFolder):
mkdir(outFolder, overwrite=None)
# Get EPSG from refRaster
epsg = get_rst_epsg(refRaster, returnIsProj=None)
"""
Start GRASS GIS Session
"""
GRS_WORKSPACE = mkdir(os.path.join(outFolder, 'grswork'))
grsb = run_grass(
GRS_WORKSPACE, grassBIN='grass78', location='resample',
srs=epsg
)
import grass.script as grass
import grass.script.setup as gsetup
gsetup.init(grsb, GRS_WORKSPACE, 'resample', 'PERMANENT')
"""
Import packages related with GRASS GIS
"""
from gasp.gt.torst import rst_to_grs, grs_to_rst
from gasp.gt.wenv.grs import rst_to_region
from gasp.gt.toshp.cff import shp_to_grs
from gasp.gt.torst import shp_to_rst, grs_to_mask
# Send Ref Raster to GRASS GIS and set region
extRst = rst_to_grs(refRaster, 'ext_rst')
rst_to_region(extRst)
# Import all bands in rstBands
grs_bands = [rst_to_grs(i, fprop(i, 'fn')) for i in rstBands]
if clipShp:
# Add clipShp to GRASS
grs_clip = shp_to_grs(clipShp, fprop(clipShp, 'fn'), asCMD=True)
# SHP to Raster
rstClip = shp_to_rst(
grs_clip, 1, None, 0, 'rst_' + grs_clip,
api='grass'
)
# Set region using
rst_to_region(rstClip)
# Set mask
grs_to_mask(rstClip)
# Export bands
return [grs_to_rst(
i, os.path.join(outFolder, i + '.tif')
) for i in grs_bands]
def get_ref_raster(refBoundBox, folder, cellsize=None):
"""
Get Reference Raster
"""
import os
from gasp.gt.prop.ff import check_isRaster
# Check if refRaster is really a Raster
isRst = check_isRaster(refBoundBox)
if not isRst:
from gasp.gt.prop.ff import check_isShp
if not check_isShp(refBoundBox):
raise ValueError((
'refRaster File has an invalid file format. Please give a file '
'with one of the following extensions: '
'shp, gml, json, kml, tif or img'
))
else:
# We have a shapefile
# Check SRS and see if it is a projected SRS
from gasp.gt.prop.prj import get_epsg_shp
epsg, isProj = get_epsg_shp(refBoundBox, returnIsProj=True)
if not epsg:
raise ValueError('Cannot get epsg code from {}'.format(refBoundBox))
if not isProj:
# A conversion between SRS is needed
from gasp.gt.prj import proj
ref_shp = proj(
refBoundBox, os.path.join(folder, 'tmp_ref_shp.shp'),
outEPSG=3857, inEPSG=epsg, gisApi='ogr2ogr'
)
epsg = 3857
else:
ref_shp = refBoundBox
# Convert to Raster
from gasp.gt.torst import shp_to_rst
refRaster = shp_to_rst(
ref_shp, None, 2 if not cellsize else cellsize,
-1, os.path.join(folder, 'ref_raster.tif'), api='gdal'
)
else:
# We have a raster
from gasp.gt.prop.prj import get_rst_epsg
epsg, isProj = get_rst_epsg(refBoundBox, returnIsProj=True)
if not epsg:
raise ValueError('Cannot get epsg code from {}'.format(refBoundBox))
# Check if Raster has a SRS with projected coordinates
if not isProj:
# We need to reproject raster
from gasp.gt.prj import reprj_rst
refRaster = reprj_rst(
refBoundBox,
os.path.join(folder, 'refrst_3857.tif'),
epsg, 3857
)
epsg = 3857
else:
refRaster = refBoundBox
return refRaster, epsg