def foldershp_to_foldershp(inFld, outFld, destiny_file_format,
file_format='.shp', useApi='ogr'):
"""
Execute shp_to_shp for every file in inFld (path to folder)
useApi options:
* ogr;
"""
import os
from gasp.pyt.oss import lst_ff, fprop
if not os.path.exists(outFld):
from gasp.pyt.oss import mkdir
mkdir(outFld)
geo_files = lst_ff(inFld, file_format=file_format)
for f in geo_files:
shp_to_shp(f, os.path.join(outFld, '{}.{}'.format(
fprop(f, 'fn'), destiny_file_format if \
destiny_file_format[0] == '.' else '.' + destiny_file_format
)), gisApi=useApi)
return outFld
def thrd_lulc_by_cell(thrd_id, df_fishnet, l_lulc, result):
# Create folder for this thread
t_folder = mkdir(os.path.join(result, 'thrd_' + str(thrd_id)))
# For each fishnet, do the job
for idx, row in df_fishnet.iterrows():
rf = mkdir(os.path.join(result, fprop(row.fishnet, 'fn')))
lulc_by_cell(int(idx), row.bound, l_lulc, row.fishnet, rf, t_folder)
def summarize_table_fields(table,
outFld,
fld_name_fld_name=None,
__upper=False):
"""
Summarize all fields in a table
"""
import os
from gasp import exec_cmd
from gasp.pyt.oss import mkdir
# List table fields:
fields = lst_fld(table)
# For each field, query data to summarize the values in the field
cmd = 'ogr2ogr {o} {i} -dialect sqlite -sql "{s};"'
if not os.path.exists(outFld):
tmp = mkdir(outFld)
for field in fields:
outTbl = os.path.join(outFld, '{}.dbf'.format(field))
outcmd = exec_cmd(
cmd.format(i=table,
o=outTbl,
s='SELECT {f_}{f} FROM {t} GROUP BY {f}'.format(
f=field,
t=os.path.splitext(os.path.basename(table))[0],
f_='' if not fld_name_fld_name else
'{}, '.format(fld_name_fld_name))))
def osm_from_geofabrik(out_folder):
"""
Get all OSM Files in GeoFabrik
"""
import os
from gasp.to.web import get_file
from gasp.cons.osm import osm_files
from gasp.pyt.oss import mkdir
main_url = "http://download.geofabrik.de/{}/{}-latest.osm.pbf"
for c in osm_files:
if c == 'russia':
get_file(
"http://download.geofabrik.de/russia-latest.osm.pbf",
os.path.join(out_folder, "russia-latest.osm.pbf")
)
continue
# Create folder for that continent
cf = mkdir(os.path.join(out_folder, c))
# Download files of every continent
for _c in osm_files[c]:
get_file(main_url.format(c, _c), os.path.join(
cf,
"{}-latest.osm.pbf".format(_c.replace('/', '_'))
), useWget=True)
return out_folder
def pub_pglyr(workspace, store, pg_table, title=None):
"""
Publish PostGIS table in geoserver
"""
import os; import requests
from gasp.pyt.char import random_str
from gasp.pyt.Xml import write_xml_tree
from gasp.pyt.oss import mkdir, del_folder
from gasp.cons.gsrv import con_gsrv
gs_con = con_gsrv()
# Create folder to write xml
wTmp = mkdir(
os.path.join(
os.path.dirname(os.path.abspath(__file__)), random_str(7)
)
)
# Create obj with data to be written in the xml
lyr_title = "Title {}".format(pg_table) if not title else title
elements = {
"featureType": {
"name" : pg_table,
"title" : lyr_title
}
}
# Write the xml
xml_file = write_xml_tree(
elements,
os.path.join(wTmp, '{}.xml'.format(pg_table))
)
# Create Geoserver Layer
url = (
'{pro}://{host}:{port}/geoserver/rest/workspaces/{wname}/'
'datastores/{store_name}/featuretypes'
).format(
host=gs_con['HOST'], port=gs_con['PORT'], wname=workspace,
store_name=store, pro=gs_con['PROTOCOL']
)
with open(xml_file, 'rb') as __xml:
r = requests.post(
url, data=__xml, headers={'content-type': 'text/xml'},
auth=(gs_con['USER'], gs_con['PASSWORD'])
)
__xml.close()
del_folder(wTmp)
return r
def pub_rst_lyr(layername, datastore, workspace, epsg_code):
"""
Publish a Raster layer
"""
import os; import requests
from gasp.pyt.char import random_str
from gasp.pyt.Xml import write_xml_tree
from gasp.pyt.oss import mkdir, del_folder
from gasp.gt.prop.prj import epsg_to_wkt
from gasp.cons.gsrv import con_gsrv
conf = con_gsrv()
url = (
'{pro}://{host}:{port}/geoserver/rest/workspaces/{work}/'
'coveragestores/{storename}/coverages'
).format(
host=conf['HOST'], port=conf['PORT'],
work=workspace, storename=datastore, pro=conf['PROTOCOL']
)
# Create obj with data to be written in the xml
xmlTree = {
"coverage" : {
"name" : layername,
"title" : layername,
"nativeCRS" : str(epsg_to_wkt(epsg_code)),
"srs" : 'EPSG:{}'.format(str(epsg_code)),
}
}
# Write XML
wTmp = mkdir(os.path.join(
os.path.dirname(os.path.abspath(__file__)),
random_str(7)
))
xml_file = write_xml_tree(
xmlTree, os.path.join(wTmp, 'rst_lyr.xml')
)
# Create layer
with open(xml_file, 'rb') as f:
r = requests.post(
url, data=f,
headers={'content-type': 'text/xml'},
auth=(conf['USER'], conf['PASSWORD'])
)
del_folder(wTmp)
return r
def tbl_fromdb_todb(from_db, to_db, tables, qForTbl=None, api='pandas'):
"""
Send PGSQL Tables from one database to other
"""
from gasp.pyt import obj_to_lst
api = 'pandas' if api != 'pandas' and api != 'psql' else api
tables = obj_to_lst(tables)
if api == 'pandas':
from gasp.sql.fm import q_to_obj
for table in tables:
if not qForTbl:
tblDf = q_to_obj(from_db, "SELECT * FROM {}".format(
table), db_api='psql')
else:
if table not in qForTbl:
tblDf = q_to_obj(from_db, "SELECT * FROM {}".format(
table), db_api='psql')
else:
tblDf = q_to_obj(from_db, qForTbl[table], db_api='psql')
df_to_db(to_db, tblDf, table, api='psql')
else:
import os
from gasp.pyt.oss import mkdir, del_folder
from gasp.sql.fm import dump_tbls
from gasp.sql.to import restore_tbls
tmpFolder = mkdir(
os.path.dirname(os.path.abspath(__file__)), randName=True
)
# Dump
sqlScript = dump_tbls(from_db, tables, os.path.join(
tmpFolder, "tables_data.sql"
))
# Restore
restore_tbls(to_db, sqlScript, tables)
del_folder(tmpFolder)
def prod_dem(_df):
for idx, row in _df.iterrows():
# Get DEM name
dem_f = 'dem_{}{}'.format(str(row.fid), demFormat)
# Get GRASS GIS Workspace
gw = mkdir(os.path.join(ref_folder, 'gw_{}'.format(str(row.fid))),
overwrite=True)
# Get mask
msk = None if not masksFolder else None if pd.isna(row.masks)\
else os.path.join(masksFolder, row.masks)
# Produce DEM
make_dem(gw,
os.path.join(countours_folder, row.countours),
attr,
os.path.join(dem_folder, dem_f),
os.path.join(ref_folder, row.ref),
method="CONTOUR",
cell_size=cellsize,
mask=msk)
def shp_diff_fm_ref(refshp, refcol, shps, out_folder, refrst, db=None):
"""
Check differences between each shp in shps and one reference shape
Dependencies:
- GRASS;
- PostgreSQL with Postgis or GeoPandas;
"""
import os
from gasp.gt.prop.ff import check_isRaster
from gasp.gt.wenv.grs import run_grass
from gasp.pyt.oss import fprop
from gasp.gt.tbl.tomtx import tbl_to_areamtx
# Check if folder exists, if not create it
if not os.path.exists(out_folder):
from gasp.pyt.oss import mkdir
mkdir(out_folder)
# Start GRASS GIS Session
gbase = run_grass(out_folder,
grassBIN='grass78',
location='shpdif',
srs=refrst)
import grass.script.setup as gsetup
gsetup.init(gbase, out_folder, 'shpdif', 'PERMANENT')
from gasp.gt.toshp.cff import shp_to_grs, grs_to_shp
from gasp.gt.torst import rst_to_grs
from gasp.gt.tbl.fld import rn_cols
from gasp.gt.toshp.rst import rst_to_polyg
# Convert to SHAPE if file is Raster
# Rename interest columns
i = 0
lstff = [refshp] + list(shps.keys())
__shps = {}
for s in lstff:
is_rst = check_isRaster(s)
if is_rst:
# To GRASS
rname = fprop(s, 'fn')
inrst = rst_to_grs(s, "rst_" + rname, as_cmd=True)
# To vector
d = rst_to_polyg(inrst,
rname,
rstColumn="lulc_{}".format(str(i)),
gisApi="grass")
else:
# To GRASS
d = shp_to_grs(s, fprop(s, 'fn'), asCMD=True)
# Change name of interest colum
rn_cols(d, {shps[s] if i else refcol: "lulc_{}".format(str(i))},
api="grass")
# Export To Shapefile
if not i:
refshp = grs_to_shp(d, os.path.join(out_folder, d + '.shp'),
'area')
refcol = "lulc_{}".format(str(i))
else:
shp = grs_to_shp(d, os.path.join(out_folder, d + '.shp'), 'area')
__shps[shp] = "lulc_{}".format(str(i))
i += 1
# Union Shapefiles
union_shape = {}
for shp in __shps:
# Optimized Union
sname = fprop(shp, 'fn')
union_shape[shp] = optimized_union_anls(
shp,
refshp,
os.path.join(out_folder, sname + '_un.shp'),
refrst,
os.path.join(out_folder, "wk_" + sname),
multiProcess=True)
# Produce confusion matrices
mtxf = tbl_to_areamtx(union_shape[shp],
"a_" + __shps[shp],
'b_' + refcol,
os.path.join(out_folder, sname + '.xlsx'),
db=db,
with_metrics=True)
return out_folder
def create_pgstore(store, workspace, pg_con):
"""
Create a store for PostGIS data
"""
import os
import requests
from gasp.pyt.char import random_str
from gasp.pyt.Xml import write_xml_tree
from gasp.pyt.oss import mkdir, del_folder
from gasp.cons.gsrv import con_gsrv
gs_con = con_gsrv()
# Create folder to write xml
wTmp = mkdir(
os.path.join(os.path.dirname(os.path.abspath(__file__)),
random_str(7)))
# Create obj with data to be written in the xml
tree_order = {
"dataStore": [
"name", "type", "enabled", "workspace", "connectionParameters",
"__default"
],
"connection:Parameters": [("entry", "key", "port"),
("entry", "key", "user"),
("entry", "key", "passwd"),
("entry", "key", "dbtype"),
("entry", "key", "host"),
("entry", "key", "database"),
("entry", "key", "schema")]
}
xml_tree = {
"dataStore": {
"name": store,
"type": "PostGIS",
"enabled": "true",
"workspace": {
"name": workspace
},
"connectionParameters": {
("entry", "key", "port"): pg_con["PORT"],
("entry", "key", "user"): pg_con["USER"],
("entry", "key", "passwd"): pg_con["PASSWORD"],
("entry", "key", "dbtype"): "postgis",
("entry", "key", "host"): pg_con["HOST"],
("entry", "key", "database"): pg_con["DATABASE"],
("entry", "key", "schema"): "public"
},
"__default": "false"
}
}
# Write xml
xml_file = write_xml_tree(xml_tree,
os.path.join(wTmp, 'pgrest.xml'),
nodes_order=tree_order)
# Create Geoserver Store
url = ('{pro}://{host}:{port}/geoserver/rest/workspaces/{wname}/'
'datastores.xml').format(host=gs_con['HOST'],
port=gs_con['PORT'],
wname=workspace,
pro=gs_con['PROTOCOL'])
with open(xml_file, 'rb') as f:
r = requests.post(url,
data=f,
headers={'content-type': 'text/xml'},
auth=(gs_con['USER'], gs_con['PASSWORD']))
f.close()
del_folder(wTmp)
return r
def joinLines_by_spatial_rel_raster(mainLines, mainId, joinLines, joinCol,
outfile, epsg):
"""
Join Attributes based on a spatial overlap.
An raster based approach
"""
import os
import pandas
from geopandas import GeoDataFrame
from gasp.gt.fmshp import shp_to_obj
from gasp.gt.toshp import df_to_shp
from gasp.gt.toshp.coord import shpext_to_boundshp
from gasp.gt.torst import shp_to_rst
from gasp.g.to import df_to_geodf
from gasp.gt.wenv.grs import run_grass
from gasp.pyt.df.joins import join_dfs
from gasp.pyt.df.agg import df_groupBy
from gasp.pyt.oss import fprop, mkdir
workspace = mkdir(os.path.join(os.path.dirname(mainLines, 'tmp_dt')))
# Create boundary file
boundary = shpext_to_boundshp(mainLines,
os.path.join(workspace, "bound.shp"), epsg)
boundRst = shp_to_rst(boundary,
None,
5,
-99,
os.path.join(workspace, "rst_base.tif"),
epsg=epsg,
api='gdal')
# Start GRASS GIS Session
gbase = run_grass(workspace, location="grs_loc", srs=boundRst)
import grass.script as grass
import grass.script.setup as gsetup
gsetup.init(gbase, workspace, "grs_loc", "PERMANENT")
from gasp.gt.nop.local import combine
from gasp.gt.prop.rst import get_rst_report_data
from gasp.gt.toshp.cff import shp_to_grs, grs_to_shp
from gasp.gt.torst import shp_to_rst
# Add data to GRASS GIS
mainVector = shp_to_grs(mainLines, fprop(mainLines, 'fn', forceLower=True))
joinVector = shp_to_grs(joinLines, fprop(joinLines, 'fn', forceLower=True))
mainRst = shp_to_rst(mainVector,
mainId,
None,
None,
"rst_" + mainVector,
api='pygrass')
joinRst = shp_to_rst(joinVector,
joinCol,
None,
None,
"rst_" + joinVector,
api='pygrass')
combRst = combine(mainRst, joinRst, "combine_rst", api="pygrass")
combine_data = get_rst_report_data(combRst, UNITS="c")
combDf = pandas.DataFrame(combine_data,
columns=["comb_cat", "rst_1", "rst_2", "ncells"])
combDf = combDf[combDf["rst_2"] != '0']
combDf["ncells"] = combDf["ncells"].astype(int)
gbdata = df_groupBy(combDf, ["rst_1"], "MAX", "ncells")
fTable = join_dfs(gbdata, combDf, ["rst_1", "ncells"], ["rst_1", "ncells"])
fTable["rst_2"] = fTable["rst_2"].astype(int)
fTable = df_groupBy(fTable, ["rst_1", "ncells"],
STAT='MIN',
STAT_FIELD="rst_2")
mainLinesCat = grs_to_shp(mainVector,
os.path.join(workspace, mainVector + '.shp'),
'line')
mainLinesDf = shp_to_obj(mainLinesCat)
resultDf = join_dfs(mainLinesDf,
fTable,
"cat",
"rst_1",
onlyCombinations=None)
resultDf.rename(columns={"rst_2": joinCol}, inplace=True)
resultDf = df_to_geodf(resultDf, "geometry", epsg)
df_to_shp(resultDf, outfile)
return outfile
def raster_based(osmdata,
nomenclature,
refRaster,
lulcRst,
overwrite=None,
dataStore=None,
roadsAPI='POSTGIS'):
"""
Convert OSM Data into Land Use/Land Cover Information
An raster based approach.
TODO: Add detailed description
"""
# ************************************************************************ #
# Python Modules from Reference Packages #
# ************************************************************************ #
import datetime
import os
import pandas
import copy
# ************************************************************************ #
# Gasp dependencies #
# ************************************************************************ #
from gasp.pyt.oss import mkdir, fprop
from gasp.gt.prop.ff import check_isRaster
from gasp.gt.prop.prj import get_rst_epsg
from gasp.gt.wenv.grs import run_grass
if roadsAPI == 'POSTGIS':
from gasp.sql.db import create_db
from gasp.gql.to.osm import osm_to_psql
from gasp.sds.osm2lulc.mod2 import roads_sqdb
from gasp.sql.fm import dump_db
from gasp.sql.db import drop_db
else:
from gasp.gt.toshp.osm import osm_to_sqdb
from gasp.sds.osm2lulc.mod2 import grs_rst_roads
from gasp.sds.osm2lulc.utils import osm_project, add_lulc_to_osmfeat, osmlulc_rsttbl
from gasp.sds.osm2lulc.utils import get_ref_raster
from gasp.sds.osm2lulc.mod1 import grs_rst
from gasp.sds.osm2lulc.m3_4 import rst_area
from gasp.sds.osm2lulc.mod5 import basic_buffer
from gasp.sds.osm2lulc.mod6 import rst_pnt_to_build
# ************************************************************************ #
# Global Settings #
# ************************************************************************ #
# Check if input parameters exists!
if not os.path.exists(os.path.dirname(lulcRst)):
raise ValueError('{} does not exist!'.format(os.path.dirname(lulcRst)))
if not os.path.exists(osmdata):
raise ValueError(
'File with OSM DATA ({}) does not exist!'.format(osmdata))
if not os.path.exists(refRaster):
raise ValueError(
'File with reference area ({}) does not exist!'.format(refRaster))
# Check if Nomenclature is valid
nomenclature = "URBAN_ATLAS" if nomenclature != "URBAN_ATLAS" and \
nomenclature != "CORINE_LAND_COVER" and \
nomenclature == "GLOBE_LAND_30" else nomenclature
time_a = datetime.datetime.now().replace(microsecond=0)
workspace = os.path.join(os.path.dirname(lulcRst),
'osmtolulc') if not dataStore else dataStore
# Check if workspace exists
if os.path.exists(workspace):
if overwrite:
mkdir(workspace)
else:
raise ValueError('Path {} already exists'.format(workspace))
else:
mkdir(workspace)
# Get Ref Raster
refRaster, epsg = get_ref_raster(refRaster, workspace, cellsize=2)
from gasp.sds.osm2lulc import PRIORITIES, osmTableData, LEGEND
__priorites = PRIORITIES[nomenclature]
__legend = LEGEND[nomenclature]
time_b = datetime.datetime.now().replace(microsecond=0)
# ************************************************************************ #
# Convert OSM file to SQLITE DB or to POSTGIS DB #
# ************************************************************************ #
if roadsAPI == 'POSTGIS':
osm_db = create_db(fprop(osmdata, 'fn', forceLower=True),
overwrite=True)
osm_db = osm_to_psql(osmdata, osm_db)
else:
osm_db = osm_to_sqdb(osmdata, os.path.join(workspace, 'osm.sqlite'))
time_c = datetime.datetime.now().replace(microsecond=0)
# ************************************************************************ #
# Add Lulc Classes to OSM_FEATURES by rule #
# ************************************************************************ #
add_lulc_to_osmfeat(osm_db, osmTableData, nomenclature, api=roadsAPI)
time_d = datetime.datetime.now().replace(microsecond=0)
# ************************************************************************ #
# Transform SRS of OSM Data #
# ************************************************************************ #
osmTableData = osm_project(
osm_db,
epsg,
api=roadsAPI,
isGlobeLand=None if nomenclature != 'GLOBE_LAND_30' else True)
time_e = datetime.datetime.now().replace(microsecond=0)
# ************************************************************************ #
# Start a GRASS GIS Session #
# ************************************************************************ #
grass_base = run_grass(workspace,
grassBIN='grass78',
location='grloc',
srs=epsg)
import grass.script as grass
import grass.script.setup as gsetup
gsetup.init(grass_base, workspace, 'grloc', 'PERMANENT')
# ************************************************************************ #
# IMPORT SOME GASP MODULES FOR GRASS GIS #
# ************************************************************************ #
from gasp.gt.torst import rst_to_grs, grs_to_rst
from gasp.gt.nop.mos import rsts_to_mosaic
from gasp.gt.wenv.grs import rst_to_region
# ************************************************************************ #
# SET GRASS GIS LOCATION EXTENT #
# ************************************************************************ #
extRst = rst_to_grs(refRaster, 'extent_raster')
rst_to_region(extRst)
time_f = datetime.datetime.now().replace(microsecond=0)
# ************************************************************************ #
# MapResults #
mergeOut = {}
# ************************************************************************ #
# ************************************************************************ #
# 1 - Selection Rule #
# ************************************************************************ #
"""
selOut = {
cls_code : rst_name, ...
}
"""
selOut, timeCheck1 = grs_rst(osm_db,
osmTableData['polygons'],
api=roadsAPI)
for cls in selOut:
mergeOut[cls] = [selOut[cls]]
time_g = datetime.datetime.now().replace(microsecond=0)
# ************************************************************************ #
# 2 - Get Information About Roads Location #
# ************************************************************************ #
"""
roads = {
cls_code : rst_name, ...
}
"""
if roadsAPI != 'POSTGIS':
roads, timeCheck2 = grs_rst_roads(
osm_db, osmTableData['lines'], osmTableData['polygons'], workspace,
1221 if nomenclature != "GLOBE_LAND_30" else 801)
else:
roadCls = 1221 if nomenclature != "GLOBE_LAND_30" else 801
roads, timeCheck2 = roads_sqdb(osm_db,
osmTableData['lines'],
osmTableData['polygons'],
apidb='POSTGIS',
asRst=roadCls)
roads = {roadCls: roads}
for cls in roads:
if cls not in mergeOut:
mergeOut[cls] = [roads[cls]]
else:
mergeOut[cls].append(roads[cls])
time_h = datetime.datetime.now().replace(microsecond=0)
# ************************************************************************ #
# 3 - Area Upper than #
# ************************************************************************ #
"""
auOut = {
cls_code : rst_name, ...
}
"""
if nomenclature != 'GLOBE_LAND_30':
auOut, timeCheck3 = rst_area(osm_db,
osmTableData['polygons'],
UPPER=True,
api=roadsAPI)
for cls in auOut:
if cls not in mergeOut:
mergeOut[cls] = [auOut[cls]]
else:
mergeOut[cls].append(auOut[cls])
time_l = datetime.datetime.now().replace(microsecond=0)
else:
timeCheck3 = None
time_l = None
# ************************************************************************ #
# 4 - Area Lower than #
# ************************************************************************ #
"""
alOut = {
cls_code : rst_name, ...
}
"""
if nomenclature != 'GLOBE_LAND_30':
alOut, timeCheck4 = rst_area(osm_db,
osmTableData['polygons'],
UPPER=None,
api=roadsAPI)
for cls in alOut:
if cls not in mergeOut:
mergeOut[cls] = [alOut[cls]]
else:
mergeOut[cls].append(alOut[cls])
time_j = datetime.datetime.now().replace(microsecond=0)
else:
timeCheck4 = None
time_j = None
# ************************************************************************ #
# 5 - Get data from lines table (railway | waterway) #
# ************************************************************************ #
"""
bfOut = {
cls_code : rst_name, ...
}
"""
bfOut, timeCheck5 = basic_buffer(osm_db,
osmTableData['lines'],
workspace,
apidb=roadsAPI)
for cls in bfOut:
if cls not in mergeOut:
mergeOut[cls] = [bfOut[cls]]
else:
mergeOut[cls].append(bfOut[cls])
time_m = datetime.datetime.now().replace(microsecond=0)
# ************************************************************************ #
# 7 - Assign untagged Buildings to tags #
# ************************************************************************ #
if nomenclature != "GLOBE_LAND_30":
buildsOut, timeCheck7 = rst_pnt_to_build(osm_db,
osmTableData['points'],
osmTableData['polygons'],
api_db=roadsAPI)
for cls in buildsOut:
if cls not in mergeOut:
mergeOut[cls] = buildsOut[cls]
else:
mergeOut[cls] += buildsOut[cls]
time_n = datetime.datetime.now().replace(microsecond=0)
else:
timeCheck7 = None
time_n = datetime.datetime.now().replace(microsecond=0)
# ************************************************************************ #
# Produce LULC Map #
# ************************************************************************ #
"""
Merge all results for one cls into one raster
mergeOut = {
cls_code : [rst_name, rst_name, ...], ...
}
into
mergeOut = {
cls_code : patched_raster, ...
}
"""
for cls in mergeOut:
if len(mergeOut[cls]) == 1:
mergeOut[cls] = mergeOut[cls][0]
else:
mergeOut[cls] = rsts_to_mosaic(mergeOut[cls],
'mosaic_{}'.format(str(cls)),
api="grass")
time_o = datetime.datetime.now().replace(microsecond=0)
"""
Merge all Class Raster using a priority rule
"""
__priorities = PRIORITIES[nomenclature]
lst_rst = []
for cls in __priorities:
if cls not in mergeOut:
continue
else:
lst_rst.append(mergeOut[cls])
outGrs = rsts_to_mosaic(lst_rst,
os.path.splitext(os.path.basename(lulcRst))[0],
api="grass")
time_p = datetime.datetime.now().replace(microsecond=0)
# Ceck if lulc Rst has an valid format
outIsRst = check_isRaster(lulcRst)
if not outIsRst:
from gasp.pyt.oss import fprop
lulcRst = os.path.join(os.path.dirname(lulcRst),
fprop(lulcRst, 'fn') + '.tif')
grs_to_rst(outGrs, lulcRst, as_cmd=True)
osmlulc_rsttbl(
nomenclature,
os.path.join(os.path.dirname(lulcRst),
os.path.basename(lulcRst) + '.vat.dbf'))
time_q = datetime.datetime.now().replace(microsecond=0)
# Dump Database if PostGIS was used
# Drop Database if PostGIS was used
if roadsAPI == 'POSTGIS':
dump_db(osm_db, os.path.join(workspace, osm_db + '.sql'), api='psql')
drop_db(osm_db)
return lulcRst, {
0: ('set_settings', time_b - time_a),
1: ('osm_to_sqdb', time_c - time_b),
2: ('cls_in_sqdb', time_d - time_c),
3: ('proj_data', time_e - time_d),
4: ('set_grass', time_f - time_e),
5: ('rule_1', time_g - time_f, timeCheck1),
6: ('rule_2', time_h - time_g, timeCheck2),
7:
None if not timeCheck3 else ('rule_3', time_l - time_h, timeCheck3),
8:
None if not timeCheck4 else ('rule_4', time_j - time_l, timeCheck4),
9: ('rule_5', time_m - time_j if timeCheck4 else time_m - time_h,
timeCheck5),
10:
None if not timeCheck7 else ('rule_7', time_n - time_m, timeCheck7),
11: ('merge_rst', time_o - time_n),
12: ('priority_rule', time_p - time_o),
13: ('export_rst', time_q - time_p)
}
Set environment variables
"""
"""
Parameters
"""
ref_raster = '/home/jasp/mrgis/landsense_pp/rst_pnse.tif'
osmtolulc = '/home/jasp/mrgis/landsense_pp/res_pnse'
tmp_folder = '/home/jasp/mrgis/landsense_pp/tmp_pnse'
results = '/home/jasp/mrgis/landsense_pp/sample_pnse'
"""
Run Script
"""
# Create Fishnets
fishnets = mkdir(os.path.join(tmp_folder, 'fishnets_shp'))
fnet = nfishnet_fm_rst(ref_raster, 500, 500, fishnets)
# List Fishnet
df_fnet = pd.DataFrame(fnet, columns=['fishnet'])
# List results
lst_lulc = lst_ff(osmtolulc, file_format='.shp')
# Produce boundaries for each fishnet
bf = mkdir(os.path.join(tmp_folder, 'boundaries'))
def produce_bound(row):
row['bound'] = shpext_to_boundshp(
row.fishnet, os.path.join(bf, os.path.basename(row.fishnet)))
def vector_based(osmdata,
nomenclature,
refRaster,
lulcShp,
overwrite=None,
dataStore=None,
RoadsAPI='POSTGIS'):
"""
Convert OSM Data into Land Use/Land Cover Information
An vector based approach.
TODO: Add a detailed description.
RoadsAPI Options:
* GRASS
* SQLITE
* POSTGIS
"""
# ************************************************************************ #
# Python Modules from Reference Packages #
# ************************************************************************ #
import datetime
import os
import copy
# ************************************************************************ #
# GASP dependencies #
# ************************************************************************ #
from gasp.pyt.oss import fprop, mkdir
from gasp.gt.wenv.grs import run_grass
if RoadsAPI == 'POSTGIS':
from gasp.sql.db import create_db
from gasp.gql.to.osm import osm_to_psql
from gasp.sql.db import drop_db
from gasp.sql.fm import dump_db
else:
from gasp.gt.toshp.osm import osm_to_sqdb
from gasp.sds.osm2lulc.utils import osm_project, add_lulc_to_osmfeat, get_ref_raster
from gasp.gt.toshp.mtos import shps_to_shp
from gasp.sds.osm2lulc.mod1 import grs_vector
if RoadsAPI == 'SQLITE' or RoadsAPI == 'POSTGIS':
from gasp.sds.osm2lulc.mod2 import roads_sqdb
else:
from gasp.sds.osm2lulc.mod2 import grs_vec_roads
from gasp.sds.osm2lulc.m3_4 import grs_vect_selbyarea
from gasp.sds.osm2lulc.mod5 import grs_vect_bbuffer
from gasp.sds.osm2lulc.mod6 import vector_assign_pntags_to_build
from gasp.gt.toshp.mtos import same_attr_to_shp
from gasp.gt.prj import def_prj
# ************************************************************************ #
# Global Settings #
# ************************************************************************ #
# Check if input parameters exists!
if not os.path.exists(os.path.dirname(lulcShp)):
raise ValueError('{} does not exist!'.format(os.path.dirname(lulcShp)))
if not os.path.exists(osmdata):
raise ValueError(
'File with OSM DATA ({}) does not exist!'.format(osmdata))
if not os.path.exists(refRaster):
raise ValueError(
'File with reference area ({}) does not exist!'.format(refRaster))
# Check if Nomenclature is valid
nomenclature = "URBAN_ATLAS" if nomenclature != "URBAN_ATLAS" and \
nomenclature != "CORINE_LAND_COVER" and \
nomenclature == "GLOBE_LAND_30" else nomenclature
time_a = datetime.datetime.now().replace(microsecond=0)
# Create workspace for temporary files
workspace = os.path.join(os.path.dirname(lulcShp),
'osmtolulc') if not dataStore else dataStore
# Check if workspace exists
if os.path.exists(workspace):
if overwrite:
mkdir(workspace)
else:
raise ValueError('Path {} already exists'.format(workspace))
else:
mkdir(workspace)
# Get Reference Raster
refRaster, epsg = get_ref_raster(refRaster, workspace, cellsize=10)
from gasp.sds.osm2lulc import osmTableData, PRIORITIES, LEGEND
__priorities = PRIORITIES[nomenclature]
__legend = LEGEND[nomenclature]
time_b = datetime.datetime.now().replace(microsecond=0)
if RoadsAPI != 'POSTGIS':
# ******************************************************************** #
# Convert OSM file to SQLITE DB #
# ******************************************************************** #
osm_db = osm_to_sqdb(osmdata, os.path.join(workspace, 'osm.sqlite'))
else:
# Convert OSM file to POSTGRESQL DB #
osm_db = create_db(fprop(osmdata, 'fn', forceLower=True),
overwrite=True)
osm_db = osm_to_psql(osmdata, osm_db)
time_c = datetime.datetime.now().replace(microsecond=0)
# ************************************************************************ #
# Add Lulc Classes to OSM_FEATURES by rule #
# ************************************************************************ #
add_lulc_to_osmfeat(osm_db,
osmTableData,
nomenclature,
api='SQLITE' if RoadsAPI != 'POSTGIS' else RoadsAPI)
time_d = datetime.datetime.now().replace(microsecond=0)
# ************************************************************************ #
# Transform SRS of OSM Data #
# ************************************************************************ #
osmTableData = osm_project(
osm_db,
epsg,
api='SQLITE' if RoadsAPI != 'POSTGIS' else RoadsAPI,
isGlobeLand=None if nomenclature != 'GLOBE_LAND_30' else True)
time_e = datetime.datetime.now().replace(microsecond=0)
# ************************************************************************ #
# Start a GRASS GIS Session #
# ************************************************************************ #
grass_base = run_grass(workspace,
grassBIN='grass78',
location='grloc',
srs=epsg)
#import grass.script as grass
import grass.script.setup as gsetup
gsetup.init(grass_base, workspace, 'grloc', 'PERMANENT')
# ************************************************************************ #
# IMPORT SOME GASP MODULES FOR GRASS GIS #
# ************************************************************************ #
from gasp.gt.gop.ovlay import erase
from gasp.gt.wenv.grs import rst_to_region
from gasp.gt.gop.genze import dissolve
from gasp.gt.tbl.grs import add_and_update, reset_table, update_table
from gasp.gt.tbl.fld import add_fields
from gasp.gt.toshp.cff import shp_to_grs, grs_to_shp
from gasp.gt.torst import rst_to_grs
# ************************************************************************ #
# SET GRASS GIS LOCATION EXTENT #
# ************************************************************************ #
extRst = rst_to_grs(refRaster, 'extent_raster')
rst_to_region(extRst)
time_f = datetime.datetime.now().replace(microsecond=0)
# ************************************************************************ #
# MapResults #
# ************************************************************************ #
osmShps = []
# ************************************************************************ #
# 1 - Selection Rule #
# ************************************************************************ #
ruleOneShp, timeCheck1 = grs_vector(osm_db,
osmTableData['polygons'],
apidb=RoadsAPI)
osmShps.append(ruleOneShp)
time_g = datetime.datetime.now().replace(microsecond=0)
# ************************************************************************ #
# 2 - Get Information About Roads Location #
# ************************************************************************ #
ruleRowShp, timeCheck2 = roads_sqdb(
osm_db,
osmTableData['lines'],
osmTableData['polygons'],
apidb=RoadsAPI
) if RoadsAPI == 'SQLITE' or RoadsAPI == 'POSTGIS' else grs_vec_roads(
osm_db, osmTableData['lines'], osmTableData['polygons'])
osmShps.append(ruleRowShp)
time_h = datetime.datetime.now().replace(microsecond=0)
# ************************************************************************ #
# 3 - Area Upper than #
# ************************************************************************ #
if nomenclature != "GLOBE_LAND_30":
ruleThreeShp, timeCheck3 = grs_vect_selbyarea(osm_db,
osmTableData['polygons'],
UPPER=True,
apidb=RoadsAPI)
osmShps.append(ruleThreeShp)
time_l = datetime.datetime.now().replace(microsecond=0)
else:
timeCheck3 = None
time_l = None
# ************************************************************************ #
# 4 - Area Lower than #
# ************************************************************************ #
if nomenclature != "GLOBE_LAND_30":
ruleFourShp, timeCheck4 = grs_vect_selbyarea(osm_db,
osmTableData['polygons'],
UPPER=False,
apidb=RoadsAPI)
osmShps.append(ruleFourShp)
time_j = datetime.datetime.now().replace(microsecond=0)
else:
timeCheck4 = None
time_j = None
# ************************************************************************ #
# 5 - Get data from lines table (railway | waterway) #
# ************************************************************************ #
ruleFiveShp, timeCheck5 = grs_vect_bbuffer(osm_db,
osmTableData["lines"],
api_db=RoadsAPI)
osmShps.append(ruleFiveShp)
time_m = datetime.datetime.now().replace(microsecond=0)
# ************************************************************************ #
# 7 - Assign untagged Buildings to tags #
# ************************************************************************ #
if nomenclature != "GLOBE_LAND_30":
ruleSeven11, ruleSeven12, timeCheck7 = vector_assign_pntags_to_build(
osm_db,
osmTableData['points'],
osmTableData['polygons'],
apidb=RoadsAPI)
if ruleSeven11:
osmShps.append(ruleSeven11)
if ruleSeven12:
osmShps.append(ruleSeven12)
time_n = datetime.datetime.now().replace(microsecond=0)
else:
timeCheck7 = None
time_n = datetime.datetime.now().replace(microsecond=0)
# ************************************************************************ #
# Produce LULC Map #
# ************************************************************************ #
"""
Get Shps with all geometries related with one class - One Shape for Classe
"""
_osmShps = []
for i in range(len(osmShps)):
if not osmShps[i]: continue
_osmShps.append(
grs_to_shp(osmShps[i],
os.path.join(workspace, osmShps[i] + '.shp'),
'auto',
lyrN=1,
asCMD=True,
asMultiPart=None))
for shp in _osmShps:
def_prj(os.path.splitext(shp)[0] + '.prj', epsg=epsg, api='epsgio')
_osmShps = same_attr_to_shp(_osmShps,
"cat",
workspace,
"osm_",
resultDict=True)
del osmShps
time_o = datetime.datetime.now().replace(microsecond=0)
"""
Merge all Classes into one feature class using a priority rule
"""
osmShps = {}
for cls in _osmShps:
if cls == '1':
osmShps[1221] = shp_to_grs(_osmShps[cls], "osm_1221", asCMD=True)
else:
osmShps[int(cls)] = shp_to_grs(_osmShps[cls],
"osm_" + cls,
asCMD=True)
# Erase overlapping areas by priority
osmNameRef = copy.deepcopy(osmShps)
for e in range(len(__priorities)):
if e + 1 == len(__priorities): break
if __priorities[e] not in osmShps:
continue
else:
for i in range(e + 1, len(__priorities)):
if __priorities[i] not in osmShps:
continue
else:
osmShps[__priorities[i]] = erase(
osmShps[__priorities[i]],
osmShps[__priorities[e]],
"{}_{}".format(osmNameRef[__priorities[i]], e),
notTbl=True,
api='pygrass')
time_p = datetime.datetime.now().replace(microsecond=0)
# Export all classes
lst_merge = []
a = None
for i in range(len(__priorities)):
if __priorities[i] not in osmShps:
continue
if not a:
reset_table(osmShps[__priorities[i]], {
'cls': 'varchar(5)',
'leg': 'varchar(75)'
}, {
'cls': str(__priorities[i]),
'leg': str(__legend[__priorities[i]])
})
a = 1
else:
add_and_update(osmShps[__priorities[i]], {'cls': 'varchar(5)'},
{'cls': str(__priorities[i])})
ds = dissolve(osmShps[__priorities[i]],
'dl_{}'.format(str(__priorities[i])),
'cls',
api="grass")
add_fields(ds, {'leg': 'varchar(75)'}, api="grass")
update_table(ds, 'leg', str(__legend[__priorities[i]]), 'leg is null')
lst_merge.append(
grs_to_shp(ds,
os.path.join(workspace, "lulc_{}.shp".format(
str(__priorities[i]))),
'auto',
lyrN=1,
asCMD=True,
asMultiPart=None))
time_q = datetime.datetime.now().replace(microsecond=0)
if fprop(lulcShp, 'ff') != '.shp':
lulcShp = os.path.join(os.path.dirname(lulcShp),
fprop(lulcShp, 'fn') + '.shp')
shps_to_shp(lst_merge, lulcShp, api='pandas')
# Check if prj of lulcShp exists and create it if necessary
prj_ff = os.path.splitext(lulcShp)[0] + '.prj'
if not os.path.exists(prj_ff):
def_prj(prj_ff, epsg=epsg, api='epsgio')
time_r = datetime.datetime.now().replace(microsecond=0)
# Dump Database if PostGIS was used
# Drop Database if PostGIS was used
if RoadsAPI == 'POSTGIS':
dump_db(osm_db, os.path.join(workspace, osm_db + '.sql'), api='psql')
drop_db(osm_db)
return lulcShp, {
0: ('set_settings', time_b - time_a),
1: ('osm_to_sqdb', time_c - time_b),
2: ('cls_in_sqdb', time_d - time_c),
3: ('proj_data', time_e - time_d),
4: ('set_grass', time_f - time_e),
5: ('rule_1', time_g - time_f, timeCheck1),
6: ('rule_2', time_h - time_g, timeCheck2),
7:
None if not timeCheck3 else ('rule_3', time_l - time_h, timeCheck3),
8:
None if not timeCheck4 else ('rule_4', time_j - time_l, timeCheck4),
9: ('rule_5', time_m - time_j if timeCheck4 else time_m - time_h,
timeCheck5),
10:
None if not timeCheck7 else ('rule_7', time_n - time_m, timeCheck7),
11: ('disj_cls', time_o - time_n),
12: ('priority_rule', time_p - time_o),
13: ('export_cls', time_q - time_p),
14: ('merge_cls', time_r - time_q)
}
def run_viewshed_by_cpu(tid,
db,
obs,
dem,
srs,
vis_basename='vis',
maxdst=None,
obselevation=None):
# Create Database
new_db = create_db("{}_{}".format(db, str(tid)), api='psql')
# Points to Database
pnt_tbl = df_to_db(new_db,
obs,
'pnt_tbl',
api='psql',
epsg=srs,
geomType='Point',
colGeom='geometry')
# Create GRASS GIS Session
workspace = mkdir(
os.path.join(os.path.dirname(dem), 'work_{}'.format(str(tid))))
loc_name = 'vis_loc'
gbase = run_grass(workspace, location=loc_name, srs=dem)
# Start GRASS GIS Session
import grass.script as grass
import grass.script.setup as gsetup
gsetup.init(gbase, workspace, loc_name, 'PERMANENT')
from gasp.gt.torst import rst_to_grs, grs_to_rst
from gasp.gt.nop.surf import grs_viewshed
from gasp.gt.deldt import del_rst
# Send DEM to GRASS GIS
grs_dem = rst_to_grs(dem, 'grs_dem', as_cmd=True)
# Produce Viewshed for each point in obs
for idx, row in obs.iterrows():
# Get Viewshed raster
vrst = grs_viewshed(grs_dem, (row.geometry.x, row.geometry.y),
'{}_{}'.format(vis_basename, str(row[obs_id])),
max_dist=maxdst,
obs_elv=obselevation)
# Export Raster to File
frst = grs_to_rst(vrst, os.path.join(workspace, vrst + '.tif'))
# Raster to Array
img = gdal.Open(frst)
num = img.ReadAsArray()
# Two Dimension to One Dimension
# Reshape Array
numone = num.reshape(num.shape[0] * num.shape[1])
# Get Indexes with visibility
visnum = np.arange(numone.shape[0]).astype(np.uint32)
visnum = visnum[numone == 1]
# Get rows indexes
visrow = visnum / num.shape[0]
visrow = visrow.astype(np.uint32)
# Get cols indexes
viscol = visnum - (visrow * num.shape[1])
# Visibility indexes to Pandas DataFrame
idxnum = np.full(visrow.shape, row[obs_id])
visdf = pd.DataFrame({
'pntid': idxnum,
'rowi': visrow,
'coli': viscol
})
# Pandas DF to database
# Create Visibility table
df_to_db(new_db,
visdf,
vis_basename,
api='psql',
colGeom=None,
append=None if not idx else True)
# Delete all variables
numone = None
visnum = None
visrow = None
viscol = None
idxnum = None
visdf = None
del img
# Delete GRASS GIS File
del_rst(vrst)
# Delete TIFF File
del_file(frst)
frst = None
def match_cellsize_and_clip(rstBands, refRaster, outFolder,
clipShp=None):
"""
Resample images to make them with the same resolution and clip
Good to resample Sentinel bands with more than 10 meters.
Dependencies:
* GRASS GIS;
* GDAL/OGR.
"""
import os
from gasp.gt.prop.prj import get_rst_epsg
from gasp.gt.wenv.grs import run_grass
from gasp.pyt.oss import fprop, mkdir
# Check if outfolder exists
if not os.path.exists(outFolder):
mkdir(outFolder, overwrite=None)
# Get EPSG from refRaster
epsg = get_rst_epsg(refRaster, returnIsProj=None)
"""
Start GRASS GIS Session
"""
GRS_WORKSPACE = mkdir(os.path.join(outFolder, 'grswork'))
grsb = run_grass(
GRS_WORKSPACE, grassBIN='grass78', location='resample',
srs=epsg
)
import grass.script as grass
import grass.script.setup as gsetup
gsetup.init(grsb, GRS_WORKSPACE, 'resample', 'PERMANENT')
"""
Import packages related with GRASS GIS
"""
from gasp.gt.torst import rst_to_grs, grs_to_rst
from gasp.gt.wenv.grs import rst_to_region
from gasp.gt.toshp.cff import shp_to_grs
from gasp.gt.torst import shp_to_rst, grs_to_mask
# Send Ref Raster to GRASS GIS and set region
extRst = rst_to_grs(refRaster, 'ext_rst')
rst_to_region(extRst)
# Import all bands in rstBands
grs_bands = [rst_to_grs(i, fprop(i, 'fn')) for i in rstBands]
if clipShp:
# Add clipShp to GRASS
grs_clip = shp_to_grs(clipShp, fprop(clipShp, 'fn'), asCMD=True)
# SHP to Raster
rstClip = shp_to_rst(
grs_clip, 1, None, 0, 'rst_' + grs_clip,
api='grass'
)
# Set region using
rst_to_region(rstClip)
# Set mask
grs_to_mask(rstClip)
# Export bands
return [grs_to_rst(
i, os.path.join(outFolder, i + '.tif')
) for i in grs_bands]
def check_shape_diff(SHAPES_TO_COMPARE, OUT_FOLDER, REPORT, DB,
GRASS_REGION_TEMPLATE):
"""
Script to check differences between pairs of Feature Classes
Suponha que temos diversas Feature Classes (FC) e que cada uma delas
possui um determinado atributo; imagine tambem que,
considerando todos os pares possiveis entre estas FC,
se pretende comparar as diferencas na distribuicao dos valores
desse atributo para cada par.
* Dependencias:
- GRASS;
- PostgreSQL;
- PostGIS.
"""
import datetime
import os
import pandas
from gasp.sql.fm import q_to_obj
from gasp.to import db_to_tbl
from gasp.sql.to import df_to_db
from gasp.gt.toshp.cff import shp_to_shp
from gasp.gt.toshp.db import dbtbl_to_shp
from gasp.gt.toshp.rst import rst_to_polyg
from gasp.gql.to import shp_to_psql
from gasp.gql.tomtx import tbl_to_area_mtx
from gasp.gt.prop.ff import check_isRaster
from gasp.pyt.oss import fprop
from gasp.sql.db import create_db
from gasp.sql.tbl import tbls_to_tbl
from gasp.sql.to import q_to_ntbl
from gasp.gql.cln import fix_geom
from gasp.to import db_to_tbl
# Check if folder exists, if not create it
if not os.path.exists(OUT_FOLDER):
from gasp.pyt.oss import mkdir
mkdir(OUT_FOLDER, overwrite=None)
else:
raise ValueError('{} already exists!'.format(OUT_FOLDER))
from gasp.gt.wenv.grs import run_grass
gbase = run_grass(OUT_FOLDER,
grassBIN='grass78',
location='shpdif',
srs=GRASS_REGION_TEMPLATE)
import grass.script as grass
import grass.script.setup as gsetup
gsetup.init(gbase, OUT_FOLDER, 'shpdif', 'PERMANENT')
from gasp.gt.toshp.cff import shp_to_grs, grs_to_shp
from gasp.gt.torst import rst_to_grs
from gasp.gt.tbl.fld import rn_cols
# Convert to SHAPE if file is Raster
i = 0
_SHP_TO_COMPARE = {}
for s in SHAPES_TO_COMPARE:
isRaster = check_isRaster(s)
if isRaster:
# To GRASS
rstName = fprop(s, 'fn')
inRst = rst_to_grs(s, "rst_" + rstName, as_cmd=True)
# To Vector
d = rst_to_polyg(inRst,
rstName,
rstColumn="lulc_{}".format(i),
gisApi="grass")
# Export Shapefile
shp = grs_to_shp(d, os.path.join(OUT_FOLDER, d + '.shp'), "area")
_SHP_TO_COMPARE[shp] = "lulc_{}".format(i)
else:
# To GRASS
grsV = shp_to_grs(s, fprop(s, 'fn'), asCMD=True)
# Change name of column with comparing value
ncol = "lulc_{}".format(str(i))
rn_cols(grsV, {SHAPES_TO_COMPARE[s]: "lulc_{}".format(str(i))},
api="grass")
# Export
shp = grs_to_shp(grsV, os.path.join(OUT_FOLDER, grsV + '_rn.shp'),
"area")
_SHP_TO_COMPARE[shp] = "lulc_{}".format(str(i))
i += 1
SHAPES_TO_COMPARE = _SHP_TO_COMPARE
__SHAPES_TO_COMPARE = SHAPES_TO_COMPARE
# Create database
create_db(DB, api='psql')
""" Union SHAPEs """
UNION_SHAPE = {}
FIX_GEOM = {}
SHPS = list(__SHAPES_TO_COMPARE.keys())
for i in range(len(SHPS)):
for e in range(i + 1, len(SHPS)):
# Optimized Union
print("Union between {} and {}".format(SHPS[i], SHPS[e]))
time_a = datetime.datetime.now().replace(microsecond=0)
__unShp = optimized_union_anls(
SHPS[i],
SHPS[e],
os.path.join(OUT_FOLDER, "un_{}_{}.shp".format(i, e)),
GRASS_REGION_TEMPLATE,
os.path.join(OUT_FOLDER, "work_{}_{}".format(i, e)),
multiProcess=True)
time_b = datetime.datetime.now().replace(microsecond=0)
print(time_b - time_a)
# Rename cols
unShp = rn_cols(
__unShp, {
"a_" + __SHAPES_TO_COMPARE[SHPS[i]]:
__SHAPES_TO_COMPARE[SHPS[i]],
"b_" + __SHAPES_TO_COMPARE[SHPS[e]]:
__SHAPES_TO_COMPARE[SHPS[e]]
})
UNION_SHAPE[(SHPS[i], SHPS[e])] = unShp
# Send data to postgresql
SYNTH_TBL = {}
for uShp in UNION_SHAPE:
# Send data to PostgreSQL
union_tbl = shp_to_psql(DB, UNION_SHAPE[uShp], api='shp2pgsql')
# Produce table with % of area equal in both maps
areaMapTbl = q_to_ntbl(
DB,
"{}_syn".format(union_tbl),
("SELECT CAST('{lulc_1}' AS text) AS lulc_1, "
"CAST('{lulc_2}' AS text) AS lulc_2, "
"round("
"CAST(SUM(g_area) / 1000000 AS numeric), 4"
") AS agree_area, round("
"CAST((SUM(g_area) / MIN(total_area)) * 100 AS numeric), 4"
") AS agree_percentage, "
"round("
"CAST(MIN(total_area) / 1000000 AS numeric), 4"
") AS total_area FROM ("
"SELECT {map1_cls}, {map2_cls}, ST_Area(geom) AS g_area, "
"CASE "
"WHEN {map1_cls} = {map2_cls} "
"THEN 1 ELSE 0 "
"END AS isthesame, total_area FROM {tbl}, ("
"SELECT SUM(ST_Area(geom)) AS total_area FROM {tbl}"
") AS foo2"
") AS foo WHERE isthesame = 1 "
"GROUP BY isthesame").format(
lulc_1=fprop(uShp[0], 'fn'),
lulc_2=fprop(uShp[1], 'fn'),
map1_cls=__SHAPES_TO_COMPARE[uShp[0]],
map2_cls=__SHAPES_TO_COMPARE[uShp[1]],
tbl=union_tbl),
api='psql')
# Produce confusion matrix for the pair in comparison
matrixTbl = tbl_to_area_mtx(DB, union_tbl,
__SHAPES_TO_COMPARE[uShp[0]],
__SHAPES_TO_COMPARE[uShp[1]],
union_tbl + '_mtx')
SYNTH_TBL[uShp] = {"TOTAL": areaMapTbl, "MATRIX": matrixTbl}
# UNION ALL TOTAL TABLES
total_table = tbls_to_tbl(DB, [SYNTH_TBL[k]["TOTAL"] for k in SYNTH_TBL],
'total_table')
# Create table with % of agreement between each pair of maps
mapsNames = q_to_obj(
DB,
("SELECT lulc FROM ("
"SELECT lulc_1 AS lulc FROM {tbl} GROUP BY lulc_1 "
"UNION ALL "
"SELECT lulc_2 AS lulc FROM {tbl} GROUP BY lulc_2"
") AS lu GROUP BY lulc ORDER BY lulc").format(tbl=total_table),
db_api='psql').lulc.tolist()
FLDS_TO_PIVOT = ["agree_percentage", "total_area"]
Q = ("SELECT * FROM crosstab('"
"SELECT CASE "
"WHEN foo.lulc_1 IS NOT NULL THEN foo.lulc_1 ELSE jtbl.tmp1 "
"END AS lulc_1, CASE "
"WHEN foo.lulc_2 IS NOT NULL THEN foo.lulc_2 ELSE jtbl.tmp2 "
"END AS lulc_2, CASE "
"WHEN foo.{valCol} IS NOT NULL THEN foo.{valCol} ELSE 0 "
"END AS agree_percentage FROM ("
"SELECT lulc_1, lulc_2, {valCol} FROM {tbl} UNION ALL "
"SELECT lulc_1, lulc_2, {valCol} FROM ("
"SELECT lulc_1 AS lulc_2, lulc_2 AS lulc_1, {valCol} "
"FROM {tbl}"
") AS tst"
") AS foo FULL JOIN ("
"SELECT lulc_1 AS tmp1, lulc_2 AS tmp2 FROM ("
"SELECT lulc_1 AS lulc_1 FROM {tbl} GROUP BY lulc_1 "
"UNION ALL "
"SELECT lulc_2 AS lulc_1 FROM {tbl} GROUP BY lulc_2"
") AS tst_1, ("
"SELECT lulc_1 AS lulc_2 FROM {tbl} GROUP BY lulc_1 "
"UNION ALL "
"SELECT lulc_2 AS lulc_2 FROM {tbl} GROUP BY lulc_2"
") AS tst_2 WHERE lulc_1 = lulc_2 GROUP BY lulc_1, lulc_2"
") AS jtbl ON foo.lulc_1 = jtbl.tmp1 AND foo.lulc_2 = jtbl.tmp2 "
"ORDER BY lulc_1, lulc_2"
"') AS ct("
"lulc_map text, {crossCols}"
")")
TOTAL_AGREE_TABLE = None
TOTAL_AREA_TABLE = None
for f in FLDS_TO_PIVOT:
if not TOTAL_AGREE_TABLE:
TOTAL_AGREE_TABLE = q_to_ntbl(
DB,
"agreement_table",
Q.format(tbl=total_table,
valCol=f,
crossCols=", ".join([
"{} numeric".format(map_) for map_ in mapsNames
])),
api='psql')
else:
TOTAL_AREA_TABLE = q_to_ntbl(DB,
"area_table",
Q.format(tbl=total_table,
valCol=f,
crossCols=", ".join([
"{} numeric".format(map_)
for map_ in mapsNames
])),
api='psql')
# Union Mapping
UNION_MAPPING = pandas.DataFrame(
[[k[0], k[1], fprop(UNION_SHAPE[k], 'fn')] for k in UNION_SHAPE],
columns=['shp_a', 'shp_b', 'union_shp'])
UNION_MAPPING = df_to_db(DB, UNION_MAPPING, 'union_map', api='psql')
# Export Results
TABLES = [UNION_MAPPING, TOTAL_AGREE_TABLE, TOTAL_AREA_TABLE
] + [SYNTH_TBL[x]["MATRIX"] for x in SYNTH_TBL]
SHEETS = ["union_map", "agreement_percentage", "area_with_data_km"] + [
"{}_{}".format(fprop(x[0], 'fn')[:15],
fprop(x[1], 'fn')[:15]) for x in SYNTH_TBL
]
db_to_tbl(DB, ["SELECT * FROM {}".format(x) for x in TABLES],
REPORT,
sheetsNames=SHEETS,
dbAPI='psql')
return REPORT
def optimized_union_anls(lyr_a,
lyr_b,
outShp,
ref_boundary,
workspace=None,
multiProcess=None):
"""
Optimized Union Analysis
Goal: optimize v.overlay performance for Union operations
"""
import os
from gasp.pyt.oss import fprop, lst_ff
from gasp.pyt.oss import cpu_cores
from gasp.gt.sample import create_fishnet
from gasp.gt.wenv.grs import run_grass
from gasp.gt.toshp import eachfeat_to_newshp
from gasp.gt.toshp.mtos import shps_to_shp
from gasp.gt.attr import split_shp_by_attr
from gasp.gt.torst import shpext_to_rst
from gasp.gt.prop.ext import get_ext
if workspace:
if not os.path.exists(workspace):
from gasp.pyt.oss import mkdir
mkdir(workspace, overwrite=True)
else:
from gasp.pyt.oss import mkdir
workspace = mkdir(os.path.join(os.path.dirname(outShp), "union_work"))
# Create Fishnet
ncpu = cpu_cores()
if ncpu == 12:
nrow = 4
ncol = 3
elif ncpu == 8:
nrow = 4
ncol = 2
else:
nrow = 2
ncol = 2
ext = get_ext(ref_boundary)
width = (ext[1] - ext[0]) / ncol
height = (ext[3] - ext[2]) / nrow
gridShp = create_fishnet(ref_boundary,
os.path.join(workspace, 'ref_grid.shp'),
width,
height,
xy_row_col=None)
# Split Fishnet in several files
cellsShp = eachfeat_to_newshp(gridShp, workspace)
if not multiProcess:
# INIT GRASS GIS Session
grsbase = run_grass(workspace, location="grs_loc", srs=ref_boundary)
import grass.script.setup as gsetup
gsetup.init(grsbase, workspace, "grs_loc", 'PERMANENT')
# Add data to GRASS GIS
from gasp.gt.toshp.cff import shp_to_grs
cellsShp = [
shp_to_grs(shp, fprop(shp, 'fn'), asCMD=True) for shp in cellsShp
]
LYR_A = shp_to_grs(lyr_a, fprop(lyr_a, 'fn'), asCMD=True)
LYR_B = shp_to_grs(lyr_b, fprop(lyr_b, 'fn'), asCMD=True)
# Clip Layers A and B for each CELL in fishnet
LYRS_A = [
clip(LYR_A, cellsShp[x], LYR_A + "_" + str(x), api_gis="grass")
for x in range(len(cellsShp))
]
LYRS_B = [
clip(LYR_B, cellsShp[x], LYR_B + "_" + str(x), api_gis="grass")
for x in range(len(cellsShp))
]
# Union SHPS
UNION_SHP = [
union(LYRS_A[i], LYRS_B[i], "un_{}".format(i), api_gis="grass")
for i in range(len(cellsShp))
]
# Export Data
from gasp.gt.toshp.cff import grs_to_shp
_UNION_SHP = [
grs_to_shp(shp, os.path.join(workspace, shp + ".shp"), "area")
for shp in UNION_SHP
]
else:
def clip_and_union(la, lb, cell, work, proc, output):
ref_rst = shpext_to_rst(cell,
os.path.join(os.path.dirname(cell),
fprop(cell, 'fn') + '.tif'),
cellsize=10)
# Start GRASS GIS Session
loc = "proc_" + str(proc)
grsbase = run_grass(work, location=loc, srs=ref_rst)
import grass.script.setup as gsetup
gsetup.init(grsbase, work, loc, 'PERMANENT')
# Import GRASS GIS modules
from gasp.gt.toshp.cff import shp_to_grs, grs_to_shp
from gasp.gt.prop.feat import feat_count
# Add data to GRASS
a = shp_to_grs(la, fprop(la, 'fn'), filterByReg=True, asCMD=True)
b = shp_to_grs(lb, fprop(lb, 'fn'), filterByReg=True, asCMD=True)
if not feat_count(a, gisApi="grass", work=work, loc=loc):
return
if not feat_count(b, gisApi="grass", work=work, loc=loc):
return
# Clip
a_clip = clip(a,
None,
"{}_clip".format(a),
api_gis="grass",
clip_by_region=True)
b_clip = clip(b,
None,
"{}_clip".format(b),
api_gis="grass",
clip_by_region=True)
# Union
u_shp = union(a_clip,
b_clip,
"un_{}".format(fprop(cell, 'fn')),
api_gis="grass")
# Export
o = grs_to_shp(u_shp, output, "area")
import multiprocessing
thrds = [
multiprocessing.Process(
target=clip_and_union,
name="th-{}".format(i),
args=(lyr_a, lyr_b, cellsShp[i],
os.path.join(workspace, "th_{}".format(i)), i,
os.path.join(workspace, "uniao_{}.shp".format(i))))
for i in range(len(cellsShp))
]
for t in thrds:
t.start()
for t in thrds:
t.join()
ff_shp = lst_ff(workspace, file_format='.shp')
_UNION_SHP = []
for i in range(len(cellsShp)):
p = os.path.join(workspace, "uniao_{}.shp".format(i))
if p in ff_shp:
_UNION_SHP.append(p)
else:
continue
# Merge all union into the same layer
MERGED_SHP = shps_to_shp(_UNION_SHP, outShp, api="ogr2ogr")
return MERGED_SHP
def osm2lulc(osmdata, nomenclature, refRaster, lulcRst,
overwrite=None, dataStore=None, roadsAPI='POSTGIS'):
"""
Convert OSM data into Land Use/Land Cover Information
A matrix based approach
roadsAPI Options:
* SQLITE
* POSTGIS
"""
# ************************************************************************ #
# Python Modules from Reference Packages #
# ************************************************************************ #
import os; import numpy; import datetime
from threading import Thread
from osgeo import gdal
# ************************************************************************ #
# Dependencies #
# ************************************************************************ #
from gasp.gt.fmrst import rst_to_array
from gasp.gt.prop.ff import check_isRaster
from gasp.gt.prop.rst import get_cellsize
from gasp.gt.prop.prj import get_rst_epsg
from gasp.pyt.oss import mkdir, copy_file
from gasp.pyt.oss import fprop
if roadsAPI == 'POSTGIS':
from gasp.sql.db import create_db
from gasp.gql.to.osm import osm_to_psql
from gasp.sds.osm2lulc.mod2 import pg_num_roads
from gasp.sql.fm import dump_db
from gasp.sql.db import drop_db
else:
from gasp.gt.toshp.osm import osm_to_sqdb
from gasp.sds.osm2lulc.mod2 import num_roads
from gasp.sds.osm2lulc.utils import osm_project, add_lulc_to_osmfeat
from gasp.sds.osm2lulc.utils import osmlulc_rsttbl
from gasp.sds.osm2lulc.utils import get_ref_raster
from gasp.sds.osm2lulc.mod1 import num_selection
from gasp.sds.osm2lulc.m3_4 import num_selbyarea
from gasp.sds.osm2lulc.mod5 import num_base_buffer
from gasp.sds.osm2lulc.mod6 import num_assign_builds
from gasp.gt.torst import obj_to_rst
# ************************************************************************ #
# Global Settings #
# ************************************************************************ #
# Check if input parameters exists!
if not os.path.exists(os.path.dirname(lulcRst)):
raise ValueError('{} does not exist!'.format(os.path.dirname(lulcRst)))
if not os.path.exists(osmdata):
raise ValueError('File with OSM DATA ({}) does not exist!'.format(osmdata))
if not os.path.exists(refRaster):
raise ValueError('File with reference area ({}) does not exist!'.format(refRaster))
# Check if Nomenclature is valid
nomenclature = "URBAN_ATLAS" if nomenclature != "URBAN_ATLAS" and \
nomenclature != "CORINE_LAND_COVER" and \
nomenclature == "GLOBE_LAND_30" else nomenclature
time_a = datetime.datetime.now().replace(microsecond=0)
workspace = os.path.join(os.path.dirname(
lulcRst), 'num_osmto') if not dataStore else dataStore
# Check if workspace exists:
if os.path.exists(workspace):
if overwrite:
mkdir(workspace, overwrite=True)
else:
raise ValueError('Path {} already exists'.format(workspace))
else:
mkdir(workspace, overwrite=None)
# Get Ref Raster and EPSG
refRaster, epsg = get_ref_raster(refRaster, workspace, cellsize=2)
CELLSIZE = get_cellsize(refRaster, gisApi='gdal')
from gasp.sds.osm2lulc import osmTableData, PRIORITIES
time_b = datetime.datetime.now().replace(microsecond=0)
# ************************************************************************ #
# Convert OSM file to SQLITE DB or to POSTGIS DB #
# ************************************************************************ #
if roadsAPI == 'POSTGIS':
osm_db = create_db(fprop(
osmdata, 'fn', forceLower=True), overwrite=True)
osm_db = osm_to_psql(osmdata, osm_db)
else:
osm_db = osm_to_sqdb(osmdata, os.path.join(workspace, 'osm.sqlite'))
time_c = datetime.datetime.now().replace(microsecond=0)
# ************************************************************************ #
# Add Lulc Classes to OSM_FEATURES by rule #
# ************************************************************************ #
add_lulc_to_osmfeat(osm_db, osmTableData, nomenclature, api=roadsAPI)
time_d = datetime.datetime.now().replace(microsecond=0)
# ************************************************************************ #
# Transform SRS of OSM Data #
# ************************************************************************ #
osmTableData = osm_project(
osm_db, epsg, api=roadsAPI,
isGlobeLand=None if nomenclature != "GLOBE_LAND_30" else True
)
time_e = datetime.datetime.now().replace(microsecond=0)
# ************************************************************************ #
# MapResults #
# ************************************************************************ #
mergeOut = {}
timeCheck = {}
RULES = [1, 2, 3, 4, 5, 7]
def run_rule(ruleID):
time_start = datetime.datetime.now().replace(microsecond=0)
_osmdb = copy_file(
osm_db, os.path.splitext(osm_db)[0] + '_r{}.sqlite'.format(ruleID)
) if roadsAPI == 'SQLITE' else None
# ******************************************************************** #
# 1 - Selection Rule #
# ******************************************************************** #
if ruleID == 1:
res, tm = num_selection(
_osmdb if _osmdb else osm_db, osmTableData['polygons'], workspace,
CELLSIZE, epsg, refRaster, api=roadsAPI
)
# ******************************************************************** #
# 2 - Get Information About Roads Location #
# ******************************************************************** #
elif ruleID == 2:
res, tm = num_roads(
_osmdb, nomenclature, osmTableData['lines'],
osmTableData['polygons'], workspace, CELLSIZE, epsg,
refRaster
) if _osmdb else pg_num_roads(
osm_db, nomenclature,
osmTableData['lines'], osmTableData['polygons'],
workspace, CELLSIZE, epsg, refRaster
)
# ******************************************************************** #
# 3 - Area Upper than #
# ******************************************************************** #
elif ruleID == 3:
if nomenclature != "GLOBE_LAND_30":
res, tm = num_selbyarea(
osm_db if not _osmdb else _osmdb,
osmTableData['polygons'], workspace,
CELLSIZE, epsg, refRaster, UPPER=True, api=roadsAPI
)
else:
return
# ******************************************************************** #
# 4 - Area Lower than #
# ******************************************************************** #
elif ruleID == 4:
if nomenclature != "GLOBE_LAND_30":
res, tm = num_selbyarea(
osm_db if not _osmdb else _osmdb,
osmTableData['polygons'], workspace,
CELLSIZE, epsg, refRaster, UPPER=False, api=roadsAPI
)
else:
return
# ******************************************************************** #
# 5 - Get data from lines table (railway | waterway) #
# ******************************************************************** #
elif ruleID == 5:
res, tm = num_base_buffer(
osm_db if not _osmdb else _osmdb,
osmTableData['lines'], workspace,
CELLSIZE, epsg, refRaster, api=roadsAPI
)
# ******************************************************************** #
# 7 - Assign untagged Buildings to tags #
# ******************************************************************** #
elif ruleID == 7:
if nomenclature != "GLOBE_LAND_30":
res, tm = num_assign_builds(
osm_db if not _osmdb else _osmdb,
osmTableData['points'], osmTableData['polygons'],
workspace, CELLSIZE, epsg, refRaster, apidb=roadsAPI
)
else:
return
time_end = datetime.datetime.now().replace(microsecond=0)
mergeOut[ruleID] = res
timeCheck[ruleID] = {'total': time_end - time_start, 'detailed': tm}
thrds = []
for r in RULES:
thrds.append(Thread(
name="to_{}".format(str(r)), target=run_rule,
args=(r,)
))
for t in thrds: t.start()
for t in thrds: t.join()
# Merge all results into one Raster
compileResults = {}
for rule in mergeOut:
for cls in mergeOut[rule]:
if cls not in compileResults:
if type(mergeOut[rule][cls]) == list:
compileResults[cls] = mergeOut[rule][cls]
else:
compileResults[cls] = [mergeOut[rule][cls]]
else:
if type(mergeOut[rule][cls]) == list:
compileResults[cls] += mergeOut[rule][cls]
else:
compileResults[cls].append(mergeOut[rule][cls])
time_m = datetime.datetime.now().replace(microsecond=0)
# All Rasters to Array
arrayRst = {}
for cls in compileResults:
for raster in compileResults[cls]:
if not raster:
continue
array = rst_to_array(raster)
if cls not in arrayRst:
arrayRst[cls] = [array.astype(numpy.uint8)]
else:
arrayRst[cls].append(array.astype(numpy.uint8))
time_n = datetime.datetime.now().replace(microsecond=0)
# Sum Rasters of each class
for cls in arrayRst:
if len(arrayRst[cls]) == 1:
sumArray = arrayRst[cls][0]
else:
sumArray = arrayRst[cls][0]
for i in range(1, len(arrayRst[cls])):
sumArray = sumArray + arrayRst[cls][i]
arrayRst[cls] = sumArray
time_o = datetime.datetime.now().replace(microsecond=0)
# Apply priority rule
__priorities = PRIORITIES[nomenclature + "_NUMPY"]
for lulcCls in __priorities:
__lulcCls = rstcls_map(lulcCls)
if __lulcCls not in arrayRst:
continue
else:
numpy.place(arrayRst[__lulcCls], arrayRst[__lulcCls] > 0,
lulcCls
)
for i in range(len(__priorities)):
lulc_i = rstcls_map(__priorities[i])
if lulc_i not in arrayRst:
continue
else:
for e in range(i+1, len(__priorities)):
lulc_e = rstcls_map(__priorities[e])
if lulc_e not in arrayRst:
continue
else:
numpy.place(arrayRst[lulc_e],
arrayRst[lulc_i] == __priorities[i], 0
)
time_p = datetime.datetime.now().replace(microsecond=0)
# Merge all rasters
startCls = 'None'
for i in range(len(__priorities)):
lulc_i = rstcls_map(__priorities[i])
if lulc_i in arrayRst:
resultSum = arrayRst[lulc_i]
startCls = i
break
if startCls == 'None':
return 'NoResults'
for i in range(startCls + 1, len(__priorities)):
lulc_i = rstcls_map(__priorities[i])
if lulc_i not in arrayRst:
continue
resultSum = resultSum + arrayRst[lulc_i]
# Save Result
outIsRst = check_isRaster(lulcRst)
if not outIsRst:
from gasp.pyt.oss import fprop
lulcRst = os.path.join(
os.path.dirname(lulcRst), fprop(lulcRst, 'fn') + '.tif'
)
numpy.place(resultSum, resultSum==0, 1)
obj_to_rst(resultSum, lulcRst, refRaster, noData=1)
osmlulc_rsttbl(nomenclature + "_NUMPY", os.path.join(
os.path.dirname(lulcRst), os.path.basename(lulcRst) + '.vat.dbf'
))
time_q = datetime.datetime.now().replace(microsecond=0)
# Dump Database if PostGIS was used
# Drop Database if PostGIS was used
if roadsAPI == 'POSTGIS':
dump_db(osm_db, os.path.join(workspace, osm_db + '.sql'), api='psql')
drop_db(osm_db)
return lulcRst, {
0 : ('set_settings', time_b - time_a),
1 : ('osm_to_sqdb', time_c - time_b),
2 : ('cls_in_sqdb', time_d - time_c),
3 : ('proj_data', time_e - time_d),
4 : ('rule_1', timeCheck[1]['total'], timeCheck[1]['detailed']),
5 : ('rule_2', timeCheck[2]['total'], timeCheck[2]['detailed']),
6 : None if 3 not in timeCheck else (
'rule_3', timeCheck[3]['total'], timeCheck[3]['detailed']),
7 : None if 4 not in timeCheck else (
'rule_4', timeCheck[4]['total'], timeCheck[4]['detailed']),
8 : ('rule_5', timeCheck[5]['total'], timeCheck[5]['detailed']),
9 : None if 7 not in timeCheck else (
'rule_7', timeCheck[7]['total'], timeCheck[7]['detailed']),
10 : ('rst_to_array', time_n - time_m),
11 : ('sum_cls', time_o - time_n),
12 : ('priority_rule', time_p - time_o),
13 : ('merge_rst', time_q - time_p)
}
def pgtables_to_layer_withStyle_by_col(pgtables,
sldData,
db,
workName=None,
storeName=None,
sldGeom='Polygon',
DATATYPE='QUANTITATIVE',
TABLE_DESIGNATION=None,
COL_DESIGNATION=None,
exclude_cols=None,
pathToSLDfiles=None):
"""
Create a new Geoserver Workspace, create a postgis store and one layer
for each table in 'pgtables'. Each layer will have
multiple styles - one style by column in it.
Parameters:
1) pgtables
- List of PSQL tables to be transformed as Geoserver Layers
2) sldData
- sldData should be a xls table with the styles specifications.
For QUANTITATIVE DATA: The table should have two sheets: one for
colors and other for intervals:
COLORS SHEET STRUCTURE (Sheet Index = 0):
cls_id | R | G | B | STROKE_R | STROKE_G | STROKE_B | STROKE_W
1 | X | X | X | X | X | X | 1
2 | X | X | X | X | X | X | 1
3 | X | X | X | X | X | X | 1
4 | X | X | X | X | X | X | 1
5 | X | X | X | X | X | X | 1
INTERVALS SHEET STRUCTURE (Sheet Index = 1):
| 0 | 1 | 2 | 3 | 4 | 5
col_0 | 0 | 5 | 10 | 15 | 20 | 25
col_1 | 0 | 6 | 12 | 18 | 24 | 30
...
col_n | 0 | 5 | 10 | 15 | 20 | 25
For CATEGORICAL DATA: The table should have only one sheet:
CATEGORICAL SHEET STRUCTURE
| R | G | B | STROKE_R | STROKE_G | STROKE_B | STROKE_W
attr_1 | X | X | X | X | X | X | 1
attr_2 | X | X | X | X | X | X | 1
...
attr_n | X | X | X | X | X | X | 1
3) pgsql_con
- Dict with parameters that will be used to connect to PostgreSQL
d = {
'HOST' : 'localhost', 'PORT' : '5432', 'USER' : 'postgres',
'PASSWORD' : 'admin', 'DATABASE' : 'db_name'
}
4) workName
- String with the name of the Geoserver workspace that will be created
5) storeName
- String with the name of the Geoserver store that will be created
6) geoserver_con
- Dict with parameters to connect to Geoserver
7) sldGeom
- Data Geometry. At the moment, the options 'Polygon' and 'Line' are
valid.
8) DATATYPE='QUANTITATIVE' | 'CATEGORICAL'
9) TABLE_DESIGNATION
- Table with the association between pgtables name and the designation
to be used to name the Geoserver Layer.
10) COL_DESIGNATION
- Table xls with association between each column and one
style that will be used to present the information of that column.
The style designation could not have blank characters.
11) exclude_cols
- One style will be created for each column in one pgtable. The columns
in 'exclude_cols' will not have a style.
12) pathToSLDfiles
- Absolute path to the folder where the SLD (Style Layer Descriptor)
will be stored.
"""
import os
from gasp.pyt import obj_to_lst
from gasp.fm import tbl_to_obj
from gasp.pyt.oss import mkdir
from gasp.sql.i import cols_name
from gasp.web.geosrv.ws import create_ws
from gasp.web.geosrv.stores import create_pgstore
from gasp.web.geosrv.lyrs import pub_pglyr
from gasp.web.geosrv.sty import create_style
from gasp.web.geosrv.sty import lst_styles
from gasp.web.geosrv.sty import del_style
from gasp.web.geosrv.sty import assign_style_to_layer
from gasp.web.geosrv.sld import write_sld
# Sanitize PGtables
pgtables = obj_to_lst(pgtables)
if not pgtables:
raise ValueError('pgtables value is not valid!')
exclude_cols = obj_to_lst(exclude_cols)
STY_DESIGNATION = tbl_to_obj(COL_DESIGNATION,
useFirstColAsIndex=True,
output='dict',
colsAsArray=True) if COL_DESIGNATION else None
LYR_DESIGNATION = tbl_to_obj(
TABLE_DESIGNATION,
useFirstColAsIndex=True,
output='dict',
colsAsArray=True) if TABLE_DESIGNATION else None
# Get intervals and colors data
if DATATYPE == 'QUANTITATIVE':
if os.path.exists(sldData):
FF = os.path.splitext(sldData)[1]
if FF == '.xls' or FF == '.xlsx':
colorsDict = tbl_to_obj(sldData,
sheet=0,
useFirstColAsIndex=True,
output='dict')
intervalsDict = tbl_to_obj(sldData,
sheet=1,
useFirstColAsIndex=True,
output='dict')
else:
raise ValueError(
('At the moment, for DATATYPE QUANTITATIVE, sldData '
'has to be a xls table'))
else:
raise ValueError(('{} is not a valid file').format(sldData))
elif DATATYPE == 'CATEGORICAL':
if os.path.exists(sldData):
if os.path.splitext(sldData)[1] == 'xls':
colorsDict = tbl_to_obj(sldData,
sheet=0,
useFirstColAsIndex=True,
output='dict')
else:
raise ValueError(
('At the moment, for DATATYPE CATEGORICAL, sldData '
'has to be a xls table'))
else:
raise ValueError(('{} is not a valid file').format(sldData))
else:
raise ValueError('{} is not avaiable at the moment'.format(DATATYPE))
# Create Workspace
workName = 'w_{}'.format(db) if not workName else workName
create_ws(workName, overwrite=True)
# Create Store
storeName = db if not storeName else storeName
create_pgstore(storeName, workName, db)
# Create folder for sld's
wTmp = mkdir(
os.path.join(os.path.dirname(sldData),
'sldFiles')) if not pathToSLDfiles else pathToSLDfiles
# List styles in geoserver
STYLES = lst_styles()
# For each table in PGTABLES
for PGTABLE in pgtables:
# Publish Postgis table
TITLE = None if not LYR_DESIGNATION else LYR_DESIGNATION[PGTABLE][0]
pub_pglyr(workName, storeName, PGTABLE, title=TITLE)
# List PGTABLE columns
pgCols = cols_name(db, PGTABLE)
# For each column
for col in pgCols:
if exclude_cols and col in exclude_cols:
continue
STYLE_NAME = '{}_{}'.format(
PGTABLE, STY_DESIGNATION[col]
[0]) if STY_DESIGNATION else '{}_{}'.format(PGTABLE, col)
if STYLE_NAME in STYLES:
del_style(STYLE_NAME)
# Create Object with association between colors and intervals
d = {}
OPACITY = str(colorsDict[1]['OPACITY'])
for i in colorsDict:
d[i] = {
'R': colorsDict[i]['R'],
'G': colorsDict[i]['G'],
'B': colorsDict[i]['B']
}
if DATATYPE == 'QUANTITATIVE':
d[i]['min'] = intervalsDict[col][i - 1]
d[i]['max'] = intervalsDict[col][i]
if 'STROKE_R' in colorsDict[i] and \
'STROKE_G' in colorsDict[i] and \
'STROKE_B' in colorsDict[i]:
d[i]['STROKE'] = {
'R': colorsDict[i]['STROKE_R'],
'G': colorsDict[i]['STROKE_G'],
'B': colorsDict[i]['STROKE_B']
}
if 'STROKE_W' in colorsDict[i]:
d[i]['STROKE']['WIDTH'] = colorsDict[i]['STROKE_W']
# Create SLD
sldFile = write_sld(col,
d,
os.path.join(wTmp, '{}.sld'.format(col)),
geometry=sldGeom,
DATA=DATATYPE,
opacity=OPACITY)
# Create Style
create_style(STYLE_NAME, sldFile)
# Apply SLD
assign_style_to_layer(STYLE_NAME, PGTABLE)