def get_capabilities():
"""
Get GetCapabilities XML Data
"""
import os
import xmltodict
from glass.pys.web import get_file
from glass.cons.gsrv import con_gsrv
from glass.pys.char import random_str
from glass.pys.oss import del_file
conparam = con_gsrv()
url = ("{}://{}:{}/geoserver/wms?request=GetCapabilities"
"&service=WMS&version=1.1.1").format(conparam["PROTOCOL"],
conparam["HOST"],
conparam["PORT"])
xml = get_file(
url,
os.path.join(os.path.dirname(os.path.abspath(__file__)),
random_str(10) + '.xml'))
with open(xml) as xmlf:
xmld = xmltodict.parse(xmlf.read())
del_file(xml)
return xmld
def create_db(newdb, overwrite=True, api='psql', use_template=True, dbset='default',
geosupport=None):
"""
Create Relational Database
APIS Available:
* psql;
* sqlite;
"""
if api == 'psql':
from glass.ng.sql.c import sqlcon
from glass.ng.prop.sql import lst_db
from glass.cons.psql import con_psql
conparam = con_psql(db_set=dbset)
dbs = lst_db()
con = sqlcon(None, sqlAPI='psql', dbset=dbset)
cs = con.cursor()
if newdb in dbs and overwrite:
cs.execute("DROP DATABASE {};".format(newdb))
cs.execute("CREATE DATABASE {}{};".format(
newdb, " TEMPLATE={}".format(conparam["TEMPLATE"]) \
if "TEMPLATE" in conparam and use_template else ""
)
)
if not use_template and geosupport:
ge = ['postgis', 'hstore', 'postgis_topology', 'postgis_raster', 'pgrouting']
for e in ge:
cs.execute("CREATE EXTENSION {};".format(e))
cs.close()
con.close()
elif api == 'sqlite':
import os
import sqlite3
try:
if os.path.exists(newdb) and overwrite:
from glass.pys.oss import del_file
del_file(newdb)
conn = sqlite3.connect(newdb)
except Error as e:
print(e)
finally:
conn.close()
else:
raise ValueError('API {} is not available'.format(api))
return newdb
def def_prj(shp, epsg=None, template=None, api='ogr'):
"""
Create/Replace the prj file of a ESRI Shapefile
API options:
* ogr;
* epsgio;
"""
import os
import shutil
prj_file = os.path.join(
os.path.dirname(shp),
'{}.prj'.format(os.path.splitext(os.path.basename(shp))[0]))
if api == 'ogr':
if epsg and not template:
s = osr.SpatialReference()
s.ImportFromEPSG(int(epsg))
s.MorphToESRI()
prj = open(prj_file, 'w')
prj.write(s.ExportToWkt())
prj.close()
return prj_file
elif not epsg and template:
prj_template = '{}.prj'.format(
os.path.splitext(os.path.basename(template))[0])
if not os.path.exists(prj_template):
return 0
try:
os.remove(prj_file)
shutil.copyfile(prj_template, prj_file)
except:
shutil.copyfile(prj_template, prj_file)
elif api == 'epsgio':
if not epsg:
raise ValueError(
("TO use epsgio option, epsg parameter must be given"))
from glass.pys.web import get_file
from glass.pys.oss import del_file
url = 'https://epsg.io/{}.wkt?download'
if os.path.exists(prj_file):
# Delete file
del_file(prj_file)
# Get new prj
get_file(url.format(str(epsg)), prj_file)
return prj_file
def add_rst_store(raster, store_name, workspace):
"""
Create a new store with a raster layer
"""
import os
import requests
from glass.pys.oss import del_file
from glass.pys.Xml import write_xml_tree
from glass.cons.gsrv import con_gsrv
conf = con_gsrv()
url = ('{pro}://{host}:{port}/geoserver/rest/workspaces/{work}/'
'coveragestores?configure=all').format(host=conf['HOST'],
port=conf['PORT'],
work=workspace,
pro=conf['PROTOCOL'])
# Create obj with data to be written in the xml
xmlTree = {
"coverageStore": {
"name": store_name,
"workspace": workspace,
"enabled": "true",
"type": "GeoTIFF",
"url": raster
}
}
treeOrder = {
"coverageStore": ["name", "workspace", "enabled", "type", "url"]
}
# Write XML
xml_file = write_xml_tree(xmlTree,
os.path.join(os.path.dirname(raster),
'new_store.xml'),
nodes_order=treeOrder)
# Send request
with open(xml_file, 'rb') as f:
r = requests.post(url,
data=f,
headers={'content-type': 'text/xml'},
auth=(conf['USER'], conf['PASSWORD']))
del_file(xml_file)
return r
def osm_to_featcls(xmlOsm,
output,
fileFormat='.shp',
useXmlName=None,
outepsg=4326):
"""
OSM to ESRI Shapefile
"""
import os
from glass.g.tbl.filter import sel_by_attr
from glass.pys.oss import fprop, del_file
# Convert xml to sqliteDB
gpkg = osm_to_gpkg(xmlOsm,
os.path.join(output,
fprop(xmlOsm, 'fn') + '.gpkg'))
# sqliteDB to Feature Class
TABLES = {
'points': 'pnt',
'lines': 'lnh',
'multilinestrings': 'mlnh',
'multipolygons': 'poly'
}
for T in TABLES:
sel_by_attr(
gpkg,
"SELECT * FROM {}".format(T),
os.path.join(
output, "{}{}{}".format(
"" if not useXmlName else fprop(xmlOsm, 'fn') + "_",
TABLES[T],
fileFormat if fileFormat[0] == '.' else "." + fileFormat)),
api_gis='ogr',
oEPSG=None if outepsg == 4326 else outepsg,
iEPSG=4326)
# Del temp DB
del_file(gpkg)
return output
def get_rst_report_data(rst, UNITS=None):
"""
Execute r.report and get reported data
"""
import os
from glass.pys.char import random_str
from glass.pys.oss import del_file
REPORT_PATH = raster_report(rst,
os.path.join(
os.path.dirname(os.path.abspath(__file__)),
"{}.txt".format(random_str(6))),
_units=UNITS)
report_data = sanitize_report(REPORT_PATH)
del_file(REPORT_PATH)
return report_data
def merge_dbs(destinationDb, dbs,
tbls_to_merge=None, ignoretbls=None):
"""
Put several database into one
For now works only with PostgreSQL
"""
import os
from glass.pys.oss import fprop, del_file
from glass.ng.sql import psql_cmd
from glass.ng.prop.sql import db_exists, lst_tbl
from glass.ng.sql.db import create_db, drop_db
from glass.ng.sql.tbl import rename_tbl, tbls_to_tbl
from glass.ng.sql.bkup import dump_tbls
from glass.ng.sql.db import restore_tbls
from glass.ng.sql.tbl import distinct_to_table, del_tables
# Prepare database
fdb = fprop(destinationDb, ['fn', 'ff'])
if fdb['fileformat'] != '':
if fdb['fileformat'] == '.sql':
newdb = create_db(fdb['filename'],
overwrite=None, api='psql')
psql_cmd(newdb, destinationDb)
destinationDb = newdb
else:
raise ValueError((
'destinationDb is a file but is not correct. The file must be'
' a SQL Script'
))
else:
if os.path.isdir(destinationDb):
raise ValueError(
'destinationdb is a dir. It must be a DB name or a SQL Script'
)
# Check if destination db exists
if not db_exists(destinationDb):
create_db(destinationDb, overwrite=None, api='psql')
# Check if dbs is a list or a dir
if type(dbs) == list:
dbs = dbs
elif os.path.isdir(dbs):
# list SQL files
from glass.pys.oss import lst_ff
dbs = lst_ff(dbs, file_format='.sql')
else:
raise ValueError(
'''
dbs value should be a list with paths
to sql files or a dir with sql files inside
'''
)
TABLES = {}
for i in range(len(dbs)):
# Check if DB is a file or a db
fp = fprop(dbs[i], ['fn', 'ff'])
DB_NAME = fp['filename']
if fp['fileformat'] == '.sql':
# Create DB
create_db(DB_NAME, overwrite=True, api='psql')
# Restore DB
psql_cmd(DB_NAME, dbs[i])
# List Tables
if not tbls_to_merge:
tbls__ = lst_tbl(DB_NAME, excludeViews=True, api='psql')
tbls = tbls__ if not ignoretbls else [t for t in tbls__ if t not in ignoretbls]
else:
tbls = tbls_to_merge
# Rename Tables
newTbls = rename_tbl(DB_NAME, {tbl : "{}_{}".format(
tbl, str(i)) for tbl in tbls})
for t in range(len(tbls)):
if tbls[t] not in TABLES:
TABLES[tbls[t]] = ["{}_{}".format(tbls[t], str(i))]
else:
TABLES[tbls[t]].append("{}_{}".format(tbls[t], str(i)))
# Dump Tables
SQL_DUMP = os.path.join(
os.path.dirname(os.path.abspath(__file__)),
'tbl_{}.sql'.format(DB_NAME)
); dump_tbls(DB_NAME, newTbls, SQL_DUMP)
# Restore Tables in the destination Database
restore_tbls(destinationDb, SQL_DUMP, newTbls)
# Delete Temp Database
if fp['fileformat'] == '.sql':
drop_db(DB_NAME)
# Delete SQL File
del_file(SQL_DUMP)
# Union of all tables
max_len = max([len(TABLES[t]) for t in TABLES])
for tbl in TABLES:
# Rename original table
NEW_TBL = "{}_{}".format(tbl, max_len)
rename_tbl(destinationDb, {tbl : NEW_TBL})
TABLES[tbl].append(NEW_TBL)
# Union
tbls_to_tbl(destinationDb, TABLES[tbl], tbl + '_tmp')
# Group By
distinct_to_table(destinationDb, tbl + '_tmp', tbl, cols=None)
# Drop unwanted tables
del_tables(destinationDb, TABLES[tbl] + [tbl + '_tmp'])
return destinationDb
1 | R. xxx | 145 | x | ... | x
2 | R. xxx | 146 | x | ... | x
3 | R. xxx | 147 | x | ... | x
4 | R. xxx | 148 | x | ... | x
5 | R. xxx | 149 | x | ... | x
FID should be the first column
"""
import xlrd
from glass.ng.rd import tbl_to_obj
from glass.pys.oss import del_file
from glass.ng.it import dict_to_xls
if overwrite:
del_file(out_path)
# XLS data to dict
data = tbl_to_obj(
xls_path, sheet=sheet, useFirstColAsIndex=True, output='dict'
)
# Split interest_column (attribute)
for fid in data:
for col in data[fid]:
if str(col) == str(interest_column):
str_lst = data[fid][col].split(rule)
data[fid][interest_column+'_1'] = str_lst[0]
data[fid][interest_column+'_2'] = str_lst[1] if len(str_lst) > 1 else ''
def rn_cols(inShp, columns, api="ogr2ogr"):
"""
Rename Columns in Shp
api options:
* ogr2ogr;
* grass;
* pygrass;
"""
if api == "ogr2ogr":
import os
from glass.pys.oss import fprop
from glass.pys.oss import del_file, lst_ff
from glass.g.tbl.filter import sel_by_attr
from glass.g.prop.col import lst_cols
# List Columns
cols = lst_cols(inShp)
for c in cols:
if c in columns:
continue
else:
columns[c] = c
columns["geometry"] = "geometry"
# Get inShp Folder
inshpfld = os.path.dirname(inShp)
# Get inShp Filename and format
inshpname = fprop(inShp, 'fn')
# Temporary output
output = os.path.join(inshpfld, inshpname + '_xtmp.shp')
# Rename columns by selecting data from input
outShp = sel_by_attr(
inShp,
"SELECT {} FROM {}".format(
", ".join(["{} AS {}".format(c, columns[c]) for c in columns]),
inshpname),
output,
api_gis='ogr')
# Delete Original file
infiles = lst_ff(inshpfld, filename=inshpname)
del_file(infiles)
# Rename Output file
oufiles = lst_ff(inshpfld, filename=inshpname + '_xtmp')
for f in oufiles:
os.rename(f, os.path.join(inshpfld, inshpname + fprop(f, 'ff')))
elif api == 'grass':
from glass.pys import execmd
for col in columns:
rcmd = execmd(
("v.db.renamecolumn map={} layer=1 column={},{}").format(
inShp, col, columns[col]))
elif api == 'pygrass':
from grass.pygrass.modules import Module
for col in columns:
func = Module("v.db.renamecolumn",
map=inShp,
column="{},{}".format(col, columns[col]),
quiet=True,
run_=False)
func()
else:
raise ValueError("{} is not available".format(api))
return inShp
def run_viewshed_by_cpu(tid, db, obs, dem, srs,
vis_basename='vis', maxdst=None, obselevation=None):
# Create Database
new_db = create_db("{}_{}".format(db, str(tid)), api='psql')
# Points to Database
pnt_tbl = df_to_db(
new_db, obs, 'pnt_tbl', api='psql',
epsg=srs, geomType='Point', colGeom='geometry')
# Create GRASS GIS Session
workspace = mkdir(os.path.join(
os.path.dirname(dem), 'work_{}'.format(str(tid))
))
loc_name = 'vis_loc'
gbase = run_grass(workspace, location=loc_name, srs=dem)
# Start GRASS GIS Session
import grass.script as grass
import grass.script.setup as gsetup
gsetup.init(gbase, workspace, loc_name, 'PERMANENT')
from glass.g.it.rst import rst_to_grs, grs_to_rst
from glass.g.rst.surf import grs_viewshed
from glass.g.deldt import del_rst
# Send DEM to GRASS GIS
grs_dem = rst_to_grs(dem, 'grs_dem', as_cmd=True)
# Produce Viewshed for each point in obs
for idx, row in obs.iterrows():
# Get Viewshed raster
vrst = grs_viewshed(
grs_dem, (row.geometry.x, row.geometry.y),
'{}_{}'.format(vis_basename, str(row[obs_id])),
max_dist=maxdst, obs_elv=obselevation
)
# Export Raster to File
frst = grs_to_rst(vrst, os.path.join(workspace, vrst + '.tif'))
# Raster to Array
img = gdal.Open(frst)
num = img.ReadAsArray()
# Two Dimension to One Dimension
# Reshape Array
numone = num.reshape(num.shape[0] * num.shape[1])
# Get Indexes with visibility
visnum = np.arange(numone.shape[0]).astype(np.uint32)
visnum = visnum[numone == 1]
# Get Indexes intervals
visint = get_minmax_fm_seq_values(visnum)
# Get rows indexes
_visint = visint.reshape(visint.shape[0] * visint.shape[1])
visrow = _visint / num.shape[1]
visrow = visrow.astype(np.uint32)
# Get cols indexes
viscol = _visint - (visrow * num.shape[1])
# Reshape
visrow = visrow.reshape(visint.shape)
viscol = viscol.reshape(visint.shape)
# Split array
irow, erow = np.vsplit(visrow.T, 1)[0]
icol, ecol = np.vsplit(viscol.T, 1)[0]
# Visibility indexes to Pandas DataFrame
idxnum = np.full(irow.shape, row[obs_id])
visdf = pd.DataFrame({
'pntid' : idxnum, 'rowi' : irow, 'rowe' : erow,
'coli': icol, 'cole' : ecol
})
# Pandas DF to database
# Create Visibility table
df_to_db(
new_db, visdf, vis_basename,
api='psql', colGeom=None,
append=None if not idx else True
)
# Delete all variables
numone = None
visnum = None
visint = None
_visint = None
visrow = None
viscol = None
irow = None
erow = None
icol = None
ecol = None
idxnum = None
visdf = None
del img
# Delete GRASS GIS File
del_rst(vrst)
# Delete TIFF File
del_file(frst)
frst = None
from glass.pys.oss import del_file
from glass.ng.wt.js import dict_to_json
if __name__ == "__main__":
# Get GIT Folder
git_path = os.path.dirname(os.path.abspath(__file__))
# Open settings file
SETTINGS = js.load(open(os.path.join(git_path, 'settings.json'), 'r'))
# Create GLASS constants files
# Add con-postgresql.json
cpsql = os.path.join(git_path, 'core', 'glass', 'cons',
'con-postgresql.json')
del_file(cpsql)
dict_to_json(
{
k: {
"HOST": SETTINGS["POSTGRES"][k]["HOST"],
"PORT": SETTINGS["POSTGRES"][k]["PORT"],
"USER": SETTINGS["POSTGRES"][k]["USER"],
"PASSWORD": SETTINGS["POSTGRES"][k]["PASSWORD"],
"TEMPLATE": SETTINGS["POSTGRES"][k]["TEMPLATE"]
}
for k in SETTINGS["POSTGRES"]
}, cpsql)
# Add con-mysql.json
cmsql = os.path.join(git_path, 'core', 'glass', 'cons', 'con-mysql.json')
def shp_to_psql(dbname,
shpData,
pgTable=None,
api="pandas",
mapCols=None,
srsEpsgCode=None,
encoding="UTF-8",
dbset='default'):
"""
Send Shapefile to PostgreSQL
if api is equal to "pandas" - GeoPandas API will be used;
if api is equal to "shp2pgsql" - shp2pgsql tool will be used.
"""
import os
from glass.pys.oss import fprop
from glass.g.prop.prj import get_shp_epsg
# If defined, srsEpsgCode must be a integer value
if srsEpsgCode:
if type(srsEpsgCode) != int:
raise ValueError('srsEpsgCode should be a integer value')
if api == "pandas":
from glass.ng.rd import tbl_to_obj
from glass.g.wt.sql import df_to_db
from glass.g.prop.feat import get_gtype
elif api == "shp2pgsql":
from glass.pys import execmd
from glass.ng.sql import psql_cmd
from glass.pys.oss import del_file
else:
raise ValueError(
'api value is not valid. options are: pandas and shp2pgsql')
# Check if shp is folder
if os.path.isdir(shpData):
from glass.pys.oss import lst_ff
shapes = lst_ff(shpData, file_format='.shp')
else:
from glass.pys import obj_to_lst
shapes = obj_to_lst(shpData)
epsgs = [get_shp_epsg(i)
for i in shapes] if not srsEpsgCode else [srsEpsgCode]
if None in epsgs:
raise ValueError(
("Cannot obtain EPSG code. Use the srsEpsgCode parameter "
"to specify the EPSG code of your data."))
tables = []
for _i in range(len(shapes)):
# Get Table name
tname = fprop(shapes[_i], 'fn', forceLower=True) if not pgTable else \
pgTable[_i] if type(pgTable) == list else pgTable if len(shapes) == 1 \
else pgTable + '_{}'.format(_i+1)
# Import data
if api == "pandas":
# SHP to DataFrame
df = tbl_to_obj(shapes[_i])
if not mapCols:
df.rename(columns={x: x.lower()
for x in df.columns.values},
inplace=True)
else:
renameD = {
x : mapCols[x].lower() if x in mapCols else \
x.lower() for x in df.columns.values
}
df.rename(columns=renameD, inplace=True)
if "geometry" in df.columns.values:
geomCol = "geometry"
elif "geom" in df.columns.values:
geomCol = "geom"
else:
raise ValueError("No Geometry found in shp")
# GeoDataFrame to PSQL
df_to_db(dbname,
df,
tname,
append=True,
api='psql',
epsg=epsgs[_i] if not srsEpsgCode else srsEpsgCode,
colGeom=geomCol,
geomType=get_gtype(shapes[_i],
name=True,
py_cls=False,
gisApi='ogr'))
else:
sql_script = os.path.join(os.path.dirname(shapes[_i]),
tname + '.sql')
cmd = ('shp2pgsql -I -s {epsg} -W {enc} '
'{shp} public.{name} > {out}').format(
epsg=epsgs[_i] if not srsEpsgCode else srsEpsgCode,
shp=shapes[_i],
name=tname,
out=sql_script,
enc=encoding)
outcmd = execmd(cmd)
psql_cmd(dbname, sql_script, dbcon=dbset)
del_file(sql_script)
tables.append(tname)
return tables[0] if len(tables) == 1 else tables