def get_capabilities():
"""
Get GetCapabilities XML Data
"""
import os
import xmltodict
from gasp.to.web import get_file
from gasp.cons.gsrv import con_gsrv
from gasp.pyt.char import random_str
from gasp.pyt.oss import del_file
conparam = con_gsrv()
url = ("{}://{}:{}/geoserver/wms?request=GetCapabilities"
"&service=WMS&version=1.1.1").format(conparam["PROTOCOL"],
conparam["HOST"],
conparam["PORT"])
xml = get_file(
url,
os.path.join(os.path.dirname(os.path.abspath(__file__)),
random_str(10) + '.xml'))
with open(xml) as xmlf:
xmld = xmltodict.parse(xmlf.read())
del_file(xml)
return xmld
def def_prj(shp, epsg=None, template=None, api='ogr'):
"""
Create/Replace the prj file of a ESRI Shapefile
API options:
* ogr;
* epsgio;
"""
import os
import shutil
prj_file = os.path.join(os.path.dirname(shp), '{}.prj'.format(
os.path.splitext(os.path.basename(shp))[0]
))
if api == 'ogr':
if epsg and not template:
s = osr.SpatialReference()
s.ImportFromEPSG(int(epsg))
s.MorphToESRI()
prj = open(prj_file, 'w')
prj.write(s.ExportToWkt())
prj.close()
return prj_file
elif not epsg and template:
prj_template = '{}.prj'.format(
os.path.splitext(os.path.basename(template))[0]
)
if not os.path.exists(prj_template):
return 0
try:
os.remove(prj_file)
shutil.copyfile(prj_template, prj_file)
except:
shutil.copyfile(prj_template, prj_file)
elif api == 'epsgio':
if not epsg:
raise ValueError((
"TO use epsgio option, epsg parameter must be given"
))
from gasp.to.web import get_file
from gasp.pyt.oss import del_file
url = 'https://epsg.io/{}.wkt?download'
if os.path.exists(prj_file):
# Delete file
del_file(prj_file)
# Get new prj
get_file(url.format(str(epsg)), prj_file)
return prj_file
def create_db(newdb, overwrite=True, api='psql', use_template=True):
"""
Create Relational Database
APIS Available:
* psql;
* sqlite;
"""
if api == 'psql':
from gasp.sql.c import sqlcon
from gasp.sql.i import lst_db
from gasp.cons.psql import con_psql
conparam = con_psql()
dbs = lst_db()
con = sqlcon(None, sqlAPI='psql')
cs = con.cursor()
if newdb in dbs and overwrite:
cs.execute("DROP DATABASE {};".format(newdb))
cs.execute("CREATE DATABASE {}{};".format(
newdb, " TEMPLATE={}".format(conparam["TEMPLATE"]) \
if "TEMPLATE" in conparam and use_template else ""
)
)
cs.close()
con.close()
elif api == 'sqlite':
import os
import sqlite3
try:
if os.path.exists(newdb) and overwrite:
from gasp.pyt.oss import del_file
del_file(newdb)
conn = sqlite3.connect(newdb)
except Error as e:
print(e)
finally:
conn.close()
else:
raise ValueError('API {} is not available'.format(api))
return newdb
def add_rst_store(raster, store_name, workspace):
"""
Create a new store with a raster layer
"""
import os
import requests
from gasp.pyt.oss import del_file
from gasp.pyt.Xml import write_xml_tree
from gasp.cons.gsrv import con_gsrv
conf = con_gsrv()
url = ('{pro}://{host}:{port}/geoserver/rest/workspaces/{work}/'
'coveragestores?configure=all').format(host=conf['HOST'],
port=conf['PORT'],
work=workspace,
pro=conf['PROTOCOL'])
# Create obj with data to be written in the xml
xmlTree = {
"coverageStore": {
"name": store_name,
"workspace": workspace,
"enabled": "true",
"type": "GeoTIFF",
"url": raster
}
}
treeOrder = {
"coverageStore": ["name", "workspace", "enabled", "type", "url"]
}
# Write XML
xml_file = write_xml_tree(xmlTree,
os.path.join(os.path.dirname(raster),
'new_store.xml'),
nodes_order=treeOrder)
# Send request
with open(xml_file, 'rb') as f:
r = requests.post(url,
data=f,
headers={'content-type': 'text/xml'},
auth=(conf['USER'], conf['PASSWORD']))
del_file(xml_file)
return r
def osm_to_featcls(xmlOsm,
output,
fileFormat='.shp',
useXmlName=None,
outepsg=4326):
"""
OSM to ESRI Shapefile
"""
import os
from gasp.gt.attr import sel_by_attr
from gasp.pyt.oss import fprop, del_file
# Convert xml to sqliteDB
gpkg = osm_to_gpkg(xmlOsm,
os.path.join(output,
fprop(xmlOsm, 'fn') + '.gpkg'))
# sqliteDB to Feature Class
TABLES = {
'points': 'pnt',
'lines': 'lnh',
'multilinestrings': 'mlnh',
'multipolygons': 'poly'
}
for T in TABLES:
sel_by_attr(
gpkg,
"SELECT * FROM {}".format(T),
os.path.join(
output, "{}{}{}".format(
"" if not useXmlName else fprop(xmlOsm, 'fn') + "_",
TABLES[T],
fileFormat if fileFormat[0] == '.' else "." + fileFormat)),
api_gis='ogr',
oEPSG=None if outepsg == 4326 else outepsg,
iEPSG=4326)
# Del temp DB
del_file(gpkg)
return output
def get_rst_report_data(rst, UNITS=None):
"""
Execute r.report and get reported data
"""
import os
from gasp.pyt.char import random_str
from gasp.pyt.oss import del_file
REPORT_PATH = raster_report(rst, os.path.join(
os.path.dirname(os.path.abspath(__file__)),
"{}.txt".format(random_str(6))
), _units=UNITS)
report_data = sanitize_report(REPORT_PATH)
del_file(REPORT_PATH)
return report_data
def osm_countries(out_shp):
"""
Get the boundary representing areas with OSM data in every
OSM PBF country file in GeoFabrik
"""
import os
import codecs
import pandas as pd
import geopandas as gpd
from shapely.geometry import MultiPolygon, Polygon
from gasp.cons.osm import osm_files
from gasp.to.web import get_file
from gasp.gt.toshp import df_to_shp
from gasp.pyt.oss import del_file
url = 'http://download.geofabrik.de/{}/{}.poly'
url_russia = 'http://download.geofabrik.de/russia.poly'
countries_boundaries = []
for c in osm_files:
for _c in osm_files[c]:
# Get poly file
ff = get_file(
url.format(c, _c) if c != 'russia' else url_russia,
os.path.join(
os.path.dirname(out_shp),
"{}_{}.poly".format(c, _c.replace('/', '_'))
if c != 'russia' else "russia.poly"))
# Get Polygon Coordinates
in_ring = False
coords = []
with open(ff, 'r') as txt:
i = 0
for l in txt.readlines():
if i == 0:
# first line is junk
i += 1
continue
elif i == 1:
# second line is the first polygon ring.
coords.append([[], []])
ring = coords[-1][0]
in_ring = True
i += 1
elif in_ring and l.strip() == 'END':
# we are at the end of a ring, perhaps with more to come.
in_ring = False
elif in_ring:
# we are in a ring and picking up new coordinates.
pnt = l.strip().split(' ')
ring.append((float(pnt[0]), float(pnt[1])))
elif not in_ring and l.strip() == 'END':
# we are at the end of the whole polygon.
break
elif not in_ring and l.startswith('!'):
# we are at the start of a polygon part hole.
coords[-1][1].append([])
ring = coords[-1][1][-1]
in_ring = True
elif not in_ring:
# we are at the start of a polygon part.
coords.append([[], []])
ring = coords[-1][0]
in_ring = True
polygon = MultiPolygon(coords)
countries_boundaries.append([c, _c, polygon])
del_file(ff)
countries_boundaries = gpd.GeoDataFrame(pd.DataFrame(
countries_boundaries, columns=['continent', 'country', 'geometry']),
crs={'init': 'epsg:4326'},
geometry='geometry')
return df_to_shp(countries_boundaries, out_shp)
def merge_dbs(destinationDb, dbs,
tbls_to_merge=None, ignoreCols=None):
"""
Put several database into one
For now works only with PostgreSQL
"""
import os
from gasp.pyt.oss import fprop, del_file
from gasp.sql import psql_cmd
from gasp.sql.i import db_exists, lst_tbl
from gasp.sql.db import create_db, drop_db
from gasp.sql.tbl import rename_tbl, tbls_to_tbl
from gasp.sql.fm import dump_tbls
from gasp.sql.to import restore_tbls
from gasp.sql.tbl import distinct_to_table, del_tables
# Prepare database
fdb = fprop(destinationDb, ['fn', 'ff'])
if os.path.isfile(destinationDb):
if fdb['fileformat'] == '.sql':
newdb = create_db(fdb['filename'],
overwrite=True, api='psql')
psql_cmd(newdb, destinationDb)
destinationDb = newdb
else:
raise ValueError((
'destinationDb is a file but is not correct. The file must be'
' a SQL Script'
))
else:
# Check if destination db exists
if not db_exists(destinationDb):
create_db(destinationDb, overwrite=None, api='psql')
# Check if dbs is a list or a dir
if type(dbs) == list:
dbs = dbs
elif os.path.isdir(dbs):
# list SQL files
from gasp.pyt.oss import lst_ff
dbs = lst_ff(dbs, file_format='.sql')
else:
raise ValueError(
'''
dbs value should be a list with paths
to sql files or a dir with sql files inside
'''
)
TABLES = {}
for i in range(len(dbs)):
# Create DB
DB_NAME = fprop(dbs[i], 'fn')
create_db(DB_NAME, overwrite=True, api='psql')
# Restore DB
psql_cmd(DB_NAME, dbs[i])
# List Tables
if not tbls_to_merge:
tbls__ = lst_tbl(DB_NAME, excludeViews=True, api='psql')
tbls = [t for t in tbls__ if t not in ignoreCols]
else:
tbls = tbls_to_merge
# Rename Tables
newTbls = rename_tbl(DB_NAME, {tbl : "{}_{}".format(
tbl, str(i)) for tbl in tbls})
for t in range(len(tbls)):
if tbls[t] not in TABLES:
TABLES[tbls[t]] = ["{}_{}".format(tbls[t], str(i))]
else:
TABLES[tbls[t]].append("{}_{}".format(tbls[t], str(i)))
# Dump Tables
SQL_DUMP = os.path.join(
os.path.dirname(dbs[i]), 'tbl_{}.sql'.format(DB_NAME)
); dump_tbls(DB_NAME, newTbls, SQL_DUMP)
# Restore Tables in the destination Database
restore_tbls(destinationDb, SQL_DUMP, newTbls)
# Delete Temp Database
drop_db(DB_NAME)
# Delete SQL File
del_file(SQL_DUMP)
# Union of all tables
max_len = max([len(TABLES[t]) for t in TABLES])
for tbl in TABLES:
# Rename original table
NEW_TBL = "{}_{}".format(tbl, max_len)
rename_tbl(destinationDb, {tbl : NEW_TBL})
TABLES[tbl].append(NEW_TBL)
# Union
tbls_to_tbl(destinationDb, TABLES[tbl], tbl + '_tmp')
# Group By
distinct_to_table(destinationDb, tbl + '_tmp', tbl, cols=None)
# Drop unwanted tables
del_tables(destinationDb, TABLES[tbl] + [tbl + '_tmp'])
return destinationDb
1 | R. xxx | 145 | x | ... | x
2 | R. xxx | 146 | x | ... | x
3 | R. xxx | 147 | x | ... | x
4 | R. xxx | 148 | x | ... | x
5 | R. xxx | 149 | x | ... | x
FID should be the first column
"""
import xlrd
from gasp.fm import tbl_to_obj
from gasp.pyt.oss import del_file
from gasp.to import dict_to_xls
if overwrite:
del_file(out_path)
# XLS data to dict
data = tbl_to_obj(
xls_path, sheet=sheet, useFirstColAsIndex=True, output='dict'
)
# Split interest_column (attribute)
for fid in data:
for col in data[fid]:
if str(col) == str(interest_column):
str_lst = data[fid][col].split(rule)
data[fid][interest_column+'_1'] = str_lst[0]
data[fid][interest_column+'_2'] = str_lst[1] if len(str_lst) > 1 else ''
def run_viewshed_by_cpu(tid,
db,
obs,
dem,
srs,
vis_basename='vis',
maxdst=None,
obselevation=None):
# Create Database
new_db = create_db("{}_{}".format(db, str(tid)), api='psql')
# Points to Database
pnt_tbl = df_to_db(new_db,
obs,
'pnt_tbl',
api='psql',
epsg=srs,
geomType='Point',
colGeom='geometry')
# Create GRASS GIS Session
workspace = mkdir(
os.path.join(os.path.dirname(dem), 'work_{}'.format(str(tid))))
loc_name = 'vis_loc'
gbase = run_grass(workspace, location=loc_name, srs=dem)
# Start GRASS GIS Session
import grass.script as grass
import grass.script.setup as gsetup
gsetup.init(gbase, workspace, loc_name, 'PERMANENT')
from gasp.gt.torst import rst_to_grs, grs_to_rst
from gasp.gt.nop.surf import grs_viewshed
from gasp.gt.deldt import del_rst
# Send DEM to GRASS GIS
grs_dem = rst_to_grs(dem, 'grs_dem', as_cmd=True)
# Produce Viewshed for each point in obs
for idx, row in obs.iterrows():
# Get Viewshed raster
vrst = grs_viewshed(grs_dem, (row.geometry.x, row.geometry.y),
'{}_{}'.format(vis_basename, str(row[obs_id])),
max_dist=maxdst,
obs_elv=obselevation)
# Export Raster to File
frst = grs_to_rst(vrst, os.path.join(workspace, vrst + '.tif'))
# Raster to Array
img = gdal.Open(frst)
num = img.ReadAsArray()
# Two Dimension to One Dimension
# Reshape Array
numone = num.reshape(num.shape[0] * num.shape[1])
# Get Indexes with visibility
visnum = np.arange(numone.shape[0]).astype(np.uint32)
visnum = visnum[numone == 1]
# Get rows indexes
visrow = visnum / num.shape[0]
visrow = visrow.astype(np.uint32)
# Get cols indexes
viscol = visnum - (visrow * num.shape[1])
# Visibility indexes to Pandas DataFrame
idxnum = np.full(visrow.shape, row[obs_id])
visdf = pd.DataFrame({
'pntid': idxnum,
'rowi': visrow,
'coli': viscol
})
# Pandas DF to database
# Create Visibility table
df_to_db(new_db,
visdf,
vis_basename,
api='psql',
colGeom=None,
append=None if not idx else True)
# Delete all variables
numone = None
visnum = None
visrow = None
viscol = None
idxnum = None
visdf = None
del img
# Delete GRASS GIS File
del_rst(vrst)
# Delete TIFF File
del_file(frst)
frst = None
def rn_cols(inShp, columns, api="ogr2ogr"):
"""
Rename Columns in Shp
api options:
* ogr2ogr;
* grass;
* pygrass;
"""
if api == "ogr2ogr":
import os
from gasp.pyt import obj_to_lst
from gasp.pyt.oss import fprop
from gasp.pyt.oss import del_file, lst_ff
from gasp.gt.attr import sel_by_attr
from gasp.gt.prop.fld import lst_cols
# List Columns
cols = lst_cols(inShp)
for c in cols:
if c in columns:
continue
else:
columns[c] = c
columns["geometry"] = "geometry"
# Get inShp Folder
inshpfld = os.path.dirname(inShp)
# Get inShp Filename and format
inshpname = fprop(inShp, 'fn')
# Temporary output
output = os.path.join(inshpfld, inshpname + '_xtmp.shp')
# Rename columns by selecting data from input
outShp = sel_by_attr(
inShp,
"SELECT {} FROM {}".format(
", ".join(["{} AS {}".format(c, columns[c]) for c in columns]),
inshpname),
output,
api_gis='ogr')
# Delete Original file
infiles = lst_ff(inshpfld, filename=inshpname)
del_file(infiles)
# Rename Output file
oufiles = lst_ff(inshpfld, filename=inshpname + '_xtmp')
for f in oufiles:
os.rename(f, os.path.join(inshpfld, inshpname + fprop(f, 'ff')))
elif api == 'grass':
from gasp import exec_cmd
for col in columns:
rcmd = exec_cmd(
("v.db.renamecolumn map={} layer=1 column={},{}").format(
inShp, col, columns[col]))
elif api == 'pygrass':
from grass.pygrass.modules import Module
for col in columns:
func = Module("v.db.renamecolumn",
map=inShp,
column="{},{}".format(col, columns[col]),
quiet=True,
run_=False)
func()
else:
raise ValueError("{} is not available".format(api))
return inShp
def shp_to_psql(dbname,
shpData,
pgTable=None,
api="pandas",
mapCols=None,
srsEpsgCode=None,
encoding="UTF-8",
dbset='default'):
"""
Send Shapefile to PostgreSQL
if api is equal to "pandas" - GeoPandas API will be used;
if api is equal to "shp2pgsql" - shp2pgsql tool will be used.
"""
import os
from gasp.pyt.oss import fprop
from gasp.gt.prop.prj import get_epsg_shp
# If defined, srsEpsgCode must be a integer value
if srsEpsgCode:
if type(srsEpsgCode) != int:
raise ValueError('srsEpsgCode should be a integer value')
if api == "pandas":
from gasp.fm import tbl_to_obj
from gasp.sql.to import df_to_db
from gasp.gt.prop.feat import get_gtype
elif api == "shp2pgsql":
from gasp import exec_cmd
from gasp.sql import psql_cmd
from gasp.pyt.oss import del_file
else:
raise ValueError(
'api value is not valid. options are: pandas and shp2pgsql')
# Check if shp is folder
if os.path.isdir(shpData):
from gasp.pyt.oss import lst_ff
shapes = lst_ff(shpData, file_format='.shp')
else:
from gasp.pyt import obj_to_lst
shapes = obj_to_lst(shpData)
epsgs = [get_epsg_shp(i)
for i in shapes] if not srsEpsgCode else [srsEpsgCode]
if None in epsgs:
raise ValueError(
("Cannot obtain EPSG code. Use the srsEpsgCode parameter "
"to specify the EPSG code of your data."))
tables = []
for _i in range(len(shapes)):
# Get Table name
tname = fprop(shapes[_i], 'fn', forceLower=True) if not pgTable else \
pgTable[_i] if type(pgTable) == list else pgTable if len(shapes) == 1 \
else pgTable + '_{}'.format(_i+1)
# Import data
if api == "pandas":
# SHP to DataFrame
df = tbl_to_obj(shapes[_i])
if not mapCols:
df.rename(columns={x: x.lower()
for x in df.columns.values},
inplace=True)
else:
renameD = {
x : mapCols[x].lower() if x in mapCols else \
x.lower() for x in df.columns.values
}
df.rename(columns=renameD, inplace=True)
if "geometry" in df.columns.values:
geomCol = "geometry"
elif "geom" in df.columns.values:
geomCol = "geom"
else:
raise ValueError("No Geometry found in shp")
# GeoDataFrame to PSQL
df_to_db(dbname,
df,
tname,
append=True,
api='psql',
epsg=epsgs[_i] if not srsEpsgCode else srsEpsgCode,
colGeom=geomCol,
geomType=get_gtype(shapes[_i],
name=True,
py_cls=False,
gisApi='ogr'))
else:
sql_script = os.path.join(os.path.dirname(shapes[_i]),
tname + '.sql')
cmd = ('shp2pgsql -I -s {epsg} -W {enc} '
'{shp} public.{name} > {out}').format(
epsg=epsgs[_i] if not srsEpsgCode else srsEpsgCode,
shp=shapes[_i],
name=tname,
out=sql_script,
enc=encoding)
outcmd = exec_cmd(cmd)
psql_cmd(dbname, sql_script, dbcon=dbset)
del_file(sql_script)
tables.append(tname)
return tables[0] if len(tables) == 1 else tables
