def unsecure(pdfs, out_res):
"""
Unsecure PDF's using qpdf
Requirements: qpdf must be installed
"""
import os
from glass.pys import execmd
if os.path.isdir(pdfs):
from glass.pys .oss import lst_ff
pdfs = lst_ff(pdfs, file_format='.pdf')
else:
from glass.pys import obj_to_lst
pdfs = obj_to_lst(pdfs)
for pdf in pdfs:
execmd("qpdf --decrypt {} {}".format(pdf, os.path.join(
out_res, os.path.basename(pdf)
)))
return out_res
def unzip(zipf, destination):
"""
Unzip some zip file
"""
from glass.pys import execmd
execmd("unzip {} -d {}".format(zipf, destination))
return destination
def rstcalc(expression, output, api='saga', grids=None):
"""
Basic Raster Calculator
"""
if api == 'saga':
# Using SAGA GIS
import os
from glass.pys import execmd
from glass.pys.oss import fprop
from glass.g.it.rst import saga_to_tif
SAGA_RASTER = os.path.join(
os.path.dirname(output),
"sag_{}.sgrd".format(fprop(output, 'fn'))
)
cmd = (
"saga_cmd grid_calculus 1 -FORMULA \"{}\" -GRIDS \"{}\" "
"-RESULT {} -RESAMPLING 0"
).format(
expression, ";".join(grids), SAGA_RASTER
)
outcmd = execmd(cmd)
# Convert to tiff
saga_to_tif(SAGA_RASTER, output)
elif api == 'pygrass':
from grass.pygrass.modules import Module
rc = Module(
'r.mapcalc',
'{} = {}'.format(output, expression),
overwrite=True, run_=False, quiet=True
)
rc()
elif api == 'grass':
from glass.pys import execmd
rcmd = execmd((
f"r.mapcalc \"{output} = {expression}\" "
"--overwrite --quiet"
))
else:
raise ValueError(f"{api} is not available!")
return output
def rst_to_grs(rst, grsRst, lmtExt=None, as_cmd=None):
"""
Raster to GRASS GIS Raster
"""
if not as_cmd:
from grass.pygrass.modules import Module
__flag = 'o' if not lmtExt else 'or'
m = Module(
"r.in.gdal",
input=rst,
output=grsRst,
flags='o',
overwrite=True,
run_=False,
quiet=True,
)
m()
else:
from glass.pys import execmd
rcmd = execmd(("r.in.gdal input={} output={} -o{} --overwrite "
"--quiet").format(rst, grsRst,
"" if not lmtExt else " -r"))
return grsRst
def update_cols(table, upcol, nval):
"""
Update a feature class table with new values
new_values = {
new_value : where statment
new_value : None # if no where statment
}
Where with OR condition
new_values and ref_values are dict with fields as keys and values as
keys values.
"""
import os
from glass.pys import execmd
from glass.pys.oss import fprop
tn = fprop(table, 'fn')
for v in nval:
q = "UPDATE {} SET {}={}{}".format(
tn, upcol,
str(v) if type(v) != str else "'{}'".format(str(v)),
"" if not nval[v] else " WHERE {}".format(
nval[v] if type(nval[v]) != list else " OR ".join(nval[v])))
ogrinfo = 'ogrinfo {} -dialect sqlite -sql "{}"'.format(table, q)
# Run command
outcmd = execmd(ogrinfo)
def del_cols(lyr, cols, api='grass', lyrn=1):
"""
Remove Columns from Tables
"""
from glass.pys import obj_to_lst
cols = obj_to_lst(cols)
if api == 'grass':
from glass.pys import execmd
rcmd = execmd(("v.db.dropcolumn map={} layer={} columns={} "
"--quiet").format(lyr, str(lyrn), ','.join(cols)))
elif api == 'pygrass':
from grass.pygrass.modules import Module
m = Module("v.db.dropcolumn",
map=lyr,
layer=lyrn,
columns=cols,
quiet=True,
run_=True)
else:
raise ValueError("API {} is not available".format(api))
return lyr
def dump_db(db, outSQL, api='psql'):
"""
DB to SQL Script
"""
from glass.pys import execmd
if api == 'psql':
from glass.cons.psql import con_psql
condb = con_psql()
cmd = "pg_dump -U {} -h {} -p {} -w {} > {}".format(
condb["USER"], condb["HOST"], condb["PORT"], db, outSQL)
elif api == 'mysql':
from glass.cons.mysql import con_mysql
condb = con_mysql()
cmd = ("mysqldump -u {} --port {} -p{} --host {} "
"{} > {}").format(condb["USER"], condb["PORT"],
condb["PASSWORD"], condb["HOST"], db, outSQL)
else:
raise ValueError('{} API is not available'.format(api))
outcmd = execmd(cmd)
return outSQL
def kernel_density(pnt_feat, popField, radius, template, outRst):
"""
Kernel density estimation. If any point is currently
in selection only selected points are taken into account.
"""
import os
from glass.g.it.rst import saga_to_tif
from glass.g.prop.rst import rst_ext, get_cellsize
from glass.pys.oss import fprop
left, right, bottom, top = rst_ext(template)
cellsize = get_cellsize(template)
SAGA_RASTER = os.path.join(os.path.dirname(outRst),
'saga_{}.sgrd'.format(fprop(outRst, 'fn')))
cmd = ("saga_cmd grid_gridding 6 -POINTS {} -POPULATION {} "
"-RADIUS {} -TARGET_DEFINITION 0 -TARGET_USER_SIZE {} "
"-TARGET_USER_XMIN {} -TARGET_USER_XMAX {} "
"-TARGET_USER_YMIN {} -TARGET_USER_YMAX {} "
"-TARGET_OUT_GRID {}").format(pnt_feat, popField, str(radius),
str(abs(cellsize)), str(left),
str(right), str(bottom), str(top),
SAGA_RASTER)
outcmd = execmd(cmd)
# Convert to tiff
saga_to_tif(SAGA_RASTER, outRst)
return outRst
def flow_accum(elevrst, flowacc, dir_rst=None, ascmd=None):
"""
Produce flow accumulation raster
if dir_rst is true, produce flow direction raster
"""
if ascmd:
from glass.pys import execmd
rcmd = execmd((f"r.watershed -sa elevation={elevrst} "
f"accumulation={flowacc} "
f"{f'drainage={dir_rst} ' if dir_rst else ''}"
f"--overwrite --quiet"))
else:
from grass.pygrass.modules import Module
gm = Module("r.watershed",
elevation=elevrst,
accumulation=flowacc,
drainage=dir_rst,
flags='sa',
run_=False,
quiet=True)
gm()
return flowacc, dir_rst
def del_rst(rstname, ascmd=True):
"""
Delete Raster map of GRASS GIS
"""
from glass.pys import obj_to_lst
rstname = obj_to_lst(rstname)
if not ascmd:
from grass.pygrass.modules import Module
add = Module("g.remove",
type='raster',
name=rstname,
quiet=True,
run_=False,
flags='f')
add()
else:
from glass.pys import execmd
rcmd = execmd(("g.remove -f type=raster name={} --quiet").format(
",".join(rstname)))
return 1
def raster_report(rst, rel, _units=None, ascmd=None):
"""
Units options:
* Options: mi, me, k, a, h, c, p
** mi: area in square miles
** me: area in square meters
** k: area in square kilometers
** a: area in acres
** h: area in hectares
** c: number of cells
** p: percent cover
"""
if not ascmd:
from grass.pygrass.modules import Module
report = Module("r.report",
map=rst,
flags="h",
output=rel,
units=_units,
run_=False,
quiet=True)
report()
else:
from glass.pys import obj_to_lst, execmd
rcmd = execmd("r.report map={} output={}{} -h".format(
rst, rel, " units={}".format(",".join(obj_to_lst(_units)))
if _units else ""))
return rel
def rseries(lst, out, meth, as_cmd=None):
"""
r.series - Makes each output cell value a function of the values
assigned to the corresponding cells in the input raster map layers.
Method Options:
average, count, median, mode, minimum, min_raster, maximum,
max_raster, stddev, range, sum, variance, diversity,
slope, offset, detcoeff, tvalue, quart1, quart3, perc90,
quantile, skewness, kurtosis
"""
if type(lst) != list:
raise ValueError("lst must be a list of rasters")
if not as_cmd:
from grass.pygrass.modules import Module
serie = Module('r.series',
input=lst,
output=out,
method=meth,
overwrite=True,
quiet=True,
run_=False)
serie()
else:
from glass.pys import execmd
rcmd = execmd(("r.series input={} output={} method={} "
"--overwrite --quiet").format(",".join(lst), out, meth))
return out
def geomtype_to_geomtype(inShp, outShp, fm_type, to_type, cmd=None):
"""
v.type - Changes type of vector features.
v.type changes the type of geometry primitives.
"""
if not cmd:
from grass.pygrass.modules import Module
m = Module("v.type",
input=inShp,
output=outShp,
from_type=fm_type,
to_type=to_type,
overwrite=True,
run_=False,
quiet=True)
m()
else:
from glass.pys import execmd
rcmd = execmd(("v.type input={} output={} from_type={} to_type={} "
"--overwrite --quiet").format(inShp, outShp, fm_type,
to_type))
return outShp
def start_grass_linux_existLocation(gisdb, grassBin=None):
"""
Method to start a GRASS GIS Session on Linux Systems
Use a existing location
"""
grassbin = grassBin if grassBin else GRASS_BIN
startcmd = grassbin + ' --config path'
outcmd = execmd(startcmd)
gisbase = outcmd.strip('\n')
# Set GISBASE environment variable
os.environ['GISBASE'] = gisbase
# the following not needed with trunk
os.environ['PATH'] += os.pathsep + os.path.join(gisbase, 'extrabin')
# add path to GRASS addons
home = os.path.expanduser("~")
os.environ['PATH'] += os.pathsep + os.path.join(home, '.grass7', 'addons',
'scripts')
# define GRASS-Python environment
gpydir = os.path.join(gisbase, "etc", "python")
sys.path.append(gpydir)
# Set GISDBASE environment variable
os.environ['GISDBASE'] = gisdb
return gisbase
def shp_to_grs(inLyr, outLyr, filterByReg=None, asCMD=None):
"""
Add Shape to GRASS GIS
"""
if not asCMD:
from grass.pygrass.modules import Module
f = 'o' if not filterByReg else 'ro'
m = Module("v.in.ogr",
input=inLyr,
output=outLyr,
flags='o',
overwrite=True,
run_=False,
quiet=True)
m()
else:
from glass.pys import execmd
rcmd = execmd(
("v.in.ogr input={} output={} -o{} --overwrite --quiet").format(
inLyr, outLyr, " -r" if filterByReg else ""))
return outLyr
def slope(demRst, slopeRst, data=None, api="pygrass"):
"""
Get Slope Raster
Data options:
* percent;
* degrees;
"""
dataf = data if data == 'percent' or data == 'degrees' else 'degrees'
if api == "pygrass":
from grass.pygrass.modules import Module
sl = Module(
"r.slope.aspect", elevation=demRst, slope=slopeRst,
format=dataf,
overwrite=True, precision="FCELL", run_=False, quiet=True
)
sl()
elif api == "grass":
from glass.pys import execmd
rcmd = execmd((
f"r.slope.aspect elevation={demRst} "
f"slope={slopeRst} format={dataf} "
f"precision=FCELL --overwrite --quiet"
))
else:
raise ValueError(f"API {api} is not available")
return slopeRst
def grsintersection(inshp, intshp, outshp, cmd=None):
"""
GRASS GIS intersection
"""
if cmd:
from glass.pys import execmd
rcmd = execmd((f"v.overlay ainput={inshp} atype=area, "
f"binput={intshp} btype=area "
f"operator=and output={outshp} --overwrite --quiet"))
else:
from grass.pygrass.modules import Module
tool = Module("v.overlay",
ainput=inshp,
atype="area",
binput=intshp,
btype="area",
operator="and",
output=outshp,
overwrite=True,
run_=False,
quiet=True)
tool()
return outshp
def viewshed(demrst, obsShp, output):
"""
This tool computes a visibility analysis using observer points from
a point shapefile.
"""
import os
from glass.pys import execmd
from glass.pys.oss import fprop
from glass.g.it.rst import saga_to_tif
SAGA_RASTER = os.path.join(
os.path.dirname(output),
"sg_{}.sgrd".format(fprop(output, 'fn'))
)
cmd = (
"saga_cmd ta_lighting 6 -ELEVATION {elv} -POINTS {pnt} "
"-VISIBILITY {out} -METHOD 0"
).format(
elv=demrst, pnt=obsShp, out=SAGA_RASTER
)
outcmd = execmd(cmd)
# Convert to Tiif
saga_to_tif(SAGA_RASTER, output)
return output
def summarize_table_fields(table,
outFld,
fld_name_fld_name=None,
__upper=False):
"""
Summarize all fields in a table
"""
import os
from glass.pys import execmd
from glass.pys.oss import mkdir
# List table fields:
fields = lst_fld(table)
# For each field, query data to summarize the values in the field
cmd = 'ogr2ogr {o} {i} -dialect sqlite -sql "{s};"'
if not os.path.exists(outFld):
tmp = mkdir(outFld)
for field in fields:
outTbl = os.path.join(outFld, '{}.dbf'.format(field))
outcmd = execmd(
cmd.format(i=table,
o=outTbl,
s='SELECT {f_}{f} FROM {t} GROUP BY {f}'.format(
f=field,
t=os.path.splitext(os.path.basename(table))[0],
f_='' if not fld_name_fld_name else
'{}, '.format(fld_name_fld_name))))
def curvature(dem, profile, tangential, ascmd=None):
"""
Returns profile and tangential curvature rasters
"""
if ascmd:
from glass.pys import execmd
rmcd = execmd((
f"r.slope.aspect elevation={dem} "
f"pcurvature={profile} tcurvature="
f"{tangential} --overwrite --quiet"
))
else:
from grass.pygrass.modules import Module
m = Module(
'r.slope.aspect', elevation=dem,
pcurvature=profile, tcurvature=tangential,
overwrite=True, run_=False, quiet=True
)
m()
return profile, tangential
def add_pnts_to_network(network,
pntLyr,
outNetwork,
__threshold=200,
asCMD=None):
"""
Connect points to GRASS GIS Network
"""
if not asCMD:
from grass.pygrass.modules import Module
m = Module("v.net",
input=network,
points=pntLyr,
operation="connect",
threshold=__threshold,
output=outNetwork,
overwrite=True,
run_=False)
m()
else:
from glass.pys import execmd
rcmd = execmd(
("v.net input={} points={} operation=connect threshold={} "
"output={} --overwrite --quiet").format(network, pntLyr,
__threshold, outNetwork))
return outNetwork
def grsunion(lyra, lyrb, oshp, cmd=None):
"""
GRASS Union
"""
if cmd:
from glass.pys import execmd
outcmd = execmd((f"v.overlay ainput={lyra} atype=area "
f"binput={lyrb} btype=area "
f"operator=or output={oshp} --overwrite --quiet"))
else:
from grass.pygrass.modules import Module
un = Module("v.overlay",
ainput=lyra,
atype="area",
binput=lyrb,
btype="area",
operator="or",
output=oshp,
overwrite=True,
run_=False,
quiet=True)
un()
return oshp
def restore_tbls(dbn, sql, tablenames=None, dbset='default'):
"""
Restore tables from a sql Script
TODO: add mysql option
"""
from glass.pys import execmd
from glass.cons.psql import con_psql
from glass.pys import obj_to_lst
condb = con_psql(db_set=dbset)
tbls = obj_to_lst(tablenames)
tblStr = "" if not tablenames else " {}".format(" ".join([
"-t {}".format(t) for t in tbls]))
outcmd = execmd((
"pg_restore -U {user} -h {host} -p {port} "
"-w{tbl} -d {db} {sqls}"
).format(
user=condb["USER"], host=condb["HOST"],
port=condb["PORT"], db=dbn, sqls=sql, tbl=tblStr
))
return tablenames
def restore_db(db, sqlScript, api='psql'):
"""
Restore Database using SQL Script
"""
from glass.pys import execmd
if api == 'psql':
from glass.cons.psql import con_psql
condb = con_psql()
cmd = 'psql -h {} -U {} -p {} -w {} < {}'.format(
condb['HOST'], condb['USER'], condb['PORT'],
db, sqlScript
)
elif api == 'mysql':
from glass.cons.mysql import con_mysql
condb = con_mysql()
cmd = 'mysql -u {} -p{} {} < {}'.format(
condb['USER'], condb['PASSWORD'], db,
sqlScript
)
else:
raise ValueError('{} API is not available'.format(api))
outcmd = execmd(cmd)
return db
def osm_to_psql(osmXml, osmdb):
"""
Use GDAL to import osmfile into PostGIS database
"""
from glass.pys import execmd
from glass.cons.psql import con_psql
from glass.ng.prop.sql import db_exists
is_db = db_exists(osmdb)
if not is_db:
from glass.ng.sql.db import create_db
create_db(osmdb, api='psql')
con = con_psql()
cmd = ("ogr2ogr -f PostgreSQL \"PG:dbname='{}' host='{}' port='{}' "
"user='******' password='******'\" {} -lco COLUM_TYPES=other_tags=hstore"
).format(osmdb, con["HOST"], con["PORT"], con["USER"],
con["PASSWORD"], osmXml)
cmdout = execmd(cmd)
return osmdb
def copy_insame_vector(inShp,
colToBePopulated,
srcColumn,
destinyLayer,
geomType="point,line,boundary,centroid",
asCMD=None):
"""
Copy Field values from one layer to another in the same GRASS Vector
"""
if not asCMD:
from grass.pygrass.modules import Module
vtodb = Module("v.to.db",
map=inShp,
layer=destinyLayer,
type=geomType,
option="query",
columns=colToBePopulated,
query_column=srcColumn,
run_=False,
quiet=True)
vtodb()
else:
from glass.pys import execmd
rcmd = execmd(
("v.to.db map={} layer={} type={} option=query columns={} "
"query_column={} --quiet").format(inShp, destinyLayer, geomType,
colToBePopulated, srcColumn))
def grow_distance(inRst, outRst, api="pygrass"):
"""
Generates a raster map containing distance to nearest raster features
"""
if api == 'pygrass':
from grass.pygrass.modules import Module
m = Module('r.grow.distance',
input=inRst,
distance=outRst,
metric='euclidean',
overwrite=True,
quiet=True,
run_=False)
m()
elif api == "grass":
from glass.pys import execmd
rcmd = execmd(("r.grow.distance input={} distance={} metric=euclidean "
"--overwrite --quiet").format(inRst, outRst))
else:
raise ValueError("API {} is not available".format(api))
return outRst
def psql_cmd(db_name, sqlfile, dbcon=None):
"""
Run a sql file do whatever is on that script
"""
import os
from glass.pys import execmd
from glass.cons.psql import con_psql
cdb = con_psql(db_set=dbcon)
if os.path.isdir(sqlfile):
from glass.pys.oss import lst_ff
sqls = lst_ff(sqlfile, file_format='.sql')
else:
sqls = [sqlfile]
cmd = 'psql -h {} -U {} -p {} -w {} < {}'
for s in sqls:
outcmd = execmd(
cmd.format(cdb['HOST'], cdb['USER'], cdb['PORT'], db_name, s))
return db_name
def shp_to_shp(inshp, outshp, gisApi='ogr', supportForSpatialLite=None):
"""
Convert a vectorial file to another with other file format
API's Available:
* ogr;
* grass;
When using gisApi='ogr' - Set supportForSpatialLite to True if outShp is
a sqlite db and if you want SpatialLite support for that database.
"""
if gisApi == 'ogr':
from glass.pys import execmd
from glass.g.prop import drv_name
out_driver = drv_name(outshp)
if out_driver == 'SQLite' and supportForSpatialLite:
splite = ' -dsco "SPATIALITE=YES"'
else:
splite = ''
cmd = 'ogr2ogr -f "{drv}" {out} {_in}{lite}'.format(drv=out_driver,
out=outshp,
_in=inshp,
lite=splite)
# Run command
cmdout = execmd(cmd)
elif gisApi == 'grass':
# TODO identify input geometry type
import os
from glass.pys.oss import fprop
from glass.g.wenv.grs import run_grass
from glass.g.prop.prj import get_epsg
# Start GRASS GIS Session
ws = os.path.dirname(outshp)
loc = f'loc_{fprop(outshp, "fn")}'
epsg = get_epsg(inshp)
gbase = run_grass(ws, location=loc, srs=epsg)
import grass.script.setup as gsetup
gsetup.init(gbase, ws, loc, 'PERMANENT')
from glass.g.it.shp import grs_to_shp, shp_to_grs
gshp = shp_to_grs(inshp, fprop(inshp, 'fn'))
grs_to_shp(gshp, outshp, 'area')
else:
raise ValueError('Sorry, API {} is not available'.format(gisApi))
return outshp
def clip(inFeat, clipFeat, outFeat, api_gis="grass", clip_by_region=None):
"""
Clip Analysis
api_gis Options:
* grass
* pygrass
* ogr2ogr
"""
from glass.pys.oss import fprop
if api_gis == "pygrass" or api_gis == "grass":
import os
from glass.g.wenv.grs import run_grass
from glass.g.prop.prj import get_epsg
epsg = get_epsg(inFeat)
work = os.path.dirname(outFeat)
refname = fprop(outFeat, 'fn')
loc = f"loc_{refname}"
grsbase = run_grass(work, location=loc, srs=epsg)
import grass.script.setup as gsetup
gsetup.init(grsbase, work, loc, 'PERMANENT')
from glass.g.it.shp import shp_to_grs, grs_to_shp
from glass.g.prop.feat import feat_count
shp = shp_to_grs(inFeat, fprop(inFeat, 'fn'))
clp = shp_to_grs(clipFeat, fprop(clipFeat, 'fn'))
# Clip
rslt = grsclip(shp,
clp,
refname,
cmd=True if api_gis == "grass" else None,
clip_by_region=clip_by_region)
# Export
grs_to_shp(rslt, outFeat, 'area')
elif api_gis == 'ogr2ogr':
from glass.pys import execmd
from glass.g.prop import drv_name
rcmd = execmd(
("ogr2ogr -f \"{}\" {} {} -clipsrc {} -clipsrclayer {}").format(
drv_name(outFeat), outFeat, inFeat, clipFeat,
fprop(clipFeat, 'fn')))
else:
raise ValueError("{} is not available!".format(api_gis))
return outFeat
