def wait(self):
"""Wait for the module to finish. Call this method if
the run() call was performed with self.false_ = False.
:return: A reference to this object
"""
if self._finished is False:
if self.stdin:
self.stdin = encode(self.stdin)
stdout, stderr = self._popen.communicate(input=self.stdin)
self.outputs["stdout"].value = decode(stdout) if stdout else ""
self.outputs["stderr"].value = decode(stderr) if stderr else ""
self.time = time.time() - self.start_time
self.returncode = self._popen.returncode
self._finished = True
if self._popen.poll():
raise CalledModuleError(
returncode=self._popen.returncode,
code=self.get_bash(),
module=self.name,
errors=stderr,
)
self._popen = None
return self
def test_rename_column(self):
"""Renaming a column"""
module = SimpleModule("v.db.renamecolumn",
map="myroads",
column=("ROAD_NAME", "roadname"))
self.assertModule(module)
m = SimpleModule("v.info", flags="c", map="myroads")
self.assertModule(m)
self.assertRegexpMatches(decode(m.outputs.stdout), "roadname")
self.assertNotRegexpMatches(decode(m.outputs.stdout), "ROAD_NAME")
def test_add_two_columns_check(self):
"""Add two column to the attribute table"""
module = SimpleModule('v.db.addcolumn', map='myroads',
columns='slope_2 double precision, \
myname varchar(15)')
self.assertModule(module)
m = SimpleModule('v.info', map='myroads', flags='c')
self.assertModule(m)
self.assertRegexpMatches(decode(m.outputs.stdout), 'slope_2')
self.assertRegexpMatches(decode(m.outputs.stdout), 'myname')
def test_rename_column(self):
"""Renaming a column"""
module = SimpleModule('v.db.renamecolumn',
map='myroads',
column=('ROAD_NAME', 'roadname'))
self.assertModule(module)
m = SimpleModule('v.info', flags='c', map='myroads')
self.assertModule(m)
self.assertRegexpMatches(decode(m.outputs.stdout), 'roadname')
self.assertNotRegexpMatches(decode(m.outputs.stdout), 'ROAD_NAME')
def decode(obj, encoding=None):
"""Decode string coming from c functions,
can be ctypes class String, bytes, or None
"""
if isinstance(obj, String):
return grassutils.decode(obj.data, encoding=encoding)
elif isinstance(obj, bytes):
return grassutils.decode(obj)
else:
# eg None
return obj
def extend(self, mapsets):
"""Add more mapsets to the search path
:param mapsets: a list of mapset's names
:type mapsets: list
"""
ms = [decode(m) for m in self.location.mapsets()]
final = [decode(m) for m in self.read()]
mapsets = [decode(m) for m in mapsets]
final.extend([m for m in mapsets if m in ms and m not in final])
self._write(final)
def test_add_two_columns_check(self):
"""Add two column to the attribute table"""
module = SimpleModule(
"v.db.addcolumn",
map="myroads",
columns="slope_2 double precision, \
myname varchar(15)",
)
self.assertModule(module)
m = SimpleModule("v.info", map="myroads", flags="c")
self.assertModule(m)
self.assertRegexpMatches(decode(m.outputs.stdout), "slope_2")
self.assertRegexpMatches(decode(m.outputs.stdout), "myname")
def project(file, source, dest):
"""Projects point (x, y) using projector"""
errors = 0
points = []
try:
ret = gcore.read_command(
"m.proj",
quiet=True,
flags="d",
proj_in=source["proj"],
proj_out=dest["proj"],
sep=";",
input=file,
)
ret = decode(ret)
except CalledModuleError:
gcore.fatal(cs2cs + " failed")
if not ret:
gcore.fatal(cs2cs + " failed")
for line in ret.splitlines():
if "*" in line:
errors += 1
else:
p_x2, p_y2, p_z2 = list(map(float, line.split(";")))
points.append((p_x2 / dest["scale"], p_y2 / dest["scale"]))
return points, errors
def project(file, source, dest):
"""Projects point (x, y) using projector"""
errors = 0
points = []
try:
ret = gcore.read_command('m.proj',
quiet=True,
flags='d',
proj_in=source['proj'],
proj_out=dest['proj'],
sep=';',
input=file)
ret = decode(ret)
except CalledModuleError:
gcore.fatal(cs2cs + ' failed')
if not ret:
gcore.fatal(cs2cs + ' failed')
for line in ret.splitlines():
if "*" in line:
errors += 1
else:
p_x2, p_y2, p_z2 = list(map(float, line.split(';')))
points.append((p_x2 / dest['scale'], p_y2 / dest['scale']))
return points, errors
def test_drop_table_with_force(self):
"""Drop table with force, the column should not be in the table"""
module = SimpleModule("v.db.droptable", map="myroads", flags="f")
self.assertModule(module)
m = SimpleModule("db.tables", flags="p")
self.assertModule(m)
self.assertNotRegexpMatches(decode(m.outputs.stdout), "myroads")
def test_drop_table_check(self):
"""Drop table check, the column should still be in the table"""
module = SimpleModule("v.db.droptable", map="myroads")
self.assertModule(module)
m = SimpleModule("db.tables", flags="p")
self.assertModule(m)
self.assertRegexpMatches(decode(m.outputs.stdout), "myroads")
def test_add_single_columned_table_check(self):
"""Add a new attribute table with single column to layer 2"""
module = SimpleModule("v.db.addtable",
map="myroads",
columns="slope double precision",
layer=2)
self.assertModule(module)
run_command("v.db.connect", flags="p", map="myroads")
m = SimpleModule("v.info", map="myroads", flags="c")
self.assertModule(m)
self.assertNotRegexpMatches(decode(m.outputs.stdout), "slope")
m = SimpleModule("v.info", map="myroads", flags="c", layer=2)
self.assertModule(m)
self.assertRegexpMatches(decode(m.outputs.stdout), "slope")
def test_add_single_columned_table_check(self):
"""Add a new attribute table with single column to layer 2"""
module = SimpleModule('v.db.addtable',
map='myroads',
columns='slope double precision',
layer=2)
self.assertModule(module)
run_command('v.db.connect', flags='p', map='myroads')
m = SimpleModule('v.info', map='myroads', flags='c')
self.assertModule(m)
self.assertNotRegexpMatches(decode(m.outputs.stdout), 'slope')
m = SimpleModule('v.info', map='myroads', flags='c', layer=2)
self.assertModule(m)
self.assertRegexpMatches(decode(m.outputs.stdout), 'slope')
def test_drop_table_with_force(self):
"""Drop table with force, the column should not be in the table"""
module = SimpleModule('db.droptable', table=self.mapName, flags='f')
self.assertModule(module)
m = SimpleModule('db.tables', flags='p')
self.assertModule(m)
self.assertNotRegexpMatches(decode(m.outputs.stdout), self.mapName)
def test_json_output(self):
import json
module = SimpleModule('g.search.modules', keyword="water", flags="j")
self.assertModule(module)
stdout = json.loads(decode(module.outputs.stdout))
self.assertEqual(len(stdout), 6, 'Six modules found')
self.assertEqual(stdout[3]['name'], 'r.water.outlet', 'r.water.outlet')
self.assertTrue('keywords' in stdout[3]['attributes'])
def test_drop_table_check(self):
"""Drop table check, the column should still be in the table"""
module = SimpleModule('db.droptable', table=self.mapName)
self.assertModule(module)
m = SimpleModule('db.tables', flags='p')
self.assertModule(m)
self.assertRegexpMatches(decode(m.outputs.stdout), self.mapName)
def _write(self, mapsets):
"""Write to SEARCH_PATH file the changes in the search path
:param mapsets: a list of mapset's names
:type mapsets: list
"""
with open(self.spath, "wb+") as f:
ms = [decode(m) for m in self.location.mapsets()]
f.write(b"\n".join([encode(m) for m in mapsets if m in ms]))
def test_add_single_column_check(self):
"""Add column to the attribute table"""
module = SimpleModule('v.db.addcolumn', map='myroads',
columns='slope double precision')
self.assertModule(module)
m = SimpleModule('v.info', map='myroads', flags='c')
self.assertModule(m)
self.assertRegexpMatches(decode(m.outputs.stdout), 'slope')
def test_tiling(self):
"""Produce tiling test"""
module = SimpleModule("r.tileset",
sourceproj="+init=epsg:4326",
maxrows=1024,
maxcols=2048)
self.assertModule(module)
self.assertMultiLineEqual(decode(module.outputs.stdout), output)
def test_json_output(self):
import json
module = SimpleModule("g.search.modules", keyword="water", flags="j")
self.assertModule(module)
stdout = json.loads(decode(module.outputs.stdout))
self.assertEqual(len(stdout), 6, "Six modules found")
self.assertEqual(stdout[3]["name"], "r.water.outlet", "r.water.outlet")
self.assertTrue("keywords" in stdout[3]["attributes"])
def read(self):
"""Return the mapsets in the search path"""
with open(self.spath, "ab+") as f:
lines = f.readlines()
if lines:
return [decode(l.strip()) for l in lines]
lns = [u'PERMANENT', ]
self._write(lns)
return lns
def get_nr_of_categories(vector, layer, rasters, rastertmp, percentile,
colprefixes, basecols, dbfdriver, c):
"""Get number of raster categories to be processed.
Perform also checks of raster and vector categories. In the case of no
raster categories, create the desired columns and exit.
:param vector: name of vector map or data source for direct OGR access
:param layer: layer number or name
:param rastertmp: name of temporary raster map
:return: number of raster categories or exit (if no categories found)
"""
# dump cats to file to avoid "too many argument" problem:
p = grass.pipe_command("r.category", map=rastertmp, sep=";", quiet=True)
cats = []
for line in p.stdout:
line = decode(line)
cats.append(line.rstrip("\r\n").split(";")[0])
p.wait()
number = len(cats)
if number < 1:
# create columns and exit
grass.warning(_("No categories found in raster map"))
for i in range(len(rasters)):
set_up_columns(
vector,
layer,
percentile,
colprefixes[i],
basecols,
dbfdriver,
flags["c"],
)
sys.exit(0)
# Check if all categories got converted
# Report categories from vector map
vect_cats = (grass.read_command("v.category",
input=vector,
option="report",
flags="g").rstrip("\n").split("\n"))
# get number of all categories in selected layer
vect_cats_n = 0 # to be modified below
for vcl in vect_cats:
if vcl.split(" ")[0] == layer and vcl.split(" ")[1] == "all":
vect_cats_n = int(vcl.split(" ")[2])
if vect_cats_n != number:
grass.warning(
_("Not all vector categories converted to raster. \
Converted {0} of {1}.".format(number, vect_cats_n)))
return number
def test_import_dbf_file(self):
"""import dbf table"""
module = SimpleModule('db.in.ogr',
input=self.dbfFile,
output=self.tableName2)
self.assertModule(module)
m = SimpleModule('db.tables', flags='p')
self.assertModule(m)
self.assertRegexpMatches(decode(m.outputs.stdout), self.tableName2)
def test_drop_single_column_check(self):
"""Drop column to the attribute table"""
module = SimpleModule("v.db.dropcolumn",
map="myroads",
columns="SHAPE_LEN")
self.assertModule(module)
m = SimpleModule("v.info", map="myroads", flags="c")
self.assertModule(m)
self.assertNotRegexpMatches(decode(m.outputs.stdout), "SHAPE_LEN")
def test_drop_single_column_check(self):
"""Drop column to the attribute table"""
module = SimpleModule('v.db.dropcolumn',
map='myroads',
columns='SHAPE_LEN')
self.assertModule(module)
m = SimpleModule('v.info', map='myroads', flags='c')
self.assertModule(m)
self.assertNotRegexpMatches(decode(m.outputs.stdout), 'SHAPE_LEN')
def test_drop_column_check(self):
"""Drop column check, the column should still be in the table"""
module = SimpleModule("db.dropcolumn",
table=self.mapName,
column=self.colName)
self.assertModule(module)
m = SimpleModule("db.columns", table=self.mapName)
self.assertModule(m)
self.assertRegexpMatches(decode(m.outputs.stdout), self.colName)
def test_add_single_column_check(self):
"""Add column to the attribute table"""
module = SimpleModule("v.db.addcolumn",
map="myroads",
columns="slope double precision")
self.assertModule(module)
m = SimpleModule("v.info", map="myroads", flags="c")
self.assertModule(m)
self.assertRegexpMatches(decode(m.outputs.stdout), "slope")
def test_drop_column_with_force(self):
"""Drop column with force, the column should not be in the table"""
module = SimpleModule("db.dropcolumn",
table=self.mapName,
column=self.colName,
flags="f")
self.assertModule(module)
m = SimpleModule("db.columns", table=self.mapName)
self.assertModule(m)
self.assertNotRegexpMatches(decode(m.outputs.stdout), self.colName)
def perform_stats(
raster,
percentile,
fi,
dbfdriver,
colprefix,
variables_dbf,
variables,
colnames,
extstat,
):
with open(sqltmp, "w") as f:
# do the stats
p = grass.pipe_command(
"r.univar",
flags="t" + extstat,
map=raster,
zones=rastertmp,
percentile=percentile,
sep=";",
)
first_line = 1
f.write("{0}\n".format(grass.db_begin_transaction(fi["driver"])))
for line in p.stdout:
if first_line:
first_line = 0
continue
vars = decode(line).rstrip("\r\n").split(";")
f.write("UPDATE %s SET" % fi["table"])
first_var = 1
for colname in colnames:
variable = colname.replace("%s_" % colprefix, "", 1)
if dbfdriver:
variable = variables_dbf[variable]
i = variables[variable]
value = vars[i]
# convert nan, +nan, -nan, inf, +inf, -inf, Infinity, +Infinity,
# -Infinity to NULL
if value.lower().endswith("nan") or "inf" in value.lower():
value = "NULL"
if not first_var:
f.write(" , ")
else:
first_var = 0
f.write(" %s=%s" % (colname, value))
f.write(" WHERE %s=%s;\n" % (fi["key"], vars[0]))
f.write("{0}\n".format(grass.db_commit_transaction(fi["driver"])))
p.wait()
def try_decode(data, encodings):
"""Try to decode data (bytes) using one of encodings
Falls back to decoding as UTF-8 with replacement for bytes.
Strings are returned unmodified.
"""
for encoding in encodings:
try:
return decode(data, encoding=encoding)
except UnicodeError:
pass
if isinstance(data, bytes):
return data.decode(encoding="utf-8", errors="replace")
return data
def test_unicode(self):
self.assertEqual(u'text', utils.decode(u'text'))
def test_bytes(self):
self.assertEqual(u'text', utils.decode(b'text'))