def test_multiple_underscores(self):
proc = start_command(
'g.region', _raster_=self.raster, stderr=PIPE)
stderr = proc.communicate()[1]
returncode = proc.poll()
self.assertEquals(returncode, 1)
self.assertIn(b'raster', stderr)
def reproject_region(region, from_proj, to_proj):
region = copy.copy(region)
print "to reproject:", region
proj_input = '{east} {north}\n{west} {south}'.format(**region)
print proj_input
proc = gcore.start_command('m.proj', input='-', separator=' , ',
proj_in=from_proj, proj_out=to_proj,
stdin=gcore.PIPE,
stdout=gcore.PIPE, stderr=gcore.PIPE)
proc.stdin.write(proj_input)
proc.stdin.close()
proc.stdin = None
proj_output, stderr = proc.communicate()
if proc.returncode:
print from_proj, to_proj, proj_input
raise RuntimeError("reprojecting region: m.proj error: " + stderr)
enws = proj_output.split(os.linesep)
print proj_output
print enws
elon, nlat, unused = enws[0].split(' ')
wlon, slat, unused = enws[1].split(' ')
region['east'] = elon
region['north'] = nlat
region['west'] = wlon
region['south'] = slat
return region
def test_suffixed_underscore(self):
proc = start_command(
'g.region', raster_=self.raster, stderr=PIPE)
stderr = proc.communicate()[1]
self.assertNotIn(b'raster_', stderr)
self.assertIn(self.raster, stderr,
msg="Raster map name should appear in the error output")
def read_grass(self, *args, **kwargs):
"""execute a grass function with error output """
kwargs['stdout'] = grass.PIPE
kwargs['stderr'] = grass.PIPE
ps = grass.start_command(*args, **kwargs)
# returns a tuple (stderr,stdout)
return ps.communicate()
def test_prefixed_underscore(self):
proc = start_command('g.region', _rast=self.raster, stderr=PIPE)
stderr = proc.communicate()[1]
self.assertNotIn('_rast', stderr)
self.assertIn(self.raster,
stderr,
msg="Raster map name should appear in the error output")
def test_suffixed_underscore(self):
proc = start_command("g.region", raster_=self.raster, stderr=PIPE)
stderr = proc.communicate()[1]
self.assertNotIn(b"raster_", stderr)
self.assertIn(
self.raster, stderr, msg="Raster map name should appear in the error output"
)
def test_multiple_underscores(self):
proc = start_command(
'g.region', _rast_=self.raster, stderr=PIPE)
stderr = proc.communicate()[1]
returncode = proc.poll()
self.assertEquals(returncode, 1,
msg="Undersocre at both sides was accepted")
self.assertIn('rast', stderr)
def test_suffixed_underscore(self):
proc = start_command(
'g.region', rast_=self.raster, stderr=PIPE)
stderr = proc.communicate()[1]
returncode = proc.poll()
self.assertEquals(returncode, 0,
msg="Undersocre as suffix was not accepted, stderr is:\n%s" % stderr)
self.assertNotIn('rast_', stderr)
def test_prefixed_underscore(self):
proc = start_command(
'g.region', _rast=self.raster, stderr=PIPE)
stderr = proc.communicate()[1]
returncode = proc.poll()
self.assertEquals(returncode, 0,
msg="Undersocre as prefix was not accepted")
self.assertNotIn('_rast', stderr)
def test_prefixed_underscore(self):
proc = start_command('g.region', _rast=self.raster, stderr=PIPE)
stderr = proc.communicate()[1]
returncode = proc.poll()
self.assertEquals(returncode,
0,
msg="Undersocre as prefix was not accepted")
self.assertNotIn('_rast', stderr)
def test_multiple_underscores(self):
proc = start_command('g.region', _rast_=self.raster, stderr=PIPE)
stderr = proc.communicate()[1]
returncode = proc.poll()
self.assertEquals(returncode,
1,
msg="Undersocre at both sides was accepted")
self.assertIn('rast', stderr)
def clean_default_db():
# clean the default db if it is sqlite
from grass.script import db as gdb
from grass.script import core as gcore
conn = gdb.db_connection()
if conn and conn['driver'] == 'sqlite':
# check if db exists
gisenv = gcore.gisenv()
database = conn['database']
database = database.replace('$GISDBASE', gisenv['GISDBASE'])
database = database.replace('$LOCATION_NAME', gisenv['LOCATION_NAME'])
database = database.replace('$MAPSET', gisenv['MAPSET'])
if os.path.exists(database):
gcore.message(_("Cleaning up default sqlite database ..."))
gcore.start_command('db.execute', sql = 'VACUUM')
# give it some time to start
import time
time.sleep(0.1)
def test_suffixed_underscore(self):
proc = start_command('g.region', rast_=self.raster, stderr=PIPE)
stderr = proc.communicate()[1]
returncode = proc.poll()
self.assertEquals(
returncode,
0,
msg="Undersocre as suffix was not accepted, stderr is:\n%s" %
stderr)
self.assertNotIn('rast_', stderr)
def clean_default_db():
# clean the default db if it is sqlite
from grass.script import core as gcore
from grass.script import db as gdb
conn = gdb.db_connection()
if conn and conn["driver"] == "sqlite":
# check if db exists
gisenv = gcore.gisenv()
database = conn["database"]
database = database.replace("$GISDBASE", gisenv["GISDBASE"])
database = database.replace("$LOCATION_NAME", gisenv["LOCATION_NAME"])
database = database.replace("$MAPSET", gisenv["MAPSET"])
if os.path.exists(database):
gcore.message(_("Cleaning up default sqlite database ..."))
gcore.start_command("db.execute", sql="VACUUM")
# give it some time to start
import time
time.sleep(0.1)
def map_exists(name, type, mapset=None):
"""Check if map exists
:param name: name of the map (without a Mapset)
:param type: data type ('raster', 'raster_3d', and 'vector')
:param mapset: Mapset (you can use dot to refer to the current Mapset)
"""
# change type to element used by find file
# otherwise, we are not checking the input,
# so anything accepted by g.findfile will work
# also supporting both short and full names
# but this can change in the
# future (this function documentation is clear about what's legal)
if type == "raster" or type == "rast":
type = "cell"
elif type == "raster_3d" or type == "rast3d" or type == "raster3d":
type = "grid3"
elif type == "vect":
type = "vector"
extra_params = {}
if mapset:
if mapset == ".":
mapset = get_current_mapset()
extra_params.update({"mapset": mapset})
# g.findfile returns non-zero when file was not found
# so we ignore return code and just focus on stdout
process = start_command(
"g.findfile",
flags="n",
element=type,
file=name,
mapset=mapset,
stdout=PIPE,
stderr=PIPE,
**extra_params,
)
output, errors = process.communicate()
info = parse_key_val(output)
# file is the key questioned in grass.script.core find_file()
# return code should be equivalent to checking the output
# g.findfile returns non-zero when nothing found but scripts are used to
# check the output, so it is unclear what should be or is the actual
# g.findfile interface
# see also #2475 for discussion about types/files
if info["file"]:
return True
else:
return False
def CalcBoundsFromPoints(self, lats, lngs):
"""Calculates the max/min lat/lng in the lists.
This method takes in a list of lats and a list of lngs, and
outputs the southwest and northeast bounds for these points.
We use this method when we have done a search for points on the
map, and we get multiple results. In the results we don't get a
bounding box so this method calculates it for us.
param: list lats: list of latitudes
param: lsit lngs: list of longitudes
returns list: a list of length 2, each holding a list of
length 2. It holds the southwest and northeast lat/lng
bounds of a map. It should look like this:
[[southwestLat, southwestLng], [northeastLat, northeastLng]]
"""
lats = [float(x) for x in lats]
lngs = [float(x) for x in lngs]
flats = list(map(float, lats))
flngs = list(map(float, lngs))
west = min(flngs)
east = max(flngs)
north = max(flats)
south = min(flats)
coords = "{bottom_left}\n{top_right}".format(
bottom_left="{} {}".format(west, south),
top_right="{} {}".format(east, north),
)
proc = grass.start_command(
"m.proj",
flags="do",
input="-",
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
)
result, error = proc.communicate(input=grass.encode(coords))
bounds = []
for row in grass.decode(result).split("\n"):
if row:
x, y, z = row.split("|")
bounds.append([float(x), float(y)])
return bounds
def proj_to_wgs84(region):
proj_in = '{east} {north}\n{west} {south}'.format(**region)
proc = gcore.start_command('m.proj', input='-', separator=' , ',
flags='od',
stdin=gcore.PIPE, stdout=gcore.PIPE,
stderr=gcore.PIPE)
proc.stdin.write(proj_in)
proc.stdin.close()
proc.stdin = None
proj_out, errors = proc.communicate()
if proc.returncode:
raise RuntimeError("m.proj error: %s" % errors)
enws = proj_out.split(os.linesep)
elon, nlat, unused = enws[0].split(' ')
wlon, slat, unused = enws[1].split(' ')
return {'east': elon, 'north': nlat,
'west': wlon, 'south': slat}
def is_map_in_mapset(name, type, mapset=None):
"""Check is map is present in the mapset (current mapset by default)
This function is different from what we would expect in GRASS
because it cares only about specific mapset, the current one by default,
and it does not care that the map is accessible in other mapset.
:param name: name of the map
:param type: data type ('raster', 'raster3d', and 'vector')
"""
if not mapset:
mapset = get_current_mapset()
# change type to element used by find file
# otherwise, we are not checking the input,
# so anything accepted by g.findfile will work but this can change in the
# future (the documentation is clear about what's legal)
# supporting both short and full names
if type == "rast" or type == "raster":
type = "cell"
elif type == "rast3d" or type == "raster3d":
type = "grid3"
elif type == "vect":
type = "vector"
# g.findfile returns non-zero when file was not found
# se we ignore return code and just focus on stdout
process = start_command(
"g.findfile",
flags="n",
element=type,
file=name,
mapset=mapset,
stdout=PIPE,
stderr=PIPE,
)
output, errors = process.communicate()
info = text_to_keyvalue(decode(output), sep="=")
# file is the key questioned in grass.script.core find_file()
# return code should be equivalent to checking the output
if info["file"]:
return True
else:
return False
def is_map_in_mapset(name, type, mapset=None):
"""Check is map is present in the mapset (current mapset by default)
This function is different from what we would expect in GRASS
because it cares only about specific mapset, the current one by default,
and it does not care that the map is accessible in other mapset.
:param name: name of the map
:param type: data type ('raster', 'raster3d', and 'vector')
"""
if not mapset:
mapset = get_current_mapset()
# change type to element used by find file
# otherwise, we are not checking the input,
# so anything accepted by g.findfile will work but this can change in the
# future (the documentation is clear about what's legal)
# supporting both short and full names
if type == 'rast' or type == 'raster':
type = 'cell'
elif type == 'rast3d' or type == 'raster3d':
type = 'grid3'
elif type == 'vect':
type = 'vector'
# g.findfile returns non-zero when file was not found
# se we ignore return code and just focus on stdout
process = start_command('g.findfile', flags='n',
element=type, file=name, mapset=mapset,
stdout=PIPE, stderr=PIPE)
output, errors = process.communicate()
info = text_to_keyvalue(output, sep='=')
# file is the key questioned in grass.script.core find_file()
# return code should be equivalent to checking the output
if info['file']:
return True
else:
return False
def main():
infile = options["input"]
output = options["output"]
method = options["method"]
dtype = options["type"]
fs = options["separator"]
x = options["x"]
y = options["y"]
z = options["z"]
value_column = options["value_column"]
vrange = options["vrange"]
vscale = options["vscale"]
percent = options["percent"]
pth = options["pth"]
trim = options["trim"]
workers = int(options["workers"])
scan_only = flags["s"]
shell_style = flags["g"]
ignore_broken = flags["i"]
if workers == 1 and "WORKERS" in os.environ:
workers = int(os.environ["WORKERS"])
if not os.path.exists(infile):
grass.fatal(_("Unable to read input file <%s>") % infile)
addl_opts = {}
if pth:
addl_opts["pth"] = "%s" % pth
if trim:
addl_opts["trim"] = "%s" % trim
if value_column:
addl_opts["value_column"] = "%s" % value_column
if vrange:
addl_opts["vrange"] = "%s" % vrange
if vscale:
addl_opts["vscale"] = "%s" % vscale
if ignore_broken:
addl_opts["flags"] = "i"
if scan_only or shell_style:
if shell_style:
doShell = "g"
else:
doShell = ""
grass.run_command(
"r.in.xyz",
flags="s" + doShell,
input=infile,
output="dummy",
sep=fs,
x=x,
y=y,
z=z,
**addl_opts,
)
sys.exit()
if dtype == "float":
data_type = "FCELL"
else:
data_type = "DCELL"
region = grass.region(region3d=True)
if region["nsres"] != region["nsres3"] or region["ewres"] != region[
"ewres3"]:
grass.run_command("g.region", flags="3p")
grass.fatal(
_("The 2D and 3D region settings are different. Can not continue.")
)
grass.verbose(
_("Region bottom=%.15g top=%.15g vertical_cell_res=%.15g (%d depths)"
) % (region["b"], region["t"], region["tbres"], region["depths"]))
grass.verbose(_("Creating slices ..."))
# to avoid a point which falls exactly on a top bound from being
# considered twice, we shrink the
# For the top slice we keep it though, as someone scanning the bounds
# may have set the bounds exactly to the data extent (a bad idea, but
# it happens..)
eps = 1.0e-15
# if there are thousands of depths hopefully this array doesn't get too
# large and so we don't have to worry much about storing/looping through
# all the finished process infos.
proc = {}
pout = {}
depths = list(range(1, 1 + region["depths"]))
for i in depths:
tmp_layer_name = "tmp.r3xyz.%d.%s" % (os.getpid(), "%05d" % i)
zrange_min = region["b"] + (region["tbres"] * (i - 1))
if i < region["depths"]:
zrange_max = region["b"] + (region["tbres"] * i) - eps
else:
zrange_max = region["b"] + (region["tbres"] * i)
# spawn depth layer import job in the background
# grass.debug("slice %d, <%s> %% %d" % (band, image[band], band % workers))
grass.message(
_("Processing horizontal slice %d of %d [%.15g,%.15g) ...") %
(i, region["depths"], zrange_min, zrange_max))
proc[i] = grass.start_command(
"r.in.xyz",
input=infile,
output=tmp_layer_name,
sep=fs,
method=method,
x=x,
y=y,
z=z,
percent=percent,
type=data_type,
zrange="%.15g,%.15g" % (zrange_min, zrange_max),
**addl_opts,
)
grass.debug("i=%d, %%=%d (workers=%d)" % (i, i % workers, workers))
# print sys.getsizeof(proc) # sizeof(proc array) [not so big]
if i % workers == 0:
# wait for the ones launched so far to finish
for p_i in depths[:i]:
pout[p_i] = proc[p_i].communicate()[0]
if proc[p_i].wait() != 0:
grass.fatal(_("Trouble importing data. Aborting."))
# wait for jSobs to finish, collect any stray output
for i in depths:
pout[i] = proc[i].communicate()[0]
if proc[i].wait() != 0:
grass.fatal(_("Trouble importing data. Aborting."))
del proc
grass.verbose(_("Assembling 3D cube ..."))
# input order: lower most strata first
slices = grass.read_command("g.list",
type="raster",
sep=",",
pattern="tmp.r3xyz.%d.*" % os.getpid()).rstrip(
os.linesep)
grass.debug(slices)
try:
grass.run_command("r.to.rast3", input=slices, output=output)
except CalledModuleError:
grass.message(_("Done. 3D raster map <%s> created.") % output)
def call_module(module,
stdin=None,
merge_stderr=False,
capture_stdout=True,
capture_stderr=True,
**kwargs):
r"""Run module with parameters given in `kwargs` and return its output.
>>> print (call_module('g.region', flags='pg')) # doctest: +ELLIPSIS
projection=...
zone=...
n=...
s=...
w=...
>>> call_module('m.proj', flags='i', input='-', stdin="50.0 41.5")
'8642890.65|6965155.61|0.00\n'
>>> call_module('g.region', aabbbccc='notexist') # doctest: +IGNORE_EXCEPTION_DETAIL
Traceback (most recent call last):
...
CalledModuleError: Module run g.region ... ended with error
If `stdin` is not set and `kwargs` contains ``input`` with value set
to ``-`` (dash), the function raises an error.
Note that ``input`` nor ``output`` parameters are used by this
function itself, these are usually module parameters which this
function just passes to it. However, when ``input`` is in parameters
the function checks if its values is correct considering value of
``stdin`` parameter.
:param str module: module name
:param stdin: string to be used as module standard input (stdin) or `None`
:param merge_stderr: if the standard error output should be merged with stdout
:param kwargs: module parameters
:returns: module standard output (stdout) as string or None if apture_stdout is False
:raises CalledModuleError: if module return code is non-zero
:raises ValueError: if the parameters are not correct
.. note::
The data read is buffered in memory, so do not use this method
if the data size is large or unlimited.
"""
# TODO: remove this:
do_doctest_gettext_workaround()
# implementation inspired by subprocess.check_output() function
if stdin:
if 'input' in kwargs and kwargs['input'] != '-':
raise ValueError(
_("input='-' must be used when stdin is specified"))
if stdin == subprocess.PIPE:
raise ValueError(_("stdin must be string or buffer, not PIPE"))
kwargs['stdin'] = subprocess.PIPE # to be able to send data to stdin
elif 'input' in kwargs and kwargs['input'] == '-':
raise ValueError(_("stdin must be used when input='-'"))
if merge_stderr and not (capture_stdout and capture_stderr):
raise ValueError(
_("You cannot merge stdout and stderr and not capture them"))
if 'stdout' in kwargs:
raise TypeError(
_("stdout argument not allowed, it could be overridden"))
if 'stderr' in kwargs:
raise TypeError(
_("stderr argument not allowed, it could be overridden"))
if capture_stdout:
kwargs['stdout'] = subprocess.PIPE
if capture_stderr:
if merge_stderr:
kwargs['stderr'] = subprocess.STDOUT
else:
kwargs['stderr'] = subprocess.PIPE
process = start_command(module, **kwargs)
# input=None means no stdin (our default)
# for no stdout, output is None which is out interface
# for stderr=STDOUT or no stderr, errors is None
# which is fine for CalledModuleError
output, errors = process.communicate(
input=encode(decode(stdin)) if stdin else None)
returncode = process.poll()
if returncode:
raise CalledModuleError(returncode, module, kwargs, errors)
return decode(output) if output else None
def call_module(module, stdin=None,
merge_stderr=False, capture_stdout=True, capture_stderr=True,
**kwargs):
r"""Run module with parameters given in `kwargs` and return its output.
>>> print call_module('g.region', flags='pg') # doctest: +ELLIPSIS
projection=...
zone=...
n=...
s=...
w=...
>>> call_module('m.proj', flags='i', input='-', stdin="50.0 41.5")
'8642890.65|6965155.61|0.00\n'
>>> call_module('g.region', aabbbccc='notexist') # doctest: +IGNORE_EXCEPTION_DETAIL
Traceback (most recent call last):
...
CalledModuleError: Module run g.region ... ended with error
If `stdin` is not set and `kwargs` contains ``input`` with value set
to ``-`` (dash), the function raises an error.
Note that ``input`` nor ``output`` parameters are used by this
function itself, these are usually module parameters which this
function just passes to it. However, when ``input`` is in parameters
the function checks if its values is correct considering value of
``stdin`` parameter.
:param str module: module name
:param stdin: string to be used as module standard input (stdin) or `None`
:param merge_stderr: if the standard error output should be merged with stdout
:param kwargs: module parameters
:returns: module standard output (stdout) as string or None if apture_stdout is False
:raises CalledModuleError: if module return code is non-zero
:raises ValueError: if the parameters are not correct
.. note::
The data read is buffered in memory, so do not use this method
if the data size is large or unlimited.
"""
# TODO: remove this:
do_doctest_gettext_workaround()
# implemenation inspired by subprocess.check_output() function
if stdin:
if 'input' in kwargs and kwargs['input'] != '-':
raise ValueError(_("input='-' must be used when stdin is specified"))
if stdin == subprocess.PIPE:
raise ValueError(_("stdin must be string or buffer, not PIPE"))
kwargs['stdin'] = subprocess.PIPE # to be able to send data to stdin
elif 'input' in kwargs and kwargs['input'] == '-':
raise ValueError(_("stdin must be used when input='-'"))
if merge_stderr and not (capture_stdout and capture_stderr):
raise ValueError(_("You cannot merge stdout and stderr and not capture them"))
if 'stdout' in kwargs:
raise TypeError(_("stdout argument not allowed, it could be overridden"))
if 'stderr' in kwargs:
raise TypeError(_("stderr argument not allowed, it could be overridden"))
if capture_stdout:
kwargs['stdout'] = subprocess.PIPE
if capture_stderr:
if merge_stderr:
kwargs['stderr'] = subprocess.STDOUT
else:
kwargs['stderr'] = subprocess.PIPE
process = start_command(module, **kwargs)
# input=None means no stdin (our default)
# for no stdout, output is None which is out interface
# for stderr=STDOUT or no stderr, errors is None
# which is fine for CalledModuleError
output, errors = process.communicate(input=stdin)
returncode = process.poll()
if returncode:
raise CalledModuleError(returncode, module, kwargs, errors)
return output
def main():
infile = options['input']
output = options['output']
method = options['method']
dtype = options['type']
fs = options['separator']
x = options['x']
y = options['y']
z = options['z']
value_column = options['value_column']
vrange = options['vrange']
vscale = options['vscale']
percent = options['percent']
pth = options['pth']
trim = options['trim']
workers = int(options['workers'])
scan_only = flags['s']
shell_style = flags['g']
ignore_broken = flags['i']
if workers is 1 and "WORKERS" in os.environ:
workers = int(os.environ["WORKERS"])
if not os.path.exists(infile):
grass.fatal(_("Unable to read input file <%s>") % infile)
addl_opts = {}
if pth:
addl_opts['pth'] = '%s' % pth
if trim:
addl_opts['trim'] = '%s' % trim
if value_column:
addl_opts['value_column'] = '%s' % value_column
if vrange:
addl_opts['vrange'] = '%s' % vrange
if vscale:
addl_opts['vscale'] = '%s' % vscale
if ignore_broken:
addl_opts['flags'] = 'i'
if scan_only or shell_style:
if shell_style:
doShell = 'g'
else:
doShell = ''
grass.run_command('r.in.xyz', flags='s' + doShell, input=infile,
output='dummy', sep=fs, x=x, y=y, z=z,
**addl_opts)
sys.exit()
if dtype == 'float':
data_type = 'FCELL'
else:
data_type = 'DCELL'
region = grass.region(region3d=True)
if region['nsres'] != region['nsres3'] or region['ewres'] != region['ewres3']:
grass.run_command('g.region', flags='3p')
grass.fatal(_("The 2D and 3D region settings are different. Can not continue."))
grass.verbose(_("Region bottom=%.15g top=%.15g vertical_cell_res=%.15g (%d depths)")
% (region['b'], region['t'], region['tbres'], region['depths']))
grass.verbose(_("Creating slices ..."))
# to avoid a point which falls exactly on a top bound from being
# considered twice, we shrink the
# For the top slice we keep it though, as someone scanning the bounds
# may have set the bounds exactly to the data extent (a bad idea, but
# it happens..)
eps = 1.0e-15
# if there are thousands of depths hopefully this array doesn't get too
# large and so we don't have to worry much about storing/looping through
# all the finished process infos.
proc = {}
pout = {}
depths = list(range(1, 1 + region['depths']))
for i in depths:
tmp_layer_name = 'tmp.r3xyz.%d.%s' % (os.getpid(), '%05d' % i)
zrange_min = region['b'] + (region['tbres'] * (i - 1))
if i < region['depths']:
zrange_max = region['b'] + (region['tbres'] * i) - eps
else:
zrange_max = region['b'] + (region['tbres'] * i)
# spawn depth layer import job in the background
#grass.debug("slice %d, <%s> %% %d" % (band, image[band], band % workers))
grass.message(_("Processing horizontal slice %d of %d [%.15g,%.15g) ...")
% (i, region['depths'], zrange_min, zrange_max))
proc[i] = grass.start_command('r.in.xyz', input=infile, output=tmp_layer_name,
sep=fs, method=method, x=x, y=y, z=z,
percent=percent, type=data_type,
zrange='%.15g,%.15g' % (zrange_min, zrange_max),
**addl_opts)
grass.debug("i=%d, %%=%d (workers=%d)" % (i, i % workers, workers))
# print sys.getsizeof(proc) # sizeof(proc array) [not so big]
if i % workers is 0:
# wait for the ones launched so far to finish
for p_i in depths[:i]:
pout[p_i] = proc[p_i].communicate()[0]
if proc[p_i].wait() is not 0:
grass.fatal(_("Trouble importing data. Aborting."))
# wait for jSobs to finish, collect any stray output
for i in depths:
pout[i] = proc[i].communicate()[0]
if proc[i].wait() is not 0:
grass.fatal(_("Trouble importing data. Aborting."))
del proc
grass.verbose(_("Assembling 3D cube ..."))
# input order: lower most strata first
slices = grass.read_command('g.list', type='raster', sep=',',
pattern='tmp.r3xyz.%d.*' % os.getpid()).rstrip(os.linesep)
grass.debug(slices)
try:
grass.run_command('r.to.rast3', input=slices, output=output)
except CalledModuleError:
grass.message(_("Done. 3D raster map <%s> created.") % output)
def main():
infile = options['input']
output = options['output']
method = options['method']
dtype = options['type']
fs = options['separator']
x = options['x']
y = options['y']
z = options['z']
value_column = options['value_column']
vrange = options['vrange']
vscale = options['vscale']
percent = options['percent']
pth = options['pth']
trim = options['trim']
workers = int(options['workers'])
scan_only = flags['s']
shell_style = flags['g']
ignore_broken = flags['i']
if workers == 1 and "WORKERS" in os.environ:
workers = int(os.environ["WORKERS"])
if not os.path.exists(infile):
grass.fatal(_("Unable to read input file <%s>") % infile)
addl_opts = {}
if pth:
addl_opts['pth'] = '%s' % pth
if trim:
addl_opts['trim'] = '%s' % trim
if value_column:
addl_opts['value_column'] = '%s' % value_column
if vrange:
addl_opts['vrange'] = '%s' % vrange
if vscale:
addl_opts['vscale'] = '%s' % vscale
if ignore_broken:
addl_opts['flags'] = 'i'
if scan_only or shell_style:
if shell_style:
doShell = 'g'
else:
doShell = ''
grass.run_command('r.in.xyz', flags='s' + doShell, input=infile,
output='dummy', sep=fs, x=x, y=y, z=z,
**addl_opts)
sys.exit()
if dtype == 'float':
data_type = 'FCELL'
else:
data_type = 'DCELL'
region = grass.region(region3d=True)
if region['nsres'] != region['nsres3'] or region['ewres'] != region['ewres3']:
grass.run_command('g.region', flags='3p')
grass.fatal(_("The 2D and 3D region settings are different. Can not continue."))
grass.verbose(_("Region bottom=%.15g top=%.15g vertical_cell_res=%.15g (%d depths)")
% (region['b'], region['t'], region['tbres'], region['depths']))
grass.verbose(_("Creating slices ..."))
# to avoid a point which falls exactly on a top bound from being
# considered twice, we shrink the
# For the top slice we keep it though, as someone scanning the bounds
# may have set the bounds exactly to the data extent (a bad idea, but
# it happens..)
eps = 1.0e-15
# if there are thousands of depths hopefully this array doesn't get too
# large and so we don't have to worry much about storing/looping through
# all the finished process infos.
proc = {}
pout = {}
depths = list(range(1, 1 + region['depths']))
for i in depths:
tmp_layer_name = 'tmp.r3xyz.%d.%s' % (os.getpid(), '%05d' % i)
zrange_min = region['b'] + (region['tbres'] * (i - 1))
if i < region['depths']:
zrange_max = region['b'] + (region['tbres'] * i) - eps
else:
zrange_max = region['b'] + (region['tbres'] * i)
# spawn depth layer import job in the background
#grass.debug("slice %d, <%s> %% %d" % (band, image[band], band % workers))
grass.message(_("Processing horizontal slice %d of %d [%.15g,%.15g) ...")
% (i, region['depths'], zrange_min, zrange_max))
proc[i] = grass.start_command('r.in.xyz', input=infile, output=tmp_layer_name,
sep=fs, method=method, x=x, y=y, z=z,
percent=percent, type=data_type,
zrange='%.15g,%.15g' % (zrange_min, zrange_max),
**addl_opts)
grass.debug("i=%d, %%=%d (workers=%d)" % (i, i % workers, workers))
# print sys.getsizeof(proc) # sizeof(proc array) [not so big]
if i % workers == 0:
# wait for the ones launched so far to finish
for p_i in depths[:i]:
pout[p_i] = proc[p_i].communicate()[0]
if proc[p_i].wait() != 0:
grass.fatal(_("Trouble importing data. Aborting."))
# wait for jSobs to finish, collect any stray output
for i in depths:
pout[i] = proc[i].communicate()[0]
if proc[i].wait() != 0:
grass.fatal(_("Trouble importing data. Aborting."))
del proc
grass.verbose(_("Assembling 3D cube ..."))
# input order: lower most strata first
slices = grass.read_command('g.list', type='raster', sep=',',
pattern='tmp.r3xyz.%d.*' % os.getpid()).rstrip(os.linesep)
grass.debug(slices)
try:
grass.run_command('r.to.rast3', input=slices, output=output)
except CalledModuleError:
grass.message(_("Done. 3D raster map <%s> created.") % output)
def RunCommand(prog,
flags="",
overwrite=False,
quiet=False,
verbose=False,
parent=None,
read=False,
parse=None,
stdin=None,
getErrorMsg=False,
**kwargs):
"""Run GRASS command
:param prog: program to run
:param flags: flags given as a string
:param overwrite, quiet, verbose: flags
:param parent: parent window for error messages
:param read: fetch stdout
:param parse: fn to parse stdout (e.g. grass.parse_key_val) or None
:param stdin: stdin or None
:param getErrorMsg: get error messages on failure
:param kwargs: program parameters
:return: returncode (read == False and getErrorMsg == False)
:return: returncode, messages (read == False and getErrorMsg == True)
:return: stdout (read == True and getErrorMsg == False)
:return: returncode, stdout, messages (read == True and getErrorMsg == True)
:return: stdout, stderr
"""
cmdString = ' '.join(
grass.make_command(prog, flags, overwrite, quiet, verbose, **kwargs))
Debug.msg(1, "gcmd.RunCommand(): %s" % cmdString)
kwargs['stderr'] = subprocess.PIPE
if read:
kwargs['stdout'] = subprocess.PIPE
if stdin:
kwargs['stdin'] = subprocess.PIPE
if parent:
messageFormat = os.getenv('GRASS_MESSAGE_FORMAT', 'gui')
os.environ['GRASS_MESSAGE_FORMAT'] = 'standard'
Debug.msg(2, "gcmd.RunCommand(): command started")
start = time.time()
ps = grass.start_command(prog, flags, overwrite, quiet, verbose, **kwargs)
if stdin:
ps.stdin.write(stdin)
ps.stdin.close()
ps.stdin = None
Debug.msg(3, "gcmd.RunCommand(): decoding string")
stdout, stderr = map(DecodeString, ps.communicate())
if parent: # restore previous settings
os.environ['GRASS_MESSAGE_FORMAT'] = messageFormat
ret = ps.returncode
Debug.msg(1, "gcmd.RunCommand(): get return code %d (%.6f sec)" % \
(ret, (time.time() - start)))
Debug.msg(3, "gcmd.RunCommand(): print error")
if ret != 0:
if stderr:
Debug.msg(2, "gcmd.RunCommand(): error %s" % stderr)
else:
Debug.msg(2, "gcmd.RunCommand(): nothing to print ???")
if parent:
GError(parent=parent,
caption=_("Error in %s") % prog,
message=stderr)
Debug.msg(3, "gcmd.RunCommand(): print read error")
if not read:
if not getErrorMsg:
return ret
else:
return ret, _formatMsg(stderr)
if stdout:
Debug.msg(2, "gcmd.RunCommand(): return stdout\n'%s'" % stdout)
else:
Debug.msg(2, "gcmd.RunCommand(): return stdout = None")
if parse:
stdout = parse(stdout)
if not getErrorMsg:
return stdout
Debug.msg(2, "gcmd.RunCommand(): return ret, stdout")
if read and getErrorMsg:
return ret, stdout, _formatMsg(stderr)
Debug.msg(2, "gcmd.RunCommand(): return result")
return stdout, _formatMsg(stderr)
def RunCommand(
prog,
flags="",
overwrite=False,
quiet=False,
verbose=False,
parent=None,
read=False,
parse=None,
stdin=None,
getErrorMsg=False,
env=None,
**kwargs,
):
"""Run GRASS command
:param prog: program to run
:param flags: flags given as a string
:param overwrite, quiet, verbose: flags
:param parent: parent window for error messages
:param read: fetch stdout
:param parse: fn to parse stdout (e.g. grass.parse_key_val) or None
:param stdin: stdin or None
:param getErrorMsg: get error messages on failure
:param env: environment (optional, uses os.environ if not provided)
:param kwargs: program parameters
The environment passed to the function (env or os.environ) is not modified (a copy is used internally).
:return: returncode (read == False and getErrorMsg == False)
:return: returncode, messages (read == False and getErrorMsg == True)
:return: stdout (read == True and getErrorMsg == False)
:return: returncode, stdout, messages (read == True and getErrorMsg == True)
:return: stdout, stderr
"""
cmdString = " ".join(
grass.make_command(prog, flags, overwrite, quiet, verbose, **kwargs)
)
Debug.msg(1, "gcmd.RunCommand(): %s" % cmdString)
kwargs["stderr"] = subprocess.PIPE
if read:
kwargs["stdout"] = subprocess.PIPE
if stdin:
kwargs["stdin"] = subprocess.PIPE
# Do not change the environment, only a local copy.
if env:
env = env.copy()
else:
env = os.environ.copy()
if parent:
env["GRASS_MESSAGE_FORMAT"] = "standard"
start = time.time()
ps = grass.start_command(prog, flags, overwrite, quiet, verbose, env=env, **kwargs)
if stdin:
ps.stdin.write(encode(stdin))
ps.stdin.close()
ps.stdin = None
stdout, stderr = ps.communicate()
stderr = decode(stderr)
stdout = decode(stdout) if read else stdout
ret = ps.returncode
Debug.msg(
1,
"gcmd.RunCommand(): get return code %d (%.6f sec)"
% (ret, (time.time() - start)),
)
if ret != 0:
if stderr:
Debug.msg(2, "gcmd.RunCommand(): error %s" % stderr)
else:
Debug.msg(2, "gcmd.RunCommand(): nothing to print ???")
if parent:
GError(parent=parent, caption=_("Error in %s") % prog, message=stderr)
if not read:
if not getErrorMsg:
return ret
else:
return ret, _formatMsg(stderr)
if stdout:
Debug.msg(3, "gcmd.RunCommand(): return stdout\n'%s'" % stdout)
else:
Debug.msg(3, "gcmd.RunCommand(): return stdout = None")
if parse:
stdout = parse(stdout)
if not getErrorMsg:
return stdout
if read and getErrorMsg:
return ret, stdout, _formatMsg(stderr)
return stdout, _formatMsg(stderr)
def RunCommand(prog, flags = "", overwrite = False, quiet = False, verbose = False,
parent = None, read = False, parse = None, stdin = None, getErrorMsg = False, **kwargs):
"""!Run GRASS command
@param prog program to run
@param flags flags given as a string
@param overwrite, quiet, verbose flags
@param parent parent window for error messages
@param read fetch stdout
@param parse fn to parse stdout (e.g. grass.parse_key_val) or None
@param stdin stdin or None
@param getErrorMsg get error messages on failure
@param kwargs program parameters
@return returncode (read == False and getErrorMsg == False)
@return returncode, messages (read == False and getErrorMsg == True)
@return stdout (read == True and getErrorMsg == False)
@return returncode, stdout, messages (read == True and getErrorMsg == True)
@return stdout, stderr
"""
cmdString = ' '.join(grass.make_command(prog, flags, overwrite,
quiet, verbose, **kwargs))
Debug.msg(1, "gcmd.RunCommand(): %s" % cmdString)
kwargs['stderr'] = subprocess.PIPE
if read:
kwargs['stdout'] = subprocess.PIPE
if stdin:
kwargs['stdin'] = subprocess.PIPE
ps = grass.start_command(prog, flags, overwrite, quiet, verbose, **kwargs)
Debug.msg(2, "gcmd.RunCommand(): command started")
if stdin:
ps.stdin.write(stdin)
ps.stdin.close()
ps.stdin = None
Debug.msg(3, "gcmd.RunCommand(): decoding string")
stdout, stderr = map(utils.DecodeString, ps.communicate())
ret = ps.returncode
Debug.msg(1, "gcmd.RunCommand(): get return code %d" % ret)
Debug.msg(3, "gcmd.RunCommand(): print error")
if ret != 0 and parent:
Debug.msg(2, "gcmd.RunCommand(): error %s" % stderr)
if (stderr == None):
Debug.msg(2, "gcmd.RunCommand(): nothing to print ???")
else:
GError(parent = parent,
message = stderr)
Debug.msg(3, "gcmd.RunCommand(): print read error")
if not read:
if not getErrorMsg:
return ret
else:
return ret, _formatMsg(stderr)
if stdout:
Debug.msg(2, "gcmd.RunCommand(): return stdout\n'%s'" % stdout)
else:
Debug.msg(2, "gcmd.RunCommand(): return stdout = None")
if parse:
stdout = parse(stdout)
if not getErrorMsg:
return stdout
Debug.msg(2, "gcmd.RunCommand(): return ret, stdout")
if read and getErrorMsg:
return ret, stdout, _formatMsg(stderr)
Debug.msg(2, "gcmd.RunCommand(): return result")
return stdout, _formatMsg(stderr)
def read2_command(*args, **kwargs):
kwargs["stdout"] = gcore.PIPE
kwargs["stderr"] = gcore.PIPE
ps = gcore.start_command(*args, **kwargs)
stdout, stderr = ps.communicate()
return ps.returncode, DecodeString(stdout), DecodeString(stderr)
def read2_command(*args, **kwargs):
kwargs['stdout'] = gcore.PIPE
kwargs['stderr'] = gcore.PIPE
ps = gcore.start_command(*args, **kwargs)
stdout, stderr = ps.communicate()
return ps.returncode, stdout, stderr
def _read2_command(*args, **kwargs):
kwargs['stdout'] = gcore.PIPE
kwargs['stderr'] = gcore.PIPE
ps = gcore.start_command(*args, **kwargs)
stdout, stderr = ps.communicate()
return ps.returncode, stdout, stderr
def RunCommand(prog, flags = "", overwrite = False, quiet = False,
verbose = False, parent = None, read = False,
parse = None, stdin = None, getErrorMsg = False, **kwargs):
"""Run GRASS command
:param prog: program to run
:param flags: flags given as a string
:param overwrite, quiet, verbose: flags
:param parent: parent window for error messages
:param read: fetch stdout
:param parse: fn to parse stdout (e.g. grass.parse_key_val) or None
:param stdin: stdin or None
:param getErrorMsg: get error messages on failure
:param kwargs: program parameters
:return: returncode (read == False and getErrorMsg == False)
:return: returncode, messages (read == False and getErrorMsg == True)
:return: stdout (read == True and getErrorMsg == False)
:return: returncode, stdout, messages (read == True and getErrorMsg == True)
:return: stdout, stderr
"""
cmdString = ' '.join(grass.make_command(prog, flags, overwrite,
quiet, verbose, **kwargs))
Debug.msg(1, "gcmd.RunCommand(): %s" % cmdString)
kwargs['stderr'] = subprocess.PIPE
if read:
kwargs['stdout'] = subprocess.PIPE
if stdin:
kwargs['stdin'] = subprocess.PIPE
if parent:
messageFormat = os.getenv('GRASS_MESSAGE_FORMAT', 'gui')
os.environ['GRASS_MESSAGE_FORMAT'] = 'standard'
start = time.time()
ps = grass.start_command(prog, flags, overwrite, quiet, verbose, **kwargs)
if stdin:
ps.stdin.write(stdin)
ps.stdin.close()
ps.stdin = None
stdout, stderr = map(DecodeString, ps.communicate())
if parent: # restore previous settings
os.environ['GRASS_MESSAGE_FORMAT'] = messageFormat
ret = ps.returncode
Debug.msg(1, "gcmd.RunCommand(): get return code %d (%.6f sec)" % \
(ret, (time.time() - start)))
if ret != 0:
if stderr:
Debug.msg(2, "gcmd.RunCommand(): error %s" % stderr)
else:
Debug.msg(2, "gcmd.RunCommand(): nothing to print ???")
if parent:
GError(parent = parent,
caption = _("Error in %s") % prog,
message = stderr)
if not read:
if not getErrorMsg:
return ret
else:
return ret, _formatMsg(stderr)
if stdout:
Debug.msg(3, "gcmd.RunCommand(): return stdout\n'%s'" % stdout)
else:
Debug.msg(3, "gcmd.RunCommand(): return stdout = None")
if parse:
stdout = parse(stdout)
if not getErrorMsg:
return stdout
if read and getErrorMsg:
return ret, stdout, _formatMsg(stderr)
return stdout, _formatMsg(stderr)
def RunCommand(prog,
flags="",
overwrite=False,
quiet=False,
verbose=False,
parent=None,
read=False,
stdin=None,
getErrorMsg=False,
**kwargs):
"""!Run GRASS command
@param prog program to run
@param flags flags given as a string
@param overwrite, quiet, verbose flags
@param parent parent window for error messages
@param read fetch stdout
@param stdin stdin or None
@param getErrorMsg get error messages on failure
@param kwargs program parameters
@return returncode (read == False and getErrorMsg == False)
@return returncode, messages (read == False and getErrorMsg == True)
@return stdout (read == True and getErrorMsg == False)
@return returncode, stdout, messages (read == True and getErrorMsg == True)
@return stdout, stderr
"""
cmdString = ' '.join(
grass.make_command(prog, flags, overwrite, quiet, verbose, **kwargs))
Debug.msg(1, "gcmd.RunCommand(): %s" % cmdString)
kwargs['stderr'] = subprocess.PIPE
if read:
kwargs['stdout'] = subprocess.PIPE
if stdin:
kwargs['stdin'] = subprocess.PIPE
ps = grass.start_command(GetRealCmd(prog), flags, overwrite, quiet,
verbose, **kwargs)
Debug.msg(2, "gcmd.RunCommand(): command started")
if stdin:
ps.stdin.write(stdin)
ps.stdin.close()
ps.stdin = None
Debug.msg(3, "gcmd.RunCommand(): decoding string")
stdout, stderr = map(DecodeString, ps.communicate())
ret = ps.returncode
Debug.msg(1, "gcmd.RunCommand(): get return code %d" % ret)
Debug.msg(3, "gcmd.RunCommand(): print error")
if ret != 0 and parent:
Debug.msg(2, "gcmd.RunCommand(): error %s" % stderr)
if (stderr == None):
Debug.msg(2, "gcmd.RunCommand(): nothing to print ???")
else:
GError(parent=parent, message=stderr)
Debug.msg(3, "gcmd.RunCommand(): print read error")
if not read:
if not getErrorMsg:
return ret
else:
return ret, _formatMsg(stderr)
if stdout:
Debug.msg(2, "gcmd.RunCommand(): return stdout\n'%s'" % stdout)
else:
Debug.msg(2, "gcmd.RunCommand(): return stdout = None")
if not getErrorMsg:
return stdout
Debug.msg(2, "gcmd.RunCommand(): return ret, stdout")
if read and getErrorMsg:
return ret, stdout, _formatMsg(stderr)
Debug.msg(2, "gcmd.RunCommand(): return result")
return stdout, _formatMsg(stderr)