def __init__(self,
cmd,
width=None,
height=None,
overlap=0,
processes=None,
split=False,
debug=False,
region=None,
move=None,
log=False,
start_row=0,
start_col=0,
out_prefix='',
*args,
**kargs):
kargs['run_'] = False
self.mset = Mapset()
self.module = Module(cmd, *args, **kargs)
self.width = width
self.height = height
self.overlap = overlap
self.processes = processes
self.region = region if region else Region()
self.start_row = start_row
self.start_col = start_col
self.out_prefix = out_prefix
self.log = log
self.move = move
self.gisrc_src = os.environ['GISRC']
self.n_mset, self.gisrc_dst = None, None
if self.move:
self.n_mset = copy_mapset(self.mset, self.move)
self.gisrc_dst = write_gisrc(self.n_mset.gisdbase,
self.n_mset.location,
self.n_mset.name)
rasters = [r for r in select(self.module.inputs, 'raster')]
if rasters:
copy_rasters(rasters,
self.gisrc_src,
self.gisrc_dst,
region=self.region)
vectors = [v for v in select(self.module.inputs, 'vector')]
if vectors:
copy_vectors(vectors, self.gisrc_src, self.gisrc_dst)
groups = [g for g in select(self.module.inputs, 'group')]
if groups:
copy_groups(groups,
self.gisrc_src,
self.gisrc_dst,
region=self.region)
self.bboxes = split_region_tiles(region=region,
width=width,
height=height,
overlap=overlap)
self.msetstr = cmd.replace('.', '') + "_%03d_%03d"
self.inlist = None
if split:
self.split()
self.debug = debug
def get_works(self):
"""Return a list of tuble with the parameters for cmd_exe function"""
works = []
reg = Region()
if self.move:
mdst, ldst, gdst = read_gisrc(self.gisrc_dst)
else:
ldst, gdst = self.mset.location, self.mset.gisdbase
cmd = self.module.get_dict()
groups = [g for g in select(self.module.inputs, 'group')]
for row, box_row in enumerate(self.bboxes):
for col, box in enumerate(box_row):
inms = None
if self.inlist:
inms = {}
cols = len(box_row)
for key in self.inlist:
indx = row * cols + col
inms[key] = "%s@%s" % (self.inlist[key][indx],
self.mset.name)
# set the computational region, prepare the region parameters
bbox = dict([(k[0], str(v)) for k, v in box.items()[:-2]])
bbox['nsres'] = '%f' % reg.nsres
bbox['ewres'] = '%f' % reg.ewres
new_mset = self.msetstr % (self.start_row + row,
self.start_col + col),
works.append((bbox, inms,
self.gisrc_src,
write_gisrc(gdst, ldst, new_mset),
cmd, groups))
return works
def get_works(self):
"""Return a list of tuble with the parameters for cmd_exe function"""
works = []
reg = Region()
if self.move:
mdst, ldst, gdst = read_gisrc(self.gisrc_dst)
else:
ldst, gdst = self.mset.location, self.mset.gisdbase
cmd = self.module.get_dict()
groups = [g for g in select(self.module.inputs, 'group')]
for row, box_row in enumerate(self.bboxes):
for col, box in enumerate(box_row):
inms = None
if self.inlist:
inms = {}
cols = len(box_row)
for key in self.inlist:
indx = row * cols + col
inms[key] = "%s@%s" % (self.inlist[key][indx],
self.mset.name)
# set the computational region, prepare the region parameters
bbox = dict([(k[0], str(v)) for k, v in box.items()[:-2]])
bbox['nsres'] = '%f' % reg.nsres
bbox['ewres'] = '%f' % reg.ewres
new_mset = self.msetstr % (self.start_row + row,
self.start_col + col),
works.append((bbox, inms, self.gisrc_src,
write_gisrc(gdst, ldst, new_mset), cmd, groups))
return works
def __init__(self, cmd, width=None, height=None, overlap=0, processes=None,
split=False, debug=False, region=None, move=None, log=False,
start_row=0, start_col=0, out_prefix='', mapset_prefix=None,
*args, **kargs):
kargs['run_'] = False
self.mset = Mapset()
self.module = Module(cmd, *args, **kargs)
self.width = width
self.height = height
self.overlap = overlap
self.processes = processes
self.region = region if region else Region()
self.start_row = start_row
self.start_col = start_col
self.out_prefix = out_prefix
self.log = log
self.move = move
self.gisrc_src = os.environ['GISRC']
self.n_mset, self.gisrc_dst = None, None
if self.move:
self.n_mset = copy_mapset(self.mset, self.move)
self.gisrc_dst = write_gisrc(self.n_mset.gisdbase,
self.n_mset.location,
self.n_mset.name)
rasters = [r for r in select(self.module.inputs, 'raster')]
if rasters:
copy_rasters(rasters, self.gisrc_src, self.gisrc_dst,
region=self.region)
vectors = [v for v in select(self.module.inputs, 'vector')]
if vectors:
copy_vectors(vectors, self.gisrc_src, self.gisrc_dst)
groups = [g for g in select(self.module.inputs, 'group')]
if groups:
copy_groups(groups, self.gisrc_src, self.gisrc_dst,
region=self.region)
self.bboxes = split_region_tiles(region=region,
width=width, height=height,
overlap=overlap)
if mapset_prefix:
self.msetstr = mapset_prefix + "_%03d_%03d"
else:
self.msetstr = cmd.replace('.', '') + "_%03d_%03d"
self.inlist = None
if split:
self.split()
self.debug = debug
def _run_test_module(self, module, results_dir, gisdbase, location):
"""Run one test file."""
self.testsuite_dirs[module.tested_dir].append(module.name)
cwd = os.path.join(results_dir, module.tested_dir, module.name)
data_dir = os.path.join(module.file_dir, 'data')
if os.path.exists(data_dir):
# TODO: link dir instead of copy tree and remove link afterwads
# (removing is good because of testsuite dir in samplecode)
# TODO: use different dir name in samplecode and test if it works
shutil.copytree(data_dir, os.path.join(cwd, 'data'),
ignore=shutil.ignore_patterns('*.svn*'))
ensure_dir(os.path.abspath(cwd))
# TODO: put this to constructor and copy here again
env = os.environ.copy()
mapset, mapset_dir = self._create_mapset(gisdbase, location, module)
gisrc = gsetup.write_gisrc(gisdbase, location, mapset)
# here is special setting of environmental variables for running tests
# some of them might be set from outside in the future and if the list
# will be long they should be stored somewhere separately
# use custom gisrc, not current session gisrc
env['GISRC'] = gisrc
# percentage in plain format is 0...10...20... ...100
env['GRASS_MESSAGE_FORMAT'] = 'plain'
stdout_path = os.path.join(cwd, 'stdout.txt')
stderr_path = os.path.join(cwd, 'stderr.txt')
stdout = open(stdout_path, 'w')
stderr = open(stderr_path, 'w')
self.reporter.start_file_test(module)
# TODO: we might clean the directory here before test if non-empty
if module.file_type == 'py':
# ignoring shebang line to use current Python
# and also pass parameters to it
# add also '-Qwarn'?
p = subprocess.Popen([sys.executable, '-tt', '-3',
module.abs_file_path],
cwd=cwd, env=env,
stdout=stdout, stderr=stderr)
elif module.file_type == 'sh':
# ignoring shebang line to pass parameters to shell
# expecting system to have sh or something compatible
# TODO: add some special checks for MS Windows
# using -x to see commands in stderr
# using -e to terminate fast
# from dash manual:
# -e errexit If not interactive, exit immediately if any
# untested command fails. The exit status of a com‐
# mand is considered to be explicitly tested if the
# command is used to control an if, elif, while, or
# until; or if the command is the left hand operand
# of an '&&' or '||' operator.
p = subprocess.Popen(['sh', '-e', '-x', module.abs_file_path],
cwd=cwd, env=env,
stdout=stdout, stderr=stderr)
else:
p = subprocess.Popen([module.abs_file_path],
cwd=cwd, env=env,
stdout=stdout, stderr=stderr)
returncode = p.wait()
stdout.close()
stderr.close()
self._file_anonymizer.anonymize([stdout_path, stderr_path])
test_summary = update_keyval_file(
os.path.join(os.path.abspath(cwd), 'test_keyvalue_result.txt'),
module=module, returncode=returncode)
self.reporter.end_file_test(module=module, cwd=cwd,
returncode=returncode,
stdout=stdout_path, stderr=stderr_path,
test_summary=test_summary)
# TODO: add some try-except or with for better error handling
os.remove(gisrc)
# TODO: only if clean up
if self.clean_mapsets:
shutil.rmtree(mapset_dir)
def _run_test_module(self, module, results_dir, gisdbase, location):
"""Run one test file."""
self.testsuite_dirs[module.tested_dir].append(module.name)
cwd = os.path.join(results_dir, module.tested_dir, module.name)
data_dir = os.path.join(module.file_dir, "data")
if os.path.exists(data_dir):
# TODO: link dir instead of copy tree and remove link afterwads
# (removing is good because of testsuite dir in samplecode)
# TODO: use different dir name in samplecode and test if it works
shutil.copytree(
data_dir,
os.path.join(cwd, "data"),
ignore=shutil.ignore_patterns("*.svn*"),
)
ensure_dir(os.path.abspath(cwd))
# TODO: put this to constructor and copy here again
env = os.environ.copy()
mapset, mapset_dir = self._create_mapset(gisdbase, location, module)
gisrc = gsetup.write_gisrc(gisdbase, location, mapset)
# here is special setting of environmental variables for running tests
# some of them might be set from outside in the future and if the list
# will be long they should be stored somewhere separately
# use custom gisrc, not current session gisrc
env["GISRC"] = gisrc
# percentage in plain format is 0...10...20... ...100
env["GRASS_MESSAGE_FORMAT"] = "plain"
stdout_path = os.path.join(cwd, "stdout.txt")
stderr_path = os.path.join(cwd, "stderr.txt")
self.reporter.start_file_test(module)
# TODO: we might clean the directory here before test if non-empty
if module.file_type == "py":
# ignoring shebang line to use current Python
# and also pass parameters to it
# add also '-Qwarn'?
if sys.version_info.major >= 3:
args = [sys.executable, "-tt", module.abs_file_path]
else:
args = [sys.executable, "-tt", "-3", module.abs_file_path]
p = subprocess.Popen(args,
cwd=cwd,
env=env,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
elif module.file_type == "sh":
# ignoring shebang line to pass parameters to shell
# expecting system to have sh or something compatible
# TODO: add some special checks for MS Windows
# using -x to see commands in stderr
# using -e to terminate fast
# from dash manual:
# -e errexit If not interactive, exit immediately if any
# untested command fails. The exit status of a com‐
# mand is considered to be explicitly tested if the
# command is used to control an if, elif, while, or
# until; or if the command is the left hand operand
# of an '&&' or '||' operator.
p = subprocess.Popen(
["sh", "-e", "-x", module.abs_file_path],
cwd=cwd,
env=env,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
else:
p = subprocess.Popen(
[module.abs_file_path],
cwd=cwd,
env=env,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
stdout, stderr = p.communicate()
returncode = p.returncode
encodings = [_get_encoding(), "utf8", "latin-1", "ascii"]
def try_decode(data, encodings):
"""Try to decode data (bytes) using one of encodings
Falls back to decoding as UTF-8 with replacement for bytes.
Strings are returned unmodified.
"""
for encoding in encodings:
try:
return decode(data, encoding=encoding)
except UnicodeError:
pass
if isinstance(data, bytes):
return data.decode(encoding="utf-8", errors="replace")
return data
stdout = try_decode(stdout, encodings=encodings)
stderr = try_decode(stderr, encodings=encodings)
with open(stdout_path, "w") as stdout_file:
stdout_file.write(stdout)
with open(stderr_path, "w") as stderr_file:
if type(stderr) == "bytes":
stderr_file.write(decode(stderr))
else:
if isinstance(stderr, str):
stderr_file.write(stderr)
else:
stderr_file.write(stderr.encode("utf8"))
self._file_anonymizer.anonymize([stdout_path, stderr_path])
test_summary = update_keyval_file(
os.path.join(os.path.abspath(cwd), "test_keyvalue_result.txt"),
module=module,
returncode=returncode,
)
self.reporter.end_file_test(
module=module,
cwd=cwd,
returncode=returncode,
stdout=stdout_path,
stderr=stderr_path,
test_summary=test_summary,
)
# TODO: add some try-except or with for better error handling
os.remove(gisrc)
# TODO: only if clean up
if self.clean_mapsets:
shutil.rmtree(mapset_dir)
def export_png_in_projection(src_mapset_name,
map_name,
output_file,
epsg_code,
routpng_flags,
compression,
wgs84_file,
use_region=True):
"""
:param use_region: use computation region and not map extent
"""
if use_region:
src_region = get_region()
src_proj_string = get_location_proj_string()
# TODO: change only location and not gisdbase?
# we rely on the tmp dir having enough space for our map
tgt_gisdbase = tempfile.mkdtemp()
# this is not needed if we use mkdtemp but why not
tgt_location = 'r.out.png.proj_location_%s' % epsg_code
# because we are using PERMANENT we don't have to create mapset explicitly
tgt_mapset_name = 'PERMANENT'
src_mapset = Mapset(name=src_mapset_name, use_current=True)
assert src_mapset.exists()
# get source (old) and set target (new) GISRC enviromental variable
# TODO: set environ only for child processes could be enough and it would
# enable (?) parallel runs
src_gisrc = os.environ['GISRC']
tgt_gisrc = gsetup.write_gisrc(tgt_gisdbase, tgt_location, tgt_mapset_name)
os.environ['GISRC'] = tgt_gisrc
# we do this only after we obtained region, so it was applied
# and we don't need it in the temporary (tgt) mapset
if os.environ.get('WIND_OVERRIDE'):
old_temp_region = os.environ['WIND_OVERRIDE']
del os.environ['WIND_OVERRIDE']
else:
old_temp_region = None
tgt_mapset = Mapset(tgt_gisdbase, tgt_location, tgt_mapset_name)
try:
# the function itself is not safe for other (backgroud) processes
# (e.g. GUI), however we already switched GISRC for us
# and child processes, so we don't influece others
gs.create_location(dbase=tgt_gisdbase,
location=tgt_location,
epsg=epsg_code,
datum=None,
datum_trans=None)
assert tgt_mapset.exists()
# we need to make the mapset change in the current GISRC (tgt)
# note that the C library for this process still holds the
# path to the old GISRC file (src)
tgt_mapset.set_as_current(gisrc=tgt_gisrc)
# setting region
if use_region:
# respecting computation region of the src location
# by previous use g.region in src location
# and m.proj and g.region now
# respecting MASK of the src location would be hard
# null values in map are usually enough
tgt_proj_string = get_location_proj_string()
tgt_region = reproject_region(src_region,
from_proj=src_proj_string,
to_proj=tgt_proj_string)
# uses g.region thus and sets region only for child processes
# which is enough now
# TODO: unlike the other branch, this keeps the current
# resolution which is not correct
set_region(tgt_region)
else:
# find out map extent to import everything
# using only classic API because of some problems with pygrass
# on ms windows
rproj_out = gs.read_command('r.proj',
input=map_name,
dbase=src_mapset.database,
location=src_mapset.location,
mapset=src_mapset.name,
output=map_name,
flags='g')
a = gs.parse_key_val(rproj_out, sep='=', vsep=' ')
gs.run_command('g.region', **a)
# map import
gs.message("Reprojecting...")
gs.run_command('r.proj',
input=map_name,
dbase=src_mapset.database,
location=src_mapset.location,
mapset=src_mapset.name,
output=map_name,
quiet=True)
# actual export
gs.message("Rendering...")
raster_to_png(map_name,
output_file,
compression=compression,
routpng_flags=routpng_flags)
# outputting file with WGS84 coordinates
if wgs84_file:
gs.verbose("Projecting coordinates to LL WGS 84...")
with open(wgs84_file, 'w') as data_file:
if use_region:
# map which is smaller than region is imported in its own
# small extent, but we export image in region, so we need
# bounds to be for region, not map
# hopefully this is consistent with r.out.png behavior
data_file.write(
map_extent_to_file_content(proj_to_wgs84(get_region()))
+ '\n')
else:
# use map to get extent
# the result is actually the same as using map
# if region is the same as map (use_region == False)
data_file.write(
map_extent_to_file_content(
get_map_extent_for_location(map_name)) + '\n')
finally:
# juts in case we need to do something in the old location
# our callers probably do
os.environ['GISRC'] = src_gisrc
if old_temp_region:
os.environ['WIND_OVERRIDE'] = old_temp_region
# set current in library
src_mapset.set_as_current(gisrc=src_gisrc)
# delete the whole gisdbase
# delete file by file to ensure that we are deleting only our things
# exception will be raised when removing non-empty directory
tgt_mapset.delete()
os.rmdir(tgt_mapset.location_path)
# dir created by tempfile.mkdtemp() needs to be romved manually
os.rmdir(tgt_gisdbase)
# we have to remove file created by tempfile.mkstemp function
# in write_gisrc function
os.remove(tgt_gisrc)
def export_png_in_projection(src_mapset_name, map_name, output_file,
epsg_code,
routpng_flags, compression, wgs84_file,
use_region=True):
"""
:param use_region: use computation region and not map extent
"""
if use_region:
src_region = get_region()
src_proj_string = get_location_proj_string()
# TODO: change only location and not gisdbase?
# we rely on the tmp dir having enough space for our map
tgt_gisdbase = tempfile.mkdtemp()
# this is not needed if we use mkdtemp but why not
tgt_location = 'r.out.png.proj_location_%s' % epsg_code
# because we are using PERMANENT we don't have to create mapset explicitly
tgt_mapset_name = 'PERMANENT'
src_mapset = Mapset(name=src_mapset_name, use_current=True)
assert src_mapset.exists()
# get source (old) and set target (new) GISRC enviromental variable
# TODO: set environ only for child processes could be enough and it would
# enable (?) parallel runs
src_gisrc = os.environ['GISRC']
tgt_gisrc = gsetup.write_gisrc(tgt_gisdbase,
tgt_location, tgt_mapset_name)
os.environ['GISRC'] = tgt_gisrc
# we do this only after we obtained region, so it was applied
# and we don't need it in the temporary (tgt) mapset
if os.environ.get('WIND_OVERRIDE'):
old_temp_region = os.environ['WIND_OVERRIDE']
del os.environ['WIND_OVERRIDE']
else:
old_temp_region = None
tgt_mapset = Mapset(tgt_gisdbase, tgt_location, tgt_mapset_name)
try:
# the function itself is not safe for other (backgroud) processes
# (e.g. GUI), however we already switched GISRC for us
# and child processes, so we don't influece others
gs.create_location(dbase=tgt_gisdbase,
location=tgt_location,
epsg=epsg_code,
datum=None,
datum_trans=None)
assert tgt_mapset.exists()
# we need to make the mapset change in the current GISRC (tgt)
# note that the C library for this process still holds the
# path to the old GISRC file (src)
tgt_mapset.set_as_current(gisrc=tgt_gisrc)
# setting region
if use_region:
# respecting computation region of the src location
# by previous use g.region in src location
# and m.proj and g.region now
# respecting MASK of the src location would be hard
# null values in map are usually enough
tgt_proj_string = get_location_proj_string()
tgt_region = reproject_region(src_region,
from_proj=src_proj_string,
to_proj=tgt_proj_string)
# uses g.region thus and sets region only for child processes
# which is enough now
# TODO: unlike the other branch, this keeps the current
# resolution which is not correct
set_region(tgt_region)
else:
# find out map extent to import everything
# using only classic API because of some problems with pygrass
# on ms windows
rproj_out = gs.read_command('r.proj', input=map_name,
dbase=src_mapset.database,
location=src_mapset.location,
mapset=src_mapset.name,
output=map_name, flags='g')
a = gs.parse_key_val(rproj_out, sep='=', vsep=' ')
gs.run_command('g.region', **a)
# map import
gs.message("Reprojecting...")
gs.run_command('r.proj', input=map_name, dbase=src_mapset.database,
location=src_mapset.location, mapset=src_mapset.name,
output=map_name, quiet=True)
# actual export
gs.message("Rendering...")
raster_to_png(map_name, output_file, compression=compression,
routpng_flags=routpng_flags)
# outputting file with WGS84 coordinates
if wgs84_file:
gs.verbose("Projecting coordinates to LL WGS 84...")
with open(wgs84_file, 'w') as data_file:
if use_region:
# map which is smaller than region is imported in its own
# small extent, but we export image in region, so we need
# bounds to be for region, not map
# hopefully this is consistent with r.out.png behavior
data_file.write(
map_extent_to_file_content(
proj_to_wgs84(get_region())) + '\n')
else:
# use map to get extent
# the result is actually the same as using map
# if region is the same as map (use_region == False)
data_file.write(
map_extent_to_file_content(
get_map_extent_for_location(map_name)) +
'\n')
finally:
# juts in case we need to do something in the old location
# our callers probably do
os.environ['GISRC'] = src_gisrc
if old_temp_region:
os.environ['WIND_OVERRIDE'] = old_temp_region
# set current in library
src_mapset.set_as_current(gisrc=src_gisrc)
# delete the whole gisdbase
# delete file by file to ensure that we are deleting only our things
# exception will be raised when removing non-empty directory
tgt_mapset.delete()
os.rmdir(tgt_mapset.location_path)
# dir created by tempfile.mkdtemp() needs to be romved manually
os.rmdir(tgt_gisdbase)
# we have to remove file created by tempfile.mkstemp function
# in write_gisrc function
os.remove(tgt_gisrc)
def __init__(
self,
cmd,
width=None,
height=None,
overlap=0,
processes=None,
split=False,
debug=False,
region=None,
move=None,
log=False,
start_row=0,
start_col=0,
out_prefix="",
mapset_prefix=None,
*args,
**kargs,
):
kargs["run_"] = False
self.mset = Mapset()
self.module = Module(cmd, *args, **kargs)
self.width = width
self.height = height
self.overlap = overlap
self.processes = processes
self.region = region if region else Region()
self.start_row = start_row
self.start_col = start_col
self.out_prefix = out_prefix
self.log = log
self.move = move
self.gisrc_src = os.environ["GISRC"]
self.n_mset, self.gisrc_dst = None, None
if self.move:
self.n_mset = copy_mapset(self.mset, self.move)
self.gisrc_dst = write_gisrc(self.n_mset.gisdbase,
self.n_mset.location,
self.n_mset.name)
rasters = [r for r in select(self.module.inputs, "raster")]
if rasters:
copy_rasters(rasters,
self.gisrc_src,
self.gisrc_dst,
region=self.region)
vectors = [v for v in select(self.module.inputs, "vector")]
if vectors:
copy_vectors(vectors, self.gisrc_src, self.gisrc_dst)
groups = [g for g in select(self.module.inputs, "group")]
if groups:
copy_groups(groups,
self.gisrc_src,
self.gisrc_dst,
region=self.region)
self.bboxes = split_region_tiles(region=region,
width=width,
height=height,
overlap=overlap)
if mapset_prefix:
self.mapset_prefix = mapset_prefix
else:
self.mapset_prefix = append_node_pid("grid_" +
legalize_vector_name(cmd))
self.msetstr = self.mapset_prefix + "_%03d_%03d"
self.inlist = None
if split:
self.split()
self.debug = debug
def _run_test_module(self, module, results_dir, gisdbase, location):
"""Run one test file."""
self.testsuite_dirs[module.tested_dir].append(module.name)
cwd = os.path.join(results_dir, module.tested_dir, module.name)
data_dir = os.path.join(module.file_dir, 'data')
if os.path.exists(data_dir):
# TODO: link dir instead of copy tree and remove link afterwads
# (removing is good because of testsuite dir in samplecode)
# TODO: use different dir name in samplecode and test if it works
shutil.copytree(data_dir, os.path.join(cwd, 'data'),
ignore=shutil.ignore_patterns('*.svn*'))
ensure_dir(os.path.abspath(cwd))
# TODO: put this to constructor and copy here again
env = os.environ.copy()
mapset, mapset_dir = self._create_mapset(gisdbase, location, module)
gisrc = gsetup.write_gisrc(gisdbase, location, mapset)
env['GISRC'] = gisrc
stdout_path = os.path.join(cwd, 'stdout.txt')
stderr_path = os.path.join(cwd, 'stderr.txt')
stdout = open(stdout_path, 'w')
stderr = open(stderr_path, 'w')
self.reporter.start_file_test(module)
# TODO: we might clean the directory here before test if non-empty
if module.file_type == 'py':
# ignoring shebang line to use current Python
# and also pass parameters to it
# add also '-Qwarn'?
p = subprocess.Popen([sys.executable, '-tt', '-3',
module.abs_file_path],
cwd=cwd, env=env,
stdout=stdout, stderr=stderr)
elif module.file_type == 'sh':
# ignoring shebang line to pass parameters to shell
# expecting system to have sh or something compatible
# TODO: add some special checks for MS Windows
# using -x to see commands in stderr
# using -e to terminate fast
# from dash manual:
# -e errexit If not interactive, exit immediately if any
# untested command fails. The exit status of a com‐
# mand is considered to be explicitly tested if the
# command is used to control an if, elif, while, or
# until; or if the command is the left hand operand
# of an '&&' or '||' operator.
p = subprocess.Popen(['sh', '-e', '-x', module.abs_file_path],
cwd=cwd, env=env,
stdout=stdout, stderr=stderr)
else:
p = subprocess.Popen([module.abs_file_path],
cwd=cwd, env=env,
stdout=stdout, stderr=stderr)
returncode = p.wait()
stdout.close()
stderr.close()
self._file_anonymizer.anonymize([stdout_path, stderr_path])
test_summary = update_keyval_file(
os.path.join(cwd, 'test_keyvalue_result.txt'),
module=module, returncode=returncode)
self.reporter.end_file_test(module=module, cwd=cwd,
returncode=returncode,
stdout=stdout_path, stderr=stderr_path,
test_summary=test_summary)
# TODO: add some try-except or with for better error handling
os.remove(gisrc)
# TODO: only if clean up
if self.clean_mapsets:
shutil.rmtree(mapset_dir)
def _run_test_module(self, module, results_dir, gisdbase, location):
"""Run one test file."""
self.testsuite_dirs[module.tested_dir].append(module.name)
cwd = os.path.join(results_dir, module.tested_dir, module.name)
data_dir = os.path.join(module.file_dir, 'data')
if os.path.exists(data_dir):
# TODO: link dir instead of copy tree and remove link afterwads
# (removing is good because of testsuite dir in samplecode)
# TODO: use different dir name in samplecode and test if it works
shutil.copytree(data_dir, os.path.join(cwd, 'data'),
ignore=shutil.ignore_patterns('*.svn*'))
ensure_dir(os.path.abspath(cwd))
# TODO: put this to constructor and copy here again
env = os.environ.copy()
mapset, mapset_dir = self._create_mapset(gisdbase, location, module)
gisrc = gsetup.write_gisrc(gisdbase, location, mapset)
# here is special setting of environmental variables for running tests
# some of them might be set from outside in the future and if the list
# will be long they should be stored somewhere separately
# use custom gisrc, not current session gisrc
env['GISRC'] = gisrc
# percentage in plain format is 0...10...20... ...100
env['GRASS_MESSAGE_FORMAT'] = 'plain'
stdout_path = os.path.join(cwd, 'stdout.txt')
stderr_path = os.path.join(cwd, 'stderr.txt')
self.reporter.start_file_test(module)
# TODO: we might clean the directory here before test if non-empty
if module.file_type == 'py':
# ignoring shebang line to use current Python
# and also pass parameters to it
# add also '-Qwarn'?
if sys.version_info.major >= 3:
args = [sys.executable, '-tt', module.abs_file_path]
else:
args = [sys.executable, '-tt', '-3', module.abs_file_path]
p = subprocess.Popen(args, cwd=cwd, env=env,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
elif module.file_type == 'sh':
# ignoring shebang line to pass parameters to shell
# expecting system to have sh or something compatible
# TODO: add some special checks for MS Windows
# using -x to see commands in stderr
# using -e to terminate fast
# from dash manual:
# -e errexit If not interactive, exit immediately if any
# untested command fails. The exit status of a com‐
# mand is considered to be explicitly tested if the
# command is used to control an if, elif, while, or
# until; or if the command is the left hand operand
# of an '&&' or '||' operator.
p = subprocess.Popen(['sh', '-e', '-x', module.abs_file_path],
cwd=cwd, env=env,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
else:
p = subprocess.Popen([module.abs_file_path],
cwd=cwd, env=env,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
stdout, stderr = p.communicate()
returncode = p.returncode
encodings = [_get_encoding(), 'utf8', 'latin-1', 'ascii']
detected = False
idx = 0
while not detected:
try:
stdout = decode(stdout, encoding=encodings[idx])
detected = True
except:
idx += 1
pass
detected = False
idx = 0
while not detected:
try:
stderr = decode(stderr, encoding=encodings[idx])
detected = True
except:
idx += 1
pass
with open(stdout_path, 'w') as stdout_file:
stdout_file.write(stdout)
with open(stderr_path, 'w') as stderr_file:
if type(stderr) == 'bytes':
stderr_file.write(decode(stderr))
else:
if isinstance(stderr, str):
stderr_file.write(stderr)
else:
stderr_file.write(stderr.encode('utf8'))
self._file_anonymizer.anonymize([stdout_path, stderr_path])
test_summary = update_keyval_file(
os.path.join(os.path.abspath(cwd), 'test_keyvalue_result.txt'),
module=module, returncode=returncode)
self.reporter.end_file_test(module=module, cwd=cwd,
returncode=returncode,
stdout=stdout_path, stderr=stderr_path,
test_summary=test_summary)
# TODO: add some try-except or with for better error handling
os.remove(gisrc)
# TODO: only if clean up
if self.clean_mapsets:
shutil.rmtree(mapset_dir)
def __init__(
self,
cmd,
width=None,
height=None,
overlap=0,
processes=None,
split=False,
debug=False,
region=None,
move=None,
log=False,
start_row=0,
start_col=0,
out_prefix="",
mapset_prefix=None,
patch_backend=None,
*args,
**kargs,
):
kargs["run_"] = False
self.mset = Mapset()
self.module = Module(cmd, *args, **kargs)
self.width = width
self.height = height
self.overlap = overlap
self.processes = processes
self.region = region if region else Region()
self.start_row = start_row
self.start_col = start_col
self.out_prefix = out_prefix
self.log = log
self.move = move
# by default RasterRow is used as previously
# if overlap > 0, r.patch won't work properly
if not patch_backend:
self.patch_backend = "RasterRow"
elif patch_backend not in ("r.patch", "RasterRow"):
raise RuntimeError(
_("Parameter patch_backend must be 'r.patch' or 'RasterRow'"))
elif patch_backend == "r.patch" and self.overlap:
raise RuntimeError(
_("Patching backend 'r.patch' doesn't work for overlap > 0"))
else:
self.patch_backend = patch_backend
self.gisrc_src = os.environ["GISRC"]
self.n_mset, self.gisrc_dst = None, None
self.estimate_tile_size()
if self.move:
self.n_mset = copy_mapset(self.mset, self.move)
self.gisrc_dst = write_gisrc(self.n_mset.gisdbase,
self.n_mset.location,
self.n_mset.name)
rasters = [r for r in select(self.module.inputs, "raster")]
if rasters:
copy_rasters(rasters,
self.gisrc_src,
self.gisrc_dst,
region=self.region)
vectors = [v for v in select(self.module.inputs, "vector")]
if vectors:
copy_vectors(vectors, self.gisrc_src, self.gisrc_dst)
groups = [g for g in select(self.module.inputs, "group")]
if groups:
copy_groups(groups,
self.gisrc_src,
self.gisrc_dst,
region=self.region)
self.bboxes = split_region_tiles(region=region,
width=self.width,
height=self.height,
overlap=overlap)
if mapset_prefix:
self.mapset_prefix = mapset_prefix
else:
self.mapset_prefix = append_node_pid("grid_" +
legalize_vector_name(cmd))
self.msetstr = self.mapset_prefix + "_%03d_%03d"
self.inlist = None
if split:
self.split()
self.debug = debug