def _run_test_module(self, module, results_dir, gisdbase, location):
"""Run one test file."""
self.testsuite_dirs[module.tested_dir].append(module.name)
cwd = os.path.join(results_dir, module.tested_dir, module.name)
data_dir = os.path.join(module.file_dir, 'data')
if os.path.exists(data_dir):
# TODO: link dir instead of copy tree and remove link afterwads
# (removing is good because of testsuite dir in samplecode)
# TODO: use different dir name in samplecode and test if it works
shutil.copytree(data_dir, os.path.join(cwd, 'data'),
ignore=shutil.ignore_patterns('*.svn*'))
ensure_dir(os.path.abspath(cwd))
# TODO: put this to constructor and copy here again
env = os.environ.copy()
mapset, mapset_dir = self._create_mapset(gisdbase, location, module)
gisrc = gsetup.write_gisrc(gisdbase, location, mapset)
# here is special setting of environmental variables for running tests
# some of them might be set from outside in the future and if the list
# will be long they should be stored somewhere separately
# use custom gisrc, not current session gisrc
env['GISRC'] = gisrc
# percentage in plain format is 0...10...20... ...100
env['GRASS_MESSAGE_FORMAT'] = 'plain'
stdout_path = os.path.join(cwd, 'stdout.txt')
stderr_path = os.path.join(cwd, 'stderr.txt')
self.reporter.start_file_test(module)
# TODO: we might clean the directory here before test if non-empty
if module.file_type == 'py':
# ignoring shebang line to use current Python
# and also pass parameters to it
# add also '-Qwarn'?
if sys.version_info.major >= 3:
args = [sys.executable, '-tt', module.abs_file_path]
else:
args = [sys.executable, '-tt', '-3', module.abs_file_path]
p = subprocess.Popen(args, cwd=cwd, env=env,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
elif module.file_type == 'sh':
# ignoring shebang line to pass parameters to shell
# expecting system to have sh or something compatible
# TODO: add some special checks for MS Windows
# using -x to see commands in stderr
# using -e to terminate fast
# from dash manual:
# -e errexit If not interactive, exit immediately if any
# untested command fails. The exit status of a com‐
# mand is considered to be explicitly tested if the
# command is used to control an if, elif, while, or
# until; or if the command is the left hand operand
# of an '&&' or '||' operator.
p = subprocess.Popen(['sh', '-e', '-x', module.abs_file_path],
cwd=cwd, env=env,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
else:
p = subprocess.Popen([module.abs_file_path],
cwd=cwd, env=env,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
stdout, stderr = p.communicate()
returncode = p.returncode
encodings = [_get_encoding(), 'utf8', 'latin-1', 'ascii']
detected = False
idx = 0
while not detected:
try:
stdout = decode(stdout, encoding=encodings[idx])
detected = True
except:
idx += 1
pass
detected = False
idx = 0
while not detected:
try:
stderr = decode(stderr, encoding=encodings[idx])
detected = True
except:
idx += 1
pass
with open(stdout_path, 'w') as stdout_file:
stdout_file.write(stdout)
with open(stderr_path, 'w') as stderr_file:
if type(stderr) == 'bytes':
stderr_file.write(decode(stderr))
else:
if isinstance(stderr, str):
stderr_file.write(stderr)
else:
stderr_file.write(stderr.encode('utf8'))
self._file_anonymizer.anonymize([stdout_path, stderr_path])
test_summary = update_keyval_file(
os.path.join(os.path.abspath(cwd), 'test_keyvalue_result.txt'),
module=module, returncode=returncode)
self.reporter.end_file_test(module=module, cwd=cwd,
returncode=returncode,
stdout=stdout_path, stderr=stderr_path,
test_summary=test_summary)
# TODO: add some try-except or with for better error handling
os.remove(gisrc)
# TODO: only if clean up
if self.clean_mapsets:
shutil.rmtree(mapset_dir)
def _run_test_module(self, module, results_dir, gisdbase, location):
"""Run one test file."""
self.testsuite_dirs[module.tested_dir].append(module.name)
cwd = os.path.join(results_dir, module.tested_dir, module.name)
data_dir = os.path.join(module.file_dir, "data")
if os.path.exists(data_dir):
# TODO: link dir instead of copy tree and remove link afterwads
# (removing is good because of testsuite dir in samplecode)
# TODO: use different dir name in samplecode and test if it works
shutil.copytree(
data_dir,
os.path.join(cwd, "data"),
ignore=shutil.ignore_patterns("*.svn*"),
)
ensure_dir(os.path.abspath(cwd))
# TODO: put this to constructor and copy here again
env = os.environ.copy()
mapset, mapset_dir = self._create_mapset(gisdbase, location, module)
gisrc = gsetup.write_gisrc(gisdbase, location, mapset)
# here is special setting of environmental variables for running tests
# some of them might be set from outside in the future and if the list
# will be long they should be stored somewhere separately
# use custom gisrc, not current session gisrc
env["GISRC"] = gisrc
# percentage in plain format is 0...10...20... ...100
env["GRASS_MESSAGE_FORMAT"] = "plain"
stdout_path = os.path.join(cwd, "stdout.txt")
stderr_path = os.path.join(cwd, "stderr.txt")
self.reporter.start_file_test(module)
# TODO: we might clean the directory here before test if non-empty
if module.file_type == "py":
# ignoring shebang line to use current Python
# and also pass parameters to it
# add also '-Qwarn'?
if sys.version_info.major >= 3:
args = [sys.executable, "-tt", module.abs_file_path]
else:
args = [sys.executable, "-tt", "-3", module.abs_file_path]
p = subprocess.Popen(args,
cwd=cwd,
env=env,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
elif module.file_type == "sh":
# ignoring shebang line to pass parameters to shell
# expecting system to have sh or something compatible
# TODO: add some special checks for MS Windows
# using -x to see commands in stderr
# using -e to terminate fast
# from dash manual:
# -e errexit If not interactive, exit immediately if any
# untested command fails. The exit status of a com‐
# mand is considered to be explicitly tested if the
# command is used to control an if, elif, while, or
# until; or if the command is the left hand operand
# of an '&&' or '||' operator.
p = subprocess.Popen(
["sh", "-e", "-x", module.abs_file_path],
cwd=cwd,
env=env,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
else:
p = subprocess.Popen(
[module.abs_file_path],
cwd=cwd,
env=env,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
stdout, stderr = p.communicate()
returncode = p.returncode
encodings = [_get_encoding(), "utf8", "latin-1", "ascii"]
def try_decode(data, encodings):
"""Try to decode data (bytes) using one of encodings
Falls back to decoding as UTF-8 with replacement for bytes.
Strings are returned unmodified.
"""
for encoding in encodings:
try:
return decode(data, encoding=encoding)
except UnicodeError:
pass
if isinstance(data, bytes):
return data.decode(encoding="utf-8", errors="replace")
return data
stdout = try_decode(stdout, encodings=encodings)
stderr = try_decode(stderr, encodings=encodings)
with open(stdout_path, "w") as stdout_file:
stdout_file.write(stdout)
with open(stderr_path, "w") as stderr_file:
if type(stderr) == "bytes":
stderr_file.write(decode(stderr))
else:
if isinstance(stderr, str):
stderr_file.write(stderr)
else:
stderr_file.write(stderr.encode("utf8"))
self._file_anonymizer.anonymize([stdout_path, stderr_path])
test_summary = update_keyval_file(
os.path.join(os.path.abspath(cwd), "test_keyvalue_result.txt"),
module=module,
returncode=returncode,
)
self.reporter.end_file_test(
module=module,
cwd=cwd,
returncode=returncode,
stdout=stdout_path,
stderr=stderr_path,
test_summary=test_summary,
)
# TODO: add some try-except or with for better error handling
os.remove(gisrc)
# TODO: only if clean up
if self.clean_mapsets:
shutil.rmtree(mapset_dir)
def txt2numpy(
tablestring,
sep=",",
names=None,
null_value=None,
fill_value=None,
comments="#",
usecols=None,
encoding=None,
structured=True,
):
"""
Can be removed when the function is included in grass core.
Read table-like output from grass modules as Numpy array;
format instructions are handed down to Numpys genfromtxt function
:param stdout: tabular stdout from GRASS GIS module call
:type stdout: str|byte
:param sep: Separator delimiting columns
:type sep: str
:param names: List of strings with names for columns
:type names: list
:param null_value: Characters representing the no-data value
:type null_value: str
:param fill_value: Value to fill no-data with
:type fill_value: str
:param comments: Character that identifies comments in the input string
:type comments: str
:param usecols: List of columns to import
:type usecols: list
:param structured: return structured array if True, un-structured otherwise
:type structured: bool
:return: numpy.ndarray
"""
from io import BytesIO
if not encoding:
encoding = grassutils._get_encoding()
if type(tablestring).__name__ == "str":
tablestring = grass.encode(tablestring, encoding=encoding)
elif type(tablestring).__name__ != "bytes":
grass.fatal("Unsupported data type")
kwargs = {
"missing_values": null_value,
"filling_values": fill_value,
"usecols": usecols,
"names": names,
"delimiter": sep,
"comments": comments,
}
if np.version.version >= "1.14":
kwargs["encoding"] = encoding
if structured:
kwargs["dtype"] = None
np_array = np.genfromtxt(BytesIO(tablestring), **kwargs)
return np_array
