def _run_test_module(self, module, results_dir, gisdbase, location): """Run one test file.""" self.testsuite_dirs[module.tested_dir].append(module.name) cwd = os.path.join(results_dir, module.tested_dir, module.name) data_dir = os.path.join(module.file_dir, 'data') if os.path.exists(data_dir): # TODO: link dir instead of copy tree and remove link afterwads # (removing is good because of testsuite dir in samplecode) # TODO: use different dir name in samplecode and test if it works shutil.copytree(data_dir, os.path.join(cwd, 'data'), ignore=shutil.ignore_patterns('*.svn*')) ensure_dir(os.path.abspath(cwd)) # TODO: put this to constructor and copy here again env = os.environ.copy() mapset, mapset_dir = self._create_mapset(gisdbase, location, module) gisrc = gsetup.write_gisrc(gisdbase, location, mapset) # here is special setting of environmental variables for running tests # some of them might be set from outside in the future and if the list # will be long they should be stored somewhere separately # use custom gisrc, not current session gisrc env['GISRC'] = gisrc # percentage in plain format is 0...10...20... ...100 env['GRASS_MESSAGE_FORMAT'] = 'plain' stdout_path = os.path.join(cwd, 'stdout.txt') stderr_path = os.path.join(cwd, 'stderr.txt') self.reporter.start_file_test(module) # TODO: we might clean the directory here before test if non-empty if module.file_type == 'py': # ignoring shebang line to use current Python # and also pass parameters to it # add also '-Qwarn'? if sys.version_info.major >= 3: args = [sys.executable, '-tt', module.abs_file_path] else: args = [sys.executable, '-tt', '-3', module.abs_file_path] p = subprocess.Popen(args, cwd=cwd, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE) elif module.file_type == 'sh': # ignoring shebang line to pass parameters to shell # expecting system to have sh or something compatible # TODO: add some special checks for MS Windows # using -x to see commands in stderr # using -e to terminate fast # from dash manual: # -e errexit If not interactive, exit immediately if any # untested command fails. The exit status of a com‐ # mand is considered to be explicitly tested if the # command is used to control an if, elif, while, or # until; or if the command is the left hand operand # of an '&&' or '||' operator. p = subprocess.Popen(['sh', '-e', '-x', module.abs_file_path], cwd=cwd, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE) else: p = subprocess.Popen([module.abs_file_path], cwd=cwd, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, stderr = p.communicate() returncode = p.returncode encodings = [_get_encoding(), 'utf8', 'latin-1', 'ascii'] detected = False idx = 0 while not detected: try: stdout = decode(stdout, encoding=encodings[idx]) detected = True except: idx += 1 pass detected = False idx = 0 while not detected: try: stderr = decode(stderr, encoding=encodings[idx]) detected = True except: idx += 1 pass with open(stdout_path, 'w') as stdout_file: stdout_file.write(stdout) with open(stderr_path, 'w') as stderr_file: if type(stderr) == 'bytes': stderr_file.write(decode(stderr)) else: if isinstance(stderr, str): stderr_file.write(stderr) else: stderr_file.write(stderr.encode('utf8')) self._file_anonymizer.anonymize([stdout_path, stderr_path]) test_summary = update_keyval_file( os.path.join(os.path.abspath(cwd), 'test_keyvalue_result.txt'), module=module, returncode=returncode) self.reporter.end_file_test(module=module, cwd=cwd, returncode=returncode, stdout=stdout_path, stderr=stderr_path, test_summary=test_summary) # TODO: add some try-except or with for better error handling os.remove(gisrc) # TODO: only if clean up if self.clean_mapsets: shutil.rmtree(mapset_dir)
def _run_test_module(self, module, results_dir, gisdbase, location): """Run one test file.""" self.testsuite_dirs[module.tested_dir].append(module.name) cwd = os.path.join(results_dir, module.tested_dir, module.name) data_dir = os.path.join(module.file_dir, "data") if os.path.exists(data_dir): # TODO: link dir instead of copy tree and remove link afterwads # (removing is good because of testsuite dir in samplecode) # TODO: use different dir name in samplecode and test if it works shutil.copytree( data_dir, os.path.join(cwd, "data"), ignore=shutil.ignore_patterns("*.svn*"), ) ensure_dir(os.path.abspath(cwd)) # TODO: put this to constructor and copy here again env = os.environ.copy() mapset, mapset_dir = self._create_mapset(gisdbase, location, module) gisrc = gsetup.write_gisrc(gisdbase, location, mapset) # here is special setting of environmental variables for running tests # some of them might be set from outside in the future and if the list # will be long they should be stored somewhere separately # use custom gisrc, not current session gisrc env["GISRC"] = gisrc # percentage in plain format is 0...10...20... ...100 env["GRASS_MESSAGE_FORMAT"] = "plain" stdout_path = os.path.join(cwd, "stdout.txt") stderr_path = os.path.join(cwd, "stderr.txt") self.reporter.start_file_test(module) # TODO: we might clean the directory here before test if non-empty if module.file_type == "py": # ignoring shebang line to use current Python # and also pass parameters to it # add also '-Qwarn'? if sys.version_info.major >= 3: args = [sys.executable, "-tt", module.abs_file_path] else: args = [sys.executable, "-tt", "-3", module.abs_file_path] p = subprocess.Popen(args, cwd=cwd, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE) elif module.file_type == "sh": # ignoring shebang line to pass parameters to shell # expecting system to have sh or something compatible # TODO: add some special checks for MS Windows # using -x to see commands in stderr # using -e to terminate fast # from dash manual: # -e errexit If not interactive, exit immediately if any # untested command fails. The exit status of a com‐ # mand is considered to be explicitly tested if the # command is used to control an if, elif, while, or # until; or if the command is the left hand operand # of an '&&' or '||' operator. p = subprocess.Popen( ["sh", "-e", "-x", module.abs_file_path], cwd=cwd, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) else: p = subprocess.Popen( [module.abs_file_path], cwd=cwd, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) stdout, stderr = p.communicate() returncode = p.returncode encodings = [_get_encoding(), "utf8", "latin-1", "ascii"] def try_decode(data, encodings): """Try to decode data (bytes) using one of encodings Falls back to decoding as UTF-8 with replacement for bytes. Strings are returned unmodified. """ for encoding in encodings: try: return decode(data, encoding=encoding) except UnicodeError: pass if isinstance(data, bytes): return data.decode(encoding="utf-8", errors="replace") return data stdout = try_decode(stdout, encodings=encodings) stderr = try_decode(stderr, encodings=encodings) with open(stdout_path, "w") as stdout_file: stdout_file.write(stdout) with open(stderr_path, "w") as stderr_file: if type(stderr) == "bytes": stderr_file.write(decode(stderr)) else: if isinstance(stderr, str): stderr_file.write(stderr) else: stderr_file.write(stderr.encode("utf8")) self._file_anonymizer.anonymize([stdout_path, stderr_path]) test_summary = update_keyval_file( os.path.join(os.path.abspath(cwd), "test_keyvalue_result.txt"), module=module, returncode=returncode, ) self.reporter.end_file_test( module=module, cwd=cwd, returncode=returncode, stdout=stdout_path, stderr=stderr_path, test_summary=test_summary, ) # TODO: add some try-except or with for better error handling os.remove(gisrc) # TODO: only if clean up if self.clean_mapsets: shutil.rmtree(mapset_dir)
def txt2numpy( tablestring, sep=",", names=None, null_value=None, fill_value=None, comments="#", usecols=None, encoding=None, structured=True, ): """ Can be removed when the function is included in grass core. Read table-like output from grass modules as Numpy array; format instructions are handed down to Numpys genfromtxt function :param stdout: tabular stdout from GRASS GIS module call :type stdout: str|byte :param sep: Separator delimiting columns :type sep: str :param names: List of strings with names for columns :type names: list :param null_value: Characters representing the no-data value :type null_value: str :param fill_value: Value to fill no-data with :type fill_value: str :param comments: Character that identifies comments in the input string :type comments: str :param usecols: List of columns to import :type usecols: list :param structured: return structured array if True, un-structured otherwise :type structured: bool :return: numpy.ndarray """ from io import BytesIO if not encoding: encoding = grassutils._get_encoding() if type(tablestring).__name__ == "str": tablestring = grass.encode(tablestring, encoding=encoding) elif type(tablestring).__name__ != "bytes": grass.fatal("Unsupported data type") kwargs = { "missing_values": null_value, "filling_values": fill_value, "usecols": usecols, "names": names, "delimiter": sep, "comments": comments, } if np.version.version >= "1.14": kwargs["encoding"] = encoding if structured: kwargs["dtype"] = None np_array = np.genfromtxt(BytesIO(tablestring), **kwargs) return np_array