def get(uri, cache_dir=None): """Entry method for download method :param str uri: A formatted remote URL of a file :param str: Absolute path to the downloaded content :param str cache_dir: path to save downloaded files """ user_base_dir = cache_dir or CONF.remote.cache_dir if user_base_dir: try: temp = tempfile.TemporaryFile(dir=os.path.abspath(user_base_dir)) temp.close() except OSError: LOG.error(_LE("Failed to write remote files to: %s") % os.path.abspath(user_base_dir)) exit(1) abs_path = download(uri, os.path.abspath(user_base_dir)) else: abs_path = download(uri) try: return extract_tar(abs_path) except (tarfile.TarError, Exception): msg = _("Not a gz file, returning abs_path") LOG.debug(msg) return abs_path
def download(uri, cache_dir=None): """A simple file downloader. A simple file downloader which returns the absolute path to where the file has been saved. In case of tar files the absolute patch excluding .tar extension is passed. :param str uri: The remote uri of the file :param str cache_dir: The directory name/handle :returns str: Absolute path to the downloaded file """ global temp_dirs global remote_dirs if not cache_dir: cache_dir = tempfile.mkdtemp() temp_dirs.append(cache_dir) remote_dirs.append(cache_dir) LOG.debug(_LI("Remote file location: %s") % remote_dirs) resp, _ = SynHTTPClient().request("GET", uri) os.chdir(cache_dir) saved_umask = os.umask(0o77) fname = uri.split("/")[-1] try: with open(fname, 'wb') as fh: fh.write(resp.content) return os.path.abspath(fname) except IOError: LOG.error(_LE("IOError in writing the downloaded file to disk.")) finally: os.umask(saved_umask)
def download(uri, cache_dir=None): """A simple file downloader. A simple file downloader which returns the absolute path to where the file has been saved. In case of tar files the absolute patch excluding .tar extension is passed. :param str uri: The remote uri of the file :param str cache_dir: The directory name/handle :returns str: Absolute path to the downloaded file """ global temp_dirs global remote_dirs if not cache_dir: cache_dir = tempfile.mkdtemp() temp_dirs.append(cache_dir) remote_dirs.append(cache_dir) LOG.debug(_LI("Remote file location: %s") % remote_dirs) resp, _ = SynHTTPClient().request("GET", uri) os.chdir(cache_dir) saved_umask = os.umask(0o77) fname = uri.split("/")[-1] try: with open(fname, 'wb') as fh: fh.write(resp.content) return os.path.abspath(fname) except IOError: LOG.error(_LE("IOError in writing the downloaded file to disk.")) finally: os.umask(saved_umask)
def get(uri, cache_dir=None): """Entry method for download method :param str uri: A formatted remote URL of a file :param str: Absolute path to the downloaded content :param str cache_dir: path to save downloaded files """ user_base_dir = cache_dir or CONF.remote.cache_dir if user_base_dir: try: temp = tempfile.TemporaryFile(dir=os.path.abspath(user_base_dir)) temp.close() except OSError: LOG.error( _LE("Failed to write remote files to: %s") % os.path.abspath(user_base_dir)) exit(1) abs_path = download(uri, os.path.abspath(user_base_dir)) else: abs_path = download(uri) try: return extract_tar(abs_path) except (tarfile.TarError, Exception): msg = _("Not a gz file, returning abs_path") LOG.debug(msg) return abs_path
def extract_tar(abs_path): """Extract a gzipped tar file from the given absolute_path :param str abs_path: The absolute path to the tar file :returns str untar_dir: The absolute path to untarred file """ work_dir, tar_file = os.path.split(abs_path) os.chdir(work_dir) try: os.mkdir("remote") except OSError: LOG.error(_LE( "path exists already, not creating remote directory.")) remote_path = os.path.abspath("remote") def safe_paths(tar_meta): """Makes sure all tar file paths are relative to the base path Orignal from https://stackoverflow.com/questions/ 10060069/safely-extract-zip-or-tar-using-python :param tarfile.TarFile tar_meta: TarFile object :returns tarfile:TarFile fh: TarFile object """ for fh in tar_meta: each_f = os.path.abspath(os.path.join(work_dir, fh.name)) if os.path.realpath(each_f).startswith(work_dir): yield fh with tarfile.open(tar_file, mode="r:gz") as tarf: tarf.extractall(path=remote_path, members=safe_paths(tarf)) os.remove(abs_path) return remote_path
def get_user_home_root(): global FOLDER user = os.environ.get("SUDO_USER") if not user: try: user = os.environ.get("USER") or os.getlogin() except OSError as e: # Refer https://mail.python.org/pipermail/python-bugs-list/ # 2002-July/012691.html LOG.error(_LE("Exception thrown in : %s") % e) user = pwd.getpwuid(os.getuid())[0] home_path = "~{0}/{1}".format(user, FOLDER) return expand_path(home_path)
def get_user_home_root(): global FOLDER user = os.environ.get("SUDO_USER") if not user: try: user = os.environ.get("USER") or os.getlogin() except OSError as e: # Refer https://mail.python.org/pipermail/python-bugs-list/ # 2002-July/012691.html LOG.error(_LE("Exception thrown in : %s") % e) user = pwd.getpwuid(os.getuid())[0] home_path = "~{0}/{1}".format(user, FOLDER) return expand_path(home_path)
def extract_tar(abs_path): """Extract a gzipped tar file from the given absolute_path :param str abs_path: The absolute path to the tar file :returns str untar_dir: The absolute path to untarred file """ work_dir, tar_file = os.path.split(abs_path) os.chdir(work_dir) try: os.mkdir("remote") except OSError: LOG.error(_LE( "path exists already, not creating remote directory.")) remote_path = os.path.abspath("remote") def safe_paths(tar_meta): """Makes sure all tar file paths are relative to the base path Orignal from https://stackoverflow.com/questions/ 10060069/safely-extract-zip-or-tar-using-python :param tarfile.TarFile tar_meta: TarFile object :returns tarfile:TarFile fh: TarFile object """ for fh in tar_meta: each_f = os.path.abspath(os.path.join(work_dir, fh.name)) if os.path.realpath(each_f).startswith(work_dir): yield fh try: with tarfile.open(tar_file, mode="r:gz") as tarf: tarf.extractall(path=remote_path, members=safe_paths(tarf)) except tarfile.ExtractError as e: LOG.error(_LE("Unable to extract the file: %s") % e) raise os.remove(abs_path) return remote_path
def dry_run(cls, list_of_tests, file_path, req_str, output, meta_vars=None): """Runs debug test to check all steps leading up to executing a test This method does not run any checks, but does parse the template files and config options. It then runs a debug test which sends no requests of its own. Note: if any external calls referenced inside the template file do make requests, the parser will still make those requests even for a dry run :param str file_path: Path of the template file :param str req_str: Request string of each template :return: None """ for k, test_class in list_of_tests: # noqa try: print("\nParsing template file...\n") test_class.create_init_request(file_path, req_str, meta_vars) except Exception as e: print("\nError in parsing template:\n \t{0}\n".format( traceback.format_exc())) LOG.error(_LE("Error in parsing template:")) output["failures"].append({ "file": file_path, "error": e.__str__() }) else: print(_("\nRequest sucessfully generated!\n")) output["successes"].append(file_path) test_cases = list(test_class.get_test_cases(file_path, req_str)) if len(test_cases) > 0: for test in test_cases: if test: cls.run_test(test, result)
def dry_run(cls, list_of_tests, file_path, req_str, output, meta_vars=None): """Runs debug test to check all steps leading up to executing a test This method does not run any checks, but does parse the template files and config options. It then runs a debug test which sends no requests of its own. Note: if any external calls referenced inside the template file do make requests, the parser will still make those requests even for a dry run :param str file_path: Path of the template file :param str req_str: Request string of each template :return: None """ for k, test_class in list_of_tests: # noqa try: print("\nParsing template file...\n") test_class.create_init_request(file_path, req_str, meta_vars) except Exception as e: print("\nError in parsing template:\n \t{0}\n".format( traceback.format_exc())) LOG.error(_LE("Error in parsing template:")) output["failures"].append({ "file": file_path, "error": e.__str__() }) else: print(_("\nRequest sucessfully generated!\n")) output["successes"].append(file_path) test_cases = list(test_class.get_test_cases(file_path, req_str)) if len(test_cases) > 0: for test in test_cases: if test: cls.run_test(test, result)
def run_given_tests(cls, list_of_tests, file_path, req_str, meta_vars=None): """Loads all the templates and runs all the given tests This method calls run_test method to run each of the tests one by one. :param list list_of_tests: A list of all the loaded tests :param str file_path: Path of the template file :param str req_str: Request string of each template :return: None """ try: template_start_time = time.time() failures = 0 errors = 0 print("\n ID \t\tTest Name \t\t\t\t\t\t Progress") for test_name, test_class in list_of_tests: test_class.test_id = cls.current_test_id cls.current_test_id += 5 log_string = "[{test_id}] : {name}".format( test_id=test_class.test_id, name=test_name) result_string = "[{test_id}] : {name}".format( test_id=cli.colorize(test_class.test_id, color="green"), name=test_name.replace("_", " ").capitalize()) if not CONF.colorize: result_string = result_string.ljust(55) else: result_string = result_string.ljust(60) LOG.debug(log_string) try: test_class.send_init_request(file_path, req_str, meta_vars) except Exception: print( _("Error in parsing template:\n %s\n") % traceback.format_exc()) LOG.error(_LE("Error in parsing template:")) break test_cases = list(test_class.get_test_cases( file_path, req_str)) if len(test_cases) > 0: p_bar = cli.ProgressBar(message=result_string, total_len=len(test_cases)) last_failures = result.stats["failures"] last_errors = result.stats["errors"] for test in test_cases: if test: cls.run_test(test, result) p_bar.increment(1) p_bar.print_bar() failures = result.stats["failures"] - last_failures errors = result.stats["errors"] - last_errors total_tests = len(test_cases) if failures > total_tests * 0.90: # More than 90 percent failure failures = cli.colorize(failures, "red") elif failures > total_tests * 0.45: # More than 45 percent failure failures = cli.colorize(failures, "yellow") elif failures > total_tests * 0.15: # More than 15 percent failure failures = cli.colorize(failures, "blue") if errors: last_failures = result.stats["failures"] last_errors = result.stats["errors"] errors = cli.colorize(errors, "red") print( _(" : %(fail)s Failure(s), %(err) Error(s)\r") % { "fail": failures, "err": errors }) else: last_failures = result.stats["failures"] print(_(" : %s Failure(s), 0 Error(s)\r") % failures) run_time = time.time() - template_start_time LOG.info(_("Run time: %s sec."), run_time) if hasattr(result, "testsRun"): num_tests = result.testsRun - result.testsRunSinceLastPrint print( _("\nRan %(num)s test(s) in %.3(time)f s\n") % { "num": num_tests, "time": run_time }) result.testsRunSinceLastPrint = result.testsRun except KeyboardInterrupt: print(_('\n\nPausing...Hit ENTER to continue, type quit to exit.')) try: response = input() if response.lower() == "quit": result.print_result(cls.start_time) cleanup.delete_temps() print(_("Exiting...")) exit(0) print(_('Resuming...')) except KeyboardInterrupt: result.print_result(cls.start_time) cleanup.delete_temps() print(_("Exiting...")) exit(0)
def run_given_tests(cls, list_of_tests, file_path, req_str, meta_vars=None): """Loads all the templates and runs all the given tests This method calls run_test method to run each of the tests one by one. :param list list_of_tests: A list of all the loaded tests :param str file_path: Path of the template file :param str req_str: Request string of each template :return: None """ try: template_start_time = time.time() failures = 0 errors = 0 print("\n ID \t\tTest Name \t\t\t\t\t\t Progress") for test_name, test_class in list_of_tests: test_class.test_id = cls.current_test_id cls.current_test_id += 5 log_string = "[{test_id}] : {name}".format( test_id=test_class.test_id, name=test_name) result_string = "[{test_id}] : {name}".format( test_id=cli.colorize( test_class.test_id, color="green"), name=test_name.replace("_", " ").capitalize()) if not CONF.colorize: result_string = result_string.ljust(55) else: result_string = result_string.ljust(60) LOG.debug(log_string) try: test_class.send_init_request(file_path, req_str, meta_vars) except Exception: print(_( "Error in parsing template:\n %s\n" ) % traceback.format_exc()) LOG.error(_LE("Error in parsing template:")) break test_cases = list( test_class.get_test_cases(file_path, req_str)) if len(test_cases) > 0: p_bar = cli.ProgressBar( message=result_string, total_len=len(test_cases)) last_failures = result.stats["failures"] last_errors = result.stats["errors"] for test in test_cases: if test: cls.run_test(test, result) p_bar.increment(1) p_bar.print_bar() failures = result.stats["failures"] - last_failures errors = result.stats["errors"] - last_errors total_tests = len(test_cases) if failures > total_tests * 0.90: # More than 90 percent failure failures = cli.colorize(failures, "red") elif failures > total_tests * 0.45: # More than 45 percent failure failures = cli.colorize(failures, "yellow") elif failures > total_tests * 0.15: # More than 15 percent failure failures = cli.colorize(failures, "blue") if errors: last_failures = result.stats["failures"] last_errors = result.stats["errors"] errors = cli.colorize(errors, "red") print(_( " : %(fail)s Failure(s), %(err)s Error(s)\r") % { "fail": failures, "err": errors}) else: last_failures = result.stats["failures"] print( _( " : %s Failure(s), 0 Error(s)\r") % failures) run_time = time.time() - template_start_time LOG.info(_("Run time: %s sec."), run_time) if hasattr(result, "testsRun"): num_tests = result.testsRun - result.testsRunSinceLastPrint print(_("\nRan %(num)s test(s) in %(time).3f s\n") % {"num": num_tests, "time": run_time}) result.testsRunSinceLastPrint = result.testsRun except KeyboardInterrupt: print(_( '\n\nPausing...Hit ENTER to continue, type quit to exit.')) try: response = input() if response.lower() == "quit": result.print_result(cls.start_time) cleanup.delete_temps() print(_("Exiting...")) exit(0) print(_('Resuming...')) except KeyboardInterrupt: result.print_result(cls.start_time) cleanup.delete_temps() print(_("Exiting...")) exit(0)