def get(uri, cache_dir=None):
"""Entry method for download method
:param str uri: A formatted remote URL of a file
:param str: Absolute path to the downloaded content
:param str cache_dir: path to save downloaded files
"""
user_base_dir = cache_dir or CONF.remote.cache_dir
if user_base_dir:
try:
temp = tempfile.TemporaryFile(dir=os.path.abspath(user_base_dir))
temp.close()
except OSError:
LOG.error(_LE("Failed to write remote files to: %s") %
os.path.abspath(user_base_dir))
exit(1)
abs_path = download(uri, os.path.abspath(user_base_dir))
else:
abs_path = download(uri)
try:
return extract_tar(abs_path)
except (tarfile.TarError, Exception):
msg = _("Not a gz file, returning abs_path")
LOG.debug(msg)
return abs_path
def download(uri, cache_dir=None):
"""A simple file downloader.
A simple file downloader which returns the absolute
path to where the file has been saved. In case of tar
files the absolute patch excluding .tar extension is
passed.
:param str uri: The remote uri of the file
:param str cache_dir: The directory name/handle
:returns str: Absolute path to the downloaded file
"""
global temp_dirs
global remote_dirs
if not cache_dir:
cache_dir = tempfile.mkdtemp()
temp_dirs.append(cache_dir)
remote_dirs.append(cache_dir)
LOG.debug(_LI("Remote file location: %s") % remote_dirs)
resp, _ = SynHTTPClient().request("GET", uri)
os.chdir(cache_dir)
saved_umask = os.umask(0o77)
fname = uri.split("/")[-1]
try:
with open(fname, 'wb') as fh:
fh.write(resp.content)
return os.path.abspath(fname)
except IOError:
LOG.error(_LE("IOError in writing the downloaded file to disk."))
finally:
os.umask(saved_umask)
def download(uri, cache_dir=None):
"""A simple file downloader.
A simple file downloader which returns the absolute
path to where the file has been saved. In case of tar
files the absolute patch excluding .tar extension is
passed.
:param str uri: The remote uri of the file
:param str cache_dir: The directory name/handle
:returns str: Absolute path to the downloaded file
"""
global temp_dirs
global remote_dirs
if not cache_dir:
cache_dir = tempfile.mkdtemp()
temp_dirs.append(cache_dir)
remote_dirs.append(cache_dir)
LOG.debug(_LI("Remote file location: %s") % remote_dirs)
resp, _ = SynHTTPClient().request("GET", uri)
os.chdir(cache_dir)
saved_umask = os.umask(0o77)
fname = uri.split("/")[-1]
try:
with open(fname, 'wb') as fh:
fh.write(resp.content)
return os.path.abspath(fname)
except IOError:
LOG.error(_LE("IOError in writing the downloaded file to disk."))
finally:
os.umask(saved_umask)
def get(uri, cache_dir=None):
"""Entry method for download method
:param str uri: A formatted remote URL of a file
:param str: Absolute path to the downloaded content
:param str cache_dir: path to save downloaded files
"""
user_base_dir = cache_dir or CONF.remote.cache_dir
if user_base_dir:
try:
temp = tempfile.TemporaryFile(dir=os.path.abspath(user_base_dir))
temp.close()
except OSError:
LOG.error(
_LE("Failed to write remote files to: %s") %
os.path.abspath(user_base_dir))
exit(1)
abs_path = download(uri, os.path.abspath(user_base_dir))
else:
abs_path = download(uri)
try:
return extract_tar(abs_path)
except (tarfile.TarError, Exception):
msg = _("Not a gz file, returning abs_path")
LOG.debug(msg)
return abs_path
def extract_tar(abs_path):
"""Extract a gzipped tar file from the given absolute_path
:param str abs_path: The absolute path to the tar file
:returns str untar_dir: The absolute path to untarred file
"""
work_dir, tar_file = os.path.split(abs_path)
os.chdir(work_dir)
try:
os.mkdir("remote")
except OSError:
LOG.error(_LE(
"path exists already, not creating remote directory."))
remote_path = os.path.abspath("remote")
def safe_paths(tar_meta):
"""Makes sure all tar file paths are relative to the base path
Orignal from https://stackoverflow.com/questions/
10060069/safely-extract-zip-or-tar-using-python
:param tarfile.TarFile tar_meta: TarFile object
:returns tarfile:TarFile fh: TarFile object
"""
for fh in tar_meta:
each_f = os.path.abspath(os.path.join(work_dir, fh.name))
if os.path.realpath(each_f).startswith(work_dir):
yield fh
with tarfile.open(tar_file, mode="r:gz") as tarf:
tarf.extractall(path=remote_path, members=safe_paths(tarf))
os.remove(abs_path)
return remote_path
def get_user_home_root():
global FOLDER
user = os.environ.get("SUDO_USER")
if not user:
try:
user = os.environ.get("USER") or os.getlogin()
except OSError as e:
# Refer https://mail.python.org/pipermail/python-bugs-list/
# 2002-July/012691.html
LOG.error(_LE("Exception thrown in : %s") % e)
user = pwd.getpwuid(os.getuid())[0]
home_path = "~{0}/{1}".format(user, FOLDER)
return expand_path(home_path)
def get_user_home_root():
global FOLDER
user = os.environ.get("SUDO_USER")
if not user:
try:
user = os.environ.get("USER") or os.getlogin()
except OSError as e:
# Refer https://mail.python.org/pipermail/python-bugs-list/
# 2002-July/012691.html
LOG.error(_LE("Exception thrown in : %s") % e)
user = pwd.getpwuid(os.getuid())[0]
home_path = "~{0}/{1}".format(user, FOLDER)
return expand_path(home_path)
def extract_tar(abs_path):
"""Extract a gzipped tar file from the given absolute_path
:param str abs_path: The absolute path to the tar file
:returns str untar_dir: The absolute path to untarred file
"""
work_dir, tar_file = os.path.split(abs_path)
os.chdir(work_dir)
try:
os.mkdir("remote")
except OSError:
LOG.error(_LE(
"path exists already, not creating remote directory."))
remote_path = os.path.abspath("remote")
def safe_paths(tar_meta):
"""Makes sure all tar file paths are relative to the base path
Orignal from https://stackoverflow.com/questions/
10060069/safely-extract-zip-or-tar-using-python
:param tarfile.TarFile tar_meta: TarFile object
:returns tarfile:TarFile fh: TarFile object
"""
for fh in tar_meta:
each_f = os.path.abspath(os.path.join(work_dir, fh.name))
if os.path.realpath(each_f).startswith(work_dir):
yield fh
try:
with tarfile.open(tar_file, mode="r:gz") as tarf:
tarf.extractall(path=remote_path, members=safe_paths(tarf))
except tarfile.ExtractError as e:
LOG.error(_LE("Unable to extract the file: %s") % e)
raise
os.remove(abs_path)
return remote_path
def dry_run(cls,
list_of_tests,
file_path,
req_str,
output,
meta_vars=None):
"""Runs debug test to check all steps leading up to executing a test
This method does not run any checks, but does parse the template files
and config options. It then runs a debug test which sends no requests
of its own.
Note: if any external calls referenced inside the template file do make
requests, the parser will still make those requests even for a dry run
:param str file_path: Path of the template file
:param str req_str: Request string of each template
:return: None
"""
for k, test_class in list_of_tests: # noqa
try:
print("\nParsing template file...\n")
test_class.create_init_request(file_path, req_str, meta_vars)
except Exception as e:
print("\nError in parsing template:\n \t{0}\n".format(
traceback.format_exc()))
LOG.error(_LE("Error in parsing template:"))
output["failures"].append({
"file": file_path,
"error": e.__str__()
})
else:
print(_("\nRequest sucessfully generated!\n"))
output["successes"].append(file_path)
test_cases = list(test_class.get_test_cases(file_path, req_str))
if len(test_cases) > 0:
for test in test_cases:
if test:
cls.run_test(test, result)
def dry_run(cls, list_of_tests, file_path, req_str, output,
meta_vars=None):
"""Runs debug test to check all steps leading up to executing a test
This method does not run any checks, but does parse the template files
and config options. It then runs a debug test which sends no requests
of its own.
Note: if any external calls referenced inside the template file do make
requests, the parser will still make those requests even for a dry run
:param str file_path: Path of the template file
:param str req_str: Request string of each template
:return: None
"""
for k, test_class in list_of_tests: # noqa
try:
print("\nParsing template file...\n")
test_class.create_init_request(file_path, req_str, meta_vars)
except Exception as e:
print("\nError in parsing template:\n \t{0}\n".format(
traceback.format_exc()))
LOG.error(_LE("Error in parsing template:"))
output["failures"].append({
"file": file_path,
"error": e.__str__()
})
else:
print(_("\nRequest sucessfully generated!\n"))
output["successes"].append(file_path)
test_cases = list(test_class.get_test_cases(file_path, req_str))
if len(test_cases) > 0:
for test in test_cases:
if test:
cls.run_test(test, result)
def run_given_tests(cls,
list_of_tests,
file_path,
req_str,
meta_vars=None):
"""Loads all the templates and runs all the given tests
This method calls run_test method to run each of the tests one
by one.
:param list list_of_tests: A list of all the loaded tests
:param str file_path: Path of the template file
:param str req_str: Request string of each template
:return: None
"""
try:
template_start_time = time.time()
failures = 0
errors = 0
print("\n ID \t\tTest Name \t\t\t\t\t\t Progress")
for test_name, test_class in list_of_tests:
test_class.test_id = cls.current_test_id
cls.current_test_id += 5
log_string = "[{test_id}] : {name}".format(
test_id=test_class.test_id, name=test_name)
result_string = "[{test_id}] : {name}".format(
test_id=cli.colorize(test_class.test_id, color="green"),
name=test_name.replace("_", " ").capitalize())
if not CONF.colorize:
result_string = result_string.ljust(55)
else:
result_string = result_string.ljust(60)
LOG.debug(log_string)
try:
test_class.send_init_request(file_path, req_str, meta_vars)
except Exception:
print(
_("Error in parsing template:\n %s\n") %
traceback.format_exc())
LOG.error(_LE("Error in parsing template:"))
break
test_cases = list(test_class.get_test_cases(
file_path, req_str))
if len(test_cases) > 0:
p_bar = cli.ProgressBar(message=result_string,
total_len=len(test_cases))
last_failures = result.stats["failures"]
last_errors = result.stats["errors"]
for test in test_cases:
if test:
cls.run_test(test, result)
p_bar.increment(1)
p_bar.print_bar()
failures = result.stats["failures"] - last_failures
errors = result.stats["errors"] - last_errors
total_tests = len(test_cases)
if failures > total_tests * 0.90:
# More than 90 percent failure
failures = cli.colorize(failures, "red")
elif failures > total_tests * 0.45:
# More than 45 percent failure
failures = cli.colorize(failures, "yellow")
elif failures > total_tests * 0.15:
# More than 15 percent failure
failures = cli.colorize(failures, "blue")
if errors:
last_failures = result.stats["failures"]
last_errors = result.stats["errors"]
errors = cli.colorize(errors, "red")
print(
_(" : %(fail)s Failure(s), %(err) Error(s)\r") %
{
"fail": failures,
"err": errors
})
else:
last_failures = result.stats["failures"]
print(_(" : %s Failure(s), 0 Error(s)\r") % failures)
run_time = time.time() - template_start_time
LOG.info(_("Run time: %s sec."), run_time)
if hasattr(result, "testsRun"):
num_tests = result.testsRun - result.testsRunSinceLastPrint
print(
_("\nRan %(num)s test(s) in %.3(time)f s\n") % {
"num": num_tests,
"time": run_time
})
result.testsRunSinceLastPrint = result.testsRun
except KeyboardInterrupt:
print(_('\n\nPausing...Hit ENTER to continue, type quit to exit.'))
try:
response = input()
if response.lower() == "quit":
result.print_result(cls.start_time)
cleanup.delete_temps()
print(_("Exiting..."))
exit(0)
print(_('Resuming...'))
except KeyboardInterrupt:
result.print_result(cls.start_time)
cleanup.delete_temps()
print(_("Exiting..."))
exit(0)
def run_given_tests(cls, list_of_tests, file_path, req_str,
meta_vars=None):
"""Loads all the templates and runs all the given tests
This method calls run_test method to run each of the tests one
by one.
:param list list_of_tests: A list of all the loaded tests
:param str file_path: Path of the template file
:param str req_str: Request string of each template
:return: None
"""
try:
template_start_time = time.time()
failures = 0
errors = 0
print("\n ID \t\tTest Name \t\t\t\t\t\t Progress")
for test_name, test_class in list_of_tests:
test_class.test_id = cls.current_test_id
cls.current_test_id += 5
log_string = "[{test_id}] : {name}".format(
test_id=test_class.test_id, name=test_name)
result_string = "[{test_id}] : {name}".format(
test_id=cli.colorize(
test_class.test_id, color="green"),
name=test_name.replace("_", " ").capitalize())
if not CONF.colorize:
result_string = result_string.ljust(55)
else:
result_string = result_string.ljust(60)
LOG.debug(log_string)
try:
test_class.send_init_request(file_path, req_str, meta_vars)
except Exception:
print(_(
"Error in parsing template:\n %s\n"
) % traceback.format_exc())
LOG.error(_LE("Error in parsing template:"))
break
test_cases = list(
test_class.get_test_cases(file_path, req_str))
if len(test_cases) > 0:
p_bar = cli.ProgressBar(
message=result_string, total_len=len(test_cases))
last_failures = result.stats["failures"]
last_errors = result.stats["errors"]
for test in test_cases:
if test:
cls.run_test(test, result)
p_bar.increment(1)
p_bar.print_bar()
failures = result.stats["failures"] - last_failures
errors = result.stats["errors"] - last_errors
total_tests = len(test_cases)
if failures > total_tests * 0.90:
# More than 90 percent failure
failures = cli.colorize(failures, "red")
elif failures > total_tests * 0.45:
# More than 45 percent failure
failures = cli.colorize(failures, "yellow")
elif failures > total_tests * 0.15:
# More than 15 percent failure
failures = cli.colorize(failures, "blue")
if errors:
last_failures = result.stats["failures"]
last_errors = result.stats["errors"]
errors = cli.colorize(errors, "red")
print(_(
" : %(fail)s Failure(s), %(err)s Error(s)\r") % {
"fail": failures, "err": errors})
else:
last_failures = result.stats["failures"]
print(
_(
" : %s Failure(s), 0 Error(s)\r") % failures)
run_time = time.time() - template_start_time
LOG.info(_("Run time: %s sec."), run_time)
if hasattr(result, "testsRun"):
num_tests = result.testsRun - result.testsRunSinceLastPrint
print(_("\nRan %(num)s test(s) in %(time).3f s\n") %
{"num": num_tests, "time": run_time})
result.testsRunSinceLastPrint = result.testsRun
except KeyboardInterrupt:
print(_(
'\n\nPausing...Hit ENTER to continue, type quit to exit.'))
try:
response = input()
if response.lower() == "quit":
result.print_result(cls.start_time)
cleanup.delete_temps()
print(_("Exiting..."))
exit(0)
print(_('Resuming...'))
except KeyboardInterrupt:
result.print_result(cls.start_time)
cleanup.delete_temps()
print(_("Exiting..."))
exit(0)
