def create_test(project_name, test_name):
test_content = (
"\n"
"description = ''\n\n"
"tags = []\n\n"
"pages = []\n\n\n"
"def setup(data):\n"
" pass\n\n\n"
"def test(data):\n"
" pass\n\n\n"
"def teardown(data):\n"
" pass\n\n")
errors = []
project = Project(project_name)
if test_name in project.tests():
errors.append('A test with that name already exists')
else:
errors = validate_project_element_name(test_name)
if not errors:
project.create_packages_for_element(test_name, project.file_types.TEST)
path = Test(project_name, test_name).path
with open(path, 'w', encoding='utf-8') as f:
f.write(test_content)
print('Test {} created for project {}'.format(test_name, project_name))
return errors
def tests(self):
"""Return a list with all the test cases in the suite"""
# TODO should use glob
tests = []
module = self.get_module()
if not hasattr(module, 'tests'):
return tests
project = Project(self.project)
if '*' in module.tests:
tests = project.tests()
else:
for test in module.tests:
if test.endswith('.*'):
this_dir = test[:-2]
path = os.path.join(project.test_directory_path,
os.sep.join(this_dir.split('.')))
this_dir_tests = file_manager.get_files_dot_path(
path, extension='.py')
this_dir_tests = [
'{}.{}'.format(this_dir, x) for x in this_dir_tests
]
tests = tests + this_dir_tests
else:
tests.append(test)
return tests
def test_get_tests(self, project_function, test_utils):
_, project_name = project_function.activate()
test_utils.create_test(project_name, 'test_one')
test_utils.create_test(project_name, 'foo.test_two')
test_utils.create_test(project_name, 'foo.bar.test_three')
test_utils.create_test(project_name, 'foo.bar.baz.test_four')
project = Project(project_name)
tests = project.tests()
expected = ['test_one', 'foo.test_two', 'foo.bar.test_three', 'foo.bar.baz.test_four']
assert tests == expected
tests = project.tests('foo')
expected = ['foo.test_two', 'foo.bar.test_three', 'foo.bar.baz.test_four']
assert tests == expected
tests = project.tests('foo/')
assert tests == expected
expected = ['foo.bar.test_three', 'foo.bar.baz.test_four']
tests = project.tests('foo/bar')
assert tests == expected
def create_test(project_name, test_name):
test_content = ('\n' 'def test(data):\n' ' pass\n')
errors = []
project = Project(project_name)
if test_name in project.tests():
errors.append('A test with that name already exists')
else:
errors = validate_project_element_name(test_name)
if not errors:
project.create_packages_for_element(test_name, project.file_types.TEST)
path = Test(project_name, test_name).path
with open(path, 'w', encoding='utf-8') as f:
f.write(test_content)
print(f'Test {test_name} created for project {project_name}')
return errors
def rename_test(project_name, test_name, new_test_name):
errors = []
project = Project(project_name)
if test_name not in project.tests():
errors.append('Test {} does not exist'.format(test_name))
else:
errors = validate_project_element_name(new_test_name)
if not errors:
old_path = Test(project_name, test_name).path
new_path = Test(project_name, new_test_name).path
project.create_packages_for_element(new_test_name, project.file_types.TEST)
errors = file_manager.rename_file(old_path, new_path)
# try to rename data file in /tests/ folder
old_data_path = os.path.splitext(old_path)[0] + '.csv'
new_data_path = os.path.splitext(new_path)[0] + '.csv'
if os.path.isfile(old_data_path):
errors = file_manager.rename_file(old_data_path, new_data_path)
return errors
class ExecutionRunner:
"""Executes tests or suites.
Three points of entry:
run_test
run_suite
run_directory
"""
def __init__(self,
project_name,
browsers=None,
processes=1,
environments=None,
interactive=False,
timestamp=None,
reports=None,
report_folder=None,
report_name=None,
tags=None,
test_functions=None):
if reports is None:
reports = []
if tags is None:
tags = []
self.project = Project(project_name)
self.cli_args = SimpleNamespace(browsers=browsers,
processes=processes,
envs=environments,
tags=tags)
self.interactive = interactive
self.timestamp = timestamp
self.reports = reports
self.report_folder = report_folder
self.report_name = report_name
self.report = {}
self.tests = []
self.is_suite = False
self.suite_name = None
self.test_name = None
self.execution_name = None
self.selected_browsers = None
self.start_time = None
self.test_functions = test_functions
self.suite = SimpleNamespace(processes=None,
browsers=None,
envs=None,
before=None,
after=None,
tags=None)
has_failed_tests = self._create_execution_has_failed_tests_flag()
self.execution = SimpleNamespace(processes=1,
browsers=[],
envs=[],
tests=[],
reportdir=None,
tags=[],
has_failed_tests=has_failed_tests)
@staticmethod
def _create_execution_has_failed_tests_flag():
"""Multiprocessing safe flag to track if any test has failed or
errored during the execution
Returns a multiprocessing.managers.ValueProxy
"""
return multiprocessing.Manager().Value('error', False)
def _select_environments(self, project_envs):
"""Define the environments to use for the test.
The test can have a list of environments set from 2 places:
- using the -e|--environments CLI argument
- suite `environments` variable
If both of these are empty try using the first env if there
are any envs defined for the project. Otherwise just return ['']
meaning: no envs will be used.
"""
if self.cli_args.envs:
# use the environments passed through command line
envs = self.cli_args.envs
elif self.suite.envs:
# use the environments defined in the suite
envs = self.suite.envs
elif project_envs:
# if there are available envs, use the first by default
envs = [sorted(project_envs)[0]]
else:
envs = []
return envs
def _create_execution_directory(self):
"""Generate the execution report directory"""
return exec_report.create_execution_directory(self.project.name,
self.execution_name,
self.timestamp)
def _define_execution_list(self):
"""Generate the execution list
Generates a list with the required combinations
for each of the following elements:
- tests
- data sets
- environments
- browsers
"""
execution_list = []
envs = self.execution.envs or [None]
envs_data = environment_manager.get_environment_data(self.project.name)
secrets = secrets_manager.get_secrets(self.project.name)
for test in self.tests:
data_sets = test_data.get_parsed_test_data(self.project.name, test)
if len(data_sets) > 1 or len(envs) > 1 or len(
self.execution.browsers) > 1:
# If the test file contain multiple data sets, envs or browsers
# then each set will have a unique id (set_name)
multiple_data_sets = True
else:
# otherwise it's just one set with set_name = ''
multiple_data_sets = False
for data_set in data_sets:
for env in envs:
data_set_env = dict(data_set)
if env in envs_data:
# add env_data to data_set
data_set_env['env'] = envs_data[env]
data_set_env['env']['name'] = env
for browser in self.execution.browsers:
if multiple_data_sets:
set_name = str(uuid.uuid4())[:6]
else:
set_name = ''
testdef = SimpleNamespace(
name=test,
data_set=data_set_env,
secrets=secrets,
browser=browser,
reportdir=self.execution.reportdir,
env=env,
set_name=set_name)
execution_list.append(testdef)
return execution_list
def _print_number_of_tests_found(self):
"""Print number of tests and test sets to console"""
test_number = len(self.tests)
set_number = len(self.execution.tests)
if test_number > 0:
msg = f'Tests found: {test_number}'
if test_number != set_number:
msg = f'{msg} ({set_number} sets)'
print(msg)
def _filter_tests_by_tags(self):
tests = []
try:
tests = tags_manager.filter_tests_by_tags(self.project.name,
self.tests,
self.execution.tags)
except tags_manager.InvalidTagExpression as e:
print(f'{e.__class__.__name__}: {e}')
self.execution.has_failed_tests.value = True
else:
if len(tests) == 0:
print("No tests found with tag(s): {}".format(', '.join(
self.execution.tags)))
return tests
def _get_elapsed_time(self, start_time):
elapsed_time = 0
if start_time:
elapsed_time = round(time.time() - self.start_time, 2)
return elapsed_time
def run_test(self, test):
"""Run a single test.
`test` can be a path to a Python file or an import path.
Both relative to the tests folder.
Example:
test = 'folder/test.py'
test = 'folder.test'
"""
if test.endswith('.py'):
filename, _ = os.path.splitext(test)
test = '.'.join(os.path.normpath(filename).split(os.sep))
self.tests = [test]
self.test_name = test
self.suite_name = test
self.execution_name = test
self._prepare()
def run_suite(self, suite):
"""Run a suite.
`suite` can be a path to a Python file or an import path.
Both relative to the suites folder.
Example:
test = 'folder/suite.py'
test = 'folder.suite'
"""
# TODO
if suite.endswith('.py'):
filename, _ = os.path.splitext(suite)
suite = '.'.join(os.path.normpath(filename).split(os.sep))
suite_obj = suite_module.Suite(self.project.name, suite)
self.tests = suite_obj.tests
if len(self.tests) == 0:
print(f'No tests found for suite {suite}')
self.suite.processes = suite_obj.processes
self.suite.browsers = suite_obj.browsers
self.suite.envs = suite_obj.environments
self.suite.tags = suite_obj.tags
module = suite_obj.get_module()
self.suite.before = getattr(module, 'before', None)
self.suite.after = getattr(module, 'after', None)
self.suite_name = suite
self.execution_name = suite
self.is_suite = True
self._prepare()
def run_directory(self, directory):
"""Run every test inside a directory.
`directory` has to be a relative path from the tests folder.
To run every test in tests folder use: directory=''
"""
self.tests = self.project.tests(directory=directory)
if len(self.tests) == 0:
print(f'No tests were found in {os.path.join("tests", directory)}')
self.is_suite = True
if directory == '':
suite_name = 'all'
else:
suite_name = '.'.join(os.path.normpath(directory).split(os.sep))
self.suite_name = suite_name
self.execution_name = suite_name
self._prepare()
def _prepare(self):
# Generate timestamp if needed.
# A timestamp is passed when the test is executed from the GUI.
# The gui uses this timestamp to fetch the test execution status later on.
# Otherwise, a new timestamp should be generated at this point.
if not self.timestamp:
self.timestamp = utils.get_timestamp()
# create the execution report directory
# The directory takes this structure:
# reports/<execution_name>/<timestamp>/
self.execution.reportdir = self._create_execution_directory()
# Filter tests by tags
self.execution.tags = self.cli_args.tags or self.suite.tags or []
if self.execution.tags:
self.tests = self._filter_tests_by_tags()
if not self.tests:
self._finalize()
else:
# get amount of processes (parallel executions), default is 1
if self.cli_args.processes > 1:
# the processes arg passed through cli has higher priority
self.execution.processes = self.cli_args.processes
elif self.suite.processes:
self.execution.processes = self.suite.processes
# select the browsers to use in this execution
# the order of precedence is:
# 1. browsers defined by CLI
# 2. browsers defined inside a suite
# 3. 'default_browser' setting key
# 4. default default is 'chrome'
self.selected_browsers = utils.choose_browser_by_precedence(
cli_browsers=self.cli_args.browsers,
suite_browsers=self.suite.browsers,
settings_default_browser=session.settings['default_browser'])
# Define the attributes for each browser.
# A browser name can be predefined ('chrome, 'chrome-headless', 'firefox', etc)
# or it can be defined by the user with the 'remote_browsers' setting.
# Remote browsers have extra details such as capabilities
#
# Each defined browser must have the following attributes:
# 'name': real name,
# 'capabilities': full capabilities defined in the remote_browsers setting
remote_browsers = settings_manager.get_remote_browsers(
session.settings)
default_browsers = gui_utils.get_supported_browsers_suggestions()
custom_browsers = self.project.custom_browsers()
self.execution.browsers = define_browsers(self.selected_browsers,
remote_browsers,
default_browsers,
custom_browsers)
# The user can define environments in the environments.json file.
# The suite/test can be executed in one or more of these environments.
# Which environments will be used is defined by this order of preference:
# 1. envs passed by CLI
# 2. envs defined inside the suite
# 3. The first env defined for the project
# 4. no envs at all
#
# Note, in the case of 4, the test might fail if it tries
# to use env variables
project_envs = environment_manager.get_envs(self.project.name)
self.execution.envs = self._select_environments(project_envs)
invalid_envs = [
e for e in self.execution.envs if e not in project_envs
]
if invalid_envs:
print(
'ERROR: the following environments do not exist for project '
f'{self.project.name}: {", ".join(invalid_envs)}')
self.execution.has_failed_tests.value = True
self._finalize()
return
# Generate the execution list
# Each test must be executed for each:
# * data set
# * environment
# * browser
# The result is a list that contains all the requested combinations
self.execution.tests = self._define_execution_list()
# Initialize reports with status 'pending'
initialize_reports_for_test_files(self.project.name,
self.execution.tests)
self._print_number_of_tests_found()
try:
self._execute()
except KeyboardInterrupt:
self.execution.has_failed_tests.value = True
self._finalize()
def _execute(self):
self.start_time = time.time()
suite_error = False
# run suite `before` function
if self.suite.before:
try:
self.suite.before.__call__()
except:
print('ERROR: suite before function failed')
print(traceback.format_exc())
if not suite_error:
if self.interactive and self.execution.processes != 1:
print(
'WARNING: to run in debug mode, processes must equal one')
if self.execution.processes == 1:
# run tests serially
for test in self.execution.tests:
run_test(session.testdir, self.project.name, test.name,
test.data_set, test.secrets, test.browser,
test.env, session.settings, test.reportdir,
test.set_name, self.test_functions,
self.execution.has_failed_tests,
self.execution.tags, self.is_suite)
else:
# run tests using multiprocessing
multiprocess_executor(self.project.name, self.execution.tests,
self.execution.has_failed_tests,
self.test_functions,
self.execution.processes,
self.execution.tags, self.is_suite)
# run suite `after` function
if self.suite.after:
try:
self.suite.after.__call__()
except:
print('ERROR: suite before function failed')
print(traceback.format_exc())
self._finalize()
def _finalize(self):
elapsed_time = self._get_elapsed_time(self.start_time)
# generate report.json
self.report = exec_report.generate_execution_report(
self.execution.reportdir, elapsed_time, self.execution.browsers,
self.execution.processes, self.execution.envs, self.execution.tags,
session.settings['remote_url'])
cli_report.report_to_cli(self.report)
cli_report.print_totals(self.report)
# generate requested reports
report_name = self.report_name or 'report'
report_folder = self.report_folder or self.execution.reportdir
if 'junit' in self.reports:
junit_report.generate_junit_report(self.project.name,
self.execution_name,
self.timestamp,
self.report_folder, report_name)
if 'json' in self.reports and (self.report_folder or self.report_name):
exec_report.save_execution_json_report(self.report, report_folder,
report_name)
if 'html' in self.reports:
html_report.generate_html_report(self.project.name,
self.suite_name, self.timestamp,
self.report_folder, report_name)
if 'html-no-images' in self.reports:
if 'html' in self.reports:
report_name = report_name + '-no-images'
html_report.generate_html_report(self.project.name,
self.suite_name,
self.timestamp,
self.report_folder,
report_name,
no_images=True)
# exit to the console with exit status code 1 in case a test fails
if self.execution.has_failed_tests.value:
sys.exit(1)