def update_decorator_metrics(self,
decorator,
start_time,
exception=None,
trace=None):
from test_junkie.objects import Limiter
self.__stats[decorator]["performance"].append(time.time() - start_time)
self.__stats[decorator]["exceptions"].append(
Limiter.parse_exception_object(exception))
self.__stats[decorator]["tracebacks"].append(
Limiter.parse_traceback(trace))
def update_metrics(self,
status,
start_time,
param=None,
class_param=None,
exception=None,
formatted_traceback=None,
runtime=None,
decorator=None):
def __get_template():
return {
"status": None,
"retry": 0,
"performance": [],
"exceptions": [],
"tracebacks": [],
DecoratorType.BEFORE_TEST: {
"performance": [],
"exceptions": [],
"tracebacks": []
},
DecoratorType.AFTER_TEST: {
"performance": [],
"exceptions": [],
"tracebacks": []
}
}
runtime = runtime if runtime is not None else time.time() - start_time
string_param = str(param)
string_class_param = str(class_param)
if string_class_param not in self.__stats:
self.__stats.update(
{string_class_param: {
string_param: __get_template()
}})
elif string_param not in self.__stats[string_class_param]:
self.__stats[string_class_param].update(
{string_param: __get_template()})
from test_junkie.objects import Limiter
if decorator is not None:
self.__stats[string_class_param][string_param][decorator][
"performance"].append(time.time() - start_time)
self.__stats[string_class_param][string_param][decorator]["exceptions"]\
.append(Limiter.parse_exception_object(exception))
self.__stats[string_class_param][string_param][decorator]["tracebacks"]\
.append(Limiter.parse_traceback(formatted_traceback))
if self.__stats[string_class_param][string_param]["status"] is None:
# while status is not set, will use one that is currently passed in
# Once its explicitly updated for the test decorator, then we wont change it
self.__stats[string_class_param][string_param][
"status"] = status
else:
self.__stats[string_class_param][string_param][
"performance"].append(runtime)
self.__stats[string_class_param][string_param]["exceptions"]\
.append(Limiter.parse_exception_object(exception))
self.__stats[string_class_param][string_param]["tracebacks"]\
.append(Limiter.parse_traceback(formatted_traceback))
self.__stats[string_class_param][string_param]["retry"] += 1
self.__stats[string_class_param][string_param]["status"] = status
self.__stats[string_class_param][string_param]["param"] = param
self.__stats[string_class_param][string_param][
"class_param"] = class_param
def __run_suite(self, suite):
def before_group_rule_failed():
for group, _result in self.__before_group_failure_records.items():
if suite.get_class_object() in _result["definition"]["suites"]:
return _result["trace"]
suite_start_time = time.time()
unsuccessful_tests = None
exception = Runner.__validate_suite_parameters(suite)
if not exception:
exception = before_group_rule_failed()
if not exception:
result = self.__group_rules.run_before_group(
suite, DecoratorType.BEFORE_GROUP)
if result is not None:
self.__before_group_failure_records.update(result)
exception = result[list(result.keys())[0]]["trace"]
if not suite.can_skip(
self.__settings) and not self.__cancel and not exception:
Runner.__process_event(event=Event.ON_CLASS_IN_PROGRESS,
suite=suite)
for suite_retry_attempt in range(1, suite.get_retry_limit() + 1):
if suite_retry_attempt == 1 or suite.get_status(
) in SuiteCategory.ALL_UN_SUCCESSFUL:
for class_param in suite.get_parameters(
process_functions=True):
LogJunkie.debug("Running suite: {}".format(
suite.get_class_object()))
LogJunkie.debug(
"Suite Retry {}/{} with Param: {}".format(
suite_retry_attempt, suite.get_retry_limit(),
class_param))
before_class_error = Runner.__run_before_class(
suite, class_param)
if suite_retry_attempt > 1:
unsuccessful_tests = suite.get_unsuccessful_tests()
LogJunkie.debug(
"There are {} unsuccessful tests that need to be retried"
.format(len(unsuccessful_tests)))
if not unsuccessful_tests:
break
tests = unsuccessful_tests
else:
tests = list(suite.get_test_objects())
while tests:
for test in list(tests):
test_start_time = time.time(
) # will use in case of a failure in context of this loop
if not self.__positive_skip_condition(test=test) and \
Runner.__runnable_tags(test=test, tag_config=self.__settings.tags):
if not test.is_parallelized():
LogJunkie.debug(
"Cant run test: {} in parallel with any other tests"
.format(
test.get_function_object()))
ParallelProcessor.wait_currently_active_tests_to_finish(
)
bad_params = Runner.__validate_test_parameters(
test)
if bad_params is not None:
tests.remove(test)
test.metrics.update_metrics(
status=TestCategory.IGNORE,
start_time=test_start_time,
exception=bad_params["exception"],
formatted_traceback=bad_params[
"trace"])
Runner.__process_event(
event=Event.ON_IGNORE,
suite=suite,
test=test,
class_param=class_param,
error=bad_params)
continue
while not self.__processor.test_qualifies(
test):
time.sleep(0.2)
if test.get_priority() is None:
continue
for param in test.get_parameters(
process_functions=True):
if unsuccessful_tests is not None and \
not test.is_qualified_for_retry(param, class_param=class_param):
# If does not qualify with current parameter, will move to the next
continue
if ((self.__processor.
test_multithreading()
and param is None) or
(self.__processor.
test_multithreading() and
test.parallelized_parameters()
and param is not None)):
while self.__processor.test_limit_reached(
):
time.sleep(0.2)
time.sleep(
Limiter.get_test_throttling())
self.__processor.run_test_in_a_thread(
Runner.__run_test, suite, test,
param, class_param,
before_class_error,
self.__cancel)
else:
Runner.__run_test(
suite=suite,
test=test,
parameter=param,
class_parameter=class_param,
before_class_error=
before_class_error,
cancel=self.__cancel)
tests.remove(test)
else:
tests.remove(test)
test.metrics.update_metrics(
status=TestCategory.SKIP,
start_time=test_start_time)
Runner.__process_event(
event=Event.ON_SKIP,
suite=suite,
test=test,
class_param=class_param)
ParallelProcessor.wait_currently_active_tests_to_finish(
)
Runner.__run_after_class(suite, class_param)
suite.metrics.update_suite_metrics(
status=SuiteCategory.FAIL
if suite.has_unsuccessful_tests() else
SuiteCategory.SUCCESS,
start_time=suite_start_time)
Runner.__process_event(event=Event.ON_CLASS_COMPLETE, suite=suite)
after_group_failed = self.__group_rules.run_after_group(suite)
if after_group_failed:
event = Event.ON_AFTER_GROUP_FAIL if isinstance(
after_group_failed["exception"],
AssertionError) else Event.ON_AFTER_GROUP_ERROR
Runner.__process_event(
event=event,
suite=suite,
error=after_group_failed["exception"],
formatted_traceback=after_group_failed["trace"])
elif self.__cancel:
suite.metrics.update_suite_metrics(status=SuiteCategory.CANCEL,
start_time=suite_start_time)
Runner.__process_event(event=Event.ON_CLASS_CANCEL, suite=suite)
elif exception or before_group_rule_failed():
suite.metrics.update_suite_metrics(status=SuiteCategory.IGNORE,
start_time=suite_start_time,
initiation_error=exception)
Runner.__process_event(event=Event.ON_CLASS_IGNORE, suite=suite)
else:
suite.metrics.update_suite_metrics(status=SuiteCategory.SKIP,
start_time=suite_start_time)
Runner.__process_event(event=Event.ON_CLASS_SKIP, suite=suite)
def run(self, **kwargs):
"""
Initiates the execution process that runs tests
:return: None
"""
self.__settings = Settings(runner_kwargs=self.__kwargs,
run_kwargs=kwargs)
initial_start_time = time.time()
resource_monitor = None
try:
if self.__settings.monitor_resources:
resource_monitor = ResourceMonitor()
resource_monitor.start()
self.__processor = ParallelProcessor(self.__settings)
with suppressed_stdout(self.__settings.quiet):
while self.__suites:
for suite in list(self.__suites):
suite_object = Builder.get_execution_roster().get(
suite, None)
if suite_object is not None:
if self.__processor.suite_multithreading(
) and suite_object.is_parallelized():
while True:
if self.__processor.suite_qualifies(
suite_object):
time.sleep(
Limiter.get_suite_throttling())
self.__executed_suites.append(
suite_object)
ParallelProcessor.run_suite_in_a_thread(
self.__run_suite, suite_object)
self.__suites.remove(suite)
break
elif suite_object.get_priority() is None:
break
else:
time.sleep(1)
else:
if not suite_object.is_parallelized():
LogJunkie.debug(
"Cant run suite: {} in parallel with any other suites. Waiting for "
"parallel suites to finish so I can run it by itself."
.format(
suite_object.get_class_object()))
ParallelProcessor.wait_currently_active_suites_to_finish(
)
self.__executed_suites.append(suite_object)
self.__run_suite(suite_object)
self.__suites.remove(suite)
else:
LogJunkie.warn(
"Suite: {} not found! Make sure that your input is correct. "
"If it is, make sure the use of Test Junkie's decorators "
"is correct.".format(suite))
self.__suites.remove(suite)
LogJunkie.debug("{} Suite(s) left in queue.".format(
len(self.__suites)))
time.sleep(0.2)
ParallelProcessor.wait_currently_active_suites_to_finish()
finally:
if self.__settings.monitor_resources:
resource_monitor.shutdown()
runtime = time.time() - initial_start_time
print("========== Test Junkie finished in {:0.2f} seconds ==========".
format(runtime))
aggregator = Aggregator(self.get_executed_suites())
Aggregator.present_console_output(aggregator)
if self.__settings.html_report:
reporter = Reporter(
monitoring_file=resource_monitor.get_file_path()
if resource_monitor is not None else None,
runtime=runtime,
aggregator=aggregator,
multi_threading_enabled=self.__processor.test_multithreading()
or self.__processor.suite_multithreading())
reporter.generate_html_report(self.__settings.html_report)
XmlReporter.create_xml_report(write_file=self.__settings.xml_report,
suites=self.get_executed_suites())
if self.__settings.monitor_resources:
resource_monitor.cleanup()
return aggregator