def discover_tests_under_paths(paths):
"""Search recursively for every test class under the given paths.
Args:
paths (iterable): list of filesystem paths to be searched.
Returns:
set: all discovered tests.
"""
loader = unittest.TestLoader()
loader.suiteClass = list
loader.loadTestsFromTestCase = lambda test: test
tests = OrderedSet()
for path in get_test_files(paths):
core_log.debug("Discovering tests in %s", path)
module = py.path.local(path).pyimport()
tests_discovered = loader.loadTestsFromModule(module)
tests_discovered = [
test for test in tests_discovered
if test.__module__ == module.__name__
]
tests_discovered = [
test for test in tests_discovered if is_test_class(test)
]
core_log.debug("Discovered %d tests in %s", len(tests_discovered),
path)
tests.update(tests_discovered)
return tests
def __init__(self, save_state, config, run_delta, run_name, requests_queue,
reply_queue, results_queue, root_test, failfast, parent_id,
skip_init, output_handlers, *args, **kwargs):
core_log.debug('Initializing test worker')
super(WorkerProcess, self).__init__()
# Current test instance, timeout and starting time
# They will be managed outside of the process
self.test = None
self.timeout = None
self.start_time = None
self.resource_manager = None
self.root_test = root_test
self.reply_queue = reply_queue
self.results_queue = results_queue
self.requests_queue = requests_queue
self.output_handlers = output_handlers
self.config = config
self.run_name = run_name
self.failfast = failfast
self.parent_id = parent_id
self.run_delta = run_delta
self.skip_init = skip_init
self.save_state = save_state
def __init__(self,
indexer=count(),
base_work_dir=ROTEST_WORK_DIR,
save_state=True,
force_initialize=False,
config=None,
parent=None,
run_data=None,
enable_debug=True,
resource_manager=None,
skip_init=False,
is_main=True):
test_method_name = self.get_test_method_name()
super(AbstractFlowComponent,
self).__init__(indexer, test_method_name, save_state,
force_initialize, config, parent, enable_debug,
resource_manager, skip_init)
self._pipes = {}
self.is_main = is_main
name = self.get_name()
core_log.debug("Initializing %r flow-component", name)
core_log.debug("Creating database entry for %r test-block", name)
self.work_dir = get_work_dir(base_work_dir, name)
self.data = CaseData(name=name, run_data=run_data)
if self.resource_manager is None:
self.resource_manager = self.create_resource_manager()
self._is_client_local = True
def run(self, result, debug=False):
"""Run the tests under the suite and update its data object.
* Notify the data object that the test suite started.
* Call the test suite run method.
* Notify the data object that the test suite ended & update its result.
Args:
result (rotest.core.result.result.Result): Holder for
test result information.
debug (bool): If suite, tests will be run without collecting errors
in a TestResult.
Returns:
rotest.core.result.result.Result. holder for test result
information.
"""
result.startComposite(self)
core_log.debug("Running %r test-suite", self.data)
result = super(TestSuite, self).run(result, debug)
result.stopComposite(self)
return result
def handle_message(self, message):
"""Process given worker message.
Handles messages from the workers' processes, and updates the
manager & workers states accordingly.
Args:
message (str): worker message object.
"""
message = self.decoder.decode(message)
core_log.debug(message)
if message.msg_id not in self.runner.workers_pool:
core_log.warn('Ignoring restarted process %r message',
message.msg_id)
return
message_type = type(message)
if message_type is RunFinished:
self._handle_done_message(message)
else:
test = get_item_by_id(self.main_test, message.test_id)
self.message_handlers[message_type](test, message)
def startTestRun(self):
"""Called once before any tests are executed."""
super(Result, self).startTestRun()
core_log.debug("Test run has started")
for result_handler in self.result_handlers:
result_handler.start_test_run()
def finalize(self):
"""Finalize the test runner.
* Removes duplicated test DB entries.
"""
core_log.debug('Finalizing test %r', self.test_item.data.name)
if self.resource_manager is not None:
self.resource_manager.disconnect()
def clear_tests_queue(self):
"""Empty the pending requests queue, preventing the tests' run."""
core_log.debug('Clearing pending tests')
try:
while True:
self.requests_queue.get(block=False)
except Empty:
pass
def terminate(self):
"""Terminate the worker process and all of its subprocesses."""
core_log.debug("Ending process %r", self.pid)
try:
process = psutil.Process(self.pid)
kill_process_tree(process)
except psutil.NoSuchProcess:
core_log.debug("Process %r not found", self.pid)
def update_worker(self, worker_pid, test):
"""Update the worker properties.
Args:
worker_pid (number): worker's process id.
test (object): the worker's current test.
"""
worker = self.workers_pool[worker_pid]
core_log.debug("Updating worker %r to run test %r", worker, test)
worker.test = test
def execute_threads_and_wait(threads, timeout):
"""Execute the threads and waits until they'll finish their jobs.
Args:
threads (list): list of Thread to be execute.
timeout (number): seconds to wait for each thread to finish.
"""
core_log.debug("executing threads.")
[thread.start() for thread in threads]
[thread.join(timeout) for thread in threads]
core_log.debug("threads execution ended.")
def pytest_sessionfinish(session, exitstatus):
"""Stop the test run and close the resource manager."""
if session.config.option.collectonly:
return
if RotestRunContext.RESULT:
RotestRunContext.RESULT.stopTestRun()
if RotestRunContext.RESOURCE_MANAGER is not None:
core_log.debug("Closing the resource manager")
RotestRunContext.RESOURCE_MANAGER.disconnect()
def test_core_logger(self):
"""Log in core logger and verify logging occurs in this logger only."""
with open(self.core_log_file_path, 'r') as core_log_file:
with open(self.test_log_file_path, 'r') as test_log_file:
log_msg = '%s TEST_CORE_LOGGER' % time.ctime()
core_log_file.seek(0, os.SEEK_END)
test_log_file.seek(0, os.SEEK_END)
core_log.debug(log_msg)
core_log_file_content = core_log_file.read()
test_log_file_content = test_log_file.read()
self.assertEquals(core_log_file_content.count(log_msg), 1)
self.assertEquals(test_log_file_content.count(log_msg), 0)
def update_timeout(self, worker_pid, timeout):
"""Update the worker timeout.
Args:
worker_pid (number): worker's process id.
timeout (number): a timeout to be applied on the worker run (in
seconds). If None is passed, no timeout will be applied.
"""
worker = self.workers_pool[worker_pid]
core_log.debug("Updating worker %r to use timeout %r", worker, timeout)
worker.start_time = datetime.datetime.now()
worker.timeout = timeout
def startTest(self, test):
"""Called when the given test is about to be run.
Args:
test (object): test item instance.
"""
if test.is_main:
super(Result, self).startTest(test)
core_log.debug("Test %r has started running", test.data)
test.start()
for result_handler in self.result_handlers:
result_handler.start_test(test)
def startComposite(self, test):
"""Called when the given TestSuite is about to be run.
This method, unlike 'startTest', does not call unittest TestResult's
'startTest', in order to avoid wrong test counting and treating
TestSuites as the actual tests.
Args:
test (rotest.core.suite.TestSuite): test item instance.
"""
core_log.debug("Test %r has started running", test.data)
test.start()
for result_handler in self.result_handlers:
result_handler.start_composite(test)
def start_server():
"""Run session manager and Django server according to the config file."""
django.setup()
for entry_point in \
pkg_resources.iter_entry_points("rotest.cli_server_actions"):
core_log.debug("Applying server entry point %s", entry_point.name)
extension_action = entry_point.load()
extension_action()
server_args = "0.0.0.0:{}".format(DJANGO_MANAGER_PORT)
if sys.platform == "win32":
sys.argv = ["-m", "django", "runserver", server_args]
call_command("runserver", server_args)
def main(*tests):
"""Run the given tests.
Args:
*tests: either suites or tests to be run.
"""
django.setup()
parser = create_client_options_parser()
arguments = parser.parse_args()
config = AttrDict(
chain(
parse_config_file(DEFAULT_CONFIG_PATH).items(),
parse_config_file(arguments.config_path).items(),
filter_valid_values(vars(arguments)),
))
# In case we're called via 'python test.py ...'
if not sys.argv[0].endswith("rotest"):
main_module = inspect.getfile(__import__("__main__"))
config.paths = (main_module, )
if len(tests) == 0:
tests = discover_tests_under_paths(config.paths)
if config.filter is not None:
tests = [
test for test in tests
if match_tags(get_tags_by_class(test), config.filter)
]
for entry_point in \
pkg_resources.iter_entry_points("rotest.cli_client_actions"):
core_log.debug("Applying entry point %s", entry_point.name)
extension_action = entry_point.load()
extension_action(tests, config)
if len(tests) == 0:
print("No test was found")
sys.exit(1)
class AlmightySuite(TestSuite):
components = tests
run_tests(test=AlmightySuite, config=config)
def parse_config_file(json_path, schema_path=DEFAULT_SCHEMA_PATH):
"""Parse configuration file to create the config dictionary.
Args:
json_path (str): path to the json config file.
schema_path (str): path of the schema file - optional.
Returns:
AttrDict. configuration dict, containing default values for run
options and other parameters.
"""
if not os.path.exists(json_path):
raise ValueError("Illegal config-path: %r" % json_path)
core_log.debug('Parsing configuration file %r', json_path)
config = parse(json_path=json_path, schema_path=schema_path)
return config
def execute(self, test_item):
"""Execute the given test item.
* Starts the main test.
* Queues sub cases identifiers into the request queue.
* Waits on the results queue for test results while
handling timed out tests, and updating console.
* Once all workers finished working return the run data.
Args:
test_item (object): test object.
Returns:
RunData. test run data.
"""
result = self._makeResult()
self.message_handler = RunnerMessageHandler(result=result,
main_test=self.test_item,
multiprocess_runner=self)
result.startTestRun()
core_log.debug('Queuing %r tests jobs', self.test_item.data.name)
self.queue_test_jobs(self.test_item)
core_log.debug('Creating %d workers processes', self.workers_number)
for _ in xrange(self.workers_number):
self.initialize_worker()
while self.finished_workers < self.workers_number:
try:
message = self.results_queue.get(timeout=self.get_timeout())
self.message_handler.handle_message(message)
except Empty:
self.handle_workers_events()
result.stopTestRun()
result.printErrors()
return self.test_item.data.run_data
def stopComposite(self, test):
"""Called when the given TestSuite has been run.
This method, unlike 'stopTest', does not call unittest TestResult's
'stopTest', in order to avoid output redirections and treating
TestSuites as the actual tests.
Args:
test (rotest.core.suite.TestSuite): test item instance.
"""
core_log.debug("Test %r has stopped running", test.data)
sub_values = [sub_test.data.success for sub_test in test
if sub_test.data.success is not None]
if len(sub_values) > 0:
test.data.success = all(sub_values)
test.data.end()
for result_handler in self.result_handlers:
result_handler.stop_composite(test)
def initialize(self, test_class):
"""Initialize the test runner.
* Generates the test object for the given class.
* Generates the test run data and links it to the test object.
Args:
test_class (type): test class inheriting from
:class:`rotest.core.case.TestCase` or
:class:`rotest.core.suite.TestSuite`.
"""
core_log.debug('Generating run data for %r', self.run_name)
run_data = RunData(run_name=self.run_name, run_delta=self.run_delta)
core_log.debug("Creating resource client")
self.resource_manager = self.create_resource_manager()
core_log.debug('Creating test object for %r', test_class.get_name())
self.test_item = test_class(
run_data=run_data,
config=self.config,
skip_init=self.skip_init,
save_state=self.save_state,
enable_debug=self.enable_debug,
resource_manager=self.resource_manager)
run_data.main_test = self.test_item.data
def __init__(self, indexer=count(), methodName='runTest',
base_work_dir=ROTEST_WORK_DIR, save_state=True,
force_initialize=False, config=None, parent=None,
run_data=None, enable_debug=True, resource_manager=None,
skip_init=False):
super(TestCase, self).__init__(indexer, methodName, save_state,
force_initialize, config, parent,
enable_debug, resource_manager,
skip_init)
self.skip_reason = None
self.skip_determined = False
name = self.get_name(methodName)
core_log.debug("Initializing %r test-case", name)
core_log.debug("Creating database entry for %r test-case", name)
self.work_dir = get_work_dir(base_work_dir, name, self)
self.data = CaseData(name=name, run_data=run_data)
core_log.debug("Initialized %r test-case successfully", name)
if self.resource_manager is None:
self.resource_manager = self.create_resource_manager()
self._is_client_local = True
def run(self, test_class):
"""Run the given test class.
* Initializes the test runner.
* Runs the tests and records its results.
* Finalizes the test runner.
Args:
test_class (type): test class inheriting from
:class:`rotest.core.case.TestCase` or
:class:`rotest.core.suite.TestSuite`.
Returns:
rotest.core.models.run_data.RunData. test's run data.
"""
if issubclass(test_class, TestCase):
AnonymousSuite.components = (test_class,)
test_class = AnonymousSuite
test_name = test_class.get_name()
core_log.debug('Initializing %r test runner', test_name)
self.initialize(test_class)
try:
core_log.debug('Running test %r', test_name)
return self.execute(self.test_item)
finally:
core_log.debug('Finalizing %r test runner', test_name)
self.finalize()
def run(self):
"""Initialize runner and run tests from queue.
Creates a test runner then pulls requests from queue,
executes them and notifies to the runner using results queue.
Once done it notifies about its termination to the manager process.
"""
core_log.debug('Worker %r started working', self.pid)
runner = WorkerRunner(config=self.config,
enable_debug=False,
failfast=self.failfast,
run_name=self.run_name,
run_delta=self.run_delta,
skip_init=self.skip_init,
save_state=self.save_state,
outputs=self.output_handlers,
reply_queue=self.reply_queue,
results_queue=self.results_queue)
runner.resource_manager = self.resource_manager
try:
for test_id in iter(self._get_tests, None):
self.assert_runner_is_alive()
test = get_item_by_id(self.root_test, test_id)
core_log.debug('Worker %r is running %r', self.pid,
test.data.name)
runner.execute(test)
core_log.debug('Worker %r done with %r', self.pid,
test.data.name)
core_log.debug('Worker %r finished working', self.pid)
runner.queue_handler.finish_run()
finally:
if (self.resource_manager is not None
and self.resource_manager.is_connected()):
runner.resource_manager.disconnect()
def tearDown(self):
"""Cleanup the runner and its workers if they are still alive."""
active_children() # Join all done processes.
core_log.debug("Waiting for runner process to end")
self.runner_process.join(timeout=self.RUNNER_JOIN_TIMEOUT)
try:
core_log.debug("Killing runner process tree")
process = psutil.Process(self.runner_process.pid)
kill_process_tree(process)
except psutil.NoSuchProcess:
core_log.debug("Process %r not found", self.runner_process.pid)
def client_action(action, client, *params):
"""Execute client's action and store the result."""
client.result = self.TIMEOUT_ERROR_CODE
try:
core_log.debug("Executing action %r on client %r.",
action.__name__, client)
client.result = action(client, *params)
core_log.debug("Action %r on client %r was executed.",
action.__name__, client)
except ServerError as ex:
core_log.debug(
"ServerError occurred while running action "
"%r on client %r. Error was: %s", action.__name__, client,
ex)
client.result = self.SERVER_ERROR_CODE
except Exception as ex:
core_log.debug(
"An error occurred while running action %r on "
"client %r. Error type: '%s'. Error was: %s",
action.__name__, client, ex.__class__.__name__, ex)
client.result = self.UNSPECIFIED_ERROR_CODE
def test_runner_crash(self):
"""Test that workers kill themselves if their runner died."""
core_log.debug("Starting runner process")
self.runner_process.start()
core_log.debug("Waiting for the workers to start")
workers_pids = [
self.pid_queue.get(timeout=self.QUEUE_GET_TIMEOUT)
for _ in xrange(self.WORKERS_NUMBER)
]
runner_process = psutil.Process(self.runner_process.pid)
self.worker_processes = runner_process.children()
core_log.debug("Killing the runner process")
self.runner_process.terminate()
runner_process.wait(timeout=self.RUNNER_KILLING_TIMEOUT)
core_log.debug("Waiting for all active workers to die")
for worker in self.worker_processes:
worker.wait(timeout=self.WORKER_SUICIDE_TIMEOUT)
core_log.debug("Validating no new workers were created")
self.assertRaises(Empty, self.pid_queue.get_nowait)
core_log.debug("Validating the number of cases ran")
self.assertEqual(
len(workers_pids), self.WORKERS_NUMBER,
"Number of cases ran was supposed to be %d"
" Got %d instead" % (self.WORKERS_NUMBER, len(workers_pids)))
# pylint: disable=wildcard-import,unused-wildcard-import
import platform
from rotest.common import core_log
from rotest.common.django_utils.settings import *
if platform.system() == 'Windows': # pragma: no cover
try:
import win32file
core_log.debug("Setting 2048 as the file descriptors limit")
win32file._setmaxstdio(2048) # pylint: disable=protected-access
except ImportError:
import warnings
warnings.warn("Cannot find package 'win32file'. "
"You must install it using 'pip install pypiwin32'")
def create_client_options_parser():
"""Create option parser for running tests.
Returns:
argparse.ArgumentParser: parser for CLI options.
"""
version = pkg_resources.get_distribution("rotest").version
parser = argparse.ArgumentParser(
description="Run tests in a module or directory.")
parser.add_argument("paths", nargs="*", default=(".", ))
parser.add_argument("--version",
action="version",
version="rotest {}".format(version))
parser.add_argument("--config",
"-c",
dest="config_path",
metavar="path",
default=DEFAULT_CONFIG_PATH,
help="Test configuration file path")
parser.add_argument("--save-state",
"-s",
action="store_true",
help="Enable saving state of resources")
parser.add_argument("--delta",
"-d",
dest="delta_iterations",
metavar="iterations",
type=int,
help="Enable run of failed tests only - enter the "
"number of times the failed tests should be run.")
parser.add_argument("--processes",
"-p",
metavar="number",
type=int,
help="Use multiprocess test runner - specify number "
"of worker processes to be created")
parser.add_argument(
"--outputs",
"-o",
type=parse_outputs_option,
help="Output handlers separated by comma. Options: {}".format(
", ".join(get_result_handlers())))
parser.add_argument("--filter",
"-f",
metavar="query",
help="Run only tests that match the filter "
"expression, e.g. 'Tag1* and not Tag13'")
parser.add_argument("--name",
"-n",
metavar="name",
dest="run_name",
help="Assign a name for current launch")
parser.add_argument("--list",
"-l",
action="store_true",
help="Print the tests hierarchy and quit")
parser.add_argument("--failfast",
"-F",
action="store_true",
dest="fail_fast",
help="Stop the run on first failure")
parser.add_argument("--debug",
"-D",
action="store_true",
help="Enter ipdb debug mode upon any test exception")
parser.add_argument("--skip-init",
"-S",
action="store_true",
help="Skip initialization and validation of resources")
parser.add_argument("--resources",
"-r",
metavar="query",
help="Specify resources to request be attributes, "
"e.g. '-r res1.group=QA,res2.comment=CI'")
for entry_point in \
pkg_resources.iter_entry_points("rotest.cli_client_parsers"):
core_log.debug("Applying entry point %s", entry_point.name)
extension_parser = entry_point.load()
extension_parser(parser)
return parser