def pytest_runtestloop(self):
"""pytest runtest loop
- Disable the master terminal reporter hooks, so we can add our own handlers
that include the slaveid in the output
- Send tests to slaves when they ask
- Log the starting of tests and test results, including slave id
- Handle clean slave shutdown when they finish their runtest loops
- Restore the master terminal reporter after testing so we get the final report
"""
# Build master collection for slave diffing and distribution
for item in self.session.items:
self.collection[item.nodeid] = item
# Fire up the workers after master collection is complete
# master and the first slave share an appliance, this is a workaround to prevent a slave
# from altering an appliance while master collection is still taking place
self._slave_audit()
try:
self.print_message("Waiting for {} slave collections".format(
len(self.slaves)),
red=True)
# Turn off the terminal reporter to suppress the builtin logstart printing
terminalreporter.disable()
while True:
# spawn/kill/replace slaves if needed
self._slave_audit()
if not self.slaves:
# All slaves are killed or errored, we're done with tests
self.print_message('all slaves have exited', yellow=True)
self.session_finished = True
if self.session_finished:
break
slaveid, event_data, event_name = self.recv()
if event_name == 'collectionfinish':
slave_collection = event_data['node_ids']
# compare slave collection to the master, all test ids must be the same
self.log.debug('diffing {} collection'.format(slaveid))
diff_err = report_collection_diff(slaveid,
self.collection.keys(),
slave_collection)
if diff_err:
self.print_message('collection differs, respawning',
slaveid,
purple=True)
self.print_message(diff_err, purple=True)
self.log.error('{}'.format(diff_err))
self.kill(slaveid)
self._start_slave(slaveid)
else:
self.ack(slaveid, event_name)
elif event_name == 'need_tests':
self.send_tests(slaveid)
self.log.info('starting master test distribution')
elif event_name == 'runtest_logstart':
self.ack(slaveid, event_name)
self.trdist.runtest_logstart(slaveid, event_data['nodeid'],
event_data['location'])
elif event_name == 'runtest_logreport':
self.ack(slaveid, event_name)
report = unserialize_report(event_data['report'])
if (report.when in ('call', 'teardown')
and report.nodeid in self.slave_tests[slaveid]):
self.slave_tests[slaveid].remove(report.nodeid)
self.trdist.runtest_logreport(slaveid, report)
elif event_name == 'internalerror':
self.ack(slaveid, event_name)
self.print_message(event_data['message'],
slaveid,
purple=True)
with SlaveDict.lock:
if slaveid in self.slaves:
# If this slave hasn't already quit, kill it with fire (signal 9)
self.slaves[slaveid].send_signal(9)
elif event_name == 'shutdown':
self.ack(slaveid, event_name)
self.monitor_shutdown(slaveid)
# total slave spawn count * 3, to allow for each slave's initial spawn
# and then each slave (on average) can fail two times
if self.slave_spawn_count >= len(self.appliances) * 3:
self.print_message('too many slave respawns, exiting',
red=True,
bold=True)
raise KeyboardInterrupt(
'Interrupted due to slave failures')
except Exception as ex:
self.log.error('Exception in runtest loop:')
self.log.exception(ex)
raise
finally:
terminalreporter.enable()
# Suppress other runtestloop calls
return True
def pytest_runtestloop(self):
"""pytest runtest loop
- Disable the master terminal reporter hooks, so we can add our own handlers
that include the slaveid in the output
- Send tests to slaves when they ask
- Log the starting of tests and test results, including slave id
- Handle clean slave shutdown when they finish their runtest loops
- Restore the master terminal reporter after testing so we get the final report
"""
# Build master collection for slave diffing and distribution
for item in self.session.items:
self.collection[item.nodeid] = item
# Fire up the workers after master collection is complete
# master and the first slave share an appliance, this is a workaround to prevent a slave
# from altering an appliance while master collection is still taking place
self._slave_audit()
try:
self.print_message("Waiting for {} slave collections".format(len(self.slaves)),
red=True)
# Turn off the terminal reporter to suppress the builtin logstart printing
terminalreporter.disable()
while True:
# spawn/kill/replace slaves if needed
self._slave_audit()
if not self.slaves:
# All slaves are killed or errored, we're done with tests
self.print_message('all slaves have exited', yellow=True)
self.session_finished = True
if self.session_finished:
break
slaveid, event_data, event_name = self.recv()
if event_name == 'collectionfinish':
slave_collection = event_data['node_ids']
# compare slave collection to the master, all test ids must be the same
self.log.debug('diffing {} collection'.format(slaveid))
diff_err = report_collection_diff(slaveid, self.collection.keys(),
slave_collection)
if diff_err:
self.print_message('collection differs, respawning', slaveid,
purple=True)
self.print_message(diff_err, purple=True)
self.log.error('{}'.format(diff_err))
self.kill(slaveid)
self._start_slave(slaveid)
else:
self.ack(slaveid, event_name)
elif event_name == 'need_tests':
self.send_tests(slaveid)
self.log.info('starting master test distribution')
elif event_name == 'runtest_logstart':
self.ack(slaveid, event_name)
self.trdist.runtest_logstart(slaveid,
event_data['nodeid'], event_data['location'])
elif event_name == 'runtest_logreport':
self.ack(slaveid, event_name)
report = unserialize_report(event_data['report'])
if report.when in ('call', 'teardown'):
self.slave_tests[slaveid].discard(report.nodeid)
self.trdist.runtest_logreport(slaveid, report)
elif event_name == 'internalerror':
self.ack(slaveid, event_name)
self.print_message(event_data['message'], slaveid, purple=True)
with SlaveDict.lock:
if slaveid in self.slaves:
# If this slave hasn't already quit, kill it with fire (signal 9)
self.slaves[slaveid].send_signal(9)
elif event_name == 'shutdown':
self.ack(slaveid, event_name)
self.monitor_shutdown(slaveid)
# total slave spawn count * 3, to allow for each slave's initial spawn
# and then each slave (on average) can fail two times
if self.slave_spawn_count >= len(self.appliances) * 3:
self.print_message('too many slave respawns, exiting',
red=True, bold=True)
raise KeyboardInterrupt('Interrupted due to slave failures')
except Exception as ex:
self.log.error('Exception in runtest loop:')
self.log.exception(ex)
raise
finally:
terminalreporter.enable()
# Suppress other runtestloop calls
return True
