async def consumer_handler(ws):
try:
async for raw_message in ws:
message = Client.decode(raw_message)
# if a message got lost, don't proceed
ignore_message_num = message['type'].lower() == MessageType.Error.value
if not ignore_message_num and message['num'] - self.last_acked_message_num > 1:
await self.send(MessageType.Ack, {'message_num': message['num'], 'status': 'out-of-order'})
logger.warn('not processing message %s since previous one(s) never arrived', message['num'])
continue
# if we've seen the message before, skip it
if self.last_acked_message_num >= message['num']:
await self.send(MessageType.Ack, {'message_num': message['num'], 'status': 'duplicate'})
logger.info('deduping message: %s', message['num'])
continue
# let subscribers send a reply to the message
for subscriber in self.subscribers:
resp = await subscriber.on_server_message(message['type'].lower(), message['payload'])
if resp is not None:
message_type, payload = resp
await self.send(message_type, payload)
# ack the message so the server knows it arrived
await self.send(MessageType.Ack, {'message_num': message['num'], 'status': 'success'})
# mark message as processed
self.last_acked_message_num = message['num']
except asyncio.CancelledError:
pass # we are shutting down
async def send(self, message_type, payload):
if self.stopping:
logger.info('client: not sending %s since shutting down'. message_type)
return
self.message_num += 1
message = Client.encode(self.message_num, message_type, payload)
await self.outgoing.put(message)
def pytest_runtestloop(session):
global fatal_error
if not settings.enabled:
logger.info('conquer not enabled')
return main.pytest_runtestloop(session)
if fatal_error:
return False
if session.testsfailed and not session.config.option.continue_on_collection_errors:
raise session.Interrupted('{} errors during collection'.format(
session.testsfailed))
if session.config.option.collectonly:
return True
print('conquer starting')
threads = []
no_of_workers = settings.client_workers
for i in range(no_of_workers):
t = Worker(args=[session, settings])
threads.append(t)
t.start()
for t in threads:
t.join()
return True
async def report(self, report):
logger.info('acking schedule %s', report.schedule_id)
await self.client.send(MessageType.Ack, {
'schedule_id': report.schedule_id,
'status': 'success'
})
logger.info('submitting report with %s item(s)', len(report.items))
await self.report_queue.put(report)
async def _report_task(self):
logger.info('initialising report task')
while True:
try:
report = await self.report_queue.get()
logger.info('sending %s completed item(s)', len(report.items))
report_data = self.serializer.serialize_report(report)
await self.client.send(MessageType.Report, report_data)
self.report_queue.task_done()
except asyncio.CancelledError:
break
async def run_task(self):
global suite_items, schedulers, report_items_by_worker
# init client
client = Client(settings)
client.subscribe(self.settings)
# init scheduler
scheduler = Scheduler(self.settings, client, suite_items, self.name)
schedulers.append(scheduler)
# connect to server
await client.start()
# work through test items
next_tests = []
report_items_by_worker[self.name] = []
while not scheduler.done:
pending_at = datetime.utcnow()
schedule = await scheduler.next()
if schedule is None:
break # happens when we are done
started_at = datetime.utcnow()
schedule_files = [item.file for item in schedule.items]
schedule_tests = itertools.chain(
*[tests_by_file[f] for f in schedule_files])
for test in schedule_tests:
test.__schedule_id__ = schedule.id
next_tests.append(test)
logger.info('preparing schedule took %sms',
str(started_at - pending_at))
while next_tests:
if len(next_tests) < 2 and not scheduler.done:
logger.info('requiring next schedule')
break # we don't know the next test yet
test = next_tests.pop(0)
next_test = next_tests[0] if next_tests else None
test.config.hook.pytest_runtest_protocol(item=test,
nextitem=next_test)
if next_test is None or test.__schedule_id__ != next_test.__schedule_id__:
report = Report(test.__schedule_id__,
report_items_by_worker[self.name],
pending_at, started_at, datetime.utcnow())
await scheduler.report(report)
report_items_by_worker[self.name] = []
# wrap things up
await scheduler.stop()
await client.stop()
async def stop(self):
logger.info('client: shutting down')
# quiescent the client
self.stopping = True
# wait for message queue to empty first
await asyncio.wait_for(self.outgoing.join(), timeout=10)
# now cancel all pending tasks
if (self.consumer_task):
self.consumer_task.cancel()
if (self.producer_task):
self.producer_task.cancel()
if (self.handle_task):
self.handle_task.cancel()
async def on_server_message(self, message_type, payload):
if message_type == MessageType.Config.value:
config_data = self.serializer.serialize_config(
self.settings, self.worker_id)
logger.info('generated config: %s', config_data)
await self.client.send(MessageType.Config, config_data)
elif message_type == MessageType.Suite.value:
suite_data = self.serializer.serialize_suite(self.suite_items)
logger.info('initialising suite with %s item(s)',
len(self.suite_items))
await self.client.send(MessageType.Suite, suite_data)
elif message_type == MessageType.Schedules.value:
for schedule_data in payload:
schedule = self.serializer.deserialize_schedule(schedule_data)
logger.info('received schedule with %s item(s)',
len(schedule.items))
await self.schedule_queue.put(schedule)
elif message_type == MessageType.Done.value:
self.more = False
await self.schedule_queue.put(None) # so we unblock 'next'
elif message_type == MessageType.Error.value:
system_exit(payload['title'], payload['body'], payload['meta'])
def deserialize(data):
items = [ScheduleItem(item['file']) for item in data['items']]
logger.info('received schedule with %s items', len(items))
return Schedule(data['id'], items)
async def _handle(self):
try:
async def consumer_handler(ws):
try:
async for raw_message in ws:
message = Client.decode(raw_message)
# if a message got lost, don't proceed
ignore_message_num = message['type'].lower() == MessageType.Error.value
if not ignore_message_num and message['num'] - self.last_acked_message_num > 1:
await self.send(MessageType.Ack, {'message_num': message['num'], 'status': 'out-of-order'})
logger.warn('not processing message %s since previous one(s) never arrived', message['num'])
continue
# if we've seen the message before, skip it
if self.last_acked_message_num >= message['num']:
await self.send(MessageType.Ack, {'message_num': message['num'], 'status': 'duplicate'})
logger.info('deduping message: %s', message['num'])
continue
# let subscribers send a reply to the message
for subscriber in self.subscribers:
resp = await subscriber.on_server_message(message['type'].lower(), message['payload'])
if resp is not None:
message_type, payload = resp
await self.send(message_type, payload)
# ack the message so the server knows it arrived
await self.send(MessageType.Ack, {'message_num': message['num'], 'status': 'success'})
# mark message as processed
self.last_acked_message_num = message['num']
except asyncio.CancelledError:
pass # we are shutting down
async def producer_handler(ws):
try:
while True:
message = await self.outgoing.get() # blocks forever until something is available
await ws.send(message)
self.outgoing.task_done()
except asyncio.CancelledError:
pass # we are shutting down
wait_before_reconnect = 0
self.connection_attempt = 1
while not self.stopping:
url = self.api_urls[0]
logger.info('connecting to %s', url)
headers = [
('X-Api-Key', str(self.api_key)),
('X-Client-Name', str(self.client_name)),
('X-Client-Version', str(self.client_version)),
('X-Connection-Attempt', str(self.connection_attempt)),
('X-Connection-ID', str(self.id)),
('X-Message-Num-Client', str(self.message_num)),
('X-Message-Num-Server', str(self.last_acked_message_num)),
('X-Message-Format', 'json'),
]
if self.system_provider:
headers.append(('X-Env', str(self.system_provider)))
if wait_before_reconnect > 0:
logger.info('retrying in %ss', wait_before_reconnect)
await asyncio.sleep(min(self.api_wait_limit, wait_before_reconnect))
try:
start = time.time()
async with websockets.connect(
url,
ping_interval=None, # don't send ping, that's the server's responsibility
max_size=None, # accept any message size
max_queue=None, # never drop a message
extra_headers=headers,
) as ws:
self.connected = True
self.connection_attempt = 1
# run consumer and producer in parallel
self.consumer_task = asyncio.ensure_future(consumer_handler(ws))
self.producer_task = asyncio.ensure_future(producer_handler(ws))
done, pending = await asyncio.wait([self.producer_task, self.consumer_task], return_when=asyncio.FIRST_COMPLETED)
# re-raise any exceptions
for task in done:
err = task.exception()
if err:
logger.info(err)
raise err
# one of them finished, let's cancel the other
for task in pending:
task.cancel()
except websockets.exceptions.InvalidStatusCode as err:
logger.info(err)
logger.warning('server error [code: %s], will try to re-connect' % err.status_code)
except websockets.exceptions.ConnectionClosed as err:
logger.info(err)
logger.warning('connection closed, will try to re-connect')
except socket.gaierror as err:
logger.info(err)
logger.warning('lost socket connection, will try to re-connect')
except ConnectionRefusedError as err:
logger.info(err)
logger.warning('connection refused, will try to re-connect')
except OSError as err:
logger.info(err)
logger.warning('connection error, will try to re-connect')
finally:
self.connected = False
if self.connection_attempt > self.api_retry_limit:
self._abort()
self.connection_attempt += 1
wait_before_reconnect = 2 ** self.connection_attempt - (time.time() - start)
self.api_urls.reverse() # we'll try the other URL next
except asyncio.CancelledError:
pass
except Exception as err:
logger.exception(err)
async def start(self):
logger.info('client: starting')
self.handle_task = asyncio.ensure_future(self._handle())