class Context():
def __init__(self, settings={}):
self.settings = settings
async def __aenter__(self):
# set up server
self.server = MockServer()
await self.server.start()
# set up client
self.settings['api_domain'] = self.server.url + '?primary'
self.settings['api_domain_fallback'] = self.server.url + '?fallback'
self.client = Client(MockSettings(self.settings))
await self.client.start()
# set up subscriber
self.subscriber = Subscriber()
self.client.subscribe(self.subscriber)
return self.client, self.server, self.subscriber
async def __aexit__(self, _type, _value, _traceback):
try:
await self.client.stop()
except Exception as err:
print(err)
try:
await self.server.stop()
except Exception as err:
print(err)
async def run_task(self):
global suite_items, schedulers, report_items_by_worker
# init client
client = Client(settings)
client.subscribe(self.settings)
# init scheduler
scheduler = Scheduler(self.settings, client, suite_items, self.name)
schedulers.append(scheduler)
# connect to server
await client.start()
# work through test items
next_tests = []
report_items_by_worker[self.name] = []
while not scheduler.done:
pending_at = datetime.utcnow()
schedule = await scheduler.next()
if schedule is None:
break # happens when we are done
started_at = datetime.utcnow()
schedule_files = [item.file for item in schedule.items]
schedule_tests = itertools.chain(
*[tests_by_file[f] for f in schedule_files])
for test in schedule_tests:
test.__schedule_id__ = schedule.id
next_tests.append(test)
logger.info('preparing schedule took %sms',
str(started_at - pending_at))
while next_tests:
if len(next_tests) < 2 and not scheduler.done:
logger.info('requiring next schedule')
break # we don't know the next test yet
test = next_tests.pop(0)
next_test = next_tests[0] if next_tests else None
test.config.hook.pytest_runtest_protocol(item=test,
nextitem=next_test)
if next_test is None or test.__schedule_id__ != next_test.__schedule_id__:
report = Report(test.__schedule_id__,
report_items_by_worker[self.name],
pending_at, started_at, datetime.utcnow())
await scheduler.report(report)
report_items_by_worker[self.name] = []
# wrap things up
await scheduler.stop()
await client.stop()
async def __aenter__(self):
# set up server
self.server = MockServer()
await self.server.start()
# set up client
self.settings['api_domain'] = self.server.url + '?primary'
self.settings['api_domain_fallback'] = self.server.url + '?fallback'
self.client = Client(MockSettings(self.settings))
await self.client.start()
# set up subscriber
self.subscriber = Subscriber()
self.client.subscribe(self.subscriber)
return self.client, self.server, self.subscriber
def test_connection_when_server_not_reachable(self, datetime_mock, caplog,
event_loop):
settings = {
'api_retry_limit': '3',
'api_wait_limit': '0',
'api_domain': 'doesnot.exist:801',
'api_domain_fallback': 'doesnot.exist:802',
}
self.client = Client(MockSettings(settings))
with pytest.raises(SystemExit):
datetime_mock.utcnow = mock.Mock(return_value=datetime(2000, 1, 1))
event_loop.run_until_complete(self.client.start())
event_loop.run_until_complete(asyncio.sleep(1))
assert warn_messages(caplog) == [
'lost socket connection, will try to re-connect',
'lost socket connection, will try to re-connect',
'lost socket connection, will try to re-connect',
'lost socket connection, will try to re-connect',
]
assert error_messages(caplog) == [
'\n'
'\n'
' '
'================================================================================\n'
'\n'
' [ERROR] [CONQUER] COULD NOT CONNECT:\n'
'\n'
' Unable to connect to server, giving up.\n'
' Please try again and contact support if the error persists.\n'
'\n'
' [Client-Name = pytest-conquer]\n'
' [Client-Version = 1.0]\n'
' [Connection-Attempt = 4]\n'
' [Connection-ID = ' + str(self.client.id) + ']\n'
' [Timestamp = 2000-01-01T00:00:00]\n'
'\n'
' '
'================================================================================\n'
'\n',
]
class TestClient():
@pytest.mark.asyncio
async def test_connection(self, caplog):
async with Context() as (client, server, subscriber):
while len(server.connections) == 0:
await asyncio.sleep(0.01)
(path, headers) = server.connections[0]
assert path == '/?primary'
assert headers['X-API-Key'] == 'some-api-key'
assert headers['X-Client-Name'] == 'pytest-conquer'
assert headers['X-Client-Version'] == '1.0'
assert headers['X-Connection-Attempt'] == '1'
assert headers['X-Connection-ID'] == server.connections[0][1][
'X-Connection-ID']
assert headers['X-Env'] == 'provider'
assert headers['X-Message-Format'] == 'json'
assert headers['X-Message-Num-Client'] == '-1'
assert headers['X-Message-Num-Server'] == '-1'
assert warn_messages(caplog) == []
@pytest.mark.skip
@pytest.mark.asyncio
async def test_connection_when_server_shuts_down(self, caplog):
async with Context() as (client, server, subscriber):
await server.stop()
assert client.connected is False
await server.start()
while client.connected is False:
await asyncio.sleep(0.01)
assert warn_messages(caplog) == [
'server error [code: 503], will try to re-connect'
]
@mock.patch('testandconquer.util.datetime')
def test_connection_when_server_not_reachable(self, datetime_mock, caplog,
event_loop):
settings = {
'api_retry_limit': '3',
'api_wait_limit': '0',
'api_domain': 'doesnot.exist:801',
'api_domain_fallback': 'doesnot.exist:802',
}
self.client = Client(MockSettings(settings))
with pytest.raises(SystemExit):
datetime_mock.utcnow = mock.Mock(return_value=datetime(2000, 1, 1))
event_loop.run_until_complete(self.client.start())
event_loop.run_until_complete(asyncio.sleep(1))
assert warn_messages(caplog) == [
'lost socket connection, will try to re-connect',
'lost socket connection, will try to re-connect',
'lost socket connection, will try to re-connect',
'lost socket connection, will try to re-connect',
]
assert error_messages(caplog) == [
'\n'
'\n'
' '
'================================================================================\n'
'\n'
' [ERROR] [CONQUER] COULD NOT CONNECT:\n'
'\n'
' Unable to connect to server, giving up.\n'
' Please try again and contact support if the error persists.\n'
'\n'
' [Client-Name = pytest-conquer]\n'
' [Client-Version = 1.0]\n'
' [Connection-Attempt = 4]\n'
' [Connection-ID = ' + str(self.client.id) + ']\n'
' [Timestamp = 2000-01-01T00:00:00]\n'
'\n'
' '
'================================================================================\n'
'\n',
]
@pytest.mark.asyncio
async def test_reconnect(self, caplog):
async with Context() as (client, server, subscriber):
await server.send(MessageType.Envs, 'some-payload')
while len(server.connections) == 0:
await asyncio.sleep(0.01)
await server.restart()
while len(server.connections) == 1:
await asyncio.sleep(0.01)
(path, headers) = server.connections[1]
assert path == '/?fallback' # different URL
assert headers['X-Connection-Attempt'] == '2'
assert headers[
'X-Message-Num-Client'] == '0' # 1 client message for ack'ing the server message
assert headers[
'X-Message-Num-Server'] == '0' # 1 server message was acked
assert warn_messages(caplog) == []
@pytest.mark.asyncio
async def test_send_message_successfully(self, caplog):
async with Context() as (client, server, subscriber):
await client.send(MessageType.Suite, 'some-payload')
await assert_received_eventually(server, [
(MessageType.Suite.value, 'some-payload'),
])
assert warn_messages(caplog) == []
@pytest.mark.asyncio
async def test_receive_message_successfully(self, caplog):
async with Context() as (client, server, subscriber):
await server.send(MessageType.Envs, 'some-payload')
await assert_received_eventually(subscriber, [
(MessageType.Envs.value, 'some-payload'),
])
assert warn_messages(caplog) == []
def test_build_url(self):
assert Client.to_url('domain.com',
'us') == 'wss://equilibrium-us.domain.com'
@pytest.mark.asyncio
async def test_ack_message(self):
async with Context() as (client, server, subscriber):
await server.send(MessageType.Envs, 'some-payload')
await assert_received_eventually(server, [
(MessageType.Ack.value, {
'message_num': 0,
'status': 'success'
}),
])
@pytest.mark.asyncio
async def test_dedup_message(self):
async with Context() as (client, server, subscriber):
await server.send(MessageType.Envs, 'some-payload')
await assert_received_eventually(server, [
(MessageType.Ack.value, {
'message_num': 0,
'status': 'success'
}),
])
server.message_num = 0 # reset numbering to replicate duplicated message
client.subscribers = None # invalidate subscribers to make sure they aren't called
await server.send(MessageType.Envs, 'some-payload')
await assert_received_eventually(server, [
(MessageType.Ack.value, {
'message_num': 0,
'status': 'duplicate'
}),
])
@pytest.mark.asyncio
async def test_out_of_order_message(self):
async with Context() as (client, server, subscriber):
server.message_num = 10 # pretend we're out of order
client.subscribers = None # invalidate subscribers to make sure they aren't called
await server.send(MessageType.Envs, 'some-payload')
await assert_received_eventually(server, [
(MessageType.Ack.value, {
'message_num': 10,
'status': 'out-of-order'
}),
])
@pytest.mark.asyncio
async def test_never_skip_out_of_order_error_message(self):
async with Context() as (client, server, subscriber):
server.message_num = 10 # pretend we're out of order
await server.send(MessageType.Error, 'some-payload')
await assert_received_eventually(server, [
(MessageType.Ack.value, {
'message_num': 10,
'status': 'success'
}),
])
def test_build_url(self):
assert Client.to_url('domain.com',
'us') == 'wss://equilibrium-us.domain.com'
async def test_successful_server_communication(config, mock_server):
os.environ['MY_CI'] = 'true'
settings = MockSettings({
'api_domain': mock_server.url,
'api_key': 'api_key',
'api_retry_limit': '1',
'api_wait_limit': '0',
'build_dir': '/app',
'build_id': config['build']['id'],
'build_job': 'job',
'enabled': True,
'vcs_branch': 'master',
'vcs_repo': 'github.com/myrepo',
'vcs_revision': 'asd43da',
'vcs_revision_message': 'my commit',
})
client = Client(settings)
client.subscribe(settings)
suite_items = [
SuiteItem(
'test',
Location('tests/IT/stub/stub_A.py', 'stub_A', 'TestClass',
'test_A', 1)),
SuiteItem('test',
Location('tests/IT/stub/stub_B.py', 'stub_B', 'TestClass',
'test_B', 1),
tags=[Tag('my_group', False)]),
SuiteItem('test',
Location('tests/IT/stub/stub_C.py', 'stub_C', 'TestClass',
'test_C', 1),
deps=[
SuiteItem(
'fixture',
Location('tests/IT/stub/stub_fixture.py', 'fixtures',
'FixtureClass', 'test_C', 0)),
]),
]
scheduler = Scheduler(settings, client, suite_items, 'my_worker_id')
# (1) CONNECT
await client.start()
# (2) SERVER REQUESTS ENV
await mock_server.send(MessageType.Envs, [{
'name': 'CI',
'conditions': ['my_CI'],
'mapping': {}
}])
# (3) CLIENT REPLIES WITH ENV
await assert_received_eventually(mock_server, [
(MessageType.Envs.value, 'CI'),
(MessageType.Ack.value, {
'message_num': 0,
'status': 'success'
}),
])
# (4) SERVER REQUESTS CONFIG
await mock_server.send(MessageType.Config, {})
# (5) CLIENT REPLIES WITH CONFIG
await assert_received_eventually(mock_server, [
(MessageType.Config.value, {
'build': {
'dir': '/app',
'id': config['build']['id'],
'job': 'job',
'node': 'random-uuid',
'pool': 0,
'project': None,
'url': None
},
'client': {
'capabilities':
['fixtures', 'lifecycle_timings', 'split_by_file'],
'messages': [
'ack', 'config', 'done', 'envs', 'error', 'report',
'schedules', 'suite'
],
'name':
'pytest-conquer',
'version':
'1.0',
'workers':
1,
'worker_id':
'my_worker_id',
},
'platform': {
'name': 'python',
'version': '3.6'
},
'runner': {
'args': ['arg1'],
'name': None,
'plugins': [],
'root': None,
'version': None
},
'system': {
'context': {},
'cpus': 3,
'os': 'Linux',
'os_version': '1.42',
'provider': 'CI',
'ram': 17179869184
},
'vcs': {
'branch': 'master',
'pr': None,
'repo': 'github.com/myrepo',
'revision': 'asd43da',
'revision_message': 'my commit',
'tag': None,
'type': 'git'
},
}),
(MessageType.Ack.value, {
'message_num': 1,
'status': 'success'
}),
])
# (6) SERVER REQUESTS SUITE
await mock_server.send(MessageType.Suite, {})
# (7) CLIENT SENDS SUITE
await assert_received_eventually(mock_server, [
(MessageType.Suite.value, {
'items': [
{
'type': 'test',
'location': {
'file': 'tests/IT/stub/stub_A.py',
'func': 'test_A',
'module': 'stub_A',
'class': 'TestClass',
'line': 1
}
},
{
'type': 'test',
'location': {
'file': 'tests/IT/stub/stub_B.py',
'func': 'test_B',
'module': 'stub_B',
'class': 'TestClass',
'line': 1
},
'tags': [{
'group': 'my_group'
}]
},
{
'type':
'test',
'location': {
'file': 'tests/IT/stub/stub_C.py',
'func': 'test_C',
'module': 'stub_C',
'class': 'TestClass',
'line': 1
},
'deps': [{
'type': 'fixture',
'location': {
'file': 'tests/IT/stub/stub_fixture.py',
'func': 'test_C',
'module': 'fixtures',
'class': 'FixtureClass'
}
}]
},
],
}),
(MessageType.Ack.value, {
'message_num': 2,
'status': 'success'
}),
])
# (8) SERVER SENDS SCHEDULE #1
await mock_server.send(MessageType.Schedules, [{
'id':
'0',
'items': [
{
'file': 'tests/IT/stub/stub_A.py'
},
{
'file': 'tests/IT/stub/stub_B.py'
},
],
}])
await assert_received_eventually(mock_server, [
(MessageType.Ack.value, {
'message_num': 3,
'status': 'success'
}),
])
assert await scheduler.next() == Schedule('0', [
ScheduleItem('tests/IT/stub/stub_A.py'),
ScheduleItem('tests/IT/stub/stub_B.py'),
])
# (9) CLIENT SENDS REPORT #1
await scheduler.report(
Report('0', [
ReportItem(
'test',
Location('tests/IT/stub/stub_A.py', 'stub_A', 'TestClass',
'test_A', 3), 'failed',
Failure('AssertionError', 'assert 1 + 1 == 4'), time, time),
ReportItem(
'test',
Location('tests/IT/stub/stub_B.py', 'stub_B', 'TestClass',
'test_B', 1), 'passed', None, time, time),
], time, time, time))
await assert_received_eventually(mock_server, [
(MessageType.Ack.value, {
'schedule_id': '0',
'status': 'success'
}),
(MessageType.Report.value, {
'schedule_id':
'0',
'items': [{
'type': 'test',
'location': {
'file': 'tests/IT/stub/stub_A.py',
'func': 'test_A',
'module': 'stub_A',
'class': 'TestClass',
'line': 3
},
'status': 'failed',
'started_at': '2000-01-01T00:00:00.000Z',
'finished_at': '2000-01-01T00:00:00.000Z',
'error': {
'type': 'AssertionError',
'message': 'assert 1 + 1 == 4'
},
}, {
'type': 'test',
'location': {
'file': 'tests/IT/stub/stub_B.py',
'func': 'test_B',
'module': 'stub_B',
'class': 'TestClass',
'line': 1
},
'status': 'passed',
'started_at': '2000-01-01T00:00:00.000Z',
'finished_at': '2000-01-01T00:00:00.000Z',
}],
'pending_at':
'2000-01-01T00:00:00.000Z',
'started_at':
'2000-01-01T00:00:00.000Z',
'finished_at':
'2000-01-01T00:00:00.000Z',
}),
])
# (10) SERVER SENDS SCHEDULE #2
await mock_server.send(MessageType.Schedules,
[{
'id': '1',
'items': [
{
'file': 'tests/IT/stub/stub_C.py'
},
],
}])
await assert_received_eventually(mock_server, [
(MessageType.Ack.value, {
'message_num': 4,
'status': 'success'
}),
])
assert await scheduler.next() == Schedule('1', [
ScheduleItem('tests/IT/stub/stub_C.py'),
])
# (12) CLIENT SENDS REPORT #2
await scheduler.report(
Report('1', [
ReportItem(
'test',
Location('tests/IT/stub/stub_C.py', 'stub_C', 'TestClass',
'test_C', 1), 'passed', None, time, time),
], time, time, time))
await assert_received_eventually(mock_server, [
(MessageType.Ack.value, {
'schedule_id': '1',
'status': 'success'
}),
(MessageType.Report.value, {
'schedule_id':
'1',
'items': [{
'type': 'test',
'location': {
'file': 'tests/IT/stub/stub_C.py',
'func': 'test_C',
'module': 'stub_C',
'class': 'TestClass',
'line': 1
},
'status': 'passed',
'started_at': '2000-01-01T00:00:00.000Z',
'finished_at': '2000-01-01T00:00:00.000Z',
}],
'pending_at':
'2000-01-01T00:00:00.000Z',
'started_at':
'2000-01-01T00:00:00.000Z',
'finished_at':
'2000-01-01T00:00:00.000Z',
}),
])
# (13) SERVER SENDS DONE
await mock_server.send(MessageType.Done, {})
await asyncio.sleep(0.1)
assert scheduler.more is False
# (14) SHUTDOWN
await scheduler.stop()
await client.stop()
async def send(self, message_type, payload):
await self.outgoing.put(
Client.encode(self.message_num, message_type, payload))
self.message_num += 1
async def consumer_handler():
async for raw_message in ws:
message = Client.decode(raw_message)
self.received.append((message['type'], message['payload']))