async def test_modify_simulator():
"""Test that we can modify a simulator"""
# This is very dangerous because we're just finding and modifying a random
# simulator. However, adding and removing a random role shouldn't really
# affect anything, so this should be fine
async with api.api() as egta:
role = '__unique_role__'
strat1 = '__unique_strategy_1__'
strat2 = '__unique_strategy_2__'
# Old sims seem frozen so > 100
for sim in await egta.get_simulators(): # pragma: no branch
if sim['id'] > 100:
break
else: # pragma: no cover
sim = None
raise AssertionError('no simulators')
try:
await sim.add_strategies({role: [strat1]})
assert ((await
sim.get_info())['role_configuration'][role] == [strat1])
await sim.add_strategy(role, strat2)
expected = [strat1, strat2]
assert ((await
sim.get_info())['role_configuration'][role] == expected)
await sim.remove_strategies({role: [strat1]})
assert ((await
sim.get_info())['role_configuration'][role] == [strat2])
finally:
await sim.remove_role(role)
assert role not in (await sim.get_info())['role_configuration']
async def test_profile_json_error():
"""Test invalid profile json triggers retry"""
async with mockserver.server() as server, \
api.api('', num_tries=3, retry_delay=0.5) as egta:
sim = await create_simulator(server, egta, 'sim', '1')
sched = await sim.create_generic_scheduler('sched', True, 0, 4, 0, 0)
await sched.add_roles({'a': 2, 'b': 2})
game = await sched.create_game()
await game.add_symgroups([('a', 2, ['1']), ('b', 2, ['5', '6'])])
prof = await sched.add_profile('a: 2 1; b: 2 5', 2)
await sched_complete(sched)
server.custom_response(lambda: '', 2)
summ = await prof.get_summary()
assert 'id' in summ
assert 'observations_count' in summ
assert 'symmetry_groups' in summ
server.custom_response(lambda: '', 3)
with pytest.raises(json.decoder.JSONDecodeError):
await prof.get_summary()
server.custom_response(lambda: '{}', 3)
with pytest.raises(jsonschema.ValidationError):
await prof.get_summary()
async def test_game_json_error():
"""Test returning invalid json in games triggers retry"""
async with mockserver.server() as server, \
api.api('', num_tries=3, retry_delay=0.5) as egta:
sim = await create_simulator(server, egta, 'sim', '1')
sched = await sim.create_generic_scheduler('sched', True, 0, 4, 0, 0)
await sched.add_roles({'a': 2, 'b': 2})
game = await sched.create_game()
await game.add_symgroups([('a', 2, ['1']), ('b', 2, ['5', '6'])])
await sched.add_profile('a: 2 1; b: 1 5, 1 6', 1)
await sched.add_profile('a: 2 1; b: 2 5', 2)
await sched_complete(sched)
server.custom_response(lambda: '', 2)
summ = await game.get_summary()
size_counts = {}
for prof in summ['profiles']:
counts = prof['observations_count']
size_counts[counts] = size_counts.get(counts, 0) + 1
assert size_counts == {1: 1, 2: 1}
server.custom_response(lambda: '', 3)
with pytest.raises(json.decoder.JSONDecodeError):
await game.get_summary()
server.custom_response(lambda: '{}', 3)
with pytest.raises(jsonschema.ValidationError):
await game.get_summary()
async def test_get_simulators():
"""Test getting simulators"""
async with mockserver.server() as server, \
api.api('', num_tries=3, retry_delay=0.5) as egta:
sim1 = server.create_simulator('foo', '1')
sim2 = server.create_simulator('bar', '1')
sim3 = server.create_simulator('bar', '2')
assert sum(1 for _ in await egta.get_simulators()) == 3
assert {0, 1, 2} == {s['id'] for s in await egta.get_simulators()}
sim = await egta.get_simulator(0)
assert sim['id'] == sim1
sim = await egta.get_simulator_fullname('foo-1')
assert sim['id'] == sim1
sim = await egta.get_simulator(2)
assert sim['id'] == sim3
sim = await egta.get_simulator_fullname('bar-1')
assert sim['id'] == sim2
sim = await egta.get_simulator_fullname('bar-2')
assert sim['id'] == sim3
with pytest.raises(requests.exceptions.HTTPError):
await egta.get_simulator(3)
with pytest.raises(AssertionError):
await egta.get_simulator_fullname('baz')
async def test_get_games():
"""Test getting games"""
async with mockserver.server() as server, \
api.api('', num_tries=3, retry_delay=0.5) as egta:
sim = await create_simulator(server, egta, 'sim', '1')
await sim.create_game('a', 5)
game2 = await egta.create_game(sim['id'], 'b', 6)
game3 = await sim.create_game('c', 3)
with pytest.raises(requests.exceptions.HTTPError):
await sim.create_game('b', 3)
assert (await egta.get_game(1))['id'] == game2['id']
assert (await egta.get_game_name('c'))['id'] == game3['id']
assert sum(1 for _ in await egta.get_games()) == 3
assert {0, 1, 2} == {g['id'] for g in await egta.get_games()}
await game2.destroy_game()
assert sum(1 for _ in await egta.get_games()) == 2
assert {0, 2} == {g['id'] for g in await egta.get_games()}
with pytest.raises(requests.exceptions.HTTPError):
await egta.get_game(3)
with pytest.raises(requests.exceptions.HTTPError):
await egta.get_game(1)
with pytest.raises(AssertionError):
await egta.get_game_name('b')
async def test_exception_in_get(game):
"""Test exception in await"""
async with mockserver.server() as server, api.api() as egta:
sim = await egta.get_simulator(
server.create_simulator("sim",
"1",
delay_dist=lambda: random.random() / 10))
strats = dict(zip(game.role_names, game.strat_names))
symgrps = list(
zip(game.role_names, game.num_role_players, game.strat_names))
await sim.add_strategies(strats)
egame = await egta.get_canon_game(sim["id"], symgrps)
profs = game.random_profiles(20)
async with eosched.eosched(game, egta, egame["id"], 0.1, 1, 10, 0,
0) as sched:
tasks = [
asyncio.ensure_future(sched.sample_payoffs(p)) for p in profs
]
await asyncio.sleep(0.1)
server.custom_response(lambda: _raise(TimeoutError))
await asyncio.sleep(0.1)
with pytest.raises(TimeoutError):
await asyncio.gather(*tasks)
# tidy up
errors = asyncio.gather(*tasks, return_exceptions=True)
errors.cancel()
with contextlib.suppress(TimeoutError, asyncio.CancelledError):
await errors
async def test_exception_open():
"""Test that exceptions are even thrown on open"""
async with mockserver.server() as server:
server.custom_response(lambda: _raise(TimeoutError))
with pytest.raises(TimeoutError):
async with api.api(''):
pass # pragma: no cover
async def test_modify_scheduler():
"""Test that we can modify a scheduler"""
async with api.api() as egta:
for sim in await egta.get_simulators(): # pragma: no branch
if next(iter(sim['role_configuration'].values()), None): # pragma: no branch pylint: disable=line-too-long
break
else: # pragma: no cover
sim = None
raise AssertionError('no simulators')
sched = game = None
try:
sched = await sim.create_generic_scheduler('__unique_scheduler__',
False, 0, 1, 0, 0)
await sched.activate()
await sched.deactivate()
role = next(iter(sim['role_configuration']))
strat = sim['role_configuration'][role][0]
symgrps = [{'role': role, 'strategy': strat, 'count': 1}]
assignment = api.symgrps_to_assignment(symgrps)
await sched.add_role(role, 1)
reqs = (await sched.get_requirements())['scheduling_requirements']
assert not reqs
prof = await sched.add_profile(symgrps, 1)
reqs = (await sched.get_requirements())['scheduling_requirements']
assert len(reqs) == 1
assert reqs[0]['requirement'] == 1
await sched.remove_profile(prof['id'])
assert (await sched.add_profile(assignment, 2))['id'] == prof['id']
reqs = (await sched.get_requirements())['scheduling_requirements']
assert len(reqs) == 1
assert reqs[0]['requirement'] == 2
await sched.remove_profile(prof['id'])
reqs = (await sched.get_requirements())['scheduling_requirements']
assert not reqs
assert (await sched.add_profile(symgrps, 3))['id'] == prof['id']
reqs = (await sched.get_requirements())['scheduling_requirements']
assert len(reqs) == 1
assert reqs[0]['requirement'] == 3
await sched.remove_all_profiles()
reqs = (await sched.get_requirements())['scheduling_requirements']
assert not reqs
await sched.remove_role(role)
game = await sched.create_game()
finally:
if sched is not None: # pragma: no branch
await sched.destroy_scheduler()
if game is not None: # pragma: no branch
await game.destroy_game()
async def test_delayed_profiles():
"""Test mock server with delayed profile scheduling"""
async with mockserver.server() as server, api.api('') as egta:
sim = await egta.get_simulator(
server.create_simulator('sim', '1', delay_dist=lambda: 0.5))
await sim.add_strategies({'1': ['a'], '2': ['b', 'c']})
sched = await sim.create_generic_scheduler('sched', True, 0, 10, 0, 0)
await sched.add_roles({'1': 8, '2': 2})
prof = await sched.add_profile('1: 8 a; 2: 1 b, 1 c', 3)
reqs = (await sched.get_requirements())['scheduling_requirements']
assert len(reqs) == 1
assert reqs[0]['current_count'] == 0
assert reqs[0]['requirement'] == 3
assert reqs[0]['id'] == prof['id']
count = 0
async for sim in egta.get_simulations():
assert sim['state'] == 'running'
count += 1
assert count == 3
await asyncio.sleep(0.5)
await sched_complete(sched)
reqs = (await sched.get_requirements())['scheduling_requirements']
assert len(reqs) == 1
assert reqs[0]['current_count'] == 3
assert reqs[0]['requirement'] == 3
assert reqs[0]['id'] == prof['id']
count = 0
async for sim in egta.get_simulations():
assert sim['state'] == 'complete'
count += 1
assert count == 3
# Test that extra sims get killed
async with mockserver.server() as server, api.api('') as egta:
sim = await egta.get_simulator(
server.create_simulator('sim', '1', delay_dist=lambda: 10))
await sim.add_strategies({'1': ['a'], '2': ['b', 'c']})
sched = await sim.create_generic_scheduler('sched', True, 0, 10, 0, 0)
await sched.add_roles({'1': 8, '2': 2})
# Need two profiles here because the other will be in the queue
await sched.add_profile('1: 8 a; 2: 1 b, 1 c', 1)
await sched.add_profile('1: 8 a; 2: 2 b', 1)
async def test_threading():
"""Test that no errors arise when multi-threading"""
async with mockserver.server() as server, \
api.api('', num_tries=3, retry_delay=0.5) as egta:
sim = await create_simulator(server, egta, 'sim', '1')
await asyncio.gather(*[
sim.add_strategies({'r{:d}'.format(i): ['s{:d}'.format(i)]})
for i in range(10)
])
async def get_eosched(server, game, name):
"""Get an eo sched"""
async with api.api() as egta:
sim = await egta.get_simulator(server.create_simulator(
name, '1', delay_dist=lambda: random.random() / 100))
await sim.add_strategies(dict(zip(game.role_names, game.strat_names)))
eogame = await sim.create_game(name, game.num_players)
await eogame.add_symgroups(list(zip(
game.role_names, game.num_role_players, game.strat_names)))
return 'eo:game:{:d},mem:2048,time:60,sleep:0.1'.format(
eogame['id'])
async def test_sched():
"""Test scheduler functionality"""
async with mockserver.server() as server:
# verify no schedulers
with stdout() as out, stderr() as err:
assert await run('-a', '', 'sched'), err.getvalue()
assert not out.getvalue()
# create one
sim_id = server.create_simulator('sim', '1')
async with api.api() as egta:
await egta.create_generic_scheduler(sim_id, 'sched', True, 1, 2, 1,
1)
with stdout() as out, stderr() as err:
assert await run('-a', '', 'sched'), err.getvalue()
sched = json.loads(out.getvalue())
with stderr() as err:
assert await run('-a', '', 'sched',
str(sched['id'])), err.getvalue()
with stderr() as err:
assert await run('-a', '', 'sched', str(sched['id']),
'-r'), err.getvalue()
with stdout() as out, stderr() as err:
assert await run('-a', '', 'sched', sched['name'],
'-n'), err.getvalue()
assert sched['id'] == json.loads(out.getvalue())['id']
# deactivate scheduler
with stderr() as err:
assert await run('-a', '', 'sched', str(sched['id']),
'--deactivate'), err.getvalue()
# verify deactivated
with stdout() as out, stderr() as err:
assert await run('-a', '', 'sched',
str(sched['id'])), err.getvalue()
assert not json.loads(out.getvalue())['active']
# delete scheduler
with stderr() as err:
assert await run('-a', '', 'sched', str(sched['id']),
'-d'), err.getvalue()
# assert no schedulers
with stdout() as out, stderr() as err:
assert await run('-a', '', 'sched'), err.getvalue()
assert not out.getvalue()
async def test_basic_profile(game):
"""Test scheduling a standard profile"""
async with mockserver.server() as server, api.api() as egta:
sim = await egta.get_simulator(
server.create_simulator("sim",
"1",
delay_dist=lambda: random.random() / 10))
strats = dict(zip(game.role_names, game.strat_names))
symgrps = list(
zip(game.role_names, game.num_role_players, game.strat_names))
await sim.add_strategies(strats)
egame = await egta.get_canon_game(sim["id"], symgrps)
profs = game.random_profiles(20)
# Schedule all new profiles and verify it works
async with eosched.eosched(game, egta, egame["id"], 0.1, 1, 10, 0,
0) as sched:
assert str(sched) == str(egame["id"])
assert game == rsgame.empty_copy(sched)
awaited = await asyncio.gather(
*[sched.sample_payoffs(p) for p in profs])
pays = np.stack(awaited)
assert np.allclose(pays[profs == 0], 0)
# Schedule old profiles and verify it still works
async with eosched.eosched(game, egta, egame["id"], 0.1, 1, 10, 0,
0) as sched:
awaited = await asyncio.gather(
*[sched.sample_payoffs(p) for p in profs])
pays = np.stack(awaited)
assert np.allclose(pays[profs == 0], 0)
# Schedule two at a time, in two batches
async with eosched.eosched(game, egta, egame["id"], 0.1, 2, 10, 0,
0) as base_sched:
sched = countsched.countsched(base_sched, 2)
awaited = await asyncio.gather(
*[sched.sample_payoffs(p) for p in profs])
pays = np.stack(awaited)
assert np.allclose(pays[profs == 0], 0)
# Try again now that everything should be scheduled
async with eosched.eosched(game, egta, egame["id"], 0.1, 2, 10, 0,
0) as base_sched:
sched = countsched.countsched(base_sched, 2)
awaited = await asyncio.gather(
*[sched.sample_payoffs(p) for p in profs])
pays = np.stack(awaited)
assert np.allclose(pays[profs == 0], 0)
async def test_modify_game():
"""Test that we can modify a game"""
async with api.api() as egta: # pragma: no branch
for sim in await egta.get_simulators(): # pragma: no branch
if next(iter(sim['role_configuration'].values()), None): # pragma: no branch pylint: disable=line-too-long
break
else: # pragma: no cover
sim = None # pytest detects that it's defined
raise AssertionError('no simulators')
game = None
try:
game = await sim.create_game('__unique_game__', 1)
summ = await game.get_summary()
assert not summ['roles']
assert not summ['profiles']
role = next(iter(sim['role_configuration']))
strat = sim['role_configuration'][role][0]
await game.add_role(role, 1)
summ = await game.get_summary()
assert len(summ['roles']) == 1
assert summ['roles'][0]['name'] == role
assert summ['roles'][0]['count'] == 1
assert not summ['roles'][0]['strategies']
await game.add_strategies({role: [strat]})
summ = await game.get_summary()
assert len(summ['roles']) == 1
assert summ['roles'][0]['name'] == role
assert summ['roles'][0]['count'] == 1
assert summ['roles'][0]['strategies'] == [strat]
await game.remove_strategies({role: [strat]})
summ = await game.get_summary()
assert len(summ['roles']) == 1
assert summ['roles'][0]['name'] == role
assert summ['roles'][0]['count'] == 1
assert not summ['roles'][0]['strategies']
await game.remove_role(role)
summ = await game.get_summary()
assert not summ['roles']
assert not summ['profiles']
finally:
if game is not None: # pragma: no branch
await game.destroy_game()
async def test_exception_in_create(game):
"""Test exception creating eo scheduler"""
async with mockserver.server() as server, api.api() as egta:
sim = await egta.get_simulator(server.create_simulator( # pragma: no branch pylint: disable=line-too-long
'sim', '1', delay_dist=lambda: random.random() / 10))
strats = dict(zip(game.role_names, game.strat_names))
symgrps = list(zip(game.role_names, game.num_role_players,
game.strat_names))
await sim.add_strategies(strats)
egame = await egta.get_canon_game(sim['id'], symgrps)
server.custom_response(lambda: _raise(TimeoutError))
with pytest.raises(TimeoutError):
async with eosched.eosched(
game, egta, egame['id'], 0.1, 1, 25, 0, 0):
pass # pragma: no cover
async def create_scheduler( # pylint: disable=too-many-arguments
game, mem, time, auth=None, count=1, sleep=600,
# pylint: disable-msg=redefined-builtin
max=100):
"""Create an egtaonline scheduler"""
game_id = int(game)
mem = int(mem)
time = int(time)
count = int(count)
sleep = float(sleep)
max_sims = int(max)
async with api.api(auth_token=auth) as egta:
game = await egta.get_game(game_id)
summ = await game.get_summary()
game = rsgame.empty_json(summ)
return ApiWrapper(game, egta, game_id, sleep, count, max_sims, mem, time)
async def test_gets():
"""Test that get functions work"""
async with api.api() as egta:
with pytest.raises(AssertionError):
await egta.get_simulator_fullname('this name is impossible I hope')
async def test_sim_name(sim):
"""Verify sim name is accurate"""
assert sim['id'] == (await
egta.get_simulator_fullname('{}-{}'.format(
sim['name'], sim['version'])))['id']
await asyncio.gather(
*[test_sim_name(sim) for sim in await egta.get_simulators()])
sched = next(iter(await egta.get_generic_schedulers()))
assert (await egta.get_scheduler(sched['id']))['id'] == sched['id']
assert ((await
egta.get_scheduler_name(sched['name']))['id'] == sched['id'])
with pytest.raises(AssertionError):
await egta.get_scheduler_name('this name is impossible I hope')
game = next(iter(await egta.get_games()))
assert (await egta.get_game(game['id']))['id'] == game['id']
assert (await egta.get_game_name(game['name']))['id'] == game['id']
with pytest.raises(AssertionError):
await egta.get_game_name('this name is impossible I hope')
folds = egta.get_simulations()
fold = await folds.__anext__()
assert (await egta.get_simulation(fold['folder']
))['folder_number'] == fold['folder']
for sort in ['job', 'folder', 'profile', 'simulator', 'state']:
assert 'folder' in await egta.get_simulations(column=sort
).__anext__()
with pytest.raises(StopAsyncIteration):
await egta.get_simulations(page_start=10**9).__anext__()
for sch in await egta.get_generic_schedulers(): # pragma: no branch
sched = await sch.get_requirements()
if sched['scheduling_requirements']: # pragma: no branch
break
prof = sched['scheduling_requirements'][0]
assert (await egta.get_profile(prof['id']))['id'] == prof['id']
async def test_large_game_failsafes():
"""Test that large games fallback to gathering profile data"""
async with mockserver.server() as server, \
api.api('', num_tries=3, retry_delay=0.5) as egta:
sim = await create_simulator(server, egta, 'sim', '1')
error = requests.exceptions.HTTPError(
'500 Server Error: Game too large!')
sched = await sim.create_generic_scheduler('sched',
True,
0,
4,
0,
0,
configuration={'k': 'v'})
await sched.add_roles({'a': 2, 'b': 2})
game = await sched.create_game()
await game.add_symgroups([('a', 2, ['1']), ('b', 2, ['5', '6'])])
await sched.add_profile('a: 2 1; b: 1 5, 1 6', 1)
await sched.add_profile('a: 2 1; b: 2 5', 2)
await sched_complete(sched)
base = await game.get_observations()
server.custom_response(lambda: _raise(error))
alternate = await game.get_observations()
assert base == alternate
size_counts = {}
for prof in alternate['profiles']:
counts = len(prof['observations'])
size_counts[counts] = size_counts.get(counts, 0) + 1
assert size_counts == {1: 1, 2: 1}
base = await game.get_full_data()
server.custom_response(lambda: _raise(error))
alternate = await game.get_full_data()
assert base == alternate
size_counts = {}
for prof in alternate['profiles']:
for obs in prof['observations']:
assert len(obs['players']) == 4
counts = len(prof['observations'])
size_counts[counts] = size_counts.get(counts, 0) + 1
assert size_counts == {1: 1, 2: 1}
async def test_canon_game(): # pylint: disable=too-many-locals
"""Test that canon game creates proper games"""
async with mockserver.server() as server, \
api.api('', num_tries=3, retry_delay=0.5) as egta:
sim = await create_simulator(server, egta, 'sim', '1')
sim2 = await create_simulator(server, egta, 'sim', '2')
symgrps = [('a', 2, ['1']), ('b', 2, ['5', '6'])]
conf = {'key': 'val'}
game1 = await sim.get_canon_game(symgrps, conf)
summ = await game1.get_summary()
assert conf == dict(summ['configuration'])
def unpack(name, count, strategies):
"""Helper to unpack dictionaries"""
return name, count, strategies
symgrp_dict = {}
for role_info in summ['roles']:
role, count, strategies = unpack(**role_info)
symgrp_dict[role] = (role, count, set(strategies))
for role, count, strats in symgrps:
_, cnt, strt = symgrp_dict[role]
assert cnt == count
assert strt == set(strats)
game2 = await egta.get_canon_game(sim['id'], symgrps, {'key': 'diff'})
assert game1['id'] != game2['id']
game3 = await egta.get_canon_game(sim['id'], [('a', 2, ['1']),
('b', 2, ['5', '7'])],
conf)
assert game1['id'] != game3['id']
game4 = await egta.get_canon_game(sim['id'], [('a', 2, ['1']),
('b', 1, ['5', '6'])],
conf)
assert game1['id'] != game4['id']
game5 = await egta.get_canon_game(sim2['id'], symgrps, conf)
assert game1['id'] != game5['id']
game6 = await egta.get_canon_game(sim['id'], symgrps, conf)
assert game1['id'] == game6['id']
async def test_exceptions():
"""Test that exceptions can be properly set"""
async with mockserver.server() as server, \
api.api('', num_tries=3, retry_delay=0.5) as egta:
sim = await create_simulator(server, egta, 'sim', '1')
await sim.add_strategies({'role': ['strategy']})
sched = await sim.create_generic_scheduler('sched', True, 0, 1, 0, 0)
await sched.add_role('role', 1)
prof = await sched.add_profile('role: 1 strategy', 1)
game = await sched.create_game('game')
await game.add_symgroup('role', 1, ['strategy'])
server.custom_response(lambda: _raise(TimeoutError), 11)
# Creations fail
with pytest.raises(TimeoutError):
await sim.create_generic_scheduler('sched_2', False, 0, 0, 0, 0)
with pytest.raises(TimeoutError):
await sched.create_game()
# Infos fail
with pytest.raises(TimeoutError):
await sim.get_info()
with pytest.raises(TimeoutError):
await sched.get_info()
with pytest.raises(TimeoutError):
await game.get_structure()
with pytest.raises(TimeoutError):
await prof.get_structure()
# Mutates fail
with pytest.raises(TimeoutError):
await sim.add_role('r')
with pytest.raises(TimeoutError):
await sim.add_strategy('role', 's')
with pytest.raises(TimeoutError):
await sched.add_role('r', 1)
with pytest.raises(TimeoutError):
await game.add_role('r', 1)
with pytest.raises(TimeoutError):
await game.add_strategy('role', 's')
# Succeed after done
assert sim['id'] == (await sim.get_info())['id']
async def test_scheduler_deactivate(game):
"""Test that scheduler ends when deactivated"""
async with mockserver.server() as server, api.api() as egta:
sim = await egta.get_simulator(server.create_simulator( # pragma: no branch pylint: disable=line-too-long
'sim', '1', delay_dist=lambda: random.random() / 10))
strats = dict(zip(game.role_names, game.strat_names))
symgrps = list(zip(game.role_names, game.num_role_players,
game.strat_names))
await sim.add_strategies(strats)
egame = await egta.get_canon_game(sim['id'], symgrps)
# Schedule all new profiles and verify it works
# This first time should have to wait to schedule more
async with eosched.eosched(
game, egta, egame['id'], 0.1, 1, 10, 0, 0) as sched:
# Deactivate scheduler
for esched in await egta.get_generic_schedulers():
await esched.deactivate()
with pytest.raises(ValueError):
await sched.sample_payoffs(game.random_profile())
async def test_exception_in_schedule(game):
"""Test exception throwin in scheduler"""
async with mockserver.server() as server, api.api() as egta:
sim = await egta.get_simulator(server.create_simulator( # pragma: no branch pylint: disable=line-too-long
'sim', '1', delay_dist=lambda: random.random() / 10))
strats = dict(zip(game.role_names, game.strat_names))
symgrps = list(zip(game.role_names, game.num_role_players,
game.strat_names))
await sim.add_strategies(strats)
egame = await egta.get_canon_game(sim['id'], symgrps)
prof = game.random_profile()
async with eosched.eosched(
game, egta, egame['id'], 0.1, 1, 25, 0, 0) as sched:
# so that enough calls to get_requirements are made
server.custom_response(lambda: _raise(TimeoutError))
await asyncio.sleep(0.1)
with pytest.raises(TimeoutError):
await sched.sample_payoffs(prof)
async def test_exception_in_get(game):
"""Test exception in await"""
async with mockserver.server() as server, api.api() as egta:
sim = await egta.get_simulator(server.create_simulator(
'sim', '1', delay_dist=lambda: random.random() / 10))
strats = dict(zip(game.role_names, game.strat_names))
symgrps = list(zip(game.role_names, game.num_role_players,
game.strat_names))
await sim.add_strategies(strats)
egame = await egta.get_canon_game(sim['id'], symgrps)
profs = game.random_profiles(20)
async with eosched.eosched(
game, egta, egame['id'], 0.1, 1, 10, 0, 0) as sched:
futures = asyncio.gather(*[
sched.sample_payoffs(p) for p in profs])
await asyncio.sleep(0.1)
server.custom_response(lambda: _raise(TimeoutError))
await asyncio.sleep(0.1)
with pytest.raises(TimeoutError):
await futures
async def test_get_schedulers():
"""Test getting schedulers"""
async with mockserver.server() as server, \
api.api('', num_tries=3, retry_delay=0.5) as egta:
sim = await create_simulator(server, egta, 'sim', '1')
await sim.create_generic_scheduler('1', False, 0, 10, 0, 0)
sched2 = await egta.create_generic_scheduler(sim['id'], '2', False, 0,
10, 0, 0)
sched3 = await sim.create_generic_scheduler('3', False, 0, 10, 0, 0)
await sim.create_generic_scheduler('4', False, 0, 10, 0, 0)
await sim.create_generic_scheduler('5', False, 0, 10, 0, 0)
with pytest.raises(requests.exceptions.HTTPError):
await sim.create_generic_scheduler('4', False, 0, 10, 0, 0)
sched = await egta.get_scheduler(2)
assert sched['id'] == sched3['id']
sched = await egta.get_scheduler_name('3')
assert sched['id'] == sched3['id']
assert sum(1 for _ in await egta.get_generic_schedulers()) == 5
assert {0, 1, 2, 3,
4} == {s['id']
for s in await egta.get_generic_schedulers()}
await sched2.destroy_scheduler()
await sched3.destroy_scheduler()
assert sum(1 for _ in await egta.get_generic_schedulers()) == 3
assert {0, 3,
4} == {s['id']
for s in await egta.get_generic_schedulers()}
with pytest.raises(requests.exceptions.HTTPError):
await egta.get_scheduler(5)
with pytest.raises(requests.exceptions.HTTPError):
await egta.get_scheduler(2)
with pytest.raises(AssertionError):
await egta.get_scheduler_name('3')
async def test_sims():
"""Test getting simulations"""
async with mockserver.server() as server:
with stdout() as out, stderr() as err:
assert await run('-a', '', 'sched'), err.getvalue()
assert not out.getvalue()
sim_id = server.create_simulator('sim', '1')
async with api.api() as egta:
sim = await egta.get_simulator(sim_id)
await sim.add_strategies({'r': ['s0', 's1']})
sched = await egta.create_generic_scheduler(
sim_id, 'sched', True, 1, 2, 1, 1)
await sched.add_role('r', 2)
# This fails because we don't implement search, so this is as if no
# job exists
assert not await run('-a', '', 'sims', '-j', '0')
await sched.add_profile('r: 1 s0, 1 s1', 1)
# This works because there's only one simulation so far and we
# don't implement search
with stdout() as out, stderr() as err:
assert await run('-a', '', 'sims', '-j', '0'), err.getvalue()
await sched.add_profile('r: 2 s0', 2)
# This fails because we don't implement search and now there are
# several simulations
assert not await run('-a', '', 'sims', '-j', '0')
with stdout() as out, stderr() as err:
assert await run('-a', '', 'sims'), err.getvalue()
sims = [json.loads(line) for line in out.getvalue()[:-1].split('\n')]
assert len(sims) == 3
with stderr() as err:
assert await run('-a', '', 'sims',
str(sims[0]['folder'])), err.getvalue()
async def test_sched_running():
"""Test running scheduler functionality"""
async with mockserver.server() as server:
# verify no schedulers
with stdout() as out, stderr() as err:
assert await run('-a', '', 'sched'), err.getvalue()
assert not out.getvalue()
# create one
sim_id = server.create_simulator('sim', '1', delay_dist=lambda: 1)
async with api.api() as egta:
sim = await egta.get_simulator(sim_id)
await sim.add_strategies({'r': ['s0', 's1']})
sched = await egta.create_generic_scheduler(
sim_id, 'sched', True, 1, 2, 1, 1)
await sched.add_role('r', 2)
with stdout() as out, stderr() as err:
assert await run('-a', '', 'sched',
'--running'), err.getvalue()
assert not out.getvalue()
await sched.add_profile('r: 1 s0, 1 s1', 1)
with stdout() as out, stderr() as err:
assert await run('-a', '', 'sched',
'--running'), err.getvalue()
lines = out.getvalue()[:-1].split('\n')
assert len(lines) == 1
running = json.loads(lines[0])
assert running['active']
# Wait to complete
await asyncio.sleep(1.5)
with stdout() as out, stderr() as err:
assert await run('-a', '', 'sched',
'--running'), err.getvalue()
assert not out.getvalue()
async def amain(*argv): # pylint: disable=too-many-statements
"""Entry point for async cli with args"""
parser = argparse.ArgumentParser(
description="""Command line access to egta online apis. This works well
in concert with `jq`.""")
parser.add_argument(
'--verbose', '-v', action='count', default=0, help="""Sets the
verbosity of commands. Output is send to standard error.""")
parser.add_argument(
'--version', '-V', action='version',
version='%(prog)s {}'.format(egtaonline.__version__))
parser_auth = parser.add_mutually_exclusive_group()
parser_auth.add_argument(
'--auth-string', '-a', metavar='<auth-string>', help="""The string
authorization token to connect to egta online.""")
parser_auth.add_argument(
'--auth-file', '-f', metavar='<auth-file>', type=argparse.FileType(),
help="""Filename that just contains the string of the auth token.""")
subparsers = parser.add_subparsers(title='Subcommands', dest='command')
subparsers.required = True
parser_sim = subparsers.add_parser(
'sim', help="""Get information on or modify a simulator.""",
description="""Operate on EGTA simulators. By defualt this will return
information about each simulator on its own line.""")
parser_sim.add_argument(
'sim_id', metavar='sim-id', nargs='?', help="""Get information from a
specific simulator instead of all of them.""")
parser_sim.add_argument(
'--sim-version', '-n', metavar='<version-name>', help="""If this is
specified then the `sim-id` is treated as the name of the simulator and
the supplied argument is treated as the version.""")
parser_sim.add_argument(
'--json', '-j', metavar='json-file', nargs='?',
type=argparse.FileType('r'), help="""Modify simulator using the
specified json file. By default this will add all of the roles and
strategies in the json file. If `delete` is specified, this will remove
only the strategies specified in the file. `-` can be used to read from
stdin.""")
parser_sim.add_argument(
'--role', '-r', metavar='<role>', help="""Modify `role` of the
simulator. By default this will add `role` to the simulator. If
`delete` is specified this will remove `role` instead. If `strategy` is
specified see strategy.""")
parser_sim.add_argument(
'--strategy', '-s', metavar='<strategy>', help="""Modify `strategy` of
the simulator. This requires that `role` is also specified. By default
this adds `strategy` to `role`. If `delete` is specified, then this
removes the strategy instead.""")
parser_sim.add_argument(
'--delete', '-d', action='store_true', help="""Triggers removal of
roles or strategies instead of addition""")
parser_sim.add_argument(
'--zip', '-z', action='store_true', help="""Download simulator zip file
to stdout.""")
parser_game = subparsers.add_parser(
'game', help="""Get information on, create, destroy, or modify a
game.""", description="""Operate on EGTA Online games. Buy default this
will return information about each game on its own line.""")
parser_game.add_argument(
'game_id', nargs='?', metavar='game-id', help="""Get data from a
specific game instead of all of them.""")
parser_game.add_argument(
'--name', '-n', action='store_true', help="""If specified then get the
game via its string name not its id number. This is much slower than
accessing via id number.""")
parser_game.add_argument(
'--fetch-conf', metavar='<configuration>', type=argparse.FileType('r'),
help="""If specified then interpret `game_id` as a simulator id and use
the file specified as a game configuration. Fetch data of the
appropriate granularity from the canonical game defined by that
simulator and this configuration. Games need specified roles and
players, these will be pulled from `--json` which must have two top
level entries, `players` and `strategies` which list the number of
players per role and the strategies per role respectively`. The
configuration will not be updated with simulator defaults, so it may be
helpful to call `sim` first to get those.""")
parser_game.add_argument(
'--json', '-j', metavar='json-file', nargs='?',
type=argparse.FileType('r'), help="""Modify game using the specified
json file. By default this will add all of the roles and strategies in
the json file. If `delete` is specified, this will remove only the
strategies specified in the file.""")
parser_game.add_argument(
'--role', '-r', metavar='<role>', help="""Modify `role` of the game. By
default this will add `role` to the game. If `delete` is specified this
will remove `role` instead. If `strategy` is specified see
strategy.""")
parser_game.add_argument(
'--count', '-c', metavar='<count>', help="""If adding a role, the count
of the role must be specified.""")
parser_game.add_argument(
'--strategy', '-s', metavar='<strategy>', help="""Modify `strategy` of
the game. This requires that `role` is also specified. By default this
adds `strategy` to `role`. If `delete` is specified, then this removes
the strategy instead.""")
parser_game.add_argument(
'--delete', '-d', action='store_true', help="""Triggers removal of
roles or strategies instead of addition""")
parser_gran = (parser_game.add_argument_group('data granularity')
.add_mutually_exclusive_group())
parser_gran.add_argument(
'--structure', action='store_true', help="""Return game information but
no profile information. (default)""")
parser_gran.add_argument(
'--summary', action='store_true', help="""Return profiles with
aggregated payoffs.""")
parser_gran.add_argument(
'--observations', action='store_true', help="""Return profiles with
observation data.""")
parser_gran.add_argument(
'--full', action='store_true', help="""Return all collected payoff
data.""")
parser_sched = subparsers.add_parser(
'sched', help="""Get information on, create, destroy, or modify a
scheduler.""", description="""Operate on EGTA Online schedulers. By
default this will return information about each generic scheduler on
its own line.""")
parser_sched.add_argument(
'--running', action='store_true', help="""If specified, filter
schedulers by ones that are currently running simulations. To be
running simulations, a scheduler has to be active, and have at least
one profile whose current count is less than the requirement for that
scheduler.""")
parser_sched.add_argument(
'sched_id', nargs='?', metavar='sched-id', help="""Get information
about a specific scheduler. This scheduler doesn't need to be generic,
but to operate on it, it must be.""")
parser_sched.add_argument(
'--name', '-n', action='store_true', help="""If specified then get the
scheduler via its string name not its id number. This is much slower
than accessing via id number, and only works for generic
schedulers.""")
parser_act = (parser_sched.add_argument_group('scheduler action')
.add_mutually_exclusive_group())
parser_act.add_argument(
'--requirements', '-r', action='store_true', help="""Get scheuler
requirements instead of just information.""")
parser_act.add_argument(
'--deactivate', action='store_true', help="""Deactivate the specified
scheduler.""")
parser_act.add_argument(
'--delete', '-d', action='store_true', help="""If specified with a
scheduler, delete it.""")
# TODO Specifying a flag to change from folder to job id is awkward
parser_sims = subparsers.add_parser(
'sims', help="""Get information about pending, scheduled, queued,
complete, or failed simulations.""", description="""Get information
about EGTA Online simulations. These are the actual scheduled
simulations instead of the simulators that generate them. If no folder
is specified, each simulation comes out on a different line, and can be
easily filtered with `head` and `jq`.""")
parser_sims.add_argument(
'folder', metavar='folder-id', nargs='?', type=int, help="""Get
information from a specific simulation instead of all of them. This
should be the folder number of the simulation of interest.""")
parser_sims.add_argument(
'-j', '--job', action='store_true', help="""Fetch the simulation with a
given PBS job id instead of its folder number.""")
parser_sims.add_argument(
'--page', '-p', metavar='<start-page>', default=1, type=int, help="""The
page to start scanning at. (default: %(default)d)""")
parser_sims.add_argument(
'--ascending', '-a', action='store_true', help="""Return results in
ascending order instead of descending.""")
parser_sims.add_argument(
'--sort-column', '-s', choices=('job', 'folder', 'profile', 'state'),
default='job', help="""Column to order results by. (default:
%(default)s)""")
parser_sims.add_argument(
'--search', default='', metavar='<search-string>', help="""The string
to filter results by. See egtaonline for examples of what this can
be.""")
parser_sims.add_argument(
'--state',
choices=['canceled', 'complete', 'failed', 'processing', 'queued',
'running'],
help="""Only select jobs with a specific state.""")
parser_sims.add_argument(
'--profile', metavar='<profile-substring>', help="""Limit results to
simulations whose profiles contain the supplied substring.""")
parser_sims.add_argument(
'--simulator', metavar='<sim-substring>', help="""Limit results to
simulations where the simulator fullname contains the supplied
substring.""")
parser_login = subparsers.add_parser(
'login', help="""Login to egtaonline by fetching your auth token.""",
description="""Download an authtoken to allow requests to
egtaonline.""")
parser_login.add_argument(
'email', metavar='<email>', help="""Email address of the auther to
login as""")
parser_login.add_argument(
'-u', '--user', action='store_true', help="""Store the auth_token in
the user directory instead of the local directory.""")
args = parser.parse_args(argv)
if args.command == 'login':
auth.login(args.email, int(args.user))
return
if args.auth_string is None and args.auth_file is not None:
args.auth_string = args.auth_file.read().strip()
logging.basicConfig(stream=sys.stderr,
level=30 - 10 * min(args.verbose, 2))
async with api.api(args.auth_string) as eoapi:
if args.command == 'sim':
return await _sim(eoapi, args)
elif args.command == 'game':
return await _game(eoapi, args)
elif args.command == 'sched':
return await _sched(eoapi, args)
elif args.command == 'sims':
return await _sims(eoapi, args)
else:
assert False # pragma: no cover
async def test_parity(): # pylint: disable=too-many-locals
"""Test that egta matches mock server"""
# Get egta data
async with api.api() as egta:
true_sim = await egta.get_simulator(183)
true_sched = await egta.get_scheduler(4997)
true_game = await egta.get_game(2536)
reqs = await true_sched.get_requirements()
async def get_prof_info(prof):
"""Get all info about a profile"""
return (await prof.get_structure(), await prof.get_summary(), await
prof.get_observations(), await prof.get_full_data())
prof_info = await asyncio.gather(
*[get_prof_info(prof) for prof in reqs['scheduling_requirements']])
game_info = await true_game.get_structure()
game_summ = await true_game.get_summary()
# Replicate in mock
async with mockserver.server() as server, api.api() as egta:
for i in range(true_sim['id']):
server.create_simulator('sim', str(i))
mock_sim = await egta.get_simulator(
server.create_simulator(true_sim['name'], true_sim['version'],
true_sim['email'],
true_sim['configuration']))
await mock_sim.add_strategies(true_sim['role_configuration'])
info = await mock_sim.get_info()
assert_dicts_types(true_sim, info)
assert_dicts_equal(true_sim, info)
await asyncio.gather(*[
mock_sim.create_generic_scheduler(str(i), False, 0, 0, 0, 0)
for i in range(true_sched['id'])
])
mock_sched = await mock_sim.create_generic_scheduler(
true_sched['name'], true_sched['active'],
true_sched['process_memory'], true_sched['size'],
true_sched['time_per_observation'],
true_sched['observations_per_simulation'], true_sched['nodes'],
dict(reqs['configuration']))
info = await mock_sched.get_info()
assert_dicts_types(true_sched, info)
assert_dicts_equal(true_sched, info)
game_sched = await mock_sim.create_generic_scheduler(
'temp', True, 0, true_sched['size'], 0, 0, 1,
dict(reqs['configuration']))
counts = prof_info[0][0]['role_configuration']
await mock_sched.add_roles(counts)
await game_sched.add_roles(counts)
await mock_sched.activate()
for prof, (info, summ, obs,
full) in zip(reqs['scheduling_requirements'], prof_info):
sprof = await game_sched.add_profile(info['assignment'],
prof['current_count'])
mprof = await mock_sched.add_profile(info['assignment'],
prof['requirement'])
await sched_complete(game_sched)
await sched_complete(mock_sched)
assert ((await sprof.get_structure()
)['observations_count'] == prof['current_count'])
assert sprof['id'] == mprof['id']
struct = await mprof.get_structure()
assert_dicts_types(info, struct)
assert_dicts_equal(info, struct, {'id'})
assert_dicts_types(summ, (await mprof.get_summary()), (), True)
assert_dicts_types(obs, (await mprof.get_observations()),
{'extended_features', 'features'}, True)
assert_dicts_types(full, (await mprof.get_full_data()),
{'extended_features', 'features', 'e', 'f'},
True)
mock_reqs = await mock_sched.get_requirements()
assert_dicts_types(mock_reqs, reqs)
await asyncio.gather(
*[mock_sim.create_game(str(i), 0) for i in range(true_game['id'])])
mock_game = await mock_sim.create_game(
true_game['name'], true_game['size'],
dict(game_summ['configuration']))
info = await mock_game.get_structure()
assert_dicts_types(game_info, info)
assert_dicts_equal(game_info, info)
symgrps = [(grp['name'], grp['count'], grp['strategies'])
for grp in game_summ['roles']]
await mock_game.add_symgroups(symgrps)
# Schedule next profiles
await asyncio.gather(*[
game_sched.add_profile(prof['symmetry_groups'],
prof['observations_count'])
for prof in game_summ['profiles']
])
await sched_complete(game_sched)
mock_summ = await mock_game.get_summary()
assert_dicts_types(game_summ, mock_summ, (), True)
async def test_simulator():
"""Test simulator api"""
async with mockserver.server() as server, \
api.api('', num_tries=3, retry_delay=0.5) as egta:
sim = await create_simulator(server, egta, 'sim', '1')
info = await sim.get_info()
validate_object(
info, {
'configuration': {
'type': 'object'
},
'created_at': {
'type': 'string'
},
'email': {
'type': 'string'
},
'id': {
'type': 'integer'
},
'name': {
'type': 'string'
},
'role_configuration': {
'type': 'object'
},
'source': {
'type': 'object'
},
'updated_at': {
'type': 'string'
},
'url': {
'type': 'string'
},
'version': {
'type': 'string'
},
})
role_conf = {'a': ['1', '2', '3', '4'], 'b': ['5', '6', '7']}
assert info['role_configuration'] == role_conf
await asyncio.sleep(1)
await sim.remove_strategy('a', '3')
new_role_conf = {'a': ['1', '2', '4'], 'b': ['5', '6', '7']}
new_info = await sim.get_info()
assert new_info['role_configuration'] == new_role_conf
assert new_info['updated_at'] != info['updated_at']
assert info['role_configuration'] == role_conf
await sim.remove_role('b')
new_role_conf = {'a': ['1', '2', '4']}
new_info = await sim.get_info()
assert new_info['role_configuration'] == new_role_conf
# Stale object didn't update
assert info['role_configuration'] == role_conf
# Add existing strategy
await sim.add_strategy('a', '1')
new_info = await sim.get_info()
assert new_info['role_configuration'] == new_role_conf
await sim.add_strategy('a', '2')
await sim.add_strategy('a', '3')
new_role_conf = {'a': ['1', '2', '3', '4']}
new_info = await sim.get_info()
assert new_info['role_configuration'] == new_role_conf
await sim.remove_strategies({'a': ['4', '5', '4']})
new_role_conf = {'a': ['1', '2', '3']}
new_info = await sim.get_info()
assert new_info['role_configuration'] == new_role_conf
await sim.remove_role('c')
with pytest.raises(KeyError):
await sim.add_strategy('c', '8')
# Shouldn't raise exception, because removals never do
await sim.remove_strategies({'c': ['8']})
async def test_get_simulations():
"""Test getting simulations"""
async with mockserver.server() as server, \
api.api('', num_tries=3, retry_delay=0.5) as egta:
assert not await agather(egta.get_simulations())
sim1 = await create_simulator(server, egta, 'sim', '1')
sched1 = await sim1.create_generic_scheduler('sched1', True, 0, 4, 0,
0)
await sched1.add_roles({'a': 2, 'b': 2})
await sched1.add_profile('a: 2 1; b: 1 6, 1 7', 2)
assert len(await agather(egta.get_simulations())) == 2
simul = next(iter(await agather(egta.get_simulations())))
validate_object(
simul, {
'folder': {
'type': 'integer'
},
'job': {
'type': 'number'
},
'profile': {
'type': 'string'
},
'simulator': {
'type': 'string'
},
'state': {
'type': 'string'
},
})
validate_object(
await egta.get_simulation(simul['folder']), {
'error_message': {
'type': 'string'
},
'folder_number': {
'type': 'integer'
},
'job': {
'type': 'string'
},
'profile': {
'type': 'string'
},
'simulator_fullname': {
'type': 'string'
},
'size': {
'type': 'integer'
},
'state': {
'type': 'string'
},
})
sim2 = await create_simulator(server, egta, 'sim', '2')
sched2 = await sim2.create_generic_scheduler('sched2', True, 0, 5, 0,
0)
await sched2.add_roles({'a': 2, 'b': 3})
await sched2.add_profile('a: 2 1; b: 1 5, 2 7', 3)
assert len(await agather(egta.get_simulations())) == 5
# Test simulations
assert is_sorted( # pragma: no branch
(f['simulator']
for f in await agather(egta.get_simulations(column='simulator'))),
reverse=True)
assert is_sorted( # pragma: no branch
(f['folder']
for f in await agather(egta.get_simulations(column='folder'))),
reverse=True)
assert is_sorted( # pragma: no branch
(f['profile']
for f in await agather(egta.get_simulations(column='profile'))),
reverse=True)
assert is_sorted( # pragma: no branch
(f['state']
for f in await agather(egta.get_simulations(column='state'))),
reverse=True)
assert is_sorted( # pragma: no branch
f['simulator'] for f in await agather(
egta.get_simulations(asc=True, column='simulator')))
assert is_sorted( # pragma: no branch
f['folder'] for f in await agather(
egta.get_simulations(asc=True, column='folder')))
assert is_sorted( # pragma: no branch
f['profile'] for f in await agather(
egta.get_simulations(asc=True, column='profile')))
assert is_sorted( # pragma: no branch
f['state'] for f in await agather(
egta.get_simulations(asc=True, column='state')))
assert not await agather(egta.get_simulations(page_start=2))
await sched2.add_profile('a: 2 1; b: 1 5, 2 6', 21)
assert len(await agather(egta.get_simulations(page_start=2))) == 1
