async def test_modify_simulator(): """Test that we can modify a simulator""" # This is very dangerous because we're just finding and modifying a random # simulator. However, adding and removing a random role shouldn't really # affect anything, so this should be fine async with api.api() as egta: role = '__unique_role__' strat1 = '__unique_strategy_1__' strat2 = '__unique_strategy_2__' # Old sims seem frozen so > 100 for sim in await egta.get_simulators(): # pragma: no branch if sim['id'] > 100: break else: # pragma: no cover sim = None raise AssertionError('no simulators') try: await sim.add_strategies({role: [strat1]}) assert ((await sim.get_info())['role_configuration'][role] == [strat1]) await sim.add_strategy(role, strat2) expected = [strat1, strat2] assert ((await sim.get_info())['role_configuration'][role] == expected) await sim.remove_strategies({role: [strat1]}) assert ((await sim.get_info())['role_configuration'][role] == [strat2]) finally: await sim.remove_role(role) assert role not in (await sim.get_info())['role_configuration']
async def test_profile_json_error(): """Test invalid profile json triggers retry""" async with mockserver.server() as server, \ api.api('', num_tries=3, retry_delay=0.5) as egta: sim = await create_simulator(server, egta, 'sim', '1') sched = await sim.create_generic_scheduler('sched', True, 0, 4, 0, 0) await sched.add_roles({'a': 2, 'b': 2}) game = await sched.create_game() await game.add_symgroups([('a', 2, ['1']), ('b', 2, ['5', '6'])]) prof = await sched.add_profile('a: 2 1; b: 2 5', 2) await sched_complete(sched) server.custom_response(lambda: '', 2) summ = await prof.get_summary() assert 'id' in summ assert 'observations_count' in summ assert 'symmetry_groups' in summ server.custom_response(lambda: '', 3) with pytest.raises(json.decoder.JSONDecodeError): await prof.get_summary() server.custom_response(lambda: '{}', 3) with pytest.raises(jsonschema.ValidationError): await prof.get_summary()
async def test_game_json_error(): """Test returning invalid json in games triggers retry""" async with mockserver.server() as server, \ api.api('', num_tries=3, retry_delay=0.5) as egta: sim = await create_simulator(server, egta, 'sim', '1') sched = await sim.create_generic_scheduler('sched', True, 0, 4, 0, 0) await sched.add_roles({'a': 2, 'b': 2}) game = await sched.create_game() await game.add_symgroups([('a', 2, ['1']), ('b', 2, ['5', '6'])]) await sched.add_profile('a: 2 1; b: 1 5, 1 6', 1) await sched.add_profile('a: 2 1; b: 2 5', 2) await sched_complete(sched) server.custom_response(lambda: '', 2) summ = await game.get_summary() size_counts = {} for prof in summ['profiles']: counts = prof['observations_count'] size_counts[counts] = size_counts.get(counts, 0) + 1 assert size_counts == {1: 1, 2: 1} server.custom_response(lambda: '', 3) with pytest.raises(json.decoder.JSONDecodeError): await game.get_summary() server.custom_response(lambda: '{}', 3) with pytest.raises(jsonschema.ValidationError): await game.get_summary()
async def test_get_simulators(): """Test getting simulators""" async with mockserver.server() as server, \ api.api('', num_tries=3, retry_delay=0.5) as egta: sim1 = server.create_simulator('foo', '1') sim2 = server.create_simulator('bar', '1') sim3 = server.create_simulator('bar', '2') assert sum(1 for _ in await egta.get_simulators()) == 3 assert {0, 1, 2} == {s['id'] for s in await egta.get_simulators()} sim = await egta.get_simulator(0) assert sim['id'] == sim1 sim = await egta.get_simulator_fullname('foo-1') assert sim['id'] == sim1 sim = await egta.get_simulator(2) assert sim['id'] == sim3 sim = await egta.get_simulator_fullname('bar-1') assert sim['id'] == sim2 sim = await egta.get_simulator_fullname('bar-2') assert sim['id'] == sim3 with pytest.raises(requests.exceptions.HTTPError): await egta.get_simulator(3) with pytest.raises(AssertionError): await egta.get_simulator_fullname('baz')
async def test_get_games(): """Test getting games""" async with mockserver.server() as server, \ api.api('', num_tries=3, retry_delay=0.5) as egta: sim = await create_simulator(server, egta, 'sim', '1') await sim.create_game('a', 5) game2 = await egta.create_game(sim['id'], 'b', 6) game3 = await sim.create_game('c', 3) with pytest.raises(requests.exceptions.HTTPError): await sim.create_game('b', 3) assert (await egta.get_game(1))['id'] == game2['id'] assert (await egta.get_game_name('c'))['id'] == game3['id'] assert sum(1 for _ in await egta.get_games()) == 3 assert {0, 1, 2} == {g['id'] for g in await egta.get_games()} await game2.destroy_game() assert sum(1 for _ in await egta.get_games()) == 2 assert {0, 2} == {g['id'] for g in await egta.get_games()} with pytest.raises(requests.exceptions.HTTPError): await egta.get_game(3) with pytest.raises(requests.exceptions.HTTPError): await egta.get_game(1) with pytest.raises(AssertionError): await egta.get_game_name('b')
async def test_exception_in_get(game): """Test exception in await""" async with mockserver.server() as server, api.api() as egta: sim = await egta.get_simulator( server.create_simulator("sim", "1", delay_dist=lambda: random.random() / 10)) strats = dict(zip(game.role_names, game.strat_names)) symgrps = list( zip(game.role_names, game.num_role_players, game.strat_names)) await sim.add_strategies(strats) egame = await egta.get_canon_game(sim["id"], symgrps) profs = game.random_profiles(20) async with eosched.eosched(game, egta, egame["id"], 0.1, 1, 10, 0, 0) as sched: tasks = [ asyncio.ensure_future(sched.sample_payoffs(p)) for p in profs ] await asyncio.sleep(0.1) server.custom_response(lambda: _raise(TimeoutError)) await asyncio.sleep(0.1) with pytest.raises(TimeoutError): await asyncio.gather(*tasks) # tidy up errors = asyncio.gather(*tasks, return_exceptions=True) errors.cancel() with contextlib.suppress(TimeoutError, asyncio.CancelledError): await errors
async def test_exception_open(): """Test that exceptions are even thrown on open""" async with mockserver.server() as server: server.custom_response(lambda: _raise(TimeoutError)) with pytest.raises(TimeoutError): async with api.api(''): pass # pragma: no cover
async def test_modify_scheduler(): """Test that we can modify a scheduler""" async with api.api() as egta: for sim in await egta.get_simulators(): # pragma: no branch if next(iter(sim['role_configuration'].values()), None): # pragma: no branch pylint: disable=line-too-long break else: # pragma: no cover sim = None raise AssertionError('no simulators') sched = game = None try: sched = await sim.create_generic_scheduler('__unique_scheduler__', False, 0, 1, 0, 0) await sched.activate() await sched.deactivate() role = next(iter(sim['role_configuration'])) strat = sim['role_configuration'][role][0] symgrps = [{'role': role, 'strategy': strat, 'count': 1}] assignment = api.symgrps_to_assignment(symgrps) await sched.add_role(role, 1) reqs = (await sched.get_requirements())['scheduling_requirements'] assert not reqs prof = await sched.add_profile(symgrps, 1) reqs = (await sched.get_requirements())['scheduling_requirements'] assert len(reqs) == 1 assert reqs[0]['requirement'] == 1 await sched.remove_profile(prof['id']) assert (await sched.add_profile(assignment, 2))['id'] == prof['id'] reqs = (await sched.get_requirements())['scheduling_requirements'] assert len(reqs) == 1 assert reqs[0]['requirement'] == 2 await sched.remove_profile(prof['id']) reqs = (await sched.get_requirements())['scheduling_requirements'] assert not reqs assert (await sched.add_profile(symgrps, 3))['id'] == prof['id'] reqs = (await sched.get_requirements())['scheduling_requirements'] assert len(reqs) == 1 assert reqs[0]['requirement'] == 3 await sched.remove_all_profiles() reqs = (await sched.get_requirements())['scheduling_requirements'] assert not reqs await sched.remove_role(role) game = await sched.create_game() finally: if sched is not None: # pragma: no branch await sched.destroy_scheduler() if game is not None: # pragma: no branch await game.destroy_game()
async def test_delayed_profiles(): """Test mock server with delayed profile scheduling""" async with mockserver.server() as server, api.api('') as egta: sim = await egta.get_simulator( server.create_simulator('sim', '1', delay_dist=lambda: 0.5)) await sim.add_strategies({'1': ['a'], '2': ['b', 'c']}) sched = await sim.create_generic_scheduler('sched', True, 0, 10, 0, 0) await sched.add_roles({'1': 8, '2': 2}) prof = await sched.add_profile('1: 8 a; 2: 1 b, 1 c', 3) reqs = (await sched.get_requirements())['scheduling_requirements'] assert len(reqs) == 1 assert reqs[0]['current_count'] == 0 assert reqs[0]['requirement'] == 3 assert reqs[0]['id'] == prof['id'] count = 0 async for sim in egta.get_simulations(): assert sim['state'] == 'running' count += 1 assert count == 3 await asyncio.sleep(0.5) await sched_complete(sched) reqs = (await sched.get_requirements())['scheduling_requirements'] assert len(reqs) == 1 assert reqs[0]['current_count'] == 3 assert reqs[0]['requirement'] == 3 assert reqs[0]['id'] == prof['id'] count = 0 async for sim in egta.get_simulations(): assert sim['state'] == 'complete' count += 1 assert count == 3 # Test that extra sims get killed async with mockserver.server() as server, api.api('') as egta: sim = await egta.get_simulator( server.create_simulator('sim', '1', delay_dist=lambda: 10)) await sim.add_strategies({'1': ['a'], '2': ['b', 'c']}) sched = await sim.create_generic_scheduler('sched', True, 0, 10, 0, 0) await sched.add_roles({'1': 8, '2': 2}) # Need two profiles here because the other will be in the queue await sched.add_profile('1: 8 a; 2: 1 b, 1 c', 1) await sched.add_profile('1: 8 a; 2: 2 b', 1)
async def test_threading(): """Test that no errors arise when multi-threading""" async with mockserver.server() as server, \ api.api('', num_tries=3, retry_delay=0.5) as egta: sim = await create_simulator(server, egta, 'sim', '1') await asyncio.gather(*[ sim.add_strategies({'r{:d}'.format(i): ['s{:d}'.format(i)]}) for i in range(10) ])
async def get_eosched(server, game, name): """Get an eo sched""" async with api.api() as egta: sim = await egta.get_simulator(server.create_simulator( name, '1', delay_dist=lambda: random.random() / 100)) await sim.add_strategies(dict(zip(game.role_names, game.strat_names))) eogame = await sim.create_game(name, game.num_players) await eogame.add_symgroups(list(zip( game.role_names, game.num_role_players, game.strat_names))) return 'eo:game:{:d},mem:2048,time:60,sleep:0.1'.format( eogame['id'])
async def test_sched(): """Test scheduler functionality""" async with mockserver.server() as server: # verify no schedulers with stdout() as out, stderr() as err: assert await run('-a', '', 'sched'), err.getvalue() assert not out.getvalue() # create one sim_id = server.create_simulator('sim', '1') async with api.api() as egta: await egta.create_generic_scheduler(sim_id, 'sched', True, 1, 2, 1, 1) with stdout() as out, stderr() as err: assert await run('-a', '', 'sched'), err.getvalue() sched = json.loads(out.getvalue()) with stderr() as err: assert await run('-a', '', 'sched', str(sched['id'])), err.getvalue() with stderr() as err: assert await run('-a', '', 'sched', str(sched['id']), '-r'), err.getvalue() with stdout() as out, stderr() as err: assert await run('-a', '', 'sched', sched['name'], '-n'), err.getvalue() assert sched['id'] == json.loads(out.getvalue())['id'] # deactivate scheduler with stderr() as err: assert await run('-a', '', 'sched', str(sched['id']), '--deactivate'), err.getvalue() # verify deactivated with stdout() as out, stderr() as err: assert await run('-a', '', 'sched', str(sched['id'])), err.getvalue() assert not json.loads(out.getvalue())['active'] # delete scheduler with stderr() as err: assert await run('-a', '', 'sched', str(sched['id']), '-d'), err.getvalue() # assert no schedulers with stdout() as out, stderr() as err: assert await run('-a', '', 'sched'), err.getvalue() assert not out.getvalue()
async def test_basic_profile(game): """Test scheduling a standard profile""" async with mockserver.server() as server, api.api() as egta: sim = await egta.get_simulator( server.create_simulator("sim", "1", delay_dist=lambda: random.random() / 10)) strats = dict(zip(game.role_names, game.strat_names)) symgrps = list( zip(game.role_names, game.num_role_players, game.strat_names)) await sim.add_strategies(strats) egame = await egta.get_canon_game(sim["id"], symgrps) profs = game.random_profiles(20) # Schedule all new profiles and verify it works async with eosched.eosched(game, egta, egame["id"], 0.1, 1, 10, 0, 0) as sched: assert str(sched) == str(egame["id"]) assert game == rsgame.empty_copy(sched) awaited = await asyncio.gather( *[sched.sample_payoffs(p) for p in profs]) pays = np.stack(awaited) assert np.allclose(pays[profs == 0], 0) # Schedule old profiles and verify it still works async with eosched.eosched(game, egta, egame["id"], 0.1, 1, 10, 0, 0) as sched: awaited = await asyncio.gather( *[sched.sample_payoffs(p) for p in profs]) pays = np.stack(awaited) assert np.allclose(pays[profs == 0], 0) # Schedule two at a time, in two batches async with eosched.eosched(game, egta, egame["id"], 0.1, 2, 10, 0, 0) as base_sched: sched = countsched.countsched(base_sched, 2) awaited = await asyncio.gather( *[sched.sample_payoffs(p) for p in profs]) pays = np.stack(awaited) assert np.allclose(pays[profs == 0], 0) # Try again now that everything should be scheduled async with eosched.eosched(game, egta, egame["id"], 0.1, 2, 10, 0, 0) as base_sched: sched = countsched.countsched(base_sched, 2) awaited = await asyncio.gather( *[sched.sample_payoffs(p) for p in profs]) pays = np.stack(awaited) assert np.allclose(pays[profs == 0], 0)
async def test_modify_game(): """Test that we can modify a game""" async with api.api() as egta: # pragma: no branch for sim in await egta.get_simulators(): # pragma: no branch if next(iter(sim['role_configuration'].values()), None): # pragma: no branch pylint: disable=line-too-long break else: # pragma: no cover sim = None # pytest detects that it's defined raise AssertionError('no simulators') game = None try: game = await sim.create_game('__unique_game__', 1) summ = await game.get_summary() assert not summ['roles'] assert not summ['profiles'] role = next(iter(sim['role_configuration'])) strat = sim['role_configuration'][role][0] await game.add_role(role, 1) summ = await game.get_summary() assert len(summ['roles']) == 1 assert summ['roles'][0]['name'] == role assert summ['roles'][0]['count'] == 1 assert not summ['roles'][0]['strategies'] await game.add_strategies({role: [strat]}) summ = await game.get_summary() assert len(summ['roles']) == 1 assert summ['roles'][0]['name'] == role assert summ['roles'][0]['count'] == 1 assert summ['roles'][0]['strategies'] == [strat] await game.remove_strategies({role: [strat]}) summ = await game.get_summary() assert len(summ['roles']) == 1 assert summ['roles'][0]['name'] == role assert summ['roles'][0]['count'] == 1 assert not summ['roles'][0]['strategies'] await game.remove_role(role) summ = await game.get_summary() assert not summ['roles'] assert not summ['profiles'] finally: if game is not None: # pragma: no branch await game.destroy_game()
async def test_exception_in_create(game): """Test exception creating eo scheduler""" async with mockserver.server() as server, api.api() as egta: sim = await egta.get_simulator(server.create_simulator( # pragma: no branch pylint: disable=line-too-long 'sim', '1', delay_dist=lambda: random.random() / 10)) strats = dict(zip(game.role_names, game.strat_names)) symgrps = list(zip(game.role_names, game.num_role_players, game.strat_names)) await sim.add_strategies(strats) egame = await egta.get_canon_game(sim['id'], symgrps) server.custom_response(lambda: _raise(TimeoutError)) with pytest.raises(TimeoutError): async with eosched.eosched( game, egta, egame['id'], 0.1, 1, 25, 0, 0): pass # pragma: no cover
async def create_scheduler( # pylint: disable=too-many-arguments game, mem, time, auth=None, count=1, sleep=600, # pylint: disable-msg=redefined-builtin max=100): """Create an egtaonline scheduler""" game_id = int(game) mem = int(mem) time = int(time) count = int(count) sleep = float(sleep) max_sims = int(max) async with api.api(auth_token=auth) as egta: game = await egta.get_game(game_id) summ = await game.get_summary() game = rsgame.empty_json(summ) return ApiWrapper(game, egta, game_id, sleep, count, max_sims, mem, time)
async def test_gets(): """Test that get functions work""" async with api.api() as egta: with pytest.raises(AssertionError): await egta.get_simulator_fullname('this name is impossible I hope') async def test_sim_name(sim): """Verify sim name is accurate""" assert sim['id'] == (await egta.get_simulator_fullname('{}-{}'.format( sim['name'], sim['version'])))['id'] await asyncio.gather( *[test_sim_name(sim) for sim in await egta.get_simulators()]) sched = next(iter(await egta.get_generic_schedulers())) assert (await egta.get_scheduler(sched['id']))['id'] == sched['id'] assert ((await egta.get_scheduler_name(sched['name']))['id'] == sched['id']) with pytest.raises(AssertionError): await egta.get_scheduler_name('this name is impossible I hope') game = next(iter(await egta.get_games())) assert (await egta.get_game(game['id']))['id'] == game['id'] assert (await egta.get_game_name(game['name']))['id'] == game['id'] with pytest.raises(AssertionError): await egta.get_game_name('this name is impossible I hope') folds = egta.get_simulations() fold = await folds.__anext__() assert (await egta.get_simulation(fold['folder'] ))['folder_number'] == fold['folder'] for sort in ['job', 'folder', 'profile', 'simulator', 'state']: assert 'folder' in await egta.get_simulations(column=sort ).__anext__() with pytest.raises(StopAsyncIteration): await egta.get_simulations(page_start=10**9).__anext__() for sch in await egta.get_generic_schedulers(): # pragma: no branch sched = await sch.get_requirements() if sched['scheduling_requirements']: # pragma: no branch break prof = sched['scheduling_requirements'][0] assert (await egta.get_profile(prof['id']))['id'] == prof['id']
async def test_large_game_failsafes(): """Test that large games fallback to gathering profile data""" async with mockserver.server() as server, \ api.api('', num_tries=3, retry_delay=0.5) as egta: sim = await create_simulator(server, egta, 'sim', '1') error = requests.exceptions.HTTPError( '500 Server Error: Game too large!') sched = await sim.create_generic_scheduler('sched', True, 0, 4, 0, 0, configuration={'k': 'v'}) await sched.add_roles({'a': 2, 'b': 2}) game = await sched.create_game() await game.add_symgroups([('a', 2, ['1']), ('b', 2, ['5', '6'])]) await sched.add_profile('a: 2 1; b: 1 5, 1 6', 1) await sched.add_profile('a: 2 1; b: 2 5', 2) await sched_complete(sched) base = await game.get_observations() server.custom_response(lambda: _raise(error)) alternate = await game.get_observations() assert base == alternate size_counts = {} for prof in alternate['profiles']: counts = len(prof['observations']) size_counts[counts] = size_counts.get(counts, 0) + 1 assert size_counts == {1: 1, 2: 1} base = await game.get_full_data() server.custom_response(lambda: _raise(error)) alternate = await game.get_full_data() assert base == alternate size_counts = {} for prof in alternate['profiles']: for obs in prof['observations']: assert len(obs['players']) == 4 counts = len(prof['observations']) size_counts[counts] = size_counts.get(counts, 0) + 1 assert size_counts == {1: 1, 2: 1}
async def test_canon_game(): # pylint: disable=too-many-locals """Test that canon game creates proper games""" async with mockserver.server() as server, \ api.api('', num_tries=3, retry_delay=0.5) as egta: sim = await create_simulator(server, egta, 'sim', '1') sim2 = await create_simulator(server, egta, 'sim', '2') symgrps = [('a', 2, ['1']), ('b', 2, ['5', '6'])] conf = {'key': 'val'} game1 = await sim.get_canon_game(symgrps, conf) summ = await game1.get_summary() assert conf == dict(summ['configuration']) def unpack(name, count, strategies): """Helper to unpack dictionaries""" return name, count, strategies symgrp_dict = {} for role_info in summ['roles']: role, count, strategies = unpack(**role_info) symgrp_dict[role] = (role, count, set(strategies)) for role, count, strats in symgrps: _, cnt, strt = symgrp_dict[role] assert cnt == count assert strt == set(strats) game2 = await egta.get_canon_game(sim['id'], symgrps, {'key': 'diff'}) assert game1['id'] != game2['id'] game3 = await egta.get_canon_game(sim['id'], [('a', 2, ['1']), ('b', 2, ['5', '7'])], conf) assert game1['id'] != game3['id'] game4 = await egta.get_canon_game(sim['id'], [('a', 2, ['1']), ('b', 1, ['5', '6'])], conf) assert game1['id'] != game4['id'] game5 = await egta.get_canon_game(sim2['id'], symgrps, conf) assert game1['id'] != game5['id'] game6 = await egta.get_canon_game(sim['id'], symgrps, conf) assert game1['id'] == game6['id']
async def test_exceptions(): """Test that exceptions can be properly set""" async with mockserver.server() as server, \ api.api('', num_tries=3, retry_delay=0.5) as egta: sim = await create_simulator(server, egta, 'sim', '1') await sim.add_strategies({'role': ['strategy']}) sched = await sim.create_generic_scheduler('sched', True, 0, 1, 0, 0) await sched.add_role('role', 1) prof = await sched.add_profile('role: 1 strategy', 1) game = await sched.create_game('game') await game.add_symgroup('role', 1, ['strategy']) server.custom_response(lambda: _raise(TimeoutError), 11) # Creations fail with pytest.raises(TimeoutError): await sim.create_generic_scheduler('sched_2', False, 0, 0, 0, 0) with pytest.raises(TimeoutError): await sched.create_game() # Infos fail with pytest.raises(TimeoutError): await sim.get_info() with pytest.raises(TimeoutError): await sched.get_info() with pytest.raises(TimeoutError): await game.get_structure() with pytest.raises(TimeoutError): await prof.get_structure() # Mutates fail with pytest.raises(TimeoutError): await sim.add_role('r') with pytest.raises(TimeoutError): await sim.add_strategy('role', 's') with pytest.raises(TimeoutError): await sched.add_role('r', 1) with pytest.raises(TimeoutError): await game.add_role('r', 1) with pytest.raises(TimeoutError): await game.add_strategy('role', 's') # Succeed after done assert sim['id'] == (await sim.get_info())['id']
async def test_scheduler_deactivate(game): """Test that scheduler ends when deactivated""" async with mockserver.server() as server, api.api() as egta: sim = await egta.get_simulator(server.create_simulator( # pragma: no branch pylint: disable=line-too-long 'sim', '1', delay_dist=lambda: random.random() / 10)) strats = dict(zip(game.role_names, game.strat_names)) symgrps = list(zip(game.role_names, game.num_role_players, game.strat_names)) await sim.add_strategies(strats) egame = await egta.get_canon_game(sim['id'], symgrps) # Schedule all new profiles and verify it works # This first time should have to wait to schedule more async with eosched.eosched( game, egta, egame['id'], 0.1, 1, 10, 0, 0) as sched: # Deactivate scheduler for esched in await egta.get_generic_schedulers(): await esched.deactivate() with pytest.raises(ValueError): await sched.sample_payoffs(game.random_profile())
async def test_exception_in_schedule(game): """Test exception throwin in scheduler""" async with mockserver.server() as server, api.api() as egta: sim = await egta.get_simulator(server.create_simulator( # pragma: no branch pylint: disable=line-too-long 'sim', '1', delay_dist=lambda: random.random() / 10)) strats = dict(zip(game.role_names, game.strat_names)) symgrps = list(zip(game.role_names, game.num_role_players, game.strat_names)) await sim.add_strategies(strats) egame = await egta.get_canon_game(sim['id'], symgrps) prof = game.random_profile() async with eosched.eosched( game, egta, egame['id'], 0.1, 1, 25, 0, 0) as sched: # so that enough calls to get_requirements are made server.custom_response(lambda: _raise(TimeoutError)) await asyncio.sleep(0.1) with pytest.raises(TimeoutError): await sched.sample_payoffs(prof)
async def test_exception_in_get(game): """Test exception in await""" async with mockserver.server() as server, api.api() as egta: sim = await egta.get_simulator(server.create_simulator( 'sim', '1', delay_dist=lambda: random.random() / 10)) strats = dict(zip(game.role_names, game.strat_names)) symgrps = list(zip(game.role_names, game.num_role_players, game.strat_names)) await sim.add_strategies(strats) egame = await egta.get_canon_game(sim['id'], symgrps) profs = game.random_profiles(20) async with eosched.eosched( game, egta, egame['id'], 0.1, 1, 10, 0, 0) as sched: futures = asyncio.gather(*[ sched.sample_payoffs(p) for p in profs]) await asyncio.sleep(0.1) server.custom_response(lambda: _raise(TimeoutError)) await asyncio.sleep(0.1) with pytest.raises(TimeoutError): await futures
async def test_get_schedulers(): """Test getting schedulers""" async with mockserver.server() as server, \ api.api('', num_tries=3, retry_delay=0.5) as egta: sim = await create_simulator(server, egta, 'sim', '1') await sim.create_generic_scheduler('1', False, 0, 10, 0, 0) sched2 = await egta.create_generic_scheduler(sim['id'], '2', False, 0, 10, 0, 0) sched3 = await sim.create_generic_scheduler('3', False, 0, 10, 0, 0) await sim.create_generic_scheduler('4', False, 0, 10, 0, 0) await sim.create_generic_scheduler('5', False, 0, 10, 0, 0) with pytest.raises(requests.exceptions.HTTPError): await sim.create_generic_scheduler('4', False, 0, 10, 0, 0) sched = await egta.get_scheduler(2) assert sched['id'] == sched3['id'] sched = await egta.get_scheduler_name('3') assert sched['id'] == sched3['id'] assert sum(1 for _ in await egta.get_generic_schedulers()) == 5 assert {0, 1, 2, 3, 4} == {s['id'] for s in await egta.get_generic_schedulers()} await sched2.destroy_scheduler() await sched3.destroy_scheduler() assert sum(1 for _ in await egta.get_generic_schedulers()) == 3 assert {0, 3, 4} == {s['id'] for s in await egta.get_generic_schedulers()} with pytest.raises(requests.exceptions.HTTPError): await egta.get_scheduler(5) with pytest.raises(requests.exceptions.HTTPError): await egta.get_scheduler(2) with pytest.raises(AssertionError): await egta.get_scheduler_name('3')
async def test_sims(): """Test getting simulations""" async with mockserver.server() as server: with stdout() as out, stderr() as err: assert await run('-a', '', 'sched'), err.getvalue() assert not out.getvalue() sim_id = server.create_simulator('sim', '1') async with api.api() as egta: sim = await egta.get_simulator(sim_id) await sim.add_strategies({'r': ['s0', 's1']}) sched = await egta.create_generic_scheduler( sim_id, 'sched', True, 1, 2, 1, 1) await sched.add_role('r', 2) # This fails because we don't implement search, so this is as if no # job exists assert not await run('-a', '', 'sims', '-j', '0') await sched.add_profile('r: 1 s0, 1 s1', 1) # This works because there's only one simulation so far and we # don't implement search with stdout() as out, stderr() as err: assert await run('-a', '', 'sims', '-j', '0'), err.getvalue() await sched.add_profile('r: 2 s0', 2) # This fails because we don't implement search and now there are # several simulations assert not await run('-a', '', 'sims', '-j', '0') with stdout() as out, stderr() as err: assert await run('-a', '', 'sims'), err.getvalue() sims = [json.loads(line) for line in out.getvalue()[:-1].split('\n')] assert len(sims) == 3 with stderr() as err: assert await run('-a', '', 'sims', str(sims[0]['folder'])), err.getvalue()
async def test_sched_running(): """Test running scheduler functionality""" async with mockserver.server() as server: # verify no schedulers with stdout() as out, stderr() as err: assert await run('-a', '', 'sched'), err.getvalue() assert not out.getvalue() # create one sim_id = server.create_simulator('sim', '1', delay_dist=lambda: 1) async with api.api() as egta: sim = await egta.get_simulator(sim_id) await sim.add_strategies({'r': ['s0', 's1']}) sched = await egta.create_generic_scheduler( sim_id, 'sched', True, 1, 2, 1, 1) await sched.add_role('r', 2) with stdout() as out, stderr() as err: assert await run('-a', '', 'sched', '--running'), err.getvalue() assert not out.getvalue() await sched.add_profile('r: 1 s0, 1 s1', 1) with stdout() as out, stderr() as err: assert await run('-a', '', 'sched', '--running'), err.getvalue() lines = out.getvalue()[:-1].split('\n') assert len(lines) == 1 running = json.loads(lines[0]) assert running['active'] # Wait to complete await asyncio.sleep(1.5) with stdout() as out, stderr() as err: assert await run('-a', '', 'sched', '--running'), err.getvalue() assert not out.getvalue()
async def amain(*argv): # pylint: disable=too-many-statements """Entry point for async cli with args""" parser = argparse.ArgumentParser( description="""Command line access to egta online apis. This works well in concert with `jq`.""") parser.add_argument( '--verbose', '-v', action='count', default=0, help="""Sets the verbosity of commands. Output is send to standard error.""") parser.add_argument( '--version', '-V', action='version', version='%(prog)s {}'.format(egtaonline.__version__)) parser_auth = parser.add_mutually_exclusive_group() parser_auth.add_argument( '--auth-string', '-a', metavar='<auth-string>', help="""The string authorization token to connect to egta online.""") parser_auth.add_argument( '--auth-file', '-f', metavar='<auth-file>', type=argparse.FileType(), help="""Filename that just contains the string of the auth token.""") subparsers = parser.add_subparsers(title='Subcommands', dest='command') subparsers.required = True parser_sim = subparsers.add_parser( 'sim', help="""Get information on or modify a simulator.""", description="""Operate on EGTA simulators. By defualt this will return information about each simulator on its own line.""") parser_sim.add_argument( 'sim_id', metavar='sim-id', nargs='?', help="""Get information from a specific simulator instead of all of them.""") parser_sim.add_argument( '--sim-version', '-n', metavar='<version-name>', help="""If this is specified then the `sim-id` is treated as the name of the simulator and the supplied argument is treated as the version.""") parser_sim.add_argument( '--json', '-j', metavar='json-file', nargs='?', type=argparse.FileType('r'), help="""Modify simulator using the specified json file. By default this will add all of the roles and strategies in the json file. If `delete` is specified, this will remove only the strategies specified in the file. `-` can be used to read from stdin.""") parser_sim.add_argument( '--role', '-r', metavar='<role>', help="""Modify `role` of the simulator. By default this will add `role` to the simulator. If `delete` is specified this will remove `role` instead. If `strategy` is specified see strategy.""") parser_sim.add_argument( '--strategy', '-s', metavar='<strategy>', help="""Modify `strategy` of the simulator. This requires that `role` is also specified. By default this adds `strategy` to `role`. If `delete` is specified, then this removes the strategy instead.""") parser_sim.add_argument( '--delete', '-d', action='store_true', help="""Triggers removal of roles or strategies instead of addition""") parser_sim.add_argument( '--zip', '-z', action='store_true', help="""Download simulator zip file to stdout.""") parser_game = subparsers.add_parser( 'game', help="""Get information on, create, destroy, or modify a game.""", description="""Operate on EGTA Online games. Buy default this will return information about each game on its own line.""") parser_game.add_argument( 'game_id', nargs='?', metavar='game-id', help="""Get data from a specific game instead of all of them.""") parser_game.add_argument( '--name', '-n', action='store_true', help="""If specified then get the game via its string name not its id number. This is much slower than accessing via id number.""") parser_game.add_argument( '--fetch-conf', metavar='<configuration>', type=argparse.FileType('r'), help="""If specified then interpret `game_id` as a simulator id and use the file specified as a game configuration. Fetch data of the appropriate granularity from the canonical game defined by that simulator and this configuration. Games need specified roles and players, these will be pulled from `--json` which must have two top level entries, `players` and `strategies` which list the number of players per role and the strategies per role respectively`. The configuration will not be updated with simulator defaults, so it may be helpful to call `sim` first to get those.""") parser_game.add_argument( '--json', '-j', metavar='json-file', nargs='?', type=argparse.FileType('r'), help="""Modify game using the specified json file. By default this will add all of the roles and strategies in the json file. If `delete` is specified, this will remove only the strategies specified in the file.""") parser_game.add_argument( '--role', '-r', metavar='<role>', help="""Modify `role` of the game. By default this will add `role` to the game. If `delete` is specified this will remove `role` instead. If `strategy` is specified see strategy.""") parser_game.add_argument( '--count', '-c', metavar='<count>', help="""If adding a role, the count of the role must be specified.""") parser_game.add_argument( '--strategy', '-s', metavar='<strategy>', help="""Modify `strategy` of the game. This requires that `role` is also specified. By default this adds `strategy` to `role`. If `delete` is specified, then this removes the strategy instead.""") parser_game.add_argument( '--delete', '-d', action='store_true', help="""Triggers removal of roles or strategies instead of addition""") parser_gran = (parser_game.add_argument_group('data granularity') .add_mutually_exclusive_group()) parser_gran.add_argument( '--structure', action='store_true', help="""Return game information but no profile information. (default)""") parser_gran.add_argument( '--summary', action='store_true', help="""Return profiles with aggregated payoffs.""") parser_gran.add_argument( '--observations', action='store_true', help="""Return profiles with observation data.""") parser_gran.add_argument( '--full', action='store_true', help="""Return all collected payoff data.""") parser_sched = subparsers.add_parser( 'sched', help="""Get information on, create, destroy, or modify a scheduler.""", description="""Operate on EGTA Online schedulers. By default this will return information about each generic scheduler on its own line.""") parser_sched.add_argument( '--running', action='store_true', help="""If specified, filter schedulers by ones that are currently running simulations. To be running simulations, a scheduler has to be active, and have at least one profile whose current count is less than the requirement for that scheduler.""") parser_sched.add_argument( 'sched_id', nargs='?', metavar='sched-id', help="""Get information about a specific scheduler. This scheduler doesn't need to be generic, but to operate on it, it must be.""") parser_sched.add_argument( '--name', '-n', action='store_true', help="""If specified then get the scheduler via its string name not its id number. This is much slower than accessing via id number, and only works for generic schedulers.""") parser_act = (parser_sched.add_argument_group('scheduler action') .add_mutually_exclusive_group()) parser_act.add_argument( '--requirements', '-r', action='store_true', help="""Get scheuler requirements instead of just information.""") parser_act.add_argument( '--deactivate', action='store_true', help="""Deactivate the specified scheduler.""") parser_act.add_argument( '--delete', '-d', action='store_true', help="""If specified with a scheduler, delete it.""") # TODO Specifying a flag to change from folder to job id is awkward parser_sims = subparsers.add_parser( 'sims', help="""Get information about pending, scheduled, queued, complete, or failed simulations.""", description="""Get information about EGTA Online simulations. These are the actual scheduled simulations instead of the simulators that generate them. If no folder is specified, each simulation comes out on a different line, and can be easily filtered with `head` and `jq`.""") parser_sims.add_argument( 'folder', metavar='folder-id', nargs='?', type=int, help="""Get information from a specific simulation instead of all of them. This should be the folder number of the simulation of interest.""") parser_sims.add_argument( '-j', '--job', action='store_true', help="""Fetch the simulation with a given PBS job id instead of its folder number.""") parser_sims.add_argument( '--page', '-p', metavar='<start-page>', default=1, type=int, help="""The page to start scanning at. (default: %(default)d)""") parser_sims.add_argument( '--ascending', '-a', action='store_true', help="""Return results in ascending order instead of descending.""") parser_sims.add_argument( '--sort-column', '-s', choices=('job', 'folder', 'profile', 'state'), default='job', help="""Column to order results by. (default: %(default)s)""") parser_sims.add_argument( '--search', default='', metavar='<search-string>', help="""The string to filter results by. See egtaonline for examples of what this can be.""") parser_sims.add_argument( '--state', choices=['canceled', 'complete', 'failed', 'processing', 'queued', 'running'], help="""Only select jobs with a specific state.""") parser_sims.add_argument( '--profile', metavar='<profile-substring>', help="""Limit results to simulations whose profiles contain the supplied substring.""") parser_sims.add_argument( '--simulator', metavar='<sim-substring>', help="""Limit results to simulations where the simulator fullname contains the supplied substring.""") parser_login = subparsers.add_parser( 'login', help="""Login to egtaonline by fetching your auth token.""", description="""Download an authtoken to allow requests to egtaonline.""") parser_login.add_argument( 'email', metavar='<email>', help="""Email address of the auther to login as""") parser_login.add_argument( '-u', '--user', action='store_true', help="""Store the auth_token in the user directory instead of the local directory.""") args = parser.parse_args(argv) if args.command == 'login': auth.login(args.email, int(args.user)) return if args.auth_string is None and args.auth_file is not None: args.auth_string = args.auth_file.read().strip() logging.basicConfig(stream=sys.stderr, level=30 - 10 * min(args.verbose, 2)) async with api.api(args.auth_string) as eoapi: if args.command == 'sim': return await _sim(eoapi, args) elif args.command == 'game': return await _game(eoapi, args) elif args.command == 'sched': return await _sched(eoapi, args) elif args.command == 'sims': return await _sims(eoapi, args) else: assert False # pragma: no cover
async def test_parity(): # pylint: disable=too-many-locals """Test that egta matches mock server""" # Get egta data async with api.api() as egta: true_sim = await egta.get_simulator(183) true_sched = await egta.get_scheduler(4997) true_game = await egta.get_game(2536) reqs = await true_sched.get_requirements() async def get_prof_info(prof): """Get all info about a profile""" return (await prof.get_structure(), await prof.get_summary(), await prof.get_observations(), await prof.get_full_data()) prof_info = await asyncio.gather( *[get_prof_info(prof) for prof in reqs['scheduling_requirements']]) game_info = await true_game.get_structure() game_summ = await true_game.get_summary() # Replicate in mock async with mockserver.server() as server, api.api() as egta: for i in range(true_sim['id']): server.create_simulator('sim', str(i)) mock_sim = await egta.get_simulator( server.create_simulator(true_sim['name'], true_sim['version'], true_sim['email'], true_sim['configuration'])) await mock_sim.add_strategies(true_sim['role_configuration']) info = await mock_sim.get_info() assert_dicts_types(true_sim, info) assert_dicts_equal(true_sim, info) await asyncio.gather(*[ mock_sim.create_generic_scheduler(str(i), False, 0, 0, 0, 0) for i in range(true_sched['id']) ]) mock_sched = await mock_sim.create_generic_scheduler( true_sched['name'], true_sched['active'], true_sched['process_memory'], true_sched['size'], true_sched['time_per_observation'], true_sched['observations_per_simulation'], true_sched['nodes'], dict(reqs['configuration'])) info = await mock_sched.get_info() assert_dicts_types(true_sched, info) assert_dicts_equal(true_sched, info) game_sched = await mock_sim.create_generic_scheduler( 'temp', True, 0, true_sched['size'], 0, 0, 1, dict(reqs['configuration'])) counts = prof_info[0][0]['role_configuration'] await mock_sched.add_roles(counts) await game_sched.add_roles(counts) await mock_sched.activate() for prof, (info, summ, obs, full) in zip(reqs['scheduling_requirements'], prof_info): sprof = await game_sched.add_profile(info['assignment'], prof['current_count']) mprof = await mock_sched.add_profile(info['assignment'], prof['requirement']) await sched_complete(game_sched) await sched_complete(mock_sched) assert ((await sprof.get_structure() )['observations_count'] == prof['current_count']) assert sprof['id'] == mprof['id'] struct = await mprof.get_structure() assert_dicts_types(info, struct) assert_dicts_equal(info, struct, {'id'}) assert_dicts_types(summ, (await mprof.get_summary()), (), True) assert_dicts_types(obs, (await mprof.get_observations()), {'extended_features', 'features'}, True) assert_dicts_types(full, (await mprof.get_full_data()), {'extended_features', 'features', 'e', 'f'}, True) mock_reqs = await mock_sched.get_requirements() assert_dicts_types(mock_reqs, reqs) await asyncio.gather( *[mock_sim.create_game(str(i), 0) for i in range(true_game['id'])]) mock_game = await mock_sim.create_game( true_game['name'], true_game['size'], dict(game_summ['configuration'])) info = await mock_game.get_structure() assert_dicts_types(game_info, info) assert_dicts_equal(game_info, info) symgrps = [(grp['name'], grp['count'], grp['strategies']) for grp in game_summ['roles']] await mock_game.add_symgroups(symgrps) # Schedule next profiles await asyncio.gather(*[ game_sched.add_profile(prof['symmetry_groups'], prof['observations_count']) for prof in game_summ['profiles'] ]) await sched_complete(game_sched) mock_summ = await mock_game.get_summary() assert_dicts_types(game_summ, mock_summ, (), True)
async def test_simulator(): """Test simulator api""" async with mockserver.server() as server, \ api.api('', num_tries=3, retry_delay=0.5) as egta: sim = await create_simulator(server, egta, 'sim', '1') info = await sim.get_info() validate_object( info, { 'configuration': { 'type': 'object' }, 'created_at': { 'type': 'string' }, 'email': { 'type': 'string' }, 'id': { 'type': 'integer' }, 'name': { 'type': 'string' }, 'role_configuration': { 'type': 'object' }, 'source': { 'type': 'object' }, 'updated_at': { 'type': 'string' }, 'url': { 'type': 'string' }, 'version': { 'type': 'string' }, }) role_conf = {'a': ['1', '2', '3', '4'], 'b': ['5', '6', '7']} assert info['role_configuration'] == role_conf await asyncio.sleep(1) await sim.remove_strategy('a', '3') new_role_conf = {'a': ['1', '2', '4'], 'b': ['5', '6', '7']} new_info = await sim.get_info() assert new_info['role_configuration'] == new_role_conf assert new_info['updated_at'] != info['updated_at'] assert info['role_configuration'] == role_conf await sim.remove_role('b') new_role_conf = {'a': ['1', '2', '4']} new_info = await sim.get_info() assert new_info['role_configuration'] == new_role_conf # Stale object didn't update assert info['role_configuration'] == role_conf # Add existing strategy await sim.add_strategy('a', '1') new_info = await sim.get_info() assert new_info['role_configuration'] == new_role_conf await sim.add_strategy('a', '2') await sim.add_strategy('a', '3') new_role_conf = {'a': ['1', '2', '3', '4']} new_info = await sim.get_info() assert new_info['role_configuration'] == new_role_conf await sim.remove_strategies({'a': ['4', '5', '4']}) new_role_conf = {'a': ['1', '2', '3']} new_info = await sim.get_info() assert new_info['role_configuration'] == new_role_conf await sim.remove_role('c') with pytest.raises(KeyError): await sim.add_strategy('c', '8') # Shouldn't raise exception, because removals never do await sim.remove_strategies({'c': ['8']})
async def test_get_simulations(): """Test getting simulations""" async with mockserver.server() as server, \ api.api('', num_tries=3, retry_delay=0.5) as egta: assert not await agather(egta.get_simulations()) sim1 = await create_simulator(server, egta, 'sim', '1') sched1 = await sim1.create_generic_scheduler('sched1', True, 0, 4, 0, 0) await sched1.add_roles({'a': 2, 'b': 2}) await sched1.add_profile('a: 2 1; b: 1 6, 1 7', 2) assert len(await agather(egta.get_simulations())) == 2 simul = next(iter(await agather(egta.get_simulations()))) validate_object( simul, { 'folder': { 'type': 'integer' }, 'job': { 'type': 'number' }, 'profile': { 'type': 'string' }, 'simulator': { 'type': 'string' }, 'state': { 'type': 'string' }, }) validate_object( await egta.get_simulation(simul['folder']), { 'error_message': { 'type': 'string' }, 'folder_number': { 'type': 'integer' }, 'job': { 'type': 'string' }, 'profile': { 'type': 'string' }, 'simulator_fullname': { 'type': 'string' }, 'size': { 'type': 'integer' }, 'state': { 'type': 'string' }, }) sim2 = await create_simulator(server, egta, 'sim', '2') sched2 = await sim2.create_generic_scheduler('sched2', True, 0, 5, 0, 0) await sched2.add_roles({'a': 2, 'b': 3}) await sched2.add_profile('a: 2 1; b: 1 5, 2 7', 3) assert len(await agather(egta.get_simulations())) == 5 # Test simulations assert is_sorted( # pragma: no branch (f['simulator'] for f in await agather(egta.get_simulations(column='simulator'))), reverse=True) assert is_sorted( # pragma: no branch (f['folder'] for f in await agather(egta.get_simulations(column='folder'))), reverse=True) assert is_sorted( # pragma: no branch (f['profile'] for f in await agather(egta.get_simulations(column='profile'))), reverse=True) assert is_sorted( # pragma: no branch (f['state'] for f in await agather(egta.get_simulations(column='state'))), reverse=True) assert is_sorted( # pragma: no branch f['simulator'] for f in await agather( egta.get_simulations(asc=True, column='simulator'))) assert is_sorted( # pragma: no branch f['folder'] for f in await agather( egta.get_simulations(asc=True, column='folder'))) assert is_sorted( # pragma: no branch f['profile'] for f in await agather( egta.get_simulations(asc=True, column='profile'))) assert is_sorted( # pragma: no branch f['state'] for f in await agather( egta.get_simulations(asc=True, column='state'))) assert not await agather(egta.get_simulations(page_start=2)) await sched2.add_profile('a: 2 1; b: 1 5, 2 6', 21) assert len(await agather(egta.get_simulations(page_start=2))) == 1