async def test_profile_json_error(): """Test invalid profile json triggers retry""" async with mockserver.server() as server, \ api.api('', num_tries=3, retry_delay=0.5) as egta: sim = await create_simulator(server, egta, 'sim', '1') sched = await sim.create_generic_scheduler('sched', True, 0, 4, 0, 0) await sched.add_roles({'a': 2, 'b': 2}) game = await sched.create_game() await game.add_symgroups([('a', 2, ['1']), ('b', 2, ['5', '6'])]) prof = await sched.add_profile('a: 2 1; b: 2 5', 2) await sched_complete(sched) server.custom_response(lambda: '', 2) summ = await prof.get_summary() assert 'id' in summ assert 'observations_count' in summ assert 'symmetry_groups' in summ server.custom_response(lambda: '', 3) with pytest.raises(json.decoder.JSONDecodeError): await prof.get_summary() server.custom_response(lambda: '{}', 3) with pytest.raises(jsonschema.ValidationError): await prof.get_summary()
async def test_game_json_error(): """Test returning invalid json in games triggers retry""" async with mockserver.server() as server, \ api.api('', num_tries=3, retry_delay=0.5) as egta: sim = await create_simulator(server, egta, 'sim', '1') sched = await sim.create_generic_scheduler('sched', True, 0, 4, 0, 0) await sched.add_roles({'a': 2, 'b': 2}) game = await sched.create_game() await game.add_symgroups([('a', 2, ['1']), ('b', 2, ['5', '6'])]) await sched.add_profile('a: 2 1; b: 1 5, 1 6', 1) await sched.add_profile('a: 2 1; b: 2 5', 2) await sched_complete(sched) server.custom_response(lambda: '', 2) summ = await game.get_summary() size_counts = {} for prof in summ['profiles']: counts = prof['observations_count'] size_counts[counts] = size_counts.get(counts, 0) + 1 assert size_counts == {1: 1, 2: 1} server.custom_response(lambda: '', 3) with pytest.raises(json.decoder.JSONDecodeError): await game.get_summary() server.custom_response(lambda: '{}', 3) with pytest.raises(jsonschema.ValidationError): await game.get_summary()
async def test_get_simulators(): """Test getting simulators""" async with mockserver.server() as server, \ api.api('', num_tries=3, retry_delay=0.5) as egta: sim1 = server.create_simulator('foo', '1') sim2 = server.create_simulator('bar', '1') sim3 = server.create_simulator('bar', '2') assert sum(1 for _ in await egta.get_simulators()) == 3 assert {0, 1, 2} == {s['id'] for s in await egta.get_simulators()} sim = await egta.get_simulator(0) assert sim['id'] == sim1 sim = await egta.get_simulator_fullname('foo-1') assert sim['id'] == sim1 sim = await egta.get_simulator(2) assert sim['id'] == sim3 sim = await egta.get_simulator_fullname('bar-1') assert sim['id'] == sim2 sim = await egta.get_simulator_fullname('bar-2') assert sim['id'] == sim3 with pytest.raises(requests.exceptions.HTTPError): await egta.get_simulator(3) with pytest.raises(AssertionError): await egta.get_simulator_fullname('baz')
async def test_get_games(): """Test getting games""" async with mockserver.server() as server, \ api.api('', num_tries=3, retry_delay=0.5) as egta: sim = await create_simulator(server, egta, 'sim', '1') await sim.create_game('a', 5) game2 = await egta.create_game(sim['id'], 'b', 6) game3 = await sim.create_game('c', 3) with pytest.raises(requests.exceptions.HTTPError): await sim.create_game('b', 3) assert (await egta.get_game(1))['id'] == game2['id'] assert (await egta.get_game_name('c'))['id'] == game3['id'] assert sum(1 for _ in await egta.get_games()) == 3 assert {0, 1, 2} == {g['id'] for g in await egta.get_games()} await game2.destroy_game() assert sum(1 for _ in await egta.get_games()) == 2 assert {0, 2} == {g['id'] for g in await egta.get_games()} with pytest.raises(requests.exceptions.HTTPError): await egta.get_game(3) with pytest.raises(requests.exceptions.HTTPError): await egta.get_game(1) with pytest.raises(AssertionError): await egta.get_game_name('b')
async def test_exception_in_get(game): """Test exception in await""" async with mockserver.server() as server, api.api() as egta: sim = await egta.get_simulator( server.create_simulator("sim", "1", delay_dist=lambda: random.random() / 10)) strats = dict(zip(game.role_names, game.strat_names)) symgrps = list( zip(game.role_names, game.num_role_players, game.strat_names)) await sim.add_strategies(strats) egame = await egta.get_canon_game(sim["id"], symgrps) profs = game.random_profiles(20) async with eosched.eosched(game, egta, egame["id"], 0.1, 1, 10, 0, 0) as sched: tasks = [ asyncio.ensure_future(sched.sample_payoffs(p)) for p in profs ] await asyncio.sleep(0.1) server.custom_response(lambda: _raise(TimeoutError)) await asyncio.sleep(0.1) with pytest.raises(TimeoutError): await asyncio.gather(*tasks) # tidy up errors = asyncio.gather(*tasks, return_exceptions=True) errors.cancel() with contextlib.suppress(TimeoutError, asyncio.CancelledError): await errors
async def test_exception_open(): """Test that exceptions are even thrown on open""" async with mockserver.server() as server: server.custom_response(lambda: _raise(TimeoutError)) with pytest.raises(TimeoutError): async with api.api(''): pass # pragma: no cover
async def test_delayed_profiles(): """Test mock server with delayed profile scheduling""" async with mockserver.server() as server, api.api('') as egta: sim = await egta.get_simulator( server.create_simulator('sim', '1', delay_dist=lambda: 0.5)) await sim.add_strategies({'1': ['a'], '2': ['b', 'c']}) sched = await sim.create_generic_scheduler('sched', True, 0, 10, 0, 0) await sched.add_roles({'1': 8, '2': 2}) prof = await sched.add_profile('1: 8 a; 2: 1 b, 1 c', 3) reqs = (await sched.get_requirements())['scheduling_requirements'] assert len(reqs) == 1 assert reqs[0]['current_count'] == 0 assert reqs[0]['requirement'] == 3 assert reqs[0]['id'] == prof['id'] count = 0 async for sim in egta.get_simulations(): assert sim['state'] == 'running' count += 1 assert count == 3 await asyncio.sleep(0.5) await sched_complete(sched) reqs = (await sched.get_requirements())['scheduling_requirements'] assert len(reqs) == 1 assert reqs[0]['current_count'] == 3 assert reqs[0]['requirement'] == 3 assert reqs[0]['id'] == prof['id'] count = 0 async for sim in egta.get_simulations(): assert sim['state'] == 'complete' count += 1 assert count == 3 # Test that extra sims get killed async with mockserver.server() as server, api.api('') as egta: sim = await egta.get_simulator( server.create_simulator('sim', '1', delay_dist=lambda: 10)) await sim.add_strategies({'1': ['a'], '2': ['b', 'c']}) sched = await sim.create_generic_scheduler('sched', True, 0, 10, 0, 0) await sched.add_roles({'1': 8, '2': 2}) # Need two profiles here because the other will be in the queue await sched.add_profile('1: 8 a; 2: 1 b, 1 c', 1) await sched.add_profile('1: 8 a; 2: 2 b', 1)
async def test_sim(): """Test sim functionality""" async with mockserver.server() as server: with stdout() as out, stderr() as err: assert await run('-a', '', 'sim'), err.getvalue() assert not out.getvalue() server.create_simulator('sim', '1') with stdout() as out, stderr() as err: assert await run('-a', '', 'sim'), err.getvalue() # get by id sim = json.loads(out.getvalue()) with stderr() as err: assert await run('-a', '', 'sim', str(sim['id'])), err.getvalue() # get by name with stdout() as out, stderr() as err: assert await run('-a', '', 'sim', sim['name'], '-n', sim['version']), err.getvalue() assert sim['id'] == json.loads(out.getvalue())['id'] assert not await run('-a', '', 'sim', '--', '-1') # add role with stderr() as err: assert await run('-a', '', 'sim', str(sim['id']), '-rr'), err.getvalue() # add strategy with stderr() as err: assert await run('-a', '', 'sim', str(sim['id']), '-rr', '-ss'), err.getvalue() # add strategies with stdin(json.dumps({'r': ['q'], 'a': ['b']})), stderr() as err: assert await run('-a', '', 'sim', str(sim['id']), '-j-'), err.getvalue() # remove strategy with stderr() as err: assert await run('-a', '', 'sim', str(sim['id']), '-drr', '-sq'), err.getvalue() # remove role with stderr() as err: assert await run('-a', '', 'sim', str(sim['id']), '-dra'), err.getvalue() # remove strategies with stdin(json.dumps({'r': ['s']})), stderr() as err: assert await run('-a', '', 'sim', str(sim['id']), '-dj-'), err.getvalue() # get zip with stdout() as out, stderr() as err: assert await run('-a', '', 'sim', str(sim['id']), '-z'), err.getvalue()
async def test_threading(): """Test that no errors arise when multi-threading""" async with mockserver.server() as server, \ api.api('', num_tries=3, retry_delay=0.5) as egta: sim = await create_simulator(server, egta, 'sim', '1') await asyncio.gather(*[ sim.add_strategies({'r{:d}'.format(i): ['s{:d}'.format(i)]}) for i in range(10) ])
async def test_brute_egta_game(game): """Test brute game with egta sched""" async with mockserver.server() as server: sched = await get_eosched(server, game, "game") with stdout() as out, stderr() as err: assert await run( "brute", sched, "--dpr", "buyers:2;sellers:2", "--min-reg" ), err.getvalue() for eqm in json.loads(out.getvalue()): game.mixture_from_json(eqm["equilibrium"])
async def test_brute_egta_game(game): """Test brute game with egta sched""" async with mockserver.server() as server: sched = await get_eosched(server, game, 'game') with stdout() as out, stderr() as err: assert await run( 'brute', sched, '--dpr', 'buyers:2;sellers:2', '--min-reg'), err.getvalue() for eqm in json.loads(out.getvalue()): game.mixture_from_json(eqm['equilibrium'])
async def test_trace_norm(): """Test tracing""" base_game = rsgame.empty([3, 2], [2, 3]) async with mockserver.server() as server: game0 = add_singleton_role(base_game, 0, 'a', 'sx', 1) sched0 = await get_eosched(server, game0, 'game0') game1 = add_singleton_role(base_game, 1, 'r00', 's0', 3) sched1 = await get_eosched(server, game1, 'game1') with stdout() as out, stderr() as err: assert await run('trace', sched0, sched1), err.getvalue() traces = json.loads(out.getvalue()) verify_trace_json(base_game, traces)
async def test_trace_norm(): """Test tracing""" base_game = rsgame.empty([3, 2], [2, 3]) async with mockserver.server() as server: game0 = add_singleton_role(base_game, 0, "a", "sx", 1) sched0 = await get_eosched(server, game0, "game0") game1 = add_singleton_role(base_game, 1, "r00", "s0", 3) sched1 = await get_eosched(server, game1, "game1") with stdout() as out, stderr() as err: assert await run("trace", sched0, sched1), err.getvalue() traces = json.loads(out.getvalue()) verify_trace_json(base_game, traces)
async def test_sched(): """Test scheduler functionality""" async with mockserver.server() as server: # verify no schedulers with stdout() as out, stderr() as err: assert await run('-a', '', 'sched'), err.getvalue() assert not out.getvalue() # create one sim_id = server.create_simulator('sim', '1') async with api.api() as egta: await egta.create_generic_scheduler(sim_id, 'sched', True, 1, 2, 1, 1) with stdout() as out, stderr() as err: assert await run('-a', '', 'sched'), err.getvalue() sched = json.loads(out.getvalue()) with stderr() as err: assert await run('-a', '', 'sched', str(sched['id'])), err.getvalue() with stderr() as err: assert await run('-a', '', 'sched', str(sched['id']), '-r'), err.getvalue() with stdout() as out, stderr() as err: assert await run('-a', '', 'sched', sched['name'], '-n'), err.getvalue() assert sched['id'] == json.loads(out.getvalue())['id'] # deactivate scheduler with stderr() as err: assert await run('-a', '', 'sched', str(sched['id']), '--deactivate'), err.getvalue() # verify deactivated with stdout() as out, stderr() as err: assert await run('-a', '', 'sched', str(sched['id'])), err.getvalue() assert not json.loads(out.getvalue())['active'] # delete scheduler with stderr() as err: assert await run('-a', '', 'sched', str(sched['id']), '-d'), err.getvalue() # assert no schedulers with stdout() as out, stderr() as err: assert await run('-a', '', 'sched'), err.getvalue() assert not out.getvalue()
async def test_basic_profile(game): """Test scheduling a standard profile""" async with mockserver.server() as server, api.api() as egta: sim = await egta.get_simulator( server.create_simulator("sim", "1", delay_dist=lambda: random.random() / 10)) strats = dict(zip(game.role_names, game.strat_names)) symgrps = list( zip(game.role_names, game.num_role_players, game.strat_names)) await sim.add_strategies(strats) egame = await egta.get_canon_game(sim["id"], symgrps) profs = game.random_profiles(20) # Schedule all new profiles and verify it works async with eosched.eosched(game, egta, egame["id"], 0.1, 1, 10, 0, 0) as sched: assert str(sched) == str(egame["id"]) assert game == rsgame.empty_copy(sched) awaited = await asyncio.gather( *[sched.sample_payoffs(p) for p in profs]) pays = np.stack(awaited) assert np.allclose(pays[profs == 0], 0) # Schedule old profiles and verify it still works async with eosched.eosched(game, egta, egame["id"], 0.1, 1, 10, 0, 0) as sched: awaited = await asyncio.gather( *[sched.sample_payoffs(p) for p in profs]) pays = np.stack(awaited) assert np.allclose(pays[profs == 0], 0) # Schedule two at a time, in two batches async with eosched.eosched(game, egta, egame["id"], 0.1, 2, 10, 0, 0) as base_sched: sched = countsched.countsched(base_sched, 2) awaited = await asyncio.gather( *[sched.sample_payoffs(p) for p in profs]) pays = np.stack(awaited) assert np.allclose(pays[profs == 0], 0) # Try again now that everything should be scheduled async with eosched.eosched(game, egta, egame["id"], 0.1, 2, 10, 0, 0) as base_sched: sched = countsched.countsched(base_sched, 2) awaited = await asyncio.gather( *[sched.sample_payoffs(p) for p in profs]) pays = np.stack(awaited) assert np.allclose(pays[profs == 0], 0)
async def test_exception_in_create(game): """Test exception creating eo scheduler""" async with mockserver.server() as server, api.api() as egta: sim = await egta.get_simulator(server.create_simulator( # pragma: no branch pylint: disable=line-too-long 'sim', '1', delay_dist=lambda: random.random() / 10)) strats = dict(zip(game.role_names, game.strat_names)) symgrps = list(zip(game.role_names, game.num_role_players, game.strat_names)) await sim.add_strategies(strats) egame = await egta.get_canon_game(sim['id'], symgrps) server.custom_response(lambda: _raise(TimeoutError)) with pytest.raises(TimeoutError): async with eosched.eosched( game, egta, egame['id'], 0.1, 1, 25, 0, 0): pass # pragma: no cover
async def test_canon_game(): # pylint: disable=too-many-locals """Test that canon game creates proper games""" async with mockserver.server() as server, \ api.api('', num_tries=3, retry_delay=0.5) as egta: sim = await create_simulator(server, egta, 'sim', '1') sim2 = await create_simulator(server, egta, 'sim', '2') symgrps = [('a', 2, ['1']), ('b', 2, ['5', '6'])] conf = {'key': 'val'} game1 = await sim.get_canon_game(symgrps, conf) summ = await game1.get_summary() assert conf == dict(summ['configuration']) def unpack(name, count, strategies): """Helper to unpack dictionaries""" return name, count, strategies symgrp_dict = {} for role_info in summ['roles']: role, count, strategies = unpack(**role_info) symgrp_dict[role] = (role, count, set(strategies)) for role, count, strats in symgrps: _, cnt, strt = symgrp_dict[role] assert cnt == count assert strt == set(strats) game2 = await egta.get_canon_game(sim['id'], symgrps, {'key': 'diff'}) assert game1['id'] != game2['id'] game3 = await egta.get_canon_game(sim['id'], [('a', 2, ['1']), ('b', 2, ['5', '7'])], conf) assert game1['id'] != game3['id'] game4 = await egta.get_canon_game(sim['id'], [('a', 2, ['1']), ('b', 1, ['5', '6'])], conf) assert game1['id'] != game4['id'] game5 = await egta.get_canon_game(sim2['id'], symgrps, conf) assert game1['id'] != game5['id'] game6 = await egta.get_canon_game(sim['id'], symgrps, conf) assert game1['id'] == game6['id']
async def test_large_game_failsafes(): """Test that large games fallback to gathering profile data""" async with mockserver.server() as server, \ api.api('', num_tries=3, retry_delay=0.5) as egta: sim = await create_simulator(server, egta, 'sim', '1') error = requests.exceptions.HTTPError( '500 Server Error: Game too large!') sched = await sim.create_generic_scheduler('sched', True, 0, 4, 0, 0, configuration={'k': 'v'}) await sched.add_roles({'a': 2, 'b': 2}) game = await sched.create_game() await game.add_symgroups([('a', 2, ['1']), ('b', 2, ['5', '6'])]) await sched.add_profile('a: 2 1; b: 1 5, 1 6', 1) await sched.add_profile('a: 2 1; b: 2 5', 2) await sched_complete(sched) base = await game.get_observations() server.custom_response(lambda: _raise(error)) alternate = await game.get_observations() assert base == alternate size_counts = {} for prof in alternate['profiles']: counts = len(prof['observations']) size_counts[counts] = size_counts.get(counts, 0) + 1 assert size_counts == {1: 1, 2: 1} base = await game.get_full_data() server.custom_response(lambda: _raise(error)) alternate = await game.get_full_data() assert base == alternate size_counts = {} for prof in alternate['profiles']: for obs in prof['observations']: assert len(obs['players']) == 4 counts = len(prof['observations']) size_counts[counts] = size_counts.get(counts, 0) + 1 assert size_counts == {1: 1, 2: 1}
async def test_exceptions(): """Test that exceptions can be properly set""" async with mockserver.server() as server, \ api.api('', num_tries=3, retry_delay=0.5) as egta: sim = await create_simulator(server, egta, 'sim', '1') await sim.add_strategies({'role': ['strategy']}) sched = await sim.create_generic_scheduler('sched', True, 0, 1, 0, 0) await sched.add_role('role', 1) prof = await sched.add_profile('role: 1 strategy', 1) game = await sched.create_game('game') await game.add_symgroup('role', 1, ['strategy']) server.custom_response(lambda: _raise(TimeoutError), 11) # Creations fail with pytest.raises(TimeoutError): await sim.create_generic_scheduler('sched_2', False, 0, 0, 0, 0) with pytest.raises(TimeoutError): await sched.create_game() # Infos fail with pytest.raises(TimeoutError): await sim.get_info() with pytest.raises(TimeoutError): await sched.get_info() with pytest.raises(TimeoutError): await game.get_structure() with pytest.raises(TimeoutError): await prof.get_structure() # Mutates fail with pytest.raises(TimeoutError): await sim.add_role('r') with pytest.raises(TimeoutError): await sim.add_strategy('role', 's') with pytest.raises(TimeoutError): await sched.add_role('r', 1) with pytest.raises(TimeoutError): await game.add_role('r', 1) with pytest.raises(TimeoutError): await game.add_strategy('role', 's') # Succeed after done assert sim['id'] == (await sim.get_info())['id']
async def test_exception_in_schedule(game): """Test exception throwin in scheduler""" async with mockserver.server() as server, api.api() as egta: sim = await egta.get_simulator(server.create_simulator( # pragma: no branch pylint: disable=line-too-long 'sim', '1', delay_dist=lambda: random.random() / 10)) strats = dict(zip(game.role_names, game.strat_names)) symgrps = list(zip(game.role_names, game.num_role_players, game.strat_names)) await sim.add_strategies(strats) egame = await egta.get_canon_game(sim['id'], symgrps) prof = game.random_profile() async with eosched.eosched( game, egta, egame['id'], 0.1, 1, 25, 0, 0) as sched: # so that enough calls to get_requirements are made server.custom_response(lambda: _raise(TimeoutError)) await asyncio.sleep(0.1) with pytest.raises(TimeoutError): await sched.sample_payoffs(prof)
async def test_scheduler_deactivate(game): """Test that scheduler ends when deactivated""" async with mockserver.server() as server, api.api() as egta: sim = await egta.get_simulator(server.create_simulator( # pragma: no branch pylint: disable=line-too-long 'sim', '1', delay_dist=lambda: random.random() / 10)) strats = dict(zip(game.role_names, game.strat_names)) symgrps = list(zip(game.role_names, game.num_role_players, game.strat_names)) await sim.add_strategies(strats) egame = await egta.get_canon_game(sim['id'], symgrps) # Schedule all new profiles and verify it works # This first time should have to wait to schedule more async with eosched.eosched( game, egta, egame['id'], 0.1, 1, 10, 0, 0) as sched: # Deactivate scheduler for esched in await egta.get_generic_schedulers(): await esched.deactivate() with pytest.raises(ValueError): await sched.sample_payoffs(game.random_profile())
async def test_get_schedulers(): """Test getting schedulers""" async with mockserver.server() as server, \ api.api('', num_tries=3, retry_delay=0.5) as egta: sim = await create_simulator(server, egta, 'sim', '1') await sim.create_generic_scheduler('1', False, 0, 10, 0, 0) sched2 = await egta.create_generic_scheduler(sim['id'], '2', False, 0, 10, 0, 0) sched3 = await sim.create_generic_scheduler('3', False, 0, 10, 0, 0) await sim.create_generic_scheduler('4', False, 0, 10, 0, 0) await sim.create_generic_scheduler('5', False, 0, 10, 0, 0) with pytest.raises(requests.exceptions.HTTPError): await sim.create_generic_scheduler('4', False, 0, 10, 0, 0) sched = await egta.get_scheduler(2) assert sched['id'] == sched3['id'] sched = await egta.get_scheduler_name('3') assert sched['id'] == sched3['id'] assert sum(1 for _ in await egta.get_generic_schedulers()) == 5 assert {0, 1, 2, 3, 4} == {s['id'] for s in await egta.get_generic_schedulers()} await sched2.destroy_scheduler() await sched3.destroy_scheduler() assert sum(1 for _ in await egta.get_generic_schedulers()) == 3 assert {0, 3, 4} == {s['id'] for s in await egta.get_generic_schedulers()} with pytest.raises(requests.exceptions.HTTPError): await egta.get_scheduler(5) with pytest.raises(requests.exceptions.HTTPError): await egta.get_scheduler(2) with pytest.raises(AssertionError): await egta.get_scheduler_name('3')
async def test_sims(): """Test getting simulations""" async with mockserver.server() as server: with stdout() as out, stderr() as err: assert await run('-a', '', 'sched'), err.getvalue() assert not out.getvalue() sim_id = server.create_simulator('sim', '1') async with api.api() as egta: sim = await egta.get_simulator(sim_id) await sim.add_strategies({'r': ['s0', 's1']}) sched = await egta.create_generic_scheduler( sim_id, 'sched', True, 1, 2, 1, 1) await sched.add_role('r', 2) # This fails because we don't implement search, so this is as if no # job exists assert not await run('-a', '', 'sims', '-j', '0') await sched.add_profile('r: 1 s0, 1 s1', 1) # This works because there's only one simulation so far and we # don't implement search with stdout() as out, stderr() as err: assert await run('-a', '', 'sims', '-j', '0'), err.getvalue() await sched.add_profile('r: 2 s0', 2) # This fails because we don't implement search and now there are # several simulations assert not await run('-a', '', 'sims', '-j', '0') with stdout() as out, stderr() as err: assert await run('-a', '', 'sims'), err.getvalue() sims = [json.loads(line) for line in out.getvalue()[:-1].split('\n')] assert len(sims) == 3 with stderr() as err: assert await run('-a', '', 'sims', str(sims[0]['folder'])), err.getvalue()
async def test_exception_in_get(game): """Test exception in await""" async with mockserver.server() as server, api.api() as egta: sim = await egta.get_simulator(server.create_simulator( 'sim', '1', delay_dist=lambda: random.random() / 10)) strats = dict(zip(game.role_names, game.strat_names)) symgrps = list(zip(game.role_names, game.num_role_players, game.strat_names)) await sim.add_strategies(strats) egame = await egta.get_canon_game(sim['id'], symgrps) profs = game.random_profiles(20) async with eosched.eosched( game, egta, egame['id'], 0.1, 1, 10, 0, 0) as sched: futures = asyncio.gather(*[ sched.sample_payoffs(p) for p in profs]) await asyncio.sleep(0.1) server.custom_response(lambda: _raise(TimeoutError)) await asyncio.sleep(0.1) with pytest.raises(TimeoutError): await futures
async def test_sched_running(): """Test running scheduler functionality""" async with mockserver.server() as server: # verify no schedulers with stdout() as out, stderr() as err: assert await run('-a', '', 'sched'), err.getvalue() assert not out.getvalue() # create one sim_id = server.create_simulator('sim', '1', delay_dist=lambda: 1) async with api.api() as egta: sim = await egta.get_simulator(sim_id) await sim.add_strategies({'r': ['s0', 's1']}) sched = await egta.create_generic_scheduler( sim_id, 'sched', True, 1, 2, 1, 1) await sched.add_role('r', 2) with stdout() as out, stderr() as err: assert await run('-a', '', 'sched', '--running'), err.getvalue() assert not out.getvalue() await sched.add_profile('r: 1 s0, 1 s1', 1) with stdout() as out, stderr() as err: assert await run('-a', '', 'sched', '--running'), err.getvalue() lines = out.getvalue()[:-1].split('\n') assert len(lines) == 1 running = json.loads(lines[0]) assert running['active'] # Wait to complete await asyncio.sleep(1.5) with stdout() as out, stderr() as err: assert await run('-a', '', 'sched', '--running'), err.getvalue() assert not out.getvalue()
async def test_parity(): # pylint: disable=too-many-locals """Test that egta matches mock server""" # Get egta data async with api.api() as egta: true_sim = await egta.get_simulator(183) true_sched = await egta.get_scheduler(4997) true_game = await egta.get_game(2536) reqs = await true_sched.get_requirements() async def get_prof_info(prof): """Get all info about a profile""" return (await prof.get_structure(), await prof.get_summary(), await prof.get_observations(), await prof.get_full_data()) prof_info = await asyncio.gather( *[get_prof_info(prof) for prof in reqs['scheduling_requirements']]) game_info = await true_game.get_structure() game_summ = await true_game.get_summary() # Replicate in mock async with mockserver.server() as server, api.api() as egta: for i in range(true_sim['id']): server.create_simulator('sim', str(i)) mock_sim = await egta.get_simulator( server.create_simulator(true_sim['name'], true_sim['version'], true_sim['email'], true_sim['configuration'])) await mock_sim.add_strategies(true_sim['role_configuration']) info = await mock_sim.get_info() assert_dicts_types(true_sim, info) assert_dicts_equal(true_sim, info) await asyncio.gather(*[ mock_sim.create_generic_scheduler(str(i), False, 0, 0, 0, 0) for i in range(true_sched['id']) ]) mock_sched = await mock_sim.create_generic_scheduler( true_sched['name'], true_sched['active'], true_sched['process_memory'], true_sched['size'], true_sched['time_per_observation'], true_sched['observations_per_simulation'], true_sched['nodes'], dict(reqs['configuration'])) info = await mock_sched.get_info() assert_dicts_types(true_sched, info) assert_dicts_equal(true_sched, info) game_sched = await mock_sim.create_generic_scheduler( 'temp', True, 0, true_sched['size'], 0, 0, 1, dict(reqs['configuration'])) counts = prof_info[0][0]['role_configuration'] await mock_sched.add_roles(counts) await game_sched.add_roles(counts) await mock_sched.activate() for prof, (info, summ, obs, full) in zip(reqs['scheduling_requirements'], prof_info): sprof = await game_sched.add_profile(info['assignment'], prof['current_count']) mprof = await mock_sched.add_profile(info['assignment'], prof['requirement']) await sched_complete(game_sched) await sched_complete(mock_sched) assert ((await sprof.get_structure() )['observations_count'] == prof['current_count']) assert sprof['id'] == mprof['id'] struct = await mprof.get_structure() assert_dicts_types(info, struct) assert_dicts_equal(info, struct, {'id'}) assert_dicts_types(summ, (await mprof.get_summary()), (), True) assert_dicts_types(obs, (await mprof.get_observations()), {'extended_features', 'features'}, True) assert_dicts_types(full, (await mprof.get_full_data()), {'extended_features', 'features', 'e', 'f'}, True) mock_reqs = await mock_sched.get_requirements() assert_dicts_types(mock_reqs, reqs) await asyncio.gather( *[mock_sim.create_game(str(i), 0) for i in range(true_game['id'])]) mock_game = await mock_sim.create_game( true_game['name'], true_game['size'], dict(game_summ['configuration'])) info = await mock_game.get_structure() assert_dicts_types(game_info, info) assert_dicts_equal(game_info, info) symgrps = [(grp['name'], grp['count'], grp['strategies']) for grp in game_summ['roles']] await mock_game.add_symgroups(symgrps) # Schedule next profiles await asyncio.gather(*[ game_sched.add_profile(prof['symmetry_groups'], prof['observations_count']) for prof in game_summ['profiles'] ]) await sched_complete(game_sched) mock_summ = await mock_game.get_summary() assert_dicts_types(game_summ, mock_summ, (), True)
async def test_get_simulations(): """Test getting simulations""" async with mockserver.server() as server, \ api.api('', num_tries=3, retry_delay=0.5) as egta: assert not await agather(egta.get_simulations()) sim1 = await create_simulator(server, egta, 'sim', '1') sched1 = await sim1.create_generic_scheduler('sched1', True, 0, 4, 0, 0) await sched1.add_roles({'a': 2, 'b': 2}) await sched1.add_profile('a: 2 1; b: 1 6, 1 7', 2) assert len(await agather(egta.get_simulations())) == 2 simul = next(iter(await agather(egta.get_simulations()))) validate_object( simul, { 'folder': { 'type': 'integer' }, 'job': { 'type': 'number' }, 'profile': { 'type': 'string' }, 'simulator': { 'type': 'string' }, 'state': { 'type': 'string' }, }) validate_object( await egta.get_simulation(simul['folder']), { 'error_message': { 'type': 'string' }, 'folder_number': { 'type': 'integer' }, 'job': { 'type': 'string' }, 'profile': { 'type': 'string' }, 'simulator_fullname': { 'type': 'string' }, 'size': { 'type': 'integer' }, 'state': { 'type': 'string' }, }) sim2 = await create_simulator(server, egta, 'sim', '2') sched2 = await sim2.create_generic_scheduler('sched2', True, 0, 5, 0, 0) await sched2.add_roles({'a': 2, 'b': 3}) await sched2.add_profile('a: 2 1; b: 1 5, 2 7', 3) assert len(await agather(egta.get_simulations())) == 5 # Test simulations assert is_sorted( # pragma: no branch (f['simulator'] for f in await agather(egta.get_simulations(column='simulator'))), reverse=True) assert is_sorted( # pragma: no branch (f['folder'] for f in await agather(egta.get_simulations(column='folder'))), reverse=True) assert is_sorted( # pragma: no branch (f['profile'] for f in await agather(egta.get_simulations(column='profile'))), reverse=True) assert is_sorted( # pragma: no branch (f['state'] for f in await agather(egta.get_simulations(column='state'))), reverse=True) assert is_sorted( # pragma: no branch f['simulator'] for f in await agather( egta.get_simulations(asc=True, column='simulator'))) assert is_sorted( # pragma: no branch f['folder'] for f in await agather( egta.get_simulations(asc=True, column='folder'))) assert is_sorted( # pragma: no branch f['profile'] for f in await agather( egta.get_simulations(asc=True, column='profile'))) assert is_sorted( # pragma: no branch f['state'] for f in await agather( egta.get_simulations(asc=True, column='state'))) assert not await agather(egta.get_simulations(page_start=2)) await sched2.add_profile('a: 2 1; b: 1 5, 2 6', 21) assert len(await agather(egta.get_simulations(page_start=2))) == 1
async def test_authfile(): """Test supplying auth file""" async with mockserver.server(): with stdin(''): assert await run('-f-', 'sim')
async def test_simulator(): """Test simulator api""" async with mockserver.server() as server, \ api.api('', num_tries=3, retry_delay=0.5) as egta: sim = await create_simulator(server, egta, 'sim', '1') info = await sim.get_info() validate_object( info, { 'configuration': { 'type': 'object' }, 'created_at': { 'type': 'string' }, 'email': { 'type': 'string' }, 'id': { 'type': 'integer' }, 'name': { 'type': 'string' }, 'role_configuration': { 'type': 'object' }, 'source': { 'type': 'object' }, 'updated_at': { 'type': 'string' }, 'url': { 'type': 'string' }, 'version': { 'type': 'string' }, }) role_conf = {'a': ['1', '2', '3', '4'], 'b': ['5', '6', '7']} assert info['role_configuration'] == role_conf await asyncio.sleep(1) await sim.remove_strategy('a', '3') new_role_conf = {'a': ['1', '2', '4'], 'b': ['5', '6', '7']} new_info = await sim.get_info() assert new_info['role_configuration'] == new_role_conf assert new_info['updated_at'] != info['updated_at'] assert info['role_configuration'] == role_conf await sim.remove_role('b') new_role_conf = {'a': ['1', '2', '4']} new_info = await sim.get_info() assert new_info['role_configuration'] == new_role_conf # Stale object didn't update assert info['role_configuration'] == role_conf # Add existing strategy await sim.add_strategy('a', '1') new_info = await sim.get_info() assert new_info['role_configuration'] == new_role_conf await sim.add_strategy('a', '2') await sim.add_strategy('a', '3') new_role_conf = {'a': ['1', '2', '3', '4']} new_info = await sim.get_info() assert new_info['role_configuration'] == new_role_conf await sim.remove_strategies({'a': ['4', '5', '4']}) new_role_conf = {'a': ['1', '2', '3']} new_info = await sim.get_info() assert new_info['role_configuration'] == new_role_conf await sim.remove_role('c') with pytest.raises(KeyError): await sim.add_strategy('c', '8') # Shouldn't raise exception, because removals never do await sim.remove_strategies({'c': ['8']})
async def test_game(tmpdir): # pylint: disable=too-many-statements """Test game functionality""" conf = str(tmpdir.join('conf.json')) with open(conf, 'w') as fil: json.dump({}, fil) async with mockserver.server() as server: with stdout() as out, stderr() as err: assert await run('-a', '', 'game'), err.getvalue() assert not out.getvalue() sim_id = server.create_simulator('sim', '1') game_spec = { 'players': { 'r': 2, }, 'strategies': { 'r': ['s0', 's1'], }, } with stdin(json.dumps(game_spec['strategies'])), stderr() as err: assert await run('-a', '', 'sim', str(sim_id), '-j-'), err.getvalue() # get canon game with stdin(json.dumps(game_spec)), stdout() as out, \ stderr() as err: assert await run('-a', '', 'game', str(sim_id), '-j-', '--fetch-conf', conf), err.getvalue() game = json.loads(out.getvalue()) # verify its now listed with games with stdout() as out, stderr() as err: assert await run('-a', '', 'game'), err.getvalue() game2 = json.loads(out.getvalue()) assert game == game2 # get game structure with stdout() as out, stderr() as err: assert await run('-a', '', 'game', str(game['id'])), err.getvalue() struct = json.loads(out.getvalue()) with stdin(json.dumps(game_spec)), stdout() as out, \ stderr() as err: assert await run('-a', '', 'game', str(sim_id), '-j-', '--fetch-conf', conf), err.getvalue() assert struct == json.loads(out.getvalue()) # get game summary with stdout() as out, stderr() as err: assert await run('-a', '', 'game', str(game['id']), '--summary'), err.getvalue() summ = json.loads(out.getvalue()) with stdin(json.dumps(game_spec)), stdout() as out, \ stderr() as err: assert await run('-a', '', 'game', str(sim_id), '-j-', '--fetch-conf', conf, '--summary'), err.getvalue() assert summ == json.loads(out.getvalue()) # get observations with stderr() as err: assert await run('-a', '', 'game', str(game['id']), '--observations'), err.getvalue() obs = json.loads(out.getvalue()) with stdin(json.dumps(game_spec)), stdout() as out, \ stderr() as err: assert await run('-a', '', 'game', str(sim_id), '-j-', '--fetch-conf', conf, '--observations'), err.getvalue() assert obs == json.loads(out.getvalue()) # get full data with stderr() as err: assert await run('-a', '', 'game', str(game['id']), '--full'), err.getvalue() full = json.loads(out.getvalue()) with stdin(json.dumps(game_spec)), stdout() as out, \ stderr() as err: assert await run('-a', '', 'game', str(sim_id), '-j-', '--fetch-conf', conf, '--full'), err.getvalue() assert full == json.loads(out.getvalue()) # test name works with stdout() as out, stderr() as err: assert await run('-a', '', 'game', game['name'], '-n'), err.getvalue() assert game['id'] == json.loads(out.getvalue())['id'] # remove strategy with stderr() as err: assert await run('-a', '', 'game', str(game['id']), '-drr', '-ss0'), err.getvalue() # remove strategys with stdin(json.dumps({'r': ['s1']})), stderr() as err: assert await run('-a', '', 'game', str(game['id']), '-dj-'), err.getvalue() # remove role with stderr() as err: assert await run('-a', '', 'game', str(game['id']), '-drr'), err.getvalue() # add role assert not await run('-a', '', 'game', str(game['id']), '-rr') with stderr() as err: assert await run('-a', '', 'game', str(game['id']), '-rr', '-c2'), err.getvalue() # add strategies with stdin(json.dumps({'r': ['s1']})), stderr() as err: assert await run('-a', '', 'game', str(game['id']), '-j-'), err.getvalue() # add strategy with stderr() as err: assert await run('-a', '', 'game', str(game['id']), '-rr', '-ss0'), err.getvalue()