def test_master_driver_shutdown_sequence(kind, master_cmd, service_cmd, client, tmpdir): spec = skein.ApplicationSpec( name="test_master_driver_shutdown_sequence_%s" % kind, master=skein.Master(script=master_cmd), services={ 'service': skein.Service( resources=skein.Resources(memory=128, vcores=1), script=service_cmd ) } ) state = 'SUCCEEDED' if kind.endswith('succeeds') else 'FAILED' if kind == 'service_succeeds': with run_application(client, spec=spec) as app: wait_for_containers(app, 1, states=['SUCCEEDED']) assert len(app.get_containers()) == 0 # App hangs around until driver completes app.shutdown() assert wait_for_completion(client, app.id) == state else: with run_application(client, spec=spec, connect=False) as app_id: # service_fails results in immediate failure # driver_succeeds results in immediate success # driver_fails results in immediate failure assert wait_for_completion(client, app_id) == state
def test_allow_failures_max_restarts(client, allow_failures): name = "test_max_restarts_allow_failures_%s" % str(allow_failures).lower() spec = skein.ApplicationSpec( name=name, master=skein.Master( script="sleep infinity" ), services={ 'myservice': skein.Service( instances=1, max_restarts=2, allow_failures=allow_failures, resources=skein.Resources(memory=128, vcores=1), script="exit 1" ) } ) with run_application(client, spec=spec) as app: if allow_failures: # Service failed 3 times, doesn't keep trying to run more wait_for_containers(app, 3, states=['FAILED']) # Check still running fine after 3 failures time.sleep(0.5) app.get_specification() # Shutdown manually app.shutdown() assert wait_for_completion(client, app.id) == 'SUCCEEDED' else: # Service failed 3 times and then terminates assert wait_for_completion(client, app.id) == 'FAILED'
def test_node_locality(client, strict): if strict: relax_locality = False nodes = ['worker.example.com'] racks = [] else: relax_locality = True nodes = ['not.a.real.host.name'] racks = ['not.a.real.rack.name'] service = skein.Service( resources=skein.Resources(memory=128, vcores=1), script='sleep infinity', nodes=nodes, racks=racks, relax_locality=relax_locality ) spec = skein.ApplicationSpec(name="test_node_locality", queue="default", services={"service": service}) with run_application(client, spec=spec) as app: wait_for_containers(app, 1, states=['RUNNING']) spec2 = app.get_specification() app.shutdown() service2 = spec2.services['service'] assert service2.nodes == nodes assert service2.racks == racks assert service2.relax_locality == relax_locality
def test_add_container(client): script = ('echo "$SKEIN_CONTAINER_ID - MYENV=$MYENV"\n' 'echo "$SKEIN_CONTAINER_ID - MYENV2=$MYENV2"\n' 'if [[ "$MYENV" == "bar" ]]; then\n' ' exit 1\n' 'else\n' ' exit 0\n' 'fi') spec = skein.ApplicationSpec(name="test_add_container", master=skein.Master(script="sleep infinity"), services={ 'test': skein.Service(instances=0, resources=skein.Resources( memory=32, vcores=1), env={ 'MYENV': 'foo', 'MYENV2': 'baz' }, max_restarts=1, script=script) }) with run_application(client, spec=spec) as app: # Add container with new overrides c = app.add_container('test') assert c.instance == 0 wait_for_containers(app, 1, states=['RUNNING', 'SUCCEEDED']) # Non-existant service with pytest.raises(ValueError): app.add_container('foobar') # Add container with override for MYENV c = app.add_container('test', {'MYENV': 'bar'}) assert c.instance == 1 # The new env var triggers a failure, should fail twice, # then fail the whole application assert wait_for_completion(client, app.id) == 'FAILED' logs = get_logs(app.id) assert "test_0 - MYENV=foo" in logs assert "test_0 - MYENV2=baz" in logs assert "test_1 - MYENV=bar" in logs assert "test_1 - MYENV2=baz" in logs assert "test_2 - MYENV=bar" in logs assert "test_2 - MYENV2=baz" in logs assert "test_3" not in logs
def ui_test_app(client, has_kerberos_enabled): if has_kerberos_enabled: pytest.skip("Testing only implemented for simple authentication") with run_application(client, spec=spec) as app: # Wait for a single container wait_for_containers(app, 1, states=['RUNNING']) try: yield app finally: try: app.shutdown() except ConnectionError: client.kill_application(app.id)
def test_webui_acls(client, has_kerberos_enabled, ui_users, checks): if has_kerberos_enabled: pytest.skip("Testing only implemented for simple authentication") service = skein.Service(resources=skein.Resources(memory=128, vcores=1), commands=['sleep infinity']) spec = skein.ApplicationSpec(name="test_webui_acls", queue="default", acls=skein.ACLs(enable=True, ui_users=ui_users), services={'sleeper': service}) with run_application(client, spec=spec) as app: # Wait for a single container initial = wait_for_containers(app, 1, states=['RUNNING']) assert initial[0].state == 'RUNNING' assert initial[0].service_name == 'sleeper' # Base url of web ui base = 'http://master.example.com:8088/proxy/%s' % app.id # Check proper subset of users allowed for user, ok in checks: resp = get_page(base + "?user.name=%s" % user) assert resp.ok == ok app.shutdown()
def test_webui(client, has_kerberos_enabled): # Smoke-tests for webui if has_kerberos_enabled: pytest.skip("Testing only implemented for simple authentication") requests = pytest.importorskip('requests') with run_application(client) as app: # Wait for a single container initial = wait_for_containers(app, 1, states=['RUNNING']) assert initial[0].state == 'RUNNING' assert initial[0].service_name == 'sleeper' # Set some key-values app.kv['foo'] = b'bar' app.kv['bad'] = b'\255\255\255' # non-unicode # Base url of web ui base = 'http://master.example.com:8088/proxy/%s' % app.id # Fails without authentication resp = requests.get(base) assert resp.status_code == 401 # With authentication resp = requests.get(base + "?user.name=testuser") assert resp.ok cookies = resp.cookies # / and /services are the same for suffix in ['', '/services']: resp = requests.get(base + suffix, cookies=cookies) assert resp.ok assert 'sleeper_0' in resp.text # list of containers assert '/testuser/sleeper.log' in resp.text # link to logs # /kv store has a few items in it resp = requests.get(base + '/kv', cookies=cookies) assert resp.ok assert 'foo' in resp.text assert 'bar' in resp.text assert 'bad' in resp.text assert '<binary value>' in resp.text # Resources are reachable resp = requests.get(base + '/favicon.ico', cookies=cookies) assert resp.ok # 404 for fake pages resp = requests.get(base + '/not-a-real-page', cookies=cookies) assert resp.status_code == 404 app.shutdown()
def test_cli_container(global_client, capsys): with run_application(global_client) as app: app_id = app.app_id ac = app.connect() wait_for_containers(ac, 1, states=['RUNNING']) # skein container scale run_command('container scale %s --service sleeper --number 3' % app_id) out, err = capsys.readouterr() assert not out assert not err wait_for_containers(ac, 3, services=['sleeper'], states=['RUNNING']) # skein container ls run_command('container ls %s' % app_id) out, err = capsys.readouterr() assert not err assert len(out.splitlines()) == 4 # skein container kill container_id = ac.containers()[0].id run_command('container kill %s --id %s' % (app_id, container_id)) out, err = capsys.readouterr() assert not out assert not err wait_for_containers(ac, 2, services=['sleeper'], states=['RUNNING']) # `skein container ls -a` run_command('container ls %s -a' % app_id) out, err = capsys.readouterr() assert not err assert container_id in out # Errors bubble up nicely run_command('container kill %s --id foobar_0' % app_id, error=True) out, err = capsys.readouterr() assert not out assert err.startswith('Error: ')
def test_dynamic_containers(client): with run_application(client) as app: initial = wait_for_containers(app, 1, states=['RUNNING']) assert initial[0].state == 'RUNNING' assert initial[0].service_name == 'sleeper' # Scale sleepers up to 3 containers new = app.scale('sleeper', 3) assert len(new) == 2 for c in new: assert c.state == 'REQUESTED' wait_for_containers(app, 3, services=['sleeper'], states=['RUNNING']) # Scale down to 1 container stopped = app.scale('sleeper', 1) assert len(stopped) == 2 # Stopped oldest 2 instances assert stopped[0].instance == 0 assert stopped[1].instance == 1 # Scale up to 2 containers new = app.scale('sleeper', 2) # Calling twice is no-op new2 = app.scale('sleeper', 2) assert len(new2) == 0 assert new[0].instance == 3 current = wait_for_containers(app, 2, services=['sleeper'], states=['RUNNING']) assert current[0].instance == 2 assert current[1].instance == 3 # Manually kill instance 3 app.kill_container('sleeper_3') current = app.get_containers() assert len(current) == 1 assert current[0].instance == 2 # Fine to kill already killed container app.kill_container('sleeper_1') # All killed containers killed = app.get_containers(states=['killed']) assert len(killed) == 3 assert [c.instance for c in killed] == [0, 1, 3] # Can't scale non-existant service with pytest.raises(ValueError): app.scale('foobar', 2) # Can't scale negative with pytest.raises(ValueError): app.scale('sleeper', -5) # Can't kill non-existant container with pytest.raises(ValueError): app.kill_container('foobar_1') with pytest.raises(ValueError): app.kill_container('sleeper_500') # Invalid container id with pytest.raises(ValueError): app.kill_container('fooooooo') # Can't get containers for non-existant service with pytest.raises(ValueError): app.get_containers(services=['sleeper', 'missing']) app.shutdown()
def test_dynamic_containers(client): spec = skein.ApplicationSpec(name="test_dynamic_containers", services={ 'sleeper': skein.Service(instances=1, resources=skein.Resources( memory=32, vcores=1), script='sleep infinity') }, master=skein.Master(script='sleep infinity')) with run_application(client, spec=spec) as app: initial = wait_for_containers(app, 1, states=['RUNNING']) assert initial[0].state == 'RUNNING' assert initial[0].service_name == 'sleeper' # Scale sleepers up to 3 containers new = app.scale('sleeper', 3) assert len(new) == 2 for c in new: assert c.state == 'REQUESTED' wait_for_containers(app, 3, services=['sleeper'], states=['RUNNING']) # Scale down to 1 container stopped = app.scale('sleeper', 1) assert len(stopped) == 2 # Stopped oldest 2 instances assert stopped[0].instance == 0 assert stopped[1].instance == 1 # Scale up to 2 containers new = app.scale('sleeper', 2) # Calling twice is no-op new2 = app.scale('sleeper', 2) assert len(new2) == 0 assert new[0].instance == 3 current = wait_for_containers(app, 2, services=['sleeper'], states=['RUNNING']) assert current[0].instance == 2 assert current[1].instance == 3 # Manually kill instance 3 app.kill_container('sleeper_3') current = app.get_containers() assert len(current) == 1 assert current[0].instance == 2 # Fine to kill already killed container app.kill_container('sleeper_1') # All killed containers killed = app.get_containers(states=['killed']) assert len(killed) == 3 assert [c.instance for c in killed] == [0, 1, 3] # All completed containers have an exit message assert all(c.exit_message for c in killed) # Add containers by delta ncurrent = len(app.get_containers()) new = app.scale('sleeper', delta=2) assert len(new) == 2 assert len(app.get_containers()) == ncurrent + 2 # Remove containers by delta ncurrent = len(app.get_containers()) assert ncurrent >= 1 res = app.scale('sleeper', delta=-1) assert len(res) == 1 assert len(app.get_containers()) == ncurrent - 1 # Removing more containers than active removes all containers ncurrent = len(app.get_containers()) res = app.scale('sleeper', delta=-(ncurrent + 2)) assert len(res) == ncurrent assert len(app.get_containers()) == 0 # Can't specify both count and delta with pytest.raises(ValueError): app.scale('sleeper', count=2, delta=2) # Must specify either count or delta with pytest.raises(ValueError): app.scale('sleeper') # Can't scale non-existant service with pytest.raises(ValueError): app.scale('foobar', 2) # Can't scale negative with pytest.raises(ValueError): app.scale('sleeper', -5) # Can't kill non-existant container with pytest.raises(ValueError): app.kill_container('foobar_1') with pytest.raises(ValueError): app.kill_container('sleeper_500') # Invalid container id with pytest.raises(ValueError): app.kill_container('fooooooo') # Can't get containers for non-existant service with pytest.raises(ValueError): app.get_containers(services=['sleeper', 'missing']) app.shutdown()