def test_master_driver_shutdown_sequence(kind, master_cmd, service_cmd,
client, tmpdir):
spec = skein.ApplicationSpec(
name="test_master_driver_shutdown_sequence_%s" % kind,
master=skein.Master(script=master_cmd),
services={
'service': skein.Service(
resources=skein.Resources(memory=128, vcores=1),
script=service_cmd
)
}
)
state = 'SUCCEEDED' if kind.endswith('succeeds') else 'FAILED'
if kind == 'service_succeeds':
with run_application(client, spec=spec) as app:
wait_for_containers(app, 1, states=['SUCCEEDED'])
assert len(app.get_containers()) == 0
# App hangs around until driver completes
app.shutdown()
assert wait_for_completion(client, app.id) == state
else:
with run_application(client, spec=spec, connect=False) as app_id:
# service_fails results in immediate failure
# driver_succeeds results in immediate success
# driver_fails results in immediate failure
assert wait_for_completion(client, app_id) == state
def test_allow_failures_max_restarts(client, allow_failures):
name = "test_max_restarts_allow_failures_%s" % str(allow_failures).lower()
spec = skein.ApplicationSpec(
name=name,
master=skein.Master(
script="sleep infinity"
),
services={
'myservice': skein.Service(
instances=1,
max_restarts=2,
allow_failures=allow_failures,
resources=skein.Resources(memory=128, vcores=1),
script="exit 1"
)
}
)
with run_application(client, spec=spec) as app:
if allow_failures:
# Service failed 3 times, doesn't keep trying to run more
wait_for_containers(app, 3, states=['FAILED'])
# Check still running fine after 3 failures
time.sleep(0.5)
app.get_specification()
# Shutdown manually
app.shutdown()
assert wait_for_completion(client, app.id) == 'SUCCEEDED'
else:
# Service failed 3 times and then terminates
assert wait_for_completion(client, app.id) == 'FAILED'
def test_node_locality(client, strict):
if strict:
relax_locality = False
nodes = ['worker.example.com']
racks = []
else:
relax_locality = True
nodes = ['not.a.real.host.name']
racks = ['not.a.real.rack.name']
service = skein.Service(
resources=skein.Resources(memory=128, vcores=1),
script='sleep infinity',
nodes=nodes,
racks=racks,
relax_locality=relax_locality
)
spec = skein.ApplicationSpec(name="test_node_locality",
queue="default",
services={"service": service})
with run_application(client, spec=spec) as app:
wait_for_containers(app, 1, states=['RUNNING'])
spec2 = app.get_specification()
app.shutdown()
service2 = spec2.services['service']
assert service2.nodes == nodes
assert service2.racks == racks
assert service2.relax_locality == relax_locality
def test_add_container(client):
script = ('echo "$SKEIN_CONTAINER_ID - MYENV=$MYENV"\n'
'echo "$SKEIN_CONTAINER_ID - MYENV2=$MYENV2"\n'
'if [[ "$MYENV" == "bar" ]]; then\n'
' exit 1\n'
'else\n'
' exit 0\n'
'fi')
spec = skein.ApplicationSpec(name="test_add_container",
master=skein.Master(script="sleep infinity"),
services={
'test':
skein.Service(instances=0,
resources=skein.Resources(
memory=32, vcores=1),
env={
'MYENV': 'foo',
'MYENV2': 'baz'
},
max_restarts=1,
script=script)
})
with run_application(client, spec=spec) as app:
# Add container with new overrides
c = app.add_container('test')
assert c.instance == 0
wait_for_containers(app, 1, states=['RUNNING', 'SUCCEEDED'])
# Non-existant service
with pytest.raises(ValueError):
app.add_container('foobar')
# Add container with override for MYENV
c = app.add_container('test', {'MYENV': 'bar'})
assert c.instance == 1
# The new env var triggers a failure, should fail twice,
# then fail the whole application
assert wait_for_completion(client, app.id) == 'FAILED'
logs = get_logs(app.id)
assert "test_0 - MYENV=foo" in logs
assert "test_0 - MYENV2=baz" in logs
assert "test_1 - MYENV=bar" in logs
assert "test_1 - MYENV2=baz" in logs
assert "test_2 - MYENV=bar" in logs
assert "test_2 - MYENV2=baz" in logs
assert "test_3" not in logs
def ui_test_app(client, has_kerberos_enabled):
if has_kerberos_enabled:
pytest.skip("Testing only implemented for simple authentication")
with run_application(client, spec=spec) as app:
# Wait for a single container
wait_for_containers(app, 1, states=['RUNNING'])
try:
yield app
finally:
try:
app.shutdown()
except ConnectionError:
client.kill_application(app.id)
def test_webui_acls(client, has_kerberos_enabled, ui_users, checks):
if has_kerberos_enabled:
pytest.skip("Testing only implemented for simple authentication")
service = skein.Service(resources=skein.Resources(memory=128, vcores=1),
commands=['sleep infinity'])
spec = skein.ApplicationSpec(name="test_webui_acls",
queue="default",
acls=skein.ACLs(enable=True,
ui_users=ui_users),
services={'sleeper': service})
with run_application(client, spec=spec) as app:
# Wait for a single container
initial = wait_for_containers(app, 1, states=['RUNNING'])
assert initial[0].state == 'RUNNING'
assert initial[0].service_name == 'sleeper'
# Base url of web ui
base = 'http://master.example.com:8088/proxy/%s' % app.id
# Check proper subset of users allowed
for user, ok in checks:
resp = get_page(base + "?user.name=%s" % user)
assert resp.ok == ok
app.shutdown()
def test_webui(client, has_kerberos_enabled):
# Smoke-tests for webui
if has_kerberos_enabled:
pytest.skip("Testing only implemented for simple authentication")
requests = pytest.importorskip('requests')
with run_application(client) as app:
# Wait for a single container
initial = wait_for_containers(app, 1, states=['RUNNING'])
assert initial[0].state == 'RUNNING'
assert initial[0].service_name == 'sleeper'
# Set some key-values
app.kv['foo'] = b'bar'
app.kv['bad'] = b'\255\255\255' # non-unicode
# Base url of web ui
base = 'http://master.example.com:8088/proxy/%s' % app.id
# Fails without authentication
resp = requests.get(base)
assert resp.status_code == 401
# With authentication
resp = requests.get(base + "?user.name=testuser")
assert resp.ok
cookies = resp.cookies
# / and /services are the same
for suffix in ['', '/services']:
resp = requests.get(base + suffix, cookies=cookies)
assert resp.ok
assert 'sleeper_0' in resp.text # list of containers
assert '/testuser/sleeper.log' in resp.text # link to logs
# /kv store has a few items in it
resp = requests.get(base + '/kv', cookies=cookies)
assert resp.ok
assert 'foo' in resp.text
assert 'bar' in resp.text
assert 'bad' in resp.text
assert '<binary value>' in resp.text
# Resources are reachable
resp = requests.get(base + '/favicon.ico', cookies=cookies)
assert resp.ok
# 404 for fake pages
resp = requests.get(base + '/not-a-real-page', cookies=cookies)
assert resp.status_code == 404
app.shutdown()
def test_cli_container(global_client, capsys):
with run_application(global_client) as app:
app_id = app.app_id
ac = app.connect()
wait_for_containers(ac, 1, states=['RUNNING'])
# skein container scale
run_command('container scale %s --service sleeper --number 3' % app_id)
out, err = capsys.readouterr()
assert not out
assert not err
wait_for_containers(ac, 3, services=['sleeper'], states=['RUNNING'])
# skein container ls
run_command('container ls %s' % app_id)
out, err = capsys.readouterr()
assert not err
assert len(out.splitlines()) == 4
# skein container kill
container_id = ac.containers()[0].id
run_command('container kill %s --id %s' % (app_id, container_id))
out, err = capsys.readouterr()
assert not out
assert not err
wait_for_containers(ac, 2, services=['sleeper'], states=['RUNNING'])
# `skein container ls -a`
run_command('container ls %s -a' % app_id)
out, err = capsys.readouterr()
assert not err
assert container_id in out
# Errors bubble up nicely
run_command('container kill %s --id foobar_0' % app_id, error=True)
out, err = capsys.readouterr()
assert not out
assert err.startswith('Error: ')
def test_dynamic_containers(client):
with run_application(client) as app:
initial = wait_for_containers(app, 1, states=['RUNNING'])
assert initial[0].state == 'RUNNING'
assert initial[0].service_name == 'sleeper'
# Scale sleepers up to 3 containers
new = app.scale('sleeper', 3)
assert len(new) == 2
for c in new:
assert c.state == 'REQUESTED'
wait_for_containers(app, 3, services=['sleeper'], states=['RUNNING'])
# Scale down to 1 container
stopped = app.scale('sleeper', 1)
assert len(stopped) == 2
# Stopped oldest 2 instances
assert stopped[0].instance == 0
assert stopped[1].instance == 1
# Scale up to 2 containers
new = app.scale('sleeper', 2)
# Calling twice is no-op
new2 = app.scale('sleeper', 2)
assert len(new2) == 0
assert new[0].instance == 3
current = wait_for_containers(app,
2,
services=['sleeper'],
states=['RUNNING'])
assert current[0].instance == 2
assert current[1].instance == 3
# Manually kill instance 3
app.kill_container('sleeper_3')
current = app.get_containers()
assert len(current) == 1
assert current[0].instance == 2
# Fine to kill already killed container
app.kill_container('sleeper_1')
# All killed containers
killed = app.get_containers(states=['killed'])
assert len(killed) == 3
assert [c.instance for c in killed] == [0, 1, 3]
# Can't scale non-existant service
with pytest.raises(ValueError):
app.scale('foobar', 2)
# Can't scale negative
with pytest.raises(ValueError):
app.scale('sleeper', -5)
# Can't kill non-existant container
with pytest.raises(ValueError):
app.kill_container('foobar_1')
with pytest.raises(ValueError):
app.kill_container('sleeper_500')
# Invalid container id
with pytest.raises(ValueError):
app.kill_container('fooooooo')
# Can't get containers for non-existant service
with pytest.raises(ValueError):
app.get_containers(services=['sleeper', 'missing'])
app.shutdown()
def test_dynamic_containers(client):
spec = skein.ApplicationSpec(name="test_dynamic_containers",
services={
'sleeper':
skein.Service(instances=1,
resources=skein.Resources(
memory=32, vcores=1),
script='sleep infinity')
},
master=skein.Master(script='sleep infinity'))
with run_application(client, spec=spec) as app:
initial = wait_for_containers(app, 1, states=['RUNNING'])
assert initial[0].state == 'RUNNING'
assert initial[0].service_name == 'sleeper'
# Scale sleepers up to 3 containers
new = app.scale('sleeper', 3)
assert len(new) == 2
for c in new:
assert c.state == 'REQUESTED'
wait_for_containers(app, 3, services=['sleeper'], states=['RUNNING'])
# Scale down to 1 container
stopped = app.scale('sleeper', 1)
assert len(stopped) == 2
# Stopped oldest 2 instances
assert stopped[0].instance == 0
assert stopped[1].instance == 1
# Scale up to 2 containers
new = app.scale('sleeper', 2)
# Calling twice is no-op
new2 = app.scale('sleeper', 2)
assert len(new2) == 0
assert new[0].instance == 3
current = wait_for_containers(app,
2,
services=['sleeper'],
states=['RUNNING'])
assert current[0].instance == 2
assert current[1].instance == 3
# Manually kill instance 3
app.kill_container('sleeper_3')
current = app.get_containers()
assert len(current) == 1
assert current[0].instance == 2
# Fine to kill already killed container
app.kill_container('sleeper_1')
# All killed containers
killed = app.get_containers(states=['killed'])
assert len(killed) == 3
assert [c.instance for c in killed] == [0, 1, 3]
# All completed containers have an exit message
assert all(c.exit_message for c in killed)
# Add containers by delta
ncurrent = len(app.get_containers())
new = app.scale('sleeper', delta=2)
assert len(new) == 2
assert len(app.get_containers()) == ncurrent + 2
# Remove containers by delta
ncurrent = len(app.get_containers())
assert ncurrent >= 1
res = app.scale('sleeper', delta=-1)
assert len(res) == 1
assert len(app.get_containers()) == ncurrent - 1
# Removing more containers than active removes all containers
ncurrent = len(app.get_containers())
res = app.scale('sleeper', delta=-(ncurrent + 2))
assert len(res) == ncurrent
assert len(app.get_containers()) == 0
# Can't specify both count and delta
with pytest.raises(ValueError):
app.scale('sleeper', count=2, delta=2)
# Must specify either count or delta
with pytest.raises(ValueError):
app.scale('sleeper')
# Can't scale non-existant service
with pytest.raises(ValueError):
app.scale('foobar', 2)
# Can't scale negative
with pytest.raises(ValueError):
app.scale('sleeper', -5)
# Can't kill non-existant container
with pytest.raises(ValueError):
app.kill_container('foobar_1')
with pytest.raises(ValueError):
app.kill_container('sleeper_500')
# Invalid container id
with pytest.raises(ValueError):
app.kill_container('fooooooo')
# Can't get containers for non-existant service
with pytest.raises(ValueError):
app.get_containers(services=['sleeper', 'missing'])
app.shutdown()
