def test_master_driver_shutdown_sequence(kind, master_cmd, service_cmd,
client, tmpdir):
spec = skein.ApplicationSpec(
name="test_master_driver_shutdown_sequence_%s" % kind,
master=skein.Master(script=master_cmd),
services={
'service': skein.Service(
resources=skein.Resources(memory=128, vcores=1),
script=service_cmd
)
}
)
state = 'SUCCEEDED' if kind.endswith('succeeds') else 'FAILED'
if kind == 'service_succeeds':
with run_application(client, spec=spec) as app:
wait_for_containers(app, 1, states=['SUCCEEDED'])
assert len(app.get_containers()) == 0
# App hangs around until driver completes
app.shutdown()
assert wait_for_completion(client, app.id) == state
else:
with run_application(client, spec=spec, connect=False) as app_id:
# service_fails results in immediate failure
# driver_succeeds results in immediate success
# driver_fails results in immediate failure
assert wait_for_completion(client, app_id) == state
def test_allow_failures_max_restarts(client, allow_failures):
name = "test_max_restarts_allow_failures_%s" % str(allow_failures).lower()
spec = skein.ApplicationSpec(
name=name,
master=skein.Master(
script="sleep infinity"
),
services={
'myservice': skein.Service(
instances=1,
max_restarts=2,
allow_failures=allow_failures,
resources=skein.Resources(memory=128, vcores=1),
script="exit 1"
)
}
)
with run_application(client, spec=spec) as app:
if allow_failures:
# Service failed 3 times, doesn't keep trying to run more
wait_for_containers(app, 3, states=['FAILED'])
# Check still running fine after 3 failures
time.sleep(0.5)
app.get_specification()
# Shutdown manually
app.shutdown()
assert wait_for_completion(client, app.id) == 'SUCCEEDED'
else:
# Service failed 3 times and then terminates
assert wait_for_completion(client, app.id) == 'FAILED'
def test_fail_on_container_failure(client, with_restarts):
script = ('if [[ "$SKEIN_CONTAINER_ID" != "test_0" ]]; then\n'
' exit 1\n'
'else\n'
' sleep infinity\n'
'fi')
spec = skein.ApplicationSpec(
name="test_fail_on_container_failure",
services={
'test':
skein.Service(instances=2,
max_restarts=2 if with_restarts else 0,
resources=skein.Resources(memory=32, vcores=1),
script=script)
})
with run_application(client, spec=spec) as app:
wait_for_completion(client, app.id) == "FAILED"
logs = get_logs(app.id)
assert "test_0" in logs
assert "test_1" in logs
assert ("test_2" in logs) == with_restarts
assert ("test_3" in logs) == with_restarts
assert "test_4" not in logs
def test_memory_limit_exceeded(kind, client):
resources = skein.Resources(memory=128, vcores=1)
# Allocate noticeably more memory than the 128 MB limit
script = 'python -c "b = bytearray(int(256e6)); import time; time.sleep(10)"'
master = services = None
if kind == 'master':
master = skein.Master(resources=resources, script=script)
search_txt = "memory limit"
else:
services = {
'service': skein.Service(resources=resources, script=script)
}
search_txt = "memory used"
spec = skein.ApplicationSpec(name="test_memory_limit_exceeded_%s" % kind,
queue="default",
master=master,
services=services)
with run_application(client, spec=spec, connect=False) as app_id:
assert wait_for_completion(client, app_id) == "FAILED"
logs = get_logs(app_id)
assert search_txt in logs
if kind == 'master':
report = client.application_report(app_id)
assert 'memory limit' in report.diagnostics
def test_container_environment(client, has_kerberos_enabled):
commands = [
'env', 'echo "LOGIN_ID=[$(whoami)]"',
'hdfs dfs -touchz /user/testuser/test_container_permissions'
]
service = skein.Service(resources=skein.Resources(memory=128, vcores=1),
commands=commands)
spec = skein.ApplicationSpec(name="test_container_permissions",
queue="default",
services={'service': service})
with run_application(client, spec=spec) as app:
assert wait_for_completion(client, app.id) == 'SUCCEEDED'
logs = get_logs(app.id)
assert "USER=testuser" in logs
assert 'SKEIN_APPMASTER_ADDRESS=' in logs
assert 'SKEIN_APPLICATION_ID=%s' % app.id in logs
assert 'SKEIN_CONTAINER_ID=service_0' in logs
assert 'SKEIN_RESOURCE_MEMORY=128' in logs
assert 'SKEIN_RESOURCE_VCORES=1' in logs
if has_kerberos_enabled:
assert "LOGIN_ID=[testuser]" in logs
assert "HADOOP_USER_NAME" not in logs
else:
assert "LOGIN_ID=[yarn]" in logs
assert "HADOOP_USER_NAME" in logs
def test_file_systems(client):
script = 'hdfs dfs -touchz /user/testuser/test_file_systems'
service = skein.Service(resources=skein.Resources(memory=128, vcores=1),
script=script)
spec = skein.ApplicationSpec(name="test_file_systems",
queue="default",
services={'service': service},
file_systems=["hdfs://master.example.com:9000"])
with run_application(client, spec=spec) as app:
assert wait_for_completion(client, app.id) == 'SUCCEEDED'
def test_shutdown_arguments(client):
status = 'killed'
diagnostics = 'This is a test diagnostic message'
with run_application(client) as app:
app.shutdown(status, diagnostics)
wait_for_completion(client, app.id) == 'KILLED'
# There's a noticeable lag in the YARN resource manager between an
# application being marked as finished and its diagnostics message being
# updated. Retry up to 5 seconds before failing.
timeout = 5
while timeout:
report = client.application_report(app.id)
if report.diagnostics:
break
time.sleep(0.1)
timeout -= 0.1
assert report.diagnostics == diagnostics
assert report.final_status == status
def test_add_container(client):
script = ('echo "$SKEIN_CONTAINER_ID - MYENV=$MYENV"\n'
'echo "$SKEIN_CONTAINER_ID - MYENV2=$MYENV2"\n'
'if [[ "$MYENV" == "bar" ]]; then\n'
' exit 1\n'
'else\n'
' exit 0\n'
'fi')
spec = skein.ApplicationSpec(name="test_add_container",
master=skein.Master(script="sleep infinity"),
services={
'test':
skein.Service(instances=0,
resources=skein.Resources(
memory=32, vcores=1),
env={
'MYENV': 'foo',
'MYENV2': 'baz'
},
max_restarts=1,
script=script)
})
with run_application(client, spec=spec) as app:
# Add container with new overrides
c = app.add_container('test')
assert c.instance == 0
wait_for_containers(app, 1, states=['RUNNING', 'SUCCEEDED'])
# Non-existant service
with pytest.raises(ValueError):
app.add_container('foobar')
# Add container with override for MYENV
c = app.add_container('test', {'MYENV': 'bar'})
assert c.instance == 1
# The new env var triggers a failure, should fail twice,
# then fail the whole application
assert wait_for_completion(client, app.id) == 'FAILED'
logs = get_logs(app.id)
assert "test_0 - MYENV=foo" in logs
assert "test_0 - MYENV2=baz" in logs
assert "test_1 - MYENV=bar" in logs
assert "test_1 - MYENV2=baz" in logs
assert "test_2 - MYENV=bar" in logs
assert "test_2 - MYENV2=baz" in logs
assert "test_3" not in logs
def test_set_log_level(client):
service = skein.Service(resources=skein.Resources(memory=128, vcores=1),
script='ls')
spec = skein.ApplicationSpec(name="test_custom_log4j_properties",
queue="default",
master=skein.Master(log_level='debug'),
services={'service': service})
with run_application(client, spec=spec) as app:
assert wait_for_completion(client, app.id) == 'SUCCEEDED'
logs = get_logs(app.id)
assert 'DEBUG' in logs
def test_memory_limit_exceeded(client):
# Allocate noticeably more memory than the 128 MB limit
service = skein.Service(
resources=skein.Resources(memory=128, vcores=1),
commands=[
'python -c "b = bytearray(int(256e6)); import time; time.sleep(10)"'
])
spec = skein.ApplicationSpec(name="test_memory_limit_exceeded",
queue="default",
services={"service": service})
with run_application(client, spec=spec) as app:
assert wait_for_completion(client, app.id) == "FAILED"
logs = get_logs(app.id)
assert "memory used" in logs
def test_custom_log4j_properties(client, tmpdir):
configpath = str(tmpdir.join("log4j.properties"))
service = skein.Service(resources=skein.Resources(memory=128, vcores=1),
script='ls')
spec = skein.ApplicationSpec(name="test_custom_log4j_properties",
queue="default",
master=skein.Master(log_config=configpath),
services={'service': service})
with open(configpath, 'w') as f:
f.write(custom_log4j_properties)
with run_application(client, spec=spec) as app:
assert wait_for_completion(client, app.id) == 'SUCCEEDED'
logs = get_logs(app.id)
assert 'CUSTOM-LOG4J-SUCCEEDED' in logs
def test_master_driver_foo(client, tmpdir):
filpath = str(tmpdir.join("dummy-file"))
with open(filpath, 'w') as fil:
fil.write('foobar')
spec = skein.ApplicationSpec(name="test_master_driver",
master=skein.Master(script='ls\nenv',
env={'FOO': 'BAR'},
files={'myfile':
filpath}))
with run_application(client, spec=spec, connect=False) as app_id:
assert wait_for_completion(client, app_id) == 'SUCCEEDED'
logs = get_logs(app_id)
assert 'FOO=BAR' in logs
assert 'myfile' in logs
def test_retries_succeeds(client):
hdfs = pytest.importorskip('pyarrow.hdfs')
spec = skein.ApplicationSpec(
name="test_application_retries_succeeds",
max_attempts=2,
master=skein.Master(script=test_retries_script_template.format(
succeed_on='02')))
with run_application(client, spec=spec, connect=False) as app_id:
assert wait_for_completion(client, app_id) == 'SUCCEEDED'
logs = get_logs(app_id)
assert 'Failing on other attempts' in logs
assert 'Application attempt 1 out of 2 failed, will retry' in logs
assert 'Succeeding on attempt 02' in logs
fs = hdfs.connect()
assert not fs.exists("/user/testuser/.skein/%s" % app_id)
def test_retries_fails(client):
hdfs = pytest.importorskip('pyarrow.hdfs')
# Global maximum is 2, checks that appmaster uses 2 instead of 10
max_attempts = 10
spec = skein.ApplicationSpec(
name="test_application_retries_fails",
max_attempts=max_attempts,
master=skein.Master(script=test_retries_script_template.format(
succeed_on='03')))
with run_application(client, spec=spec, connect=False) as app_id:
assert wait_for_completion(client, app_id) == 'FAILED'
logs = get_logs(app_id)
assert logs.count('Failing on other attempts') == 2
assert 'Application attempt 1 out of 2 failed' in logs
fs = hdfs.connect()
assert not fs.exists("/user/testuser/.skein/%s" % app_id)
def test_container_environment(runon, client, has_kerberos_enabled):
script = ('set -e\n'
'env\n'
'echo "LOGIN_ID=[$(whoami)]"\n'
'hdfs dfs -touchz /user/testuser/test_container_permissions\n'
'yarn application -list')
kwargs = dict(resources=skein.Resources(memory=512, vcores=1),
script=script)
services = master = None
if runon == 'service':
services = {'service': skein.Service(**kwargs)}
else:
master = skein.Master(**kwargs)
spec = skein.ApplicationSpec(name="test_container_permissions_%s" % runon,
queue="default",
services=services,
master=master)
with run_application(client, spec=spec, connect=False) as app_id:
assert wait_for_completion(client, app_id) == 'SUCCEEDED'
logs = get_logs(app_id)
assert "USER=testuser" in logs
assert 'SKEIN_APPMASTER_ADDRESS=' in logs
assert 'SKEIN_APPLICATION_ID=%s' % app_id in logs
if runon == 'service':
assert 'SKEIN_CONTAINER_ID=service_0' in logs
assert 'SKEIN_RESOURCE_MEMORY=512' in logs
assert 'SKEIN_RESOURCE_VCORES=1' in logs
assert 'CLASSPATH' not in logs
if has_kerberos_enabled:
assert "LOGIN_ID=[testuser]" in logs
assert "HADOOP_USER_NAME" not in logs
else:
assert "LOGIN_ID=[yarn]" in logs
assert "HADOOP_USER_NAME" in logs
