def test_master_driver_shutdown_sequence(kind, master_cmd, service_cmd, client, tmpdir): spec = skein.ApplicationSpec( name="test_master_driver_shutdown_sequence_%s" % kind, master=skein.Master(script=master_cmd), services={ 'service': skein.Service( resources=skein.Resources(memory=128, vcores=1), script=service_cmd ) } ) state = 'SUCCEEDED' if kind.endswith('succeeds') else 'FAILED' if kind == 'service_succeeds': with run_application(client, spec=spec) as app: wait_for_containers(app, 1, states=['SUCCEEDED']) assert len(app.get_containers()) == 0 # App hangs around until driver completes app.shutdown() assert wait_for_completion(client, app.id) == state else: with run_application(client, spec=spec, connect=False) as app_id: # service_fails results in immediate failure # driver_succeeds results in immediate success # driver_fails results in immediate failure assert wait_for_completion(client, app_id) == state
def test_allow_failures_max_restarts(client, allow_failures): name = "test_max_restarts_allow_failures_%s" % str(allow_failures).lower() spec = skein.ApplicationSpec( name=name, master=skein.Master( script="sleep infinity" ), services={ 'myservice': skein.Service( instances=1, max_restarts=2, allow_failures=allow_failures, resources=skein.Resources(memory=128, vcores=1), script="exit 1" ) } ) with run_application(client, spec=spec) as app: if allow_failures: # Service failed 3 times, doesn't keep trying to run more wait_for_containers(app, 3, states=['FAILED']) # Check still running fine after 3 failures time.sleep(0.5) app.get_specification() # Shutdown manually app.shutdown() assert wait_for_completion(client, app.id) == 'SUCCEEDED' else: # Service failed 3 times and then terminates assert wait_for_completion(client, app.id) == 'FAILED'
def test_fail_on_container_failure(client, with_restarts): script = ('if [[ "$SKEIN_CONTAINER_ID" != "test_0" ]]; then\n' ' exit 1\n' 'else\n' ' sleep infinity\n' 'fi') spec = skein.ApplicationSpec( name="test_fail_on_container_failure", services={ 'test': skein.Service(instances=2, max_restarts=2 if with_restarts else 0, resources=skein.Resources(memory=32, vcores=1), script=script) }) with run_application(client, spec=spec) as app: wait_for_completion(client, app.id) == "FAILED" logs = get_logs(app.id) assert "test_0" in logs assert "test_1" in logs assert ("test_2" in logs) == with_restarts assert ("test_3" in logs) == with_restarts assert "test_4" not in logs
def test_memory_limit_exceeded(kind, client): resources = skein.Resources(memory=128, vcores=1) # Allocate noticeably more memory than the 128 MB limit script = 'python -c "b = bytearray(int(256e6)); import time; time.sleep(10)"' master = services = None if kind == 'master': master = skein.Master(resources=resources, script=script) search_txt = "memory limit" else: services = { 'service': skein.Service(resources=resources, script=script) } search_txt = "memory used" spec = skein.ApplicationSpec(name="test_memory_limit_exceeded_%s" % kind, queue="default", master=master, services=services) with run_application(client, spec=spec, connect=False) as app_id: assert wait_for_completion(client, app_id) == "FAILED" logs = get_logs(app_id) assert search_txt in logs if kind == 'master': report = client.application_report(app_id) assert 'memory limit' in report.diagnostics
def test_container_environment(client, has_kerberos_enabled): commands = [ 'env', 'echo "LOGIN_ID=[$(whoami)]"', 'hdfs dfs -touchz /user/testuser/test_container_permissions' ] service = skein.Service(resources=skein.Resources(memory=128, vcores=1), commands=commands) spec = skein.ApplicationSpec(name="test_container_permissions", queue="default", services={'service': service}) with run_application(client, spec=spec) as app: assert wait_for_completion(client, app.id) == 'SUCCEEDED' logs = get_logs(app.id) assert "USER=testuser" in logs assert 'SKEIN_APPMASTER_ADDRESS=' in logs assert 'SKEIN_APPLICATION_ID=%s' % app.id in logs assert 'SKEIN_CONTAINER_ID=service_0' in logs assert 'SKEIN_RESOURCE_MEMORY=128' in logs assert 'SKEIN_RESOURCE_VCORES=1' in logs if has_kerberos_enabled: assert "LOGIN_ID=[testuser]" in logs assert "HADOOP_USER_NAME" not in logs else: assert "LOGIN_ID=[yarn]" in logs assert "HADOOP_USER_NAME" in logs
def test_file_systems(client): script = 'hdfs dfs -touchz /user/testuser/test_file_systems' service = skein.Service(resources=skein.Resources(memory=128, vcores=1), script=script) spec = skein.ApplicationSpec(name="test_file_systems", queue="default", services={'service': service}, file_systems=["hdfs://master.example.com:9000"]) with run_application(client, spec=spec) as app: assert wait_for_completion(client, app.id) == 'SUCCEEDED'
def test_shutdown_arguments(client): status = 'killed' diagnostics = 'This is a test diagnostic message' with run_application(client) as app: app.shutdown(status, diagnostics) wait_for_completion(client, app.id) == 'KILLED' # There's a noticeable lag in the YARN resource manager between an # application being marked as finished and its diagnostics message being # updated. Retry up to 5 seconds before failing. timeout = 5 while timeout: report = client.application_report(app.id) if report.diagnostics: break time.sleep(0.1) timeout -= 0.1 assert report.diagnostics == diagnostics assert report.final_status == status
def test_add_container(client): script = ('echo "$SKEIN_CONTAINER_ID - MYENV=$MYENV"\n' 'echo "$SKEIN_CONTAINER_ID - MYENV2=$MYENV2"\n' 'if [[ "$MYENV" == "bar" ]]; then\n' ' exit 1\n' 'else\n' ' exit 0\n' 'fi') spec = skein.ApplicationSpec(name="test_add_container", master=skein.Master(script="sleep infinity"), services={ 'test': skein.Service(instances=0, resources=skein.Resources( memory=32, vcores=1), env={ 'MYENV': 'foo', 'MYENV2': 'baz' }, max_restarts=1, script=script) }) with run_application(client, spec=spec) as app: # Add container with new overrides c = app.add_container('test') assert c.instance == 0 wait_for_containers(app, 1, states=['RUNNING', 'SUCCEEDED']) # Non-existant service with pytest.raises(ValueError): app.add_container('foobar') # Add container with override for MYENV c = app.add_container('test', {'MYENV': 'bar'}) assert c.instance == 1 # The new env var triggers a failure, should fail twice, # then fail the whole application assert wait_for_completion(client, app.id) == 'FAILED' logs = get_logs(app.id) assert "test_0 - MYENV=foo" in logs assert "test_0 - MYENV2=baz" in logs assert "test_1 - MYENV=bar" in logs assert "test_1 - MYENV2=baz" in logs assert "test_2 - MYENV=bar" in logs assert "test_2 - MYENV2=baz" in logs assert "test_3" not in logs
def test_set_log_level(client): service = skein.Service(resources=skein.Resources(memory=128, vcores=1), script='ls') spec = skein.ApplicationSpec(name="test_custom_log4j_properties", queue="default", master=skein.Master(log_level='debug'), services={'service': service}) with run_application(client, spec=spec) as app: assert wait_for_completion(client, app.id) == 'SUCCEEDED' logs = get_logs(app.id) assert 'DEBUG' in logs
def test_memory_limit_exceeded(client): # Allocate noticeably more memory than the 128 MB limit service = skein.Service( resources=skein.Resources(memory=128, vcores=1), commands=[ 'python -c "b = bytearray(int(256e6)); import time; time.sleep(10)"' ]) spec = skein.ApplicationSpec(name="test_memory_limit_exceeded", queue="default", services={"service": service}) with run_application(client, spec=spec) as app: assert wait_for_completion(client, app.id) == "FAILED" logs = get_logs(app.id) assert "memory used" in logs
def test_custom_log4j_properties(client, tmpdir): configpath = str(tmpdir.join("log4j.properties")) service = skein.Service(resources=skein.Resources(memory=128, vcores=1), script='ls') spec = skein.ApplicationSpec(name="test_custom_log4j_properties", queue="default", master=skein.Master(log_config=configpath), services={'service': service}) with open(configpath, 'w') as f: f.write(custom_log4j_properties) with run_application(client, spec=spec) as app: assert wait_for_completion(client, app.id) == 'SUCCEEDED' logs = get_logs(app.id) assert 'CUSTOM-LOG4J-SUCCEEDED' in logs
def test_master_driver_foo(client, tmpdir): filpath = str(tmpdir.join("dummy-file")) with open(filpath, 'w') as fil: fil.write('foobar') spec = skein.ApplicationSpec(name="test_master_driver", master=skein.Master(script='ls\nenv', env={'FOO': 'BAR'}, files={'myfile': filpath})) with run_application(client, spec=spec, connect=False) as app_id: assert wait_for_completion(client, app_id) == 'SUCCEEDED' logs = get_logs(app_id) assert 'FOO=BAR' in logs assert 'myfile' in logs
def test_retries_succeeds(client): hdfs = pytest.importorskip('pyarrow.hdfs') spec = skein.ApplicationSpec( name="test_application_retries_succeeds", max_attempts=2, master=skein.Master(script=test_retries_script_template.format( succeed_on='02'))) with run_application(client, spec=spec, connect=False) as app_id: assert wait_for_completion(client, app_id) == 'SUCCEEDED' logs = get_logs(app_id) assert 'Failing on other attempts' in logs assert 'Application attempt 1 out of 2 failed, will retry' in logs assert 'Succeeding on attempt 02' in logs fs = hdfs.connect() assert not fs.exists("/user/testuser/.skein/%s" % app_id)
def test_retries_fails(client): hdfs = pytest.importorskip('pyarrow.hdfs') # Global maximum is 2, checks that appmaster uses 2 instead of 10 max_attempts = 10 spec = skein.ApplicationSpec( name="test_application_retries_fails", max_attempts=max_attempts, master=skein.Master(script=test_retries_script_template.format( succeed_on='03'))) with run_application(client, spec=spec, connect=False) as app_id: assert wait_for_completion(client, app_id) == 'FAILED' logs = get_logs(app_id) assert logs.count('Failing on other attempts') == 2 assert 'Application attempt 1 out of 2 failed' in logs fs = hdfs.connect() assert not fs.exists("/user/testuser/.skein/%s" % app_id)
def test_container_environment(runon, client, has_kerberos_enabled): script = ('set -e\n' 'env\n' 'echo "LOGIN_ID=[$(whoami)]"\n' 'hdfs dfs -touchz /user/testuser/test_container_permissions\n' 'yarn application -list') kwargs = dict(resources=skein.Resources(memory=512, vcores=1), script=script) services = master = None if runon == 'service': services = {'service': skein.Service(**kwargs)} else: master = skein.Master(**kwargs) spec = skein.ApplicationSpec(name="test_container_permissions_%s" % runon, queue="default", services=services, master=master) with run_application(client, spec=spec, connect=False) as app_id: assert wait_for_completion(client, app_id) == 'SUCCEEDED' logs = get_logs(app_id) assert "USER=testuser" in logs assert 'SKEIN_APPMASTER_ADDRESS=' in logs assert 'SKEIN_APPLICATION_ID=%s' % app_id in logs if runon == 'service': assert 'SKEIN_CONTAINER_ID=service_0' in logs assert 'SKEIN_RESOURCE_MEMORY=512' in logs assert 'SKEIN_RESOURCE_VCORES=1' in logs assert 'CLASSPATH' not in logs if has_kerberos_enabled: assert "LOGIN_ID=[testuser]" in logs assert "HADOOP_USER_NAME" not in logs else: assert "LOGIN_ID=[yarn]" in logs assert "HADOOP_USER_NAME" in logs