def test_await_process_completion_normal(self):
task_id = get_random_task_id()
stdout_name = ensure_directory('build/stdout.' + str(task_id))
stderr_name = ensure_directory('build/stderr.' + str(task_id))
stdout = open(stdout_name, 'w+')
stderr = open(stderr_name, 'w+')
try:
command = 'sleep 2'
process = subprocess.Popen(command,
shell=True,
stdout=stdout,
stderr=stderr)
process_info = process, stdout, stderr
shutdown_grace_period_ms = 1000
stop_signal = Event()
ce.await_process_completion(stop_signal, process_info,
shutdown_grace_period_ms)
self.assertFalse(stop_signal.isSet())
self.assertEqual(0, process.returncode)
self.assertTrue(stdout.closed)
self.assertTrue(stderr.closed)
finally:
cleanup_output(stdout_name, stderr_name)
def test_await_process_completion_killed(self):
task_id = tu.get_random_task_id()
stdout_name = tu.ensure_directory('build/stdout.{}'.format(task_id))
stderr_name = tu.ensure_directory('build/stderr.{}'.format(task_id))
tu.redirect_stdout_to_file(stdout_name)
tu.redirect_stderr_to_file(stderr_name)
try:
command = 'sleep 100'
process = subprocess.Popen(command,
preexec_fn=os.setpgrp,
shell=True)
shutdown_grace_period_ms = 2000
stop_signal = Event()
sleep_and_set_stop_signal_task(stop_signal, 2)
ce.await_process_completion(process, stop_signal,
shutdown_grace_period_ms)
self.assertTrue(process.returncode < 0)
finally:
tu.cleanup_output(stdout_name, stderr_name)
def manage_task_runner(self, command, assertions_fn, stop_signal=Event()):
driver = FakeMesosExecutorDriver()
task_id = get_random_task_id()
task = {'task_id': {'value': task_id},
'data': encode_data(json.dumps({'command': command}).encode('utf8'))}
stdout_name = 'build/stdout.' + str(task_id)
stderr_name = 'build/stderr.' + str(task_id)
completed_signal = Event()
max_message_length = 300
progress_sample_interval_ms = 100
sandbox_location = '/location/to/task/sandbox/{}'.format(task_id)
progress_output_name = stdout_name
progress_regex_string = '\^\^\^\^JOB-PROGRESS: (\d*)(?: )?(.*)'
config = cc.ExecutorConfig(max_message_length=max_message_length,
progress_output_name=progress_output_name,
progress_regex_string=progress_regex_string,
progress_sample_interval_ms=progress_sample_interval_ms,
sandbox_location=sandbox_location)
try:
ce.manage_task(driver, task, stop_signal, completed_signal, config, stdout_name, stderr_name)
self.assertTrue(completed_signal.isSet())
assertions_fn(driver, task_id, sandbox_location)
finally:
cleanup_output(stdout_name, stderr_name)
def test_kill_task_terminate(self):
task_id = get_random_task_id()
stdout_name = ensure_directory('build/stdout.' + str(task_id))
stderr_name = ensure_directory('build/stderr.' + str(task_id))
stdout = open(stdout_name, 'w+')
stderr = open(stderr_name, 'w+')
try:
command = 'sleep 100'
process = subprocess.Popen(command,
shell=True,
stdout=stdout,
stderr=stderr)
process_info = process, stdout, stderr
shutdown_grace_period_ms = 2000
ce.kill_task(process_info, shutdown_grace_period_ms)
# await process termination
while process.poll() is None:
time.sleep(0.01)
self.assertEqual(-1 * signal.SIGTERM, process.poll())
self.assertTrue(stdout.closed)
self.assertTrue(stderr.closed)
finally:
cleanup_output(stdout_name, stderr_name)
def test_await_process_completion_killed(self):
task_id = get_random_task_id()
stdout_name = 'build/stdout.' + str(task_id)
stderr_name = 'build/stderr.' + str(task_id)
stdout = open(stdout_name, 'w+')
stderr = open(stderr_name, 'w+')
try:
command = 'sleep 100'
process = subprocess.Popen(command, shell=True, stdout=stdout, stderr=stderr)
process_info = process, stdout, stderr
shutdown_grace_period_ms = 2000
stop_signal = Event()
def sleep_and_set_stop_signal():
time.sleep(2 * cook.RUNNING_POLL_INTERVAL_SECS)
stop_signal.set()
thread = Thread(target=sleep_and_set_stop_signal, args=())
thread.start()
ce.await_process_completion(stop_signal, process_info, shutdown_grace_period_ms)
self.assertTrue(process.returncode < 0)
self.assertTrue(stdout.closed)
self.assertTrue(stderr.closed)
finally:
cleanup_output(stdout_name, stderr_name)
def test_kill_task_terminate_with_sigkill(self):
task_id = tu.get_random_task_id()
stdout_name = tu.ensure_directory('build/stdout.{}'.format(task_id))
stderr_name = tu.ensure_directory('build/stderr.{}'.format(task_id))
tu.redirect_stdout_to_file(stdout_name)
tu.redirect_stderr_to_file(stderr_name)
try:
command = "trap '' TERM SIGTERM; sleep 200"
process = cs.launch_process(command, {})
shutdown_grace_period_ms = 1000
group_id = cs.find_process_group(process.pid)
self.assertGreater(len(find_process_ids_in_group(group_id)), 0)
cs.kill_process(process, shutdown_grace_period_ms)
# await process termination
for i in range(1, 10 * shutdown_grace_period_ms):
if process.poll() is None:
time.sleep(0.01)
if process.poll() is None:
process.kill()
self.assertTrue(
((-1 * signal.SIGKILL) == process.poll())
or ((128 + signal.SIGKILL) == process.poll()),
'Process exited with code {}'.format(process.poll()))
self.assertEqual(len(find_process_ids_in_group(group_id)), 0)
finally:
tu.cleanup_output(stdout_name, stderr_name)
def manage_task_runner(self,
command,
assertions_fn,
stop_signal=None,
task_id=None,
config=None,
driver=None):
if driver is None:
driver = tu.FakeMesosExecutorDriver()
if stop_signal is None:
stop_signal = Event()
if task_id is None:
task_id = tu.get_random_task_id()
task = {
'task_id': {
'value': task_id
},
'data':
pm.encode_data(json.dumps({
'command': command
}).encode('utf8'))
}
stdout_name = tu.ensure_directory('build/stdout.{}'.format(task_id))
stderr_name = tu.ensure_directory('build/stderr.{}'.format(task_id))
tu.redirect_stdout_to_file(stdout_name)
tu.redirect_stderr_to_file(stderr_name)
completed_signal = Event()
if config is None:
sandbox_directory = '/location/to/task/sandbox/{}'.format(task_id)
config = cc.ExecutorConfig(
max_message_length=300,
progress_output_name=stdout_name,
progress_regex_string=
'\^\^\^\^JOB-PROGRESS:\s+([0-9]*\.?[0-9]+)($|\s+.*)',
progress_sample_interval_ms=100,
sandbox_directory=sandbox_directory)
else:
sandbox_directory = config.sandbox_directory
try:
ce.manage_task(driver, task, stop_signal, completed_signal, config)
self.assertTrue(completed_signal.isSet())
assertions_fn(driver, task_id, sandbox_directory)
finally:
tu.cleanup_output(stdout_name, stderr_name)
def test_launch_task_interactive_output(self):
task_id = tu.get_random_task_id()
command = 'echo "Start"; echo "Hello"; sleep 100; echo "World"; echo "Done"; '
task = {
'task_id': {
'value': task_id
},
'data':
pm.encode_data(json.dumps({
'command': command
}).encode('utf8'))
}
stdout_name = tu.ensure_directory('build/stdout.{}'.format(task_id))
stderr_name = tu.ensure_directory('build/stderr.{}'.format(task_id))
tu.redirect_stdout_to_file(stdout_name)
tu.redirect_stderr_to_file(stderr_name)
try:
process = ce.launch_task(task, os.environ)
self.assertIsNotNone(process)
# let the process run for up to 50 seconds
for _ in range(5000):
if cs.is_process_running(process):
time.sleep(0.01)
with open(stdout_name) as f:
stdout_content = f.read()
if 'Start' in stdout_content and 'Hello' in stdout_content:
break
try:
with open(stdout_name) as f:
stdout_content = f.read()
logging.info(
'Contents of stdout: {}'.format(stdout_content))
self.assertTrue("Start" in stdout_content)
self.assertTrue("Hello" in stdout_content)
self.assertFalse("World" in stdout_content)
self.assertFalse("Done" in stdout_content)
finally:
if process.poll() is None:
logging.info('Killing launched process')
process.kill()
finally:
tu.cleanup_output(stdout_name, stderr_name)
def test_cleanup_process(self):
task_id = get_random_task_id()
stdout_name = ensure_directory('build/stdout.' + str(task_id))
stderr_name = ensure_directory('build/stderr.' + str(task_id))
stdout = open(stdout_name, 'w+')
stderr = open(stderr_name, 'w+')
try:
process_info = None, stdout, stderr
ce.cleanup_process(process_info)
self.assertTrue(stdout.closed)
self.assertTrue(stderr.closed)
finally:
cleanup_output(stdout_name, stderr_name)
def test_launch_task(self):
task_id = get_random_task_id()
command = 'echo "Hello World"; echo "Error Message" >&2'
task = {
'task_id': {
'value': task_id
},
'data':
encode_data(json.dumps({
'command': command
}).encode('utf8'))
}
stdout_name = ensure_directory('build/stdout.' + str(task_id))
stderr_name = ensure_directory('build/stderr.' + str(task_id))
if not os.path.isdir("build"):
os.mkdir("build")
try:
process, stdout, stderr = ce.launch_task(task, stdout_name,
stderr_name)
self.assertIsNotNone(process)
for i in range(100):
if process.poll() is None:
time.sleep(0.01)
stdout.close()
stderr.close()
if process.poll() is None:
process.kill()
self.assertEqual(0, process.poll())
with open(stdout_name) as f:
stdout_content = f.read()
self.assertEqual("Hello World\n", stdout_content)
with open(stderr_name) as f:
stderr_content = f.read()
self.assertEqual("Error Message\n", stderr_content)
finally:
cleanup_output(stdout_name, stderr_name)
def test_launch_task(self):
task_id = tu.get_random_task_id()
command = 'echo "Hello World"; echo "Error Message" >&2'
task = {
'task_id': {
'value': task_id
},
'data':
pm.encode_data(json.dumps({
'command': command
}).encode('utf8'))
}
stdout_name = tu.ensure_directory('build/stdout.{}'.format(task_id))
stderr_name = tu.ensure_directory('build/stderr.{}'.format(task_id))
tu.redirect_stdout_to_file(stdout_name)
tu.redirect_stderr_to_file(stderr_name)
try:
process = ce.launch_task(task, os.environ)
self.assertIsNotNone(process)
for _ in range(100):
if cs.is_process_running(process):
time.sleep(0.01)
if process.poll() is None:
process.kill()
tu.close_sys_outputs()
self.assertEqual(0, process.poll())
with open(stdout_name) as f:
stdout_content = f.read()
self.assertTrue("Hello World\n" in stdout_content)
with open(stderr_name) as f:
stderr_content = f.read()
self.assertTrue("Error Message\n" in stderr_content)
finally:
tu.cleanup_output(stdout_name, stderr_name)
def test_kill_task_terminate_with_sigterm(self):
task_id = tu.get_random_task_id()
stdout_name = tu.ensure_directory('build/stdout.{}'.format(task_id))
stderr_name = tu.ensure_directory('build/stderr.{}'.format(task_id))
tu.redirect_stdout_to_file(stdout_name)
tu.redirect_stderr_to_file(stderr_name)
try:
command = "bash -c 'function handle_term { echo GOT TERM; }; trap handle_term SIGTERM TERM; sleep 200'"
process = cs.launch_process(command, {})
shutdown_grace_period_ms = 1000
group_id = cs.find_process_group(process.pid)
self.assertGreater(len(find_process_ids_in_group(group_id)), 0)
cs.kill_process(process, shutdown_grace_period_ms)
# await process termination
for i in range(1, 10 * shutdown_grace_period_ms):
if process.poll() is None:
time.sleep(0.01)
if process.poll() is None:
process.kill()
self.assertTrue(
((-1 * signal.SIGTERM) == process.poll())
or ((128 + signal.SIGTERM) == process.poll()),
'Process exited with code {}'.format(process.poll()))
self.assertEqual(0, len(find_process_ids_in_group(group_id)))
with open(stdout_name) as f:
file_contents = f.read()
self.assertTrue('GOT TERM' in file_contents)
finally:
tu.cleanup_output(stdout_name, stderr_name)
def process_launch_and_kill_helper(self, kill_fn):
task_id = tu.get_random_task_id()
stdout_name = tu.ensure_directory('build/stdout.{}'.format(task_id))
stderr_name = tu.ensure_directory('build/stderr.{}'.format(task_id))
tu.redirect_stdout_to_file(stdout_name)
tu.redirect_stderr_to_file(stderr_name)
try:
start_time = time.time()
command = 'echo "A.$(sleep 30)" & echo "B.$(sleep 30)" & echo "C.$(sleep 30)" &'
environment = {}
process = cs.launch_process(command, environment)
group_id = cs.find_process_group(process.pid)
self.assertGreater(group_id, 0)
child_process_ids = tu.wait_for(
lambda: find_process_ids_in_group(group_id),
lambda data: len(data) >= 7,
default_value=[])
self.assertGreaterEqual(len(child_process_ids), 7)
self.assertLessEqual(len(child_process_ids), 10)
kill_fn(process.pid)
child_process_ids = tu.wait_for(
lambda: find_process_ids_in_group(group_id),
lambda data: len(data) == 0,
default_value=[])
self.assertEqual(0, len(child_process_ids))
# ensure the test ran in under 30 seconds
self.assertLess(time.time() - start_time, 20)
finally:
tu.cleanup_output(stdout_name, stderr_name)
def test_executor_launch_task_and_disconnect(self):
task_id = tu.get_random_task_id()
stdout_name = tu.ensure_directory('build/stdout.{}'.format(task_id))
stderr_name = tu.ensure_directory('build/stderr.{}'.format(task_id))
output_name = tu.ensure_directory('build/output.' + str(task_id))
tu.redirect_stdout_to_file(stdout_name)
tu.redirect_stderr_to_file(stderr_name)
try:
config = cc.ExecutorConfig()
stop_signal = Event()
executor = ce.CookExecutor(stop_signal, config)
driver = tu.FakeMesosExecutorDriver()
command = 'echo "Start" >> {}; sleep 100; echo "Done." >> {}; '.format(
output_name, output_name)
task = {
'task_id': {
'value': task_id
},
'data':
pm.encode_data(
json.dumps({
'command': command
}).encode('utf8'))
}
executor.launchTask(driver, task)
# let the process run for up to 10 seconds
for _ in range(1000):
time.sleep(0.01)
if os.path.isfile(output_name):
with open(output_name) as f:
content = f.read()
if 'Start' in content:
break
executor.disconnected(driver)
self.assertTrue(executor.disconnect_signal.isSet())
self.assertTrue(executor.stop_signal.isSet())
executor.await_completion()
logging.info('Task completed')
if os.path.isfile(output_name):
with open(output_name) as f:
file_contents = f.read()
self.assertTrue('Start' in file_contents)
self.assertTrue('Done' not in file_contents)
else:
self.fail('{} does not exist.'.format(stderr_name))
expected_statuses = [{
'task_id': {
'value': task_id
},
'state': cook.TASK_STARTING
}, {
'task_id': {
'value': task_id
},
'state': cook.TASK_RUNNING
}, {
'task_id': {
'value': task_id
},
'state': cook.TASK_KILLED
}]
tu.assert_statuses(self, expected_statuses, driver.statuses)
expected_message_0 = {
'sandbox-directory': '',
'task-id': task_id,
'type': 'directory'
}
expected_message_1 = {'exit-code': -15, 'task-id': task_id}
tu.assert_messages(self, [expected_message_0, expected_message_1],
[], driver.messages)
finally:
tu.cleanup_output(stdout_name, stderr_name)
tu.cleanup_file(output_name)
