def test_kill_task_terminate_with_sigkill(self): task_id = tu.get_random_task_id() stdout_name = tu.ensure_directory('build/stdout.{}'.format(task_id)) stderr_name = tu.ensure_directory('build/stderr.{}'.format(task_id)) tu.redirect_stdout_to_file(stdout_name) tu.redirect_stderr_to_file(stderr_name) try: command = "trap '' TERM SIGTERM; sleep 200" process = cs.launch_process(command, {}) shutdown_grace_period_ms = 1000 group_id = cs.find_process_group(process.pid) self.assertGreater(len(find_process_ids_in_group(group_id)), 0) cs.kill_process(process, shutdown_grace_period_ms) # await process termination for i in range(1, 10 * shutdown_grace_period_ms): if process.poll() is None: time.sleep(0.01) if process.poll() is None: process.kill() self.assertTrue( ((-1 * signal.SIGKILL) == process.poll()) or ((128 + signal.SIGKILL) == process.poll()), 'Process exited with code {}'.format(process.poll())) self.assertEqual(len(find_process_ids_in_group(group_id)), 0) finally: tu.cleanup_output(stdout_name, stderr_name)
def test_await_process_completion_killed(self): task_id = tu.get_random_task_id() stdout_name = tu.ensure_directory('build/stdout.{}'.format(task_id)) stderr_name = tu.ensure_directory('build/stderr.{}'.format(task_id)) tu.redirect_stdout_to_file(stdout_name) tu.redirect_stderr_to_file(stderr_name) try: command = 'sleep 100' process = subprocess.Popen(command, preexec_fn=os.setpgrp, shell=True) shutdown_grace_period_ms = 2000 stop_signal = Event() sleep_and_set_stop_signal_task(stop_signal, 2) ce.await_process_completion(process, stop_signal, shutdown_grace_period_ms) self.assertTrue(process.returncode < 0) finally: tu.cleanup_output(stdout_name, stderr_name)
def manage_task_runner(self, command, assertions_fn, stop_signal=None, task_id=None, config=None, driver=None): if driver is None: driver = tu.FakeMesosExecutorDriver() if stop_signal is None: stop_signal = Event() if task_id is None: task_id = tu.get_random_task_id() task = { 'task_id': { 'value': task_id }, 'data': pm.encode_data(json.dumps({ 'command': command }).encode('utf8')) } stdout_name = tu.ensure_directory('build/stdout.{}'.format(task_id)) stderr_name = tu.ensure_directory('build/stderr.{}'.format(task_id)) tu.redirect_stdout_to_file(stdout_name) tu.redirect_stderr_to_file(stderr_name) completed_signal = Event() if config is None: sandbox_directory = '/location/to/task/sandbox/{}'.format(task_id) config = cc.ExecutorConfig( max_message_length=300, progress_output_name=stdout_name, progress_regex_string= '\^\^\^\^JOB-PROGRESS:\s+([0-9]*\.?[0-9]+)($|\s+.*)', progress_sample_interval_ms=100, sandbox_directory=sandbox_directory) else: sandbox_directory = config.sandbox_directory try: ce.manage_task(driver, task, stop_signal, completed_signal, config) self.assertTrue(completed_signal.isSet()) assertions_fn(driver, task_id, sandbox_directory) finally: tu.cleanup_output(stdout_name, stderr_name)
def test_launch_task_interactive_output(self): task_id = tu.get_random_task_id() command = 'echo "Start"; echo "Hello"; sleep 100; echo "World"; echo "Done"; ' task = { 'task_id': { 'value': task_id }, 'data': pm.encode_data(json.dumps({ 'command': command }).encode('utf8')) } stdout_name = tu.ensure_directory('build/stdout.{}'.format(task_id)) stderr_name = tu.ensure_directory('build/stderr.{}'.format(task_id)) tu.redirect_stdout_to_file(stdout_name) tu.redirect_stderr_to_file(stderr_name) try: process = ce.launch_task(task, os.environ) self.assertIsNotNone(process) # let the process run for up to 50 seconds for _ in range(5000): if cs.is_process_running(process): time.sleep(0.01) with open(stdout_name) as f: stdout_content = f.read() if 'Start' in stdout_content and 'Hello' in stdout_content: break try: with open(stdout_name) as f: stdout_content = f.read() logging.info( 'Contents of stdout: {}'.format(stdout_content)) self.assertTrue("Start" in stdout_content) self.assertTrue("Hello" in stdout_content) self.assertFalse("World" in stdout_content) self.assertFalse("Done" in stdout_content) finally: if process.poll() is None: logging.info('Killing launched process') process.kill() finally: tu.cleanup_output(stdout_name, stderr_name)
def test_launch_task(self): task_id = tu.get_random_task_id() command = 'echo "Hello World"; echo "Error Message" >&2' task = { 'task_id': { 'value': task_id }, 'data': pm.encode_data(json.dumps({ 'command': command }).encode('utf8')) } stdout_name = tu.ensure_directory('build/stdout.{}'.format(task_id)) stderr_name = tu.ensure_directory('build/stderr.{}'.format(task_id)) tu.redirect_stdout_to_file(stdout_name) tu.redirect_stderr_to_file(stderr_name) try: process = ce.launch_task(task, os.environ) self.assertIsNotNone(process) for _ in range(100): if cs.is_process_running(process): time.sleep(0.01) if process.poll() is None: process.kill() tu.close_sys_outputs() self.assertEqual(0, process.poll()) with open(stdout_name) as f: stdout_content = f.read() self.assertTrue("Hello World\n" in stdout_content) with open(stderr_name) as f: stderr_content = f.read() self.assertTrue("Error Message\n" in stderr_content) finally: tu.cleanup_output(stdout_name, stderr_name)
def test_kill_task_terminate_with_sigterm(self): task_id = tu.get_random_task_id() stdout_name = tu.ensure_directory('build/stdout.{}'.format(task_id)) stderr_name = tu.ensure_directory('build/stderr.{}'.format(task_id)) tu.redirect_stdout_to_file(stdout_name) tu.redirect_stderr_to_file(stderr_name) try: command = "bash -c 'function handle_term { echo GOT TERM; }; trap handle_term SIGTERM TERM; sleep 200'" process = cs.launch_process(command, {}) shutdown_grace_period_ms = 1000 group_id = cs.find_process_group(process.pid) self.assertGreater(len(find_process_ids_in_group(group_id)), 0) cs.kill_process(process, shutdown_grace_period_ms) # await process termination for i in range(1, 10 * shutdown_grace_period_ms): if process.poll() is None: time.sleep(0.01) if process.poll() is None: process.kill() self.assertTrue( ((-1 * signal.SIGTERM) == process.poll()) or ((128 + signal.SIGTERM) == process.poll()), 'Process exited with code {}'.format(process.poll())) self.assertEqual(0, len(find_process_ids_in_group(group_id))) with open(stdout_name) as f: file_contents = f.read() self.assertTrue('GOT TERM' in file_contents) finally: tu.cleanup_output(stdout_name, stderr_name)
def process_launch_and_kill_helper(self, kill_fn): task_id = tu.get_random_task_id() stdout_name = tu.ensure_directory('build/stdout.{}'.format(task_id)) stderr_name = tu.ensure_directory('build/stderr.{}'.format(task_id)) tu.redirect_stdout_to_file(stdout_name) tu.redirect_stderr_to_file(stderr_name) try: start_time = time.time() command = 'echo "A.$(sleep 30)" & echo "B.$(sleep 30)" & echo "C.$(sleep 30)" &' environment = {} process = cs.launch_process(command, environment) group_id = cs.find_process_group(process.pid) self.assertGreater(group_id, 0) child_process_ids = tu.wait_for( lambda: find_process_ids_in_group(group_id), lambda data: len(data) >= 7, default_value=[]) self.assertGreaterEqual(len(child_process_ids), 7) self.assertLessEqual(len(child_process_ids), 10) kill_fn(process.pid) child_process_ids = tu.wait_for( lambda: find_process_ids_in_group(group_id), lambda data: len(data) == 0, default_value=[]) self.assertEqual(0, len(child_process_ids)) # ensure the test ran in under 30 seconds self.assertLess(time.time() - start_time, 20) finally: tu.cleanup_output(stdout_name, stderr_name)
def test_executor_launch_task_and_disconnect(self): task_id = tu.get_random_task_id() stdout_name = tu.ensure_directory('build/stdout.{}'.format(task_id)) stderr_name = tu.ensure_directory('build/stderr.{}'.format(task_id)) output_name = tu.ensure_directory('build/output.' + str(task_id)) tu.redirect_stdout_to_file(stdout_name) tu.redirect_stderr_to_file(stderr_name) try: config = cc.ExecutorConfig() stop_signal = Event() executor = ce.CookExecutor(stop_signal, config) driver = tu.FakeMesosExecutorDriver() command = 'echo "Start" >> {}; sleep 100; echo "Done." >> {}; '.format( output_name, output_name) task = { 'task_id': { 'value': task_id }, 'data': pm.encode_data( json.dumps({ 'command': command }).encode('utf8')) } executor.launchTask(driver, task) # let the process run for up to 10 seconds for _ in range(1000): time.sleep(0.01) if os.path.isfile(output_name): with open(output_name) as f: content = f.read() if 'Start' in content: break executor.disconnected(driver) self.assertTrue(executor.disconnect_signal.isSet()) self.assertTrue(executor.stop_signal.isSet()) executor.await_completion() logging.info('Task completed') if os.path.isfile(output_name): with open(output_name) as f: file_contents = f.read() self.assertTrue('Start' in file_contents) self.assertTrue('Done' not in file_contents) else: self.fail('{} does not exist.'.format(stderr_name)) expected_statuses = [{ 'task_id': { 'value': task_id }, 'state': cook.TASK_STARTING }, { 'task_id': { 'value': task_id }, 'state': cook.TASK_RUNNING }, { 'task_id': { 'value': task_id }, 'state': cook.TASK_KILLED }] tu.assert_statuses(self, expected_statuses, driver.statuses) expected_message_0 = { 'sandbox-directory': '', 'task-id': task_id, 'type': 'directory' } expected_message_1 = {'exit-code': -15, 'task-id': task_id} tu.assert_messages(self, [expected_message_0, expected_message_1], [], driver.messages) finally: tu.cleanup_output(stdout_name, stderr_name) tu.cleanup_file(output_name)