def test_kill_task_terminate(self): task_id = get_random_task_id() stdout_name = ensure_directory('build/stdout.' + str(task_id)) stderr_name = ensure_directory('build/stderr.' + str(task_id)) stdout = open(stdout_name, 'w+') stderr = open(stderr_name, 'w+') try: command = 'sleep 100' process = subprocess.Popen(command, shell=True, stdout=stdout, stderr=stderr) process_info = process, stdout, stderr shutdown_grace_period_ms = 2000 ce.kill_task(process_info, shutdown_grace_period_ms) # await process termination while process.poll() is None: time.sleep(0.01) self.assertEqual(-1 * signal.SIGTERM, process.poll()) self.assertTrue(stdout.closed) self.assertTrue(stderr.closed) finally: cleanup_output(stdout_name, stderr_name)
def test_await_process_completion_killed(self): task_id = tu.get_random_task_id() stdout_name = tu.ensure_directory('build/stdout.{}'.format(task_id)) stderr_name = tu.ensure_directory('build/stderr.{}'.format(task_id)) tu.redirect_stdout_to_file(stdout_name) tu.redirect_stderr_to_file(stderr_name) try: command = 'sleep 100' process = subprocess.Popen(command, preexec_fn=os.setpgrp, shell=True) shutdown_grace_period_ms = 2000 stop_signal = Event() sleep_and_set_stop_signal_task(stop_signal, 2) ce.await_process_completion(process, stop_signal, shutdown_grace_period_ms) self.assertTrue(process.returncode < 0) finally: tu.cleanup_output(stdout_name, stderr_name)
def test_kill_task_terminate_with_sigkill(self): task_id = tu.get_random_task_id() stdout_name = tu.ensure_directory('build/stdout.{}'.format(task_id)) stderr_name = tu.ensure_directory('build/stderr.{}'.format(task_id)) tu.redirect_stdout_to_file(stdout_name) tu.redirect_stderr_to_file(stderr_name) try: command = "trap '' TERM SIGTERM; sleep 200" process = cs.launch_process(command, {}) shutdown_grace_period_ms = 1000 group_id = cs.find_process_group(process.pid) self.assertGreater(len(find_process_ids_in_group(group_id)), 0) cs.kill_process(process, shutdown_grace_period_ms) # await process termination for i in range(1, 10 * shutdown_grace_period_ms): if process.poll() is None: time.sleep(0.01) if process.poll() is None: process.kill() self.assertTrue( ((-1 * signal.SIGKILL) == process.poll()) or ((128 + signal.SIGKILL) == process.poll()), 'Process exited with code {}'.format(process.poll())) self.assertEqual(len(find_process_ids_in_group(group_id)), 0) finally: tu.cleanup_output(stdout_name, stderr_name)
def manage_task_runner(self, command, assertions_fn, stop_signal=None, task_id=None, config=None, driver=None): if driver is None: driver = tu.FakeMesosExecutorDriver() if stop_signal is None: stop_signal = Event() if task_id is None: task_id = tu.get_random_task_id() task = { 'task_id': { 'value': task_id }, 'data': pm.encode_data(json.dumps({ 'command': command }).encode('utf8')) } stdout_name = tu.ensure_directory('build/stdout.{}'.format(task_id)) stderr_name = tu.ensure_directory('build/stderr.{}'.format(task_id)) tu.redirect_stdout_to_file(stdout_name) tu.redirect_stderr_to_file(stderr_name) completed_signal = Event() if config is None: sandbox_directory = '/location/to/task/sandbox/{}'.format(task_id) config = cc.ExecutorConfig( max_message_length=300, progress_output_name=stdout_name, progress_regex_string= '\^\^\^\^JOB-PROGRESS:\s+([0-9]*\.?[0-9]+)($|\s+.*)', progress_sample_interval_ms=100, sandbox_directory=sandbox_directory) else: sandbox_directory = config.sandbox_directory try: ce.manage_task(driver, task, stop_signal, completed_signal, config) self.assertTrue(completed_signal.isSet()) assertions_fn(driver, task_id, sandbox_directory) finally: tu.cleanup_output(stdout_name, stderr_name)
def test_launch_task_interactive_output(self): task_id = tu.get_random_task_id() command = 'echo "Start"; echo "Hello"; sleep 100; echo "World"; echo "Done"; ' task = { 'task_id': { 'value': task_id }, 'data': pm.encode_data(json.dumps({ 'command': command }).encode('utf8')) } stdout_name = tu.ensure_directory('build/stdout.{}'.format(task_id)) stderr_name = tu.ensure_directory('build/stderr.{}'.format(task_id)) tu.redirect_stdout_to_file(stdout_name) tu.redirect_stderr_to_file(stderr_name) try: process = ce.launch_task(task, os.environ) self.assertIsNotNone(process) # let the process run for up to 50 seconds for _ in range(5000): if cs.is_process_running(process): time.sleep(0.01) with open(stdout_name) as f: stdout_content = f.read() if 'Start' in stdout_content and 'Hello' in stdout_content: break try: with open(stdout_name) as f: stdout_content = f.read() logging.info( 'Contents of stdout: {}'.format(stdout_content)) self.assertTrue("Start" in stdout_content) self.assertTrue("Hello" in stdout_content) self.assertFalse("World" in stdout_content) self.assertFalse("Done" in stdout_content) finally: if process.poll() is None: logging.info('Killing launched process') process.kill() finally: tu.cleanup_output(stdout_name, stderr_name)
def test_cleanup_process(self): task_id = get_random_task_id() stdout_name = ensure_directory('build/stdout.' + str(task_id)) stderr_name = ensure_directory('build/stderr.' + str(task_id)) stdout = open(stdout_name, 'w+') stderr = open(stderr_name, 'w+') try: process_info = None, stdout, stderr ce.cleanup_process(process_info) self.assertTrue(stdout.closed) self.assertTrue(stderr.closed) finally: cleanup_output(stdout_name, stderr_name)
def test_collect_progress_updates_with_empty_regex(self): file_name = ensure_directory('build/collect_progress_test.' + get_random_task_id()) progress_regex_string = '' config = cc.ExecutorConfig(progress_output_name=file_name, progress_regex_string=progress_regex_string) stop_signal = Event() completed_signal = Event() file = open(file_name, 'w+') file.flush() progress_watcher = cp.ProgressWatcher(config, stop_signal, completed_signal) try: def read_progress_states(): for _ in progress_watcher.retrieve_progress_states(): pass Thread(target=read_progress_states, args=()).start() file.write("Stage One complete\n") file.flush() file.write("^^^^JOB-PROGRESS: 25 Twenty-Fine percent\n") file.flush() file.write("Stage Two complete\n") file.flush() file.write("^^^^JOB-PROGRESS: 50 Fifty percent\n") file.flush() time.sleep(0.10) self.assertIsNone(progress_watcher.current_progress()) file.write("Stage Three complete\n") file.flush() time.sleep(0.10) self.assertIsNone(progress_watcher.current_progress()) file.write("^^^^JOB-PROGRESS: 55 Fifty-five percent\n") file.flush() time.sleep(0.10) self.assertIsNone(progress_watcher.current_progress()) file.write("Stage Four complete\n") file.flush() file.write("^^^^JOB-PROGRESS: 100 Hundred percent\n") file.flush() time.sleep(0.10) self.assertIsNone(progress_watcher.current_progress()) finally: completed_signal.set() file.close() if os.path.isfile(file_name): os.remove(file_name)
def assertions(driver, task_id, sandbox_directory): expected_statuses = [{ 'task_id': { 'value': task_id }, 'state': cook.TASK_STARTING }, { 'task_id': { 'value': task_id }, 'state': cook.TASK_RUNNING }, { 'task_id': { 'value': task_id }, 'state': cook.TASK_FINISHED }] tu.assert_statuses(self, expected_statuses, driver.statuses) expected_message_0 = { 'sandbox-directory': sandbox_directory, 'task-id': task_id, 'type': 'directory' } expected_message_1 = {'exit-code': 0, 'task-id': task_id} tu.assert_messages(self, [expected_message_0, expected_message_1], [], driver.messages) stdout_name = tu.ensure_directory('build/stdout.' + str(task_id)) if os.path.isfile(stdout_name): with open(stdout_name) as f: file_contents = f.read() self.assertEqual(num_iterations * 25, file_contents.count('X')) else: self.fail('{} does not exist.'.format(stdout_name)) stderr_name = tu.ensure_directory('build/stderr.' + str(task_id)) if os.path.isfile(stderr_name): with open(stderr_name) as f: file_contents = f.read() self.assertEqual(num_iterations * 25, file_contents.count('X')) else: self.fail('{} does not exist.'.format(stderr_name))
def test_launch_task(self): task_id = get_random_task_id() command = 'echo "Hello World"; echo "Error Message" >&2' task = { 'task_id': { 'value': task_id }, 'data': encode_data(json.dumps({ 'command': command }).encode('utf8')) } stdout_name = ensure_directory('build/stdout.' + str(task_id)) stderr_name = ensure_directory('build/stderr.' + str(task_id)) if not os.path.isdir("build"): os.mkdir("build") try: process, stdout, stderr = ce.launch_task(task, stdout_name, stderr_name) self.assertIsNotNone(process) for i in range(100): if process.poll() is None: time.sleep(0.01) stdout.close() stderr.close() if process.poll() is None: process.kill() self.assertEqual(0, process.poll()) with open(stdout_name) as f: stdout_content = f.read() self.assertEqual("Hello World\n", stdout_content) with open(stderr_name) as f: stderr_content = f.read() self.assertEqual("Error Message\n", stderr_content) finally: cleanup_output(stdout_name, stderr_name)
def test_manage_task_successful_exit_despite_faulty_driver(self): def assertions(driver, task_id, sandbox_directory): expected_statuses = [{ 'task_id': { 'value': task_id }, 'state': cook.TASK_STARTING }, { 'task_id': { 'value': task_id }, 'state': cook.TASK_RUNNING }, { 'task_id': { 'value': task_id }, 'state': cook.TASK_FINISHED }] tu.assert_statuses(self, expected_statuses, driver.statuses) expected_core_messages = [{ 'sandbox-directory': sandbox_directory, 'task-id': task_id, 'type': 'directory' }, { 'exit-code': 0, 'task-id': task_id }] expected_progress_messages = [ { 'progress-message': 'ninety percent', 'progress-percent': 90, 'progress-sequence': 1, 'task-id': task_id }, # retried because the previous send fails { 'progress-message': 'ninety percent', 'progress-percent': 90, 'progress-sequence': 1, 'task-id': task_id } ] tu.assert_messages(self, expected_core_messages, expected_progress_messages, driver.messages) test_file_name = tu.ensure_directory('build/file.' + tu.get_random_task_id()) command = ('echo "^^^^JOB-PROGRESS: 90 ninety percent"'.format( test_file_name)) socket_error = OSError( 'socket.error') # socket.error is an alias of OSError self.manage_task_runner( command, assertions, driver=tu.ErrorMesosExecutorDriver(socket_error))
def test_launch_task(self): task_id = tu.get_random_task_id() command = 'echo "Hello World"; echo "Error Message" >&2' task = { 'task_id': { 'value': task_id }, 'data': pm.encode_data(json.dumps({ 'command': command }).encode('utf8')) } stdout_name = tu.ensure_directory('build/stdout.{}'.format(task_id)) stderr_name = tu.ensure_directory('build/stderr.{}'.format(task_id)) tu.redirect_stdout_to_file(stdout_name) tu.redirect_stderr_to_file(stderr_name) try: process = ce.launch_task(task, os.environ) self.assertIsNotNone(process) for _ in range(100): if cs.is_process_running(process): time.sleep(0.01) if process.poll() is None: process.kill() tu.close_sys_outputs() self.assertEqual(0, process.poll()) with open(stdout_name) as f: stdout_content = f.read() self.assertTrue("Hello World\n" in stdout_content) with open(stderr_name) as f: stderr_content = f.read() self.assertTrue("Error Message\n" in stderr_content) finally: tu.cleanup_output(stdout_name, stderr_name)
def test_collect_progress_updates_dev_null(self): file_name = tu.ensure_directory('build/collect_progress_test.' + tu.get_random_task_id()) progress_regex = '\^\^\^\^JOB-PROGRESS:\s+([0-9]*\.?[0-9]+)($|\s+.*)' location = '/dev/null' stop = Event() completed = Event() termination = Event() file = open(file_name, 'w+') file.flush() counter = cp.ProgressSequenceCounter() dn_watcher = cp.ProgressWatcher(location, 'dn', counter, 1024, progress_regex, stop, completed, termination) out_watcher = cp.ProgressWatcher(file_name, 'so', counter, 1024, progress_regex, stop, completed, termination) try: def print_to_file(): file.write('Stage One complete\n') file.write('^^^^JOB-PROGRESS: 100 100-percent\n') file.flush() file.close() completed.set() print_thread = Thread(target=print_to_file, args=()) print_thread.start() progress_states = [{ 'progress-message': b' 100-percent', 'progress-percent': 100, 'progress-sequence': 1 }] for actual_progress_state in out_watcher.retrieve_progress_states( ): expected_progress_state = progress_states.pop(0) self.assertEqual(expected_progress_state, actual_progress_state) self.assertEqual(expected_progress_state, out_watcher.current_progress()) self.assertFalse(progress_states) iterable = dn_watcher.retrieve_progress_states() exhausted = object() self.assertEqual(exhausted, next(iterable, exhausted)) self.assertIsNone(dn_watcher.current_progress()) print_thread.join() finally: completed.set() tu.cleanup_file(file_name)
def assertions(driver, task_id, sandbox_directory): expected_statuses = [{ 'task_id': { 'value': task_id }, 'state': cook.TASK_STARTING }, { 'task_id': { 'value': task_id }, 'state': cook.TASK_RUNNING }, { 'task_id': { 'value': task_id }, 'state': cook.TASK_FINISHED }] tu.assert_statuses(self, expected_statuses, driver.statuses) logging.info('Messages: {}'.format(driver.messages)) self.assertLess(2, len(driver.messages)) actual_encoded_message_0 = driver.messages[0] expected_message_0 = { 'sandbox-directory': sandbox_directory, 'task-id': task_id, 'type': 'directory' } tu.assert_message(self, expected_message_0, actual_encoded_message_0) found_exit_message = False for index in range(1, len(driver.messages)): actual_encoded_message = driver.messages[index] actual_message = tu.parse_message(actual_encoded_message) if 'exit-code' in actual_message: found_exit_message = True expected_message = {'exit-code': 0, 'task-id': task_id} tu.assert_message(self, expected_message, actual_encoded_message) break self.assertTrue(found_exit_message) stderr_name = tu.ensure_directory('build/stderr.' + str(task_id)) if os.path.isfile(stderr_name): with open(stderr_name) as f: file_contents = f.read() self.assertEqual(num_iterations * 25, file_contents.count('X')) else: self.fail('{} does not exist.'.format(stderr_name))
def manage_task_runner(self, command, assertions_fn, stop_signal=Event()): driver = FakeMesosExecutorDriver() task_id = get_random_task_id() task = { 'task_id': { 'value': task_id }, 'data': encode_data(json.dumps({ 'command': command }).encode('utf8')) } stdout_name = ensure_directory('build/stdout.' + str(task_id)) stderr_name = ensure_directory('build/stderr.' + str(task_id)) completed_signal = Event() max_message_length = 300 progress_sample_interval_ms = 100 sandbox_directory = '/location/to/task/sandbox/{}'.format(task_id) progress_output_name = stdout_name progress_regex_string = '\^\^\^\^JOB-PROGRESS: (\d*)(?: )?(.*)' config = cc.ExecutorConfig( max_message_length=max_message_length, progress_output_name=progress_output_name, progress_regex_string=progress_regex_string, progress_sample_interval_ms=progress_sample_interval_ms, sandbox_directory=sandbox_directory) try: ce.manage_task(driver, task, stop_signal, completed_signal, config, stdout_name, stderr_name) self.assertTrue(completed_signal.isSet()) assertions_fn(driver, task_id, sandbox_directory) finally: cleanup_output(stdout_name, stderr_name)
def test_kill_task_terminate_with_sigterm(self): task_id = tu.get_random_task_id() stdout_name = tu.ensure_directory('build/stdout.{}'.format(task_id)) stderr_name = tu.ensure_directory('build/stderr.{}'.format(task_id)) tu.redirect_stdout_to_file(stdout_name) tu.redirect_stderr_to_file(stderr_name) try: command = "bash -c 'function handle_term { echo GOT TERM; }; trap handle_term SIGTERM TERM; sleep 200'" process = cs.launch_process(command, {}) shutdown_grace_period_ms = 1000 group_id = cs.find_process_group(process.pid) self.assertGreater(len(find_process_ids_in_group(group_id)), 0) cs.kill_process(process, shutdown_grace_period_ms) # await process termination for i in range(1, 10 * shutdown_grace_period_ms): if process.poll() is None: time.sleep(0.01) if process.poll() is None: process.kill() self.assertTrue( ((-1 * signal.SIGTERM) == process.poll()) or ((128 + signal.SIGTERM) == process.poll()), 'Process exited with code {}'.format(process.poll())) self.assertEqual(0, len(find_process_ids_in_group(group_id))) with open(stdout_name) as f: file_contents = f.read() self.assertTrue('GOT TERM' in file_contents) finally: tu.cleanup_output(stdout_name, stderr_name)
def process_launch_and_kill_helper(self, kill_fn): task_id = tu.get_random_task_id() stdout_name = tu.ensure_directory('build/stdout.{}'.format(task_id)) stderr_name = tu.ensure_directory('build/stderr.{}'.format(task_id)) tu.redirect_stdout_to_file(stdout_name) tu.redirect_stderr_to_file(stderr_name) try: start_time = time.time() command = 'echo "A.$(sleep 30)" & echo "B.$(sleep 30)" & echo "C.$(sleep 30)" &' environment = {} process = cs.launch_process(command, environment) group_id = cs.find_process_group(process.pid) self.assertGreater(group_id, 0) child_process_ids = tu.wait_for( lambda: find_process_ids_in_group(group_id), lambda data: len(data) >= 7, default_value=[]) self.assertGreaterEqual(len(child_process_ids), 7) self.assertLessEqual(len(child_process_ids), 10) kill_fn(process.pid) child_process_ids = tu.wait_for( lambda: find_process_ids_in_group(group_id), lambda data: len(data) == 0, default_value=[]) self.assertEqual(0, len(child_process_ids)) # ensure the test ran in under 30 seconds self.assertLess(time.time() - start_time, 20) finally: tu.cleanup_output(stdout_name, stderr_name)
def test_await_process_completion_killed(self): task_id = get_random_task_id() stdout_name = ensure_directory('build/stdout.' + str(task_id)) stderr_name = ensure_directory('build/stderr.' + str(task_id)) stdout = open(stdout_name, 'w+') stderr = open(stderr_name, 'w+') try: command = 'sleep 100' process = subprocess.Popen(command, shell=True, stdout=stdout, stderr=stderr) process_info = process, stdout, stderr shutdown_grace_period_ms = 2000 stop_signal = Event() def sleep_and_set_stop_signal(): time.sleep(2 * cook.RUNNING_POLL_INTERVAL_SECS) stop_signal.set() thread = Thread(target=sleep_and_set_stop_signal, args=()) thread.start() ce.await_process_completion(stop_signal, process_info, shutdown_grace_period_ms) self.assertTrue(process.returncode < 0) self.assertTrue(stdout.closed) self.assertTrue(stderr.closed) finally: cleanup_output(stdout_name, stderr_name)
def test_collect_progress_updates_faulty_regex(self): file_name = tu.ensure_directory( 'build/collect_progress_updates_skip_faulty.' + tu.get_random_task_id()) progress_regex = '\^\^\^\^JOB-PROGRESS: (\S+)(?: )?(.*)' stop = Event() completed = Event() termination = Event() file = open(file_name, 'w+') file.flush() counter = cp.ProgressSequenceCounter() watcher = cp.ProgressWatcher(file_name, 'test', counter, 1024, progress_regex, stop, completed, termination) try: def print_to_file(): file.write('^^^^JOB-PROGRESS: ABCDEF string percent\n') file.write('^^^^JOB-PROGRESS: F50 Fifty percent\n') file.write( '^^^^JOB-PROGRESS: 1019101010101010101010101018101101010101010110171010110 Sixty percent\n' ) file.write('^^^^JOB-PROGRESS: 75 75% percent\n') file.flush() file.close() completed.set() print_thread = Thread(target=print_to_file, args=()) print_thread.start() progress_states = [{ 'progress-message': b'75% percent', 'progress-percent': 75, 'progress-sequence': 1 }] for actual_progress_state in watcher.retrieve_progress_states(): expected_progress_state = progress_states.pop(0) self.assertEqual(expected_progress_state, actual_progress_state) self.assertEqual(expected_progress_state, watcher.current_progress()) self.assertFalse(progress_states) print_thread.join() finally: completed.set() tu.cleanup_file(file_name)
def test_manage_task_involved_command_successful_exit(self): def assertions(driver, task_id, sandbox_directory): expected_statuses = [{ 'task_id': { 'value': task_id }, 'state': cook.TASK_STARTING }, { 'task_id': { 'value': task_id }, 'state': cook.TASK_RUNNING }, { 'task_id': { 'value': task_id }, 'state': cook.TASK_FINISHED }] tu.assert_statuses(self, expected_statuses, driver.statuses) expected_core_messages = [{ 'sandbox-directory': sandbox_directory, 'task-id': task_id, 'type': 'directory' }, { 'exit-code': 0, 'task-id': task_id }] expected_progress_messages = [{ 'progress-message': 'line count is 20', 'progress-percent': 90, 'progress-sequence': 1, 'task-id': task_id }] tu.assert_messages(self, expected_core_messages, expected_progress_messages, driver.messages) test_file_name = tu.ensure_directory('build/file.' + tu.get_random_task_id()) command = ( 'mkdir -p build; touch {0}; for i in $(seq 20); do echo $i >> {0}; done; ' 'LINE_COUNT=`wc -l < {0} | tr -d \'[:space:]\'`; cat {0}; rm -rfv {0}; ' 'echo "^^^^JOB-PROGRESS: 90 line count is $LINE_COUNT"'.format( test_file_name)) self.manage_task_runner(command, assertions)
def test_collect_progress_updates_with_empty_regex(self): file_name = tu.ensure_directory('build/collect_progress_test.' + tu.get_random_task_id()) progress_regex = '' stop = Event() completed = Event() termination = Event() file = open(file_name, 'w+') file.flush() counter = cp.ProgressSequenceCounter() watcher = cp.ProgressWatcher(file_name, 'test', counter, 1024, progress_regex, stop, completed, termination) try: def print_to_file(): file.write('Stage One complete\n') file.write('^^^^JOB-PROGRESS: 25 Twenty-Five percent\n') file.write('Stage Two complete\n') file.write('^^^^JOB-PROGRESS: 50 Fifty percent\n') file.write('Stage Three complete\n') file.write('^^^^JOB-PROGRESS: 55.0 Fifty-five percent\n') file.write('Stage Four complete\n') file.write('^^^^JOB-PROGRESS: 100 100-percent\n') file.flush() file.close() completed.set() print_thread = Thread(target=print_to_file, args=()) print_thread.start() progress_states = [] for actual_progress_state in watcher.retrieve_progress_states(): expected_progress_state = progress_states.pop(0) self.assertEqual(expected_progress_state, actual_progress_state) self.assertEqual(expected_progress_state, watcher.current_progress()) self.assertFalse(progress_states) self.assertIsNone(watcher.current_progress()) finally: completed.set() tu.cleanup_file(file_name)
def test_progress_watcher_tail_with_read_limit(self): file_name = ensure_directory('build/tail_progress_test.' + get_random_task_id()) config = cc.ExecutorConfig(max_bytes_read_per_line=10, progress_output_name=file_name) stop_signal = Event() completed_signal = Event() tail_sleep_ms = 25 try: def write_to_file(): file = open(file_name, 'w+') file.write("abcd\n") file.flush() file.write("abcdefghijkl\n") file.flush() file.write("abcdefghijklmnopqrstuvwxyz\n") file.flush() file.close() time.sleep(0.15) completed_signal.set() Thread(target=write_to_file, args=()).start() progress_watcher = cp.ProgressWatcher(config, stop_signal, completed_signal) collected_data = [] for line in progress_watcher.tail(tail_sleep_ms): collected_data.append(line.strip()) logging.debug('collected_data = {}'.format(collected_data)) expected_data = [ 'abcd', 'abcdefghij', 'kl', 'abcdefghij', 'klmnopqrst', 'uvwxyz' ] self.assertEqual(expected_data, collected_data) finally: if os.path.isfile(file_name): os.remove(file_name)
def test_watcher_tail_with_read_limit(self): file_name = tu.ensure_directory('build/tail_progress_test.' + tu.get_random_task_id()) stop = Event() completed = Event() termination = Event() tail_sleep_ms = 25 try: def write_to_file(): file = open(file_name, 'w+') file.write('abcd\n') file.flush() file.write('abcdefghijkl\n') file.flush() file.write('abcdefghijklmnopqrstuvwxyz\n') file.flush() file.close() time.sleep(0.15) completed.set() Thread(target=write_to_file, args=()).start() counter = cp.ProgressSequenceCounter() watcher = cp.ProgressWatcher(file_name, 'test', counter, 10, '', stop, completed, termination) collected_data = [] for line in watcher.tail(tail_sleep_ms): collected_data.append(line.strip()) logging.debug('collected_data = {}'.format(collected_data)) expected_data = [ b'abcd', b'abcdefghij', b'kl', b'abcdefghij', b'klmnopqrst', b'uvwxyz' ] self.assertEqual(expected_data, collected_data) finally: tu.cleanup_file(file_name)
def test_watcher_tail_lot_of_writes(self): file_name = tu.ensure_directory('build/tail_progress_test.' + tu.get_random_task_id()) items_to_write = 250000 stop = Event() completed = Event() termination = Event() tail_sleep_ms = 25 try: def write_to_file(): file = open(file_name, 'w+') for item in range(items_to_write): file.write('line-{}\n'.format(item)) if item % 100 == 0: file.flush() file.flush() file.close() time.sleep(0.15) completed.set() Thread(target=write_to_file, args=()).start() counter = cp.ProgressSequenceCounter() watcher = cp.ProgressWatcher(file_name, 'test', counter, 1024, '', stop, completed, termination) collected_data = [] for line in watcher.tail(tail_sleep_ms): collected_data.append(line.strip()) logging.info('Items read: {}'.format(len(collected_data))) if items_to_write != len(collected_data): for index in range(len(collected_data)): logging.info('{}: {}'.format(index, collected_data[index])) self.assertEqual(items_to_write, len(collected_data)) expected_data = list( map(lambda x: str.encode('line-{}'.format(x)), range(items_to_write))) self.assertEqual(expected_data, collected_data) finally: tu.cleanup_file(file_name)
def assertions(driver, task_id, sandbox_directory): expected_statuses = [{ 'task_id': { 'value': task_id }, 'state': cook.TASK_STARTING }, { 'task_id': { 'value': task_id }, 'state': cook.TASK_RUNNING }, { 'task_id': { 'value': task_id }, 'state': cook.TASK_FINISHED }] tu.assert_statuses(self, expected_statuses, driver.statuses) expected_core_messages = [{ 'sandbox-directory': sandbox_directory, 'task-id': task_id, 'type': 'directory' }, { 'exit-code': 0, 'task-id': task_id }] expected_progress_messages = [{ 'progress-message': '', 'progress-percent': 50, 'progress-sequence': 1, 'task-id': task_id }] tu.assert_messages(self, expected_core_messages, expected_progress_messages, driver.messages) stdout_name = tu.ensure_directory('build/stdout.' + str(task_id)) if not os.path.isfile(stdout_name): self.fail('{} does not exist.'.format(stdout_name))
def test_progress_watcher_tail_lot_of_writes(self): file_name = ensure_directory('build/tail_progress_test.' + get_random_task_id()) config = cc.ExecutorConfig(progress_output_name=file_name) items_to_write = 250000 stop_signal = Event() completed_signal = Event() tail_sleep_ms = 25 try: def write_to_file(): file = open(file_name, 'w+') for item in range(items_to_write): file.write("line-{}\n".format(item)) file.flush() file.close() time.sleep(0.15) completed_signal.set() Thread(target=write_to_file, args=()).start() progress_watcher = cp.ProgressWatcher(config, stop_signal, completed_signal) collected_data = [] for line in progress_watcher.tail(tail_sleep_ms): collected_data.append(line.strip()) logging.info('Items read: {}'.format(len(collected_data))) if items_to_write != len(collected_data): for index in range(len(collected_data)): logging.info('{}: {}'.format(index, collected_data[index])) self.assertEqual(items_to_write, len(collected_data)) expected_data = list( map(lambda x: 'line-' + str(x), range(items_to_write))) self.assertEqual(expected_data, collected_data) finally: if os.path.isfile(file_name): os.remove(file_name)
def test_progress_watcher_tail(self): file_name = ensure_directory('build/tail_progress_test.' + get_random_task_id()) config = cc.ExecutorConfig(progress_output_name=file_name) items_to_write = 12 stop_signal = Event() completed_signal = Event() write_sleep_ms = 50 tail_sleep_ms = 25 try: def write_to_file(): file = open(file_name, 'w+') for item in range(items_to_write): time.sleep(write_sleep_ms / 1000.0) file.write("{}\n".format(item)) file.flush() file.close() time.sleep(0.15) completed_signal.set() Thread(target=write_to_file, args=()).start() progress_watcher = cp.ProgressWatcher(config, stop_signal, completed_signal) collected_data = [] for line in progress_watcher.tail(tail_sleep_ms): collected_data.append(line.strip()) self.assertEqual(items_to_write, len(collected_data)) self.assertEqual( list(map(lambda x: str(x), range(items_to_write))), collected_data) finally: if os.path.isfile(file_name): os.remove(file_name)
def test_watcher_tail(self): file_name = tu.ensure_directory('build/tail_progress_test.' + tu.get_random_task_id()) items_to_write = 12 stop = Event() completed = Event() termination = Event() write_sleep_ms = 50 tail_sleep_ms = 25 try: def write_to_file(): file = open(file_name, 'w+') for item in range(items_to_write): time.sleep(write_sleep_ms / 1000.0) file.write('{}\n'.format(item)) file.flush() file.close() time.sleep(0.15) completed.set() Thread(target=write_to_file, args=()).start() counter = cp.ProgressSequenceCounter() watcher = cp.ProgressWatcher(file_name, 'test', counter, 1024, '', stop, completed, termination) collected_data = [] for line in watcher.tail(tail_sleep_ms): collected_data.append(line.strip()) self.assertEqual(items_to_write, len(collected_data)) self.assertEqual( list(map(lambda x: str.encode(str(x)), range(items_to_write))), collected_data) finally: tu.cleanup_file(file_name)
def assertions(driver, task_id, _): expected_statuses = [{ 'task_id': { 'value': task_id }, 'state': cook.TASK_STARTING }, { 'task_id': { 'value': task_id }, 'state': cook.TASK_RUNNING }, { 'task_id': { 'value': task_id }, 'state': cook.TASK_FINISHED }] tu.assert_statuses(self, expected_statuses, driver.statuses) expected_message_0 = { 'sandbox-directory': '/location/to/task/sandbox/{}'.format(task_id), 'task-id': task_id, 'type': 'directory' } expected_message_1 = {'exit-code': 0, 'task-id': task_id} tu.assert_messages(self, [expected_message_0, expected_message_1], [], driver.messages) stdout_name = tu.ensure_directory( 'build/stdout.{}'.format(task_id)) with open(stdout_name) as f: file_contents = f.read() self.assertTrue('FEE=FIE' in file_contents) self.assertTrue('FOO=BAR' in file_contents) self.assertTrue('PROGRESS_OUTPUT_FILE=foobar' in file_contents)
def test_executor_launch_task_and_disconnect(self): task_id = tu.get_random_task_id() stdout_name = tu.ensure_directory('build/stdout.{}'.format(task_id)) stderr_name = tu.ensure_directory('build/stderr.{}'.format(task_id)) output_name = tu.ensure_directory('build/output.' + str(task_id)) tu.redirect_stdout_to_file(stdout_name) tu.redirect_stderr_to_file(stderr_name) try: config = cc.ExecutorConfig() stop_signal = Event() executor = ce.CookExecutor(stop_signal, config) driver = tu.FakeMesosExecutorDriver() command = 'echo "Start" >> {}; sleep 100; echo "Done." >> {}; '.format( output_name, output_name) task = { 'task_id': { 'value': task_id }, 'data': pm.encode_data( json.dumps({ 'command': command }).encode('utf8')) } executor.launchTask(driver, task) # let the process run for up to 10 seconds for _ in range(1000): time.sleep(0.01) if os.path.isfile(output_name): with open(output_name) as f: content = f.read() if 'Start' in content: break executor.disconnected(driver) self.assertTrue(executor.disconnect_signal.isSet()) self.assertTrue(executor.stop_signal.isSet()) executor.await_completion() logging.info('Task completed') if os.path.isfile(output_name): with open(output_name) as f: file_contents = f.read() self.assertTrue('Start' in file_contents) self.assertTrue('Done' not in file_contents) else: self.fail('{} does not exist.'.format(stderr_name)) expected_statuses = [{ 'task_id': { 'value': task_id }, 'state': cook.TASK_STARTING }, { 'task_id': { 'value': task_id }, 'state': cook.TASK_RUNNING }, { 'task_id': { 'value': task_id }, 'state': cook.TASK_KILLED }] tu.assert_statuses(self, expected_statuses, driver.statuses) expected_message_0 = { 'sandbox-directory': '', 'task-id': task_id, 'type': 'directory' } expected_message_1 = {'exit-code': -15, 'task-id': task_id} tu.assert_messages(self, [expected_message_0, expected_message_1], [], driver.messages) finally: tu.cleanup_output(stdout_name, stderr_name) tu.cleanup_file(output_name)
def test_manage_task_progress_in_progress_stderr_and_stdout_progress(self): max_message_length = 35 def assertions(driver, task_id, sandbox_directory): expected_statuses = [{ 'task_id': { 'value': task_id }, 'state': cook.TASK_STARTING }, { 'task_id': { 'value': task_id }, 'state': cook.TASK_RUNNING }, { 'task_id': { 'value': task_id }, 'state': cook.TASK_FINISHED }] tu.assert_statuses(self, expected_statuses, driver.statuses) expected_core_messages = [{ 'sandbox-directory': sandbox_directory, 'task-id': task_id, 'type': 'directory' }, { 'exit-code': 0, 'task-id': task_id }] expected_progress_messages = [{ 'progress-message': 'Fifty percent in progress file', 'progress-percent': 50, 'progress-sequence': 1, 'task-id': task_id }, { 'progress-message': 'Fifty-five percent in stdout', 'progress-percent': 55, 'progress-sequence': 2, 'task-id': task_id }, { 'progress-message': 'Sixty percent in stderr', 'progress-percent': 60, 'progress-sequence': 3, 'task-id': task_id }, { 'progress-message': 'Sixty-five percent in stdout wit...', 'progress-percent': 65, 'progress-sequence': 4, 'task-id': task_id }] tu.assert_messages(self, expected_core_messages, expected_progress_messages, driver.messages) stop_signal = Event() sleep_and_set_stop_signal_task(stop_signal, 60) task_id = tu.get_random_task_id() progress_name = tu.ensure_directory( 'build/progress.{}'.format(task_id)) stderr_name = tu.ensure_directory('build/stderr.{}'.format(task_id)) stdout_name = tu.ensure_directory('build/stdout.{}'.format(task_id)) config = tu.FakeExecutorConfig({ 'max_bytes_read_per_line': 1024, 'max_message_length': max_message_length, 'progress_output_env_variable': 'DEFAULT_PROGRESS_FILE_ENV_VARIABLE', 'progress_output_name': progress_name, 'progress_regex_string': '\^\^\^\^JOB-PROGRESS:\s+([0-9]*\.?[0-9]+)($|\s+.*)', 'progress_sample_interval_ms': 10, 'sandbox_directory': '/sandbox/directory/for/{}'.format(task_id), 'shutdown_grace_period_ms': 60000, 'stderr_file': stderr_name, 'stdout_file': stdout_name }) command = 'echo "Hello World"; ' \ 'echo "^^^^JOB-PROGRESS: 50 Fifty percent in progress file" >> {}; ' \ 'sleep 0.25; ' \ 'echo "^^^^JOB-PROGRESS: 55 Fifty-five percent in stdout" >> {}; ' \ 'sleep 0.25; ' \ 'echo "^^^^JOB-PROGRESS: 60 Sixty percent in stderr" >> {}; ' \ 'sleep 0.25; ' \ 'echo "^^^^JOB-PROGRESS: 65 Sixty-five percent in stdout with a long message" >> {}; ' \ 'sleep 0.25; ' \ 'echo "Exiting..."; ' \ 'exit 0'.format(progress_name, stdout_name, stderr_name, stdout_name) try: self.manage_task_runner(command, assertions, stop_signal=stop_signal, task_id=task_id, config=config) stop_signal.set() finally: tu.cleanup_file(progress_name)