def test_collect_progress_updates_dev_null(self): file_name = tu.ensure_directory('build/collect_progress_test.' + tu.get_random_task_id()) progress_regex = '\^\^\^\^JOB-PROGRESS:\s+([0-9]*\.?[0-9]+)($|\s+.*)' location = '/dev/null' stop = Event() completed = Event() termination = Event() file = open(file_name, 'w+') file.flush() counter = cp.ProgressSequenceCounter() dn_watcher = cp.ProgressWatcher(location, 'dn', counter, 1024, progress_regex, stop, completed, termination) out_watcher = cp.ProgressWatcher(file_name, 'so', counter, 1024, progress_regex, stop, completed, termination) try: def print_to_file(): file.write('Stage One complete\n') file.write('^^^^JOB-PROGRESS: 100 100-percent\n') file.flush() file.close() completed.set() print_thread = Thread(target=print_to_file, args=()) print_thread.start() progress_states = [{ 'progress-message': b' 100-percent', 'progress-percent': 100, 'progress-sequence': 1 }] for actual_progress_state in out_watcher.retrieve_progress_states( ): expected_progress_state = progress_states.pop(0) self.assertEqual(expected_progress_state, actual_progress_state) self.assertEqual(expected_progress_state, out_watcher.current_progress()) self.assertFalse(progress_states) iterable = dn_watcher.retrieve_progress_states() exhausted = object() self.assertEqual(exhausted, next(iterable, exhausted)) self.assertIsNone(dn_watcher.current_progress()) print_thread.join() finally: completed.set() tu.cleanup_file(file_name)
def test_collect_progress_updates_faulty_regex(self): file_name = tu.ensure_directory( 'build/collect_progress_updates_skip_faulty.' + tu.get_random_task_id()) progress_regex = '\^\^\^\^JOB-PROGRESS: (\S+)(?: )?(.*)' stop = Event() completed = Event() termination = Event() file = open(file_name, 'w+') file.flush() counter = cp.ProgressSequenceCounter() watcher = cp.ProgressWatcher(file_name, 'test', counter, 1024, progress_regex, stop, completed, termination) try: def print_to_file(): file.write('^^^^JOB-PROGRESS: ABCDEF string percent\n') file.write('^^^^JOB-PROGRESS: F50 Fifty percent\n') file.write( '^^^^JOB-PROGRESS: 1019101010101010101010101018101101010101010110171010110 Sixty percent\n' ) file.write('^^^^JOB-PROGRESS: 75 75% percent\n') file.flush() file.close() completed.set() print_thread = Thread(target=print_to_file, args=()) print_thread.start() progress_states = [{ 'progress-message': b'75% percent', 'progress-percent': 75, 'progress-sequence': 1 }] for actual_progress_state in watcher.retrieve_progress_states(): expected_progress_state = progress_states.pop(0) self.assertEqual(expected_progress_state, actual_progress_state) self.assertEqual(expected_progress_state, watcher.current_progress()) self.assertFalse(progress_states) print_thread.join() finally: completed.set() tu.cleanup_file(file_name)
def test_collect_progress_updates_with_empty_regex(self): file_name = tu.ensure_directory('build/collect_progress_test.' + tu.get_random_task_id()) progress_regex = '' stop = Event() completed = Event() termination = Event() file = open(file_name, 'w+') file.flush() counter = cp.ProgressSequenceCounter() watcher = cp.ProgressWatcher(file_name, 'test', counter, 1024, progress_regex, stop, completed, termination) try: def print_to_file(): file.write('Stage One complete\n') file.write('^^^^JOB-PROGRESS: 25 Twenty-Five percent\n') file.write('Stage Two complete\n') file.write('^^^^JOB-PROGRESS: 50 Fifty percent\n') file.write('Stage Three complete\n') file.write('^^^^JOB-PROGRESS: 55.0 Fifty-five percent\n') file.write('Stage Four complete\n') file.write('^^^^JOB-PROGRESS: 100 100-percent\n') file.flush() file.close() completed.set() print_thread = Thread(target=print_to_file, args=()) print_thread.start() progress_states = [] for actual_progress_state in watcher.retrieve_progress_states(): expected_progress_state = progress_states.pop(0) self.assertEqual(expected_progress_state, actual_progress_state) self.assertEqual(expected_progress_state, watcher.current_progress()) self.assertFalse(progress_states) self.assertIsNone(watcher.current_progress()) finally: completed.set() tu.cleanup_file(file_name)
def test_watcher_tail_with_read_limit(self): file_name = tu.ensure_directory('build/tail_progress_test.' + tu.get_random_task_id()) stop = Event() completed = Event() termination = Event() tail_sleep_ms = 25 try: def write_to_file(): file = open(file_name, 'w+') file.write('abcd\n') file.flush() file.write('abcdefghijkl\n') file.flush() file.write('abcdefghijklmnopqrstuvwxyz\n') file.flush() file.close() time.sleep(0.15) completed.set() Thread(target=write_to_file, args=()).start() counter = cp.ProgressSequenceCounter() watcher = cp.ProgressWatcher(file_name, 'test', counter, 10, '', stop, completed, termination) collected_data = [] for line in watcher.tail(tail_sleep_ms): collected_data.append(line.strip()) logging.debug('collected_data = {}'.format(collected_data)) expected_data = [ b'abcd', b'abcdefghij', b'kl', b'abcdefghij', b'klmnopqrst', b'uvwxyz' ] self.assertEqual(expected_data, collected_data) finally: tu.cleanup_file(file_name)
def test_watcher_tail_lot_of_writes(self): file_name = tu.ensure_directory('build/tail_progress_test.' + tu.get_random_task_id()) items_to_write = 250000 stop = Event() completed = Event() termination = Event() tail_sleep_ms = 25 try: def write_to_file(): file = open(file_name, 'w+') for item in range(items_to_write): file.write('line-{}\n'.format(item)) if item % 100 == 0: file.flush() file.flush() file.close() time.sleep(0.15) completed.set() Thread(target=write_to_file, args=()).start() counter = cp.ProgressSequenceCounter() watcher = cp.ProgressWatcher(file_name, 'test', counter, 1024, '', stop, completed, termination) collected_data = [] for line in watcher.tail(tail_sleep_ms): collected_data.append(line.strip()) logging.info('Items read: {}'.format(len(collected_data))) if items_to_write != len(collected_data): for index in range(len(collected_data)): logging.info('{}: {}'.format(index, collected_data[index])) self.assertEqual(items_to_write, len(collected_data)) expected_data = list( map(lambda x: str.encode('line-{}'.format(x)), range(items_to_write))) self.assertEqual(expected_data, collected_data) finally: tu.cleanup_file(file_name)
def test_watcher_tail(self): file_name = tu.ensure_directory('build/tail_progress_test.' + tu.get_random_task_id()) items_to_write = 12 stop = Event() completed = Event() termination = Event() write_sleep_ms = 50 tail_sleep_ms = 25 try: def write_to_file(): file = open(file_name, 'w+') for item in range(items_to_write): time.sleep(write_sleep_ms / 1000.0) file.write('{}\n'.format(item)) file.flush() file.close() time.sleep(0.15) completed.set() Thread(target=write_to_file, args=()).start() counter = cp.ProgressSequenceCounter() watcher = cp.ProgressWatcher(file_name, 'test', counter, 1024, '', stop, completed, termination) collected_data = [] for line in watcher.tail(tail_sleep_ms): collected_data.append(line.strip()) self.assertEqual(items_to_write, len(collected_data)) self.assertEqual( list(map(lambda x: str.encode(str(x)), range(items_to_write))), collected_data) finally: tu.cleanup_file(file_name)
def test_executor_launch_task_and_disconnect(self): task_id = tu.get_random_task_id() stdout_name = tu.ensure_directory('build/stdout.{}'.format(task_id)) stderr_name = tu.ensure_directory('build/stderr.{}'.format(task_id)) output_name = tu.ensure_directory('build/output.' + str(task_id)) tu.redirect_stdout_to_file(stdout_name) tu.redirect_stderr_to_file(stderr_name) try: config = cc.ExecutorConfig() stop_signal = Event() executor = ce.CookExecutor(stop_signal, config) driver = tu.FakeMesosExecutorDriver() command = 'echo "Start" >> {}; sleep 100; echo "Done." >> {}; '.format( output_name, output_name) task = { 'task_id': { 'value': task_id }, 'data': pm.encode_data( json.dumps({ 'command': command }).encode('utf8')) } executor.launchTask(driver, task) # let the process run for up to 10 seconds for _ in range(1000): time.sleep(0.01) if os.path.isfile(output_name): with open(output_name) as f: content = f.read() if 'Start' in content: break executor.disconnected(driver) self.assertTrue(executor.disconnect_signal.isSet()) self.assertTrue(executor.stop_signal.isSet()) executor.await_completion() logging.info('Task completed') if os.path.isfile(output_name): with open(output_name) as f: file_contents = f.read() self.assertTrue('Start' in file_contents) self.assertTrue('Done' not in file_contents) else: self.fail('{} does not exist.'.format(stderr_name)) expected_statuses = [{ 'task_id': { 'value': task_id }, 'state': cook.TASK_STARTING }, { 'task_id': { 'value': task_id }, 'state': cook.TASK_RUNNING }, { 'task_id': { 'value': task_id }, 'state': cook.TASK_KILLED }] tu.assert_statuses(self, expected_statuses, driver.statuses) expected_message_0 = { 'sandbox-directory': '', 'task-id': task_id, 'type': 'directory' } expected_message_1 = {'exit-code': -15, 'task-id': task_id} tu.assert_messages(self, [expected_message_0, expected_message_1], [], driver.messages) finally: tu.cleanup_output(stdout_name, stderr_name) tu.cleanup_file(output_name)
def test_manage_task_progress_in_progress_stderr_and_stdout_progress(self): max_message_length = 35 def assertions(driver, task_id, sandbox_directory): expected_statuses = [{ 'task_id': { 'value': task_id }, 'state': cook.TASK_STARTING }, { 'task_id': { 'value': task_id }, 'state': cook.TASK_RUNNING }, { 'task_id': { 'value': task_id }, 'state': cook.TASK_FINISHED }] tu.assert_statuses(self, expected_statuses, driver.statuses) expected_core_messages = [{ 'sandbox-directory': sandbox_directory, 'task-id': task_id, 'type': 'directory' }, { 'exit-code': 0, 'task-id': task_id }] expected_progress_messages = [{ 'progress-message': 'Fifty percent in progress file', 'progress-percent': 50, 'progress-sequence': 1, 'task-id': task_id }, { 'progress-message': 'Fifty-five percent in stdout', 'progress-percent': 55, 'progress-sequence': 2, 'task-id': task_id }, { 'progress-message': 'Sixty percent in stderr', 'progress-percent': 60, 'progress-sequence': 3, 'task-id': task_id }, { 'progress-message': 'Sixty-five percent in stdout wit...', 'progress-percent': 65, 'progress-sequence': 4, 'task-id': task_id }] tu.assert_messages(self, expected_core_messages, expected_progress_messages, driver.messages) stop_signal = Event() sleep_and_set_stop_signal_task(stop_signal, 60) task_id = tu.get_random_task_id() progress_name = tu.ensure_directory( 'build/progress.{}'.format(task_id)) stderr_name = tu.ensure_directory('build/stderr.{}'.format(task_id)) stdout_name = tu.ensure_directory('build/stdout.{}'.format(task_id)) config = tu.FakeExecutorConfig({ 'max_bytes_read_per_line': 1024, 'max_message_length': max_message_length, 'progress_output_env_variable': 'DEFAULT_PROGRESS_FILE_ENV_VARIABLE', 'progress_output_name': progress_name, 'progress_regex_string': '\^\^\^\^JOB-PROGRESS:\s+([0-9]*\.?[0-9]+)($|\s+.*)', 'progress_sample_interval_ms': 10, 'sandbox_directory': '/sandbox/directory/for/{}'.format(task_id), 'shutdown_grace_period_ms': 60000, 'stderr_file': stderr_name, 'stdout_file': stdout_name }) command = 'echo "Hello World"; ' \ 'echo "^^^^JOB-PROGRESS: 50 Fifty percent in progress file" >> {}; ' \ 'sleep 0.25; ' \ 'echo "^^^^JOB-PROGRESS: 55 Fifty-five percent in stdout" >> {}; ' \ 'sleep 0.25; ' \ 'echo "^^^^JOB-PROGRESS: 60 Sixty percent in stderr" >> {}; ' \ 'sleep 0.25; ' \ 'echo "^^^^JOB-PROGRESS: 65 Sixty-five percent in stdout with a long message" >> {}; ' \ 'sleep 0.25; ' \ 'echo "Exiting..."; ' \ 'exit 0'.format(progress_name, stdout_name, stderr_name, stdout_name) try: self.manage_task_runner(command, assertions, stop_signal=stop_signal, task_id=task_id, config=config) stop_signal.set() finally: tu.cleanup_file(progress_name)
def test_collect_progress_updates_lots_of_writes(self): file_name = tu.ensure_directory('build/collect_progress_test.' + tu.get_random_task_id()) progress_regex = 'progress: ([0-9]*\.?[0-9]+), (.*)' items_to_write = 250000 stop = Event() completed = Event() termination = Event() def write_to_file(): target_file = open(file_name, 'w+') unit_progress_granularity = int(items_to_write / 100) for item in range(items_to_write): remainder = (item + 1) % unit_progress_granularity if remainder == 0: progress_percent = math.ceil(item / unit_progress_granularity) target_file.write( 'progress: {0}, completed-{0}-percent\n'.format( progress_percent)) target_file.flush() target_file.write('{}\n'.format(item)) target_file.flush() target_file.close() time.sleep(0.15) write_thread = Thread(target=write_to_file, args=()) write_thread.daemon = True write_thread.start() counter = cp.ProgressSequenceCounter() watcher = cp.ProgressWatcher(file_name, 'test', counter, 1024, progress_regex, stop, completed, termination) try: progress_states = list( map( lambda x: { 'progress-message': 'completed-{}-percent'.format(x).encode(), 'progress-percent': x, 'progress-sequence': x }, range(1, 101))) for actual_progress_state in watcher.retrieve_progress_states(): expected_progress_state = progress_states.pop(0) self.assertEqual(expected_progress_state, actual_progress_state) self.assertEqual(expected_progress_state, watcher.current_progress()) if not progress_states: completed.set() self.assertFalse(progress_states) write_thread.join() finally: completed.set() tu.cleanup_file(file_name)
def test_progress_updates_early_termination(self): file_name = tu.ensure_directory('build/collect_progress_test.' + tu.get_random_task_id()) progress_regex = '\^\^\^\^JOB-PROGRESS:\s+([0-9]*\.?[0-9]+)($|\s+.*)' stop = Event() completed = Event() termination = Event() termination_trigger = Event() file = open(file_name, 'w+') file.flush() counter = cp.ProgressSequenceCounter() watcher = cp.ProgressWatcher(file_name, 'test', counter, 1024, progress_regex, stop, completed, termination) try: def print_to_file(): file.write('Stage One complete\n') file.write('^^^^JOB-PROGRESS: 25 Twenty-Five\n') file.write('^^^^JOB-PROGRESS: 50 Fifty\n') file.flush() logging.info('Awaiting termination_trigger') termination_trigger.wait() logging.info('termination_trigger has been set') termination.set() file.write('Stage Three complete\n') file.write('^^^^JOB-PROGRESS: 55 Fifty-five\n') file.write('Stage Four complete\n') file.write('^^^^JOB-PROGRESS: 100 Hundred\n') file.flush() file.close() completed.set() print_thread = Thread(target=print_to_file, args=()) print_thread.daemon = True print_thread.start() progress_states = [{ 'progress-message': b' Twenty-Five', 'progress-percent': 25, 'progress-sequence': 1 }, { 'progress-message': b' Fifty', 'progress-percent': 50, 'progress-sequence': 2 }] for actual_progress_state in watcher.retrieve_progress_states(): expected_progress_state = progress_states.pop(0) self.assertEqual(expected_progress_state, actual_progress_state) self.assertEqual(expected_progress_state, watcher.current_progress()) if expected_progress_state['progress-percent'] == 50: termination_trigger.set() self.assertFalse(progress_states) print_thread.join() finally: completed.set() tu.cleanup_file(file_name)
def test_collect_progress_updates_two_capture_groups(self): file_name = tu.ensure_directory('build/collect_progress_test.' + tu.get_random_task_id()) progress_regex = '\^\^\^\^JOB-PROGRESS:\s+([0-9]*\.?[0-9]+)($|\s+.*)' stop = Event() completed = Event() termination = Event() file = open(file_name, 'w+') file.flush() counter = cp.ProgressSequenceCounter() watcher = cp.ProgressWatcher(file_name, 'test', counter, 1024, progress_regex, stop, completed, termination) try: def print_to_file(): file.write('Stage One complete\n') file.write('^^^^JOB-PROGRESS: 25 Twenty-Five\n') file.write('^^^^JOB-PROGRESS: 50 Fifty\n') file.write('Stage Three complete\n') file.write('^^^^JOB-PROGRESS: 55.0 Fifty-five\n') file.write('^^^^JOB-PROGRESS: 65.8 Sixty-six\n') file.write('Stage Four complete\n') file.write('^^^^JOB-PROGRESS: 100 Hundred\n') file.write('^^^^JOB-PROGRESS: 100.1 Over a hundred\n') file.flush() file.close() print_thread = Thread(target=print_to_file, args=()) print_thread.start() progress_states = [{ 'progress-message': b' Twenty-Five', 'progress-percent': 25, 'progress-sequence': 1 }, { 'progress-message': b' Fifty', 'progress-percent': 50, 'progress-sequence': 2 }, { 'progress-message': b' Fifty-five', 'progress-percent': 55, 'progress-sequence': 3 }, { 'progress-message': b' Sixty-six', 'progress-percent': 66, 'progress-sequence': 4 }, { 'progress-message': b' Hundred', 'progress-percent': 100, 'progress-sequence': 5 }] for actual_progress_state in watcher.retrieve_progress_states(): expected_progress_state = progress_states.pop(0) self.assertEqual(expected_progress_state, actual_progress_state) self.assertEqual(expected_progress_state, watcher.current_progress()) if not progress_states: completed.set() self.assertFalse(progress_states) print_thread.join() finally: completed.set() tu.cleanup_file(file_name)