def test_collect_progress_updates_dev_null(self):
file_name = tu.ensure_directory('build/collect_progress_test.' +
tu.get_random_task_id())
progress_regex = '\^\^\^\^JOB-PROGRESS:\s+([0-9]*\.?[0-9]+)($|\s+.*)'
location = '/dev/null'
stop = Event()
completed = Event()
termination = Event()
file = open(file_name, 'w+')
file.flush()
counter = cp.ProgressSequenceCounter()
dn_watcher = cp.ProgressWatcher(location, 'dn', counter, 1024,
progress_regex, stop, completed,
termination)
out_watcher = cp.ProgressWatcher(file_name, 'so', counter, 1024,
progress_regex, stop, completed,
termination)
try:
def print_to_file():
file.write('Stage One complete\n')
file.write('^^^^JOB-PROGRESS: 100 100-percent\n')
file.flush()
file.close()
completed.set()
print_thread = Thread(target=print_to_file, args=())
print_thread.start()
progress_states = [{
'progress-message': b' 100-percent',
'progress-percent': 100,
'progress-sequence': 1
}]
for actual_progress_state in out_watcher.retrieve_progress_states(
):
expected_progress_state = progress_states.pop(0)
self.assertEqual(expected_progress_state,
actual_progress_state)
self.assertEqual(expected_progress_state,
out_watcher.current_progress())
self.assertFalse(progress_states)
iterable = dn_watcher.retrieve_progress_states()
exhausted = object()
self.assertEqual(exhausted, next(iterable, exhausted))
self.assertIsNone(dn_watcher.current_progress())
print_thread.join()
finally:
completed.set()
tu.cleanup_file(file_name)
def test_collect_progress_updates_faulty_regex(self):
file_name = tu.ensure_directory(
'build/collect_progress_updates_skip_faulty.' +
tu.get_random_task_id())
progress_regex = '\^\^\^\^JOB-PROGRESS: (\S+)(?: )?(.*)'
stop = Event()
completed = Event()
termination = Event()
file = open(file_name, 'w+')
file.flush()
counter = cp.ProgressSequenceCounter()
watcher = cp.ProgressWatcher(file_name, 'test', counter, 1024,
progress_regex, stop, completed,
termination)
try:
def print_to_file():
file.write('^^^^JOB-PROGRESS: ABCDEF string percent\n')
file.write('^^^^JOB-PROGRESS: F50 Fifty percent\n')
file.write(
'^^^^JOB-PROGRESS: 1019101010101010101010101018101101010101010110171010110 Sixty percent\n'
)
file.write('^^^^JOB-PROGRESS: 75 75% percent\n')
file.flush()
file.close()
completed.set()
print_thread = Thread(target=print_to_file, args=())
print_thread.start()
progress_states = [{
'progress-message': b'75% percent',
'progress-percent': 75,
'progress-sequence': 1
}]
for actual_progress_state in watcher.retrieve_progress_states():
expected_progress_state = progress_states.pop(0)
self.assertEqual(expected_progress_state,
actual_progress_state)
self.assertEqual(expected_progress_state,
watcher.current_progress())
self.assertFalse(progress_states)
print_thread.join()
finally:
completed.set()
tu.cleanup_file(file_name)
def test_collect_progress_updates_with_empty_regex(self):
file_name = tu.ensure_directory('build/collect_progress_test.' +
tu.get_random_task_id())
progress_regex = ''
stop = Event()
completed = Event()
termination = Event()
file = open(file_name, 'w+')
file.flush()
counter = cp.ProgressSequenceCounter()
watcher = cp.ProgressWatcher(file_name, 'test', counter, 1024,
progress_regex, stop, completed,
termination)
try:
def print_to_file():
file.write('Stage One complete\n')
file.write('^^^^JOB-PROGRESS: 25 Twenty-Five percent\n')
file.write('Stage Two complete\n')
file.write('^^^^JOB-PROGRESS: 50 Fifty percent\n')
file.write('Stage Three complete\n')
file.write('^^^^JOB-PROGRESS: 55.0 Fifty-five percent\n')
file.write('Stage Four complete\n')
file.write('^^^^JOB-PROGRESS: 100 100-percent\n')
file.flush()
file.close()
completed.set()
print_thread = Thread(target=print_to_file, args=())
print_thread.start()
progress_states = []
for actual_progress_state in watcher.retrieve_progress_states():
expected_progress_state = progress_states.pop(0)
self.assertEqual(expected_progress_state,
actual_progress_state)
self.assertEqual(expected_progress_state,
watcher.current_progress())
self.assertFalse(progress_states)
self.assertIsNone(watcher.current_progress())
finally:
completed.set()
tu.cleanup_file(file_name)
def test_watcher_tail_with_read_limit(self):
file_name = tu.ensure_directory('build/tail_progress_test.' +
tu.get_random_task_id())
stop = Event()
completed = Event()
termination = Event()
tail_sleep_ms = 25
try:
def write_to_file():
file = open(file_name, 'w+')
file.write('abcd\n')
file.flush()
file.write('abcdefghijkl\n')
file.flush()
file.write('abcdefghijklmnopqrstuvwxyz\n')
file.flush()
file.close()
time.sleep(0.15)
completed.set()
Thread(target=write_to_file, args=()).start()
counter = cp.ProgressSequenceCounter()
watcher = cp.ProgressWatcher(file_name, 'test', counter, 10, '',
stop, completed, termination)
collected_data = []
for line in watcher.tail(tail_sleep_ms):
collected_data.append(line.strip())
logging.debug('collected_data = {}'.format(collected_data))
expected_data = [
b'abcd', b'abcdefghij', b'kl', b'abcdefghij', b'klmnopqrst',
b'uvwxyz'
]
self.assertEqual(expected_data, collected_data)
finally:
tu.cleanup_file(file_name)
def test_watcher_tail_lot_of_writes(self):
file_name = tu.ensure_directory('build/tail_progress_test.' +
tu.get_random_task_id())
items_to_write = 250000
stop = Event()
completed = Event()
termination = Event()
tail_sleep_ms = 25
try:
def write_to_file():
file = open(file_name, 'w+')
for item in range(items_to_write):
file.write('line-{}\n'.format(item))
if item % 100 == 0:
file.flush()
file.flush()
file.close()
time.sleep(0.15)
completed.set()
Thread(target=write_to_file, args=()).start()
counter = cp.ProgressSequenceCounter()
watcher = cp.ProgressWatcher(file_name, 'test', counter, 1024, '',
stop, completed, termination)
collected_data = []
for line in watcher.tail(tail_sleep_ms):
collected_data.append(line.strip())
logging.info('Items read: {}'.format(len(collected_data)))
if items_to_write != len(collected_data):
for index in range(len(collected_data)):
logging.info('{}: {}'.format(index, collected_data[index]))
self.assertEqual(items_to_write, len(collected_data))
expected_data = list(
map(lambda x: str.encode('line-{}'.format(x)),
range(items_to_write)))
self.assertEqual(expected_data, collected_data)
finally:
tu.cleanup_file(file_name)
def test_watcher_tail(self):
file_name = tu.ensure_directory('build/tail_progress_test.' +
tu.get_random_task_id())
items_to_write = 12
stop = Event()
completed = Event()
termination = Event()
write_sleep_ms = 50
tail_sleep_ms = 25
try:
def write_to_file():
file = open(file_name, 'w+')
for item in range(items_to_write):
time.sleep(write_sleep_ms / 1000.0)
file.write('{}\n'.format(item))
file.flush()
file.close()
time.sleep(0.15)
completed.set()
Thread(target=write_to_file, args=()).start()
counter = cp.ProgressSequenceCounter()
watcher = cp.ProgressWatcher(file_name, 'test', counter, 1024, '',
stop, completed, termination)
collected_data = []
for line in watcher.tail(tail_sleep_ms):
collected_data.append(line.strip())
self.assertEqual(items_to_write, len(collected_data))
self.assertEqual(
list(map(lambda x: str.encode(str(x)), range(items_to_write))),
collected_data)
finally:
tu.cleanup_file(file_name)
def test_executor_launch_task_and_disconnect(self):
task_id = tu.get_random_task_id()
stdout_name = tu.ensure_directory('build/stdout.{}'.format(task_id))
stderr_name = tu.ensure_directory('build/stderr.{}'.format(task_id))
output_name = tu.ensure_directory('build/output.' + str(task_id))
tu.redirect_stdout_to_file(stdout_name)
tu.redirect_stderr_to_file(stderr_name)
try:
config = cc.ExecutorConfig()
stop_signal = Event()
executor = ce.CookExecutor(stop_signal, config)
driver = tu.FakeMesosExecutorDriver()
command = 'echo "Start" >> {}; sleep 100; echo "Done." >> {}; '.format(
output_name, output_name)
task = {
'task_id': {
'value': task_id
},
'data':
pm.encode_data(
json.dumps({
'command': command
}).encode('utf8'))
}
executor.launchTask(driver, task)
# let the process run for up to 10 seconds
for _ in range(1000):
time.sleep(0.01)
if os.path.isfile(output_name):
with open(output_name) as f:
content = f.read()
if 'Start' in content:
break
executor.disconnected(driver)
self.assertTrue(executor.disconnect_signal.isSet())
self.assertTrue(executor.stop_signal.isSet())
executor.await_completion()
logging.info('Task completed')
if os.path.isfile(output_name):
with open(output_name) as f:
file_contents = f.read()
self.assertTrue('Start' in file_contents)
self.assertTrue('Done' not in file_contents)
else:
self.fail('{} does not exist.'.format(stderr_name))
expected_statuses = [{
'task_id': {
'value': task_id
},
'state': cook.TASK_STARTING
}, {
'task_id': {
'value': task_id
},
'state': cook.TASK_RUNNING
}, {
'task_id': {
'value': task_id
},
'state': cook.TASK_KILLED
}]
tu.assert_statuses(self, expected_statuses, driver.statuses)
expected_message_0 = {
'sandbox-directory': '',
'task-id': task_id,
'type': 'directory'
}
expected_message_1 = {'exit-code': -15, 'task-id': task_id}
tu.assert_messages(self, [expected_message_0, expected_message_1],
[], driver.messages)
finally:
tu.cleanup_output(stdout_name, stderr_name)
tu.cleanup_file(output_name)
def test_manage_task_progress_in_progress_stderr_and_stdout_progress(self):
max_message_length = 35
def assertions(driver, task_id, sandbox_directory):
expected_statuses = [{
'task_id': {
'value': task_id
},
'state': cook.TASK_STARTING
}, {
'task_id': {
'value': task_id
},
'state': cook.TASK_RUNNING
}, {
'task_id': {
'value': task_id
},
'state': cook.TASK_FINISHED
}]
tu.assert_statuses(self, expected_statuses, driver.statuses)
expected_core_messages = [{
'sandbox-directory': sandbox_directory,
'task-id': task_id,
'type': 'directory'
}, {
'exit-code': 0,
'task-id': task_id
}]
expected_progress_messages = [{
'progress-message': 'Fifty percent in progress file',
'progress-percent': 50,
'progress-sequence': 1,
'task-id': task_id
}, {
'progress-message': 'Fifty-five percent in stdout',
'progress-percent': 55,
'progress-sequence': 2,
'task-id': task_id
}, {
'progress-message': 'Sixty percent in stderr',
'progress-percent': 60,
'progress-sequence': 3,
'task-id': task_id
}, {
'progress-message': 'Sixty-five percent in stdout wit...',
'progress-percent': 65,
'progress-sequence': 4,
'task-id': task_id
}]
tu.assert_messages(self, expected_core_messages,
expected_progress_messages, driver.messages)
stop_signal = Event()
sleep_and_set_stop_signal_task(stop_signal, 60)
task_id = tu.get_random_task_id()
progress_name = tu.ensure_directory(
'build/progress.{}'.format(task_id))
stderr_name = tu.ensure_directory('build/stderr.{}'.format(task_id))
stdout_name = tu.ensure_directory('build/stdout.{}'.format(task_id))
config = tu.FakeExecutorConfig({
'max_bytes_read_per_line':
1024,
'max_message_length':
max_message_length,
'progress_output_env_variable':
'DEFAULT_PROGRESS_FILE_ENV_VARIABLE',
'progress_output_name':
progress_name,
'progress_regex_string':
'\^\^\^\^JOB-PROGRESS:\s+([0-9]*\.?[0-9]+)($|\s+.*)',
'progress_sample_interval_ms':
10,
'sandbox_directory':
'/sandbox/directory/for/{}'.format(task_id),
'shutdown_grace_period_ms':
60000,
'stderr_file':
stderr_name,
'stdout_file':
stdout_name
})
command = 'echo "Hello World"; ' \
'echo "^^^^JOB-PROGRESS: 50 Fifty percent in progress file" >> {}; ' \
'sleep 0.25; ' \
'echo "^^^^JOB-PROGRESS: 55 Fifty-five percent in stdout" >> {}; ' \
'sleep 0.25; ' \
'echo "^^^^JOB-PROGRESS: 60 Sixty percent in stderr" >> {}; ' \
'sleep 0.25; ' \
'echo "^^^^JOB-PROGRESS: 65 Sixty-five percent in stdout with a long message" >> {}; ' \
'sleep 0.25; ' \
'echo "Exiting..."; ' \
'exit 0'.format(progress_name, stdout_name, stderr_name, stdout_name)
try:
self.manage_task_runner(command,
assertions,
stop_signal=stop_signal,
task_id=task_id,
config=config)
stop_signal.set()
finally:
tu.cleanup_file(progress_name)
def test_collect_progress_updates_lots_of_writes(self):
file_name = tu.ensure_directory('build/collect_progress_test.' +
tu.get_random_task_id())
progress_regex = 'progress: ([0-9]*\.?[0-9]+), (.*)'
items_to_write = 250000
stop = Event()
completed = Event()
termination = Event()
def write_to_file():
target_file = open(file_name, 'w+')
unit_progress_granularity = int(items_to_write / 100)
for item in range(items_to_write):
remainder = (item + 1) % unit_progress_granularity
if remainder == 0:
progress_percent = math.ceil(item /
unit_progress_granularity)
target_file.write(
'progress: {0}, completed-{0}-percent\n'.format(
progress_percent))
target_file.flush()
target_file.write('{}\n'.format(item))
target_file.flush()
target_file.close()
time.sleep(0.15)
write_thread = Thread(target=write_to_file, args=())
write_thread.daemon = True
write_thread.start()
counter = cp.ProgressSequenceCounter()
watcher = cp.ProgressWatcher(file_name, 'test', counter, 1024,
progress_regex, stop, completed,
termination)
try:
progress_states = list(
map(
lambda x: {
'progress-message':
'completed-{}-percent'.format(x).encode(),
'progress-percent':
x,
'progress-sequence':
x
}, range(1, 101)))
for actual_progress_state in watcher.retrieve_progress_states():
expected_progress_state = progress_states.pop(0)
self.assertEqual(expected_progress_state,
actual_progress_state)
self.assertEqual(expected_progress_state,
watcher.current_progress())
if not progress_states:
completed.set()
self.assertFalse(progress_states)
write_thread.join()
finally:
completed.set()
tu.cleanup_file(file_name)
def test_progress_updates_early_termination(self):
file_name = tu.ensure_directory('build/collect_progress_test.' +
tu.get_random_task_id())
progress_regex = '\^\^\^\^JOB-PROGRESS:\s+([0-9]*\.?[0-9]+)($|\s+.*)'
stop = Event()
completed = Event()
termination = Event()
termination_trigger = Event()
file = open(file_name, 'w+')
file.flush()
counter = cp.ProgressSequenceCounter()
watcher = cp.ProgressWatcher(file_name, 'test', counter, 1024,
progress_regex, stop, completed,
termination)
try:
def print_to_file():
file.write('Stage One complete\n')
file.write('^^^^JOB-PROGRESS: 25 Twenty-Five\n')
file.write('^^^^JOB-PROGRESS: 50 Fifty\n')
file.flush()
logging.info('Awaiting termination_trigger')
termination_trigger.wait()
logging.info('termination_trigger has been set')
termination.set()
file.write('Stage Three complete\n')
file.write('^^^^JOB-PROGRESS: 55 Fifty-five\n')
file.write('Stage Four complete\n')
file.write('^^^^JOB-PROGRESS: 100 Hundred\n')
file.flush()
file.close()
completed.set()
print_thread = Thread(target=print_to_file, args=())
print_thread.daemon = True
print_thread.start()
progress_states = [{
'progress-message': b' Twenty-Five',
'progress-percent': 25,
'progress-sequence': 1
}, {
'progress-message': b' Fifty',
'progress-percent': 50,
'progress-sequence': 2
}]
for actual_progress_state in watcher.retrieve_progress_states():
expected_progress_state = progress_states.pop(0)
self.assertEqual(expected_progress_state,
actual_progress_state)
self.assertEqual(expected_progress_state,
watcher.current_progress())
if expected_progress_state['progress-percent'] == 50:
termination_trigger.set()
self.assertFalse(progress_states)
print_thread.join()
finally:
completed.set()
tu.cleanup_file(file_name)
def test_collect_progress_updates_two_capture_groups(self):
file_name = tu.ensure_directory('build/collect_progress_test.' +
tu.get_random_task_id())
progress_regex = '\^\^\^\^JOB-PROGRESS:\s+([0-9]*\.?[0-9]+)($|\s+.*)'
stop = Event()
completed = Event()
termination = Event()
file = open(file_name, 'w+')
file.flush()
counter = cp.ProgressSequenceCounter()
watcher = cp.ProgressWatcher(file_name, 'test', counter, 1024,
progress_regex, stop, completed,
termination)
try:
def print_to_file():
file.write('Stage One complete\n')
file.write('^^^^JOB-PROGRESS: 25 Twenty-Five\n')
file.write('^^^^JOB-PROGRESS: 50 Fifty\n')
file.write('Stage Three complete\n')
file.write('^^^^JOB-PROGRESS: 55.0 Fifty-five\n')
file.write('^^^^JOB-PROGRESS: 65.8 Sixty-six\n')
file.write('Stage Four complete\n')
file.write('^^^^JOB-PROGRESS: 100 Hundred\n')
file.write('^^^^JOB-PROGRESS: 100.1 Over a hundred\n')
file.flush()
file.close()
print_thread = Thread(target=print_to_file, args=())
print_thread.start()
progress_states = [{
'progress-message': b' Twenty-Five',
'progress-percent': 25,
'progress-sequence': 1
}, {
'progress-message': b' Fifty',
'progress-percent': 50,
'progress-sequence': 2
}, {
'progress-message': b' Fifty-five',
'progress-percent': 55,
'progress-sequence': 3
}, {
'progress-message': b' Sixty-six',
'progress-percent': 66,
'progress-sequence': 4
}, {
'progress-message': b' Hundred',
'progress-percent': 100,
'progress-sequence': 5
}]
for actual_progress_state in watcher.retrieve_progress_states():
expected_progress_state = progress_states.pop(0)
self.assertEqual(expected_progress_state,
actual_progress_state)
self.assertEqual(expected_progress_state,
watcher.current_progress())
if not progress_states:
completed.set()
self.assertFalse(progress_states)
print_thread.join()
finally:
completed.set()
tu.cleanup_file(file_name)
