def test_kill_task_terminate(self):
task_id = get_random_task_id()
stdout_name = ensure_directory('build/stdout.' + str(task_id))
stderr_name = ensure_directory('build/stderr.' + str(task_id))
stdout = open(stdout_name, 'w+')
stderr = open(stderr_name, 'w+')
try:
command = 'sleep 100'
process = subprocess.Popen(command,
shell=True,
stdout=stdout,
stderr=stderr)
process_info = process, stdout, stderr
shutdown_grace_period_ms = 2000
ce.kill_task(process_info, shutdown_grace_period_ms)
# await process termination
while process.poll() is None:
time.sleep(0.01)
self.assertEqual(-1 * signal.SIGTERM, process.poll())
self.assertTrue(stdout.closed)
self.assertTrue(stderr.closed)
finally:
cleanup_output(stdout_name, stderr_name)
def test_await_process_completion_killed(self):
task_id = tu.get_random_task_id()
stdout_name = tu.ensure_directory('build/stdout.{}'.format(task_id))
stderr_name = tu.ensure_directory('build/stderr.{}'.format(task_id))
tu.redirect_stdout_to_file(stdout_name)
tu.redirect_stderr_to_file(stderr_name)
try:
command = 'sleep 100'
process = subprocess.Popen(command,
preexec_fn=os.setpgrp,
shell=True)
shutdown_grace_period_ms = 2000
stop_signal = Event()
sleep_and_set_stop_signal_task(stop_signal, 2)
ce.await_process_completion(process, stop_signal,
shutdown_grace_period_ms)
self.assertTrue(process.returncode < 0)
finally:
tu.cleanup_output(stdout_name, stderr_name)
def test_kill_task_terminate_with_sigkill(self):
task_id = tu.get_random_task_id()
stdout_name = tu.ensure_directory('build/stdout.{}'.format(task_id))
stderr_name = tu.ensure_directory('build/stderr.{}'.format(task_id))
tu.redirect_stdout_to_file(stdout_name)
tu.redirect_stderr_to_file(stderr_name)
try:
command = "trap '' TERM SIGTERM; sleep 200"
process = cs.launch_process(command, {})
shutdown_grace_period_ms = 1000
group_id = cs.find_process_group(process.pid)
self.assertGreater(len(find_process_ids_in_group(group_id)), 0)
cs.kill_process(process, shutdown_grace_period_ms)
# await process termination
for i in range(1, 10 * shutdown_grace_period_ms):
if process.poll() is None:
time.sleep(0.01)
if process.poll() is None:
process.kill()
self.assertTrue(
((-1 * signal.SIGKILL) == process.poll())
or ((128 + signal.SIGKILL) == process.poll()),
'Process exited with code {}'.format(process.poll()))
self.assertEqual(len(find_process_ids_in_group(group_id)), 0)
finally:
tu.cleanup_output(stdout_name, stderr_name)
def manage_task_runner(self,
command,
assertions_fn,
stop_signal=None,
task_id=None,
config=None,
driver=None):
if driver is None:
driver = tu.FakeMesosExecutorDriver()
if stop_signal is None:
stop_signal = Event()
if task_id is None:
task_id = tu.get_random_task_id()
task = {
'task_id': {
'value': task_id
},
'data':
pm.encode_data(json.dumps({
'command': command
}).encode('utf8'))
}
stdout_name = tu.ensure_directory('build/stdout.{}'.format(task_id))
stderr_name = tu.ensure_directory('build/stderr.{}'.format(task_id))
tu.redirect_stdout_to_file(stdout_name)
tu.redirect_stderr_to_file(stderr_name)
completed_signal = Event()
if config is None:
sandbox_directory = '/location/to/task/sandbox/{}'.format(task_id)
config = cc.ExecutorConfig(
max_message_length=300,
progress_output_name=stdout_name,
progress_regex_string=
'\^\^\^\^JOB-PROGRESS:\s+([0-9]*\.?[0-9]+)($|\s+.*)',
progress_sample_interval_ms=100,
sandbox_directory=sandbox_directory)
else:
sandbox_directory = config.sandbox_directory
try:
ce.manage_task(driver, task, stop_signal, completed_signal, config)
self.assertTrue(completed_signal.isSet())
assertions_fn(driver, task_id, sandbox_directory)
finally:
tu.cleanup_output(stdout_name, stderr_name)
def test_launch_task_interactive_output(self):
task_id = tu.get_random_task_id()
command = 'echo "Start"; echo "Hello"; sleep 100; echo "World"; echo "Done"; '
task = {
'task_id': {
'value': task_id
},
'data':
pm.encode_data(json.dumps({
'command': command
}).encode('utf8'))
}
stdout_name = tu.ensure_directory('build/stdout.{}'.format(task_id))
stderr_name = tu.ensure_directory('build/stderr.{}'.format(task_id))
tu.redirect_stdout_to_file(stdout_name)
tu.redirect_stderr_to_file(stderr_name)
try:
process = ce.launch_task(task, os.environ)
self.assertIsNotNone(process)
# let the process run for up to 50 seconds
for _ in range(5000):
if cs.is_process_running(process):
time.sleep(0.01)
with open(stdout_name) as f:
stdout_content = f.read()
if 'Start' in stdout_content and 'Hello' in stdout_content:
break
try:
with open(stdout_name) as f:
stdout_content = f.read()
logging.info(
'Contents of stdout: {}'.format(stdout_content))
self.assertTrue("Start" in stdout_content)
self.assertTrue("Hello" in stdout_content)
self.assertFalse("World" in stdout_content)
self.assertFalse("Done" in stdout_content)
finally:
if process.poll() is None:
logging.info('Killing launched process')
process.kill()
finally:
tu.cleanup_output(stdout_name, stderr_name)
def test_cleanup_process(self):
task_id = get_random_task_id()
stdout_name = ensure_directory('build/stdout.' + str(task_id))
stderr_name = ensure_directory('build/stderr.' + str(task_id))
stdout = open(stdout_name, 'w+')
stderr = open(stderr_name, 'w+')
try:
process_info = None, stdout, stderr
ce.cleanup_process(process_info)
self.assertTrue(stdout.closed)
self.assertTrue(stderr.closed)
finally:
cleanup_output(stdout_name, stderr_name)
def test_collect_progress_updates_with_empty_regex(self):
file_name = ensure_directory('build/collect_progress_test.' +
get_random_task_id())
progress_regex_string = ''
config = cc.ExecutorConfig(progress_output_name=file_name,
progress_regex_string=progress_regex_string)
stop_signal = Event()
completed_signal = Event()
file = open(file_name, 'w+')
file.flush()
progress_watcher = cp.ProgressWatcher(config, stop_signal,
completed_signal)
try:
def read_progress_states():
for _ in progress_watcher.retrieve_progress_states():
pass
Thread(target=read_progress_states, args=()).start()
file.write("Stage One complete\n")
file.flush()
file.write("^^^^JOB-PROGRESS: 25 Twenty-Fine percent\n")
file.flush()
file.write("Stage Two complete\n")
file.flush()
file.write("^^^^JOB-PROGRESS: 50 Fifty percent\n")
file.flush()
time.sleep(0.10)
self.assertIsNone(progress_watcher.current_progress())
file.write("Stage Three complete\n")
file.flush()
time.sleep(0.10)
self.assertIsNone(progress_watcher.current_progress())
file.write("^^^^JOB-PROGRESS: 55 Fifty-five percent\n")
file.flush()
time.sleep(0.10)
self.assertIsNone(progress_watcher.current_progress())
file.write("Stage Four complete\n")
file.flush()
file.write("^^^^JOB-PROGRESS: 100 Hundred percent\n")
file.flush()
time.sleep(0.10)
self.assertIsNone(progress_watcher.current_progress())
finally:
completed_signal.set()
file.close()
if os.path.isfile(file_name):
os.remove(file_name)
def assertions(driver, task_id, sandbox_directory):
expected_statuses = [{
'task_id': {
'value': task_id
},
'state': cook.TASK_STARTING
}, {
'task_id': {
'value': task_id
},
'state': cook.TASK_RUNNING
}, {
'task_id': {
'value': task_id
},
'state': cook.TASK_FINISHED
}]
tu.assert_statuses(self, expected_statuses, driver.statuses)
expected_message_0 = {
'sandbox-directory': sandbox_directory,
'task-id': task_id,
'type': 'directory'
}
expected_message_1 = {'exit-code': 0, 'task-id': task_id}
tu.assert_messages(self, [expected_message_0, expected_message_1],
[], driver.messages)
stdout_name = tu.ensure_directory('build/stdout.' + str(task_id))
if os.path.isfile(stdout_name):
with open(stdout_name) as f:
file_contents = f.read()
self.assertEqual(num_iterations * 25,
file_contents.count('X'))
else:
self.fail('{} does not exist.'.format(stdout_name))
stderr_name = tu.ensure_directory('build/stderr.' + str(task_id))
if os.path.isfile(stderr_name):
with open(stderr_name) as f:
file_contents = f.read()
self.assertEqual(num_iterations * 25,
file_contents.count('X'))
else:
self.fail('{} does not exist.'.format(stderr_name))
def test_launch_task(self):
task_id = get_random_task_id()
command = 'echo "Hello World"; echo "Error Message" >&2'
task = {
'task_id': {
'value': task_id
},
'data':
encode_data(json.dumps({
'command': command
}).encode('utf8'))
}
stdout_name = ensure_directory('build/stdout.' + str(task_id))
stderr_name = ensure_directory('build/stderr.' + str(task_id))
if not os.path.isdir("build"):
os.mkdir("build")
try:
process, stdout, stderr = ce.launch_task(task, stdout_name,
stderr_name)
self.assertIsNotNone(process)
for i in range(100):
if process.poll() is None:
time.sleep(0.01)
stdout.close()
stderr.close()
if process.poll() is None:
process.kill()
self.assertEqual(0, process.poll())
with open(stdout_name) as f:
stdout_content = f.read()
self.assertEqual("Hello World\n", stdout_content)
with open(stderr_name) as f:
stderr_content = f.read()
self.assertEqual("Error Message\n", stderr_content)
finally:
cleanup_output(stdout_name, stderr_name)
def test_manage_task_successful_exit_despite_faulty_driver(self):
def assertions(driver, task_id, sandbox_directory):
expected_statuses = [{
'task_id': {
'value': task_id
},
'state': cook.TASK_STARTING
}, {
'task_id': {
'value': task_id
},
'state': cook.TASK_RUNNING
}, {
'task_id': {
'value': task_id
},
'state': cook.TASK_FINISHED
}]
tu.assert_statuses(self, expected_statuses, driver.statuses)
expected_core_messages = [{
'sandbox-directory': sandbox_directory,
'task-id': task_id,
'type': 'directory'
}, {
'exit-code': 0,
'task-id': task_id
}]
expected_progress_messages = [
{
'progress-message': 'ninety percent',
'progress-percent': 90,
'progress-sequence': 1,
'task-id': task_id
},
# retried because the previous send fails
{
'progress-message': 'ninety percent',
'progress-percent': 90,
'progress-sequence': 1,
'task-id': task_id
}
]
tu.assert_messages(self, expected_core_messages,
expected_progress_messages, driver.messages)
test_file_name = tu.ensure_directory('build/file.' +
tu.get_random_task_id())
command = ('echo "^^^^JOB-PROGRESS: 90 ninety percent"'.format(
test_file_name))
socket_error = OSError(
'socket.error') # socket.error is an alias of OSError
self.manage_task_runner(
command,
assertions,
driver=tu.ErrorMesosExecutorDriver(socket_error))
def test_launch_task(self):
task_id = tu.get_random_task_id()
command = 'echo "Hello World"; echo "Error Message" >&2'
task = {
'task_id': {
'value': task_id
},
'data':
pm.encode_data(json.dumps({
'command': command
}).encode('utf8'))
}
stdout_name = tu.ensure_directory('build/stdout.{}'.format(task_id))
stderr_name = tu.ensure_directory('build/stderr.{}'.format(task_id))
tu.redirect_stdout_to_file(stdout_name)
tu.redirect_stderr_to_file(stderr_name)
try:
process = ce.launch_task(task, os.environ)
self.assertIsNotNone(process)
for _ in range(100):
if cs.is_process_running(process):
time.sleep(0.01)
if process.poll() is None:
process.kill()
tu.close_sys_outputs()
self.assertEqual(0, process.poll())
with open(stdout_name) as f:
stdout_content = f.read()
self.assertTrue("Hello World\n" in stdout_content)
with open(stderr_name) as f:
stderr_content = f.read()
self.assertTrue("Error Message\n" in stderr_content)
finally:
tu.cleanup_output(stdout_name, stderr_name)
def test_collect_progress_updates_dev_null(self):
file_name = tu.ensure_directory('build/collect_progress_test.' +
tu.get_random_task_id())
progress_regex = '\^\^\^\^JOB-PROGRESS:\s+([0-9]*\.?[0-9]+)($|\s+.*)'
location = '/dev/null'
stop = Event()
completed = Event()
termination = Event()
file = open(file_name, 'w+')
file.flush()
counter = cp.ProgressSequenceCounter()
dn_watcher = cp.ProgressWatcher(location, 'dn', counter, 1024,
progress_regex, stop, completed,
termination)
out_watcher = cp.ProgressWatcher(file_name, 'so', counter, 1024,
progress_regex, stop, completed,
termination)
try:
def print_to_file():
file.write('Stage One complete\n')
file.write('^^^^JOB-PROGRESS: 100 100-percent\n')
file.flush()
file.close()
completed.set()
print_thread = Thread(target=print_to_file, args=())
print_thread.start()
progress_states = [{
'progress-message': b' 100-percent',
'progress-percent': 100,
'progress-sequence': 1
}]
for actual_progress_state in out_watcher.retrieve_progress_states(
):
expected_progress_state = progress_states.pop(0)
self.assertEqual(expected_progress_state,
actual_progress_state)
self.assertEqual(expected_progress_state,
out_watcher.current_progress())
self.assertFalse(progress_states)
iterable = dn_watcher.retrieve_progress_states()
exhausted = object()
self.assertEqual(exhausted, next(iterable, exhausted))
self.assertIsNone(dn_watcher.current_progress())
print_thread.join()
finally:
completed.set()
tu.cleanup_file(file_name)
def assertions(driver, task_id, sandbox_directory):
expected_statuses = [{
'task_id': {
'value': task_id
},
'state': cook.TASK_STARTING
}, {
'task_id': {
'value': task_id
},
'state': cook.TASK_RUNNING
}, {
'task_id': {
'value': task_id
},
'state': cook.TASK_FINISHED
}]
tu.assert_statuses(self, expected_statuses, driver.statuses)
logging.info('Messages: {}'.format(driver.messages))
self.assertLess(2, len(driver.messages))
actual_encoded_message_0 = driver.messages[0]
expected_message_0 = {
'sandbox-directory': sandbox_directory,
'task-id': task_id,
'type': 'directory'
}
tu.assert_message(self, expected_message_0,
actual_encoded_message_0)
found_exit_message = False
for index in range(1, len(driver.messages)):
actual_encoded_message = driver.messages[index]
actual_message = tu.parse_message(actual_encoded_message)
if 'exit-code' in actual_message:
found_exit_message = True
expected_message = {'exit-code': 0, 'task-id': task_id}
tu.assert_message(self, expected_message,
actual_encoded_message)
break
self.assertTrue(found_exit_message)
stderr_name = tu.ensure_directory('build/stderr.' + str(task_id))
if os.path.isfile(stderr_name):
with open(stderr_name) as f:
file_contents = f.read()
self.assertEqual(num_iterations * 25,
file_contents.count('X'))
else:
self.fail('{} does not exist.'.format(stderr_name))
def manage_task_runner(self, command, assertions_fn, stop_signal=Event()):
driver = FakeMesosExecutorDriver()
task_id = get_random_task_id()
task = {
'task_id': {
'value': task_id
},
'data':
encode_data(json.dumps({
'command': command
}).encode('utf8'))
}
stdout_name = ensure_directory('build/stdout.' + str(task_id))
stderr_name = ensure_directory('build/stderr.' + str(task_id))
completed_signal = Event()
max_message_length = 300
progress_sample_interval_ms = 100
sandbox_directory = '/location/to/task/sandbox/{}'.format(task_id)
progress_output_name = stdout_name
progress_regex_string = '\^\^\^\^JOB-PROGRESS: (\d*)(?: )?(.*)'
config = cc.ExecutorConfig(
max_message_length=max_message_length,
progress_output_name=progress_output_name,
progress_regex_string=progress_regex_string,
progress_sample_interval_ms=progress_sample_interval_ms,
sandbox_directory=sandbox_directory)
try:
ce.manage_task(driver, task, stop_signal, completed_signal, config,
stdout_name, stderr_name)
self.assertTrue(completed_signal.isSet())
assertions_fn(driver, task_id, sandbox_directory)
finally:
cleanup_output(stdout_name, stderr_name)
def test_kill_task_terminate_with_sigterm(self):
task_id = tu.get_random_task_id()
stdout_name = tu.ensure_directory('build/stdout.{}'.format(task_id))
stderr_name = tu.ensure_directory('build/stderr.{}'.format(task_id))
tu.redirect_stdout_to_file(stdout_name)
tu.redirect_stderr_to_file(stderr_name)
try:
command = "bash -c 'function handle_term { echo GOT TERM; }; trap handle_term SIGTERM TERM; sleep 200'"
process = cs.launch_process(command, {})
shutdown_grace_period_ms = 1000
group_id = cs.find_process_group(process.pid)
self.assertGreater(len(find_process_ids_in_group(group_id)), 0)
cs.kill_process(process, shutdown_grace_period_ms)
# await process termination
for i in range(1, 10 * shutdown_grace_period_ms):
if process.poll() is None:
time.sleep(0.01)
if process.poll() is None:
process.kill()
self.assertTrue(
((-1 * signal.SIGTERM) == process.poll())
or ((128 + signal.SIGTERM) == process.poll()),
'Process exited with code {}'.format(process.poll()))
self.assertEqual(0, len(find_process_ids_in_group(group_id)))
with open(stdout_name) as f:
file_contents = f.read()
self.assertTrue('GOT TERM' in file_contents)
finally:
tu.cleanup_output(stdout_name, stderr_name)
def process_launch_and_kill_helper(self, kill_fn):
task_id = tu.get_random_task_id()
stdout_name = tu.ensure_directory('build/stdout.{}'.format(task_id))
stderr_name = tu.ensure_directory('build/stderr.{}'.format(task_id))
tu.redirect_stdout_to_file(stdout_name)
tu.redirect_stderr_to_file(stderr_name)
try:
start_time = time.time()
command = 'echo "A.$(sleep 30)" & echo "B.$(sleep 30)" & echo "C.$(sleep 30)" &'
environment = {}
process = cs.launch_process(command, environment)
group_id = cs.find_process_group(process.pid)
self.assertGreater(group_id, 0)
child_process_ids = tu.wait_for(
lambda: find_process_ids_in_group(group_id),
lambda data: len(data) >= 7,
default_value=[])
self.assertGreaterEqual(len(child_process_ids), 7)
self.assertLessEqual(len(child_process_ids), 10)
kill_fn(process.pid)
child_process_ids = tu.wait_for(
lambda: find_process_ids_in_group(group_id),
lambda data: len(data) == 0,
default_value=[])
self.assertEqual(0, len(child_process_ids))
# ensure the test ran in under 30 seconds
self.assertLess(time.time() - start_time, 20)
finally:
tu.cleanup_output(stdout_name, stderr_name)
def test_await_process_completion_killed(self):
task_id = get_random_task_id()
stdout_name = ensure_directory('build/stdout.' + str(task_id))
stderr_name = ensure_directory('build/stderr.' + str(task_id))
stdout = open(stdout_name, 'w+')
stderr = open(stderr_name, 'w+')
try:
command = 'sleep 100'
process = subprocess.Popen(command,
shell=True,
stdout=stdout,
stderr=stderr)
process_info = process, stdout, stderr
shutdown_grace_period_ms = 2000
stop_signal = Event()
def sleep_and_set_stop_signal():
time.sleep(2 * cook.RUNNING_POLL_INTERVAL_SECS)
stop_signal.set()
thread = Thread(target=sleep_and_set_stop_signal, args=())
thread.start()
ce.await_process_completion(stop_signal, process_info,
shutdown_grace_period_ms)
self.assertTrue(process.returncode < 0)
self.assertTrue(stdout.closed)
self.assertTrue(stderr.closed)
finally:
cleanup_output(stdout_name, stderr_name)
def test_collect_progress_updates_faulty_regex(self):
file_name = tu.ensure_directory(
'build/collect_progress_updates_skip_faulty.' +
tu.get_random_task_id())
progress_regex = '\^\^\^\^JOB-PROGRESS: (\S+)(?: )?(.*)'
stop = Event()
completed = Event()
termination = Event()
file = open(file_name, 'w+')
file.flush()
counter = cp.ProgressSequenceCounter()
watcher = cp.ProgressWatcher(file_name, 'test', counter, 1024,
progress_regex, stop, completed,
termination)
try:
def print_to_file():
file.write('^^^^JOB-PROGRESS: ABCDEF string percent\n')
file.write('^^^^JOB-PROGRESS: F50 Fifty percent\n')
file.write(
'^^^^JOB-PROGRESS: 1019101010101010101010101018101101010101010110171010110 Sixty percent\n'
)
file.write('^^^^JOB-PROGRESS: 75 75% percent\n')
file.flush()
file.close()
completed.set()
print_thread = Thread(target=print_to_file, args=())
print_thread.start()
progress_states = [{
'progress-message': b'75% percent',
'progress-percent': 75,
'progress-sequence': 1
}]
for actual_progress_state in watcher.retrieve_progress_states():
expected_progress_state = progress_states.pop(0)
self.assertEqual(expected_progress_state,
actual_progress_state)
self.assertEqual(expected_progress_state,
watcher.current_progress())
self.assertFalse(progress_states)
print_thread.join()
finally:
completed.set()
tu.cleanup_file(file_name)
def test_manage_task_involved_command_successful_exit(self):
def assertions(driver, task_id, sandbox_directory):
expected_statuses = [{
'task_id': {
'value': task_id
},
'state': cook.TASK_STARTING
}, {
'task_id': {
'value': task_id
},
'state': cook.TASK_RUNNING
}, {
'task_id': {
'value': task_id
},
'state': cook.TASK_FINISHED
}]
tu.assert_statuses(self, expected_statuses, driver.statuses)
expected_core_messages = [{
'sandbox-directory': sandbox_directory,
'task-id': task_id,
'type': 'directory'
}, {
'exit-code': 0,
'task-id': task_id
}]
expected_progress_messages = [{
'progress-message': 'line count is 20',
'progress-percent': 90,
'progress-sequence': 1,
'task-id': task_id
}]
tu.assert_messages(self, expected_core_messages,
expected_progress_messages, driver.messages)
test_file_name = tu.ensure_directory('build/file.' +
tu.get_random_task_id())
command = (
'mkdir -p build; touch {0}; for i in $(seq 20); do echo $i >> {0}; done; '
'LINE_COUNT=`wc -l < {0} | tr -d \'[:space:]\'`; cat {0}; rm -rfv {0}; '
'echo "^^^^JOB-PROGRESS: 90 line count is $LINE_COUNT"'.format(
test_file_name))
self.manage_task_runner(command, assertions)
def test_collect_progress_updates_with_empty_regex(self):
file_name = tu.ensure_directory('build/collect_progress_test.' +
tu.get_random_task_id())
progress_regex = ''
stop = Event()
completed = Event()
termination = Event()
file = open(file_name, 'w+')
file.flush()
counter = cp.ProgressSequenceCounter()
watcher = cp.ProgressWatcher(file_name, 'test', counter, 1024,
progress_regex, stop, completed,
termination)
try:
def print_to_file():
file.write('Stage One complete\n')
file.write('^^^^JOB-PROGRESS: 25 Twenty-Five percent\n')
file.write('Stage Two complete\n')
file.write('^^^^JOB-PROGRESS: 50 Fifty percent\n')
file.write('Stage Three complete\n')
file.write('^^^^JOB-PROGRESS: 55.0 Fifty-five percent\n')
file.write('Stage Four complete\n')
file.write('^^^^JOB-PROGRESS: 100 100-percent\n')
file.flush()
file.close()
completed.set()
print_thread = Thread(target=print_to_file, args=())
print_thread.start()
progress_states = []
for actual_progress_state in watcher.retrieve_progress_states():
expected_progress_state = progress_states.pop(0)
self.assertEqual(expected_progress_state,
actual_progress_state)
self.assertEqual(expected_progress_state,
watcher.current_progress())
self.assertFalse(progress_states)
self.assertIsNone(watcher.current_progress())
finally:
completed.set()
tu.cleanup_file(file_name)
def test_progress_watcher_tail_with_read_limit(self):
file_name = ensure_directory('build/tail_progress_test.' +
get_random_task_id())
config = cc.ExecutorConfig(max_bytes_read_per_line=10,
progress_output_name=file_name)
stop_signal = Event()
completed_signal = Event()
tail_sleep_ms = 25
try:
def write_to_file():
file = open(file_name, 'w+')
file.write("abcd\n")
file.flush()
file.write("abcdefghijkl\n")
file.flush()
file.write("abcdefghijklmnopqrstuvwxyz\n")
file.flush()
file.close()
time.sleep(0.15)
completed_signal.set()
Thread(target=write_to_file, args=()).start()
progress_watcher = cp.ProgressWatcher(config, stop_signal,
completed_signal)
collected_data = []
for line in progress_watcher.tail(tail_sleep_ms):
collected_data.append(line.strip())
logging.debug('collected_data = {}'.format(collected_data))
expected_data = [
'abcd', 'abcdefghij', 'kl', 'abcdefghij', 'klmnopqrst',
'uvwxyz'
]
self.assertEqual(expected_data, collected_data)
finally:
if os.path.isfile(file_name):
os.remove(file_name)
def test_watcher_tail_with_read_limit(self):
file_name = tu.ensure_directory('build/tail_progress_test.' +
tu.get_random_task_id())
stop = Event()
completed = Event()
termination = Event()
tail_sleep_ms = 25
try:
def write_to_file():
file = open(file_name, 'w+')
file.write('abcd\n')
file.flush()
file.write('abcdefghijkl\n')
file.flush()
file.write('abcdefghijklmnopqrstuvwxyz\n')
file.flush()
file.close()
time.sleep(0.15)
completed.set()
Thread(target=write_to_file, args=()).start()
counter = cp.ProgressSequenceCounter()
watcher = cp.ProgressWatcher(file_name, 'test', counter, 10, '',
stop, completed, termination)
collected_data = []
for line in watcher.tail(tail_sleep_ms):
collected_data.append(line.strip())
logging.debug('collected_data = {}'.format(collected_data))
expected_data = [
b'abcd', b'abcdefghij', b'kl', b'abcdefghij', b'klmnopqrst',
b'uvwxyz'
]
self.assertEqual(expected_data, collected_data)
finally:
tu.cleanup_file(file_name)
def test_watcher_tail_lot_of_writes(self):
file_name = tu.ensure_directory('build/tail_progress_test.' +
tu.get_random_task_id())
items_to_write = 250000
stop = Event()
completed = Event()
termination = Event()
tail_sleep_ms = 25
try:
def write_to_file():
file = open(file_name, 'w+')
for item in range(items_to_write):
file.write('line-{}\n'.format(item))
if item % 100 == 0:
file.flush()
file.flush()
file.close()
time.sleep(0.15)
completed.set()
Thread(target=write_to_file, args=()).start()
counter = cp.ProgressSequenceCounter()
watcher = cp.ProgressWatcher(file_name, 'test', counter, 1024, '',
stop, completed, termination)
collected_data = []
for line in watcher.tail(tail_sleep_ms):
collected_data.append(line.strip())
logging.info('Items read: {}'.format(len(collected_data)))
if items_to_write != len(collected_data):
for index in range(len(collected_data)):
logging.info('{}: {}'.format(index, collected_data[index]))
self.assertEqual(items_to_write, len(collected_data))
expected_data = list(
map(lambda x: str.encode('line-{}'.format(x)),
range(items_to_write)))
self.assertEqual(expected_data, collected_data)
finally:
tu.cleanup_file(file_name)
def assertions(driver, task_id, sandbox_directory):
expected_statuses = [{
'task_id': {
'value': task_id
},
'state': cook.TASK_STARTING
}, {
'task_id': {
'value': task_id
},
'state': cook.TASK_RUNNING
}, {
'task_id': {
'value': task_id
},
'state': cook.TASK_FINISHED
}]
tu.assert_statuses(self, expected_statuses, driver.statuses)
expected_core_messages = [{
'sandbox-directory': sandbox_directory,
'task-id': task_id,
'type': 'directory'
}, {
'exit-code': 0,
'task-id': task_id
}]
expected_progress_messages = [{
'progress-message': '',
'progress-percent': 50,
'progress-sequence': 1,
'task-id': task_id
}]
tu.assert_messages(self, expected_core_messages,
expected_progress_messages, driver.messages)
stdout_name = tu.ensure_directory('build/stdout.' + str(task_id))
if not os.path.isfile(stdout_name):
self.fail('{} does not exist.'.format(stdout_name))
def test_progress_watcher_tail_lot_of_writes(self):
file_name = ensure_directory('build/tail_progress_test.' +
get_random_task_id())
config = cc.ExecutorConfig(progress_output_name=file_name)
items_to_write = 250000
stop_signal = Event()
completed_signal = Event()
tail_sleep_ms = 25
try:
def write_to_file():
file = open(file_name, 'w+')
for item in range(items_to_write):
file.write("line-{}\n".format(item))
file.flush()
file.close()
time.sleep(0.15)
completed_signal.set()
Thread(target=write_to_file, args=()).start()
progress_watcher = cp.ProgressWatcher(config, stop_signal,
completed_signal)
collected_data = []
for line in progress_watcher.tail(tail_sleep_ms):
collected_data.append(line.strip())
logging.info('Items read: {}'.format(len(collected_data)))
if items_to_write != len(collected_data):
for index in range(len(collected_data)):
logging.info('{}: {}'.format(index, collected_data[index]))
self.assertEqual(items_to_write, len(collected_data))
expected_data = list(
map(lambda x: 'line-' + str(x), range(items_to_write)))
self.assertEqual(expected_data, collected_data)
finally:
if os.path.isfile(file_name):
os.remove(file_name)
def test_progress_watcher_tail(self):
file_name = ensure_directory('build/tail_progress_test.' +
get_random_task_id())
config = cc.ExecutorConfig(progress_output_name=file_name)
items_to_write = 12
stop_signal = Event()
completed_signal = Event()
write_sleep_ms = 50
tail_sleep_ms = 25
try:
def write_to_file():
file = open(file_name, 'w+')
for item in range(items_to_write):
time.sleep(write_sleep_ms / 1000.0)
file.write("{}\n".format(item))
file.flush()
file.close()
time.sleep(0.15)
completed_signal.set()
Thread(target=write_to_file, args=()).start()
progress_watcher = cp.ProgressWatcher(config, stop_signal,
completed_signal)
collected_data = []
for line in progress_watcher.tail(tail_sleep_ms):
collected_data.append(line.strip())
self.assertEqual(items_to_write, len(collected_data))
self.assertEqual(
list(map(lambda x: str(x), range(items_to_write))),
collected_data)
finally:
if os.path.isfile(file_name):
os.remove(file_name)
def test_watcher_tail(self):
file_name = tu.ensure_directory('build/tail_progress_test.' +
tu.get_random_task_id())
items_to_write = 12
stop = Event()
completed = Event()
termination = Event()
write_sleep_ms = 50
tail_sleep_ms = 25
try:
def write_to_file():
file = open(file_name, 'w+')
for item in range(items_to_write):
time.sleep(write_sleep_ms / 1000.0)
file.write('{}\n'.format(item))
file.flush()
file.close()
time.sleep(0.15)
completed.set()
Thread(target=write_to_file, args=()).start()
counter = cp.ProgressSequenceCounter()
watcher = cp.ProgressWatcher(file_name, 'test', counter, 1024, '',
stop, completed, termination)
collected_data = []
for line in watcher.tail(tail_sleep_ms):
collected_data.append(line.strip())
self.assertEqual(items_to_write, len(collected_data))
self.assertEqual(
list(map(lambda x: str.encode(str(x)), range(items_to_write))),
collected_data)
finally:
tu.cleanup_file(file_name)
def assertions(driver, task_id, _):
expected_statuses = [{
'task_id': {
'value': task_id
},
'state': cook.TASK_STARTING
}, {
'task_id': {
'value': task_id
},
'state': cook.TASK_RUNNING
}, {
'task_id': {
'value': task_id
},
'state': cook.TASK_FINISHED
}]
tu.assert_statuses(self, expected_statuses, driver.statuses)
expected_message_0 = {
'sandbox-directory':
'/location/to/task/sandbox/{}'.format(task_id),
'task-id': task_id,
'type': 'directory'
}
expected_message_1 = {'exit-code': 0, 'task-id': task_id}
tu.assert_messages(self, [expected_message_0, expected_message_1],
[], driver.messages)
stdout_name = tu.ensure_directory(
'build/stdout.{}'.format(task_id))
with open(stdout_name) as f:
file_contents = f.read()
self.assertTrue('FEE=FIE' in file_contents)
self.assertTrue('FOO=BAR' in file_contents)
self.assertTrue('PROGRESS_OUTPUT_FILE=foobar' in file_contents)
def test_executor_launch_task_and_disconnect(self):
task_id = tu.get_random_task_id()
stdout_name = tu.ensure_directory('build/stdout.{}'.format(task_id))
stderr_name = tu.ensure_directory('build/stderr.{}'.format(task_id))
output_name = tu.ensure_directory('build/output.' + str(task_id))
tu.redirect_stdout_to_file(stdout_name)
tu.redirect_stderr_to_file(stderr_name)
try:
config = cc.ExecutorConfig()
stop_signal = Event()
executor = ce.CookExecutor(stop_signal, config)
driver = tu.FakeMesosExecutorDriver()
command = 'echo "Start" >> {}; sleep 100; echo "Done." >> {}; '.format(
output_name, output_name)
task = {
'task_id': {
'value': task_id
},
'data':
pm.encode_data(
json.dumps({
'command': command
}).encode('utf8'))
}
executor.launchTask(driver, task)
# let the process run for up to 10 seconds
for _ in range(1000):
time.sleep(0.01)
if os.path.isfile(output_name):
with open(output_name) as f:
content = f.read()
if 'Start' in content:
break
executor.disconnected(driver)
self.assertTrue(executor.disconnect_signal.isSet())
self.assertTrue(executor.stop_signal.isSet())
executor.await_completion()
logging.info('Task completed')
if os.path.isfile(output_name):
with open(output_name) as f:
file_contents = f.read()
self.assertTrue('Start' in file_contents)
self.assertTrue('Done' not in file_contents)
else:
self.fail('{} does not exist.'.format(stderr_name))
expected_statuses = [{
'task_id': {
'value': task_id
},
'state': cook.TASK_STARTING
}, {
'task_id': {
'value': task_id
},
'state': cook.TASK_RUNNING
}, {
'task_id': {
'value': task_id
},
'state': cook.TASK_KILLED
}]
tu.assert_statuses(self, expected_statuses, driver.statuses)
expected_message_0 = {
'sandbox-directory': '',
'task-id': task_id,
'type': 'directory'
}
expected_message_1 = {'exit-code': -15, 'task-id': task_id}
tu.assert_messages(self, [expected_message_0, expected_message_1],
[], driver.messages)
finally:
tu.cleanup_output(stdout_name, stderr_name)
tu.cleanup_file(output_name)
def test_manage_task_progress_in_progress_stderr_and_stdout_progress(self):
max_message_length = 35
def assertions(driver, task_id, sandbox_directory):
expected_statuses = [{
'task_id': {
'value': task_id
},
'state': cook.TASK_STARTING
}, {
'task_id': {
'value': task_id
},
'state': cook.TASK_RUNNING
}, {
'task_id': {
'value': task_id
},
'state': cook.TASK_FINISHED
}]
tu.assert_statuses(self, expected_statuses, driver.statuses)
expected_core_messages = [{
'sandbox-directory': sandbox_directory,
'task-id': task_id,
'type': 'directory'
}, {
'exit-code': 0,
'task-id': task_id
}]
expected_progress_messages = [{
'progress-message': 'Fifty percent in progress file',
'progress-percent': 50,
'progress-sequence': 1,
'task-id': task_id
}, {
'progress-message': 'Fifty-five percent in stdout',
'progress-percent': 55,
'progress-sequence': 2,
'task-id': task_id
}, {
'progress-message': 'Sixty percent in stderr',
'progress-percent': 60,
'progress-sequence': 3,
'task-id': task_id
}, {
'progress-message': 'Sixty-five percent in stdout wit...',
'progress-percent': 65,
'progress-sequence': 4,
'task-id': task_id
}]
tu.assert_messages(self, expected_core_messages,
expected_progress_messages, driver.messages)
stop_signal = Event()
sleep_and_set_stop_signal_task(stop_signal, 60)
task_id = tu.get_random_task_id()
progress_name = tu.ensure_directory(
'build/progress.{}'.format(task_id))
stderr_name = tu.ensure_directory('build/stderr.{}'.format(task_id))
stdout_name = tu.ensure_directory('build/stdout.{}'.format(task_id))
config = tu.FakeExecutorConfig({
'max_bytes_read_per_line':
1024,
'max_message_length':
max_message_length,
'progress_output_env_variable':
'DEFAULT_PROGRESS_FILE_ENV_VARIABLE',
'progress_output_name':
progress_name,
'progress_regex_string':
'\^\^\^\^JOB-PROGRESS:\s+([0-9]*\.?[0-9]+)($|\s+.*)',
'progress_sample_interval_ms':
10,
'sandbox_directory':
'/sandbox/directory/for/{}'.format(task_id),
'shutdown_grace_period_ms':
60000,
'stderr_file':
stderr_name,
'stdout_file':
stdout_name
})
command = 'echo "Hello World"; ' \
'echo "^^^^JOB-PROGRESS: 50 Fifty percent in progress file" >> {}; ' \
'sleep 0.25; ' \
'echo "^^^^JOB-PROGRESS: 55 Fifty-five percent in stdout" >> {}; ' \
'sleep 0.25; ' \
'echo "^^^^JOB-PROGRESS: 60 Sixty percent in stderr" >> {}; ' \
'sleep 0.25; ' \
'echo "^^^^JOB-PROGRESS: 65 Sixty-five percent in stdout with a long message" >> {}; ' \
'sleep 0.25; ' \
'echo "Exiting..."; ' \
'exit 0'.format(progress_name, stdout_name, stderr_name, stdout_name)
try:
self.manage_task_runner(command,
assertions,
stop_signal=stop_signal,
task_id=task_id,
config=config)
stop_signal.set()
finally:
tu.cleanup_file(progress_name)
