def test_send_progress_does_not_trim_unknown_field(self):
driver = tu.FakeMesosExecutorDriver()
task_id = tu.get_random_task_id()
max_message_length = 30
poll_interval_ms = 10
send_progress_message = self.send_progress_message_helper(
driver, max_message_length)
progress_updater = cp.ProgressUpdater(task_id, max_message_length,
poll_interval_ms,
send_progress_message)
progress_data_0 = {
'progress-message':
b' pm',
'progress-sequence':
1,
'unknown':
'Unknown field has a really long lorem ipsum dolor sit amet exceed limit text'
}
progress_updater.send_progress_update(progress_data_0)
self.assertEqual(1, len(driver.messages))
actual_encoded_message_0 = driver.messages[0]
expected_message_0 = {
'progress-message':
'pm',
'progress-sequence':
1,
'task-id':
task_id,
'unknown':
'Unknown field has a really long lorem ipsum dolor sit amet exceed limit text'
}
tu.assert_message(self, expected_message_0, actual_encoded_message_0)
def test_send_progress_update_trims_progress_message(self):
driver = tu.FakeMesosExecutorDriver()
task_id = tu.get_random_task_id()
max_message_length = 30
poll_interval_ms = 10
send_progress_message = self.send_progress_message_helper(
driver, max_message_length)
progress_updater = cp.ProgressUpdater(task_id, max_message_length,
poll_interval_ms,
send_progress_message)
progress_data_0 = {
'progress-message':
b' Progress message-0 is really long lorem ipsum dolor sit amet text',
'progress-sequence': 1
}
progress_updater.send_progress_update(progress_data_0)
self.assertEqual(1, len(driver.messages))
actual_encoded_message_0 = driver.messages[0]
expected_message_0 = {
'progress-message': 'Progress message-0 is reall...',
'progress-sequence': 1,
'task-id': task_id
}
tu.assert_message(self, expected_message_0, actual_encoded_message_0)
def test_send_progress_update(self):
driver = tu.FakeMesosExecutorDriver()
task_id = tu.get_random_task_id()
max_message_length = 30
poll_interval_ms = 100
send_progress_message = self.send_progress_message_helper(
driver, max_message_length)
progress_updater = cp.ProgressUpdater(task_id, max_message_length,
poll_interval_ms,
send_progress_message)
progress_data_0 = {
'progress-message': b' Progress message-0',
'progress-sequence': 1
}
progress_updater.send_progress_update(progress_data_0)
self.assertEqual(1, len(driver.messages))
actual_encoded_message_0 = driver.messages[0]
expected_message_0 = {
'progress-message': 'Progress message-0',
'progress-sequence': 1,
'task-id': task_id
}
tu.assert_message(self, expected_message_0, actual_encoded_message_0)
progress_data_1 = {
'progress-message': b' Progress message-1',
'progress-sequence': 2
}
progress_updater.send_progress_update(progress_data_1)
self.assertEqual(1, len(driver.messages))
time.sleep(poll_interval_ms / 1000.0)
progress_data_2 = {
'progress-message': b' Progress message-2',
'progress-sequence': 3
}
progress_updater.send_progress_update(progress_data_2)
self.assertEqual(2, len(driver.messages))
actual_encoded_message_2 = driver.messages[1]
expected_message_2 = {
'progress-message': 'Progress message-2',
'progress-sequence': 3,
'task-id': task_id
}
tu.assert_message(self, expected_message_2, actual_encoded_message_2)
def test_send_progress_does_not_trim_unknown_field(self):
driver = FakeMesosExecutorDriver()
task_id = get_random_task_id()
max_message_length = 100
poll_interval_ms = 10
progress_updater = cp.ProgressUpdater(driver, task_id,
max_message_length,
poll_interval_ms,
ce.send_message)
progress_data_0 = {
'unknown':
'Unknown field has a really long lorem ipsum dolor sit amet exceed limit text'
}
progress_updater.send_progress_update(progress_data_0)
self.assertEqual(0, len(driver.messages))
def test_send_progress_update(self):
driver = FakeMesosExecutorDriver()
task_id = get_random_task_id()
max_message_length = 100
poll_interval_ms = 100
progress_updater = cp.ProgressUpdater(driver, task_id,
max_message_length,
poll_interval_ms,
ce.send_message)
progress_data_0 = {'progress-message': 'Progress message-0'}
progress_updater.send_progress_update(progress_data_0)
self.assertEqual(1, len(driver.messages))
actual_encoded_message_0 = driver.messages[0]
expected_message_0 = {
'progress-message': 'Progress message-0',
'task-id': task_id
}
assert_message(self, expected_message_0, actual_encoded_message_0)
progress_data_1 = {'progress-message': 'Progress message-1'}
progress_updater.send_progress_update(progress_data_1)
self.assertEqual(1, len(driver.messages))
time.sleep(poll_interval_ms / 1000.0)
progress_data_2 = {'progress-message': 'Progress message-2'}
progress_updater.send_progress_update(progress_data_2)
self.assertEqual(2, len(driver.messages))
actual_encoded_message_2 = driver.messages[1]
expected_message_2 = {
'progress-message': 'Progress message-2',
'task-id': task_id
}
assert_message(self, expected_message_2, actual_encoded_message_2)
def test_send_progress_update_trims_progress_message(self):
driver = FakeMesosExecutorDriver()
task_id = get_random_task_id()
max_message_length = 100
poll_interval_ms = 10
progress_updater = cp.ProgressUpdater(driver, task_id,
max_message_length,
poll_interval_ms,
ce.send_message)
progress_data_0 = {
'progress-message':
'Progress message-0 is really long lorem ipsum dolor sit amet text'
}
progress_updater.send_progress_update(progress_data_0)
self.assertEqual(1, len(driver.messages))
actual_encoded_message_0 = driver.messages[0]
expected_message_0 = {
'progress-message':
'Progress message-0 is really long lorem ipsum dolor...',
'task-id': task_id
}
assert_message(self, expected_message_0, actual_encoded_message_0)
def manage_task(driver, task, stop_signal, completed_signal, config,
stdout_name, stderr_name):
"""Manages the execution of a task waiting for it to terminate normally or be killed.
It also sends the task status updates, sandbox location and exit code back to the scheduler.
Progress updates are tracked on a separate thread and are also sent to the scheduler.
Setting the stop_signal will trigger termination of the task and associated cleanup.
Returns
-------
Nothing
"""
task_id = get_task_id(task)
try:
# not yet started to run the task
update_status(driver, task_id, cook.TASK_STARTING)
sandbox_message = json.dumps({
'sandbox-directory': config.sandbox_directory,
'task-id': task_id
})
send_message(driver, sandbox_message, config.max_message_length)
process_info = launch_task(task, stdout_name, stderr_name)
if process_info:
# task has begun running successfully
update_status(driver, task_id, cook.TASK_RUNNING)
else:
# task launch failed, report an error
logging.info('Error in launching task')
update_status(driver, task_id, cook.TASK_ERROR)
return
process, _, _ = process_info
task_completed_signal = Event(
) # event to track task execution completion
progress_watcher = cp.ProgressWatcher(config, stop_signal,
task_completed_signal)
progress_updater = cp.ProgressUpdater(
driver, task_id, config.max_message_length,
config.progress_sample_interval_ms, send_message)
progress_complete_event = cp.launch_progress_tracker(
progress_watcher, progress_updater)
await_process_completion(stop_signal, process_info,
config.shutdown_grace_period_ms)
task_completed_signal.set()
# propagate the exit code
exit_code = process.returncode
logging.info('Process completed with exit code {}'.format(exit_code))
exit_message = json.dumps({'exit-code': exit_code, 'task-id': task_id})
send_message(driver, exit_message, config.max_message_length)
# await progress updater termination if executor is terminating normally
if not stop_signal.isSet():
logging.info('Awaiting progress updater completion')
progress_complete_event.wait()
logging.info('Progress updater completed')
# force send the latest progress state if available
cp.force_send_progress_update(progress_watcher, progress_updater)
# task either completed successfully or aborted with an error
task_state = get_task_state(exit_code)
update_status(driver, task_id, task_state)
except Exception:
# task aborted with an error
logging.exception('Error in executing task')
update_status(driver, task_id, cook.TASK_FAILED)
finally:
# ensure completed_signal is set so driver can stop
completed_signal.set()
def manage_task(driver, task, stop_signal, completed_signal, config):
"""Manages the execution of a task waiting for it to terminate normally or be killed.
It also sends the task status updates, sandbox location and exit code back to the scheduler.
Progress updates are tracked on a separate thread and are also sent to the scheduler.
Setting the stop_signal will trigger termination of the task and associated cleanup.
Returns
-------
Nothing
"""
launched_process = None
task_id = get_task_id(task)
cio.print_and_log('Starting task {}'.format(task_id))
status_updater = StatusUpdater(driver, task_id)
inner_os_error_handler = functools.partial(os_error_handler, stop_signal,
status_updater)
try:
# not yet started to run the task
status_updater.update_status(cook.TASK_STARTING)
# Use MESOS_DIRECTORY instead of MESOS_SANDBOX, to report the sandbox location outside of the container
sandbox_message = {
'sandbox-directory': config.mesos_directory,
'task-id': task_id,
'type': 'directory'
}
send_message(driver, inner_os_error_handler, sandbox_message)
environment = retrieve_process_environment(config, task, os.environ)
launched_process = launch_task(task, environment)
if launched_process:
# task has begun running successfully
status_updater.update_status(cook.TASK_RUNNING)
cio.print_and_log('Forked command at {}'.format(
launched_process.pid))
else:
# task launch failed, report an error
logging.error('Error in launching task')
status_updater.update_status(cook.TASK_ERROR,
reason=cook.REASON_TASK_INVALID)
return
task_completed_signal = Event(
) # event to track task execution completion
sequence_counter = cp.ProgressSequenceCounter()
send_progress_message = functools.partial(send_message, driver,
inner_os_error_handler)
max_message_length = config.max_message_length
sample_interval_ms = config.progress_sample_interval_ms
progress_updater = cp.ProgressUpdater(task_id, max_message_length,
sample_interval_ms,
send_progress_message)
progress_termination_signal = Event()
def launch_progress_tracker(progress_location, location_tag):
logging.info('Location {} tagged as [tag={}]'.format(
progress_location, location_tag))
progress_tracker = cp.ProgressTracker(
config, stop_signal, task_completed_signal, sequence_counter,
progress_updater, progress_termination_signal,
progress_location, location_tag, inner_os_error_handler)
progress_tracker.start()
return progress_tracker
progress_locations = {
config.progress_output_name: 'progress',
config.stderr_file(): 'stderr',
config.stdout_file(): 'stdout'
}
logging.info('Progress will be tracked from {} locations'.format(
len(progress_locations)))
progress_trackers = [
launch_progress_tracker(l, progress_locations[l])
for l in progress_locations
]
await_process_completion(launched_process, stop_signal,
config.shutdown_grace_period_ms)
task_completed_signal.set()
progress_termination_timer = Timer(
config.shutdown_grace_period_ms / 1000.0,
progress_termination_signal.set)
progress_termination_timer.daemon = True
progress_termination_timer.start()
# propagate the exit code
exit_code = launched_process.returncode
cio.print_and_log('Command exited with status {} (pid: {})'.format(
exit_code, launched_process.pid))
exit_message = {'exit-code': exit_code, 'task-id': task_id}
send_message(driver, inner_os_error_handler, exit_message)
# await progress updater termination if executor is terminating normally
if not stop_signal.isSet():
logging.info('Awaiting completion of progress updaters')
[progress_tracker.wait() for progress_tracker in progress_trackers]
logging.info('Progress updaters completed')
# force send the latest progress state if available
[
progress_tracker.force_send_progress_update()
for progress_tracker in progress_trackers
]
# task either completed successfully or aborted with an error
task_state = get_task_state(exit_code)
output_task_completion(task_id, task_state)
status_updater.update_status(task_state)
except Exception as exception:
if cu.is_out_of_memory_error(exception):
inner_os_error_handler(exception)
else:
# task aborted with an error
logging.exception('Error in executing task')
output_task_completion(task_id, cook.TASK_FAILED)
status_updater.update_status(
cook.TASK_FAILED, reason=cook.REASON_EXECUTOR_TERMINATED)
finally:
# ensure completed_signal is set so driver can stop
completed_signal.set()
if launched_process and cs.is_process_running(launched_process):
cs.send_signal(launched_process.pid, signal.SIGKILL)
