def send_message(driver, error_handler, message): """Sends the message, if it is smaller than the max length, using the driver. Note: This function must rethrow any OSError exceptions that it encounters. Parameters ---------- driver: MesosExecutorDriver The driver to send the message to. error_handler: fn(os_error) OSError exception handler for out of memory situations. message: dictionary The raw message to send. Returns ------- whether the message was successfully sent """ try: logging.info('Sending framework message {}'.format(message)) message_string = json.dumps(message).encode('utf8') encoded_message = pm.encode_data(message_string) driver.sendFrameworkMessage(encoded_message) return True except Exception as exception: if cu.is_out_of_memory_error(exception): error_handler(exception) else: logging.exception( 'Exception while sending message {}'.format(message)) return False
def retrieve_progress_states(self): """Generates the progress states by tailing the target_file. It tails a target file (using the tail() method) and uses the provided regex to find a match for a progress message. The regex is expected to generate two components in the match: the progress percent as an int and a progress message string. When such a message is found, this method yields the current progress as a dictionary. Note: This function must rethrow any OSError exceptions that it encounters. Returns ------- An incrementally generated list of progress states. """ last_unprocessed_report = None if self.progress_regex_string: sleep_time_ms = 50 for line in self.tail(sleep_time_ms): try: progress_report = self.match_progress_update(line) if progress_report is not None: if self.task_completed_signal.isSet(): last_unprocessed_report = progress_report elif self.__update_progress(progress_report): yield self.progress except Exception as exception: if cu.is_out_of_memory_error(exception): raise exception else: logging.exception('Skipping "%s" as a progress entry', line) if last_unprocessed_report is not None: if self.__update_progress(last_unprocessed_report): yield self.progress
def track_progress(self): """Retrieves and sends progress updates using send_progress_update_fn. It sets the progress_complete_event before returning.""" try: for current_progress in self.watcher.retrieve_progress_states(): self.updater.send_progress_update(current_progress) except Exception as exception: if cu.is_out_of_memory_error(exception): self.os_error_handler(exception) else: logging.exception('Exception while tracking progress [tag=%s]', self.location_tag) finally: self.progress_complete_event.set()
def os_error_handler(stop_signal, status_updater, os_error): """Exception handler for OSError. Parameters ---------- stop_signal: threading.Event Event that determines if the process was requested to terminate. status_updater: StatusUpdater Wrapper object that sends task status messages. os_error: OSError The current executor config. Returns ------- Nothing """ stop_signal.set() logging.exception('OSError generated, requesting process to terminate') reason = cook.REASON_CONTAINER_LIMITATION_MEMORY if cu.is_out_of_memory_error( os_error) else None status_updater.update_status(cook.TASK_FAILED, reason=reason) cu.print_memory_usage()
def manage_task(driver, task, stop_signal, completed_signal, config): """Manages the execution of a task waiting for it to terminate normally or be killed. It also sends the task status updates, sandbox location and exit code back to the scheduler. Progress updates are tracked on a separate thread and are also sent to the scheduler. Setting the stop_signal will trigger termination of the task and associated cleanup. Returns ------- Nothing """ launched_process = None task_id = get_task_id(task) cio.print_and_log('Starting task {}'.format(task_id)) status_updater = StatusUpdater(driver, task_id) inner_os_error_handler = functools.partial(os_error_handler, stop_signal, status_updater) try: # not yet started to run the task status_updater.update_status(cook.TASK_STARTING) # Use MESOS_DIRECTORY instead of MESOS_SANDBOX, to report the sandbox location outside of the container sandbox_message = { 'sandbox-directory': config.mesos_directory, 'task-id': task_id, 'type': 'directory' } send_message(driver, inner_os_error_handler, sandbox_message) environment = retrieve_process_environment(config, task, os.environ) launched_process = launch_task(task, environment) if launched_process: # task has begun running successfully status_updater.update_status(cook.TASK_RUNNING) cio.print_and_log('Forked command at {}'.format( launched_process.pid)) else: # task launch failed, report an error logging.error('Error in launching task') status_updater.update_status(cook.TASK_ERROR, reason=cook.REASON_TASK_INVALID) return task_completed_signal = Event( ) # event to track task execution completion sequence_counter = cp.ProgressSequenceCounter() send_progress_message = functools.partial(send_message, driver, inner_os_error_handler) max_message_length = config.max_message_length sample_interval_ms = config.progress_sample_interval_ms progress_updater = cp.ProgressUpdater(task_id, max_message_length, sample_interval_ms, send_progress_message) progress_termination_signal = Event() def launch_progress_tracker(progress_location, location_tag): logging.info('Location {} tagged as [tag={}]'.format( progress_location, location_tag)) progress_tracker = cp.ProgressTracker( config, stop_signal, task_completed_signal, sequence_counter, progress_updater, progress_termination_signal, progress_location, location_tag, inner_os_error_handler) progress_tracker.start() return progress_tracker progress_locations = { config.progress_output_name: 'progress', config.stderr_file(): 'stderr', config.stdout_file(): 'stdout' } logging.info('Progress will be tracked from {} locations'.format( len(progress_locations))) progress_trackers = [ launch_progress_tracker(l, progress_locations[l]) for l in progress_locations ] await_process_completion(launched_process, stop_signal, config.shutdown_grace_period_ms) task_completed_signal.set() progress_termination_timer = Timer( config.shutdown_grace_period_ms / 1000.0, progress_termination_signal.set) progress_termination_timer.daemon = True progress_termination_timer.start() # propagate the exit code exit_code = launched_process.returncode cio.print_and_log('Command exited with status {} (pid: {})'.format( exit_code, launched_process.pid)) exit_message = {'exit-code': exit_code, 'task-id': task_id} send_message(driver, inner_os_error_handler, exit_message) # await progress updater termination if executor is terminating normally if not stop_signal.isSet(): logging.info('Awaiting completion of progress updaters') [progress_tracker.wait() for progress_tracker in progress_trackers] logging.info('Progress updaters completed') # force send the latest progress state if available [ progress_tracker.force_send_progress_update() for progress_tracker in progress_trackers ] # task either completed successfully or aborted with an error task_state = get_task_state(exit_code) output_task_completion(task_id, task_state) status_updater.update_status(task_state) except Exception as exception: if cu.is_out_of_memory_error(exception): inner_os_error_handler(exception) else: # task aborted with an error logging.exception('Error in executing task') output_task_completion(task_id, cook.TASK_FAILED) status_updater.update_status( cook.TASK_FAILED, reason=cook.REASON_EXECUTOR_TERMINATED) finally: # ensure completed_signal is set so driver can stop completed_signal.set() if launched_process and cs.is_process_running(launched_process): cs.send_signal(launched_process.pid, signal.SIGKILL)