def launch_task(self, task_id, executable, *args, **kwargs): """Handle the actual submission of the task to the executor layer. If the app task has the executors attributes not set (default=='all') the task is launched on a randomly selected executor from the list of executors. This behavior could later be updated to support binding to executors based on user specified criteria. If the app task specifies a particular set of executors, it will be targeted at those specific executors. Args: task_id (uuid string) : A uuid string that uniquely identifies the task executable (callable) : A callable object args (list of positional args) kwargs (arbitrary keyword arguments) Returns: Future that tracks the execution of the submitted executable """ hit, memo_fu = self.memoizer.check_memo(task_id, self.tasks[task_id]) if hit: self.handle_update(task_id, memo_fu, memo_cbk=True) return memo_fu executor_label = self.tasks[task_id]["executor"] try: executor = self.executors[executor_label] except Exception as e: logger.exception( "Task {} requested invalid executor {}: config is\n{}".format( task_id, executor_label, self._config)) if self.db_logger_config is not None and self.db_logger_config.get( 'enable_remote_monitoring', False): executable = app_monitor.monitor_wrapper(executable, task_id, self.db_logger_config, self.run_id) exec_fu = executor.submit(executable, *args, **kwargs) self.tasks[task_id]['status'] = States.running self.tasks[task_id]['time_started'] = str( datetime.now().strftime('%Y-%m-%d %H:%M:%S')) if self.db_logger_config is not None and self.db_logger_config.get( 'enable_es_logging', False): task_log_info = { 'task_' + k: v for k, v in self.tasks[task_id].items() } task_log_info['task_status_name'] = self.tasks[task_id][ 'status'].name self.db_logger.info("Task Launch", extra=task_log_info) exec_fu.retries_left = self._config.retries - \ self.tasks[task_id]['fail_count'] exec_fu.add_done_callback(partial(self.handle_update, task_id)) logger.info("Task {} launched on executor {}".format( task_id, executor.label)) return exec_fu
def launch_task(self, task_id, executable, *args, **kwargs): """Handle the actual submission of the task to the executor layer. If the app task has the executors attributes not set (default=='all') the task is launched on a randomly selected executor from the list of executors. This behavior could later be updated to support binding to executors based on user specified criteria. If the app task specifies a particular set of executors, it will be targeted at those specific executors. Args: task_id (uuid string) : A uuid string that uniquely identifies the task executable (callable) : A callable object args (list of positional args) kwargs (arbitrary keyword arguments) Returns: Future that tracks the execution of the submitted executable """ self.tasks[task_id]['time_submitted'] = time.time() hit, memo_fu = self.memoizer.check_memo(task_id, self.tasks[task_id]) if hit: logger.info("Reusing cached result for task {}".format(task_id)) try: self.handle_exec_update(task_id, memo_fu) except Exception as e: logger.error( "handle_exec_update raised an exception {} which will be ignored" .format(e)) try: self.handle_app_update(task_id, memo_fu, memo_cbk=True) except Exception as e: logger.error( "handle_app_update raised an exception {} which will be ignored" .format(e)) return memo_fu executor_label = self.tasks[task_id]["executor"] try: executor = self.executors[executor_label] except Exception: logger.exception( "Task {} requested invalid executor {}: config is\n{}".format( task_id, executor_label, self._config)) if self.monitoring_config is not None: executable = app_monitor.monitor_wrapper(executable, task_id, self.monitoring_config, self.run_id) with self.submitter_lock: exec_fu = executor.submit(executable, *args, **kwargs) self.tasks[task_id]['status'] = States.running if self.monitoring_config is not None: task_log_info = self._create_task_log_info(task_id) self.db_logger.info("Task Launch", extra=task_log_info) exec_fu.retries_left = self._config.retries - \ self.tasks[task_id]['fail_count'] logger.info("Task {} launched on executor {}".format( task_id, executor.label)) try: exec_fu.add_done_callback(partial(self.handle_exec_update, task_id)) except Exception as e: logger.error( "add_done_callback got an exception {} which will be ignored". format(e)) return exec_fu