def launch_task(self, task_id, executable, *args, **kwargs):
"""Handle the actual submission of the task to the executor layer.
If the app task has the executors attributes not set (default=='all')
the task is launched on a randomly selected executor from the
list of executors. This behavior could later be updated to support
binding to executors based on user specified criteria.
If the app task specifies a particular set of executors, it will be
targeted at those specific executors.
Args:
task_id (uuid string) : A uuid string that uniquely identifies the task
executable (callable) : A callable object
args (list of positional args)
kwargs (arbitrary keyword arguments)
Returns:
Future that tracks the execution of the submitted executable
"""
hit, memo_fu = self.memoizer.check_memo(task_id, self.tasks[task_id])
if hit:
self.handle_update(task_id, memo_fu, memo_cbk=True)
return memo_fu
executor_label = self.tasks[task_id]["executor"]
try:
executor = self.executors[executor_label]
except Exception as e:
logger.exception(
"Task {} requested invalid executor {}: config is\n{}".format(
task_id, executor_label, self._config))
if self.db_logger_config is not None and self.db_logger_config.get(
'enable_remote_monitoring', False):
executable = app_monitor.monitor_wrapper(executable, task_id,
self.db_logger_config,
self.run_id)
exec_fu = executor.submit(executable, *args, **kwargs)
self.tasks[task_id]['status'] = States.running
self.tasks[task_id]['time_started'] = str(
datetime.now().strftime('%Y-%m-%d %H:%M:%S'))
if self.db_logger_config is not None and self.db_logger_config.get(
'enable_es_logging', False):
task_log_info = {
'task_' + k: v
for k, v in self.tasks[task_id].items()
}
task_log_info['task_status_name'] = self.tasks[task_id][
'status'].name
self.db_logger.info("Task Launch", extra=task_log_info)
exec_fu.retries_left = self._config.retries - \
self.tasks[task_id]['fail_count']
exec_fu.add_done_callback(partial(self.handle_update, task_id))
logger.info("Task {} launched on executor {}".format(
task_id, executor.label))
return exec_fu
def launch_task(self, task_id, executable, *args, **kwargs):
"""Handle the actual submission of the task to the executor layer.
If the app task has the executors attributes not set (default=='all')
the task is launched on a randomly selected executor from the
list of executors. This behavior could later be updated to support
binding to executors based on user specified criteria.
If the app task specifies a particular set of executors, it will be
targeted at those specific executors.
Args:
task_id (uuid string) : A uuid string that uniquely identifies the task
executable (callable) : A callable object
args (list of positional args)
kwargs (arbitrary keyword arguments)
Returns:
Future that tracks the execution of the submitted executable
"""
self.tasks[task_id]['time_submitted'] = time.time()
hit, memo_fu = self.memoizer.check_memo(task_id, self.tasks[task_id])
if hit:
logger.info("Reusing cached result for task {}".format(task_id))
try:
self.handle_exec_update(task_id, memo_fu)
except Exception as e:
logger.error(
"handle_exec_update raised an exception {} which will be ignored"
.format(e))
try:
self.handle_app_update(task_id, memo_fu, memo_cbk=True)
except Exception as e:
logger.error(
"handle_app_update raised an exception {} which will be ignored"
.format(e))
return memo_fu
executor_label = self.tasks[task_id]["executor"]
try:
executor = self.executors[executor_label]
except Exception:
logger.exception(
"Task {} requested invalid executor {}: config is\n{}".format(
task_id, executor_label, self._config))
if self.monitoring_config is not None:
executable = app_monitor.monitor_wrapper(executable, task_id,
self.monitoring_config,
self.run_id)
with self.submitter_lock:
exec_fu = executor.submit(executable, *args, **kwargs)
self.tasks[task_id]['status'] = States.running
if self.monitoring_config is not None:
task_log_info = self._create_task_log_info(task_id)
self.db_logger.info("Task Launch", extra=task_log_info)
exec_fu.retries_left = self._config.retries - \
self.tasks[task_id]['fail_count']
logger.info("Task {} launched on executor {}".format(
task_id, executor.label))
try:
exec_fu.add_done_callback(partial(self.handle_exec_update,
task_id))
except Exception as e:
logger.error(
"add_done_callback got an exception {} which will be ignored".
format(e))
return exec_fu