def submit(self, func, *args, parsl_sites='all', fn_hash=None, cache=False, **kwargs): """Add task to the dataflow system. >>> IF all deps are met : >>> send to the runnable queue and launch the task >>> ELSE: >>> post the task in the pending queue Args: - func : A function object - *args : Args to the function KWargs : - parsl_sites (List|String) : List of sites this call could go to. Default='all' - fn_hash (Str) : Hash of the function and inputs Default=None - cache (Bool) : To enable memoization or not - kwargs (dict) : Rest of the kwargs to the fn passed as dict. Returns: (AppFuture) [DataFutures,] """ task_id = self.task_count self.task_count += 1 # Get the dep count and a list of dependencies for the task dep_cnt, depends = self._count_all_deps(task_id, args, kwargs) task_def = { 'depends': depends, 'sites': parsl_sites, 'func': func, 'func_name': func.__name__, 'args': args, 'kwargs': kwargs, 'fn_hash': fn_hash, 'memoize': cache, 'callback': None, 'dep_cnt': dep_cnt, 'exec_fu': None, 'checkpoint': None, 'fail_count': 0, 'fail_history': [], 'env': None, 'status': States.unsched, 'app_fu': None } if task_id in self.tasks: raise DuplicateTaskError( "Task {0} in pending list".format(task_id)) else: self.tasks[task_id] = task_def # Extract stdout and stderr to pass to AppFuture: task_stdout = kwargs.get('stdout', None) task_stderr = kwargs.get('stderr', None) logger.info("Task {} submitted for App {}, waiting on tasks {}".format( task_id, task_def['func_name'], [fu.tid for fu in depends])) # Handle three cases here: # No pending deps # - But has failures -> dep_fail # - No failures -> running # Has pending deps -> pending if dep_cnt == 0: new_args, kwargs, exceptions = self.sanitize_and_wrap( task_id, args, kwargs) self.tasks[task_id]['args'] = new_args self.tasks[task_id]['kwargs'] = kwargs if not exceptions: self.tasks[task_id]['exec_fu'] = self.launch_task( task_id, func, *new_args, **kwargs) self.tasks[task_id]['app_fu'] = AppFuture( self.tasks[task_id]['exec_fu'], tid=task_id, stdout=task_stdout, stderr=task_stderr) self.tasks[task_id]['status'] = States.running logger.debug("Task {} launched with AppFut:{}".format( task_id, task_def['app_fu'])) else: self.tasks[task_id]['exec_fu'] = None app_fu = AppFuture(self.tasks[task_id]['exec_fu'], tid=task_id, stdout=task_stdout, stderr=task_stderr) app_fu.set_exception( DependencyError(exceptions, "Failures in input dependencies", None)) self.tasks[task_id]['app_fu'] = app_fu self.tasks[task_id]['status'] = States.dep_fail logger.debug( "Task {} failed due to failure in parent task(s):{}". format(task_id, task_def['app_fu'])) else: # Send to pending, create the AppFuture with no parent and have it set # when an executor future is available. self.tasks[task_id]['app_fu'] = AppFuture(None, tid=task_id, stdout=task_stdout, stderr=task_stderr) self.tasks[task_id]['status'] = States.pending logger.debug("Task {} launched with AppFut:{}".format( task_id, task_def['app_fu'])) return task_def['app_fu']
def submit (self, func, *args, parsl_sites='all', **kwargs): ''' Add task to the dataflow system. Args: func : A function object *args : Args to the function KWargs : Standard kwargs to the func as provided by the user parsl_sites : List of sites as defined in the config, Default :'all' This is the only kwarg that is passed in by the app definition. If all deps are met : send to the runnable queue and launch the task Else: post the task in the pending queue Returns: (AppFuture) [DataFutures,] ''' task_id = self.task_count self.task_count += 1 # Get the dep count and a list of dependencies for the task dep_cnt, depends = self._count_all_deps(task_id, args, kwargs) task_def = { 'depends' : depends, 'sites' : parsl_sites, 'func' : func, 'func_name' : func.__name__, 'args' : args, 'kwargs' : kwargs, 'callback' : None, 'dep_cnt' : dep_cnt, 'exec_fu' : None, 'status' : States.unsched, 'app_fu' : None } if task_id in self.tasks: raise DuplicateTaskError("Task {0} in pending list".format(task_id)) else: self.tasks[task_id] = task_def # Extract stdout and stderr to pass to AppFuture: task_stdout = kwargs.get('stdout', None) task_stderr = kwargs.get('stderr', None) if dep_cnt == 0 : # Set to running new_args, kwargs, exceptions = self.sanitize_and_wrap(task_id, args, kwargs) if not exceptions: self.tasks[task_id]['exec_fu'] = self.launch_task(task_id, func, *new_args, **kwargs) self.tasks[task_id]['app_fu'] = AppFuture(self.tasks[task_id]['exec_fu'], tid=task_id, stdout=task_stdout, stderr=task_stderr) self.tasks[task_id]['status'] = States.running else: self.tasks[task_id]['exec_fu'] = None app_fu = AppFuture(self.tasks[task_id]['exec_fu'], tid=task_id, stdout=task_stdout, stderr=task_stderr) app_fu.set_exception(DependencyError(exceptions, "Failures in input dependencies", None)) self.tasks[task_id]['app_fu'] = app_fu self.tasks[task_id]['status'] = States.dep_fail else: # Send to pending, create the AppFuture with no parent and have it set # when an executor future is available. self.tasks[task_id]['app_fu'] = AppFuture(None, tid=task_id, stdout=task_stdout, stderr=task_stderr) self.tasks[task_id]['status'] = States.pending logger.debug("Task:%s Launched with AppFut:%s", task_id, task_def['app_fu']) return task_def['app_fu']