def submit(self, func, resource_specification, *args, **kwargs): """Submits work to the the outgoing_q. The outgoing_q is an external process listens on this queue for new work. This method behaves like a submit call as described here `Python docs: <https://docs.python.org/3/library/concurrent.futures.html#concurrent.futures.ThreadPoolExecutor>`_ Args: - func (callable) : Callable function - *args (list) : List of arbitrary positional arguments. Kwargs: - **kwargs (dict) : A dictionary of arbitrary keyword args for func. Returns: Future """ if resource_specification: logger.error( "Ignoring the resource specification. " "Parsl resource specification is not supported in HighThroughput Executor. " "Please check WorkQueueExecutor if resource specification is needed." ) raise UnsupportedFeatureError('resource specification', 'HighThroughput Executor', 'WorkQueue Executor') if self.bad_state_is_set: raise self.executor_exception self._task_counter += 1 task_id = self._task_counter # handle people sending blobs gracefully args_to_print = args if logger.getEffectiveLevel() >= logging.DEBUG: args_to_print = tuple([ arg if len(repr(arg)) < 100 else (repr(arg)[:100] + '...') for arg in args ]) logger.debug("Pushing function {} to queue with args {}".format( func, args_to_print)) self.tasks[task_id] = Future() try: fn_buf = pack_apply_message(func, args, kwargs, buffer_threshold=1024 * 1024) except TypeError: raise SerializationError(func.__name__) msg = {"task_id": task_id, "buffer": fn_buf} # Post task to the the outgoing queue self.outgoing_q.put(msg) # Return the future return self.tasks[task_id]
def submit( self, func: Callable, resource_specification: Dict[str, Any], *args: Any, **kwargs: Any, ): """Wrap a callable in a Flux job and submit it to Flux. :param func: The callable to submit as a job to Flux :param resource_specification: A mapping defining the resources to allocate to the Flux job. Only the following keys are checked for: - num_tasks: the number of tasks to launch (MPI ranks for an MPI job), default 1 - cores_per_task: cores per task, default 1 - gpus_per_task: gpus per task, default 1 - num_nodes: if > 0, evenly distribute the allocated cores/gpus across the given number of nodes. Does *not* give the job exclusive access to those nodes; this option only affects distribution. :param args: positional arguments for the callable :param kwargs: keyword arguments for the callable """ # protect self._task_id_counter and shutdown/submit race with self._submission_lock: if self._stop_event.is_set(): raise RuntimeError("`shutdown()` already called") task_id = str(next(self._task_id_counter)) infile = os.path.join(self.working_dir, f"{task_id}_in{os.extsep}pkl") outfile = os.path.join(self.working_dir, f"{task_id}_out{os.extsep}pkl") try: fn_buf = pack_apply_message(func, args, kwargs, buffer_threshold=1024 * 1024) except TypeError: raise SerializationError(func.__name__) with open(infile, "wb") as infile_handle: infile_handle.write(fn_buf) future = FluxFutureWrapper() self._submission_queue.put( _FluxJobInfo(future, task_id, infile, outfile, resource_specification)) return future