def submit(self, func, resource_specification, *args, **kwargs):
"""Submits work to the the outgoing_q.
The outgoing_q is an external process listens on this
queue for new work. This method behaves like a
submit call as described here `Python docs: <https://docs.python.org/3/library/concurrent.futures.html#concurrent.futures.ThreadPoolExecutor>`_
Args:
- func (callable) : Callable function
- *args (list) : List of arbitrary positional arguments.
Kwargs:
- **kwargs (dict) : A dictionary of arbitrary keyword args for func.
Returns:
Future
"""
if resource_specification:
logger.error(
"Ignoring the resource specification. "
"Parsl resource specification is not supported in HighThroughput Executor. "
"Please check WorkQueueExecutor if resource specification is needed."
)
raise UnsupportedFeatureError('resource specification',
'HighThroughput Executor',
'WorkQueue Executor')
if self.bad_state_is_set:
raise self.executor_exception
self._task_counter += 1
task_id = self._task_counter
# handle people sending blobs gracefully
args_to_print = args
if logger.getEffectiveLevel() >= logging.DEBUG:
args_to_print = tuple([
arg if len(repr(arg)) < 100 else (repr(arg)[:100] + '...')
for arg in args
])
logger.debug("Pushing function {} to queue with args {}".format(
func, args_to_print))
self.tasks[task_id] = Future()
try:
fn_buf = pack_apply_message(func,
args,
kwargs,
buffer_threshold=1024 * 1024)
except TypeError:
raise SerializationError(func.__name__)
msg = {"task_id": task_id, "buffer": fn_buf}
# Post task to the the outgoing queue
self.outgoing_q.put(msg)
# Return the future
return self.tasks[task_id]
def submit(
self,
func: Callable,
resource_specification: Dict[str, Any],
*args: Any,
**kwargs: Any,
):
"""Wrap a callable in a Flux job and submit it to Flux.
:param func: The callable to submit as a job to Flux
:param resource_specification: A mapping defining the resources to allocate to the Flux job.
Only the following keys are checked for:
- num_tasks: the number of tasks to launch (MPI ranks for an MPI job), default 1
- cores_per_task: cores per task, default 1
- gpus_per_task: gpus per task, default 1
- num_nodes: if > 0, evenly distribute the allocated cores/gpus
across the given number of nodes. Does *not* give the job exclusive
access to those nodes; this option only affects distribution.
:param args: positional arguments for the callable
:param kwargs: keyword arguments for the callable
"""
# protect self._task_id_counter and shutdown/submit race
with self._submission_lock:
if self._stop_event.is_set():
raise RuntimeError("`shutdown()` already called")
task_id = str(next(self._task_id_counter))
infile = os.path.join(self.working_dir,
f"{task_id}_in{os.extsep}pkl")
outfile = os.path.join(self.working_dir,
f"{task_id}_out{os.extsep}pkl")
try:
fn_buf = pack_apply_message(func,
args,
kwargs,
buffer_threshold=1024 * 1024)
except TypeError:
raise SerializationError(func.__name__)
with open(infile, "wb") as infile_handle:
infile_handle.write(fn_buf)
future = FluxFutureWrapper()
self._submission_queue.put(
_FluxJobInfo(future, task_id, infile, outfile,
resource_specification))
return future