def submit(self, func, resource_specification, *args, **kwargs):
"""Submits work to the the outgoing_q.
The outgoing_q is an external process listens on this
queue for new work. This method behaves like a
submit call as described here `Python docs: <https://docs.python.org/3/library/concurrent.futures.html#concurrent.futures.ThreadPoolExecutor>`_
Args:
- func (callable) : Callable function
- *args (list) : List of arbitrary positional arguments.
Kwargs:
- **kwargs (dict) : A dictionary of arbitrary keyword args for func.
Returns:
Future
"""
if resource_specification:
logger.error(
"Ignoring the resource specification. "
"Parsl resource specification is not supported in HighThroughput Executor. "
"Please check WorkQueueExecutor if resource specification is needed."
)
raise UnsupportedFeatureError('resource specification',
'HighThroughput Executor',
'WorkQueue Executor')
if self.bad_state_is_set:
raise self.executor_exception
self._task_counter += 1
task_id = self._task_counter
# handle people sending blobs gracefully
args_to_print = args
if logger.getEffectiveLevel() >= logging.DEBUG:
args_to_print = tuple([
arg if len(repr(arg)) < 100 else (repr(arg)[:100] + '...')
for arg in args
])
logger.debug("Pushing function {} to queue with args {}".format(
func, args_to_print))
self.tasks[task_id] = Future()
try:
fn_buf = pack_apply_message(func,
args,
kwargs,
buffer_threshold=1024 * 1024)
except TypeError:
raise SerializationError(func.__name__)
msg = {"task_id": task_id, "buffer": fn_buf}
# Post task to the the outgoing queue
self.outgoing_q.put(msg)
# Return the future
return self.tasks[task_id]
def _serialize_function(self, fn_path, parsl_fn, parsl_fn_args,
parsl_fn_kwargs):
"""Takes the function application parsl_fn(*parsl_fn_args, **parsl_fn_kwargs)
and serializes it to the file fn_path."""
# Either build a dictionary with the source of the function, or pickle
# the function directly:
if self.source:
function_info = {
"source code": inspect.getsource(parsl_fn),
"name": parsl_fn.__name__,
"args": parsl_fn_args,
"kwargs": parsl_fn_kwargs
}
else:
function_info = {
"byte code":
pack_apply_message(parsl_fn,
parsl_fn_args,
parsl_fn_kwargs,
buffer_threshold=1024 * 1024)
}
with open(fn_path, "wb") as f_out:
pickle.dump(function_info, f_out)
def submit(self, func, resource_specification, *args, **kwargs):
""" TODO: docstring """
if resource_specification:
logger.error(
"Ignoring the resource specification. "
"Parsl resource specification is not supported in LowLatency Executor. "
"Please check WorkQueueExecutor if resource specification is needed."
)
raise UnsupportedFeatureError('resource specification',
'LowLatency Executor',
'WorkQueue Executor')
if self.bad_state_is_set:
raise self.executor_exception
self._task_counter += 1
task_id = self._task_counter
logger.debug("Pushing function {} to queue with args {}".format(
func, args))
self.tasks[task_id] = Future()
fn_buf = pack_apply_message(func,
args,
kwargs,
buffer_threshold=1024 * 1024)
# Post task to the the outgoing queue
self.outgoing_q.put(task_id, fn_buf)
# Return the future
return self.tasks[task_id]
def submit(
self,
func: Callable,
resource_specification: Dict[str, Any],
*args: Any,
**kwargs: Any,
):
"""Wrap a callable in a Flux job and submit it to Flux.
:param func: The callable to submit as a job to Flux
:param resource_specification: A mapping defining the resources to allocate to the Flux job.
Only the following keys are checked for:
- num_tasks: the number of tasks to launch (MPI ranks for an MPI job), default 1
- cores_per_task: cores per task, default 1
- gpus_per_task: gpus per task, default 1
- num_nodes: if > 0, evenly distribute the allocated cores/gpus
across the given number of nodes. Does *not* give the job exclusive
access to those nodes; this option only affects distribution.
:param args: positional arguments for the callable
:param kwargs: keyword arguments for the callable
"""
# protect self._task_id_counter and shutdown/submit race
with self._submission_lock:
if self._stop_event.is_set():
raise RuntimeError("`shutdown()` already called")
task_id = str(next(self._task_id_counter))
infile = os.path.join(self.working_dir,
f"{task_id}_in{os.extsep}pkl")
outfile = os.path.join(self.working_dir,
f"{task_id}_out{os.extsep}pkl")
try:
fn_buf = pack_apply_message(func,
args,
kwargs,
buffer_threshold=1024 * 1024)
except TypeError:
raise SerializationError(func.__name__)
with open(infile, "wb") as infile_handle:
infile_handle.write(fn_buf)
future = FluxFutureWrapper()
self._submission_queue.put(
_FluxJobInfo(future, task_id, infile, outfile,
resource_specification))
return future
def submit(self, func, *args, **kwargs):
"""Submits work to the the outgoing_q.
The outgoing_q is an external process listens on this
queue for new work. This method is simply pass through and behaves like a
submit call as described here `Python docs: <https://docs.python.org/3/library/concurrent.futures.html#concurrent.futures.ThreadPoolExecutor>`_
Args:
- func (callable) : Callable function
- args (list) : List of arbitrary positional arguments.
Kwargs:
- kwargs (dict) : A dictionary of arbitrary keyword args for func.
Returns:
Future
"""
task_id = uuid.uuid4()
logger.debug("Pushing function {} to queue with args {}".format(
func, args))
self.tasks[task_id] = Future()
fn_buf = pack_apply_message(func,
args,
kwargs,
buffer_threshold=1024 * 1024)
msg = {"task_id": task_id, "buffer": fn_buf}
# Post task to the the outgoing queue
self.outgoing_q.put(msg)
# Return the future
return self.tasks[task_id]
