def submit(self, func, resource_specification, *args, **kwargs):
"""Submits work to the the outgoing_q.
The outgoing_q is an external process listens on this
queue for new work. This method behaves like a
submit call as described here `Python docs: <https://docs.python.org/3/library/concurrent.futures.html#concurrent.futures.ThreadPoolExecutor>`_
Args:
- func (callable) : Callable function
- *args (list) : List of arbitrary positional arguments.
Kwargs:
- **kwargs (dict) : A dictionary of arbitrary keyword args for func.
Returns:
Future
"""
if resource_specification:
logger.error(
"Ignoring the resource specification. "
"Parsl resource specification is not supported in HighThroughput Executor. "
"Please check WorkQueueExecutor if resource specification is needed."
)
raise UnsupportedFeatureError('resource specification',
'HighThroughput Executor',
'WorkQueue Executor')
if self.bad_state_is_set:
raise self.executor_exception
self._task_counter += 1
task_id = self._task_counter
# handle people sending blobs gracefully
args_to_print = args
if logger.getEffectiveLevel() >= logging.DEBUG:
args_to_print = tuple([
arg if len(repr(arg)) < 100 else (repr(arg)[:100] + '...')
for arg in args
])
logger.debug("Pushing function {} to queue with args {}".format(
func, args_to_print))
self.tasks[task_id] = Future()
try:
fn_buf = pack_apply_message(func,
args,
kwargs,
buffer_threshold=1024 * 1024)
except TypeError:
raise SerializationError(func.__name__)
msg = {"task_id": task_id, "buffer": fn_buf}
# Post task to the the outgoing queue
self.outgoing_q.put(msg)
# Return the future
return self.tasks[task_id]
def submit(self, func, resource_specification, *args, **kwargs):
""" TODO: docstring """
if resource_specification:
logger.error(
"Ignoring the resource specification. "
"Parsl resource specification is not supported in LowLatency Executor. "
"Please check WorkQueueExecutor if resource specification is needed."
)
raise UnsupportedFeatureError('resource specification',
'LowLatency Executor',
'WorkQueue Executor')
if self.bad_state_is_set:
raise self.executor_exception
self._task_counter += 1
task_id = self._task_counter
logger.debug("Pushing function {} to queue with args {}".format(
func, args))
self.tasks[task_id] = Future()
fn_buf = pack_apply_message(func,
args,
kwargs,
buffer_threshold=1024 * 1024)
# Post task to the the outgoing queue
self.outgoing_q.put(task_id, fn_buf)
# Return the future
return self.tasks[task_id]
def submit(self, func, resource_specification, *args, **kwargs):
"""Submits work to the thread pool.
This method is simply pass through and behaves like a submit call as described
here `Python docs: <https://docs.python.org/3/library/concurrent.futures.html#concurrent.futures.ThreadPoolExecutor>`_
"""
if resource_specification:
logger.error("Ignoring the resource specification. "
"Parsl resource specification is not supported in ThreadPool Executor. "
"Please check WorkQueue Executor if resource specification is needed.")
raise UnsupportedFeatureError('resource specification', 'ThreadPool Executor', 'WorkQueue Executor')
return self.executor.submit(func, *args, **kwargs)