def submit(self, func, *args, **kwargs):
''' Submits work to the the outgoing_q, an external process listens on this
queue for new work. This method is simply pass through and behaves like a
submit call as described here `Python docs: <https://docs.python.org/3/library/concurrent.futures.html#concurrent.futures.ThreadPoolExecutor>`_
Args:
- func (callable) : Callable function
- *args (list) : List of arbitrary positional arguments.
Kwargs:
- **kwargs (dict) : A dictionary of arbitrary keyword args for func.
Returns:
Future
'''
task_id = uuid.uuid4()
logger.debug("Before pushing to queue : func:%s func_args:%s", func,
args)
self.tasks[task_id] = Future()
fn_buf = pack_apply_message(func,
args,
kwargs,
buffer_threshold=1024 * 1024,
item_threshold=1024)
msg = {"task_id": task_id, "buffer": fn_buf}
# Post task to the the outgoing queue
self.outgoing_q.put(msg)
# Return the future
return self.tasks[task_id]
def _serialize_function(self, fn_path, parsl_fn, parsl_fn_args,
parsl_fn_kwargs):
"""Takes the function application parsl_fn(*parsl_fn_args, **parsl_fn_kwargs)
and serializes it to the file fn_path."""
# Either build a dictionary with the source of the function, or pickle
# the function directly:
if self.source:
function_info = {
"source code": inspect.getsource(parsl_fn),
"name": parsl_fn.__name__,
"args": parsl_fn_args,
"kwargs": parsl_fn_kwargs
}
else:
function_info = {
"byte code":
pack_apply_message(parsl_fn,
parsl_fn_args,
parsl_fn_kwargs,
buffer_threshold=1024 * 1024,
item_threshold=1024)
}
with open(fn_path, "wb") as f_out:
pickle.dump(function_info, f_out)
def simple_executor(f_all, args_all, kwargs_all, num_tasks):
serialization_times = []
exec_times = []
results = []
for i in range(num_tasks):
task_id = i
start_time = time.time()
buf = pack_apply_message(f=next(f_all),
args=next(args_all),
kwargs=next(kwargs_all),
buffer_threshold=1024 * 1024,
item_threshold=1024)
serialization_times.append(time.time() - start_time)
start_time = time.time()
user_ns = locals()
user_ns.update({'__builtins__': __builtins__})
f, args, kwargs = unpack_apply_message(buf, user_ns, copy=False)
result = execute_task(f, args, kwargs, user_ns)
exec_times.append(time.time() - start_time)
results.append(result)
avg_serialization_time = sum(serialization_times) / len(
serialization_times) * 10**6
avg_execution_time = sum(exec_times) / len(exec_times) * 10**6
return {
"avg_serialization_time": avg_serialization_time,
"avg_execution_time": avg_execution_time,
"results": results
}
def submit(self, func, resource_specification, *args, **kwargs):
""" TODO: docstring """
if resource_specification:
logger.error(
"Ignoring the resource specification. "
"Parsl resource specification is not supported in LowLatency Executor. "
"Please check WorkQueueExecutor if resource specification is needed."
)
raise UnsupportedFeatureError('resource specification',
'LowLatency Executor',
'WorkQueue Executor')
if self.bad_state_is_set:
raise self.executor_exception
self._task_counter += 1
task_id = self._task_counter
logger.debug("Pushing function {} to queue with args {}".format(
func, args))
self.tasks[task_id] = Future()
fn_buf = pack_apply_message(func,
args,
kwargs,
buffer_threshold=1024 * 1024,
item_threshold=1024)
# Post task to the the outgoing queue
self.outgoing_q.put(task_id, fn_buf)
# Return the future
return self.tasks[task_id]
def submit(self, func, resource_specification, *args, **kwargs):
"""Submits work to the the outgoing_q.
The outgoing_q is an external process listens on this
queue for new work. This method behaves like a
submit call as described here `Python docs: <https://docs.python.org/3/library/concurrent.futures.html#concurrent.futures.ThreadPoolExecutor>`_
Args:
- func (callable) : Callable function
- *args (list) : List of arbitrary positional arguments.
Kwargs:
- **kwargs (dict) : A dictionary of arbitrary keyword args for func.
Returns:
Future
"""
if resource_specification:
logger.error("Ignoring the resource specification. "
"Parsl resource specification is not supported in HighThroughput Executor. "
"Please check WorkQueueExecutor if resource specification is needed.")
raise UnsupportedFeatureError('resource specification', 'HighThroughput Executor', 'WorkQueue Executor')
if self.bad_state_is_set:
raise self.executor_exception
self._task_counter += 1
task_id = self._task_counter
# handle people sending blobs gracefully
args_to_print = args
if logger.getEffectiveLevel() >= logging.DEBUG:
args_to_print = tuple([arg if len(repr(arg)) < 100 else (repr(arg)[:100] + '...') for arg in args])
logger.debug("Pushing function {} to queue with args {}".format(func, args_to_print))
self.tasks[task_id] = Future()
try:
fn_buf = pack_apply_message(func, args, kwargs,
buffer_threshold=1024 * 1024,
item_threshold=1024)
except TypeError:
raise SerializationError(func.__name__)
msg = {"task_id": task_id,
"buffer": fn_buf}
# Post task to the the outgoing queue
self.outgoing_q.put(msg)
# Return the future
return self.tasks[task_id]
def submit(self, func, *args, **kwargs):
"""Submits work to the the outgoing_q.
The outgoing_q is an external process listens on this
queue for new work. This method behaves like a
submit call as described here `Python docs: <https://docs.python.org/3/library/concurrent.futures.html#concurrent.futures.ThreadPoolExecutor>`_
Args:
- func (callable) : Callable function
- *args (list) : List of arbitrary positional arguments.
Kwargs:
- **kwargs (dict) : A dictionary of arbitrary keyword args for func.
Returns:
Future
"""
if self._executor_bad_state.is_set():
raise self._executor_exception
self._task_counter += 1
task_id = self._task_counter
# handle people sending blobs gracefully
args_to_print = args
if logger.getEffectiveLevel() >= logging.DEBUG:
args_to_print = tuple([
arg if len(repr(arg)) < 100 else (repr(arg)[:100] + '...')
for arg in args
])
logger.debug("Pushing function {} to queue with args {}".format(
func, args_to_print))
self.tasks[task_id] = Future()
fn_buf = pack_apply_message(func,
args,
kwargs,
buffer_threshold=1024 * 1024,
item_threshold=1024)
msg = {"task_id": task_id, "buffer": fn_buf}
# Post task to the the outgoing queue
self.outgoing_q.put(msg)
# Return the future
return self.tasks[task_id]
def submit(self, func, *args, **kwargs):
""" TODO: docstring """
self._task_counter += 1
task_id = self._task_counter
logger.debug("Pushing function {} to queue with args {}".format(
func, args))
self.tasks[task_id] = Future()
fn_buf = pack_apply_message(func,
args,
kwargs,
buffer_threshold=1024 * 1024,
item_threshold=1024)
# Post task to the the outgoing queue
self.outgoing_q.put(task_id, fn_buf)
# Return the future
return self.tasks[task_id]
def submit(self, func, *args, **kwargs):
"""Processes the Parsl app by its arguments and submits the function
information to the task queue, to be executed using the Work Queue
system. The args and kwargs are processed for input and output files to
the Parsl app, so that the files are appropriately specified for the Work
Queue task.
Parameters
----------
func : function
Parsl app to be submitted to the Work Queue system
args : list
Arguments to the Parsl app
kwargs : dict
Keyword arguments to the Parsl app
"""
self.task_counter += 1
task_id = self.task_counter
input_files = []
output_files = []
std_files = []
# Add input files from the "inputs" keyword argument
func_inputs = kwargs.get("inputs", [])
for inp in func_inputs:
if isinstance(inp, File):
input_files.append(self.create_name_tuple(inp, "in"))
for kwarg, inp in kwargs.items():
# Add appropriate input and output files from "stdout" and "stderr" keyword arguments
if kwarg == "stdout" or kwarg == "stderr":
if (isinstance(inp, tuple) and len(inp) > 1
and isinstance(inp[0], str)
and isinstance(inp[1], str)) or isinstance(inp, str):
if isinstance(inp, tuple):
inp = inp[0]
if not os.path.exists(
os.path.join(".",
os.path.split(inp)[0])):
continue
# Create "std" files instead of input or output files
if inp in self.registered_files:
input_files.append(
(inp, os.path.basename(inp) + "-1", False, "std"))
output_files.append(
(inp, os.path.basename(inp), False, "std"))
else:
output_files.append(
(inp, os.path.basename(inp), False, "std"))
self.registered_files.add(inp)
# Add to input file if passed-in argument is a File object
elif isinstance(inp, File):
input_files.append(self.create_name_tuple(inp, "in"))
# Add to input file if passed-in argument is a File object
for inp in args:
if isinstance(inp, File):
input_files.append(self.create_name_tuple(inp, "in"))
# Add output files from the "outputs" keyword argument
func_outputs = kwargs.get("outputs", [])
for output in func_outputs:
if isinstance(output, File):
output_files.append(self.create_name_tuple(output, "out"))
if not self.submit_process.is_alive():
raise ExecutorError(self, "Workqueue Submit Process is not alive")
# Create a Future object and have it be mapped from the task ID in the tasks dictionary
fu = Future()
self.tasks_lock.acquire()
self.tasks[str(task_id)] = fu
self.tasks_lock.release()
logger.debug("Creating task {} for function {} with args {}".format(
task_id, func, args))
# Pickle the result into object to pass into message buffer
function_data_file = os.path.join(
self.function_data_dir, "task_" + str(task_id) + "_function_data")
function_result_file = os.path.join(
self.function_data_dir,
"task_" + str(task_id) + "_function_result")
logger.debug("Creating Task {} with executable at: {}".format(
task_id, function_data_file))
logger.debug("Creating Task {} with result to be found at: {}".format(
task_id, function_result_file))
# Obtain function information and put into dictionary
if self.source:
# Obtain function information and put into dictionary
source_code = inspect.getsource(func)
name = func.__name__
function_info = {
"source code": source_code,
"name": name,
"args": args,
"kwargs": kwargs
}
# Pack the function data into file
f = open(function_data_file, "wb")
pickle.dump(function_info, f)
f.close()
else:
# Serialize function information
function_info = pack_apply_message(func,
args,
kwargs,
buffer_threshold=1024 * 1024,
item_threshold=1024)
# Pack the function data into file
f = open(function_data_file, "wb")
pickle.dump(function_info, f)
f.close()
# Create message to put into the message queue
logger.debug("Placing task {} on message queue".format(task_id))
msg = {
"task_id": task_id,
"data_loc": function_data_file,
"result_loc": function_result_file,
"input_files": input_files,
"output_files": output_files,
"std_files": std_files
}
self.task_queue.put_nowait(msg)
return fu
def submit(self, func, *args, **kwargs):
"""Submit.
We haven't yet decided on what the args to this can be,
whether it should just be func, args, kwargs or be the partially evaluated
fn
"""
self.task_counter += 1
task_id = self.task_counter
input_files = []
output_files = []
std_files = []
func_inputs = kwargs.get("inputs", [])
for inp in func_inputs:
if isinstance(inp, File):
input_files.append(self.create_name_tuple(inp, "in"))
for kwarg, inp in kwargs.items():
if kwarg == "stdout" or kwarg == "stderr":
if (isinstance(inp, tuple) and len(inp) > 1
and isinstance(inp[0], str)
and isinstance(inp[1], str)) or isinstance(inp, str):
if isinstance(inp, tuple):
inp = inp[0]
print(os.path.split(inp))
if not os.path.exists(
os.path.join(".",
os.path.split(inp)[0])):
continue
if inp in self.registered_files:
input_files.append(
(inp, os.path.basename(inp) + "-1", False, "std"))
output_files.append(
(inp, os.path.basename(inp), False, "std"))
else:
output_files.append(
(inp, os.path.basename(inp), False, "std"))
self.registered_files.add(inp)
elif isinstance(inp, File):
input_files.append(self.create_name_tuple(inp, "in"))
for inp in args:
if isinstance(inp, File):
input_files.append(self.create_name_tuple(inp, "in"))
func_outputs = kwargs.get("outputs", [])
for output in func_outputs:
if isinstance(output, File):
output_files.append(self.create_name_tuple(output, "out"))
if not self.submit_process.is_alive():
raise ExecutorError(self, "Workqueue Submit Process is not alive")
fu = Future()
self.tasks_lock.acquire()
self.tasks[str(task_id)] = fu
self.tasks_lock.release()
logger.debug("Creating task {} for function {} with args {}".format(
task_id, func, args))
# Pickle the result into object to pass into message buffer
# TODO Try/Except Block
function_data_file = os.path.join(
self.function_data_dir, "task_" + str(task_id) + "_function_data")
function_result_file = os.path.join(
self.function_data_dir,
"task_" + str(task_id) + "_function_result")
logger.debug("Creating Task {} with executable at: {}".format(
task_id, function_data_file))
logger.debug("Creating Task {} with result to be found at: {}".format(
task_id, function_result_file))
f = open(function_data_file, "wb")
fn_buf = pack_apply_message(func,
args,
kwargs,
buffer_threshold=1024 * 1024,
item_threshold=1024)
pickle.dump(fn_buf, f)
f.close()
logger.debug("Placing task {} on message queue".format(task_id))
msg = {
"task_id": task_id,
"data_loc": function_data_file,
"result_loc": function_result_file,
"input_files": input_files,
"output_files": output_files,
"std_files": std_files
}
self.task_queue.put_nowait(msg)
return fu
def dealer_executor(f_all,
args_all,
kwargs_all,
num_tasks,
return_dict,
port=5559,
interchange=True,
warmup=10):
label = "DEALER-INTERCHANGE-REP" if interchange else "DEALER-REP"
logger.info("Starting executor:{}".format(label))
serialization_times = []
deserialization_times = []
send_times = {}
exec_times = {}
results = []
context = zmq.Context()
dealer = context.socket(zmq.DEALER)
dealer.bind("tcp://*:{}".format(port))
poller = zmq.Poller()
poller.register(dealer, zmq.POLLIN)
num_send = 0
num_recv = 0
while True:
socks = dict(poller.poll(1))
if num_send < num_tasks:
task_id = num_send
task_id_bytes = task_id.to_bytes(4, "little")
start_time = time.time()
buf = pack_apply_message(f=next(f_all),
args=next(args_all),
kwargs=next(kwargs_all),
buffer_threshold=1024 * 1024,
item_threshold=1024)
serialization_times.append(time.time() - start_time)
logger.debug("Manager sending task {}".format(task_id))
send_times[task_id] = time.time()
dealer.send_multipart([b"", task_id_bytes] + buf)
num_send += 1
if dealer in socks and socks[dealer] == zmq.POLLIN:
buf = dealer.recv_multipart()
recv_time = time.time()
start_time = time.time()
msg = deserialize_object(buf[2:])[0]
deserialization_times.append(time.time() - start_time)
logger.debug("Got message {}".format(msg))
task_id = int.from_bytes(buf[1], "little")
results.append(msg["result"])
if num_recv >= warmup:
# Ignore the first `warmup` tasks
exec_times[task_id] = recv_time - send_times[task_id]
num_recv += 1
logger.debug("Dealer received result {}".format(task_id))
if num_recv == num_tasks:
break
avg_serialization_time = sum(serialization_times) / len(
serialization_times) * 10**6
avg_execution_time = sum(exec_times.values()) / len(exec_times) * 10**6
return_dict["avg_serialization_time"] = avg_serialization_time
return_dict["avg_execution_time"] = avg_execution_time
return_dict["results"] = results
def add2nums(num1, num2):
return num1 + num2
for req in range(10):
# time.sleep(4)
print("Sending request {}".format(req))
task_id = str(uuid.uuid4())
args = {"num1": 2, "num2": 2}
kwargs = {}
obj = pack_apply_message(add2nums, args, kwargs)
worker.send_multipart([
pickle.dumps(task_id),
pickle.dumps(obj),
pickle.dumps("INVOKE"), b'B'
])
worker.send_multipart([
pickle.dumps(task_id),
pickle.dumps(obj),
pickle.dumps("INVOKE"), b'A'
])
total = 0
while True:
print("Listening for new result")
