def worker(worker_id, task_url, debug=True, logdir="workers", uid="1"):
""" TODO: docstring
TODO : Cleanup debug, logdir and uid to function correctly
"""
start_file_logger('{}/{}/worker_{}.log'.format(logdir, uid, worker_id),
0,
level=logging.DEBUG if debug is True else logging.INFO)
logger.info("Starting worker {}".format(worker_id))
task_ids_received = []
message_q = zmq_pipes.WorkerMessages(task_url)
while True:
print("Worker loop iteration starting")
task_id, buf = message_q.get()
task_ids_received.append(task_id)
user_ns = locals()
user_ns.update({'__builtins__': __builtins__})
f, args, kwargs = unpack_apply_message(buf, user_ns, copy=False)
logger.debug("Worker {} received task {}".format(worker_id, task_id))
result = execute_task(f, args, kwargs, user_ns)
logger.debug("Worker {} completed task {}".format(worker_id, task_id))
reply = {"result": result, "worker_id": worker_id}
message_q.put(task_id, serialize(reply))
logger.debug("Result sent")
def execute_task(bufs):
"""Deserialize the buffer and execute the task.
Returns the result or throws exception.
"""
user_ns = locals()
user_ns.update({'__builtins__': __builtins__})
f, args, kwargs = unpack_apply_message(bufs, user_ns, copy=False)
# We might need to look into callability of the function from itself
# since we change it's name in the new namespace
prefix = "parsl_"
fname = prefix + "f"
argname = prefix + "args"
kwargname = prefix + "kwargs"
resultname = prefix + "result"
user_ns.update({
fname: f,
argname: args,
kwargname: kwargs,
resultname: resultname
})
code = "{0} = {1}(*{2}, **{3})".format(resultname, fname, argname,
kwargname)
exec(code, user_ns, user_ns)
return user_ns.get(resultname)
def execute_task(bufs):
"""Deserialize the buffer and execute the task.
Returns the serialized result or exception.
"""
user_ns = locals()
user_ns.update({'__builtins__': __builtins__})
f, args, kwargs = unpack_apply_message(bufs, user_ns, copy=False)
fname = getattr(f, '__name__', 'f')
prefix = "parsl_"
fname = prefix + "f"
argname = prefix + "args"
kwargname = prefix + "kwargs"
resultname = prefix + "result"
user_ns.update({
fname: f,
argname: args,
kwargname: kwargs,
resultname: resultname
})
code = "{0} = {1}(*{2}, **{3})".format(resultname, fname, argname,
kwargname)
try:
logger.debug("[RUNNER] Executing: {0}".format(code))
exec(code, user_ns, user_ns)
except Exception as e:
logger.warning("Caught exception; will raise it: {}".format(e))
raise e
else:
logger.debug("[RUNNER] Result: {0}".format(
user_ns.get(resultname)))
return user_ns.get(resultname)
def execute_task(bufs):
"""Deserialize the buffer and execute the task.
Returns the result or throws exception.
"""
user_ns = locals()
user_ns.update({'__builtins__': __builtins__})
f, args, kwargs = unpack_apply_message(bufs, user_ns, copy=False)
# We might need to look into callability of the function from itself
# since we change it's name in the new namespace
prefix = "parsl_"
fname = prefix + "f"
argname = prefix + "args"
kwargname = prefix + "kwargs"
resultname = prefix + "result"
user_ns.update({
fname: f,
argname: args,
kwargname: kwargs,
resultname: resultname
})
code = "{0} = {1}(*{2}, **{3})".format(resultname, fname, argname,
kwargname)
try:
# logger.debug("[RUNNER] Executing: {0}".format(code))
exec(code, user_ns, user_ns)
except Exception as e:
logger.warning("Caught exception; will raise it: {}".format(e),
exc_info=True)
raise e
else:
# logger.debug("[RUNNER] Result: {0}".format(user_ns.get(resultname)))
return user_ns.get(resultname)
def unpack_byte_code_function(function_info, user_namespace):
from parsl.serialize import unpack_apply_message
func, args, kwargs = unpack_apply_message(function_info["byte code"],
user_namespace,
copy=False)
return (func, 'parsl_function_name', args, kwargs)