def WorkQueueSubmitThread(task_queue=multiprocessing.Queue(),
queue_lock=threading.Lock(),
launch_cmd=None,
env=None,
collector_queue=multiprocessing.Queue(),
see_worker_output=False,
data_dir=".",
full=False,
cancel_value=multiprocessing.Value('i', 1),
port=WORK_QUEUE_DEFAULT_PORT,
wq_log_dir=None,
project_password=None,
project_password_file=None,
project_name=None):
"""Thread to handle Parsl app submissions to the Work Queue objects.
Takes in Parsl functions submitted using submit(), and creates a
Work Queue task with the appropriate specifications, which is then
submitted to Work Queue. After tasks are completed, processes the
exit status and exit code of the task, and sends results to the
Work Queue collector thread.
"""
logger.debug("Starting WorkQueue Submit/Wait Process")
# Enable debugging flags and create logging file
if wq_log_dir is not None:
logger.debug("Setting debugging flags and creating logging file")
wq_debug_log = os.path.join(wq_log_dir, "debug_log")
cctools_debug_flags_set("all")
cctools_debug_config_file(wq_debug_log)
# Create WorkQueue queue object
logger.debug("Creating WorkQueue Object")
try:
logger.debug("Listening on port {}".format(port))
q = WorkQueue(port)
except Exception as e:
logger.error("Unable to create WorkQueue object: {}".format(e))
raise e
# Specify WorkQueue queue attributes
if project_name:
q.specify_name(project_name)
if project_password:
q.specify_password(project_password)
elif project_password_file:
q.specify_password_file(project_password_file)
# Only write logs when the wq_log_dir is specified, which it most likely will be
if wq_log_dir is not None:
wq_master_log = os.path.join(wq_log_dir, "master_log")
wq_trans_log = os.path.join(wq_log_dir, "transaction_log")
if full:
wq_resource_log = os.path.join(wq_log_dir, "resource_logs")
q.enable_monitoring_full(dirname=wq_resource_log)
q.specify_log(wq_master_log)
q.specify_transactions_log(wq_trans_log)
wq_tasks = set()
orig_ppid = os.getppid()
continue_running = True
while (continue_running):
# Monitor the task queue
ppid = os.getppid()
if ppid != orig_ppid:
logger.debug("new Process")
continue_running = False
continue
# Submit tasks
while task_queue.qsize() > 0:
if cancel_value.value == 0:
logger.debug("cancel value set to cancel")
continue_running = False
break
# Obtain task from task_queue
try:
item = task_queue.get(timeout=1)
logger.debug("Removing task from queue")
except queue.Empty:
continue
parsl_id = item["task_id"]
# Extract information about the task
function_data_loc = item["data_loc"]
function_data_loc_remote = function_data_loc.split("/")[-1]
function_result_loc = item["result_loc"]
function_result_loc_remote = function_result_loc.split("/")[-1]
input_files = item["input_files"]
output_files = item["output_files"]
std_files = item["std_files"]
full_script_name = workqueue_worker.__file__
script_name = full_script_name.split("/")[-1]
remapping_string = ""
std_string = ""
# Parse input file information
logger.debug("Looking at input")
for item in input_files:
if item[3] == "std":
std_string += "mv " + item[1] + " " + item[0] + "; "
else:
remapping_string += item[0] + ":" + item[1] + ","
logger.debug(remapping_string)
# Parse output file information
logger.debug("Looking at output")
for item in output_files:
remapping_string += item[0] + ":" + item[1] + ","
logger.debug(remapping_string)
if len(input_files) + len(output_files) > 0:
remapping_string = "-r " + remapping_string
remapping_string = remapping_string[:-1]
# Create command string
logger.debug(launch_cmd)
command_str = launch_cmd.format(
input_file=function_data_loc_remote,
output_file=function_result_loc_remote,
remapping_string=remapping_string)
command_str = std_string + command_str
logger.debug(command_str)
# Create WorkQueue task for the command
logger.debug("Sending task {} with command: {}".format(
parsl_id, command_str))
try:
t = Task(command_str)
except Exception as e:
logger.error("Unable to create task: {}".format(e))
continue
# Specify environment variables for the task
if env is not None:
for var in env:
t.specify_environment_variable(var, env[var])
# Specify script, and data/result files for task
t.specify_file(full_script_name,
script_name,
WORK_QUEUE_INPUT,
cache=True)
t.specify_file(function_data_loc,
function_data_loc_remote,
WORK_QUEUE_INPUT,
cache=False)
t.specify_file(function_result_loc,
function_result_loc_remote,
WORK_QUEUE_OUTPUT,
cache=False)
t.specify_tag(str(parsl_id))
logger.debug("Parsl ID: {}".format(t.id))
# Specify all input/output files for task
for item in input_files:
t.specify_file(item[0],
item[1],
WORK_QUEUE_INPUT,
cache=item[2])
for item in output_files:
t.specify_file(item[0],
item[1],
WORK_QUEUE_OUTPUT,
cache=item[2])
for item in std_files:
t.specify_file(item[0],
item[1],
WORK_QUEUE_OUTPUT,
cache=item[2])
# Submit the task to the WorkQueue object
logger.debug("Submitting task {} to WorkQueue".format(parsl_id))
try:
wq_id = q.submit(t)
wq_tasks.add(wq_id)
except Exception as e:
logger.error("Unable to create task: {}".format(e))
msg = {
"tid": parsl_id,
"result_received": False,
"reason": "Workqueue Task Start Failure",
"status": 1
}
collector_queue.put_nowait(msg)
continue
logger.debug("Task {} submitted to WorkQueue with id {}".format(
parsl_id, wq_id))
if cancel_value.value == 0:
continue_running = False
# If the queue is not empty wait on the WorkQueue queue for a task
task_found = True
if not q.empty() and continue_running:
while task_found is True:
if cancel_value.value == 0:
continue_running = False
task_found = False
continue
# Obtain the task from the queue
t = q.wait(1)
if t is None:
task_found = False
continue
else:
parsl_tid = t.tag
logger.debug(
"Completed WorkQueue task {}, parsl task {}".format(
t.id, parsl_tid))
status = t.return_status
task_result = t.result
msg = None
# Task failure
if status != 0 or (task_result != WORK_QUEUE_RESULT_SUCCESS
and task_result !=
WORK_QUEUE_RESULT_OUTPUT_MISSING):
logger.debug(
"Wrapper Script status: {}\nWorkQueue Status: {}".
format(status, task_result))
# Wrapper script failure
if status != 0:
logger.debug(
"WorkQueue task {} failed with status {}".
format(t.id, status))
reason = "Wrapper Script Failure: "
if status == 1:
reason += "problem parsing command line options"
elif status == 2:
reason += "problem loading function data"
elif status == 3:
reason += "problem remapping file names"
elif status == 4:
reason += "problem writing out function result"
else:
reason += "unable to process wrapper script failure with status = {}".format(
status)
reason += "\nTrace:\n" + str(t.output)
logger.debug(
"WorkQueue runner script failed for task {} because {}\n"
.format(parsl_tid, reason))
# WorkQueue system failure
else:
reason = "WorkQueue System Failure: "
if task_result == 1:
reason += "missing input file"
elif task_result == 2:
reason += "unable to generate output file"
elif task_result == 4:
reason += "stdout has been truncated"
elif task_result == 1 << 3:
reason += "task terminated with a signal"
elif task_result == 2 << 3:
reason += "task used more resources than requested"
elif task_result == 3 << 3:
reason += "task ran past the specified end time"
elif task_result == 4 << 3:
reason += "result could not be classified"
elif task_result == 5 << 3:
reason += "task failed, but not a task error"
elif task_result == 6 << 3:
reason += "unable to complete after specified number of retries"
elif task_result == 7 << 3:
reason += "task ran for more than the specified time"
elif task_result == 8 << 3:
reason += "task needed more space to complete task"
else:
reason += "unable to process Work Queue system failure"
msg = {
"tid": parsl_tid,
"result_received": False,
"reason": reason,
"status": status
}
collector_queue.put_nowait(msg)
# Task Success
else:
# Print the output from the task
if see_worker_output:
print(t.output)
# Load result into result file
result_loc = os.path.join(
data_dir,
"task_" + str(parsl_tid) + "_function_result")
logger.debug(
"Looking for result in {}".format(result_loc))
f = open(result_loc, "rb")
result = pickle.load(f)
f.close()
msg = {
"tid": parsl_tid,
"result_received": True,
"result": result
}
wq_tasks.remove(t.id)
collector_queue.put_nowait(msg)
if continue_running is False:
logger.debug("Exiting WorkQueue Master Thread event loop")
break
# Remove all WorkQueue tasks that remain in the queue object
for wq_task in wq_tasks:
logger.debug("Cancelling WorkQueue Task {}".format(wq_task))
q.cancel_by_taskid(wq_task)
logger.debug("Exiting WorkQueue Monitoring Process")
return 0
def WorkQueueSubmitThread(task_queue=multiprocessing.Queue(),
queue_lock=threading.Lock(),
launch_cmd=None,
env=None,
collector_queue=multiprocessing.Queue(),
see_worker_output=False,
data_dir=".",
full=False,
cancel_value=multiprocessing.Value('i', 1),
port=WORK_QUEUE_DEFAULT_PORT,
wq_log_dir=None,
project_password=None,
project_password_file=None,
project_name=None):
logger.debug("Starting WorkQueue Submit/Wait Process")
orig_ppid = os.getppid()
wq_tasks = set()
continue_running = True
if wq_log_dir is not None:
wq_debug_log = os.path.join(wq_log_dir, "debug")
cctools_debug_flags_set("all")
cctools_debug_config_file(wq_debug_log)
logger.debug("Creating Workqueue Object")
try:
q = WorkQueue(port)
except Exception as e:
logger.error("Unable to create Workqueue object: {}", format(e))
raise e
if project_name:
q.specify_name(project_name)
if project_password:
q.specify_password(project_password)
elif project_password_file:
q.specify_password_file(project_password_file)
# Only write Logs when the log_dir is specified, which is most likely always will be
if wq_log_dir is not None:
wq_master_log = os.path.join(wq_log_dir, "master_log")
wq_trans_log = os.path.join(wq_log_dir, "transaction_log")
if full:
wq_resource_log = os.path.join(wq_log_dir, "resource_logs")
q.enable_monitoring_full(dirname=wq_resource_log)
q.specify_log(wq_master_log)
q.specify_transactions_log(wq_trans_log)
while (continue_running):
# Monitor the Task Queue
ppid = os.getppid()
if ppid != orig_ppid:
continue_running = False
continue
# Submit Tasks
while task_queue.qsize() > 0:
if cancel_value.value == 0:
continue_running = False
break
try:
# item = task_queue.get_nowait()
item = task_queue.get(timeout=1)
logger.debug("Removing task from queue")
except queue.Empty:
continue
parsl_id = item["task_id"]
function_data_loc = item["data_loc"]
function_result_loc = item["result_loc"]
function_result_loc_remote = function_result_loc.split("/")[-1]
function_data_loc_remote = function_data_loc.split("/")[-1]
input_files = item["input_files"]
output_files = item["output_files"]
std_files = item["std_files"]
full_script_name = workqueue_worker.__file__
script_name = full_script_name.split("/")[-1]
remapping_string = ""
std_string = ""
logger.debug("looking at input")
for item in input_files:
if item[3] == "std":
std_string += "mv " + item[1] + " " + item[0] + "; "
else:
remapping_string += item[0] + ":" + item[1] + ","
logger.debug(remapping_string)
logger.debug("looking at output")
for item in output_files:
remapping_string += item[0] + ":" + item[1] + ","
logger.debug(remapping_string)
if len(input_files) + len(output_files) > 0:
remapping_string = "-r " + remapping_string
remapping_string = remapping_string[:-1]
logger.debug(launch_cmd)
command_str = launch_cmd.format(
input_file=function_data_loc_remote,
output_file=function_result_loc_remote,
remapping_string=remapping_string)
logger.debug(command_str)
command_str = std_string + command_str
logger.debug(command_str)
logger.debug("Sending task {} with command: {}".format(
parsl_id, command_str))
try:
t = Task(command_str)
except Exception as e:
logger.error("Unable to create task: {}".format(e))
continue
if env is not None:
for var in env:
t.specify_environment_variable(var, env[var])
t.specify_file(full_script_name,
script_name,
WORK_QUEUE_INPUT,
cache=True)
t.specify_file(function_result_loc,
function_result_loc_remote,
WORK_QUEUE_OUTPUT,
cache=False)
t.specify_file(function_data_loc,
function_data_loc_remote,
WORK_QUEUE_INPUT,
cache=False)
t.specify_tag(str(parsl_id))
for item in input_files:
t.specify_file(item[0],
item[1],
WORK_QUEUE_INPUT,
cache=item[2])
for item in output_files:
t.specify_file(item[0],
item[1],
WORK_QUEUE_OUTPUT,
cache=item[2])
for item in std_files:
t.specify_file(item[0],
item[1],
WORK_QUEUE_OUTPUT,
cache=item[2])
logger.debug("Submitting task {} to workqueue".format(parsl_id))
try:
wq_id = q.submit(t)
wq_tasks.add(wq_id)
except Exception as e:
logger.error("Unable to create task: {}".format(e))
msg = {
"tid": parsl_id,
"result_received": False,
"reason": "Workqueue Task Start Failure",
"status": 1
}
collector_queue.put_nowait(msg)
continue
logger.debug("Task {} submitted workqueue with id {}".format(
parsl_id, wq_id))
if cancel_value.value == 0:
continue_running = False
# Wait for Tasks
task_found = True
# If the queue is not empty wait on the workqueue queue for a task
if not q.empty() and continue_running:
while task_found is True:
if cancel_value.value == 0:
continue_running = False
task_found = False
continue
t = q.wait(1)
if t is None:
task_found = False
continue
else:
parsl_tid = t.tag
logger.debug(
"Completed workqueue task {}, parsl task {}".format(
t.id, parsl_tid))
status = t.return_status
task_result = t.result
msg = None
if status != 0 or (task_result != WORK_QUEUE_RESULT_SUCCESS
and task_result !=
WORK_QUEUE_RESULT_OUTPUT_MISSING):
if task_result == WORK_QUEUE_RESULT_SUCCESS:
logger.debug(
"Workqueue task {} failed with status {}".
format(t.id, status))
reason = "Wrapper Script Failure: "
if status == 1:
reason += "command line parsing"
if status == 2:
reason += "problem loading function data"
if status == 3:
reason += "problem remapping file names"
if status == 4:
reason += "problem writing out function result"
reason += "\nTrace:\n" + t.output
logger.debug(
"Workqueue runner script failed for task {} because {}\n"
.format(parsl_tid, reason))
else:
reason = "Workqueue system failure\n"
msg = {
"tid": parsl_tid,
"result_received": False,
"reason": reason,
"status": status
}
collector_queue.put_nowait(msg)
else:
if see_worker_output:
print(t.output)
result_loc = os.path.join(
data_dir,
"task_" + str(parsl_tid) + "_function_result")
logger.debug(
"Looking for result in {}".format(result_loc))
f = open(result_loc, "rb")
result = pickle.load(f)
f.close()
msg = {
"tid": parsl_tid,
"result_received": True,
"result": result
}
wq_tasks.remove(t.id)
collector_queue.put_nowait(msg)
if continue_running is False:
logger.debug("Exiting WorkQueue Master Thread event loop")
break
for wq_task in wq_tasks:
logger.debug("Cancelling Workqueue Task {}".format(wq_task))
q.cancel_by_taskid(wq_task)
logger.debug("Exiting WorkQueue Monitoring Process")
return 0