def create_tasks(cls, params):
"""Create the tasks dictionary from the parameters.
Args:
params (dict): Parameters.
Returns:
(dict): The tasks.
"""
out_dir = path(params["out_dir"])
if not out_dir.exists():
out_dir.makedirs()
tasks = {}
for itask, task_name in enumerate(params["task_names"]):
out_pth = out_dir.joinpath("{}.pkl".format(task_name))
# Make the task dicts for this sample.
task = {
"out_pth": str(out_pth),
"script_pth": params["script_pth"],
"task_name": task_name,
"seed": abs(hash(task_name)),
"complete": False}
tasks[task_name] = task
mark_complete(tasks)
update_tasks_file(params["tasks_pth"], tasks)
return tasks
def manage_tasks(self, tasks):
"""Receives completed tasks from the clients and updates the
tasks file.
Args:
tasks (dict): Tasks.
"""
t0 = Time.time()
tasks_pth = path(self.params().get("tasks_pth", ""))
n_tasks = len(tasks)
n_done0 = n_tasks - self.taskq().qsize()
n_done = n_done0
util.info("[Server] Tasks queued. {}/{} ({:.1f}%%) complete.".format(
n_done, n_tasks, n_done / n_tasks * 100))
save_time = Time.time()
# Set up a thread that joins self.taskq and only returns once
# all the tasks have completed. The while loops continues as
# long as taskq_thread is alive.
taskq_thread = Thread(name="taskq", target=self._taskq_worker,
args=(self.taskq,))
taskq_thread.start()
while taskq_thread.is_alive():
# Wait for a done task to arrive.
task = self.doneq().get()
# Update the master tasks dict.
task_name = task["task_name"]
tasks[task_name].update(task)
if tasks_pth and save_time.delta() > 10.:
# Save task to disk.
update_tasks_file(tasks_pth, tasks, overwrite=True)
save_time = Time.time()
# Report progress.
n_done += 1
percent = float(n_done) / n_tasks * 100.
dt = t0.delta()
time_per_task = dt / float(n_done - n_done0)
n_left = n_tasks - n_done
t_left = Time(time_per_task * n_left)
util.info("[Server] Task `{}` complete:\n\t\t {}/{} ({:.2f}%) {} "
"\n\t\t Time left: {}.".format(
task_name, n_done, n_tasks, percent, dt, t_left))
# Save tasks to disk one last time.
update_tasks_file(tasks_pth, tasks, overwrite=True)
# Wait for the clients to d/c.
clients = []
while not self.activeq().empty():
clients.append(self.activeq().get())
util.debug("[Server] Waiting for clients to disconnect:\n\t{}.".format(
"\n\t".join(clients)))
self.activeq().join()
time.sleep(0.5)
self.activeq().close()
self.taskq().close()
self.doneq().close()
util.info("[Server] Tasks completed.")
