Пример #1
0
    def create_tasks(cls, params):
        """Create the tasks dictionary from the parameters.

        Args:
            params (dict): Parameters.

        Returns:
            (dict):  The tasks.

        """
        out_dir = path(params["out_dir"])
        if not out_dir.exists():
            out_dir.makedirs()
        tasks = {}
        for itask, task_name in enumerate(params["task_names"]):
            out_pth = out_dir.joinpath("{}.pkl".format(task_name))
            # Make the task dicts for this sample.
            task = {
                "out_pth": str(out_pth),
                "script_pth": params["script_pth"],
                "task_name": task_name,
                "seed": abs(hash(task_name)),
                "complete": False}
            tasks[task_name] = task
        mark_complete(tasks)
        update_tasks_file(params["tasks_pth"], tasks)
        return tasks
Пример #2
0
    def manage_tasks(self, tasks):
        """Receives completed tasks from the clients and updates the
        tasks file.

        Args:
            tasks (dict): Tasks.

        """
        t0 = Time.time()
        tasks_pth = path(self.params().get("tasks_pth", ""))
        n_tasks = len(tasks)
        n_done0 = n_tasks - self.taskq().qsize()
        n_done = n_done0
        util.info("[Server] Tasks queued. {}/{} ({:.1f}%%) complete.".format(
            n_done, n_tasks, n_done / n_tasks * 100))
        save_time = Time.time()
        # Set up a thread that joins self.taskq and only returns once
        # all the tasks have completed. The while loops continues as
        # long as taskq_thread is alive.
        taskq_thread = Thread(name="taskq", target=self._taskq_worker,
                              args=(self.taskq,))
        taskq_thread.start()
        while taskq_thread.is_alive():
            # Wait for a done task to arrive.
            task = self.doneq().get()
            # Update the master tasks dict.
            task_name = task["task_name"]
            tasks[task_name].update(task)
            if tasks_pth and save_time.delta() > 10.:
                # Save task to disk.
                update_tasks_file(tasks_pth, tasks, overwrite=True)
                save_time = Time.time()
            # Report progress.
            n_done += 1
            percent = float(n_done) / n_tasks * 100.
            dt = t0.delta()
            time_per_task = dt / float(n_done - n_done0)
            n_left = n_tasks - n_done
            t_left = Time(time_per_task * n_left)
            util.info("[Server] Task `{}` complete:\n\t\t {}/{} ({:.2f}%) {} "
                      "\n\t\t Time left: {}.".format(
                          task_name, n_done, n_tasks, percent, dt, t_left))
        # Save tasks to disk one last time.
        update_tasks_file(tasks_pth, tasks, overwrite=True)
        # Wait for the clients to d/c.
        clients = []
        while not self.activeq().empty():
            clients.append(self.activeq().get())
        util.debug("[Server] Waiting for clients to disconnect:\n\t{}.".format(
            "\n\t".join(clients)))
        self.activeq().join()
        time.sleep(0.5)
        self.activeq().close()
        self.taskq().close()
        self.doneq().close()
        util.info("[Server] Tasks completed.")