def process_msg(self, msg): try: def func(k): return msg["keys"].get(k, 0) status_key = max(msg["keys"], key=func) typ = self.worker.types.get(status_key, object).__name__ keyname = key_split(status_key) d = { "nbytes": msg["total"], "duration": msg["duration"], "bandwidth": msg["bandwidth"], "count": len(msg["keys"]), "type": typ, "type-color": color_of(typ), "key": keyname, "key-color": color_of(keyname), "start": msg["start"], "stop": msg["stop"], } return d except Exception as e: logger.exception(e) raise
def transition(self, key, start, finish, *args, **kwargs): """Run whenever a task changes state Parameters ---------- key : string start : string Start state of the transition. One of released, waiting, processing, memory, error. finish : string Final state of the transition. *args, **kwargs : More options passed when transitioning This may include worker ID, compute time, etc. """ if key not in self.scheduler.tasks: return kwargs["key"] = key startstops = kwargs.get("startstops", []) for startstop in startstops: color = colors[startstop["action"]] if type(color) is not str: color = color(kwargs) data = { "key": key, "name": key_split(key), "color": color, **kwargs, **startstop, } self.socket.send("transition", data)
def task_stream_append(lists, msg, workers): key = msg["key"] name = key_split(key) startstops = msg.get("startstops", []) for startstop in startstops: color = colors[startstop["action"]] if type(color) is not str: color = color(msg) lists["start"].append( (startstop["start"] + startstop["stop"]) / 2 * 1000) lists["duration"].append(1000 * (startstop["stop"] - startstop["start"])) lists["key"].append(key) lists["name"].append(prefix[startstop["action"]] + name) lists["color"].append(color) lists["alpha"].append(alphas[startstop["action"]]) lists["worker"].append(msg["worker"]) worker_thread = "%s-%d" % (msg["worker"], msg["thread"]) lists["worker_thread"].append(worker_thread) if worker_thread not in workers: workers[worker_thread] = len(workers) / 2 lists["y"].append(workers[worker_thread]) return len(startstops)
def task_status(self): tasks = self.client.get_task_stream(start=time() - 10) task_keys = [key_split(t['key']) for t in tasks] task_counts = dict() for k in task_keys: if k not in task_counts: task_counts[k] = 0 task_counts[k] += 1 return task_counts
def rectangles(msgs, workers=None, start_boundary=0): if workers is None: workers = {} L_start = [] L_duration = [] L_duration_text = [] L_key = [] L_name = [] L_color = [] L_alpha = [] L_worker = [] L_worker_thread = [] L_y = [] for msg in msgs: key = msg["key"] name = key_split(key) startstops = msg.get("startstops", []) try: worker_thread = "%s-%d" % (msg["worker"], msg["thread"]) except Exception: continue logger.warning("Message contained bad information: %s", msg, exc_info=True) worker_thread = "" if worker_thread not in workers: workers[worker_thread] = len(workers) / 2 for startstop in startstops: if startstop["start"] < start_boundary: continue color = colors[startstop["action"]] if type(color) is not str: color = color(msg) L_start.append((startstop["start"] + startstop["stop"]) / 2 * 1000) L_duration.append(1000 * (startstop["stop"] - startstop["start"])) L_duration_text.append( format_time(startstop["stop"] - startstop["start"])) L_key.append(key) L_name.append(prefix[startstop["action"]] + name) L_color.append(color) L_alpha.append(alphas[startstop["action"]]) L_worker.append(msg["worker"]) L_worker_thread.append(worker_thread) L_y.append(workers[worker_thread]) return { "start": L_start, "duration": L_duration, "duration_text": L_duration_text, "key": L_key, "name": L_name, "color": L_color, "alpha": L_alpha, "worker": L_worker, "worker_thread": L_worker_thread, "y": L_y, }
def color_of_message(msg): if msg["status"] == "OK": split = key_split(msg["key"]) return color_of(split) else: return "black"