def getDirectories(structure): if structure == "ranked": return jsonify(Directory.dirs_ranked) elif structure == "resolved": dirs = {} buffered_network_paths = Config.get("buffered_network_paths", {}) buffered_network_hosts = Config.get("buffered_network_hosts", {}) for dir in Directory.dirs_flat: if isinstance(dir, str) and dir not in buffered_network_paths: smb, file, unc = Filesystem.networkpath(Directory.dirs_flat[dir]) host = Filesystem.get_host_from_url(smb) buffered_network_paths[dir] = smb Config.set("buffered_network_paths." + dir, smb) buffered_network_hosts[dir] = host Config.set("buffered_network_hosts." + dir, host) dirs[dir] = buffered_network_paths[dir] return jsonify(dirs) else: return jsonify(Directory.dirs_flat)
def plot(self, uids, name): dot = Digraph(name="Produksjonssystem", format="png") dot.graph_attr["bgcolor"] = "transparent" node_ranks = {} for rank in Directory.dirs_ranked: node_ranks[rank["id"]] = [] # remember edges so that we don't plot them twice edges = {} for uid in uids: pipeline = None for p in self.pipelines: if p[0].uid == uid: pipeline = p break if not pipeline: continue group_pipeline = pipeline[0].get_current_group_pipeline() title = group_pipeline.get_group_title() pipeline_id = group_pipeline.get_group_id() # re.sub(r"[^a-z\d]", "", title.lower()) queue = group_pipeline.get_queue() queue_created = len([book for book in queue if Pipeline.get_main_event(book) == "created"]) if queue else 0 queue_deleted = len([book for book in queue if Pipeline.get_main_event(book) == "deleted"]) if queue else 0 queue_modified = len([book for book in queue if Pipeline.get_main_event(book) == "modified"]) if queue else 0 queue_triggered = len([book for book in queue if Pipeline.get_main_event(book) == "triggered"]) if queue else 0 queue_autotriggered = len([book for book in queue if Pipeline.get_main_event(book) == "autotriggered"]) if queue else 0 queue_string = [] if queue_created: queue_string.append("nye:"+str(queue_created)) if queue_modified: queue_string.append("endret:"+str(queue_modified)) if queue_deleted: queue_string.append("slettet:"+str(queue_deleted)) if queue_triggered: queue_string.append("trigget:"+str(queue_triggered)) if queue_autotriggered: queue_string.append("autotrigget:"+str(queue_autotriggered)) queue_string = ", ".join(queue_string) queue_size = 0 if queue: queue_size = len(queue) if not group_pipeline.should_handle_autotriggered_books(): queue_size -= queue_autotriggered book = Metadata.pipeline_book_shortname(group_pipeline) relpath_in = None netpath_in = "" rank_in = None if pipeline[0].dir_in: for rank in Directory.dirs_ranked: for dir in rank["dirs"]: if os.path.normpath(pipeline[0].dir_in) == os.path.normpath(rank["dirs"][dir]): rank_in = rank["id"] break if pipeline[0].dir_in and not pipeline[0].dir_base: relpath_in = os.path.basename(os.path.dirname(pipeline[0].dir_in)) elif pipeline[0].dir_in and pipeline[0].dir_base: base_path = Filesystem.get_base_path(pipeline[0].dir_in, pipeline[0].dir_base) relpath_in = os.path.relpath(pipeline[0].dir_in, base_path) if "master" in pipeline[0].dir_base and pipeline[0].dir_base["master"] == base_path: pass else: if pipeline[0].dir_in not in self.buffered_network_paths: smb, file, unc = Filesystem.networkpath(pipeline[0].dir_in) host = Filesystem.get_host_from_url(smb) self.buffered_network_paths[pipeline[0].dir_in] = smb self.buffered_network_hosts[pipeline[0].dir_in] = host netpath_in = self.buffered_network_hosts[pipeline[0].dir_in] if not netpath_in: netpath_in = self.buffered_network_paths[pipeline[0].dir_in] book_count_in = self.get_book_count(pipeline[0].dir_in) label_in = "< <font point-size='24'>{}</font>{}{} >".format( relpath_in, "\n<br/><i><font point-size='20'>{} {}</font></i>".format(book_count_in, "bok" if book_count_in == 1 else "bøker"), "\n<br/><i><font point-size='20'>{}</font></i>".format(netpath_in.replace("\\", "\\\\")) if netpath_in else "") relpath_out = None netpath_out = "" rank_out = None if pipeline[0].dir_out: for rank in Directory.dirs_ranked: for dir in rank["dirs"]: if os.path.normpath(pipeline[0].dir_out) == os.path.normpath(rank["dirs"][dir]): rank_out = rank["id"] break if pipeline[0].dir_out and not pipeline[0].dir_base: relpath_out = os.path.basename(os.path.dirname(pipeline[0].dir_out)) elif pipeline[0].dir_out and pipeline[0].dir_base: base_path = Filesystem.get_base_path(pipeline[0].dir_out, pipeline[0].dir_base) relpath_out = os.path.relpath(pipeline[0].dir_out, base_path) if "master" in pipeline[0].dir_base and pipeline[0].dir_base["master"] == base_path: pass else: if pipeline[0].dir_out not in self.buffered_network_paths: smb, file, unc = Filesystem.networkpath(pipeline[0].dir_out) host = Filesystem.get_host_from_url(smb) self.buffered_network_paths[pipeline[0].dir_out] = unc self.buffered_network_hosts[pipeline[0].dir_out] = host netpath_out = self.buffered_network_hosts[pipeline[0].dir_out] if not netpath_out: netpath_out = self.buffered_network_paths[pipeline[0].dir_out] book_count_out = self.get_book_count(pipeline[0].dir_out, pipeline[0].parentdirs) label_out = "< <font point-size='24'>{}</font>{}{} >".format( relpath_out, "\n<br/><i><font point-size='20'>{} {}</font></i>".format(book_count_out, "bok" if book_count_out == 1 else "bøker"), "\n<br/><i><font point-size='20'>{}</font></i>".format(netpath_out.replace("\\", "\\\\")) if netpath_out else "") if rank_out: node_ranks[rank_out].append(pipeline_id) elif rank_in: next_rank = self.next_rank(rank_in) if next_rank: node_ranks[next_rank].append(pipeline_id) else: node_ranks[rank_in].append(pipeline_id) state = group_pipeline.get_state() status = group_pipeline.get_status() progress_text = group_pipeline.get_progress() pipeline_label = "< <font point-size='26'>{}</font>{} >".format( title, "".join(["\n<br/><i><font point-size='22'>{}</font></i>".format(val) for val in [queue_string, progress_text, status] if val])) fillcolor = "lightskyblue1" if book or queue_size: fillcolor = "lightslateblue" elif state == "considering": fillcolor = "lightskyblue3" elif not group_pipeline.running: fillcolor = "white" elif isinstance(group_pipeline, DummyPipeline): fillcolor = "snow" dot.attr("node", shape="box", style="filled", fillcolor=fillcolor) dot.node(pipeline_id, pipeline_label.replace("\\", "\\\\")) if relpath_in: fillcolor = "wheat" if not pipeline[0].dir_in_obj or not pipeline[0].dir_in_obj.is_available(): fillcolor = "white" dot.attr("node", shape="folder", style="filled", fillcolor=fillcolor) dot.node(pipeline[1], label_in) if pipeline[1] not in edges: edges[pipeline[1]] = [] if pipeline_id not in edges[pipeline[1]]: edges[pipeline[1]].append(pipeline_id) dot.edge(pipeline[1], pipeline_id) node_ranks[rank_in].append(pipeline[1]) if relpath_out: fillcolor = "wheat" if not pipeline[0].dir_out_obj or not pipeline[0].dir_out_obj.is_available(): fillcolor = "white" dot.attr("node", shape="folder", style="filled", fillcolor=fillcolor) dot.node(pipeline[2], label_out) if pipeline_id not in edges: edges[pipeline_id] = [] if pipeline[2] not in edges[pipeline_id]: edges[pipeline_id].append(pipeline[2]) dot.edge(pipeline_id, pipeline[2]) node_ranks[rank_out].append(pipeline[2]) for rank in node_ranks: subgraph = Digraph("cluster_" + rank, graph_attr={"style": "dotted"}) subgraph.graph_attr["bgcolor"] = "#FFFFFFAA" if node_ranks[rank]: subgraph.attr("node", shape="none", style="filled", fillcolor="transparent") subgraph.node("_ranklabel_" + rank, "< <i><font point-size='28'>{}</font></i> >".format(" <br/>".join(str(self.rank_name(rank)).split(" ")))) for dir in node_ranks[rank]: subgraph.node(dir) dot.subgraph(subgraph) dot.render(os.path.join(self.report_dir, name + "_")) # there seems to be some race condition when doing this across a mounted network drive, # so if we get an exception we retry a few times and hope that it works. # see: https://github.com/nlbdev/produksjonssystem/issues/81 for t in reversed(range(10)): try: shutil.copyfile(os.path.join(self.report_dir, name + "_.png"), os.path.join(self.report_dir, name + ".png")) with open(os.path.join(self.report_dir, name + ".js"), "w") as javascript_file: javascript_file.write("setTime(\"{}\");".format(datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"))) break except Exception as e: logging.debug(" Unable to copy plot image: {}".format(os.path.join(self.report_dir, name + "_.png"))) time.sleep(0.5) if t == 0: raise e dashboard_file = os.path.join(self.report_dir, name + ".html") if not os.path.isfile(dashboard_file): dashboard_template = os.path.normpath(os.path.join(os.path.dirname(os.path.realpath(__file__)), '../../dashboard.html')) if not os.path.exists(self.report_dir): os.makedirs(self.report_dir) shutil.copyfile(dashboard_template, dashboard_file)