def load( cls: Type["RelationDAG"], database: Database, extend_relations: List[Relation], ignore_relations: List[Relation], ignore_tables: List[Table], ) -> "RelationDAG": """ Create a RelationDAG The data loaded from this method is sourced from the database and from the user config """ graph = DiGraph(name="RelationDAG") # Get actual table instances tables = database.tables.__dict__.values() # Create relations from table data and add the ones specified in settings relations = Relation.from_tables(tables) + extend_relations # Create graph graph.add_nodes_from(tables) graph.add_edges_from([r.edge for r in relations]) # Remove excluded entities (tables and relations) from the created graph graph.remove_edges_from([r.edge for r in ignore_relations]) graph.remove_nodes_from(ignore_tables) # Create RelationDAG instance return cls.from_graph(graph)
def get_inheritance_graph(self, profiles=None): """ Determine the class inheritance hierarchy (class definition needs to adhere to strict inheritance hierarchy) :return: g - A networkx DiGraph of the class hierarchy, with a common ancestor __root__ """ # Determine class inheritance hierarchy (bfs on a directed graph) if not profiles: log.info(f"No profiles specified - using all profiles for ORM.") elif not isinstance(profiles, Iterable): profiles = (profiles, ) g = DiGraph() g.add_node("__root__") class_list = list(self.session.query(CIMClass).all()) classes = {} for c in class_list: if (c.namespace.short, c.name) in classes: raise ValueError("Duplicate class identity: %s_%s." % (c.namespace.short, c.name)) classes[(c.namespace.short, c.name)] = c nodes = classes.keys() g.add_nodes_from(nodes) for key, instance in classes.items(): if instance: parent = instance.parent if parent is None: g.add_edge("__root__", key) else: parent_key = (parent.namespace.short, parent.name) g.add_edge(parent_key, key) return g, classes
def map(self): if not self.g: g = DiGraph() classnames = [ _[0] for _ in self.session.query(CIMClass.name).all() ] classes = self.session.query(CIMClass).all() enums = self.session.query(CIMEnum).all() enumnames = [_[0] for _ in self.session.query(CIMEnum.name).all()] propnames = [_[0] for _ in self.session.query(CIMProp.name).all()] g.add_nodes_from(classnames) g.add_nodes_from(enumnames) g.add_nodes_from(propnames) for node in classes + enums: try: for prop in node.all_props.values(): if prop.range: g.add_edge(node.name, prop.range.name, label=prop.label) else: g.add_edge(node.name, prop.name, label=prop.label) except AttributeError: pass self.g = g return self.g
def filter_non_dependencies(nodes, get_deps_func): node_set = set(nodes) G = DiGraph() G.add_nodes_from(nodes) # process the edges based on the dependency function for n in G: deps = get_deps_func(n) logging.info('%s depends on %s' % (n, deps)) for d in deps: if d in G: G.add_edge(n, d) # now filter the nodes and return them filtered_pkgs = { node for node, in_degree in G.in_degree_iter() if in_degree == 0 } # now find any strongly connected components with size greater than 1 # these will all have in degree > 0, but should still be included glist = [ g for g in strongly_connected_component_subgraphs(G, copy=False) if g.number_of_nodes() > 1 ] for g in glist: # only counts if it was the original list nodes = [n for n in g.nodes() if n in node_set] if len(nodes) > 0: logging.debug('Strongly connected component: %s' % repr(nodes)) for n in nodes: filtered_pkgs.add(n) return filtered_pkgs
def check_pipeline_structure(cls, values): graph = DiGraph() graph.add_nodes_from([v.node_id for v in values.get('nodes')]) graph.add_edges_from([(e.from_, e.to) for e in values.get('links')]) validate_pipeline_structure(graph) return values
def internalize_calls(cfg: LocalGraph) -> LocalGraph: """ Transform external callees into symbolic internal nodes. A symbolic node will bear an identifier and a single label, both equal to the callee's symbolic name. Of course, these new nodes will be isolated from the rest of the graph. Therefore, this method is of practical use only when the user is planning an attempt at name resolution by modifying the internal graph. :param cfg: a local graph :return: a new local graph, with all external calls converted into symbolic nodes """ # Gather all the callees' names external_nodes_ids = set(map(attrgetter('callee'), cfg.external_calls)) # Create a new local graph containing only the symbolic nodes foreign_graph = DiGraph() foreign_graph.add_nodes_from((i, { 'labels': [i], 'external': True }) for i in external_nodes_ids) external = LocalGraph(external_nodes_ids, foreign_graph, [], external_nodes_ids) # Merge the new graph with the original and return the result return cfg.merge(external)
def find_dependents(zip, dependency_regex, entry_points, targets): from networkx import DiGraph from networkx.algorithms.simple_paths import all_simple_paths nodes = zip.namelist() dependency_graph = DiGraph() dependency_graph.add_nodes_from(nodes) for node in nodes: with zip.open(node, 'r') as zipped_file: content = zipped_file.read() for match in dependency_regex.finditer(content): dependency_graph.add_edge(node, match.group()) dependency_graph.add_edge(node, match.group() + 'bin') dependents = set() actual_targets = set() for source in entry_points: for target in targets: for path in all_simple_paths(dependency_graph, source, target): actual_targets.add(path[-1]) for dependent in path[0:-1]: dependents.add(dependent) return list(dependents), list(actual_targets)
def _graph(candidates, pairs): """ helper function for run(), evaluates winner of pairs given ordered pairs slower than _faster_comp(), produces same results """ # This is the standard graph based way to do ranked pairs g = DiGraph() g.add_nodes_from(candidates) # The strongest victories are added in order, unless there is a cycle, # in which case they are skipped edges = set() for (i, j) in pairs: if (j, i) not in edges: g.add_edge(i, j) # if a cycle exists, the edge is removed, otherwise continue try: find_cycle(g) g.remove_edge(i, j) except no_cycle: pass edges.add((i, j)) # We then find the source of the graph, which is the winner winners = set() for c in candidates: try: next(g.in_edges(c).__iter__())[0] except StopIteration: winners.add(c) return winners
def __init__(self, ligands): graph = DiGraph() graph.add_nodes_from(ligands) for i in range(len(ligands)-1): graph.add_edge(ligands[i], ligands[i+1]) self.graph = graph self.ligands = ligands
def ad_graph(f): # asynchronous dynamics as graph adG = DiGraph() adf = sd_to_ad(f) adG.add_nodes_from(f.keys()) adG.add_edges_from([(k, v) for k in adf.keys() for v in adf[k]]) return adG
def reduce_digraphs(ga, gb): """ ga : a -> r, DiGraph gb : b -> r, DiGraph Gout : a -> b, DiGraph """ # identify non empty neighborhoods of ga, i.e. a a_left = [left for left in ga.nodes() if ga.neighbors(left)] # identify non empty neighborhoods of ga, i.e. a b_left = [left for left in gb.nodes() if gb.neighbors(left)] gout = DiGraph() gout.add_nodes_from(a_left) gout.add_nodes_from(b_left) for a in a_left: r_ga = ga.neighbors(a) for b in b_left: r_gb = gb.neighbors(b) r_intersection = list(set(r_ga) & set(r_gb)) ar_vec = array([ga[a][r]['weight'] for r in r_intersection]) br_vec = array([gb[b][r]['weight'] for r in r_intersection]) gout.add_edge(a, b, {'weight': dot(ar_vec, br_vec)}) return gout
def test_get_paths_two_connected_nodes(self): expected = [[1, 2]] graph = DiGraph() graph.add_nodes_from(range(1, 3)) graph.add_edge(1, 2) actual = get_paths(graph) self.assertEqual(expected, actual)
def filter_non_dependencies(nodes, get_deps_func): node_set = set(nodes) G = DiGraph() G.add_nodes_from(nodes) # process the edges based on the dependency function for n in G: deps = get_deps_func(n) logging.info('%s depends on %s' % (n, deps)) for d in deps: if d in G: G.add_edge(n, d) # now filter the nodes and return them filtered_pkgs = {node for node, in_degree in G.in_degree_iter() if in_degree == 0} # now find any strongly connected components with size greater than 1 # these will all have in degree > 0, but should still be included glist = [g for g in strongly_connected_component_subgraphs(G, copy=False) if g.number_of_nodes() > 1] for g in glist: # only counts if it was the original list nodes = [n for n in g.nodes() if n in node_set] if len(nodes) > 0: logging.debug('Strongly connected component: %s' % repr(nodes)) for n in nodes: filtered_pkgs.add(n) return filtered_pkgs
def to_graph(self) -> DiGraph: graph = DiGraph() graph.add_nodes_from([v.id for v in self.nodes]) graph.add_edges_from([(e.from_node_id, e.to_node_id) for e in self.links]) return graph
def add_nodes_from(self, nodes, **attr): H=copy.deepcopy(self) self.clear() if not H.nodes(): DiGraph.add_nodes_from(self, nodes, **attr) self.names=names=sorted(nodes) for i, n in enumerate(self.names): self.node[i]={'name': n, 'pmf': Pmf()} self.node[i]['pmf'].Set(1,self.p) self.node[i]['pmf'].Set(0, 1-self.p) self.remove_node(n) self.edge[i]={} self.indep_vars+=[i] self.SetProbs() return DiGraph.add_nodes_from(self, nodes, **attr) #ind_vars=[var for var in H.indep_vars] #DiGraph.add_nodes_from(self, ind_vars) self.names=names=sorted(set(H.names + nodes)) for i, n in enumerate(names): try: self.node[i], self.edge[i]=H.node[i], H.edge[i] except: self.node[i]={'name': n, 'pmf': Pmf()} self.node[i]['pmf'].Set(1,self.p) self.node[i]['pmf'].Set(0, 1-self.p) self.remove_node(n) self.edge[i]={} self.indep_vars+=[i] self.SetProbs()
def setNetwork(flights): """ Setting network and hierarchy """ aircraftNetwork = DiGraph() aircraftNetwork.add_nodes_from(flights) return aircraftNetwork
def set_node_label(graph: nx.DiGraph, dgxp_file: str = DGXP_FILE, ppi_file: str = PPI_FILE, ppi_columns: str = COLUMNS, sep=SEPARATOR, dgxp_columns=DGXPCOLUMNS, threshold: float = THRESHOLD): """ Set the attribute LABEL of nodes according to the fold-change to +1 or -1. :param dgxp_file: :param sep: :param threshold: :param dgxp_columns: :param graph: NetworkX graph """ gene_to_fc_dict = create_gene_to_fold_change_dict(dgxp_file, ppi_file, ppi_columns, sep, dgxp_columns, threshold) # create a container of tuples to add nodes with label fold change to graph all_tup = [] for node, attr_dict in gene_to_fc_dict.items(): tup = (node, attr_dict) all_tup.append(tup) graph.add_nodes_from(nodes_for_adding=all_tup) for node in graph.nodes(): if graph[node][LABEL] is None: raise KeyError(f"The node {node} has not been labeled.")
def __init__(self, proteins, datatables): graph = DiGraph() graph.add_nodes_from(proteins) self.data = datatables for i in range(len(proteins)-1): graph.add_edge(proteins[i], proteins[i+1]) self.graph = graph self.proteins = proteins
def test_get_paths_with_two_possibilities(self): expected = [[1, 2, 3], [1, 3, 2]] graph = DiGraph() graph.add_nodes_from(range(1, 4)) graph.add_edge(1, 2) graph.add_edge(1, 3) actual = get_paths(graph) self.assertEqual(expected, actual)
def load_graph(input_stream: TextIO) -> DiGraph: graph = DiGraph() n = int(input_stream.readline()) graph.add_nodes_from(range(1, n + 1)) for _ in range(int(input_stream.readline())): edge_src, edge_dest = input_stream.readline().split(' ') graph.add_edge(int(edge_src), int(edge_dest)) return graph
def return_graph(self): index = 0 digraph = DiGraph() digraph.add_nodes_from([i for i in self.states]) for i in self.states: for j in self.transition[i].keys(): if self.transition[i][j] is not None: digraph.add_edge(i, j, weight=self.transition[i][j]) return digraph
def init_graph(self): #Represent the graph using adjacency lists g = DiGraph() g.add_nodes_from(self.vocabulary) #The keys (terms) become nodes in the graph for x in self.vocabulary: for y in self.vocabulary: if self.conditional_probability(x,y) >= 0.8 and self.conditional_probability(y,x) < 0.8: g.add_edge(x,y) return g
def update_from_db(self, session): # type: (Session) -> None # Only allow one thread at a time to construct a fresh graph. with self._update_lock: checkpoint, checkpoint_time = self._get_checkpoint(session) if checkpoint == self.checkpoint: self._logger.debug("Checkpoint hasn't changed. Not Updating.") return self._logger.debug("Checkpoint changed; updating!") start_time = datetime.utcnow() user_metadata = self._get_user_metadata(session) groups, disabled_groups = self._get_groups(session, user_metadata) permissions = self._get_permissions(session) group_grants = self._get_group_grants(session) group_service_accounts = self._get_group_service_accounts(session) service_account_grants = all_service_account_permissions(session) nodes = self._get_nodes(groups, user_metadata) edges = self._get_edges(session) edges_without_np_owner = [ (n1, n2) for n1, n2, r in edges if GROUP_EDGE_ROLES[r["role"]] != "np-owner" ] graph = DiGraph() graph.add_nodes_from(nodes) graph.add_edges_from(edges) rgraph = graph.reverse() # We need a separate graph without np-owner edges to construct the mapping of # permissions to users with that grant. permission_graph = DiGraph() permission_graph.add_nodes_from(nodes) permission_graph.add_edges_from(edges_without_np_owner) grants_by_permission = self._get_grants_by_permission( permission_graph, group_grants, service_account_grants, user_metadata) with self.lock: self._graph = graph self._rgraph = rgraph self.checkpoint = checkpoint self.checkpoint_time = checkpoint_time self.user_metadata = user_metadata self._groups = groups self._disabled_groups = disabled_groups self._permissions = permissions self._group_grants = group_grants self._group_service_accounts = group_service_accounts self._service_account_grants = service_account_grants self._grants_by_permission = grants_by_permission duration = datetime.utcnow() - start_time stats.log_rate("graph_update_ms", int(duration.total_seconds() * 1000))
def mock_graph(mock_user_friends): graph = DiGraph() for row in mock_user_friends: user = row["screen_name"] friends = row["friend_names"] graph.add_node(user) graph.add_nodes_from(friends) graph.add_edges_from([(user, friend) for friend in friends]) return graph
def __init__(self, transitions): graph = DiGraph() graph.add_nodes_from(transitions.keys()) for node in graph.nodes: edges = transitions[node] edges = ((node, edge) for edge in edges) graph.add_edges_from(edges) self._graph = graph
def create_entry_and_exit_nodes(graph: nx.DiGraph, items): graph.add_nodes_from([START, END]) items = sorted(items) graph.add_edge(START, items[0]) for node in (set(graph.nodes()) - {START}) - nx.descendants(graph, START): graph.add_edge(START, node) graph.add_edge(items[-1], END) for node in (set(graph.nodes()) - {END}) - nx.ancestors(graph, END): graph.add_edge(node, END)
def build_networkXGraph_from_spaCy_depGraph(sentence): """ Given a spaCy-parsed sentence, return the relative networkXGraph. """ g = DiGraph() tokens = list(sentence) g.add_nodes_from(tokens) _add_edges_from_spaCy_depGraph(g, sentence.root) return g
def build_digraph(fasta_dict, k): """Build a digraph from the data given by the fasta dict, using k and our is_edge predicate. """ d = DiGraph() d.add_nodes_from(fasta_dict.items()) for (s, t) in permutations(fasta_dict.items(), 2): if is_edge(s, t, k): d.add_edge(s, t) return d
def test_get_coord_dict(): g = DiGraph() g.add_nodes_from(['1', '23X', 'X', 'OX', 'A', 'TU']) coord_dict, leftovers = get_coord_dict(g, 'cocotools/coords/pht00_rhesus.tsv') nt.assert_equal(coord_dict, {'1': [20.31, -10.8], '23X': [2.41, -30.15], 'OX': [1.57, -1.8], 'TU': [5.21, 1.8]}) nt.assert_equal(leftovers, ['A', 'X'])
def add_string_to_graph(graph: nx.DiGraph, string: str, sp, bpe_dropout: Optional[float]): num_nodes_before = len(graph) token_indices = sp.encode( string, enable_sampling=bpe_dropout is not None, alpha=bpe_dropout) nodes = list( range(num_nodes_before, num_nodes_before+len(token_indices))) graph.add_nodes_from(nodes) for n1, n2 in zip(nodes[:-1], nodes[1:]): graph.add_edge(n1, n2) return nodes, token_indices
def example_tree(): """creates a tree/networkx.DiGraph of a syntactic parse tree""" tree = DiGraph() tree.add_nodes_from(['S', 'NP-1', 'N-1', 'Jeff', 'VP', 'V', 'ate', 'NP-2', 'D', 'the', 'N-2', 'apple']) tree.add_edges_from([('S', 'NP-1'), ('NP-1', 'N-1'), ('N-1', 'Jeff'), ('S', 'VP'), ('VP', 'V'), ('V', 'ate'), ('VP', 'NP-2'), ('NP-2', 'D'), ('D', 'the'), ('NP-2', 'N-2'), ('N-2', 'apple')]) return tree
def render_trees(parsed_dir_path: Path, target_dir_path: Path, work_distribution: list): for ecli in work_distribution: parsed_files_glob = parsed_dir_path.joinpath(ecli).glob('*.xml') for parsed_file_path in parsed_files_glob: if parsed_file_path.is_file( ) and parsed_file_path.stat().st_size != 0: json_dir_path = target_dir_path.joinpath(ecli) if not json_dir_path.is_dir(): mkdir(str(json_dir_path)) json_file_name = parsed_file_path.name + '.json' json_file_path = json_dir_path.joinpath(json_file_name) # draw_spring(tree) if not json_file_path.is_file( ) or json_file_path.stat().st_size != 0: tree = DiGraph() xml_tree = parse(str(parsed_file_path)) sentence = SENTENCE_XPATH(xml_tree)[0].text nodes = [] edges = [] for xml_node in XML_NODES(xml_tree): lemma = xml_node.get('lemma') pos = xml_node.get('pos') if lemma is None: lemma = '...' node_id = int(xml_node.attrib['id']) node_attributes = {'name': lemma, 'pos': pos} node = (node_id, node_attributes) nodes.append(node) parent_node_id = int(xml_node.getparent().get('id')) edges.append((parent_node_id, node_id)) tree.add_nodes_from(nodes) tree.add_edges_from(edges) tree_json = json_graph.tree_data(tree, root=0) wrapper_json = { 'origin': parsed_file_path.as_uri(), 'sentence': sentence, 'tree': tree_json } with json_file_path.open(mode='wt') as json_file: dump(wrapper_json, json_file, indent=True) info("Rendered parse tree to '{json_file_path}'. ".format( json_file_path=json_file_path)) else: error("Empty or non-existent XML parse tree file at " "'{parsed_file_path}'. ".format( parsed_file_path=parsed_file_path))
def to_directed(self): from networkx import DiGraph G = DiGraph() G.name = self.name G.add_nodes_from(self.n) G.add_edges_from(((u, v, deepcopy(data)) for u, nbrs in self.a for v, data in nbrs.items())) G.graph = deepcopy(self.data) G._nodedata = deepcopy(self._nodedata) G.node = G._nodedata # hack to pass test return G
def update_from_db(self, session): # Only allow one thread at a time to construct a fresh graph. with self.update_lock: checkpoint, checkpoint_time = self._get_checkpoint(session) if checkpoint == self.checkpoint: self.logger.debug("Checkpoint hasn't changed. Not Updating.") return self.logger.debug("Checkpoint changed; updating!") new_graph = DiGraph() new_graph.add_nodes_from(self._get_nodes_from_db(session)) new_graph.add_edges_from(self._get_edges_from_db(session)) rgraph = new_graph.reverse() users = set() groups = set() for (node_type, node_name) in new_graph.nodes(): if node_type == "User": users.add(node_name) elif node_type == "Group": groups.add(node_name) user_metadata = self._get_user_metadata(session) permission_metadata = self._get_permission_metadata(session) service_account_permissions = all_service_account_permissions( session) group_metadata = self._get_group_metadata(session, permission_metadata) group_service_accounts = self._get_group_service_accounts(session) permission_tuples = self._get_permission_tuples(session) group_tuples = self._get_group_tuples(session) disabled_group_tuples = self._get_group_tuples(session, enabled=False) with self.lock: self._graph = new_graph self._rgraph = rgraph self.checkpoint = checkpoint self.checkpoint_time = checkpoint_time self.users = users self.groups = groups self.permissions = { perm.permission for perm_list in itervalues(permission_metadata) for perm in perm_list } self.user_metadata = user_metadata self.group_metadata = group_metadata self.group_service_accounts = group_service_accounts self.permission_metadata = permission_metadata self.service_account_permissions = service_account_permissions self.permission_tuples = permission_tuples self.group_tuples = group_tuples self.disabled_group_tuples = disabled_group_tuples
def post_parse(grammar): # Create `G` G = DiGraph() G.add_nodes_from(grammar.prods) for lhs, rhs in grammar.prods.items(): for tok in rhs: if tok in grammar.prods: G.add_edge(lhs, tok) # DEBUG: from ._debug import Drawer # DEBUG # DEBUG: drawer = Drawer(G, grammar.start) # DEBUG # Inlining for root, _ in list(bfs_edges(G, grammar.start)): while True: nodes = [d for _, d in bfs_edges(G, root)] nodes = [root] + nodes edges = [] for n in nodes: edges.extend(G.edges([n])) for ns, nd in reversed(edges): # DEBUG: drawer.draw(G, (ns, nd)) # DEBUG # Skip if `nd` has a self-loop if G.has_edge(nd, nd): continue # Skip if `C` consists of multiple nodes g = G.subgraph(n for n in G.nodes_iter() if n != ns) if len(next((c for c in sccs(g) if nd in c))) != 1: continue # Update grammar expr = [] alter = Token.alter() for tok in grammar.prods[ns]: expr.append(tok) if tok == nd: expr.extend(list(grammar.prods[nd]) + [alter]) grammar.prods[ns] = Expr(expr) # Update G G.remove_edge(ns, nd) for _, dst in G.edges_iter(nd): G.add_edge(ns, dst) # DEBUG: drawer.draw(G) # DEBUG break # Back to `for ns, nd in ...` else: # DEBUG: drawer.draw(G) # DEBUG break # Back to `for root, _ in ...` return {nd for _, nd in G.edges_iter()} # Unexpanded nonterminals
def to_directed(self): from networkx import DiGraph G=DiGraph() G.name=self.name G.add_nodes_from(self) G.add_edges_from( ((u,v,deepcopy(data)) for u,nbrs in self.adjacency_iter() for v,data in nbrs.iteritems()) ) G.graph=deepcopy(self.graph) G.node=deepcopy(self.node) return G
def render_trees(parsed_dir_path: Path, target_dir_path: Path, work_distribution: list): for ecli in work_distribution: parsed_files_glob = parsed_dir_path.joinpath(ecli).glob('*.xml') for parsed_file_path in parsed_files_glob: if parsed_file_path.is_file() and parsed_file_path.stat( ).st_size != 0: json_dir_path = target_dir_path.joinpath(ecli) if not json_dir_path.is_dir(): mkdir(str(json_dir_path)) json_file_name = parsed_file_path.name + '.json' json_file_path = json_dir_path.joinpath(json_file_name) # draw_spring(tree) if not json_file_path.is_file() or json_file_path.stat( ).st_size != 0: tree = DiGraph() xml_tree = parse(str(parsed_file_path)) sentence = SENTENCE_XPATH(xml_tree)[0].text nodes = [] edges = [] for xml_node in XML_NODES(xml_tree): lemma = xml_node.get('lemma') pos = xml_node.get('pos') if lemma is None: lemma = '...' node_id = int(xml_node.attrib['id']) node_attributes = {'name': lemma, 'pos': pos} node = (node_id, node_attributes) nodes.append(node) parent_node_id = int(xml_node.getparent().get('id')) edges.append((parent_node_id, node_id)) tree.add_nodes_from(nodes) tree.add_edges_from(edges) tree_json = json_graph.tree_data(tree, root=0) wrapper_json = {'origin': parsed_file_path.as_uri(), 'sentence': sentence, 'tree': tree_json} with json_file_path.open(mode='wt') as json_file: dump(wrapper_json, json_file, indent=True) info("Rendered parse tree to '{json_file_path}'. ".format( json_file_path=json_file_path)) else: error("Empty or non-existent XML parse tree file at " "'{parsed_file_path}'. ".format( parsed_file_path=parsed_file_path))
def to_directed(self): from networkx import DiGraph G = DiGraph() G.name = self.name G.add_nodes_from(self.n) G.add_edges_from(((u, v, deepcopy(data)) for u, nbrs in self.a for v, data in nbrs.items())) G.graph = deepcopy(self.data) G._nodedata = deepcopy(self._nodedata) G.node = G._nodedata # hack to pass test return G
def add_nodes_from(self, nodes, **attr): H=DiGraph() H.add_nodes_from(self.names) h_names=sorted(H.nodes()) H.add_edges_from([(h_names[e[0]], h_names[e[1]], self.edge[e[0]][e[1]]) for e in self.edges()]) causes={h_names[v]: {h_names[item]: self.node[v]['causes'][item] for item in self.node[v]['causes']} for v in self.dep_vars} self.clear() self.indep_vars=[] self.dep_vars=[] if not H.nodes(): DiGraph.add_nodes_from(self, nodes, **attr) self.names=names=sorted(nodes) for i, n in enumerate(self.names): self.node[i]={'name': n, 'pmf': Pmf()} self.node[i]['pmf'].Set(1,self.p) self.node[i]['pmf'].Set(0, 1-self.p) self.remove_node(n) self.edge[i]={} self.indep_vars+=[i] self.SetProbs() return #DiGraph.add_nodes_from(self, nodes, **attr) #ind_vars=[var for var in H.indep_vars] #DiGraph.add_nodes_from(self, ind_vars) self.names=names=sorted(set(H.nodes() + nodes)) for i, n in enumerate(names): if n in H.nodes(): self.node[i], self.edge[i]=H.node[n], {names.index(item): H.edge[n][item] for item in H.edge[n]} self.node[i]['causes']={names.index(item): causes[n][item] for item in causes[n]} if n in causes else {} self.node[i]['name']=n self.node[i]['pmf']=Pmf() if not self.node[i]['causes']: self.node[i]['pmf'].Set(1,self.p) self.node[i]['pmf'].Set(0, 1-self.p) self.indep_vars+=[i] else: self.dep_vars+=[i] else: self.node[i]={'name': n, 'pmf': Pmf()} self.node[i]['pmf'].Set(1,self.p) self.node[i]['pmf'].Set(0, 1-self.p) #self.remove_node(n) self.edge[i]={} self.indep_vars+=[i] self.SetProbs()
def update_from_db(self, session): # Only allow one thread at a time to construct a fresh graph. with self.update_lock: checkpoint, checkpoint_time = self._get_checkpoint(session) if checkpoint == self.checkpoint: self.logger.debug("Checkpoint hasn't changed. Not Updating.") return self.logger.debug("Checkpoint changed; updating!") new_graph = DiGraph() new_graph.add_nodes_from(self._get_nodes_from_db(session)) new_graph.add_edges_from(self._get_edges_from_db(session)) rgraph = new_graph.reverse() users = set() groups = set() for (node_type, node_name) in new_graph.nodes(): if node_type == "User": users.add(node_name) elif node_type == "Group": groups.add(node_name) user_metadata = self._get_user_metadata(session) permission_metadata = self._get_permission_metadata(session) service_account_permissions = all_service_account_permissions(session) group_metadata = self._get_group_metadata(session, permission_metadata) group_service_accounts = self._get_group_service_accounts(session) permission_tuples = self._get_permission_tuples(session) group_tuples = self._get_group_tuples(session) disabled_group_tuples = self._get_group_tuples(session, enabled=False) with self.lock: self._graph = new_graph self._rgraph = rgraph self.checkpoint = checkpoint self.checkpoint_time = checkpoint_time self.users = users self.groups = groups self.permissions = {perm.permission for perm_list in permission_metadata.values() for perm in perm_list} self.user_metadata = user_metadata self.group_metadata = group_metadata self.group_service_accounts = group_service_accounts self.permission_metadata = permission_metadata self.service_account_permissions = service_account_permissions self.permission_tuples = permission_tuples self.group_tuples = group_tuples self.disabled_group_tuples = disabled_group_tuples
def to_directed(self): """Return a directed representation of the graph. A new directed graph is returned with the same name, same nodes, and with each edge (u,v,data) replaced by two directed edges (u,v,data) and (v,u,data). """ from networkx import DiGraph G=DiGraph() G.add_nodes_from(self) G.add_edges_from( ((u,v,data) for u,nbrs in self.adjacency_iter() for v,data in nbrs.iteritems()) ) return G
def update_from_db(self, session): # type: (Session) -> None # Only allow one thread at a time to construct a fresh graph. with self._update_lock: checkpoint, checkpoint_time = self._get_checkpoint(session) if checkpoint == self.checkpoint: self._logger.debug("Checkpoint hasn't changed. Not Updating.") return self._logger.debug("Checkpoint changed; updating!") start_time = datetime.utcnow() user_metadata = self._get_user_metadata(session) groups, disabled_groups = self._get_groups(session, user_metadata) permissions = self._get_permissions(session) group_grants = self._get_group_grants(session) group_service_accounts = self._get_group_service_accounts(session) service_account_grants = all_service_account_permissions(session) graph = DiGraph() graph.add_nodes_from(self._get_nodes(groups, user_metadata)) graph.add_edges_from(self._get_edges(session)) rgraph = graph.reverse() grants_by_permission = self._get_grants_by_permission( graph, group_grants, service_account_grants ) with self.lock: self._graph = graph self._rgraph = rgraph self.checkpoint = checkpoint self.checkpoint_time = checkpoint_time self.user_metadata = user_metadata self._groups = groups self._disabled_groups = disabled_groups self._permissions = permissions self._group_grants = group_grants self._group_service_accounts = group_service_accounts self._service_account_grants = service_account_grants self._grants_by_permission = grants_by_permission duration = datetime.utcnow() - start_time stats.log_rate("graph_update_ms", int(duration.total_seconds() * 1000))
def update_from_db(self, session): checkpoint, checkpoint_time = self._get_checkpoint(session) if checkpoint == self.checkpoint: logging.debug("Checkpoint hasn't changed. Not Updating.") return logging.debug("Checkpoint changed; updating!") new_graph = DiGraph() new_graph.add_nodes_from(self._get_nodes_from_db(session)) new_graph.add_edges_from(self._get_edges_from_db(session)) rgraph = new_graph.reverse() users = set() groups = set() for (node_type, node_name) in new_graph.nodes(): if node_type == "User": users.add(node_name) elif node_type == "Group": groups.add(node_name) user_metadata = self._get_user_metadata(session) permission_metadata = self._get_permission_metadata(session) group_metadata = self._get_group_metadata(session, permission_metadata) with self.lock: self._graph = new_graph self._rgraph = rgraph self.checkpoint = checkpoint self.checkpoint_time = checkpoint_time self.users = users self.groups = groups self.permissions = {perm.permission for perm_list in permission_metadata.values() for perm in perm_list} self.user_metadata = user_metadata self.group_metadata = group_metadata self.permission_metadata = permission_metadata
def deterministic_topological_ordering(nodes, links, start_node): """ Topological sort that is deterministic because it sorts (alphabetically) candidates to check """ graph = DiGraph() graph.add_nodes_from(nodes) for link in links: graph.add_edge(*link) if not is_directed_acyclic_graph(graph): raise NetworkXUnfeasible task_names = sorted(graph.successors(start_node)) task_set = set(task_names) graph.remove_node(start_node) result = [start_node] while task_names: for name in task_names: if graph.in_degree(name) == 0: result.append(name) # it is OK to modify task_names because we break out # of loop below task_names.remove(name) new_successors = [t for t in graph.successors(name) if t not in task_set] task_names.extend(new_successors) task_names.sort() task_set.update(set(new_successors)) graph.remove_node(name) break return result
def get_bin_group(grps, bin_grps, rt, graph): level = grps.successors(rt) if not level: bin_grps.append(graph) return bin_grps elif len(level) == 2: graph.add_nodes_from(level) graph.add_edge(rt, level[0], grps.get_edge_data(rt, level[0])) graph.add_edge(rt, level[1], grps.get_edge_data(rt, level[1])) if grps.successors(level[0]): get_bin_group(grps, bin_grps, rt=level[0], graph=graph) else: get_bin_group(grps, bin_grps, rt=level[1], graph=graph) else: level.sort() for i in xrange(0, len(level), 2): g = DiGraph(graph) g.add_nodes_from([level[i], level[i + 1]]) g.add_edge(rt, level[i], grps.get_edge_data(rt, level[i])) g.add_edge(rt, level[i + 1], grps.get_edge_data(rt, level[i + 1])) if grps.successors(level[i]): get_bin_group(grps, bin_grps, rt=level[i], graph=g) else: get_bin_group(grps, bin_grps, rt=level[i + 1], graph=g)
def example_invalid_subtrees(): is1 = DiGraph() is1.add_node('NP-2') is1.add_edges_from([('D', 'the'), ('N-2', 'apple')]) is2 = DiGraph() is2.add_node('D') is3 = DiGraph() is3.add_nodes_from(['D', 'N-2']) is4 = DiGraph() is4.add_edges_from([('NP-2', 'D'), ('D', 'the')]) is5 = DiGraph() is5.add_nodes_from(['the']) is6 = DiGraph() is6.add_nodes_from(['the', 'apple']) is7 = DiGraph() is7.add_edges_from([('the', 'apple')]) return [is1, is2, is3, is4, is5, is6, is7]
def load(self,fname, verbose=True, **kwargs): """ Load a data file. The expected data format is three columns (comma seperated by default) with source, target, flux. No header should be included and the node IDs have to run contuously from 0 to Number_of_nodes-1. Parameters ---------- fname : str Path to the file verbose : bool Print information about the data. True by Default kwargs : dict Default parameters can be changed here. Supported key words are dtype : float (default) delimiter : "," (default) return_graph : bool If True, the graph is returned (False by default). Returns: -------- The graph is saved internally in self.graph. """ delimiter = kwargs["delimiter"] if "delimiter" in kwargs.keys() else " " data = np.genfromtxt(fname, delimiter=delimiter, dtype=int, unpack=False) source, target = data[:,0], data[:,1] if data.shape[1] > 2: flux = data[:,2] else: flux = np.ones_like(source) nodes = set(source) | set(target) self.nodes = len(nodes) lines = len(flux) if set(range(self.nodes)) != nodes: new_node_ID = {old:new for new,old in enumerate(nodes)} map_new_node_ID = np.vectorize(new_node_ID.__getitem__) source = map_new_node_ID(source) target = map_new_node_ID(target) if verbose: print "\nThe node IDs have to run continuously from 0 to Number_of_nodes-1." print "Node IDs have been changed according to the requirement.\n-----------------------------------\n" print 'Lines: ',lines , ', Nodes: ', self.nodes print '-----------------------------------\nData Structure:\n\nsource, target, weight \n' for ii in range(7): print "%i, %i, %1.2e" %(source[ii], target[ii], flux[ii]) print '-----------------------------------\n' G = DiGraph() # Empty, directed Graph G.add_nodes_from(range(self.nodes)) for ii in xrange(lines): u, v, w = int(source[ii]), int(target[ii]), float(flux[ii]) if u != v: # ignore self loops assert not G.has_edge(u,v), "Edge appeared twice - not supported" G.add_edge(u,v,weight=w) else: if verbose: print "ignore self loop at node", u symmetric = True for s,t,w in G.edges(data=True): w1 = G[s][t]["weight"] try: w2 = G[t][s]["weight"] except KeyError: symmetric = False G.add_edge(t,s,weight=w1) w2 = w1 if w1 != w2: symmetric = False G[s][t]["weight"] += G[t][s]["weight"] G[s][t]["weight"] /= 2 G[t][s]["weight"] = G[s][t]["weight"] if verbose: if not symmetric: print "The network has been symmetricised." ccs = strongly_connected_component_subgraphs(G) ccs = sorted(ccs, key=len, reverse=True) G_GSCC = ccs[0] if G_GSCC.number_of_nodes() != G.number_of_nodes(): G = G_GSCC if verbose: print "\n--------------------------------------------------------------------------" print "The network has been restricted to the giant strongly connected component." self.nodes = G.number_of_nodes() for u, v, data in G.edges(data=True): weight = G.out_degree(u,weight='weight') data['transition_rate'] = 1.*data['weight']/weight for u, v, data in G.edges(data=True): data['effective_distance'] = 1. - log(data['transition_rate']) if verbose: print "\n--------------------------------------------------------------------------" print "\nnode ID, out-weight, normalized out-weight, sum of effective distances \n " for ii in range(7): out_edges = G.out_edges(ii, data=True) out_weight, effective_distance, transition_rate = 0, 0, 0 for u, v, data in out_edges: out_weight += data["weight"] effective_distance += data["effective_distance"] transition_rate += data["transition_rate"] print " %i %1.2e %2.3f %1.2e " %(ii,out_weight, transition_rate, effective_distance) print "\n ... graph is saved in self.graph" return G
def synthesize(tsys, exprtab, init_flags='ALL_ENV_EXIST_SYS_INIT'): assert init_flags.upper() == 'ALL_ENV_EXIST_SYS_INIT', 'Only the initial condition interpretation ALL_ENV_EXIST_SYS_INIT is supported.' W, Y_list, X_list = get_winning_set(tsys, return_intermediates=True) initial_states = get_initial_states(W, tsys, exprtab, init_flags) if initial_states is None: return None goalnames = ['SYSGOAL'+str(i) for i in range(tsys.num_sgoals)] for goalmode in range(tsys.num_sgoals): Y_list[goalmode][0] = set([s for s in W if goalnames[goalmode] in tsys.G.node[s]['sat']]) strategy = DiGraph() next_id = len(initial_states) workset = list(range(next_id)) strategy.add_nodes_from([(i, {'state': s, 'mode': 0, 'initial': True}) for (i,s) in enumerate(initial_states)]) while len(workset) > 0: nd = workset.pop() j = 0 while j < len(Y_list[strategy.node[nd]['mode']]): if strategy.node[nd]['state'] in Y_list[strategy.node[nd]['mode']][j]: break j += 1 if j == 0: assert goalnames[strategy.node[nd]['mode']] in tsys.G.node[strategy.node[nd]['state']]['sat'] original_mode = strategy.node[nd]['mode'] while goalnames[strategy.node[nd]['mode']] in tsys.G.node[strategy.node[nd]['state']]['sat']: strategy.node[nd]['mode'] = (strategy.node[nd]['mode'] + 1) % tsys.num_sgoals if strategy.node[nd]['mode'] == original_mode: break if strategy.node[nd]['mode'] != original_mode: repeat_found = False for possible_repeat, attr in strategy.nodes_iter(data=True): if (possible_repeat != nd and attr['mode'] == strategy.node[nd]['mode'] and attr['state'] == strategy.node[nd]['state']): repeat_found = True for (u,v) in strategy.in_edges_iter(nd): strategy.add_edge(u, possible_repeat) strategy.remove_edges_from(strategy.in_edges(nd)) strategy.remove_node(nd) break if repeat_found: continue j = 0 while j < len(Y_list[strategy.node[nd]['mode']]): if strategy.node[nd]['state'] in Y_list[strategy.node[nd]['mode']][j]: break j += 1 if j == 0: assert goalnames[strategy.node[nd]['mode']] in tsys.G.node[strategy.node[nd]['state']]['sat'] for envpost in tsys.envtrans[strategy.node[nd]['state']]: next_state = None for succ_nd in tsys.G.successors(strategy.node[nd]['state']): if (tuple([succ_nd[i] for i in tsys.ind_uncontrolled]) == envpost and ((j > 0 and succ_nd in Y_list[strategy.node[nd]['mode']][j-1]) or (j == 0 and succ_nd in W))): next_state = succ_nd break if next_state is None: assert j > 0 if j == 0: import pdb; pdb.set_trace() blocking_index = None blocking_sets = X_list[strategy.node[nd]['mode']][j-1] for k in range(len(blocking_sets)): if strategy.node[nd]['state'] in blocking_sets[k]: blocking_index = k break assert blocking_index is not None for succ_nd in tsys.G.successors(strategy.node[nd]['state']): if (tuple([succ_nd[i] for i in tsys.ind_uncontrolled]) == envpost and succ_nd in blocking_sets[blocking_index]): next_state = succ_nd break assert next_state is not None foundmatch = False for candidate, cattr in strategy.nodes_iter(data=True): if cattr['state'] == next_state and cattr['mode'] == strategy.node[nd]['mode']: strategy.add_edge(nd, candidate) foundmatch = True break if not foundmatch: if j == 0: new_mode = (strategy.node[nd]['mode'] + 1) % tsys.num_sgoals else: new_mode = strategy.node[nd]['mode'] workset.append(next_id) strategy.add_node(next_id, {'state': next_state, 'mode': new_mode, 'initial': False}) strategy.add_edge(nd, next_id) next_id += 1 return strategy
def calc_reactions(org, fluxResults): """ Adding Results from FLUX-Analysis @param org: organism @param fluxResults: Results from PyNetMet FLUX-Analysis Calculation as String @return Graph with added Attributes """ changeDic = {} reactions = fluxResults.reacs for reaction, flux in zip(reactions, fluxResults.flux): if not reaction.name.endswith("_transp"): changeDic[reaction.name] = float('%.3g' % flux) vList = list(changeDic.values()) for i in range(len(vList)): vList[i] = math.sqrt(math.pow(vList[i], 2)) vList = sorted(vList) oldMin = vList[0] oldMax = vList[-1] oldRange = oldMax - oldMin newMin = 1 newMax = 10 newRange = newMax - newMin enzymes = fluxResults.reacs nodeDic = OrderedDict() nodecounter = 0 graph = DiGraph() objective = None if len(org.obj) > 0: objective = org.obj[0][0] for enzyme in enzymes: if not enzyme.name.endswith("_transp"): nodecounter += 1 nodeDic[enzyme.name] = nodecounter if enzyme.name == objective: graph.add_node(nodeDic[enzyme.name]) for substrate in enzyme.substrates: nodecounter += 1 if substrate not in nodeDic: nodeDic[substrate] = nodecounter attr = {"label": substrate, "shape": "oval", "color": str((nodecounter % 8)+1)} graph.add_node(nodeDic[substrate], **attr) if enzyme.name == objective: graph.add_nodes_from([(nodeDic[substrate], {"shape":"box"})]) for product in enzyme.products: nodecounter += 1 if product not in nodeDic: nodeDic[product] = nodecounter attr = {"label": product, "shape": "oval", "color": str((nodecounter % 8)+1)} graph.add_node(nodeDic[product], **attr) if enzyme.name == objective: graph.add_nodes_from([(nodeDic[product], {"shape":"box"})]) # Check if enzyme is objective for substrate in enzyme.substrates: for product in enzyme.products: flux = changeDic[enzyme.name] color = "black" if flux < 0: color = "red" elif flux > 0: color = "green" value = flux if value != 0: value = (math.sqrt(math.pow(value, 2)) - oldMin) / oldRange * newRange + newMin else: value = 1 attr = {"color": color, "penwidth": value, "label": u"{} ({})".format(enzyme.name, flux), "object": enzyme} if enzyme.name == objective: attr["style"] = "dashed" graph.add_edge(nodeDic[substrate], nodeDic[product], attr) if enzyme.name == objective: graph.add_edge(nodeDic[substrate], nodeDic[enzyme.name]) graph.add_edge(nodeDic[enzyme.name], nodeDic[product]) if enzyme.reversible: graph.add_edge(nodeDic[product], nodeDic[substrate], label=enzyme.name, object=enzyme) return graph, nodeDic
class PulseProgramUi(PulseProgramWidget, PulseProgramBase): pulseProgramChanged = QtCore.pyqtSignal() contextDictChanged = QtCore.pyqtSignal(object) definitionWords = ['counter', 'var', 'shutter', 'parameter', 'masked_shutter', 'trigger', 'address', 'exitcode', 'const'] builtinWords = [] for key, val in SymbolTable().items(): #Extract the builtin words which should be highlighted if type(val).__name__ == 'Builtin': builtinWords.append(key) SourceMode = enum('pp', 'ppp') def __init__(self, config, parameterdict, channelNameData): PulseProgramWidget.__init__(self) PulseProgramBase.__init__(self) self.dependencyGraph = DiGraph() self.pulseProgram = PulseProgram.PulseProgram() self.sourceCodeEdits = dict() self.pppCodeEdits = dict() self.config = config self.variableTableModel = None self.globalVariablesChanged = None self.channelNameData = channelNameData self.pppCompileException = None self.globaldict = parameterdict self.project = getProject() self.defaultPPPDir = self.project.configDir+'/PulseProgramsPlus' if not os.path.exists(self.defaultPPPDir): os.makedirs(self.defaultPPPDir) examplePPPDir = os.path.realpath(os.path.join(os.path.dirname(__file__), '..', 'config/PulseProgramsPlus')) #/IonControl/config/PulseProgramsPlus directory for basename in os.listdir(examplePPPDir): if basename.endswith('.ppp'): pathname = os.path.join(examplePPPDir, basename) if os.path.isfile(pathname): shutil.copy(pathname, self.defaultPPPDir) #Copy over all example PPP pulse programs self.defaultRAMDir = self.project.configDir+'/RAMFiles' if not os.path.exists(self.defaultRAMDir): exampleRAMDir = os.path.realpath(os.path.join(os.path.dirname(__file__), '..', 'config/RAMFiles')) #/IonControl/config/RAMFiles directory shutil.copytree(exampleRAMDir, self.defaultRAMDir) #Copy over all example RAM files def setupUi(self, experimentname, parent): super(PulseProgramUi, self).setupUi(parent) self.setCentralWidget(None) #No central widget self.experimentname = experimentname self.configname = 'PulseProgramUi.'+self.experimentname self.contextDict = self.config.get( self.configname+'.contextdict', dict() ) self.populateDependencyGraph() for context in self.contextDict.values(): # set the global dict as this field does not survive pickling context.setGlobaldict(self.globaldict) self.currentContext = self.config.get(self.configname + '.currentContext', PulseProgramContext(self.globaldict)) self.currentContext.setGlobaldict(self.globaldict) self.configParams = self.config.get(self.configname, ConfiguredParams()) self.currentContextName = self.configParams.lastContextName self.filenameComboBox.addItems([key for key, path in self.configParams.recentFiles.items() if os.path.exists(path)]) self.contextComboBox.addItems(sorted(self.contextDict.keys())) self.parentComboBox.addItems([''] + sorted(self.contextDict.keys())) self.ramFilenameComboBox.addItems([''] + [key for key, path in self.configParams.recentRamFiles.items() if os.path.exists(path)]) self.writeRamCheckbox.setChecked(self.currentContext.writeRam) #setup documentation list definitionDict, builtinDict, encodingDict = self.getDocs() self.addDocs(definitionDict, "Variable Definitions") self.addDocs(builtinDict, "Pulse Program Commands") self.addDocs(encodingDict, "Encodings") #connect actions self.actionOpen.triggered.connect( self.onLoad ) self.actionSave.triggered.connect( self.onSave ) self.actionReset.triggered.connect(self.onReset) self.loadButton.setDefaultAction( self.actionOpen ) self.saveButton.setDefaultAction( self.actionSave ) self.resetButton.setDefaultAction( self.actionReset ) self.loadButtonRam.clicked.connect( self.onLoadRamFile ) self.writeRamCheckbox.clicked.connect( self.onWriteRamCheckbox ) self.shutterTableView.setHorizontalHeader( RotatedHeaderView(QtCore.Qt.Horizontal, self.shutterTableView) ) self.triggerTableView.setHorizontalHeader( RotatedHeaderView(QtCore.Qt.Horizontal, self.triggerTableView ) ) self.counterTableView.setHorizontalHeader( RotatedHeaderView(QtCore.Qt.Horizontal, self.counterTableView ) ) self.reloadContextButton.clicked.connect( self.onReloadContext ) self.saveContextButton.clicked.connect( self.onSaveContext ) self.deleteContextButton.clicked.connect( self.onDeleteContext ) self.contextComboBox.currentIndexChanged[str].connect( self.onLoadContext ) self.parentComboBox.currentIndexChanged[str].connect(self.onSetParent) self.linkAllButton.clicked.connect(self.onLinkAllToParent) self.unlinkAllButton.clicked.connect(self.onUnlinkAllFromParent) self.filenameComboBox.currentIndexChanged[str].connect( self.onFilenameChange ) self.ramFilenameComboBox.currentIndexChanged[str].connect( self.onRamFilenameChange ) self.removeCurrent.clicked.connect( self.onRemoveCurrent ) self.removeCurrentRamFile.clicked.connect( self.onRemoveCurrentRamFile ) self.variableTableModel = VariableTableModel( self.currentContext.parameters, self.config, self.currentContextName ) if self.globalVariablesChanged: self.globalVariablesChanged.connect(self.variableTableModel.recalculateDependent ) self.variableView.setModel(self.variableTableModel) self.variableView.resizeColumnToContents(0) self.variableView.clicked.connect(self.onVariableViewClicked) self.filter = KeyListFilter( [], [QtCore.Qt.Key_B, QtCore.Qt.Key_W, QtCore.Qt.Key_R] ) self.filter.controlKeyPressed.connect( self.onControl ) self.variableView.installEventFilter(self.filter) self.shutterTableModel = ShutterTableModel( self.currentContext.shutters, self.channelNameData[0], size=48, globalDict=self.globaldict, channelSignal=self.channelNameData[1] ) self.triggerTableModel = TriggerTableModel( self.currentContext.triggers, self.channelNameData[2], size=32, globalDict=self.globaldict, channelSignal=self.channelNameData[3] ) self.counterTableModel = CounterTableModel( self.currentContext.counters, self.channelNameData[4], globalDict=self.globaldict ) self.delegates = list() for model, view in [ (self.shutterTableModel, self.shutterTableView), (self.triggerTableModel, self.triggerTableView), (self.counterTableModel, self.counterTableView)]: view.setModel(model) delegate = MagnitudeSpinBoxDelegate(globalDict=self.globaldict) self.delegates.append(delegate) # we need to keep those around, otherwise python will crash. view.setItemDelegateForColumn(model.numericDataColumn, delegate) if model.tristate: view.setItemDelegateForColumn(model.numericMaskColumn, delegate) view.clicked.connect(model.onClicked) view.resizeColumnsToContents() view.setupUi(self.globaldict) if self.globalVariablesChanged: self.globalVariablesChanged.connect( model.recalculateAllDependents ) self.counterIdDelegate = MagnitudeSpinBoxDelegate() self.counterTableView.setItemDelegateForColumn(self.counterTableModel.idColumn, self.counterIdDelegate) try: self.loadContext(self.currentContext) if self.configParams.lastContextName: index = self.contextComboBox.findText(self.configParams.lastContextName) with BlockSignals(self.contextComboBox) as w: w.setCurrentIndex(index) except: logging.getLogger(__name__).exception("Loading of previous context failed") #self.contextComboBox.editTextChanged.connect( self.updateSaveStatus ) self.contextComboBox.lineEdit().editingFinished.connect( self.updateSaveStatus ) self.variableTableModel.contentsChanged.connect( self.updateSaveStatus ) self.counterTableModel.contentsChanged.connect( self.updateSaveStatus ) self.shutterTableModel.contentsChanged.connect( self.updateSaveStatus ) self.triggerTableModel.contentsChanged.connect( self.updateSaveStatus ) self.setContextMenuPolicy( QtCore.Qt.ActionsContextMenu ) self.autoSaveAction = QtWidgets.QAction("Automatically save configuration", self) self.autoSaveAction.setCheckable(True) self.autoSaveAction.setChecked( self.configParams.autoSaveContext ) self.autoSaveAction.triggered.connect( self.onAutoSave ) self.addAction( self.autoSaveAction ) #background color context menu setBackgroundColorAction = QtGui.QAction("Set Background Color", self) setBackgroundColorAction.triggered.connect(self.onSetBackgroundColor) self.addAction(setBackgroundColorAction) removeBackgroundColorAction = QtGui.QAction("Remove Background Color", self) removeBackgroundColorAction.triggered.connect(self.onRemoveBackgroundColor) self.addAction(removeBackgroundColorAction) self.initMenu() self.tabifyDockWidget(self.shutterDock, self.triggerDock) self.tabifyDockWidget(self.triggerDock, self.counterDock) self.restoreLayout() def populateDependencyGraph(self): self.dependencyGraph = DiGraph() self.dependencyGraph.add_nodes_from(self.contextDict.keys()) for name, context in self.contextDict.items(): if context.parentContext is not None: try: self.dependencySetParent(name, context.parentContext) except CyclicDependencyError as ce: context.parentContext = ce.parent def dependencySetParent(self, child, parent): parent = parent if parent else None oldParentEdges = self.dependencyGraph.out_edges([child]) _, oldParent = first(oldParentEdges, (None, None)) if parent != oldParent: self.dependencyGraph.remove_edges_from(oldParentEdges) if parent: self.dependencyGraph.add_edge(child, parent) cycles = simple_cycles(self.dependencyGraph) try: cycle = next(cycles) # StopIteration is raised if there are cycles self.dependencyGraph.remove_edge(child, parent) self.dependencyGraph.add_edges_from(oldParentEdges) raise CyclicDependencyError(oldParent) except StopIteration: pass def restoreLayout(self): """Restore layout from config settings""" windowState = self.config.get(self.configname+".state") if windowState: self.restoreState(windowState) docSplitterState = self.config.get(self.configname+'.docSplitter') if docSplitterState: self.docSplitter.restoreState(docSplitterState) def initMenu(self): self.menuView.clear() dockList = self.findChildren(QtWidgets.QDockWidget) for dock in dockList: self.menuView.addAction(dock.toggleViewAction()) def onAutoSave(self, checked): self.configParams.autoSaveContext = checked if checked: self.onSaveContext() def loadContext(self, newContext ): previousContext = self.currentContext self.currentContext = copy.deepcopy(newContext) #changeMode = self.currentContext.pulseProgramMode != previousContext.pulseProgramMode if self.currentContext.pulseProgramFile != previousContext.pulseProgramFile or len(self.sourceCodeEdits)==0: self.currentContext.pulseProgramFile = self.project.findFile(self.currentContext.pulseProgramFile) self.adaptiveLoadFile(self.currentContext.pulseProgramFile) if self.currentContext.ramFile != previousContext.ramFile or (self.currentContext.ramFile): self.currentContext.ramFile = self.project.findFile(self.currentContext.ramFile) self.loadRamFile(self.currentContext.ramFile) self.mergeVariablesIntoContext( self.pulseProgram.variabledict ) self.updateDisplayContext() self.updateSaveStatus(isSaved=True) def onReloadContext(self): self.loadContext( self.contextDict[str(self.contextComboBox.currentText())] ) self.updateSaveStatus() def onSaveContext(self): name = str(self.contextComboBox.currentText()) isNewContext = not name in self.contextDict self.contextDict[ name ] = copy.deepcopy(self.currentContext) if self.contextComboBox.findText(name)<0: with BlockSignals(self.contextComboBox) as w: w.addItem(name) with BlockSignals(self.parentComboBox) as w: w.addItem(name) if isNewContext: self.contextDictChanged.emit(list(self.contextDict.keys())) self.updateSaveStatus(isSaved=True) self.currentContextName = name def onDeleteContext(self): name = str(self.contextComboBox.currentText()) index = self.contextComboBox.findText(name) if index>=0: self.contextDict.pop(name) self.contextComboBox.removeItem( index ) self.parentComboBox.removeItem(self.parentComboBox.findText(name)) self.contextDictChanged.emit(list(self.contextDict.keys())) self.updateSaveStatus() self.currentContextName = None def onLoadContext(self): name = str(self.contextComboBox.currentText()) self.currentContextName = name if name in self.contextDict: self.loadContext( self.contextDict[name] ) else: self.onSaveContext() def onSetParent(self, parent): try: parent = parent if parent else None self.dependencySetParent(self.currentContextName, parent) self.currentContext.parentContext = parent self.variableTableModel.setContextHasParent(bool(parent)) except CyclicDependencyError as ce: parent = ce.parent self.currentContext.parentContext = parent self.parentComboBox.setCurrentIndex(self.parentComboBox.findText(parent if parent else '')) self.currentContext.parentContext = parent self.setParentData(self.currentContext.parentContext) def loadContextByName(self, name): if name in self.contextDict: self.loadContext( self.contextDict[name] ) with BlockSignals(self.contextComboBox) as w: w.setCurrentIndex( w.findText( name )) def updatepppDisplay(self): for pppTab in list(self.pppCodeEdits.values()): self.sourceTabs.removeTab( self.sourceTabs.indexOf(pppTab) ) self.pppCodeEdits = dict() if self.currentContext.pulseProgramMode == 'ppp': for name, text in [(self.pppSourceFile, self.pppSource)]: textEdit = PulseProgramSourceEdit(mode='ppp') encodingStrings = [encoding for encoding in EncodingDict.keys() if type(encoding) == str] textEdit.setupUi(textEdit, extraKeywords1=self.definitionWords+encodingStrings, extraKeywords2=self.builtinWords) textEdit.setPlainText(text) self.pppCodeEdits[name] = textEdit self.sourceTabs.addTab( textEdit, name ) def updateppDisplay(self): for pppTab in list(self.sourceCodeEdits.values()): self.sourceTabs.removeTab( self.sourceTabs.indexOf(pppTab) ) self.sourceCodeEdits = dict() for name, text in self.pulseProgram.source.items(): textEdit = PulseProgramSourceEdit() textEdit.setupUi(textEdit, extraKeywords1=self.definitionWords, extraKeywords2=[key for key in OPS]) textEdit.setPlainText(text) self.sourceCodeEdits[name] = textEdit self.sourceTabs.addTab( textEdit, name ) textEdit.setReadOnly( self.currentContext.pulseProgramMode!='pp' ) def updateDisplayContext(self): self.setParentData(self.currentContext.parentContext) self.variableTableModel.setVariables(self.currentContext.parameters, self.currentContextName) self.variableTableModel.setContextHasParent(bool(self.currentContext.parentContext)) with BlockSignals(self.parentComboBox) as w: w.setCurrentIndex(self.parentComboBox.findText(self.currentContext.parentContext if self.currentContext.parentContext else '')) self.variableView.resizeColumnsToContents() self.shutterTableModel.setDataDict(self.currentContext.shutters) self.triggerTableModel.setDataDict(self.currentContext.triggers) self.counterTableModel.setDataDict(self.currentContext.counters) self.writeRamCheckbox.setChecked(self.currentContext.writeRam) def documentationString(self): messages = [ "PulseProgram {0}".format( self.configParams.lastLoadFilename ) ] r = "\n".join(messages) return "\n".join( [r, self.pulseProgram.currentVariablesText()]) def description(self): desc = dict() desc["PulseProgram"] = self.configParams.lastLoadFilename desc.update( self.pulseProgram.variables() ) return desc def onFilenameChange(self, name ): name = str(name) if name in self.configParams.recentFiles and self.configParams.recentFiles[name]!=self.currentContext.pulseProgramFile: self.adaptiveLoadFile(self.configParams.recentFiles[name]) if str(self.filenameComboBox.currentText())!=name: with BlockSignals(self.filenameComboBox) as w: w.setCurrentIndex( self.filenameComboBox.findText( name )) self.updateSaveStatus() def onRamFilenameChange(self, name ): name = str(name) if name in self.configParams.recentRamFiles and self.configParams.recentRamFiles[name]!=self.currentContext.ramFile: self.loadRamFile(self.configParams.recentRamFiles[name]) if str(self.ramFilenameComboBox.currentText())!=name: with BlockSignals(self.ramFilenameComboBox) as w: w.setCurrentIndex( self.ramFilenameComboBox.findText( name )) self.updateSaveStatus() def onOk(self): pass def onLoadRamFile(self): path, _ = QtWidgets.QFileDialog.getOpenFileName(self, 'Open RAM file', self.defaultRAMDir, filter='*.yml') if path: self.loadRamFile(path) self.updateSaveStatus() @QtCore.pyqtProperty(list) def ramData(self): return self.loadRamData(filename=self.currentContext.ramFile) @QtCore.pyqtProperty(bool) def writeRam(self): return self.currentContext.writeRam @file_data_cache(maxsize=3) def loadRamData(self, filename=''): """load in the data from a RAM file""" with open(filename, 'r') as f: yamldata = yaml.load(f) return [self.pulseProgram.convertParameter(Q(float(ramValue['value']), ramValue['unit']), ramValue['encoding']) for ramValue in yamldata] def loadRamFile(self, path): if path and os.path.exists(path): self.currentContext.ramFile = path filename = os.path.basename(path) if filename not in self.configParams.recentRamFiles: self.ramFilenameComboBox.addItem(filename) self.configParams.recentRamFiles[filename]=path with BlockSignals(self.ramFilenameComboBox) as w: w.setCurrentIndex( self.ramFilenameComboBox.findText(filename)) else: self.currentContext.ramFile = '' with BlockSignals(self.ramFilenameComboBox) as w: w.setCurrentIndex( self.ramFilenameComboBox.findText('')) def onWriteRamCheckbox(self): self.currentContext.writeRam = self.writeRamCheckbox.isChecked() self.updateSaveStatus() def onRemoveCurrentRamFile(self): text = str(self.ramFilenameComboBox.currentText()) if text in self.configParams.recentRamFiles: self.configParams.recentRamFiles.pop(text) self.ramFilenameComboBox.removeItem(self.ramFilenameComboBox.currentIndex()) self.currentContext.ramFile = '' self.updateSaveStatus() def onLoad(self): path, _ = QtWidgets.QFileDialog.getOpenFileName(self, 'Open Pulse Programmer file', self.defaultPPPDir, filter='*.ppp *.pp') if path!="": self.adaptiveLoadFile(path) self.updateSaveStatus() def adaptiveLoadFile(self, path): if path: _, ext = os.path.splitext(path) self.currentContext.pulseProgramFile = path if ext==".ppp": self.currentContext.pulseProgramMode = 'ppp' self.loadpppFile(path) else: self.currentContext.pulseProgramMode = 'pp' self.updatepppDisplay() self.loadppFile(path) self.configParams.lastLoadFilename = path def onReset(self): if self.configParams.lastLoadFilename is not None: self.variabledict = VariableDictionary( self.pulseProgram.variabledict, self.parameterdict ) self.adaptiveLoadFile(self.configParams.lastLoadFilename) def loadpppFile(self, path): self.pppSourcePath = path _, self.pppSourceFile = os.path.split(path) with open(path, "r") as f: self.pppSource = f.read() self.updatepppDisplay() ppFilename = getPpFileName(path) if self.compileppp(ppFilename): self.loadppFile(ppFilename, cache=False) filename = os.path.basename(path) if filename not in self.configParams.recentFiles: self.filenameComboBox.addItem(filename) self.configParams.recentFiles[filename]=path with BlockSignals(self.filenameComboBox) as w: w.setCurrentIndex( self.filenameComboBox.findText(filename)) def saveppp(self, path): if self.pppSource and path: with open(path, 'w') as f: f.write( self.pppSource ) def compileppp(self, savefilename): self.pppSource = self.pppSource.expandtabs(4) success = False try: compiler = pppCompiler() ppCode = compiler.compileString( self.pppSource ) self.pppReverseLineLookup = compiler.reverseLineLookup self.pppCompileException = None with open(savefilename, "w") as f: f.write(ppCode) success = True self.pppCodeEdits[self.pppSourceFile].clearHighlightError() except CompileException as e: self.pppCodeEdits[self.pppSourceFile].highlightError( e.message(), e.lineno(), col=e.col()) except ParseException as e: e.__class__ = CompileException # cast to CompileException. Is possible because CompileException does ONLY add behavior self.pppCodeEdits[self.pppSourceFile].highlightError( e.message(), e.lineno(), col=e.col()) return success def loadppFile(self, path, cache=True): self.pulseProgram.loadSource(path, docompile=False) self.updateppDisplay() try: self.pulseProgram.compileCode() except PulseProgram.ppexception as compileexception: self.sourceCodeEdits[ compileexception.file ].highlightError(str(compileexception), compileexception.line, compileexception.context ) if cache: filename = os.path.basename(path) if filename not in self.configParams.recentFiles: self.filenameComboBox.addItem(filename) self.configParams.recentFiles[filename]=path self.filenameComboBox.setCurrentIndex( self.filenameComboBox.findText(filename)) # Merge any new values into the current context self.mergeVariablesIntoContext( self.pulseProgram.variabledict ) self.updateDisplayContext() self.pulseProgramChanged.emit() def mergeVariablesIntoContext( self, variabledict ): """Merge variables into context, propagating variable changes up the parentContext list""" # Follow the parenting links parent = self.currentContext.parentContext parents = [] while parent is not None and parent in self.contextDict and parent not in parents: parentContext = self.contextDict[parent] if parentContext.pulseProgramFile != self.currentContext.pulseProgramFile: break # Stop if the parent's pulse program does not match the current parents.append(parent) # Prevents infinite loops parent = parentContext.parentContext # Propagate variable changes up the parentContext tree in reverse order so the variable # linking up the tree doesn't break. linkNewToParent = False for parent in reversed(parents): self.contextDict[parent].merge( variabledict, linkNewToParent=linkNewToParent) linkNewToParent = True # Finally, merge into the current context self.currentContext.merge( variabledict, linkNewToParent=linkNewToParent ) def onRemoveCurrent(self): text = str(self.filenameComboBox.currentText()) if text in self.configParams.recentFiles: self.configParams.recentFiles.pop(text) self.filenameComboBox.removeItem(self.filenameComboBox.currentIndex()) def onSave(self): self.onApply() if self.currentContext.pulseProgramMode=='pp': self.pulseProgram.saveSource() else: self.saveppp(self.pppSourcePath) def onApply(self): if self.currentContext.pulseProgramMode=='pp': try: positionCache = dict() for name, textEdit in self.sourceCodeEdits.items(): self.pulseProgram.source[name] = str(textEdit.toPlainText()) positionCache[name] = ( textEdit.textEdit.cursorPosition(), textEdit.textEdit.scrollPosition() ) self.pulseProgram.loadFromMemory() self.updateppDisplay() for name, textEdit in self.sourceCodeEdits.items(): textEdit.clearHighlightError() if name in positionCache: cursorpos, scrollpos = positionCache[name] textEdit.textEdit.setCursorPosition( *cursorpos ) textEdit.textEdit.setScrollPosition( scrollpos ) except PulseProgram.ppexception as ppex: textEdit = self.sourceCodeEdits[ ppex.file ].highlightError(str(ppex), ppex.line, ppex.context ) else: positionCache = dict() for name, textEdit in self.pppCodeEdits.items(): self.pppSource = str(textEdit.toPlainText()) positionCache[name] = ( textEdit.textEdit.cursorPosition(), textEdit.textEdit.scrollPosition() ) ppFilename = getPpFileName( self.pppSourcePath ) if self.compileppp(ppFilename): self.loadppFile(ppFilename, cache=False) for name, textEdit in self.pppCodeEdits.items(): textEdit.clearHighlightError() if name in positionCache: cursorpos, scrollpos = positionCache[name] textEdit.textEdit.setCursorPosition( *cursorpos ) textEdit.textEdit.setScrollPosition( scrollpos ) def onAccept(self): self.saveConfig() def onReject(self): pass def saveConfig(self): """Save the pulse program configuration state""" self.configParams.lastContextName = str(self.contextComboBox.currentText()) self.config[self.configname+".state"] = self.saveState() #Arrangement of dock widgets self.config[self.configname] = self.configParams self.config[self.configname+'.contextdict'] = self.contextDict self.config[self.configname+'.currentContext'] = self.currentContext self.config[self.configname+'.docSplitter'] = self.docSplitter.saveState() self.variableTableModel.saveConfig() def getPulseProgramBinary(self,parameters=dict(),override=dict()): # need to update variables self.pulseProgram.updateVariables( self.) substitutes = dict(self.currentContext.parameters.valueView.items()) for model in [self.shutterTableModel, self.triggerTableModel, self.counterTableModel]: substitutes.update( model.getVariables() ) substitutes.update(override) self.pulseProgram.updateVariables(substitutes) return self.pulseProgram.toBinary() def exitcode(self, number): return self.pulseProgram.exitcode(number) def getVariableValue(self, name): return self.variableTableModel.getVariableValue(name) def variableScanCode(self, variablename, values, extendedReturn=False): tempparameters = copy.deepcopy( self.currentContext.parameters ) updatecode = list() numVariablesPerUpdate = 0 for currentval in values: upd_names, upd_values = tempparameters.setValue(variablename, currentval) numVariablesPerUpdate = len(upd_names) upd_names.append( variablename ) upd_values.append( currentval ) updatecode.extend( self.pulseProgram.multiVariableUpdateCode( upd_names, upd_values ) ) logging.getLogger(__name__).info("{0}: {1}".format(upd_names, upd_values)) if extendedReturn: return updatecode, numVariablesPerUpdate return updatecode def updateSaveStatus(self, isSaved=None): try: if isSaved is None: currentText = str(self.contextComboBox.currentText()) if not currentText: self.contextSaveStatus = True elif currentText in self.contextDict: self.contextSaveStatus = self.contextDict[currentText]==self.currentContext else: self.contextSaveStatus = False if self.configParams.autoSaveContext and not self.contextSaveStatus: self.onSaveContext() self.contextSaveStatus = True else: self.contextSaveStatus = isSaved self.saveContextButton.setEnabled( not self.contextSaveStatus ) except Exception: pass def onControl(self, key): if key==QtCore.Qt.Key_B: self.onBold() elif key==QtCore.Qt.Key_W: self.onSetBackgroundColor() elif key==QtCore.Qt.Key_R: self.onRemoveBackgroundColor() def onBold(self): indexes = self.variableView.selectedIndexes() for index in indexes: self.variableTableModel.toggleBold( index ) def onSetBackgroundColor(self): indexes = self.variableView.selectedIndexes() if indexes: color = QtGui.QColorDialog.getColor() if not color.isValid(): color = None for index in indexes: self.variableTableModel.setBackgroundColor(index, color) def onRemoveBackgroundColor(self): indexes = self.variableView.selectedIndexes() for index in indexes: self.variableTableModel.removeBackgroundColor(index) def lineOfInstruction(self, binaryelement ): ppline = self.pulseProgram.lineOfInstruction(binaryelement) pppline = self.pppReverseLineLookup.get(ppline, None) if hasattr(self, 'pppReverseLineLookup') else None return ppline, pppline def setTimingViolations(self, linelist): edit = self.pppCodeEdits.get(self.pppSourceFile) if edit: edit.highlightTimingViolation( [l[1]-1 for l in linelist] ) def getDocs(self): """Assemble the pulse program function documentation into dictionaries""" definitionDocPath = os.path.join(os.path.dirname(__file__), '..', r'docs/manual/pppDefinitionDocs.include') definitionDict = self.readDocFile(definitionDocPath) encodingDocPath = os.path.join(os.path.dirname(__file__), '..', r'docs/manual/pppEncodingDocs.include') encodingDict = self.readDocFile(encodingDocPath) builtinDict = OrderedDict() symbolTable = SymbolTable() for name in self.builtinWords: builtinDict[name] = symbolTable[name].doc or "This should be documentation for builtin word {0}".format(name) return definitionDict, builtinDict, encodingDict def readDocFile(self, filename): """Read in the rst file 'filename' """ docdict = OrderedDict() try: with open(filename, 'r') as f: docs = f.read() sepdocs = docs.split('.. py:data:: ') for doc in sepdocs: if doc: doclines = doc.splitlines() name = doclines.pop(0) for line in doclines: if line: documentation = line.strip() #Take the first line with content as the documentation to display in the program break docdict[name] = documentation except Exception as e: logging.getLogger(__name__).warning("Unable to load documentation: {0}".format(e)) return docdict def addDocs(self, docDict, category): """Add the documentation dictionary docDict to the docTree under 'category' """ categoryItem = QtWidgets.QTreeWidgetItem(self.docTreeWidget, [category]) self.docTreeWidget.addTopLevelItem(categoryItem) for name, documentation in docDict.items(): nameItem = QtWidgets.QTreeWidgetItem(categoryItem, [name]) label = QtWidgets.QLabel(documentation) label.setWordWrap(True) docItem = QtWidgets.QTreeWidgetItem(nameItem) self.docTreeWidget.setItemWidget(docItem, 0, label) def onVariableViewClicked(self, index): if index.column() == 1 and self.currentContext.parentContext: var = self.currentContext.parameters.at(index.row()) if var.hasParent: var.useParentValue ^= True try: self.setParentData(self.currentContext.parentContext, var) except KeyError: var.hasParent = False self.variableTableModel.onClicked(index) self.updateSaveStatus() def onLinkAllToParent(self): """ Link all variables to their parents """ self.variableTableModel.beginResetModel() if self.currentContext.parentContext: for var in self.currentContext.parameters.values(): if var.hasParent: var.useParentValue = True self.setParentData(self.currentContext.parentContext) self.variableTableModel.endResetModel() self.updateSaveStatus() def onUnlinkAllFromParent(self): """ Unlink all variables from their parents """ self.variableTableModel.beginResetModel() for var in self.currentContext.parameters.values(): var.useParentValue = False self.setParentData(self.currentContext.parentContext) self.variableTableModel.endResetModel() self.updateSaveStatus() def setParentData(self, parentContext, var=None): for var in [var] if var is not None else self.currentContext.parameters.values(): try: var.hasParent = bool(parentContext) if var.useParentValue and var.hasParent: rootName = self.findControllingNode(var.name) self.setParentValue(var.name, rootName) var.parentObject = self.contextDict[rootName].parameters[var.name] else: self.currentContext.parameters.setStrValue(var.name, var.strvalue) var.parentObject = None except KeyError: var.hasParent = False def findControllingNode(self, paramName): current, parent = self.currentContextName, self.currentContext.parentContext var = self.currentContext.parameters[paramName] child = None if not (parent and var.useParentValue): return None for child, parent in dfs_edges(self.dependencyGraph, parent): try: var = self.contextDict[parent].parameters[paramName] if not (var.useParentValue and var.hasParent): return child except KeyError: return child return parent def setParentValue(self, paramName, contextName): self.currentContext.parameters.setParentValue(paramName, self.contextDict[contextName].parameters[paramName].value) self.currentContext.parameters.setParentStrValue(paramName, self.contextDict[contextName].parameters[paramName].strvalue)
class MacroManager(object): """This class manages the macros specified in the configuration file. The parameters of each section, along with their dependencies are passed to the class. Then, it verifies that the dependencies are correct (they form a DAG and respect the sections dependencies) and creates an ordered list of the macros to be used when replacing their actual values in a given combination. """ def __init__(self): """Create a new MacroManager object.""" self.dep_graph = DiGraph() self.ds_macros = set([]) self.xp_macros = set([]) self.__define_test_macros() def __define_test_macros(self): """Define values and dependencies of test macros. A set of macros are defined by default, including input and output directories of datasets and experiments and their identifiers. """ self.test_macros = { "data_base_dir": "/tests/data", "out_base_dir": "/tests/out", "data_dir": "/tests/data/0", # data_base_dir/ds_id "out_dir": "/tests/out/0", # data_base_dir/comb_id "comb_id": 0, "ds_id": 0, "xp.input": "/tests/data/0", # data_dir "xp.output": "/tests/out/0" # out_dir } self.ds_params = set([]) self.xp_params = set([]) self.dep_graph.add_nodes_from(self.test_macros.keys()) self.add_dependency("data_base_dir", "data_dir") self.add_dependency("ds_id", "data_dir") self.add_dependency("data_dir", "xp.input") self.add_dependency("out_base_dir", "out_dir") self.add_dependency("comb_id", "out_dir") self.add_dependency("out_dir", "xp.output") self.sorted_test_macros = topological_sort(self.dep_graph) def update_test_macros(self, ds_id=None, comb_id=None): """Update test macros with dataset and/or combination ids. Args: ds_id (int, optional): The dataset identifier. comb_id (int, optional): The combination identifier. """ if ds_id: if "data_dir" in self.test_macros: self.test_macros["data_dir"] = \ self.test_macros["data_base_dir"] + "/" + str(ds_id) if "xp.input" in self.test_macros: self.test_macros["xp.input"] = \ self.test_macros["data_dir"] if comb_id: if "out_dir" in self.test_macros: self.test_macros["out_dir"] = \ self.test_macros["out_base_dir"] + "/" + str(comb_id) if "xp.output" in self.test_macros: self.test_macros["xp.output"] = \ self.test_macros["out_dir"] def __filter_unused_test_macros(self): for m in reversed(self.sorted_test_macros): if not self.dep_graph.successors(m): self.dep_graph.remove_node(m) self.sorted_test_macros.remove(m) del self.test_macros[m] def add_ds_params(self, params): """Add the list of dataset parameters. Args: params (dict): The list of dataset parameters. """ self.ds_params = self.ds_params.union(params) def add_xp_params(self, params): """Add the list of experiment parameters. Args: params (dict): The list of experiment parameters. """ self.xp_params = self.xp_params.union(params) def add_dependency(self, m1, m2): """Include a new macro dependency: m1 -> m2. This means that to obtain the value of m2 we use the value of m1. Args: m1 (string): The name of the param used. m2 (string): The name of the param being specified. Raises: MacroException: If the order of sections (test -> ds -> xp) is not respected. """ # Check if dependency is correct if m1 in self.ds_params: if m2 in self.test_macros: logger.error("Not allowed dependency: ds -> test") raise MacroException("Not allowed dependency: ds -> test") elif m1 in self.xp_params: if m2 in self.test_macros: logger.error("Not allowed dependency: xp -> test") raise MacroException("Not allowed dependency: xp -> test") elif m2 in self.ds_params: logger.error("Not allowed dependency: xp -> ds") raise MacroException("Not allowed dependency: xp -> ds") # Add dependency self.dep_graph.add_edge(m1, m2) def sort_macros(self): """Sort macros respecting dependencies. Raises: MacroException: If there are cycles in dependencies between macros. """ # Filter out unused test variables self.__filter_unused_test_macros() # Sort ds and xp macros try: self.sorted_ds_macros = \ topological_sort(self.dep_graph.subgraph(self.ds_params)) self.sorted_xp_macros = \ topological_sort(self.dep_graph.subgraph(self.xp_params)) except NetworkXUnfeasible: raise MacroException("Macros do not follow a DAG") logger.info("Dependencies = " + str(self.dep_graph.edges())) logger.info("Test macros = " + str(self.sorted_test_macros)) logger.info("Dataset macros = " + str(self.sorted_ds_macros)) logger.info("Experiment macros = " + str(self.sorted_xp_macros)) def _replace_macros_from_list(self, list_macros, value): """Replace the macros given in the list within the value if present. Args: list_macros (dict): The list of macros to replace and their respective values. value (string): The value where to do the replacement. """ new_value = value for m in list_macros: new_value = new_value.replace("${" + m + "}", str(list_macros[m])) return new_value def replace_ds_macros(self, comb): """Replace macros in ds combination. Args: comb (dict): The combination of parameters. """ list_macros = self.test_macros for m in self.sorted_ds_macros: comb[m] = self._replace_macros_from_list(list_macros, comb[m]) list_macros[m] = comb[m] def replace_xp_macros(self, comb): """Replace macros in xp combination. Args: comb (dict): The combination of parameters. """ list_macros = self.test_macros for m in self.sorted_ds_macros: comb[m] = self._replace_macros_from_list(list_macros, comb[m]) list_macros[m] = comb[m] for m in self.sorted_xp_macros: comb[m] = self._replace_macros_from_list(list_macros, comb[m]) list_macros[m] = comb[m]
def make_warehouse(): G = DiGraph() # Nodes for all states in scope. # G.add_nodes_from([(x, y, d) for x in range(0, 9) for y in range(0, 5) for d in ['N', 'S', 'E', 'W']]) ##### Turns ##### # Squares/Corners # G.add_edges_from([((0, y, 'W'), (0, y, 'E')) for y in range(0,5)], weight = 1, inst = "u") # Column 0 G.add_edges_from([((8, y, 'E'), (8, y, 'W')) for y in range(0,5)], weight = 1, inst = "u") # Column 8 G.add_edges_from([((x, y, 'W'), (x, y, 'E')) for x in [3, 6] for y in range(1,4)], weight = 1, inst = "u") # Column 2, 5 G.add_edges_from([((x, y, 'E'), (x, y, 'W')) for x in [2, 5] for y in range(1,4)], weight = 1, inst = "u") # Column 2, 5 # Middle Spots # midRights = [('N', 'E'), ('E', 'S'), ('W', 'N'), ('S', 'W')] G.add_edges_from([((x, y, ds), (x, y, dt)) for x in [1, 4, 7] for y in range(1, 4) for ds, dt in midRights], weight = 1, inst = "r") G.add_edges_from([((x, y, ds), (x, y, dt)) for x in [1, 4, 7] for y in range(1, 4) for dt, ds in midRights], weight = 1, inst = "l") # Top Edges # topRights = [('N', 'E'), ('E', 'S'), ('S', 'W'), ('W', 'E')] topLefts = [('N', 'W'), ('W', 'S'), ('S', 'E'), ('E', 'W')] G.add_edges_from([((x, 0, ds), (x, 0, dt)) for x in [1, 4, 7] for ds, dt in topRights], weight = 1, inst = "r") # Column 1 Top - Rights G.add_edges_from([((x, 0, ds), (x, 0, dt)) for x in [1, 4, 7] for ds, dt in topLefts], weight = 1, inst = "l") # Column 1 Top - Lefts # Bottom Edges # bottomRights = [('N', 'E'), ('W', 'N'), ('S', 'W'), ('E', 'W')] bottomLefts = [('N', 'W'), ('W', 'E'), ('E', 'N'), ('S', 'E')] G.add_edges_from([((x, 4, ds), (x, 4, dt)) for x in [1, 4, 7] for ds, dt in bottomRights], weight = 1, inst = "r") # Column 1 Bottom - Rights G.add_edges_from([((x, 4, ds), (x, 4, dt)) for x in [1, 4, 7] for ds, dt in bottomLefts], weight = 1, inst = "l") # Column 1 Bottom - Lefts ##### Paths ##### # Columns 0 and 8 # for y in range(0, 5): G.add_edge((0, y, 'E'), (1, y, 'E'), weight = 1, inst = 't') # Col 0 to Col 1 G.add_edge((8, y, 'W'), (7, y, 'W'), weight = 1, inst = 't') # Col 8 to Col 7 # Columns 2 and 5 G.add_edges_from([((x, y, 'W'), (x-1, y, 'W')) for x in [2, 5] for y in range(1, 4)], weight = 1, inst = 't') # Columns 3 and 6 # G.add_edges_from([((x, y, 'E'), (x+1, y, 'E')) for x in [3, 6] for y in range(1, 4)], weight = 1, inst = 't') # Columns 1, 4, and 7 # for x in [1, 4, 7]: for y in range(0, 5): if y != 0: G.add_edge((x, y, "N"), (x, y-1, "N"), weight = 1, inst = 't') if y != 4: G.add_edge((x, y, "S"), (x, y+1, "S"), weight = 1, inst = 't') for y in range(1, 4): G.add_edge((x, y, "W"), (x-1, y, "W"), weight = 1, inst = 't') G.add_edge((x, y, "E"), (x+1, y, "E"), weight = 1, inst = 't') for y in [0, 4]: if x in [1, 4]: G.add_edge((x, y, "E"), (x+3, y, "E"), weight = 1, inst = 't') if x in [4, 7]: G.add_edge((x, y, "W"), (x-3, y, "W"), weight = 1, inst = 't') if x == 1: G.add_edge((x, y, "W"), (x-1, y, "W"), weight = 1, inst = 't') if x == 7: G.add_edge((x, y, "W"), (x+1, y, "E"), weight = 1, inst = 't') return G