def load(
cls: Type["RelationDAG"],
database: Database,
extend_relations: List[Relation],
ignore_relations: List[Relation],
ignore_tables: List[Table],
) -> "RelationDAG":
"""
Create a RelationDAG
The data loaded from this method is sourced
from the database and from the user config
"""
graph = DiGraph(name="RelationDAG")
# Get actual table instances
tables = database.tables.__dict__.values()
# Create relations from table data and add the ones specified in settings
relations = Relation.from_tables(tables) + extend_relations
# Create graph
graph.add_nodes_from(tables)
graph.add_edges_from([r.edge for r in relations])
# Remove excluded entities (tables and relations) from the created graph
graph.remove_edges_from([r.edge for r in ignore_relations])
graph.remove_nodes_from(ignore_tables)
# Create RelationDAG instance
return cls.from_graph(graph)
def get_inheritance_graph(self, profiles=None):
"""
Determine the class inheritance hierarchy (class definition needs to adhere to strict inheritance hierarchy)
:return: g - A networkx DiGraph of the class hierarchy, with a common ancestor __root__
"""
# Determine class inheritance hierarchy (bfs on a directed graph)
if not profiles:
log.info(f"No profiles specified - using all profiles for ORM.")
elif not isinstance(profiles, Iterable):
profiles = (profiles, )
g = DiGraph()
g.add_node("__root__")
class_list = list(self.session.query(CIMClass).all())
classes = {}
for c in class_list:
if (c.namespace.short, c.name) in classes:
raise ValueError("Duplicate class identity: %s_%s." %
(c.namespace.short, c.name))
classes[(c.namespace.short, c.name)] = c
nodes = classes.keys()
g.add_nodes_from(nodes)
for key, instance in classes.items():
if instance:
parent = instance.parent
if parent is None:
g.add_edge("__root__", key)
else:
parent_key = (parent.namespace.short, parent.name)
g.add_edge(parent_key, key)
return g, classes
def map(self):
if not self.g:
g = DiGraph()
classnames = [
_[0] for _ in self.session.query(CIMClass.name).all()
]
classes = self.session.query(CIMClass).all()
enums = self.session.query(CIMEnum).all()
enumnames = [_[0] for _ in self.session.query(CIMEnum.name).all()]
propnames = [_[0] for _ in self.session.query(CIMProp.name).all()]
g.add_nodes_from(classnames)
g.add_nodes_from(enumnames)
g.add_nodes_from(propnames)
for node in classes + enums:
try:
for prop in node.all_props.values():
if prop.range:
g.add_edge(node.name,
prop.range.name,
label=prop.label)
else:
g.add_edge(node.name, prop.name, label=prop.label)
except AttributeError:
pass
self.g = g
return self.g
def filter_non_dependencies(nodes, get_deps_func):
node_set = set(nodes)
G = DiGraph()
G.add_nodes_from(nodes)
# process the edges based on the dependency function
for n in G:
deps = get_deps_func(n)
logging.info('%s depends on %s' % (n, deps))
for d in deps:
if d in G:
G.add_edge(n, d)
# now filter the nodes and return them
filtered_pkgs = {
node
for node, in_degree in G.in_degree_iter() if in_degree == 0
}
# now find any strongly connected components with size greater than 1
# these will all have in degree > 0, but should still be included
glist = [
g for g in strongly_connected_component_subgraphs(G, copy=False)
if g.number_of_nodes() > 1
]
for g in glist:
# only counts if it was the original list
nodes = [n for n in g.nodes() if n in node_set]
if len(nodes) > 0:
logging.debug('Strongly connected component: %s' % repr(nodes))
for n in nodes:
filtered_pkgs.add(n)
return filtered_pkgs
def check_pipeline_structure(cls, values):
graph = DiGraph()
graph.add_nodes_from([v.node_id for v in values.get('nodes')])
graph.add_edges_from([(e.from_, e.to) for e in values.get('links')])
validate_pipeline_structure(graph)
return values
def internalize_calls(cfg: LocalGraph) -> LocalGraph:
"""
Transform external callees into symbolic internal nodes.
A symbolic node will bear an identifier and a single label, both equal to the callee's symbolic name. Of course,
these new nodes will be isolated from the rest of the graph. Therefore, this method is of practical use only when
the user is planning an attempt at name resolution by modifying the internal graph.
:param cfg: a local graph
:return: a new local graph, with all external calls converted into symbolic nodes
"""
# Gather all the callees' names
external_nodes_ids = set(map(attrgetter('callee'), cfg.external_calls))
# Create a new local graph containing only the symbolic nodes
foreign_graph = DiGraph()
foreign_graph.add_nodes_from((i, {
'labels': [i],
'external': True
}) for i in external_nodes_ids)
external = LocalGraph(external_nodes_ids, foreign_graph, [],
external_nodes_ids)
# Merge the new graph with the original and return the result
return cfg.merge(external)
def find_dependents(zip, dependency_regex, entry_points, targets):
from networkx import DiGraph
from networkx.algorithms.simple_paths import all_simple_paths
nodes = zip.namelist()
dependency_graph = DiGraph()
dependency_graph.add_nodes_from(nodes)
for node in nodes:
with zip.open(node, 'r') as zipped_file:
content = zipped_file.read()
for match in dependency_regex.finditer(content):
dependency_graph.add_edge(node, match.group())
dependency_graph.add_edge(node, match.group() + 'bin')
dependents = set()
actual_targets = set()
for source in entry_points:
for target in targets:
for path in all_simple_paths(dependency_graph, source, target):
actual_targets.add(path[-1])
for dependent in path[0:-1]:
dependents.add(dependent)
return list(dependents), list(actual_targets)
def _graph(candidates, pairs):
"""
helper function for run(), evaluates winner of pairs given ordered pairs
slower than _faster_comp(), produces same results
"""
# This is the standard graph based way to do ranked pairs
g = DiGraph()
g.add_nodes_from(candidates)
# The strongest victories are added in order, unless there is a cycle,
# in which case they are skipped
edges = set()
for (i, j) in pairs:
if (j, i) not in edges:
g.add_edge(i, j)
# if a cycle exists, the edge is removed, otherwise continue
try:
find_cycle(g)
g.remove_edge(i, j)
except no_cycle:
pass
edges.add((i, j))
# We then find the source of the graph, which is the winner
winners = set()
for c in candidates:
try:
next(g.in_edges(c).__iter__())[0]
except StopIteration:
winners.add(c)
return winners
def __init__(self, ligands):
graph = DiGraph()
graph.add_nodes_from(ligands)
for i in range(len(ligands)-1):
graph.add_edge(ligands[i], ligands[i+1])
self.graph = graph
self.ligands = ligands
def ad_graph(f):
# asynchronous dynamics as graph
adG = DiGraph()
adf = sd_to_ad(f)
adG.add_nodes_from(f.keys())
adG.add_edges_from([(k, v) for k in adf.keys() for v in adf[k]])
return adG
def reduce_digraphs(ga, gb):
"""
ga : a -> r, DiGraph
gb : b -> r, DiGraph
Gout : a -> b, DiGraph
"""
# identify non empty neighborhoods of ga, i.e. a
a_left = [left for left in ga.nodes() if ga.neighbors(left)]
# identify non empty neighborhoods of ga, i.e. a
b_left = [left for left in gb.nodes() if gb.neighbors(left)]
gout = DiGraph()
gout.add_nodes_from(a_left)
gout.add_nodes_from(b_left)
for a in a_left:
r_ga = ga.neighbors(a)
for b in b_left:
r_gb = gb.neighbors(b)
r_intersection = list(set(r_ga) & set(r_gb))
ar_vec = array([ga[a][r]['weight'] for r in r_intersection])
br_vec = array([gb[b][r]['weight'] for r in r_intersection])
gout.add_edge(a, b, {'weight': dot(ar_vec, br_vec)})
return gout
def test_get_paths_two_connected_nodes(self):
expected = [[1, 2]]
graph = DiGraph()
graph.add_nodes_from(range(1, 3))
graph.add_edge(1, 2)
actual = get_paths(graph)
self.assertEqual(expected, actual)
def filter_non_dependencies(nodes, get_deps_func):
node_set = set(nodes)
G = DiGraph()
G.add_nodes_from(nodes)
# process the edges based on the dependency function
for n in G:
deps = get_deps_func(n)
logging.info('%s depends on %s' % (n, deps))
for d in deps:
if d in G:
G.add_edge(n, d)
# now filter the nodes and return them
filtered_pkgs = {node for node, in_degree in G.in_degree_iter() if in_degree == 0}
# now find any strongly connected components with size greater than 1
# these will all have in degree > 0, but should still be included
glist = [g for g in strongly_connected_component_subgraphs(G, copy=False) if g.number_of_nodes() > 1]
for g in glist:
# only counts if it was the original list
nodes = [n for n in g.nodes() if n in node_set]
if len(nodes) > 0:
logging.debug('Strongly connected component: %s' % repr(nodes))
for n in nodes:
filtered_pkgs.add(n)
return filtered_pkgs
def to_graph(self) -> DiGraph:
graph = DiGraph()
graph.add_nodes_from([v.id for v in self.nodes])
graph.add_edges_from([(e.from_node_id, e.to_node_id)
for e in self.links])
return graph
def add_nodes_from(self, nodes, **attr):
H=copy.deepcopy(self)
self.clear()
if not H.nodes():
DiGraph.add_nodes_from(self, nodes, **attr)
self.names=names=sorted(nodes)
for i, n in enumerate(self.names):
self.node[i]={'name': n, 'pmf': Pmf()}
self.node[i]['pmf'].Set(1,self.p)
self.node[i]['pmf'].Set(0, 1-self.p)
self.remove_node(n)
self.edge[i]={}
self.indep_vars+=[i]
self.SetProbs()
return
DiGraph.add_nodes_from(self, nodes, **attr)
#ind_vars=[var for var in H.indep_vars]
#DiGraph.add_nodes_from(self, ind_vars)
self.names=names=sorted(set(H.names + nodes))
for i, n in enumerate(names):
try:
self.node[i], self.edge[i]=H.node[i], H.edge[i]
except:
self.node[i]={'name': n, 'pmf': Pmf()}
self.node[i]['pmf'].Set(1,self.p)
self.node[i]['pmf'].Set(0, 1-self.p)
self.remove_node(n)
self.edge[i]={}
self.indep_vars+=[i]
self.SetProbs()
def setNetwork(flights):
"""
Setting network and hierarchy
"""
aircraftNetwork = DiGraph()
aircraftNetwork.add_nodes_from(flights)
return aircraftNetwork
def set_node_label(graph: nx.DiGraph,
dgxp_file: str = DGXP_FILE,
ppi_file: str = PPI_FILE,
ppi_columns: str = COLUMNS,
sep=SEPARATOR,
dgxp_columns=DGXPCOLUMNS,
threshold: float = THRESHOLD):
"""
Set the attribute LABEL of nodes according to the fold-change to +1 or -1.
:param dgxp_file:
:param sep:
:param threshold:
:param dgxp_columns:
:param graph: NetworkX graph
"""
gene_to_fc_dict = create_gene_to_fold_change_dict(dgxp_file, ppi_file,
ppi_columns, sep,
dgxp_columns, threshold)
# create a container of tuples to add nodes with label fold change to graph
all_tup = []
for node, attr_dict in gene_to_fc_dict.items():
tup = (node, attr_dict)
all_tup.append(tup)
graph.add_nodes_from(nodes_for_adding=all_tup)
for node in graph.nodes():
if graph[node][LABEL] is None:
raise KeyError(f"The node {node} has not been labeled.")
def __init__(self, proteins, datatables):
graph = DiGraph()
graph.add_nodes_from(proteins)
self.data = datatables
for i in range(len(proteins)-1):
graph.add_edge(proteins[i], proteins[i+1])
self.graph = graph
self.proteins = proteins
def test_get_paths_with_two_possibilities(self):
expected = [[1, 2, 3], [1, 3, 2]]
graph = DiGraph()
graph.add_nodes_from(range(1, 4))
graph.add_edge(1, 2)
graph.add_edge(1, 3)
actual = get_paths(graph)
self.assertEqual(expected, actual)
def load_graph(input_stream: TextIO) -> DiGraph:
graph = DiGraph()
n = int(input_stream.readline())
graph.add_nodes_from(range(1, n + 1))
for _ in range(int(input_stream.readline())):
edge_src, edge_dest = input_stream.readline().split(' ')
graph.add_edge(int(edge_src), int(edge_dest))
return graph
def return_graph(self):
index = 0
digraph = DiGraph()
digraph.add_nodes_from([i for i in self.states])
for i in self.states:
for j in self.transition[i].keys():
if self.transition[i][j] is not None:
digraph.add_edge(i, j, weight=self.transition[i][j])
return digraph
def init_graph(self): #Represent the graph using adjacency lists g = DiGraph() g.add_nodes_from(self.vocabulary) #The keys (terms) become nodes in the graph for x in self.vocabulary: for y in self.vocabulary: if self.conditional_probability(x,y) >= 0.8 and self.conditional_probability(y,x) < 0.8: g.add_edge(x,y) return g
def update_from_db(self, session):
# type: (Session) -> None
# Only allow one thread at a time to construct a fresh graph.
with self._update_lock:
checkpoint, checkpoint_time = self._get_checkpoint(session)
if checkpoint == self.checkpoint:
self._logger.debug("Checkpoint hasn't changed. Not Updating.")
return
self._logger.debug("Checkpoint changed; updating!")
start_time = datetime.utcnow()
user_metadata = self._get_user_metadata(session)
groups, disabled_groups = self._get_groups(session, user_metadata)
permissions = self._get_permissions(session)
group_grants = self._get_group_grants(session)
group_service_accounts = self._get_group_service_accounts(session)
service_account_grants = all_service_account_permissions(session)
nodes = self._get_nodes(groups, user_metadata)
edges = self._get_edges(session)
edges_without_np_owner = [
(n1, n2) for n1, n2, r in edges
if GROUP_EDGE_ROLES[r["role"]] != "np-owner"
]
graph = DiGraph()
graph.add_nodes_from(nodes)
graph.add_edges_from(edges)
rgraph = graph.reverse()
# We need a separate graph without np-owner edges to construct the mapping of
# permissions to users with that grant.
permission_graph = DiGraph()
permission_graph.add_nodes_from(nodes)
permission_graph.add_edges_from(edges_without_np_owner)
grants_by_permission = self._get_grants_by_permission(
permission_graph, group_grants, service_account_grants,
user_metadata)
with self.lock:
self._graph = graph
self._rgraph = rgraph
self.checkpoint = checkpoint
self.checkpoint_time = checkpoint_time
self.user_metadata = user_metadata
self._groups = groups
self._disabled_groups = disabled_groups
self._permissions = permissions
self._group_grants = group_grants
self._group_service_accounts = group_service_accounts
self._service_account_grants = service_account_grants
self._grants_by_permission = grants_by_permission
duration = datetime.utcnow() - start_time
stats.log_rate("graph_update_ms",
int(duration.total_seconds() * 1000))
def mock_graph(mock_user_friends):
graph = DiGraph()
for row in mock_user_friends:
user = row["screen_name"]
friends = row["friend_names"]
graph.add_node(user)
graph.add_nodes_from(friends)
graph.add_edges_from([(user, friend) for friend in friends])
return graph
def __init__(self, transitions):
graph = DiGraph()
graph.add_nodes_from(transitions.keys())
for node in graph.nodes:
edges = transitions[node]
edges = ((node, edge) for edge in edges)
graph.add_edges_from(edges)
self._graph = graph
def create_entry_and_exit_nodes(graph: nx.DiGraph, items):
graph.add_nodes_from([START, END])
items = sorted(items)
graph.add_edge(START, items[0])
for node in (set(graph.nodes()) - {START}) - nx.descendants(graph, START):
graph.add_edge(START, node)
graph.add_edge(items[-1], END)
for node in (set(graph.nodes()) - {END}) - nx.ancestors(graph, END):
graph.add_edge(node, END)
def build_networkXGraph_from_spaCy_depGraph(sentence):
"""
Given a spaCy-parsed sentence, return the relative networkXGraph.
"""
g = DiGraph()
tokens = list(sentence)
g.add_nodes_from(tokens)
_add_edges_from_spaCy_depGraph(g, sentence.root)
return g
def build_digraph(fasta_dict, k):
"""Build a digraph from the data given by the fasta dict, using k and our
is_edge predicate.
"""
d = DiGraph()
d.add_nodes_from(fasta_dict.items())
for (s, t) in permutations(fasta_dict.items(), 2):
if is_edge(s, t, k):
d.add_edge(s, t)
return d
def test_get_coord_dict():
g = DiGraph()
g.add_nodes_from(['1', '23X', 'X', 'OX', 'A', 'TU'])
coord_dict, leftovers = get_coord_dict(g,
'cocotools/coords/pht00_rhesus.tsv')
nt.assert_equal(coord_dict, {'1': [20.31, -10.8],
'23X': [2.41, -30.15],
'OX': [1.57, -1.8],
'TU': [5.21, 1.8]})
nt.assert_equal(leftovers, ['A', 'X'])
def add_string_to_graph(graph: nx.DiGraph, string: str, sp, bpe_dropout: Optional[float]):
num_nodes_before = len(graph)
token_indices = sp.encode(
string, enable_sampling=bpe_dropout is not None, alpha=bpe_dropout)
nodes = list(
range(num_nodes_before, num_nodes_before+len(token_indices)))
graph.add_nodes_from(nodes)
for n1, n2 in zip(nodes[:-1], nodes[1:]):
graph.add_edge(n1, n2)
return nodes, token_indices
def example_tree():
"""creates a tree/networkx.DiGraph of a syntactic parse tree"""
tree = DiGraph()
tree.add_nodes_from(['S', 'NP-1', 'N-1', 'Jeff', 'VP', 'V', 'ate', 'NP-2', 'D',
'the', 'N-2', 'apple'])
tree.add_edges_from([('S', 'NP-1'), ('NP-1', 'N-1'), ('N-1', 'Jeff'),
('S', 'VP'), ('VP', 'V'), ('V', 'ate'),
('VP', 'NP-2'), ('NP-2', 'D'), ('D', 'the'),
('NP-2', 'N-2'), ('N-2', 'apple')])
return tree
def render_trees(parsed_dir_path: Path, target_dir_path: Path,
work_distribution: list):
for ecli in work_distribution:
parsed_files_glob = parsed_dir_path.joinpath(ecli).glob('*.xml')
for parsed_file_path in parsed_files_glob:
if parsed_file_path.is_file(
) and parsed_file_path.stat().st_size != 0:
json_dir_path = target_dir_path.joinpath(ecli)
if not json_dir_path.is_dir():
mkdir(str(json_dir_path))
json_file_name = parsed_file_path.name + '.json'
json_file_path = json_dir_path.joinpath(json_file_name)
# draw_spring(tree)
if not json_file_path.is_file(
) or json_file_path.stat().st_size != 0:
tree = DiGraph()
xml_tree = parse(str(parsed_file_path))
sentence = SENTENCE_XPATH(xml_tree)[0].text
nodes = []
edges = []
for xml_node in XML_NODES(xml_tree):
lemma = xml_node.get('lemma')
pos = xml_node.get('pos')
if lemma is None:
lemma = '...'
node_id = int(xml_node.attrib['id'])
node_attributes = {'name': lemma, 'pos': pos}
node = (node_id, node_attributes)
nodes.append(node)
parent_node_id = int(xml_node.getparent().get('id'))
edges.append((parent_node_id, node_id))
tree.add_nodes_from(nodes)
tree.add_edges_from(edges)
tree_json = json_graph.tree_data(tree, root=0)
wrapper_json = {
'origin': parsed_file_path.as_uri(),
'sentence': sentence,
'tree': tree_json
}
with json_file_path.open(mode='wt') as json_file:
dump(wrapper_json, json_file, indent=True)
info("Rendered parse tree to '{json_file_path}'. ".format(
json_file_path=json_file_path))
else:
error("Empty or non-existent XML parse tree file at "
"'{parsed_file_path}'. ".format(
parsed_file_path=parsed_file_path))
def to_directed(self):
from networkx import DiGraph
G = DiGraph()
G.name = self.name
G.add_nodes_from(self.n)
G.add_edges_from(((u, v, deepcopy(data)) for u, nbrs in self.a
for v, data in nbrs.items()))
G.graph = deepcopy(self.data)
G._nodedata = deepcopy(self._nodedata)
G.node = G._nodedata # hack to pass test
return G
def update_from_db(self, session):
# Only allow one thread at a time to construct a fresh graph.
with self.update_lock:
checkpoint, checkpoint_time = self._get_checkpoint(session)
if checkpoint == self.checkpoint:
self.logger.debug("Checkpoint hasn't changed. Not Updating.")
return
self.logger.debug("Checkpoint changed; updating!")
new_graph = DiGraph()
new_graph.add_nodes_from(self._get_nodes_from_db(session))
new_graph.add_edges_from(self._get_edges_from_db(session))
rgraph = new_graph.reverse()
users = set()
groups = set()
for (node_type, node_name) in new_graph.nodes():
if node_type == "User":
users.add(node_name)
elif node_type == "Group":
groups.add(node_name)
user_metadata = self._get_user_metadata(session)
permission_metadata = self._get_permission_metadata(session)
service_account_permissions = all_service_account_permissions(
session)
group_metadata = self._get_group_metadata(session,
permission_metadata)
group_service_accounts = self._get_group_service_accounts(session)
permission_tuples = self._get_permission_tuples(session)
group_tuples = self._get_group_tuples(session)
disabled_group_tuples = self._get_group_tuples(session,
enabled=False)
with self.lock:
self._graph = new_graph
self._rgraph = rgraph
self.checkpoint = checkpoint
self.checkpoint_time = checkpoint_time
self.users = users
self.groups = groups
self.permissions = {
perm.permission
for perm_list in itervalues(permission_metadata)
for perm in perm_list
}
self.user_metadata = user_metadata
self.group_metadata = group_metadata
self.group_service_accounts = group_service_accounts
self.permission_metadata = permission_metadata
self.service_account_permissions = service_account_permissions
self.permission_tuples = permission_tuples
self.group_tuples = group_tuples
self.disabled_group_tuples = disabled_group_tuples
def post_parse(grammar):
# Create `G`
G = DiGraph()
G.add_nodes_from(grammar.prods)
for lhs, rhs in grammar.prods.items():
for tok in rhs:
if tok in grammar.prods:
G.add_edge(lhs, tok)
# DEBUG: from ._debug import Drawer # DEBUG
# DEBUG: drawer = Drawer(G, grammar.start) # DEBUG
# Inlining
for root, _ in list(bfs_edges(G, grammar.start)):
while True:
nodes = [d for _, d in bfs_edges(G, root)]
nodes = [root] + nodes
edges = []
for n in nodes:
edges.extend(G.edges([n]))
for ns, nd in reversed(edges):
# DEBUG: drawer.draw(G, (ns, nd)) # DEBUG
# Skip if `nd` has a self-loop
if G.has_edge(nd, nd):
continue
# Skip if `C` consists of multiple nodes
g = G.subgraph(n for n in G.nodes_iter() if n != ns)
if len(next((c for c in sccs(g) if nd in c))) != 1:
continue
# Update grammar
expr = []
alter = Token.alter()
for tok in grammar.prods[ns]:
expr.append(tok)
if tok == nd:
expr.extend(list(grammar.prods[nd]) + [alter])
grammar.prods[ns] = Expr(expr)
# Update G
G.remove_edge(ns, nd)
for _, dst in G.edges_iter(nd):
G.add_edge(ns, dst)
# DEBUG: drawer.draw(G) # DEBUG
break # Back to `for ns, nd in ...`
else:
# DEBUG: drawer.draw(G) # DEBUG
break # Back to `for root, _ in ...`
return {nd for _, nd in G.edges_iter()} # Unexpanded nonterminals
def to_directed(self):
from networkx import DiGraph
G=DiGraph()
G.name=self.name
G.add_nodes_from(self)
G.add_edges_from( ((u,v,deepcopy(data))
for u,nbrs in self.adjacency_iter()
for v,data in nbrs.iteritems()) )
G.graph=deepcopy(self.graph)
G.node=deepcopy(self.node)
return G
def render_trees(parsed_dir_path: Path, target_dir_path: Path,
work_distribution: list):
for ecli in work_distribution:
parsed_files_glob = parsed_dir_path.joinpath(ecli).glob('*.xml')
for parsed_file_path in parsed_files_glob:
if parsed_file_path.is_file() and parsed_file_path.stat(
).st_size != 0:
json_dir_path = target_dir_path.joinpath(ecli)
if not json_dir_path.is_dir():
mkdir(str(json_dir_path))
json_file_name = parsed_file_path.name + '.json'
json_file_path = json_dir_path.joinpath(json_file_name)
# draw_spring(tree)
if not json_file_path.is_file() or json_file_path.stat(
).st_size != 0:
tree = DiGraph()
xml_tree = parse(str(parsed_file_path))
sentence = SENTENCE_XPATH(xml_tree)[0].text
nodes = []
edges = []
for xml_node in XML_NODES(xml_tree):
lemma = xml_node.get('lemma')
pos = xml_node.get('pos')
if lemma is None:
lemma = '...'
node_id = int(xml_node.attrib['id'])
node_attributes = {'name': lemma, 'pos': pos}
node = (node_id, node_attributes)
nodes.append(node)
parent_node_id = int(xml_node.getparent().get('id'))
edges.append((parent_node_id, node_id))
tree.add_nodes_from(nodes)
tree.add_edges_from(edges)
tree_json = json_graph.tree_data(tree, root=0)
wrapper_json = {'origin': parsed_file_path.as_uri(),
'sentence': sentence,
'tree': tree_json}
with json_file_path.open(mode='wt') as json_file:
dump(wrapper_json, json_file, indent=True)
info("Rendered parse tree to '{json_file_path}'. ".format(
json_file_path=json_file_path))
else:
error("Empty or non-existent XML parse tree file at "
"'{parsed_file_path}'. ".format(
parsed_file_path=parsed_file_path))
def to_directed(self):
from networkx import DiGraph
G = DiGraph()
G.name = self.name
G.add_nodes_from(self.n)
G.add_edges_from(((u, v, deepcopy(data))
for u, nbrs in self.a
for v, data in nbrs.items()))
G.graph = deepcopy(self.data)
G._nodedata = deepcopy(self._nodedata)
G.node = G._nodedata # hack to pass test
return G
def add_nodes_from(self, nodes, **attr):
H=DiGraph()
H.add_nodes_from(self.names)
h_names=sorted(H.nodes())
H.add_edges_from([(h_names[e[0]], h_names[e[1]], self.edge[e[0]][e[1]]) for e in self.edges()])
causes={h_names[v]: {h_names[item]: self.node[v]['causes'][item] for item in self.node[v]['causes']} for v in self.dep_vars}
self.clear()
self.indep_vars=[]
self.dep_vars=[]
if not H.nodes():
DiGraph.add_nodes_from(self, nodes, **attr)
self.names=names=sorted(nodes)
for i, n in enumerate(self.names):
self.node[i]={'name': n, 'pmf': Pmf()}
self.node[i]['pmf'].Set(1,self.p)
self.node[i]['pmf'].Set(0, 1-self.p)
self.remove_node(n)
self.edge[i]={}
self.indep_vars+=[i]
self.SetProbs()
return
#DiGraph.add_nodes_from(self, nodes, **attr)
#ind_vars=[var for var in H.indep_vars]
#DiGraph.add_nodes_from(self, ind_vars)
self.names=names=sorted(set(H.nodes() + nodes))
for i, n in enumerate(names):
if n in H.nodes():
self.node[i], self.edge[i]=H.node[n], {names.index(item): H.edge[n][item] for item in H.edge[n]}
self.node[i]['causes']={names.index(item): causes[n][item] for item in causes[n]} if n in causes else {}
self.node[i]['name']=n
self.node[i]['pmf']=Pmf()
if not self.node[i]['causes']:
self.node[i]['pmf'].Set(1,self.p)
self.node[i]['pmf'].Set(0, 1-self.p)
self.indep_vars+=[i]
else: self.dep_vars+=[i]
else:
self.node[i]={'name': n, 'pmf': Pmf()}
self.node[i]['pmf'].Set(1,self.p)
self.node[i]['pmf'].Set(0, 1-self.p)
#self.remove_node(n)
self.edge[i]={}
self.indep_vars+=[i]
self.SetProbs()
def update_from_db(self, session):
# Only allow one thread at a time to construct a fresh graph.
with self.update_lock:
checkpoint, checkpoint_time = self._get_checkpoint(session)
if checkpoint == self.checkpoint:
self.logger.debug("Checkpoint hasn't changed. Not Updating.")
return
self.logger.debug("Checkpoint changed; updating!")
new_graph = DiGraph()
new_graph.add_nodes_from(self._get_nodes_from_db(session))
new_graph.add_edges_from(self._get_edges_from_db(session))
rgraph = new_graph.reverse()
users = set()
groups = set()
for (node_type, node_name) in new_graph.nodes():
if node_type == "User":
users.add(node_name)
elif node_type == "Group":
groups.add(node_name)
user_metadata = self._get_user_metadata(session)
permission_metadata = self._get_permission_metadata(session)
service_account_permissions = all_service_account_permissions(session)
group_metadata = self._get_group_metadata(session, permission_metadata)
group_service_accounts = self._get_group_service_accounts(session)
permission_tuples = self._get_permission_tuples(session)
group_tuples = self._get_group_tuples(session)
disabled_group_tuples = self._get_group_tuples(session, enabled=False)
with self.lock:
self._graph = new_graph
self._rgraph = rgraph
self.checkpoint = checkpoint
self.checkpoint_time = checkpoint_time
self.users = users
self.groups = groups
self.permissions = {perm.permission
for perm_list in permission_metadata.values()
for perm in perm_list}
self.user_metadata = user_metadata
self.group_metadata = group_metadata
self.group_service_accounts = group_service_accounts
self.permission_metadata = permission_metadata
self.service_account_permissions = service_account_permissions
self.permission_tuples = permission_tuples
self.group_tuples = group_tuples
self.disabled_group_tuples = disabled_group_tuples
def to_directed(self):
"""Return a directed representation of the graph.
A new directed graph is returned with the same name, same nodes,
and with each edge (u,v,data) replaced by two directed edges
(u,v,data) and (v,u,data).
"""
from networkx import DiGraph
G=DiGraph()
G.add_nodes_from(self)
G.add_edges_from( ((u,v,data)
for u,nbrs in self.adjacency_iter()
for v,data in nbrs.iteritems()) )
return G
def update_from_db(self, session):
# type: (Session) -> None
# Only allow one thread at a time to construct a fresh graph.
with self._update_lock:
checkpoint, checkpoint_time = self._get_checkpoint(session)
if checkpoint == self.checkpoint:
self._logger.debug("Checkpoint hasn't changed. Not Updating.")
return
self._logger.debug("Checkpoint changed; updating!")
start_time = datetime.utcnow()
user_metadata = self._get_user_metadata(session)
groups, disabled_groups = self._get_groups(session, user_metadata)
permissions = self._get_permissions(session)
group_grants = self._get_group_grants(session)
group_service_accounts = self._get_group_service_accounts(session)
service_account_grants = all_service_account_permissions(session)
graph = DiGraph()
graph.add_nodes_from(self._get_nodes(groups, user_metadata))
graph.add_edges_from(self._get_edges(session))
rgraph = graph.reverse()
grants_by_permission = self._get_grants_by_permission(
graph, group_grants, service_account_grants
)
with self.lock:
self._graph = graph
self._rgraph = rgraph
self.checkpoint = checkpoint
self.checkpoint_time = checkpoint_time
self.user_metadata = user_metadata
self._groups = groups
self._disabled_groups = disabled_groups
self._permissions = permissions
self._group_grants = group_grants
self._group_service_accounts = group_service_accounts
self._service_account_grants = service_account_grants
self._grants_by_permission = grants_by_permission
duration = datetime.utcnow() - start_time
stats.log_rate("graph_update_ms", int(duration.total_seconds() * 1000))
def update_from_db(self, session):
checkpoint, checkpoint_time = self._get_checkpoint(session)
if checkpoint == self.checkpoint:
logging.debug("Checkpoint hasn't changed. Not Updating.")
return
logging.debug("Checkpoint changed; updating!")
new_graph = DiGraph()
new_graph.add_nodes_from(self._get_nodes_from_db(session))
new_graph.add_edges_from(self._get_edges_from_db(session))
rgraph = new_graph.reverse()
users = set()
groups = set()
for (node_type, node_name) in new_graph.nodes():
if node_type == "User":
users.add(node_name)
elif node_type == "Group":
groups.add(node_name)
user_metadata = self._get_user_metadata(session)
permission_metadata = self._get_permission_metadata(session)
group_metadata = self._get_group_metadata(session, permission_metadata)
with self.lock:
self._graph = new_graph
self._rgraph = rgraph
self.checkpoint = checkpoint
self.checkpoint_time = checkpoint_time
self.users = users
self.groups = groups
self.permissions = {perm.permission
for perm_list in permission_metadata.values()
for perm in perm_list}
self.user_metadata = user_metadata
self.group_metadata = group_metadata
self.permission_metadata = permission_metadata
def deterministic_topological_ordering(nodes, links, start_node):
"""
Topological sort that is deterministic because it sorts (alphabetically)
candidates to check
"""
graph = DiGraph()
graph.add_nodes_from(nodes)
for link in links:
graph.add_edge(*link)
if not is_directed_acyclic_graph(graph):
raise NetworkXUnfeasible
task_names = sorted(graph.successors(start_node))
task_set = set(task_names)
graph.remove_node(start_node)
result = [start_node]
while task_names:
for name in task_names:
if graph.in_degree(name) == 0:
result.append(name)
# it is OK to modify task_names because we break out
# of loop below
task_names.remove(name)
new_successors = [t for t in graph.successors(name)
if t not in task_set]
task_names.extend(new_successors)
task_names.sort()
task_set.update(set(new_successors))
graph.remove_node(name)
break
return result
def get_bin_group(grps, bin_grps, rt, graph):
level = grps.successors(rt)
if not level:
bin_grps.append(graph)
return bin_grps
elif len(level) == 2:
graph.add_nodes_from(level)
graph.add_edge(rt, level[0], grps.get_edge_data(rt, level[0]))
graph.add_edge(rt, level[1], grps.get_edge_data(rt, level[1]))
if grps.successors(level[0]):
get_bin_group(grps, bin_grps, rt=level[0], graph=graph)
else:
get_bin_group(grps, bin_grps, rt=level[1], graph=graph)
else:
level.sort()
for i in xrange(0, len(level), 2):
g = DiGraph(graph)
g.add_nodes_from([level[i], level[i + 1]])
g.add_edge(rt, level[i], grps.get_edge_data(rt, level[i]))
g.add_edge(rt, level[i + 1], grps.get_edge_data(rt, level[i + 1]))
if grps.successors(level[i]):
get_bin_group(grps, bin_grps, rt=level[i], graph=g)
else:
get_bin_group(grps, bin_grps, rt=level[i + 1], graph=g)
def example_invalid_subtrees():
is1 = DiGraph()
is1.add_node('NP-2')
is1.add_edges_from([('D', 'the'), ('N-2', 'apple')])
is2 = DiGraph()
is2.add_node('D')
is3 = DiGraph()
is3.add_nodes_from(['D', 'N-2'])
is4 = DiGraph()
is4.add_edges_from([('NP-2', 'D'), ('D', 'the')])
is5 = DiGraph()
is5.add_nodes_from(['the'])
is6 = DiGraph()
is6.add_nodes_from(['the', 'apple'])
is7 = DiGraph()
is7.add_edges_from([('the', 'apple')])
return [is1, is2, is3, is4, is5, is6, is7]
def load(self,fname, verbose=True, **kwargs):
"""
Load a data file. The expected data format is three columns
(comma seperated by default) with source, target, flux.
No header should be included and the node IDs have to run contuously
from 0 to Number_of_nodes-1.
Parameters
----------
fname : str
Path to the file
verbose : bool
Print information about the data. True by Default
kwargs : dict
Default parameters can be changed here. Supported key words are
dtype : float (default)
delimiter : "," (default)
return_graph : bool
If True, the graph is returned (False by default).
Returns:
--------
The graph is saved internally in self.graph.
"""
delimiter = kwargs["delimiter"] if "delimiter" in kwargs.keys() else " "
data = np.genfromtxt(fname, delimiter=delimiter, dtype=int, unpack=False)
source, target = data[:,0], data[:,1]
if data.shape[1] > 2:
flux = data[:,2]
else:
flux = np.ones_like(source)
nodes = set(source) | set(target)
self.nodes = len(nodes)
lines = len(flux)
if set(range(self.nodes)) != nodes:
new_node_ID = {old:new for new,old in enumerate(nodes)}
map_new_node_ID = np.vectorize(new_node_ID.__getitem__)
source = map_new_node_ID(source)
target = map_new_node_ID(target)
if verbose:
print "\nThe node IDs have to run continuously from 0 to Number_of_nodes-1."
print "Node IDs have been changed according to the requirement.\n-----------------------------------\n"
print 'Lines: ',lines , ', Nodes: ', self.nodes
print '-----------------------------------\nData Structure:\n\nsource, target, weight \n'
for ii in range(7):
print "%i, %i, %1.2e" %(source[ii], target[ii], flux[ii])
print '-----------------------------------\n'
G = DiGraph() # Empty, directed Graph
G.add_nodes_from(range(self.nodes))
for ii in xrange(lines):
u, v, w = int(source[ii]), int(target[ii]), float(flux[ii])
if u != v: # ignore self loops
assert not G.has_edge(u,v), "Edge appeared twice - not supported"
G.add_edge(u,v,weight=w)
else:
if verbose:
print "ignore self loop at node", u
symmetric = True
for s,t,w in G.edges(data=True):
w1 = G[s][t]["weight"]
try:
w2 = G[t][s]["weight"]
except KeyError:
symmetric = False
G.add_edge(t,s,weight=w1)
w2 = w1
if w1 != w2:
symmetric = False
G[s][t]["weight"] += G[t][s]["weight"]
G[s][t]["weight"] /= 2
G[t][s]["weight"] = G[s][t]["weight"]
if verbose:
if not symmetric:
print "The network has been symmetricised."
ccs = strongly_connected_component_subgraphs(G)
ccs = sorted(ccs, key=len, reverse=True)
G_GSCC = ccs[0]
if G_GSCC.number_of_nodes() != G.number_of_nodes():
G = G_GSCC
if verbose:
print "\n--------------------------------------------------------------------------"
print "The network has been restricted to the giant strongly connected component."
self.nodes = G.number_of_nodes()
for u, v, data in G.edges(data=True):
weight = G.out_degree(u,weight='weight')
data['transition_rate'] = 1.*data['weight']/weight
for u, v, data in G.edges(data=True):
data['effective_distance'] = 1. - log(data['transition_rate'])
if verbose:
print "\n--------------------------------------------------------------------------"
print "\nnode ID, out-weight, normalized out-weight, sum of effective distances \n "
for ii in range(7):
out_edges = G.out_edges(ii, data=True)
out_weight, effective_distance, transition_rate = 0, 0, 0
for u, v, data in out_edges:
out_weight += data["weight"]
effective_distance += data["effective_distance"]
transition_rate += data["transition_rate"]
print " %i %1.2e %2.3f %1.2e " %(ii,out_weight, transition_rate, effective_distance)
print "\n ... graph is saved in self.graph"
return G
def synthesize(tsys, exprtab, init_flags='ALL_ENV_EXIST_SYS_INIT'):
assert init_flags.upper() == 'ALL_ENV_EXIST_SYS_INIT', 'Only the initial condition interpretation ALL_ENV_EXIST_SYS_INIT is supported.'
W, Y_list, X_list = get_winning_set(tsys, return_intermediates=True)
initial_states = get_initial_states(W, tsys, exprtab, init_flags)
if initial_states is None:
return None
goalnames = ['SYSGOAL'+str(i) for i in range(tsys.num_sgoals)]
for goalmode in range(tsys.num_sgoals):
Y_list[goalmode][0] = set([s for s in W if goalnames[goalmode] in tsys.G.node[s]['sat']])
strategy = DiGraph()
next_id = len(initial_states)
workset = list(range(next_id))
strategy.add_nodes_from([(i, {'state': s, 'mode': 0, 'initial': True})
for (i,s) in enumerate(initial_states)])
while len(workset) > 0:
nd = workset.pop()
j = 0
while j < len(Y_list[strategy.node[nd]['mode']]):
if strategy.node[nd]['state'] in Y_list[strategy.node[nd]['mode']][j]:
break
j += 1
if j == 0:
assert goalnames[strategy.node[nd]['mode']] in tsys.G.node[strategy.node[nd]['state']]['sat']
original_mode = strategy.node[nd]['mode']
while goalnames[strategy.node[nd]['mode']] in tsys.G.node[strategy.node[nd]['state']]['sat']:
strategy.node[nd]['mode'] = (strategy.node[nd]['mode'] + 1) % tsys.num_sgoals
if strategy.node[nd]['mode'] == original_mode:
break
if strategy.node[nd]['mode'] != original_mode:
repeat_found = False
for possible_repeat, attr in strategy.nodes_iter(data=True):
if (possible_repeat != nd
and attr['mode'] == strategy.node[nd]['mode']
and attr['state'] == strategy.node[nd]['state']):
repeat_found = True
for (u,v) in strategy.in_edges_iter(nd):
strategy.add_edge(u, possible_repeat)
strategy.remove_edges_from(strategy.in_edges(nd))
strategy.remove_node(nd)
break
if repeat_found:
continue
j = 0
while j < len(Y_list[strategy.node[nd]['mode']]):
if strategy.node[nd]['state'] in Y_list[strategy.node[nd]['mode']][j]:
break
j += 1
if j == 0:
assert goalnames[strategy.node[nd]['mode']] in tsys.G.node[strategy.node[nd]['state']]['sat']
for envpost in tsys.envtrans[strategy.node[nd]['state']]:
next_state = None
for succ_nd in tsys.G.successors(strategy.node[nd]['state']):
if (tuple([succ_nd[i] for i in tsys.ind_uncontrolled]) == envpost
and ((j > 0 and succ_nd in Y_list[strategy.node[nd]['mode']][j-1])
or (j == 0 and succ_nd in W))):
next_state = succ_nd
break
if next_state is None:
assert j > 0
if j == 0:
import pdb; pdb.set_trace()
blocking_index = None
blocking_sets = X_list[strategy.node[nd]['mode']][j-1]
for k in range(len(blocking_sets)):
if strategy.node[nd]['state'] in blocking_sets[k]:
blocking_index = k
break
assert blocking_index is not None
for succ_nd in tsys.G.successors(strategy.node[nd]['state']):
if (tuple([succ_nd[i] for i in tsys.ind_uncontrolled]) == envpost
and succ_nd in blocking_sets[blocking_index]):
next_state = succ_nd
break
assert next_state is not None
foundmatch = False
for candidate, cattr in strategy.nodes_iter(data=True):
if cattr['state'] == next_state and cattr['mode'] == strategy.node[nd]['mode']:
strategy.add_edge(nd, candidate)
foundmatch = True
break
if not foundmatch:
if j == 0:
new_mode = (strategy.node[nd]['mode'] + 1) % tsys.num_sgoals
else:
new_mode = strategy.node[nd]['mode']
workset.append(next_id)
strategy.add_node(next_id, {'state': next_state,
'mode': new_mode,
'initial': False})
strategy.add_edge(nd, next_id)
next_id += 1
return strategy
def calc_reactions(org, fluxResults):
"""
Adding Results from FLUX-Analysis
@param org: organism
@param fluxResults: Results from PyNetMet FLUX-Analysis Calculation as String
@return Graph with added Attributes
"""
changeDic = {}
reactions = fluxResults.reacs
for reaction, flux in zip(reactions, fluxResults.flux):
if not reaction.name.endswith("_transp"):
changeDic[reaction.name] = float('%.3g' % flux)
vList = list(changeDic.values())
for i in range(len(vList)):
vList[i] = math.sqrt(math.pow(vList[i], 2))
vList = sorted(vList)
oldMin = vList[0]
oldMax = vList[-1]
oldRange = oldMax - oldMin
newMin = 1
newMax = 10
newRange = newMax - newMin
enzymes = fluxResults.reacs
nodeDic = OrderedDict()
nodecounter = 0
graph = DiGraph()
objective = None
if len(org.obj) > 0:
objective = org.obj[0][0]
for enzyme in enzymes:
if not enzyme.name.endswith("_transp"):
nodecounter += 1
nodeDic[enzyme.name] = nodecounter
if enzyme.name == objective:
graph.add_node(nodeDic[enzyme.name])
for substrate in enzyme.substrates:
nodecounter += 1
if substrate not in nodeDic:
nodeDic[substrate] = nodecounter
attr = {"label": substrate, "shape": "oval", "color": str((nodecounter % 8)+1)}
graph.add_node(nodeDic[substrate], **attr)
if enzyme.name == objective:
graph.add_nodes_from([(nodeDic[substrate], {"shape":"box"})])
for product in enzyme.products:
nodecounter += 1
if product not in nodeDic:
nodeDic[product] = nodecounter
attr = {"label": product, "shape": "oval", "color": str((nodecounter % 8)+1)}
graph.add_node(nodeDic[product], **attr)
if enzyme.name == objective:
graph.add_nodes_from([(nodeDic[product], {"shape":"box"})])
# Check if enzyme is objective
for substrate in enzyme.substrates:
for product in enzyme.products:
flux = changeDic[enzyme.name]
color = "black"
if flux < 0:
color = "red"
elif flux > 0:
color = "green"
value = flux
if value != 0:
value = (math.sqrt(math.pow(value, 2)) - oldMin) / oldRange * newRange + newMin
else:
value = 1
attr = {"color": color, "penwidth": value, "label": u"{} ({})".format(enzyme.name, flux), "object": enzyme}
if enzyme.name == objective:
attr["style"] = "dashed"
graph.add_edge(nodeDic[substrate], nodeDic[product], attr)
if enzyme.name == objective:
graph.add_edge(nodeDic[substrate], nodeDic[enzyme.name])
graph.add_edge(nodeDic[enzyme.name], nodeDic[product])
if enzyme.reversible:
graph.add_edge(nodeDic[product], nodeDic[substrate], label=enzyme.name, object=enzyme)
return graph, nodeDic
class PulseProgramUi(PulseProgramWidget, PulseProgramBase):
pulseProgramChanged = QtCore.pyqtSignal()
contextDictChanged = QtCore.pyqtSignal(object)
definitionWords = ['counter', 'var', 'shutter', 'parameter', 'masked_shutter', 'trigger', 'address', 'exitcode', 'const']
builtinWords = []
for key, val in SymbolTable().items(): #Extract the builtin words which should be highlighted
if type(val).__name__ == 'Builtin':
builtinWords.append(key)
SourceMode = enum('pp', 'ppp')
def __init__(self, config, parameterdict, channelNameData):
PulseProgramWidget.__init__(self)
PulseProgramBase.__init__(self)
self.dependencyGraph = DiGraph()
self.pulseProgram = PulseProgram.PulseProgram()
self.sourceCodeEdits = dict()
self.pppCodeEdits = dict()
self.config = config
self.variableTableModel = None
self.globalVariablesChanged = None
self.channelNameData = channelNameData
self.pppCompileException = None
self.globaldict = parameterdict
self.project = getProject()
self.defaultPPPDir = self.project.configDir+'/PulseProgramsPlus'
if not os.path.exists(self.defaultPPPDir):
os.makedirs(self.defaultPPPDir)
examplePPPDir = os.path.realpath(os.path.join(os.path.dirname(__file__), '..', 'config/PulseProgramsPlus')) #/IonControl/config/PulseProgramsPlus directory
for basename in os.listdir(examplePPPDir):
if basename.endswith('.ppp'):
pathname = os.path.join(examplePPPDir, basename)
if os.path.isfile(pathname):
shutil.copy(pathname, self.defaultPPPDir) #Copy over all example PPP pulse programs
self.defaultRAMDir = self.project.configDir+'/RAMFiles'
if not os.path.exists(self.defaultRAMDir):
exampleRAMDir = os.path.realpath(os.path.join(os.path.dirname(__file__), '..', 'config/RAMFiles')) #/IonControl/config/RAMFiles directory
shutil.copytree(exampleRAMDir, self.defaultRAMDir) #Copy over all example RAM files
def setupUi(self, experimentname, parent):
super(PulseProgramUi, self).setupUi(parent)
self.setCentralWidget(None) #No central widget
self.experimentname = experimentname
self.configname = 'PulseProgramUi.'+self.experimentname
self.contextDict = self.config.get( self.configname+'.contextdict', dict() )
self.populateDependencyGraph()
for context in self.contextDict.values(): # set the global dict as this field does not survive pickling
context.setGlobaldict(self.globaldict)
self.currentContext = self.config.get(self.configname + '.currentContext', PulseProgramContext(self.globaldict))
self.currentContext.setGlobaldict(self.globaldict)
self.configParams = self.config.get(self.configname, ConfiguredParams())
self.currentContextName = self.configParams.lastContextName
self.filenameComboBox.addItems([key for key, path in self.configParams.recentFiles.items() if os.path.exists(path)])
self.contextComboBox.addItems(sorted(self.contextDict.keys()))
self.parentComboBox.addItems([''] + sorted(self.contextDict.keys()))
self.ramFilenameComboBox.addItems([''] + [key for key, path in self.configParams.recentRamFiles.items() if os.path.exists(path)])
self.writeRamCheckbox.setChecked(self.currentContext.writeRam)
#setup documentation list
definitionDict, builtinDict, encodingDict = self.getDocs()
self.addDocs(definitionDict, "Variable Definitions")
self.addDocs(builtinDict, "Pulse Program Commands")
self.addDocs(encodingDict, "Encodings")
#connect actions
self.actionOpen.triggered.connect( self.onLoad )
self.actionSave.triggered.connect( self.onSave )
self.actionReset.triggered.connect(self.onReset)
self.loadButton.setDefaultAction( self.actionOpen )
self.saveButton.setDefaultAction( self.actionSave )
self.resetButton.setDefaultAction( self.actionReset )
self.loadButtonRam.clicked.connect( self.onLoadRamFile )
self.writeRamCheckbox.clicked.connect( self.onWriteRamCheckbox )
self.shutterTableView.setHorizontalHeader( RotatedHeaderView(QtCore.Qt.Horizontal, self.shutterTableView) )
self.triggerTableView.setHorizontalHeader( RotatedHeaderView(QtCore.Qt.Horizontal, self.triggerTableView ) )
self.counterTableView.setHorizontalHeader( RotatedHeaderView(QtCore.Qt.Horizontal, self.counterTableView ) )
self.reloadContextButton.clicked.connect( self.onReloadContext )
self.saveContextButton.clicked.connect( self.onSaveContext )
self.deleteContextButton.clicked.connect( self.onDeleteContext )
self.contextComboBox.currentIndexChanged[str].connect( self.onLoadContext )
self.parentComboBox.currentIndexChanged[str].connect(self.onSetParent)
self.linkAllButton.clicked.connect(self.onLinkAllToParent)
self.unlinkAllButton.clicked.connect(self.onUnlinkAllFromParent)
self.filenameComboBox.currentIndexChanged[str].connect( self.onFilenameChange )
self.ramFilenameComboBox.currentIndexChanged[str].connect( self.onRamFilenameChange )
self.removeCurrent.clicked.connect( self.onRemoveCurrent )
self.removeCurrentRamFile.clicked.connect( self.onRemoveCurrentRamFile )
self.variableTableModel = VariableTableModel( self.currentContext.parameters, self.config, self.currentContextName )
if self.globalVariablesChanged:
self.globalVariablesChanged.connect(self.variableTableModel.recalculateDependent )
self.variableView.setModel(self.variableTableModel)
self.variableView.resizeColumnToContents(0)
self.variableView.clicked.connect(self.onVariableViewClicked)
self.filter = KeyListFilter( [], [QtCore.Qt.Key_B, QtCore.Qt.Key_W, QtCore.Qt.Key_R] )
self.filter.controlKeyPressed.connect( self.onControl )
self.variableView.installEventFilter(self.filter)
self.shutterTableModel = ShutterTableModel( self.currentContext.shutters, self.channelNameData[0], size=48, globalDict=self.globaldict, channelSignal=self.channelNameData[1] )
self.triggerTableModel = TriggerTableModel( self.currentContext.triggers, self.channelNameData[2], size=32, globalDict=self.globaldict, channelSignal=self.channelNameData[3] )
self.counterTableModel = CounterTableModel( self.currentContext.counters, self.channelNameData[4], globalDict=self.globaldict )
self.delegates = list()
for model, view in [ (self.shutterTableModel, self.shutterTableView),
(self.triggerTableModel, self.triggerTableView),
(self.counterTableModel, self.counterTableView)]:
view.setModel(model)
delegate = MagnitudeSpinBoxDelegate(globalDict=self.globaldict)
self.delegates.append(delegate) # we need to keep those around, otherwise python will crash.
view.setItemDelegateForColumn(model.numericDataColumn, delegate)
if model.tristate:
view.setItemDelegateForColumn(model.numericMaskColumn, delegate)
view.clicked.connect(model.onClicked)
view.resizeColumnsToContents()
view.setupUi(self.globaldict)
if self.globalVariablesChanged:
self.globalVariablesChanged.connect( model.recalculateAllDependents )
self.counterIdDelegate = MagnitudeSpinBoxDelegate()
self.counterTableView.setItemDelegateForColumn(self.counterTableModel.idColumn, self.counterIdDelegate)
try:
self.loadContext(self.currentContext)
if self.configParams.lastContextName:
index = self.contextComboBox.findText(self.configParams.lastContextName)
with BlockSignals(self.contextComboBox) as w:
w.setCurrentIndex(index)
except:
logging.getLogger(__name__).exception("Loading of previous context failed")
#self.contextComboBox.editTextChanged.connect( self.updateSaveStatus )
self.contextComboBox.lineEdit().editingFinished.connect( self.updateSaveStatus )
self.variableTableModel.contentsChanged.connect( self.updateSaveStatus )
self.counterTableModel.contentsChanged.connect( self.updateSaveStatus )
self.shutterTableModel.contentsChanged.connect( self.updateSaveStatus )
self.triggerTableModel.contentsChanged.connect( self.updateSaveStatus )
self.setContextMenuPolicy( QtCore.Qt.ActionsContextMenu )
self.autoSaveAction = QtWidgets.QAction("Automatically save configuration", self)
self.autoSaveAction.setCheckable(True)
self.autoSaveAction.setChecked( self.configParams.autoSaveContext )
self.autoSaveAction.triggered.connect( self.onAutoSave )
self.addAction( self.autoSaveAction )
#background color context menu
setBackgroundColorAction = QtGui.QAction("Set Background Color", self)
setBackgroundColorAction.triggered.connect(self.onSetBackgroundColor)
self.addAction(setBackgroundColorAction)
removeBackgroundColorAction = QtGui.QAction("Remove Background Color", self)
removeBackgroundColorAction.triggered.connect(self.onRemoveBackgroundColor)
self.addAction(removeBackgroundColorAction)
self.initMenu()
self.tabifyDockWidget(self.shutterDock, self.triggerDock)
self.tabifyDockWidget(self.triggerDock, self.counterDock)
self.restoreLayout()
def populateDependencyGraph(self):
self.dependencyGraph = DiGraph()
self.dependencyGraph.add_nodes_from(self.contextDict.keys())
for name, context in self.contextDict.items():
if context.parentContext is not None:
try:
self.dependencySetParent(name, context.parentContext)
except CyclicDependencyError as ce:
context.parentContext = ce.parent
def dependencySetParent(self, child, parent):
parent = parent if parent else None
oldParentEdges = self.dependencyGraph.out_edges([child])
_, oldParent = first(oldParentEdges, (None, None))
if parent != oldParent:
self.dependencyGraph.remove_edges_from(oldParentEdges)
if parent:
self.dependencyGraph.add_edge(child, parent)
cycles = simple_cycles(self.dependencyGraph)
try:
cycle = next(cycles)
# StopIteration is raised if there are cycles
self.dependencyGraph.remove_edge(child, parent)
self.dependencyGraph.add_edges_from(oldParentEdges)
raise CyclicDependencyError(oldParent)
except StopIteration:
pass
def restoreLayout(self):
"""Restore layout from config settings"""
windowState = self.config.get(self.configname+".state")
if windowState: self.restoreState(windowState)
docSplitterState = self.config.get(self.configname+'.docSplitter')
if docSplitterState: self.docSplitter.restoreState(docSplitterState)
def initMenu(self):
self.menuView.clear()
dockList = self.findChildren(QtWidgets.QDockWidget)
for dock in dockList:
self.menuView.addAction(dock.toggleViewAction())
def onAutoSave(self, checked):
self.configParams.autoSaveContext = checked
if checked:
self.onSaveContext()
def loadContext(self, newContext ):
previousContext = self.currentContext
self.currentContext = copy.deepcopy(newContext)
#changeMode = self.currentContext.pulseProgramMode != previousContext.pulseProgramMode
if self.currentContext.pulseProgramFile != previousContext.pulseProgramFile or len(self.sourceCodeEdits)==0:
self.currentContext.pulseProgramFile = self.project.findFile(self.currentContext.pulseProgramFile)
self.adaptiveLoadFile(self.currentContext.pulseProgramFile)
if self.currentContext.ramFile != previousContext.ramFile or (self.currentContext.ramFile):
self.currentContext.ramFile = self.project.findFile(self.currentContext.ramFile)
self.loadRamFile(self.currentContext.ramFile)
self.mergeVariablesIntoContext( self.pulseProgram.variabledict )
self.updateDisplayContext()
self.updateSaveStatus(isSaved=True)
def onReloadContext(self):
self.loadContext( self.contextDict[str(self.contextComboBox.currentText())] )
self.updateSaveStatus()
def onSaveContext(self):
name = str(self.contextComboBox.currentText())
isNewContext = not name in self.contextDict
self.contextDict[ name ] = copy.deepcopy(self.currentContext)
if self.contextComboBox.findText(name)<0:
with BlockSignals(self.contextComboBox) as w:
w.addItem(name)
with BlockSignals(self.parentComboBox) as w:
w.addItem(name)
if isNewContext:
self.contextDictChanged.emit(list(self.contextDict.keys()))
self.updateSaveStatus(isSaved=True)
self.currentContextName = name
def onDeleteContext(self):
name = str(self.contextComboBox.currentText())
index = self.contextComboBox.findText(name)
if index>=0:
self.contextDict.pop(name)
self.contextComboBox.removeItem( index )
self.parentComboBox.removeItem(self.parentComboBox.findText(name))
self.contextDictChanged.emit(list(self.contextDict.keys()))
self.updateSaveStatus()
self.currentContextName = None
def onLoadContext(self):
name = str(self.contextComboBox.currentText())
self.currentContextName = name
if name in self.contextDict:
self.loadContext( self.contextDict[name] )
else:
self.onSaveContext()
def onSetParent(self, parent):
try:
parent = parent if parent else None
self.dependencySetParent(self.currentContextName, parent)
self.currentContext.parentContext = parent
self.variableTableModel.setContextHasParent(bool(parent))
except CyclicDependencyError as ce:
parent = ce.parent
self.currentContext.parentContext = parent
self.parentComboBox.setCurrentIndex(self.parentComboBox.findText(parent if parent else ''))
self.currentContext.parentContext = parent
self.setParentData(self.currentContext.parentContext)
def loadContextByName(self, name):
if name in self.contextDict:
self.loadContext( self.contextDict[name] )
with BlockSignals(self.contextComboBox) as w:
w.setCurrentIndex( w.findText( name ))
def updatepppDisplay(self):
for pppTab in list(self.pppCodeEdits.values()):
self.sourceTabs.removeTab( self.sourceTabs.indexOf(pppTab) )
self.pppCodeEdits = dict()
if self.currentContext.pulseProgramMode == 'ppp':
for name, text in [(self.pppSourceFile, self.pppSource)]:
textEdit = PulseProgramSourceEdit(mode='ppp')
encodingStrings = [encoding for encoding in EncodingDict.keys() if type(encoding) == str]
textEdit.setupUi(textEdit, extraKeywords1=self.definitionWords+encodingStrings, extraKeywords2=self.builtinWords)
textEdit.setPlainText(text)
self.pppCodeEdits[name] = textEdit
self.sourceTabs.addTab( textEdit, name )
def updateppDisplay(self):
for pppTab in list(self.sourceCodeEdits.values()):
self.sourceTabs.removeTab( self.sourceTabs.indexOf(pppTab) )
self.sourceCodeEdits = dict()
for name, text in self.pulseProgram.source.items():
textEdit = PulseProgramSourceEdit()
textEdit.setupUi(textEdit, extraKeywords1=self.definitionWords, extraKeywords2=[key for key in OPS])
textEdit.setPlainText(text)
self.sourceCodeEdits[name] = textEdit
self.sourceTabs.addTab( textEdit, name )
textEdit.setReadOnly( self.currentContext.pulseProgramMode!='pp' )
def updateDisplayContext(self):
self.setParentData(self.currentContext.parentContext)
self.variableTableModel.setVariables(self.currentContext.parameters, self.currentContextName)
self.variableTableModel.setContextHasParent(bool(self.currentContext.parentContext))
with BlockSignals(self.parentComboBox) as w:
w.setCurrentIndex(self.parentComboBox.findText(self.currentContext.parentContext
if self.currentContext.parentContext else ''))
self.variableView.resizeColumnsToContents()
self.shutterTableModel.setDataDict(self.currentContext.shutters)
self.triggerTableModel.setDataDict(self.currentContext.triggers)
self.counterTableModel.setDataDict(self.currentContext.counters)
self.writeRamCheckbox.setChecked(self.currentContext.writeRam)
def documentationString(self):
messages = [ "PulseProgram {0}".format( self.configParams.lastLoadFilename ) ]
r = "\n".join(messages)
return "\n".join( [r, self.pulseProgram.currentVariablesText()])
def description(self):
desc = dict()
desc["PulseProgram"] = self.configParams.lastLoadFilename
desc.update( self.pulseProgram.variables() )
return desc
def onFilenameChange(self, name ):
name = str(name)
if name in self.configParams.recentFiles and self.configParams.recentFiles[name]!=self.currentContext.pulseProgramFile:
self.adaptiveLoadFile(self.configParams.recentFiles[name])
if str(self.filenameComboBox.currentText())!=name:
with BlockSignals(self.filenameComboBox) as w:
w.setCurrentIndex( self.filenameComboBox.findText( name ))
self.updateSaveStatus()
def onRamFilenameChange(self, name ):
name = str(name)
if name in self.configParams.recentRamFiles and self.configParams.recentRamFiles[name]!=self.currentContext.ramFile:
self.loadRamFile(self.configParams.recentRamFiles[name])
if str(self.ramFilenameComboBox.currentText())!=name:
with BlockSignals(self.ramFilenameComboBox) as w:
w.setCurrentIndex( self.ramFilenameComboBox.findText( name ))
self.updateSaveStatus()
def onOk(self):
pass
def onLoadRamFile(self):
path, _ = QtWidgets.QFileDialog.getOpenFileName(self, 'Open RAM file', self.defaultRAMDir, filter='*.yml')
if path:
self.loadRamFile(path)
self.updateSaveStatus()
@QtCore.pyqtProperty(list)
def ramData(self):
return self.loadRamData(filename=self.currentContext.ramFile)
@QtCore.pyqtProperty(bool)
def writeRam(self):
return self.currentContext.writeRam
@file_data_cache(maxsize=3)
def loadRamData(self, filename=''):
"""load in the data from a RAM file"""
with open(filename, 'r') as f:
yamldata = yaml.load(f)
return [self.pulseProgram.convertParameter(Q(float(ramValue['value']), ramValue['unit']), ramValue['encoding']) for ramValue in yamldata]
def loadRamFile(self, path):
if path and os.path.exists(path):
self.currentContext.ramFile = path
filename = os.path.basename(path)
if filename not in self.configParams.recentRamFiles:
self.ramFilenameComboBox.addItem(filename)
self.configParams.recentRamFiles[filename]=path
with BlockSignals(self.ramFilenameComboBox) as w:
w.setCurrentIndex( self.ramFilenameComboBox.findText(filename))
else:
self.currentContext.ramFile = ''
with BlockSignals(self.ramFilenameComboBox) as w:
w.setCurrentIndex( self.ramFilenameComboBox.findText(''))
def onWriteRamCheckbox(self):
self.currentContext.writeRam = self.writeRamCheckbox.isChecked()
self.updateSaveStatus()
def onRemoveCurrentRamFile(self):
text = str(self.ramFilenameComboBox.currentText())
if text in self.configParams.recentRamFiles:
self.configParams.recentRamFiles.pop(text)
self.ramFilenameComboBox.removeItem(self.ramFilenameComboBox.currentIndex())
self.currentContext.ramFile = ''
self.updateSaveStatus()
def onLoad(self):
path, _ = QtWidgets.QFileDialog.getOpenFileName(self, 'Open Pulse Programmer file', self.defaultPPPDir, filter='*.ppp *.pp')
if path!="":
self.adaptiveLoadFile(path)
self.updateSaveStatus()
def adaptiveLoadFile(self, path):
if path:
_, ext = os.path.splitext(path)
self.currentContext.pulseProgramFile = path
if ext==".ppp":
self.currentContext.pulseProgramMode = 'ppp'
self.loadpppFile(path)
else:
self.currentContext.pulseProgramMode = 'pp'
self.updatepppDisplay()
self.loadppFile(path)
self.configParams.lastLoadFilename = path
def onReset(self):
if self.configParams.lastLoadFilename is not None:
self.variabledict = VariableDictionary( self.pulseProgram.variabledict, self.parameterdict )
self.adaptiveLoadFile(self.configParams.lastLoadFilename)
def loadpppFile(self, path):
self.pppSourcePath = path
_, self.pppSourceFile = os.path.split(path)
with open(path, "r") as f:
self.pppSource = f.read()
self.updatepppDisplay()
ppFilename = getPpFileName(path)
if self.compileppp(ppFilename):
self.loadppFile(ppFilename, cache=False)
filename = os.path.basename(path)
if filename not in self.configParams.recentFiles:
self.filenameComboBox.addItem(filename)
self.configParams.recentFiles[filename]=path
with BlockSignals(self.filenameComboBox) as w:
w.setCurrentIndex( self.filenameComboBox.findText(filename))
def saveppp(self, path):
if self.pppSource and path:
with open(path, 'w') as f:
f.write( self.pppSource )
def compileppp(self, savefilename):
self.pppSource = self.pppSource.expandtabs(4)
success = False
try:
compiler = pppCompiler()
ppCode = compiler.compileString( self.pppSource )
self.pppReverseLineLookup = compiler.reverseLineLookup
self.pppCompileException = None
with open(savefilename, "w") as f:
f.write(ppCode)
success = True
self.pppCodeEdits[self.pppSourceFile].clearHighlightError()
except CompileException as e:
self.pppCodeEdits[self.pppSourceFile].highlightError( e.message(), e.lineno(), col=e.col())
except ParseException as e:
e.__class__ = CompileException # cast to CompileException. Is possible because CompileException does ONLY add behavior
self.pppCodeEdits[self.pppSourceFile].highlightError( e.message(), e.lineno(), col=e.col())
return success
def loadppFile(self, path, cache=True):
self.pulseProgram.loadSource(path, docompile=False)
self.updateppDisplay()
try:
self.pulseProgram.compileCode()
except PulseProgram.ppexception as compileexception:
self.sourceCodeEdits[ compileexception.file ].highlightError(str(compileexception), compileexception.line, compileexception.context )
if cache:
filename = os.path.basename(path)
if filename not in self.configParams.recentFiles:
self.filenameComboBox.addItem(filename)
self.configParams.recentFiles[filename]=path
self.filenameComboBox.setCurrentIndex( self.filenameComboBox.findText(filename))
# Merge any new values into the current context
self.mergeVariablesIntoContext( self.pulseProgram.variabledict )
self.updateDisplayContext()
self.pulseProgramChanged.emit()
def mergeVariablesIntoContext( self, variabledict ):
"""Merge variables into context, propagating variable changes up the parentContext list"""
# Follow the parenting links
parent = self.currentContext.parentContext
parents = []
while parent is not None and parent in self.contextDict and parent not in parents:
parentContext = self.contextDict[parent]
if parentContext.pulseProgramFile != self.currentContext.pulseProgramFile:
break # Stop if the parent's pulse program does not match the current
parents.append(parent) # Prevents infinite loops
parent = parentContext.parentContext
# Propagate variable changes up the parentContext tree in reverse order so the variable
# linking up the tree doesn't break.
linkNewToParent = False
for parent in reversed(parents):
self.contextDict[parent].merge( variabledict, linkNewToParent=linkNewToParent)
linkNewToParent = True
# Finally, merge into the current context
self.currentContext.merge( variabledict, linkNewToParent=linkNewToParent )
def onRemoveCurrent(self):
text = str(self.filenameComboBox.currentText())
if text in self.configParams.recentFiles:
self.configParams.recentFiles.pop(text)
self.filenameComboBox.removeItem(self.filenameComboBox.currentIndex())
def onSave(self):
self.onApply()
if self.currentContext.pulseProgramMode=='pp':
self.pulseProgram.saveSource()
else:
self.saveppp(self.pppSourcePath)
def onApply(self):
if self.currentContext.pulseProgramMode=='pp':
try:
positionCache = dict()
for name, textEdit in self.sourceCodeEdits.items():
self.pulseProgram.source[name] = str(textEdit.toPlainText())
positionCache[name] = ( textEdit.textEdit.cursorPosition(),
textEdit.textEdit.scrollPosition() )
self.pulseProgram.loadFromMemory()
self.updateppDisplay()
for name, textEdit in self.sourceCodeEdits.items():
textEdit.clearHighlightError()
if name in positionCache:
cursorpos, scrollpos = positionCache[name]
textEdit.textEdit.setCursorPosition( *cursorpos )
textEdit.textEdit.setScrollPosition( scrollpos )
except PulseProgram.ppexception as ppex:
textEdit = self.sourceCodeEdits[ ppex.file ].highlightError(str(ppex), ppex.line, ppex.context )
else:
positionCache = dict()
for name, textEdit in self.pppCodeEdits.items():
self.pppSource = str(textEdit.toPlainText())
positionCache[name] = ( textEdit.textEdit.cursorPosition(),
textEdit.textEdit.scrollPosition() )
ppFilename = getPpFileName( self.pppSourcePath )
if self.compileppp(ppFilename):
self.loadppFile(ppFilename, cache=False)
for name, textEdit in self.pppCodeEdits.items():
textEdit.clearHighlightError()
if name in positionCache:
cursorpos, scrollpos = positionCache[name]
textEdit.textEdit.setCursorPosition( *cursorpos )
textEdit.textEdit.setScrollPosition( scrollpos )
def onAccept(self):
self.saveConfig()
def onReject(self):
pass
def saveConfig(self):
"""Save the pulse program configuration state"""
self.configParams.lastContextName = str(self.contextComboBox.currentText())
self.config[self.configname+".state"] = self.saveState() #Arrangement of dock widgets
self.config[self.configname] = self.configParams
self.config[self.configname+'.contextdict'] = self.contextDict
self.config[self.configname+'.currentContext'] = self.currentContext
self.config[self.configname+'.docSplitter'] = self.docSplitter.saveState()
self.variableTableModel.saveConfig()
def getPulseProgramBinary(self,parameters=dict(),override=dict()):
# need to update variables self.pulseProgram.updateVariables( self.)
substitutes = dict(self.currentContext.parameters.valueView.items())
for model in [self.shutterTableModel, self.triggerTableModel, self.counterTableModel]:
substitutes.update( model.getVariables() )
substitutes.update(override)
self.pulseProgram.updateVariables(substitutes)
return self.pulseProgram.toBinary()
def exitcode(self, number):
return self.pulseProgram.exitcode(number)
def getVariableValue(self, name):
return self.variableTableModel.getVariableValue(name)
def variableScanCode(self, variablename, values, extendedReturn=False):
tempparameters = copy.deepcopy( self.currentContext.parameters )
updatecode = list()
numVariablesPerUpdate = 0
for currentval in values:
upd_names, upd_values = tempparameters.setValue(variablename, currentval)
numVariablesPerUpdate = len(upd_names)
upd_names.append( variablename )
upd_values.append( currentval )
updatecode.extend( self.pulseProgram.multiVariableUpdateCode( upd_names, upd_values ) )
logging.getLogger(__name__).info("{0}: {1}".format(upd_names, upd_values))
if extendedReturn:
return updatecode, numVariablesPerUpdate
return updatecode
def updateSaveStatus(self, isSaved=None):
try:
if isSaved is None:
currentText = str(self.contextComboBox.currentText())
if not currentText:
self.contextSaveStatus = True
elif currentText in self.contextDict:
self.contextSaveStatus = self.contextDict[currentText]==self.currentContext
else:
self.contextSaveStatus = False
if self.configParams.autoSaveContext and not self.contextSaveStatus:
self.onSaveContext()
self.contextSaveStatus = True
else:
self.contextSaveStatus = isSaved
self.saveContextButton.setEnabled( not self.contextSaveStatus )
except Exception:
pass
def onControl(self, key):
if key==QtCore.Qt.Key_B:
self.onBold()
elif key==QtCore.Qt.Key_W:
self.onSetBackgroundColor()
elif key==QtCore.Qt.Key_R:
self.onRemoveBackgroundColor()
def onBold(self):
indexes = self.variableView.selectedIndexes()
for index in indexes:
self.variableTableModel.toggleBold( index )
def onSetBackgroundColor(self):
indexes = self.variableView.selectedIndexes()
if indexes:
color = QtGui.QColorDialog.getColor()
if not color.isValid():
color = None
for index in indexes:
self.variableTableModel.setBackgroundColor(index, color)
def onRemoveBackgroundColor(self):
indexes = self.variableView.selectedIndexes()
for index in indexes:
self.variableTableModel.removeBackgroundColor(index)
def lineOfInstruction(self, binaryelement ):
ppline = self.pulseProgram.lineOfInstruction(binaryelement)
pppline = self.pppReverseLineLookup.get(ppline, None) if hasattr(self, 'pppReverseLineLookup') else None
return ppline, pppline
def setTimingViolations(self, linelist):
edit = self.pppCodeEdits.get(self.pppSourceFile)
if edit:
edit.highlightTimingViolation( [l[1]-1 for l in linelist] )
def getDocs(self):
"""Assemble the pulse program function documentation into dictionaries"""
definitionDocPath = os.path.join(os.path.dirname(__file__), '..', r'docs/manual/pppDefinitionDocs.include')
definitionDict = self.readDocFile(definitionDocPath)
encodingDocPath = os.path.join(os.path.dirname(__file__), '..', r'docs/manual/pppEncodingDocs.include')
encodingDict = self.readDocFile(encodingDocPath)
builtinDict = OrderedDict()
symbolTable = SymbolTable()
for name in self.builtinWords:
builtinDict[name] = symbolTable[name].doc or "This should be documentation for builtin word {0}".format(name)
return definitionDict, builtinDict, encodingDict
def readDocFile(self, filename):
"""Read in the rst file 'filename' """
docdict = OrderedDict()
try:
with open(filename, 'r') as f:
docs = f.read()
sepdocs = docs.split('.. py:data:: ')
for doc in sepdocs:
if doc:
doclines = doc.splitlines()
name = doclines.pop(0)
for line in doclines:
if line:
documentation = line.strip() #Take the first line with content as the documentation to display in the program
break
docdict[name] = documentation
except Exception as e:
logging.getLogger(__name__).warning("Unable to load documentation: {0}".format(e))
return docdict
def addDocs(self, docDict, category):
"""Add the documentation dictionary docDict to the docTree under 'category' """
categoryItem = QtWidgets.QTreeWidgetItem(self.docTreeWidget, [category])
self.docTreeWidget.addTopLevelItem(categoryItem)
for name, documentation in docDict.items():
nameItem = QtWidgets.QTreeWidgetItem(categoryItem, [name])
label = QtWidgets.QLabel(documentation)
label.setWordWrap(True)
docItem = QtWidgets.QTreeWidgetItem(nameItem)
self.docTreeWidget.setItemWidget(docItem, 0, label)
def onVariableViewClicked(self, index):
if index.column() == 1 and self.currentContext.parentContext:
var = self.currentContext.parameters.at(index.row())
if var.hasParent:
var.useParentValue ^= True
try:
self.setParentData(self.currentContext.parentContext, var)
except KeyError:
var.hasParent = False
self.variableTableModel.onClicked(index)
self.updateSaveStatus()
def onLinkAllToParent(self):
""" Link all variables to their parents """
self.variableTableModel.beginResetModel()
if self.currentContext.parentContext:
for var in self.currentContext.parameters.values():
if var.hasParent:
var.useParentValue = True
self.setParentData(self.currentContext.parentContext)
self.variableTableModel.endResetModel()
self.updateSaveStatus()
def onUnlinkAllFromParent(self):
""" Unlink all variables from their parents """
self.variableTableModel.beginResetModel()
for var in self.currentContext.parameters.values():
var.useParentValue = False
self.setParentData(self.currentContext.parentContext)
self.variableTableModel.endResetModel()
self.updateSaveStatus()
def setParentData(self, parentContext, var=None):
for var in [var] if var is not None else self.currentContext.parameters.values():
try:
var.hasParent = bool(parentContext)
if var.useParentValue and var.hasParent:
rootName = self.findControllingNode(var.name)
self.setParentValue(var.name, rootName)
var.parentObject = self.contextDict[rootName].parameters[var.name]
else:
self.currentContext.parameters.setStrValue(var.name, var.strvalue)
var.parentObject = None
except KeyError:
var.hasParent = False
def findControllingNode(self, paramName):
current, parent = self.currentContextName, self.currentContext.parentContext
var = self.currentContext.parameters[paramName]
child = None
if not (parent and var.useParentValue):
return None
for child, parent in dfs_edges(self.dependencyGraph, parent):
try:
var = self.contextDict[parent].parameters[paramName]
if not (var.useParentValue and var.hasParent):
return child
except KeyError:
return child
return parent
def setParentValue(self, paramName, contextName):
self.currentContext.parameters.setParentValue(paramName,
self.contextDict[contextName].parameters[paramName].value)
self.currentContext.parameters.setParentStrValue(paramName,
self.contextDict[contextName].parameters[paramName].strvalue)
class MacroManager(object):
"""This class manages the macros specified in the configuration file.
The parameters of each section, along with their dependencies are passed to
the class. Then, it verifies that the dependencies are correct (they form a
DAG and respect the sections dependencies) and creates an ordered list of
the macros to be used when replacing their actual values in a given
combination.
"""
def __init__(self):
"""Create a new MacroManager object."""
self.dep_graph = DiGraph()
self.ds_macros = set([])
self.xp_macros = set([])
self.__define_test_macros()
def __define_test_macros(self):
"""Define values and dependencies of test macros.
A set of macros are defined by default, including input and output
directories of datasets and experiments and their identifiers.
"""
self.test_macros = {
"data_base_dir": "/tests/data",
"out_base_dir": "/tests/out",
"data_dir": "/tests/data/0", # data_base_dir/ds_id
"out_dir": "/tests/out/0", # data_base_dir/comb_id
"comb_id": 0,
"ds_id": 0,
"xp.input": "/tests/data/0", # data_dir
"xp.output": "/tests/out/0" # out_dir
}
self.ds_params = set([])
self.xp_params = set([])
self.dep_graph.add_nodes_from(self.test_macros.keys())
self.add_dependency("data_base_dir", "data_dir")
self.add_dependency("ds_id", "data_dir")
self.add_dependency("data_dir", "xp.input")
self.add_dependency("out_base_dir", "out_dir")
self.add_dependency("comb_id", "out_dir")
self.add_dependency("out_dir", "xp.output")
self.sorted_test_macros = topological_sort(self.dep_graph)
def update_test_macros(self, ds_id=None, comb_id=None):
"""Update test macros with dataset and/or combination ids.
Args:
ds_id (int, optional):
The dataset identifier.
comb_id (int, optional):
The combination identifier.
"""
if ds_id:
if "data_dir" in self.test_macros:
self.test_macros["data_dir"] = \
self.test_macros["data_base_dir"] + "/" + str(ds_id)
if "xp.input" in self.test_macros:
self.test_macros["xp.input"] = \
self.test_macros["data_dir"]
if comb_id:
if "out_dir" in self.test_macros:
self.test_macros["out_dir"] = \
self.test_macros["out_base_dir"] + "/" + str(comb_id)
if "xp.output" in self.test_macros:
self.test_macros["xp.output"] = \
self.test_macros["out_dir"]
def __filter_unused_test_macros(self):
for m in reversed(self.sorted_test_macros):
if not self.dep_graph.successors(m):
self.dep_graph.remove_node(m)
self.sorted_test_macros.remove(m)
del self.test_macros[m]
def add_ds_params(self, params):
"""Add the list of dataset parameters.
Args:
params (dict):
The list of dataset parameters.
"""
self.ds_params = self.ds_params.union(params)
def add_xp_params(self, params):
"""Add the list of experiment parameters.
Args:
params (dict):
The list of experiment parameters.
"""
self.xp_params = self.xp_params.union(params)
def add_dependency(self, m1, m2):
"""Include a new macro dependency: m1 -> m2. This means that to obtain
the value of m2 we use the value of m1.
Args:
m1 (string):
The name of the param used.
m2 (string):
The name of the param being specified.
Raises:
MacroException:
If the order of sections (test -> ds -> xp) is not respected.
"""
# Check if dependency is correct
if m1 in self.ds_params:
if m2 in self.test_macros:
logger.error("Not allowed dependency: ds -> test")
raise MacroException("Not allowed dependency: ds -> test")
elif m1 in self.xp_params:
if m2 in self.test_macros:
logger.error("Not allowed dependency: xp -> test")
raise MacroException("Not allowed dependency: xp -> test")
elif m2 in self.ds_params:
logger.error("Not allowed dependency: xp -> ds")
raise MacroException("Not allowed dependency: xp -> ds")
# Add dependency
self.dep_graph.add_edge(m1, m2)
def sort_macros(self):
"""Sort macros respecting dependencies.
Raises:
MacroException:
If there are cycles in dependencies between macros.
"""
# Filter out unused test variables
self.__filter_unused_test_macros()
# Sort ds and xp macros
try:
self.sorted_ds_macros = \
topological_sort(self.dep_graph.subgraph(self.ds_params))
self.sorted_xp_macros = \
topological_sort(self.dep_graph.subgraph(self.xp_params))
except NetworkXUnfeasible:
raise MacroException("Macros do not follow a DAG")
logger.info("Dependencies = " + str(self.dep_graph.edges()))
logger.info("Test macros = " + str(self.sorted_test_macros))
logger.info("Dataset macros = " + str(self.sorted_ds_macros))
logger.info("Experiment macros = " + str(self.sorted_xp_macros))
def _replace_macros_from_list(self, list_macros, value):
"""Replace the macros given in the list within the value if present.
Args:
list_macros (dict):
The list of macros to replace and their respective values.
value (string):
The value where to do the replacement.
"""
new_value = value
for m in list_macros:
new_value = new_value.replace("${" + m + "}", str(list_macros[m]))
return new_value
def replace_ds_macros(self, comb):
"""Replace macros in ds combination.
Args:
comb (dict):
The combination of parameters.
"""
list_macros = self.test_macros
for m in self.sorted_ds_macros:
comb[m] = self._replace_macros_from_list(list_macros, comb[m])
list_macros[m] = comb[m]
def replace_xp_macros(self, comb):
"""Replace macros in xp combination.
Args:
comb (dict):
The combination of parameters.
"""
list_macros = self.test_macros
for m in self.sorted_ds_macros:
comb[m] = self._replace_macros_from_list(list_macros, comb[m])
list_macros[m] = comb[m]
for m in self.sorted_xp_macros:
comb[m] = self._replace_macros_from_list(list_macros, comb[m])
list_macros[m] = comb[m]
def make_warehouse():
G = DiGraph()
# Nodes for all states in scope. #
G.add_nodes_from([(x, y, d) for x in range(0, 9) for y in range(0, 5) for d in ['N', 'S', 'E', 'W']])
##### Turns #####
# Squares/Corners #
G.add_edges_from([((0, y, 'W'), (0, y, 'E')) for y in range(0,5)], weight = 1, inst = "u") # Column 0
G.add_edges_from([((8, y, 'E'), (8, y, 'W')) for y in range(0,5)], weight = 1, inst = "u") # Column 8
G.add_edges_from([((x, y, 'W'), (x, y, 'E')) for x in [3, 6] for y in range(1,4)], weight = 1, inst = "u") # Column 2, 5
G.add_edges_from([((x, y, 'E'), (x, y, 'W')) for x in [2, 5] for y in range(1,4)], weight = 1, inst = "u") # Column 2, 5
# Middle Spots #
midRights = [('N', 'E'), ('E', 'S'), ('W', 'N'), ('S', 'W')]
G.add_edges_from([((x, y, ds), (x, y, dt)) for x in [1, 4, 7] for y in range(1, 4) for ds, dt in midRights], weight = 1, inst = "r")
G.add_edges_from([((x, y, ds), (x, y, dt)) for x in [1, 4, 7] for y in range(1, 4) for dt, ds in midRights], weight = 1, inst = "l")
# Top Edges #
topRights = [('N', 'E'), ('E', 'S'), ('S', 'W'), ('W', 'E')]
topLefts = [('N', 'W'), ('W', 'S'), ('S', 'E'), ('E', 'W')]
G.add_edges_from([((x, 0, ds), (x, 0, dt)) for x in [1, 4, 7] for ds, dt in topRights], weight = 1, inst = "r") # Column 1 Top - Rights
G.add_edges_from([((x, 0, ds), (x, 0, dt)) for x in [1, 4, 7] for ds, dt in topLefts], weight = 1, inst = "l") # Column 1 Top - Lefts
# Bottom Edges #
bottomRights = [('N', 'E'), ('W', 'N'), ('S', 'W'), ('E', 'W')]
bottomLefts = [('N', 'W'), ('W', 'E'), ('E', 'N'), ('S', 'E')]
G.add_edges_from([((x, 4, ds), (x, 4, dt)) for x in [1, 4, 7] for ds, dt in bottomRights], weight = 1, inst = "r") # Column 1 Bottom - Rights
G.add_edges_from([((x, 4, ds), (x, 4, dt)) for x in [1, 4, 7] for ds, dt in bottomLefts], weight = 1, inst = "l") # Column 1 Bottom - Lefts
##### Paths #####
# Columns 0 and 8 #
for y in range(0, 5):
G.add_edge((0, y, 'E'), (1, y, 'E'), weight = 1, inst = 't') # Col 0 to Col 1
G.add_edge((8, y, 'W'), (7, y, 'W'), weight = 1, inst = 't') # Col 8 to Col 7
# Columns 2 and 5
G.add_edges_from([((x, y, 'W'), (x-1, y, 'W')) for x in [2, 5] for y in range(1, 4)], weight = 1, inst = 't')
# Columns 3 and 6 #
G.add_edges_from([((x, y, 'E'), (x+1, y, 'E')) for x in [3, 6] for y in range(1, 4)], weight = 1, inst = 't')
# Columns 1, 4, and 7 #
for x in [1, 4, 7]:
for y in range(0, 5):
if y != 0:
G.add_edge((x, y, "N"), (x, y-1, "N"), weight = 1, inst = 't')
if y != 4:
G.add_edge((x, y, "S"), (x, y+1, "S"), weight = 1, inst = 't')
for y in range(1, 4):
G.add_edge((x, y, "W"), (x-1, y, "W"), weight = 1, inst = 't')
G.add_edge((x, y, "E"), (x+1, y, "E"), weight = 1, inst = 't')
for y in [0, 4]:
if x in [1, 4]:
G.add_edge((x, y, "E"), (x+3, y, "E"), weight = 1, inst = 't')
if x in [4, 7]:
G.add_edge((x, y, "W"), (x-3, y, "W"), weight = 1, inst = 't')
if x == 1:
G.add_edge((x, y, "W"), (x-1, y, "W"), weight = 1, inst = 't')
if x == 7:
G.add_edge((x, y, "W"), (x+1, y, "E"), weight = 1, inst = 't')
return G
