def load(self, fname, verbose=True, **kwargs):
"""
Load a data file. The expected data format is three columns
(comma seperated by default) with source, target, flux.
No header should be included and the node IDs have to run contuously
from 0 to Number_of_nodes-1.
Parameters
----------
fname : str
Path to the file
verbose : bool
Print information about the data. True by Default
kwargs : dict
Default parameters can be changed here. Supported key words are
dtype : float (default)
delimiter : "," (default)
return_graph : bool
If True, the graph is returned (False by default).
Returns:
--------
The graph is saved internally in self.graph.
"""
delimiter = kwargs["delimiter"] if "delimiter" in kwargs.keys(
) else " "
data = np.genfromtxt(fname,
delimiter=delimiter,
dtype=int,
unpack=False)
source, target = data[:, 0], data[:, 1]
if data.shape[1] > 2:
flux = data[:, 2]
else:
flux = np.ones_like(source)
nodes = set(source) | set(target)
self.nodes = len(nodes)
lines = len(flux)
if set(range(self.nodes)) != nodes:
new_node_ID = {old: new for new, old in enumerate(nodes)}
map_new_node_ID = np.vectorize(new_node_ID.__getitem__)
source = map_new_node_ID(source)
target = map_new_node_ID(target)
if verbose:
print(
"\nThe node IDs have to run continuously from 0 to Number_of_nodes-1."
)
print(
"Node IDs have been changed according to the requirement.\n-----------------------------------\n"
)
print('Lines: ', lines, ', Nodes: ', self.nodes)
print(
'-----------------------------------\nData Structure:\n\nsource, target, weight \n'
)
for ii in range(7):
print("%i, %i, %1.2e" %
(source[ii], target[ii], flux[ii]))
print('-----------------------------------\n')
G = DiGraph() # Empty, directed Graph
G.add_nodes_from(range(self.nodes))
for ii in range(lines):
u, v, w = int(source[ii]), int(target[ii]), float(flux[ii])
if u != v: # ignore self loops
assert not G.has_edge(u,
v), "Edge appeared twice - not supported"
G.add_edge(u, v, weight=w)
else:
if verbose:
print("ignore self loop at node", u)
symmetric = True
for s, t, w in G.edges(data=True):
w1 = G[s][t]["weight"]
try:
w2 = G[t][s]["weight"]
except KeyError:
symmetric = False
G.add_edge(t, s, weight=w1)
w2 = w1
if w1 != w2:
symmetric = False
G[s][t]["weight"] += G[t][s]["weight"]
G[s][t]["weight"] /= 2
G[t][s]["weight"] = G[s][t]["weight"]
if verbose:
if not symmetric:
print("The network has been symmetricised.")
ccs = [G.subgraph(c).copy() for c in strongly_connected_components(G)]
ccs = sorted(ccs, key=len, reverse=True)
G_GSCC = ccs[0]
if G_GSCC.number_of_nodes() != G.number_of_nodes():
G = G_GSCC
if verbose:
print(
"\n--------------------------------------------------------------------------"
)
print(
"The network has been restricted to the giant strongly connected component."
)
self.nodes = G.number_of_nodes()
for u, v, data in G.edges(data=True):
weight = G.out_degree(u, weight='weight')
data['transition_rate'] = 1. * data['weight'] / weight
for u, v, data in G.edges(data=True):
data['effective_distance'] = 1. - log(data['transition_rate'])
if verbose:
print(
"\n--------------------------------------------------------------------------"
)
print(
"\nnode ID, out-weight, normalized out-weight, sum of effective distances \n "
)
for ii in range(7):
out_edges = G.out_edges(ii, data=True)
out_weight, effective_distance, transition_rate = 0, 0, 0
for u, v, data in out_edges:
out_weight += data["weight"]
effective_distance += data["effective_distance"]
transition_rate += data["transition_rate"]
print(
" %i %1.2e %2.3f %1.2e " %
(ii, out_weight, transition_rate, effective_distance))
print("\n ... graph is saved in self.graph")
return G
class TaskGraph(object):
"""
A tasks graph builder.
Build an operations flow graph
"""
def __init__(self, name):
self.name = name
self._id = generate_uuid(variant='uuid')
self._graph = DiGraph()
def __repr__(self):
return '{name}(id={self._id}, name={self.name}, graph={self._graph!r})'.format(
name=self.__class__.__name__, self=self)
@property
def id(self):
"""
Represents the id of the graph
:return: graph id
"""
return self._id
# graph traversal methods
@property
def tasks(self):
"""
An iterator on tasks added to the graph
:yields: Iterator over all tasks in the graph
"""
for _, data in self._graph.nodes_iter(data=True):
yield data['task']
def topological_order(self, reverse=False):
"""
Returns topological sort on the graph
:param reverse: whether to reverse the sort
:return: a list which represents the topological sort
"""
for task_id in topological_sort(self._graph, reverse=reverse):
yield self.get_task(task_id)
def get_dependencies(self, dependent_task):
"""
Iterates over the task's dependencies
:param BaseTask dependent_task: The task whose dependencies are requested
:yields: Iterator over all tasks which dependency_task depends on
:raise: TaskNotInGraphError if dependent_task is not in the graph
"""
if not self.has_tasks(dependent_task):
raise TaskNotInGraphError('Task id: {0}'.format(dependent_task.id))
for _, dependency_id in self._graph.out_edges_iter(dependent_task.id):
yield self.get_task(dependency_id)
def get_dependents(self, dependency_task):
"""
Iterates over the task's dependents
:param BaseTask dependency_task: The task whose dependents are requested
:yields: Iterator over all tasks which depend on dependency_task
:raise: TaskNotInGraphError if dependency_task is not in the graph
"""
if not self.has_tasks(dependency_task):
raise TaskNotInGraphError('Task id: {0}'.format(
dependency_task.id))
for dependent_id, _ in self._graph.in_edges_iter(dependency_task.id):
yield self.get_task(dependent_id)
# task methods
def get_task(self, task_id):
"""
Get a task instance that's been inserted to the graph by the task's id
:param basestring task_id: The task's id
:return: Requested task
:rtype: BaseTask
:raise: TaskNotInGraphError if no task found in the graph with the given id
"""
if not self._graph.has_node(task_id):
raise TaskNotInGraphError('Task id: {0}'.format(task_id))
data = self._graph.node[task_id]
return data['task']
@_filter_out_empty_tasks
def add_tasks(self, *tasks):
"""
Add a task to the graph
:param BaseTask task: The task
:return: A list of added tasks
:rtype: list
"""
assert all([
isinstance(task, (api_task.BaseTask, Iterable)) for task in tasks
])
return_tasks = []
for task in tasks:
if isinstance(task, Iterable):
return_tasks += self.add_tasks(*task)
elif not self.has_tasks(task):
self._graph.add_node(task.id, task=task)
return_tasks.append(task)
return return_tasks
@_filter_out_empty_tasks
def remove_tasks(self, *tasks):
"""
Remove the provided task from the graph
:param BaseTask task: The task
:return: A list of removed tasks
:rtype: list
"""
return_tasks = []
for task in tasks:
if isinstance(task, Iterable):
return_tasks += self.remove_tasks(*task)
elif self.has_tasks(task):
self._graph.remove_node(task.id)
return_tasks.append(task)
return return_tasks
@_filter_out_empty_tasks
def has_tasks(self, *tasks):
"""
Check whether a task is in the graph or not
:param BaseTask task: The task
:return: True if all tasks are in the graph, otherwise True
:rtype: list
"""
assert all(isinstance(t, (api_task.BaseTask, Iterable)) for t in tasks)
return_value = True
for task in tasks:
if isinstance(task, Iterable):
return_value &= self.has_tasks(*task)
else:
return_value &= self._graph.has_node(task.id)
return return_value
def add_dependency(self, dependent, dependency):
"""
Add a dependency for one item (task, sequence or parallel) on another
The dependent will only be executed after the dependency terminates
If either of the items is either a sequence or a parallel,
multiple dependencies may be added
:param BaseTask|_TasksArrangement dependent: The dependent (task, sequence or parallel)
:param BaseTask|_TasksArrangement dependency: The dependency (task, sequence or parallel)
:return: True if the dependency between the two hadn't already existed, otherwise False
:rtype: bool
:raise TaskNotInGraphError if either the dependent or dependency are tasks which
are not in the graph
"""
if not (self.has_tasks(dependent) and self.has_tasks(dependency)):
raise TaskNotInGraphError()
if self.has_dependency(dependent, dependency):
return
if isinstance(dependent, Iterable):
for dependent_task in dependent:
self.add_dependency(dependent_task, dependency)
else:
if isinstance(dependency, Iterable):
for dependency_task in dependency:
self.add_dependency(dependent, dependency_task)
else:
self._graph.add_edge(dependent.id, dependency.id)
def has_dependency(self, dependent, dependency):
"""
Check whether one item (task, sequence or parallel) depends on another
Note that if either of the items is either a sequence or a parallel,
and some of the dependencies exist in the graph but not all of them,
this method will return False
:param BaseTask|_TasksArrangement dependent: The dependent (task, sequence or parallel)
:param BaseTask|_TasksArrangement dependency: The dependency (task, sequence or parallel)
:return: True if the dependency between the two exists, otherwise False
:rtype: bool
:raise TaskNotInGraphError if either the dependent or dependency are tasks
which are not in the graph
"""
if not (dependent and dependency):
return False
elif not (self.has_tasks(dependent) and self.has_tasks(dependency)):
raise TaskNotInGraphError()
return_value = True
if isinstance(dependent, Iterable):
for dependent_task in dependent:
return_value &= self.has_dependency(dependent_task, dependency)
else:
if isinstance(dependency, Iterable):
for dependency_task in dependency:
return_value &= self.has_dependency(
dependent, dependency_task)
else:
return_value &= self._graph.has_edge(dependent.id,
dependency.id)
return return_value
def remove_dependency(self, dependent, dependency):
"""
Remove a dependency for one item (task, sequence or parallel) on another
Note that if either of the items is either a sequence or a parallel, and some of
the dependencies exist in the graph but not all of them, this method will not remove
any of the dependencies and return False
:param BaseTask|_TasksArrangement dependent: The dependent (task, sequence or parallel)
:param BaseTask|_TasksArrangement dependency: The dependency (task, sequence or parallel)
:return: False if the dependency between the two hadn't existed, otherwise True
:rtype: bool
:raise TaskNotInGraphError if either the dependent or dependency are tasks
which are not in the graph
"""
if not (self.has_tasks(dependent) and self.has_tasks(dependency)):
raise TaskNotInGraphError()
if not self.has_dependency(dependent, dependency):
return
if isinstance(dependent, Iterable):
for dependent_task in dependent:
self.remove_dependency(dependent_task, dependency)
elif isinstance(dependency, Iterable):
for dependency_task in dependency:
self.remove_dependency(dependent, dependency_task)
else:
self._graph.remove_edge(dependent.id, dependency.id)
@_filter_out_empty_tasks
def sequence(self, *tasks):
"""
Create and insert a sequence into the graph, effectively each task i depends on i-1
:param tasks: an iterable of dependencies
:return: the provided tasks
"""
if tasks:
self.add_tasks(*tasks)
for i in xrange(1, len(tasks)):
self.add_dependency(tasks[i], tasks[i - 1])
return tasks
class TSN(object):
"""
Time-Space-Network Object with generator and updater functions.
Parameters
----------
flights : list,
list of :class:`classes.Flight` objects
"""
def __init__(self, flights):
self.flights = flights
self.last_arr = max([f.arrival for f in flights]) # last arrival
self.data = Expand()
self.G = DiGraph(directed=True, n_res=8)
# Init update params
self.it = None
self.k_make = None
self.airline = None
self.duals = None
self.first_flight = None
def _build(self):
self._build_data()
self._build_graph()
return self
def _build_data(self):
airports_dep = [f.origin for f in self.flights]
airports_arrival = [f.destination for f in self.flights]
self.data.airports = list(set(airports_arrival + airports_dep))
self.data.airports.sort()
def _build_graph(self):
# Construct an Activity-on-Arc network using flights as activities
flights = self.flights
airports = self.data.airports
self.G.add_node('Source',
pos=(-1, len(airports) + 1),
i_d=-1,
airport='Source')
self.G.add_node('Sink',
pos=(self.last_arr + 1, -2),
i_d=-1,
airport='Sink')
list(map(self._init_graph, flights))
self._add_edges()
log.info("Generated TSN with {} edges {} nodes".format(
len(self.G.edges()), len(self.G.nodes())))
# plot_graph(self.G)
def _init_graph(self, f):
""" Populate graph using flights from csv file"""
label_dep = "{}_{}".format(f.origin, f.departure) # label
# Arrival node label
label_arr = "{}_{}".format(f.destination, f.arrival) # label
self._init_node(f, label_dep, label_arr) # Init nodes
self._init_edge(f, label_dep, label_arr) # Init edges
def _init_node(self, f, label_dep, label_arr):
# Add a Departure Node
self.G.add_node(label_dep,
pos=(f.departure, self.data.airports.index(f.origin)),
airport=f.origin)
# Add an Arrival Node
self.G.add_node(label_arr,
pos=(f.arrival,
self.data.airports.index(f.destination)),
airport=f.destination)
def _init_edge(self, f, label_dep, label_arr):
# Add all flight edges
self.G.add_edge(label_dep, label_arr, data=f._full_dict(), weight=0)
# Add edge from Source to departure and arrival nodes
data_source = {
'origin': 'Source',
'destination': f.origin,
'departure': -1,
'arrival': f.departure,
'type': f._classify_flight(0, f.departure)
}
self.G.add_edge('Source',
label_dep,
data=data_source,
weight=self._edge_weight('Source', label_dep))
# Add edge from arrival to Sink
data_sink = {
'origin': f.destination,
'destination': 'Sink',
'departure': f.arrival,
'arrival': self.last_arr,
'type': f._classify_flight(f.arrival, self.last_arr)
}
self.G.add_edge(label_arr,
'Sink',
data=data_sink,
weight=self._edge_weight(label_arr, 'Sink'))
def _add_edges(self):
# Add edges if not already exist with their respective data
self._add_ground_edges()
self._add_remaining_edges()
def _add_ground_edges(self):
nodes = sorted(
[
node for node in self.G.nodes(data=True)
if node[0] not in ["Source", "Sink"]
],
key=lambda x: x[1]['pos'][0],
)
for n in nodes:
# first element in tuple has string 'Airport_time'
# Second element in tuple has node data
n_name, n_data = n[0], n[1]
n_time = float(n_name.split('_')[1])
ground_nodes_n = sorted(
[
k for k in nodes if float(k[0].split('_')[1]) > n_time
and n_data['airport'] == k[1]['airport']
],
key=lambda x: x[1]['pos'][0],
)
# ground_nodes_n.sort()
if ground_nodes_n:
m = ground_nodes_n[0]
# for m in ground_nodes_n:
path = None
m_name, m_data = m[0], m[1]
m_time = float(m_name.split('_')[1])
if not self.G.has_edge(*(n_name, m_name)):
try:
path = astar_path(self.G, n_name, m_name)
except NetworkXNoPath:
self._add_edge(n_name, n_data, n_time, m_name, m_data,
m_time)
if path and any(
p.split('_')[0] != n_data['airport']
for p in path):
# if edge doesn't exist, add it
self._add_edge(n_name, n_data, n_time, m_name, m_data,
m_time)
def _add_remaining_edges(self):
nodes = sorted(
[
node for node in self.G.nodes(data=True)
if node[0] not in ["Source", "Sink"]
],
key=lambda x: x[1]['pos'][0],
)
for n in nodes:
# first element in tuple has string 'Airport_time'
# Second element in tuple has node data
n_name, n_data = n[0], n[1]
n_time = float(n_name.split('_')[1])
nodes_n = sorted(
[k for k in nodes if float(k[0].split('_')[1]) > n_time],
key=lambda x: x[1]['pos'][0],
)
# nodes_n.sort()
for m in nodes_n:
m_name, m_data = m[0], m[1]
m_time = float(m_name.split('_')[1])
if (not has_path(self.G, n_name, m_name)):
# if path doesn't exist add edge
f = [
f for f in self.flights
if (f.origin == n_name and f.destination == m_name
and f.arrival - f.departure == n_time -
m_time and f.arrival < n_time)
]
if f:
self._add_edge(n_name, n_data, n_time, m_name, m_data,
m_time)
def _add_edge(self, n_name, n_data, n_time, m_name, m_data, m_time):
data = {
'origin': n_data['airport'],
'destination': m_data['airport'],
'departure': n_time,
'arrival': m_time,
'type': Flight._classify_flight(n_time, m_time)
}
self.G.add_edge(n_name,
m_name,
data=data,
weight=self._edge_weight(n_name, m_name,
{'data': data}))
################
# Updating TSN #
################
def _update_TSN(self, G, it, k_type, k_make, airline, duals, first_flight,
drop_edges):
"""
Updates TSN network and returns preprocessed version,
with less edges
"""
edges = G.edges(data=True)
self.it = it
self.k_make = k_make
self.airline = airline
self.duals = duals
self.first_flight = first_flight
list(map(self._update_edge_attrs, edges))
if drop_edges:
return self._drop_edges(G, k_type)
else:
return G
@staticmethod
def _drop_edges(G, k_type):
"""
Creates a copy of the TSN graph and removes
unnecessary (Source, *) edges.
"""
G = copy.deepcopy(G)
count, edges = 0, G.edges(data=True)
number_edges = len(edges)
edges_to_remove = []
for edge in edges:
edge_data = edge[2]
i_airport = edge[0].split('_')[0]
j_airport = edge[1].split('_')[0]
if ((((k_type == 1 and edge_data['data']['type'] > k_type) or
(k_type == 2 and edge_data['data']['type'] != k_type))
and i_airport != j_airport) and i_airport != 'Source'
and j_airport != 'Sink'):
edges_to_remove.append(edge[0:2])
count += 1
G.remove_edges_from(edges_to_remove)
log.info('Removed {}/{} edges.'.format(count, number_edges))
return G
def _edge_weight(self, i, j, edge_data={}):
"""
Weight function for edge between two pair of nodes.
Parameters
----------
i : string,
tail node in the form 'LETTER_INTEGER'
j : string,
head node in the form 'LETTER_INTEGER'
Returns
-------
int
value with appropriate weight
"""
if i == 'Source':
return 0 # float(j.split('_')[1])
elif j == 'Sink':
return 0 # (self.last_arr - float(i.split('_')[1]))
else:
i_airport, i_time = i.split('_')
j_airport, j_time = j.split('_')
i_time, j_time = float(i_time), float(j_time)
# if i_airport == j_airport:
# return (j_time - i_time)
# else:
if self.k_make:
try: # SCHEDULED CONNECTION
flight_dual = list(
w[0] for w in self.duals
if w[1]._full_dict() == edge_data['data'])[0]
cost = (OPERATING_COSTS[self.k_make]['standard']
if i_airport != j_airport else 0)
return cost * (j_time - i_time) - flight_dual
except IndexError: # NON-SCHEDULED CONNECTION
cost_delay, delay = 0, 0
if i_airport != j_airport:
closest_flight = self._get_flight_copy(
self.flights, edge_data)
delay = (edge_data['data']['departure'] -
closest_flight.departure)
flight_dual = list(w[0] for w in self.duals
if w[1]._full_dict() ==
closest_flight._full_dict())[0]
cost = OPERATING_COSTS[self.k_make]['standard']
cost_delay = OPERATING_COSTS[self.k_make]['copy']
return (cost * (j_time - i_time) + cost_delay * delay -
flight_dual)
else:
return (OPERATING_COSTS[self.k_make]['ground'] *
(j_time - i_time))
else:
return 0
@staticmethod
def _get_flight_copy(flights, edge_data):
previous_flights = [
f for f in flights if
(f._full_dict()['origin'] == edge_data['data']['origin'] and
f._full_dict()['destination'] == edge_data['data']['destination']
and f._full_dict()['departure'] < edge_data['data']['departure'])
]
if previous_flights:
return max(previous_flights, key=lambda x: x.departure)
else:
return
def _update_edge_attrs(self, edge):
"""
Update edge attributes using dual values from the solution of the
relaxed master problem.
Parameters
----------
edge : edge
edge to update.
it : int
iteration number.
duals : list of tuples
(dual, classes.Flight). dual values from the master problem
and schedule.
first_flight : object, :class:`classes.Flight`
first flight scheduled.
"""
def __update_weight(edge):
edge_data = edge[2]
weight = self._edge_weight(*edge)
edge_data['weight'] = weight
def __update_res_cost(edge):
edge[2]['res_cost'] = np.zeros(self.G.graph['n_res'])
__update_weight(edge)
__update_res_cost(edge)
class TaskGraph(object):
"""
Task graph builder.
"""
def __init__(self, name):
self.name = name
self._id = generate_uuid(variant='uuid')
self._graph = DiGraph()
def __repr__(self):
return u'{name}(id={self._id}, name={self.name}, graph={self._graph!r})'.format( # pylint: disable=redundant-keyword-arg
name=self.__class__.__name__, self=self)
@property
def id(self):
"""
ID of the graph
"""
return self._id
# graph traversal methods
@property
def tasks(self):
"""
Iterator over tasks in the graph.
"""
for _, data in self._graph.nodes(data=True):
yield data['task']
def topological_order(self, reverse=False):
"""
Topological sort of the graph.
:param reverse: whether to reverse the sort
:return: list which represents the topological sort
"""
task_ids = topological_sort(self._graph)
if reverse:
task_ids = reversed(tuple(task_ids))
for task_id in task_ids:
yield self.get_task(task_id)
def get_dependencies(self, dependent_task):
"""
Iterates over the task's dependencies.
:param dependent_task: task whose dependencies are requested
:raises ~aria.orchestrator.workflows.api.task_graph.TaskNotInGraphError: if
``dependent_task`` is not in the graph
"""
if not self.has_tasks(dependent_task):
raise TaskNotInGraphError(u'Task id: {0}'.format(dependent_task.id))
for _, dependency_id in self._graph.out_edges(dependent_task.id):
yield self.get_task(dependency_id)
def get_dependents(self, dependency_task):
"""
Iterates over the task's dependents.
:param dependency_task: task whose dependents are requested
:raises ~aria.orchestrator.workflows.api.task_graph.TaskNotInGraphError: if
``dependency_task`` is not in the graph
"""
if not self.has_tasks(dependency_task):
raise TaskNotInGraphError(u'Task id: {0}'.format(dependency_task.id))
for dependent_id, _ in self._graph.in_edges(dependency_task.id):
yield self.get_task(dependent_id)
# task methods
def get_task(self, task_id):
"""
Get a task instance that's been inserted to the graph by the task's ID.
:param basestring task_id: task ID
:raises ~aria.orchestrator.workflows.api.task_graph.TaskNotInGraphError: if no task found in
the graph with the given ID
"""
if not self._graph.has_node(task_id):
raise TaskNotInGraphError(u'Task id: {0}'.format(task_id))
data = self._graph.node[task_id]
return data['task']
@_filter_out_empty_tasks
def add_tasks(self, *tasks):
"""
Adds a task to the graph.
:param task: task
:return: list of added tasks
:rtype: list
"""
assert all([isinstance(task, (api_task.BaseTask, Iterable)) for task in tasks])
return_tasks = []
for task in tasks:
if isinstance(task, Iterable):
return_tasks += self.add_tasks(*task)
elif not self.has_tasks(task):
self._graph.add_node(task.id, task=task)
return_tasks.append(task)
return return_tasks
@_filter_out_empty_tasks
def remove_tasks(self, *tasks):
"""
Removes the provided task from the graph.
:param task: task
:return: list of removed tasks
:rtype: list
"""
return_tasks = []
for task in tasks:
if isinstance(task, Iterable):
return_tasks += self.remove_tasks(*task)
elif self.has_tasks(task):
self._graph.remove_node(task.id)
return_tasks.append(task)
return return_tasks
@_filter_out_empty_tasks
def has_tasks(self, *tasks):
"""
Checks whether a task is in the graph.
:param task: task
:return: ``True`` if all tasks are in the graph, otherwise ``False``
:rtype: list
"""
assert all(isinstance(t, (api_task.BaseTask, Iterable)) for t in tasks)
return_value = True
for task in tasks:
if isinstance(task, Iterable):
return_value &= self.has_tasks(*task)
else:
return_value &= self._graph.has_node(task.id)
return return_value
def add_dependency(self, dependent, dependency):
"""
Adds a dependency for one item (task, sequence or parallel) on another.
The dependent will only be executed after the dependency terminates. If either of the items
is either a sequence or a parallel, multiple dependencies may be added.
:param dependent: dependent (task, sequence or parallel)
:param dependency: dependency (task, sequence or parallel)
:return: ``True`` if the dependency between the two hadn't already existed, otherwise
``False``
:rtype: bool
:raises ~aria.orchestrator.workflows.api.task_graph.TaskNotInGraphError: if either the
dependent or dependency are tasks which are not in the graph
"""
if not (self.has_tasks(dependent) and self.has_tasks(dependency)):
raise TaskNotInGraphError()
if self.has_dependency(dependent, dependency):
return
if isinstance(dependent, Iterable):
for dependent_task in dependent:
self.add_dependency(dependent_task, dependency)
else:
if isinstance(dependency, Iterable):
for dependency_task in dependency:
self.add_dependency(dependent, dependency_task)
else:
self._graph.add_edge(dependent.id, dependency.id)
def has_dependency(self, dependent, dependency):
"""
Checks whether one item (task, sequence or parallel) depends on another.
Note that if either of the items is either a sequence or a parallel, and some of the
dependencies exist in the graph but not all of them, this method will return ``False``.
:param dependent: dependent (task, sequence or parallel)
:param dependency: dependency (task, sequence or parallel)
:return: ``True`` if the dependency between the two exists, otherwise ``False``
:rtype: bool
:raises ~aria.orchestrator.workflows.api.task_graph.TaskNotInGraphError: if either the
dependent or dependency are tasks which are not in the graph
"""
if not (dependent and dependency):
return False
elif not (self.has_tasks(dependent) and self.has_tasks(dependency)):
raise TaskNotInGraphError()
return_value = True
if isinstance(dependent, Iterable):
for dependent_task in dependent:
return_value &= self.has_dependency(dependent_task, dependency)
else:
if isinstance(dependency, Iterable):
for dependency_task in dependency:
return_value &= self.has_dependency(dependent, dependency_task)
else:
return_value &= self._graph.has_edge(dependent.id, dependency.id)
return return_value
def remove_dependency(self, dependent, dependency):
"""
Removes a dependency for one item (task, sequence or parallel) on another.
Note that if either of the items is either a sequence or a parallel, and some of the
dependencies exist in the graph but not all of them, this method will not remove any of the
dependencies and return ``False``.
:param dependent: dependent (task, sequence or parallel)
:param dependency: dependency (task, sequence or parallel)
:return: ``False`` if the dependency between the two hadn't existed, otherwise ``True``
:rtype: bool
:raises ~aria.orchestrator.workflows.api.task_graph.TaskNotInGraphError: if either the
dependent or dependency are tasks which are not in the graph
"""
if not (self.has_tasks(dependent) and self.has_tasks(dependency)):
raise TaskNotInGraphError()
if not self.has_dependency(dependent, dependency):
return
if isinstance(dependent, Iterable):
for dependent_task in dependent:
self.remove_dependency(dependent_task, dependency)
elif isinstance(dependency, Iterable):
for dependency_task in dependency:
self.remove_dependency(dependent, dependency_task)
else:
self._graph.remove_edge(dependent.id, dependency.id)
@_filter_out_empty_tasks
def sequence(self, *tasks):
"""
Creates and inserts a sequence into the graph, effectively each task i depends on i-1.
:param tasks: iterable of dependencies
:return: provided tasks
"""
if tasks:
self.add_tasks(*tasks)
for i in xrange(1, len(tasks)):
self.add_dependency(tasks[i], tasks[i-1])
return tasks
def load(self,fname, verbose=True, **kwargs):
"""
Load a data file. The expected data format is three columns
(comma seperated by default) with source, target, flux.
No header should be included and the node IDs have to run contuously
from 0 to Number_of_nodes-1.
Parameters
----------
fname : str
Path to the file
verbose : bool
Print information about the data. True by Default
kwargs : dict
Default parameters can be changed here. Supported key words are
dtype : float (default)
delimiter : "," (default)
return_graph : bool
If True, the graph is returned (False by default).
Returns:
--------
The graph is saved internally in self.graph.
"""
delimiter = kwargs["delimiter"] if "delimiter" in kwargs.keys() else " "
data = np.genfromtxt(fname, delimiter=delimiter, dtype=int, unpack=False)
source, target = data[:,0], data[:,1]
if data.shape[1] > 2:
flux = data[:,2]
else:
flux = np.ones_like(source)
nodes = set(source) | set(target)
self.nodes = len(nodes)
lines = len(flux)
if set(range(self.nodes)) != nodes:
new_node_ID = {old:new for new,old in enumerate(nodes)}
map_new_node_ID = np.vectorize(new_node_ID.__getitem__)
source = map_new_node_ID(source)
target = map_new_node_ID(target)
if verbose:
print "\nThe node IDs have to run continuously from 0 to Number_of_nodes-1."
print "Node IDs have been changed according to the requirement.\n-----------------------------------\n"
print 'Lines: ',lines , ', Nodes: ', self.nodes
print '-----------------------------------\nData Structure:\n\nsource, target, weight \n'
for ii in range(7):
print "%i, %i, %1.2e" %(source[ii], target[ii], flux[ii])
print '-----------------------------------\n'
G = DiGraph() # Empty, directed Graph
G.add_nodes_from(range(self.nodes))
for ii in xrange(lines):
u, v, w = int(source[ii]), int(target[ii]), float(flux[ii])
if u != v: # ignore self loops
assert not G.has_edge(u,v), "Edge appeared twice - not supported"
G.add_edge(u,v,weight=w)
else:
if verbose:
print "ignore self loop at node", u
symmetric = True
for s,t,w in G.edges(data=True):
w1 = G[s][t]["weight"]
try:
w2 = G[t][s]["weight"]
except KeyError:
symmetric = False
G.add_edge(t,s,weight=w1)
w2 = w1
if w1 != w2:
symmetric = False
G[s][t]["weight"] += G[t][s]["weight"]
G[s][t]["weight"] /= 2
G[t][s]["weight"] = G[s][t]["weight"]
if verbose:
if not symmetric:
print "The network has been symmetricised."
ccs = strongly_connected_component_subgraphs(G)
ccs = sorted(ccs, key=len, reverse=True)
G_GSCC = ccs[0]
if G_GSCC.number_of_nodes() != G.number_of_nodes():
G = G_GSCC
if verbose:
print "\n--------------------------------------------------------------------------"
print "The network has been restricted to the giant strongly connected component."
self.nodes = G.number_of_nodes()
for u, v, data in G.edges(data=True):
weight = G.out_degree(u,weight='weight')
data['transition_rate'] = 1.*data['weight']/weight
for u, v, data in G.edges(data=True):
data['effective_distance'] = 1. - log(data['transition_rate'])
if verbose:
print "\n--------------------------------------------------------------------------"
print "\nnode ID, out-weight, normalized out-weight, sum of effective distances \n "
for ii in range(7):
out_edges = G.out_edges(ii, data=True)
out_weight, effective_distance, transition_rate = 0, 0, 0
for u, v, data in out_edges:
out_weight += data["weight"]
effective_distance += data["effective_distance"]
transition_rate += data["transition_rate"]
print " %i %1.2e %2.3f %1.2e " %(ii,out_weight, transition_rate, effective_distance)
print "\n ... graph is saved in self.graph"
return G
def is_anti_reflexive(graph: DiGraph) -> bool:
return all(not graph.has_edge(node, node) for node in graph.nodes)
def digraph2condensationgraph(digraph: networkx.DiGraph) -> networkx.DiGraph:
"""
Creates the condensation graph from *digraph*.
The condensation graph is similar to the SCC graph but it replaces
cascades between SCCs by single edges.
Its nodes are therefore non-trivial SCCs or inputs.
As for the SCC graph, nodes are tuples of names that belong to the SCC.
The condensation graph is cycle-free and does distinguish between inputs
and constants.
The graph has no additional data.
**arguments**:
* *digraph*: directed graph
**returns**:
* *condensation_graph*: the condensation graph
**example**::
>>> cgraph = digraph2condensationgraph(igraph)
>>> cgraph.nodes()
[('Ash1', 'Cbf1'), ('Gal4',), ('Gal80',), ('Cbf1','Swi5)]
>>> cgraph.edges()
[(('Gal4',), ('Ash1', 'Cbf1')), (('Gal4',), ('Gal80',)),
(('Gal80',),('Cbf1','Swi5))]
"""
sccs = sorted([
tuple(sorted(scc))
for scc in networkx.strongly_connected_components(digraph)
])
cascades = [
scc for scc in sccs
if (len(scc) == 1) and not digraph.has_edge(scc[0], scc[0])
]
noncascades = [scc for scc in sccs if scc not in cascades]
cgraph = networkx.DiGraph()
cgraph.add_nodes_from(noncascades)
# rgraph is a copy of Digraph with edges leaving noncascade components removed.
# will use rgraph to decide if there is a cascade path between U and W (i.e. edge in cgraph)
rgraph = networkx.DiGraph(digraph.edges())
for U, W in itertools.product(noncascades, noncascades):
if U == W:
continue
rgraph = digraph.copy()
for X in noncascades:
if not X == U and not X == W:
rgraph.remove_nodes_from(X)
if has_path(rgraph, U, W):
cgraph.add_edge(U, W)
# annotate each node with its depth in the hierarchy and an integer ID
for ID, target in enumerate(cgraph.nodes()):
depth = 1
for source in networkx.ancestors(cgraph, target):
for p in networkx.all_simple_paths(cgraph, source, target):
depth = max(depth, len(p))
cgraph.nodes[target]["depth"] = depth
cgraph.nodes[target]["id"] = ID
return cgraph
}
# start the scanning process
switch = []
# direction: up. We will change the rows
prev = None
flag = 1
for coord in range(int(comr), -1, -1):
r = coord
c = comc
if prev == None:
prev = testcase[r, c]
else:
if g.has_edge(prev, testcase[r, c]):
prev = testcase[r, c]
continue
else:
print('invalid!')
flag = -1
break
if flag != -1:
print('Passed the Up direction test!')
# direction: down
# direction: down
prev = None
flag = 1
for coord in range(int(comr), np.shape(testcase)[0], 1):
r = coord
def is_not_transitive(graph: DiGraph) -> bool:
return any(not graph.has_edge(x, z)
for x, y in get_set_combination(graph.nodes)
if x != y and graph.has_edge(x, y) for z in graph.adj[y]
if y != z)
def perception_text(
self, perception: PerceptualRepresentation[
DevelopmentalPrimitivePerceptionFrame]
) -> str:
"""
Turns a perception into a list of items in the perceptions frames.
"""
output_text: List[str] = []
check_state(
len(perception.frames) in (1, 2),
"Only know how to handle 1 or 2 frame "
"perceptions for now",
)
perception_is_dynamic = len(perception.frames) > 1
# first, we build an index of objects to their properties.
# This will be used so that when we list the objects,
# we can easily list their properties in brackets right after them.
def extract_subject(prop: PropertyPerception) -> ObjectPerception:
return prop.perceived_object
first_frame_properties = _index_to_setmultidict(
perception.frames[0].property_assertions, extract_subject)
second_frame_properties = (_index_to_setmultidict(
perception.frames[1].property_assertions, extract_subject)
if perception_is_dynamic else
immutablesetmultidict())
# Next, we determine what objects persist between both frames
# and which do not.
first_frame_objects = perception.frames[0].perceived_objects
second_frame_objects = (perception.frames[1].perceived_objects
if perception_is_dynamic else immutableset())
static_objects = (
first_frame_objects.intersection(second_frame_objects)
if perception_is_dynamic else first_frame_objects)
all_objects = first_frame_objects.union(second_frame_objects)
# For objects, properties, and relations we will use arrows to indicate
# when something beings or ceased to exist between frames.
# Since the logic will be the same for all three types,
# we pull it out into a function.
def compute_arrow(
item: Any, static_items: AbstractSet[Any],
first_frame_items: AbstractSet[Any]) -> Tuple[str, str]:
if item in static_items:
# item doesn't change - no arrow
return ("", "")
elif item in first_frame_items:
# item ceases to exist
return ("", " ---> Ø")
else:
# item beings to exist in the second frame
return ("Ø ---> ", "")
# the logic for rendering objects, which will be used in the loop below.
# This needs to be an inner function so it can access the frame property maps, etc.
def render_object(obj: ObjectPerception) -> str:
obj_text = f"<i>{obj.debug_handle}</i>"
first_frame_obj_properties = first_frame_properties[obj]
second_frame_obj_properties = second_frame_properties[obj]
static_properties = (second_frame_obj_properties.intersection(
first_frame_obj_properties) if second_frame_obj_properties else
first_frame_obj_properties)
# logic for rendering properties, for use in the loop below.
def render_property(prop: PropertyPerception) -> str:
(prop_prefix,
prop_suffix) = compute_arrow(prop, static_properties,
first_frame_obj_properties)
prop_string: str
if isinstance(prop, HasColor):
prop_string = (
f'<span style="background-color: {prop.color}; '
f'color: {prop.color.inverse()}; border: 1px solid black;">'
f"color={prop.color.hex}</span>")
elif isinstance(prop, HasBinaryProperty):
prop_string = prop.binary_property.handle
else:
raise RuntimeError(f"Cannot render property: {prop}")
return f"{prop_prefix}{prop_string}{prop_suffix}"
all_properties: ImmutableSet[PropertyPerception] = immutableset(
flatten(
[first_frame_obj_properties, second_frame_obj_properties]))
prop_strings = [render_property(prop) for prop in all_properties]
if prop_strings:
return f"{obj_text}[{'; '.join(prop_strings)}]"
else:
return obj_text
# Here we process the relations between the two scenes to determine all relations.
# This has to be done before rending objects so we can use the PART_OF relation to order
# the objects.
first_frame_relations = perception.frames[0].relations
second_frame_relations = (perception.frames[1].relations
if perception_is_dynamic else immutableset())
static_relations = (
second_frame_relations.intersection(first_frame_relations)
if perception_is_dynamic else first_frame_relations)
all_relations = first_frame_relations.union(second_frame_relations)
# Here we add the perceived objects to a NetworkX DiGraph with PART_OF relations being the
# edges between objects. This allows us to do pre-order traversal of the Graph to make a
# nested <ul></ul> for the objects rather than a flat list.
graph = DiGraph()
root = ObjectPerception("root", axes=WORLD_AXES)
graph.add_node(root)
expressed_relations = set()
axis_to_object: Dict[GeonAxis, ObjectPerception] = {}
for object_ in all_objects:
graph.add_node(object_)
graph.add_edge(root, object_)
for axis in object_.axes.all_axes:
axis_to_object[axis] = object_
for relation_ in all_relations:
if relation_.relation_type == PART_OF:
graph.add_edge(relation_.second_slot, relation_.first_slot)
if graph.has_edge(root, relation_.first_slot):
graph.remove_edge(root, relation_.first_slot)
expressed_relations.add(relation_)
# Next, we render objects, together with their properties, using preorder DFS Traversal
# We also add in `In Region` relationships at this step for objects which have them.
output_text.append(
"\n\t\t\t\t\t<h5>Perceived Objects</h5>\n\t\t\t\t\t<ul>")
visited = set()
region_relations = immutableset(region for region in all_relations
if region.relation_type == IN_REGION)
# This loop doesn't quite get the tab spacing right. It could at the cost of increased
# complexity. Would need to track the "depth" we are currently at.
axis_info = perception.frames[0].axis_info
def dfs_walk(node, depth=0):
visited.add(node)
if not node == root:
(obj_prefix,
obj_suffix) = compute_arrow(node, static_objects,
first_frame_objects)
output_text.append(
f"\t" * (6 + depth) +
f"<li>{obj_prefix}{render_object(node)}{obj_suffix}<ul>")
if node.geon:
output_text.append(
f"\t\t\t\t\t\t<li>Geon: {self._render_geon(node.geon, indent_dept=7)}</li>"
)
# Handle Region Relations
for region_relation in region_relations:
if region_relation.first_slot == node:
(relation_prefix, relation_suffix) = compute_arrow(
region_relation, static_relations,
first_frame_relations)
relation_str = self._render_relation(
axis_info, region_relation)
output_text.append(
f"\t\t\t\t\t\t<li>{relation_prefix}"
f"{relation_str}{relation_suffix}</li>")
expressed_relations.add(region_relation)
for succ in graph.successors(node):
if succ not in visited:
depth = depth + 6
dfs_walk(succ, depth)
depth = depth - 6
output_text.append("\t" * (6 + depth) + f"</ul></li>")
dfs_walk(root)
output_text.append("\t\t\t\t\t</ul>")
# Finally we render remaining relations between objects
remaining_relations = immutableset(
relation for relation in all_relations
if relation not in expressed_relations)
if remaining_relations:
output_text.append(
"\t\t\t\t\t<h5>Other Relations</h5>\n\t\t\t\t\t<ul>")
for relation in remaining_relations:
(relation_prefix,
relation_suffix) = compute_arrow(relation, static_relations,
first_frame_relations)
single_size_relation: Optional[Tuple[
Any, str, Any]] = self._get_single_size_relation(
relation, all_relations)
if single_size_relation:
relation_text = f"{single_size_relation[0]} {single_size_relation[1]} {single_size_relation[2]}"
size_output = f"\t\t\t\t\t\t<li>{relation_prefix}{relation_text}{relation_suffix}</li>"
if size_output not in output_text:
output_text.append(size_output)
else:
output_text.append(
f"\t\t\t\t\t\t<li>{relation_prefix}{relation}{relation_suffix}</li>"
)
output_text.append("\t\t\t\t\t</ul>")
if perception.during:
output_text.append("\t\t\t\t\t<h5>During the action</h5>")
output_text.append(
self._render_during(perception.during, indent_depth=5))
if axis_info and axis_info.axes_facing:
output_text.append(("\t\t\t\t\t<h5>Axis Facings</h5>"))
output_text.append(("\t\t\t\t\t<ul>"))
for object_ in axis_info.axes_facing:
output_text.append(
f"\t\t\t\t\t\t<li>{object_.debug_handle} faced by:\n\t\t\t\t\t\t<ul>"
)
for axis in axis_info.axes_facing[object_]:
output_text.append(
f"\t\t\t\t\t\t\t<li>{axis} possessed by {axis_to_object[axis]}</li>"
)
output_text.append("\t\t\t\t\t\t</ul>")
output_text.append("\t\t\t\t\t</ul>")
return "\n".join(output_text)
def is_anti_symmetric(graph: DiGraph) -> bool:
return all(not graph.has_edge(y, x) for x, y in graph.edges if x != y
if graph.has_edge(x, y))
def is_symmetric(graph: DiGraph) -> bool:
return all(
graph.has_edge(y, x) for x, y in graph.edges
if x != y and graph.has_edge(x, y))
def is_not_reflexive(graph: DiGraph) -> bool:
if is_reflexive(graph):
return False
return any(graph.has_edge(node, node) for node in graph.nodes)
class CollectNodes(Visitor):
def __init__(self, call_deps=False):
self.graph = DiGraph()
self.modified = set()
self.used = set()
self.undefined = set()
self.sources = set()
self.targets = set()
self.context_names = set()
self.call_deps = call_deps
visitDefault = collect_
def visitName(self, node):
if isinstance(node.ctx, _ast.Store):
self.modified.add(node.id)
elif isinstance(node.ctx, _ast.Load):
self.used.update(node.id)
if not self.graph.has_node(node.id):
self.graph.add_node(node.id)
if isinstance(node.ctx, _ast.Load):
self.undefined.add(node.id)
for ctx_var in self.context_names:
if not self.graph.has_edge(node.id, ctx_var):
self.graph.add_edge(node.id, ctx_var)
return {node.id}
def visitalias(self, node):
name = node.asname if node.asname else node.name
if '.' in name:
name = name.split('.', 1)[0]
if not self.graph.has_node(name):
self.graph.add_node(name)
return {name}
def visitCall(self, node):
left = self.visit(node.func)
right = set()
for attr in ('args', 'keywords'):
for child in getattr(node, attr):
if child:
right.update(self.visit(child))
for attr in ('starargs', 'kwargs'):
child = getattr(node, attr)
if child:
right.update(self.visit(child))
for src in left | right:
if not self.graph.has_node(src):
self.undefined.add(src)
if self.call_deps:
add_edges(self.graph, left, right)
add_edges(self.graph, right, left)
right.update(left)
return right
def visitSubscript(self, node):
if isinstance(node.ctx, _ast.Load):
return collect_(self, node)
else:
sources = self.visit(node.slice)
targets = self.visit(node.value)
self.modified.update(targets)
add_edges(self.graph, targets, sources)
return targets
def handle_generators(self, generators):
defined = set()
required = set()
for generator in generators:
get_symbols(generator, _ast.Load)
required.update(get_symbols(generator, _ast.Load) - defined)
defined.update(get_symbols(generator, _ast.Store))
return defined, required
def visitListComp(self, node):
defined, required = self.handle_generators(node.generators)
required.update(get_symbols(node.elt, _ast.Load) - defined)
for symbol in required:
if not self.graph.has_node(symbol):
self.graph.add_node(symbol)
self.undefined.add(symbol)
return required
def visitSetComp(self, node):
defined, required = self.handle_generators(node.generators)
required.update(get_symbols(node.elt, _ast.Load) - defined)
for symbol in required:
if not self.graph.has_node(symbol):
self.graph.add_node(symbol)
self.undefined.add(symbol)
return required
def visitDictComp(self, node):
defined, required = self.handle_generators(node.generators)
required.update(get_symbols(node.key, _ast.Load) - defined)
required.update(get_symbols(node.value, _ast.Load) - defined)
for symbol in required:
if not self.graph.has_node(symbol):
self.graph.add_node(symbol)
self.undefined.add(symbol)
return required
def bidirectionalize(graph: nx.DiGraph) -> None:
graph.add_edges_from([
(node2, node1, reversed_edge_attrs(eattrs))
for node1, node2, eattrs in graph.out_edges(data=True)
if not graph.has_edge(node2, node1)
])
def restore_alphabet(input_file_path=None):
"""
This function restores the original alphabet from given dictionary.
Here dictionary is considered as passed in txt-file.
Commands:
python main.py - simple test, which shows result via console
python main.py /tmp/alphabet.txt - run script with data from
specific file. Results are put in 'result.txt'.
:param input_file_path: file with dictionary.
:return: list of chars, which are considered to be the wanted alphabet
"""
with open(input_file_path, 'r') as dictionary:
dg = DiGraph()
# get first word
previous_word = dictionary.readline()
previous_word = previous_word.strip()
previous_position = 1
dg.add_nodes_from(previous_word)
# get second word, if it is one
current_word = dictionary.readline()
while current_word:
current_word = current_word.strip()
current_position = previous_position + 1
# look for the first non-equal chars,
# which are situated on the same positions in
# previous word and current word
index = 0
length_previous = len(previous_word)
length_current = len(current_word)
while (index < length_previous and index < length_current):
if previous_word[index] == current_word[index]:
index += 1
else:
# if the pair is found, and there is no opposite rule of
# any kind, add chars as nodes and an edge,
# which demonstrates their order.
if dg.has_edge(current_word[index], previous_word[index]):
raise Exception(
"Wrong words order!\n"
"This pair of words violates rule: {} -> {}.\n"
"Position: {}.\n"
"Words: '{}' and '{}'.\n"
"String numbers in dictionary: {} and {}."
"".format(previous_word[index],
current_word[index], index + 1,
previous_word, current_word,
previous_position, current_position))
dg.add_edge(previous_word[index], current_word[index])
break
else:
if length_previous > length_current:
raise Exception("Wrong words order!\n"
"Words: '{}' and '{}'.\n"
"String numbers in dictionary: {} and {}."
"".format(previous_word, current_word,
previous_position,
current_position))
dg.add_nodes_from(current_word[index:])
# move to the next word from dictionary
previous_word = current_word
previous_position = current_position
current_word = dictionary.readline()
# use topological sort to get the correct order of chars
return topological_sort(dg)
class GraphBuilder():
"""
A class for building the reaction graph.
"""
def __init__(self) -> None:
self._reaction_graph = DiGraph()
def _add_node(self, node_id: str, type: NodeType, label: str) -> None:
"""
Adding a node if the node is not already in the graph.
Args:
node_id: The ID of a graph node e.g. a domain will have the ID A_[dom]
type: The type of a node e.g. component, domain, residue
label: The label of the node e.g. the domain name (dom)
Returns:
"""
if not self._reaction_graph.has_node(node_id):
logger.debug('Adding new node node_id: {0} label: {1}, type: {2}'.format(node_id, label, type))
self._reaction_graph.add_node(node_id, label=label, type=type.value)
def _add_edge(self, source: str, target: str, interaction: EdgeType, width: EdgeWith) -> None:
"""
Adding an edge to the graph.
Note:
Internal edges are edges within the different levels of an specification.
External edges are edges between specific resolution levels of two specifications
e.g. between component and domain, domain and domain and so on.
Args:
source: The source of an edge e.g. component node, domain node, residue node.
target: The target of an edge e.g. component node, domain node, residue node.
interaction (EdgeType): The type of an interaction.
width (EdgeWith): The width of an edge.
Returns:
None
"""
if not self._reaction_graph.has_edge(source, target):
logger.debug('Adding new edge source: {0} target: {1}, interaction: {2}, width: {3}'.format(source,
target,
interaction.value,
width))
self._reaction_graph.add_edge(source, target, interaction=interaction.value, width=width.value)
elif width == EdgeWith.external:
logger.debug('Adding replace inner edge with external edge source: {0} target: {1}, interaction: {2}, width: {3}'.format(source,
target,
interaction.value,
width))
self._reaction_graph.add_edge(source, target, interaction=interaction.value, width=width.value)
def add_external_edge(self, source: Spec, target: Spec, type: EdgeType) -> None:
"""
Adding an external edge.
Note:
An external edges is an edge between two specific resolution levels of two specifications respectively
e.g. between component and domain, domain and domain and so on.
Args:
source: The source specification.
target: The target specification.
type (EdgeType): The type of this edge e.g. interaction, modification etc.
Returns:
None
"""
logger.info('Adding external edge source: {0} target: {1}, interaction: {2}'.format(get_node_id(source, source.resolution),
get_node_id(target, target.resolution),
type))
self._add_edge(get_node_id(source, source.resolution), get_node_id(target, target.resolution),
interaction=type, width=EdgeWith.external)
def add_spec_information(self, specification: Spec) -> None:
"""
Adding specification information to the reaction graph.
Args:
specification: The specification of a reaction reactant
Returns:
None
"""
def _add_spec_nodes() -> None:
logger.info('Adding component node -> id: {0}, label: {1}'.format(get_node_id(specification, NodeType.component),
get_node_label(specification, NodeType.component)))
self._add_node(node_id=get_node_id(specification, NodeType.component), type=NodeType.component,
label=get_node_label(specification, NodeType.component))
if specification.locus.domain:
logger.info('Adding domain node -> id: {0}, label: {1}'.format(get_node_id(specification, NodeType.domain),
get_node_label(specification, NodeType.domain)))
self._add_node(get_node_id(specification, NodeType.domain), type=NodeType.domain,
label=get_node_label(specification, NodeType.domain))
if specification.locus.residue:
logger.info('Adding residue node id: {0}, label: {1}'.format(get_node_id(specification, NodeType.residue),
get_node_label(specification, NodeType.residue)))
self._add_node(get_node_id(specification, NodeType.residue), type=NodeType.residue,
label=get_node_label(specification, NodeType.residue))
def _add_spec_edges() -> None:
"""
Adding internal edges between nodes.
Note:
Internal edges are edges within the different levels of an specification.
Returns:
None
"""
if specification.locus.domain:
logger.info('Adding internal edge component -> domain source: {} target: {}'.format(get_node_id(specification, NodeType.component),
get_node_id(specification, NodeType.domain)))
self._add_edge(get_node_id(specification, NodeType.component),
get_node_id(specification, NodeType.domain), interaction=EdgeType.interaction,
width=EdgeWith.internal)
if specification.locus.residue:
logger.info('Adding internal edge domain -> residue source: {} target: {}'.format(get_node_id(specification, NodeType.domain),
get_node_id(specification, NodeType.residue)))
self._add_edge(get_node_id(specification, NodeType.domain), get_node_id(specification, NodeType.residue),
interaction=EdgeType.interaction, width=EdgeWith.internal)
elif specification.locus.residue:
logger.info('Adding internal edge component -> residue source: {} target: {}'.format(get_node_id(specification, NodeType.component),
get_node_id(specification, NodeType.residue)))
self._add_edge(get_node_id(specification, NodeType.component), get_node_id(specification, NodeType.residue),
interaction=EdgeType.interaction, width=EdgeWith.internal)
logger.info('Adding nodes for {}'.format(specification))
_add_spec_nodes()
logger.info('Adding internal edges for {}'.format(specification))
_add_spec_edges()
def get_graph(self) -> DiGraph:
"""
Returning the reaction graph
Returns:
The reaction graph (DiGraph).
"""
return self._reaction_graph
class CollectNodes(Visitor):
def __init__(self, call_deps=False):
self.graph = DiGraph()
self.modified = set()
self.used = set()
self.undefined = set()
self.sources = set()
self.targets = set()
self.context_names = set()
self.call_deps = call_deps
visitDefault = collect_
def visitName(self, node):
if isinstance(node.ctx, _ast.Store):
self.modified.add(node.id)
elif isinstance(node.ctx, _ast.Load):
self.used.update(node.id)
if not self.graph.has_node(node.id):
self.graph.add_node(node.id)
if isinstance(node.ctx, _ast.Load):
self.undefined.add(node.id)
for ctx_var in self.context_names:
if not self.graph.has_edge(node.id, ctx_var):
self.graph.add_edge(node.id, ctx_var)
return {node.id}
def visitalias(self, node):
name = node.asname if node.asname else node.name
if '.' in name:
name = name.split('.', 1)[0]
if not self.graph.has_node(name):
self.graph.add_node(name)
return {name}
def visitCall(self, node):
left = self.visit(node.func)
right = set()
for attr in ('args', 'keywords'):
for child in getattr(node, attr):
if child:
right.update(self.visit(child))
for attr in ('starargs', 'kwargs'):
child = getattr(node, attr)
if child:
right.update(self.visit(child))
for src in left | right:
if not self.graph.has_node(src):
self.undefined.add(src)
if self.call_deps:
add_edges(self.graph, left, right)
add_edges(self.graph, right, left)
right.update(left)
return right
def visitSubscript(self, node):
if isinstance(node.ctx, _ast.Load):
return collect_(self, node)
else:
sources = self.visit(node.slice)
targets = self.visit(node.value)
self.modified.update(targets)
add_edges(self.graph, targets, sources)
return targets
def handle_generators(self, generators):
defined = set()
required = set()
for generator in generators:
get_symbols(generator, _ast.Load)
required.update(get_symbols(generator, _ast.Load) - defined)
defined.update(get_symbols(generator, _ast.Store))
return defined, required
def visitListComp(self, node):
defined, required = self.handle_generators(node.generators)
required.update(get_symbols(node.elt, _ast.Load) - defined)
for symbol in required:
if not self.graph.has_node(symbol):
self.graph.add_node(symbol)
self.undefined.add(symbol)
return required
def visitSetComp(self, node):
defined, required = self.handle_generators(node.generators)
required.update(get_symbols(node.elt, _ast.Load) - defined)
for symbol in required:
if not self.graph.has_node(symbol):
self.graph.add_node(symbol)
self.undefined.add(symbol)
return required
def visitDictComp(self, node):
defined, required = self.handle_generators(node.generators)
required.update(get_symbols(node.key, _ast.Load) - defined)
required.update(get_symbols(node.value, _ast.Load) - defined)
for symbol in required:
if not self.graph.has_node(symbol):
self.graph.add_node(symbol)
self.undefined.add(symbol)
return required
class ExecutionGraph:
def __init__(self, ctx: Context):
self.graph = DiGraph()
self.ctx = ctx
ctx.logger.info("Preparing execution plan...")
comp_ids = self._effective_component_ids()
for comp_id in comp_ids:
self._add_component(comp_id)
if ctx.flags.debug:
self.ctx.logger.debug("Resolved execution order: {}".format(
list(self.topologically_ordered_comp_ids())))
def _add_deps(self, dependencies, comp_id):
for dep in dependencies:
self.graph.add_node(dep)
if not self.graph.has_edge(dep, comp_id):
self.ctx.logger.progress("Adding dependency: {} -> {}".format(
comp_id, dep))
self.graph.add_edge(dep, comp_id)
if not is_directed_acyclic_graph(self.graph):
raise CyclicDependencyError(
"Dependency {} -> {} forms a cycle!".format(comp_id, dep))
self._add_component(dep)
def _effective_component_ids(self):
if self.ctx.components is None:
all_components = self.ctx.registry.component_ids()
self.ctx.logger.debug(
"No components have been explicitly specified. Will execute all: {}"
.format(", ".join(all_components)))
return all_components
else:
requested = self.ctx.components
self.ctx.logger.info("Requested components: {}".format(
", ".join(requested)))
return requested
def _add_component(self, comp_id):
self.graph.add_node(comp_id)
if not self.ctx.registry.component_ids().__contains__(comp_id):
raise MissingDependencyError(
"No component with id '{}' is registered! "
"You might need to add '--experimental' or '-e'".format(
comp_id))
self._add_deps(
prerequisites_of(self.ctx.registry.find_collector(comp_id)),
comp_id)
self._add_deps(
prerequisites_of(self.ctx.registry.find_validator(comp_id)),
comp_id)
for reactor in self.ctx.registry.find_reactors(comp_id):
self._add_deps(prerequisites_of(reactor), comp_id)
def topologically_ordered_comp_ids(self):
return topological_sort(self.graph)
class TransformTree:
'''
A feature complete if not particularly optimized implementation of a transform graph.
Few allowances are made for thread safety, caching, or enforcing graph structure.
'''
def __init__(self, base_frame='world'):
self._transforms = DiGraph()
self._parents = {}
self._paths = {}
self._is_changed = False
self.base_frame = base_frame
def update(self,
frame_to,
frame_from = None,
**kwargs):
'''
Update a transform in the tree.
Arguments
---------
frame_from: hashable object, usually a string (eg 'world').
If left as None it will be set to self.base_frame
frame_to: hashable object, usually a string (eg 'mesh_0')
Additional kwargs (can be used in combinations)
---------
matrix: (4,4) array
quaternion: (4) quatenion
axis: (3) array
angle: float, radians
translation: (3) array
'''
if frame_from is None:
frame_from = self.base_frame
matrix = np.eye(4)
if 'matrix' in kwargs:
# a matrix takes precedence over other options
matrix = kwargs['matrix']
elif 'quaternion' in kwargs:
matrix = quaternion_matrix(kwargs['quaternion'])
elif ('axis' in kwargs) and ('angle' in kwargs):
matrix = rotation_matrix(kwargs['angle'],
kwargs['axis'])
else:
raise ValueError('Couldn\'t update transform!')
if 'translation' in kwargs:
# translation can be used in conjunction with any of the methods of
# specifying transforms. In the case a matrix and translation are passed,
# we add the translations together rather than picking one.
matrix[0:3,3] += kwargs['translation']
if self._transforms.has_edge(frame_from, frame_to):
self._transforms.edge[frame_from][frame_to]['matrix'] = matrix
self._transforms.edge[frame_from][frame_to]['time'] = time.time()
else:
# since the connectivity has changed, throw out previously computed
# paths through the transform graph so queries compute new shortest paths
# we could only throw out transforms that are connected to the new edge,
# but this is less bookeeping at the expensive of being slower.
self._paths = {}
self._transforms.add_edge(frame_from,
frame_to,
matrix = matrix,
time = time.time())
self._is_changed = True
def get(self,
frame_to,
frame_from = None):
'''
Get the transform from one frame to another, assuming they are connected
in the transform tree.
If the frames are not connected a NetworkXNoPath error will be raised.
Arguments
---------
frame_from: hashable object, usually a string (eg 'world').
If left as None it will be set to self.base_frame
frame_to: hashable object, usually a string (eg 'mesh_0')
Returns
---------
transform: (4,4) homogenous transformation matrix
'''
if frame_from is None:
frame_from = self.base_frame
transform = np.eye(4)
path, inverted = self._get_path(frame_from, frame_to)
for i in range(len(path) - 1):
matrix = self._transforms.get_edge_data(path[i],
path[i+1])['matrix']
transform = np.dot(transform, matrix)
if inverted:
transform = np.linalg.inv(transform)
return transform
def clear(self):
self._transforms = DiGraph()
self._paths = {}
def _get_path(self,
frame_from,
frame_to):
'''
Find a path between two frames, either from cached paths or
from the transform graph.
Arguments
---------
frame_from: a frame key, usually a string
example: 'world'
frame_to: a frame key, usually a string
example: 'mesh_0'
Returns
----------
path: (n) list of frame keys
example: ['mesh_finger', 'mesh_hand', 'world']
inverted: boolean flag, whether the path is traversing stored
matrices forwards or backwards.
'''
try:
return self._paths[(frame_from, frame_to)]
except KeyError:
return self._generate_path(frame_from, frame_to)
def _generate_path(self,
frame_from,
frame_to):
'''
Generate a path between two frames.
Arguments
---------
frame_from: a frame key, usually a string
example: 'world'
frame_to: a frame key, usually a string
example: 'mesh_0'
Returns
----------
path: (n) list of frame keys
example: ['mesh_finger', 'mesh_hand', 'world']
inverted: boolean flag, whether the path is traversing stored
matrices forwards or backwards.
'''
try:
path = shortest_path(self._transforms, frame_from, frame_to)
inverted = False
except NetworkXNoPath:
path = shortest_path(self._transforms, frame_to, frame_from)
inverted = True
self._paths[(frame_from, frame_to)] = (path, inverted)
return path, inverted
def from_equilibrium_graph(cls, graph: nx.DiGraph, ph_values: np.ndarray):
"""Instantiate a titration curve specified using microequilibrium pKas
Parameters
----------
graph - a directed graph of microequilibria.
ph_values - 1D-array of pH values that correspond to the curve
Notes
-----
In the equilibrium graph every node is a state.
Edges between states have a pKa associated with them.
Every edge has direction from Deprotonated -> Protonated.
Can be generated using `micro_pKas_to_equilibrium_graph`
"""
# First state has least protons bound, set to 0 to be reference to other states
reference = list(nx.algorithms.dag.dag_longest_path(graph))[0]
all_nodes = list(deepcopy(graph.nodes))
all_nodes.remove(reference)
all_nodes.insert(0, reference)
augmented_graph = add_reverse_equilibrium_arrows(graph)
augmented_graph = add_Ka_equil_graph(augmented_graph)
instance = cls()
instance.augmented_graph = augmented_graph
energies: List[np.ndarray] = list()
nbound: List[int] = [0]
energies.append(free_energy_from_pka(0, 0.0, ph_values))
# Every node is a state
for s, state in enumerate(all_nodes[1:], start=1):
# Least number of equilibria to pass through to reach a state
# If there are more than one path, the shortest one is the one that uses pKas closer to 7
# Which should be the most relevant range, and likely the applicable range of most techniques
path = nx.shortest_path(augmented_graph,
reference,
all_nodes[s],
weight="pKa7")
# The number of protons is equal to the number of equilibria traversed
bound_protons = len(path) - 1
sumpKa = 0
# Add pKa along edges of the path
for edge in range(bound_protons):
sumpKa += augmented_graph[path[edge]][path[edge + 1]]["pKa"]
# For reverse paths, deduct one proton
if not graph.has_edge(path[edge], path[edge + 1]):
bound_protons -= 1
# Free energy calculated according to Ullmann (2003).
energies.append(
free_energy_from_pka(bound_protons, sumpKa, ph_values))
nbound.append(bound_protons)
instance.free_energies = np.asarray(energies)
instance.populations = populations_from_free_energies(
instance.free_energies)
instance.ph_values = ph_values
instance.state_ids = all_nodes
instance.charges = np.asarray(nbound)
instance.mean_charge = instance.charges @ instance.populations
# Set lowest value to 0
instance.mean_charge -= int(round(min(instance.mean_charge)))
return instance
