def load_data(cls, compiled_net, context, batch_index):
"""Load data from the sources of the model and adds them to the compiled net.
Parameters
----------
context : ComputationContext
compiled_net : nx.DiGraph
batch_index : int
Returns
-------
output_net : nx.DiGraph
"""
# Make a shallow copy of the graph
loaded_net = nx.DiGraph(compiled_net)
loaded_net = ObservedLoader.load(context, loaded_net, batch_index)
loaded_net = AdditionalNodesLoader.load(context, loaded_net,
batch_index)
loaded_net = RandomStateLoader.load(context, loaded_net, batch_index)
loaded_net = PoolLoader.load(context, loaded_net, batch_index)
# Add cache from the contect
loaded_net.graph['_executor_cache'] = context.caches['executor']
return loaded_net
def compile(cls, source_net, outputs=None):
"""Compile the structure of the output net.
Does not insert any data into the net.
Parameters
----------
source_net : nx.DiGraph
Can be acquired from `model.source_net`
outputs : list of node names
Returns
-------
output_net : nx.DiGraph
output_net codes the execution of the model
"""
if outputs is None:
outputs = source_net.nodes()
if not outputs:
logger.warning("Compiling for no outputs!")
outputs = outputs if isinstance(outputs, list) else [outputs]
compiled_net = nx.DiGraph(outputs=outputs,
name=source_net.graph['name'],
observed=source_net.graph['observed'])
compiled_net = OutputCompiler.compile(source_net, compiled_net)
compiled_net = ObservedCompiler.compile(source_net, compiled_net)
compiled_net = AdditionalNodesCompiler.compile(source_net,
compiled_net)
compiled_net = RandomStateCompiler.compile(source_net, compiled_net)
compiled_net = ReduceCompiler.compile(source_net, compiled_net)
return compiled_net
def get_execution_order(cls, G):
"""Return a list of nodes to execute.
This method returns the minimal list of nodes that need to be executed in
graph G in order to return the requested outputs.
The ordering of the nodes is fixed.
Parameters
----------
G : nx.DiGraph
Returns
-------
nodes : list
nodes that require execution
"""
# Get the cache dict if it exists
cache = G.graph.get('_executor_cache', {})
output_nodes = G.graph['outputs']
# Filter those output nodes who have an operation to run
needed = tuple(
sorted(node for node in output_nodes
if 'operation' in G.node[node]))
if needed not in cache:
# Resolve the nodes that need to be executed in the graph
nodes_to_execute = set(needed)
if 'sort_order' not in cache:
cache['sort_order'] = nx_constant_topological_sort(G)
sort_order = cache['sort_order']
# Resolve the dependencies of needed
dep_graph = nx.DiGraph(G.edges())
for node in sort_order:
attr = G.node[node]
if attr.keys() >= {'operation', 'output'}:
raise ValueError(
'Generative graph has both op and output present')
# Remove those nodes from the dependency graph whose outputs are present
if 'output' in attr:
dep_graph.remove_node(node)
elif 'operation' not in attr:
raise ValueError(
'Generative graph has no op or output present')
# Add the dependencies of the needed nodes
for needed_node in needed:
nodes_to_execute.update(nx.ancestors(dep_graph, needed_node))
# Turn in to a sorted list and cache
cache[needed] = [n for n in sort_order if n in nodes_to_execute]
return cache[needed]
def __init__(self, source_net=None):
"""Initialize the graph.
Parameters
----------
source_net : nx.DiGraph, optional
"""
self.source_net = source_net or nx.DiGraph()
def copy(self):
"""Return a copy of the graph."""
kopy = self.__class__()
# Copy the source net
kopy.source_net = nx.DiGraph(self.source_net)
return kopy