def __init__(self, processors, sendee=None, sending=True):
super(ChainProcessor, self).__init__(sendee, sending=sending)
processors = tuple(processors)
if not processors:
raise ValueError("expected at least one element in processors chain, got zero")
# front of the chain, where we push stuff
self.head = processors[0]
# create chain of processors, linking each processor's send function to
# each successor's process() function....
gb = GraphBuilder()
nodes, starts, ends = gb.add_graph(processors[0].graph)
assert len(starts) == 1 and len(ends) == 1
if len(processors) > 1:
succ_iter = iter(processors)
succ_iter.next()
for pred, succ in izip(processors, succ_iter):
pred.set_sendee(succ.process)
nodes, new_starts, new_ends = gb.add_graph(succ.graph)
gb.new_arc(ends[0], new_starts[0])
starts, ends = new_starts, new_ends
assert len(starts) == 1 and len(ends) == 1
assert succ == processors[-1]
# set up the list that will collect what the final element pushes
self.collector = list()
processors[-1].set_sendee(self.collector.append)
self._graph = FrozenGraph(gb)
def test2(num_obs):
# Each of the 2 nodes contains a 4-node order-2 Hmm; the nodes are connected in single chain
dimension = 2
obs_gen = make_data_generator(dimension)
obs_list = [obs_gen.next() for i in xrange(num_obs)]
# GmmMgr setup
num_models = 20
models = make_standard_gmms(dimension, num_models)
gmm_mgr1 = GmmMgr(models[0:10])
gmm_mgr2 = GmmMgr(models[10:20])
# Hmm setup
# Make two Hmms with 4 states and order 2 (self loop, forward 1)
num_states = 4
seed(0)
hmm0 = make_forward_hmm(gmm_mgr1, num_states, 2, exact=True)
hmm1 = make_forward_hmm(gmm_mgr1, num_states, 2, exact=True)
hmm_mgr = HmmMgr((hmm0, hmm1))
# TrainingGraph setup
gb = GraphBuilder()
node_id0 = gb.new_node((0, 0))
node_id1 = gb.new_node((1, 1))
arc_id = gb.new_arc(node_id0, node_id1)
gr0 = FrozenGraph(gb)
tg0 = TrainingGraph(gr0, hmm_mgr, dict())
valid, ret = validate_training_graph(tg0, gmm_mgr1, hmm_mgr, obs_list, 1,
gmm_mgr2)
return ret
def test1(num_obs):
# 1 node contains a 4-node order-2 Hmm
dimension = 2
obs_gen = make_data_generator(dimension)
obs_list = [obs_gen.next() for i in xrange(num_obs)]
# GmmMgr setup
num_models = 20
models = make_standard_gmms(dimension, num_models)
gmm_mgr1 = GmmMgr(models[0:10])
gmm_mgr2 = GmmMgr(models[10:20])
# Hmm setup
# Make one Hmm with 4 states and order 2 (self loop, forward 1)
num_states = 4
seed(0)
hmm0 = make_forward_hmm(gmm_mgr1, num_states, 2, exact=True)
hmm_mgr = HmmMgr((hmm0, ))
# TrainingGraph setup
gb = GraphBuilder()
node_id0 = gb.new_node((0, 0))
gr0 = FrozenGraph(gb)
tg0 = TrainingGraph(gr0, hmm_mgr, dict())
valid, ret = validate_training_graph(tg0, gmm_mgr1, hmm_mgr, obs_list, 1,
gmm_mgr2)
return ret
def test4(num_passes, num_obs):
# Each of the 4 nodes contains a 4 (or 6)-node order-3 Hmm; the nodes are connected in a
# diamond pattern
ret = ""
dimension = 2
# Data generator setup and data generation
obs_gen = make_data_generator(dimension)
obs_list = [obs_gen.next() for i in xrange(num_obs)]
# GmmMgr setup
num_models = 10
models = make_standard_gmms(dimension, num_models)
gmm_mgr = GmmMgr(models)
# Hmm setup
# Make three Hmms with 4 (or 6) states and order 3 (self loop, forward 1, forward 2)
num_states = 4
seed(0)
hmm0 = make_forward_hmm(gmm_mgr, num_states, 3, exact=True)
hmm1 = make_forward_hmm(gmm_mgr, num_states + 2, 3, exact=True)
hmm2 = make_forward_hmm(gmm_mgr, num_states, 3, exact=True)
hmm_mgr = HmmMgr((hmm0, hmm1, hmm2))
# TrainingGraph setup
gb = GraphBuilder()
# Note that here we are using the same HMM in two different TG nodes
node_id0 = gb.new_node((0, 0))
node_id1 = gb.new_node((1, 1))
node_id2 = gb.new_node((2, 2))
node_id3 = gb.new_node((3, 0))
arc_id = gb.new_arc(node_id0, node_id1)
arc_id = gb.new_arc(node_id0, node_id2)
arc_id = gb.new_arc(node_id1, node_id3)
arc_id = gb.new_arc(node_id2, node_id3)
gr0 = FrozenGraph(gb)
spd = {}
spd[(0, 1)] = (0.4, 0.3, 0.8)
spd[(0, 2)] = (0.6, 0.7, 0.2)
tg0 = TrainingGraph(gr0, hmm_mgr, spd)
# Now adapt original TrainingGraph
for i in xrange(num_passes):
gmm_mgr.set_adaptation_state("INITIALIZING")
gmm_mgr.clear_all_accumulators()
tg0.begin_training()
gmm_mgr.set_adaptation_state("ACCUMULATING")
for obs in obs_list:
tg0.train_one_sequence(obs)
tg0.end_training()
gmm_mgr.set_adaptation_state("APPLYING")
gmm_mgr.apply_all_accumulators()
gmm_mgr.set_adaptation_state("NOT_ADAPTING")
ret = tg0.to_string(full=True)
return ret
def _test11():
# A reduced version of test10
ret = ""
# GmmMgr setup
num_states = 2
dimension = 2
models = []
for i in xrange(num_states):
dm = DummyModel(dimension, 1.0)
models.append(dm)
gmm_mgr = GmmMgr(models)
gb = GraphBuilder()
node_id0 = gb.new_node((0, 0))
node_id1 = gb.new_node((1, 1))
node_id2 = gb.new_node((2, 1))
node_id3 = gb.new_node((3, 1))
node_id4 = gb.new_node((4, 2))
# The topology here is slightly complex than the previous example
arc_id = gb.new_arc(node_id0, node_id1)
arc_id = gb.new_arc(node_id1, node_id4)
arc_id = gb.new_arc(node_id0, node_id2)
arc_id = gb.new_arc(node_id2, node_id3)
arc_id = gb.new_arc(node_id3, node_id4)
arc_id = gb.new_arc(node_id2, node_id4)
gr0 = FrozenGraph(gb)
# Make two Hmms with 3 states and order 2 (self loop, forward 1)
# The models in the middle are special and can skip.
seed(0)
hmm0 = make_forward_hmm(gmm_mgr, num_states, order=2, exact=False)
hmm1 = Hmm(1)
trans = array(((0.0, 0.5, 0.5), (0.0, 0.5, 0.5), (0.0, 0.0, 0.0)))
hmm1.build_model(gmm_mgr, (0, ), 1, 1, trans)
hmm2 = make_forward_hmm(gmm_mgr, num_states, order=2, exact=True)
hmm_mgr = HmmMgr((hmm0, hmm1, hmm2))
spd = {}
spd[(0, 1)] = (0.4, )
spd[(0, 2)] = (0.6, )
spd[(2, 3)] = (0.4, )
spd[(2, 4)] = (0.6, )
tg0 = TrainingGraph(gr0, hmm_mgr, split_prob_dict=spd)
if do_display:
tg0.dot_display()
tg0.dot_display(expand_hmms=True)
with DebugPrint("bwt_ctsh") if True else DebugPrint():
result_hmm = tg0.convert_to_standalone_hmm()
ret += "\n\n========= TG CONVERTED TO Hmm =========\n\n" + result_hmm.to_string(
full=True)
return ret
def _sequence_to_linear_graph(s):
gb = GraphBuilder()
start = gb.new_node()
for item in s:
end = gb.new_node()
gb.new_arc(start, end, item)
start = end
return FrozenGraph(gb)
def __init__(self, sendee=None, sending=True, label=None):
self._sendee = None
self._sending = sending
if sendee is not None:
self.set_sendee(sendee)
self._label = label if label is not None else type(self).__name__
gb = GraphBuilder()
node = gb.new_node(self._label)
self._graph = FrozenGraph(gb)
def _test9():
# Like test8, but now HMMs have multiple inputs and outputs.
ret = ""
# GmmMgr setup
num_states = 3
dimension = 2
models = []
for i in xrange(num_states):
dm = DummyModel(dimension, 1.0)
models.append(dm)
gmm_mgr = GmmMgr(models)
gb = GraphBuilder()
node_id0 = gb.new_node((0, 0))
node_id1 = gb.new_node((1, 1))
node_id2 = gb.new_node((2, 1))
node_id3 = gb.new_node((3, 1))
node_id4 = gb.new_node((4, 1))
node_id5 = gb.new_node((5, 2))
arc_id = gb.new_arc(node_id0, node_id1)
arc_id = gb.new_arc(node_id1, node_id2)
arc_id = gb.new_arc(node_id2, node_id3)
arc_id = gb.new_arc(node_id3, node_id4)
arc_id = gb.new_arc(node_id4, node_id5)
gr0 = FrozenGraph(gb)
# Make two Hmms with 3 states and order 3 (self loop, forward 1, forward 2)
# The models in the middle are special and can skip directly
seed(0)
hmm0 = make_forward_hmm(gmm_mgr, num_states, order=3, exact=True)
hmm1 = Hmm(1)
trans = array(((0.0, 0.0, 0.0, 0.5, 0.5, 0.0,
0.0), (0.0, 0.0, 0.0, 0.5, 0.0, 0.5,
0.0), (0.0, 0.0, 0.0, 0.5, 0.0, 0.0,
0.5), (0.0, 0.0, 0.0, 0.5, 0.35, 0.1, 0.05),
(0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
0.0), (0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
0.0), (0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0)))
hmm1.build_model(gmm_mgr, (0, ), 3, 3, trans)
hmm2 = make_forward_hmm(gmm_mgr, num_states, order=3, exact=True)
hmm_mgr = HmmMgr((hmm0, hmm1, hmm2))
with DebugPrint("bwt_vrfy") if False else DebugPrint():
tg0 = TrainingGraph(gr0, hmm_mgr, split_prob_dict=dict())
result_hmm = tg0.convert_to_standalone_hmm()
ret += "\n\n========= TG CONVERTED TO Hmm =========\n\n" + result_hmm.to_string(
full=True)
return ret
def build_it(linesstring):
builder = SetGraphBuilder()
boxen = set()
for arc in StringIO.StringIO(linesstring):
parts = arc.split()
if not parts or parts[0].startswith('#'):
continue
if len(parts) == 1:
builder.add_node(*parts)
elif len(parts) == 2:
builder.add_arc(*parts)
else:
assert False, str(parts)
return FrozenGraph(builder)
def test5(num_obs, do_display=False):
# A test in which one of the HMMs has a transition from an input directly to
# an output, so it can behave as an epsilon. This node is between two other
# nodes in a linear arrangement.
# Data generator setup and data generation
dimension = 2
obs_gen = make_data_generator(dimension)
obs_list = [obs_gen.next() for i in xrange(num_obs)]
# GmmMgr setup
num_models = 20
models = make_standard_gmms(dimension, num_models)
gmm_mgr1 = GmmMgr(models[0:10])
gmm_mgr2 = GmmMgr(models[10:20])
# Hmm setup
# Make two Hmms with 2 states and order 2 (self loop, forward 1) The model
# in the middle is special in that it can skip directly from the input state
# to the output state.
seed(0)
num_states = 2
hmm0 = make_forward_hmm(gmm_mgr1, num_states, 2, exact=False)
hmm1 = Hmm(1)
trans = array(((0.0, 0.5, 0.5), (0.0, 0.5, 0.5), (0.0, 0.0, 0.0)))
hmm1.build_model(gmm_mgr1, (0, ), 1, 1, trans)
hmm2 = make_forward_hmm(gmm_mgr1, num_states, 2, exact=False)
hmm_mgr = HmmMgr((hmm0, hmm1, hmm2))
# TrainingGraph setup
gb = GraphBuilder()
node_id0 = gb.new_node((0, 0))
node_id1 = gb.new_node((1, 1))
# node_id2 = gb.new_node((2,2))
arc_id = gb.new_arc(node_id0, node_id1)
# arc_id = gb.new_arc(node_id1, node_id2)
gr0 = FrozenGraph(gb)
tg0 = TrainingGraph(gr0, hmm_mgr, split_prob_dict=dict())
if do_display:
tg0.dot_display()
tg0.dot_display(expand_hmms=True)
valid, ret = validate_training_graph(tg0, gmm_mgr1, hmm_mgr, obs_list, 1,
gmm_mgr2)
return ret
def test3(num_obs):
# Each of the 4 nodes contains a 4 (or 6)-node order-3 Hmm; the nodes are connected in a
# diamond pattern
dimension = 2
obs_gen = make_data_generator(dimension)
obs_list = [obs_gen.next() for i in xrange(num_obs)]
# GmmMgr setup
num_states = 4
num_models = 20
models = make_standard_gmms(dimension, num_models)
gmm_mgr1 = GmmMgr(models[0:10])
gmm_mgr2 = GmmMgr(models[10:20])
# Hmm setup
# Make four Hmms with 4 (or 6) states and order 3 (self loop, forward 1, forward 2)
seed(0)
hmm0 = make_forward_hmm(gmm_mgr1, num_states, 3, exact=True)
# NB: the asymetry between the two successors is a key part of this test; otherwise,
# there are no differences between the transition probs going to these successors,
# which is the tricky case
hmm1 = make_forward_hmm(gmm_mgr1, num_states + 2, 3, exact=True)
hmm2 = make_forward_hmm(gmm_mgr1, num_states, 3, exact=True)
hmm3 = make_forward_hmm(gmm_mgr1, num_states, 3, exact=True)
hmm_mgr = HmmMgr((hmm0, hmm1, hmm2, hmm3))
# TrainingGraph setup
gb = GraphBuilder()
node_id0 = gb.new_node((0, 0))
node_id1 = gb.new_node((1, 1))
node_id2 = gb.new_node((2, 2))
node_id3 = gb.new_node((3, 3))
arc_id = gb.new_arc(node_id0, node_id1)
arc_id = gb.new_arc(node_id0, node_id2)
arc_id = gb.new_arc(node_id1, node_id3)
arc_id = gb.new_arc(node_id2, node_id3)
gr0 = FrozenGraph(gb)
spd = {}
spd[(0, 1)] = (0.4, 0.3, 0.8)
spd[(0, 2)] = (0.6, 0.7, 0.2)
tg0 = TrainingGraph(gr0, hmm_mgr, spd)
valid, ret = validate_training_graph(tg0, gmm_mgr1, hmm_mgr, obs_list, 1,
gmm_mgr2)
return ret
def labels_to_lattice(labels):
"""
From a sequence of labels, build a linear lattice with labels on the arcs.
The result will have node labels which are guaranteed to be unique.
>>> label_dict = {'A': 1, 'B': 4, 'C': 9}
>>> labels_to_lattice(('A', 'B', 'C'))
FrozenGraph(GraphTables(((0, 1, 2, 3), (0, 1, 2), (1, 2, 3), ('A', 'B', 'C'))))
"""
gb = SetGraphBuilder()
counter = itertools.count()
start = gb.add_node(counter.next())
for l in labels:
end = gb.add_node(counter.next())
gb.add_arc(start, end, l)
start = end
return FrozenGraph(gb)
def __init__(self, stream):
docgen = YamldataGenerator(stream)
reader = YamldataReader(docgen, stream_type=self.STREAM_TYPE, stream_version=self.STREAM_VERSION, header_only=True)
self.runtime_objects = IndexedObjectSet(docgen)
self.dataflow_graph = FrozenGraph(SerializedGraphTables(docgen))
# until we've sorted out the yaml issues
# make a linear graph with arc labels indexing into runtime_objects
builder = GraphBuilder()
startid = builder.new_node_label_is_id()
for arc_label, obj in enumerate(self.runtime_objects):
endid = builder.new_node_label_is_id()
arcid = builder.new_arc(startid, endid, arc_label)
startid = endid
self.graph = FrozenGraph(builder)
for i in self.runtime_objects:
self.last = i[-1]
def _test10():
# Like test9, but now HMMs are arranged in a diamond pattern so inter-HMM
# probabilities come into play
ret = ""
# GmmMgr setup
num_states = 3
dimension = 2
models = []
for i in xrange(num_states):
dm = DummyModel(dimension, 1.0)
models.append(dm)
gmm_mgr = GmmMgr(models)
gb = GraphBuilder()
node_id0 = gb.new_node((0, 0))
node_id1 = gb.new_node((1, 1))
node_id2 = gb.new_node((2, 1))
node_id3 = gb.new_node((3, 1))
node_id4 = gb.new_node((4, 1))
node_id5 = gb.new_node((5, 2))
# The topology here is more complex than previous examples
arc_id = gb.new_arc(node_id0, node_id1)
arc_id = gb.new_arc(node_id1, node_id5)
arc_id = gb.new_arc(node_id0, node_id2)
arc_id = gb.new_arc(node_id2, node_id3)
arc_id = gb.new_arc(node_id3, node_id4)
arc_id = gb.new_arc(node_id3, node_id5)
arc_id = gb.new_arc(node_id4, node_id5)
gr0 = FrozenGraph(gb)
# Make two Hmms with 3 states and order 3 (self loop, forward 1, forward 2)
# The models in the middle are special and can skip.
seed(0)
hmm0 = make_forward_hmm(gmm_mgr, num_states, order=3, exact=True)
hmm1 = Hmm(1)
trans = array(((0.0, 0.0, 0.0, 0.5, 0.5, 0.0,
0.0), (0.0, 0.0, 0.0, 0.5, 0.0, 0.5,
0.0), (0.0, 0.0, 0.0, 0.5, 0.0, 0.0,
0.5), (0.0, 0.0, 0.0, 0.5, 0.35, 0.1, 0.05),
(0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
0.0), (0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
0.0), (0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0)))
hmm1.build_model(gmm_mgr, (0, ), 3, 3, trans)
hmm2 = make_forward_hmm(gmm_mgr, num_states, order=3, exact=True)
hmm_mgr = HmmMgr((hmm0, hmm1, hmm2))
spd = {}
spd[(0, 1)] = (0.4, 0.3, 0.8)
spd[(0, 2)] = (0.6, 0.7, 0.2)
spd[(3, 4)] = (0.4, 0.3, 0.8)
spd[(3, 5)] = (0.6, 0.7, 0.2)
tg0 = TrainingGraph(gr0, hmm_mgr, split_prob_dict=spd)
with DebugPrint("bwt_ctsh") if True else DebugPrint():
result_hmm = tg0.convert_to_standalone_hmm()
ret += "\n\n========= TG CONVERTED TO Hmm =========\n\n" + result_hmm.to_string(
full=True)
return ret
def _lattice_nbest_align(l1, l2, cost_fn=None, substring_cost=False):
if not (isinstance(l1, FrozenGraph) and isinstance(l2, FrozenGraph)):
raise ValueError("lattice_nbest_align needs two FrozenGraph arguments")
if cost_fn is None:
cost_fn = _std_cost
else:
cost_fn = _make_safe_cost_fn(cost_fn)
l1_canon = l1.get_canonical_DAG()
l2_canon = l2.get_canonical_DAG()
len1 = l1_canon.get_num_arcs()
len2 = l2_canon.get_num_arcs()
assert l1_canon.is_lattice()
assert l2_canon.is_lattice()
terms = l1_canon.get_terminals()
lat_start1 = terms[0][0]
lat_end1 = terms[1][0]
terms = l2_canon.get_terminals()
lat_start2 = terms[0][0]
lat_end2 = terms[1][0]
# The lattices to be aligned, l1 and l2, have labels on the arcs;
# any labels on their nodes will be ignored. We begin by finding
# a topological ordering of the lattice arcs, so that each arc is
# assigned a non-negative index. We construct a 2-D lattice, g,
# with nodes representing pairs of arcs in the l1 and l2. An
# additional row and column on the top and left of g represent an
# initial position prior consuming any tokens from l1 and l2,
# respectively. The arcs in g are labeled with triples giving the
# orginal labels from l1 and l2 (or None if the arc's start node
# is on the left side or top row) and the local cost of the edit
# represented by the arc. There are two slightly tricky parts.
# First, because the alignment is done on lattices, the cost
# lattice may have links which cross several rows or columns,
# depending on the adjacencies of l1 and l2. Second, there's an
# offset of 1 between the arc numbering for l1 and l2 and the node
# numbering in g, because of the initial row and column. Thus,
# the node in g corresponding to the arc pair <a1, a2> is at
# indices <a1+1, a2+1>. One egregious (but very handy) abuse of
# this arrangement is the occasional use of -1 as an ersatz arcID,
# which will be converted to a 0 index into g. Finally, because
# the graph iterpath function requires a single start and end
# node, we tie all the potential end nodes in g to a special end
# node with "ground" arcs. A node in g is a potential end node if
# the pair of arcs it represents are each incident on the terminal
# node of their respective lattices.
gb = GraphBuilder()
# Initialize cost lattice nodes
node_array = [[gb.new_node() for j in range(len2 + 1)]
for i in range(len1 + 1)]
# We use this single node to tie all end nodes together
end_node = gb.new_node()
# Add first row of arcs
for i in xrange(len1):
start, end, x = l1_canon.get_arc(i)
insert_cost = cost_fn(x, None)
if end == lat_end1 and len2 == 0:
gb.new_arc(node_array[i + 1][0], end_node, (None, None, 0))
# print "Added ground arc for l1 arc from %d to %d with label %s" % (start, end, x)
pred_arcs = l1_canon.get_node_in_arcs(start)
if start == lat_start1:
pred_arcs.append(-1)
for arc in pred_arcs:
gb.new_arc(node_array[arc + 1][0], node_array[i + 1][0],
(x, None, insert_cost))
# print ("Processed l1 arc from %d to %d with label %s - added %d arcs"
# % (start, end, x, len(pred_arcs)))
# Add first column of arcs
for j in xrange(len2):
start, end, y = l2_canon.get_arc(j)
delete_cost = 0 if substring_cost else cost_fn(None, y)
if end == lat_end2 and len1 == 0:
gb.new_arc(node_array[0][j + 1], end_node, (None, None, 0))
# print "Added ground arc for l1 arc from %d to %d with label %s" % (start, end, y)
pred_arcs = l2_canon.get_node_in_arcs(start)
if start == lat_start2:
pred_arcs.append(-1)
for arc in pred_arcs:
gb.new_arc(node_array[0][arc + 1], node_array[0][j + 1],
(None, y, delete_cost))
# print ("Processed l1 arc from %d to %d with label %s - added %d arcs"
# % (start, end, y, len(pred_arcs)))
# Construct remainder of cost lattice
for i in xrange(len1):
for j in xrange(len2):
start1, end1, x = l1_canon.get_arc(i)
start2, end2, y = l2_canon.get_arc(j)
pred_arcs1 = l1_canon.get_node_in_arcs(start1)
pred_arcs2 = l2_canon.get_node_in_arcs(start2)
if start1 == lat_start1:
pred_arcs1.append(-1)
if start2 == lat_start2:
pred_arcs2.append(-1)
insert_cost = cost_fn(x, None)
delete_cost = 0 if (substring_cost and i == len1 - 1) else cost_fn(
None, y)
subst_cost = cost_fn(x, y)
num_added = 0
if end1 == lat_end1 and end2 == lat_end2:
gb.new_arc(node_array[i + 1][j + 1], end_node, (None, None, 0))
# print "Added ground arc for l1,l1 arcs with labels %s, %s" % (x,y)
for arc in pred_arcs1:
gb.new_arc(node_array[arc + 1][j + 1],
node_array[i + 1][j + 1], (x, None, insert_cost))
num_added += 1
for arc in pred_arcs2:
gb.new_arc(node_array[i + 1][arc + 1],
node_array[i + 1][j + 1], (None, y, delete_cost))
num_added += 1
for arc1 in pred_arcs1:
for arc2 in pred_arcs2:
gb.new_arc(node_array[arc1 + 1][arc2 + 1],
node_array[i + 1][j + 1], (x, y, subst_cost))
num_added += 1
# print ("Processed l1 arc from %d to %d with label %s "
# "and l1 arc from %d to %d with label %s - added %d arcs"
# % (start1, end1, x, start2, end2, y, num_added))
g = FrozenGraph(gb)
assert g.is_lattice()
# print "g.get_terminals() = %s, %s" % g.get_terminals()
def graph_cost_fn(label):
return label[2]
def iter_helper(path):
arc_labels = [path[2].get_arc_label(arc) for arc in path[1][:-1]]
return (path[0], tuple(arc_labels))
return imap(iter_helper,
g.iterpaths(graph_cost_fn, node_array[0][0], end_node))
class Channel(processor):
"""
Signal processing object pushes data through objects in a graph
>>> docfile = '''
... ---
... - __onyx_yaml__meta_version : '1'
... __onyx_yaml__stream_type : GraphBasedProcessor
... __onyx_yaml__stream_version : '0'
... ...
... ---
... - __onyx_yaml__meta_version : '1'
... __onyx_yaml__stream_type : IndexedObjectSet
... __onyx_yaml__stream_version : '0'
... __onyx_yaml__stream_options : implicit_index=True
...
... -
... # format for IndexedObjectSet, where presence of index field depends on value of implicit_index
... # [index] module factory args-string
... - onyx.signalprocessing.spectrum PreEmphasis
... - onyx.signalprocessing.window Sliding length=25000*usec shift=10000*usec
... - onyx.signalprocessing.window Hamming
...
... ...
...
... ---
... - __onyx_yaml__meta_version : '1'
... __onyx_yaml__stream_type : SerializedGraphTables
... __onyx_yaml__stream_version : '0'
...
... -
... # canonic format for SerializedGraphTables
... - num_nodes 3
... # fields are: node-id node-label
... - 0 -0
... - 1 -1
... - 2 -2
... - num_arcs 3
... # fields are: arc-id start-node-id end-node-id arc-label
... - 0 0 1 x
... - 1 0 2 y
... - 2 1 2 z
... '''
# for now we use a trivial version
>>> docfile = '''
... ---
... - __onyx_yaml__meta_version : '1'
... __onyx_yaml__stream_type : GraphBasedProcessor
... __onyx_yaml__stream_version : '0'
...
... ...
...
... ---
... - __onyx_yaml__meta_version : '1'
... __onyx_yaml__stream_type : IndexedObjectSet
... __onyx_yaml__stream_version : '0'
... __onyx_yaml__stream_options : implicit_index=True
... -
... # format for IndexedObjectSet, where presence of index field depends on value of implicit_index
... # [index] module factory args-string
... - onyx.signalprocessing.spectrum PreEmphasis
... - onyx.signalprocessing.spectrum PreEmphasis
... - onyx.signalprocessing.window Sliding length=25000*usec shift=10000*usec
... #- onyx.signalprocessing.spectrum PreEmphasis
... #- onyx.signalprocessing.spectrum PreEmphasis
... #- onyx.signalprocessing.spectrum PreEmphasis
... ...
...
... ---
... - __onyx_yaml__meta_version : '1'
... __onyx_yaml__stream_type : SerializedGraphTables
... __onyx_yaml__stream_version : '0'
... -
... # canonic format for SerializedGraphTables
... - num_nodes 3
... # fields are: node-id node-label
... - 0 -0
... - 1 -1
... - 2 -2
... - num_arcs 3
... # fields are: arc-id start-node-id end-node-id arc-label
... - 0 0 1 x
... - 1 0 2 y
... - 2 1 2 z
... '''
>>> docfile = cStringIO.StringIO(docfile)
>>> chan = Channel(docfile)
XXX this is a broken lie
>>> chan.graph
FrozenGraph(GraphTables(((0, 1, 2, 3), (0, 1, 2), (1, 2, 3), (0, 1, 2))))
>>> chan.config(samplerate=44100)
win 1102 hop 367
>>> res = list()
>>> chan.set_recipient(res.append)
>>> chan.send(23)
>>> res
[]
>>> chan.send_many(xrange(10))
>>> res
[]
>>> chan.send_many(xrange(1105))
>>> map(floatutils.float_to_readable_string, res[0])
['+(+0004)0x7000000000000', '+(+0005)0x7000000000000', '-(+0004)0x51a0000000000', '-(+0005)0x48d0000000000', '+(+0004)0xc263f00000000', '+(+0002)0x8780000000000', '+(+0002)0xca06400000000', '+(+0003)0x0646400000000', '+(+0003)0x2789600000000', '+(+0003)0x48cc800000000', '+(+0003)0x6a0fa00000000', '+(+0001)0x2d4b000000000', '-(+0004)0x09b5100000000', '-(+0002)0x7d4e000000000', '+(+0003)0xd61c200000000', '+(+0002)0x8780000000000', '+(+0002)0xca06400000000', '+(+0003)0x0646400000000', '+(+0003)0x2789600000000', '+(+0003)0x48cc800000000', '+(+0003)0x6a0fa00000000', '+(+0003)0x8b52c00000000', '+(+0003)0xac95e00000000', '+(+0003)0xcdd9000000000', '+(+0003)0xef1c200000000', '+(+0004)0x082fa00000000', '+(+0004)0x18d1300000000', '+(+0004)0x2972c00000000', '+(+0004)0x3a14500000000', '+(+0004)0x4ab5e00000000', '+(+0004)0x5b57700000000', '+(+0004)0x6bf9000000000', '+(+0004)0x7c9a900000000', '+(+0004)0x8d3c200000000', '+(+0004)0x9dddb00000000', '+(+0004)0xae7f400000000', '+(+0004)0xbf20d00000000', '+(+0004)0xcfc2600000000', '+(+0004)0xe063f00000000', '+(+0004)0xf105800000000', '+(+0005)0x00d3880000000', '+(+0005)0x0924500000000', '+(+0005)0x1175180000000', '+(+0005)0x19c5e00000000', '+(+0005)0x2216a80000000', '+(+0005)0x2a67700000000', '+(+0005)0x32b8380000000', '+(+0005)0x3b09000000000', '+(+0005)0x4359c80000000', '+(+0005)0x4baa900000000', '+(+0005)0x53fb580000000', '+(+0005)0x5c4c200000000', '+(+0005)0x649ce80000000', '+(+0005)0x6cedb00000000', '+(+0005)0x753e780000000', '+(+0005)0x7d8f400000000', '+(+0005)0x85e0080000000', '+(+0005)0x8e30d00000000', '+(+0005)0x9681980000000', '+(+0005)0x9ed2600000000', '+(+0005)0xa723280000000', '+(+0005)0xaf73f00000000', '+(+0005)0xb7c4b80000000', '+(+0005)0xc015800000000', '+(+0005)0xc866480000000', '+(+0005)0xd0b7100000000', '+(+0005)0xd907d80000000', '+(+0005)0xe158a00000000', '+(+0005)0xe9a9680000000', '+(+0005)0xf1fa300000000', '+(+0005)0xfa4af80000000', '+(+0006)0x014de00000000', '+(+0006)0x0576440000000', '+(+0006)0x099ea80000000', '+(+0006)0x0dc70c0000000', '+(+0006)0x11ef700000000', '+(+0006)0x1617d40000000', '+(+0006)0x1a40380000000', '+(+0006)0x1e689c0000000', '+(+0006)0x2291000000000', '+(+0006)0x26b9640000000', '+(+0006)0x2ae1c80000000', '+(+0006)0x2f0a2c0000000', '+(+0006)0x3332900000000', '+(+0006)0x375af40000000', '+(+0006)0x3b83580000000', '+(+0006)0x3fabbc0000000', '+(+0006)0x43d4200000000', '+(+0006)0x47fc840000000', '+(+0006)0x4c24e80000000', '+(+0006)0x504d4c0000000', '+(+0006)0x5475b00000000', '+(+0006)0x589e140000000', '+(+0006)0x5cc6780000000', '+(+0006)0x60eedc0000000', '+(+0006)0x6517400000000', '+(+0006)0x693fa40000000', '+(+0006)0x6d68080000000', '+(+0006)0x71906c0000000', '+(+0006)0x75b8d00000000', '+(+0006)0x79e1340000000', '+(+0006)0x7e09980000000', '+(+0006)0x8231fc0000000', '+(+0006)0x865a600000000', '+(+0006)0x8a82c40000000', '+(+0006)0x8eab280000000', '+(+0006)0x92d38c0000000', '+(+0006)0x96fbf00000000', '+(+0006)0x9b24540000000', '+(+0006)0x9f4cb80000000', '+(+0006)0xa3751c0000000', '+(+0006)0xa79d800000000', '+(+0006)0xabc5e40000000', '+(+0006)0xafee480000000', '+(+0006)0xb416ac0000000', '+(+0006)0xb83f100000000', '+(+0006)0xbc67740000000', '+(+0006)0xc08fd80000000', '+(+0006)0xc4b83c0000000', '+(+0006)0xc8e0a00000000', '+(+0006)0xcd09040000000', '+(+0006)0xd131680000000', '+(+0006)0xd559cc0000000', '+(+0006)0xd982300000000', '+(+0006)0xddaa940000000', '+(+0006)0xe1d2f80000000', '+(+0006)0xe5fb5c0000000', '+(+0006)0xea23c00000000', '+(+0006)0xee4c240000000', '+(+0006)0xf274880000000', '+(+0006)0xf69cec0000000', '+(+0006)0xfac5500000000', '+(+0006)0xfeedb40000000', '+(+0007)0x018b0c0000000', '+(+0007)0x039f3e0000000', '+(+0007)0x05b3700000000', '+(+0007)0x07c7a20000000', '+(+0007)0x09dbd40000000', '+(+0007)0x0bf0060000000', '+(+0007)0x0e04380000000', '+(+0007)0x10186a0000000', '+(+0007)0x122c9c0000000', '+(+0007)0x1440ce0000000', '+(+0007)0x1655000000000', '+(+0007)0x1869320000000', '+(+0007)0x1a7d640000000', '+(+0007)0x1c91960000000', '+(+0007)0x1ea5c80000000', '+(+0007)0x20b9fa0000000', '+(+0007)0x22ce2c0000000', '+(+0007)0x24e25e0000000', '+(+0007)0x26f6900000000', '+(+0007)0x290ac20000000', '+(+0007)0x2b1ef40000000', '+(+0007)0x2d33260000000', '+(+0007)0x2f47580000000', '+(+0007)0x315b8a0000000', '+(+0007)0x336fbc0000000', '+(+0007)0x3583ee0000000', '+(+0007)0x3798200000000', '+(+0007)0x39ac520000000', '+(+0007)0x3bc0840000000', '+(+0007)0x3dd4b60000000', '+(+0007)0x3fe8e80000000', '+(+0007)0x41fd1a0000000', '+(+0007)0x44114c0000000', '+(+0007)0x46257e0000000', '+(+0007)0x4839b00000000', '+(+0007)0x4a4de20000000', '+(+0007)0x4c62140000000', '+(+0007)0x4e76460000000', '+(+0007)0x508a780000000', '+(+0007)0x529eaa0000000', '+(+0007)0x54b2dc0000000', '+(+0007)0x56c70e0000000', '+(+0007)0x58db400000000', '+(+0007)0x5aef720000000', '+(+0007)0x5d03a40000000', '+(+0007)0x5f17d60000000', '+(+0007)0x612c080000000', '+(+0007)0x63403a0000000', '+(+0007)0x65546c0000000', '+(+0007)0x67689e0000000', '+(+0007)0x697cd00000000', '+(+0007)0x6b91020000000', '+(+0007)0x6da5340000000', '+(+0007)0x6fb9660000000', '+(+0007)0x71cd980000000', '+(+0007)0x73e1ca0000000', '+(+0007)0x75f5fc0000000', '+(+0007)0x780a2e0000000', '+(+0007)0x7a1e600000000', '+(+0007)0x7c32920000000', '+(+0007)0x7e46c40000000', '+(+0007)0x805af60000000', '+(+0007)0x826f280000000', '+(+0007)0x84835a0000000', '+(+0007)0x86978c0000000', '+(+0007)0x88abbe0000000', '+(+0007)0x8abff00000000', '+(+0007)0x8cd4220000000', '+(+0007)0x8ee8540000000', '+(+0007)0x90fc860000000', '+(+0007)0x9310b80000000', '+(+0007)0x9524ea0000000', '+(+0007)0x97391c0000000', '+(+0007)0x994d4e0000000', '+(+0007)0x9b61800000000', '+(+0007)0x9d75b20000000', '+(+0007)0x9f89e40000000', '+(+0007)0xa19e160000000', '+(+0007)0xa3b2480000000', '+(+0007)0xa5c67a0000000', '+(+0007)0xa7daac0000000', '+(+0007)0xa9eede0000000', '+(+0007)0xac03100000000', '+(+0007)0xae17420000000', '+(+0007)0xb02b740000000', '+(+0007)0xb23fa60000000', '+(+0007)0xb453d80000000', '+(+0007)0xb6680a0000000', '+(+0007)0xb87c3c0000000', '+(+0007)0xba906e0000000', '+(+0007)0xbca4a00000000', '+(+0007)0xbeb8d20000000', '+(+0007)0xc0cd040000000', '+(+0007)0xc2e1360000000', '+(+0007)0xc4f5680000000', '+(+0007)0xc7099a0000000', '+(+0007)0xc91dcc0000000', '+(+0007)0xcb31fe0000000', '+(+0007)0xcd46300000000', '+(+0007)0xcf5a620000000', '+(+0007)0xd16e940000000', '+(+0007)0xd382c60000000', '+(+0007)0xd596f80000000', '+(+0007)0xd7ab2a0000000', '+(+0007)0xd9bf5c0000000', '+(+0007)0xdbd38e0000000', '+(+0007)0xdde7c00000000', '+(+0007)0xdffbf20000000', '+(+0007)0xe210240000000', '+(+0007)0xe424560000000', '+(+0007)0xe638880000000', '+(+0007)0xe84cba0000000', '+(+0007)0xea60ec0000000', '+(+0007)0xec751e0000000', '+(+0007)0xee89500000000', '+(+0007)0xf09d820000000', '+(+0007)0xf2b1b40000000', '+(+0007)0xf4c5e60000000', '+(+0007)0xf6da180000000', '+(+0007)0xf8ee4a0000000', '+(+0007)0xfb027c0000000', '+(+0007)0xfd16ae0000000', '+(+0007)0xff2ae00000000', '+(+0008)0x009f890000000', '+(+0008)0x01a9a20000000', '+(+0008)0x02b3bb0000000', '+(+0008)0x03bdd40000000', '+(+0008)0x04c7ed0000000', '+(+0008)0x05d2060000000', '+(+0008)0x06dc1f0000000', '+(+0008)0x07e6380000000', '+(+0008)0x08f0510000000', '+(+0008)0x09fa6a0000000', '+(+0008)0x0b04830000000', '+(+0008)0x0c0e9c0000000', '+(+0008)0x0d18b50000000', '+(+0008)0x0e22ce0000000', '+(+0008)0x0f2ce70000000', '+(+0008)0x1037000000000', '+(+0008)0x1141190000000', '+(+0008)0x124b320000000', '+(+0008)0x13554b0000000', '+(+0008)0x145f640000000', '+(+0008)0x15697d0000000', '+(+0008)0x1673960000000', '+(+0008)0x177daf0000000', '+(+0008)0x1887c80000000', '+(+0008)0x1991e10000000', '+(+0008)0x1a9bfa0000000', '+(+0008)0x1ba6130000000', '+(+0008)0x1cb02c0000000', '+(+0008)0x1dba450000000', '+(+0008)0x1ec45e0000000', '+(+0008)0x1fce770000000', '+(+0008)0x20d8900000000', '+(+0008)0x21e2a90000000', '+(+0008)0x22ecc20000000', '+(+0008)0x23f6db0000000', '+(+0008)0x2500f40000000', '+(+0008)0x260b0d0000000', '+(+0008)0x2715260000000', '+(+0008)0x281f3f0000000', '+(+0008)0x2929580000000', '+(+0008)0x2a33710000000', '+(+0008)0x2b3d8a0000000', '+(+0008)0x2c47a30000000', '+(+0008)0x2d51bc0000000', '+(+0008)0x2e5bd50000000', '+(+0008)0x2f65ee0000000', '+(+0008)0x3070070000000', '+(+0008)0x317a200000000', '+(+0008)0x3284390000000', '+(+0008)0x338e520000000', '+(+0008)0x34986b0000000', '+(+0008)0x35a2840000000', '+(+0008)0x36ac9d0000000', '+(+0008)0x37b6b60000000', '+(+0008)0x38c0cf0000000', '+(+0008)0x39cae80000000', '+(+0008)0x3ad5010000000', '+(+0008)0x3bdf1a0000000', '+(+0008)0x3ce9330000000', '+(+0008)0x3df34c0000000', '+(+0008)0x3efd650000000', '+(+0008)0x40077e0000000', '+(+0008)0x4111970000000', '+(+0008)0x421bb00000000', '+(+0008)0x4325c90000000', '+(+0008)0x442fe20000000', '+(+0008)0x4539fb0000000', '+(+0008)0x4644140000000', '+(+0008)0x474e2d0000000', '+(+0008)0x4858460000000', '+(+0008)0x49625f0000000', '+(+0008)0x4a6c780000000', '+(+0008)0x4b76910000000', '+(+0008)0x4c80aa0000000', '+(+0008)0x4d8ac30000000', '+(+0008)0x4e94dc0000000', '+(+0008)0x4f9ef50000000', '+(+0008)0x50a90e0000000', '+(+0008)0x51b3270000000', '+(+0008)0x52bd400000000', '+(+0008)0x53c7590000000', '+(+0008)0x54d1720000000', '+(+0008)0x55db8b0000000', '+(+0008)0x56e5a40000000', '+(+0008)0x57efbd0000000', '+(+0008)0x58f9d60000000', '+(+0008)0x5a03ef0000000', '+(+0008)0x5b0e080000000', '+(+0008)0x5c18210000000', '+(+0008)0x5d223a0000000', '+(+0008)0x5e2c530000000', '+(+0008)0x5f366c0000000', '+(+0008)0x6040850000000', '+(+0008)0x614a9e0000000', '+(+0008)0x6254b70000000', '+(+0008)0x635ed00000000', '+(+0008)0x6468e90000000', '+(+0008)0x6573020000000', '+(+0008)0x667d1b0000000', '+(+0008)0x6787340000000', '+(+0008)0x68914d0000000', '+(+0008)0x699b660000000', '+(+0008)0x6aa57f0000000', '+(+0008)0x6baf980000000', '+(+0008)0x6cb9b10000000', '+(+0008)0x6dc3ca0000000', '+(+0008)0x6ecde30000000', '+(+0008)0x6fd7fc0000000', '+(+0008)0x70e2150000000', '+(+0008)0x71ec2e0000000', '+(+0008)0x72f6470000000', '+(+0008)0x7400600000000', '+(+0008)0x750a790000000', '+(+0008)0x7614920000000', '+(+0008)0x771eab0000000', '+(+0008)0x7828c40000000', '+(+0008)0x7932dd0000000', '+(+0008)0x7a3cf60000000', '+(+0008)0x7b470f0000000', '+(+0008)0x7c51280000000', '+(+0008)0x7d5b410000000', '+(+0008)0x7e655a0000000', '+(+0008)0x7f6f730000000', '+(+0008)0x80798c0000000', '+(+0008)0x8183a50000000', '+(+0008)0x828dbe0000000', '+(+0008)0x8397d70000000', '+(+0008)0x84a1f00000000', '+(+0008)0x85ac090000000', '+(+0008)0x86b6220000000', '+(+0008)0x87c03b0000000', '+(+0008)0x88ca540000000', '+(+0008)0x89d46d0000000', '+(+0008)0x8ade860000000', '+(+0008)0x8be89f0000000', '+(+0008)0x8cf2b80000000', '+(+0008)0x8dfcd10000000', '+(+0008)0x8f06ea0000000', '+(+0008)0x9011030000000', '+(+0008)0x911b1c0000000', '+(+0008)0x9225350000000', '+(+0008)0x932f4e0000000', '+(+0008)0x9439670000000', '+(+0008)0x9543800000000', '+(+0008)0x964d990000000', '+(+0008)0x9757b20000000', '+(+0008)0x9861cb0000000', '+(+0008)0x996be40000000', '+(+0008)0x9a75fd0000000', '+(+0008)0x9b80160000000', '+(+0008)0x9c8a2f0000000', '+(+0008)0x9d94480000000', '+(+0008)0x9e9e610000000', '+(+0008)0x9fa87a0000000', '+(+0008)0xa0b2930000000', '+(+0008)0xa1bcac0000000', '+(+0008)0xa2c6c50000000', '+(+0008)0xa3d0de0000000', '+(+0008)0xa4daf70000000', '+(+0008)0xa5e5100000000', '+(+0008)0xa6ef290000000', '+(+0008)0xa7f9420000000', '+(+0008)0xa9035b0000000', '+(+0008)0xaa0d740000000', '+(+0008)0xab178d0000000', '+(+0008)0xac21a60000000', '+(+0008)0xad2bbf0000000', '+(+0008)0xae35d80000000', '+(+0008)0xaf3ff10000000', '+(+0008)0xb04a0a0000000', '+(+0008)0xb154230000000', '+(+0008)0xb25e3c0000000', '+(+0008)0xb368550000000', '+(+0008)0xb4726e0000000', '+(+0008)0xb57c870000000', '+(+0008)0xb686a00000000', '+(+0008)0xb790b90000000', '+(+0008)0xb89ad20000000', '+(+0008)0xb9a4eb0000000', '+(+0008)0xbaaf040000000', '+(+0008)0xbbb91d0000000', '+(+0008)0xbcc3360000000', '+(+0008)0xbdcd4f0000000', '+(+0008)0xbed7680000000', '+(+0008)0xbfe1810000000', '+(+0008)0xc0eb9a0000000', '+(+0008)0xc1f5b30000000', '+(+0008)0xc2ffcc0000000', '+(+0008)0xc409e50000000', '+(+0008)0xc513fe0000000', '+(+0008)0xc61e170000000', '+(+0008)0xc728300000000', '+(+0008)0xc832490000000', '+(+0008)0xc93c620000000', '+(+0008)0xca467b0000000', '+(+0008)0xcb50940000000', '+(+0008)0xcc5aad0000000', '+(+0008)0xcd64c60000000', '+(+0008)0xce6edf0000000', '+(+0008)0xcf78f80000000', '+(+0008)0xd083110000000', '+(+0008)0xd18d2a0000000', '+(+0008)0xd297430000000', '+(+0008)0xd3a15c0000000', '+(+0008)0xd4ab750000000', '+(+0008)0xd5b58e0000000', '+(+0008)0xd6bfa70000000', '+(+0008)0xd7c9c00000000', '+(+0008)0xd8d3d90000000', '+(+0008)0xd9ddf20000000', '+(+0008)0xdae80b0000000', '+(+0008)0xdbf2240000000', '+(+0008)0xdcfc3d0000000', '+(+0008)0xde06560000000', '+(+0008)0xdf106f0000000', '+(+0008)0xe01a880000000', '+(+0008)0xe124a10000000', '+(+0008)0xe22eba0000000', '+(+0008)0xe338d30000000', '+(+0008)0xe442ec0000000', '+(+0008)0xe54d050000000', '+(+0008)0xe6571e0000000', '+(+0008)0xe761370000000', '+(+0008)0xe86b500000000', '+(+0008)0xe975690000000', '+(+0008)0xea7f820000000', '+(+0008)0xeb899b0000000', '+(+0008)0xec93b40000000', '+(+0008)0xed9dcd0000000', '+(+0008)0xeea7e60000000', '+(+0008)0xefb1ff0000000', '+(+0008)0xf0bc180000000', '+(+0008)0xf1c6310000000', '+(+0008)0xf2d04a0000000', '+(+0008)0xf3da630000000', '+(+0008)0xf4e47c0000000', '+(+0008)0xf5ee950000000', '+(+0008)0xf6f8ae0000000', '+(+0008)0xf802c70000000', '+(+0008)0xf90ce00000000', '+(+0008)0xfa16f90000000', '+(+0008)0xfb21120000000', '+(+0008)0xfc2b2b0000000', '+(+0008)0xfd35440000000', '+(+0008)0xfe3f5d0000000', '+(+0008)0xff49760000000', '+(+0009)0x0029c78000000', '+(+0009)0x00aed40000000', '+(+0009)0x0133e08000000', '+(+0009)0x01b8ed0000000', '+(+0009)0x023df98000000', '+(+0009)0x02c3060000000', '+(+0009)0x0348128000000', '+(+0009)0x03cd1f0000000', '+(+0009)0x04522b8000000', '+(+0009)0x04d7380000000', '+(+0009)0x055c448000000', '+(+0009)0x05e1510000000', '+(+0009)0x06665d8000000', '+(+0009)0x06eb6a0000000', '+(+0009)0x0770768000000', '+(+0009)0x07f5830000000', '+(+0009)0x087a8f8000000', '+(+0009)0x08ff9c0000000', '+(+0009)0x0984a88000000', '+(+0009)0x0a09b50000000', '+(+0009)0x0a8ec18000000', '+(+0009)0x0b13ce0000000', '+(+0009)0x0b98da8000000', '+(+0009)0x0c1de70000000', '+(+0009)0x0ca2f38000000', '+(+0009)0x0d28000000000', '+(+0009)0x0dad0c8000000', '+(+0009)0x0e32190000000', '+(+0009)0x0eb7258000000', '+(+0009)0x0f3c320000000', '+(+0009)0x0fc13e8000000', '+(+0009)0x10464b0000000', '+(+0009)0x10cb578000000', '+(+0009)0x1150640000000', '+(+0009)0x11d5708000000', '+(+0009)0x125a7d0000000', '+(+0009)0x12df898000000', '+(+0009)0x1364960000000', '+(+0009)0x13e9a28000000', '+(+0009)0x146eaf0000000', '+(+0009)0x14f3bb8000000', '+(+0009)0x1578c80000000', '+(+0009)0x15fdd48000000', '+(+0009)0x1682e10000000', '+(+0009)0x1707ed8000000', '+(+0009)0x178cfa0000000', '+(+0009)0x1812068000000', '+(+0009)0x1897130000000', '+(+0009)0x191c1f8000000', '+(+0009)0x19a12c0000000', '+(+0009)0x1a26388000000', '+(+0009)0x1aab450000000', '+(+0009)0x1b30518000000', '+(+0009)0x1bb55e0000000', '+(+0009)0x1c3a6a8000000', '+(+0009)0x1cbf770000000', '+(+0009)0x1d44838000000', '+(+0009)0x1dc9900000000', '+(+0009)0x1e4e9c8000000', '+(+0009)0x1ed3a90000000', '+(+0009)0x1f58b58000000', '+(+0009)0x1fddc20000000', '+(+0009)0x2062ce8000000', '+(+0009)0x20e7db0000000', '+(+0009)0x216ce78000000', '+(+0009)0x21f1f40000000', '+(+0009)0x2277008000000', '+(+0009)0x22fc0d0000000', '+(+0009)0x2381198000000', '+(+0009)0x2406260000000', '+(+0009)0x248b328000000', '+(+0009)0x25103f0000000', '+(+0009)0x25954b8000000', '+(+0009)0x261a580000000', '+(+0009)0x269f648000000', '+(+0009)0x2724710000000', '+(+0009)0x27a97d8000000', '+(+0009)0x282e8a0000000', '+(+0009)0x28b3968000000', '+(+0009)0x2938a30000000', '+(+0009)0x29bdaf8000000', '+(+0009)0x2a42bc0000000', '+(+0009)0x2ac7c88000000', '+(+0009)0x2b4cd50000000', '+(+0009)0x2bd1e18000000', '+(+0009)0x2c56ee0000000', '+(+0009)0x2cdbfa8000000', '+(+0009)0x2d61070000000', '+(+0009)0x2de6138000000', '+(+0009)0x2e6b200000000', '+(+0009)0x2ef02c8000000', '+(+0009)0x2f75390000000', '+(+0009)0x2ffa458000000', '+(+0009)0x307f520000000', '+(+0009)0x31045e8000000', '+(+0009)0x31896b0000000', '+(+0009)0x320e778000000', '+(+0009)0x3293840000000', '+(+0009)0x3318908000000', '+(+0009)0x339d9d0000000', '+(+0009)0x3422a98000000', '+(+0009)0x34a7b60000000', '+(+0009)0x352cc28000000', '+(+0009)0x35b1cf0000000', '+(+0009)0x3636db8000000', '+(+0009)0x36bbe80000000', '+(+0009)0x3740f48000000', '+(+0009)0x37c6010000000', '+(+0009)0x384b0d8000000', '+(+0009)0x38d01a0000000', '+(+0009)0x3955268000000', '+(+0009)0x39da330000000', '+(+0009)0x3a5f3f8000000', '+(+0009)0x3ae44c0000000', '+(+0009)0x3b69588000000', '+(+0009)0x3bee650000000', '+(+0009)0x3c73718000000', '+(+0009)0x3cf87e0000000', '+(+0009)0x3d7d8a8000000', '+(+0009)0x3e02970000000', '+(+0009)0x3e87a38000000', '+(+0009)0x3f0cb00000000', '+(+0009)0x3f91bc8000000', '+(+0009)0x4016c90000000', '+(+0009)0x409bd58000000', '+(+0009)0x4120e20000000', '+(+0009)0x41a5ee8000000', '+(+0009)0x422afb0000000', '+(+0009)0x42b0078000000', '+(+0009)0x4335140000000', '+(+0009)0x43ba208000000', '+(+0009)0x443f2d0000000', '+(+0009)0x44c4398000000', '+(+0009)0x4549460000000', '+(+0009)0x45ce528000000', '+(+0009)0x46535f0000000', '+(+0009)0x46d86b8000000', '+(+0009)0x475d780000000', '+(+0009)0x47e2848000000', '+(+0009)0x4867910000000', '+(+0009)0x48ec9d8000000', '+(+0009)0x4971aa0000000', '+(+0009)0x49f6b68000000', '+(+0009)0x4a7bc30000000', '+(+0009)0x4b00cf8000000', '+(+0009)0x4b85dc0000000', '+(+0009)0x4c0ae88000000', '+(+0009)0x4c8ff50000000', '+(+0009)0x4d15018000000', '+(+0009)0x4d9a0e0000000', '+(+0009)0x4e1f1a8000000', '+(+0009)0x4ea4270000000', '+(+0009)0x4f29338000000', '+(+0009)0x4fae400000000', '+(+0009)0x50334c8000000', '+(+0009)0x50b8590000000', '+(+0009)0x513d658000000', '+(+0009)0x51c2720000000', '+(+0009)0x52477e8000000', '+(+0009)0x52cc8b0000000', '+(+0009)0x5351978000000', '+(+0009)0x53d6a40000000', '+(+0009)0x545bb08000000', '+(+0009)0x54e0bd0000000', '+(+0009)0x5565c98000000', '+(+0009)0x55ead60000000', '+(+0009)0x566fe28000000', '+(+0009)0x56f4ef0000000', '+(+0009)0x5779fb8000000', '+(+0009)0x57ff080000000', '+(+0009)0x5884148000000', '+(+0009)0x5909210000000', '+(+0009)0x598e2d8000000', '+(+0009)0x5a133a0000000', '+(+0009)0x5a98468000000', '+(+0009)0x5b1d530000000', '+(+0009)0x5ba25f8000000', '+(+0009)0x5c276c0000000', '+(+0009)0x5cac788000000', '+(+0009)0x5d31850000000', '+(+0009)0x5db6918000000', '+(+0009)0x5e3b9e0000000', '+(+0009)0x5ec0aa8000000', '+(+0009)0x5f45b70000000', '+(+0009)0x5fcac38000000', '+(+0009)0x604fd00000000', '+(+0009)0x60d4dc8000000', '+(+0009)0x6159e90000000', '+(+0009)0x61def58000000', '+(+0009)0x6264020000000', '+(+0009)0x62e90e8000000', '+(+0009)0x636e1b0000000', '+(+0009)0x63f3278000000', '+(+0009)0x6478340000000', '+(+0009)0x64fd408000000', '+(+0009)0x65824d0000000', '+(+0009)0x6607598000000', '+(+0009)0x668c660000000', '+(+0009)0x6711728000000', '+(+0009)0x67967f0000000', '+(+0009)0x681b8b8000000', '+(+0009)0x68a0980000000', '+(+0009)0x6925a48000000', '+(+0009)0x69aab10000000', '+(+0009)0x6a2fbd8000000', '+(+0009)0x6ab4ca0000000', '+(+0009)0x6b39d68000000', '+(+0009)0x6bbee30000000', '+(+0009)0x6c43ef8000000', '+(+0009)0x6cc8fc0000000', '+(+0009)0x6d4e088000000', '+(+0009)0x6dd3150000000', '+(+0009)0x6e58218000000', '+(+0009)0x6edd2e0000000', '+(+0009)0x6f623a8000000', '+(+0009)0x6fe7470000000', '+(+0009)0x706c538000000', '+(+0009)0x70f1600000000', '+(+0009)0x71766c8000000', '+(+0009)0x71fb790000000', '+(+0009)0x7280858000000', '+(+0009)0x7305920000000', '+(+0009)0x738a9e8000000', '+(+0009)0x740fab0000000', '+(+0009)0x7494b78000000', '+(+0009)0x7519c40000000', '+(+0009)0x759ed08000000', '+(+0009)0x7623dd0000000', '+(+0009)0x76a8e98000000', '+(+0009)0x772df60000000', '+(+0009)0x77b3028000000', '+(+0009)0x78380f0000000', '+(+0009)0x78bd1b8000000', '+(+0009)0x7942280000000', '+(+0009)0x79c7348000000', '+(+0009)0x7a4c410000000', '+(+0009)0x7ad14d8000000', '+(+0009)0x7b565a0000000', '+(+0009)0x7bdb668000000', '+(+0009)0x7c60730000000', '+(+0009)0x7ce57f8000000', '+(+0009)0x7d6a8c0000000', '+(+0009)0x7def988000000', '+(+0009)0x7e74a50000000', '+(+0009)0x7ef9b18000000', '+(+0009)0x7f7ebe0000000', '+(+0009)0x8003ca8000000', '+(+0009)0x8088d70000000', '+(+0009)0x810de38000000', '+(+0009)0x8192f00000000', '+(+0009)0x8217fc8000000', '+(+0009)0x829d090000000', '+(+0009)0x8322158000000', '+(+0009)0x83a7220000000', '+(+0009)0x842c2e8000000', '+(+0009)0x84b13b0000000', '+(+0009)0x8536478000000', '+(+0009)0x85bb540000000', '+(+0009)0x8640608000000', '+(+0009)0x86c56d0000000', '+(+0009)0x874a798000000', '+(+0009)0x87cf860000000', '+(+0009)0x8854928000000', '+(+0009)0x88d99f0000000', '+(+0009)0x895eab8000000', '+(+0009)0x89e3b80000000', '+(+0009)0x8a68c48000000', '+(+0009)0x8aedd10000000', '+(+0009)0x8b72dd8000000', '+(+0009)0x8bf7ea0000000', '+(+0009)0x8c7cf68000000', '+(+0009)0x8d02030000000', '+(+0009)0x8d870f8000000', '+(+0009)0x8e0c1c0000000', '+(+0009)0x8e91288000000', '+(+0009)0x8f16350000000', '+(+0009)0x8f9b418000000', '+(+0009)0x90204e0000000', '+(+0009)0x90a55a8000000', '+(+0009)0x912a670000000', '+(+0009)0x91af738000000', '+(+0009)0x9234800000000', '+(+0009)0x92b98c8000000', '+(+0009)0x933e990000000', '+(+0009)0x93c3a58000000', '+(+0009)0x9448b20000000', '+(+0009)0x94cdbe8000000', '+(+0009)0x9552cb0000000', '+(+0009)0x95d7d78000000', '+(+0009)0x965ce40000000', '+(+0009)0x96e1f08000000', '+(+0009)0x9766fd0000000', '+(+0009)0x97ec098000000', '+(+0009)0x9871160000000', '+(+0009)0x98f6228000000', '+(+0009)0x997b2f0000000', '+(+0009)0x9a003b8000000', '+(+0009)0x9a85480000000', '+(+0009)0x9b0a548000000', '+(+0009)0x9b8f610000000', '+(+0009)0x9c146d8000000', '+(+0009)0x9c997a0000000', '+(+0009)0x9d1e868000000', '+(+0009)0x9da3930000000', '+(+0009)0x9e289f8000000', '+(+0009)0x9eadac0000000', '+(+0009)0x9f32b88000000', '+(+0009)0x9fb7c50000000', '+(+0009)0xa03cd18000000', '+(+0009)0xa0c1de0000000', '+(+0009)0xa146ea8000000', '+(+0009)0xa1cbf70000000', '+(+0009)0xa251038000000', '+(+0009)0xa2d6100000000', '+(+0009)0xa35b1c8000000', '+(+0009)0xa3e0290000000', '+(+0009)0xa465358000000', '+(+0009)0xa4ea420000000', '+(+0009)0xa56f4e8000000', '+(+0009)0xa5f45b0000000', '+(+0009)0xa679678000000', '+(+0009)0xa6fe740000000', '+(+0009)0xa783808000000', '+(+0009)0xa8088d0000000', '+(+0009)0xa88d998000000', '+(+0009)0xa912a60000000', '+(+0009)0xa997b28000000', '+(+0009)0xaa1cbf0000000', '+(+0009)0xaaa1cb8000000', '+(+0009)0xab26d80000000', '+(+0009)0xababe48000000', '+(+0009)0xac30f10000000', '+(+0009)0xacb5fd8000000', '+(+0009)0xad3b0a0000000', '+(+0009)0xadc0168000000', '+(+0009)0xae45230000000', '+(+0009)0xaeca2f8000000', '+(+0009)0xaf4f3c0000000', '+(+0009)0xafd4488000000', '+(+0009)0xb059550000000', '+(+0009)0xb0de618000000', '+(+0009)0xb1636e0000000', '+(+0009)0xb1e87a8000000', '+(+0009)0xb26d870000000', '+(+0009)0xb2f2938000000', '+(+0009)0xb377a00000000', '+(+0009)0xb3fcac8000000', '+(+0009)0xb481b90000000', '+(+0009)0xb506c58000000', '+(+0009)0xb58bd20000000', '+(+0009)0xb610de8000000', '+(+0009)0xb695eb0000000', '+(+0009)0xb71af78000000', '+(+0009)0xb7a0040000000', '+(+0009)0xb825108000000', '+(+0009)0xb8aa1d0000000', '+(+0009)0xb92f298000000', '+(+0009)0xb9b4360000000', '+(+0009)0xba39428000000', '+(+0009)0xbabe4f0000000', '+(+0009)0xbb435b8000000', '+(+0009)0xbbc8680000000', '+(+0009)0xbc4d748000000', '+(+0009)0xbcd2810000000', '+(+0009)0xbd578d8000000', '+(+0009)0xbddc9a0000000', '+(+0009)0xbe61a68000000', '+(+0009)0xbee6b30000000', '+(+0009)0xbf6bbf8000000', '+(+0009)0xbff0cc0000000', '+(+0009)0xc075d88000000', '+(+0009)0xc0fae50000000', '+(+0009)0xc17ff18000000', '+(+0009)0xc204fe0000000', '+(+0009)0xc28a0a8000000', '+(+0009)0xc30f170000000', '+(+0009)0xc394238000000', '+(+0009)0xc419300000000', '+(+0009)0xc49e3c8000000', '+(+0009)0xc523490000000', '+(+0009)0xc5a8558000000', '+(+0009)0xc62d620000000', '+(+0009)0xc6b26e8000000', '+(+0009)0xc7377b0000000', '+(+0009)0xc7bc878000000', '+(+0009)0xc841940000000', '+(+0009)0xc8c6a08000000', '+(+0009)0xc94bad0000000', '+(+0009)0xc9d0b98000000', '+(+0009)0xca55c60000000', '+(+0009)0xcadad28000000', '+(+0009)0xcb5fdf0000000', '+(+0009)0xcbe4eb8000000', '+(+0009)0xcc69f80000000', '+(+0009)0xccef048000000', '+(+0009)0xcd74110000000', '+(+0009)0xcdf91d8000000', '+(+0009)0xce7e2a0000000', '+(+0009)0xcf03368000000', '+(+0009)0xcf88430000000', '+(+0009)0xd00d4f8000000', '+(+0009)0xd0925c0000000', '+(+0009)0xd117688000000', '+(+0009)0xd19c750000000', '+(+0009)0xd221818000000', '+(+0009)0xd2a68e0000000', '+(+0009)0xd32b9a8000000', '+(+0009)0xd3b0a70000000', '+(+0009)0xd435b38000000', '+(+0009)0xd4bac00000000', '+(+0009)0xd53fcc8000000', '+(+0009)0xd5c4d90000000', '+(+0009)0xd649e58000000', '+(+0009)0xd6cef20000000', '+(+0009)0xd753fe8000000', '+(+0009)0xd7d90b0000000', '+(+0009)0xd85e178000000', '+(+0009)0xd8e3240000000', '+(+0009)0xd968308000000', '+(+0009)0xd9ed3d0000000', '+(+0009)0xda72498000000', '+(+0009)0xdaf7560000000', '+(+0009)0xdb7c628000000', '+(+0009)0xdc016f0000000', '+(+0009)0xdc867b8000000', '+(+0009)0xdd0b880000000', '+(+0009)0xdd90948000000', '+(+0009)0xde15a10000000', '+(+0009)0xde9aad8000000', '+(+0009)0xdf1fba0000000', '+(+0009)0xdfa4c68000000', '+(+0009)0xe029d30000000', '+(+0009)0xe0aedf8000000', '+(+0009)0xe133ec0000000', '+(+0009)0xe1b8f88000000', '+(+0009)0xe23e050000000', '+(+0009)0xe2c3118000000', '+(+0009)0xe3481e0000000', '+(+0009)0xe3cd2a8000000', '+(+0009)0xe452370000000', '+(+0009)0xe4d7438000000', '+(+0009)0xe55c500000000', '+(+0009)0xe5e15c8000000', '+(+0009)0xe666690000000', '+(+0009)0xe6eb758000000', '+(+0009)0xe770820000000', '+(+0009)0xe7f58e8000000', '+(+0009)0xe87a9b0000000', '+(+0009)0xe8ffa78000000', '+(+0009)0xe984b40000000', '+(+0009)0xea09c08000000', '+(+0009)0xea8ecd0000000', '+(+0009)0xeb13d98000000', '+(+0009)0xeb98e60000000', '+(+0009)0xec1df28000000', '+(+0009)0xeca2ff0000000', '+(+0009)0xed280b8000000', '+(+0009)0xedad180000000', '+(+0009)0xee32248000000', '+(+0009)0xeeb7310000000', '+(+0009)0xef3c3d8000000', '+(+0009)0xefc14a0000000', '+(+0009)0xf046568000000', '+(+0009)0xf0cb630000000', '+(+0009)0xf1506f8000000', '+(+0009)0xf1d57c0000000', '+(+0009)0xf25a888000000', '+(+0009)0xf2df950000000', '+(+0009)0xf364a18000000', '+(+0009)0xf3e9ae0000000', '+(+0009)0xf46eba8000000', '+(+0009)0xf4f3c70000000', '+(+0009)0xf578d38000000', '+(+0009)0xf5fde00000000', '+(+0009)0xf682ec8000000', '+(+0009)0xf707f90000000', '+(+0009)0xf78d058000000', '+(+0009)0xf812120000000', '+(+0009)0xf8971e8000000', '+(+0009)0xf91c2b0000000', '+(+0009)0xf9a1378000000', '+(+0009)0xfa26440000000', '+(+0009)0xfaab508000000', '+(+0009)0xfb305d0000000', '+(+0009)0xfbb5698000000', '+(+0009)0xfc3a760000000', '+(+0009)0xfcbf828000000', '+(+0009)0xfd448f0000000', '+(+0009)0xfdc99b8000000', '+(+0009)0xfe4ea80000000', '+(+0009)0xfed3b48000000', '+(+0009)0xff58c10000000', '+(+0009)0xffddcd8000000', '+(+0010)0x00316d0000000', '+(+0010)0x0073f34000000', '+(+0010)0x00b6798000000', '+(+0010)0x00f8ffc000000', '+(+0010)0x013b860000000', '+(+0010)0x017e0c4000000', '+(+0010)0x01c0928000000', '+(+0010)0x020318c000000', '+(+0010)0x02459f0000000', '+(+0010)0x0288254000000', '+(+0010)0x02caab8000000', '+(+0010)0x030d31c000000', '+(+0010)0x034fb80000000', '+(+0010)0x03923e4000000', '+(+0010)0x03d4c48000000', '+(+0010)0x04174ac000000', '+(+0010)0x0459d10000000', '+(+0010)0x049c574000000', '+(+0010)0x04dedd8000000', '+(+0010)0x052163c000000', '+(+0010)0x0563ea0000000', '+(+0010)0x05a6704000000', '+(+0010)0x05e8f68000000', '+(+0010)0x062b7cc000000', '+(+0010)0x066e030000000', '+(+0010)0x06b0894000000', '+(+0010)0x06f30f8000000', '+(+0010)0x073595c000000', '+(+0010)0x07781c0000000', '+(+0010)0x07baa24000000', '+(+0010)0x07fd288000000', '+(+0010)0x083faec000000', '+(+0010)0x0882350000000', '+(+0010)0x08c4bb4000000', '+(+0010)0x0907418000000', '+(+0010)0x0949c7c000000', '+(+0010)0x098c4e0000000', '+(+0010)0x09ced44000000', '+(+0010)0x0a115a8000000', '+(+0010)0x0a53e0c000000', '+(+0010)0x0a96670000000', '+(+0010)0x0ad8ed4000000', '+(+0010)0x0b1b738000000', '+(+0010)0x0b5df9c000000', '+(+0010)0x0ba0800000000', '+(+0010)0x0be3064000000', '+(+0010)0x0c258c8000000', '+(+0010)0x0c6812c000000', '+(+0010)0x0caa990000000', '+(+0010)0x0ced1f4000000', '+(+0010)0x0d2fa58000000', '+(+0010)0x0d722bc000000', '+(+0010)0x0db4b20000000', '+(+0010)0x0df7384000000', '+(+0010)0x0e39be8000000', '+(+0010)0x0e7c44c000000', '+(+0010)0x0ebecb0000000', '+(+0010)0x0f01514000000', '+(+0010)0x0f43d78000000', '+(+0010)0x0f865dc000000', '+(+0010)0x0fc8e40000000', '+(+0010)0x100b6a4000000', '+(+0010)0x104df08000000', '+(+0010)0x109076c000000', '+(+0010)0x10d2fd0000000', '+(+0010)0x1115834000000', '+(+0010)0x1158098000000', '+(+0010)0x119a8fc000000', '+(+0010)0x11dd160000000', '+(+0010)0x121f9c4000000', '+(+0010)0x1262228000000', '+(+0010)0x12a4a8c000000', '+(+0010)0x12e72f0000000', '+(+0010)0x1329b54000000', '+(+0010)0x136c3b8000000', '+(+0010)0x13aec1c000000', '+(+0010)0x13f1480000000', '+(+0010)0x1433ce4000000', '+(+0010)0x1476548000000', '+(+0010)0x14b8dac000000', '+(+0010)0x14fb610000000', '+(+0010)0x153de74000000', '+(+0010)0x15806d8000000', '+(+0010)0x15c2f3c000000', '+(+0010)0x16057a0000000', '+(+0010)0x1648004000000', '+(+0010)0x168a868000000', '+(+0010)0x16cd0cc000000', '+(+0010)0x170f930000000', '+(+0010)0x1752194000000', '+(+0010)0x17949f8000000', '+(+0010)0x17d725c000000', '+(+0010)0x1819ac0000000', '+(+0010)0x185c324000000', '+(+0010)0x189eb88000000', '+(+0010)0x18e13ec000000', '+(+0010)0x1923c50000000', '+(+0010)0x19664b4000000', '+(+0010)0x19a8d18000000', '+(+0010)0x19eb57c000000', '+(+0010)0x1a2dde0000000', '+(+0010)0x1a70644000000', '+(+0010)0x1ab2ea8000000', '+(+0010)0x1af570c000000', '+(+0010)0x1b37f70000000', '+(+0010)0x1b7a7d4000000', '+(+0010)0x1bbd038000000']
"""
STREAM_TYPE = 'GraphBasedProcessor'
STREAM_VERSION = '0'
def __init__(self, stream):
docgen = YamldataGenerator(stream)
reader = YamldataReader(docgen, stream_type=self.STREAM_TYPE, stream_version=self.STREAM_VERSION, header_only=True)
self.runtime_objects = IndexedObjectSet(docgen)
self.dataflow_graph = FrozenGraph(SerializedGraphTables(docgen))
# until we've sorted out the yaml issues
# make a linear graph with arc labels indexing into runtime_objects
builder = GraphBuilder()
startid = builder.new_node_label_is_id()
for arc_label, obj in enumerate(self.runtime_objects):
endid = builder.new_node_label_is_id()
arcid = builder.new_arc(startid, endid, arc_label)
startid = endid
self.graph = FrozenGraph(builder)
for i in self.runtime_objects:
self.last = i[-1]
def config(self, samplerate):
for obj in self.runtime_objects:
# print obj
# obj[-1].send_signals(samplerate=samplerate)
obj[-1].config(samplerate=samplerate)
self.sends = tuple(x[-1].send for x in self.runtime_objects)
self.send = self.sends[0]
nextobj = iter(self.sends)
nextobj.next()
for obj1, obj2 in izip(self.runtime_objects, nextobj):
obj1[-1].set_recipient(obj2)
def set_recipient(self, recip):
self.last.set_recipient(recip)
def serialize(self, stream):
self.runtime_objects.serialize(stream)
self.dataflow_graph.serialize(stream)
def go(wordnames, do_display=False):
"""
A first example of composing finite CFGs.
"""
comlexcfg = make_recognizer(StringIO(comlextop))
#for non_terminal in sorted(comlexcfg.non_terminals):
# print non_terminal
#for terminal in sorted(comlexcfg.terminals):
# print terminal
phonecfg = make_recognizer(StringIO(comlexphones))
wordrecognizer = comlexcfg.recognizer(wordnames)
links, (start_id,
is_sentential) = explore_finite(comlexcfg.recognizer(wordnames))
g1 = FrozenGraph(make_initialized_set_graph_builder(links))
printgraph(g1, 'Pronunciation lattice')
do_display and display(g1, '\\n'.join(wordnames))
breadth_first = deque()
global_start = start_id = unstarted = object()
seen_symbols = set()
links = set()
links2 = set()
send_arg = None
count = -1
while True:
is_sentential, end_id, legal, exception = wordrecognizer(send_arg)
if global_start is unstarted:
global_start = end_id
if exception is not None: raise exception
if not legal: assert is_sentential
#print 'legal:', ' '.join(legal)
if start_id is not unstarted:
links.add(
((start_id, was_sentential), (end_id, is_sentential), symbol))
count += 1
count = 0
substart2 = start_id, substart
links2.add(((start_id, was_sentential), (substart2, False), '(-'))
for (sub_start_id, sub_was_sentential), (
sub_end_id, sub_is_sentential), subsymbol in sublinks:
sub_start_id = start_id, sub_start_id
sub_end_id = start_id, sub_end_id
## if sub_start_id == substart:
## links2.add(((start_id, was_sentential), (sub_start_id, False), '(-'))
links2.add((
(sub_start_id, False),
#((end_id, is_sentential) if sub_is_sentential else (sub_end_id, False)),
(sub_end_id, False),
subsymbol))
if sub_is_sentential:
links2.add(
((sub_end_id, False), (end_id, is_sentential), '(-'))
for symbol in sorted(legal):
breadth_first.appendleft(
((end_id, is_sentential), symbol,
explore_finite(phonecfg.recognizer([symbol]))))
sublinks, (sub_start_id, sub_is_sentential) = explore_finite(
phonecfg.recognizer([symbol]))
if symbol not in seen_symbols:
seen_symbols.add(symbol)
subgraph = FrozenGraph(
make_initialized_set_graph_builder(sublinks))
printgraph(subgraph, 'Phoneme %s' % (symbol, ))
do_display and display(subgraph, symbol)
if not breadth_first:
break
(start_id, was_sentential), symbol, (sublinks, (
substart, substart_sentential)) = breadth_first.pop()
send_arg = start_id, symbol
g = FrozenGraph(make_initialized_set_graph_builder(links))
None and do_display and display(g)
g3 = FrozenGraph(make_initialized_set_graph_builder(links2))
printgraph(g3, 'HMM graph')
do_display and display(g3)
def go(do_display=False):
"""
Generate a dependency graph, and display it if optional do_display is True.
>>> go(do_display=False)
digraph {
node [shape=box];
ranksep=0.4;
{rank=same; "n05";}
{rank=same; "n01"; "n02";}
{rank=same; "n04"; "n06";}
{rank=same; "n00"; "n03"; "n08"; "n10";}
{rank=same; "n07"; "n09";}
n00 [label="Acoustic Models", style=bold, shape=box];
n01 [label="Graph / Lattice", style=bold, shape=octagon];
n02 [label="Serialization", style=bold, shape=octagon];
n03 [label="Audio", style=bold, shape=box];
n04 [label="Dataflow", style=bold, shape=octagon];
n05 [label=" Utilities / Containers ", style=bold, shape=octagon];
n06 [label="CFG", style=bold, shape=octagon];
n07 [label="Decoding", style=bold, shape=box];
n08 [label="Lexicon", style=bold, shape=box];
n09 [label="HTK Files", style=bold, shape=octagon];
n10 [label="Signal Processing", style=bold, shape=box];
n00 -> n01;
n00 -> n02;
n03 -> n04;
n03 -> n05;
n06 -> n01;
n06 -> n05;
n04 -> n01;
n04 -> n05;
n07 -> n06;
n07 -> n04;
n07 -> n08;
n01 -> n02;
n01 -> n05;
n09 -> n00;
n09 -> n04;
n09 -> n08;
n08 -> n06;
n02 -> n05;
n10 -> n04;
}
"""
g = FrozenGraph(make_initialized_set_graph_builder(dependencies))
# make the rank sub graphs
ranks = dict_of(set)
for id in xrange(g.num_nodes):
name, rank, color = g.get_node_label(id)
ranks[rank].add('n%02d' %(id,))
rankglobals = list()
for rank, names in sorted(ranks.iteritems()):
rankglobals.append('{rank=same; "' + '"; "'.join(sorted(names)) + '";}')
# log it
globals=['node [shape=box];', 'ranksep=0.4;'] + rankglobals
node_label_callback=lambda x, *_: str(x[0])
#node_attributes_callback=lambda x, *_: ['color=%s' % (x[2],)]
node_attributes_callback=lambda x, *_: ['style=bold', 'shape=octagon'] if x[2] else ['style=bold', 'shape=box']
for line in g.dot_iter(globals=globals, node_label_callback=node_label_callback, node_attributes_callback=node_attributes_callback):
print line,
# display it
do_display and g.dot_display(globals=globals, node_label_callback=node_label_callback, node_attributes_callback=node_attributes_callback)
def build_model_lattice(label_lattice, model_dict, epsilon_index):
"""
From a lattice with labels on the arcs and a dict mapping labels to model
indices, build a lattice with (node-index, model index) pairs on the nodes,
usable for constructing a TrainingGraph.
The resulting lattice may have new epsilon nodes as the new start and end
nodes; these will be given epsilon_index as their model indices. Note that
this function requires that label_lattice have unique labels on nodes. XXX
maybe do this node-labeling ourselves here?
>>> label_dict = {'A': 1, 'B': 4, 'C': 9}
>>> lat = labels_to_lattice(('A', 'B', 'C'))
>>> lat
FrozenGraph(GraphTables(((0, 1, 2, 3), (0, 1, 2), (1, 2, 3), ('A', 'B', 'C'))))
>>> result = build_model_lattice(lat, label_dict, 15)
>>> print result
FrozenGraph(GraphTables((((0, 1), (1, 4), (2, 9)), (0, 1), (1, 2), (None, None))))
# >>> result.dot_display()
"""
if not label_lattice.is_lattice() or label_lattice.has_self_loop():
raise ValueError("label_lattice is not a lattice or has a self loop")
counter = itertools.count()
# we need our node labels to be pairs of ints in which the first int is
# unique and the second is the index of the model from the callers
# label_dict
def model_node_labeler(pred_node_label, arc_label, succ_node_label):
if not model_dict.has_key(arc_label):
raise KeyError("Failed on lookup of label %s" % (arc_label))
model_index = model_dict[arc_label]
return (counter.next(), model_index)
def empty_arc_labeler(in_arc_label, node_label, out_arc_label):
return None
line_graph = label_lattice.get_line_graph(model_node_labeler,
empty_arc_labeler)
starts, ends = line_graph.get_terminals()
num_starts = len(starts)
num_ends = len(ends)
# If we started with a lattice, the line graph must have some terminals
assert num_starts >= 1 and num_ends >= 1
start_labels = (line_graph.get_label(node_id) for node_id in starts)
end_labels = (line_graph.get_label(node_id) for node_id in ends)
gb = SetGraphBuilder(line_graph)
# Tie terminals together with epsilons if necessary
if num_starts > 1:
new_start_label = gb.add_node((counter.next(), epsilon_index))
for node_label in start_labels:
gb.add_arc(new_start_label, node_label)
if num_ends > 1:
new_end_label = gb.add_node((counter.next(), epsilon_index))
for node_label in end_labels:
gb.new_arc(node_label, new_end_label)
return FrozenGraph(gb)
def _lattice_nbest_align(l1, l2, cost_fn = None, substring_cost = False):
if not (isinstance(l1, FrozenGraph) and isinstance(l2, FrozenGraph)):
raise ValueError("lattice_nbest_align needs two FrozenGraph arguments")
if cost_fn is None:
cost_fn = _std_cost
else:
cost_fn = _make_safe_cost_fn(cost_fn)
l1_canon = l1.get_canonical_DAG()
l2_canon = l2.get_canonical_DAG()
len1 = l1_canon.get_num_arcs()
len2 = l2_canon.get_num_arcs()
assert l1_canon.is_lattice()
assert l2_canon.is_lattice()
terms = l1_canon.get_terminals()
lat_start1 = terms[0][0]
lat_end1 = terms[1][0]
terms = l2_canon.get_terminals()
lat_start2 = terms[0][0]
lat_end2 = terms[1][0]
# The lattices to be aligned, l1 and l2, have labels on the arcs;
# any labels on their nodes will be ignored. We begin by finding
# a topological ordering of the lattice arcs, so that each arc is
# assigned a non-negative index. We construct a 2-D lattice, g,
# with nodes representing pairs of arcs in the l1 and l2. An
# additional row and column on the top and left of g represent an
# initial position prior consuming any tokens from l1 and l2,
# respectively. The arcs in g are labeled with triples giving the
# orginal labels from l1 and l2 (or None if the arc's start node
# is on the left side or top row) and the local cost of the edit
# represented by the arc. There are two slightly tricky parts.
# First, because the alignment is done on lattices, the cost
# lattice may have links which cross several rows or columns,
# depending on the adjacencies of l1 and l2. Second, there's an
# offset of 1 between the arc numbering for l1 and l2 and the node
# numbering in g, because of the initial row and column. Thus,
# the node in g corresponding to the arc pair <a1, a2> is at
# indices <a1+1, a2+1>. One egregious (but very handy) abuse of
# this arrangement is the occasional use of -1 as an ersatz arcID,
# which will be converted to a 0 index into g. Finally, because
# the graph iterpath function requires a single start and end
# node, we tie all the potential end nodes in g to a special end
# node with "ground" arcs. A node in g is a potential end node if
# the pair of arcs it represents are each incident on the terminal
# node of their respective lattices.
gb = GraphBuilder()
# Initialize cost lattice nodes
node_array = [[gb.new_node() for j in range(len2+1)] for i in range(len1+1)]
# We use this single node to tie all end nodes together
end_node = gb.new_node()
# Add first row of arcs
for i in xrange(len1):
start, end, x = l1_canon.get_arc(i)
insert_cost = cost_fn(x, None)
if end == lat_end1 and len2 == 0:
gb.new_arc(node_array[i+1][0], end_node, (None, None, 0))
# print "Added ground arc for l1 arc from %d to %d with label %s" % (start, end, x)
pred_arcs = l1_canon.get_node_in_arcs(start)
if start == lat_start1:
pred_arcs.append(-1)
for arc in pred_arcs:
gb.new_arc(node_array[arc+1][0], node_array[i+1][0], (x, None, insert_cost))
# print ("Processed l1 arc from %d to %d with label %s - added %d arcs"
# % (start, end, x, len(pred_arcs)))
# Add first column of arcs
for j in xrange(len2):
start, end, y = l2_canon.get_arc(j)
delete_cost = 0 if substring_cost else cost_fn(None, y)
if end == lat_end2 and len1 == 0:
gb.new_arc(node_array[0][j+1], end_node, (None, None, 0))
# print "Added ground arc for l1 arc from %d to %d with label %s" % (start, end, y)
pred_arcs = l2_canon.get_node_in_arcs(start)
if start == lat_start2:
pred_arcs.append(-1)
for arc in pred_arcs:
gb.new_arc(node_array[0][arc+1], node_array[0][j+1], (None, y, delete_cost))
# print ("Processed l1 arc from %d to %d with label %s - added %d arcs"
# % (start, end, y, len(pred_arcs)))
# Construct remainder of cost lattice
for i in xrange(len1):
for j in xrange(len2):
start1, end1, x = l1_canon.get_arc(i)
start2, end2, y = l2_canon.get_arc(j)
pred_arcs1 = l1_canon.get_node_in_arcs(start1)
pred_arcs2 = l2_canon.get_node_in_arcs(start2)
if start1 == lat_start1:
pred_arcs1.append(-1)
if start2 == lat_start2:
pred_arcs2.append(-1)
insert_cost = cost_fn(x, None)
delete_cost = 0 if (substring_cost and i == len1 - 1) else cost_fn(None, y)
subst_cost = cost_fn(x, y)
num_added = 0
if end1 == lat_end1 and end2 == lat_end2:
gb.new_arc(node_array[i+1][j+1], end_node, (None, None, 0))
# print "Added ground arc for l1,l1 arcs with labels %s, %s" % (x,y)
for arc in pred_arcs1:
gb.new_arc(node_array[arc+1][j+1], node_array[i+1][j+1], (x, None, insert_cost))
num_added += 1
for arc in pred_arcs2:
gb.new_arc(node_array[i+1][arc+1], node_array[i+1][j+1], (None, y, delete_cost))
num_added += 1
for arc1 in pred_arcs1:
for arc2 in pred_arcs2:
gb.new_arc(node_array[arc1+1][arc2+1], node_array[i+1][j+1], (x, y, subst_cost))
num_added += 1
# print ("Processed l1 arc from %d to %d with label %s "
# "and l1 arc from %d to %d with label %s - added %d arcs"
# % (start1, end1, x, start2, end2, y, num_added))
g = FrozenGraph(gb)
assert g.is_lattice()
# print "g.get_terminals() = %s, %s" % g.get_terminals()
def graph_cost_fn(label): return label[2]
def iter_helper(path):
arc_labels = [path[2].get_arc_label(arc) for arc in path[1][:-1]]
return(path[0], tuple(arc_labels))
return imap(iter_helper,
g.iterpaths(graph_cost_fn, node_array[0][0], end_node))
