def test_serialisation():
import wallaroo
m1 = Message(1, 1)
m2 = Message(1, 1)
s1 = State()
s2 = State()
s3 = State()
s4 = State()
t1 = Tag(1)
t2 = Tag(2, OneToN('1', 2))
ts1 = TagState(1)
ts2 = TagState(2, OneToN('2', 2))
wt1 = wallaroo.computation("tag1")(t1)
wt2 = wallaroo.computation("tag2_to2")(t2)
wts1 = wallaroo.state_computation("tagstate1", State)(ts1)
wts2 = wallaroo.state_computation("tagstate2_to2", State)(ts2)
res = wt1.compute(m1)
res = wts1.compute(res, s1)
res = wt2.compute(res) # returns list of [message, message,...]
res = res[0] # choose first message
res = wts2.compute(res, s2) # returns list of [message, message,...]
r1 = res[0] # choose first message
assert (r1.value == 1)
assert (r1.key == "1.0.0")
assert (r1.tags == ['tag__1', 'tagstate__1', 'tag__2', 'tagstate__2'])
assert (r1.states == [None, (None, ("1", 1)), None, (None, ('1.0', 1))])
# serialise, deserialse, then run again
ds_wt1 = loads(dumps(wt1))
ds_wt2 = loads(dumps(wt2))
ds_wts1 = loads(dumps(wts1))
ds_wts2 = loads(dumps(wts2))
res = ds_wt1.compute(m2)
res = ds_wts1.compute(res, s3)
res = ds_wt2.compute(res) # returns list of [message, message, ...]
res = res[0] # choose first message
res = ds_wts2.compute(res, s4) # returns list of [message, message, ...]
r2 = res[0] # choose first message
assert (r2.value == 1)
assert (r2.key == "1.0.0")
assert (r2.tags == ['tag__1', 'tagstate__1', 'tag__2', 'tagstate__2'])
assert (r2.states == [None, (None, ("1", 1)), None, (None, ('1.0', 1))])
def test_serialisation():
import wallaroo
m1 = Message(1,1)
m2 = Message(1,1)
s1 = State()
s2 = State()
s3 = State()
s4 = State()
t1 = Tag(1)
t2 = Tag(2, OneToN(2))
ts1 = TagState(1)
ts2 = TagState(2, OneToN(2))
wt1 = wallaroo.computation("tag1")(t1)
wt2 = wallaroo.computation("tag2_to2")(t2)
wts1 = wallaroo.state_computation("tagstate1")(ts1)
wts2 = wallaroo.state_computation("tagstate2_to2")(ts2)
res = wt1.compute(m1) # returns data
res = wts1.compute(res, s1) # returns (data, flag)
res = wt2.compute(res[0]) # returns [data]
res = wts2.compute(res[0], s2) # returns ([data], flag)
r1 = res[0]
print('r1', str(r1))
assert(r1.value == 1)
assert(r1.key == "1.0.0")
assert(r1.tags == ['tag_1', 'tagstate_1', 'tag_2', 'tagstate_2'])
assert(r1.states == [None, (None, ("1", 1)), None, (None, ('1.0', 1))])
# serialise, deserialse, then run again
ds_wt1 = loads(dumps(wt1))
ds_wt2 = loads(dumps(wt2))
ds_wts1 = loads(dumps(wts1))
ds_wts2 = loads(dumps(wts2))
res = ds_wt1.compute(m2) # returns data
res = ds_wts1.compute(res, s3) # returns (data, flag)
res = ds_wt2.compute(res[0]) # returns [data]
res = ds_wts2.compute(res[0], s4) # returns ([data], flag)
r2 = res[0]
assert(r2.value == 1)
assert(r2.key == "1.0.0")
assert(r2.tags == ['tag_1', 'tagstate_1', 'tag_2', 'tagstate_2'])
assert(r2.states == [None, (None, ("1", 1)), None, (None, ('1.0', 1))])
def _executor(self, comp, state, partition, ab):
stateless = wallaroo.computation(name=self.comp_name)
stateful = wallaroo.state_computation(name=self.comp_name)
if state:
if partition:
ab.to_state_partition(stateful(comp), state, self.comp_name,
*partition)
else:
ab.to_stateful(stateful(comp), state, self.comp_name)
else:
ab.to(stateless(comp))
def __init__(self, topology):
print("Topology({!r})".format(topology))
c = Counter()
self.steps = []
for node in topology:
c[node] += 1
if node == 'to_stateless':
f = components.Tag('{}{}'.format(node, c[node]))
comp = wallaroo.computation(f.__name__)(f)
self.steps.append(('to', comp, f.__name__))
elif node == 'to_state':
f = components.TagState('{}{}'.format(node, c[node]))
comp = wallaroo.state_computation(f.__name__, components.State)(f)
self.steps.append(('to', comp, f.__name__))
elif node == 'key_by':
comp = wallaroo.key_extractor(components.key_extractor)
self.steps.append(('key_by', comp, 'key-by'))
else:
raise ValueError("Unknown topology node type: {!r}. Please use "
"'to', 'to_parallel', 'to_stateful', or "
"'to_state_partition'".format(node))
def __init__(self, cmds):
print("Topology({!r})".format(cmds))
c = Counter()
self.steps = []
# build topology from cmds in a 2-pass process
topology = []
# 1st pass: collapse commands into steps (pre, comp, post)
current_node = Node()
for cmd in cmds:
if cmd in _PRE: # new step
if not current_node.is_null(
): # check this isn't the first cmd
topology.append(current_node)
current_node = Node()
c[cmd] += 1
current_node.pre = cmd
current_node.pre_id = c[cmd]
elif cmd in _COMPS: # check if new
if current_node.comp: # new if node.computation exists
topology.append(current_node)
current_node = Node()
c[cmd] += 1
current_node.comp = cmd
current_node.id = c[cmd]
elif cmd in _POST:
if current_node.post is not None:
raise ValueError("Can't have two modifiers in a row. You "
"used '--{} --{}'".format(
current_node.post, cmd))
c[cmd] += 1
current_node.post = cmd
current_node.post_id = c[cmd]
# Append the last current_node, since we won't reach another
# check in the loop for this
topology.append(current_node)
# Now build the steps
for node in topology:
# configure the node name
node_name = "{}{}{}".format(
('{}{}_'.format(node.pre, node.pre_id) if node.pre else ''),
'{}{}'.format(node.comp, node.id),
('_{}{}'.format(node.post, node.post_id) if node.post else ''))
# configure node.pre: key_by
if node.pre == 'key_by':
key_by = wallaroo.key_extractor(components.key_extractor)
else:
key_by = None
# configure node.post: flow_modifier
if node.post:
if node.post == 'multi':
# multi: 1-to-2, adding '.x' to msg.key, x in [0,1]
# e.g. 1 -> [1.0, 1.1]
flow_mod = components.OneToN(node.post_id, 2)
elif node.post == 'filter':
# filter: filter out keys ending with 1
# e.g. 1, 0.1, 1.1, 1.1.1, ...
flow_mod = components.FilterBy(
node.post_id, lambda msg: msg.key.endswith('1'))
else:
raise ValueError("Invalid flow modifier: {}".format(
node.post))
else:
flow_mod = None
# create the step as a wallaroo wrapped object
if node.comp == 'stateless':
f = components.Tag(node_name, flow_mod=flow_mod)
if node.post == 'multi':
comp = wallaroo.computation_multi(f.__name__)(f)
else:
comp = wallaroo.computation(f.__name__)(f)
elif node.comp == 'state':
f = components.TagState(node_name, flow_mod=flow_mod)
if node.post == 'multi':
comp = wallaroo.state_computation_multi(
f.__name__, components.State)(f)
else:
comp = wallaroo.state_computation(f.__name__,
components.State)(f)
# append the Step object of the computation to self.steps
self.steps.append(Step(key_by, 'to', comp, f.__name__))