def test_batch_aggregate(self):
topo = Topology()
s = topo.source(U.Sequence(iterations=122))
w = s.batch(size=10)
a = R.Aggregate.invoke(w, ARSCHEMA)
a.acount = a.count()
a.acount_all = a.count_all()
a.amax = a.max('seq')
r = a.stream
tester = Tester(topo)
# Mimic the aggregate processing
expected = []
for i in range(0, 120, 10):
expected.append({
'acount': 10,
'acount_all': 10,
'amax': i + 10 - 1
})
tester.contents(r, expected)
tester.tuple_count(r, 12)
tester.test(self.test_ctxtype, self.test_config)
def test_sequence(self):
topo = Topology()
s = topo.source(U.Sequence(iterations=122))
tester = Tester(topo)
tester.tuple_check(s, lambda x: 'seq' in x and 'ts' in x)
tester.tuple_count(s, 122)
tester.test(self.test_ctxtype, self.test_config)
def test_filter_none(self):
topo = Topology()
s = topo.source(U.Sequence(iterations=4))
matches = R.Filter.matching(s, filter=None)
tester = Tester(topo)
tester.tuple_count(matches, 4)
tester.test(self.test_ctxtype, self.test_config)
def test_single_output(self):
topo = Topology()
s = topo.source(U.Sequence(iterations=4))
matches = R.Filter.matching(s, filter='seq<2ul')
tester = Tester(topo)
tester.tuple_count(matches, 2)
tester.test(self.test_ctxtype, self.test_config)
def test_throttle(self):
topo = Topology()
s = topo.source(U.Sequence(iterations=40))
s = s.map(U.Throttle(rate=2.0, precise=True))
tester = Tester(topo)
tester.tuple_count(s, 40)
tester.tuple_check(s, ThrottleCheck())
tester.test(self.test_ctxtype, self.test_config)
def test_delay(self):
topo = Topology()
s = topo.source(U.Sequence(iterations=223))
s = s.map(U.Delay(delay=0.4))
tester = Tester(topo)
tester.tuple_count(s, 223)
tester.tuple_check(s, lambda t: (time.time() - t['ts'].time()) > 0.35)
tester.test(self.test_ctxtype, self.test_config)
def test_sequence_period(self):
topo = Topology()
s = topo.source(U.Sequence(iterations=67, period=0.1))
E = U.SEQUENCE_SCHEMA.extend(StreamSchema('tuple<float64 d>'))
s = s.map(_Delta(), schema=E)
tester = Tester(topo)
tester.tuple_check(s, lambda x: x['d'] > 0.08)
tester.tuple_count(s, 67 - 1)
tester.test(self.test_ctxtype, self.test_config)
def test_transform_schema(self):
topo = Topology()
s = topo.source(U.Sequence(iterations=10))
A = U.SEQUENCE_SCHEMA.extend(StreamSchema('tuple<rstring a>'))
fo = R.Functor.map(s, A)
fo.a = fo.output(fo.outputs[0], '"string value"')
r = fo.outputs[0]
r.print()
tester = Tester(topo)
tester.tuple_count(r, 10)
tester.test(self.test_ctxtype, self.test_config)
def test_transform_filter(self):
topo = Topology()
s = topo.source(U.Sequence(iterations=5))
fo = R.Functor.map(s,
StreamSchema('tuple<uint64 seq>'),
filter='seq>=2ul')
r = fo.outputs[0]
r.print()
tester = Tester(topo)
tester.tuple_count(r, 3)
tester.test(self.test_ctxtype, self.test_config)
def test_pair(self):
topo = Topology()
s = topo.source(U.Sequence(iterations=932))
rschema = U.SEQUENCE_SCHEMA.extend(
StreamSchema('tuple<float64 score>'))
r0 = s.map(lambda t: (t['seq'], t['ts'], 1.0), schema=rschema)
r1 = s.map(lambda t: (t['seq'], t['ts'], 2.0), schema=rschema)
r = U.pair(r0, r1)
tester = Tester(topo)
tester.tuple_count(r, 932 * 2)
tester.tuple_check(r, PairCheck())
tester.test(self.test_ctxtype, self.test_config)
def test_spray(self):
topo = Topology()
s = topo.source(U.Sequence(iterations=2442))
outs = []
for so in U.spray(s, count=7):
outs.append(
so.map(lambda x: (x['seq'], x['ts']),
schema=U.SEQUENCE_SCHEMA))
s = outs[0].union(set(outs))
tester = Tester(topo)
tester.tuple_count(s, 2442)
tester.test(self.test_ctxtype, self.test_config)
def test_transform_two_outputs(self):
topo = Topology()
s = topo.source(U.Sequence(iterations=10))
fo = R.Functor.map(s, [
StreamSchema('tuple<uint64 seq>'),
StreamSchema('tuple<timestamp ts>')
])
seq = fo.outputs[0]
ts = fo.outputs[1]
seq.print()
ts.print()
tester = Tester(topo)
tester.tuple_count(seq, 10)
tester.tuple_count(ts, 10)
tester.test(self.test_ctxtype, self.test_config)
def test_union(self):
topo = Topology()
s = topo.source(U.Sequence(iterations=932))
A = U.SEQUENCE_SCHEMA.extend(StreamSchema('tuple<int32 a, int32 c>'))
B = U.SEQUENCE_SCHEMA.extend(StreamSchema('tuple<int32 c, int32 b>'))
F = U.SEQUENCE_SCHEMA.extend(StreamSchema('tuple<int32 c>'))
r0 = s.map(lambda t: (t['seq'], t['ts'], 89, t['seq'] + 19), schema=A)
r1 = s.map(lambda t: (t['seq'], t['ts'], t['seq'] + 5, 32), schema=B)
r = U.union([r0, r1], schema=F)
r19 = r.filter(lambda t: t['c'] == t['seq'] + 19)
r5 = r.filter(lambda t: t['c'] == t['seq'] + 5)
tester = Tester(topo)
tester.tuple_count(r, 932 * 2)
tester.tuple_count(r19, 932)
tester.tuple_count(r5, 932)
tester.test(self.test_ctxtype, self.test_config)
def test_transform_schema_two_outputs(self):
topo = Topology()
s = topo.source(U.Sequence(iterations=2))
fo = R.Functor.map(s, [
StreamSchema('tuple<uint64 seq, rstring a>'),
StreamSchema('tuple<timestamp ts, int32 b>')
])
fo.a = fo.output(fo.outputs[0], '"string value"')
fo.b = fo.output(fo.outputs[1], 99)
a = fo.outputs[0]
b = fo.outputs[1]
a.print()
b.print()
tester = Tester(topo)
tester.tuple_count(a, 2)
tester.tuple_count(b, 2)
tester.test(self.test_ctxtype, self.test_config)
def test_filename_from_stream(self):
topo = Topology()
s = topo.source(U.Sequence(iterations=5))
F = U.SEQUENCE_SCHEMA.extend(StreamSchema('tuple<rstring filename>'))
fo = R.Functor.map(s, F)
fo.filename = fo.output(fo.outputs[0], '"myfile_{id}.txt"')
to_file = fo.outputs[0]
config = {
'format': Format.txt.name,
'tuples_per_file': 5,
'close_mode': CloseMode.count.name,
'write_punctuations': True,
'suppress': 'filename, ts'
}
fsink = files.FileSink(
file=streamsx.spl.op.Expression.expression('"' + self.dir + '/"+' +
'filename'),
**config)
to_file.for_each(fsink)
tester = Tester(topo)
tester.tuple_count(to_file, 5)
tester.test(self.test_ctxtype, self.test_config)
