def test_batch_aggregate(self): topo = Topology() s = topo.source(U.Sequence(iterations=122)) w = s.batch(size=10) a = R.Aggregate.invoke(w, ARSCHEMA) a.acount = a.count() a.acount_all = a.count_all() a.amax = a.max('seq') r = a.stream tester = Tester(topo) # Mimic the aggregate processing expected = [] for i in range(0, 120, 10): expected.append({ 'acount': 10, 'acount_all': 10, 'amax': i + 10 - 1 }) tester.contents(r, expected) tester.tuple_count(r, 12) tester.test(self.test_ctxtype, self.test_config)
def test_sequence(self): topo = Topology() s = topo.source(U.Sequence(iterations=122)) tester = Tester(topo) tester.tuple_check(s, lambda x: 'seq' in x and 'ts' in x) tester.tuple_count(s, 122) tester.test(self.test_ctxtype, self.test_config)
def test_filter_none(self): topo = Topology() s = topo.source(U.Sequence(iterations=4)) matches = R.Filter.matching(s, filter=None) tester = Tester(topo) tester.tuple_count(matches, 4) tester.test(self.test_ctxtype, self.test_config)
def test_single_output(self): topo = Topology() s = topo.source(U.Sequence(iterations=4)) matches = R.Filter.matching(s, filter='seq<2ul') tester = Tester(topo) tester.tuple_count(matches, 2) tester.test(self.test_ctxtype, self.test_config)
def test_throttle(self): topo = Topology() s = topo.source(U.Sequence(iterations=40)) s = s.map(U.Throttle(rate=2.0, precise=True)) tester = Tester(topo) tester.tuple_count(s, 40) tester.tuple_check(s, ThrottleCheck()) tester.test(self.test_ctxtype, self.test_config)
def test_delay(self): topo = Topology() s = topo.source(U.Sequence(iterations=223)) s = s.map(U.Delay(delay=0.4)) tester = Tester(topo) tester.tuple_count(s, 223) tester.tuple_check(s, lambda t: (time.time() - t['ts'].time()) > 0.35) tester.test(self.test_ctxtype, self.test_config)
def test_sequence_period(self): topo = Topology() s = topo.source(U.Sequence(iterations=67, period=0.1)) E = U.SEQUENCE_SCHEMA.extend(StreamSchema('tuple<float64 d>')) s = s.map(_Delta(), schema=E) tester = Tester(topo) tester.tuple_check(s, lambda x: x['d'] > 0.08) tester.tuple_count(s, 67 - 1) tester.test(self.test_ctxtype, self.test_config)
def test_transform_schema(self): topo = Topology() s = topo.source(U.Sequence(iterations=10)) A = U.SEQUENCE_SCHEMA.extend(StreamSchema('tuple<rstring a>')) fo = R.Functor.map(s, A) fo.a = fo.output(fo.outputs[0], '"string value"') r = fo.outputs[0] r.print() tester = Tester(topo) tester.tuple_count(r, 10) tester.test(self.test_ctxtype, self.test_config)
def test_transform_filter(self): topo = Topology() s = topo.source(U.Sequence(iterations=5)) fo = R.Functor.map(s, StreamSchema('tuple<uint64 seq>'), filter='seq>=2ul') r = fo.outputs[0] r.print() tester = Tester(topo) tester.tuple_count(r, 3) tester.test(self.test_ctxtype, self.test_config)
def test_pair(self): topo = Topology() s = topo.source(U.Sequence(iterations=932)) rschema = U.SEQUENCE_SCHEMA.extend( StreamSchema('tuple<float64 score>')) r0 = s.map(lambda t: (t['seq'], t['ts'], 1.0), schema=rschema) r1 = s.map(lambda t: (t['seq'], t['ts'], 2.0), schema=rschema) r = U.pair(r0, r1) tester = Tester(topo) tester.tuple_count(r, 932 * 2) tester.tuple_check(r, PairCheck()) tester.test(self.test_ctxtype, self.test_config)
def test_spray(self): topo = Topology() s = topo.source(U.Sequence(iterations=2442)) outs = [] for so in U.spray(s, count=7): outs.append( so.map(lambda x: (x['seq'], x['ts']), schema=U.SEQUENCE_SCHEMA)) s = outs[0].union(set(outs)) tester = Tester(topo) tester.tuple_count(s, 2442) tester.test(self.test_ctxtype, self.test_config)
def test_transform_two_outputs(self): topo = Topology() s = topo.source(U.Sequence(iterations=10)) fo = R.Functor.map(s, [ StreamSchema('tuple<uint64 seq>'), StreamSchema('tuple<timestamp ts>') ]) seq = fo.outputs[0] ts = fo.outputs[1] seq.print() ts.print() tester = Tester(topo) tester.tuple_count(seq, 10) tester.tuple_count(ts, 10) tester.test(self.test_ctxtype, self.test_config)
def test_union(self): topo = Topology() s = topo.source(U.Sequence(iterations=932)) A = U.SEQUENCE_SCHEMA.extend(StreamSchema('tuple<int32 a, int32 c>')) B = U.SEQUENCE_SCHEMA.extend(StreamSchema('tuple<int32 c, int32 b>')) F = U.SEQUENCE_SCHEMA.extend(StreamSchema('tuple<int32 c>')) r0 = s.map(lambda t: (t['seq'], t['ts'], 89, t['seq'] + 19), schema=A) r1 = s.map(lambda t: (t['seq'], t['ts'], t['seq'] + 5, 32), schema=B) r = U.union([r0, r1], schema=F) r19 = r.filter(lambda t: t['c'] == t['seq'] + 19) r5 = r.filter(lambda t: t['c'] == t['seq'] + 5) tester = Tester(topo) tester.tuple_count(r, 932 * 2) tester.tuple_count(r19, 932) tester.tuple_count(r5, 932) tester.test(self.test_ctxtype, self.test_config)
def test_transform_schema_two_outputs(self): topo = Topology() s = topo.source(U.Sequence(iterations=2)) fo = R.Functor.map(s, [ StreamSchema('tuple<uint64 seq, rstring a>'), StreamSchema('tuple<timestamp ts, int32 b>') ]) fo.a = fo.output(fo.outputs[0], '"string value"') fo.b = fo.output(fo.outputs[1], 99) a = fo.outputs[0] b = fo.outputs[1] a.print() b.print() tester = Tester(topo) tester.tuple_count(a, 2) tester.tuple_count(b, 2) tester.test(self.test_ctxtype, self.test_config)
def test_filename_from_stream(self): topo = Topology() s = topo.source(U.Sequence(iterations=5)) F = U.SEQUENCE_SCHEMA.extend(StreamSchema('tuple<rstring filename>')) fo = R.Functor.map(s, F) fo.filename = fo.output(fo.outputs[0], '"myfile_{id}.txt"') to_file = fo.outputs[0] config = { 'format': Format.txt.name, 'tuples_per_file': 5, 'close_mode': CloseMode.count.name, 'write_punctuations': True, 'suppress': 'filename, ts' } fsink = files.FileSink( file=streamsx.spl.op.Expression.expression('"' + self.dir + '/"+' + 'filename'), **config) to_file.for_each(fsink) tester = Tester(topo) tester.tuple_count(to_file, 5) tester.test(self.test_ctxtype, self.test_config)