def test_mixed_toolkits(self): topo = Topology() streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('testtkpy')) streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('tk17')) data = ['A'] bop = op.Source(topo, "spl.utility::Beacon", 'tuple<rstring a>', {'iterations': 1}) bop.a = bop.output('"A"') sv = op.Map( "com.ibm.streamsx.topology.pytest.pyvers::StreamsxVersion", bop.stream, 'tuple<rstring a, rstring v1, rstring v2>') m17f = op.Map( "com.ibm.streamsx.topology.pytest.tk17::M17F", sv.stream, 'tuple<rstring a, rstring v1, rstring v2, rstring f1, rstring f2>') m17c = op.Map( "com.ibm.streamsx.topology.pytest.tk17::M17C", m17f.stream, 'tuple<rstring a, rstring v1, rstring v2, rstring f1, rstring f2, rstring c1, rstring c2, int32 x>', {'x': 27}) tester = Tester(topo) tester.contents(m17c.stream, [{'a':'A', 'f1':'1.7', 'f2':'F', 'v1':'aggregate', 'v2':'True', 'c1':'1.7', 'c2':'C', 'x':27}]) tester.test(self.test_ctxtype, self.test_config)
def test_mixed_toolkits(self): topo = Topology() streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('testtkpy')) streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('tk17')) data = ['A'] bop = op.Source(topo, "spl.utility::Beacon", 'tuple<rstring a>', {'iterations': 1}) bop.a = bop.output('"A"') sv = op.Map("com.ibm.streamsx.topology.pytest.pyvers::StreamsxVersion", bop.stream, 'tuple<rstring a, rstring v1, rstring v2>') m17f = op.Map( "com.ibm.streamsx.topology.pytest.tk17::M17F", sv.stream, 'tuple<rstring a, rstring v1, rstring v2, rstring f1, rstring f2>') m17c = op.Map( "com.ibm.streamsx.topology.pytest.tk17::M17C", m17f.stream, 'tuple<rstring a, rstring v1, rstring v2, rstring f1, rstring f2, rstring c1, rstring c2, int32 x>', {'x': 27}) tester = Tester(topo) tester.contents(m17c.stream, [{ 'a': 'A', 'f1': '1.7', 'f2': 'F', 'v1': 'aggregate', 'v2': 'True', 'c1': '1.7', 'c2': 'C', 'x': 27 }]) tester.test(self.test_ctxtype, self.test_config)
def test_enter_exit(self): topo = Topology() streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('testtkpy')) source = op.Source(topo, 'com.ibm.streamsx.topology.pytest.checkpoint::EnterExitSource', schema.StreamSchema('tuple<rstring from, int32 enter, int32 exit>').as_tuple(), params={'period':0.1}) source.stream.set_consistent(ConsistentRegionConfig.periodic(5, drain_timeout=40, reset_timeout=40, max_consecutive_attempts=6)) transit = op.Map('com.ibm.streamsx.topology.pytest.checkpoint::EnterExitMap', source.stream, schema.StreamSchema('tuple<rstring from, int32 enter, int32 exit>').as_tuple()) tester = Tester(topo) tester.resets(10) # On each operator, __enter__ and __exit__ should be called once for # each reset. Also __enter__ should be called at startup and __exit__ # at shutdown. It is hard to verify the final __exit__ call (and that # is handled by python rather than our code), so # the test is valid if the number of __enter__ calls is one more than # the number of resets, and the number of __exit__ calls is equal to # number of resets. The tuples on the two streams indicate the number # of times __enter__ and __exit__ have been called. # We are looking for two specific tuples: # ('source', 6, 5) and ('transit', 6, 5) tester.eventual_result(source.stream, lambda tuple_ : True if tuple_[1] >= 6 and tuple_[1] == tuple_[2] + 1 else Fale if tuple_[1] != tuple_[2] + 1 else None) tester.eventual_result(transit.stream, lambda tuple_ : True if tuple_[1] >= 6 and tuple_[1] == tuple_[2] + 1 else Fale if tuple_[1] != tuple_[2] + 1 else None) job_config = streamsx.topology.context.JobConfig(tracing='debug') job_config.add(self.test_config) tester.test(self.test_ctxtype, self.test_config)
def test_filter_map(self): topo = Topology() topo.checkpoint_period = timedelta(seconds=1) streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('testtkpy')) timeCounter = op.Source( topo, "com.ibm.streamsx.topology.pytest.checkpoint::TimeCounter", schema.StreamSchema('tuple<int32 f>').as_tuple(), params={ 'iterations': 30, 'period': 0.1 }) evenFilter = op.Map( "com.ibm.streamsx.topology.pytest.checkpoint::StatefulEvenFilter", timeCounter.stream, None, params={}) hpo = op.Map( "com.ibm.streamsx.topology.pytest.checkpoint::StatefulHalfPlusOne", evenFilter.stream, None, params={}) s = hpo.stream tester = Tester(topo) tester.tuple_count(s, 15) tester.contents(s, list(zip(range(1, 16)))) tester.test(self.test_ctxtype, self.test_config)
def _run_app(self, kind, opi='M'): schema = 'tuple<rstring a, int32 b>' topo = Topology('TESPL' + str(uuid.uuid4().hex)) streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('testtkpy')) if opi == 'M': data = [1,2,3] se = topo.source(data) se = se.map(lambda x : {'a':'hello', 'b':x} , schema=schema) prim = op.Map( "com.ibm.streamsx.topology.pytest.pyexceptions::" + kind, se, params={'tf':self.tf}) res = prim.stream elif opi == 'S': prim = op.Source( topo, "com.ibm.streamsx.topology.pytest.pyexceptions::" + kind, schema=schema, params={'tf':self.tf}) res = prim.stream elif opi == 'E': data = [1,2,3] se = topo.source(data) se = se.map(lambda x : {'a':'hello', 'b':x} , schema=schema) prim = op.Sink( "com.ibm.streamsx.topology.pytest.pyexceptions::" + kind, se, params={'tf':self.tf}) res = None tester = Tester(topo) tester.run_for(3) ok = tester.test(self.test_ctxtype, self.test_config, assert_on_fail=False) self.assertFalse(ok)
def test_source(self): topo = Topology() streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('testtkpy')) bop = op.Source( topo, "com.ibm.streamsx.topology.pytest.checkpoint::TimeCounter", schema.StreamSchema('tuple<int32 f>').as_tuple(), params={ 'iterations': 30, 'period': 0.1 }) s = bop.stream s.set_consistent( ConsistentRegionConfig.operator_driven(drain_timeout=40, reset_timeout=40, max_consecutive_attempts=3)) tester = Tester(topo) self.assertFalse( tester.test(self.test_ctxtype, self.test_config, assert_on_fail=False))
def test_single_output_port_punct_forward(self): """Operator with receives and forwards window marker.""" topo = Topology() streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('testtkpy')) s = topo.source([1, 2, 3, 4]) s = s.punctor(func=(lambda t: 2 < t), before=False) s = s.map(lambda x: (x, ), schema='tuple<int32 z>') s1 = op.Map( "com.ibm.streamsx.topology.pytest.pyprimitives::SingleOutputPort", s) # has no on_punct bop = op.Map( "com.ibm.streamsx.topology.pytest.pyprimitives::SingleOutputPortPunctForward", s) # implements on_punct r = bop.stream op.Sink("com.ibm.streamsx.topology.pytest.pyprimitives::VerifyPosInt", r) r.print(write_punctuations=True) #self.test_config['topology.keepArtifacts'] = True self.tester = Tester(topo) self.tester.tuple_count(r, 4) self.tester.punct_count(r, 2) self.tester.test(self.test_ctxtype, self.test_config)
def test_input_by_position(self): """Operator with input by position""" topo = Topology() streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('testtkpy')) s = topo.source([3642, -393]) s = s.map(lambda x: (x, x * 2, x + 4), schema='tuple<int64 d, int64 e, int64 f>') bop = op.Map( "com.ibm.streamsx.topology.pytest.pyprimitives::InputByPosition", s) r = bop.stream self.tester = Tester(topo) self.tester.tuple_count(r, 2) self.tester.contents(r, [{ 'd': 3642, 'e': (3642 * 2) + 89, 'f': -92 }, { 'd': -393, 'e': (-393 * 2) + 89, 'f': -92 }]) self.tester.test(self.test_ctxtype, self.test_config)
def test_multi_output_ports_punct(self): """Operator with multiple output port emits window marker after forwarding the tuple.""" topo = Topology() streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('testtkpy')) s = topo.source([9237, -24]) s = s.map(lambda x: (x, ), schema='tuple<int64 v>') bop = op.Invoke( topo, "com.ibm.streamsx.topology.pytest.pyprimitives::MultiOutputPortsPunct", s, schemas=['tuple<int64 v1>', 'tuple<int32 v2>', 'tuple<int16 v3>']) r = bop.outputs r[0].print(tag='0', write_punctuations=True) r[1].print(tag='1', write_punctuations=True) r[2].print(tag='2', write_punctuations=True) self.tester = Tester(topo) self.tester.tuple_count(s, 2) self.tester.punct_count(r[0], 2) self.tester.punct_count(r[1], 2) self.tester.punct_count(r[2], 2) self.tester.contents(r[0], [{'v1': 9237}, {'v1': -24}]) self.tester.contents(r[1], [{'v2': 9237 + 921}, {'v2': -24 + 921}]) self.tester.contents(r[2], [{'v3': 9237 - 407}, {'v3': -24 - 407}]) self.tester.test(self.test_ctxtype, self.test_config)
def test_blob_type(self): topo = Topology() streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('testtkpy')) data = ['Hello', 'Blob', 'Did', 'you', 'reset' ] s = topo.source(data) s = s.as_string() toBlob = op.Map( "com.ibm.streamsx.topology.pytest.pytypes::ToBlob", s, 'tuple<blob b>') toBlob = op.Map( "com.ibm.streamsx.topology.pysamples.positional::Noop", toBlob.stream, 'tuple<blob b>') bt = op.Map( "com.ibm.streamsx.topology.pytest.pytypes::BlobTest", toBlob.stream, 'tuple<rstring string>', {'keep': True}) bt2 = op.Map( "com.ibm.streamsx.topology.pytest.pytypes::BlobTest", toBlob.stream, 'tuple<rstring string>', {'keep': False}) tester = Tester(topo) tester.contents(bt.stream, data) self.test_config['topology.keepArtifacts'] = True; tester.test(self.test_ctxtype, self.test_config)
def test_filer_for_each(self): topo = Topology() streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('testtkpy')) s = topo.source([(1, 2, 3), (1, 2, 201), (2, 2, 301), (3, 4, 401), (5, 6, 78), (8, 6, 501), (803, 9324, 901)]) sch = 'tuple<int32 a, int32 b, int32 c>' s = s.map(lambda x: x, schema=sch) bop = op.Map("com.ibm.streamsx.topology.pytest.pykwargs::KWFilter", s) op.Sink("com.ibm.streamsx.topology.pytest.pykwargs::KWForEach", bop.stream) tester = Tester(topo) tester.contents(bop.stream, [{ 'a': 1, 'b': 2, 'c': 201 }, { 'a': 3, 'b': 4, 'c': 401 }, { 'a': 803, 'b': 9324, 'c': 901 }]) tester.test(self.test_ctxtype, self.test_config)
def test_primitive_foreach(self): iterations = 3000 topo = Topology() topo.checkpoint_period = timedelta(seconds=1) streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('testtkpy')) timeCounter = op.Source( topo, "com.ibm.streamsx.topology.pytest.checkpoint::TimeCounter", schema.StreamSchema('tuple<int32 f>').as_tuple(), params={ 'iterations': iterations, 'period': 0.01 }) timeCounter.stream.set_consistent( ConsistentRegionConfig.periodic(5, drain_timeout=40, reset_timeout=40, max_consecutive_attempts=6)) fizzbuzz = op.Map( "com.ibm.streamsx.topology.pytest.checkpoint::FizzBuzzPrimitive", timeCounter.stream, schema.StreamSchema('tuple<int32 f, rstring c>').as_tuple()) verify = op.Sink("com.ibm.streamsx.topology.pytest.checkpoint::Verify", fizzbuzz.stream) s = fizzbuzz.stream tester = Tester(topo) tester.resets() tester.tuple_count(s, iterations) tester.test(self.test_ctxtype, self.test_config)
def test_source(self): iterations = 3000 topo = Topology() streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('testtkpy')) bop = op.Source( topo, "com.ibm.streamsx.topology.pytest.checkpoint::TimeCounter", schema.StreamSchema('tuple<int32 f>').as_tuple(), params={ 'iterations': iterations, 'period': 0.01 }) s = bop.stream s.set_consistent( ConsistentRegionConfig.periodic(5, drain_timeout=40, reset_timeout=40, max_consecutive_attempts=6)) tester = Tester(topo) tester.resets() tester.tuple_count(s, iterations) tester.contents(s, list(zip(range(0, iterations)))) # job_config = streamsx.topology.context.JobConfig(tracing='debug') # job_config.add(self.test_config) tester.test(self.test_ctxtype, self.test_config)
def test_blob_type(self): topo = Topology() streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('testtkpy')) data = ['Hello', 'Blob', 'Did', 'you', 'reset'] s = topo.source(data) s = s.as_string() toBlob = op.Map("com.ibm.streamsx.topology.pytest.pytypes::ToBlob", s, 'tuple<blob b>') toBlob = op.Map("com.ibm.streamsx.topology.pysamples.positional::Noop", toBlob.stream, 'tuple<blob b>') toBlob = op.Map("com.ibm.streamsx.topology.pytest.pykwargs::KWNoop", toBlob.stream, 'tuple<blob b>') bt = op.Map("com.ibm.streamsx.topology.pytest.pytypes::BlobTest", toBlob.stream, 'tuple<rstring string>', {'keep': True}) bt2 = op.Map("com.ibm.streamsx.topology.pytest.pytypes::BlobTest", toBlob.stream, 'tuple<rstring string>', {'keep': False}) tester = Tester(topo) tester.contents(bt.stream, data) self.test_config['topology.keepArtifacts'] = True tester.test(self.test_ctxtype, self.test_config)
def test_mt(self): topo = Topology() N = 1000 streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('testtkpy')) b1 = op.Source(topo, "spl.utility::Beacon", schema.StreamSchema('tuple<int32 f>').as_tuple(), params={'iterations':N}) b1.f = b1.output('(int32)IterationCount()') b2 = op.Source(topo, "spl.utility::Beacon", schema.StreamSchema('tuple<int32 f>').as_tuple(), params={'iterations':N}) b2.f = b2.output(str(N) + ' + (int32)IterationCount()') b3 = op.Source(topo, "spl.utility::Beacon", schema.StreamSchema('tuple<int32 f>').as_tuple(), params={'iterations':N}) b3.f = b3.output(str(2*N) + ' + (int32)IterationCount()') s1 = b1.stream.low_latency() s2 = b2.stream.low_latency() s3 = b3.stream.low_latency() s = s1.union({s2, s3}) f = op.Map("com.ibm.streamsx.topology.pytest.mt::MTFilter", s) m = op.Map("com.ibm.streamsx.topology.pytest.mt::MTMap", f.stream) op.Sink("com.ibm.streamsx.topology.pytest.mt::MTForEach", f.stream) cr = m.stream.flat_map() tester = Tester(topo) tester.tuple_count(m.stream, 3*N) tester.contents(cr, range(3*N), ordered=False) tester.test(self.test_ctxtype, self.test_config)
def test_noports(self): """Operator with no inputs or outputs""" topo = Topology() streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('testtkpy')) bop = op.Invoke(topo, "com.ibm.streamsx.topology.pytest.pyprimitives::NoPorts", params = {'mn': 'mymetric', 'iv':89}) self.tester = Tester(topo) self.tester.local_check = self._noports_check self.tester.test(self.test_ctxtype, self.test_config)
def test_fn_source(self): count = 37 topo = Topology() streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('testtkpy')) bop = op.Source(topo, "com.ibm.streamsx.topology.pysamples.sources::Range37", schema.StreamSchema('tuple<int64 c>').as_tuple()) r = bop.stream self.tester = Tester(topo) self.tester.tuple_count(r, count) self.tester.contents(r, list(zip(range(count)))) self.tester.test(self.test_ctxtype, self.test_config)
def test_verify_operator_pip_install(self): """ Verify pint is installed by the operator module """ topo = Topology() streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('testtkpy_pip_op')) s = topo.source(['a']) s = s.as_string() fp = op.Map("com.ibm.streamsx.topology.pytest.pypip::find_a_pint", s) tester = Tester(topo) tester.contents(fp.stream, ['RTOP_PintImported']) tester.test(self.test_ctxtype, self.test_config)
def test_class_source(self): count = 43 topo = Topology() streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('testtkpy')) streamsx.spl.toolkit.add_toolkit_dependency(topo, 'spl', '[1.0,2.0)') bop = op.Source(topo, "com.ibm.streamsx.topology.pysamples.sources::Range", schema.StreamSchema('tuple<int64 c>').as_tuple(), params={'count':count}) r = bop.stream self.tester = Tester(topo) self.tester.tuple_count(r, count) self.tester.contents(r, list(zip(range(count)))) self.tester.test(self.test_ctxtype, self.test_config)
def test_map_foreach(self): topo = Topology() topo.checkpoint_period = timedelta(seconds=1) streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('testtkpy')) timeCounter = op.Source(topo, "com.ibm.streamsx.topology.pytest.checkpoint::TimeCounter", schema.StreamSchema('tuple<int32 f>').as_tuple(), params={'iterations':30,'period':0.1}) fizzbuzz = op.Map("com.ibm.streamsx.topology.pytest.checkpoint::FizzBuzzMap", timeCounter.stream, schema.StreamSchema('tuple<int32 f, rstring c>').as_tuple()) verify = op.Sink("com.ibm.streamsx.topology.pytest.checkpoint::Verify", fizzbuzz.stream) s = fizzbuzz.stream tester = Tester(topo) tester.tuple_count(s, 30) tester.test(self.test_ctxtype, self.test_config)
def test_single_input_port(self): """Operator with one input port""" topo = Topology() streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('testtkpy')) s = topo.source([1043]) s = s.map(lambda x : (x,), schema='tuple<uint64 v>') bop = op.Sink("com.ibm.streamsx.topology.pytest.pyprimitives::SingleInputPort", s, name="SIP_OP") self.tester = Tester(topo) self.tester.local_check = self._single_input_port_check self.tester.test(self.test_ctxtype, self.test_config)
def test_source(self): topo = Topology() topo.checkpoint_period = timedelta(seconds=1) streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('testtkpy')) bop = op.Source(topo, "com.ibm.streamsx.topology.pytest.checkpoint::TimeCounter", schema.StreamSchema('tuple<int32 f>').as_tuple(), params={'iterations':30,'period':0.1}) # streamsx.topology.context.submit('TOOLKIT', topo) s = bop.stream tester = Tester(topo) tester.tuple_count(s, 30) tester.contents(s, list(zip(range(0,30)))) tester.test(self.test_ctxtype, self.test_config)
def test_multi_input_ports(self): """Operator with three input ports""" topo = Topology() streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('testtkpy')) s0 = topo.source([9054]).map(lambda x : (x,), schema='tuple<uint64 v>') s1 = topo.source([345]).map(lambda x : (x,), schema='tuple<int64 v>') s2 = topo.source([-953]).map(lambda x : (x,), schema='tuple<int32 v>') bop = op.Invoke(topo, "com.ibm.streamsx.topology.pytest.pyprimitives::MultiInputPort", [s0,s1,s2], name="MIP_OP") self.tester = Tester(topo) self.tester.local_check = self._multi_input_port_check self.tester.test(self.test_ctxtype, self.test_config)
def test_source(self): topo = Topology("test") topo.checkpoint_period = timedelta(seconds=1) streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('testtkpy')) bop = op.Source(topo, "com.ibm.streamsx.topology.pytest.checkpoint::TimeCounter", schema.StreamSchema('tuple<int32 f>').as_tuple(), params={'iterations':30,'period':0.1}) s = bop.stream tester = Tester(topo) tester.tuple_count(s, 30) #tester.contents(s, range(0,30)) # why doesn't this work? tester.contents(s, list(zip(range(0,30)))) tester.test(self.test_ctxtype, self.test_config, always_collect_logs=True)
def test_suppress_metric(self): schema = 'tuple<int32 a, int32 b, int32 c, int32 d>' topo = Topology() streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('testtkpy')) # no metric st = op.Source( topo, kind='com.ibm.streamsx.topology.pytest.pysource::SparseTuple', schema=schema, name='NOMETRIC_ST') sf = op.Source( topo, kind='com.ibm.streamsx.topology.pysamples.sources::Range37', schema=schema, name='NOMETRIC_SF') s = st.stream.union({sf.stream}) sm = op.Map('com.ibm.streamsx.topology.pysamples.positional::Noop', s, name='NOMETRIC_MF') sm = op.Map('com.ibm.streamsx.topology.pysamples.positional::AddSeq', sm.stream, name='NOMETRIC_MC') # With metric schema = 'tuple<rstring a, int32 b>' ms = op.Source( topo, kind= 'com.ibm.streamsx.topology.pytest.pyexceptions::SuppressNextSource', schema=schema, name='HASMETRIC_S_1') mm = op.Map( kind='com.ibm.streamsx.topology.pytest.pyexceptions::SuppressMap', stream=ms.stream, name='HASMETRIC_M_0') mf = op.Map( kind= 'com.ibm.streamsx.topology.pytest.pyexceptions::SuppressFilter', stream=ms.stream, name='HASMETRIC_F_0') self.tester = Tester(topo) self.tester.local_check = self.check_suppress_metric # Add filters to avoid the test operators having # names of NOMETIC/HASMETRIC self.tester.tuple_count(sm.stream.filter(lambda _: True), 38) self.tester.tuple_count(ms.stream.filter(lambda _: True), 2) self.tester.tuple_count(mm.stream.filter(lambda _: True), 2) self.tester.tuple_count(mf.stream.filter(lambda _: True), 2) self.tester.test(self.test_ctxtype, self.test_config)
def test_source(self): topo = Topology() streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('testtkpy')) bop = op.Source(topo, "com.ibm.streamsx.topology.pytest.checkpoint::TimeCounter", schema.StreamSchema('tuple<int32 f>').as_tuple(), params={'iterations':30,'period':0.1}) s = bop.stream s.set_consistent(ConsistentRegionConfig.operator_driven(drain_timeout=40, reset_timeout=40, max_consecutive_attempts=3)) tester = Tester(topo) self.assertFalse(tester.test(self.test_ctxtype, self.test_config, assert_on_fail=False))
def test_verify_toolkit_pip_install(self): """ Verify pint is installed by requirements.txt in toolkit """ topo = Topology() streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('testtkpy_pip_toolkit')) s = topo.source(['a']) s = s.as_string() fp = op.Map( "com.ibm.streamsx.topology.pytest.pypip::find_a_pint_toolkit", s) tester = Tester(topo) tester.contents(fp.stream, ['RTTK_PintImported']) tester.test(self.test_ctxtype, self.test_config)
def test_filer_for_each(self): topo = Topology() streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('testtkpy')) s = topo.source([(1,2,3), (1,2,201), (2,2,301), (3,4,401), (5,6,78), (8,6,501), (803, 9324, 901)]) sch = 'tuple<int32 a, int32 b, int32 c>' s = s.map(lambda x : x, schema=sch) bop = op.Map("com.ibm.streamsx.topology.pytest.pykwargs::KWFilter", s) op.Sink("com.ibm.streamsx.topology.pytest.pykwargs::KWForEach", bop.stream) tester = Tester(topo) tester.contents(bop.stream, [{'a':1, 'b':2, 'c':201}, {'a':3, 'b':4, 'c':401}, {'a':803, 'b':9324, 'c':901}]) tester.test(self.test_ctxtype, self.test_config)
def test_filter_map(self): topo = Topology() topo.checkpoint_period = timedelta(seconds=1) streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('testtkpy')) timeCounter = op.Source(topo, "com.ibm.streamsx.topology.pytest.checkpoint::TimeCounter", schema.StreamSchema('tuple<int32 f>').as_tuple(), params={'iterations':30,'period':0.1}) evenFilter = op.Map("com.ibm.streamsx.topology.pytest.checkpoint::StatefulEvenFilter", timeCounter.stream, None, params={}) hpo = op.Map("com.ibm.streamsx.topology.pytest.checkpoint::StatefulHalfPlusOne", evenFilter.stream, None, params={}) s = hpo.stream tester = Tester(topo) tester.tuple_count(s, 15) tester.contents(s, list(zip(range(1,16)))) tester.test(self.test_ctxtype, self.test_config)
def test_with_pint(self): schema = 'tuple<float64 temp>' topo = Topology() streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('testtkpy_remote')) s = topo.source([0, 100, 28.5]) s = s.map(lambda t: {'temp': t}, schema=schema) fh = op.Map("com.ibm.streamsx.topology.pytest.temps::ToFahrenheit", s) r = fh.stream.map(lambda x: x['temp']) tester = Tester(topo) # We round off to ints because pint temp conversion # is actually incorrect! tester.contents(r, [31.0, 211.0, 83.0]) tester.test(self.test_ctxtype, self.test_config)
def test_map_return(self): """Simple test of returning values from a map.""" topo = Topology() streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('testtkpy')) s = topo.source(range(20)) s = s.map(lambda v: (v, ), schema='tuple<int32 val>') values = op.Map( "com.ibm.streamsx.topology.pytest.pytypes::MapReturnValues", s, 'tuple<rstring how, int32 val>') tester = Tester(topo) tester.contents(values.stream, MRV_EXPECTED) tester.test(self.test_ctxtype, self.test_config)
def test_single_input_port(self): """Operator with one input port""" topo = Topology() streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('testtkpy')) s = topo.source([1043]) s = s.map(lambda x: (x, ), schema='tuple<uint64 v>') bop = op.Sink( "com.ibm.streamsx.topology.pytest.pyprimitives::SingleInputPort", s, name="SIP_OP") self.tester = Tester(topo) self.tester.local_check = self._single_input_port_check self.tester.test(self.test_ctxtype, self.test_config)
def test_only_output_port(self): """Operator with single output port and no inputs.""" count = 106 topo = Topology() streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('testtkpy')) bop = op.Source(topo, "com.ibm.streamsx.topology.pytest.pyprimitives::OutputOnly", schema='tuple<int64 c>', params={'count':count}) r = bop.stream self.tester = Tester(topo) self.tester.tuple_count(r, count) self.tester.contents(r, list({'c':i+501} for i in range(count))) self.tester.test(self.test_ctxtype, self.test_config)
def test_map_return(self): """Simple test of returning values from a map.""" topo = Topology() streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('testtkpy')) s = topo.source(range(20)) s = s.map(lambda v : (v,), schema='tuple<int32 val>') values = op.Map( "com.ibm.streamsx.topology.pytest.pytypes::MapReturnValues", s, 'tuple<rstring how, int32 val>') tester = Tester(topo) tester.contents(values.stream, MRV_EXPECTED) tester.test(self.test_ctxtype, self.test_config)
def test_noports(self): """Operator with no inputs or outputs""" topo = Topology() streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('testtkpy')) bop = op.Invoke( topo, "com.ibm.streamsx.topology.pytest.pyprimitives::NoPorts", params={ 'mn': 'mymetric', 'iv': 89 }) self.tester = Tester(topo) self.tester.local_check = self._noports_check self.tester.test(self.test_ctxtype, self.test_config)
def test_with_pint(self): schema='tuple<float64 temp>' topo = Topology() streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('testtkpy_remote')) s = topo.source([0, 100, 28.5]) s = s.map(lambda t : {'temp':t}, schema=schema) fh = op.Map( "com.ibm.streamsx.topology.pytest.temps::ToFahrenheit", s) r = fh.stream.map(lambda x : x['temp']) tester = Tester(topo) # We round off to ints because pint temp conversion # is actually incorrect! tester.contents(r, [32.0, 212.0, 83.0]) tester.test(self.test_ctxtype, self.test_config)
def test_multi_input_ports(self): """Operator with three input ports""" topo = Topology() streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('testtkpy')) s0 = topo.source([9054]).map(lambda x: (x, ), schema='tuple<uint64 v>') s1 = topo.source([345]).map(lambda x: (x, ), schema='tuple<int64 v>') s2 = topo.source([-953]).map(lambda x: (x, ), schema='tuple<int32 v>') bop = op.Invoke( topo, "com.ibm.streamsx.topology.pytest.pyprimitives::MultiInputPort", [s0, s1, s2], name="MIP_OP") self.tester = Tester(topo) self.tester.local_check = self._multi_input_port_check self.tester.test(self.test_ctxtype, self.test_config)
def test_single_output_port(self): """Operator with single output port.""" topo = Topology() streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('testtkpy')) s = topo.source([9237, -24]) s = s.map(lambda x : (x,), schema='tuple<int64 v>') bop = op.Map("com.ibm.streamsx.topology.pytest.pyprimitives::SingleOutputPort", s) r = bop.stream self.tester = Tester(topo) self.tester.tuple_count(s, 2) self.tester.contents(s, [{'v':9237}, {'v':-24}]) self.tester.test(self.test_ctxtype, self.test_config)
def test_input_by_position(self): """Operator with input by position""" topo = Topology() streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('testtkpy')) s = topo.source([3642, -393]) s = s.map(lambda x : (x,x*2,x+4), schema='tuple<int64 d, int64 e, int64 f>') bop = op.Map("com.ibm.streamsx.topology.pytest.pyprimitives::InputByPosition", s) r = bop.stream self.tester = Tester(topo) self.tester.tuple_count(r, 2) self.tester.contents(r, [{'d':3642, 'e':(3642*2)+89, 'f':-92}, {'d':-393, 'e':(-393*2)+89, 'f':-92}]) self.tester.test(self.test_ctxtype, self.test_config)
def test_map_blob_type(self): topo = Topology() streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('testtkpy')) data = ['Hello', 'Blob', 'Did', 'you', 'reset'] s = topo.source(data) s = s.as_string() toBlob = op.Map("com.ibm.streamsx.topology.pytest.pytypes::ToMapBlob", s, 'tuple<map<rstring,blob> lb>') bt = op.Map("com.ibm.streamsx.topology.pytest.pytypes::MapBlobTest", toBlob.stream, 'tuple<rstring string>') tester = Tester(topo) tester.contents(bt.stream, data) tester.test(self.test_ctxtype, self.test_config)
def test_single_toolkit(self): topo = Topology() streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('testtkpy')) bop = op.Source(topo, "spl.utility::Beacon", 'tuple<rstring a>', {'iterations': 1}) bop.a = bop.output('"A"') sv = op.Map( "com.ibm.streamsx.topology.pytest.pyvers::StreamsxVersion", bop.stream, 'tuple<rstring a, rstring v1, rstring v2>') tester = Tester(topo) tester.contents(sv.stream, [{'a':'A', 'v1':'aggregate', 'v2':'True'}]) tester.test(self.test_ctxtype, self.test_config)
def test_dict_output_ports(self): """Operator with multiple output port submitting dict objects.""" topo = Topology() streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('testtkpy')) s = topo.source([9237, -24]) s = s.map(lambda x: (x, x * 2, x + 4), schema='tuple<int64 d, int64 e, int64 f>') bop = op.Invoke( topo, "com.ibm.streamsx.topology.pytest.pyprimitives::DictOutputPorts", s, schemas=['tuple<int64 d, int64 e, int64 f>'] * 2) r = bop.outputs self.tester = Tester(topo) self.tester.tuple_count(r[0], 2) self.tester.tuple_count(r[1], 4) self.tester.contents(r[0], [{ 'd': 9237, 'e': (9237 * 2), 'f': 9237 + 4 }, { 'd': -24, 'e': (-24 * 2), 'f': -24 + 4 }]) self.tester.contents(r[1], [{ 'd': 9237 + 7, 'f': (9237 * 2) + 777, 'e': 9237 + 4 + 77 }, { 'd': 9237, 'e': (9237 * 2), 'f': 9237 + 4 }, { 'd': -24 + 7, 'f': (-24 * 2) + 777, 'e': -24 + 4 + 77 }, { 'd': -24, 'e': (-24 * 2), 'f': -24 + 4 }]) self.tester.test(self.test_ctxtype, self.test_config)
def test_filter_map(self): iterations = 3000 topo = Topology() streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('testtkpy')) timeCounter = op.Source(topo, "com.ibm.streamsx.topology.pytest.checkpoint::TimeCounter", schema.StreamSchema('tuple<int32 f>').as_tuple(), params={'iterations':iterations,'period':0.01}) timeCounter.stream.set_consistent(ConsistentRegionConfig.periodic(5, drain_timeout=40, reset_timeout=40, max_consecutive_attempts=6)) evenFilter = op.Map("com.ibm.streamsx.topology.pytest.checkpoint::StatefulEvenFilter", timeCounter.stream, None, params={}) hpo = op.Map("com.ibm.streamsx.topology.pytest.checkpoint::StatefulHalfPlusOne", evenFilter.stream, None, params={}) s = hpo.stream tester = Tester(topo) tester.resets() tester.tuple_count(s, iterations/2) tester.contents(s, list(zip(range(1,int((iterations/2)+1))))) tester.test(self.test_ctxtype, self.test_config)
def test_single_toolkit(self): topo = Topology() streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('testtkpy')) bop = op.Source(topo, "spl.utility::Beacon", 'tuple<rstring a>', {'iterations': 1}) bop.a = bop.output('"A"') sv = op.Map("com.ibm.streamsx.topology.pytest.pyvers::StreamsxVersion", bop.stream, 'tuple<rstring a, rstring v1, rstring v2>') tester = Tester(topo) tester.contents(sv.stream, [{ 'a': 'A', 'v1': 'aggregate', 'v2': 'True' }]) tester.test(self.test_ctxtype, self.test_config)
def test_optional_blob_type(self): Tester.require_streams_version(self, '4.3') topo = Topology() streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('testtkpy')) data = ['Hello', 'Blob', 'Did', 'you', 'reset'] s = topo.source(data) s = s.as_string() toBlob = op.Map("com.ibm.streamsx.topology.pytest.pytypes::ToBlob", s, 'tuple<optional<blob> ob>') bt = op.Map("com.ibm.streamsx.topology.pytest.pytypes::BlobTest", toBlob.stream, 'tuple<rstring string>', {'keep': True}) tester = Tester(topo) tester.contents(bt.stream, data) tester.test(self.test_ctxtype, self.test_config)
def test_suppress_metric(self): schema = 'tuple<int32 a, int32 b, int32 c, int32 d>' topo = Topology() streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('testtkpy')) # no metric st = op.Source(topo, kind='com.ibm.streamsx.topology.pytest.pysource::SparseTuple', schema=schema, name='NOMETRIC_ST') sf = op.Source(topo, kind='com.ibm.streamsx.topology.pysamples.sources::Range37', schema=schema, name='NOMETRIC_SF') s = st.stream.union({sf.stream}) sm = op.Map('com.ibm.streamsx.topology.pysamples.positional::Noop', s, name='NOMETRIC_MF') sm = op.Map('com.ibm.streamsx.topology.pysamples.positional::AddSeq', sm.stream, name='NOMETRIC_MC') # With metric schema = 'tuple<rstring a, int32 b>' ms = op.Source(topo, kind='com.ibm.streamsx.topology.pytest.pyexceptions::SuppressNextSource', schema=schema, name='HASMETRIC_S_1') mm = op.Map( kind='com.ibm.streamsx.topology.pytest.pyexceptions::SuppressMap', stream=ms.stream, name='HASMETRIC_M_0') mf = op.Map( kind='com.ibm.streamsx.topology.pytest.pyexceptions::SuppressFilter', stream=ms.stream, name='HASMETRIC_F_0') self.tester = Tester(topo) self.tester.local_check = self.check_suppress_metric # Add filters to avoid the test operators having # names of NOMETIC/HASMETRIC self.tester.tuple_count(sm.stream.filter(lambda _ : True), 38) self.tester.tuple_count(ms.stream.filter(lambda _ : True), 2) self.tester.tuple_count(mm.stream.filter(lambda _ : True), 2) self.tester.tuple_count(mf.stream.filter(lambda _ : True), 2) self.tester.test(self.test_ctxtype, self.test_config)
def test_primitive_foreach(self): iterations=3000 topo = Topology() topo.checkpoint_period = timedelta(seconds=1) streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('testtkpy')) timeCounter = op.Source(topo, "com.ibm.streamsx.topology.pytest.checkpoint::TimeCounter", schema.StreamSchema('tuple<int32 f>').as_tuple(), params={'iterations':iterations,'period':0.01}) timeCounter.stream.set_consistent(ConsistentRegionConfig.periodic(5, drain_timeout=40, reset_timeout=40, max_consecutive_attempts=6)) fizzbuzz = op.Map("com.ibm.streamsx.topology.pytest.checkpoint::FizzBuzzPrimitive", timeCounter.stream, schema.StreamSchema('tuple<int32 f, rstring c>').as_tuple()) verify = op.Sink("com.ibm.streamsx.topology.pytest.checkpoint::Verify", fizzbuzz.stream) s = fizzbuzz.stream tester = Tester(topo) tester.resets() tester.tuple_count(s, iterations) tester.test(self.test_ctxtype, self.test_config)
def test_multi_output_ports(self): """Operator with multiple output port.""" topo = Topology() streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('testtkpy')) s = topo.source([9237, -24]) s = s.map(lambda x : (x,), schema='tuple<int64 v>') bop = op.Invoke(topo, "com.ibm.streamsx.topology.pytest.pyprimitives::MultiOutputPorts", s, schemas=['tuple<int64 v1>', 'tuple<int32 v2>', 'tuple<int16 v3>']) r = bop.outputs self.tester = Tester(topo) self.tester.tuple_count(s, 2) self.tester.contents(r[0], [{'v1':9237}, {'v1':-24}]) self.tester.contents(r[1], [{'v2':9237+921}, {'v2':-24+921}]) self.tester.contents(r[2], [{'v3':9237-407}, {'v3':-24-407}]) self.tester.test(self.test_ctxtype, self.test_config)
def test_single_output_port(self): """Operator with single output port.""" topo = Topology() streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('testtkpy')) s = topo.source([9237, -24]) s = s.map(lambda x: (x, ), schema='tuple<int64 v>') bop = op.Map( "com.ibm.streamsx.topology.pytest.pyprimitives::SingleOutputPort", s) r = bop.stream self.tester = Tester(topo) self.tester.tuple_count(s, 2) self.tester.contents(s, [{'v': 9237}, {'v': -24}]) self.tester.test(self.test_ctxtype, self.test_config)
def test_dict_output_ports(self): """Operator with multiple output port submitting dict objects.""" topo = Topology() streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('testtkpy')) s = topo.source([9237, -24]) s = s.map(lambda x : (x,x*2,x+4), schema='tuple<int64 d, int64 e, int64 f>') bop = op.Invoke(topo, "com.ibm.streamsx.topology.pytest.pyprimitives::DictOutputPorts", s, schemas=['tuple<int64 d, int64 e, int64 f>']*2) r = bop.outputs self.tester = Tester(topo) self.tester.tuple_count(r[0], 2) self.tester.tuple_count(r[1], 4) self.tester.contents(r[0], [{'d':9237, 'e':(9237*2), 'f':9237+4}, {'d':-24, 'e':(-24*2), 'f':-24+4}]) self.tester.contents(r[1], [{'d':9237+7, 'f':(9237*2)+777, 'e':9237+4+77}, {'d':9237, 'e':(9237*2), 'f':9237+4}, {'d':-24+7, 'f':(-24*2)+777, 'e':-24+4+77}, {'d':-24, 'e':(-24*2), 'f':-24+4}]) self.tester.test(self.test_ctxtype, self.test_config)
def test_only_output_port(self): """Operator with single output port and no inputs.""" count = 106 topo = Topology() streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('testtkpy')) bop = op.Source( topo, "com.ibm.streamsx.topology.pytest.pyprimitives::OutputOnly", schema='tuple<int64 c>', params={'count': count}) r = bop.stream self.tester = Tester(topo) self.tester.tuple_count(r, count) self.tester.contents(r, list({'c': i + 501} for i in range(count))) self.tester.test(self.test_ctxtype, self.test_config)
def test_source(self): iterations = 3000 topo = Topology() streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('testtkpy')) bop = op.Source(topo, "com.ibm.streamsx.topology.pytest.checkpoint::TimeCounter", schema.StreamSchema('tuple<int32 f>').as_tuple(), params={'iterations':iterations,'period':0.01}) s = bop.stream s.set_consistent(ConsistentRegionConfig.periodic(5, drain_timeout=40, reset_timeout=40, max_consecutive_attempts=6)) tester = Tester(topo) tester.resets() tester.tuple_count(s, iterations) tester.contents(s, list(zip(range(0,iterations)))) # job_config = streamsx.topology.context.JobConfig(tracing='debug') # job_config.add(self.test_config) tester.test(self.test_ctxtype, self.test_config)
def test_primitive_submit(self): """Simple test of submitting values from a primitive operator.""" topo = Topology() streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('testtkpy')) values = op.Source( topo, kind="com.ibm.streamsx.topology.pytest.pytypes::PrimitiveReturnValues", schema='tuple<rstring how, int32 val>') expected = copy.deepcopy(MRV_EXPECTED) # no assignment from input tuple so all automatically # assigned values will be zero for d in expected: if d['val'] < 100: d['val'] = 0 tester = Tester(topo) tester.contents(values.stream, expected) tester.test(self.test_ctxtype, self.test_config)
def test_map_blob_type(self): topo = Topology() streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('testtkpy')) data = ['Hello', 'Blob', 'Did', 'you', 'reset' ] s = topo.source(data) s = s.as_string() toBlob = op.Map( "com.ibm.streamsx.topology.pytest.pytypes::ToMapBlob", s, 'tuple<map<rstring,blob> lb>') bt = op.Map( "com.ibm.streamsx.topology.pytest.pytypes::MapBlobTest", toBlob.stream, 'tuple<rstring string>') tester = Tester(topo) tester.contents(bt.stream, data) tester.test(self.test_ctxtype, self.test_config)
def test_not_extracting(self): self.assertFalse(spl.extracting()) topo = Topology() streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('testtkpy_pip_op')) s = topo.source(['a', 'b', 'c']) s = s.as_string() fpe = op.Map( "com.ibm.streamsx.topology.pytest.pypip::check_not_extracting", s) fpa = op.Map( "com.ibm.streamsx.topology.pytest.pypip::check_ec_active", fpe.stream) fplc = op.Map( "com.ibm.streamsx.topology.pytest.pypip::check_protected_import", fpa.stream) tester = Tester(topo) tester.contents(fplc.stream, ['a', 'b', 'c']) tester.test(self.test_ctxtype, self.test_config)