def _test_punct_file(self, topo, s, test_name, expected_content, expected_tuple_count, expected_punct_count=None): s = s.map(lambda x: (x, ), schema='tuple<int32 z>') op_params = { 'file': 'punct_file_' + test_name, 'writePunctuations': True, 'flushOnPunctuation': True } op.Sink("spl.adapter::FileSink", s, params=op_params) # Copy the config, since it's shared across all tests, and not every test needs a data # directory. cfg = self.test_config.copy() jc = JobConfig(data_directory=os.getcwd()) jc.add(cfg) tester = Tester(topo) tester.tuple_count(s, expected_tuple_count) if expected_punct_count is not None: tester.punct_count(s, expected_punct_count) tester.test(self.test_ctxtype, cfg) path = os.path.join(os.getcwd(), 'punct_file_' + test_name) # Validate the contents of the file. with open(path, 'r') as f: file_contents = f.read() self.assertEqual(expected_content, file_contents) os.remove(path)
def test_sliding_time_stv_no_default(self): topo = Topology() b = op.Source(topo, "spl.utility::Beacon", 'tuple<uint64 seq>', params={'iterations': 12}) b.seq = b.output('IterationCount()') s = b.stream wtime = topo.create_submission_parameter(name='secs', type_=int) window = s.lastSeconds(wtime).trigger(1) agg = op.Map('spl.relational::Aggregate', window, schema='tuple<uint64 sum, uint64 max>') agg.sum = agg.output('Sum(seq)') agg.max = agg.output('Max(seq)') jc = JobConfig() jc.submission_parameters['secs'] = 2 jc.add(self.test_config) tester = Tester(topo) tester.tuple_count(agg.stream, 12) tester.test(self.test_ctxtype, self.test_config)
def test_sliding_count_stv_no_default(self): step = 1 topo = Topology() b = op.Source(topo, "spl.utility::Beacon", 'tuple<uint64 seq>', params={'iterations': 12}) b.seq = b.output('IterationCount()') s = b.stream count = topo.create_submission_parameter('count', type_=int) window = s.last(count).trigger(step) agg = op.Map('spl.relational::Aggregate', window, schema='tuple<uint64 sum, uint64 max>') agg.sum = agg.output('Sum(seq)') agg.max = agg.output('Max(seq)') expected = [] for i in range(4 + step - 2, 12, step): expected.append({'sum': sum(range(i - 3, i + 1)), 'max': i}) jc = JobConfig() jc.submission_parameters['count'] = 4 jc.add(self.test_config) tester = Tester(topo) tester.contents(agg.stream, expected) tester.test(self.test_ctxtype, self.test_config)
def _multi_injection(self, raw_overlay=None): topo = Topology() streamsx.spl.toolkit.add_toolkit(topo, TestEmInject._TK) context = _rand_path() name = _rand_path() job_name = _rand_path() s1 = endpoint.inject(topo, name=name+'N1', context=context+'C1', monitor=self._monitor) s2 = endpoint.inject(topo, name=name+'N2', context=context+'C2', monitor=self._monitor) jc = JobConfig() self._job_name = _rand_path() jc.job_name = self._job_name jc.add(self.test_config) # Force separate PEs for the inject operators if raw_overlay: jc.raw_overlay = raw_overlay else: s1.colocate(s2) self._path = '/' + context + 'C1/' + name + 'N1/ports/output/0/inject' self.tester = Tester(topo) self.tester.local_check = self._multi_inject self.tester.contents(s1, [{'seq1':i} for i in range(self.N)]) self.tester.contents(s2, [{'seq2':i} for i in range(self.N)]) self.tester.test(self.test_ctxtype, self.test_config) self._check_no_endpoint()
def test_image_name_image_tag(self): topo = Topology("test_image_name_image_tag") heartbeat = topo.source(lambda: itertools.count()) heartbeat.print() image_name = 'py-tst' image_tag = 'v1.0' cfg = {ConfigParams.SSL_VERIFY: False} jc = JobConfig() jc.raw_overlay = { 'edgeConfig': { 'imageName': image_name, 'imageTag': image_tag, 'pipPackages': ['pandas', 'numpy'], 'rpms': ['atlas-devel'] } } jc.add(cfg) try: submission_result = submit(ContextTypes.EDGE, topo.graph, cfg) print(str(submission_result)) self.assertTrue(submission_result is not None) self.assertTrue(self._is_not_blank(submission_result.image)) self.assertTrue(self._is_not_blank(submission_result.imageDigest)) self.assertTrue(image_name in submission_result.image) self.assertTrue(image_tag in submission_result.image) except RuntimeError as e: print(str(e)) self.skipTest("Skip test, CPD does not support EDGE.")
def test_topo_types_explicit_set(self): topo = Topology() sp_str = topo.create_submission_parameter('sp_str', type_=str) sp_int = topo.create_submission_parameter('sp_int', type_=int) sp_float = topo.create_submission_parameter('sp_float', type_=float) sp_bool = topo.create_submission_parameter('sp_bool', type_=bool) s = topo.source(range(17)) s = s.filter( lambda v: isinstance(sp_str(), str) and sp_str() == 'SeeYa') s = s.filter(lambda v: isinstance(sp_int(), int) and sp_int() == 10) s = s.filter( lambda v: isinstance(sp_float(), float) and sp_float() == -0.5) s = s.filter( lambda v: isinstance(sp_bool(), bool) and sp_bool() is True) jc = JobConfig() jc.submission_parameters['sp_str'] = 'SeeYa' jc.submission_parameters['sp_int'] = 10 jc.submission_parameters['sp_float'] = -0.5 jc.submission_parameters['sp_bool'] = True jc.add(self.test_config) tester = Tester(topo) tester.tuple_count(s, 17) tester.test(self.test_ctxtype, self.test_config)
def test_colocation(self): topo = Topology() s1 = topo.source([], name='S1') s2 = topo.source([], name='S2') s1f = s1.filter(lambda x: True, name='S1F') s2f = s2.filter(lambda x: True, name='S2F') s1e = s1f.for_each(lambda x: None, name='S1E') s2e = s2f.for_each(lambda x: None, name='S2E') # S1 -> S1F -> S1E # S2 -> S2F -> S2E s2e.colocate(s1) # S1(X) -> S1F -> S1E # S2 -> S2F -> S2E(X) s2f.colocate([s2, s1f]) # S1(X) -> S1F(Y) -> S1E # S2(Y) -> S2F(Y) -> S2E(X) s1f.colocate(s1e) # S1(X) -> S1F(Y) -> S1E(Y) # S2(Y) -> S2F(Y) -> S2E(X) beacon = Source(topo, "spl.utility::Beacon", 'tuple<uint64 seq>', params={ 'period': 0.02, 'iterations': 100 }, name='BeaconColo') beacon.seq = beacon.output('IterationCount()') beacon.colocate(s2) # Now a colocation independent of the other sets s3 = topo.source([], name='S3') s3f = s3.filter(lambda x: True, name='S3F') s3e = s3f.for_each(lambda x: None, name='S3E') s3.colocate(s3e) # and then join to an existing set #print("MERGE!!!") s3.colocate(s1) self.tester = Tester(topo) self.tester.local_check = self.check_colocations jc = JobConfig() jc.raw_overlay = {'deploymentConfig': {'fusionScheme': 'legacy'}} jc.add(self.test_config) sr = self.tester.test(self.test_ctxtype, self.test_config)
def test_colocation(self): topo = Topology() s1 = topo.source([], name='S1') s2 = topo.source([], name='S2') s1f = s1.filter(lambda x : True, name='S1F') s2f = s2.filter(lambda x : True, name='S2F') s1e = s1f.for_each(lambda x : None, name='S1E') s2e = s2f.for_each(lambda x : None, name='S2E') # S1 -> S1F -> S1E # S2 -> S2F -> S2E s2e.colocate(s1) # S1(X) -> S1F -> S1E # S2 -> S2F -> S2E(X) s2f.colocate([s2,s1f]) # S1(X) -> S1F(Y) -> S1E # S2(Y) -> S2F(Y) -> S2E(X) s1f.colocate(s1e) # S1(X) -> S1F(Y) -> S1E(Y) # S2(Y) -> S2F(Y) -> S2E(X) beacon = Source(topo, "spl.utility::Beacon", 'tuple<uint64 seq>', params = {'period': 0.02, 'iterations':100}, name = 'BeaconColo') beacon.seq = beacon.output('IterationCount()') beacon.colocate(s2) # Now a colocation independent of the other sets s3 = topo.source([], name='S3') s3f = s3.filter(lambda x : True, name='S3F') s3e = s3f.for_each(lambda x : None, name='S3E') s3.colocate(s3e) # and then join to an existing set #print("MERGE!!!") s3.colocate(s1) self.tester = Tester(topo) self.tester.local_check = self.check_colocations jc = JobConfig() jc.raw_overlay = {'deploymentConfig': {'fusionScheme':'legacy'}} jc.add(self.test_config) sr = self.tester.test(self.test_ctxtype, self.test_config)
def test_run_with_name_overwrite(self): topo = Topology() topo.source(['a']) name_not_used = 'JOB:' + str(uuid.uuid4()) name = 'JOB:' + str(uuid.uuid4()) cfg = {} jc = JobConfig(job_name=name_not_used) jc.add(cfg) job, sr = run(topo, config=cfg, job_name=name, verify=self._verify) self.assertIs(job, sr.job) self.assertEqual(name, job.name) job.cancel()
def test_spl(self): """ Test passing as an SPL parameter. """ N = 22 G = 'hey' t = ''.join(random.choice('0123456789abcdef') for x in range(20)) topic = 'topology/test/python/' + t topo = Topology() spTopic = topo.create_submission_parameter('mytopic') spGreet = topo.create_submission_parameter('greeting') self.assertIsNone(spTopic()) self.assertIsNone(spGreet()) sch = StreamSchema('tuple<uint64 seq, rstring s>') b = op.Source(topo, "spl.utility::Beacon", sch, params={ 'initDelay': 10.0, 'period': 0.02, 'iterations': N }) b.seq = b.output('IterationCount()') b.s = b.output(spGreet) p = op.Sink("com.ibm.streamsx.topology.topic::Publish", b.stream, params={'topic': topic}) s = op.Source(topo, "com.ibm.streamsx.topology.topic::Subscribe", sch, params={ 'streamType': sch, 'topic': spTopic }) jc = JobConfig() jc.submission_parameters['mytopic'] = topic jc.submission_parameters['greeting'] = G jc.add(self.test_config) tester = Tester(topo) tester.tuple_count(s.stream, N) #tester.run_for(300) tester.contents(s.stream, [{'seq': i, 's': G} for i in range(N)]) tester.test(self.test_ctxtype, self.test_config)
def test_topo_with_def_and_type(self): topo = Topology() s = topo.source(range(38)) lower = topo.create_submission_parameter('lower', default=0) upper = topo.create_submission_parameter('upper', default=30) s = s.filter(lambda v: v < lower() or v > upper()) jc = JobConfig() jc.submission_parameters['lower'] = 5 jc.add(self.test_config) tester = Tester(topo) tester.contents(s, [0, 1, 2, 3, 4, 31, 32, 33, 34, 35, 36, 37]) tester.test(self.test_ctxtype, self.test_config)
def test_topo_with_def_and_type(self): topo = Topology() s = topo.source(range(38)) lower = topo.create_submission_parameter('lower', default=0) upper = topo.create_submission_parameter('upper', default=30) s = s.filter(lambda v: v < lower() or v > upper()) jc = JobConfig() jc.submission_parameters['lower'] = 5 jc.add(self.test_config) tester = Tester(topo) tester.contents(s, [0,1,2,3,4,31,32,33,34,35,36,37]) tester.test(self.test_ctxtype, self.test_config)
def test_UnicodeJobName(self): """ Test unicode topo names """ job_name = '你好世界' topo = Topology() jc = JobConfig(job_name=job_name) hw = topo.source(["Hello", "Tester"]) tester = Tester(topo) tester.contents(hw, ["Hello", "Tester"]) jc.add(self.test_config) self.assertIs(jc, self.test_config[ConfigParams.JOB_CONFIG]) sr = tester.test(self.test_ctxtype, self.test_config) self.assertEqual(job_name, tester.result['submission_result']['name'])
def test_from_topology(self): topo = Topology('SabTest', namespace='mynamespace') s = topo.source([1, 2]) es = s.for_each(lambda x: None) cfg = {} jc = JobConfig(job_name='ABCD', job_group='XXG', preload=True) jc.add(cfg) bb = streamsx.topology.context.submit('BUNDLE', topo, cfg) self.assertIn('bundlePath', bb) self.assertIn('jobConfigPath', bb) with open(bb['jobConfigPath']) as json_data: jct = JobConfig.from_overlays(json.load(json_data)) self.assertEqual(jc.job_name, jct.job_name) self.assertEqual(jc.job_group, jct.job_group) self.assertEqual(jc.preload, jct.preload) os.remove(bb['bundlePath']) os.remove(bb['jobConfigPath'])
def test_from_topology(self): topo = Topology('SabTest', namespace='mynamespace') s = topo.source([1,2]) es = s.for_each(lambda x : None) cfg = {} jc = JobConfig(job_name='ABCD', job_group='XXG', preload=True) jc.add(cfg) bb = streamsx.topology.context.submit('BUNDLE', topo, cfg) self.assertIn('bundlePath', bb) self.assertIn('jobConfigPath', bb) with open(bb['jobConfigPath']) as json_data: jct = JobConfig.from_overlays(json.load(json_data)) self.assertEqual(jc.job_name, jct.job_name) self.assertEqual(jc.job_group, jct.job_group) self.assertEqual(jc.preload, jct.preload) os.remove(bb['bundlePath']) os.remove(bb['jobConfigPath'])
def test_UnicodeJobName(self): """ Test unicode topo names """ job_name = '你好世界' topo = Topology() jc = JobConfig(job_name=job_name) # When tracing is info some extra code is invoked # to trace all Python packages. Ensure it is exercised. jc.tracing = 'info' hw = topo.source(["Hello", "Tester"]) tester = Tester(topo) tester.contents(hw, ["Hello", "Tester"]) jc.add(self.test_config) self.assertIs(jc, self.test_config[ConfigParams.JOB_CONFIG]) sr = tester.test(self.test_ctxtype, self.test_config) self.assertEqual(job_name, tester.result['submission_result']['name'])
def test_UnicodeJobName(self): """ Test unicode topo names """ job_name = '你好世界' topo = Topology() jc = JobConfig(job_name=job_name) # When tracing is info some extra code is invoked # to trace all Python packages. Ensure it is exercised. jc.tracing = 'info' hw = topo.source(["Hello", "Tester"]) tester = Tester(topo) tester.contents(hw, ["Hello", "Tester"]) jc.add(self.test_config) self.assertIs(jc, self.test_config[ConfigParams.JOB_CONFIG]) sr = tester.test(self.test_ctxtype, self.test_config) self.assertEqual(job_name, tester.result['submission_result']['name'])
def test_parallel(self): topo = Topology() sp_w1 = topo.create_submission_parameter('w1', type_=int) sp_w2 = topo.create_submission_parameter('w2', type_=int) s = topo.source(range(67)).set_parallel(sp_w1) s = s.filter(lambda v: v % sp_w1() == 0) s = s.end_parallel() s = s.parallel(width=sp_w2) s = s.filter(lambda v: v % sp_w2() == 0) s = s.end_parallel() jc = JobConfig() jc.submission_parameters['w1'] = 3 jc.submission_parameters['w2'] = 5 jc.add(self.test_config) tester = Tester(topo) tester.contents(s, [0, 15, 30, 45, 60] * 3, ordered=False) tester.test(self.test_ctxtype, self.test_config)
def test_parallel(self): topo = Topology() sp_w1 = topo.create_submission_parameter('w1', type_=int) sp_w2 = topo.create_submission_parameter('w2', type_=int) s = topo.source(range(67)).set_parallel(sp_w1) s = s.filter(lambda v : v % sp_w1() == 0) s = s.end_parallel() s = s.parallel(width=sp_w2) s = s.filter(lambda v : v % sp_w2() == 0) s = s.end_parallel() jc = JobConfig() jc.submission_parameters['w1'] = 3 jc.submission_parameters['w2'] = 5 jc.add(self.test_config) tester = Tester(topo) tester.contents(s,[0,15,30,45,60]*3, ordered=False) tester.test(self.test_ctxtype, self.test_config)
def test_BasicTimeCountWindow_stv_no_default(self): # Ensure tuples are evicted no later than (window_span*tolerance + window_span) tolerance = 0.20 window_span = 2.0 max_evict = window_span * tolerance + window_span topo = Topology() s = topo.source(TimeCounter(iterations=100, period=0.1)) s = s.map(lambda x: (x, time.time())) wtime = topo.create_submission_parameter('secs', type_=int) s = s.lastSeconds(wtime).trigger(20).aggregate( TupleTimespanCheck(max_evict)) jc = JobConfig() jc.submission_parameters['secs'] = 2 jc.add(self.test_config) tester = Tester(topo) tester.tuple_check(s, lambda val: val) tester.test(self.test_ctxtype, self.test_config)
def test_topo(self): topo = Topology() s = topo.source(range(38)) lower = topo.create_submission_parameter('lower') upper = topo.create_submission_parameter('upper') addin = topo.create_submission_parameter('addin') s = s.filter(lambda v: v < int(lower()) or v > int(upper())) m = s.filter(lambda v : v < 3) m = m.map(AddIt(addin)) jc = JobConfig() jc.submission_parameters['lower'] = 7 jc.submission_parameters['upper'] = 33 jc.submission_parameters['addin'] = 'Yeah!' jc.add(self.test_config) tester = Tester(topo) tester.contents(s, [0,1,2,3,4,5,6,34,35,36,37]) tester.contents(m, ['0-Yeah!','1-Yeah!','2-Yeah!']) tester.test(self.test_ctxtype, self.test_config)
def test_topo_types_explicit_set(self): topo = Topology() sp_str = topo.create_submission_parameter('sp_str', type_=str) sp_int = topo.create_submission_parameter('sp_int', type_=int) sp_float = topo.create_submission_parameter('sp_float', type_=float) sp_bool = topo.create_submission_parameter('sp_bool', type_=bool) s = topo.source(range(17)) s = s.filter(lambda v : isinstance(sp_str(), str) and sp_str() == 'SeeYa') s = s.filter(lambda v : isinstance(sp_int(), int) and sp_int() == 10) s = s.filter(lambda v : isinstance(sp_float(), float) and sp_float() == -0.5) s = s.filter(lambda v : isinstance(sp_bool(), bool) and sp_bool() is True) jc = JobConfig() jc.submission_parameters['sp_str'] = 'SeeYa' jc.submission_parameters['sp_int'] = 10 jc.submission_parameters['sp_float'] = -0.5 jc.submission_parameters['sp_bool'] = True jc.add(self.test_config) tester = Tester(topo) tester.tuple_count(s, 17) tester.test(self.test_ctxtype, self.test_config)
def test_topo(self): topo = Topology() s = topo.source(range(38)) lower = topo.create_submission_parameter('lower') upper = topo.create_submission_parameter('upper') addin = topo.create_submission_parameter('addin') s = s.filter(lambda v: v < int(lower()) or v > int(upper())) m = s.filter(lambda v: v < 3) m = m.map(AddIt(addin)) jc = JobConfig() jc.submission_parameters['lower'] = 7 jc.submission_parameters['upper'] = 33 jc.submission_parameters['addin'] = 'Yeah!' jc.add(self.test_config) tester = Tester(topo) tester.contents(s, [0, 1, 2, 3, 4, 5, 6, 34, 35, 36, 37]) tester.contents(m, ['0-Yeah!', '1-Yeah!', '2-Yeah!']) tester.test(self.test_ctxtype, self.test_config)
def test_spl(self): """ Test passing as an SPL parameter. """ N=22 G='hey' t = ''.join(random.choice('0123456789abcdef') for x in range(20)) topic = 'topology/test/python/' + t topo = Topology() spTopic = topo.create_submission_parameter('mytopic') spGreet = topo.create_submission_parameter('greeting') self.assertIsNone(spTopic()) self.assertIsNone(spGreet()) sch = StreamSchema('tuple<uint64 seq, rstring s>') b = op.Source(topo, "spl.utility::Beacon", sch, params = {'initDelay': 10.0, 'period': 0.02, 'iterations':N}) b.seq = b.output('IterationCount()') b.s = b.output(spGreet) p = op.Sink("com.ibm.streamsx.topology.topic::Publish", b.stream, params={'topic': topic}) s = op.Source(topo, "com.ibm.streamsx.topology.topic::Subscribe", sch, params = {'streamType': sch, 'topic': spTopic}) jc = JobConfig() jc.submission_parameters['mytopic'] = topic jc.submission_parameters['greeting'] = G jc.add(self.test_config) tester = Tester(topo) tester.tuple_count(s.stream, N) #tester.run_for(300) tester.contents(s.stream, [{'seq':i, 's':G} for i in range(N)]) tester.test(self.test_ctxtype, self.test_config)
from streamsx.mqtt import MQTTSource, MQTTSink import streamsx.topology.context as context from streamsx.topology.context import ContextTypes, JobConfig from streamsx.topology.topology import Topology from streamsx.topology.schema import CommonSchema mqtt_server_uri = 'tcp://172.16.33.10:1883' topology = Topology() s = 'Each character will be an MQTT message' data = topology.source([c for c in s]).as_string() # publish to MQTT data.for_each(MQTTSink(server_uri=mqtt_server_uri, topic='topic'), name='MQTTpublish') # subscribe for data and print to stdout received = topology.source( MQTTSource(mqtt_server_uri, schema=CommonSchema.String, topics='topic')) received.print() job_config = JobConfig(job_name='MQTTBasic', tracing='info') submission_config = {} # add the job_config into the submission config: job_config.add(submission_config) print('submission_config: ' + str(submission_config)) context.submit(ContextTypes.DISTRIBUTED, topology, config=submission_config) # the Streams Job keeps running and must be cancelled manually