def test_primitive_foreach(self): iterations = 3000 topo = Topology() topo.checkpoint_period = timedelta(seconds=1) streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('testtkpy')) timeCounter = op.Source( topo, "com.ibm.streamsx.topology.pytest.checkpoint::TimeCounter", schema.StreamSchema('tuple<int32 f>').as_tuple(), params={ 'iterations': iterations, 'period': 0.01 }) timeCounter.stream.set_consistent( ConsistentRegionConfig.periodic(5, drain_timeout=40, reset_timeout=40, max_consecutive_attempts=6)) fizzbuzz = op.Map( "com.ibm.streamsx.topology.pytest.checkpoint::FizzBuzzPrimitive", timeCounter.stream, schema.StreamSchema('tuple<int32 f, rstring c>').as_tuple()) verify = op.Sink("com.ibm.streamsx.topology.pytest.checkpoint::Verify", fizzbuzz.stream) s = fizzbuzz.stream tester = Tester(topo) tester.resets() tester.tuple_count(s, iterations) tester.test(self.test_ctxtype, self.test_config)
def test_WindowPunctuation(self): """Trigger an aggregation 4 times. Ensure that window punctuations are submitted each time by writing them to an output file, and then verifying that the file contains the correct contents.""" topo = Topology() s = topo.source([1,2,3,4]) # Aggregate and write to file. s = s.last(1).trigger(1).aggregate(lambda x: x[0]+7) # Ensure map/flat_map/filter passes window marks through. s = s.flat_map(lambda x : [x]) s = s.filter(lambda x : True) s = s.map(lambda x : (x,), schema='tuple<int32 z>') op_params = {'file' : 'punct_file', 'writePunctuations' : True, 'flushOnPunctuation' : True} op.Sink("spl.adapter::FileSink", s, params = op_params) # Copy the config, since it's shared across all tests, and not every test needs a data # directory. cfg = self.test_config.copy() jc = context.JobConfig(data_directory=os.getcwd()) jc.add(cfg) tester = Tester(topo) tester.test(self.test_ctxtype, cfg) path = os.path.join(os.getcwd(), 'punct_file') # Validate the contents of the file. with open(path, 'r') as f: file_contents = f.read() self.assertEqual(expected_contents, file_contents) os.remove(path)
def test_single_output_port_punct_forward(self): """Operator with receives and forwards window marker.""" topo = Topology() streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('testtkpy')) s = topo.source([1, 2, 3, 4]) s = s.punctor(func=(lambda t: 2 < t), before=False) s = s.map(lambda x: (x, ), schema='tuple<int32 z>') s1 = op.Map( "com.ibm.streamsx.topology.pytest.pyprimitives::SingleOutputPort", s) # has no on_punct bop = op.Map( "com.ibm.streamsx.topology.pytest.pyprimitives::SingleOutputPortPunctForward", s) # implements on_punct r = bop.stream op.Sink("com.ibm.streamsx.topology.pytest.pyprimitives::VerifyPosInt", r) r.print(write_punctuations=True) #self.test_config['topology.keepArtifacts'] = True self.tester = Tester(topo) self.tester.tuple_count(r, 4) self.tester.punct_count(r, 2) self.tester.test(self.test_ctxtype, self.test_config)
def test_filer_for_each(self): topo = Topology() streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('testtkpy')) s = topo.source([(1, 2, 3), (1, 2, 201), (2, 2, 301), (3, 4, 401), (5, 6, 78), (8, 6, 501), (803, 9324, 901)]) sch = 'tuple<int32 a, int32 b, int32 c>' s = s.map(lambda x: x, schema=sch) bop = op.Map("com.ibm.streamsx.topology.pytest.pykwargs::KWFilter", s) op.Sink("com.ibm.streamsx.topology.pytest.pykwargs::KWForEach", bop.stream) tester = Tester(topo) tester.contents(bop.stream, [{ 'a': 1, 'b': 2, 'c': 201 }, { 'a': 3, 'b': 4, 'c': 401 }, { 'a': 803, 'b': 9324, 'c': 901 }]) tester.test(self.test_ctxtype, self.test_config)
def wikiFeed(inetToolkit, buildType, port): # Sumbit request Build Server and Submit. schemaTicker = 'tuple<rstring dataId, rstring bot, rstring domain, int32 lenOld,int32 lenNew, rstring SSEdata, rstring SSEevent>' # # Define the application # topo = Topology("WikiFeed") tk.add_toolkit(topo, inetToolkit) source = topo.source(wikiStream) # Only one type of event, verity that I got it right. event = source.filter(lambda t: t["SSEevent"] == "message", name="eventFilter") event.print(name="eth") eventTuple = event.map(lambda t: t, schema=schemaTicker) eventWin = eventTuple.last(100).trigger(1) rawRequest = op.Sink("com.ibm.streamsx.inet.rest::HTTPTupleView", stream=eventWin, params={ 'port': 8081, 'context': 'gdaxEth', 'contextResourceBase': '/base' }, name="TupleView") # # Compile & Submit the Topology to Streams instance # streams_conf = common.build_streams_config("StreamingTurbine", credential.serviceCredentials) context.submit(context.ContextTypes.STREAMING_ANALYTICS_SERVICE, topo, config=streams_conf)
def _test_punct_file(self, topo, s, test_name, expected_content, expected_tuple_count, expected_punct_count=None): s = s.map(lambda x: (x, ), schema='tuple<int32 z>') op_params = { 'file': 'punct_file_' + test_name, 'writePunctuations': True, 'flushOnPunctuation': True } op.Sink("spl.adapter::FileSink", s, params=op_params) # Copy the config, since it's shared across all tests, and not every test needs a data # directory. cfg = self.test_config.copy() jc = JobConfig(data_directory=os.getcwd()) jc.add(cfg) tester = Tester(topo) tester.tuple_count(s, expected_tuple_count) if expected_punct_count is not None: tester.punct_count(s, expected_punct_count) tester.test(self.test_ctxtype, cfg) path = os.path.join(os.getcwd(), 'punct_file_' + test_name) # Validate the contents of the file. with open(path, 'r') as f: file_contents = f.read() self.assertEqual(expected_content, file_contents) os.remove(path)
def test_map_foreach(self): topo = Topology() topo.checkpoint_period = timedelta(seconds=1) streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('testtkpy')) timeCounter = op.Source(topo, "com.ibm.streamsx.topology.pytest.checkpoint::TimeCounter", schema.StreamSchema('tuple<int32 f>').as_tuple(), params={'iterations':30,'period':0.1}) fizzbuzz = op.Map("com.ibm.streamsx.topology.pytest.checkpoint::FizzBuzzMap", timeCounter.stream, schema.StreamSchema('tuple<int32 f, rstring c>').as_tuple()) verify = op.Sink("com.ibm.streamsx.topology.pytest.checkpoint::Verify", fizzbuzz.stream) s = fizzbuzz.stream tester = Tester(topo) tester.tuple_count(s, 30) tester.test(self.test_ctxtype, self.test_config)
def test_single_input_port(self): """Operator with one input port""" topo = Topology() streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('testtkpy')) s = topo.source([1043]) s = s.map(lambda x: (x, ), schema='tuple<uint64 v>') bop = op.Sink( "com.ibm.streamsx.topology.pytest.pyprimitives::SingleInputPort", s, name="SIP_OP") self.tester = Tester(topo) self.tester.local_check = self._single_input_port_check self.tester.test(self.test_ctxtype, self.test_config)
def test_spl(self): """ Test passing as an SPL parameter. """ N = 22 G = 'hey' t = ''.join(random.choice('0123456789abcdef') for x in range(20)) topic = 'topology/test/python/' + t topo = Topology() spTopic = topo.create_submission_parameter('mytopic') spGreet = topo.create_submission_parameter('greeting') self.assertIsNone(spTopic()) self.assertIsNone(spGreet()) sch = StreamSchema('tuple<uint64 seq, rstring s>') b = op.Source(topo, "spl.utility::Beacon", sch, params={ 'initDelay': 10.0, 'period': 0.02, 'iterations': N }) b.seq = b.output('IterationCount()') b.s = b.output(spGreet) p = op.Sink("com.ibm.streamsx.topology.topic::Publish", b.stream, params={'topic': topic}) s = op.Source(topo, "com.ibm.streamsx.topology.topic::Subscribe", sch, params={ 'streamType': sch, 'topic': spTopic }) jc = JobConfig() jc.submission_parameters['mytopic'] = topic jc.submission_parameters['greeting'] = G jc.add(self.test_config) tester = Tester(topo) tester.tuple_count(s.stream, N) #tester.run_for(300) tester.contents(s.stream, [{'seq': i, 's': G} for i in range(N)]) tester.test(self.test_ctxtype, self.test_config)
def gdaxFeed(inetToolkit, buildType, port): # Sumbit request Build Server and Submit. schemaTicker = 'tuple<rstring ttype, float32 price, float32 low_24h, float32 best_ask, rstring side, float32 best_bid, float32 open_24h, rstring product_id, int32 sequence, int32 trade_id, rstring time, float32 last_size, float32 volume_24h, float32 volume_30d, float32 high_24h>' # # Define the application # topo = Topology("GdaxFeed") tk.add_toolkit(topo, inetToolkit) source = topo.source(gdaxData) # Split out the securities : ETH-USD, LTC-USD, BTC-USD eth = source.filter(lambda t: t["product_id"] == "ETH-USD", name="ethFilter") ltc = source.filter(lambda t: t["product_id"] == "LTC-USD", name="ltcFilter") btc = source.filter(lambda t: t["product_id"] == "LTC-USD", name="btcFilter") eth.print(name="eth") ltc.print(name="ltc") btc.print(name="btc") ethTuple = eth.map(lambda t: t, schema=schemaTicker) #ethWin = ethTuple.last(datetime.timedelta(minutes=2)) ethWin = ethTuple.last(100).trigger(1) rawRequest = op.Sink("com.ibm.streamsx.inet.rest::HTTPTupleView", stream=ethWin, params={ 'port': 8080, 'context': 'gdaxEth', 'contextResourceBase': '/base' }, name="TupleView") # # Compile & Submit the Topology to Streams instance # streams_conf = common.build_streams_config("StreamingTurbine", credential.serviceCredentials) context.submit(context.ContextTypes.STREAMING_ANALYTICS_SERVICE, topo, config=streams_conf) def test(): print("enter") tmp = gdaxData() while True: print(tmp.__next__())
def _publish(self, topo, N, topic, width=None, allow_filter=False): b = op.Source(topo, "spl.utility::Beacon", SCHEMA, params={ 'initDelay': 10.0, 'iterations': N }) b.seq = b.output('IterationCount()') ps = b.stream if width: ps = ps.parallel(width=width) p = op.Sink('com.ibm.streamsx.topology.topic::Publish', ps, params={'topic': topic}, name='MSP') if allow_filter: p.params['allowFilter'] = True
def _run_app(self, kind, e, opi='M'): schema = 'tuple<rstring a, int32 b>' topo = Topology('TSESPL' + str(uuid.uuid4().hex)) streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('testtkpy')) if opi == 'M': data = [1, 2, 3] se = topo.source(data) se = se.map(lambda x: {'a': 'hello', 'b': x}, schema=schema) prim = op.Map("com.ibm.streamsx.topology.pytest.pyexceptions::" + kind, se, params={'tf': self.tf}) res = prim.stream elif opi == 'S': prim = op.Source( topo, "com.ibm.streamsx.topology.pytest.pyexceptions::" + kind, schema=schema, params={'tf': self.tf}) res = prim.stream elif opi == 'E': data = [1, 2, 3] se = topo.source(data) se = se.map(lambda x: {'a': 'hello', 'b': x}, schema=schema) prim = op.Sink("com.ibm.streamsx.topology.pytest.pyexceptions::" + kind, se, params={'tf': self.tf}) res = None tester = Tester(topo) if res is not None: tester.tuple_count(res, len(e)) if e: tester.contents(res, e) else: tester.run_for(5) tester.test(self.test_ctxtype, self.test_config)
def test_map_foreach(self): iterations = 3000 topo = Topology() streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('testtkpy')) timeCounter = op.Source( topo, "com.ibm.streamsx.topology.pytest.checkpoint::TimeCounter", schema.StreamSchema('tuple<int32 f>').as_tuple(), params={ 'iterations': iterations, 'period': 0.01 }) timeCounter.stream.set_consistent( ConsistentRegionConfig.periodic(5, drain_timeout=40, reset_timeout=40, max_consecutive_attempts=6)) fizzbuzz = op.Map( "com.ibm.streamsx.topology.pytest.checkpoint::FizzBuzzMap", timeCounter.stream, schema.StreamSchema('tuple<int32 f, rstring c>').as_tuple()) verify = op.Sink("com.ibm.streamsx.topology.pytest.checkpoint::Verify", fizzbuzz.stream) s = fizzbuzz.stream tester = Tester(topo) tester.resets() tester.tuple_count(s, iterations) # Find the expected results. fizz = lambda x: (x[0], x[1] + 'fizz' if x[0] % 3 == 0 else x[1]) buzz = lambda x: (x[0], x[1] + 'buzz' if x[0] % 5 == 0 else x[1]) expected = list( map(buzz, (map(fizz, (map(lambda x: (x, ''), range(iterations))))))) tester.contents(s, expected) tester.test(self.test_ctxtype, self.test_config)
def test_mt(self): topo = Topology() N = 1000 streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('testtkpy')) b1 = op.Source(topo, "spl.utility::Beacon", schema.StreamSchema('tuple<int32 f>').as_tuple(), params={'iterations': N}) b1.f = b1.output('(int32)IterationCount()') b2 = op.Source(topo, "spl.utility::Beacon", schema.StreamSchema('tuple<int32 f>').as_tuple(), params={'iterations': N}) b2.f = b2.output(str(N) + ' + (int32)IterationCount()') b3 = op.Source(topo, "spl.utility::Beacon", schema.StreamSchema('tuple<int32 f>').as_tuple(), params={'iterations': N}) b3.f = b3.output(str(2 * N) + ' + (int32)IterationCount()') s1 = b1.stream.low_latency() s2 = b2.stream.low_latency() s3 = b3.stream.low_latency() s = s1.union({s2, s3}) f = op.Map("com.ibm.streamsx.topology.pytest.mt::MTFilter", s) m = op.Map("com.ibm.streamsx.topology.pytest.mt::MTMap", f.stream) op.Sink("com.ibm.streamsx.topology.pytest.mt::MTForEach", f.stream) cr = m.stream.flat_map() tester = Tester(topo) tester.tuple_count(m.stream, 3 * N) tester.contents(cr, range(3 * N), ordered=False) tester.test(self.test_ctxtype, self.test_config)
def _run_app(self, kind, opi='M'): schema = 'tuple<rstring a, int32 b>' topo = Topology('TESPL' + str(uuid.uuid4().hex)) streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('testtkpy')) if opi == 'M': data = [1, 2, 3] se = topo.source(data) se = se.map(lambda x: {'a': 'hello', 'b': x}, schema=schema) prim = op.Map("com.ibm.streamsx.topology.pytest.pyexceptions::" + kind, se, params={'tf': self.tf}) res = prim.stream elif opi == 'S': prim = op.Source( topo, "com.ibm.streamsx.topology.pytest.pyexceptions::" + kind, schema=schema, params={'tf': self.tf}) res = prim.stream elif opi == 'E': data = [1, 2, 3] se = topo.source(data) se = se.map(lambda x: {'a': 'hello', 'b': x}, schema=schema) prim = op.Sink("com.ibm.streamsx.topology.pytest.pyexceptions::" + kind, se, params={'tf': self.tf}) res = None tester = Tester(topo) tester.run_for(3) ok = tester.test(self.test_ctxtype, self.test_config, assert_on_fail=False) self.assertFalse(ok)