def test_feedback_loop(self): topo = Topology() data = ['A','B', 'A', 'A', 'X', 'C', 'C', 'D', 'A', 'A', 'E'] expected = ['B', 'X', 'C', 'C', 'D', 'A', 'A', 'E'] s = topo.source(data) s = s.filter(lambda t : time.sleep(1) or True).as_string(); feedback = PendingStream(topo) df = op.Invoke(topo, 'spl.utility::DynamicFilter', inputs = [s, feedback.stream], schemas= [schema.CommonSchema.String]) df.params['key'] = df.attribute(s, 'string') df.params['addKey'] = df.attribute(feedback.stream, 'string') delayed_out = op.Map('spl.utility::Delay', df.outputs[0], params={'delay': 0.05}).stream x = delayed_out.filter(lambda s : s == 'X').map(lambda s : 'A').as_string() i = topo.source(['B', 'X', 'C', 'D', 'E']).as_string() x = x.union({i}) feedback.complete(x) result = delayed_out result.print() #streamsx.topology.context.submit('TOOLKIT', topo) tester = Tester(topo) tester.contents(result, expected) tester.test(self.test_ctxtype, self.test_config)
def test_multi_output_ports_punct(self): """Operator with multiple output port emits window marker after forwarding the tuple.""" topo = Topology() streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('testtkpy')) s = topo.source([9237, -24]) s = s.map(lambda x: (x, ), schema='tuple<int64 v>') bop = op.Invoke( topo, "com.ibm.streamsx.topology.pytest.pyprimitives::MultiOutputPortsPunct", s, schemas=['tuple<int64 v1>', 'tuple<int32 v2>', 'tuple<int16 v3>']) r = bop.outputs r[0].print(tag='0', write_punctuations=True) r[1].print(tag='1', write_punctuations=True) r[2].print(tag='2', write_punctuations=True) self.tester = Tester(topo) self.tester.tuple_count(s, 2) self.tester.punct_count(r[0], 2) self.tester.punct_count(r[1], 2) self.tester.punct_count(r[2], 2) self.tester.contents(r[0], [{'v1': 9237}, {'v1': -24}]) self.tester.contents(r[1], [{'v2': 9237 + 921}, {'v2': -24 + 921}]) self.tester.contents(r[2], [{'v3': 9237 - 407}, {'v3': -24 - 407}]) self.tester.test(self.test_ctxtype, self.test_config)
def test_param_types(self): ri,tkl = _index_tk() self.assertEqual(0, ri) topo = Topology() streamsx.spl.toolkit.add_toolkit(topo, tkl) p = {} p['b'] = False p['u8'] = streamsx.spl.types.uint8(78) p['u16'] = streamsx.spl.types.uint16(78) p['u32'] = streamsx.spl.types.uint32(78) p['u64'] = streamsx.spl.types.uint64(78) p['i8'] = streamsx.spl.types.int8(78) p['i16'] = streamsx.spl.types.int16(78) p['i32'] = 78 p['i64'] = streamsx.spl.types.int64(78) p['f32'] = streamsx.spl.types.float32(78) p['f64'] = 78.9 s = op.Invoke(topo, "testtopo::PT", params = p, name="PY") if np is not None: p = {} p['b'] = np.bool_(False) p['u8'] = np.uint8(78) p['u16'] = np.uint16(78) p['u32'] = np.uint32(78) p['u64'] = np.uint64(78) p['i8'] = np.int8(78) p['i16'] = np.int16(78) p['i32'] = np.int32(89) p['i64'] = np.int64(78) p['f32'] = np.float32(78.03) p['f64'] = np.float64(78.9) snp = op.Invoke(topo, "testtopo::PT", params = p, name="Numpy") cfg = {} if not 'STREAMS_INSTALL' in os.environ: cfg[streamsx.topology.context.ConfigParams.SSL_VERIFY] = False rc = streamsx.topology.context.submit('BUNDLE', topo, cfg) self.assertEqual(0, rc['return_code']) os.remove(rc['bundlePath']) os.remove(rc['jobConfigPath']) shutil.rmtree(tkl)
def _get_spl_app(cmd_args): ns, name = cmd_args.main_composite.rsplit('::', 1) ns += '._spl' topo = Topology(name=name, namespace=ns) if cmd_args.toolkits is not None: for tk_path in cmd_args.toolkits: tk.add_toolkit(topo, tk_path) op.Invoke(topo, cmd_args.main_composite) return (topo, {})
def test_noports(self): """Operator with no inputs or outputs""" topo = Topology() streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('testtkpy')) bop = op.Invoke( topo, "com.ibm.streamsx.topology.pytest.pyprimitives::NoPorts", params={ 'mn': 'mymetric', 'iv': 89 }) self.tester = Tester(topo) self.tester.local_check = self._noports_check self.tester.test(self.test_ctxtype, self.test_config)
def test_flight_path_encounter(self): print ('\n---------'+str(self)) name = 'test_flight_path_encounter' topo = Topology(name) toolkit.add_toolkit(topo, self.geospatial_toolkit_home) self._index_toolkit(_get_test_tk_path()) toolkit.add_toolkit(topo, _get_test_tk_path()) datagen = op.Invoke(topo, kind='test::GenFlightPathData', schemas=[FlighPathEncounterSchema.EncounterEvents]) planes_stream = datagen.outputs[0] events = planes_stream.map(geo.FlightPathEncounter(north_latitude=52.6,south_latitude=52.4,west_longitude=13.3,east_longitude=13.5,num_latitude_divs=5,num_longitude_divs=5,search_radius=10000,altitude_search_radius=400,time_search_interval=600000), schema=FlighPathEncounterSchema.EncounterEvents) dump = op.Invoke(topo, inputs=[events], kind='test::DumpData', schemas=CommonSchema.String) res = dump.outputs[0] res.print() if (("TestDistributed" in str(self)) or ("TestStreamingAnalytics" in str(self))): tester = Tester(topo) tester.tuple_count(res, 1, exact=True) tester.test(self.test_ctxtype, self.test_config, always_collect_logs=True) else: # build only self._build_only(name, topo)
def test_multi_input_ports(self): """Operator with three input ports""" topo = Topology() streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('testtkpy')) s0 = topo.source([9054]).map(lambda x: (x, ), schema='tuple<uint64 v>') s1 = topo.source([345]).map(lambda x: (x, ), schema='tuple<int64 v>') s2 = topo.source([-953]).map(lambda x: (x, ), schema='tuple<int32 v>') bop = op.Invoke( topo, "com.ibm.streamsx.topology.pytest.pyprimitives::MultiInputPort", [s0, s1, s2], name="MIP_OP") self.tester = Tester(topo) self.tester.local_check = self._multi_input_port_check self.tester.test(self.test_ctxtype, self.test_config)
def test_dict_output_ports(self): """Operator with multiple output port submitting dict objects.""" topo = Topology() streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('testtkpy')) s = topo.source([9237, -24]) s = s.map(lambda x: (x, x * 2, x + 4), schema='tuple<int64 d, int64 e, int64 f>') bop = op.Invoke( topo, "com.ibm.streamsx.topology.pytest.pyprimitives::DictOutputPorts", s, schemas=['tuple<int64 d, int64 e, int64 f>'] * 2) r = bop.outputs self.tester = Tester(topo) self.tester.tuple_count(r[0], 2) self.tester.tuple_count(r[1], 4) self.tester.contents(r[0], [{ 'd': 9237, 'e': (9237 * 2), 'f': 9237 + 4 }, { 'd': -24, 'e': (-24 * 2), 'f': -24 + 4 }]) self.tester.contents(r[1], [{ 'd': 9237 + 7, 'f': (9237 * 2) + 777, 'e': 9237 + 4 + 77 }, { 'd': 9237, 'e': (9237 * 2), 'f': 9237 + 4 }, { 'd': -24 + 7, 'f': (-24 * 2) + 777, 'e': -24 + 4 + 77 }, { 'd': -24, 'e': (-24 * 2), 'f': -24 + 4 }]) self.tester.test(self.test_ctxtype, self.test_config)
def test_region_match(self): print ('\n---------'+str(self)) name = 'test_region_match' topo = Topology(name) toolkit.add_toolkit(topo, self.geospatial_toolkit_home) self._index_toolkit(_get_test_tk_path()) toolkit.add_toolkit(topo, _get_test_tk_path()) datagen = op.Invoke(topo, kind='test::GenRegionData', schemas=[RegionMatchSchema.Devices,RegionMatchSchema.Regions]) device_stream = datagen.outputs[0] region_stream = datagen.outputs[1] res = geo.region_match(stream=device_stream, region_stream=region_stream) res.print() if (("TestDistributed" in str(self)) or ("TestStreamingAnalytics" in str(self))): tester = Tester(topo) tester.tuple_count(res, 4, exact=True) tester.test(self.test_ctxtype, self.test_config, always_collect_logs=True) else: # build only self._build_only(name, topo)
def test_multi_output_ports(self): """Operator with multiple output port.""" topo = Topology() streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('testtkpy')) s = topo.source([9237, -24]) s = s.map(lambda x: (x, ), schema='tuple<int64 v>') bop = op.Invoke( topo, "com.ibm.streamsx.topology.pytest.pyprimitives::MultiOutputPorts", s, schemas=['tuple<int64 v1>', 'tuple<int32 v2>', 'tuple<int16 v3>']) r = bop.outputs self.tester = Tester(topo) self.tester.tuple_count(s, 2) self.tester.contents(r[0], [{'v1': 9237}, {'v1': -24}]) self.tester.contents(r[1], [{'v2': 9237 + 921}, {'v2': -24 + 921}]) self.tester.contents(r[2], [{'v3': 9237 - 407}, {'v3': -24 - 407}]) self.tester.test(self.test_ctxtype, self.test_config)
def test_attributes_category_params_resource_tags(self): topo = Topology() b = op.Source( topo, "spl.utility::Beacon", 'tuple<uint64 seq, rstring category, boolean params, int32 resource_tags>', params={'iterations': 3}) # update operator parameters b.params['period'] = 0.01 b.params.update({'initDelay': 1.0}) # the SPL operator's category b.category = 'sources_category' # output assignments: b.seq = b.output('IterationCount()') b.category = b.output('"category"') # an rstring literal b.params = b.output('true') b.resource_tags = b.output(spltypes.int32(42)) beacon_stream = b.stream functor_params = { 'filter': op.Expression.expression('seq % 2ul == 0ul') } f = op.Invoke( topo, "spl.relational::Functor", inputs=beacon_stream, schemas=[ 'tuple<uint64 seq, rstring category, boolean params, int32 resource_tags, rstring extension>', 'tuple<uint64 seq>' ], params=functor_params, name=None) f.category = 'analytics_category' # output streams fs0 and fs1 fs0 = f.outputs[0] fs1 = f.outputs[1] # output assignments for port 0 f.category = f.output(fs0, f.attribute(beacon_stream, 'category')) f.extension = f.output(fs0, '"extension_attr_val"') # output assignments for port 1 f.seq = f.output(fs1, op.Expression.expression("seq * 10ul")) # test that property access works self.assertEqual(b.category, 'sources_category') self.assertDictEqual(b.params, { 'iterations': 3, 'period': 0.01, 'initDelay': 1.0 }) self.assertSetEqual(b.resource_tags, set()) self.assertEqual(f.category, 'analytics_category') self.assertDictEqual(f.params, functor_params) self.assertSetEqual(f.resource_tags, set()) tester = Tester(topo) tester.tuple_count(fs0, 2) tester.contents(fs0, [{ 'seq': 0, 'category': 'category', 'params': True, 'resource_tags': 42, 'extension': 'extension_attr_val' }, { 'seq': 2, 'category': 'category', 'params': True, 'resource_tags': 42, 'extension': 'extension_attr_val' }], ordered=True) tester.test(self.test_ctxtype, self.test_config)