def test_feedback_loop(self):
topo = Topology()
data = ['A','B', 'A', 'A', 'X', 'C', 'C', 'D', 'A', 'A', 'E']
expected = ['B', 'X', 'C', 'C', 'D', 'A', 'A', 'E']
s = topo.source(data)
s = s.filter(lambda t : time.sleep(1) or True).as_string();
feedback = PendingStream(topo)
df = op.Invoke(topo, 'spl.utility::DynamicFilter',
inputs = [s, feedback.stream],
schemas= [schema.CommonSchema.String])
df.params['key'] = df.attribute(s, 'string')
df.params['addKey'] = df.attribute(feedback.stream, 'string')
delayed_out = op.Map('spl.utility::Delay', df.outputs[0], params={'delay': 0.05}).stream
x = delayed_out.filter(lambda s : s == 'X').map(lambda s : 'A').as_string()
i = topo.source(['B', 'X', 'C', 'D', 'E']).as_string()
x = x.union({i})
feedback.complete(x)
result = delayed_out
result.print()
#streamsx.topology.context.submit('TOOLKIT', topo)
tester = Tester(topo)
tester.contents(result, expected)
tester.test(self.test_ctxtype, self.test_config)
def test_multi_output_ports_punct(self):
"""Operator with multiple output port emits window marker after forwarding the tuple."""
topo = Topology()
streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('testtkpy'))
s = topo.source([9237, -24])
s = s.map(lambda x: (x, ), schema='tuple<int64 v>')
bop = op.Invoke(
topo,
"com.ibm.streamsx.topology.pytest.pyprimitives::MultiOutputPortsPunct",
s,
schemas=['tuple<int64 v1>', 'tuple<int32 v2>', 'tuple<int16 v3>'])
r = bop.outputs
r[0].print(tag='0', write_punctuations=True)
r[1].print(tag='1', write_punctuations=True)
r[2].print(tag='2', write_punctuations=True)
self.tester = Tester(topo)
self.tester.tuple_count(s, 2)
self.tester.punct_count(r[0], 2)
self.tester.punct_count(r[1], 2)
self.tester.punct_count(r[2], 2)
self.tester.contents(r[0], [{'v1': 9237}, {'v1': -24}])
self.tester.contents(r[1], [{'v2': 9237 + 921}, {'v2': -24 + 921}])
self.tester.contents(r[2], [{'v3': 9237 - 407}, {'v3': -24 - 407}])
self.tester.test(self.test_ctxtype, self.test_config)
def test_param_types(self):
ri,tkl = _index_tk()
self.assertEqual(0, ri)
topo = Topology()
streamsx.spl.toolkit.add_toolkit(topo, tkl)
p = {}
p['b'] = False
p['u8'] = streamsx.spl.types.uint8(78)
p['u16'] = streamsx.spl.types.uint16(78)
p['u32'] = streamsx.spl.types.uint32(78)
p['u64'] = streamsx.spl.types.uint64(78)
p['i8'] = streamsx.spl.types.int8(78)
p['i16'] = streamsx.spl.types.int16(78)
p['i32'] = 78
p['i64'] = streamsx.spl.types.int64(78)
p['f32'] = streamsx.spl.types.float32(78)
p['f64'] = 78.9
s = op.Invoke(topo, "testtopo::PT", params = p, name="PY")
if np is not None:
p = {}
p['b'] = np.bool_(False)
p['u8'] = np.uint8(78)
p['u16'] = np.uint16(78)
p['u32'] = np.uint32(78)
p['u64'] = np.uint64(78)
p['i8'] = np.int8(78)
p['i16'] = np.int16(78)
p['i32'] = np.int32(89)
p['i64'] = np.int64(78)
p['f32'] = np.float32(78.03)
p['f64'] = np.float64(78.9)
snp = op.Invoke(topo, "testtopo::PT", params = p, name="Numpy")
cfg = {}
if not 'STREAMS_INSTALL' in os.environ:
cfg[streamsx.topology.context.ConfigParams.SSL_VERIFY] = False
rc = streamsx.topology.context.submit('BUNDLE', topo, cfg)
self.assertEqual(0, rc['return_code'])
os.remove(rc['bundlePath'])
os.remove(rc['jobConfigPath'])
shutil.rmtree(tkl)
def _get_spl_app(cmd_args):
ns, name = cmd_args.main_composite.rsplit('::', 1)
ns += '._spl'
topo = Topology(name=name, namespace=ns)
if cmd_args.toolkits is not None:
for tk_path in cmd_args.toolkits:
tk.add_toolkit(topo, tk_path)
op.Invoke(topo, cmd_args.main_composite)
return (topo, {})
def test_noports(self):
"""Operator with no inputs or outputs"""
topo = Topology()
streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('testtkpy'))
bop = op.Invoke(
topo,
"com.ibm.streamsx.topology.pytest.pyprimitives::NoPorts",
params={
'mn': 'mymetric',
'iv': 89
})
self.tester = Tester(topo)
self.tester.local_check = self._noports_check
self.tester.test(self.test_ctxtype, self.test_config)
def test_flight_path_encounter(self):
print ('\n---------'+str(self))
name = 'test_flight_path_encounter'
topo = Topology(name)
toolkit.add_toolkit(topo, self.geospatial_toolkit_home)
self._index_toolkit(_get_test_tk_path())
toolkit.add_toolkit(topo, _get_test_tk_path())
datagen = op.Invoke(topo, kind='test::GenFlightPathData', schemas=[FlighPathEncounterSchema.EncounterEvents])
planes_stream = datagen.outputs[0]
events = planes_stream.map(geo.FlightPathEncounter(north_latitude=52.6,south_latitude=52.4,west_longitude=13.3,east_longitude=13.5,num_latitude_divs=5,num_longitude_divs=5,search_radius=10000,altitude_search_radius=400,time_search_interval=600000), schema=FlighPathEncounterSchema.EncounterEvents)
dump = op.Invoke(topo, inputs=[events], kind='test::DumpData', schemas=CommonSchema.String)
res = dump.outputs[0]
res.print()
if (("TestDistributed" in str(self)) or ("TestStreamingAnalytics" in str(self))):
tester = Tester(topo)
tester.tuple_count(res, 1, exact=True)
tester.test(self.test_ctxtype, self.test_config, always_collect_logs=True)
else:
# build only
self._build_only(name, topo)
def test_multi_input_ports(self):
"""Operator with three input ports"""
topo = Topology()
streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('testtkpy'))
s0 = topo.source([9054]).map(lambda x: (x, ), schema='tuple<uint64 v>')
s1 = topo.source([345]).map(lambda x: (x, ), schema='tuple<int64 v>')
s2 = topo.source([-953]).map(lambda x: (x, ), schema='tuple<int32 v>')
bop = op.Invoke(
topo,
"com.ibm.streamsx.topology.pytest.pyprimitives::MultiInputPort",
[s0, s1, s2],
name="MIP_OP")
self.tester = Tester(topo)
self.tester.local_check = self._multi_input_port_check
self.tester.test(self.test_ctxtype, self.test_config)
def test_dict_output_ports(self):
"""Operator with multiple output port submitting dict objects."""
topo = Topology()
streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('testtkpy'))
s = topo.source([9237, -24])
s = s.map(lambda x: (x, x * 2, x + 4),
schema='tuple<int64 d, int64 e, int64 f>')
bop = op.Invoke(
topo,
"com.ibm.streamsx.topology.pytest.pyprimitives::DictOutputPorts",
s,
schemas=['tuple<int64 d, int64 e, int64 f>'] * 2)
r = bop.outputs
self.tester = Tester(topo)
self.tester.tuple_count(r[0], 2)
self.tester.tuple_count(r[1], 4)
self.tester.contents(r[0], [{
'd': 9237,
'e': (9237 * 2),
'f': 9237 + 4
}, {
'd': -24,
'e': (-24 * 2),
'f': -24 + 4
}])
self.tester.contents(r[1], [{
'd': 9237 + 7,
'f': (9237 * 2) + 777,
'e': 9237 + 4 + 77
}, {
'd': 9237,
'e': (9237 * 2),
'f': 9237 + 4
}, {
'd': -24 + 7,
'f': (-24 * 2) + 777,
'e': -24 + 4 + 77
}, {
'd': -24,
'e': (-24 * 2),
'f': -24 + 4
}])
self.tester.test(self.test_ctxtype, self.test_config)
def test_region_match(self):
print ('\n---------'+str(self))
name = 'test_region_match'
topo = Topology(name)
toolkit.add_toolkit(topo, self.geospatial_toolkit_home)
self._index_toolkit(_get_test_tk_path())
toolkit.add_toolkit(topo, _get_test_tk_path())
datagen = op.Invoke(topo, kind='test::GenRegionData', schemas=[RegionMatchSchema.Devices,RegionMatchSchema.Regions])
device_stream = datagen.outputs[0]
region_stream = datagen.outputs[1]
res = geo.region_match(stream=device_stream, region_stream=region_stream)
res.print()
if (("TestDistributed" in str(self)) or ("TestStreamingAnalytics" in str(self))):
tester = Tester(topo)
tester.tuple_count(res, 4, exact=True)
tester.test(self.test_ctxtype, self.test_config, always_collect_logs=True)
else:
# build only
self._build_only(name, topo)
def test_multi_output_ports(self):
"""Operator with multiple output port."""
topo = Topology()
streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('testtkpy'))
s = topo.source([9237, -24])
s = s.map(lambda x: (x, ), schema='tuple<int64 v>')
bop = op.Invoke(
topo,
"com.ibm.streamsx.topology.pytest.pyprimitives::MultiOutputPorts",
s,
schemas=['tuple<int64 v1>', 'tuple<int32 v2>', 'tuple<int16 v3>'])
r = bop.outputs
self.tester = Tester(topo)
self.tester.tuple_count(s, 2)
self.tester.contents(r[0], [{'v1': 9237}, {'v1': -24}])
self.tester.contents(r[1], [{'v2': 9237 + 921}, {'v2': -24 + 921}])
self.tester.contents(r[2], [{'v3': 9237 - 407}, {'v3': -24 - 407}])
self.tester.test(self.test_ctxtype, self.test_config)
def test_attributes_category_params_resource_tags(self):
topo = Topology()
b = op.Source(
topo,
"spl.utility::Beacon",
'tuple<uint64 seq, rstring category, boolean params, int32 resource_tags>',
params={'iterations': 3})
# update operator parameters
b.params['period'] = 0.01
b.params.update({'initDelay': 1.0})
# the SPL operator's category
b.category = 'sources_category'
# output assignments:
b.seq = b.output('IterationCount()')
b.category = b.output('"category"') # an rstring literal
b.params = b.output('true')
b.resource_tags = b.output(spltypes.int32(42))
beacon_stream = b.stream
functor_params = {
'filter': op.Expression.expression('seq % 2ul == 0ul')
}
f = op.Invoke(
topo,
"spl.relational::Functor",
inputs=beacon_stream,
schemas=[
'tuple<uint64 seq, rstring category, boolean params, int32 resource_tags, rstring extension>',
'tuple<uint64 seq>'
],
params=functor_params,
name=None)
f.category = 'analytics_category'
# output streams fs0 and fs1
fs0 = f.outputs[0]
fs1 = f.outputs[1]
# output assignments for port 0
f.category = f.output(fs0, f.attribute(beacon_stream, 'category'))
f.extension = f.output(fs0, '"extension_attr_val"')
# output assignments for port 1
f.seq = f.output(fs1, op.Expression.expression("seq * 10ul"))
# test that property access works
self.assertEqual(b.category, 'sources_category')
self.assertDictEqual(b.params, {
'iterations': 3,
'period': 0.01,
'initDelay': 1.0
})
self.assertSetEqual(b.resource_tags, set())
self.assertEqual(f.category, 'analytics_category')
self.assertDictEqual(f.params, functor_params)
self.assertSetEqual(f.resource_tags, set())
tester = Tester(topo)
tester.tuple_count(fs0, 2)
tester.contents(fs0, [{
'seq': 0,
'category': 'category',
'params': True,
'resource_tags': 42,
'extension': 'extension_attr_val'
}, {
'seq': 2,
'category': 'category',
'params': True,
'resource_tags': 42,
'extension': 'extension_attr_val'
}],
ordered=True)
tester.test(self.test_ctxtype, self.test_config)
