def _test_punct_file(self,
topo,
s,
test_name,
expected_content,
expected_tuple_count,
expected_punct_count=None):
s = s.map(lambda x: (x, ), schema='tuple<int32 z>')
op_params = {
'file': 'punct_file_' + test_name,
'writePunctuations': True,
'flushOnPunctuation': True
}
op.Sink("spl.adapter::FileSink", s, params=op_params)
# Copy the config, since it's shared across all tests, and not every test needs a data
# directory.
cfg = self.test_config.copy()
jc = JobConfig(data_directory=os.getcwd())
jc.add(cfg)
tester = Tester(topo)
tester.tuple_count(s, expected_tuple_count)
if expected_punct_count is not None:
tester.punct_count(s, expected_punct_count)
tester.test(self.test_ctxtype, cfg)
path = os.path.join(os.getcwd(), 'punct_file_' + test_name)
# Validate the contents of the file.
with open(path, 'r') as f:
file_contents = f.read()
self.assertEqual(expected_content, file_contents)
os.remove(path)
def test_sliding_time_stv_no_default(self):
topo = Topology()
b = op.Source(topo,
"spl.utility::Beacon",
'tuple<uint64 seq>',
params={'iterations': 12})
b.seq = b.output('IterationCount()')
s = b.stream
wtime = topo.create_submission_parameter(name='secs', type_=int)
window = s.lastSeconds(wtime).trigger(1)
agg = op.Map('spl.relational::Aggregate',
window,
schema='tuple<uint64 sum, uint64 max>')
agg.sum = agg.output('Sum(seq)')
agg.max = agg.output('Max(seq)')
jc = JobConfig()
jc.submission_parameters['secs'] = 2
jc.add(self.test_config)
tester = Tester(topo)
tester.tuple_count(agg.stream, 12)
tester.test(self.test_ctxtype, self.test_config)
def test_sliding_count_stv_no_default(self):
step = 1
topo = Topology()
b = op.Source(topo,
"spl.utility::Beacon",
'tuple<uint64 seq>',
params={'iterations': 12})
b.seq = b.output('IterationCount()')
s = b.stream
count = topo.create_submission_parameter('count', type_=int)
window = s.last(count).trigger(step)
agg = op.Map('spl.relational::Aggregate',
window,
schema='tuple<uint64 sum, uint64 max>')
agg.sum = agg.output('Sum(seq)')
agg.max = agg.output('Max(seq)')
expected = []
for i in range(4 + step - 2, 12, step):
expected.append({'sum': sum(range(i - 3, i + 1)), 'max': i})
jc = JobConfig()
jc.submission_parameters['count'] = 4
jc.add(self.test_config)
tester = Tester(topo)
tester.contents(agg.stream, expected)
tester.test(self.test_ctxtype, self.test_config)
def _multi_injection(self, raw_overlay=None):
topo = Topology()
streamsx.spl.toolkit.add_toolkit(topo, TestEmInject._TK)
context = _rand_path()
name = _rand_path()
job_name = _rand_path()
s1 = endpoint.inject(topo, name=name+'N1', context=context+'C1', monitor=self._monitor)
s2 = endpoint.inject(topo, name=name+'N2', context=context+'C2', monitor=self._monitor)
jc = JobConfig()
self._job_name = _rand_path()
jc.job_name = self._job_name
jc.add(self.test_config)
# Force separate PEs for the inject operators
if raw_overlay:
jc.raw_overlay = raw_overlay
else:
s1.colocate(s2)
self._path = '/' + context + 'C1/' + name + 'N1/ports/output/0/inject'
self.tester = Tester(topo)
self.tester.local_check = self._multi_inject
self.tester.contents(s1, [{'seq1':i} for i in range(self.N)])
self.tester.contents(s2, [{'seq2':i} for i in range(self.N)])
self.tester.test(self.test_ctxtype, self.test_config)
self._check_no_endpoint()
def test_image_name_image_tag(self):
topo = Topology("test_image_name_image_tag")
heartbeat = topo.source(lambda: itertools.count())
heartbeat.print()
image_name = 'py-tst'
image_tag = 'v1.0'
cfg = {ConfigParams.SSL_VERIFY: False}
jc = JobConfig()
jc.raw_overlay = {
'edgeConfig': {
'imageName': image_name,
'imageTag': image_tag,
'pipPackages': ['pandas', 'numpy'],
'rpms': ['atlas-devel']
}
}
jc.add(cfg)
try:
submission_result = submit(ContextTypes.EDGE, topo.graph, cfg)
print(str(submission_result))
self.assertTrue(submission_result is not None)
self.assertTrue(self._is_not_blank(submission_result.image))
self.assertTrue(self._is_not_blank(submission_result.imageDigest))
self.assertTrue(image_name in submission_result.image)
self.assertTrue(image_tag in submission_result.image)
except RuntimeError as e:
print(str(e))
self.skipTest("Skip test, CPD does not support EDGE.")
def test_topo_types_explicit_set(self):
topo = Topology()
sp_str = topo.create_submission_parameter('sp_str', type_=str)
sp_int = topo.create_submission_parameter('sp_int', type_=int)
sp_float = topo.create_submission_parameter('sp_float', type_=float)
sp_bool = topo.create_submission_parameter('sp_bool', type_=bool)
s = topo.source(range(17))
s = s.filter(
lambda v: isinstance(sp_str(), str) and sp_str() == 'SeeYa')
s = s.filter(lambda v: isinstance(sp_int(), int) and sp_int() == 10)
s = s.filter(
lambda v: isinstance(sp_float(), float) and sp_float() == -0.5)
s = s.filter(
lambda v: isinstance(sp_bool(), bool) and sp_bool() is True)
jc = JobConfig()
jc.submission_parameters['sp_str'] = 'SeeYa'
jc.submission_parameters['sp_int'] = 10
jc.submission_parameters['sp_float'] = -0.5
jc.submission_parameters['sp_bool'] = True
jc.add(self.test_config)
tester = Tester(topo)
tester.tuple_count(s, 17)
tester.test(self.test_ctxtype, self.test_config)
def test_colocation(self):
topo = Topology()
s1 = topo.source([], name='S1')
s2 = topo.source([], name='S2')
s1f = s1.filter(lambda x: True, name='S1F')
s2f = s2.filter(lambda x: True, name='S2F')
s1e = s1f.for_each(lambda x: None, name='S1E')
s2e = s2f.for_each(lambda x: None, name='S2E')
# S1 -> S1F -> S1E
# S2 -> S2F -> S2E
s2e.colocate(s1)
# S1(X) -> S1F -> S1E
# S2 -> S2F -> S2E(X)
s2f.colocate([s2, s1f])
# S1(X) -> S1F(Y) -> S1E
# S2(Y) -> S2F(Y) -> S2E(X)
s1f.colocate(s1e)
# S1(X) -> S1F(Y) -> S1E(Y)
# S2(Y) -> S2F(Y) -> S2E(X)
beacon = Source(topo,
"spl.utility::Beacon",
'tuple<uint64 seq>',
params={
'period': 0.02,
'iterations': 100
},
name='BeaconColo')
beacon.seq = beacon.output('IterationCount()')
beacon.colocate(s2)
# Now a colocation independent of the other sets
s3 = topo.source([], name='S3')
s3f = s3.filter(lambda x: True, name='S3F')
s3e = s3f.for_each(lambda x: None, name='S3E')
s3.colocate(s3e)
# and then join to an existing set
#print("MERGE!!!")
s3.colocate(s1)
self.tester = Tester(topo)
self.tester.local_check = self.check_colocations
jc = JobConfig()
jc.raw_overlay = {'deploymentConfig': {'fusionScheme': 'legacy'}}
jc.add(self.test_config)
sr = self.tester.test(self.test_ctxtype, self.test_config)
def test_colocation(self):
topo = Topology()
s1 = topo.source([], name='S1')
s2 = topo.source([], name='S2')
s1f = s1.filter(lambda x : True, name='S1F')
s2f = s2.filter(lambda x : True, name='S2F')
s1e = s1f.for_each(lambda x : None, name='S1E')
s2e = s2f.for_each(lambda x : None, name='S2E')
# S1 -> S1F -> S1E
# S2 -> S2F -> S2E
s2e.colocate(s1)
# S1(X) -> S1F -> S1E
# S2 -> S2F -> S2E(X)
s2f.colocate([s2,s1f])
# S1(X) -> S1F(Y) -> S1E
# S2(Y) -> S2F(Y) -> S2E(X)
s1f.colocate(s1e)
# S1(X) -> S1F(Y) -> S1E(Y)
# S2(Y) -> S2F(Y) -> S2E(X)
beacon = Source(topo, "spl.utility::Beacon",
'tuple<uint64 seq>',
params = {'period': 0.02, 'iterations':100},
name = 'BeaconColo')
beacon.seq = beacon.output('IterationCount()')
beacon.colocate(s2)
# Now a colocation independent of the other sets
s3 = topo.source([], name='S3')
s3f = s3.filter(lambda x : True, name='S3F')
s3e = s3f.for_each(lambda x : None, name='S3E')
s3.colocate(s3e)
# and then join to an existing set
#print("MERGE!!!")
s3.colocate(s1)
self.tester = Tester(topo)
self.tester.local_check = self.check_colocations
jc = JobConfig()
jc.raw_overlay = {'deploymentConfig': {'fusionScheme':'legacy'}}
jc.add(self.test_config)
sr = self.tester.test(self.test_ctxtype, self.test_config)
def test_run_with_name_overwrite(self):
topo = Topology()
topo.source(['a'])
name_not_used = 'JOB:' + str(uuid.uuid4())
name = 'JOB:' + str(uuid.uuid4())
cfg = {}
jc = JobConfig(job_name=name_not_used)
jc.add(cfg)
job, sr = run(topo, config=cfg, job_name=name, verify=self._verify)
self.assertIs(job, sr.job)
self.assertEqual(name, job.name)
job.cancel()
def test_spl(self):
"""
Test passing as an SPL parameter.
"""
N = 22
G = 'hey'
t = ''.join(random.choice('0123456789abcdef') for x in range(20))
topic = 'topology/test/python/' + t
topo = Topology()
spTopic = topo.create_submission_parameter('mytopic')
spGreet = topo.create_submission_parameter('greeting')
self.assertIsNone(spTopic())
self.assertIsNone(spGreet())
sch = StreamSchema('tuple<uint64 seq, rstring s>')
b = op.Source(topo,
"spl.utility::Beacon",
sch,
params={
'initDelay': 10.0,
'period': 0.02,
'iterations': N
})
b.seq = b.output('IterationCount()')
b.s = b.output(spGreet)
p = op.Sink("com.ibm.streamsx.topology.topic::Publish",
b.stream,
params={'topic': topic})
s = op.Source(topo,
"com.ibm.streamsx.topology.topic::Subscribe",
sch,
params={
'streamType': sch,
'topic': spTopic
})
jc = JobConfig()
jc.submission_parameters['mytopic'] = topic
jc.submission_parameters['greeting'] = G
jc.add(self.test_config)
tester = Tester(topo)
tester.tuple_count(s.stream, N)
#tester.run_for(300)
tester.contents(s.stream, [{'seq': i, 's': G} for i in range(N)])
tester.test(self.test_ctxtype, self.test_config)
def test_topo_with_def_and_type(self):
topo = Topology()
s = topo.source(range(38))
lower = topo.create_submission_parameter('lower', default=0)
upper = topo.create_submission_parameter('upper', default=30)
s = s.filter(lambda v: v < lower() or v > upper())
jc = JobConfig()
jc.submission_parameters['lower'] = 5
jc.add(self.test_config)
tester = Tester(topo)
tester.contents(s, [0, 1, 2, 3, 4, 31, 32, 33, 34, 35, 36, 37])
tester.test(self.test_ctxtype, self.test_config)
def test_topo_with_def_and_type(self):
topo = Topology()
s = topo.source(range(38))
lower = topo.create_submission_parameter('lower', default=0)
upper = topo.create_submission_parameter('upper', default=30)
s = s.filter(lambda v: v < lower() or v > upper())
jc = JobConfig()
jc.submission_parameters['lower'] = 5
jc.add(self.test_config)
tester = Tester(topo)
tester.contents(s, [0,1,2,3,4,31,32,33,34,35,36,37])
tester.test(self.test_ctxtype, self.test_config)
def test_UnicodeJobName(self):
""" Test unicode topo names
"""
job_name = '你好世界'
topo = Topology()
jc = JobConfig(job_name=job_name)
hw = topo.source(["Hello", "Tester"])
tester = Tester(topo)
tester.contents(hw, ["Hello", "Tester"])
jc.add(self.test_config)
self.assertIs(jc, self.test_config[ConfigParams.JOB_CONFIG])
sr = tester.test(self.test_ctxtype, self.test_config)
self.assertEqual(job_name, tester.result['submission_result']['name'])
def test_from_topology(self):
topo = Topology('SabTest', namespace='mynamespace')
s = topo.source([1, 2])
es = s.for_each(lambda x: None)
cfg = {}
jc = JobConfig(job_name='ABCD', job_group='XXG', preload=True)
jc.add(cfg)
bb = streamsx.topology.context.submit('BUNDLE', topo, cfg)
self.assertIn('bundlePath', bb)
self.assertIn('jobConfigPath', bb)
with open(bb['jobConfigPath']) as json_data:
jct = JobConfig.from_overlays(json.load(json_data))
self.assertEqual(jc.job_name, jct.job_name)
self.assertEqual(jc.job_group, jct.job_group)
self.assertEqual(jc.preload, jct.preload)
os.remove(bb['bundlePath'])
os.remove(bb['jobConfigPath'])
def test_from_topology(self):
topo = Topology('SabTest', namespace='mynamespace')
s = topo.source([1,2])
es = s.for_each(lambda x : None)
cfg = {}
jc = JobConfig(job_name='ABCD', job_group='XXG', preload=True)
jc.add(cfg)
bb = streamsx.topology.context.submit('BUNDLE', topo, cfg)
self.assertIn('bundlePath', bb)
self.assertIn('jobConfigPath', bb)
with open(bb['jobConfigPath']) as json_data:
jct = JobConfig.from_overlays(json.load(json_data))
self.assertEqual(jc.job_name, jct.job_name)
self.assertEqual(jc.job_group, jct.job_group)
self.assertEqual(jc.preload, jct.preload)
os.remove(bb['bundlePath'])
os.remove(bb['jobConfigPath'])
def test_UnicodeJobName(self):
""" Test unicode topo names
"""
job_name = '你好世界'
topo = Topology()
jc = JobConfig(job_name=job_name)
# When tracing is info some extra code is invoked
# to trace all Python packages. Ensure it is exercised.
jc.tracing = 'info'
hw = topo.source(["Hello", "Tester"])
tester = Tester(topo)
tester.contents(hw, ["Hello", "Tester"])
jc.add(self.test_config)
self.assertIs(jc, self.test_config[ConfigParams.JOB_CONFIG])
sr = tester.test(self.test_ctxtype, self.test_config)
self.assertEqual(job_name, tester.result['submission_result']['name'])
def test_UnicodeJobName(self):
""" Test unicode topo names
"""
job_name = '你好世界'
topo = Topology()
jc = JobConfig(job_name=job_name)
# When tracing is info some extra code is invoked
# to trace all Python packages. Ensure it is exercised.
jc.tracing = 'info'
hw = topo.source(["Hello", "Tester"])
tester = Tester(topo)
tester.contents(hw, ["Hello", "Tester"])
jc.add(self.test_config)
self.assertIs(jc, self.test_config[ConfigParams.JOB_CONFIG])
sr = tester.test(self.test_ctxtype, self.test_config)
self.assertEqual(job_name, tester.result['submission_result']['name'])
def test_parallel(self):
topo = Topology()
sp_w1 = topo.create_submission_parameter('w1', type_=int)
sp_w2 = topo.create_submission_parameter('w2', type_=int)
s = topo.source(range(67)).set_parallel(sp_w1)
s = s.filter(lambda v: v % sp_w1() == 0)
s = s.end_parallel()
s = s.parallel(width=sp_w2)
s = s.filter(lambda v: v % sp_w2() == 0)
s = s.end_parallel()
jc = JobConfig()
jc.submission_parameters['w1'] = 3
jc.submission_parameters['w2'] = 5
jc.add(self.test_config)
tester = Tester(topo)
tester.contents(s, [0, 15, 30, 45, 60] * 3, ordered=False)
tester.test(self.test_ctxtype, self.test_config)
def test_parallel(self):
topo = Topology()
sp_w1 = topo.create_submission_parameter('w1', type_=int)
sp_w2 = topo.create_submission_parameter('w2', type_=int)
s = topo.source(range(67)).set_parallel(sp_w1)
s = s.filter(lambda v : v % sp_w1() == 0)
s = s.end_parallel()
s = s.parallel(width=sp_w2)
s = s.filter(lambda v : v % sp_w2() == 0)
s = s.end_parallel()
jc = JobConfig()
jc.submission_parameters['w1'] = 3
jc.submission_parameters['w2'] = 5
jc.add(self.test_config)
tester = Tester(topo)
tester.contents(s,[0,15,30,45,60]*3, ordered=False)
tester.test(self.test_ctxtype, self.test_config)
def test_BasicTimeCountWindow_stv_no_default(self):
# Ensure tuples are evicted no later than (window_span*tolerance + window_span)
tolerance = 0.20
window_span = 2.0
max_evict = window_span * tolerance + window_span
topo = Topology()
s = topo.source(TimeCounter(iterations=100, period=0.1))
s = s.map(lambda x: (x, time.time()))
wtime = topo.create_submission_parameter('secs', type_=int)
s = s.lastSeconds(wtime).trigger(20).aggregate(
TupleTimespanCheck(max_evict))
jc = JobConfig()
jc.submission_parameters['secs'] = 2
jc.add(self.test_config)
tester = Tester(topo)
tester.tuple_check(s, lambda val: val)
tester.test(self.test_ctxtype, self.test_config)
def test_topo(self):
topo = Topology()
s = topo.source(range(38))
lower = topo.create_submission_parameter('lower')
upper = topo.create_submission_parameter('upper')
addin = topo.create_submission_parameter('addin')
s = s.filter(lambda v: v < int(lower()) or v > int(upper()))
m = s.filter(lambda v : v < 3)
m = m.map(AddIt(addin))
jc = JobConfig()
jc.submission_parameters['lower'] = 7
jc.submission_parameters['upper'] = 33
jc.submission_parameters['addin'] = 'Yeah!'
jc.add(self.test_config)
tester = Tester(topo)
tester.contents(s, [0,1,2,3,4,5,6,34,35,36,37])
tester.contents(m, ['0-Yeah!','1-Yeah!','2-Yeah!'])
tester.test(self.test_ctxtype, self.test_config)
def test_topo_types_explicit_set(self):
topo = Topology()
sp_str = topo.create_submission_parameter('sp_str', type_=str)
sp_int = topo.create_submission_parameter('sp_int', type_=int)
sp_float = topo.create_submission_parameter('sp_float', type_=float)
sp_bool = topo.create_submission_parameter('sp_bool', type_=bool)
s = topo.source(range(17))
s = s.filter(lambda v : isinstance(sp_str(), str) and sp_str() == 'SeeYa')
s = s.filter(lambda v : isinstance(sp_int(), int) and sp_int() == 10)
s = s.filter(lambda v : isinstance(sp_float(), float) and sp_float() == -0.5)
s = s.filter(lambda v : isinstance(sp_bool(), bool) and sp_bool() is True)
jc = JobConfig()
jc.submission_parameters['sp_str'] = 'SeeYa'
jc.submission_parameters['sp_int'] = 10
jc.submission_parameters['sp_float'] = -0.5
jc.submission_parameters['sp_bool'] = True
jc.add(self.test_config)
tester = Tester(topo)
tester.tuple_count(s, 17)
tester.test(self.test_ctxtype, self.test_config)
def test_topo(self):
topo = Topology()
s = topo.source(range(38))
lower = topo.create_submission_parameter('lower')
upper = topo.create_submission_parameter('upper')
addin = topo.create_submission_parameter('addin')
s = s.filter(lambda v: v < int(lower()) or v > int(upper()))
m = s.filter(lambda v: v < 3)
m = m.map(AddIt(addin))
jc = JobConfig()
jc.submission_parameters['lower'] = 7
jc.submission_parameters['upper'] = 33
jc.submission_parameters['addin'] = 'Yeah!'
jc.add(self.test_config)
tester = Tester(topo)
tester.contents(s, [0, 1, 2, 3, 4, 5, 6, 34, 35, 36, 37])
tester.contents(m, ['0-Yeah!', '1-Yeah!', '2-Yeah!'])
tester.test(self.test_ctxtype, self.test_config)
def test_spl(self):
"""
Test passing as an SPL parameter.
"""
N=22
G='hey'
t = ''.join(random.choice('0123456789abcdef') for x in range(20))
topic = 'topology/test/python/' + t
topo = Topology()
spTopic = topo.create_submission_parameter('mytopic')
spGreet = topo.create_submission_parameter('greeting')
self.assertIsNone(spTopic())
self.assertIsNone(spGreet())
sch = StreamSchema('tuple<uint64 seq, rstring s>')
b = op.Source(topo, "spl.utility::Beacon", sch,
params = {'initDelay': 10.0, 'period': 0.02, 'iterations':N})
b.seq = b.output('IterationCount()')
b.s = b.output(spGreet)
p = op.Sink("com.ibm.streamsx.topology.topic::Publish", b.stream,
params={'topic': topic})
s = op.Source(topo, "com.ibm.streamsx.topology.topic::Subscribe", sch,
params = {'streamType': sch, 'topic': spTopic})
jc = JobConfig()
jc.submission_parameters['mytopic'] = topic
jc.submission_parameters['greeting'] = G
jc.add(self.test_config)
tester = Tester(topo)
tester.tuple_count(s.stream, N)
#tester.run_for(300)
tester.contents(s.stream, [{'seq':i, 's':G} for i in range(N)])
tester.test(self.test_ctxtype, self.test_config)
from streamsx.mqtt import MQTTSource, MQTTSink
import streamsx.topology.context as context
from streamsx.topology.context import ContextTypes, JobConfig
from streamsx.topology.topology import Topology
from streamsx.topology.schema import CommonSchema
mqtt_server_uri = 'tcp://172.16.33.10:1883'
topology = Topology()
s = 'Each character will be an MQTT message'
data = topology.source([c for c in s]).as_string()
# publish to MQTT
data.for_each(MQTTSink(server_uri=mqtt_server_uri, topic='topic'),
name='MQTTpublish')
# subscribe for data and print to stdout
received = topology.source(
MQTTSource(mqtt_server_uri, schema=CommonSchema.String, topics='topic'))
received.print()
job_config = JobConfig(job_name='MQTTBasic', tracing='info')
submission_config = {}
# add the job_config into the submission config:
job_config.add(submission_config)
print('submission_config: ' + str(submission_config))
context.submit(ContextTypes.DISTRIBUTED, topology, config=submission_config)
# the Streams Job keeps running and must be cancelled manually
