def test_mt(self):
topo = Topology()
N = 1000
streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('testtkpy'))
b1 = op.Source(topo, "spl.utility::Beacon", schema.StreamSchema('tuple<int32 f>').as_tuple(), params={'iterations':N})
b1.f = b1.output('(int32)IterationCount()')
b2 = op.Source(topo, "spl.utility::Beacon", schema.StreamSchema('tuple<int32 f>').as_tuple(), params={'iterations':N})
b2.f = b2.output(str(N) + ' + (int32)IterationCount()')
b3 = op.Source(topo, "spl.utility::Beacon", schema.StreamSchema('tuple<int32 f>').as_tuple(), params={'iterations':N})
b3.f = b3.output(str(2*N) + ' + (int32)IterationCount()')
s1 = b1.stream.low_latency()
s2 = b2.stream.low_latency()
s3 = b3.stream.low_latency()
s = s1.union({s2, s3})
f = op.Map("com.ibm.streamsx.topology.pytest.mt::MTFilter", s)
m = op.Map("com.ibm.streamsx.topology.pytest.mt::MTMap", f.stream)
op.Sink("com.ibm.streamsx.topology.pytest.mt::MTForEach", f.stream)
cr = m.stream.flat_map()
tester = Tester(topo)
tester.tuple_count(m.stream, 3*N)
tester.contents(cr, range(3*N), ordered=False)
tester.test(self.test_ctxtype, self.test_config)
def test_good(self):
for dt in SPL_TYPES:
if dt in GOOD_DATA:
data = GOOD_DATA[dt]
topo = Topology()
schema = StreamSchema('tuple<' + dt + ' a>')
s = topo.source(data)
c = s.map(lambda x : (x,), schema=schema)
#c.print(tag=dt)
if dt.startswith('float'):
expected = [{'a':float(d)} for d in data]
elif dt.startswith('int'):
expected = [{'a':int(d)} for d in data]
elif dt == 'decimal32':
ctx = decimal.Context(prec=7, rounding=decimal.ROUND_HALF_EVEN)
expected = [{'a':decimal.Decimal(str(d)).normalize(ctx)} for d in data]
elif dt == 'decimal64':
ctx = decimal.Context(prec=16, rounding=decimal.ROUND_HALF_EVEN)
expected = [{'a':decimal.Decimal(str(d)).normalize(ctx)} for d in data]
elif dt == 'decimal128':
ctx = decimal.Context(prec=34, rounding=decimal.ROUND_HALF_EVEN)
expected = [{'a':decimal.Decimal(str(d)).normalize(ctx)} for d in data]
elif dt.startswith('complex'):
expected = [{'a':complex(d)} for d in data]
elif dt == 'timestamp':
expected = [{'a': d if isinstance(d, Timestamp) else Timestamp.from_datetime(d)} for d in data]
tester = Tester(topo)
tester.tuple_count(c, len(data))
tester.contents(c, expected)
tester.test(self.test_ctxtype, self.test_config)
def test_app_log(self):
topo = Topology()
s = topo.source(['logmsg1', 'logmsg2你好'])
s.for_each(_log_msg)
tester = Tester(topo)
tester.tuple_count(s, 2)
tester.test(self.test_ctxtype, self.test_config)
class TestSources(unittest.TestCase):
"""
Test @spl.source decorated operators
"""
@classmethod
def setUpClass(cls):
"""Extract Python operators in toolkit"""
streamsx.scripts.extract.main(['-i', '../testtkpy', '--make-toolkit'])
def setUp(self):
Tester.setup_standalone(self)
def test_class_source(self):
count = 43
topo = Topology()
streamsx.spl.toolkit.add_toolkit(topo, '../testtkpy')
bop = op.Source(topo, "com.ibm.streamsx.topology.pysamples.sources::Range", schema.StreamSchema('tuple<int64 c>').as_tuple(), params={'count':count})
r = bop.stream
self.tester = Tester(topo)
self.tester.tuple_count(r, count)
self.tester.contents(r, list(zip(range(count))))
self.tester.test(self.test_ctxtype, self.test_config)
def test_fn_source(self):
count = 37
topo = Topology()
streamsx.spl.toolkit.add_toolkit(topo, '../testtkpy')
bop = op.Source(topo, "com.ibm.streamsx.topology.pysamples.sources::Range37", schema.StreamSchema('tuple<int64 c>').as_tuple())
r = bop.stream
self.tester = Tester(topo)
self.tester.tuple_count(r, count)
self.tester.contents(r, list(zip(range(count))))
self.tester.test(self.test_ctxtype, self.test_config)
def test_at_least(self):
""" Test the at least tuple count.
"""
topo = Topology()
s = topo.source(rands)
tester = Tester(topo)
tester.tuple_count(s, 100, exact=False)
tester.test(self.test_ctxtype, self.test_config)
def test_no_tuples(self):
""" Test exact count with zero tuples.
"""
topo = Topology()
s = topo.source([])
tester = Tester(topo)
tester.tuple_count(s, 0)
tester.test(self.test_ctxtype, self.test_config)
def test_app_log_direct(self):
topo = Topology()
s = topo.source([40,30,20,99])
at = s.filter(lambda x : x != 99)
at.for_each(_log_msg_direct)
tester = Tester(topo)
tester.tuple_count(s, 4)
tester.test(self.test_ctxtype, self.test_config)
def test_as_tuple_for_each(self):
topo = Topology()
st = self._create_stream(topo)
st.for_each(check_is_tuple_for_each(self.is_named()))
tester = Tester(topo)
tester.tuple_count(st, 3)
tester.test(self.test_ctxtype, self.test_config)
def test_count_bad(self):
N=10
topo = Topology()
s = topo.source(range(N))
tester = Tester(topo)
tester.tuple_count(s, N+1)
ok = tester.test(self.test_ctxtype, self.test_config, assert_on_fail=False)
self.assertFalse(ok)
def test_as_tuple_filter(self):
topo = Topology()
s = self._create_stream(topo)
s = s.filter(check_is_tuple_filter(self.is_named()))
tester = Tester(topo)
tester.tuple_count(s, 2)
tester.test(self.test_ctxtype, self.test_config)
def test_at_least_no_tuples(self):
""" Test at least count with zero tuples.
(kind of a pointless condition, always true).
"""
topo = Topology()
s = topo.source([])
tester = Tester(topo)
tester.tuple_count(s, 0, exact=False)
tester.test(self.test_ctxtype, self.test_config)
def test_batch_count(self):
topo = Topology()
s = topo.source(lambda : range(20))
b = s.batch(4)
r = b.aggregate(lambda items : sum(items))
tester = Tester(topo)
tester.contents(r, [0+1+2+3,4+5+6+7,8+9+10+11,12+13+14+15,16+17+18+19])
tester.tuple_count(r, 5)
tester.test(self.test_ctxtype, self.test_config)
def test_count_bad_conflicting(self):
N=10
topo = Topology()
s = topo.source(range(N))
tester = Tester(topo)
# Add one that fails and one that never becomes valid
tester.tuple_count(s.map(), int(N/2))
tester.tuple_count(s, N+1)
ok = tester.test(self.test_ctxtype, self.test_config, assert_on_fail=False)
self.assertFalse(ok)
def test_TopologySourceFn(self):
topo = Topology('test_TopologySourceFn')
hw = topo.source(s4)
tester = Tester(topo)
tester.contents(hw, s4())
tester.tuple_count(hw, len(s4()))
self.test_config['topology.keepArtifacts'] = True
tester.test(self.test_ctxtype, self.test_config)
self.result = tester.result['submission_result']
verifyArtifacts(self)
class TestPubSub(unittest.TestCase):
""" Test publish/subscribe
"""
def setUp(self):
Tester.setup_distributed(self)
def _check_topics(self):
ins = self.tester.streams_connection.get_instance(self.tester.submission_result['instanceId'])
topics = ins.get_published_topics()
# Don't assume this is the only app running so
# other topics may exist
sts = 0
stp = 0
for pt in topics:
if self.topic_spl == pt.topic:
sts += 1
if pt.schema is not None:
self.assertEqual(StSc('tuple<uint64 seq>'), pt.schema)
elif self.topic_python == pt.topic:
if pt.schema is not None:
self.assertEqual(CmnSc.Python.value, pt.schema)
stp += 1
self.assertEqual(1, sts)
self.assertEqual(1, stp)
def test_published_topics(self):
"""Test a published stream is available through get_published_topics.
"""
tspl = ''.join(random.choice('0123456789abcdef') for x in range(20))
tpy = ''.join(random.choice('0123456789abcdef') for x in range(20))
self.topic_spl = 'topology/test/python/' + tspl
self.topic_python = 'topology/test/python/' + tpy
self.assertNotEqual(self.topic_spl, self.topic_python)
topo = Topology()
s = op.Source(topo, "spl.utility::Beacon",
'tuple<uint64 seq>',
params = {'period': 0.02})
s.seq = s.output('IterationCount()')
s.stream.publish(topic=self.topic_spl)
s = s.stream.map(lambda x : x)
s.publish(topic=self.topic_python)
# Publish twice to ensure its only listed once
s = s.filter(lambda x : True)
s.publish(topic=self.topic_python)
self.tester = Tester(topo)
self.tester.local_check = self._check_topics
self.tester.tuple_count(s, 100, exact=False)
self.tester.test(self.test_ctxtype, self.test_config)
def test_bad_pe(self):
"""Test a failure in a PE is caught as a test failure"""
topo = Topology()
s = topo.source(rands)
# intentional addition of a string with an int
# to cause a PE failure
s = s.map(lambda t : t + 'a string')
tester = Tester(topo)
tester.tuple_count(s, 0, exact=False)
tp = tester.test(self.test_ctxtype, self.test_config, assert_on_fail=False)
self.assertFalse(tp)
def test_map_foreach(self):
topo = Topology()
topo.checkpoint_period = timedelta(seconds=1)
streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('testtkpy'))
timeCounter = op.Source(topo, "com.ibm.streamsx.topology.pytest.checkpoint::TimeCounter", schema.StreamSchema('tuple<int32 f>').as_tuple(), params={'iterations':30,'period':0.1})
fizzbuzz = op.Map("com.ibm.streamsx.topology.pytest.checkpoint::FizzBuzzMap", timeCounter.stream, schema.StreamSchema('tuple<int32 f, rstring c>').as_tuple())
verify = op.Sink("com.ibm.streamsx.topology.pytest.checkpoint::Verify", fizzbuzz.stream)
s = fizzbuzz.stream
tester = Tester(topo)
tester.tuple_count(s, 30)
tester.test(self.test_ctxtype, self.test_config)
def test_checker(self):
""" Test the per-tuple checker.
"""
topo = Topology()
s = topo.source(rands)
s = s.filter(lambda r : r > 0.8)
s = s.map(lambda r : r + 7.0 )
tester = Tester(topo)
tester.tuple_count(s, 200, exact=False)
tester.tuple_check(s, lambda r : r > 7.8)
tester.test(self.test_ctxtype, self.test_config)
def test_at_least(self):
""" Test the at least tuple count.
"""
if self.test_ctxtype == context.ContextTypes.STANDALONE:
return unittest.skip("Standalone tests must complete")
topo = Topology()
s = topo.source(rands)
tester = Tester(topo)
tester.tuple_count(s, 100, exact=False)
tester.test(self.test_ctxtype, self.test_config)
def test_eventual_result_bad(self):
N=500000
topo = Topology()
s = topo.source(range(N))
w = s.batch(datetime.timedelta(milliseconds=300))
a = w.aggregate(lambda t : (len(t), sum(t)))
tester = Tester(topo)
tester.tuple_count(s, N)
tester.eventual_result(a, _EROK(int(N/4)))
ok = tester.test(self.test_ctxtype, self.test_config, assert_on_fail=False)
self.assertFalse(ok)
def test_batch_time(self):
topo = Topology()
s = topo.source(lambda : map(_delay, range(50)), name='A')
b = s.batch(datetime.timedelta(seconds=2))
r = b.aggregate(lambda x : x)
rf = r.flat_map()
tester = Tester(topo)
tester.tuple_count(rf, 50)
tester.run_for((50*0.2) + 20)
tester.tuple_check(r, _BatchTimeCheck())
tester.test(self.test_ctxtype, self.test_config)
def test_at_least_no_tuples(self):
""" Test at least count with zero tuples.
(kind of a pointless condition, always true).
"""
if self.test_ctxtype == context.ContextTypes.STANDALONE:
return unittest.skip("Standalone tests must complete")
topo = Topology()
s = topo.source([])
tester = Tester(topo)
tester.tuple_count(s, 0, exact=False)
tester.test(self.test_ctxtype, self.test_config)
class Test(unittest.TestCase):
""" Test invocations of composite operators in local Streams instance """
@classmethod
def setUpClass(self):
print (str(self))
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
def setUp(self):
Tester.setup_distributed(self)
self.inet_toolkit_location = "../../com.ibm.streamsx.inet"
def _add_toolkits(self, topo, test_toolkit):
tk.add_toolkit(topo, test_toolkit)
if self.inet_toolkit_location is not None:
tk.add_toolkit(topo, self.inet_toolkit_location)
def _build_launch_app(self, name, composite_name, parameters, num_result_tuples, test_toolkit):
print ("------ "+name+" ------")
topo = Topology(name)
self._add_toolkits(topo, test_toolkit)
params = parameters
# Call the test composite
test_op = op.Source(topo, composite_name, 'tuple<rstring result>', params=params)
self.tester = Tester(topo)
self.tester.run_for(30)
self.tester.tuple_count(test_op.stream, num_result_tuples, exact=True)
cfg = {}
# change trace level
job_config = streamsx.topology.context.JobConfig(tracing='info')
job_config.add(cfg)
if ("TestCloud" not in str(self)):
cfg[streamsx.topology.context.ConfigParams.SSL_VERIFY] = False
# Run the test
test_res = self.tester.test(self.test_ctxtype, cfg, assert_on_fail=True, always_collect_logs=True)
print (str(self.tester.result))
assert test_res, name+" FAILED ("+self.tester.result["application_logs"]+")"
# ------------------------------------
def test_inet_source(self):
self._build_launch_app("test_inet_source", "com.ibm.streamsx.inet.test::InetSourceTestComp", {}, 3, 'inet_test')
# ------------------------------------
def test_http_request(self):
self._build_launch_app("test_http_request", "com.ibm.streamsx.inet.test::HTTPRequestTestComp", {}, 8, 'inet_test')
def test_source(self):
topo = Topology()
topo.checkpoint_period = timedelta(seconds=1)
streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('testtkpy'))
bop = op.Source(topo, "com.ibm.streamsx.topology.pytest.checkpoint::TimeCounter", schema.StreamSchema('tuple<int32 f>').as_tuple(), params={'iterations':30,'period':0.1})
# streamsx.topology.context.submit('TOOLKIT', topo)
s = bop.stream
tester = Tester(topo)
tester.tuple_count(s, 30)
tester.contents(s, list(zip(range(0,30))))
tester.test(self.test_ctxtype, self.test_config)
def test_eventual_result_ok(self):
N=500000
topo = Topology()
s = topo.source(range(N))
w = s.batch(datetime.timedelta(milliseconds=300))
a = w.aggregate(lambda t : (len(t), sum(t)))
tester = Tester(topo)
tester.tuple_count(s, N)
tester.eventual_result(a, _EROK(N))
# Ensure we perform dependency checking for the check function
import fns_test2_test
tester.eventual_result(s, fns_test2_test.tc_dep)
tester.test(self.test_ctxtype, self.test_config)
def test_filter_map(self):
topo = Topology()
topo.checkpoint_period = timedelta(seconds=1)
streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('testtkpy'))
timeCounter = op.Source(topo, "com.ibm.streamsx.topology.pytest.checkpoint::TimeCounter", schema.StreamSchema('tuple<int32 f>').as_tuple(), params={'iterations':30,'period':0.1})
evenFilter = op.Map("com.ibm.streamsx.topology.pytest.checkpoint::StatefulEvenFilter", timeCounter.stream, None, params={})
hpo = op.Map("com.ibm.streamsx.topology.pytest.checkpoint::StatefulHalfPlusOne", evenFilter.stream, None, params={})
s = hpo.stream
tester = Tester(topo)
tester.tuple_count(s, 15)
tester.contents(s, list(zip(range(1,16))))
tester.test(self.test_ctxtype, self.test_config)
def test_bad_pe(self):
"""Test a failure in a PE is caught as a test failure"""
topo = Topology()
s = topo.source(rands)
# intentional addition of a string with an int
# to cause a PE failure
s = s.map(lambda t: t + 'a string')
tester = Tester(topo)
tester.tuple_count(s, 0, exact=False)
tp = tester.test(self.test_ctxtype,
self.test_config,
assert_on_fail=False)
self.assertFalse(tp)
def test_schemas_for_each(self):
"""Test various schemas are handled by for_each
"""
for schema in schemas:
with self.subTest(schema=schema):
topo = Topology('test_schemas_for_each')
b = beacon(topo, schema)
b.for_each(lambda tuple : None)
tester = Tester(topo)
tester.tuple_count(b, 100)
tester.test(self.test_ctxtype, self.test_config)
def test_checker(self):
""" Test the per-tuple checker.
"""
topo = Topology()
s = topo.source(rands)
s = s.filter(lambda r : r > 0.8)
s = s.map(lambda r : r + 7.0 )
tester = Tester(topo)
tester.tuple_count(s, 200, exact=False)
if self.test_ctxtype == context.ContextTypes.STANDALONE:
tester.run_for(20)
tester.tuple_check(s, lambda r : r > 7.8)
tester.test(self.test_ctxtype, self.test_config)
def test_bad_from_string(self):
for dt in SPL_TYPES:
topo = Topology()
schema = StreamSchema('tuple<' + dt + ' a>')
s = topo.source(['ABC'])
c = s.map(lambda x : (x,), schema=schema)
e = c.filter(lambda t : True)
#e.print(tag=dt)
tester = Tester(topo)
tester.tuple_count(e, 1)
tr = tester.test(self.test_ctxtype, self.test_config, assert_on_fail=False)
self.assertFalse(tr, msg=dt)
def test_request_get_fixed_url(self):
topo = Topology('test_request_get_fixed_url')
url_sample = 'http://httpbin.org/get'
s = topo.source(['fixed-url-test']).as_string()
res_http = inet.request_get(s, url_sample)
res_http.print()
tester = Tester(topo)
tester.tuple_count(res_http, 1)
tester.run_for(60)
tester.test(self.test_ctxtype,
self.test_config,
always_collect_logs=True)
def test_all_hdsf_operators(self):
hdfs_cfg_file = os.environ['HDFS_SITE_XML']
# credentials is the path to the HDSF *configuration file 'hdfs-site.xml'
topo = Topology('test_all_hdsf_operators')
if self.hdfs_toolkit_location is not None:
tk.add_toolkit(topo, self.hdfs_toolkit_location)
# creates an input stream
fileSinkInputStream = topo.source(
['This line will be written into a HDFS file.']).as_string()
# writes a line into a HDFS file (HDFS2FileSink)
fileSinkResults = hdfs.write(fileSinkInputStream,
credentials=hdfs_cfg_file,
file='pytest1/sample4%FILENUM.txt')
fileSinkResults.print(name='printFileSinkResults')
# scans an HDFS directory and return file names (HDFS2DirectoryScan)
scannedFileNames = hdfs.scan(topo,
credentials=hdfs_cfg_file,
directory='pytest1',
pattern='sample.*txt',
init_delay=10)
scannedFileNames.print(name='printScannedFileNames')
# reads lines from a HDFS file (HDFS2FileSource)
readLines = hdfs.read(scannedFileNames, credentials=hdfs_cfg_file)
readLines.print(name='printReadLines')
# copies files from HDFS into local disk "/tmp/" (HDFS2FileCopy)
copyFileResults = hdfs.copy(scannedFileNames,
credentials=hdfs_cfg_file,
direction='copyToLocalFile',
hdfsFile=None,
hdfsFileAttrName='fileName',
localFile='/tmp/')
copyFileResults.print(name='printCopyFileResults')
tester = Tester(topo)
tester.tuple_count(readLines, 1, exact=False)
# tester.run_for(80)
cfg = {}
job_config = streamsx.topology.context.JobConfig(tracing='info')
job_config.add(cfg)
cfg[streamsx.topology.context.ConfigParams.SSL_VERIFY] = False
# Run the test
tester.test(self.test_ctxtype, cfg, always_collect_logs=True)
def test_batch_punct_aggregate(self):
topo = Topology("test_batch_punct_aggregate")
s = topo.source(generate_numbers_for_named_tuple_schema)
s = s.punctor(func=(lambda t: True == t.punct_flag),
before=False,
replace=False)
window = s.batch('punct')
a = window.aggregate(SumMaxValues())
a.print(write_punctuations=True)
tester = Tester(topo)
tester.tuple_count(a, 2)
tester.punct_count(a, 2)
tester.test(self.test_ctxtype, self.test_config)
def test_spray(self):
topo = Topology()
s = U.sequence(topo, iterations=2442)
outs = []
for so in U.spray(s, count=7):
outs.append(
so.map(lambda x: (x['seq'], x['ts']),
schema=U.SEQUENCE_SCHEMA))
s = outs[0].union(set(outs))
tester = Tester(topo)
tester.tuple_count(s, 2442)
tester.test(self.test_ctxtype, self.test_config)
def test_object_view(self):
""" Test a view of Python objects.
"""
topo = Topology()
s = topo.source(rands)
throttle = op.Map('spl.utility::Throttle', s, params={'rate': 25.0})
s = throttle.stream
self._ov = s.view()
self._expected_type = float
tester = Tester(topo)
tester.local_check = self._object_view
tester.tuple_count(s, 1000, exact=False)
tester.test(self.test_ctxtype, self.test_config)
def test_spl(self):
"""
Test passing as an SPL parameter.
"""
N = 22
G = 'hey'
t = ''.join(random.choice('0123456789abcdef') for x in range(20))
topic = 'topology/test/python/' + t
topo = Topology()
spTopic = topo.create_submission_parameter('mytopic')
spGreet = topo.create_submission_parameter('greeting')
self.assertIsNone(spTopic())
self.assertIsNone(spGreet())
sch = StreamSchema('tuple<uint64 seq, rstring s>')
b = op.Source(topo,
"spl.utility::Beacon",
sch,
params={
'initDelay': 10.0,
'period': 0.02,
'iterations': N
})
b.seq = b.output('IterationCount()')
b.s = b.output(spGreet)
p = op.Sink("com.ibm.streamsx.topology.topic::Publish",
b.stream,
params={'topic': topic})
s = op.Source(topo,
"com.ibm.streamsx.topology.topic::Subscribe",
sch,
params={
'streamType': sch,
'topic': spTopic
})
jc = JobConfig()
jc.submission_parameters['mytopic'] = topic
jc.submission_parameters['greeting'] = G
jc.add(self.test_config)
tester = Tester(topo)
tester.tuple_count(s.stream, N)
#tester.run_for(300)
tester.contents(s.stream, [{'seq': i, 's': G} for i in range(N)])
tester.test(self.test_ctxtype, self.test_config)
def test_batch_count_stv(self):
topo = Topology()
s = topo.source(lambda: range(20))
wcount = topo.create_submission_parameter('count', 4)
b = s.batch(wcount)
r = b.aggregate(lambda items: sum(items))
tester = Tester(topo)
tester.contents(r, [
0 + 1 + 2 + 3, 4 + 5 + 6 + 7, 8 + 9 + 10 + 11, 12 + 13 + 14 + 15,
16 + 17 + 18 + 19
])
tester.tuple_count(r, 5)
tester.test(self.test_ctxtype, self.test_config)
def test_mt_last_aggregate(self):
topo = Topology()
N = 1000
s1 = topo.source(range(N))
s2 = topo.source(range(N))
s3 = topo.source(range(N))
self.add_stateful(topo, [s1,s2,s3])
s = s1.union({s2,s3})
s = s.last(1).trigger(1).aggregate(lambda tuples : tuples).flat_map()
tester = Tester(topo)
tester.tuple_count(s, N*3)
tester.test(self.test_ctxtype, self.test_config)
def test_pair(self):
topo = Topology()
s = topo.source(U.Sequence(iterations=932))
rschema = U.SEQUENCE_SCHEMA.extend(
StreamSchema('tuple<float64 score>'))
r0 = s.map(lambda t: (t['seq'], t['ts'], 1.0), schema=rschema)
r1 = s.map(lambda t: (t['seq'], t['ts'], 2.0), schema=rschema)
r = U.pair(r0, r1)
tester = Tester(topo)
tester.tuple_count(r, 932 * 2)
tester.tuple_check(r, PairCheck())
tester.test(self.test_ctxtype, self.test_config)
def test_bad_from_string(self):
for dt in SPL_TYPES:
topo = Topology()
schema = StreamSchema('tuple<' + dt + ' a>')
s = topo.source(['ABC'])
c = s.map(lambda x: (x, ), schema=schema)
e = c.filter(lambda t: True)
#e.print(tag=dt)
tester = Tester(topo)
tester.tuple_count(e, 1)
tr = tester.test(self.test_ctxtype,
self.test_config,
assert_on_fail=False)
self.assertFalse(tr, msg=dt)
def test_calls(self):
self.td = tempfile.mkdtemp()
topo = Topology()
s = topo.source(EESource(self.td))
s = s.map(EEMap(self.td))
s = s.flat_map(EEFlatMap(self.td))
s = s.filter(EEFilter(self.td))
s.for_each(EEForEach(self.td))
tester = Tester(topo)
tester.tuple_count(s, 101)
tester.test(self.test_ctxtype, self.test_config)
for op in ['EESource', 'EEMap', 'EEFlatMap', 'EEFilter', 'EEForEach']:
self.assertTrue(os.path.isfile(os.path.join(self.td, op + "_ENTER")))
self.assertTrue(os.path.isfile(os.path.join(self.td, op + "_EXIT")))
def test_calls(self):
self.td = tempfile.mkdtemp()
topo = Topology()
s = topo.source(EESource(self.td))
s = s.map(EEMap(self.td))
s = s.flat_map(EEFlatMap(self.td))
s = s.filter(EEFilter(self.td))
s.for_each(EEForEach(self.td))
tester = Tester(topo)
tester.tuple_count(s, 101)
tester.test(self.test_ctxtype, self.test_config)
for op in ['EESource', 'EEMap', 'EEFlatMap', 'EEFilter', 'EEForEach']:
self.assertTrue(os.path.isfile(os.path.join(self.td, op + "_ENTER")))
self.assertTrue(os.path.isfile(os.path.join(self.td, op + "_EXIT")))
def test_gate(self):
N = 137
D = 0.75
G = 17
topo = Topology()
s = topo.source(range(N))
c = PendingStream(topo)
g = U.gate(s, c.stream, max_unacked=G)
g = g.map(lambda _: time.time())
r = g.map(U.Delay(delay=D))
c.complete(r)
tester = Tester(topo)
tester.tuple_count(r, N)
tester.tuple_check(r, GateCheck(G, D))
tester.test(self.test_ctxtype, self.test_config)
def test_string_view(self):
""" Test a view of strings
"""
topo = Topology()
s = topo.source(rands)
throttle = op.Map('spl.utility::Throttle', s, params={'rate': 25.0})
s = throttle.stream
s = s.map(lambda t: "ABC" + str(t))
s = s.as_string()
self._ov = s.view()
self._expected_type = str
tester = Tester(topo)
tester.local_check = self._object_view
tester.tuple_count(s, 1000, exact=False)
tester.test(self.test_ctxtype, self.test_config)
def test_read_file_from_application_dir(self):
topo = Topology()
script_dir = os.path.dirname(os.path.realpath(__file__))
sample_file = os.path.join(script_dir, 'data.csv')
topo.add_file_dependency(sample_file,
'etc') # add sample file to etc dir in bundle
fn = os.path.join('etc',
'data.csv') # file name relative to application dir
sch = 'tuple<rstring a, int32 b>'
#fn = streamsx.spl.op.Expression.expression('getApplicationDir()+"'+'/'+fn+'"')
r = topo.source(files.CSVReader(schema=sch, file=fn))
r.print()
tester = Tester(topo)
tester.tuple_count(r, 3)
tester.test(self.test_ctxtype, self.test_config)
def test_basic(self):
print('\n---------' + str(self))
name = 'test_basic'
topo = Topology(name)
toolkit.add_toolkit(topo, self.sttgateway_toolkit_home)
if (("TestDistributed" in str(self))
or ("TestStreamingAnalytics" in str(self))):
tester = Tester(topo)
tester.tuple_count(res, 1, exact=True)
tester.test(self.test_ctxtype,
self.test_config,
always_collect_logs=True)
else:
# build only
self._build_only(name, topo)
def test_checker(self):
""" Test the per-tuple checker.
"""
topo = Topology()
s = topo.source(rands)
s = s.filter(lambda r: r > 0.8)
s = s.map(lambda r: r + 7.0)
tester = Tester(topo)
tester.tuple_count(s, 200, exact=False)
if self.test_ctxtype == context.ContextTypes.STANDALONE:
tester.run_for(20)
tester.tuple_check(s, lambda r: r > 7.8)
# Ensure we perform dependency checking for the check function
import fns_test2_test
tester.tuple_check(s, fns_test2_test.tc_dep)
tester.test(self.test_ctxtype, self.test_config)
def _run_app(self, fn=None, data=None, n=None, e=None):
topo = Topology('TSE' + str(uuid.uuid4().hex))
if data is None:
data = [1, 2, 3]
se = topo.source(data)
if fn is not None:
se = fn(se)
tester = Tester(topo)
if n is not None:
tester.tuple_count(se, n)
if e is not None:
tester.contents(se, e)
tester.run_for(3)
tester.test(self.test_ctxtype, self.test_config)
def test_SPLBeaconFilter(self):
"""Test a Source and a Map operator.
Including an output clause.
"""
topo = Topology('test_SPLBeaconFilter')
s = op.Source(topo, "spl.utility::Beacon",
'tuple<uint64 seq>',
params = {'period': 0.2, 'iterations':100})
s.seq = s.output('IterationCount()')
f = op.Map('spl.relational::Filter', s.stream,
params = {'filter': op.Expression.expression('seq % 2ul == 0ul')})
tester = Tester(topo)
tester.tuple_count(f.stream, 50)
tester.test(self.test_ctxtype, self.test_config)
def test_transform_two_outputs(self):
topo = Topology()
s = topo.source(U.Sequence(iterations=10))
fo = R.Functor.map(s, [
StreamSchema('tuple<uint64 seq>'),
StreamSchema('tuple<timestamp ts>')
])
seq = fo.outputs[0]
ts = fo.outputs[1]
seq.print()
ts.print()
tester = Tester(topo)
tester.tuple_count(seq, 10)
tester.tuple_count(ts, 10)
tester.test(self.test_ctxtype, self.test_config)
def test_pair_matched(self):
topo = Topology()
s = topo.source([0, 1, 2])
rs = 'tuple<rstring id, int32 v>'
s = s.map(lambda t : (str(t),t + 7), schema=rs)
# "Reorder" tuples as well
r0 = s.map(lambda t : ((int(t['id']) + 1)%3, t['v'] * 2), schema=rs)
r1 = s.map(lambda t : (t['id'], t['v'] * 3), schema=rs)
r = U.pair(r0, r1, matching='id')
tester = Tester(topo)
tester.tuple_count(r, 6)
tester.tuple_check(r, PairMatchedCheck())
tester.test(self.test_ctxtype, self.test_config)
def test_ResourceTags(self):
topo = Topology()
h1 = topo.source(host_name)
h1.resource_tags.add('host1')
h2 = topo.source(host_name)
h2.resource_tags.add('host2')
h = h1.union({h2})
h.print()
h = h.map(RemoveDup())
tester = Tester(topo)
tester.tuple_count(h, 2)
sr = tester.test(self.test_ctxtype, self.test_config)
def test_filter_map(self):
iterations = 3000
topo = Topology()
streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('testtkpy'))
timeCounter = op.Source(topo, "com.ibm.streamsx.topology.pytest.checkpoint::TimeCounter", schema.StreamSchema('tuple<int32 f>').as_tuple(), params={'iterations':iterations,'period':0.01})
timeCounter.stream.set_consistent(ConsistentRegionConfig.periodic(5, drain_timeout=40, reset_timeout=40, max_consecutive_attempts=6))
evenFilter = op.Map("com.ibm.streamsx.topology.pytest.checkpoint::StatefulEvenFilter", timeCounter.stream, None, params={})
hpo = op.Map("com.ibm.streamsx.topology.pytest.checkpoint::StatefulHalfPlusOne", evenFilter.stream, None, params={})
s = hpo.stream
tester = Tester(topo)
tester.resets()
tester.tuple_count(s, iterations/2)
tester.contents(s, list(zip(range(1,int((iterations/2)+1)))))
tester.test(self.test_ctxtype, self.test_config)
def test_primitive_foreach(self):
iterations=3000
topo = Topology("test_primitive_foreach_EXPECT_SC_FAIL")
topo.checkpoint_period = timedelta(seconds=1)
streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('testtkpy'))
timeCounter = op.Source(topo, "com.ibm.streamsx.topology.pytest.checkpoint::TimeCounter", schema.StreamSchema('tuple<int32 f>').as_tuple(), params={'iterations':iterations,'period':0.01})
timeCounter.stream.set_consistent(ConsistentRegionConfig.periodic(5, drain_timeout=40, reset_timeout=40, max_consecutive_attempts=6))
fizzbuzz = op.Map("com.ibm.streamsx.topology.pytest.checkpoint::FizzBuzzPrimitive", timeCounter.stream, schema.StreamSchema('tuple<int32 f, rstring c>').as_tuple())
verify = op.Sink("com.ibm.streamsx.topology.pytest.checkpoint::Verify", fizzbuzz.stream)
s = fizzbuzz.stream
tester = Tester(topo)
tester.resets()
tester.tuple_count(s, iterations)
tester.test(self.test_ctxtype, self.test_config)
def _build_launch_validate(self, name, composite_name):
topo = Topology(name)
self._add_toolkits(topo)
params = {}
# Call the test composite
test_op = op.Source(topo,
composite_name,
'tuple<rstring result>',
params=params)
tester = Tester(topo)
tester.tuple_count(test_op.stream, 1, exact=True)
tester.contents(test_op.stream, [{'result': 'TEST_RESULT_PASS'}])
tester.test(self.test_ctxtype, self.test_config)
def test_good(self):
for dt in SPL_TYPES:
if dt in GOOD_DATA:
data = GOOD_DATA[dt]
topo = Topology()
schema = StreamSchema('tuple<' + dt + ' a>')
s = topo.source(data)
c = s.map(lambda x: (x, ), schema=schema)
#c.print(tag=dt)
if dt.startswith('float'):
expected = [{'a': float(d)} for d in data]
elif dt.startswith('int'):
expected = [{'a': int(d)} for d in data]
elif dt == 'decimal32':
ctx = decimal.Context(prec=7,
rounding=decimal.ROUND_HALF_EVEN)
expected = [{
'a': decimal.Decimal(str(d)).normalize(ctx)
} for d in data]
elif dt == 'decimal64':
ctx = decimal.Context(prec=16,
rounding=decimal.ROUND_HALF_EVEN)
expected = [{
'a': decimal.Decimal(str(d)).normalize(ctx)
} for d in data]
elif dt == 'decimal128':
ctx = decimal.Context(prec=34,
rounding=decimal.ROUND_HALF_EVEN)
expected = [{
'a': decimal.Decimal(str(d)).normalize(ctx)
} for d in data]
elif dt.startswith('complex'):
expected = [{'a': complex(d)} for d in data]
elif dt == 'timestamp':
expected = [{
'a':
d if isinstance(d, Timestamp) else
Timestamp.from_datetime(d)
} for d in data]
tester = Tester(topo)
tester.tuple_count(c, len(data))
tester.contents(c, expected)
tester.test(self.test_ctxtype, self.test_config)
def test_topo_types_from_default(self):
topo = Topology()
sp_str = topo.create_submission_parameter('sp_str', default='Hi')
sp_int = topo.create_submission_parameter('sp_int', default=89)
sp_float = topo.create_submission_parameter('sp_float', default=0.5)
sp_bool = topo.create_submission_parameter('sp_bool', default=False)
s = topo.source(range(17))
s = s.filter(lambda v: isinstance(sp_str(), str) and sp_str() == 'Hi')
s = s.filter(lambda v: isinstance(sp_int(), int) and sp_int() == 89)
s = s.filter(
lambda v: isinstance(sp_float(), float) and sp_float() == 0.5)
s = s.filter(
lambda v: isinstance(sp_bool(), bool) and sp_bool() is False)
tester = Tester(topo)
tester.tuple_count(s, 17)
tester.test(self.test_ctxtype, self.test_config)
def test_structured(self):
topo = Topology()
s = topo.source(V_Sensor())
d = s.map(lambda v: v._asdict())
tester = Tester(topo)
tester.tuple_count(s, 2)
tester.tuple_check(s, lambda v: isinstance(v, tuple))
tester.contents(d, [{
'sensor_id': 101,
'ts': 10,
'reading': 3.0
}, {
'sensor_id': 202,
'ts': 20,
'reading': 7.0
}])
tester.test(self.test_ctxtype, self.test_config)
def test_union(self):
topo = Topology()
s = U.sequence(topo, iterations=932)
A = U.SEQUENCE_SCHEMA.extend(StreamSchema('tuple<int32 a, int32 c>'))
B = U.SEQUENCE_SCHEMA.extend(StreamSchema('tuple<int32 c, int32 b>'))
F = U.SEQUENCE_SCHEMA.extend(StreamSchema('tuple<int32 c>'))
r0 = s.map(lambda t : (t['seq'], t['ts'], 89, t['seq']+19), schema=A)
r1 = s.map(lambda t : (t['seq'], t['ts'], t['seq']+5, 32), schema=B)
r = U.union([r0,r1], schema=F)
r19 = r.filter(lambda t : t['c'] == t['seq'] + 19)
r5 = r.filter(lambda t : t['c'] == t['seq'] + 5)
tester = Tester(topo)
tester.tuple_count(r, 932*2)
tester.tuple_count(r19, 932)
tester.tuple_count(r5, 932)
tester.test(self.test_ctxtype, self.test_config)
def test_schemas_map(self):
"""Test various schemas are handled by map
"""
for schema in schemas:
with self.subTest(schema=schema):
topo = Topology('test_schemas_map')
b = beacon(topo, schema)
f = b.map(lambda tuple: tuple)
f2 = b.map(lambda tuple: tuple)
f2 = op.Map('spl.relational::Filter', f2).stream
f = f.union({f2})
tester = Tester(topo)
tester.tuple_count(f, 200)
tester.test(self.test_ctxtype, self.test_config)
