def test_source(self):
iterations = 3000
topo = Topology()
streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('testtkpy'))
bop = op.Source(
topo,
"com.ibm.streamsx.topology.pytest.checkpoint::TimeCounter",
schema.StreamSchema('tuple<int32 f>').as_tuple(),
params={
'iterations': iterations,
'period': 0.01
})
s = bop.stream
s.set_consistent(
ConsistentRegionConfig.periodic(5,
drain_timeout=40,
reset_timeout=40,
max_consecutive_attempts=6))
tester = Tester(topo)
tester.resets()
tester.tuple_count(s, iterations)
tester.contents(s, list(zip(range(0, iterations))))
# job_config = streamsx.topology.context.JobConfig(tracing='debug')
# job_config.add(self.test_config)
tester.test(self.test_ctxtype, self.test_config)
def test_enter_exit(self):
topo = Topology()
streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('testtkpy'))
source = op.Source(topo, 'com.ibm.streamsx.topology.pytest.checkpoint::EnterExitSource', schema.StreamSchema('tuple<rstring from, int32 enter, int32 exit>').as_tuple(), params={'period':0.1})
source.stream.set_consistent(ConsistentRegionConfig.periodic(5, drain_timeout=40, reset_timeout=40, max_consecutive_attempts=6))
transit = op.Map('com.ibm.streamsx.topology.pytest.checkpoint::EnterExitMap', source.stream, schema.StreamSchema('tuple<rstring from, int32 enter, int32 exit>').as_tuple())
tester = Tester(topo)
tester.resets(10)
# On each operator, __enter__ and __exit__ should be called once for
# each reset. Also __enter__ should be called at startup and __exit__
# at shutdown. It is hard to verify the final __exit__ call (and that
# is handled by python rather than our code), so
# the test is valid if the number of __enter__ calls is one more than
# the number of resets, and the number of __exit__ calls is equal to
# number of resets. The tuples on the two streams indicate the number
# of times __enter__ and __exit__ have been called.
# We are looking for two specific tuples:
# ('source', 6, 5) and ('transit', 6, 5)
tester.eventual_result(source.stream, lambda tuple_ : True if tuple_[1] >= 6 and tuple_[1] == tuple_[2] + 1 else Fale if tuple_[1] != tuple_[2] + 1 else None)
tester.eventual_result(transit.stream, lambda tuple_ : True if tuple_[1] >= 6 and tuple_[1] == tuple_[2] + 1 else Fale if tuple_[1] != tuple_[2] + 1 else None)
job_config = streamsx.topology.context.JobConfig(tracing='debug')
job_config.add(self.test_config)
tester.test(self.test_ctxtype, self.test_config)
def test_enter_exit(self):
topo = Topology()
streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('testtkpy'))
source = op.Source(topo, 'com.ibm.streamsx.topology.pytest.checkpoint::EnterExitSource', schema.StreamSchema('tuple<rstring from, int32 enter, int32 exit>').as_tuple(), params={'period':0.1})
source.stream.set_consistent(ConsistentRegionConfig.periodic(5, drain_timeout=40, reset_timeout=40, max_consecutive_attempts=6))
transit = op.Map('com.ibm.streamsx.topology.pytest.checkpoint::EnterExitMap', source.stream, schema.StreamSchema('tuple<rstring from, int32 enter, int32 exit>').as_tuple())
tester = Tester(topo)
tester.resets(10)
# On each operator, __enter__ and __exit__ should be called once for
# each reset. Also __enter__ should be called at startup and __exit__
# at shutdown. It is hard to verify the final __exit__ call (and that
# is handled by python rather than our code), so
# the test is valid if the number of __enter__ calls is one more than
# the number of resets, and the number of __exit__ calls is equal to
# number of resets. The tuples on the two streams indicate the number
# of times __enter__ and __exit__ have been called.
# We are looking for two specific tuples:
# ('source', 6, 5) and ('transit', 6, 5)
tester.eventual_result(source.stream, lambda tuple_ : True if tuple_[1] >= 6 and tuple_[1] == tuple_[2] + 1 else Fale if tuple_[1] != tuple_[2] + 1 else None)
tester.eventual_result(transit.stream, lambda tuple_ : True if tuple_[1] >= 6 and tuple_[1] == tuple_[2] + 1 else Fale if tuple_[1] != tuple_[2] + 1 else None)
job_config = streamsx.topology.context.JobConfig(tracing='debug')
job_config.add(self.test_config)
tester.test(self.test_ctxtype, self.test_config)
def test_primitive_foreach(self):
iterations = 3000
topo = Topology()
topo.checkpoint_period = timedelta(seconds=1)
streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('testtkpy'))
timeCounter = op.Source(
topo,
"com.ibm.streamsx.topology.pytest.checkpoint::TimeCounter",
schema.StreamSchema('tuple<int32 f>').as_tuple(),
params={
'iterations': iterations,
'period': 0.01
})
timeCounter.stream.set_consistent(
ConsistentRegionConfig.periodic(5,
drain_timeout=40,
reset_timeout=40,
max_consecutive_attempts=6))
fizzbuzz = op.Map(
"com.ibm.streamsx.topology.pytest.checkpoint::FizzBuzzPrimitive",
timeCounter.stream,
schema.StreamSchema('tuple<int32 f, rstring c>').as_tuple())
verify = op.Sink("com.ibm.streamsx.topology.pytest.checkpoint::Verify",
fizzbuzz.stream)
s = fizzbuzz.stream
tester = Tester(topo)
tester.resets()
tester.tuple_count(s, iterations)
tester.test(self.test_ctxtype, self.test_config)
def test_aggregate(self):
iterations = 3000
reset_count = 5
topo = Topology()
# Generate integers from [0,3000)
s = topo.source(TimeCounter(iterations=iterations, period=0.01))
if self.is_cr():
s.set_consistent(
ConsistentRegionConfig.periodic(5,
drain_timeout=40,
reset_timeout=40,
max_consecutive_attempts=6))
# Filter the odd ones
s = s.filter(StatefulEvenFilter())
# Halve the even ones and add one. Now have integers [1,(iterations/2))
s = s.map(StatefulHalfPlusOne())
sc = s.last(10).trigger(3).aggregate(StatefulAverage())
st = s.last(17).trigger(timedelta(seconds=2)).aggregate(
StatefulAverage())
# non-stateful aggregation functions
nsc = s.last(19).trigger(13).aggregate(lambda tuples: sum(tuples))
tester = Tester(topo)
if self.is_cr():
tester.resets(reset_count)
# Find the expected results.
# mimic the processing using Python builtins
iv = filter(StatefulEvenFilter(), range(iterations))
iv = list(map(StatefulHalfPlusOne(), iv))
# Expected stateful averages sc,st
sagg = StatefulAverage()
ers = [
sagg(iv[0:i + 3][-10:]) for i in range(0, 3 * int(len(iv) / 3), 3)
]
tester.contents(sc, ers)
tester.tuple_check(st, TimedStatefulAverageChecker())
# Must eventually aggregate on the last 17 items in iv
# but only if cr otherwise the final marker stops
# the window before the final trigger
if self.is_cr():
tester.eventual_result(
st, lambda av: True if av[1] == sagg(iv[-17:])[1] else None)
# Expected non-stateful averages nsc
ernsc = [
sum(iv[0:i + 13][-19:])
for i in range(0, 13 * int(len(iv) / 13), 13)
]
tester.contents(nsc, ernsc)
tester.test(self.test_ctxtype, self.test_config)
def test_periodic_period(self):
# No period
with self.assertRaises(ValueError):
config = ConsistentRegionConfig(ConsistentRegionConfig.Trigger.PERIODIC)
# zero (float)
with self.assertRaises(ValueError):
config = ConsistentRegionConfig.periodic(0.0)
# negative (float)
with self.assertRaises(ValueError):
config = ConsistentRegionConfig.periodic(-1.0)
# timedelta
config = ConsistentRegionConfig.periodic(timedelta(seconds=41))
self.assertEqual(config.period, timedelta(seconds=41))
# zero (timedelta)
with self.assertRaises(ValueError):
config = ConsistentRegionConfig.periodic(timedelta(seconds=0))
# negative (timedelta)
with self.assertRaises(ValueError):
config = ConsistentRegionConfig.periodic(timedelta(seconds=-8))
# can cast to float
config = ConsistentRegionConfig.periodic("4")
self.assertEqual(config.period, "4")
# cannot cast to float
with self.assertRaises(ValueError):
config = ConsistentRegionConfig.periodic("table")
def test_periodic(self):
config = ConsistentRegionConfig.periodic(14.0)
self.assertEqual(config.trigger, ConsistentRegionConfig.Trigger.PERIODIC)
self.assertEqual(config.period, 14.0)
# verify the correct defaults have been applied
self.assertEqual(config.drain_timeout, self._DEFAULT_DRAIN_TIMEOUT)
self.assertEqual(config.reset_timeout, self._DEFAULT_RESET_TIMEOUT)
self.assertEqual(config.max_consecutive_attempts, self._DEFAULT_ATTEMPTS)
def _test_insert_consistent_region(self):
print('\n---------' + str(self))
name = 'test_insert_consistent_region'
topo = Topology(name)
self._add_toolkits(topo, None)
# configuration of consistent region trigger period
trigger_period = 10
num_expected_tuples = 8000
num_resets = 2
run_for = 120 # in seconds
beacon = op.Source(topo,
"spl.utility::Beacon",
'tuple<int64 id, rstring val>',
params={
'period': 0.01,
'iterations': num_expected_tuples
})
beacon.id = beacon.output('(int64)IterationCount()')
beacon.val = beacon.output(spltypes.rstring('CR_TEST'))
beacon.stream.set_consistent(
ConsistentRegionConfig.periodic(trigger_period))
es.insert(beacon.stream,
connection=self.connection,
database=self.database,
table='StreamsCRTable',
primary_key='id',
partitioning_key='id',
front_end_connection_flag=False,
user=self.es_user,
password=self.es_password,
truststore=self.es_truststore,
truststore_password=self.es_truststore_password,
keystore=self.es_keystore,
keystore_password=self.es_keystore_password)
#self._build_only(name, topo)
tester = Tester(topo)
tester.run_for(run_for)
tester.resets(num_resets) # minimum number of resets for each region
cfg = {}
# change trace level
job_config = streamsx.topology.context.JobConfig(tracing='warn')
job_config.add(cfg)
cfg[streamsx.topology.context.ConfigParams.SSL_VERIFY] = False
tester.test(self.test_ctxtype, cfg, always_collect_logs=True)
print(str(tester.result))
def test_enter_exit(self):
iterations = 3000
reset_count = 5
topo = Topology()
s = topo.source(TimeCounter(iterations=iterations, period=0.01))
s.set_consistent(ConsistentRegionConfig.periodic(5, drain_timeout=40, reset_timeout=40, max_consecutive_attempts=6))
v = VerifyEnterExit(reset_count + 1, "VerifyEnterExit")
tester = Tester(topo)
tester.contents(s, range(0,iterations))
tester.resets(reset_count)
tester.add_condition(s, v)
tester.test(self.test_ctxtype, self.test_config)
def test_for_each(self):
iterations = 3000
reset_count = 5
topo = Topology()
s = topo.source(TimeCounter(iterations=iterations, period=0.01))
if self.is_cr():
s.set_consistent(ConsistentRegionConfig.periodic(5, drain_timeout=40, reset_timeout=40, max_consecutive_attempts=6))
s.for_each(VerifyNumericOrder())
tester = Tester(topo)
tester.contents(s, range(0,iterations))
if self.is_cr():
tester.resets(reset_count)
tester.test(self.test_ctxtype, self.test_config)
def test_aggregate(self):
iterations = 3000
reset_count = 5
topo = Topology()
# Generate integers from [0,3000)
s = topo.source(TimeCounter(iterations=iterations, period=0.01))
if self.is_cr():
s.set_consistent(ConsistentRegionConfig.periodic(5, drain_timeout=40, reset_timeout=40, max_consecutive_attempts=6))
# Filter the odd ones
s = s.filter(StatefulEvenFilter())
# Halve the even ones and add one. Now have integers [1,(iterations/2))
s = s.map(StatefulHalfPlusOne())
sc = s.last(10).trigger(3).aggregate(StatefulAverage())
st = s.last(17).trigger(datetime.timedelta(seconds=2)).aggregate(StatefulAverage())
# non-stateful aggregation functions
nsc = s.last(19).trigger(13).aggregate(lambda tuples : sum(tuples))
tester = Tester(topo)
if self.is_cr():
tester.resets(reset_count)
# Find the expected results.
# mimic the processing using Python builtins
iv = filter(StatefulEvenFilter(), range(iterations))
iv = list(map(StatefulHalfPlusOne(), iv))
# Expected stateful averages sc,st
sagg = StatefulAverage()
ers = [ sagg(iv[0:i+3][-10:]) for i in range(0, 3*int(len(iv)/3), 3) ]
tester.contents(sc, ers)
tester.tuple_check(st, TimedStatefulAverageChecker())
# Must eventually aggregate on the last 17 items in iv
# but only if cr otherwise the final marker stops
# the window before the final trigger
if self.is_cr():
tester.eventual_result(st, lambda av : True if av[1] == sagg(iv[-17:])[1] else None)
# Expected non-stateful averages nsc
ernsc = [ sum(iv[0:i+13][-19:]) for i in range(0, 13*int(len(iv)/13), 13) ]
tester.contents(nsc, ernsc)
tester.test(self.test_ctxtype, self.test_config)
def test_primitive_foreach(self):
iterations=3000
topo = Topology()
topo.checkpoint_period = timedelta(seconds=1)
streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('testtkpy'))
timeCounter = op.Source(topo, "com.ibm.streamsx.topology.pytest.checkpoint::TimeCounter", schema.StreamSchema('tuple<int32 f>').as_tuple(), params={'iterations':iterations,'period':0.01})
timeCounter.stream.set_consistent(ConsistentRegionConfig.periodic(5, drain_timeout=40, reset_timeout=40, max_consecutive_attempts=6))
fizzbuzz = op.Map("com.ibm.streamsx.topology.pytest.checkpoint::FizzBuzzPrimitive", timeCounter.stream, schema.StreamSchema('tuple<int32 f, rstring c>').as_tuple())
verify = op.Sink("com.ibm.streamsx.topology.pytest.checkpoint::Verify", fizzbuzz.stream)
s = fizzbuzz.stream
tester = Tester(topo)
tester.resets()
tester.tuple_count(s, iterations)
tester.test(self.test_ctxtype, self.test_config)
def test_filter_map(self):
iterations = 3000
topo = Topology()
streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('testtkpy'))
timeCounter = op.Source(topo, "com.ibm.streamsx.topology.pytest.checkpoint::TimeCounter", schema.StreamSchema('tuple<int32 f>').as_tuple(), params={'iterations':iterations,'period':0.01})
timeCounter.stream.set_consistent(ConsistentRegionConfig.periodic(5, drain_timeout=40, reset_timeout=40, max_consecutive_attempts=6))
evenFilter = op.Map("com.ibm.streamsx.topology.pytest.checkpoint::StatefulEvenFilter", timeCounter.stream, None, params={})
hpo = op.Map("com.ibm.streamsx.topology.pytest.checkpoint::StatefulHalfPlusOne", evenFilter.stream, None, params={})
s = hpo.stream
tester = Tester(topo)
tester.resets()
tester.tuple_count(s, iterations/2)
tester.contents(s, list(zip(range(1,int((iterations/2)+1)))))
tester.test(self.test_ctxtype, self.test_config)
def test_filter_map(self):
iterations = 3000
topo = Topology()
streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('testtkpy'))
timeCounter = op.Source(topo, "com.ibm.streamsx.topology.pytest.checkpoint::TimeCounter", schema.StreamSchema('tuple<int32 f>').as_tuple(), params={'iterations':iterations,'period':0.01})
timeCounter.stream.set_consistent(ConsistentRegionConfig.periodic(5, drain_timeout=40, reset_timeout=40, max_consecutive_attempts=6))
evenFilter = op.Map("com.ibm.streamsx.topology.pytest.checkpoint::StatefulEvenFilter", timeCounter.stream, None, params={})
hpo = op.Map("com.ibm.streamsx.topology.pytest.checkpoint::StatefulHalfPlusOne", evenFilter.stream, None, params={})
s = hpo.stream
tester = Tester(topo)
tester.resets()
tester.tuple_count(s, iterations/2)
tester.contents(s, list(zip(range(1,int((iterations/2)+1)))))
tester.test(self.test_ctxtype, self.test_config)
def test_source(self):
iterations = 3000
topo = Topology()
streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('testtkpy'))
bop = op.Source(topo, "com.ibm.streamsx.topology.pytest.checkpoint::TimeCounter", schema.StreamSchema('tuple<int32 f>').as_tuple(), params={'iterations':iterations,'period':0.01})
s = bop.stream
s.set_consistent(ConsistentRegionConfig.periodic(5, drain_timeout=40, reset_timeout=40, max_consecutive_attempts=6))
tester = Tester(topo)
tester.resets()
tester.tuple_count(s, iterations)
tester.contents(s, list(zip(range(0,iterations))))
# job_config = streamsx.topology.context.JobConfig(tracing='debug')
# job_config.add(self.test_config)
tester.test(self.test_ctxtype, self.test_config)
def test_aggregate_partitioned_stateful(self):
iterations = 3000
reset_count = 5
topo = Topology()
# Generate integers from [0,3000)
s = topo.source(TimeCounter(iterations=iterations, period=0.01))
if (self.is_cr()):
s.set_consistent(
ConsistentRegionConfig.periodic(5,
drain_timeout=40,
reset_timeout=40,
max_consecutive_attempts=6))
# Find the collatz (hotpo) sequence length
s = s.map(lambda x: CollatzLength(x + 1))
count = 10
trigger = 3
sc = s.last(count).trigger(trigger).partition(
GreatestSoFar()).aggregate(max)
#sc.print()
#streamsx.topology.context.submit('TOOLKIT', topo)
tester = Tester(topo)
if (self.is_cr()):
tester.resets(reset_count)
# Find the expected results.
# mimic the processing using Python builtins
iv = list(map(lambda x: CollatzLength(x + 1), range(iterations)))
def q(l):
partitions = [[], []]
gsf = GreatestSoFar()
for i in l:
partition = gsf(i)
partitions[partition].append(i)
if len(partitions[partition]) % trigger == 0:
yield max(partitions[partition][-count:])
tester.contents(sc, q(iv))
tester.test(self.test_ctxtype, self.test_config)
def test_hash_adder(self):
iterations=3000
reset_count=5
topo = Topology()
s = topo.source(TimeCounter(iterations=iterations, period=0.01))
if self.is_cr():
s.set_consistent(ConsistentRegionConfig.periodic(5, drain_timeout=40, reset_timeout=40, max_consecutive_attempts=6))
width = 3
s = s.parallel(width, Routing.HASH_PARTITIONED, StatefulStupidHash())
s = s.map(lambda x: x + 23)
s = s.end_parallel()
expected = [v + 23 for v in range(iterations)]
tester = Tester(topo)
if self.is_cr():
tester.resets(reset_count)
tester.contents(s, expected, ordered=width==1)
tester.test(self.test_ctxtype, self.test_config)
def test_map_foreach(self):
iterations = 3000
topo = Topology()
streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('testtkpy'))
timeCounter = op.Source(topo, "com.ibm.streamsx.topology.pytest.checkpoint::TimeCounter", schema.StreamSchema('tuple<int32 f>').as_tuple(), params={'iterations':iterations,'period':0.01})
timeCounter.stream.set_consistent(ConsistentRegionConfig.periodic(5 , drain_timeout=40, reset_timeout=40, max_consecutive_attempts=6))
fizzbuzz = op.Map("com.ibm.streamsx.topology.pytest.checkpoint::FizzBuzzMap", timeCounter.stream, schema.StreamSchema('tuple<int32 f, rstring c>').as_tuple())
verify = op.Sink("com.ibm.streamsx.topology.pytest.checkpoint::Verify", fizzbuzz.stream)
s = fizzbuzz.stream
tester = Tester(topo)
tester.resets()
tester.tuple_count(s, iterations)
# Find the expected results.
fizz=lambda x: (x[0], x[1]+'fizz' if x[0] % 3 == 0 else x[1])
buzz=lambda x: (x[0], x[1]+'buzz' if x[0] % 5 == 0 else x[1])
expected = list (map (buzz, (map (fizz, (map (lambda x: (x,''), range(iterations)))))))
tester.contents(s, expected)
tester.test(self.test_ctxtype, self.test_config)
def test_aggregate_partitioned(self):
iterations = 3000
reset_count = 5
topo = Topology()
# Generate integers from [0,3000)
s = topo.source(TimeCounter(iterations=iterations, period=0.01))
if (self.is_cr()):
s.set_consistent(
ConsistentRegionConfig.periodic(5,
drain_timeout=40,
reset_timeout=40,
max_consecutive_attempts=6))
# Filter the odd ones
s = s.filter(StatefulEvenFilter())
# Halve the even ones and add one. Now have integers [1,(iterations/2))
s = s.map(StatefulHalfPlusOne())
sc = s.last(10).trigger(3).partition(lambda x: x % 2).aggregate(
StatefulAverage())
tester = Tester(topo)
if (self.is_cr()):
tester.resets(reset_count)
# Find the expected results.
# mimic the processing using Python builtins
iv = filter(StatefulEvenFilter(), range(iterations))
iv = list(map(StatefulHalfPlusOne(), iv))
# Expected stateful averages sc,st
sagg = StatefulAverage()
ers = [
sagg(iv[i % 2:i + 5 - i % 2:2][-10:])
for i in range(0, 3 * int(len(iv) / 3), 3)
]
tester.contents(sc, ers)
tester.test(self.test_ctxtype, self.test_config)
def test_map_foreach(self):
iterations = 3000
topo = Topology()
streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('testtkpy'))
timeCounter = op.Source(topo, "com.ibm.streamsx.topology.pytest.checkpoint::TimeCounter", schema.StreamSchema('tuple<int32 f>').as_tuple(), params={'iterations':iterations,'period':0.01})
timeCounter.stream.set_consistent(ConsistentRegionConfig.periodic(5 , drain_timeout=40, reset_timeout=40, max_consecutive_attempts=6))
fizzbuzz = op.Map("com.ibm.streamsx.topology.pytest.checkpoint::FizzBuzzMap", timeCounter.stream, schema.StreamSchema('tuple<int32 f, rstring c>').as_tuple())
verify = op.Sink("com.ibm.streamsx.topology.pytest.checkpoint::Verify", fizzbuzz.stream)
s = fizzbuzz.stream
tester = Tester(topo)
tester.resets()
tester.tuple_count(s, iterations)
# Find the expected results.
fizz=lambda x: (x[0], x[1]+'fizz' if x[0] % 3 == 0 else x[1])
buzz=lambda x: (x[0], x[1]+'buzz' if x[0] % 5 == 0 else x[1])
expected = list (map (buzz, (map (fizz, (map (lambda x: (x,''), range(iterations)))))))
tester.contents(s, expected)
tester.test(self.test_ctxtype, self.test_config)
def test_flat_map(self):
count = 1000
reset_count = 5
topo = Topology();
lines = topo.source(ListIterator(["All work","and no play","makes Jack","a dull boy"], period=0.01, count=count))
if self.is_cr():
lines.set_consistent(ConsistentRegionConfig.periodic(5, drain_timeout=40, reset_timeout=40, max_consecutive_attempts=6))
words = lines.flat_map(StatefulSplit())
tester = Tester(topo)
if self.is_cr():
tester.resets(reset_count)
# Find the expected results.
flat_contents = ["All","work","and","no","play","makes","Jack","a","dull","boy"]
# repeat the contents 1000 times.
expected = []
for i in range(count):
expected.extend(flat_contents)
tester.contents(words, expected)
tester.test(self.test_ctxtype, self.test_config)
def add_stateful(self, topo, streams):
for s in streams:
s.set_consistent(ConsistentRegionConfig.periodic(0.5))