def test_periodic_period(self):
# No period
with self.assertRaises(ValueError):
config = ConsistentRegionConfig(
ConsistentRegionConfig.Trigger.PERIODIC)
# zero (float)
with self.assertRaises(ValueError):
config = ConsistentRegionConfig.periodic(0.0)
# negative (float)
with self.assertRaises(ValueError):
config = ConsistentRegionConfig.periodic(-1.0)
# timedelta
config = ConsistentRegionConfig.periodic(timedelta(seconds=41))
self.assertEqual(config.period, timedelta(seconds=41))
# zero (timedelta)
with self.assertRaises(ValueError):
config = ConsistentRegionConfig.periodic(timedelta(seconds=0))
# negative (timedelta)
with self.assertRaises(ValueError):
config = ConsistentRegionConfig.periodic(timedelta(seconds=-8))
# can cast to float
config = ConsistentRegionConfig.periodic("4")
self.assertEqual(config.period, "4")
# cannot cast to float
with self.assertRaises(ValueError):
config = ConsistentRegionConfig.periodic("table")
def test_primitive_foreach(self):
iterations = 3000
topo = Topology()
topo.checkpoint_period = timedelta(seconds=1)
streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('testtkpy'))
timeCounter = op.Source(
topo,
"com.ibm.streamsx.topology.pytest.checkpoint::TimeCounter",
schema.StreamSchema('tuple<int32 f>').as_tuple(),
params={
'iterations': iterations,
'period': 0.01
})
timeCounter.stream.set_consistent(
ConsistentRegionConfig.periodic(5,
drain_timeout=40,
reset_timeout=40,
max_consecutive_attempts=6))
fizzbuzz = op.Map(
"com.ibm.streamsx.topology.pytest.checkpoint::FizzBuzzPrimitive",
timeCounter.stream,
schema.StreamSchema('tuple<int32 f, rstring c>').as_tuple())
verify = op.Sink("com.ibm.streamsx.topology.pytest.checkpoint::Verify",
fizzbuzz.stream)
s = fizzbuzz.stream
tester = Tester(topo)
tester.resets()
tester.tuple_count(s, iterations)
tester.test(self.test_ctxtype, self.test_config)
def test_flat_map(self):
count = 1000
reset_count = 5
topo = Topology()
lines = topo.source(
ListIterator(
["All work", "and no play", "makes Jack", "a dull boy"],
period=0.01,
count=count))
lines.set_consistent(
ConsistentRegionConfig.periodic(5,
drain_timeout=40,
reset_timeout=40,
max_consecutive_attempts=6))
words = lines.flat_map(StatefulSplit())
tester = Tester(topo)
tester.resets(reset_count)
# Find the expected results.
flat_contents = [
"All", "work", "and", "no", "play", "makes", "Jack", "a", "dull",
"boy"
]
# repeat the contents 1000 times.
expected = []
for i in range(count):
expected.extend(flat_contents)
tester.contents(words, expected)
tester.test(self.test_ctxtype, self.test_config)
def test_source(self):
iterations = 3000
topo = Topology()
streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('testtkpy'))
bop = op.Source(
topo,
"com.ibm.streamsx.topology.pytest.checkpoint::TimeCounter",
schema.StreamSchema('tuple<int32 f>').as_tuple(),
params={
'iterations': iterations,
'period': 0.01
})
s = bop.stream
s.set_consistent(
ConsistentRegionConfig.periodic(5,
drain_timeout=40,
reset_timeout=40,
max_consecutive_attempts=6))
tester = Tester(topo)
tester.resets()
tester.tuple_count(s, iterations)
tester.contents(s, list(zip(range(0, iterations))))
# job_config = streamsx.topology.context.JobConfig(tracing='debug')
# job_config.add(self.test_config)
tester.test(self.test_ctxtype, self.test_config)
def test_source(self):
topo = Topology()
streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('testtkpy'))
bop = op.Source(
topo,
"com.ibm.streamsx.topology.pytest.checkpoint::TimeCounter",
schema.StreamSchema('tuple<int32 f>').as_tuple(),
params={
'iterations': 30,
'period': 0.1
})
s = bop.stream
s.set_consistent(
ConsistentRegionConfig.operator_driven(drain_timeout=40,
reset_timeout=40,
max_consecutive_attempts=3))
tester = Tester(topo)
self.assertFalse(
tester.test(self.test_ctxtype,
self.test_config,
assert_on_fail=False))
def test_beacon(self):
# An operator-driven consistent region can be used with a source
# that supports it, such as Beacon
iterations = 5000
topo = Topology()
beacon = op.Source(topo,
"spl.utility::Beacon",
schema.StreamSchema('tuple<int32 f>').as_tuple(),
params={
'iterations': iterations,
'period': 0.01,
'triggerCount': streamsx.spl.types.uint32(500)
})
beacon.f = beacon.output('(int32)IterationCount()')
s = beacon.stream
s.set_consistent(
ConsistentRegionConfig.operator_driven(drain_timeout=40,
reset_timeout=40,
max_consecutive_attempts=4))
tester = Tester(topo)
# For operator-driven regions, the resetter uses a random interval
# from 10-40 seconds for resets. Only one is likely to be completed
# while processing tuples for this test.
tester.resets(1)
tester.tuple_count(s, iterations)
tester.contents(s, list(zip(range(0, iterations))))
tester.test(self.test_ctxtype, self.test_config)
def test_op_driven_all(self):
config = ConsistentRegionConfig.operator_driven(
max_consecutive_attempts=14, drain_timeout=1, reset_timeout=2)
self.assertEqual(config.trigger,
ConsistentRegionConfig.Trigger.OPERATOR_DRIVEN)
self.assertEqual(config.drain_timeout, 1)
self.assertEqual(config.reset_timeout, 2)
self.assertEqual(config.max_consecutive_attempts, 14)
def test_op_driven_simple_attempts(self):
config = ConsistentRegionConfig.operator_driven(
max_consecutive_attempts=14)
self.assertEqual(config.trigger,
ConsistentRegionConfig.Trigger.OPERATOR_DRIVEN)
self.assertEqual(config.drain_timeout, self._DEFAULT_DRAIN_TIMEOUT)
self.assertEqual(config.reset_timeout, self._DEFAULT_RESET_TIMEOUT)
self.assertEqual(config.max_consecutive_attempts, 14)
def test_op_driven(self):
config = ConsistentRegionConfig.operator_driven()
self.assertEqual(config.trigger,
ConsistentRegionConfig.Trigger.OPERATOR_DRIVEN)
# verify the correct defaults have been applied
self.assertEqual(config.drain_timeout, self._DEFAULT_DRAIN_TIMEOUT)
self.assertEqual(config.reset_timeout, self._DEFAULT_RESET_TIMEOUT)
self.assertEqual(config.max_consecutive_attempts,
self._DEFAULT_ATTEMPTS)
def test_periodic(self):
config = ConsistentRegionConfig.periodic(14.0)
self.assertEqual(config.trigger,
ConsistentRegionConfig.Trigger.PERIODIC)
self.assertEqual(config.period, 14.0)
# verify the correct defaults have been applied
self.assertEqual(config.drain_timeout, self._DEFAULT_DRAIN_TIMEOUT)
self.assertEqual(config.reset_timeout, self._DEFAULT_RESET_TIMEOUT)
self.assertEqual(config.max_consecutive_attempts,
self._DEFAULT_ATTEMPTS)
def test_op_driven_reset_timeout(self):
# negative
with self.assertRaises(ValueError):
config = ConsistentRegionConfig.operator_driven(reset_timeout=-1)
# timedelta, positive
config = ConsistentRegionConfig.operator_driven(
reset_timeout=timedelta(seconds=1))
self.assertEqual(config.reset_timeout, timedelta(seconds=1))
# timedelta, zero
with self.assertRaises(ValueError):
config = ConsistentRegionConfig.operator_driven(
reset_timeout=timedelta(seconds=0))
# timedelta, negative
with self.assertRaises(ValueError):
config = ConsistentRegionConfig.operator_driven(
reset_timeout=timedelta(seconds=-1))
# can cast to float
config = ConsistentRegionConfig.operator_driven(reset_timeout="8.2")
self.assertEqual(config.reset_timeout, "8.2")
# cannot cast to float
with self.assertRaises(ValueError):
config = ConsistentRegionConfig.operator_driven(
reset_timeout="clogged")
def test_source(self):
topo = Topology()
s = topo.source(TimeCounter(iterations=30, period=0.1))
s.set_consistent(
ConsistentRegionConfig.operator_driven(drain_timeout=40,
reset_timeout=40,
max_consecutive_attempts=3))
tester = Tester(topo)
self.assertFalse(
tester.test(self.test_ctxtype,
self.test_config,
assert_on_fail=False))
def test_enter_exit(self):
topo = Topology()
streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('testtkpy'))
source = op.Source(
topo,
'com.ibm.streamsx.topology.pytest.checkpoint::EnterExitSource',
schema.StreamSchema(
'tuple<rstring from, int32 enter, int32 exit>').as_tuple(),
params={'period': 0.1})
source.stream.set_consistent(
ConsistentRegionConfig.periodic(5,
drain_timeout=40,
reset_timeout=40,
max_consecutive_attempts=6))
transit = op.Map(
'com.ibm.streamsx.topology.pytest.checkpoint::EnterExitMap',
source.stream,
schema.StreamSchema(
'tuple<rstring from, int32 enter, int32 exit>').as_tuple())
tester = Tester(topo)
tester.resets(10)
# On each operator, __enter__ and __exit__ should be called once for
# each reset. Also __enter__ should be called at startup and __exit__
# at shutdown. It is hard to verify the final __exit__ call (and that
# is handled by python rather than our code), so
# the test is valid if the number of __enter__ calls is one more than
# the number of resets, and the number of __exit__ calls is equal to
# number of resets. The tuples on the two streams indicate the number
# of times __enter__ and __exit__ have been called.
# We are looking for two specific tuples:
# ('source', 6, 5) and ('transit', 6, 5)
tester.eventual_result(
source.stream, lambda tuple_: True
if tuple_[1] >= 6 and tuple_[1] == tuple_[2] + 1 else Fale
if tuple_[1] != tuple_[2] + 1 else None)
tester.eventual_result(
transit.stream, lambda tuple_: True
if tuple_[1] >= 6 and tuple_[1] == tuple_[2] + 1 else Fale
if tuple_[1] != tuple_[2] + 1 else None)
job_config = streamsx.topology.context.JobConfig(tracing='debug')
job_config.add(self.test_config)
tester.test(self.test_ctxtype, self.test_config)
def test_for_each(self):
iterations = 3000
reset_count = 5
topo = Topology()
s = topo.source(TimeCounter(iterations=iterations, period=0.01))
s.set_consistent(
ConsistentRegionConfig.periodic(5,
drain_timeout=40,
reset_timeout=40,
max_consecutive_attempts=6))
s.for_each(VerifyNumericOrder())
tester = Tester(topo)
tester.contents(s, range(0, iterations))
tester.resets(reset_count)
tester.test(self.test_ctxtype, self.test_config)
def test_op_driven_attempts(self):
# negative
with self.assertRaises(ValueError):
config = ConsistentRegionConfig.operator_driven(
max_consecutive_attempts=-1)
# zero
with self.assertRaises(ValueError):
config = ConsistentRegionConfig.operator_driven(
max_consecutive_attempts=0)
# one
config = ConsistentRegionConfig.operator_driven(
max_consecutive_attempts=1)
self.assertEqual(config.max_consecutive_attempts, 1)
# exactly 0x7FFFFFFF
config = ConsistentRegionConfig.operator_driven(
max_consecutive_attempts=0x7FFFFFFF)
self.assertEqual(config.max_consecutive_attempts, 0x7FFFFFFF)
# greater than 0x7FFFFFF
with self.assertRaises(ValueError):
config = ConsistentRegionConfig.operator_driven(
max_consecutive_attempts=0x80000000)
# float literal, exactly integral
config = ConsistentRegionConfig.operator_driven(
max_consecutive_attempts=12.0)
self.assertEqual(config.max_consecutive_attempts, 12.0)
# not exactly integral
with self.assertRaises(ValueError):
config = ConsistentRegionConfig.operator_driven(
max_consecutive_attempts=12.2)
# a string that can be cast to a valid integral value
config = ConsistentRegionConfig.operator_driven(
max_consecutive_attempts="14")
self.assertEqual(config.max_consecutive_attempts, "14")
def test_enter_exit(self):
iterations = 3000
reset_count = 5
topo = Topology()
s = topo.source(TimeCounter(iterations=iterations, period=0.01))
s.set_consistent(
ConsistentRegionConfig.periodic(5,
drain_timeout=40,
reset_timeout=40,
max_consecutive_attempts=6))
v = VerifyEnterExit(reset_count + 1, "VerifyEnterExit")
tester = Tester(topo)
tester.contents(s, range(0, iterations))
tester.resets(reset_count)
tester.add_condition(s, v)
tester.test(self.test_ctxtype, self.test_config)
def test_flat_map(self):
topo = Topology()
lines = topo.source(
ListIterator(
["mary had a little lamb", "its fleece was white as snow"]))
# slow things down so checkpoints can be taken.
lines = lines.filter(StatefulDelay(0.5))
words = lines.flat_map(StatefulSplit())
words.set_consistent(
ConsistentRegionConfig.operator_driven(drain_timeout=40,
reset_timeout=40,
max_consecutive_attempts=3))
tester = Tester(topo)
self.assertFalse(
tester.test(self.test_ctxtype,
self.test_config,
assert_on_fail=False))
def test_map(self):
topo = Topology()
# Generate integers from [0,30)
s = topo.source(TimeCounter(iterations=30, period=0.1))
# Filter the odd ones
s = s.filter(StatefulEvenFilter())
# Halve the even ones and add one. Now have integers [1,15)
s = s.map(StatefulHalfPlusOne())
s.set_consistent(
ConsistentRegionConfig.operator_driven(drain_timeout=40,
reset_timeout=40,
max_consecutive_attempts=3))
s = s.last(10).trigger(2).aggregate(StatefulAverage())
tester = Tester(topo)
self.assertFalse(
tester.test(self.test_ctxtype,
self.test_config,
assert_on_fail=False))
def test_source(self):
iterations = 3000
reset_count = 5
topo = Topology()
s = topo.source(TimeCounter(iterations=iterations, period=0.01))
s.set_consistent(
ConsistentRegionConfig.periodic(5,
drain_timeout=40,
reset_timeout=40,
max_consecutive_attempts=6))
tester = Tester(topo)
tester.contents(s, range(0, iterations))
tester.resets(reset_count)
# cfg={}
# job_config = streamsx.topology.context.JobConfig(tracing='debug')
# job_config.add(self.test_config)
tester.test(self.test_ctxtype, self.test_config)
def test_hash_adder(self):
iterations = 3000
reset_count = 5
topo = Topology()
s = topo.source(TimeCounter(iterations=iterations, period=0.01))
s.set_consistent(
ConsistentRegionConfig.periodic(5,
drain_timeout=40,
reset_timeout=40,
max_consecutive_attempts=6))
width = 3
s = s.parallel(width, Routing.HASH_PARTITIONED, StatefulStupidHash())
s = s.map(lambda x: x + 23)
s = s.end_parallel()
expected = [v + 23 for v in range(iterations)]
tester = Tester(topo)
tester.resets(reset_count)
tester.contents(s, expected, ordered=width == 1)
tester.test(self.test_ctxtype, self.test_config)
def test_map_foreach(self):
iterations = 3000
topo = Topology()
streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('testtkpy'))
timeCounter = op.Source(
topo,
"com.ibm.streamsx.topology.pytest.checkpoint::TimeCounter",
schema.StreamSchema('tuple<int32 f>').as_tuple(),
params={
'iterations': iterations,
'period': 0.01
})
timeCounter.stream.set_consistent(
ConsistentRegionConfig.periodic(5,
drain_timeout=40,
reset_timeout=40,
max_consecutive_attempts=6))
fizzbuzz = op.Map(
"com.ibm.streamsx.topology.pytest.checkpoint::FizzBuzzMap",
timeCounter.stream,
schema.StreamSchema('tuple<int32 f, rstring c>').as_tuple())
verify = op.Sink("com.ibm.streamsx.topology.pytest.checkpoint::Verify",
fizzbuzz.stream)
s = fizzbuzz.stream
tester = Tester(topo)
tester.resets()
tester.tuple_count(s, iterations)
# Find the expected results.
fizz = lambda x: (x[0], x[1] + 'fizz' if x[0] % 3 == 0 else x[1])
buzz = lambda x: (x[0], x[1] + 'buzz' if x[0] % 5 == 0 else x[1])
expected = list(
map(buzz, (map(fizz, (map(lambda x:
(x, ''), range(iterations)))))))
tester.contents(s, expected)
tester.test(self.test_ctxtype, self.test_config)
def test_aggregate(self):
# If the number of iterations is changed, keep it a multiple of six,
# or improve the expected results generation below.
iterations = 3000
reset_count = 5
topo = Topology()
# Generate integers from [0,3000)
s = topo.source(TimeCounter(iterations=iterations, period=0.01))
s.set_consistent(
ConsistentRegionConfig.periodic(5,
drain_timeout=40,
reset_timeout=40,
max_consecutive_attempts=6))
# Filter the odd ones
s = s.filter(StatefulEvenFilter())
# Halve the even ones and add one. Now have integers [1,(iterations/2))
s = s.map(StatefulHalfPlusOne())
s = s.last(10).trigger(3).aggregate(StatefulAverage())
tester = Tester(topo)
tester.resets(reset_count)
# Find the expected results.
# The first three values (until the window fills) are special.
expected = [2.0, 3.5, 5.0]
# The rest start at 7.5 and increase by three until 1495.5.
# Assuming that the number of iterations is a multiple of six,
# the final trigger happens at the last tuple. There will be
# ten values in the window, from (iterations/2 - 9) to (iterations/2).
# The average is then ((iterations/2 - 9) + (iterations/2))/2.
# For 3000 iterations, that works out to 1495.5
end = float(iterations) / 2.0 - 4.5
expected.extend(
itertools.takewhile(lambda x: x <= end, itertools.count(7.5, 3)))
tester.contents(s, expected)
tester.test(self.test_ctxtype, self.test_config)
def test_filter_map(self):
iterations = 3000
topo = Topology()
streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('testtkpy'))
timeCounter = op.Source(
topo,
"com.ibm.streamsx.topology.pytest.checkpoint::TimeCounter",
schema.StreamSchema('tuple<int32 f>').as_tuple(),
params={
'iterations': iterations,
'period': 0.01
})
timeCounter.stream.set_consistent(
ConsistentRegionConfig.periodic(5,
drain_timeout=40,
reset_timeout=40,
max_consecutive_attempts=6))
evenFilter = op.Map(
"com.ibm.streamsx.topology.pytest.checkpoint::StatefulEvenFilter",
timeCounter.stream,
None,
params={})
hpo = op.Map(
"com.ibm.streamsx.topology.pytest.checkpoint::StatefulHalfPlusOne",
evenFilter.stream,
None,
params={})
s = hpo.stream
tester = Tester(topo)
tester.resets()
tester.tuple_count(s, iterations / 2)
tester.contents(s, list(zip(range(1, int((iterations / 2) + 1)))))
tester.test(self.test_ctxtype, self.test_config)
def test_no_trigger(self):
with self.assertRaises(ValueError):
config = ConsistentRegionConfig()