def test_op_driven_attempts(self):
# negative
with self.assertRaises(ValueError):
config = ConsistentRegionConfig.operator_driven(max_consecutive_attempts=-1)
# zero
with self.assertRaises(ValueError):
config = ConsistentRegionConfig.operator_driven(max_consecutive_attempts=0)
# one
config = ConsistentRegionConfig.operator_driven(max_consecutive_attempts=1)
self.assertEqual(config.max_consecutive_attempts, 1)
# exactly 0x7FFFFFFF
config = ConsistentRegionConfig.operator_driven(max_consecutive_attempts=0x7FFFFFFF)
self.assertEqual(config.max_consecutive_attempts, 0x7FFFFFFF)
# greater than 0x7FFFFFF
with self.assertRaises(ValueError):
config = ConsistentRegionConfig.operator_driven(max_consecutive_attempts=0x80000000)
# float literal, exactly integral
config = ConsistentRegionConfig.operator_driven(max_consecutive_attempts=12.0)
self.assertEqual(config.max_consecutive_attempts, 12.0)
# not exactly integral
with self.assertRaises(ValueError):
config = ConsistentRegionConfig.operator_driven(max_consecutive_attempts=12.2)
# a string that can be cast to a valid integral value
config = ConsistentRegionConfig.operator_driven(max_consecutive_attempts="14")
self.assertEqual(config.max_consecutive_attempts, "14")
def test_op_driven(self):
config = ConsistentRegionConfig.operator_driven()
self.assertEqual(config.trigger, ConsistentRegionConfig.Trigger.OPERATOR_DRIVEN)
# verify the correct defaults have been applied
self.assertEqual(config.drain_timeout, self._DEFAULT_DRAIN_TIMEOUT)
self.assertEqual(config.reset_timeout, self._DEFAULT_RESET_TIMEOUT)
self.assertEqual(config.max_consecutive_attempts, self._DEFAULT_ATTEMPTS)
def test_opdriven_source(self):
topo = Topology()
s = topo.source(TimeCounter(iterations=30, period=0.1))
s.set_consistent(ConsistentRegionConfig.operator_driven(drain_timeout=40, reset_timeout=40, max_consecutive_attempts=3))
tester = Tester(topo)
self.assertFalse(tester.test(self.test_ctxtype, self.test_config, assert_on_fail=False))
def test_beacon(self):
# An operator-driven consistent region can be used with a source
# that supports it, such as Beacon
iterations = 5000
topo = Topology()
beacon = op.Source(topo,
"spl.utility::Beacon",
schema.StreamSchema('tuple<int32 f>').as_tuple(),
params={
'iterations': iterations,
'period': 0.01,
'triggerCount': streamsx.spl.types.uint32(500)
})
beacon.f = beacon.output('(int32)IterationCount()')
s = beacon.stream
s.set_consistent(
ConsistentRegionConfig.operator_driven(drain_timeout=40,
reset_timeout=40,
max_consecutive_attempts=4))
tester = Tester(topo)
# For operator-driven regions, the resetter uses a random interval
# from 10-40 seconds for resets. Only one is likely to be completed
# while processing tuples for this test.
tester.resets(1)
tester.tuple_count(s, iterations)
tester.contents(s, list(zip(range(0, iterations))))
tester.test(self.test_ctxtype, self.test_config)
def test_source(self):
topo = Topology()
streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('testtkpy'))
bop = op.Source(
topo,
"com.ibm.streamsx.topology.pytest.checkpoint::TimeCounter",
schema.StreamSchema('tuple<int32 f>').as_tuple(),
params={
'iterations': 30,
'period': 0.1
})
s = bop.stream
s.set_consistent(
ConsistentRegionConfig.operator_driven(drain_timeout=40,
reset_timeout=40,
max_consecutive_attempts=3))
tester = Tester(topo)
self.assertFalse(
tester.test(self.test_ctxtype,
self.test_config,
assert_on_fail=False))
def test_opdriven_flat_map(self):
topo = Topology();
lines = topo.source(ListIterator(["mary had a little lamb", "its fleece was white as snow"]))
# slow things down so checkpoints can be taken.
lines = lines.filter(StatefulDelay(0.5))
words = lines.flat_map(StatefulSplit())
words.set_consistent(ConsistentRegionConfig.operator_driven(drain_timeout=40, reset_timeout=40, max_consecutive_attempts=3))
tester = Tester(topo)
self.assertFalse(tester.test(self.test_ctxtype, self.test_config, assert_on_fail=False))
def test_source(self):
topo = Topology()
streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('testtkpy'))
bop = op.Source(topo, "com.ibm.streamsx.topology.pytest.checkpoint::TimeCounter", schema.StreamSchema('tuple<int32 f>').as_tuple(), params={'iterations':30,'period':0.1})
s = bop.stream
s.set_consistent(ConsistentRegionConfig.operator_driven(drain_timeout=40, reset_timeout=40, max_consecutive_attempts=3))
tester = Tester(topo)
self.assertFalse(tester.test(self.test_ctxtype, self.test_config, assert_on_fail=False))
def test_opdriven_map(self):
topo = Topology()
# Generate integers from [0,30)
s = topo.source(TimeCounter(iterations=30, period=0.1))
# Filter the odd ones
s = s.filter(StatefulEvenFilter())
# Halve the even ones and add one. Now have integers [1,15)
s = s.map(StatefulHalfPlusOne())
s.set_consistent(ConsistentRegionConfig.operator_driven(drain_timeout=40, reset_timeout=40, max_consecutive_attempts=3))
s = s.last(10).trigger(2).aggregate(StatefulAverage())
tester = Tester(topo)
self.assertFalse(tester.test(self.test_ctxtype, self.test_config, assert_on_fail=False))
def test_beacon(self):
# An operator-driven consistent region can be used with a source
# that supports it, such as Beacon
iterations = 5000
topo = Topology()
beacon = op.Source(topo, "spl.utility::Beacon", schema.StreamSchema('tuple<int32 f>').as_tuple(), params={'iterations':iterations,'period':0.01,'triggerCount':streamsx.spl.types.uint32(500)})
beacon.f = beacon.output('(int32)IterationCount()')
s = beacon.stream
s.set_consistent(ConsistentRegionConfig.operator_driven(drain_timeout=40, reset_timeout=40, max_consecutive_attempts=4))
tester = Tester(topo)
# For operator-driven regions, the resetter uses a random interval
# from 10-40 seconds for resets. Only one is likely to be completed
# while processing tuples for this test.
tester.resets(1)
tester.tuple_count(s, iterations)
tester.contents(s, list(zip(range(0,iterations))))
tester.test(self.test_ctxtype, self.test_config)
def test_op_driven_drain_timeout(self):
with self.assertRaises(ValueError):
config = ConsistentRegionConfig.operator_driven(drain_timeout=-1)
# timedelta, positive
config = ConsistentRegionConfig.operator_driven(drain_timeout=timedelta(seconds=1))
self.assertEqual(config.drain_timeout, timedelta(seconds=1))
# timedelta, zero
with self.assertRaises(ValueError):
config = ConsistentRegionConfig.operator_driven(drain_timeout=timedelta(seconds=0))
# timedelta, negative
with self.assertRaises(ValueError):
config = ConsistentRegionConfig.operator_driven(drain_timeout=timedelta(seconds=-1))
# can cast to float
config = ConsistentRegionConfig.operator_driven(drain_timeout="8.2")
self.assertEqual(config.drain_timeout, "8.2")
# cannot cast to float
with self.assertRaises(ValueError):
config = ConsistentRegionConfig.operator_driven(drain_timeout="clogged")
def test_op_driven_all(self):
config = ConsistentRegionConfig.operator_driven(max_consecutive_attempts=14, drain_timeout=1, reset_timeout=2)
self.assertEqual(config.trigger, ConsistentRegionConfig.Trigger.OPERATOR_DRIVEN)
self.assertEqual(config.drain_timeout, 1)
self.assertEqual(config.reset_timeout, 2)
self.assertEqual(config.max_consecutive_attempts, 14)
def test_op_driven_simple_attempts(self):
config = ConsistentRegionConfig.operator_driven(max_consecutive_attempts=14)
self.assertEqual(config.trigger, ConsistentRegionConfig.Trigger.OPERATOR_DRIVEN)
self.assertEqual(config.drain_timeout, self._DEFAULT_DRAIN_TIMEOUT)
self.assertEqual(config.reset_timeout, self._DEFAULT_RESET_TIMEOUT)
self.assertEqual(config.max_consecutive_attempts, 14)