def test_enter_exit(self):
topo = Topology()
streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('testtkpy'))
source = op.Source(topo, 'com.ibm.streamsx.topology.pytest.checkpoint::EnterExitSource', schema.StreamSchema('tuple<rstring from, int32 enter, int32 exit>').as_tuple(), params={'period':0.1})
source.stream.set_consistent(ConsistentRegionConfig.periodic(5, drain_timeout=40, reset_timeout=40, max_consecutive_attempts=6))
transit = op.Map('com.ibm.streamsx.topology.pytest.checkpoint::EnterExitMap', source.stream, schema.StreamSchema('tuple<rstring from, int32 enter, int32 exit>').as_tuple())
tester = Tester(topo)
tester.resets(10)
# On each operator, __enter__ and __exit__ should be called once for
# each reset. Also __enter__ should be called at startup and __exit__
# at shutdown. It is hard to verify the final __exit__ call (and that
# is handled by python rather than our code), so
# the test is valid if the number of __enter__ calls is one more than
# the number of resets, and the number of __exit__ calls is equal to
# number of resets. The tuples on the two streams indicate the number
# of times __enter__ and __exit__ have been called.
# We are looking for two specific tuples:
# ('source', 6, 5) and ('transit', 6, 5)
tester.eventual_result(source.stream, lambda tuple_ : True if tuple_[1] >= 6 and tuple_[1] == tuple_[2] + 1 else Fale if tuple_[1] != tuple_[2] + 1 else None)
tester.eventual_result(transit.stream, lambda tuple_ : True if tuple_[1] >= 6 and tuple_[1] == tuple_[2] + 1 else Fale if tuple_[1] != tuple_[2] + 1 else None)
job_config = streamsx.topology.context.JobConfig(tracing='debug')
job_config.add(self.test_config)
tester.test(self.test_ctxtype, self.test_config)
def test_enter_exit(self):
topo = Topology()
streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('testtkpy'))
source = op.Source(topo, 'com.ibm.streamsx.topology.pytest.checkpoint::EnterExitSource', schema.StreamSchema('tuple<rstring from, int32 enter, int32 exit>').as_tuple(), params={'period':0.1})
source.stream.set_consistent(ConsistentRegionConfig.periodic(5, drain_timeout=40, reset_timeout=40, max_consecutive_attempts=6))
transit = op.Map('com.ibm.streamsx.topology.pytest.checkpoint::EnterExitMap', source.stream, schema.StreamSchema('tuple<rstring from, int32 enter, int32 exit>').as_tuple())
tester = Tester(topo)
tester.resets(10)
# On each operator, __enter__ and __exit__ should be called once for
# each reset. Also __enter__ should be called at startup and __exit__
# at shutdown. It is hard to verify the final __exit__ call (and that
# is handled by python rather than our code), so
# the test is valid if the number of __enter__ calls is one more than
# the number of resets, and the number of __exit__ calls is equal to
# number of resets. The tuples on the two streams indicate the number
# of times __enter__ and __exit__ have been called.
# We are looking for two specific tuples:
# ('source', 6, 5) and ('transit', 6, 5)
tester.eventual_result(source.stream, lambda tuple_ : True if tuple_[1] >= 6 and tuple_[1] == tuple_[2] + 1 else Fale if tuple_[1] != tuple_[2] + 1 else None)
tester.eventual_result(transit.stream, lambda tuple_ : True if tuple_[1] >= 6 and tuple_[1] == tuple_[2] + 1 else Fale if tuple_[1] != tuple_[2] + 1 else None)
job_config = streamsx.topology.context.JobConfig(tracing='debug')
job_config.add(self.test_config)
tester.test(self.test_ctxtype, self.test_config)
def test_aggregate(self):
iterations = 3000
reset_count = 5
topo = Topology()
# Generate integers from [0,3000)
s = topo.source(TimeCounter(iterations=iterations, period=0.01))
if self.is_cr():
s.set_consistent(
ConsistentRegionConfig.periodic(5,
drain_timeout=40,
reset_timeout=40,
max_consecutive_attempts=6))
# Filter the odd ones
s = s.filter(StatefulEvenFilter())
# Halve the even ones and add one. Now have integers [1,(iterations/2))
s = s.map(StatefulHalfPlusOne())
sc = s.last(10).trigger(3).aggregate(StatefulAverage())
st = s.last(17).trigger(timedelta(seconds=2)).aggregate(
StatefulAverage())
# non-stateful aggregation functions
nsc = s.last(19).trigger(13).aggregate(lambda tuples: sum(tuples))
tester = Tester(topo)
if self.is_cr():
tester.resets(reset_count)
# Find the expected results.
# mimic the processing using Python builtins
iv = filter(StatefulEvenFilter(), range(iterations))
iv = list(map(StatefulHalfPlusOne(), iv))
# Expected stateful averages sc,st
sagg = StatefulAverage()
ers = [
sagg(iv[0:i + 3][-10:]) for i in range(0, 3 * int(len(iv) / 3), 3)
]
tester.contents(sc, ers)
tester.tuple_check(st, TimedStatefulAverageChecker())
# Must eventually aggregate on the last 17 items in iv
# but only if cr otherwise the final marker stops
# the window before the final trigger
if self.is_cr():
tester.eventual_result(
st, lambda av: True if av[1] == sagg(iv[-17:])[1] else None)
# Expected non-stateful averages nsc
ernsc = [
sum(iv[0:i + 13][-19:])
for i in range(0, 13 * int(len(iv) / 13), 13)
]
tester.contents(nsc, ernsc)
tester.test(self.test_ctxtype, self.test_config)
def test_eventual_result_bad(self):
N=500000
topo = Topology()
s = topo.source(range(N))
w = s.batch(datetime.timedelta(milliseconds=300))
a = w.aggregate(lambda t : (len(t), sum(t)))
tester = Tester(topo)
tester.tuple_count(s, N)
tester.eventual_result(a, _EROK(int(N/4)))
ok = tester.test(self.test_ctxtype, self.test_config, assert_on_fail=False)
self.assertFalse(ok)
def test_eventual_result_bad(self):
N=500000
topo = Topology()
s = topo.source(range(N))
w = s.batch(datetime.timedelta(milliseconds=300))
a = w.aggregate(lambda t : (len(t), sum(t)))
tester = Tester(topo)
tester.tuple_count(s, N)
tester.eventual_result(a, _EROK(int(N/4)))
ok = tester.test(self.test_ctxtype, self.test_config, assert_on_fail=False)
self.assertFalse(ok)
def test_eventual_result_ok(self):
N = 500000
topo = Topology()
s = topo.source(range(N))
w = s.batch(datetime.timedelta(milliseconds=300))
a = w.aggregate(lambda t: (len(t), sum(t)))
tester = Tester(topo)
tester.tuple_count(s, N)
tester.eventual_result(a, _EROK(N))
# Ensure we perform dependency checking for the check function
import fns_test2_test
tester.eventual_result(s, fns_test2_test.tc_dep)
tester.test(self.test_ctxtype, self.test_config)
def test_eventual_result_ok(self):
N=500000
topo = Topology()
s = topo.source(range(N))
w = s.batch(datetime.timedelta(milliseconds=300))
a = w.aggregate(lambda t : (len(t), sum(t)))
tester = Tester(topo)
tester.tuple_count(s, N)
tester.eventual_result(a, _EROK(N))
# Ensure we perform dependency checking for the check function
import fns_test2_test
tester.eventual_result(s, fns_test2_test.tc_dep)
tester.test(self.test_ctxtype, self.test_config)
def test_aggregate(self):
iterations = 3000
reset_count = 5
topo = Topology()
# Generate integers from [0,3000)
s = topo.source(TimeCounter(iterations=iterations, period=0.01))
if self.is_cr():
s.set_consistent(ConsistentRegionConfig.periodic(5, drain_timeout=40, reset_timeout=40, max_consecutive_attempts=6))
# Filter the odd ones
s = s.filter(StatefulEvenFilter())
# Halve the even ones and add one. Now have integers [1,(iterations/2))
s = s.map(StatefulHalfPlusOne())
sc = s.last(10).trigger(3).aggregate(StatefulAverage())
st = s.last(17).trigger(datetime.timedelta(seconds=2)).aggregate(StatefulAverage())
# non-stateful aggregation functions
nsc = s.last(19).trigger(13).aggregate(lambda tuples : sum(tuples))
tester = Tester(topo)
if self.is_cr():
tester.resets(reset_count)
# Find the expected results.
# mimic the processing using Python builtins
iv = filter(StatefulEvenFilter(), range(iterations))
iv = list(map(StatefulHalfPlusOne(), iv))
# Expected stateful averages sc,st
sagg = StatefulAverage()
ers = [ sagg(iv[0:i+3][-10:]) for i in range(0, 3*int(len(iv)/3), 3) ]
tester.contents(sc, ers)
tester.tuple_check(st, TimedStatefulAverageChecker())
# Must eventually aggregate on the last 17 items in iv
# but only if cr otherwise the final marker stops
# the window before the final trigger
if self.is_cr():
tester.eventual_result(st, lambda av : True if av[1] == sagg(iv[-17:])[1] else None)
# Expected non-stateful averages nsc
ernsc = [ sum(iv[0:i+13][-19:]) for i in range(0, 13*int(len(iv)/13), 13) ]
tester.contents(nsc, ernsc)
tester.test(self.test_ctxtype, self.test_config)
