def test_sequence(self):
topo = Topology()
s = U.sequence(topo, iterations=122)
tester = Tester(topo)
tester.tuple_check(s, lambda x: 'seq' in x and 'ts' in x)
tester.tuple_count(s, 122)
tester.test(self.test_ctxtype, self.test_config)
def test_aggregate(self):
iterations = 3000
reset_count = 5
topo = Topology()
# Generate integers from [0,3000)
s = topo.source(TimeCounter(iterations=iterations, period=0.01))
if self.is_cr():
s.set_consistent(
ConsistentRegionConfig.periodic(5,
drain_timeout=40,
reset_timeout=40,
max_consecutive_attempts=6))
# Filter the odd ones
s = s.filter(StatefulEvenFilter())
# Halve the even ones and add one. Now have integers [1,(iterations/2))
s = s.map(StatefulHalfPlusOne())
sc = s.last(10).trigger(3).aggregate(StatefulAverage())
st = s.last(17).trigger(timedelta(seconds=2)).aggregate(
StatefulAverage())
# non-stateful aggregation functions
nsc = s.last(19).trigger(13).aggregate(lambda tuples: sum(tuples))
tester = Tester(topo)
if self.is_cr():
tester.resets(reset_count)
# Find the expected results.
# mimic the processing using Python builtins
iv = filter(StatefulEvenFilter(), range(iterations))
iv = list(map(StatefulHalfPlusOne(), iv))
# Expected stateful averages sc,st
sagg = StatefulAverage()
ers = [
sagg(iv[0:i + 3][-10:]) for i in range(0, 3 * int(len(iv) / 3), 3)
]
tester.contents(sc, ers)
tester.tuple_check(st, TimedStatefulAverageChecker())
# Must eventually aggregate on the last 17 items in iv
# but only if cr otherwise the final marker stops
# the window before the final trigger
if self.is_cr():
tester.eventual_result(
st, lambda av: True if av[1] == sagg(iv[-17:])[1] else None)
# Expected non-stateful averages nsc
ernsc = [
sum(iv[0:i + 13][-19:])
for i in range(0, 13 * int(len(iv) / 13), 13)
]
tester.contents(nsc, ernsc)
tester.test(self.test_ctxtype, self.test_config)
def test_throttle(self):
topo = Topology()
s = topo.source(U.Sequence(iterations=40))
s = s.map(U.Throttle(rate=2.0, precise=True))
tester = Tester(topo)
tester.tuple_count(s, 40)
tester.tuple_check(s, ThrottleCheck())
tester.test(self.test_ctxtype, self.test_config)
def test_delay(self):
topo = Topology()
s = U.sequence(topo, iterations=223)
s = U.delay(s, delay=0.4)
tester = Tester(topo)
tester.tuple_count(s, 223)
tester.tuple_check(s, lambda t : (time.time() - t['ts'].time()) > 0.35)
tester.test(self.test_ctxtype, self.test_config)
def test_string(self):
topo = Topology()
s = topo.source(V_Str())
tester = Tester(topo)
tester.tuple_count(s, 2)
tester.tuple_check(s, lambda v: isinstance(v, str))
tester.contents(s, ['Hello', 'World'])
tester.test(self.test_ctxtype, self.test_config)
def test_delay(self):
topo = Topology()
s = topo.source(U.Sequence(iterations=223))
s = s.map(U.Delay(delay=0.4))
tester = Tester(topo)
tester.tuple_count(s, 223)
tester.tuple_check(s, lambda t: (time.time() - t['ts'].time()) > 0.35)
tester.test(self.test_ctxtype, self.test_config)
def test_sequence_period(self):
topo = Topology()
s = U.sequence(topo, iterations=67, period=0.1)
E = U.SEQUENCE_SCHEMA.extend(StreamSchema('tuple<float64 d>'))
s = s.map(_Delta(), schema=E)
tester = Tester(topo)
tester.tuple_check(s, lambda x: x['d'] > 0.08)
tester.tuple_count(s, 67-1)
tester.test(self.test_ctxtype, self.test_config)
def test_checker(self):
""" Test the per-tuple checker.
"""
topo = Topology()
s = topo.source(rands)
s = s.filter(lambda r : r > 0.8)
s = s.map(lambda r : r + 7.0 )
tester = Tester(topo)
tester.tuple_count(s, 200, exact=False)
tester.tuple_check(s, lambda r : r > 7.8)
tester.test(self.test_ctxtype, self.test_config)
def test_checker(self):
""" Test the per-tuple checker.
"""
topo = Topology()
s = topo.source(rands)
s = s.filter(lambda r : r > 0.8)
s = s.map(lambda r : r + 7.0 )
tester = Tester(topo)
tester.tuple_count(s, 200, exact=False)
tester.tuple_check(s, lambda r : r > 7.8)
tester.test(self.test_ctxtype, self.test_config)
def test_BasicCountTimeWindow(self):
# Aggregate every 0.5 seconds
aggregate_period = 0.5
# Check that each aggregation is triggered at the right time, with a maximum %20 error
tolerance = 0.20
topo = Topology()
s = topo.source(TimeCounter(iterations = 10))
s = s.last(1).trigger(datetime.timedelta(seconds=aggregate_period)).aggregate(TriggerDiff())
tester = Tester(topo)
tester.tuple_check(s, lambda val: within_tolerance(val, tolerance, aggregate_period))
tester.test(self.test_ctxtype, self.test_config)
def test_BasicCountTimeWindow(self):
# Aggregate every 0.5 seconds
aggregate_period = 0.5
# Check that each aggregation is triggered at the right time, with a maximum %20 error
tolerance = 0.20
topo = Topology()
s = topo.source(TimeCounter(iterations = 10))
s = s.last(1).trigger(datetime.timedelta(seconds=aggregate_period)).aggregate(TriggerDiff())
tester = Tester(topo)
tester.tuple_check(s, lambda val: within_tolerance(val, tolerance, aggregate_period))
tester.test(self.test_ctxtype, self.test_config)
def test_batch_time(self):
topo = Topology()
s = topo.source(lambda : map(_delay, range(50)), name='A')
b = s.batch(datetime.timedelta(seconds=2))
r = b.aggregate(lambda x : x)
rf = r.flat_map()
tester = Tester(topo)
tester.tuple_count(rf, 50)
tester.run_for((50*0.2) + 20)
tester.tuple_check(r, _BatchTimeCheck())
tester.test(self.test_ctxtype, self.test_config)
def test_batch_time(self):
topo = Topology()
s = topo.source(lambda : map(_delay, range(50)), name='A')
b = s.batch(datetime.timedelta(seconds=2))
r = b.aggregate(lambda x : x)
rf = r.flat_map()
tester = Tester(topo)
tester.tuple_count(rf, 50)
tester.run_for((50*0.2) + 20)
tester.tuple_check(r, _BatchTimeCheck())
tester.test(self.test_ctxtype, self.test_config)
def test_checker(self):
""" Test the per-tuple checker.
"""
topo = Topology()
s = topo.source(rands)
s = s.filter(lambda r : r > 0.8)
s = s.map(lambda r : r + 7.0 )
tester = Tester(topo)
tester.tuple_count(s, 200, exact=False)
if self.test_ctxtype == context.ContextTypes.STANDALONE:
tester.run_for(20)
tester.tuple_check(s, lambda r : r > 7.8)
tester.test(self.test_ctxtype, self.test_config)
def test_checker(self):
""" Test the per-tuple checker.
"""
topo = Topology()
s = topo.source(rands)
s = s.filter(lambda r: r > 0.8)
s = s.map(lambda r: r + 7.0)
tester = Tester(topo)
tester.tuple_count(s, 200, exact=False)
if self.test_ctxtype == context.ContextTypes.STANDALONE:
tester.run_for(20)
tester.tuple_check(s, lambda r: r > 7.8)
tester.test(self.test_ctxtype, self.test_config)
def test_pair(self):
topo = Topology()
s = U.sequence(topo, iterations=932)
rschema = U.SEQUENCE_SCHEMA.extend(StreamSchema('tuple<float64 score>'))
r0 = s.map(lambda t : (t['seq'], t['ts'], 1.0), schema=rschema)
r1 = s.map(lambda t : (t['seq'], t['ts'], 2.0), schema=rschema)
r = U.pair(r0, r1)
tester = Tester(topo)
tester.tuple_count(r, 932*2)
tester.tuple_check(r, PairCheck())
tester.test(self.test_ctxtype, self.test_config)
def test_BasicTimeCountWindow(self):
# Ensure tuples are evicted no later than (window_span*tolerance + window_span)
tolerance = 0.20
window_span = 2.0
max_evict = window_span*tolerance + window_span
topo = Topology()
s = topo.source(TimeCounter(iterations = 100, period = 0.1))
s = s.map(lambda x: (x, time.time()))
s = s.last(datetime.timedelta(seconds=window_span)).trigger(20).aggregate(TupleTimespanCheck(max_evict))
tester = Tester(topo)
tester.tuple_check(s, lambda val: val)
tester.test(self.test_ctxtype, self.test_config)
def test_BasicTimeCountWindow(self):
# Ensure tuples are evicted no later than (window_span*tolerance + window_span)
tolerance = 0.20
window_span = 2.0
max_evict = window_span*tolerance + window_span
topo = Topology()
s = topo.source(TimeCounter(iterations = 100, period = 0.1))
s = s.map(lambda x: (x, time.time()))
s = s.last(datetime.timedelta(seconds=window_span)).trigger(20).aggregate(TupleTimespanCheck(max_evict))
tester = Tester(topo)
tester.tuple_check(s, lambda val: val)
tester.test(self.test_ctxtype, self.test_config)
def test_split(self):
N = 3
T = N * 1000 * 2
topo = Topology()
s = topo.source(lambda: itertools.islice(
iter(lambda: random.randint(-T, T), None), T))
streams = s.split(N, lambda x: x + 2, names=['high', 'medium', 'low'])
self.assertEqual(N, len(streams))
self.assertIsInstance(streams, tuple)
self.assertEqual(('high', 'medium', 'low'), type(streams)._fields)
self.assertIs(streams.high, streams[0])
self.assertIs(streams.medium, streams[1])
self.assertIs(streams.low, streams[2])
tester = Tester(topo)
for i in range(N):
tester.tuple_count(streams[i], 500, exact=False)
tester.tuple_check(streams[0], lambda x:
(x + 2) >= 0 and (x + 2) % N == 0)
tester.tuple_check(streams[1], lambda x:
(x + 2) >= 0 and (x + 2) % N == 1)
tester.tuple_check(streams[2], lambda x:
(x + 2) >= 0 and (x + 2) % N == 2)
single = s.split(1, lambda x: x)
self.assertEqual(1, len(single))
tester.tuple_count(single[0], 1000, exact=False)
tester.tuple_check(single[0], lambda x: x >= 0)
tester.tuple_count(s, T, exact=False)
tester.test(self.test_ctxtype, self.test_config)
def test_gate(self):
N = 137
D = 0.75
G = 17
topo = Topology()
s = topo.source(range(N))
c = PendingStream(topo)
g = U.gate(s, c.stream, max_unacked=G)
g = g.map(lambda _: time.time())
r = g.map(U.Delay(delay=D))
c.complete(r)
tester = Tester(topo)
tester.tuple_count(r, N)
tester.tuple_check(r, GateCheck(G, D))
tester.test(self.test_ctxtype, self.test_config)
def test_checker(self):
""" Test the per-tuple checker.
"""
topo = Topology()
s = topo.source(rands)
s = s.filter(lambda r : r > 0.8)
s = s.map(lambda r : r + 7.0 )
tester = Tester(topo)
tester.tuple_count(s, 200, exact=False)
if self.test_ctxtype == context.ContextTypes.STANDALONE:
tester.run_for(20)
tester.tuple_check(s, lambda r : r > 7.8)
# Ensure we perform dependency checking for the check function
import fns_test2_test
tester.tuple_check(s, fns_test2_test.tc_dep)
tester.test(self.test_ctxtype, self.test_config)
def test_pair_matched(self):
topo = Topology()
s = topo.source([0, 1, 2])
rs = 'tuple<rstring id, int32 v>'
s = s.map(lambda t : (str(t),t + 7), schema=rs)
# "Reorder" tuples as well
r0 = s.map(lambda t : ((int(t['id']) + 1)%3, t['v'] * 2), schema=rs)
r1 = s.map(lambda t : (t['id'], t['v'] * 3), schema=rs)
r = U.pair(r0, r1, matching='id')
tester = Tester(topo)
tester.tuple_count(r, 6)
tester.tuple_check(r, PairMatchedCheck())
tester.test(self.test_ctxtype, self.test_config)
def test_checker(self):
""" Test the per-tuple checker.
"""
topo = Topology()
s = topo.source(rands)
s = s.filter(lambda r: r > 0.8)
s = s.map(lambda r: r + 7.0)
tester = Tester(topo)
tester.tuple_count(s, 200, exact=False)
if self.test_ctxtype == context.ContextTypes.STANDALONE:
tester.run_for(20)
tester.tuple_check(s, lambda r: r > 7.8)
# Ensure we perform dependency checking for the check function
import fns_test2_test
tester.tuple_check(s, fns_test2_test.tc_dep)
tester.test(self.test_ctxtype, self.test_config)
def test_aggregate(self):
iterations = 3000
reset_count = 5
topo = Topology()
# Generate integers from [0,3000)
s = topo.source(TimeCounter(iterations=iterations, period=0.01))
if self.is_cr():
s.set_consistent(ConsistentRegionConfig.periodic(5, drain_timeout=40, reset_timeout=40, max_consecutive_attempts=6))
# Filter the odd ones
s = s.filter(StatefulEvenFilter())
# Halve the even ones and add one. Now have integers [1,(iterations/2))
s = s.map(StatefulHalfPlusOne())
sc = s.last(10).trigger(3).aggregate(StatefulAverage())
st = s.last(17).trigger(datetime.timedelta(seconds=2)).aggregate(StatefulAverage())
# non-stateful aggregation functions
nsc = s.last(19).trigger(13).aggregate(lambda tuples : sum(tuples))
tester = Tester(topo)
if self.is_cr():
tester.resets(reset_count)
# Find the expected results.
# mimic the processing using Python builtins
iv = filter(StatefulEvenFilter(), range(iterations))
iv = list(map(StatefulHalfPlusOne(), iv))
# Expected stateful averages sc,st
sagg = StatefulAverage()
ers = [ sagg(iv[0:i+3][-10:]) for i in range(0, 3*int(len(iv)/3), 3) ]
tester.contents(sc, ers)
tester.tuple_check(st, TimedStatefulAverageChecker())
# Must eventually aggregate on the last 17 items in iv
# but only if cr otherwise the final marker stops
# the window before the final trigger
if self.is_cr():
tester.eventual_result(st, lambda av : True if av[1] == sagg(iv[-17:])[1] else None)
# Expected non-stateful averages nsc
ernsc = [ sum(iv[0:i+13][-19:]) for i in range(0, 13*int(len(iv)/13), 13) ]
tester.contents(nsc, ernsc)
tester.test(self.test_ctxtype, self.test_config)
def test_structured(self):
topo = Topology()
s = topo.source(V_Sensor())
d = s.map(lambda v: v._asdict())
tester = Tester(topo)
tester.tuple_count(s, 2)
tester.tuple_check(s, lambda v: isinstance(v, tuple))
tester.contents(d, [{
'sensor_id': 101,
'ts': 10,
'reading': 3.0
}, {
'sensor_id': 202,
'ts': 20,
'reading': 7.0
}])
tester.test(self.test_ctxtype, self.test_config)
def test_BasicTimeCountWindow_stv_no_default(self):
# Ensure tuples are evicted no later than (window_span*tolerance + window_span)
tolerance = 0.20
window_span = 2.0
max_evict = window_span * tolerance + window_span
topo = Topology()
s = topo.source(TimeCounter(iterations=100, period=0.1))
s = s.map(lambda x: (x, time.time()))
wtime = topo.create_submission_parameter('secs', type_=int)
s = s.lastSeconds(wtime).trigger(20).aggregate(
TupleTimespanCheck(max_evict))
jc = JobConfig()
jc.submission_parameters['secs'] = 2
jc.add(self.test_config)
tester = Tester(topo)
tester.tuple_check(s, lambda val: val)
tester.test(self.test_ctxtype, self.test_config)
def test_timestamp(self):
ts_schema = StreamSchema('tuple<int32 a, timestamp ts>').as_tuple(named=True)
ts1 = Timestamp(133001, 302245576, 56)
ts2s = Timestamp(23543463, 876265298, 32)
dt1 = ts2s.datetime()
ts2 = Timestamp.from_datetime(dt1)
self.assertEqual(ts2s.seconds, ts2s.seconds);
topo = Topology()
s = topo.source([(1,ts1), (2,dt1)])
s = s.map(lambda x : x, schema=ts_schema)
as_ts = s.map(lambda x : x.ts.tuple())
s.print()
tester = Tester(topo)
tester.tuple_check(s, ts_check)
tester.tuple_count(s, 2)
tester.contents(as_ts, [ts1.tuple(), ts2.tuple()])
tester.test(self.test_ctxtype, self.test_config)
def test_timestamp(self):
ts_schema = StreamSchema('tuple<int32 a, timestamp ts>').as_tuple(named=True)
ts1 = Timestamp(133001, 302245576, 56)
ts2s = Timestamp(23543463, 876265298, 32)
dt1 = ts2s.datetime()
ts2 = Timestamp.from_datetime(dt1)
self.assertEqual(ts2s.seconds, ts2s.seconds);
topo = Topology()
s = topo.source([(1,ts1), (2,dt1)])
s = s.map(lambda x : x, schema=ts_schema)
as_ts = s.map(lambda x : x.ts.tuple())
s.print()
tester = Tester(topo)
tester.tuple_check(s, ts_check)
tester.tuple_count(s, 2)
tester.contents(as_ts, [ts1.tuple(), ts2.tuple()])
tester.test(self.test_ctxtype, self.test_config)
class TestPubSub(unittest.TestCase):
""" Test basic pub-sub in SPL
"""
def setUp(self):
Tester.setup_distributed(self)
def _publish(self, topo, N, topic, width=None, allow_filter=False):
b = op.Source(topo, "spl.utility::Beacon",
SCHEMA,
params = {'initDelay':10.0, 'iterations':N})
b.seq = b.output('IterationCount()')
ps = b.stream
if width:
ps = ps.parallel(width=width)
p = op.Sink('com.ibm.streamsx.topology.topic::Publish',
ps,
params = {'topic': topic},
name='MSP')
if allow_filter:
p.params['allowFilter'] = True
def _subscribe(self, topo, topic, direct=True, drop=None, filtered=False, extra=None):
s = op.Source(topo,
"com.ibm.streamsx.topology.topic::FilteredSubscribe" if filtered else "com.ibm.streamsx.topology.topic::Subscribe",
SCHEMA,
params = {'topic':topic, 'streamType':SCHEMA},
name='MSS')
if extra:
s.params.update(extra)
if not direct:
s.params['connect'] = op.Expression.expression('com.ibm.streamsx.topology.topic::Buffered')
if drop:
s.params['bufferFullPolicy'] = op.Expression.expression('Sys.' + drop)
return s.stream.filter(slowme)
return s.stream
def _get_single_sub_op(self):
job = self.tester.submission_result.job
self.assertEqual('healthy', job.health)
for op in job.get_operators():
if op.name.startswith('MSS') and op.operatorKind == 'spl.relational::Filter':
mss = op
return mss
def _get_single_sub_metrics(self, mss):
nDropped = None
nProcessed = None
ip = mss.get_input_ports()[0]
while nDropped is None or nProcessed is None:
if nDropped is None:
metrics = ip.get_metrics(name='nTuplesDropped')
if metrics:
nDropped = metrics[0]
if nProcessed is None:
metrics = ip.get_metrics(name='nTuplesProcessed')
if metrics:
nProcessed = metrics[0]
return nDropped, nProcessed
def check_single_sub(self):
"""
Check we get all the tuples with none dropped
with a single subcriber.
"""
mss = self._get_single_sub_op()
nDropped, nProcessed = self._get_single_sub_metrics(mss)
while nProcessed.value < self.N:
self.assertEqual(0, nDropped.value)
time.sleep(2)
nDropped, nProcessed = self._get_single_sub_metrics(mss)
self.assertEqual(0, nDropped.value)
self.assertEqual(self.N, nProcessed.value)
def check_single_sub_drop(self):
"""
Check we get all the tuples with none dropped
with a single subcriber.
"""
mss = self._get_single_sub_op()
nDropped, nProcessed = self._get_single_sub_metrics(mss)
while nDropped.value + nProcessed.value < self.N:
time.sleep(2)
nDropped, nProcessed = self._get_single_sub_metrics(mss)
self.assertEqual(self.N, nDropped.value + nProcessed.value)
self.assertTrue(nDropped.value > 0)
def test_One2One(self):
"""Publish->Subscribe
"""
N = 2466
topic = random_topic()
topo = Topology()
# Subscriber
s = self._subscribe(topo, topic)
# Publisher
self._publish(topo, N, topic)
self.tester = Tester(topo)
self.tester.run_for(15)
self.N = N
self.tester.tuple_count(s, N)
self.tester.local_check = self.check_single_sub
self.tester.test(self.test_ctxtype, self.test_config)
def test_One2OneNonDirect(self):
"""Publish->Subscribe with a buffered subscriber.
"""
N = 3252
topic = random_topic()
topo = Topology()
# Subscriber
s = self._subscribe(topo, topic, direct=False)
# Publisher
self._publish(topo, N, topic)
self.tester = Tester(topo)
self.tester.run_for(15)
self.N = N
self.tester.tuple_count(s, N)
self.tester.local_check = self.check_single_sub
self.tester.test(self.test_ctxtype, self.test_config)
def test_One2OneNonDirectDropFirst(self):
"""Publish->Subscribe with a buffered subscriber.
"""
N = 5032
topic = random_topic()
topo = Topology()
# Subscriber
s = self._subscribe(topo, topic, direct=False, drop='DropFirst')
# Publisher
self._publish(topo, N, topic)
self.tester = Tester(topo)
self.tester.run_for(15)
self.N = N
# 1000-2 for window & final mark
self.tester.tuple_count(s, 998, exact=False)
self.tester.local_check = self.check_single_sub_drop
self.tester.test(self.test_ctxtype, self.test_config)
def test_One2OneNonDirectDropLast(self):
"""Publish->Subscribe with a buffered subscriber.
"""
N = 5032
topic = random_topic()
topo = Topology()
# Subscriber
s = self._subscribe(topo, topic, direct=False, drop='DropLast')
# Publisher
self._publish(topo, N, topic)
self.tester = Tester(topo)
self.tester.run_for(15)
self.N = N
self.tester.tuple_count(s, 1000, exact=False)
self.tester.local_check = self.check_single_sub_drop
self.tester.test(self.test_ctxtype, self.test_config)
def test_UDPMany2One(self):
"""
UDP publishers to a single subscriber.
"""
N = 17342
for pw in (1,5):
topic = random_topic()
topo = Topology()
# Subscriber
s = self._subscribe(topo, topic)
# Publisher
self._publish(topo, N, topic, width=pw)
self.tester = Tester(topo)
self.tester.run_for(15)
self.tester.tuple_count(s, N)
self.N = N
self.tester.local_check = self.check_single_sub
self.tester.test(self.test_ctxtype, self.test_config)
def test_Many2One(self):
"""
Many non-UDP publishers to a single subscriber.
"""
N = 17342
topic = random_topic()
topo = Topology()
# Subscriber
s = self._subscribe(topo, topic)
# Publisher
M=3
for i in range(M):
self._publish(topo, N, topic)
self.tester = Tester(topo)
self.tester.run_for(15)
self.tester.tuple_count(s, N*M)
self.N = N*M
self.tester.local_check = self.check_single_sub
self.tester.test(self.test_ctxtype, self.test_config)
def test_allow_filter_subscribe(self):
N = 99
topic = random_topic()
topo = Topology()
# Non-Filter Subscriber
s = self._subscribe(topo, topic)
# Publisher
self._publish(topo, N, topic, allow_filter=True)
self.tester = Tester(topo)
self.tester.run_for(15)
self.N = N
self.tester.tuple_count(s, N)
self.tester.local_check = self.check_single_sub
self.tester.test(self.test_ctxtype, self.test_config)
def test_allow_filter_filtered_subscribe(self):
N = 201
F = 87
pd = os.path.dirname(os.path.dirname(__file__))
ttk = os.path.join(pd, 'testtk')
for af,d in [(False,False), (False,True), (True,False), (True,True)]:
#af = c[0]
#d = c[1]
topic = random_topic()
topo = Topology()
streamsx.spl.toolkit.add_toolkit(topo, ttk)
extra = {}
extra['remoteFilter'] = 'seq < 87'
extra['localFilterFunction'] = op.Expression.expression('testspl::affs')
s = self._subscribe(topo, topic, direct=d, filtered=True, extra=extra)
sf = s.filter(check_lt_87)
# Publisher
self._publish(topo, N, topic, allow_filter=af)
self.tester = Tester(topo)
self.tester.run_for(15)
self.N = F
self.tester.tuple_count(s, F)
self.tester.tuple_check(s, check_lt_87)
#self.tester.local_check = self.check_single_sub
self.tester.test(self.test_ctxtype, self.test_config)
class TestDistributedPubSub(unittest.TestCase):
""" Test basic pub-sub in SPL
"""
def setUp(self):
Tester.setup_distributed(self)
self.test_config[ConfigParams.SSL_VERIFY] = False
def _publish(self, topo, N, topic, width=None, allow_filter=False):
b = op.Source(topo,
"spl.utility::Beacon",
SCHEMA,
params={
'initDelay': 10.0,
'iterations': N
})
b.seq = b.output('IterationCount()')
ps = b.stream
if width:
ps = ps.parallel(width=width)
p = op.Sink('com.ibm.streamsx.topology.topic::Publish',
ps,
params={'topic': topic},
name='MSP')
if allow_filter:
p.params['allowFilter'] = True
def _subscribe(self,
topo,
topic,
direct=True,
drop=None,
filtered=False,
extra=None):
s = op.Source(
topo,
"com.ibm.streamsx.topology.topic::FilteredSubscribe"
if filtered else "com.ibm.streamsx.topology.topic::Subscribe",
SCHEMA,
params={
'topic': topic,
'streamType': SCHEMA
},
name='MSS')
if extra:
s.params.update(extra)
if not direct:
s.params['connect'] = op.Expression.expression(
'com.ibm.streamsx.topology.topic::Buffered')
if drop:
s.params['bufferFullPolicy'] = op.Expression.expression(
'Sys.' + drop)
return s.stream.filter(slowme)
return s.stream
def _get_single_sub_op(self):
job = self.tester.submission_result.job
self.assertEqual('healthy', job.health)
for op in job.get_operators():
if op.name.startswith(
'MSS') and op.operatorKind == 'spl.relational::Filter':
mss = op
return mss
def _get_single_sub_metrics(self, mss):
nDropped = None
nProcessed = None
ip = mss.get_input_ports()[0]
while nDropped is None or nProcessed is None:
if nDropped is None:
metrics = ip.get_metrics(name='nTuplesDropped')
if metrics:
nDropped = metrics[0]
if nProcessed is None:
metrics = ip.get_metrics(name='nTuplesProcessed')
if metrics:
nProcessed = metrics[0]
return nDropped, nProcessed
def check_single_sub(self):
"""
Check we get all the tuples with none dropped
with a single subcriber.
"""
mss = self._get_single_sub_op()
nDropped, nProcessed = self._get_single_sub_metrics(mss)
while nProcessed.value < self.N:
self.assertEqual(0, nDropped.value)
time.sleep(2)
nDropped, nProcessed = self._get_single_sub_metrics(mss)
self.assertEqual(0, nDropped.value)
self.assertEqual(self.N, nProcessed.value)
def check_single_sub_drop(self):
"""
Check we get all the tuples with none dropped
with a single subcriber.
"""
mss = self._get_single_sub_op()
nDropped, nProcessed = self._get_single_sub_metrics(mss)
while nDropped.value + nProcessed.value < self.N:
time.sleep(2)
nDropped, nProcessed = self._get_single_sub_metrics(mss)
self.assertEqual(self.N, nDropped.value + nProcessed.value)
self.assertTrue(nDropped.value > 0)
def test_One2One(self):
"""Publish->Subscribe
"""
N = 2466
topic = random_topic()
topo = Topology()
# Subscriber
s = self._subscribe(topo, topic)
# Publisher
self._publish(topo, N, topic)
self.tester = Tester(topo)
self.tester.run_for(15)
self.N = N
self.tester.tuple_count(s, N)
self.tester.local_check = self.check_single_sub
self.tester.test(self.test_ctxtype, self.test_config)
def test_One2OneNonDirect(self):
"""Publish->Subscribe with a buffered subscriber.
"""
N = 3252
topic = random_topic()
topo = Topology()
# Subscriber
s = self._subscribe(topo, topic, direct=False)
# Publisher
self._publish(topo, N, topic)
self.tester = Tester(topo)
self.tester.run_for(15)
self.N = N
self.tester.tuple_count(s, N)
self.tester.local_check = self.check_single_sub
self.tester.test(self.test_ctxtype, self.test_config)
def test_One2OneNonDirectDropFirst(self):
"""Publish->Subscribe with a buffered subscriber.
"""
N = 5032
topic = random_topic()
topo = Topology()
# Subscriber
s = self._subscribe(topo, topic, direct=False, drop='DropFirst')
# Publisher
self._publish(topo, N, topic)
self.tester = Tester(topo)
self.tester.run_for(15)
self.N = N
# 1000-2 for window & final mark
self.tester.tuple_count(s, 998, exact=False)
self.tester.local_check = self.check_single_sub_drop
self.tester.test(self.test_ctxtype, self.test_config)
def test_One2OneNonDirectDropLast(self):
"""Publish->Subscribe with a buffered subscriber.
"""
N = 5032
topic = random_topic()
topo = Topology()
# Subscriber
s = self._subscribe(topo, topic, direct=False, drop='DropLast')
# Publisher
self._publish(topo, N, topic)
self.tester = Tester(topo)
self.tester.run_for(15)
self.N = N
self.tester.tuple_count(s, 1000, exact=False)
self.tester.local_check = self.check_single_sub_drop
self.tester.test(self.test_ctxtype, self.test_config)
def test_UDPMany2One(self):
"""
UDP publishers to a single subscriber.
"""
N = 17342
for pw in (1, 5):
topic = random_topic()
topo = Topology()
# Subscriber
s = self._subscribe(topo, topic)
# Publisher
self._publish(topo, N, topic, width=pw)
self.tester = Tester(topo)
self.tester.run_for(15)
self.tester.tuple_count(s, N)
self.N = N
self.tester.local_check = self.check_single_sub
self.tester.test(self.test_ctxtype, self.test_config)
def test_Many2One(self):
"""
Many non-UDP publishers to a single subscriber.
"""
N = 17342
topic = random_topic()
topo = Topology()
# Subscriber
s = self._subscribe(topo, topic)
# Publisher
M = 3
for i in range(M):
self._publish(topo, N, topic)
self.tester = Tester(topo)
self.tester.run_for(15)
self.tester.tuple_count(s, N * M)
self.N = N * M
self.tester.local_check = self.check_single_sub
self.tester.test(self.test_ctxtype, self.test_config)
def test_allow_filter_subscribe(self):
N = 99
topic = random_topic()
topo = Topology()
# Non-Filter Subscriber
s = self._subscribe(topo, topic)
# Publisher
self._publish(topo, N, topic, allow_filter=True)
self.tester = Tester(topo)
self.tester.run_for(15)
self.N = N
self.tester.tuple_count(s, N)
self.tester.local_check = self.check_single_sub
self.tester.test(self.test_ctxtype, self.test_config)
def test_allow_filter_filtered_subscribe(self):
N = 201
F = 87
pd = os.path.dirname(os.path.dirname(__file__))
ttk = os.path.join(pd, 'testtk')
for af, d in [(False, False), (False, True), (True, False),
(True, True)]:
#af = c[0]
#d = c[1]
topic = random_topic()
topo = Topology()
streamsx.spl.toolkit.add_toolkit(topo, ttk)
extra = {}
extra['remoteFilter'] = 'seq < 87'
extra['localFilterFunction'] = op.Expression.expression(
'testspl::affs')
s = self._subscribe(topo,
topic,
direct=d,
filtered=True,
extra=extra)
sf = s.filter(check_lt_87)
# Publisher
self._publish(topo, N, topic, allow_filter=af)
self.tester = Tester(topo)
self.tester.run_for(15)
self.N = F
self.tester.tuple_count(s, F)
self.tester.tuple_check(s, check_lt_87)
#self.tester.local_check = self.check_single_sub
self.tester.test(self.test_ctxtype, self.test_config)
