def test_deduplicate(self):
topo = Topology()
s = topo.source([1, 2, 1, 4, 5, 2])
s = s.map(lambda v: {'a': v}, schema='tuple<int32 a>')
s = s.map(U.Deduplicate(count=3))
s = s.map(lambda v: v['a'])
tester = Tester(topo)
tester.contents(s, [1, 2, 4, 5, 2])
tester.test(self.test_ctxtype, self.test_config)
def test_source(self):
topo = Topology("test")
topo.checkpoint_period = timedelta(seconds=1)
s = topo.source(TimeCounter(iterations=30, period=0.1))
tester = Tester(topo)
tester.contents(s, range(0, 30))
tester.test(self.test_ctxtype,
self.test_config,
always_collect_logs=True)
def test_TopologySourceFn(self):
topo = Topology('test_TopologySourceFn')
hw = topo.source(s4)
tester = Tester(topo)
tester.contents(hw, s4())
tester.tuple_count(hw, len(s4()))
self.test_config['topology.keepArtifacts'] = True
tester.test(self.test_ctxtype, self.test_config)
self.result = tester.result['submission_result']
verifyArtifacts(self)
def test_sequence_period(self):
topo = Topology()
s = topo.source(U.Sequence(iterations=67, period=0.1))
E = U.SEQUENCE_SCHEMA.extend(StreamSchema('tuple<float64 d>'))
s = s.map(_Delta(), schema=E)
tester = Tester(topo)
tester.tuple_check(s, lambda x: x['d'] > 0.08)
tester.tuple_count(s, 67 - 1)
tester.test(self.test_ctxtype, self.test_config)
def test_structured_as_dict(self):
topo = Topology()
s = topo.source([('a',1),('b', 7),('c', 2),('d', 9)])
s = s.map(lambda x: x, schema = "tuple<rstring a, int32 b>")
s = s.last(3).trigger(2).aggregate(lambda items: (items[0]['a'],items[1]['b']))
tester = Tester(topo)
tester.contents(s, [('a',7), ('b',2)] )
tester.test(self.test_ctxtype, self.test_config)
def test_subscribe_buffered_capacity(self):
topic = ''.join(random.choice('0123456789abcdef') for x in range(20))
topo = Topology()
N = 789
s = topo.subscribe(topic=topic, connect=SubscribeConnection.Buffered, buffer_capacity=N)
self._buf_size = N
self.tester = Tester(topo)
self.tester.local_check = self._check_buffer
self.tester.test(self.test_ctxtype, self.test_config)
def test_object_to_string(self):
topo = Topology()
s = topo.source([93, 'hello', True])
st = s.map(lambda x: x, schema=CommonSchema.String)
st2 = s.map(schema=unicode if sys.version_info.major == 2 else str)
tester = Tester(topo)
tester.contents(st, ['93', 'hello', 'True'])
tester.contents(st2, ['93', 'hello', 'True'])
tester.test(self.test_ctxtype, self.test_config)
def test_DictInputWindow(self):
topo = Topology()
s = topo.source([1, 2, 3, 4])
s = s.map(lambda x: ('a', x), schema="tuple<rstring a, int32 b>")
# Canned aggregate
s = s.last(3).trigger(4).aggregate(lambda x: 0),
tester = Tester(topo)
tester.test(self.test_ctxtype, self.test_config)
def test_python_operator(self):
topo = Topology('test_python_operator')
s = topo.source(['0','1','2','3','4','five','6','7','8','9']).as_string()
num_stream = s.map(lambda t: {'num': int(t)}, schema=NumbersSchema)
num_stream.catch_exceptions()
tester = Tester(topo)
tester.tuple_count(num_stream, 10-1)
tester.test(self.test_ctxtype, self.test_config)
def test_local_capture(self):
topo = Topology("test_local_capture")
n = topo.source([1, 2, 4])
x = 93
n = n.map(lambda v: v + x)
tester = Tester(topo)
tester.contents(n, [94, 95, 97])
tester.tuple_count(n, 3)
tester.test(self.test_ctxtype, self.test_config)
def test_setup_checkpoint(self):
topo = Topology("test")
topo.checkpoint_period = timedelta(seconds=1)
s = topo.source(dillable_source(3))
tester = Tester(topo)
tester.contents(s, [1, 2, 3])
tester.test(self.test_ctxtype,
self.test_config,
always_collect_logs=True)
def test_TopologyFilterNonMatching(self):
topo = Topology("test_TopologyFilterNonMatching")
hw = topo.source(['Hello', 'World'])
hwf, nm = hw.filter((lambda t: "Wor" in t), non_matching=True)
tester = Tester(topo)
self.test_config['topology.keepArtifacts'] = True
tester.contents(hwf, ['World'])
tester.contents(nm, ['Hello'])
tester.test(self.test_ctxtype, self.test_config)
def test_as_tuple_hash(self):
topo = Topology()
s = self._create_stream(topo)
s = s.parallel(width=2, routing=Routing.HASH_PARTITIONED, func=check_is_tuple_hash)
s = s.map(check_is_tuple_map)
s = s.end_parallel()
tester = Tester(topo)
tester.contents(s, [(7,'2Hi!-Map'), (14,'4Hi!-Map'), (21,'6Hi!-Map')], ordered=False)
tester.test(self.test_ctxtype, self.test_config)
def test_class_source(self):
count = 43
topo = Topology()
streamsx.spl.toolkit.add_toolkit(topo, '../testtkpy')
bop = op.Source(topo, "com.ibm.streamsx.topology.pysamples.sources::Range", schema.StreamSchema('tuple<int64 c>').as_tuple(), params={'count':count})
r = bop.stream
self.tester = Tester(topo)
self.tester.tuple_count(r, count)
self.tester.contents(r, list(zip(range(count))))
self.tester.test(self.test_ctxtype, self.test_config)
def test_object_to_json(self):
topo = Topology()
s = topo.source([{'a': 7}, {'b': 8}, {'c': 9}])
st = s.map(lambda x: x, schema=CommonSchema.Json)
st2 = s.map(schema=json)
tester = Tester(topo)
tester.contents(st, [{'a': 7}, {'b': 8}, {'c': 9}])
tester.contents(st2, [{'a': 7}, {'b': 8}, {'c': 9}])
tester.test(self.test_ctxtype, self.test_config)
def test_as_string(self):
topo = Topology()
s = topo.source(['String!', 3, 42.0])
s = s.as_string()
s2 = s.as_string()
self.assertIs(s, s2)
tester = Tester(topo)
tester.contents(s, ['String!', '3', '42.0'])
tester.test(self.test_ctxtype, self.test_config)
def test_count_bad(self):
N = 10
topo = Topology()
s = topo.source(range(N))
tester = Tester(topo)
tester.tuple_count(s, N + 1)
ok = tester.test(self.test_ctxtype,
self.test_config,
assert_on_fail=False)
self.assertFalse(ok)
def test_BasicCountCountWindow(self):
topo = Topology()
s = topo.source([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15])
# Need a float cast to make the value consistent with Python 2/3
s = s.last(10).trigger(2).aggregate(
lambda x: float(sum(x)) / float(len(x)))
tester = Tester(topo)
tester.contents(s, [1.5, 2.5, 3.5, 4.5, 5.5, 7.5, 9.5])
tester.test(self.test_ctxtype, self.test_config)
def test_TopologySetParallel(self):
topo = Topology("test_TopologySetParallel")
s = topo.source([1])
s.set_parallel(5)
s = s.end_parallel()
tester = Tester(topo)
tester.contents(s, [1, 1, 1, 1, 1])
tester.test(self.test_ctxtype, self.test_config)
print(tester.result)
def test_batch_count(self):
topo = Topology()
s = topo.source(lambda : range(20))
b = s.batch(4)
r = b.aggregate(lambda items : sum(items))
tester = Tester(topo)
tester.contents(r, [0+1+2+3,4+5+6+7,8+9+10+11,12+13+14+15,16+17+18+19])
tester.tuple_count(r, 5)
tester.test(self.test_ctxtype, self.test_config)
def test_fn_source(self):
count = 37
topo = Topology()
streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('testtkpy'))
bop = op.Source(topo, "com.ibm.streamsx.topology.pysamples.sources::Range37", schema.StreamSchema('tuple<int64 c>').as_tuple())
r = bop.stream
self.tester = Tester(topo)
self.tester.tuple_count(r, count)
self.tester.contents(r, list(zip(range(count))))
self.tester.test(self.test_ctxtype, self.test_config)
def test_subscribe_buffered_drop(self):
topic = ''.join(random.choice('0123456789abcdef') for x in range(20))
topo = Topology()
N = 563
s = topo.subscribe(topic=topic, connect=SubscribeConnection.Buffered, buffer_capacity=N, buffer_full_policy = streamsx.types.CongestionPolicy.DropLast)
self._buf_size = N
self.tester = Tester(topo)
self.tester.local_check = self._check_buffer
self.tester.test(self.test_ctxtype, self.test_config)
def test_as_tuple_map_to_string(self):
topo = Topology()
s = self._create_stream(topo)
st = s.map(check_is_tuple_map_to_string(self.is_named()),
schema=CommonSchema.String)
tester = Tester(topo)
tester.contents(
st, ['112Hi!-MapString', '224Hi!-MapString', '336Hi!-MapString'])
tester.test(self.test_ctxtype, self.test_config)
def test_punct_replaces_tuple(self):
topo = Topology("test_punct_replaces_tuple")
s = topo.source(generate_numbers_for_named_tuple_schema)
s = s.punctor(func=(lambda t: True == t.punct_flag), replace=True)
s = s.map(lambda x: (x.value, ), schema='tuple<int32 z>')
s.print(write_punctuations=True)
tester = Tester(topo)
tester.tuple_count(s, 8)
tester.punct_count(s, 2)
tester.test(self.test_ctxtype, self.test_config)
def test_add_pip_package_whl_from_url(self):
topo = Topology()
topo.add_pip_package(
'https://github.com/IBMStreams/streamsx.topology/raw/develop/test/python/topology/test_package_whl/whl/tstexamplepkg-1.0-py3-none-any.whl',
name='tstexamplepkg')
s = topo.source([1])
s = s.map(lambda x: __import__('tstexamplepkg').__name__)
tester = Tester(topo)
tester.contents(s, ['tstexamplepkg'])
tester.test(self.test_ctxtype, self.test_config)
def test_ByRef(self):
topo = Topology()
s = topo.source(['ByRef', 3, list(('a', 42))])
s = s.map(lambda x: x)
s = s.flat_map(lambda x: x if isinstance(x, list) else [x])
s.for_each(CheckForEach())
tester = Tester(topo)
tester.contents(s, ['ByRef', 3, 'a', 42])
tester.test(self.test_ctxtype, self.test_config)
def _runTest(self, topo, s):
#s.print()
#self.test_config = {
# ConfigParams.SSL_VERIFY: False,
# 'topology.keepArtifacts': True
#}
tester = Tester(topo)
tester.tuple_count(s, 10)
tester.test(self.test_ctxtype, self.test_config)
def test_basic(self):
topo = Topology("Basic")
self.tester = Tester(topo)
tk.add_toolkit(topo, inetToolkit)
# Loop back not natural in a directed graph, need
# to have place holder while the graph gets built,
# At the end of the graph, connect back to the
# begining.
pending_source = PendingStream(topo)
# Directed graph has start this loop loop does not.
# Need to give topology an "onRamp" so it can build the graph.
rsp = pending_source.stream.map(lambda t : t)
ss = topo.source([], name="onRamp")
rsp = ss.union({rsp})
# FormatResponse :
rspFormatted = rsp.map(lambda x : json.dumps(x) ).as_string();
rawRequest = op.Map("com.ibm.streamsx.inet.rest::HTTPRequestProcess",
stream=rspFormatted,
schema='tuple<int64 key, rstring request, rstring method, rstring pathInfo >',
params={'port': PORT,
'webTimeout':5.0,
'responseJsonAttributeName':'string',
'context':'base',
'contextResourceBase':'opt/base' },
name = "RequestProcess")
rawRequest.stream.sink(webEntryLog) ## log what we have received.
# determine what to work on
onlyTuple = rawRequest.stream.filter(lambda t : t["pathInfo"]=="/Tuple",
name="inputFilter")
# do the work
upperDone = onlyTuple.transform(upperString,
name="upperCase")
#
self.tester.tuple_count(upperDone, 1)
# loopback to sending
pending_source.complete(upperDone) # loopback
## All done building the graph......
self.test_config['topology.keepArtifacts']=True
# setup the code that will invoke this test.
self.tester.local_check = self.basic_request
# submit the application for test
self.tester.test(self.test_ctxtype, self.test_config)
def test_map_foreach(self):
topo = Topology()
topo.checkpoint_period = timedelta(seconds=1)
streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('testtkpy'))
timeCounter = op.Source(topo, "com.ibm.streamsx.topology.pytest.checkpoint::TimeCounter", schema.StreamSchema('tuple<int32 f>').as_tuple(), params={'iterations':30,'period':0.1})
fizzbuzz = op.Map("com.ibm.streamsx.topology.pytest.checkpoint::FizzBuzzMap", timeCounter.stream, schema.StreamSchema('tuple<int32 f, rstring c>').as_tuple())
verify = op.Sink("com.ibm.streamsx.topology.pytest.checkpoint::Verify", fizzbuzz.stream)
s = fizzbuzz.stream
tester = Tester(topo)
tester.tuple_count(s, 30)
tester.test(self.test_ctxtype, self.test_config)
def test_no_func_flat_map(self):
topo = Topology()
s = topo.source(['World', 'Cup', '2018'])
s1 = s.flat_map()
s2 = s.flat_map(name='JustFlatten!')
tester = Tester(topo)
tester.contents(s1, 'WorldCup2018')
tester.tuple_count(s1, 12)
tester.contents(s2, 'WorldCup2018')
tester.tuple_count(s2, 12)
tester.test(self.test_ctxtype, self.test_config)
