def test_primitive_foreach(self):
iterations = 3000
topo = Topology()
topo.checkpoint_period = timedelta(seconds=1)
streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('testtkpy'))
timeCounter = op.Source(
topo,
"com.ibm.streamsx.topology.pytest.checkpoint::TimeCounter",
schema.StreamSchema('tuple<int32 f>').as_tuple(),
params={
'iterations': iterations,
'period': 0.01
})
timeCounter.stream.set_consistent(
ConsistentRegionConfig.periodic(5,
drain_timeout=40,
reset_timeout=40,
max_consecutive_attempts=6))
fizzbuzz = op.Map(
"com.ibm.streamsx.topology.pytest.checkpoint::FizzBuzzPrimitive",
timeCounter.stream,
schema.StreamSchema('tuple<int32 f, rstring c>').as_tuple())
verify = op.Sink("com.ibm.streamsx.topology.pytest.checkpoint::Verify",
fizzbuzz.stream)
s = fizzbuzz.stream
tester = Tester(topo)
tester.resets()
tester.tuple_count(s, iterations)
tester.test(self.test_ctxtype, self.test_config)
def test_WindowPunctuation(self):
"""Trigger an aggregation 4 times. Ensure that window punctuations are submitted each time
by writing them to an output file, and then verifying that the file contains the correct
contents."""
topo = Topology()
s = topo.source([1,2,3,4])
# Aggregate and write to file.
s = s.last(1).trigger(1).aggregate(lambda x: x[0]+7)
# Ensure map/flat_map/filter passes window marks through.
s = s.flat_map(lambda x : [x])
s = s.filter(lambda x : True)
s = s.map(lambda x : (x,), schema='tuple<int32 z>')
op_params = {'file' : 'punct_file', 'writePunctuations' : True, 'flushOnPunctuation' : True}
op.Sink("spl.adapter::FileSink", s, params = op_params)
# Copy the config, since it's shared across all tests, and not every test needs a data
# directory.
cfg = self.test_config.copy()
jc = context.JobConfig(data_directory=os.getcwd())
jc.add(cfg)
tester = Tester(topo)
tester.test(self.test_ctxtype, cfg)
path = os.path.join(os.getcwd(), 'punct_file')
# Validate the contents of the file.
with open(path, 'r') as f:
file_contents = f.read()
self.assertEqual(expected_contents, file_contents)
os.remove(path)
def test_single_output_port_punct_forward(self):
"""Operator with receives and forwards window marker."""
topo = Topology()
streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('testtkpy'))
s = topo.source([1, 2, 3, 4])
s = s.punctor(func=(lambda t: 2 < t), before=False)
s = s.map(lambda x: (x, ), schema='tuple<int32 z>')
s1 = op.Map(
"com.ibm.streamsx.topology.pytest.pyprimitives::SingleOutputPort",
s) # has no on_punct
bop = op.Map(
"com.ibm.streamsx.topology.pytest.pyprimitives::SingleOutputPortPunctForward",
s) # implements on_punct
r = bop.stream
op.Sink("com.ibm.streamsx.topology.pytest.pyprimitives::VerifyPosInt",
r)
r.print(write_punctuations=True)
#self.test_config['topology.keepArtifacts'] = True
self.tester = Tester(topo)
self.tester.tuple_count(r, 4)
self.tester.punct_count(r, 2)
self.tester.test(self.test_ctxtype, self.test_config)
def test_filer_for_each(self):
topo = Topology()
streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('testtkpy'))
s = topo.source([(1, 2, 3), (1, 2, 201), (2, 2, 301), (3, 4, 401),
(5, 6, 78), (8, 6, 501), (803, 9324, 901)])
sch = 'tuple<int32 a, int32 b, int32 c>'
s = s.map(lambda x: x, schema=sch)
bop = op.Map("com.ibm.streamsx.topology.pytest.pykwargs::KWFilter", s)
op.Sink("com.ibm.streamsx.topology.pytest.pykwargs::KWForEach",
bop.stream)
tester = Tester(topo)
tester.contents(bop.stream, [{
'a': 1,
'b': 2,
'c': 201
}, {
'a': 3,
'b': 4,
'c': 401
}, {
'a': 803,
'b': 9324,
'c': 901
}])
tester.test(self.test_ctxtype, self.test_config)
def wikiFeed(inetToolkit, buildType, port):
# Sumbit request Build Server and Submit.
schemaTicker = 'tuple<rstring dataId, rstring bot, rstring domain, int32 lenOld,int32 lenNew, rstring SSEdata, rstring SSEevent>'
#
# Define the application
#
topo = Topology("WikiFeed")
tk.add_toolkit(topo, inetToolkit)
source = topo.source(wikiStream)
# Only one type of event, verity that I got it right.
event = source.filter(lambda t: t["SSEevent"] == "message",
name="eventFilter")
event.print(name="eth")
eventTuple = event.map(lambda t: t, schema=schemaTicker)
eventWin = eventTuple.last(100).trigger(1)
rawRequest = op.Sink("com.ibm.streamsx.inet.rest::HTTPTupleView",
stream=eventWin,
params={
'port': 8081,
'context': 'gdaxEth',
'contextResourceBase': '/base'
},
name="TupleView")
#
# Compile & Submit the Topology to Streams instance
#
streams_conf = common.build_streams_config("StreamingTurbine",
credential.serviceCredentials)
context.submit(context.ContextTypes.STREAMING_ANALYTICS_SERVICE,
topo,
config=streams_conf)
def _test_punct_file(self,
topo,
s,
test_name,
expected_content,
expected_tuple_count,
expected_punct_count=None):
s = s.map(lambda x: (x, ), schema='tuple<int32 z>')
op_params = {
'file': 'punct_file_' + test_name,
'writePunctuations': True,
'flushOnPunctuation': True
}
op.Sink("spl.adapter::FileSink", s, params=op_params)
# Copy the config, since it's shared across all tests, and not every test needs a data
# directory.
cfg = self.test_config.copy()
jc = JobConfig(data_directory=os.getcwd())
jc.add(cfg)
tester = Tester(topo)
tester.tuple_count(s, expected_tuple_count)
if expected_punct_count is not None:
tester.punct_count(s, expected_punct_count)
tester.test(self.test_ctxtype, cfg)
path = os.path.join(os.getcwd(), 'punct_file_' + test_name)
# Validate the contents of the file.
with open(path, 'r') as f:
file_contents = f.read()
self.assertEqual(expected_content, file_contents)
os.remove(path)
def test_map_foreach(self):
topo = Topology()
topo.checkpoint_period = timedelta(seconds=1)
streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('testtkpy'))
timeCounter = op.Source(topo, "com.ibm.streamsx.topology.pytest.checkpoint::TimeCounter", schema.StreamSchema('tuple<int32 f>').as_tuple(), params={'iterations':30,'period':0.1})
fizzbuzz = op.Map("com.ibm.streamsx.topology.pytest.checkpoint::FizzBuzzMap", timeCounter.stream, schema.StreamSchema('tuple<int32 f, rstring c>').as_tuple())
verify = op.Sink("com.ibm.streamsx.topology.pytest.checkpoint::Verify", fizzbuzz.stream)
s = fizzbuzz.stream
tester = Tester(topo)
tester.tuple_count(s, 30)
tester.test(self.test_ctxtype, self.test_config)
def test_single_input_port(self):
"""Operator with one input port"""
topo = Topology()
streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('testtkpy'))
s = topo.source([1043])
s = s.map(lambda x: (x, ), schema='tuple<uint64 v>')
bop = op.Sink(
"com.ibm.streamsx.topology.pytest.pyprimitives::SingleInputPort",
s,
name="SIP_OP")
self.tester = Tester(topo)
self.tester.local_check = self._single_input_port_check
self.tester.test(self.test_ctxtype, self.test_config)
def test_spl(self):
"""
Test passing as an SPL parameter.
"""
N = 22
G = 'hey'
t = ''.join(random.choice('0123456789abcdef') for x in range(20))
topic = 'topology/test/python/' + t
topo = Topology()
spTopic = topo.create_submission_parameter('mytopic')
spGreet = topo.create_submission_parameter('greeting')
self.assertIsNone(spTopic())
self.assertIsNone(spGreet())
sch = StreamSchema('tuple<uint64 seq, rstring s>')
b = op.Source(topo,
"spl.utility::Beacon",
sch,
params={
'initDelay': 10.0,
'period': 0.02,
'iterations': N
})
b.seq = b.output('IterationCount()')
b.s = b.output(spGreet)
p = op.Sink("com.ibm.streamsx.topology.topic::Publish",
b.stream,
params={'topic': topic})
s = op.Source(topo,
"com.ibm.streamsx.topology.topic::Subscribe",
sch,
params={
'streamType': sch,
'topic': spTopic
})
jc = JobConfig()
jc.submission_parameters['mytopic'] = topic
jc.submission_parameters['greeting'] = G
jc.add(self.test_config)
tester = Tester(topo)
tester.tuple_count(s.stream, N)
#tester.run_for(300)
tester.contents(s.stream, [{'seq': i, 's': G} for i in range(N)])
tester.test(self.test_ctxtype, self.test_config)
def gdaxFeed(inetToolkit, buildType, port):
# Sumbit request Build Server and Submit.
schemaTicker = 'tuple<rstring ttype, float32 price, float32 low_24h, float32 best_ask, rstring side, float32 best_bid, float32 open_24h, rstring product_id, int32 sequence, int32 trade_id, rstring time, float32 last_size, float32 volume_24h, float32 volume_30d, float32 high_24h>'
#
# Define the application
#
topo = Topology("GdaxFeed")
tk.add_toolkit(topo, inetToolkit)
source = topo.source(gdaxData)
# Split out the securities : ETH-USD, LTC-USD, BTC-USD
eth = source.filter(lambda t: t["product_id"] == "ETH-USD",
name="ethFilter")
ltc = source.filter(lambda t: t["product_id"] == "LTC-USD",
name="ltcFilter")
btc = source.filter(lambda t: t["product_id"] == "LTC-USD",
name="btcFilter")
eth.print(name="eth")
ltc.print(name="ltc")
btc.print(name="btc")
ethTuple = eth.map(lambda t: t, schema=schemaTicker)
#ethWin = ethTuple.last(datetime.timedelta(minutes=2))
ethWin = ethTuple.last(100).trigger(1)
rawRequest = op.Sink("com.ibm.streamsx.inet.rest::HTTPTupleView",
stream=ethWin,
params={
'port': 8080,
'context': 'gdaxEth',
'contextResourceBase': '/base'
},
name="TupleView")
#
# Compile & Submit the Topology to Streams instance
#
streams_conf = common.build_streams_config("StreamingTurbine",
credential.serviceCredentials)
context.submit(context.ContextTypes.STREAMING_ANALYTICS_SERVICE,
topo,
config=streams_conf)
def test():
print("enter")
tmp = gdaxData()
while True:
print(tmp.__next__())
def _publish(self, topo, N, topic, width=None, allow_filter=False):
b = op.Source(topo,
"spl.utility::Beacon",
SCHEMA,
params={
'initDelay': 10.0,
'iterations': N
})
b.seq = b.output('IterationCount()')
ps = b.stream
if width:
ps = ps.parallel(width=width)
p = op.Sink('com.ibm.streamsx.topology.topic::Publish',
ps,
params={'topic': topic},
name='MSP')
if allow_filter:
p.params['allowFilter'] = True
def _run_app(self, kind, e, opi='M'):
schema = 'tuple<rstring a, int32 b>'
topo = Topology('TSESPL' + str(uuid.uuid4().hex))
streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('testtkpy'))
if opi == 'M':
data = [1, 2, 3]
se = topo.source(data)
se = se.map(lambda x: {'a': 'hello', 'b': x}, schema=schema)
prim = op.Map("com.ibm.streamsx.topology.pytest.pyexceptions::" +
kind,
se,
params={'tf': self.tf})
res = prim.stream
elif opi == 'S':
prim = op.Source(
topo,
"com.ibm.streamsx.topology.pytest.pyexceptions::" + kind,
schema=schema,
params={'tf': self.tf})
res = prim.stream
elif opi == 'E':
data = [1, 2, 3]
se = topo.source(data)
se = se.map(lambda x: {'a': 'hello', 'b': x}, schema=schema)
prim = op.Sink("com.ibm.streamsx.topology.pytest.pyexceptions::" +
kind,
se,
params={'tf': self.tf})
res = None
tester = Tester(topo)
if res is not None:
tester.tuple_count(res, len(e))
if e:
tester.contents(res, e)
else:
tester.run_for(5)
tester.test(self.test_ctxtype, self.test_config)
def test_map_foreach(self):
iterations = 3000
topo = Topology()
streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('testtkpy'))
timeCounter = op.Source(
topo,
"com.ibm.streamsx.topology.pytest.checkpoint::TimeCounter",
schema.StreamSchema('tuple<int32 f>').as_tuple(),
params={
'iterations': iterations,
'period': 0.01
})
timeCounter.stream.set_consistent(
ConsistentRegionConfig.periodic(5,
drain_timeout=40,
reset_timeout=40,
max_consecutive_attempts=6))
fizzbuzz = op.Map(
"com.ibm.streamsx.topology.pytest.checkpoint::FizzBuzzMap",
timeCounter.stream,
schema.StreamSchema('tuple<int32 f, rstring c>').as_tuple())
verify = op.Sink("com.ibm.streamsx.topology.pytest.checkpoint::Verify",
fizzbuzz.stream)
s = fizzbuzz.stream
tester = Tester(topo)
tester.resets()
tester.tuple_count(s, iterations)
# Find the expected results.
fizz = lambda x: (x[0], x[1] + 'fizz' if x[0] % 3 == 0 else x[1])
buzz = lambda x: (x[0], x[1] + 'buzz' if x[0] % 5 == 0 else x[1])
expected = list(
map(buzz, (map(fizz, (map(lambda x:
(x, ''), range(iterations)))))))
tester.contents(s, expected)
tester.test(self.test_ctxtype, self.test_config)
def test_mt(self):
topo = Topology()
N = 1000
streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('testtkpy'))
b1 = op.Source(topo,
"spl.utility::Beacon",
schema.StreamSchema('tuple<int32 f>').as_tuple(),
params={'iterations': N})
b1.f = b1.output('(int32)IterationCount()')
b2 = op.Source(topo,
"spl.utility::Beacon",
schema.StreamSchema('tuple<int32 f>').as_tuple(),
params={'iterations': N})
b2.f = b2.output(str(N) + ' + (int32)IterationCount()')
b3 = op.Source(topo,
"spl.utility::Beacon",
schema.StreamSchema('tuple<int32 f>').as_tuple(),
params={'iterations': N})
b3.f = b3.output(str(2 * N) + ' + (int32)IterationCount()')
s1 = b1.stream.low_latency()
s2 = b2.stream.low_latency()
s3 = b3.stream.low_latency()
s = s1.union({s2, s3})
f = op.Map("com.ibm.streamsx.topology.pytest.mt::MTFilter", s)
m = op.Map("com.ibm.streamsx.topology.pytest.mt::MTMap", f.stream)
op.Sink("com.ibm.streamsx.topology.pytest.mt::MTForEach", f.stream)
cr = m.stream.flat_map()
tester = Tester(topo)
tester.tuple_count(m.stream, 3 * N)
tester.contents(cr, range(3 * N), ordered=False)
tester.test(self.test_ctxtype, self.test_config)
def _run_app(self, kind, opi='M'):
schema = 'tuple<rstring a, int32 b>'
topo = Topology('TESPL' + str(uuid.uuid4().hex))
streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('testtkpy'))
if opi == 'M':
data = [1, 2, 3]
se = topo.source(data)
se = se.map(lambda x: {'a': 'hello', 'b': x}, schema=schema)
prim = op.Map("com.ibm.streamsx.topology.pytest.pyexceptions::" +
kind,
se,
params={'tf': self.tf})
res = prim.stream
elif opi == 'S':
prim = op.Source(
topo,
"com.ibm.streamsx.topology.pytest.pyexceptions::" + kind,
schema=schema,
params={'tf': self.tf})
res = prim.stream
elif opi == 'E':
data = [1, 2, 3]
se = topo.source(data)
se = se.map(lambda x: {'a': 'hello', 'b': x}, schema=schema)
prim = op.Sink("com.ibm.streamsx.topology.pytest.pyexceptions::" +
kind,
se,
params={'tf': self.tf})
res = None
tester = Tester(topo)
tester.run_for(3)
ok = tester.test(self.test_ctxtype,
self.test_config,
assert_on_fail=False)
self.assertFalse(ok)
