def test_pipe(self):
topo = Topology()
streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('testtkpy'))
timeCounter = op.Source(
topo,
"com.ibm.streamsx.topology.pytest.checkpoint::TimeCounter",
schema.StreamSchema('tuple<int32 f>').as_tuple(),
params={
'iterations': 30,
'period': 0.1
})
evenFilter = op.Map(
"com.ibm.streamsx.topology.pytest.checkpoint::StatefulEvenFilter",
timeCounter.stream,
None,
params={})
hpo = op.Map(
"com.ibm.streamsx.topology.pytest.checkpoint::StatefulHalfPlusOne",
evenFilter.stream,
None,
params={})
hpo.stream.set_consistent(
ConsistentRegionConfig.operator_driven(drain_timeout=40,
reset_timeout=40,
max_consecutive_attempts=3))
s = hpo.stream
tester = Tester(topo)
self.assertFalse(
tester.test(self.test_ctxtype,
self.test_config,
assert_on_fail=False))
def test_mixed_toolkits(self):
topo = Topology()
streamsx.spl.toolkit.add_toolkit(topo, '../testtkpy')
streamsx.spl.toolkit.add_toolkit(topo, '../tk17')
data = ['A']
bop = op.Source(topo, "spl.utility::Beacon", 'tuple<rstring a>',
{'iterations': 1})
bop.a = bop.output('"A"')
sv = op.Map("com.ibm.streamsx.topology.pytest.pyvers::StreamsxVersion",
bop.stream, 'tuple<rstring a, rstring v1, rstring v2>')
m17f = op.Map(
"com.ibm.streamsx.topology.pytest.tk17::M17F", sv.stream,
'tuple<rstring a, rstring v1, rstring v2, rstring f1, rstring f2>')
m17c = op.Map(
"com.ibm.streamsx.topology.pytest.tk17::M17C", m17f.stream,
'tuple<rstring a, rstring v1, rstring v2, rstring f1, rstring f2, rstring c1, rstring c2, int32 x>',
{'x': 27})
tester = Tester(topo)
tester.contents(m17c.stream, [{
'a': 'A',
'f1': '1.7',
'f2': 'F',
'v1': 'aggregate',
'v2': 'True',
'c1': '1.7',
'c2': 'C',
'x': 27
}])
tester.test(self.test_ctxtype, self.test_config)
def test_suppress_metric(self):
schema = 'tuple<int32 a, int32 b, int32 c, int32 d>'
topo = Topology()
streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('testtkpy'))
# no metric
st = op.Source(
topo,
kind='com.ibm.streamsx.topology.pytest.pysource::SparseTuple',
schema=schema,
name='NOMETRIC_ST')
sf = op.Source(
topo,
kind='com.ibm.streamsx.topology.pysamples.sources::Range37',
schema=schema,
name='NOMETRIC_SF')
s = st.stream.union({sf.stream})
sm = op.Map('com.ibm.streamsx.topology.pysamples.positional::Noop',
s,
name='NOMETRIC_MF')
sm = op.Map('com.ibm.streamsx.topology.pysamples.positional::AddSeq',
sm.stream,
name='NOMETRIC_MC')
# With metric
schema = 'tuple<rstring a, int32 b>'
ms = op.Source(
topo,
kind=
'com.ibm.streamsx.topology.pytest.pyexceptions::SuppressNextSource',
schema=schema,
name='HASMETRIC_S_1')
mm = op.Map(
kind='com.ibm.streamsx.topology.pytest.pyexceptions::SuppressMap',
stream=ms.stream,
name='HASMETRIC_M_0')
mf = op.Map(
kind=
'com.ibm.streamsx.topology.pytest.pyexceptions::SuppressFilter',
stream=ms.stream,
name='HASMETRIC_F_0')
self.tester = Tester(topo)
self.tester.local_check = self.check_suppress_metric
# Add filters to avoid the test operators having
# names of NOMETIC/HASMETRIC
self.tester.tuple_count(sm.stream.filter(lambda _: True), 38)
self.tester.tuple_count(ms.stream.filter(lambda _: True), 2)
self.tester.tuple_count(mm.stream.filter(lambda _: True), 2)
self.tester.tuple_count(mf.stream.filter(lambda _: True), 2)
self.tester.test(self.test_ctxtype, self.test_config)
def test_filter_map(self):
topo = Topology()
topo.checkpoint_period = timedelta(seconds=1)
streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('testtkpy'))
timeCounter = op.Source(topo, "com.ibm.streamsx.topology.pytest.checkpoint::TimeCounter", schema.StreamSchema('tuple<int32 f>').as_tuple(), params={'iterations':30,'period':0.1})
evenFilter = op.Map("com.ibm.streamsx.topology.pytest.checkpoint::StatefulEvenFilter", timeCounter.stream, None, params={})
hpo = op.Map("com.ibm.streamsx.topology.pytest.checkpoint::StatefulHalfPlusOne", evenFilter.stream, None, params={})
s = hpo.stream
tester = Tester(topo)
tester.tuple_count(s, 15)
tester.contents(s, list(zip(range(1,16))))
tester.test(self.test_ctxtype, self.test_config)
def test_NotByRef(self):
topo = Topology()
s = topo.source(['ByRef', 3, list(('a', 42))])
f = op.Map('spl.relational::Filter', s)
s = f.stream.map(lambda x: x)
f = op.Map('spl.relational::Filter', s)
s = f.stream.flat_map(lambda x: x if isinstance(x, list) else [x])
f = op.Map('spl.relational::Filter', s)
f.stream.for_each(CheckForEach())
tester = Tester(topo)
tester.contents(s, ['ByRef', 3, 'a', 42])
tester.test(self.test_ctxtype, self.test_config)
def test_enter_exit(self):
topo = Topology()
streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('testtkpy'))
source = op.Source(topo, 'com.ibm.streamsx.topology.pytest.checkpoint::EnterExitSource', schema.StreamSchema('tuple<rstring from, int32 enter, int32 exit>').as_tuple(), params={'period':0.1})
source.stream.set_consistent(ConsistentRegionConfig.periodic(5, drain_timeout=40, reset_timeout=40, max_consecutive_attempts=6))
transit = op.Map('com.ibm.streamsx.topology.pytest.checkpoint::EnterExitMap', source.stream, schema.StreamSchema('tuple<rstring from, int32 enter, int32 exit>').as_tuple())
tester = Tester(topo)
tester.resets(10)
# On each operator, __enter__ and __exit__ should be called once for
# each reset. Also __enter__ should be called at startup and __exit__
# at shutdown. It is hard to verify the final __exit__ call (and that
# is handled by python rather than our code), so
# the test is valid if the number of __enter__ calls is one more than
# the number of resets, and the number of __exit__ calls is equal to
# number of resets. The tuples on the two streams indicate the number
# of times __enter__ and __exit__ have been called.
# We are looking for two specific tuples:
# ('source', 6, 5) and ('transit', 6, 5)
tester.eventual_result(source.stream, lambda tuple_ : True if tuple_[1] >= 6 and tuple_[1] == tuple_[2] + 1 else Fale if tuple_[1] != tuple_[2] + 1 else None)
tester.eventual_result(transit.stream, lambda tuple_ : True if tuple_[1] >= 6 and tuple_[1] == tuple_[2] + 1 else Fale if tuple_[1] != tuple_[2] + 1 else None)
job_config = streamsx.topology.context.JobConfig(tracing='debug')
job_config.add(self.test_config)
tester.test(self.test_ctxtype, self.test_config)
def test_timestamp_event_time_attribute(self):
topo = Topology('test_timestamp_event_time_attribute')
ts1 = Timestamp(1608196, 235000000, 0)
ts2 = Timestamp(1608199, 876265298, 0)
ts_schema = StreamSchema('tuple<int64 num, timestamp ts>').as_tuple(
named=True)
s = topo.source([(1, ts1), (2, ts2)])
# map() named "event_time_source" transforms to structured schema
s = s.map(lambda x: x, schema=ts_schema, name='event_time_source')
# add event-time annotation for attribute ts to the "event_time_source"
s = s.set_event_time('ts')
# use SPL function getEventTime() to get the event time of the input tuple
# copies the event-time timestamp value to a new output attribute "eventtime"
f = op.Map('spl.relational::Functor',
s,
schema=StreamSchema('tuple<timestamp eventtime>').as_tuple(
named=True))
f.eventtime = f.output('getEventTime(event_time_source)')
# map to Python schema (prepare for content comparision)
as_ts = f.stream.map(lambda x: x.eventtime)
tester = Tester(topo)
tester.tuple_count(s, 2)
tester.contents(as_ts, [ts1, ts2])
tester.test(self.test_ctxtype, self.test_config)
def test_map_blob_type(self):
topo = Topology()
streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('testtkpy'))
data = ['Hello', 'Blob', 'Did', 'you', 'reset']
s = topo.source(data)
s = s.as_string()
toBlob = op.Map("com.ibm.streamsx.topology.pytest.pytypes::ToMapBlob",
s, 'tuple<map<rstring,blob> lb>')
bt = op.Map("com.ibm.streamsx.topology.pytest.pytypes::MapBlobTest",
toBlob.stream, 'tuple<rstring string>')
tester = Tester(topo)
tester.contents(bt.stream, data)
tester.test(self.test_ctxtype, self.test_config)
def test_primitive_foreach(self):
iterations = 3000
topo = Topology()
topo.checkpoint_period = timedelta(seconds=1)
streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('testtkpy'))
timeCounter = op.Source(
topo,
"com.ibm.streamsx.topology.pytest.checkpoint::TimeCounter",
schema.StreamSchema('tuple<int32 f>').as_tuple(),
params={
'iterations': iterations,
'period': 0.01
})
timeCounter.stream.set_consistent(
ConsistentRegionConfig.periodic(5,
drain_timeout=40,
reset_timeout=40,
max_consecutive_attempts=6))
fizzbuzz = op.Map(
"com.ibm.streamsx.topology.pytest.checkpoint::FizzBuzzPrimitive",
timeCounter.stream,
schema.StreamSchema('tuple<int32 f, rstring c>').as_tuple())
verify = op.Sink("com.ibm.streamsx.topology.pytest.checkpoint::Verify",
fizzbuzz.stream)
s = fizzbuzz.stream
tester = Tester(topo)
tester.resets()
tester.tuple_count(s, iterations)
tester.test(self.test_ctxtype, self.test_config)
def test_map_attr(self):
"""Test a Source and a Map operator.
Including an output clause.
"""
topo = Topology('test_SPLBeaconFilter')
beacon = op.Source(topo,
"spl.utility::Beacon",
'tuple<uint64 seq>',
params={
'period': 0.02,
'iterations': 27
})
beacon.seq = beacon.output('IterationCount()')
f = op.Map(
'spl.relational::Functor',
beacon.stream,
schema='tuple<uint64 a>',
params={'filter': op.Expression.expression('seq % 4ul == 0ul')})
f.a = f.output(f.attribute('seq'))
s = f.stream.map(lambda x: x['a'])
tester = Tester(topo)
tester.contents(s, [0, 4, 8, 12, 16, 20, 24])
tester.test(self.test_ctxtype, self.test_config)
def test_SPLHashFunc(self):
"""
Test hashing works when the schema is a general SPL one
using an explicit hash function.
"""
raw = []
for v in range(20):
raw.append(''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(v)))
data = []
for v in range(7):
data.extend(raw)
random.shuffle(data)
for width in (1,4):
with self.subTest(width=width):
topo = Topology("test_SPLHash" + str(width))
s = topo.source(data)
s = s.as_string()
f = op.Map('spl.relational::Functor', s,
schema = 'tuple<rstring string, rstring s2>')
f.s2 = f.output('string + "_1234"')
s = f.stream
s = s.parallel(width, Routing.HASH_PARTITIONED, s2_hash)
s = s.map(AddChannel())
s = s.end_parallel()
s = s.map(CheckSameChannel(lambda t : t[0]['s2']))
expected = []
for v in data:
expected.append(v + '_1234')
tester = Tester(topo)
tester.contents(s, expected, ordered=width==1)
tester.test(self.test_ctxtype, self.test_config)
def test_sliding_count(self):
for step in [1, 3]:
with self.subTest(step=step):
topo = Topology()
b = op.Source(topo,
"spl.utility::Beacon",
'tuple<uint64 seq>',
params={'iterations': 12})
b.seq = b.output('IterationCount()')
s = b.stream
agg = op.Map('spl.relational::Aggregate',
s.last(4).trigger(step),
schema='tuple<uint64 sum, uint64 max>')
agg.sum = agg.output('Sum(seq)')
agg.max = agg.output('Max(seq)')
expected = []
for i in range(4 + step - 2, 12, step):
expected.append({
'sum': sum(range(i - 3, i + 1)),
'max': i
})
tester = Tester(topo)
tester.contents(agg.stream, expected)
tester.test(self.test_ctxtype, self.test_config)
def test_sliding_count_stv_no_default(self):
step = 1
topo = Topology()
b = op.Source(topo,
"spl.utility::Beacon",
'tuple<uint64 seq>',
params={'iterations': 12})
b.seq = b.output('IterationCount()')
s = b.stream
count = topo.create_submission_parameter('count', type_=int)
window = s.last(count).trigger(step)
agg = op.Map('spl.relational::Aggregate',
window,
schema='tuple<uint64 sum, uint64 max>')
agg.sum = agg.output('Sum(seq)')
agg.max = agg.output('Max(seq)')
expected = []
for i in range(4 + step - 2, 12, step):
expected.append({'sum': sum(range(i - 3, i + 1)), 'max': i})
jc = JobConfig()
jc.submission_parameters['count'] = 4
jc.add(self.test_config)
tester = Tester(topo)
tester.contents(agg.stream, expected)
tester.test(self.test_ctxtype, self.test_config)
def test_stream_alias(self):
"""
test a stream alias to ensure the SPL expression
is consistent with hand-coded SPL expression.
"""
topo = Topology('test_stream_alias')
s = op.Source(topo,
"spl.utility::Beacon",
'tuple<uint64 seq>',
params={
'period': 0.02,
'iterations': 27
},
name='SomeName')
s.seq = s.output('IterationCount()')
stream = s.stream.aliased_as('IN')
f = op.Map(
'spl.relational::Functor',
stream,
schema='tuple<uint64 a>',
params={'filter': op.Expression.expression('IN.seq % 4ul == 0ul')})
f.a = f.output(f.attribute('seq'))
s = f.stream.map(lambda x: x['a'])
tester = Tester(topo)
tester.contents(s, [0, 4, 8, 12, 16, 20, 24])
tester.test(self.test_ctxtype, self.test_config)
def test_sliding_time_stv_no_default(self):
topo = Topology()
b = op.Source(topo,
"spl.utility::Beacon",
'tuple<uint64 seq>',
params={'iterations': 12})
b.seq = b.output('IterationCount()')
s = b.stream
wtime = topo.create_submission_parameter(name='secs', type_=int)
window = s.lastSeconds(wtime).trigger(1)
agg = op.Map('spl.relational::Aggregate',
window,
schema='tuple<uint64 sum, uint64 max>')
agg.sum = agg.output('Sum(seq)')
agg.max = agg.output('Max(seq)')
jc = JobConfig()
jc.submission_parameters['secs'] = 2
jc.add(self.test_config)
tester = Tester(topo)
tester.tuple_count(agg.stream, 12)
tester.test(self.test_ctxtype, self.test_config)
def main():
"""
This demonstrates the invocation of SPL operators from
the SPL standard toolkit.
Example:
python3 files.py
Output:
Capitalized words from the contents of files in /tmp/work
"""
# Create the container for the topology that will hold the streams of tuples.
topo = Topology("Files")
# Invoke an SPL DirectoryScan operator as a source.
# This one scans /tmp/work for files.
# Note the full kind of the operator is required.
files = op.Source(topo, "spl.adapter::DirectoryScan",
schema=CommonSchema.String, params = {'directory': '/tmp/work'})
# Follow it with a FileSource operator
# If no schema is provided then the input schema is used.
lines = op.Map("spl.adapter::FileSource", files.stream)
# Feed the lines into a Python function
lines = lines.stream.map(string.capwords)
# Sink lines by printing each of its tuples to standard output
lines.print()
# Now execute the topology by submitting to a standalone context.
streamsx.topology.context.submit("STANDALONE", topo.graph)
def test_filer_for_each(self):
topo = Topology()
streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('testtkpy'))
s = topo.source([(1, 2, 3), (1, 2, 201), (2, 2, 301), (3, 4, 401),
(5, 6, 78), (8, 6, 501), (803, 9324, 901)])
sch = 'tuple<int32 a, int32 b, int32 c>'
s = s.map(lambda x: x, schema=sch)
bop = op.Map("com.ibm.streamsx.topology.pytest.pykwargs::KWFilter", s)
op.Sink("com.ibm.streamsx.topology.pytest.pykwargs::KWForEach",
bop.stream)
tester = Tester(topo)
tester.contents(bop.stream, [{
'a': 1,
'b': 2,
'c': 201
}, {
'a': 3,
'b': 4,
'c': 401
}, {
'a': 803,
'b': 9324,
'c': 901
}])
tester.test(self.test_ctxtype, self.test_config)
def test_mixed_types(self):
creds_file = os.environ['DB2_CREDENTIALS']
with open(creds_file) as data_file:
credentials = json.load(data_file)
topo = Topology('test_mixed_types')
if self.jdbc_toolkit_home is not None:
streamsx.spl.toolkit.add_toolkit(topo, self.jdbc_toolkit_home)
pulse = op.Source(topo, "spl.utility::Beacon", 'tuple<rstring A, rstring B>', params = {'iterations':1})
pulse.A = pulse.output('"hello"')
pulse.B = pulse.output('"world"')
sample_schema = StreamSchema('tuple<rstring A, rstring B>')
query_schema = StreamSchema('tuple<rstring sql>')
sql_create = 'CREATE TABLE RUN_SAMPLE (A CHAR(10), B CHAR(10))'
create_table = db.run_statement(pulse.stream, credentials, schema=sample_schema, sql=sql_create)
sql_insert = 'INSERT INTO RUN_SAMPLE (A, B) VALUES (?, ?)'
inserts = db.run_statement(create_table, credentials, schema=sample_schema, sql=sql_insert, sql_params="A, B")
query = op.Map('spl.relational::Functor', inserts, schema=query_schema)
query.sql = query.output('"SELECT A, B FROM RUN_SAMPLE"')
res_sql = db.run_statement(query.stream, credentials, schema=sample_schema, sql_attribute='sql')
res_sql.print()
sql_drop = 'DROP TABLE RUN_SAMPLE'
drop_table = db.run_statement(res_sql, credentials, sql=sql_drop)
tester = Tester(topo)
tester.tuple_count(drop_table, 1)
#tester.run_for(60)
tester.test(self.test_ctxtype, self.test_config)
def test_window_spl_aggregate(self):
topo = Topology('test_window_spl_aggregate')
ts1 = Timestamp(1608196, 235000000, 0)
ts2 = Timestamp(1608199, 876265298, 0)
ts3 = Timestamp(1608506, 123456078, 0)
ts4 = Timestamp(1608507, 654326980, 0)
ts_schema = StreamSchema('tuple<int64 num, timestamp ts>').as_tuple(
named=True)
s = topo.source([(1, ts1), (2, ts2), (3, ts3), (4, ts4)])
# map() named "event_time_source" transforms to structured schema
s = s.map(lambda x: x, schema=ts_schema, name='event_time_source')
# add event-time annotation for attribute ts to the "event_time_source"
s = s.set_event_time('ts')
agg_schema = StreamSchema(
'tuple<uint64 sum, rstring pane_timing>').as_tuple(named=True)
# timeInterval window
win = s.time_interval(interval_duration=5.0, discard_age=30.0)
agg = op.Map('spl.relational::Aggregate', win, schema=agg_schema)
agg.sum = agg.output('Sum((uint64)num)')
agg.pane_timing = agg.output('(rstring)paneTiming()')
#agg.stream.print()
result = agg.stream.map(lambda x: x.sum)
result_pane_timing = agg.stream.map(lambda x: x.pane_timing)
tester = Tester(topo)
tester.tuple_count(result, 2)
tester.contents(result, [3, 7])
tester.contents(result_pane_timing,
['paneOnComplete', 'paneOnComplete'])
tester.test(self.test_ctxtype, self.test_config)
def test_feedback_loop(self):
topo = Topology()
data = ['A','B', 'A', 'A', 'X', 'C', 'C', 'D', 'A', 'A', 'E']
expected = ['B', 'X', 'C', 'C', 'D', 'A', 'A', 'E']
s = topo.source(data)
s = s.filter(lambda t : time.sleep(1) or True).as_string();
feedback = PendingStream(topo)
df = op.Invoke(topo, 'spl.utility::DynamicFilter',
inputs = [s, feedback.stream],
schemas= [schema.CommonSchema.String])
df.params['key'] = df.attribute(s, 'string')
df.params['addKey'] = df.attribute(feedback.stream, 'string')
delayed_out = op.Map('spl.utility::Delay', df.outputs[0], params={'delay': 0.05}).stream
x = delayed_out.filter(lambda s : s == 'X').map(lambda s : 'A').as_string()
i = topo.source(['B', 'X', 'C', 'D', 'E']).as_string()
x = x.union({i})
feedback.complete(x)
result = delayed_out
result.print()
#streamsx.topology.context.submit('TOOLKIT', topo)
tester = Tester(topo)
tester.contents(result, expected)
tester.test(self.test_ctxtype, self.test_config)
def test_pure_loop(self):
topo = Topology()
feedback = PendingStream(topo)
df = op.Map('spl.utility::Custom',
feedback.stream,
schema=schema.CommonSchema.String)
delayed_out = op.Map('spl.utility::Delay', df.stream, params={'delay': 0.05}).stream
feedback.complete(delayed_out)
sr = streamsx.topology.context.submit('BUNDLE', topo)
self.assertEqual(0, sr['return_code'])
os.remove(sr.bundlePath)
def test_input_by_position(self):
"""Operator with input by position"""
topo = Topology()
streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('testtkpy'))
s = topo.source([3642, -393])
s = s.map(lambda x: (x, x * 2, x + 4),
schema='tuple<int64 d, int64 e, int64 f>')
bop = op.Map(
"com.ibm.streamsx.topology.pytest.pyprimitives::InputByPosition",
s)
r = bop.stream
self.tester = Tester(topo)
self.tester.tuple_count(r, 2)
self.tester.contents(r, [{
'd': 3642,
'e': (3642 * 2) + 89,
'f': -92
}, {
'd': -393,
'e': (-393 * 2) + 89,
'f': -92
}])
self.tester.test(self.test_ctxtype, self.test_config)
def test_spl_types_compile_only(self):
topo = Topology('test_spl_types_compile_only')
beacon = op.Source(topo, "spl.utility::Beacon", 'tuple<rstring ticks>')
beacon.ticks = beacon.output('"1608196235"')
# event-time annotation TYPE=uint64
f1 = op.Map('spl.relational::Functor',
beacon.stream,
schema='tuple<uint64 tsEventTimeUInt64>')
f1.tsEventTimeUInt64 = f1.output('(uint64)ticks')
f1 = f1.stream.set_event_time('tsEventTimeUInt64',
lag=1.0,
minimum_gap=0.2)
# event-time annotation TYPE=int64
lag = topo.create_submission_parameter('lag', 0.4)
gap = topo.create_submission_parameter('gap', 0.2)
f2 = op.Map('spl.relational::Functor',
beacon.stream,
schema='tuple<int64 tsEventTimeInt64>')
f2.tsEventTimeInt64 = f2.output('(int64)ticks')
f2 = f2.stream.set_event_time('tsEventTimeInt64',
lag=lag,
minimum_gap=gap,
resolution='Milliseconds')
# event-time annotation TYPE=timestamp
f3 = op.Map('spl.relational::Functor',
beacon.stream,
schema='tuple<timestamp tsEventTime>')
f3.tsEventTime = f3.output(
'toTimestamp((int64)ticks, Sys.Milliseconds)')
f3 = f3.stream.set_event_time('tsEventTime')
# timeInterval window
win = f3.time_interval(interval_duration=10.0, creation_period=1.0)
agg = op.Map('spl.relational::Aggregate',
win,
schema='tuple<uint64 pi, timestamp start, timestamp end>')
agg.pi = agg.output('paneIndex()')
agg.start = agg.output('intervalStart()')
agg.end = agg.output('intervalEnd()')
sr = streamsx.topology.context.submit('BUNDLE', topo)
self.assertEqual(0, sr['return_code'])
os.remove(sr.bundlePath)
os.remove(sr.jobConfigPath)
def test_map_attr_opt(self):
"""Test a Source and a Map operator with optional types.
Including with operator parameters and output clauses.
"""
if ("TestSas" in str(self)) or 'CP4D_URL' in os.environ:
print('Testing with Streams supporting optional types')
else:
if 'STREAMS_INSTALL' in os.environ:
Tester.require_streams_version(self, '4.3')
else:
self.skipTest("STREAMS_INSTALL not set")
topo = Topology('test_map_attr_opt')
this_dir = os.path.dirname(os.path.realpath(__file__))
spl_dir = os.path.join(os.path.dirname(os.path.dirname(this_dir)),
'spl')
tk_dir = os.path.join(spl_dir, 'testtkopt')
streamsx.spl.toolkit.add_toolkit(topo, tk_dir)
schema = 'tuple<' \
'rstring r, ' \
'optional<rstring> orv, ' \
'optional<rstring> ornv, ' \
'int32 i32, ' \
'optional<int32> oi32v, ' \
'optional<int32> oi32nv>'
s = op.Source(topo,
"testgen::TypeLiteralTester",
schema,
params={
'r': 'a string',
'orv': 'optional string',
'ornv': None,
'i32': 123,
'oi32v': 456,
'oi32nv': streamsx.spl.types.null()
})
f = op.Map('spl.relational::Functor', s.stream, schema=schema)
f.orv = f.output("null")
f.ornv = f.output('"string value"')
f.oi32v = f.output(streamsx.spl.types.null())
f.oi32nv = f.output('789')
tester = Tester(topo)
tester.contents(s.stream, [{
'r': 'a string',
'orv': 'optional string',
'ornv': None,
'i32': 123,
'oi32v': 456,
'oi32nv': None
}])
tester.contents(f.stream, [{
'r': 'a string',
'orv': None,
'ornv': 'string value',
'i32': 123,
'oi32v': None,
'oi32nv': 789
}])
tester.test(self.test_ctxtype, self.test_config)
def test_filter_map(self):
iterations = 3000
topo = Topology()
streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('testtkpy'))
timeCounter = op.Source(topo, "com.ibm.streamsx.topology.pytest.checkpoint::TimeCounter", schema.StreamSchema('tuple<int32 f>').as_tuple(), params={'iterations':iterations,'period':0.01})
timeCounter.stream.set_consistent(ConsistentRegionConfig.periodic(5, drain_timeout=40, reset_timeout=40, max_consecutive_attempts=6))
evenFilter = op.Map("com.ibm.streamsx.topology.pytest.checkpoint::StatefulEvenFilter", timeCounter.stream, None, params={})
hpo = op.Map("com.ibm.streamsx.topology.pytest.checkpoint::StatefulHalfPlusOne", evenFilter.stream, None, params={})
s = hpo.stream
tester = Tester(topo)
tester.resets()
tester.tuple_count(s, iterations/2)
tester.contents(s, list(zip(range(1,int((iterations/2)+1)))))
tester.test(self.test_ctxtype, self.test_config)
def test_optional_blob_type(self):
Tester.require_streams_version(self, '4.3')
topo = Topology()
streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('testtkpy'))
data = ['Hello', 'Blob', 'Did', 'you', 'reset']
s = topo.source(data)
s = s.as_string()
toBlob = op.Map("com.ibm.streamsx.topology.pytest.pytypes::ToBlob", s,
'tuple<optional<blob> ob>')
bt = op.Map("com.ibm.streamsx.topology.pytest.pytypes::BlobTest",
toBlob.stream, 'tuple<rstring string>', {'keep': True})
tester = Tester(topo)
tester.contents(bt.stream, data)
tester.test(self.test_ctxtype, self.test_config)
def reflectPost(self, expected_requests, local_check_function):
"""Reflect base : reflecting back post.
Ran into a problem with reflecting back the header, if you include a Content-Length
in the header a corrected Content-Length is *not* sent. When the data arrives
back at the client the data is too short - thus the buildRsp()
"""
topo = Topology("Reflect")
self.tester = Tester(topo)
tk.add_toolkit(topo, inetToolkit)
pending_source = PendingStream(topo)
rsp = pending_source.stream.map(lambda t: t)
ss = topo.source([], name="onRamp")
rsp = ss.union({rsp})
# FormatResponse :
rspFormatted = rsp.map(lambda x: json.dumps(x)).as_string()
rawRequest = op.Map(
"com.ibm.streamsx.inet.rest::HTTPRequestProcess",
stream=rspFormatted,
schema=
'tuple<int64 key, rstring request, rstring contentType, map<rstring, rstring> header, rstring response, rstring method,rstring pathInfo, int32 status, rstring statusMessage>',
params={
'port': PORT,
'webTimeout': 5.0,
'responseJsonAttributeName': 'string',
'context': 'Reflect',
'contextResourceBase': 'opt/Reflect'
},
name="RequestProcess")
rawRequest.stream.sink(webEntryLog) ## log what we have received.
# wait for
lastAct = rawRequest.stream.filter(sleepMacbeth)
# do the work
getReflect = rawRequest.stream.transform(buildResponse("RSP:"),
name="buildResponse")
getReflect.sink(webExitLog) ## log what we are sending back
# do tests on what we have processed.
self.tester.tuple_count(getReflect, expected_requests)
# loopback to sending
pending_source.complete(getReflect) # loopback
## All done building the graph......
# setup the code that will invoke this test.
self.tester.local_check = local_check_function
# enable tracing info.
job_config = streamsx.topology.context.JobConfig(job_name='Reflect',
tracing="info")
job_config.add(self.test_config)
# submit the application for test
self.tester.test(self.test_ctxtype, self.test_config)
def _test_instance_submit(self):
""" Test submitting a bundle from an Instance.
Tests all four mechanisms.
"""
sab_name = 'ISJ_'+uuid.uuid4().hex
topo = topology.Topology(sab_name, namespace='myinstancens')
s = op.Source(topo, "spl.utility::Beacon",
'tuple<uint64 seq>',
params = {'period': 0.02, 'iterations':100})
s.seq = s.output('IterationCount()')
f = op.Map('spl.relational::Filter', s.stream,
params = {'filter': op.Expression.expression('seq % 2ul == 0ul')})
bb = streamsx.topology.context.submit('BUNDLE', topo, {})
self.assertIn('bundlePath', bb)
self.assertIn('jobConfigPath', bb)
sc = self.sc
instances = sc.get_instances()
if len(instances) == 1:
instance = instances[0]
else:
instance = sc.get_instance(os.environ['STREAMS_INSTANCE_ID'])
job = instance.submit_job(bb['bundlePath'])
self.assertIsInstance(job, Job)
self.assertEqual('myinstancens::'+sab_name, job.applicationName)
job.cancel()
with open(bb['jobConfigPath']) as fp:
jc = JobConfig.from_overlays(json.load(fp))
jn = 'JN_'+uuid.uuid4().hex
jc.job_name = jn
job = instance.submit_job(bb['bundlePath'], jc)
self.assertIsInstance(job, Job)
self.assertEqual('myinstancens::'+sab_name, job.applicationName)
self.assertEqual(jn, job.name)
job.cancel()
ab = instance.upload_bundle(bb['bundlePath'])
self.assertIsInstance(ab, ApplicationBundle)
job = ab.submit_job()
self.assertIsInstance(job, Job)
self.assertEqual('myinstancens::'+sab_name, job.applicationName)
job.cancel()
jn = 'JN_'+uuid.uuid4().hex
jc.job_name = jn
job = ab.submit_job(jc)
self.assertIsInstance(job, Job)
self.assertEqual('myinstancens::'+sab_name, job.applicationName)
self.assertEqual(jn, job.name)
job.cancel()
os.remove(bb['bundlePath'])
os.remove(bb['jobConfigPath'])
def smokePending(inetToolkit, buildType, port):
# Specify the Schema going in and out of the Streams operator.
schemaRequest = 'tuple<int64 key, rstring request, rstring method, rstring pathInfo >'
schemaResponse = 'tuple<int64 key, rstring request, rstring method, rstring pathInfo, rstring response >'
topo = Topology("SmokePending")
# Add extrenal spl functions, this is what is being testing my HTTPRequestProcess()
tk.add_toolkit(topo, inetToolkit)
# 'pending_source' is the loopback point.
pending_source = PendingStream(topo)
# Convert to Streams' tuple
rsp = pending_source.stream.map(lambda t: t, schema=schemaResponse)
rspFormatted = rsp.map(lambda t: json.dumps(t)).as_string()
rspFormatted.sink(webLog("Output response")) ## log what we have received.
rawRequest = op.Map("com.ibm.streamsx.inet.rest::HTTPRequestProcess",
stream=rsp,
schema = schemaRequest,
params={'port': 8080,
'webTimeout': 5.0,
#'responseJsonAttributeName': 'string',
'context':'myStreams',
'contextResourceBase': '/base'},
name="TupleRequest")
# write out data
rawRequest.stream.sink(webLog('Input request')) ## log what we have received.
# determine what to work on
# Filter does not change the type
upperDo = rawRequest.stream.filter(lambda t: t["pathInfo"] == "/upper", name="upperFilter")
lowerDo = rawRequest.stream.filter(lambda t: t["pathInfo"] == "/lower", name="lowerFilter")
# do some processing
upperDone = upperDo.map(upperString, schema = schemaResponse, name="upperProcess")
lowerDone = lowerDo.map(lowerString, schema = schemaResponse, name="lowerProcess")
##
processingDone = upperDone.union({lowerDone})
hack = processingDone.map(lambda t: t, schema=schemaResponse)
pending_source.complete(hack) # close the loop
# hack make it so I do not get this error.
"""
pending_source.complete(processingDone) # close the loop
SEVERE: Streaming Analytics service (StreamingTurbine): The submitted archive tk5385353137138356122.zip failed to scripts with status failed.
Exception in thread "main" java.lang.IllegalStateException: Error submitting archive for compilation:
"tk5385353137138356122/SmokePending/SmokePending.spl:5:1: CDISP0011E ERROR: A syntax error exists at the '}' token in 'graph'."
"""
#
# Compile & Submit the Topology to Streams instance
#
streams_conf = common.build_streams_config("StreamingTurbine", credential.serviceCredentials)
context.submit(context.ContextTypes.STREAMING_ANALYTICS_SERVICE, topo, config=streams_conf)
def test_custom_literal(self):
schema = StreamSchema('tuple<int32 a, rstring b>')
topo = Topology()
s = topo.source([(1,'ABC'), (2,'DEF')])
s = s.map(lambda x : x, schema=schema)
fmt = op.Map('spl.utility::Format', s, 'tuple<blob data>',
{'format':TestFormats.csv})
fmt.data = fmt.output('Output()')
parse = op.Map('spl.utility::Parse', fmt.stream, schema,
{'format':TestFormats.csv, 'parsing': TestParseOption.fast})
ts = parse.stream
tester = Tester(topo)
tester.tuple_count(ts, 2)
tester.contents(ts, [{'a':1,'b':'ABC'},{'a':2,'b':'DEF'}])
tester.test(self.test_ctxtype, self.test_config)