def _test_punct_file(self,
topo,
s,
test_name,
expected_content,
expected_tuple_count,
expected_punct_count=None):
s = s.map(lambda x: (x, ), schema='tuple<int32 z>')
op_params = {
'file': 'punct_file_' + test_name,
'writePunctuations': True,
'flushOnPunctuation': True
}
op.Sink("spl.adapter::FileSink", s, params=op_params)
# Copy the config, since it's shared across all tests, and not every test needs a data
# directory.
cfg = self.test_config.copy()
jc = JobConfig(data_directory=os.getcwd())
jc.add(cfg)
tester = Tester(topo)
tester.tuple_count(s, expected_tuple_count)
if expected_punct_count is not None:
tester.punct_count(s, expected_punct_count)
tester.test(self.test_ctxtype, cfg)
path = os.path.join(os.getcwd(), 'punct_file_' + test_name)
# Validate the contents of the file.
with open(path, 'r') as f:
file_contents = f.read()
self.assertEqual(expected_content, file_contents)
os.remove(path)
def test_for_each(self):
topo = Topology('test_for_each')
s = topo.source([1, 2, 3, 4])
s = s.punctor(func=(lambda t: 4 == t), before=False)
s.for_each(FEClass(), name='SINK_PUNCT', process_punct=True)
tester = Tester(topo)
tester.punct_count(s, 1)
tester.test(self.test_ctxtype, self.test_config)
def test_punct_replaces_tuple(self):
topo = Topology("test_punct_replaces_tuple")
s = topo.source(generate_numbers_for_named_tuple_schema)
s = s.punctor(func=(lambda t: True == t.punct_flag), replace=True)
s = s.map(lambda x: (x.value, ), schema='tuple<int32 z>')
s.print(write_punctuations=True)
tester = Tester(topo)
tester.tuple_count(s, 8)
tester.punct_count(s, 2)
tester.test(self.test_ctxtype, self.test_config)
def test_batch_punct_aggregate_empty_window(self):
topo = Topology("test_batch_punct_aggregate_empty_window")
s = topo.source(generate_puncts_schema)
s = s.punctor(func=(lambda t: True == t.punct_flag), replace=True)
window = s.batch('punct')
a = window.aggregate(SumMaxValues())
a.print(write_punctuations=True)
tester = Tester(topo)
tester.tuple_count(a, 0)
tester.punct_count(a, 3)
tester.test(self.test_ctxtype, self.test_config)
def test_batch_punct_aggregate(self):
topo = Topology("test_batch_punct_aggregate")
s = topo.source(generate_numbers_for_named_tuple_schema)
s = s.punctor(func=(lambda t: True == t.punct_flag),
before=False,
replace=False)
window = s.batch('punct')
a = window.aggregate(SumMaxValues())
a.print(write_punctuations=True)
tester = Tester(topo)
tester.tuple_count(a, 2)
tester.punct_count(a, 2)
tester.test(self.test_ctxtype, self.test_config)
def test_batch_punct_spl_aggregate(self):
topo = Topology("test_batch_punct_spl_aggregate")
s = topo.source(generate_numbers_for_named_tuple_schema)
s = s.punctor(func=(lambda t: True == t.punct_flag),
before=False,
replace=False)
s = s.map(lambda x: (x.value, ), schema='tuple<uint64 seq>')
agg = op.Map('spl.relational::Aggregate',
s.batch('punct'),
schema='tuple<uint64 sum, uint64 max>')
agg.sum = agg.output('Sum(seq)')
agg.max = agg.output('Max(seq)')
s = agg.stream
s.print(write_punctuations=True)
tester = Tester(topo)
tester.tuple_count(s, 2)
tester.punct_count(s, 2)
tester.test(self.test_ctxtype, self.test_config)
class TestPrimitivesOutputs(unittest.TestCase):
_multiprocess_can_split_ = True
@classmethod
def setUpClass(cls):
"""Extract Python operators in toolkit"""
stu._extract_tk('testtkpy')
def setUp(self):
Tester.setup_standalone(self)
def test_single_output_port(self):
"""Operator with single output port."""
topo = Topology()
streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('testtkpy'))
s = topo.source([9237, -24])
s = s.map(lambda x: (x, ), schema='tuple<int64 v>')
bop = op.Map(
"com.ibm.streamsx.topology.pytest.pyprimitives::SingleOutputPort",
s)
r = bop.stream
self.tester = Tester(topo)
self.tester.tuple_count(s, 2)
self.tester.contents(s, [{'v': 9237}, {'v': -24}])
self.tester.test(self.test_ctxtype, self.test_config)
def test_single_output_port_punct(self):
"""Operator with single output port emits window marker before forwarding the tuple."""
topo = Topology()
streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('testtkpy'))
s = topo.source([9237, -24])
s = s.map(lambda x: (x, ), schema='tuple<int64 v>')
bop = op.Map(
"com.ibm.streamsx.topology.pytest.pyprimitives::SingleOutputPortPunct",
s)
r = bop.stream
r.print(write_punctuations=True)
self.tester = Tester(topo)
self.tester.tuple_count(r, 2)
self.tester.punct_count(r, 2)
self.tester.contents(s, [{'v': 9237}, {'v': -24}])
self.tester.test(self.test_ctxtype, self.test_config)
def test_single_output_port_punct_forward(self):
"""Operator with receives and forwards window marker."""
topo = Topology()
streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('testtkpy'))
s = topo.source([1, 2, 3, 4])
s = s.punctor(func=(lambda t: 2 < t), before=False)
s = s.map(lambda x: (x, ), schema='tuple<int32 z>')
s1 = op.Map(
"com.ibm.streamsx.topology.pytest.pyprimitives::SingleOutputPort",
s) # has no on_punct
bop = op.Map(
"com.ibm.streamsx.topology.pytest.pyprimitives::SingleOutputPortPunctForward",
s) # implements on_punct
r = bop.stream
op.Sink("com.ibm.streamsx.topology.pytest.pyprimitives::VerifyPosInt",
r)
r.print(write_punctuations=True)
#self.test_config['topology.keepArtifacts'] = True
self.tester = Tester(topo)
self.tester.tuple_count(r, 4)
self.tester.punct_count(r, 2)
self.tester.test(self.test_ctxtype, self.test_config)
def test_multi_output_ports(self):
"""Operator with multiple output port."""
topo = Topology()
streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('testtkpy'))
s = topo.source([9237, -24])
s = s.map(lambda x: (x, ), schema='tuple<int64 v>')
bop = op.Invoke(
topo,
"com.ibm.streamsx.topology.pytest.pyprimitives::MultiOutputPorts",
s,
schemas=['tuple<int64 v1>', 'tuple<int32 v2>', 'tuple<int16 v3>'])
r = bop.outputs
self.tester = Tester(topo)
self.tester.tuple_count(s, 2)
self.tester.contents(r[0], [{'v1': 9237}, {'v1': -24}])
self.tester.contents(r[1], [{'v2': 9237 + 921}, {'v2': -24 + 921}])
self.tester.contents(r[2], [{'v3': 9237 - 407}, {'v3': -24 - 407}])
self.tester.test(self.test_ctxtype, self.test_config)
def test_multi_output_ports_punct(self):
"""Operator with multiple output port emits window marker after forwarding the tuple."""
topo = Topology()
streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('testtkpy'))
s = topo.source([9237, -24])
s = s.map(lambda x: (x, ), schema='tuple<int64 v>')
bop = op.Invoke(
topo,
"com.ibm.streamsx.topology.pytest.pyprimitives::MultiOutputPortsPunct",
s,
schemas=['tuple<int64 v1>', 'tuple<int32 v2>', 'tuple<int16 v3>'])
r = bop.outputs
r[0].print(tag='0', write_punctuations=True)
r[1].print(tag='1', write_punctuations=True)
r[2].print(tag='2', write_punctuations=True)
self.tester = Tester(topo)
self.tester.tuple_count(s, 2)
self.tester.punct_count(r[0], 2)
self.tester.punct_count(r[1], 2)
self.tester.punct_count(r[2], 2)
self.tester.contents(r[0], [{'v1': 9237}, {'v1': -24}])
self.tester.contents(r[1], [{'v2': 9237 + 921}, {'v2': -24 + 921}])
self.tester.contents(r[2], [{'v3': 9237 - 407}, {'v3': -24 - 407}])
self.tester.test(self.test_ctxtype, self.test_config)
def test_dict_output_ports(self):
"""Operator with multiple output port submitting dict objects."""
topo = Topology()
streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('testtkpy'))
s = topo.source([9237, -24])
s = s.map(lambda x: (x, x * 2, x + 4),
schema='tuple<int64 d, int64 e, int64 f>')
bop = op.Invoke(
topo,
"com.ibm.streamsx.topology.pytest.pyprimitives::DictOutputPorts",
s,
schemas=['tuple<int64 d, int64 e, int64 f>'] * 2)
r = bop.outputs
self.tester = Tester(topo)
self.tester.tuple_count(r[0], 2)
self.tester.tuple_count(r[1], 4)
self.tester.contents(r[0], [{
'd': 9237,
'e': (9237 * 2),
'f': 9237 + 4
}, {
'd': -24,
'e': (-24 * 2),
'f': -24 + 4
}])
self.tester.contents(r[1], [{
'd': 9237 + 7,
'f': (9237 * 2) + 777,
'e': 9237 + 4 + 77
}, {
'd': 9237,
'e': (9237 * 2),
'f': 9237 + 4
}, {
'd': -24 + 7,
'f': (-24 * 2) + 777,
'e': -24 + 4 + 77
}, {
'd': -24,
'e': (-24 * 2),
'f': -24 + 4
}])
self.tester.test(self.test_ctxtype, self.test_config)
def test_input_by_position(self):
"""Operator with input by position"""
topo = Topology()
streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('testtkpy'))
s = topo.source([3642, -393])
s = s.map(lambda x: (x, x * 2, x + 4),
schema='tuple<int64 d, int64 e, int64 f>')
bop = op.Map(
"com.ibm.streamsx.topology.pytest.pyprimitives::InputByPosition",
s)
r = bop.stream
self.tester = Tester(topo)
self.tester.tuple_count(r, 2)
self.tester.contents(r, [{
'd': 3642,
'e': (3642 * 2) + 89,
'f': -92
}, {
'd': -393,
'e': (-393 * 2) + 89,
'f': -92
}])
self.tester.test(self.test_ctxtype, self.test_config)
def test_only_output_port(self):
"""Operator with single output port and no inputs."""
count = 106
topo = Topology()
streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('testtkpy'))
bop = op.Source(
topo,
"com.ibm.streamsx.topology.pytest.pyprimitives::OutputOnly",
schema='tuple<int64 c>',
params={'count': count})
r = bop.stream
self.tester = Tester(topo)
self.tester.tuple_count(r, count)
self.tester.contents(r, list({'c': i + 501} for i in range(count)))
self.tester.test(self.test_ctxtype, self.test_config)
