def test_WindowPunctuation(self):
"""Trigger an aggregation 4 times. Ensure that window punctuations are submitted each time
by writing them to an output file, and then verifying that the file contains the correct
contents."""
topo = Topology()
s = topo.source([1,2,3,4])
# Aggregate and write to file.
s = s.last(1).trigger(1).aggregate(lambda x: x[0]+7)
# Ensure map/flat_map/filter passes window marks through.
s = s.flat_map(lambda x : [x])
s = s.filter(lambda x : True)
s = s.map(lambda x : (x,), schema='tuple<int32 z>')
op_params = {'file' : 'punct_file', 'writePunctuations' : True, 'flushOnPunctuation' : True}
op.Sink("spl.adapter::FileSink", s, params = op_params)
# Copy the config, since it's shared across all tests, and not every test needs a data
# directory.
cfg = self.test_config.copy()
jc = context.JobConfig(data_directory=os.getcwd())
jc.add(cfg)
tester = Tester(topo)
tester.test(self.test_ctxtype, cfg)
path = os.path.join(os.getcwd(), 'punct_file')
# Validate the contents of the file.
with open(path, 'r') as f:
file_contents = f.read()
self.assertEqual(expected_contents, file_contents)
os.remove(path)
def setUp(self):
Tester.setup_distributed(self)
# setup test config
self.test_config = {}
job_config = context.JobConfig(tracing='info')
job_config.add(self.test_config)
self.test_config[context.ConfigParams.SSL_VERIFY] = False
def submitToStreams(topology, streams_cfg, streams_instance):
"""Cancel if same name job is active and submit.
Args:
toplogy: streams application topology
streams_cfg : connection information - from get_instance()
streams_instance : from get_instance()
Notes:
Create local copy of the streams config so this can be thread-safe
"""
local_streams_cfg = dict(streams_cfg)
# Cancel the job from the instance if it is already running...
for job in streams_instance.get_jobs():
if job.name == topology.name:
print("Cancelling old job:", job.name)
job.cancel()
# Set the job config
job_config = context.JobConfig(job_name=topology.name, tracing="debug")
job_config.add(local_streams_cfg)
# Actually submit the job
print("Building and submitting new job:", topology.name)
submission_result = context.submit('DISTRIBUTED', topology,
local_streams_cfg)
return submission_result
def _job_config_args(cmd_args, app):
cfg = app.cfg
if cmd_args.job_config_overlays:
with open(cmd_args.job_config_overlays) as fd:
ctx.JobConfig.from_overlays(json.load(fd)).add(cfg)
elif not ctx.ConfigParams.JOB_CONFIG in cfg:
ctx.JobConfig().add(cfg)
jc = cfg[ctx.ConfigParams.JOB_CONFIG]
if cmd_args.job_name:
jc.job_name = str(cmd_args.job_name)
if cmd_args.preload:
jc.preload = True
if cmd_args.trace:
jc.tracing = cmd_args.trace
if cmd_args.submission_parameters:
jc.submission_parameters.update(cmd_args.submission_parameters)
def _build_launch_validate(self, name, composite_name):
print("------ " + name + " ------")
topo = Topology(name)
self._add_toolkits(topo)
params = {}
# Call the test composite
test_op = op.Source(topo,
composite_name,
'tuple<rstring result>',
params=params)
tester = Tester(topo)
tester.tuple_count(test_op.stream, 1, exact=True)
tester.contents(test_op.stream, [{'result': 'TEST_RESULT_PASS'}])
self.test_config[context.ConfigParams.SSL_VERIFY] = False
job_config = context.JobConfig(tracing='info')
job_config.add(self.test_config)
tester.test(self.test_ctxtype,
self.test_config,
always_collect_logs=True)
def _launch(main):
cfg = {}
job_config = context.JobConfig(tracing='info')
job_config.add(cfg)
cfg[context.ConfigParams.SSL_VERIFY] = False
rc = context.submit('DISTRIBUTED', main, cfg)