def test_WindowPunctuation(self): """Trigger an aggregation 4 times. Ensure that window punctuations are submitted each time by writing them to an output file, and then verifying that the file contains the correct contents.""" topo = Topology() s = topo.source([1,2,3,4]) # Aggregate and write to file. s = s.last(1).trigger(1).aggregate(lambda x: x[0]+7) # Ensure map/flat_map/filter passes window marks through. s = s.flat_map(lambda x : [x]) s = s.filter(lambda x : True) s = s.map(lambda x : (x,), schema='tuple<int32 z>') op_params = {'file' : 'punct_file', 'writePunctuations' : True, 'flushOnPunctuation' : True} op.Sink("spl.adapter::FileSink", s, params = op_params) # Copy the config, since it's shared across all tests, and not every test needs a data # directory. cfg = self.test_config.copy() jc = context.JobConfig(data_directory=os.getcwd()) jc.add(cfg) tester = Tester(topo) tester.test(self.test_ctxtype, cfg) path = os.path.join(os.getcwd(), 'punct_file') # Validate the contents of the file. with open(path, 'r') as f: file_contents = f.read() self.assertEqual(expected_contents, file_contents) os.remove(path)
def setUp(self): Tester.setup_distributed(self) # setup test config self.test_config = {} job_config = context.JobConfig(tracing='info') job_config.add(self.test_config) self.test_config[context.ConfigParams.SSL_VERIFY] = False
def submitToStreams(topology, streams_cfg, streams_instance): """Cancel if same name job is active and submit. Args: toplogy: streams application topology streams_cfg : connection information - from get_instance() streams_instance : from get_instance() Notes: Create local copy of the streams config so this can be thread-safe """ local_streams_cfg = dict(streams_cfg) # Cancel the job from the instance if it is already running... for job in streams_instance.get_jobs(): if job.name == topology.name: print("Cancelling old job:", job.name) job.cancel() # Set the job config job_config = context.JobConfig(job_name=topology.name, tracing="debug") job_config.add(local_streams_cfg) # Actually submit the job print("Building and submitting new job:", topology.name) submission_result = context.submit('DISTRIBUTED', topology, local_streams_cfg) return submission_result
def _job_config_args(cmd_args, app): cfg = app.cfg if cmd_args.job_config_overlays: with open(cmd_args.job_config_overlays) as fd: ctx.JobConfig.from_overlays(json.load(fd)).add(cfg) elif not ctx.ConfigParams.JOB_CONFIG in cfg: ctx.JobConfig().add(cfg) jc = cfg[ctx.ConfigParams.JOB_CONFIG] if cmd_args.job_name: jc.job_name = str(cmd_args.job_name) if cmd_args.preload: jc.preload = True if cmd_args.trace: jc.tracing = cmd_args.trace if cmd_args.submission_parameters: jc.submission_parameters.update(cmd_args.submission_parameters)
def _build_launch_validate(self, name, composite_name): print("------ " + name + " ------") topo = Topology(name) self._add_toolkits(topo) params = {} # Call the test composite test_op = op.Source(topo, composite_name, 'tuple<rstring result>', params=params) tester = Tester(topo) tester.tuple_count(test_op.stream, 1, exact=True) tester.contents(test_op.stream, [{'result': 'TEST_RESULT_PASS'}]) self.test_config[context.ConfigParams.SSL_VERIFY] = False job_config = context.JobConfig(tracing='info') job_config.add(self.test_config) tester.test(self.test_ctxtype, self.test_config, always_collect_logs=True)
def _launch(main): cfg = {} job_config = context.JobConfig(tracing='info') job_config.add(cfg) cfg[context.ConfigParams.SSL_VERIFY] = False rc = context.submit('DISTRIBUTED', main, cfg)