def _test_submit_sab(self):
sab_name = 'Sab_' + uuid.uuid4().hex
topo = topology.Topology(sab_name, namespace='mynamespace')
s = topo.source([1, 2])
es = s.for_each(lambda x: None)
bb = streamsx.topology.context.submit('BUNDLE', topo, {})
self.assertIn('bundlePath', bb)
self.assertIn('jobConfigPath', bb)
sas = self.sc.get_streaming_analytics()
sr = sas.submit_job(bundle=bb['bundlePath'])
job_id = sr.get('id', sr.get('jobId'))
self.assertIsNotNone(job_id)
self.assertIn('name', sr)
self.assertIn('application', sr)
self.assertEqual('mynamespace::' + sab_name, sr['application'])
cr = sas.cancel_job(job_id=job_id)
jn = 'SABTEST:' + uuid.uuid4().hex
jc = streamsx.topology.context.JobConfig(job_name=jn)
sr = sas.submit_job(bundle=bb['bundlePath'], job_config=jc)
job_id = sr.get('id', sr.get('jobId'))
self.assertIsNotNone(job_id)
self.assertIn('application', sr)
self.assertEqual('mynamespace::' + sab_name, sr['application'])
self.assertIn('name', sr)
self.assertEqual(jn, sr['name'])
cr = sas.cancel_job(job_id=job_id)
os.remove(bb['bundlePath'])
os.remove(bb['jobConfigPath'])
def test_job_refresh(self):
top = topology.Topology()
src = top.source(['Hello'])
self.tester = Tester(top)
self.tester.tuple_count(src, 1)
self.tester.local_check = self._verify_job_refresh
self.tester.test(self.test_ctxtype, self.test_config)
# Job was cancelled by test wait for health to change
timeout = 10
while hasattr(self.job, 'health') and 'healthy' == self.job.health:
time.sleep(0.2)
timeout -= 1
try:
self.job.refresh()
except exceptions.HTTPError:
self.job = None
break
self.assertGreaterEqual(
timeout,
0,
msg='Timeout exceeded while waiting for job to cancel')
if hasattr(self.job, 'health'):
self.assertNotEqual('healthy', self.job.health)
def test_basic_view_support(self):
self.logger.debug("Beginning test: test_basic_view_support.")
top = topology.Topology('basicViewTest')
# Send only one tuple
stream = top.source(DelayedTupleSourceWithLastTuple(['hello'], 20))
view = stream.view()
# Temporary workaround for Bluemix TLS issue with views
stream.publish(schema=schema.CommonSchema.String,
topic="__test_topic::test_basic_view_support")
self.logger.debug(
"Begging compilation and submission of basic_view_support topology."
)
self._submit(top)
time.sleep(5)
queue = view.start_data_fetch()
try:
view_tuple_value = queue.get(block=True, timeout=20.0)
except:
logger.exception("Timed out while waiting for tuple.")
raise
finally:
view.stop_data_fetch()
self.logger.debug("Returned view value in basic_view_support is " +
view_tuple_value)
self.assertTrue(view_tuple_value.startswith('hello'))
def build_ingest_app(config, streams_config):
gerrit_stream = gerrit.GerritEvents(**config['gerrit'])
ingest_topo = topology.Topology(INGEST_JOB_NAME)
event_stream = ingest_topo.source(gerrit_stream)
event_stream.publish('openstack_review_stream')
# ship it off to service to build and run
context.submit('ANALYTICS_SERVICE', ingest_topo, config=streams_config)
def _test_instance_submit(self):
""" Test submitting a bundle from an Instance.
Tests all four mechanisms.
"""
sab_name = 'ISJ_'+uuid.uuid4().hex
topo = topology.Topology(sab_name, namespace='myinstancens')
s = op.Source(topo, "spl.utility::Beacon",
'tuple<uint64 seq>',
params = {'period': 0.02, 'iterations':100})
s.seq = s.output('IterationCount()')
f = op.Map('spl.relational::Filter', s.stream,
params = {'filter': op.Expression.expression('seq % 2ul == 0ul')})
bb = streamsx.topology.context.submit('BUNDLE', topo, {})
self.assertIn('bundlePath', bb)
self.assertIn('jobConfigPath', bb)
sc = self.sc
instances = sc.get_instances()
if len(instances) == 1:
instance = instances[0]
else:
instance = sc.get_instance(os.environ['STREAMS_INSTANCE_ID'])
job = instance.submit_job(bb['bundlePath'])
self.assertIsInstance(job, Job)
self.assertEqual('myinstancens::'+sab_name, job.applicationName)
job.cancel()
with open(bb['jobConfigPath']) as fp:
jc = JobConfig.from_overlays(json.load(fp))
jn = 'JN_'+uuid.uuid4().hex
jc.job_name = jn
job = instance.submit_job(bb['bundlePath'], jc)
self.assertIsInstance(job, Job)
self.assertEqual('myinstancens::'+sab_name, job.applicationName)
self.assertEqual(jn, job.name)
job.cancel()
ab = instance.upload_bundle(bb['bundlePath'])
self.assertIsInstance(ab, ApplicationBundle)
job = ab.submit_job()
self.assertIsInstance(job, Job)
self.assertEqual('myinstancens::'+sab_name, job.applicationName)
job.cancel()
jn = 'JN_'+uuid.uuid4().hex
jc.job_name = jn
job = ab.submit_job(jc)
self.assertIsInstance(job, Job)
self.assertEqual('myinstancens::'+sab_name, job.applicationName)
self.assertEqual(jn, job.name)
job.cancel()
os.remove(bb['bundlePath'])
os.remove(bb['jobConfigPath'])
def test_basic_view_support(self):
self.logger.debug("Beginning test: test_basic_view_support.")
top = topology.Topology('basicViewTest')
# Send only one tuple
stream = top.source(DelayedTupleSourceWithLastTuple(['hello'], 20))
self._view = stream.view()
# Temporary workaround for Bluemix TLS issue with views
stream.publish(schema=schema.CommonSchema.String, topic="__test_topic::test_basic_view_support")
self.logger.debug("Beginning compilation and submission of basic_view_support topology.")
tester = Tester(top)
tester.local_check = self._verify_basic_view
tester.test(self.test_ctxtype, self.test_config)
def test_basic_calls(self):
"""
Test the basic rest apis.
"""
top = topology.Topology()
src = top.source(['Rest', 'tester'])
src = src.filter(lambda x : True)
src.view()
src = src.map(lambda x : x)
self.tester = Tester(top)
self.tester.tuple_count(src, 2)
self.tester.local_check = self._call_rest_apis
self.tester.test(self.test_ctxtype, self.test_config)
def build_filter_app(config, streams_config):
def comments_only(data):
jdata = json.loads(data)
print(jdata, file=sys.stderr)
return jdata["type"] == "comment-added"
# define our streams app
filter_topo = topology.Topology(FILTER_JOB_NAME)
event_stream2 = filter_topo.subscribe('openstack_review_stream')
comments_stream = event_stream2.filter(comments_only)
comments_stream.publish('comments')
c_view = comments_stream.view(name=COMMENT_VIEW_NAME)
# ship it off to service to build and run
context.submit('ANALYTICS_SERVICE', filter_topo, config=streams_config)
