def test_opdriven_source(self):
topo = Topology()
s = topo.source(TimeCounter(iterations=30, period=0.1))
s.set_consistent(ConsistentRegionConfig.operator_driven(drain_timeout=40, reset_timeout=40, max_consecutive_attempts=3))
tester = Tester(topo)
self.assertFalse(tester.test(self.test_ctxtype, self.test_config, assert_on_fail=False))
def test_SPLHashFunc(self):
"""
Test hashing works when the schema is a general SPL one
using an explicit hash function.
"""
raw = []
for v in range(20):
raw.append(''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(v)))
data = []
for v in range(7):
data.extend(raw)
random.shuffle(data)
for width in (1,4):
with self.subTest(width=width):
topo = Topology("test_SPLHash" + str(width))
s = topo.source(data)
s = s.as_string()
f = op.Map('spl.relational::Functor', s,
schema = 'tuple<rstring string, rstring s2>')
f.s2 = f.output('string + "_1234"')
s = f.stream
s = s.parallel(width, Routing.HASH_PARTITIONED, s2_hash)
s = s.map(AddChannel())
s = s.end_parallel()
s = s.map(CheckSameChannel(lambda t : t[0]['s2']))
expected = []
for v in data:
expected.append(v + '_1234')
tester = Tester(topo)
tester.contents(s, expected, ordered=width==1)
tester.test(self.test_ctxtype, self.test_config)
print(tester.result)
def test_verify_no_pint(self):
""" Verify pint is not installed on the service """
topo = Topology()
s = topo.source(down_a_pint_source)
tester = Tester(topo)
tester.contents(s, ['NoPintsForYou'])
tester.test(self.test_ctxtype, self.test_config)
def test_TopologyIndirectPackage(self):
topo = Topology("test_TopologyIndirectPackage")
hw = topo.source(["Hello", "World!"])
hwf = hw.transform(test2_pkg_helpers.imported_package)
tester = Tester(topo)
tester.contents(hwf, ["HelloIP"])
tester.test(self.test_ctxtype, self.test_config)
def test_TopologyImportPackage(self):
topo = Topology("test_TopologyImportPackage")
hw = topo.source(test_package.test_subpackage.test_module.SourceTuples(["Hello", "World!"]))
hwf = hw.filter(test_package.test_subpackage.test_module.filter)
tester = Tester(topo)
tester.contents(hwf, ["Hello"])
tester.test(self.test_ctxtype, self.test_config)
def test_StringHash(self):
"""
Test hashing works when the schema is tuple<rstring string>.
"""
raw = []
for v in range(20):
raw.append(''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(v)))
data = []
for v in range(7):
data.extend(raw)
random.shuffle(data)
for width in (1,3):
with self.subTest(width=width):
topo = Topology("test_StringHash" + str(width))
s = topo.source(data)
s = s.as_string()
s = s.parallel(width, Routing.HASH_PARTITIONED)
s = s.map(AddChannel())
s = s.end_parallel()
s = s.map(CheckSameChannel())
tester = Tester(topo)
tester.contents(s, data, ordered=width==1)
tester.test(self.test_ctxtype, self.test_config)
print(tester.result)
def test_TopologySourceItertools(self):
topo = Topology('test_TopologySourceItertools')
hw = topo.source(itertools.repeat(9, 3))
hw = hw.filter(test_functions.check_asserts_disabled)
tester = Tester(topo)
tester.contents(hw, [9, 9, 9])
tester.test(self.test_ctxtype, self.test_config)
def test_feedback_loop(self):
topo = Topology()
data = ['A','B', 'A', 'A', 'X', 'C', 'C', 'D', 'A', 'A', 'E']
expected = ['B', 'X', 'C', 'C', 'D', 'A', 'A', 'E']
s = topo.source(data)
s = s.filter(lambda t : time.sleep(1) or True).as_string();
feedback = PendingStream(topo)
df = op.Invoke(topo, 'spl.utility::DynamicFilter',
inputs = [s, feedback.stream],
schemas= [schema.CommonSchema.String])
df.params['key'] = df.attribute(s, 'string')
df.params['addKey'] = df.attribute(feedback.stream, 'string')
delayed_out = op.Map('spl.utility::Delay', df.outputs[0], params={'delay': 0.05}).stream
x = delayed_out.filter(lambda s : s == 'X').map(lambda s : 'A').as_string()
i = topo.source(['B', 'X', 'C', 'D', 'E']).as_string()
x = x.union({i})
feedback.complete(x)
result = delayed_out
result.print()
#streamsx.topology.context.submit('TOOLKIT', topo)
tester = Tester(topo)
tester.contents(result, expected)
tester.test(self.test_ctxtype, self.test_config)
def _run_app(self, kind, opi='M'):
schema = 'tuple<rstring a, int32 b>'
topo = Topology('TESPL' + str(uuid.uuid4().hex))
streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('testtkpy'))
if opi == 'M':
data = [1,2,3]
se = topo.source(data)
se = se.map(lambda x : {'a':'hello', 'b':x} , schema=schema)
prim = op.Map(
"com.ibm.streamsx.topology.pytest.pyexceptions::" + kind,
se, params={'tf':self.tf})
res = prim.stream
elif opi == 'S':
prim = op.Source(
topo,
"com.ibm.streamsx.topology.pytest.pyexceptions::" + kind,
schema=schema, params={'tf':self.tf})
res = prim.stream
elif opi == 'E':
data = [1,2,3]
se = topo.source(data)
se = se.map(lambda x : {'a':'hello', 'b':x} , schema=schema)
prim = op.Sink(
"com.ibm.streamsx.topology.pytest.pyexceptions::" + kind,
se, params={'tf':self.tf})
res = None
tester = Tester(topo)
tester.run_for(3)
ok = tester.test(self.test_ctxtype, self.test_config, assert_on_fail=False)
self.assertFalse(ok)
def setUp(self):
Tester.setup_distributed(self)
# Tester.setup_standalone(self)
"""TEST: test_basic validate that the operator can talk
def test_blob_type(self):
topo = Topology()
streamsx.spl.toolkit.add_toolkit(topo, '../testtkpy')
data = ['Hello', 'Blob', 'Did', 'you', 'reset' ]
s = topo.source(data)
s = s.as_string()
toBlob = op.Map(
"com.ibm.streamsx.topology.pytest.pytypes::ToBlob",
s,
'tuple<blob b>')
toBlob = op.Map(
"com.ibm.streamsx.topology.pysamples.positional::Noop",
toBlob.stream,
'tuple<blob b>')
bt = op.Map(
"com.ibm.streamsx.topology.pytest.pytypes::BlobTest",
toBlob.stream,
'tuple<rstring string>',
{'keep': True})
bt2 = op.Map(
"com.ibm.streamsx.topology.pytest.pytypes::BlobTest",
toBlob.stream,
'tuple<rstring string>',
{'keep': False})
tester = Tester(topo)
tester.contents(bt.stream, data)
self.test_config['topology.keepArtifacts'] = True;
tester.test(self.test_ctxtype, self.test_config)
def test_mixed_toolkits(self):
topo = Topology()
streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('testtkpy'))
streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('tk17'))
data = ['A']
bop = op.Source(topo, "spl.utility::Beacon",
'tuple<rstring a>',
{'iterations': 1})
bop.a = bop.output('"A"')
sv = op.Map(
"com.ibm.streamsx.topology.pytest.pyvers::StreamsxVersion",
bop.stream,
'tuple<rstring a, rstring v1, rstring v2>')
m17f = op.Map(
"com.ibm.streamsx.topology.pytest.tk17::M17F",
sv.stream,
'tuple<rstring a, rstring v1, rstring v2, rstring f1, rstring f2>')
m17c = op.Map(
"com.ibm.streamsx.topology.pytest.tk17::M17C",
m17f.stream,
'tuple<rstring a, rstring v1, rstring v2, rstring f1, rstring f2, rstring c1, rstring c2, int32 x>',
{'x': 27})
tester = Tester(topo)
tester.contents(m17c.stream, [{'a':'A', 'f1':'1.7', 'f2':'F', 'v1':'aggregate', 'v2':'True', 'c1':'1.7', 'c2':'C', 'x':27}])
tester.test(self.test_ctxtype, self.test_config)
def test_app_log(self):
topo = Topology()
s = topo.source(['logmsg1', 'logmsg2你好'])
s.for_each(_log_msg)
tester = Tester(topo)
tester.tuple_count(s, 2)
tester.test(self.test_ctxtype, self.test_config)
def test_WindowPunctuation(self):
"""Trigger an aggregation 4 times. Ensure that window punctuations are submitted each time
by writing them to an output file, and then verifying that the file contains the correct
contents."""
topo = Topology()
s = topo.source([1,2,3,4])
# Aggregate and write to file.
s = s.last(1).trigger(1).aggregate(lambda x: x[0]+7)
# Ensure map/flat_map/filter passes window marks through.
s = s.flat_map(lambda x : [x])
s = s.filter(lambda x : True)
s = s.map(lambda x : (x,), schema='tuple<int32 z>')
op_params = {'file' : 'punct_file', 'writePunctuations' : True, 'flushOnPunctuation' : True}
op.Sink("spl.adapter::FileSink", s, params = op_params)
# Copy the config, since it's shared across all tests, and not every test needs a data
# directory.
cfg = self.test_config.copy()
jc = context.JobConfig(data_directory=os.getcwd())
jc.add(cfg)
tester = Tester(topo)
tester.test(self.test_ctxtype, cfg)
path = os.path.join(os.getcwd(), 'punct_file')
# Validate the contents of the file.
with open(path, 'r') as f:
file_contents = f.read()
self.assertEqual(expected_contents, file_contents)
os.remove(path)
def setUp(self):
Tester.setup_streaming_analytics(self, force_remote_build=True)
self.sc = StreamingAnalyticsConnection()
self.test_config[ConfigParams.STREAMS_CONNECTION]=self.sc
self.is_v2 = False
if _IAMConstants.V2_REST_URL in self.sc.session.auth._credentials:
self.is_v2 = True
def test_string_to_string(self):
topo = Topology()
s = topo.source([False, 'b', 19]).as_string()
st = s.map(lambda x: x + '3', schema=CommonSchema.String)
tester = Tester(topo)
tester.contents(st, ['False3', 'b3', '193'])
tester.test(self.test_ctxtype, self.test_config)
def test_json_to_schema(self):
topo = Topology()
s = topo.source([{'a':7}, {'b':8}, {'c':9}]).as_json()
st = s.map(lambda x : (next(iter(x)), x[next(iter(x))]), schema='tuple<rstring y, int32 x>')
tester = Tester(topo)
tester.contents(st, [{'y':'a', 'x':7}, {'y':'b', 'x':8}, {'y':'c', 'x':9}])
tester.test(self.test_ctxtype, self.test_config)
def test_string_to_json(self):
topo = Topology()
s = topo.source(['a', 79, 'c']).as_string()
st = s.map(lambda x: x if x == 'c' else {'v': x + 'd'}, schema=CommonSchema.Json)
tester = Tester(topo)
tester.contents(st, [{'v': 'ad'}, {'v': '79d'}, 'c'])
tester.test(self.test_ctxtype, self.test_config)
def test_string_to_schema(self):
topo = Topology()
s = topo.source(['a', 'b', 'c']).as_string()
st = s.map(lambda x : (x+'struct!',), schema='tuple<rstring y>')
tester = Tester(topo)
tester.contents(st, [{'y':'astruct!'}, {'y':'bstruct!'}, {'y':'cstruct!'}])
tester.test(self.test_ctxtype, self.test_config)
def test_string_to_schema_dict(self):
topo = Topology()
s = topo.source(['a', 'b', 'c']).as_string()
st = s.map(lambda x : {'z': x+'dict!'}, schema='tuple<rstring z>')
tester = Tester(topo)
tester.contents(st, [{'z':'adict!'}, {'z':'bdict!'}, {'z':'cdict!'}])
tester.test(self.test_ctxtype, self.test_config)
def test_object_to_string(self):
topo = Topology()
s = topo.source([93,'hello',True])
st = s.map(lambda x : x, schema=CommonSchema.String)
tester = Tester(topo)
tester.contents(st, ['93','hello','True'])
tester.test(self.test_ctxtype, self.test_config)
def test_object_to_json(self):
topo = Topology()
s = topo.source([{'a': 7}, {'b': 8}, {'c': 9}])
st = s.map(lambda x: x, schema=CommonSchema.Json)
tester = Tester(topo)
tester.contents(st, [{'a': 7}, {'b': 8}, {'c': 9}])
tester.test(self.test_ctxtype, self.test_config)
def test_json_to_json(self):
topo = Topology()
s = topo.source([{'a': True}, {'a': 8}, {'a': 'third'}]).as_json()
st = s.map(lambda x : {'yy': x['a']}, schema=CommonSchema.Json)
tester = Tester(topo)
tester.contents(st, [{'yy': True}, {'yy': 8}, {'yy': 'third'}])
tester.test(self.test_ctxtype, self.test_config)
def test_as_tuple_for_each(self):
topo = Topology()
st = self._create_stream(topo)
st.for_each(check_is_tuple_for_each(self.is_named()))
tester = Tester(topo)
tester.tuple_count(st, 3)
tester.test(self.test_ctxtype, self.test_config)
def test_object_to_schema_dict(self):
topo = Topology()
s = topo.source([1,2,3])
st = s.map(lambda x : {'x':x}, schema='tuple<int32 x>')
tester = Tester(topo)
tester.contents(st, [{'x':1}, {'x':2}, {'x':3}])
tester.test(self.test_ctxtype, self.test_config)
def test_as_tuple_flat_map(self):
topo = Topology()
s = self._create_stream(topo)
st = s.flat_map(check_is_tuple_flat_map(self.is_named()))
tester = Tester(topo)
tester.contents(st, [(9,'2Hi!-FlatMap'), (18,'4Hi!-FlatMap'), (27,'6Hi!-FlatMap')])
tester.test(self.test_ctxtype, self.test_config)
def test_as_tuple_map_to_string(self):
topo = Topology()
s = self._create_stream(topo)
st = s.map(check_is_tuple_map_to_string(self.is_named()), schema=CommonSchema.String)
tester = Tester(topo)
tester.contents(st, ['112Hi!-MapString', '224Hi!-MapString', '336Hi!-MapString'])
tester.test(self.test_ctxtype, self.test_config)
def test_as_tuple_map_to_schema(self):
topo = Topology()
s = self._create_stream(topo)
st = s.map(check_is_tuple_map_to_schema(self.is_named()), schema=StreamSchema('tuple<int32 y, rstring txt>'))
tester = Tester(topo)
tester.contents(st, [{'y':13, 'txt':'2Hi!-MapSPL'}, {'y':26, 'txt':'4Hi!-MapSPL'}, {'y':39, 'txt':'6Hi!-MapSPL'}])
tester.test(self.test_ctxtype, self.test_config)
def test_as_tuple_filter(self):
topo = Topology()
s = self._create_stream(topo)
s = s.filter(check_is_tuple_filter(self.is_named()))
tester = Tester(topo)
tester.tuple_count(s, 2)
tester.test(self.test_ctxtype, self.test_config)
def test_json_to_string(self):
topo = Topology()
s = topo.source([{'a': True}, {'a': 8}, {'a': 'third'}]).as_json()
st = s.map(lambda x : x['a'], schema=CommonSchema.String)
tester = Tester(topo)
tester.contents(st, ['True', '8', 'third'])
tester.test(self.test_ctxtype, self.test_config)
def test_lambda_module_refs(self):
topo = Topology("test_lambda_module_refs")
sr = topo.source(_rand_msg, name='RM')
#sevs = hw.split(3, lambda m: M.get(m, -1), names=['high', 'medium', 'low'], name='SeveritySplit')
tt = 33
s1 = topo.source(lambda: [tt], name="S1")
s2 = topo.source(lambda: [streamsx.ec.is_active()], name="S2")
s3 = topo.source(lambda: [random.randint(0, 99)], name="S3")
s1.for_each(lambda x: tt, name='S1E')
s2.for_each(lambda x: streamsx.ec.is_active(), name='S2E')
s3.for_each(lambda x: None, name='S3E')
srm1 = sr.map(lambda x: tt, name='SMR1')
srm2 = sr.map(lambda x: streamsx.ec.is_active() and ia1234,
name='SMR2')
srm3 = sr.map(lambda x: random.randint(0, 100), name='SMR3')
srf1 = srm1.filter(lambda x: tt, name='SRF1')
srf2 = srm2.filter(
lambda x: streamsx.ec.is_active() and datetime.now(), name='SRF2')
srf3 = srm3.filter(lambda x: random.randint(0, 2), name='SRF3')
tester = Tester(topo)
# Mostly testing this runs without failing due to
# unresolved modules
tester.contents(s1, [33])
tester.contents(s2, [True])
tester.tuple_count(s3, 1)
tester.tuple_count(srf1, 100, exact=False)
tester.tuple_count(srf2, 100, exact=False)
tester.tuple_count(srf3, 30, exact=False)
tester.test(self.test_ctxtype, self.test_config)
class TestDistributedRestFeatures(unittest.TestCase):
@classmethod
def setUpClass(cls):
"""
Initialize the logger and get the SWS username, password, and REST URL.
:return: None
"""
cls.is_v2 = None
cls.logger = logger
def setUp(self):
Tester.setup_distributed(self)
self.sc = StreamsConnection()
self.sc.session.verify = False
self.test_config[ConfigParams.STREAMS_CONNECTION] = self.sc
def test_username_and_password(self):
self.logger.debug("Beginning test: test_username_and_password.")
# Ensure, at minimum, that the StreamsContext can connect and retrieve valid data from the SWS resources path
resources = self.sc.get_resources()
self.logger.debug("Number of retrieved resources is: " +
str(len(resources)))
self.assertGreater(
len(resources),
0,
msg="Returned zero resources from the \"resources\" endpoint.")
def test_streamsconnection_samplecode(self):
self.logger.debug("Beginning test: test_streamsconnection_samplecode.")
domains = self.sc.get_domains()
if domains is not None:
self.assertGreater(len(domains),
0,
msg="Should have more than 0 domains.")
instances = self.sc.get_instances()
self.assertGreater(len(instances),
0,
msg="Should have more than 0 instances.")
jobs_count = 0
for instance in instances:
jobs_count += len(instance.get_jobs())
def _verify_basic_view(self):
q = self._view.start_data_fetch()
try:
view_tuple_value = q.get(block=True, timeout=25.0)
except:
logger.exception("Timed out while waiting for tuple.")
raise
self._view.stop_data_fetch()
self.logger.debug("Returned view value in basic_view_support is " +
view_tuple_value)
self.assertTrue(view_tuple_value.startswith('hello'))
def test_basic_view_support(self):
self.logger.debug("Beginning test: test_basic_view_support.")
top = topology.Topology()
# Send only one tuple
stream = top.source(DelayedTupleSourceWithLastTuple(['hello'], 20))
self._view = stream.view(start=True, buffer_time=60)
# Temporary workaround for Bluemix TLS issue with views
#stream.publish(schema=schema.CommonSchema.String, topic="__test_topic::test_basic_view_support")
self.logger.debug(
"Beginning compilation and submission of basic_view_support topology."
)
tester = Tester(top)
tester.local_check = self._verify_basic_view
tester.test(self.test_ctxtype, self.test_config)
def _verify_job_refresh(self):
result = self.tester.submission_result
self.job = self.sc.get_instance(result['instanceId']).get_job(
result['jobId'])
self.assertEqual('healthy', self.job.health)
def test_job_refresh(self):
top = topology.Topology()
src = top.source(['Hello'])
self.tester = Tester(top)
self.tester.tuple_count(src, 1)
self.tester.local_check = self._verify_job_refresh
self.tester.test(self.test_ctxtype, self.test_config)
# Job was cancelled by test wait for health to change
timeout = 10
while hasattr(self.job, 'health') and 'healthy' == self.job.health:
time.sleep(0.2)
timeout -= 1
try:
self.job.refresh()
except exceptions.HTTPError:
self.job = None
break
self.assertGreaterEqual(
timeout,
0,
msg='Timeout exceeded while waiting for job to cancel')
if hasattr(self.job, 'health'):
self.assertNotEqual('healthy', self.job.health)
def _call_rest_apis(self):
job = self.tester.submission_result.job
self.assertIsInstance(job, Job)
primitives_caller.check_job(self, job)
instance = job.get_instance()
self.assertIsInstance(instance, Instance)
primitives_caller.check_instance(self, instance)
domain = instance.get_domain()
if domain is not None:
self.assertIsInstance(domain, Domain)
primitives_caller.check_domain(self, domain)
nops = job.get_operators(name='.*BASIC.$')
self.assertEqual(2, len(nops))
nops = job.get_operators(name='.*BASICD$')
self.assertEqual(1, len(nops))
self.assertTrue(nops[0].name.endswith('BASICD'))
def test_basic_calls(self):
"""
Test the basic rest apis.
"""
top = topology.Topology()
src = top.source(['Rest', 'tester'])
src = src.filter(lambda x: True, name='BASICC')
src.view()
src = src.map(lambda x: x, name='BASICD')
self.tester = Tester(top)
self.tester.tuple_count(src, 2)
self.tester.local_check = self._call_rest_apis
self.tester.test(self.test_ctxtype, self.test_config)
# Underscore as the local evironment must match the remote environment
# such as OS version and architecture type.
def _test_instance_submit(self):
""" Test submitting a bundle from an Instance.
Tests all four mechanisms.
"""
sab_name = 'ISJ_' + uuid.uuid4().hex
topo = topology.Topology(sab_name, namespace='myinstancens')
s = op.Source(topo,
"spl.utility::Beacon",
'tuple<uint64 seq>',
params={
'period': 0.02,
'iterations': 100
})
s.seq = s.output('IterationCount()')
f = op.Map(
'spl.relational::Filter',
s.stream,
params={'filter': op.Expression.expression('seq % 2ul == 0ul')})
bb = streamsx.topology.context.submit('BUNDLE', topo, {})
self.assertIn('bundlePath', bb)
self.assertIn('jobConfigPath', bb)
sc = self.sc
instances = sc.get_instances()
if len(instances) == 1:
instance = instances[0]
else:
instance = sc.get_instance(os.environ['STREAMS_INSTANCE_ID'])
job = instance.submit_job(bb['bundlePath'])
self.assertIsInstance(job, Job)
self.assertEqual('myinstancens::' + sab_name, job.applicationName)
job.cancel()
with open(bb['jobConfigPath']) as fp:
jc = JobConfig.from_overlays(json.load(fp))
jn = 'JN_' + uuid.uuid4().hex
jc.job_name = jn
job = instance.submit_job(bb['bundlePath'], jc)
self.assertIsInstance(job, Job)
self.assertEqual('myinstancens::' + sab_name, job.applicationName)
self.assertEqual(jn, job.name)
job.cancel()
ab = instance.upload_bundle(bb['bundlePath'])
self.assertIsInstance(ab, ApplicationBundle)
job = ab.submit_job()
self.assertIsInstance(job, Job)
self.assertEqual('myinstancens::' + sab_name, job.applicationName)
job.cancel()
jn = 'JN_' + uuid.uuid4().hex
jc.job_name = jn
job = ab.submit_job(jc)
self.assertIsInstance(job, Job)
self.assertEqual('myinstancens::' + sab_name, job.applicationName)
self.assertEqual(jn, job.name)
job.cancel()
os.remove(bb['bundlePath'])
os.remove(bb['jobConfigPath'])
class TestTester(unittest.TestCase):
_multiprocess_can_split_ = True
def setUp(self):
Tester.setup_standalone(self)
def test_at_least(self):
""" Test the at least tuple count.
"""
if self.test_ctxtype == context.ContextTypes.STANDALONE:
return unittest.skip("Standalone tests must complete")
topo = Topology()
s = topo.source(rands)
tester = Tester(topo)
tester.tuple_count(s, 100, exact=False)
tester.test(self.test_ctxtype, self.test_config)
def test_no_tuples(self):
""" Test exact count with zero tuples.
"""
topo = Topology()
s = topo.source([])
tester = Tester(topo)
tester.tuple_count(s, 0)
tester.test(self.test_ctxtype, self.test_config)
def test_at_least_no_tuples(self):
""" Test at least count with zero tuples.
(kind of a pointless condition, always true).
"""
if self.test_ctxtype == context.ContextTypes.STANDALONE:
return unittest.skip("Standalone tests must complete")
topo = Topology()
s = topo.source([])
tester = Tester(topo)
tester.tuple_count(s, 0, exact=False)
tester.test(self.test_ctxtype, self.test_config)
def test_checker(self):
""" Test the per-tuple checker.
"""
topo = Topology()
s = topo.source(rands)
s = s.filter(lambda r: r > 0.8)
s = s.map(lambda r: r + 7.0)
tester = Tester(topo)
tester.tuple_count(s, 200, exact=False)
if self.test_ctxtype == context.ContextTypes.STANDALONE:
tester.run_for(20)
tester.tuple_check(s, lambda r: r > 7.8)
# Ensure we perform dependency checking for the check function
import fns_test2_test
tester.tuple_check(s, fns_test2_test.tc_dep)
tester.test(self.test_ctxtype, self.test_config)
def test_local_check(self):
""" Test the at least tuple count.
"""
if self.test_ctxtype == context.ContextTypes.STANDALONE:
return unittest.skip("Standalone tests don't support local check")
topo = Topology()
s = topo.source(rands)
self.my_local_called = False
self.tester = Tester(topo)
self.tester.tuple_count(s, 100, exact=False)
self.tester.local_check = self.my_local
self.tester.test(self.test_ctxtype, self.test_config)
self.assertTrue(self.my_local_called)
def my_local(self):
self.assertTrue(hasattr(self.tester, 'submission_result'))
self.assertTrue(hasattr(self.tester, 'streams_connection'))
self.assertIs(self.tester.streams_connection,
self.tester.submission_result.job.rest_client._sc)
self.my_local_called = True
def test_bad_pe(self):
"""Test a failure in a PE is caught as a test failure"""
topo = Topology()
s = topo.source(rands)
# intentional addition of a string with an int
# to cause a PE failure
s = s.map(lambda t: t + 'a string')
tester = Tester(topo)
tester.tuple_count(s, 0, exact=False)
tp = tester.test(self.test_ctxtype,
self.test_config,
assert_on_fail=False)
self.assertFalse(tp)
def test_run_for(self):
topo = Topology()
s = topo.source([1, 2, 3])
self.tester = Tester(topo)
self.tester.tuple_count(s, 3)
self.tester.run_for(120)
if self.test_ctxtype == context.ContextTypes.STANDALONE:
self.rf_start = time.time()
else:
self.tester.local_check = self.get_start_time
self.tester.test(self.test_ctxtype, self.test_config)
now = time.time()
test_duration = now - self.rf_start
self.assertTrue(test_duration >= 120)
def test_eventual_result_ok(self):
N = 500000
topo = Topology()
s = topo.source(range(N))
w = s.batch(datetime.timedelta(milliseconds=300))
a = w.aggregate(lambda t: (len(t), sum(t)))
tester = Tester(topo)
tester.tuple_count(s, N)
tester.eventual_result(a, _EROK(N))
# Ensure we perform dependency checking for the check function
import fns_test2_test
tester.eventual_result(s, fns_test2_test.tc_dep)
tester.test(self.test_ctxtype, self.test_config)
def test_eventual_result_bad(self):
N = 500000
topo = Topology()
s = topo.source(range(N))
w = s.batch(datetime.timedelta(milliseconds=300))
a = w.aggregate(lambda t: (len(t), sum(t)))
tester = Tester(topo)
tester.tuple_count(s, N)
tester.eventual_result(a, _EROK(int(N / 4)))
ok = tester.test(self.test_ctxtype,
self.test_config,
assert_on_fail=False)
self.assertFalse(ok)
def test_count_bad(self):
N = 10
topo = Topology()
s = topo.source(range(N))
tester = Tester(topo)
tester.tuple_count(s, N + 1)
ok = tester.test(self.test_ctxtype,
self.test_config,
assert_on_fail=False)
self.assertFalse(ok)
def test_count_bad_conflicting(self):
N = 10
topo = Topology()
s = topo.source(range(N))
tester = Tester(topo)
# Add one that fails and one that never becomes valid
tester.tuple_count(s.map(), int(N / 2))
tester.tuple_count(s, N + 1)
ok = tester.test(self.test_ctxtype,
self.test_config,
assert_on_fail=False)
self.assertFalse(ok)
def get_start_time(self):
job = self.tester.submission_result.job
self.rf_start = job.submitTime / 1000.0
def setUp(self):
self.dir = tempfile.mkdtemp()
Tester.setup_standalone(self)
class CommonTests(unittest.TestCase):
@classmethod
def setUpClass(cls):
"""
Initialize the logger and get the SWS username, password, and REST URL.
:return: None
"""
if cls is CommonTests:
raise unittest.SkipTest("Skipping base tests.")
def test_username_and_password(self):
self.logger.debug("Beginning test: test_username_and_password.")
# Ensure, at minimum, that the StreamsContext can connect and retrieve valid data from the SWS resources path
resources = self.sc.get_resources()
self.logger.debug("Number of retrieved resources is: " +
str(len(resources)))
self.assertGreater(
len(resources),
0,
msg="Returned zero resources from the \"resources\" endpoint.")
def test_streamsconnection_samplecode(self):
self.logger.debug("Beginning test: test_streamsconnection_samplecode.")
domains = self.sc.get_domains()
self.assertGreater(len(domains),
0,
msg="Should have more than 0 domains.")
instances = self.sc.get_instances()
self.assertGreater(len(instances),
0,
msg="Should have more than 0 instances.")
jobs_count = 0
for instance in instances:
jobs_count += len(instance.get_jobs())
def _verify_basic_view(self):
q = self._view.start_data_fetch()
try:
view_tuple_value = q.get(block=True, timeout=25.0)
except:
logger.exception("Timed out while waiting for tuple.")
raise
self._view.stop_data_fetch()
self.logger.debug("Returned view value in basic_view_support is " +
view_tuple_value)
self.assertTrue(view_tuple_value.startswith('hello'))
def test_basic_view_support(self):
self.logger.debug("Beginning test: test_basic_view_support.")
top = topology.Topology('basicViewTest')
# Send only one tuple
stream = top.source(DelayedTupleSourceWithLastTuple(['hello'], 20))
self._view = stream.view(start=True, buffer_time=60)
# Temporary workaround for Bluemix TLS issue with views
#stream.publish(schema=schema.CommonSchema.String, topic="__test_topic::test_basic_view_support")
self.logger.debug(
"Beginning compilation and submission of basic_view_support topology."
)
tester = Tester(top)
tester.local_check = self._verify_basic_view
tester.test(self.test_ctxtype, self.test_config)
def _verify_job_refresh(self):
result = self.tester.submission_result
self.job = self.sc.get_instance(result['instanceId']).get_job(
result['jobId'])
self.assertEqual('healthy', self.job.health)
def test_job_refresh(self):
top = topology.Topology('jobRefreshTest')
src = top.source(['Hello'])
self.tester = Tester(top)
self.tester.tuple_count(src, 1)
self.tester.local_check = self._verify_job_refresh
self.tester.test(self.test_ctxtype, self.test_config)
# Job was cancelled by test wait for health to change
timeout = 10
while hasattr(self.job, 'health') and 'healthy' == self.job.health:
time.sleep(0.2)
timeout -= 1
self.job.refresh()
self.assertGreaterEqual(
timeout,
0,
msg='Timeout exceeded while waiting for job to cancel')
if hasattr(self.job, 'health'):
self.assertNotEqual('healthy', self.job.health)
def _call_rest_apis(self):
job = self.tester.submission_result.job
self.assertIsInstance(job, Job)
primitives_caller.check_job(self, job)
instance = job.get_instance()
self.assertIsInstance(instance, Instance)
primitives_caller.check_instance(self, instance)
domain = instance.get_domain()
self.assertIsInstance(domain, Domain)
primitives_caller.check_domain(self, domain)
def test_basic_calls(self):
"""
Test the basic rest apis.
"""
top = topology.Topology()
src = top.source(['Rest', 'tester'])
src = src.filter(lambda x: True)
src.view()
src = src.map(lambda x: x)
self.tester = Tester(top)
self.tester.tuple_count(src, 2)
self.tester.local_check = self._call_rest_apis
self.tester.test(self.test_ctxtype, self.test_config)
def setUp(self):
Tester.setup_streaming_analytics(self, force_remote_build=True)
# Ensure the old name still works.
self.test_ctxtype = "ANALYTICS_SERVICE"
def test_sys_argv(self):
topo = Topology()
s = topo.source(get_sys_argv)
tester = Tester(topo)
tester.contents(s, [''])
tester.test(self.test_ctxtype, self.test_config)
def setUp(self):
Tester.setup_standalone(self)
self.result = {}
def setUp(self):
Tester.setup_distributed(self)
self.test_config[ConfigParams.SSL_VERIFY] = False
def test_read_write(self):
topo = Topology()
s = topo.source(range(13))
sch = 'tuple<rstring a, int32 b>'
s = s.map(lambda v: ('A' + str(v), v + 7), schema=sch)
fn = os.path.join(self.dir, 'data.csv')
files.csv_writer(s, fn)
tester = Tester(topo)
tester.tuple_count(s, 13)
tester.test(self.test_ctxtype, self.test_config)
self.assertTrue(os.path.isfile(fn))
topo = Topology()
r = files.csv_reader(topo, schema=sch, file=fn)
expected = [{'a': 'A' + str(v), 'b': v + 7} for v in range(13)]
tester = Tester(topo)
tester.contents(r, expected)
tester.test(self.test_ctxtype, self.test_config)
def setUp(self):
Tester.setup_distributed(self)
def test_checker(self):
""" Test the per-tuple checker.
"""
topo = Topology()
s = topo.source(rands)
s = s.filter(lambda r: r > 0.8)
s = s.map(lambda r: r + 7.0)
tester = Tester(topo)
tester.tuple_count(s, 200, exact=False)
if self.test_ctxtype == context.ContextTypes.STANDALONE:
tester.run_for(20)
tester.tuple_check(s, lambda r: r > 7.8)
# Ensure we perform dependency checking for the check function
import fns_test2_test
tester.tuple_check(s, fns_test2_test.tc_dep)
tester.test(self.test_ctxtype, self.test_config)
class TestDistributedPubSub(unittest.TestCase):
""" Test basic pub-sub in SPL
"""
def setUp(self):
Tester.setup_distributed(self)
self.test_config[ConfigParams.SSL_VERIFY] = False
def _publish(self, topo, N, topic, width=None, allow_filter=False):
b = op.Source(topo,
"spl.utility::Beacon",
SCHEMA,
params={
'initDelay': 10.0,
'iterations': N
})
b.seq = b.output('IterationCount()')
ps = b.stream
if width:
ps = ps.parallel(width=width)
p = op.Sink('com.ibm.streamsx.topology.topic::Publish',
ps,
params={'topic': topic},
name='MSP')
if allow_filter:
p.params['allowFilter'] = True
def _subscribe(self,
topo,
topic,
direct=True,
drop=None,
filtered=False,
extra=None):
s = op.Source(
topo,
"com.ibm.streamsx.topology.topic::FilteredSubscribe"
if filtered else "com.ibm.streamsx.topology.topic::Subscribe",
SCHEMA,
params={
'topic': topic,
'streamType': SCHEMA
},
name='MSS')
if extra:
s.params.update(extra)
if not direct:
s.params['connect'] = op.Expression.expression(
'com.ibm.streamsx.topology.topic::Buffered')
if drop:
s.params['bufferFullPolicy'] = op.Expression.expression(
'Sys.' + drop)
return s.stream.filter(slowme)
return s.stream
def _get_single_sub_op(self):
job = self.tester.submission_result.job
self.assertEqual('healthy', job.health)
for op in job.get_operators():
if op.name.startswith(
'MSS') and op.operatorKind == 'spl.relational::Filter':
mss = op
return mss
def _get_single_sub_metrics(self, mss):
nDropped = None
nProcessed = None
ip = mss.get_input_ports()[0]
while nDropped is None or nProcessed is None:
if nDropped is None:
metrics = ip.get_metrics(name='nTuplesDropped')
if metrics:
nDropped = metrics[0]
if nProcessed is None:
metrics = ip.get_metrics(name='nTuplesProcessed')
if metrics:
nProcessed = metrics[0]
return nDropped, nProcessed
def check_single_sub(self):
"""
Check we get all the tuples with none dropped
with a single subcriber.
"""
mss = self._get_single_sub_op()
nDropped, nProcessed = self._get_single_sub_metrics(mss)
while nProcessed.value < self.N:
self.assertEqual(0, nDropped.value)
time.sleep(2)
nDropped, nProcessed = self._get_single_sub_metrics(mss)
self.assertEqual(0, nDropped.value)
self.assertEqual(self.N, nProcessed.value)
def check_single_sub_drop(self):
"""
Check we get all the tuples with none dropped
with a single subcriber.
"""
mss = self._get_single_sub_op()
nDropped, nProcessed = self._get_single_sub_metrics(mss)
while nDropped.value + nProcessed.value < self.N:
time.sleep(2)
nDropped, nProcessed = self._get_single_sub_metrics(mss)
self.assertEqual(self.N, nDropped.value + nProcessed.value)
self.assertTrue(nDropped.value > 0)
def test_One2One(self):
"""Publish->Subscribe
"""
N = 2466
topic = random_topic()
topo = Topology()
# Subscriber
s = self._subscribe(topo, topic)
# Publisher
self._publish(topo, N, topic)
self.tester = Tester(topo)
self.tester.run_for(15)
self.N = N
self.tester.tuple_count(s, N)
self.tester.local_check = self.check_single_sub
self.tester.test(self.test_ctxtype, self.test_config)
def test_One2OneNonDirect(self):
"""Publish->Subscribe with a buffered subscriber.
"""
N = 3252
topic = random_topic()
topo = Topology()
# Subscriber
s = self._subscribe(topo, topic, direct=False)
# Publisher
self._publish(topo, N, topic)
self.tester = Tester(topo)
self.tester.run_for(15)
self.N = N
self.tester.tuple_count(s, N)
self.tester.local_check = self.check_single_sub
self.tester.test(self.test_ctxtype, self.test_config)
def test_One2OneNonDirectDropFirst(self):
"""Publish->Subscribe with a buffered subscriber.
"""
N = 5032
topic = random_topic()
topo = Topology()
# Subscriber
s = self._subscribe(topo, topic, direct=False, drop='DropFirst')
# Publisher
self._publish(topo, N, topic)
self.tester = Tester(topo)
self.tester.run_for(15)
self.N = N
# 1000-2 for window & final mark
self.tester.tuple_count(s, 998, exact=False)
self.tester.local_check = self.check_single_sub_drop
self.tester.test(self.test_ctxtype, self.test_config)
def test_One2OneNonDirectDropLast(self):
"""Publish->Subscribe with a buffered subscriber.
"""
N = 5032
topic = random_topic()
topo = Topology()
# Subscriber
s = self._subscribe(topo, topic, direct=False, drop='DropLast')
# Publisher
self._publish(topo, N, topic)
self.tester = Tester(topo)
self.tester.run_for(15)
self.N = N
self.tester.tuple_count(s, 1000, exact=False)
self.tester.local_check = self.check_single_sub_drop
self.tester.test(self.test_ctxtype, self.test_config)
def test_UDPMany2One(self):
"""
UDP publishers to a single subscriber.
"""
N = 17342
for pw in (1, 5):
topic = random_topic()
topo = Topology()
# Subscriber
s = self._subscribe(topo, topic)
# Publisher
self._publish(topo, N, topic, width=pw)
self.tester = Tester(topo)
self.tester.run_for(15)
self.tester.tuple_count(s, N)
self.N = N
self.tester.local_check = self.check_single_sub
self.tester.test(self.test_ctxtype, self.test_config)
def test_Many2One(self):
"""
Many non-UDP publishers to a single subscriber.
"""
N = 17342
topic = random_topic()
topo = Topology()
# Subscriber
s = self._subscribe(topo, topic)
# Publisher
M = 3
for i in range(M):
self._publish(topo, N, topic)
self.tester = Tester(topo)
self.tester.run_for(15)
self.tester.tuple_count(s, N * M)
self.N = N * M
self.tester.local_check = self.check_single_sub
self.tester.test(self.test_ctxtype, self.test_config)
def test_allow_filter_subscribe(self):
N = 99
topic = random_topic()
topo = Topology()
# Non-Filter Subscriber
s = self._subscribe(topo, topic)
# Publisher
self._publish(topo, N, topic, allow_filter=True)
self.tester = Tester(topo)
self.tester.run_for(15)
self.N = N
self.tester.tuple_count(s, N)
self.tester.local_check = self.check_single_sub
self.tester.test(self.test_ctxtype, self.test_config)
def test_allow_filter_filtered_subscribe(self):
N = 201
F = 87
pd = os.path.dirname(os.path.dirname(__file__))
ttk = os.path.join(pd, 'testtk')
for af, d in [(False, False), (False, True), (True, False),
(True, True)]:
#af = c[0]
#d = c[1]
topic = random_topic()
topo = Topology()
streamsx.spl.toolkit.add_toolkit(topo, ttk)
extra = {}
extra['remoteFilter'] = 'seq < 87'
extra['localFilterFunction'] = op.Expression.expression(
'testspl::affs')
s = self._subscribe(topo,
topic,
direct=d,
filtered=True,
extra=extra)
sf = s.filter(check_lt_87)
# Publisher
self._publish(topo, N, topic, allow_filter=af)
self.tester = Tester(topo)
self.tester.run_for(15)
self.N = F
self.tester.tuple_count(s, F)
self.tester.tuple_check(s, check_lt_87)
#self.tester.local_check = self.check_single_sub
self.tester.test(self.test_ctxtype, self.test_config)
def setUp(self):
Tester.setup_distributed(self)
self.result = {}
def setUp(self):
# setup test config
#self.test_config = {}
#job_config = context.JobConfig(tracing='info')
#job_config.add(self.test_config)
Tester.setup_streaming_analytics(self, force_remote_build=False)
def setUp(self):
Tester.setup_streaming_analytics(self)
def setUp(self):
self.tf = None
Tester.setup_distributed(self)
class TestPubSub(unittest.TestCase):
""" Test publish/subscribe
"""
_multiprocess_can_split_ = True
def setUp(self):
Tester.setup_distributed(self)
def _check_topics(self):
ins = self.tester.submission_result.job.get_instance()
# topics are created dynamically so give some time
# for them to be created
for _ in range(10):
topics = ins.get_published_topics()
if topics:
break
time.sleep(2)
# Don't assume this is the only app running so
# other topics may exist
sts = 0
stp = 0
for pt in topics:
if self.topic_spl == pt.topic:
sts += 1
if pt.schema is not None:
self.assertEqual(StSc('tuple<uint64 seq>'), pt.schema)
elif self.topic_python == pt.topic:
if pt.schema is not None:
self.assertEqual(CmnSc.Python.value, pt.schema)
stp += 1
self.assertEqual(1, sts)
self.assertEqual(1, stp)
def test_published_topics(self):
"""Test a published stream is available through get_published_topics.
"""
tspl = ''.join(random.choice('0123456789abcdef') for x in range(20))
tpy = ''.join(random.choice('0123456789abcdef') for x in range(20))
self.topic_spl = 'topology/test/python/' + tspl
self.topic_python = 'topology/test/python/' + tpy
self.assertNotEqual(self.topic_spl, self.topic_python)
topo = Topology()
beacon = op.Source(topo,
"spl.utility::Beacon",
'tuple<uint64 seq>',
params={'period': 0.02})
beacon.seq = beacon.output('IterationCount()')
beacon.stream.publish(topic=self.topic_spl)
s = beacon.stream.map(lambda x: x)
s.publish(topic=self.topic_python)
# Publish twice to ensure its only listed once
s = s.filter(lambda x: True)
s.publish(topic=self.topic_python)
self.tester = Tester(topo)
self.tester.local_check = self._check_topics
self.tester.tuple_count(s, 100, exact=False)
self.tester.test(self.test_ctxtype, self.test_config)
def _check_buffer(self):
job = self.tester.submission_result.job
seen_sub = False
seen_qs = False
for op in job.get_operators():
if op.operatorKind == 'spl.relational::Filter':
seen_sub = True
ip = op.get_input_ports()[0]
for _ in range(5):
for m in ip.get_metrics():
if m.name == 'queueSize':
seen_qs = True
self.assertEqual(self._buf_size, m.value)
if seen_qs:
break
time.sleep(2)
self.assertTrue(seen_sub)
self.assertTrue(seen_qs)
def test_subscribe_direct_explicit(self):
topic = ''.join(random.choice('0123456789abcdef') for x in range(20))
topo = Topology()
s = topo.subscribe(topic=topic, connect=SubscribeConnection.Direct)
self._buf_size = 0
self.tester = Tester(topo)
self.tester.local_check = self._check_buffer
self.tester.test(self.test_ctxtype, self.test_config)
def test_subscribe_buffered(self):
topic = ''.join(random.choice('0123456789abcdef') for x in range(20))
topo = Topology()
s = topo.subscribe(topic=topic, connect=SubscribeConnection.Buffered)
self._buf_size = 1000
self.tester = Tester(topo)
self.tester.local_check = self._check_buffer
self.tester.test(self.test_ctxtype, self.test_config)
def test_subscribe_buffered_capacity(self):
topic = ''.join(random.choice('0123456789abcdef') for x in range(20))
topo = Topology()
N = 789
s = topo.subscribe(topic=topic,
connect=SubscribeConnection.Buffered,
buffer_capacity=N)
self._buf_size = N
self.tester = Tester(topo)
self.tester.local_check = self._check_buffer
self.tester.test(self.test_ctxtype, self.test_config)
def test_subscribe_buffered_drop(self):
topic = ''.join(random.choice('0123456789abcdef') for x in range(20))
topo = Topology()
N = 563
s = topo.subscribe(
topic=topic,
connect=SubscribeConnection.Buffered,
buffer_capacity=N,
buffer_full_policy=streamsx.types.CongestionPolicy.DropLast)
self._buf_size = N
self.tester = Tester(topo)
self.tester.local_check = self._check_buffer
self.tester.test(self.test_ctxtype, self.test_config)
def setup_app(topo, cls, suppress=False):
topo.checkpoint_period = timedelta(seconds=1)
s = topo.source(cls(3, suppress))
tester = Tester(topo)
tester.contents(s, [1, 2, 3])
return tester
def setUp(self):
self.tf = _create_tf()
Tester.setup_standalone(self)
def setUp(self):
Tester.setup_streaming_analytics(self, force_remote_build=True)
class TestSuppressMetric(TestBaseExceptions):
@classmethod
def setUpClass(cls):
"""Extract Python operators in toolkit"""
stu._extract_tk('testtkpy')
def setUp(self):
self.tf = None
Tester.setup_distributed(self)
def test_suppress_metric(self):
schema = 'tuple<int32 a, int32 b, int32 c, int32 d>'
topo = Topology()
streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('testtkpy'))
# no metric
st = op.Source(
topo,
kind='com.ibm.streamsx.topology.pytest.pysource::SparseTuple',
schema=schema,
name='NOMETRIC_ST')
sf = op.Source(
topo,
kind='com.ibm.streamsx.topology.pysamples.sources::Range37',
schema=schema,
name='NOMETRIC_SF')
s = st.stream.union({sf.stream})
sm = op.Map('com.ibm.streamsx.topology.pysamples.positional::Noop',
s,
name='NOMETRIC_MF')
sm = op.Map('com.ibm.streamsx.topology.pysamples.positional::AddSeq',
sm.stream,
name='NOMETRIC_MC')
# With metric
schema = 'tuple<rstring a, int32 b>'
ms = op.Source(
topo,
kind=
'com.ibm.streamsx.topology.pytest.pyexceptions::SuppressNextSource',
schema=schema,
name='HASMETRIC_S_1')
mm = op.Map(
kind='com.ibm.streamsx.topology.pytest.pyexceptions::SuppressMap',
stream=ms.stream,
name='HASMETRIC_M_0')
mf = op.Map(
kind=
'com.ibm.streamsx.topology.pytest.pyexceptions::SuppressFilter',
stream=ms.stream,
name='HASMETRIC_F_0')
self.tester = Tester(topo)
self.tester.local_check = self.check_suppress_metric
# Add filters to avoid the test operators having
# names of NOMETIC/HASMETRIC
self.tester.tuple_count(sm.stream.filter(lambda _: True), 38)
self.tester.tuple_count(ms.stream.filter(lambda _: True), 2)
self.tester.tuple_count(mm.stream.filter(lambda _: True), 2)
self.tester.tuple_count(mf.stream.filter(lambda _: True), 2)
self.tester.test(self.test_ctxtype, self.test_config)
def check_suppress_metric(self):
job = self.tester.submission_result.job
hms = []
for op in job.get_operators():
seen_suppress_metric = False
for m in op.get_metrics():
if m.name == 'nExceptionsSuppressed':
seen_suppress_metric = True
if 'NOMETRIC_' in op.name:
self.assertFalse(seen_suppress_metric, msg=op.name)
elif 'HASMETRIC_' in op.name:
hms.append(op)
self.assertTrue(seen_suppress_metric, msg=op.name)
self.assertEqual(3, len(hms))
for _ in range(10):
ok = True
for op in hms:
exp = int(op.name.split('_')[2])
m = op.get_metrics(name='nExceptionsSuppressed')[0]
if m.value != exp:
ok = False
if ok:
break
time.sleep(2)
for op in hms:
exp = int(op.name.split('_')[2])
m = op.get_metrics(name='nExceptionsSuppressed')[0]
self.assertEqual(exp, m.value, msg=op.name)
class TestSources(unittest.TestCase):
"""
Test @spl.source decorated operators
"""
_multiprocess_can_split_ = True
@classmethod
def setUpClass(cls):
"""Extract Python operators in toolkit"""
stu._extract_tk('testtkpy')
def setUp(self):
Tester.setup_standalone(self)
def test_class_source(self):
count = 43
topo = Topology()
streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('testtkpy'))
streamsx.spl.toolkit.add_toolkit_dependency(topo, 'spl', '[1.0,2.0)')
bop = op.Source(topo, "com.ibm.streamsx.topology.pysamples.sources::Range", schema.StreamSchema('tuple<int64 c>').as_tuple(), params={'count':count})
r = bop.stream
self.tester = Tester(topo)
self.tester.tuple_count(r, count)
self.tester.contents(r, list(zip(range(count))))
self.tester.test(self.test_ctxtype, self.test_config)
def test_fn_source(self):
count = 37
topo = Topology()
streamsx.spl.toolkit.add_toolkit(topo, stu._tk_dir('testtkpy'))
bop = op.Source(topo, "com.ibm.streamsx.topology.pysamples.sources::Range37", schema.StreamSchema('tuple<int64 c>').as_tuple())
r = bop.stream
self.tester = Tester(topo)
self.tester.tuple_count(r, count)
self.tester.contents(r, list(zip(range(count))))
self.tester.test(self.test_ctxtype, self.test_config)
def setUp(self):
Tester.setup_standalone(self)
def test_aggregate(self):
iterations = 3000
reset_count = 5
topo = Topology()
# Generate integers from [0,3000)
s = topo.source(TimeCounter(iterations=iterations, period=0.01))
if self.is_cr():
s.set_consistent(ConsistentRegionConfig.periodic(5, drain_timeout=40, reset_timeout=40, max_consecutive_attempts=6))
# Filter the odd ones
s = s.filter(StatefulEvenFilter())
# Halve the even ones and add one. Now have integers [1,(iterations/2))
s = s.map(StatefulHalfPlusOne())
sc = s.last(10).trigger(3).aggregate(StatefulAverage())
st = s.last(17).trigger(datetime.timedelta(seconds=2)).aggregate(StatefulAverage())
# non-stateful aggregation functions
nsc = s.last(19).trigger(13).aggregate(lambda tuples : sum(tuples))
tester = Tester(topo)
if self.is_cr():
tester.resets(reset_count)
# Find the expected results.
# mimic the processing using Python builtins
iv = filter(StatefulEvenFilter(), range(iterations))
iv = list(map(StatefulHalfPlusOne(), iv))
# Expected stateful averages sc,st
sagg = StatefulAverage()
ers = [ sagg(iv[0:i+3][-10:]) for i in range(0, 3*int(len(iv)/3), 3) ]
tester.contents(sc, ers)
tester.tuple_check(st, TimedStatefulAverageChecker())
# Must eventually aggregate on the last 17 items in iv
# but only if cr otherwise the final marker stops
# the window before the final trigger
if self.is_cr():
tester.eventual_result(st, lambda av : True if av[1] == sagg(iv[-17:])[1] else None)
# Expected non-stateful averages nsc
ernsc = [ sum(iv[0:i+13][-19:]) for i in range(0, 13*int(len(iv)/13), 13) ]
tester.contents(nsc, ernsc)
tester.test(self.test_ctxtype, self.test_config)
def setUp(self):
Tester.setup_distributed(self)
self.sc = StreamsConnection()
self.sc.session.verify = False
self.test_config[ConfigParams.STREAMS_CONNECTION] = self.sc
class TestDistributedSuppressMetric(unittest.TestCase):
_multiprocess_can_split_ = True
def setUp(self):
Tester.setup_distributed(self)
def test_suppress_metric(self):
topo = Topology()
# No metric
s1 = topo.source(EESMSource(), name='NOMETRIC_SA')
s2 = topo.source([], name='NOMETRIC_SB')
s3 = topo.source(range(0), name='NOMETRIC_SB')
s = s1.union({s2,s3})
s = s.filter(EESMClass(), name='NOMETRIC_FA')
s = s.filter(lambda x : False, name='NOMETRIC_FA')
s = s.filter(ascii, name='NOMETRIC_FA')
s = s.map(EESMClass(), name='NOMETRIC_MA')
s = s.map(lambda x : False, name='NOMETRIC_MB')
s = s.map(ascii, name='NOMETRIC_MC')
s.for_each(EESMClass(), name='NOMETRIC_EA')
s.for_each(lambda x : False, name='NOMETRIC_EB')
s.for_each(ascii, name='NOMETRIC_EC')
# Has metric
m = topo.source(EESMSourceM(), name='HASMETRIC_S_7')
m = m.filter(EESMClassM(5), name='HASMETRIC_F_3')
m = m.map(EESMClassM(7), name='HASMETRIC_M_2')
m.for_each(EESMClassM(4), name='HASMETRIC_E_3')
self.tester = Tester(topo)
self.tester.local_check = self.check_suppress_metric
self.tester.contents(m, [1,2,4,8,11,13,16,17,19])
self.tester.test(self.test_ctxtype, self.test_config)
def check_suppress_metric(self):
job = self.tester.submission_result.job
hms = []
for op in job.get_operators():
seen_suppress_metric = False
for m in op.get_metrics():
if m.name == 'nExceptionsSuppressed':
seen_suppress_metric = True
if 'NOMETRIC_' in op.name:
self.assertFalse(seen_suppress_metric, msg=op.name)
elif 'HASMETRIC_' in op.name and len(op.name) == 13:
# Add length to avoid injected test operators.
hms.append(op)
self.assertTrue(seen_suppress_metric, msg=op.name)
self.assertEqual(4, len(hms))
for _ in range(10):
ok = True
for op in hms:
exp = int(op.name.split('_')[2])
m = op.get_metrics(name='nExceptionsSuppressed')[0]
if m.value != exp:
ok = False
if ok:
break
time.sleep(2)
for op in hms:
exp = int(op.name.split('_')[2])
m = op.get_metrics(name='nExceptionsSuppressed')[0]
self.assertEqual(exp, m.value, msg=op.name)
class TestTester(unittest.TestCase):
def setUp(self):
Tester.setup_distributed(self)
def test_at_least(self):
""" Test the at least tuple count.
"""
topo = Topology()
s = topo.source(rands)
tester = Tester(topo)
tester.tuple_count(s, 100, exact=False)
tester.test(self.test_ctxtype, self.test_config)
def test_no_tuples(self):
""" Test exact count with zero tuples.
"""
topo = Topology()
s = topo.source([])
tester = Tester(topo)
tester.tuple_count(s, 0)
tester.test(self.test_ctxtype, self.test_config)
def test_at_least_no_tuples(self):
""" Test at least count with zero tuples.
(kind of a pointless condition, always true).
"""
topo = Topology()
s = topo.source([])
tester = Tester(topo)
tester.tuple_count(s, 0, exact=False)
tester.test(self.test_ctxtype, self.test_config)
def test_checker(self):
""" Test the per-tuple checker.
"""
topo = Topology()
s = topo.source(rands)
s = s.filter(lambda r: r > 0.8)
s = s.map(lambda r: r + 7.0)
tester = Tester(topo)
tester.tuple_count(s, 200, exact=False)
tester.tuple_check(s, lambda r: r > 7.8)
tester.test(self.test_ctxtype, self.test_config)
def test_local_check(self):
""" Test the at least tuple count.
"""
topo = Topology()
s = topo.source(rands)
self.my_local_called = False
self.tester = Tester(topo)
self.tester.tuple_count(s, 100, exact=False)
self.tester.local_check = self.my_local
self.tester.test(self.test_ctxtype, self.test_config)
self.assertTrue(self.my_local_called)
def my_local(self):
self.assertTrue(hasattr(self.tester, 'submission_result'))
self.assertTrue(hasattr(self.tester, 'streams_connection'))
self.my_local_called = True
def test_bad_pe(self):
"""Test a failure in a PE is caught as a test failure"""
topo = Topology()
s = topo.source(rands)
# intentional addition of a string with an int
# to cause a PE failure
s = s.map(lambda t: t + 'a string')
tester = Tester(topo)
tester.tuple_count(s, 0, exact=False)
tp = tester.test(self.test_ctxtype,
self.test_config,
assert_on_fail=False)
self.assertFalse(tp)
def setUp(self):
self.td = None
Tester.setup_standalone(self)
class TestSources(unittest.TestCase):
"""
Test @spl.source decorated operators
"""
def setUp(self):
Tester.setup_standalone(self)
def test_class_source(self):
count = 43
topo = Topology()
streamsx.spl.toolkit.add_toolkit(topo, '../testtkpy')
bop = op.Source(topo, "com.ibm.streamsx.topology.pysamples.sources::Range", schema.StreamSchema('tuple<int64 c>').as_tuple(), params={'count':count})
r = bop.stream
self.tester = Tester(topo)
self.tester.tuple_count(r, count)
self.tester.contents(r, list(zip(range(count))))
self.tester.test(self.test_ctxtype, self.test_config)
def test_fn_source(self):
count = 37
topo = Topology()
streamsx.spl.toolkit.add_toolkit(topo, '../testtkpy')
bop = op.Source(topo, "com.ibm.streamsx.topology.pysamples.sources::Range37", schema.StreamSchema('tuple<int64 c>').as_tuple())
r = bop.stream
self.tester = Tester(topo)
self.tester.tuple_count(r, count)
self.tester.contents(r, list(zip(range(count))))
self.tester.test(self.test_ctxtype, self.test_config)
