def test_single_output(self):
topo = Topology()
s = topo.source(U.Sequence(iterations=4))
matches = R.Filter.matching(s, filter='seq<2ul')
tester = Tester(topo)
tester.tuple_count(matches, 2)
tester.test(self.test_ctxtype, self.test_config)
def test_sequence(self):
topo = Topology()
s = U.sequence(topo, iterations=122)
tester = Tester(topo)
tester.tuple_check(s, lambda x: 'seq' in x and 'ts' in x)
tester.tuple_count(s, 122)
tester.test(self.test_ctxtype, self.test_config)
def test_schemas_ok(self):
topo = Topology()
evstr.subscribe(topo, 'T1', CommonSchema.String)
evstr.subscribe(topo, 'T1', CommonSchema.Json)
evstr.subscribe(topo, 'T1', MsgSchema.StringMessage)
evstr.subscribe(topo, 'T1', MsgSchema.BinaryMessage)
evstr.subscribe(topo, 'T1', MsgSchema.StringMessageMeta)
evstr.subscribe(topo, 'T1', MsgSchema.BinaryMessageMeta)
def test_download_latest_with_target_dir(self):
topology = Topology()
target_dir = 'pypi.streamsx.nlp.tests-' + str(
uuid.uuid4()) + '/nlp-toolkit'
location = toolkits.download_toolkit('com.ibm.streamsx.nlp',
target_dir=target_dir)
print('\ntoolkit location: ' + location)
streamsx.spl.toolkit.add_toolkit(topology, location)
def test_creds(self):
creds_file = os.environ['EVENTSTREAMS_CREDENTIALS']
with open(creds_file) as data_file:
credentials = json.load(data_file)
topo = Topology()
stream = topo.source(['Hello', 'World']).as_json()
evstr.publish(stream, 'Topic', credentials=credentials)
evstr.publish(stream, 'Topic', credentials='eventstreams')
def test_download_samples(self):
topology = Topology()
location = toolkits.download_toolkit('samples',
repository_name='streamsx.nlp')
print('\nsamples location: ' + location)
files = os.listdir(location)
for name in files:
print(name)
def test_filter_none(self):
topo = Topology()
s = topo.source(U.Sequence(iterations=4))
matches = R.Filter.matching(s, filter=None)
tester = Tester(topo)
tester.tuple_count(matches, 4)
tester.test(self.test_ctxtype, self.test_config)
def test_source_argcount(self):
topo = Topology()
topo.source(a_0)
topo.source(A_0())
self.assertRaises(TypeError, topo.source, a_1)
self.assertRaises(TypeError, topo.source, A_1())
topo.source(ao_1)
topo.source(AO_1())
def test_map(self):
topo = Topology()
s = topo.source(s_none)
s.map(m_none)
sr = s.map(m_int)
self.assertEqual(CommonSchema.String, sr.oport.schema)
sr = s.map(m_str)
self.assertEqual(_normalize(SensorReading), sr.oport.schema)
s.map(m_any)
s.map(m_sensor)
s = topo.source(s_int)
s.map(m_none)
s.map(m_int)
self.assertRaises(TypeError, s.map, m_str)
s.map(m_any)
self.assertRaises(TypeError, s.map, m_sensor)
s = topo.source(s_str)
s.map(m_none)
self.assertRaises(TypeError, s.map, m_int)
s.map(m_str)
s.map(m_any)
self.assertRaises(TypeError, s.map, m_sensor)
s = topo.source(s_any)
s.map(m_none)
s.map(m_int)
s.map(m_str)
s.map(m_any)
s.map(m_sensor)
s = topo.source(s_sensor)
s.map(m_none)
self.assertRaises(TypeError, s.map, m_int)
self.assertRaises(TypeError, s.map, m_str)
s.map(m_any)
s.map(m_sensor)
s = topo.source(s_p)
s.map(m_none)
self.assertRaises(TypeError, s.map, m_int)
self.assertRaises(TypeError, s.map, m_str)
s.map(m_any)
self.assertRaises(TypeError, s.map, m_sensor)
sr = s.map(m_p)
self.assertEqual(CommonSchema.Python, sr.oport.schema)
self.assertRaises(TypeError, s.map, m_s)
# Ensure we maintain the hint as well as the schema
sr.map(m_p)
self.assertRaises(TypeError, sr.map, m_s)
sr.map(m_p2s).map(m_s)
s = topo.source(s_s)
s.map(m_p)
s.map(m_s)
def test_MQTTSource_schemas(self):
s = MQTTSource(server_uri='tcp://server:1833', topics='topic1', schema=CommonSchema.Json, data_attribute_name='ignored')
# topology.source() calls our populate()
Topology().source(s)
self.assertEqual(s._op.params['dataAttributeName'], 'jsonString')
s = MQTTSource(server_uri='tcp://server:1833', topics='topic1', schema=CommonSchema.String, data_attribute_name='ignored')
Topology().source(s)
self.assertEqual(s._op.params['dataAttributeName'], 'string')
s = MQTTSource(server_uri='tcp://server:1833', topics='topic1', schema=CommonSchema.Binary, data_attribute_name='ignored')
Topology().source(s)
self.assertEqual(s._op.params['dataAttributeName'], 'binary')
s = MQTTSource(server_uri='tcp://server:1833', topics='topic1', schema=MqttDataTuple)
Topology().source(s)
self.assertNotIn('dataAttributeName', s._op.params)
s = MQTTSource(server_uri='tcp://server:1833', topics='topic1', schema=[MqttDataTuple], data_attribute_name='data')
Topology().source(s)
self.assertEqual(s._op.params['dataAttributeName'], 'data')
s = MQTTSource(server_uri='tcp://server:1833', topics='topic1', schema='tuple<rstring data>')
Topology().source(s)
self.assertNotIn('dataAttributeName', s._op.params)
s = MQTTSource(server_uri='tcp://server:1833', topics='topic1', schema=StreamSchema('tuple<rstring data>'))
Topology().source(s)
self.assertNotIn('dataAttributeName', s._op.params)
def test_throttle(self):
topo = Topology()
s = topo.source(U.Sequence(iterations=40))
s = s.map(U.Throttle(rate=2.0, precise=True))
tester = Tester(topo)
tester.tuple_count(s, 40)
tester.tuple_check(s, ThrottleCheck())
tester.test(self.test_ctxtype, self.test_config)
def test_delay(self):
topo = Topology()
s = topo.source(U.Sequence(iterations=223))
s = s.map(U.Delay(delay=0.4))
tester = Tester(topo)
tester.tuple_count(s, 223)
tester.tuple_check(s, lambda t: (time.time() - t['ts'].time()) > 0.35)
tester.test(self.test_ctxtype, self.test_config)
def main():
topo = Topology("patient_data")
patientData = topo.source(Observations)
heartRate = patientData.filter(
lambda tuple: (tuple['reading']['readingType']['code'] == '8867-4'))
heartRate.sink(print)
streamsx.topology.context.submit("STANDALONE", topo)
def main():
t = Topology("IFFT_Sample")
readings = t.source(signal_generator.Readings(50)).transform(
TumblingWindow(10))
fftStream = readings.transform(fftpack.fft)
ifftStream = fftStream.transform(fftpack.ifft)
ifftStream.sink(print)
streamsx.topology.context.submit("STANDALONE", t.graph)
def test_download_with_url_and_target_dir(self):
topology = Topology()
target_dir = 'pypi.streamsx.eventstreams.tests-' + str(
uuid.uuid4()) + '/messagehub-toolkit'
ver = '1.9.0'
url = 'https://github.com/IBMStreams/streamsx.messagehub/releases/download/v' + ver + '/com.ibm.streamsx.messagehub-' + ver + '.tgz'
location = evstr.download_toolkit(url=url, target_dir=target_dir)
print('toolkit location: ' + location)
streamsx.spl.toolkit.add_toolkit(topology, location)
def test_xml_creds(self):
xml_file = os.environ['HDFS_SITE_XML']
topo = Topology()
hdfs.scan(topo, credentials=xml_file, directory='a_dir')
hdfs.scan(topo,
credentials=xml_file,
directory='a_dir',
pattern='*.txt',
init_delay=datetime.timedelta(seconds=5))
def main():
ref_signal = signal.hann(10)
t = Topology("FFTConvolve_Sample")
readings = t.source(signal_generator.Readings(10000)).transform(TumblingWindow(1000))
convolveStream = readings.transform(signal_functions.FFTConvolve(ref_signal))
convolveStream.sink(print)
streamsx.topology.context.submit("STANDALONE", t.graph)
def test_delay(self):
topo = Topology()
s = U.sequence(topo, iterations=223)
s = U.delay(s, delay=0.4)
tester = Tester(topo)
tester.tuple_count(s, 223)
tester.tuple_check(s, lambda t : (time.time() - t['ts'].time()) > 0.35)
tester.test(self.test_ctxtype, self.test_config)
def test_basic_xml_injection(self):
name = 'test_basic_xml_injection'
topo = Topology(name)
res = endpoint.inject(topo,
name='jsoninject',
schema=CommonSchema.XML,
monitor='xml',
context='sample')
self._build_only(name, topo)
def test_get_job(self):
topo = Topology("job_in_result_test")
topo.source(["foo"])
tester = Tester(topo)
self.tester = tester
tester.local_check = self._correct_job_ids
tester.test(self.test_ctxtype, self.test_config)
def test_basic_expose(self):
name = 'test_basic_expose'
topo = Topology(name)
s = topo.source([{'a': 'Hello'}, {'a': 'World'}, {'a': '!'}]).as_json()
endpoint.expose(s.last(10).trigger(1),
name='tupleview',
context='sample',
monitor=None)
self._build_only(name, topo)
def test_basic_json_injection(self):
name = 'test_basic_json_injection'
topo = Topology(name)
res = endpoint.inject(topo,
name='jsoninject',
monitor=None,
context='sample')
res.print()
self._build_only(name, topo)
def test_string(self):
topo = Topology()
s = topo.source(V_Str())
tester = Tester(topo)
tester.tuple_count(s, 2)
tester.tuple_check(s, lambda v: isinstance(v, str))
tester.contents(s, ['Hello', 'World'])
tester.test(self.test_ctxtype, self.test_config)
def test_flat_map(self):
topo = Topology()
s = topo.source(s_none)
s.flat_map(fm_none)
s.flat_map(fm_int)
sr = s.flat_map(fm_str)
self.assertEqual(_normalize(SensorReading), sr.oport.schema)
sr.flat_map(fm_sensor)
s.flat_map(fm_any)
s.flat_map(fm_sensor)
s = topo.source(s_int)
s.flat_map(fm_none)
s.flat_map(fm_int)
self.assertRaises(TypeError, s.flat_map, fm_str)
s.flat_map(fm_any)
self.assertRaises(TypeError, s.flat_map, fm_sensor)
s = topo.source(s_str)
s.flat_map(fm_none)
self.assertRaises(TypeError, s.flat_map, fm_int)
sr = s.flat_map(fm_str)
self.assertEqual(_normalize(SensorReading), sr.oport.schema)
sr.flat_map(fm_sensor)
s.flat_map(fm_any)
self.assertRaises(TypeError, s.flat_map, fm_sensor)
s = topo.source(s_any)
s.flat_map(fm_none)
s.flat_map(fm_int)
sr = s.flat_map(fm_str)
self.assertEqual(_normalize(SensorReading), sr.oport.schema)
sr.flat_map(fm_sensor)
s.flat_map(fm_any)
s.flat_map(fm_sensor)
s = topo.source(s_sensor)
s.flat_map(fm_none)
self.assertRaises(TypeError, s.flat_map, fm_int)
self.assertRaises(TypeError, s.flat_map, fm_str)
s.flat_map(fm_any)
s.flat_map(fm_sensor)
s = topo.source(s_p)
s.flat_map(fm_none)
self.assertRaises(TypeError, s.flat_map, fm_int)
self.assertRaises(TypeError, s.flat_map, fm_str)
s.flat_map(fm_any)
self.assertRaises(TypeError, s.flat_map, fm_sensor)
s.flat_map(fm_p)
self.assertRaises(TypeError, s.flat_map, fm_s)
s = topo.source(s_s)
s.flat_map(fm_p)
s.flat_map(fm_s)
def test_sequence_period(self):
topo = Topology()
s = U.sequence(topo, iterations=67, period=0.1)
E = U.SEQUENCE_SCHEMA.extend(StreamSchema('tuple<float64 d>'))
s = s.map(_Delta(), schema=E)
tester = Tester(topo)
tester.tuple_check(s, lambda x: x['d'] > 0.08)
tester.tuple_count(s, 67-1)
tester.test(self.test_ctxtype, self.test_config)
def test_deduplicate(self):
topo = Topology()
s = topo.source([1, 2, 1, 4, 5, 2])
s = s.map(lambda v: {'a': v}, schema='tuple<int32 a>')
s = s.map(U.Deduplicate(count=3))
s = s.map(lambda v: v['a'])
tester = Tester(topo)
tester.contents(s, [1, 2, 4, 5, 2])
tester.test(self.test_ctxtype, self.test_config)
def test_bad_lib_param(self):
creds_file = os.environ['DB2_CREDENTIALS']
with open(creds_file) as data_file:
credentials = json.load(data_file)
topo = Topology()
s = topo.source(['DROP TABLE STR_SAMPLE']).as_string()
# expect ValueError because driver class is not default and jdbc_driver_lib is missing
self.assertRaises(ValueError, db.run_statement, s, credentials, jdbc_driver_class='com.any.DBDriver')
# expect ValueError because jdbc_driver_lib is not a valid file
self.assertRaises(ValueError, db.run_statement, s, credentials, jdbc_driver_class='com.any.DBDriver', jdbc_driver_lib='_any_invalid_file_')
def _get_spl_app(cmd_args):
ns, name = cmd_args.main_composite.rsplit('::', 1)
ns += '._spl'
topo = Topology(name=name, namespace=ns)
if cmd_args.toolkits is not None:
for tk_path in cmd_args.toolkits:
tk.add_toolkit(topo, tk_path)
op.Invoke(topo, cmd_args.main_composite)
return (topo, {})
def test_basic_string_injection(self):
name = 'test_basic_string_injection'
topo = Topology(name)
res = endpoint.inject(topo,
name='jsoninject',
schema=CommonSchema.String,
monitor='sample',
context='sample')
res.print()
self._build_only(name, topo)
def test_cloud_creds(self):
creds_file = os.environ['ANALYTICS_ENGINE']
with open(creds_file) as data_file:
credentials = json.load(data_file)
topo = Topology()
hdfs.scan(topo, credentials, 'a_dir')
hdfs.scan(topo,
credentials=credentials,
directory='a_dir',
init_delay=10.0)
