def test_hbase_scan(self):
topo = Topology('test_hbase_scan')
print('\n--------------------------- \nTest : ' + topo.name +
'\n--------------------------- ')
if self.hbase_toolkit_location is not None:
tk.add_toolkit(topo, self.hbase_toolkit_location)
tester = Tester(topo)
# hbase.scan creates site.xml file with the use of environment variable HADOOP_HOST_PORT / HBASE_SITE_XML
scanned_rows = hbase.scan(topo,
table_name=_get_table_name(),
max_versions=1,
init_delay=2)
scanned_rows.print(name='printScan')
tester.tuple_count(scanned_rows, 2, exact=False)
cfg = {}
job_config = streamsx.topology.context.JobConfig(tracing='info')
job_config.add(cfg)
cfg[streamsx.topology.context.ConfigParams.SSL_VERIFY] = False
# Run the test
tester.test(self.test_ctxtype, cfg, always_collect_logs=True)
def test_hbase_scan_connection_param_dict(self):
topo = Topology('test_hbase_scan_connection_param_dict')
print('\n--------------------------- \nTest : ' + topo.name +
'\n--------------------------- ')
if self.hbase_toolkit_location is not None:
tk.add_toolkit(topo, self.hbase_toolkit_location)
tester = Tester(topo)
# set dict as connection parameter with values from HADOOP_HOST_PORT
hp = os.environ['HADOOP_HOST_PORT'].split(":", 1)
ext_connection = {}
ext_connection['host'] = hp[0]
ext_connection['port'] = hp[1]
scanned_rows = hbase.scan(topo,
table_name=_get_table_name(),
max_versions=1,
init_delay=2,
connection=ext_connection)
scanned_rows.print(name='printScan')
tester.tuple_count(scanned_rows, 2, exact=False)
cfg = {}
job_config = streamsx.topology.context.JobConfig(tracing='info')
job_config.add(cfg)
cfg[streamsx.topology.context.ConfigParams.SSL_VERIFY] = False
# Run the test
tester.test(self.test_ctxtype, cfg, always_collect_logs=True)
def test_hbase_scan_connection_param_file(self):
topo = Topology('test_hbase_scan_connection_param_file')
print('\n--------------------------- \nTest : ' + topo.name +
'\n--------------------------- ')
if self.hbase_toolkit_location is not None:
tk.add_toolkit(topo, self.hbase_toolkit_location)
tester = Tester(topo)
# set HBASE_SITE_XML file as connection parameter
scanned_rows = hbase.scan(topo,
table_name=_get_table_name(),
max_versions=1,
init_delay=2,
connection=os.environ['HBASE_SITE_XML'])
scanned_rows.print(name='printScan')
tester.tuple_count(scanned_rows, 2, exact=False)
cfg = {}
job_config = streamsx.topology.context.JobConfig(tracing='info')
job_config.add(cfg)
cfg[streamsx.topology.context.ConfigParams.SSL_VERIFY] = False
# Run the test
tester.test(self.test_ctxtype, cfg, always_collect_logs=True)
def test_buildonly_produce(self):
txtmsg_schema = StreamSchema('tuple<rstring msg>')
errmsg_schema = StreamSchema('tuple<rstring errorMessage>')
java_class_lib_paths = []
java_class_lib_paths.append("./streamsx/jms/tests/libs/activemq/lib")
java_class_lib_paths.append(
"./streamsx/jms/tests/libs/activemq/lib/optional")
path_to_connection_doc = "./streamsx/jms/tests/connectionDocument.xml" # tests are supposed to be run from the package directory
topo = Topology('buildonly_produce')
toolkit.add_toolkit(topo, "../../streamsx.jms/com.ibm.streamsx.jms")
txtmsg_source = Source(topo,
'spl.utility::Beacon',
txtmsg_schema,
params={'period': 0.3},
name="DataGenerator")
txtmsg_source.msg = txtmsg_source.output(
'"Message #" + (rstring)IterationCount()')
txtmsg_stream = txtmsg_source.stream
txtmsg_stream.print()
errmsg_stream = jms.produce(stream=txtmsg_stream,
schema=errmsg_schema,
java_class_libs=java_class_lib_paths,
connection="localActiveMQ",
access="accessToTextMessages",
connection_document=path_to_connection_doc,
name="JMS_Producer")
errmsg_stream.print()
self._build_only('test_buildonly_produce', topo)
def job_status_service():
topo = Topology('JobStatusService')
add_toolkit(topo, monitoring_toolkit)
r = op.main_composite(
kind='com.ibm.streamsx.monitoring.jobs.services::JobStatusService',
toolkits=[monitoring_toolkit])
_launch(r[0])
def wikiFeed(inetToolkit, buildType, port):
# Sumbit request Build Server and Submit.
schemaTicker = 'tuple<rstring dataId, rstring bot, rstring domain, int32 lenOld,int32 lenNew, rstring SSEdata, rstring SSEevent>'
#
# Define the application
#
topo = Topology("WikiFeed")
tk.add_toolkit(topo, inetToolkit)
source = topo.source(wikiStream)
# Only one type of event, verity that I got it right.
event = source.filter(lambda t: t["SSEevent"] == "message",
name="eventFilter")
event.print(name="eth")
eventTuple = event.map(lambda t: t, schema=schemaTicker)
eventWin = eventTuple.last(100).trigger(1)
rawRequest = op.Sink("com.ibm.streamsx.inet.rest::HTTPTupleView",
stream=eventWin,
params={
'port': 8081,
'context': 'gdaxEth',
'contextResourceBase': '/base'
},
name="TupleView")
#
# Compile & Submit the Topology to Streams instance
#
streams_conf = common.build_streams_config("StreamingTurbine",
credential.serviceCredentials)
context.submit(context.ContextTypes.STREAMING_ANALYTICS_SERVICE,
topo,
config=streams_conf)
def test_hbase_get(self):
topo = Topology('test_hbase_get')
print('\n--------------------------- \nTest : ' + topo.name +
'\n--------------------------- ')
if self.hbase_toolkit_location is not None:
tk.add_toolkit(topo, self.hbase_toolkit_location)
s = _create_stream_for_get(topo)
tester = Tester(topo)
# hbase.get creates site.xml file with the use of environment variable HADOOP_HOST_PORT / HBASE_SITE_XML
get_rows = hbase.get(s,
table_name=_get_table_name(),
row_attr_name='who')
get_rows.print(name='printGet')
tester.tuple_count(get_rows, 2, exact=False)
# tester.run_for(60)
cfg = {}
job_config = streamsx.topology.context.JobConfig(tracing='info')
job_config.add(cfg)
cfg[streamsx.topology.context.ConfigParams.SSL_VERIFY] = False
# Run the test
tester.test(self.test_ctxtype, cfg, always_collect_logs=True)
def failed_pe_service():
topo = Topology('FailedPEService')
add_toolkit(topo, monitoring_toolkit)
r = op.main_composite(
kind='com.ibm.streamsx.monitoring.jobs.services::FailedPEService',
toolkits=[monitoring_toolkit])
_launch(r[0])
def test_close_on_tuples(self):
ae_service_creds_file = os.environ['ANALYTICS_ENGINE']
with open(ae_service_creds_file) as data_file:
credentials = json.load(data_file)
topo = Topology('test_hdfs_uri')
if self.hdfs_toolkit_location is not None:
tk.add_toolkit(topo, self.hdfs_toolkit_location)
s = topo.source([
'Hello World!', 'Hello', 'World', 'Hello World!', 'Hello', 'World'
]).as_string()
result = hdfs.write(s,
credentials=credentials,
file='pytest/write_test%FILENUM.txt',
tuples_per_file=3)
result.print()
tester = Tester(topo)
tester.tuple_count(result, 2, exact=True)
#tester.run_for(60)
cfg = {}
job_config = streamsx.topology.context.JobConfig(tracing='info')
job_config.add(cfg)
cfg[streamsx.topology.context.ConfigParams.SSL_VERIFY] = False
# Run the test
tester.test(self.test_ctxtype, cfg, always_collect_logs=True)
def metrics_ingest_service():
topo = Topology('MetricsIngestService')
add_toolkit(topo, monitoring_toolkit)
r = op.main_composite(
kind=
'com.ibm.streamsx.monitoring.metrics.services::MetricsIngestService',
toolkits=[monitoring_toolkit])
_launch(r[0])
def reflectPost(self, expected_requests, local_check_function):
"""Reflect base : reflecting back post.
Ran into a problem with reflecting back the header, if you include a Content-Length
in the header a corrected Content-Length is *not* sent. When the data arrives
back at the client the data is too short - thus the buildRsp()
"""
topo = Topology("Reflect")
self.tester = Tester(topo)
tk.add_toolkit(topo, inetToolkit)
pending_source = PendingStream(topo)
rsp = pending_source.stream.map(lambda t: t)
ss = topo.source([], name="onRamp")
rsp = ss.union({rsp})
# FormatResponse :
rspFormatted = rsp.map(lambda x: json.dumps(x)).as_string()
rawRequest = op.Map(
"com.ibm.streamsx.inet.rest::HTTPRequestProcess",
stream=rspFormatted,
schema=
'tuple<int64 key, rstring request, rstring contentType, map<rstring, rstring> header, rstring response, rstring method,rstring pathInfo, int32 status, rstring statusMessage>',
params={
'port': PORT,
'webTimeout': 5.0,
'responseJsonAttributeName': 'string',
'context': 'Reflect',
'contextResourceBase': 'opt/Reflect'
},
name="RequestProcess")
rawRequest.stream.sink(webEntryLog) ## log what we have received.
# wait for
lastAct = rawRequest.stream.filter(sleepMacbeth)
# do the work
getReflect = rawRequest.stream.transform(buildResponse("RSP:"),
name="buildResponse")
getReflect.sink(webExitLog) ## log what we are sending back
# do tests on what we have processed.
self.tester.tuple_count(getReflect, expected_requests)
# loopback to sending
pending_source.complete(getReflect) # loopback
## All done building the graph......
# setup the code that will invoke this test.
self.tester.local_check = local_check_function
# enable tracing info.
job_config = streamsx.topology.context.JobConfig(job_name='Reflect',
tracing="info")
job_config.add(self.test_config)
# submit the application for test
self.tester.test(self.test_ctxtype, self.test_config)
def smokePending(inetToolkit, buildType, port):
# Specify the Schema going in and out of the Streams operator.
schemaRequest = 'tuple<int64 key, rstring request, rstring method, rstring pathInfo >'
schemaResponse = 'tuple<int64 key, rstring request, rstring method, rstring pathInfo, rstring response >'
topo = Topology("SmokePending")
# Add extrenal spl functions, this is what is being testing my HTTPRequestProcess()
tk.add_toolkit(topo, inetToolkit)
# 'pending_source' is the loopback point.
pending_source = PendingStream(topo)
# Convert to Streams' tuple
rsp = pending_source.stream.map(lambda t: t, schema=schemaResponse)
rspFormatted = rsp.map(lambda t: json.dumps(t)).as_string()
rspFormatted.sink(webLog("Output response")) ## log what we have received.
rawRequest = op.Map("com.ibm.streamsx.inet.rest::HTTPRequestProcess",
stream=rsp,
schema = schemaRequest,
params={'port': 8080,
'webTimeout': 5.0,
#'responseJsonAttributeName': 'string',
'context':'myStreams',
'contextResourceBase': '/base'},
name="TupleRequest")
# write out data
rawRequest.stream.sink(webLog('Input request')) ## log what we have received.
# determine what to work on
# Filter does not change the type
upperDo = rawRequest.stream.filter(lambda t: t["pathInfo"] == "/upper", name="upperFilter")
lowerDo = rawRequest.stream.filter(lambda t: t["pathInfo"] == "/lower", name="lowerFilter")
# do some processing
upperDone = upperDo.map(upperString, schema = schemaResponse, name="upperProcess")
lowerDone = lowerDo.map(lowerString, schema = schemaResponse, name="lowerProcess")
##
processingDone = upperDone.union({lowerDone})
hack = processingDone.map(lambda t: t, schema=schemaResponse)
pending_source.complete(hack) # close the loop
# hack make it so I do not get this error.
"""
pending_source.complete(processingDone) # close the loop
SEVERE: Streaming Analytics service (StreamingTurbine): The submitted archive tk5385353137138356122.zip failed to scripts with status failed.
Exception in thread "main" java.lang.IllegalStateException: Error submitting archive for compilation:
"tk5385353137138356122/SmokePending/SmokePending.spl:5:1: CDISP0011E ERROR: A syntax error exists at the '}' token in 'graph'."
"""
#
# Compile & Submit the Topology to Streams instance
#
streams_conf = common.build_streams_config("StreamingTurbine", credential.serviceCredentials)
context.submit(context.ContextTypes.STREAMING_ANALYTICS_SERVICE, topo, config=streams_conf)
def sample_app():
topo = Topology('SystemMonitorSample')
add_toolkit(topo, monitoring_toolkit)
add_toolkit(topo, sample_toolkit)
r = op.main_composite(
kind=
'com.ibm.streamsx.monitoring.system.sample.SystemMonitorSource::Monitor',
toolkits=[monitoring_toolkit, sample_toolkit])
_launch(r[0])
def sample_app():
topo = Topology('MetricsSample')
add_toolkit(topo, monitoring_toolkit)
add_toolkit(topo, sample_toolkit)
r = op.main_composite(
kind=
'com.ibm.streamsx.monitoring.metrics.sample.MetricsSource::SampleJob',
toolkits=[monitoring_toolkit, sample_toolkit])
_launch(r[0])
def sample_app():
topo = Topology('CrashSample')
add_toolkit(topo, monitoring_toolkit)
add_toolkit(topo, sample_toolkit)
r = op.main_composite(
kind=
'com.ibm.streamsx.monitoring.jobs.sample.JobStatusMonitor::SampleJob',
toolkits=[monitoring_toolkit, sample_toolkit])
_launch(r[0])
def reflectPost(self, expected_requests, local_check_function ):
"""Reflect base : Reflect back the data that is send as well as sending amount
specified in the repeat paramter.
Ran into a problem with reflecting back the header, if you include a Content-Length
in the header a corrected Content-Length is *not* sent. When the data arrives
back at the client the data is too short - thus the buildRsp()
"""
topo = Topology("Reflect")
self.tester = Tester(topo)
tk.add_toolkit(topo, inetToolkit)
pending_source = PendingStream(topo)
rsp = pending_source.stream.map(lambda t : t)
ss = topo.source([], name="onRamp")
rsp = ss.union({rsp})
# FormatResponse :
rspFormatted = rsp.map(lambda x : json.dumps(x) ).as_string();
rawRequest = op.Map("com.ibm.streamsx.inet.rest::HTTPRequestProcess",
stream=rspFormatted,
schema='tuple<int64 key, rstring request, rstring contentType, map<rstring, rstring> header, rstring response, rstring method,rstring pathInfo, int32 status, rstring statusMessage>',
params={'port': PORT,
'webTimeout':5.0,
'responseJsonAttributeName':'string',
'context':'Reflect',
'contextResourceBase': 'opt/Reflect'},
name = "RequestProcess")
rawRequest.stream.sink(webEntryLog) ## log what we have received.
# wait for
lastAct = rawRequest.stream.filter(sleepMacbeth)
# do the work
getReflect = rawRequest.stream.transform(buildResponse("RSP:"),
name="buildResponse")
getReflect.sink(webExitLog) ## log what we are sending back
# do tests on what we have processed.
self.tester.tuple_count(getReflect, expected_requests)
# loopback to sending
pending_source.complete(getReflect) # loopback
## All done building the graph......
# setup the code that will invoke this test.
self.tester.local_check = local_check_function
# enable tracing info.
job_config = streamsx.topology.context.JobConfig(job_name='Reflect', tracing="info")
job_config.add(self.test_config)
# submit the application for test
self.tester.test(self.test_ctxtype, self.test_config)
"""
def _get_spl_app(cmd_args):
ns, name = cmd_args.main_composite.rsplit('::', 1)
ns += '._spl'
topo = Topology(name=name, namespace=ns)
if cmd_args.toolkits is not None:
for tk_path in cmd_args.toolkits:
tk.add_toolkit(topo, tk_path)
op.Invoke(topo, cmd_args.main_composite)
return (topo, {})
def test_basic(self):
topo = Topology("Basic")
self.tester = Tester(topo)
tk.add_toolkit(topo, inetToolkit)
# Loop back not natural in a directed graph, need
# to have place holder while the graph gets built,
# At the end of the graph, connect back to the
# begining.
pending_source = PendingStream(topo)
# Directed graph has start this loop loop does not.
# Need to give topology an "onRamp" so it can build the graph.
rsp = pending_source.stream.map(lambda t : t)
ss = topo.source([], name="onRamp")
rsp = ss.union({rsp})
# FormatResponse :
rspFormatted = rsp.map(lambda x : json.dumps(x) ).as_string();
rawRequest = op.Map("com.ibm.streamsx.inet.rest::HTTPRequestProcess",
stream=rspFormatted,
schema='tuple<int64 key, rstring request, rstring method, rstring pathInfo >',
params={'port': PORT,
'webTimeout':5.0,
'responseJsonAttributeName':'string',
'context':'base',
'contextResourceBase':'opt/base' },
name = "RequestProcess")
rawRequest.stream.sink(webEntryLog) ## log what we have received.
# determine what to work on
onlyTuple = rawRequest.stream.filter(lambda t : t["pathInfo"]=="/Tuple",
name="inputFilter")
# do the work
upperDone = onlyTuple.transform(upperString,
name="upperCase")
#
self.tester.tuple_count(upperDone, 1)
# loopback to sending
pending_source.complete(upperDone) # loopback
## All done building the graph......
self.test_config['topology.keepArtifacts']=True
# setup the code that will invoke this test.
self.tester.local_check = self.basic_request
# submit the application for test
self.tester.test(self.test_ctxtype, self.test_config)
def _add_local_toolkits(toolkit_paths, dependencies, topo, verify_arg):
""" A helper function that given the local toolkit paths, and the dependencies (ie toolkits that the apps depends on),
finds the latest dependency w/ correct version (could be local or on the buildserver) and checks whether it exists locally, and if so adds it.
If the latest dependency w/ correct version is located on the buildserver, don't need to do anything
Arguments:
toolkit_paths {List} -- A list of local toolkit directories, each could be a toolkit directory (has info.xml directly inside),
or a directory consisting of toolkits (no info.xml directly inside)
dependencies {Dictionary} -- A dictionary consisting of elements where the key is the name of the dependency, and the value is its version.
topo {topology Object} -- [description]
verify_arg {type} -- Disable SSL verification.
"""
build_server = BuildService.of_endpoint(verify=verify_arg)
local_toolkits = _get_all_local_toolkits(toolkit_paths)
remote_toolkits = build_server.get_toolkits()
all_toolkits = local_toolkits + remote_toolkits
# Iterate through all dependencies (toolkits) needed to build sab file, and check if the dependency already exist on the build server
for dependency_name, dependency_version in dependencies.items():
# All the toolkits that match dependency_name regardless of version
matching_toolkits = [toolkit for toolkit in all_toolkits if toolkit.name == dependency_name]
if not matching_toolkits:
# No toolkits exists on the remote buildserver or locally that can satsify this dependency, continue and let SPL compiler throw error
continue
# Toolkit with the highest verion that still satisfies the dependency_version
latest_compatible_toolkit = None
# Go through all toolkits, find the highest/latest version, given by latest_compatible_toolkit
for tk in matching_toolkits:
# Check if current toolkit tk satisfies the version requirement
if _check_correct_version(tk, dependency_version):
# if we have a toolkit that already satisfies the version requirement, and tk also satisfies its
# compare tk against the highest version we have seen thus far
if latest_compatible_toolkit:
latest_version_so_far = pkg_resources.parse_version(latest_compatible_toolkit.version)
curr_tk_version = pkg_resources.parse_version(tk.version)
# if it is a later version, update our toolkit
if curr_tk_version > latest_version_so_far:
latest_compatible_toolkit = tk
else:
latest_compatible_toolkit = tk
# Check if latest_compatible_toolkit is local, bc we then need to add it
if isinstance(latest_compatible_toolkit, _LocalToolkit):
# Latest toolkit is local, upload it
add_toolkit(topo, latest_compatible_toolkit.path)
def reflect(self, expected_requests, local_check_function ):
"""Reflect base : request back to request, check results in Python VM.
Basis for other tests where the processing of the results are done in PYTHON.
- expected_requests : number of requests that the local_check_function will make
"""
topo = Topology("Reflect")
self.tester = Tester(topo)
tk.add_toolkit(topo, inetToolkit)
pending_source = PendingStream(topo)
rsp = pending_source.stream.map(lambda t : t)
ss = topo.source([], name="onRamp")
rsp = ss.union({rsp})
# FormatResponse :
rspFormatted = rsp.map(lambda x : json.dumps(x) ).as_string();
rawRequest = op.Map("com.ibm.streamsx.inet.rest::HTTPRequestProcess",
stream=rspFormatted,
schema='tuple<int64 key, rstring request, rstring method, rstring pathInfo >',
params={'port': PORT,
'webTimeout':5.0,
'responseJsonAttributeName':'string',
'context':'Reflect',
'contextResourceBase':'opt/Reflect' },
name = "RequestProcess")
rawRequest.stream.sink(webEntryLog) ## log what we have received.
# wait for
lastAct = rawRequest.stream.filter(sleepMacbeth)
# do the work
getReflect = lastAct.transform(reflectResult("RAW:"),
name="reflectResult")
# do tests on what we have processed.
self.tester.tuple_count(getReflect, expected_requests)
# loopback to sending
pending_source.complete(getReflect) # loopback
## All done building the graph......
# setup the code that will invoke this test.
self.tester.local_check = local_check_function
# enable tracing info.
job_config = streamsx.topology.context.JobConfig(job_name='Reflect', tracing="info")
job_config.add(self.test_config)
# submit the application for test
self.tester.test(self.test_ctxtype, self.test_config)
def test_all_hdsf_operators(self):
hdfs_cfg_file = os.environ['HDFS_SITE_XML']
# credentials is the path to the HDSF *configuration file 'hdfs-site.xml'
topo = Topology('test_all_hdsf_operators')
if self.hdfs_toolkit_location is not None:
tk.add_toolkit(topo, self.hdfs_toolkit_location)
# creates an input stream
fileSinkInputStream = topo.source(
['This line will be written into a HDFS file.']).as_string()
# writes a line into a HDFS file (HDFS2FileSink)
fileSinkResults = hdfs.write(fileSinkInputStream,
credentials=hdfs_cfg_file,
file='pytest1/sample4%FILENUM.txt')
fileSinkResults.print(name='printFileSinkResults')
# scans an HDFS directory and return file names (HDFS2DirectoryScan)
scannedFileNames = hdfs.scan(topo,
credentials=hdfs_cfg_file,
directory='pytest1',
pattern='sample.*txt',
init_delay=10)
scannedFileNames.print(name='printScannedFileNames')
# reads lines from a HDFS file (HDFS2FileSource)
readLines = hdfs.read(scannedFileNames, credentials=hdfs_cfg_file)
readLines.print(name='printReadLines')
# copies files from HDFS into local disk "/tmp/" (HDFS2FileCopy)
copyFileResults = hdfs.copy(scannedFileNames,
credentials=hdfs_cfg_file,
direction='copyToLocalFile',
hdfsFile=None,
hdfsFileAttrName='fileName',
localFile='/tmp/')
copyFileResults.print(name='printCopyFileResults')
tester = Tester(topo)
tester.tuple_count(readLines, 1, exact=False)
# tester.run_for(80)
cfg = {}
job_config = streamsx.topology.context.JobConfig(tracing='info')
job_config.add(cfg)
cfg[streamsx.topology.context.ConfigParams.SSL_VERIFY] = False
# Run the test
tester.test(self.test_ctxtype, cfg, always_collect_logs=True)
def gdaxFeed(inetToolkit, buildType, port):
# Sumbit request Build Server and Submit.
schemaTicker = 'tuple<rstring ttype, float32 price, float32 low_24h, float32 best_ask, rstring side, float32 best_bid, float32 open_24h, rstring product_id, int32 sequence, int32 trade_id, rstring time, float32 last_size, float32 volume_24h, float32 volume_30d, float32 high_24h>'
#
# Define the application
#
topo = Topology("GdaxFeed")
tk.add_toolkit(topo, inetToolkit)
source = topo.source(gdaxData)
# Split out the securities : ETH-USD, LTC-USD, BTC-USD
eth = source.filter(lambda t: t["product_id"] == "ETH-USD",
name="ethFilter")
ltc = source.filter(lambda t: t["product_id"] == "LTC-USD",
name="ltcFilter")
btc = source.filter(lambda t: t["product_id"] == "LTC-USD",
name="btcFilter")
eth.print(name="eth")
ltc.print(name="ltc")
btc.print(name="btc")
ethTuple = eth.map(lambda t: t, schema=schemaTicker)
#ethWin = ethTuple.last(datetime.timedelta(minutes=2))
ethWin = ethTuple.last(100).trigger(1)
rawRequest = op.Sink("com.ibm.streamsx.inet.rest::HTTPTupleView",
stream=ethWin,
params={
'port': 8080,
'context': 'gdaxEth',
'contextResourceBase': '/base'
},
name="TupleView")
#
# Compile & Submit the Topology to Streams instance
#
streams_conf = common.build_streams_config("StreamingTurbine",
credential.serviceCredentials)
context.submit(context.ContextTypes.STREAMING_ANALYTICS_SERVICE,
topo,
config=streams_conf)
def test():
print("enter")
tmp = gdaxData()
while True:
print(tmp.__next__())
def test_basic(self):
print('\n---------' + str(self))
name = 'test_basic'
topo = Topology(name)
toolkit.add_toolkit(topo, self.sttgateway_toolkit_home)
if (("TestDistributed" in str(self))
or ("TestStreamingAnalytics" in str(self))):
tester = Tester(topo)
tester.tuple_count(res, 1, exact=True)
tester.test(self.test_ctxtype,
self.test_config,
always_collect_logs=True)
else:
# build only
self._build_only(name, topo)
def _get_spl_app(cmd_args):
if '::' in cmd_args.main_composite:
ns, name = cmd_args.main_composite.rsplit('::', 1)
ns += '._spl'
else:
raise ValueError('--main-composite requires a namespace qualified name: ' + str(cmd_args.main_composite))
topo = Topology(name=name, namespace=ns)
if cmd_args.toolkits is not None:
for tk_path in cmd_args.toolkits:
tk.add_toolkit(topo, tk_path)
if cmd_args.create_bundle:
# Mimic what the build service does by indexing
# any required toolkits including Python operator extraction
# but only if we can write to it.
if os.access(tk_path, os.W_OK):
streamsx.scripts.extract.main(['-i', tk_path, '--make-toolkit'])
op.Invoke(topo, cmd_args.main_composite)
return _App(topo, {})
def test_region_match(self):
print ('\n---------'+str(self))
name = 'test_region_match'
topo = Topology(name)
toolkit.add_toolkit(topo, self.geospatial_toolkit_home)
self._index_toolkit(_get_test_tk_path())
toolkit.add_toolkit(topo, _get_test_tk_path())
datagen = op.Invoke(topo, kind='test::GenRegionData', schemas=[RegionMatchSchema.Devices,RegionMatchSchema.Regions])
device_stream = datagen.outputs[0]
region_stream = datagen.outputs[1]
res = geo.region_match(stream=device_stream, region_stream=region_stream)
res.print()
if (("TestDistributed" in str(self)) or ("TestStreamingAnalytics" in str(self))):
tester = Tester(topo)
tester.tuple_count(res, 4, exact=True)
tester.test(self.test_ctxtype, self.test_config, always_collect_logs=True)
else:
# build only
self._build_only(name, topo)
def test_hdfs_uri(self):
ae_service_creds_file = os.environ['ANALYTICS_ENGINE']
with open(ae_service_creds_file) as data_file:
credentials = json.load(data_file)
# credentials is dict
topo = Topology('test_hdfs_uri')
if self.hdfs_toolkit_location is not None:
tk.add_toolkit(topo, self.hdfs_toolkit_location)
# creates an input stream
fileSinkInputStream = topo.source(
['This line will be written into a HDFS file.']).as_string()
result = hdfs.write(fileSinkInputStream,
credentials=credentials,
file='pytest/sample%FILENUM.txt')
result.print()
scanned_files = hdfs.scan(topo,
credentials=credentials,
directory='pytest',
pattern='sample.*txt',
init_delay=10)
scanned_files.print()
lines = hdfs.read(scanned_files, credentials=credentials)
lines.print()
tester = Tester(topo)
tester.tuple_count(lines, 1, exact=True)
#tester.run_for(60)
cfg = {}
job_config = streamsx.topology.context.JobConfig(tracing='info')
job_config.add(cfg)
cfg[streamsx.topology.context.ConfigParams.SSL_VERIFY] = False
# Run the test
tester.test(self.test_ctxtype, cfg, always_collect_logs=True)
def test_buildonly_consume(self):
txtmsg_schema = StreamSchema('tuple<rstring msg>')
errmsg_schema = StreamSchema('tuple<rstring errorMessage>')
java_class_lib_paths = []
java_class_lib_paths.append("./streamsx/jms/tests/libs/activemq/lib")
java_class_lib_paths.append(
"./streamsx/jms/tests/libs/activemq/lib/optional")
path_to_connection_doc = "./streamsx/jms/tests/connectionDocument.xml" # tests are supposed to be run from the package directory
topo = Topology('buildonly_consume')
toolkit.add_toolkit(topo, "../../streamsx.jms/com.ibm.streamsx.jms")
outputs = jms.consume(topo,
schemas=[txtmsg_schema, errmsg_schema],
java_class_libs=java_class_lib_paths,
connection="localActiveMQ",
access="accessToTextMessages",
connection_document=path_to_connection_doc,
name="JMS_Consumer")
txtmsg_stream = outputs[0]
txtmsg_stream.print()
self._build_only('test_buildonly_consume', topo)
def test_flight_path_encounter(self):
print ('\n---------'+str(self))
name = 'test_flight_path_encounter'
topo = Topology(name)
toolkit.add_toolkit(topo, self.geospatial_toolkit_home)
self._index_toolkit(_get_test_tk_path())
toolkit.add_toolkit(topo, _get_test_tk_path())
datagen = op.Invoke(topo, kind='test::GenFlightPathData', schemas=[FlighPathEncounterSchema.EncounterEvents])
planes_stream = datagen.outputs[0]
events = planes_stream.map(geo.FlightPathEncounter(north_latitude=52.6,south_latitude=52.4,west_longitude=13.3,east_longitude=13.5,num_latitude_divs=5,num_longitude_divs=5,search_radius=10000,altitude_search_radius=400,time_search_interval=600000), schema=FlighPathEncounterSchema.EncounterEvents)
dump = op.Invoke(topo, inputs=[events], kind='test::DumpData', schemas=CommonSchema.String)
res = dump.outputs[0]
res.print()
if (("TestDistributed" in str(self)) or ("TestStreamingAnalytics" in str(self))):
tester = Tester(topo)
tester.tuple_count(res, 1, exact=True)
tester.test(self.test_ctxtype, self.test_config, always_collect_logs=True)
else:
# build only
self._build_only(name, topo)
def test_basic(self):
topo = Topology("TupleBasic")
self.tester = Tester(topo)
tk.add_toolkit(topo, inetToolkit)
# Loop back not natural in a directed graph, need
# to have place holder while the graph gets built,
# At the end of the graph, connect back to the
# begining.
pending_source = PendingStream(topo)
# Within a topology a directed graph has a start, this loop does not.
# Create an 'onRamp' that creates and input (that does nothing) in order
# that a graph graph can be built.
rsp = pending_source.stream
ss = topo.source([], name="onRamp")
rsp = ss.union({rsp})
# FormatResponse :
rspFormatted = rsp.map(lambda x : json.dumps(x) ).as_string();
# Convert the tuple json object that we have been working with to
# 'tuple' that is native to Streams.
toTuple = op.Map("com.ibm.streamsx.json::JSONToTuple",
stream=rspFormatted,
schema='tuple<int64 key, rstring response>',
params = {'ignoreParsingError':True},
name = "JSONToTuple")
# Get to the Streams.
rspFormatted = toTuple.stream
rawRequest = op.Map("com.ibm.streamsx.inet.rest::HTTPRequestProcess",
stream = rspFormatted,
schema ='tuple<int64 key, rstring request, rstring method, rstring pathInfo >',
params={'port': PORT,
'webTimeout':5.0,
'contextResourceBase':'opt/base',
'context':CONTEXTBASE},
name = "HttpRequestProcess")
rawRequest.stream.sink(webEntryLog) ## log what we have received.
# determine what to work on
onlyTuple = rawRequest.stream.filter(lambda t : t["pathInfo"]=="/Tuple",
name="inputFilter")
# do the work
upperDone = onlyTuple.transform(upperString,
name="upperCase")
#
self.tester.tuple_count(upperDone, 1)
# loopback to sending
pending_source.complete(upperDone) # loopback
## All done building the graph......
self.test_config['topology.keepArtifacts']=True
# setup the code that will invoke this test.
self.tester.local_check = self.basic_client
# enable tracing info.
job_config = streamsx.topology.context.JobConfig(job_name='Tuple', tracing="trace")
job_config.add(self.test_config)
# submit the application for test
self.tester.test(self.test_ctxtype, self.test_config)
def _add_toolkits(self, topo, toolkit_name):
if toolkit_name is not None:
tk.add_toolkit(topo, toolkit_name)
if self.eventstore_toolkit_location is not None:
tk.add_toolkit(topo, self.eventstore_toolkit_location)
def _add_toolkits(self, topo, test_toolkit):
tk.add_toolkit(topo, test_toolkit)
if self.avro_toolkit_location is not None:
tk.add_toolkit(topo, self.avro_toolkit_location)
if self.json_toolkit_location is not None:
tk.add_toolkit(topo, self.json_toolkit_location)
def _add_toolkits(self, topo, test_toolkit):
tk.add_toolkit(topo, test_toolkit)
if self.inet_toolkit_location is not None:
tk.add_toolkit(topo, self.inet_toolkit_location)
def _add_toolkits(self, topo, test_toolkit):
tk.add_toolkit(topo, test_toolkit)
if self.elasticsearch_toolkit_location is not None:
tk.add_toolkit(topo, self.elasticsearch_toolkit_location)
def test_pathInfo(self):
"""TEST test_routine, validate that the pathInfo can be used for routing.
The context defines the base of incoming messages, beyond that it is in
the pathInfo. If the context is 'base' and the request url is
http://node:port/base/fred/route/my/request, the pathInfo
will be /fred/route/my/request.
"""
topo = Topology("PathInfo")
self.tester = Tester(topo)
tk.add_toolkit(topo, inetToolkit)
pending_source = PendingStream(topo)
rsp = pending_source.stream.map(lambda t : t)
ss = topo.source([], name="onRamp")
rsp = ss.union({rsp})
# FormatResponse :
rspFormatted = rsp.map(lambda x : json.dumps(x) ).as_string();
rawRequest = op.Map("com.ibm.streamsx.inet.rest::HTTPRequestProcess",
stream=rspFormatted,
schema='tuple<int64 key, rstring request, rstring method, rstring pathInfo >',
params={'port': PORT,
'webTimeout':5.0,
'responseJsonAttributeName':'string',
'context':'base',
'contextResourceBase':'opt/base' },
name = "RequestProcess")
rawRequest.stream.sink(webEntryLog) ## log what we have received.
# determine what to work on
onlyUpper = rawRequest.stream.filter(lambda t : t["pathInfo"]=="/Upper",
name="upperFilter")
onlyLower = rawRequest.stream.filter(lambda t : t["pathInfo"]=="/Lower",
name="lowerFilter")
onlyLong = rawRequest.stream.filter(lambda t : t["pathInfo"]=="/L/O/N/G/L/O/W/E/R",
name="longFilter")
onlyCount = rawRequest.stream.filter(lambda t : t["pathInfo"]=="/Count",
name="countFilter")
# do the work
upperDone = onlyUpper.transform(upperString,
name="upperCase")
lowerDone = onlyLower.transform(lowerString,
name="lowerCase")
longDone = onlyLong.transform(lowerString,
name="longCase")
longCount = onlyCount.transform(upperString,
name="countCase")
# do tests on what we have processed.
self.tester.tuple_count(lowerDone, 1)
self.tester.tuple_count(upperDone, 1)
self.tester.tuple_count(longDone, 1)
self.tester.tuple_count(longCount, 10)
# union
unionResult = upperDone.union({lowerDone, longDone, longCount})
# loopback to sending
pending_source.complete(unionResult) # loopback
## All done building the graph......
# setup the code that will invoke this test.
self.tester.local_check = self.pathInfo_request
# submit the application for test
self.tester.test(self.test_ctxtype, self.test_config)
def test_reflect(self):
topo = Topology("TupleReflect")
self.tester = Tester(topo)
tk.add_toolkit(topo, inetToolkit)
# Looping
pending_source = PendingStream(topo)
# Build the onramp.
rsp = pending_source.stream
ss = topo.source([], name="onRamp")
rsp = ss.union({rsp})
# FormatResponse :
rspFormatted = rsp.map(lambda x : json.dumps(x) ).as_string();
# Convert to 'tuple' that is native to Streams, stripping of everything: key, response.
toTuple = op.Map("com.ibm.streamsx.json::JSONToTuple",
stream=rspFormatted,
schema='tuple<int64 key, rstring response>',
params = {'ignoreParsingError':True},
name = "JSONToTuple")
# Get to the Streams.
rspFormatted = toTuple.stream
# Output : return to web,
# Input : get from web
rawRequest = op.Map("com.ibm.streamsx.inet.rest::HTTPRequestProcess",
stream = rspFormatted,
#schema ='tuple<int64 key, rstring request, rstring method, rstring pathInfo >',
schema='tuple<int64 key, rstring request, rstring contentType, map<rstring, rstring> header, rstring response, rstring method,rstring pathInfo, int32 status, rstring statusMessage>',
params={'port': PORT,
'webTimeout':5.0,
'contextResourceBase':'opt/base',
'context':CONTEXTBASE},
name = "HttpRequestProcess")
rawRequest.stream.sink(webEntryLog) ## log what we have received.
# determine what to work on
onlyTuple = rawRequest.stream.filter(lambda t : t["pathInfo"]=="/Tuple",
name="inputFilter")
# do the work
reflected = onlyTuple.transform(reflect2Result("REF:"), name="refectStream")
#
self.tester.tuple_count(reflected, 3)
# loopback to sending
pending_source.complete(reflected) # loopback
## All done building the graph......
## Set parameters
self.test_config['topology.keepArtifacts']=True
# setup the code that will invoke this test.
self.tester.local_check = self.reflect_client
# enable tracing info.
job_config = streamsx.topology.context.JobConfig(job_name='Tuple', tracing="trace")
job_config.add(self.test_config)
# submit the application for test
self.tester.test(self.test_ctxtype, self.test_config)
def reflectPost(self, expected_requests, local_check_function ):
"""
Reflect back after a set number of seconds.
"""
topo = Topology("Reflect")
self.tester = Tester(topo)
tk.add_toolkit(topo, inetToolkit)
pending_source = PendingStream(topo)
rsp = pending_source.stream.map(lambda t : t)
ss = topo.source([], name="onRamp")
rsp = ss.union({rsp})
# FormatResponse :
rspFormatted = rsp.map(lambda x : json.dumps(x) ).as_string();
rawRequest = op.Map("com.ibm.streamsx.inet.rest::HTTPRequestProcess",
stream=rspFormatted,
schema='tuple<int64 key, rstring request, rstring contentType, map<rstring, rstring> header, rstring response, rstring method,rstring pathInfo, int32 status, rstring statusMessage>',
params={'port': PORT,
'webTimeout':15.0,
'responseJsonAttributeName':'string',
'context':'Reflect',
'contextResourceBase': 'opt/Reflect'},
name = "RequestProcess")
rawRequest.stream.sink(webEntryLog) ## log what we have received.
onlyFirst = rawRequest.stream.filter(lambda t : t["pathInfo"]=="/first",
name="firstFilter")
onlySecond = rawRequest.stream.filter(lambda t : t["pathInfo"]=="/second",
name="secondFilter")
onlyThird = rawRequest.stream.filter(lambda t : t["pathInfo"]=="/third",
name="thirdFilter")
onlyFourth = rawRequest.stream.filter(lambda t : t["pathInfo"]=="/fourth",
name="fourthFilter")
# do the work
firstReflect = onlyFirst.transform(sleepResponse("SLP1:"),
name="firstResponse")
secondReflect = onlySecond.transform(sleepResponse("SLP2:"),
name="secondResponse")
thirdReflect = onlyThird.transform(sleepResponse("SLP3:"),
name="thirdResponse")
fourthReflect = onlyFourth.transform(sleepResponse("SLP4:"),
name="fourthResponse")
allReflect = firstReflect.union({secondReflect,thirdReflect,fourthReflect})
allReflect.sink(webExitLog) ## log what we are sending back
# do tests on what we have processed.
self.tester.tuple_count(allReflect, expected_requests)
# loopback to sending
pending_source.complete(allReflect) # loopback
## All done building the graph......
# setup the code that will invoke this test.
self.tester.local_check = local_check_function
# enable tracing info.
job_config = streamsx.topology.context.JobConfig(job_name='Async', tracing="info")
job_config.add(self.test_config)
# submit the application for test
self.tester.test(self.test_ctxtype, self.test_config)
"""
