def test_hbase_scan(self): topo = Topology('test_hbase_scan') print('\n--------------------------- \nTest : ' + topo.name + '\n--------------------------- ') if self.hbase_toolkit_location is not None: tk.add_toolkit(topo, self.hbase_toolkit_location) tester = Tester(topo) # hbase.scan creates site.xml file with the use of environment variable HADOOP_HOST_PORT / HBASE_SITE_XML scanned_rows = hbase.scan(topo, table_name=_get_table_name(), max_versions=1, init_delay=2) scanned_rows.print(name='printScan') tester.tuple_count(scanned_rows, 2, exact=False) cfg = {} job_config = streamsx.topology.context.JobConfig(tracing='info') job_config.add(cfg) cfg[streamsx.topology.context.ConfigParams.SSL_VERIFY] = False # Run the test tester.test(self.test_ctxtype, cfg, always_collect_logs=True)
def test_hbase_scan_connection_param_dict(self): topo = Topology('test_hbase_scan_connection_param_dict') print('\n--------------------------- \nTest : ' + topo.name + '\n--------------------------- ') if self.hbase_toolkit_location is not None: tk.add_toolkit(topo, self.hbase_toolkit_location) tester = Tester(topo) # set dict as connection parameter with values from HADOOP_HOST_PORT hp = os.environ['HADOOP_HOST_PORT'].split(":", 1) ext_connection = {} ext_connection['host'] = hp[0] ext_connection['port'] = hp[1] scanned_rows = hbase.scan(topo, table_name=_get_table_name(), max_versions=1, init_delay=2, connection=ext_connection) scanned_rows.print(name='printScan') tester.tuple_count(scanned_rows, 2, exact=False) cfg = {} job_config = streamsx.topology.context.JobConfig(tracing='info') job_config.add(cfg) cfg[streamsx.topology.context.ConfigParams.SSL_VERIFY] = False # Run the test tester.test(self.test_ctxtype, cfg, always_collect_logs=True)
def test_hbase_scan_connection_param_file(self): topo = Topology('test_hbase_scan_connection_param_file') print('\n--------------------------- \nTest : ' + topo.name + '\n--------------------------- ') if self.hbase_toolkit_location is not None: tk.add_toolkit(topo, self.hbase_toolkit_location) tester = Tester(topo) # set HBASE_SITE_XML file as connection parameter scanned_rows = hbase.scan(topo, table_name=_get_table_name(), max_versions=1, init_delay=2, connection=os.environ['HBASE_SITE_XML']) scanned_rows.print(name='printScan') tester.tuple_count(scanned_rows, 2, exact=False) cfg = {} job_config = streamsx.topology.context.JobConfig(tracing='info') job_config.add(cfg) cfg[streamsx.topology.context.ConfigParams.SSL_VERIFY] = False # Run the test tester.test(self.test_ctxtype, cfg, always_collect_logs=True)
def test_buildonly_produce(self): txtmsg_schema = StreamSchema('tuple<rstring msg>') errmsg_schema = StreamSchema('tuple<rstring errorMessage>') java_class_lib_paths = [] java_class_lib_paths.append("./streamsx/jms/tests/libs/activemq/lib") java_class_lib_paths.append( "./streamsx/jms/tests/libs/activemq/lib/optional") path_to_connection_doc = "./streamsx/jms/tests/connectionDocument.xml" # tests are supposed to be run from the package directory topo = Topology('buildonly_produce') toolkit.add_toolkit(topo, "../../streamsx.jms/com.ibm.streamsx.jms") txtmsg_source = Source(topo, 'spl.utility::Beacon', txtmsg_schema, params={'period': 0.3}, name="DataGenerator") txtmsg_source.msg = txtmsg_source.output( '"Message #" + (rstring)IterationCount()') txtmsg_stream = txtmsg_source.stream txtmsg_stream.print() errmsg_stream = jms.produce(stream=txtmsg_stream, schema=errmsg_schema, java_class_libs=java_class_lib_paths, connection="localActiveMQ", access="accessToTextMessages", connection_document=path_to_connection_doc, name="JMS_Producer") errmsg_stream.print() self._build_only('test_buildonly_produce', topo)
def job_status_service(): topo = Topology('JobStatusService') add_toolkit(topo, monitoring_toolkit) r = op.main_composite( kind='com.ibm.streamsx.monitoring.jobs.services::JobStatusService', toolkits=[monitoring_toolkit]) _launch(r[0])
def wikiFeed(inetToolkit, buildType, port): # Sumbit request Build Server and Submit. schemaTicker = 'tuple<rstring dataId, rstring bot, rstring domain, int32 lenOld,int32 lenNew, rstring SSEdata, rstring SSEevent>' # # Define the application # topo = Topology("WikiFeed") tk.add_toolkit(topo, inetToolkit) source = topo.source(wikiStream) # Only one type of event, verity that I got it right. event = source.filter(lambda t: t["SSEevent"] == "message", name="eventFilter") event.print(name="eth") eventTuple = event.map(lambda t: t, schema=schemaTicker) eventWin = eventTuple.last(100).trigger(1) rawRequest = op.Sink("com.ibm.streamsx.inet.rest::HTTPTupleView", stream=eventWin, params={ 'port': 8081, 'context': 'gdaxEth', 'contextResourceBase': '/base' }, name="TupleView") # # Compile & Submit the Topology to Streams instance # streams_conf = common.build_streams_config("StreamingTurbine", credential.serviceCredentials) context.submit(context.ContextTypes.STREAMING_ANALYTICS_SERVICE, topo, config=streams_conf)
def test_hbase_get(self): topo = Topology('test_hbase_get') print('\n--------------------------- \nTest : ' + topo.name + '\n--------------------------- ') if self.hbase_toolkit_location is not None: tk.add_toolkit(topo, self.hbase_toolkit_location) s = _create_stream_for_get(topo) tester = Tester(topo) # hbase.get creates site.xml file with the use of environment variable HADOOP_HOST_PORT / HBASE_SITE_XML get_rows = hbase.get(s, table_name=_get_table_name(), row_attr_name='who') get_rows.print(name='printGet') tester.tuple_count(get_rows, 2, exact=False) # tester.run_for(60) cfg = {} job_config = streamsx.topology.context.JobConfig(tracing='info') job_config.add(cfg) cfg[streamsx.topology.context.ConfigParams.SSL_VERIFY] = False # Run the test tester.test(self.test_ctxtype, cfg, always_collect_logs=True)
def failed_pe_service(): topo = Topology('FailedPEService') add_toolkit(topo, monitoring_toolkit) r = op.main_composite( kind='com.ibm.streamsx.monitoring.jobs.services::FailedPEService', toolkits=[monitoring_toolkit]) _launch(r[0])
def test_close_on_tuples(self): ae_service_creds_file = os.environ['ANALYTICS_ENGINE'] with open(ae_service_creds_file) as data_file: credentials = json.load(data_file) topo = Topology('test_hdfs_uri') if self.hdfs_toolkit_location is not None: tk.add_toolkit(topo, self.hdfs_toolkit_location) s = topo.source([ 'Hello World!', 'Hello', 'World', 'Hello World!', 'Hello', 'World' ]).as_string() result = hdfs.write(s, credentials=credentials, file='pytest/write_test%FILENUM.txt', tuples_per_file=3) result.print() tester = Tester(topo) tester.tuple_count(result, 2, exact=True) #tester.run_for(60) cfg = {} job_config = streamsx.topology.context.JobConfig(tracing='info') job_config.add(cfg) cfg[streamsx.topology.context.ConfigParams.SSL_VERIFY] = False # Run the test tester.test(self.test_ctxtype, cfg, always_collect_logs=True)
def metrics_ingest_service(): topo = Topology('MetricsIngestService') add_toolkit(topo, monitoring_toolkit) r = op.main_composite( kind= 'com.ibm.streamsx.monitoring.metrics.services::MetricsIngestService', toolkits=[monitoring_toolkit]) _launch(r[0])
def reflectPost(self, expected_requests, local_check_function): """Reflect base : reflecting back post. Ran into a problem with reflecting back the header, if you include a Content-Length in the header a corrected Content-Length is *not* sent. When the data arrives back at the client the data is too short - thus the buildRsp() """ topo = Topology("Reflect") self.tester = Tester(topo) tk.add_toolkit(topo, inetToolkit) pending_source = PendingStream(topo) rsp = pending_source.stream.map(lambda t: t) ss = topo.source([], name="onRamp") rsp = ss.union({rsp}) # FormatResponse : rspFormatted = rsp.map(lambda x: json.dumps(x)).as_string() rawRequest = op.Map( "com.ibm.streamsx.inet.rest::HTTPRequestProcess", stream=rspFormatted, schema= 'tuple<int64 key, rstring request, rstring contentType, map<rstring, rstring> header, rstring response, rstring method,rstring pathInfo, int32 status, rstring statusMessage>', params={ 'port': PORT, 'webTimeout': 5.0, 'responseJsonAttributeName': 'string', 'context': 'Reflect', 'contextResourceBase': 'opt/Reflect' }, name="RequestProcess") rawRequest.stream.sink(webEntryLog) ## log what we have received. # wait for lastAct = rawRequest.stream.filter(sleepMacbeth) # do the work getReflect = rawRequest.stream.transform(buildResponse("RSP:"), name="buildResponse") getReflect.sink(webExitLog) ## log what we are sending back # do tests on what we have processed. self.tester.tuple_count(getReflect, expected_requests) # loopback to sending pending_source.complete(getReflect) # loopback ## All done building the graph...... # setup the code that will invoke this test. self.tester.local_check = local_check_function # enable tracing info. job_config = streamsx.topology.context.JobConfig(job_name='Reflect', tracing="info") job_config.add(self.test_config) # submit the application for test self.tester.test(self.test_ctxtype, self.test_config)
def smokePending(inetToolkit, buildType, port): # Specify the Schema going in and out of the Streams operator. schemaRequest = 'tuple<int64 key, rstring request, rstring method, rstring pathInfo >' schemaResponse = 'tuple<int64 key, rstring request, rstring method, rstring pathInfo, rstring response >' topo = Topology("SmokePending") # Add extrenal spl functions, this is what is being testing my HTTPRequestProcess() tk.add_toolkit(topo, inetToolkit) # 'pending_source' is the loopback point. pending_source = PendingStream(topo) # Convert to Streams' tuple rsp = pending_source.stream.map(lambda t: t, schema=schemaResponse) rspFormatted = rsp.map(lambda t: json.dumps(t)).as_string() rspFormatted.sink(webLog("Output response")) ## log what we have received. rawRequest = op.Map("com.ibm.streamsx.inet.rest::HTTPRequestProcess", stream=rsp, schema = schemaRequest, params={'port': 8080, 'webTimeout': 5.0, #'responseJsonAttributeName': 'string', 'context':'myStreams', 'contextResourceBase': '/base'}, name="TupleRequest") # write out data rawRequest.stream.sink(webLog('Input request')) ## log what we have received. # determine what to work on # Filter does not change the type upperDo = rawRequest.stream.filter(lambda t: t["pathInfo"] == "/upper", name="upperFilter") lowerDo = rawRequest.stream.filter(lambda t: t["pathInfo"] == "/lower", name="lowerFilter") # do some processing upperDone = upperDo.map(upperString, schema = schemaResponse, name="upperProcess") lowerDone = lowerDo.map(lowerString, schema = schemaResponse, name="lowerProcess") ## processingDone = upperDone.union({lowerDone}) hack = processingDone.map(lambda t: t, schema=schemaResponse) pending_source.complete(hack) # close the loop # hack make it so I do not get this error. """ pending_source.complete(processingDone) # close the loop SEVERE: Streaming Analytics service (StreamingTurbine): The submitted archive tk5385353137138356122.zip failed to scripts with status failed. Exception in thread "main" java.lang.IllegalStateException: Error submitting archive for compilation: "tk5385353137138356122/SmokePending/SmokePending.spl:5:1: CDISP0011E ERROR: A syntax error exists at the '}' token in 'graph'." """ # # Compile & Submit the Topology to Streams instance # streams_conf = common.build_streams_config("StreamingTurbine", credential.serviceCredentials) context.submit(context.ContextTypes.STREAMING_ANALYTICS_SERVICE, topo, config=streams_conf)
def sample_app(): topo = Topology('SystemMonitorSample') add_toolkit(topo, monitoring_toolkit) add_toolkit(topo, sample_toolkit) r = op.main_composite( kind= 'com.ibm.streamsx.monitoring.system.sample.SystemMonitorSource::Monitor', toolkits=[monitoring_toolkit, sample_toolkit]) _launch(r[0])
def sample_app(): topo = Topology('MetricsSample') add_toolkit(topo, monitoring_toolkit) add_toolkit(topo, sample_toolkit) r = op.main_composite( kind= 'com.ibm.streamsx.monitoring.metrics.sample.MetricsSource::SampleJob', toolkits=[monitoring_toolkit, sample_toolkit]) _launch(r[0])
def sample_app(): topo = Topology('CrashSample') add_toolkit(topo, monitoring_toolkit) add_toolkit(topo, sample_toolkit) r = op.main_composite( kind= 'com.ibm.streamsx.monitoring.jobs.sample.JobStatusMonitor::SampleJob', toolkits=[monitoring_toolkit, sample_toolkit]) _launch(r[0])
def reflectPost(self, expected_requests, local_check_function ): """Reflect base : Reflect back the data that is send as well as sending amount specified in the repeat paramter. Ran into a problem with reflecting back the header, if you include a Content-Length in the header a corrected Content-Length is *not* sent. When the data arrives back at the client the data is too short - thus the buildRsp() """ topo = Topology("Reflect") self.tester = Tester(topo) tk.add_toolkit(topo, inetToolkit) pending_source = PendingStream(topo) rsp = pending_source.stream.map(lambda t : t) ss = topo.source([], name="onRamp") rsp = ss.union({rsp}) # FormatResponse : rspFormatted = rsp.map(lambda x : json.dumps(x) ).as_string(); rawRequest = op.Map("com.ibm.streamsx.inet.rest::HTTPRequestProcess", stream=rspFormatted, schema='tuple<int64 key, rstring request, rstring contentType, map<rstring, rstring> header, rstring response, rstring method,rstring pathInfo, int32 status, rstring statusMessage>', params={'port': PORT, 'webTimeout':5.0, 'responseJsonAttributeName':'string', 'context':'Reflect', 'contextResourceBase': 'opt/Reflect'}, name = "RequestProcess") rawRequest.stream.sink(webEntryLog) ## log what we have received. # wait for lastAct = rawRequest.stream.filter(sleepMacbeth) # do the work getReflect = rawRequest.stream.transform(buildResponse("RSP:"), name="buildResponse") getReflect.sink(webExitLog) ## log what we are sending back # do tests on what we have processed. self.tester.tuple_count(getReflect, expected_requests) # loopback to sending pending_source.complete(getReflect) # loopback ## All done building the graph...... # setup the code that will invoke this test. self.tester.local_check = local_check_function # enable tracing info. job_config = streamsx.topology.context.JobConfig(job_name='Reflect', tracing="info") job_config.add(self.test_config) # submit the application for test self.tester.test(self.test_ctxtype, self.test_config) """
def _get_spl_app(cmd_args): ns, name = cmd_args.main_composite.rsplit('::', 1) ns += '._spl' topo = Topology(name=name, namespace=ns) if cmd_args.toolkits is not None: for tk_path in cmd_args.toolkits: tk.add_toolkit(topo, tk_path) op.Invoke(topo, cmd_args.main_composite) return (topo, {})
def test_basic(self): topo = Topology("Basic") self.tester = Tester(topo) tk.add_toolkit(topo, inetToolkit) # Loop back not natural in a directed graph, need # to have place holder while the graph gets built, # At the end of the graph, connect back to the # begining. pending_source = PendingStream(topo) # Directed graph has start this loop loop does not. # Need to give topology an "onRamp" so it can build the graph. rsp = pending_source.stream.map(lambda t : t) ss = topo.source([], name="onRamp") rsp = ss.union({rsp}) # FormatResponse : rspFormatted = rsp.map(lambda x : json.dumps(x) ).as_string(); rawRequest = op.Map("com.ibm.streamsx.inet.rest::HTTPRequestProcess", stream=rspFormatted, schema='tuple<int64 key, rstring request, rstring method, rstring pathInfo >', params={'port': PORT, 'webTimeout':5.0, 'responseJsonAttributeName':'string', 'context':'base', 'contextResourceBase':'opt/base' }, name = "RequestProcess") rawRequest.stream.sink(webEntryLog) ## log what we have received. # determine what to work on onlyTuple = rawRequest.stream.filter(lambda t : t["pathInfo"]=="/Tuple", name="inputFilter") # do the work upperDone = onlyTuple.transform(upperString, name="upperCase") # self.tester.tuple_count(upperDone, 1) # loopback to sending pending_source.complete(upperDone) # loopback ## All done building the graph...... self.test_config['topology.keepArtifacts']=True # setup the code that will invoke this test. self.tester.local_check = self.basic_request # submit the application for test self.tester.test(self.test_ctxtype, self.test_config)
def _add_local_toolkits(toolkit_paths, dependencies, topo, verify_arg): """ A helper function that given the local toolkit paths, and the dependencies (ie toolkits that the apps depends on), finds the latest dependency w/ correct version (could be local or on the buildserver) and checks whether it exists locally, and if so adds it. If the latest dependency w/ correct version is located on the buildserver, don't need to do anything Arguments: toolkit_paths {List} -- A list of local toolkit directories, each could be a toolkit directory (has info.xml directly inside), or a directory consisting of toolkits (no info.xml directly inside) dependencies {Dictionary} -- A dictionary consisting of elements where the key is the name of the dependency, and the value is its version. topo {topology Object} -- [description] verify_arg {type} -- Disable SSL verification. """ build_server = BuildService.of_endpoint(verify=verify_arg) local_toolkits = _get_all_local_toolkits(toolkit_paths) remote_toolkits = build_server.get_toolkits() all_toolkits = local_toolkits + remote_toolkits # Iterate through all dependencies (toolkits) needed to build sab file, and check if the dependency already exist on the build server for dependency_name, dependency_version in dependencies.items(): # All the toolkits that match dependency_name regardless of version matching_toolkits = [toolkit for toolkit in all_toolkits if toolkit.name == dependency_name] if not matching_toolkits: # No toolkits exists on the remote buildserver or locally that can satsify this dependency, continue and let SPL compiler throw error continue # Toolkit with the highest verion that still satisfies the dependency_version latest_compatible_toolkit = None # Go through all toolkits, find the highest/latest version, given by latest_compatible_toolkit for tk in matching_toolkits: # Check if current toolkit tk satisfies the version requirement if _check_correct_version(tk, dependency_version): # if we have a toolkit that already satisfies the version requirement, and tk also satisfies its # compare tk against the highest version we have seen thus far if latest_compatible_toolkit: latest_version_so_far = pkg_resources.parse_version(latest_compatible_toolkit.version) curr_tk_version = pkg_resources.parse_version(tk.version) # if it is a later version, update our toolkit if curr_tk_version > latest_version_so_far: latest_compatible_toolkit = tk else: latest_compatible_toolkit = tk # Check if latest_compatible_toolkit is local, bc we then need to add it if isinstance(latest_compatible_toolkit, _LocalToolkit): # Latest toolkit is local, upload it add_toolkit(topo, latest_compatible_toolkit.path)
def reflect(self, expected_requests, local_check_function ): """Reflect base : request back to request, check results in Python VM. Basis for other tests where the processing of the results are done in PYTHON. - expected_requests : number of requests that the local_check_function will make """ topo = Topology("Reflect") self.tester = Tester(topo) tk.add_toolkit(topo, inetToolkit) pending_source = PendingStream(topo) rsp = pending_source.stream.map(lambda t : t) ss = topo.source([], name="onRamp") rsp = ss.union({rsp}) # FormatResponse : rspFormatted = rsp.map(lambda x : json.dumps(x) ).as_string(); rawRequest = op.Map("com.ibm.streamsx.inet.rest::HTTPRequestProcess", stream=rspFormatted, schema='tuple<int64 key, rstring request, rstring method, rstring pathInfo >', params={'port': PORT, 'webTimeout':5.0, 'responseJsonAttributeName':'string', 'context':'Reflect', 'contextResourceBase':'opt/Reflect' }, name = "RequestProcess") rawRequest.stream.sink(webEntryLog) ## log what we have received. # wait for lastAct = rawRequest.stream.filter(sleepMacbeth) # do the work getReflect = lastAct.transform(reflectResult("RAW:"), name="reflectResult") # do tests on what we have processed. self.tester.tuple_count(getReflect, expected_requests) # loopback to sending pending_source.complete(getReflect) # loopback ## All done building the graph...... # setup the code that will invoke this test. self.tester.local_check = local_check_function # enable tracing info. job_config = streamsx.topology.context.JobConfig(job_name='Reflect', tracing="info") job_config.add(self.test_config) # submit the application for test self.tester.test(self.test_ctxtype, self.test_config)
def test_all_hdsf_operators(self): hdfs_cfg_file = os.environ['HDFS_SITE_XML'] # credentials is the path to the HDSF *configuration file 'hdfs-site.xml' topo = Topology('test_all_hdsf_operators') if self.hdfs_toolkit_location is not None: tk.add_toolkit(topo, self.hdfs_toolkit_location) # creates an input stream fileSinkInputStream = topo.source( ['This line will be written into a HDFS file.']).as_string() # writes a line into a HDFS file (HDFS2FileSink) fileSinkResults = hdfs.write(fileSinkInputStream, credentials=hdfs_cfg_file, file='pytest1/sample4%FILENUM.txt') fileSinkResults.print(name='printFileSinkResults') # scans an HDFS directory and return file names (HDFS2DirectoryScan) scannedFileNames = hdfs.scan(topo, credentials=hdfs_cfg_file, directory='pytest1', pattern='sample.*txt', init_delay=10) scannedFileNames.print(name='printScannedFileNames') # reads lines from a HDFS file (HDFS2FileSource) readLines = hdfs.read(scannedFileNames, credentials=hdfs_cfg_file) readLines.print(name='printReadLines') # copies files from HDFS into local disk "/tmp/" (HDFS2FileCopy) copyFileResults = hdfs.copy(scannedFileNames, credentials=hdfs_cfg_file, direction='copyToLocalFile', hdfsFile=None, hdfsFileAttrName='fileName', localFile='/tmp/') copyFileResults.print(name='printCopyFileResults') tester = Tester(topo) tester.tuple_count(readLines, 1, exact=False) # tester.run_for(80) cfg = {} job_config = streamsx.topology.context.JobConfig(tracing='info') job_config.add(cfg) cfg[streamsx.topology.context.ConfigParams.SSL_VERIFY] = False # Run the test tester.test(self.test_ctxtype, cfg, always_collect_logs=True)
def gdaxFeed(inetToolkit, buildType, port): # Sumbit request Build Server and Submit. schemaTicker = 'tuple<rstring ttype, float32 price, float32 low_24h, float32 best_ask, rstring side, float32 best_bid, float32 open_24h, rstring product_id, int32 sequence, int32 trade_id, rstring time, float32 last_size, float32 volume_24h, float32 volume_30d, float32 high_24h>' # # Define the application # topo = Topology("GdaxFeed") tk.add_toolkit(topo, inetToolkit) source = topo.source(gdaxData) # Split out the securities : ETH-USD, LTC-USD, BTC-USD eth = source.filter(lambda t: t["product_id"] == "ETH-USD", name="ethFilter") ltc = source.filter(lambda t: t["product_id"] == "LTC-USD", name="ltcFilter") btc = source.filter(lambda t: t["product_id"] == "LTC-USD", name="btcFilter") eth.print(name="eth") ltc.print(name="ltc") btc.print(name="btc") ethTuple = eth.map(lambda t: t, schema=schemaTicker) #ethWin = ethTuple.last(datetime.timedelta(minutes=2)) ethWin = ethTuple.last(100).trigger(1) rawRequest = op.Sink("com.ibm.streamsx.inet.rest::HTTPTupleView", stream=ethWin, params={ 'port': 8080, 'context': 'gdaxEth', 'contextResourceBase': '/base' }, name="TupleView") # # Compile & Submit the Topology to Streams instance # streams_conf = common.build_streams_config("StreamingTurbine", credential.serviceCredentials) context.submit(context.ContextTypes.STREAMING_ANALYTICS_SERVICE, topo, config=streams_conf) def test(): print("enter") tmp = gdaxData() while True: print(tmp.__next__())
def test_basic(self): print('\n---------' + str(self)) name = 'test_basic' topo = Topology(name) toolkit.add_toolkit(topo, self.sttgateway_toolkit_home) if (("TestDistributed" in str(self)) or ("TestStreamingAnalytics" in str(self))): tester = Tester(topo) tester.tuple_count(res, 1, exact=True) tester.test(self.test_ctxtype, self.test_config, always_collect_logs=True) else: # build only self._build_only(name, topo)
def _get_spl_app(cmd_args): if '::' in cmd_args.main_composite: ns, name = cmd_args.main_composite.rsplit('::', 1) ns += '._spl' else: raise ValueError('--main-composite requires a namespace qualified name: ' + str(cmd_args.main_composite)) topo = Topology(name=name, namespace=ns) if cmd_args.toolkits is not None: for tk_path in cmd_args.toolkits: tk.add_toolkit(topo, tk_path) if cmd_args.create_bundle: # Mimic what the build service does by indexing # any required toolkits including Python operator extraction # but only if we can write to it. if os.access(tk_path, os.W_OK): streamsx.scripts.extract.main(['-i', tk_path, '--make-toolkit']) op.Invoke(topo, cmd_args.main_composite) return _App(topo, {})
def test_region_match(self): print ('\n---------'+str(self)) name = 'test_region_match' topo = Topology(name) toolkit.add_toolkit(topo, self.geospatial_toolkit_home) self._index_toolkit(_get_test_tk_path()) toolkit.add_toolkit(topo, _get_test_tk_path()) datagen = op.Invoke(topo, kind='test::GenRegionData', schemas=[RegionMatchSchema.Devices,RegionMatchSchema.Regions]) device_stream = datagen.outputs[0] region_stream = datagen.outputs[1] res = geo.region_match(stream=device_stream, region_stream=region_stream) res.print() if (("TestDistributed" in str(self)) or ("TestStreamingAnalytics" in str(self))): tester = Tester(topo) tester.tuple_count(res, 4, exact=True) tester.test(self.test_ctxtype, self.test_config, always_collect_logs=True) else: # build only self._build_only(name, topo)
def test_hdfs_uri(self): ae_service_creds_file = os.environ['ANALYTICS_ENGINE'] with open(ae_service_creds_file) as data_file: credentials = json.load(data_file) # credentials is dict topo = Topology('test_hdfs_uri') if self.hdfs_toolkit_location is not None: tk.add_toolkit(topo, self.hdfs_toolkit_location) # creates an input stream fileSinkInputStream = topo.source( ['This line will be written into a HDFS file.']).as_string() result = hdfs.write(fileSinkInputStream, credentials=credentials, file='pytest/sample%FILENUM.txt') result.print() scanned_files = hdfs.scan(topo, credentials=credentials, directory='pytest', pattern='sample.*txt', init_delay=10) scanned_files.print() lines = hdfs.read(scanned_files, credentials=credentials) lines.print() tester = Tester(topo) tester.tuple_count(lines, 1, exact=True) #tester.run_for(60) cfg = {} job_config = streamsx.topology.context.JobConfig(tracing='info') job_config.add(cfg) cfg[streamsx.topology.context.ConfigParams.SSL_VERIFY] = False # Run the test tester.test(self.test_ctxtype, cfg, always_collect_logs=True)
def test_buildonly_consume(self): txtmsg_schema = StreamSchema('tuple<rstring msg>') errmsg_schema = StreamSchema('tuple<rstring errorMessage>') java_class_lib_paths = [] java_class_lib_paths.append("./streamsx/jms/tests/libs/activemq/lib") java_class_lib_paths.append( "./streamsx/jms/tests/libs/activemq/lib/optional") path_to_connection_doc = "./streamsx/jms/tests/connectionDocument.xml" # tests are supposed to be run from the package directory topo = Topology('buildonly_consume') toolkit.add_toolkit(topo, "../../streamsx.jms/com.ibm.streamsx.jms") outputs = jms.consume(topo, schemas=[txtmsg_schema, errmsg_schema], java_class_libs=java_class_lib_paths, connection="localActiveMQ", access="accessToTextMessages", connection_document=path_to_connection_doc, name="JMS_Consumer") txtmsg_stream = outputs[0] txtmsg_stream.print() self._build_only('test_buildonly_consume', topo)
def test_flight_path_encounter(self): print ('\n---------'+str(self)) name = 'test_flight_path_encounter' topo = Topology(name) toolkit.add_toolkit(topo, self.geospatial_toolkit_home) self._index_toolkit(_get_test_tk_path()) toolkit.add_toolkit(topo, _get_test_tk_path()) datagen = op.Invoke(topo, kind='test::GenFlightPathData', schemas=[FlighPathEncounterSchema.EncounterEvents]) planes_stream = datagen.outputs[0] events = planes_stream.map(geo.FlightPathEncounter(north_latitude=52.6,south_latitude=52.4,west_longitude=13.3,east_longitude=13.5,num_latitude_divs=5,num_longitude_divs=5,search_radius=10000,altitude_search_radius=400,time_search_interval=600000), schema=FlighPathEncounterSchema.EncounterEvents) dump = op.Invoke(topo, inputs=[events], kind='test::DumpData', schemas=CommonSchema.String) res = dump.outputs[0] res.print() if (("TestDistributed" in str(self)) or ("TestStreamingAnalytics" in str(self))): tester = Tester(topo) tester.tuple_count(res, 1, exact=True) tester.test(self.test_ctxtype, self.test_config, always_collect_logs=True) else: # build only self._build_only(name, topo)
def test_basic(self): topo = Topology("TupleBasic") self.tester = Tester(topo) tk.add_toolkit(topo, inetToolkit) # Loop back not natural in a directed graph, need # to have place holder while the graph gets built, # At the end of the graph, connect back to the # begining. pending_source = PendingStream(topo) # Within a topology a directed graph has a start, this loop does not. # Create an 'onRamp' that creates and input (that does nothing) in order # that a graph graph can be built. rsp = pending_source.stream ss = topo.source([], name="onRamp") rsp = ss.union({rsp}) # FormatResponse : rspFormatted = rsp.map(lambda x : json.dumps(x) ).as_string(); # Convert the tuple json object that we have been working with to # 'tuple' that is native to Streams. toTuple = op.Map("com.ibm.streamsx.json::JSONToTuple", stream=rspFormatted, schema='tuple<int64 key, rstring response>', params = {'ignoreParsingError':True}, name = "JSONToTuple") # Get to the Streams. rspFormatted = toTuple.stream rawRequest = op.Map("com.ibm.streamsx.inet.rest::HTTPRequestProcess", stream = rspFormatted, schema ='tuple<int64 key, rstring request, rstring method, rstring pathInfo >', params={'port': PORT, 'webTimeout':5.0, 'contextResourceBase':'opt/base', 'context':CONTEXTBASE}, name = "HttpRequestProcess") rawRequest.stream.sink(webEntryLog) ## log what we have received. # determine what to work on onlyTuple = rawRequest.stream.filter(lambda t : t["pathInfo"]=="/Tuple", name="inputFilter") # do the work upperDone = onlyTuple.transform(upperString, name="upperCase") # self.tester.tuple_count(upperDone, 1) # loopback to sending pending_source.complete(upperDone) # loopback ## All done building the graph...... self.test_config['topology.keepArtifacts']=True # setup the code that will invoke this test. self.tester.local_check = self.basic_client # enable tracing info. job_config = streamsx.topology.context.JobConfig(job_name='Tuple', tracing="trace") job_config.add(self.test_config) # submit the application for test self.tester.test(self.test_ctxtype, self.test_config)
def _add_toolkits(self, topo, toolkit_name): if toolkit_name is not None: tk.add_toolkit(topo, toolkit_name) if self.eventstore_toolkit_location is not None: tk.add_toolkit(topo, self.eventstore_toolkit_location)
def _add_toolkits(self, topo, test_toolkit): tk.add_toolkit(topo, test_toolkit) if self.avro_toolkit_location is not None: tk.add_toolkit(topo, self.avro_toolkit_location) if self.json_toolkit_location is not None: tk.add_toolkit(topo, self.json_toolkit_location)
def _add_toolkits(self, topo, test_toolkit): tk.add_toolkit(topo, test_toolkit) if self.inet_toolkit_location is not None: tk.add_toolkit(topo, self.inet_toolkit_location)
def _add_toolkits(self, topo, test_toolkit): tk.add_toolkit(topo, test_toolkit) if self.elasticsearch_toolkit_location is not None: tk.add_toolkit(topo, self.elasticsearch_toolkit_location)
def test_pathInfo(self): """TEST test_routine, validate that the pathInfo can be used for routing. The context defines the base of incoming messages, beyond that it is in the pathInfo. If the context is 'base' and the request url is http://node:port/base/fred/route/my/request, the pathInfo will be /fred/route/my/request. """ topo = Topology("PathInfo") self.tester = Tester(topo) tk.add_toolkit(topo, inetToolkit) pending_source = PendingStream(topo) rsp = pending_source.stream.map(lambda t : t) ss = topo.source([], name="onRamp") rsp = ss.union({rsp}) # FormatResponse : rspFormatted = rsp.map(lambda x : json.dumps(x) ).as_string(); rawRequest = op.Map("com.ibm.streamsx.inet.rest::HTTPRequestProcess", stream=rspFormatted, schema='tuple<int64 key, rstring request, rstring method, rstring pathInfo >', params={'port': PORT, 'webTimeout':5.0, 'responseJsonAttributeName':'string', 'context':'base', 'contextResourceBase':'opt/base' }, name = "RequestProcess") rawRequest.stream.sink(webEntryLog) ## log what we have received. # determine what to work on onlyUpper = rawRequest.stream.filter(lambda t : t["pathInfo"]=="/Upper", name="upperFilter") onlyLower = rawRequest.stream.filter(lambda t : t["pathInfo"]=="/Lower", name="lowerFilter") onlyLong = rawRequest.stream.filter(lambda t : t["pathInfo"]=="/L/O/N/G/L/O/W/E/R", name="longFilter") onlyCount = rawRequest.stream.filter(lambda t : t["pathInfo"]=="/Count", name="countFilter") # do the work upperDone = onlyUpper.transform(upperString, name="upperCase") lowerDone = onlyLower.transform(lowerString, name="lowerCase") longDone = onlyLong.transform(lowerString, name="longCase") longCount = onlyCount.transform(upperString, name="countCase") # do tests on what we have processed. self.tester.tuple_count(lowerDone, 1) self.tester.tuple_count(upperDone, 1) self.tester.tuple_count(longDone, 1) self.tester.tuple_count(longCount, 10) # union unionResult = upperDone.union({lowerDone, longDone, longCount}) # loopback to sending pending_source.complete(unionResult) # loopback ## All done building the graph...... # setup the code that will invoke this test. self.tester.local_check = self.pathInfo_request # submit the application for test self.tester.test(self.test_ctxtype, self.test_config)
def test_reflect(self): topo = Topology("TupleReflect") self.tester = Tester(topo) tk.add_toolkit(topo, inetToolkit) # Looping pending_source = PendingStream(topo) # Build the onramp. rsp = pending_source.stream ss = topo.source([], name="onRamp") rsp = ss.union({rsp}) # FormatResponse : rspFormatted = rsp.map(lambda x : json.dumps(x) ).as_string(); # Convert to 'tuple' that is native to Streams, stripping of everything: key, response. toTuple = op.Map("com.ibm.streamsx.json::JSONToTuple", stream=rspFormatted, schema='tuple<int64 key, rstring response>', params = {'ignoreParsingError':True}, name = "JSONToTuple") # Get to the Streams. rspFormatted = toTuple.stream # Output : return to web, # Input : get from web rawRequest = op.Map("com.ibm.streamsx.inet.rest::HTTPRequestProcess", stream = rspFormatted, #schema ='tuple<int64 key, rstring request, rstring method, rstring pathInfo >', schema='tuple<int64 key, rstring request, rstring contentType, map<rstring, rstring> header, rstring response, rstring method,rstring pathInfo, int32 status, rstring statusMessage>', params={'port': PORT, 'webTimeout':5.0, 'contextResourceBase':'opt/base', 'context':CONTEXTBASE}, name = "HttpRequestProcess") rawRequest.stream.sink(webEntryLog) ## log what we have received. # determine what to work on onlyTuple = rawRequest.stream.filter(lambda t : t["pathInfo"]=="/Tuple", name="inputFilter") # do the work reflected = onlyTuple.transform(reflect2Result("REF:"), name="refectStream") # self.tester.tuple_count(reflected, 3) # loopback to sending pending_source.complete(reflected) # loopback ## All done building the graph...... ## Set parameters self.test_config['topology.keepArtifacts']=True # setup the code that will invoke this test. self.tester.local_check = self.reflect_client # enable tracing info. job_config = streamsx.topology.context.JobConfig(job_name='Tuple', tracing="trace") job_config.add(self.test_config) # submit the application for test self.tester.test(self.test_ctxtype, self.test_config)
def reflectPost(self, expected_requests, local_check_function ): """ Reflect back after a set number of seconds. """ topo = Topology("Reflect") self.tester = Tester(topo) tk.add_toolkit(topo, inetToolkit) pending_source = PendingStream(topo) rsp = pending_source.stream.map(lambda t : t) ss = topo.source([], name="onRamp") rsp = ss.union({rsp}) # FormatResponse : rspFormatted = rsp.map(lambda x : json.dumps(x) ).as_string(); rawRequest = op.Map("com.ibm.streamsx.inet.rest::HTTPRequestProcess", stream=rspFormatted, schema='tuple<int64 key, rstring request, rstring contentType, map<rstring, rstring> header, rstring response, rstring method,rstring pathInfo, int32 status, rstring statusMessage>', params={'port': PORT, 'webTimeout':15.0, 'responseJsonAttributeName':'string', 'context':'Reflect', 'contextResourceBase': 'opt/Reflect'}, name = "RequestProcess") rawRequest.stream.sink(webEntryLog) ## log what we have received. onlyFirst = rawRequest.stream.filter(lambda t : t["pathInfo"]=="/first", name="firstFilter") onlySecond = rawRequest.stream.filter(lambda t : t["pathInfo"]=="/second", name="secondFilter") onlyThird = rawRequest.stream.filter(lambda t : t["pathInfo"]=="/third", name="thirdFilter") onlyFourth = rawRequest.stream.filter(lambda t : t["pathInfo"]=="/fourth", name="fourthFilter") # do the work firstReflect = onlyFirst.transform(sleepResponse("SLP1:"), name="firstResponse") secondReflect = onlySecond.transform(sleepResponse("SLP2:"), name="secondResponse") thirdReflect = onlyThird.transform(sleepResponse("SLP3:"), name="thirdResponse") fourthReflect = onlyFourth.transform(sleepResponse("SLP4:"), name="fourthResponse") allReflect = firstReflect.union({secondReflect,thirdReflect,fourthReflect}) allReflect.sink(webExitLog) ## log what we are sending back # do tests on what we have processed. self.tester.tuple_count(allReflect, expected_requests) # loopback to sending pending_source.complete(allReflect) # loopback ## All done building the graph...... # setup the code that will invoke this test. self.tester.local_check = local_check_function # enable tracing info. job_config = streamsx.topology.context.JobConfig(job_name='Async', tracing="info") job_config.add(self.test_config) # submit the application for test self.tester.test(self.test_ctxtype, self.test_config) """