def execute(self): # copy the build(Release folder) and config yaml to output fileutils.mkdir(self.output + '/Release') with socketProcessMutex: self.copytree(TEST_SUBJECT_DIR + '/Release', self.output + '/Release') filecopy.filecopy(self.input + '/sample.yaml', self.output + '/sample.yaml') # create the correlator helper and start the correlator and an # engine receive listening on the Echo Channel correlator = CorrelatorHelper(self, name='mycorrelator') correlator.start(logfile='mycorrelator.log', inputLog='mycorrelator.input.log', config=[self.output + '/sample.yaml']) correlator.injectEPL( ['ConnectivityPluginsControl.mon', 'ConnectivityPlugins.mon'], filedir=PROJECT.APAMA_HOME + '/monitors') correlator.receive(filename='receive.evt', channels=['EchoChannel']) # inject the simple monitor into the correlator correlator.injectEPL(filenames=[self.input + '/DemoApp.mon']) #self.wait(3) # wait for receipt msg towards transport # we could use correlator.flush() here instead self.waitForSignal('mycorrelator.log', expr="Towards Host:", condition="==8")
def execute(self): # since we can't specify locations other than APAMA_WORK/HOME and cwd, # have to copy input file to output dir import shutil for f in ['input.txt']: shutil.copyfile(self.input + '/' + f, self.output + '/' + f) # Can specify which port the correlator runs on using '-X CORR_PORT=15903' on the # PySys command line else it will be randomly allocated correlator = CorrelatorHelper( self, name='testcorrelator', port=self.CORR_PORT if hasattr(self, 'CORR_PORT') else None) correlator.start(arguments=[ '--connectivityConfig', self.input + '/connectivity.yaml' ]) correlator.receive('received.evt', channels=['received']) correlator.injectMonitorscript( self.project.APAMA_HOME + '/monitors/ConnectivityPluginsControl.mon') correlator.injectMonitorscript(self.project.APAMA_HOME + '/monitors/ConnectivityPlugins.mon') correlator.injectMonitorscript('Test.mon') # wait until the output is on disk self.waitForGrep('received.evt', expr="final message") self.waitForGrep('output.txt', expr="final message")
def execute(self): correlator = CorrelatorHelper(self, name='correlator') correlator.start(logfile='correlator.log', config=os.path.join(PROJECT.TEST_SUBJECT_DIR, 'initialization.yaml')) correlator.flush() correlator.injectEPL(filenames='testUtils.mon', filedir=PROJECT.UTILS_DIR) # Start test results receiver correlator.receive(filename='TestResult.evt', channels=['TestResult'], logChannels=True) # Inject test correlator.injectEPL(filenames=['test.mon']) # wait for all events to be processed correlator.flush() # wait for test to complete self.waitForSignal('TestResult.evt', expr="TestComplete", condition="==1", timeout=10)
def execute(self): correlator = CorrelatorHelper(self, name='correlator') correlator.start(logfile='correlator.log', config=os.path.join(PROJECT.TEST_SUBJECT_DIR, 'initialization.yaml')) correlator.flush() correlator.injectEPL(filenames='testUtils.mon', filedir=PROJECT.UTILS_DIR) # Start test results receiver correlator.receive(filename='TestResult.evt', channels=['TestResult'], logChannels=True) # Set the log level to DEBUG so that we can see when the listeners are killed correlator.setApplicationLogLevel(verbosity='DEBUG') # Inject test correlator.injectEPL(filenames=['test.mon']) # wait for all events to be processed correlator.flush() # wait for test to complete self.waitForSignal('TestResult.evt', expr="TestComplete", condition="==1", timeout=10) # Output the engine_inspect result to a file to check for any remaining subscribed channels correlator.inspect(filename='preTerminateInspect.txt', arguments=['-x']) correlator.sendEventStrings('utils.KeepAliveUntilTerminated()') # Output the engine_inspect result to a file to check for non-terminated listeners correlator.inspect(filename='postTerminateInspect.txt', raw=True, arguments=['-x'])
def startAnalyticsBuilderCorrelator(self, blockSourceDir=None, Xclock=True, numWorkers=4, injectBlocks = True, **kwargs): """ Start a correlator with the EPL for Analytics Builder loaded. :param blockSourceDir: A location of blocks to include, or a list of locations :param Xclock: Externally clock correlator (on by default). :param numWorkers: Number of workers for Analytics Builder runtime (4 by default). :param injectBlocks: if false, don't inject the actual block EPL (use if there are dependencies), returns blockOutput directory. Also skips applicationInitialized call. :param \**kwargs: extra kwargs are passed to startCorrelator """ # Build and extract the block extension: if blockSourceDir == None: blockSourceDir = self.input if not isinstance(blockSourceDir, list): blockSourceDir = [blockSourceDir] blockOutput = self.output+'/block-output-' blockOutputDirs=[] blockSrcOutput = self.output+'/block-src-' for blockDir in blockSourceDir: blockOutput=blockSrcOutput + os.path.basename(blockDir) self.buildExtensionDirectory(blockDir, blockOutput) blockOutputDirs.append(Path(blockOutput)) # Start the correlator: corr = CorrelatorHelper(self) arguments=kwargs.get('arguments', []) arguments.append(f'-DanalyticsBuilder.numWorkerThreads={numWorkers}') arguments.append(f'-DanalyticsBuilder.timedelay_secs=0.1') kwargs['arguments']=arguments logfile=kwargs.get('logfile', 'correlator.log') kwargs['logfile']=logfile corr.start(Xclock=Xclock, **kwargs) corr.logfile = logfile corr.injectEPL([self.project.APAMA_HOME+'/monitors/'+i+'.mon' for i in ['ScenarioService', 'data_storage/MemoryStore', 'JSONPlugin', 'AnyExtractor', 'ManagementImpl', 'Management', 'ConnectivityPluginsControl', 'ConnectivityPlugins', 'HTTPClientEvents', 'AutomaticOnApplicationInitialized']]) corr.injectCDP(self.project.ANALYTICS_BUILDER_SDK+'/block-api/framework/analyticsbuilder-framework.cdp') self.injectCumulocityEvents(corr) corr.injectCDP(self.project.ANALYTICS_BUILDER_SDK + '/block-api/framework/cumulocity-inventoryLookup-events.cdp') corr.injectEPL(self.project.ANALYTICS_BUILDER_SDK+'/testframework/resources/TestHelpers.mon') for blockOutput in blockOutputDirs: corr.send(sorted(list(blockOutput.rglob('*.evt')))) self.analyticsBuilderCorrelator = corr corr.receive('output.evt', channels=['TestOutput']) corr.injectTestEventLogger(channels=['TestOutput']) if not injectBlocks: return blockOutputDirs self.preInjectBlock(corr) # inject block files: for blockOutput in blockOutputDirs: corr.injectEPL(sorted(list(blockOutput.rglob('*.mon')))) corr.send(sorted(list(blockOutput.rglob('*.evt')))) # now done corr.sendEventStrings('com.apama.connectivity.ApplicationInitialized()') corr.flush(count=10) return corr
def execute(self): # create the correlator helper, start the correlator and attach an # engine_receive process listening to a test channel. The helper will # automatically get an available port that will be used for all # operations against it correlator = CorrelatorHelper(self, name='testcorrelator') self.project.APAMA_WORK = self.descriptor.testDir + '/../../EPLApp' correlator.start(logfile='testcorrelator.log', config=[ os.path.join(self.project.APAMA_WORK, 'initialization.yaml') ]) receiveProcess = correlator.receive(filename='receive.evt', channels=['EchoChannel'], logChannels=True) correlator.applicationEventLogging(enable=True) # not strictly necessary in this testcase, but a useful example of waiting # for a log message self.waitForSignal('testcorrelator.out', expr="AverageReadings\(\"S001\",105\.25,825\)", process=correlator.process, errorExpr=[' (ERROR|FATAL) .*']) # wait for all events to be processed correlator.flush()
def startApplication(self): correlator = CorrelatorHelper(self.parent, name='defaultCorrelator') correlator.start(logfile='correlator.log') receiver = correlator.receive(filename='receive.evt', logChannels=True) correlator.injectEPL(filenames=['Cumulocity_EventDefinitions.mon'], filedir=os.path.join( self.project.APAMA_MONITORS_DIR, 'cumulocity', '10.5')) correlator.injectEPL(filenames=['GAPDemo.mon'], filedir=self.project.MONITORS_DIR) self.parent.waitForSignal( 'receive.evt', expr="com.apama.cumulocity.FindManagedObject", condition="==1") self.correlator = correlator return correlator
def execute(self): # create the correlator helper, start the correlator and attach an # engine_receive process listening to a test channel. The helper will # automatically get an available port that will be used for all # operations against it correlator = CorrelatorHelper(self, name='testcorrelator') correlator.start(logfile='testcorrelator.log', config=PROJECT.TEST_SUBJECT_DIR + '/initialization.yaml') receiveProcess = correlator.receive(filename='receive.evt', channels=['output'], logChannels=True) correlator.applicationEventLogging(enable=True) # send in the events contained in the test.evt file (directory defaults # to the testcase input) correlator.send(filenames=['test.evt']) # wait for all events to be processed correlator.flush() # wait until the receiver writes the expected events to disk self.waitForSignal('receive.evt', expr="Msg", condition="==1")
def startAnalyticsBuilderCorrelator(self, blockSourceDir=None, Xclock=True, numWorkers=4, **kwargs): """ Start a correlator with the EPL for Analytics Builder loaded. :param blockSourceDir: A location of blocks to include. :param Xclock: Externally clock correlator (on by default). :param numWorkers: Number of workers for Analytics Builder runtime (4 by default). """ # Build and extract the block extension: if blockSourceDir == None: blockSourceDir = self.input blockOutput = self.output + '/block-output.zip' self.runAnalyticsBuilderScript([ 'build', 'extension', '--input', blockSourceDir, '--output', blockOutput ]) with zipfile.ZipFile(blockOutput, 'r') as zf: blockOutput = Path(self.output + '/block-output/') os.mkdir(blockOutput) zf.extractall(blockOutput) # Start the correlator: corr = CorrelatorHelper(self) arguments = kwargs.get('arguments', []) arguments.append(f'-DanalyticsBuilder.numWorkerThreads={numWorkers}') kwargs['arguments'] = arguments logfile = kwargs.get('logfile', 'correlator.log') kwargs['logfile'] = logfile corr.start(Xclock=Xclock, **kwargs) corr.logfile = logfile corr.injectEPL([ self.project.APAMA_HOME + '/monitors/' + i + '.mon' for i in [ 'ScenarioService', 'data_storage/MemoryStore', 'JSONPlugin', 'AnyExtractor', 'ManagementImpl', 'Management', 'ConnectivityPluginsControl', 'ConnectivityPlugins', 'HTTPClientEvents', 'AutomaticOnApplicationInitialized' ] ]) corr.injectCDP(self.project.ANALYTICS_BUILDER_SDK + '/block-api/framework/analyticsbuilder-framework.cdp') self.injectCumulocityEvents(corr) corr.injectCDP( self.project.ANALYTICS_BUILDER_SDK + '/block-api/framework/cumulocity-inventoryLookup-events.cdp') corr.injectEPL(self.project.ANALYTICS_BUILDER_SDK + '/testframework/resources/TestHelpers.mon') # inject block files: corr.injectEPL(sorted(list(blockOutput.rglob('*.mon')))) corr.send(sorted(list(blockOutput.rglob('*.evt')))) # now done corr.sendEventStrings( 'com.apama.connectivity.ApplicationInitialized()') corr.flush(10) self.analyticsBuilderCorrelator = corr corr.receive('output.evt', channels=['TestOutput']) corr.injectTestEventLogger(channels=['TestOutput']) return corr
class CycleMonitorTest(BaseTest): def setup(self): '''Override BaseTest setup. ''' BaseTest.setup(self) self.stations=[] self.jython_classpath = os.path.join(PROJECT.JYTHON_HOME, 'jython.jar') self.addToJythonClassPath(os.path.join(PROJECT.APAMA_HOME,'lib','ap-util.jar')) self.addToJythonClassPath(os.path.join(PROJECT.APAMA_HOME,'lib','ap-client.jar')) def addToJythonClassPath(self, path): '''Path builder for for jython. ''' self.jython_classpath = '%s%s%s' % (self.jython_classpath, ENVSEPERATOR, path) def addStation(self,id,name,lat,lng): '''Add a station to the list of station, return the station object. ''' station = Station(id,name,lat,lng) self.stations.append(station) return station def dumpStations(self, file): '''Write the current list of stations to file in JSON. ''' with open(os.path.join(self.output, file), 'w') as fp: json.dump([station.__dict__ for station in self.stations], fp, indent=4) def dumpStationsAndPoll(self, file): '''Write the current list of stations to file in JSON, and force a poll on the adapter through the correlator. ''' self.dumpStations(file) with open(os.path.join(self.output, 'poll.mon'),'w') as fp: fp.write('monitor Poll { action onload{ send com.jtech.source.Poll() to "CITY-BIKES"; } }') self.correlator.injectMonitorscript('poll.mon', filedir=self.output) def startCorrelator(self, url=None, city='London', xclock=True): '''Start a correlator and set logging, return correlator object handle. ''' self.correlator = CorrelatorHelper(self) self.log.info('Allocating port %s for correlator'%self.correlator.port) arguments = [] arguments.extend(['--inputLog',os.path.join(self.output,'correlator-input.log')]) if not(url is None): replaceDict = {'${apama.project}': PROJECT.root, '${app.city}': city, '${app.data.url}': url} replace(os.path.join(PROJECT.root, 'config', 'connectors.yaml'), os.path.join(self.output, 'connectors.yaml'), replaceDict) arguments.extend(['--connectivityConfig',os.path.join(self.output, 'connectors.yaml')]) if xclock: arguments.append('-Xclock') self.correlator.start(logfile='correlator.log', arguments=arguments) self.correlator.receive('correlator_input.log', channels=['com.apama.input']) self.correlator.receive('correlator_output.log') self.setApplicationLogFile(self.correlator, 'application.log', 'com.jtech') self.setApplicationLogLevel(self.correlator, 'DEBUG', 'com.jtech') def initialiseApplication(self, correlator): '''Initialise the application. ''' parser=FileListParser(os.path.join(self.project.root, 'build-filelists.xml'), self.project.root, {'${apama.home}':PROJECT.APAMA_HOME}) self.inject(correlator, parser.getFileList('apama.scenarios')) self.inject(correlator, parser.getFileList('apama.adbc')) self.inject(correlator, parser.getFileList('apama.connectivity')) self.inject(correlator, parser.getFileList('utils')) self.inject(correlator, parser.getFileList('configuration')) self.inject(correlator, parser.getFileList('source')) self.inject(correlator, parser.getFileList('strategy')) correlator.send(filenames='adbc.evt',filedir=self.output) self.waitForSignal('correlator_output.log', expr='com.jtech.source.Start', timeout=20) def startADBCAdapter(self, correlator, insert=None): '''Start IAF running the ADBC adapter for sqlite, return process handle. ''' schema = os.path.join(PROJECT.root,'test','utils','scripts','schema.sql') if not insert: insert = os.path.join(PROJECT.root,'test','utils','scripts','insert.sql') else: insert = os.path.join(self.input, insert) self.sqlite(schema) self.sqlite(insert) self.adbcAdapter = IAFHelper(self) database_url = 'jdbc:sqlite:%s' % os.path.join(self.output, 'store.db') replaceDict = {'${basedir}':'%s' % PROJECT.root, '${apama.home}':'%s' % PROJECT.APAMA_HOME, '${correlator.port}':'%d' % correlator.port, '${adbc.database-url}': database_url.replace('\\', '\\\\'), '${adbc.driver}': 'org.sqlite.JDBC', '${adbc.driver-jar}':'sqlite-jdbc-3.7.2.jar', '${adbc.username}':'', '${adbc.password}':''} for include in ['ADBC-static-codecs.xml', 'ADBC-static.xml', 'ADBC-application.xml', 'adbc.evt']: replace(os.path.join(PROJECT.root, 'config', include), os.path.join(self.output, include), replaceDict) return self.adbcAdapter.start(configdir=os.path.join(PROJECT.root, 'config'), configname='adbc.xml', logfile='adbc_adapter.log', replace=replaceDict) def startHTTPServer(self, dir=None): '''Start HTTP server serving files from output directory, setting port as self.httpPort. ''' command = sys.executable displayName = 'http-server' if dir is None: dir=self.output instances = self.getInstanceCount(displayName) dstdout = os.path.join(self.output, 'http-server.out') dstderr = os.path.join(self.output, 'http-server.err') if instances: dstdout = '%s.%d' % (dstdout, instances) if instances: dstderr = '%s.%d' % (dstderr, instances) self.httpPort = allocateTCPPort() self.log.info('Allocating port %s for HTTP server'%self.httpPort) arguments = [] arguments.append(os.path.join(PROJECT.root,'python','com','jtech','httpserver.py')) arguments.append('%s'%dir) arguments.append('%d'%self.httpPort) return self.startProcess(command, arguments, os.environ, self.output, BACKGROUND, 300, dstdout, dstderr, displayName) def startScenarioPrinter(self, correlator, filedir=None): if filedir is None: filedir=os.path.join(PROJECT.root,'test','utils','scripts') self.startJython(script='printer.py', filedir=filedir, scriptArgs=['%d'%correlator.port]) def startScenarioPrinter(self, correlator, dvname='DV_StationAlert', fields=None): if fields is None: fields=['id','ratio','type'] filedir=os.path.join(PROJECT.root,'test','utils','scripts') self.startJython(script='printer.py', filedir=filedir, scriptArgs=['%d'%correlator.port,dvname,','.join(fields)]) def startJython(self, script, filedir=None, scriptArgs=None): command = os.path.join(PROJECT.APAMA_COMMON_JRE, 'bin', 'java') displayName = 'jython' if filedir is None: filedir=self.input # set the default stdout and stderr instances = self.getInstanceCount(displayName) dstdout = "%s/jython.out"%self.output dstderr = "%s/jython.err"%self.output if instances: dstdout = "%s.%d" % (dstdout, instances) if instances: dstderr = "%s.%d" % (dstderr, instances) # setup args args=[] args.extend(['-classpath', self.jython_classpath]) args.append('-Dpython.home=%s' % PROJECT.JYTHON_HOME) args.append('-Dpython.path=%s' % os.path.join(PROJECT.root,'python')) args.append('org.python.util.jython') args.append(os.path.join(self.input, os.path.join(filedir,script))) if scriptArgs is not None: args.extend(scriptArgs) # run the process and return the handle return self.startProcess(command, args, os.environ, self.output, BACKGROUND, 0, dstdout, dstderr, displayName) def setApplicationLogFile(self, correlator, logfile, package): '''Set the correlator application log file. ''' correlator.manage(arguments=['-r', 'setApplicationLogFile %s %s'%(os.path.join(self.output,logfile),package)]) def setApplicationLogLevel(self, correlator, level, package): '''Set the correlator application log level. ''' correlator.manage(arguments=['-r', 'setApplicationLogLevel %s %s'%(level,package)]) def inject(self, correlator, fileList, **xargs): '''Inject a file list. ''' for entry in fileList: filedir,filenames=entry correlator.injectMonitorscript(filenames=filenames, filedir=filedir) def sqlite(self, filename): '''Run sqlite using the input script specified. ''' command = PROJECT.SQLITE3_EXE displayName = 'sqlite3' instances = self.getInstanceCount(displayName) dstdout = os.path.join(self.output, 'sqlite.out') dstderr = os.path.join(self.output, 'sqlite.err') if instances: dstdout = '%s.%d' % (dstdout, instances) if instances: dstderr = '%s.%d' % (dstderr, instances) arguments = [] arguments.append('-echo') arguments.extend(['-init', filename]) arguments.append('store.db') arguments.append('.quit') self.startProcess(command, arguments, os.environ, self.output, FOREGROUND, 300, dstdout, dstderr, displayName) def sendLiteral(self, correlator, literal): with open(os.path.join(self.output,'sendliteral.evt'),'w') as file: file.write(literal) correlator.send(os.path.join(self.output,'sendliteral.evt'))