def execute(self):
# copy the build(Release folder) and config yaml to output
fileutils.mkdir(self.output + '/Release')
with socketProcessMutex:
self.copytree(TEST_SUBJECT_DIR + '/Release',
self.output + '/Release')
filecopy.filecopy(self.input + '/sample.yaml',
self.output + '/sample.yaml')
# create the correlator helper and start the correlator and an
# engine receive listening on the Echo Channel
correlator = CorrelatorHelper(self, name='mycorrelator')
correlator.start(logfile='mycorrelator.log',
inputLog='mycorrelator.input.log',
config=[self.output + '/sample.yaml'])
correlator.injectEPL(
['ConnectivityPluginsControl.mon', 'ConnectivityPlugins.mon'],
filedir=PROJECT.APAMA_HOME + '/monitors')
correlator.receive(filename='receive.evt', channels=['EchoChannel'])
# inject the simple monitor into the correlator
correlator.injectEPL(filenames=[self.input + '/DemoApp.mon'])
#self.wait(3)
# wait for receipt msg towards transport
# we could use correlator.flush() here instead
self.waitForSignal('mycorrelator.log',
expr="Towards Host:",
condition="==8")
def execute(self):
# since we can't specify locations other than APAMA_WORK/HOME and cwd,
# have to copy input file to output dir
import shutil
for f in ['input.txt']:
shutil.copyfile(self.input + '/' + f, self.output + '/' + f)
# Can specify which port the correlator runs on using '-X CORR_PORT=15903' on the
# PySys command line else it will be randomly allocated
correlator = CorrelatorHelper(
self,
name='testcorrelator',
port=self.CORR_PORT if hasattr(self, 'CORR_PORT') else None)
correlator.start(arguments=[
'--connectivityConfig', self.input + '/connectivity.yaml'
])
correlator.receive('received.evt', channels=['received'])
correlator.injectMonitorscript(
self.project.APAMA_HOME +
'/monitors/ConnectivityPluginsControl.mon')
correlator.injectMonitorscript(self.project.APAMA_HOME +
'/monitors/ConnectivityPlugins.mon')
correlator.injectMonitorscript('Test.mon')
# wait until the output is on disk
self.waitForGrep('received.evt', expr="final message")
self.waitForGrep('output.txt', expr="final message")
def execute(self):
correlator = CorrelatorHelper(self, name='correlator')
correlator.start(logfile='correlator.log',
config=os.path.join(PROJECT.TEST_SUBJECT_DIR,
'initialization.yaml'))
correlator.flush()
correlator.injectEPL(filenames='testUtils.mon',
filedir=PROJECT.UTILS_DIR)
# Start test results receiver
correlator.receive(filename='TestResult.evt',
channels=['TestResult'],
logChannels=True)
# Inject test
correlator.injectEPL(filenames=['test.mon'])
# wait for all events to be processed
correlator.flush()
# wait for test to complete
self.waitForSignal('TestResult.evt',
expr="TestComplete",
condition="==1",
timeout=10)
def execute(self):
correlator = CorrelatorHelper(self, name='correlator')
correlator.start(logfile='correlator.log', config=os.path.join(PROJECT.TEST_SUBJECT_DIR, 'initialization.yaml'))
correlator.flush()
correlator.injectEPL(filenames='testUtils.mon', filedir=PROJECT.UTILS_DIR)
# Start test results receiver
correlator.receive(filename='TestResult.evt', channels=['TestResult'], logChannels=True)
# Set the log level to DEBUG so that we can see when the listeners are killed
correlator.setApplicationLogLevel(verbosity='DEBUG')
# Inject test
correlator.injectEPL(filenames=['test.mon'])
# wait for all events to be processed
correlator.flush()
# wait for test to complete
self.waitForSignal('TestResult.evt', expr="TestComplete", condition="==1", timeout=10)
# Output the engine_inspect result to a file to check for any remaining subscribed channels
correlator.inspect(filename='preTerminateInspect.txt', arguments=['-x'])
correlator.sendEventStrings('utils.KeepAliveUntilTerminated()')
# Output the engine_inspect result to a file to check for non-terminated listeners
correlator.inspect(filename='postTerminateInspect.txt', raw=True, arguments=['-x'])
def startAnalyticsBuilderCorrelator(self, blockSourceDir=None, Xclock=True, numWorkers=4, injectBlocks = True, **kwargs):
"""
Start a correlator with the EPL for Analytics Builder loaded.
:param blockSourceDir: A location of blocks to include, or a list of locations
:param Xclock: Externally clock correlator (on by default).
:param numWorkers: Number of workers for Analytics Builder runtime (4 by default).
:param injectBlocks: if false, don't inject the actual block EPL (use if there are dependencies), returns blockOutput directory. Also skips applicationInitialized call.
:param \**kwargs: extra kwargs are passed to startCorrelator
"""
# Build and extract the block extension:
if blockSourceDir == None: blockSourceDir = self.input
if not isinstance(blockSourceDir, list):
blockSourceDir = [blockSourceDir]
blockOutput = self.output+'/block-output-'
blockOutputDirs=[]
blockSrcOutput = self.output+'/block-src-'
for blockDir in blockSourceDir:
blockOutput=blockSrcOutput + os.path.basename(blockDir)
self.buildExtensionDirectory(blockDir, blockOutput)
blockOutputDirs.append(Path(blockOutput))
# Start the correlator:
corr = CorrelatorHelper(self)
arguments=kwargs.get('arguments', [])
arguments.append(f'-DanalyticsBuilder.numWorkerThreads={numWorkers}')
arguments.append(f'-DanalyticsBuilder.timedelay_secs=0.1')
kwargs['arguments']=arguments
logfile=kwargs.get('logfile', 'correlator.log')
kwargs['logfile']=logfile
corr.start(Xclock=Xclock, **kwargs)
corr.logfile = logfile
corr.injectEPL([self.project.APAMA_HOME+'/monitors/'+i+'.mon' for i in ['ScenarioService', 'data_storage/MemoryStore', 'JSONPlugin', 'AnyExtractor', 'ManagementImpl', 'Management', 'ConnectivityPluginsControl', 'ConnectivityPlugins', 'HTTPClientEvents', 'AutomaticOnApplicationInitialized']])
corr.injectCDP(self.project.ANALYTICS_BUILDER_SDK+'/block-api/framework/analyticsbuilder-framework.cdp')
self.injectCumulocityEvents(corr)
corr.injectCDP(self.project.ANALYTICS_BUILDER_SDK + '/block-api/framework/cumulocity-inventoryLookup-events.cdp')
corr.injectEPL(self.project.ANALYTICS_BUILDER_SDK+'/testframework/resources/TestHelpers.mon')
for blockOutput in blockOutputDirs:
corr.send(sorted(list(blockOutput.rglob('*.evt'))))
self.analyticsBuilderCorrelator = corr
corr.receive('output.evt', channels=['TestOutput'])
corr.injectTestEventLogger(channels=['TestOutput'])
if not injectBlocks:
return blockOutputDirs
self.preInjectBlock(corr)
# inject block files:
for blockOutput in blockOutputDirs:
corr.injectEPL(sorted(list(blockOutput.rglob('*.mon'))))
corr.send(sorted(list(blockOutput.rglob('*.evt'))))
# now done
corr.sendEventStrings('com.apama.connectivity.ApplicationInitialized()')
corr.flush(count=10)
return corr
def execute(self):
# create the correlator helper, start the correlator and attach an
# engine_receive process listening to a test channel. The helper will
# automatically get an available port that will be used for all
# operations against it
correlator = CorrelatorHelper(self, name='testcorrelator')
self.project.APAMA_WORK = self.descriptor.testDir + '/../../EPLApp'
correlator.start(logfile='testcorrelator.log',
config=[
os.path.join(self.project.APAMA_WORK,
'initialization.yaml')
])
receiveProcess = correlator.receive(filename='receive.evt',
channels=['EchoChannel'],
logChannels=True)
correlator.applicationEventLogging(enable=True)
# not strictly necessary in this testcase, but a useful example of waiting
# for a log message
self.waitForSignal('testcorrelator.out',
expr="AverageReadings\(\"S001\",105\.25,825\)",
process=correlator.process,
errorExpr=[' (ERROR|FATAL) .*'])
# wait for all events to be processed
correlator.flush()
def startApplication(self):
correlator = CorrelatorHelper(self.parent, name='defaultCorrelator')
correlator.start(logfile='correlator.log')
receiver = correlator.receive(filename='receive.evt', logChannels=True)
correlator.injectEPL(filenames=['Cumulocity_EventDefinitions.mon'],
filedir=os.path.join(
self.project.APAMA_MONITORS_DIR, 'cumulocity',
'10.5'))
correlator.injectEPL(filenames=['GAPDemo.mon'],
filedir=self.project.MONITORS_DIR)
self.parent.waitForSignal(
'receive.evt',
expr="com.apama.cumulocity.FindManagedObject",
condition="==1")
self.correlator = correlator
return correlator
def execute(self):
# create the correlator helper, start the correlator and attach an
# engine_receive process listening to a test channel. The helper will
# automatically get an available port that will be used for all
# operations against it
correlator = CorrelatorHelper(self, name='testcorrelator')
correlator.start(logfile='testcorrelator.log',
config=PROJECT.TEST_SUBJECT_DIR +
'/initialization.yaml')
receiveProcess = correlator.receive(filename='receive.evt',
channels=['output'],
logChannels=True)
correlator.applicationEventLogging(enable=True)
# send in the events contained in the test.evt file (directory defaults
# to the testcase input)
correlator.send(filenames=['test.evt'])
# wait for all events to be processed
correlator.flush()
# wait until the receiver writes the expected events to disk
self.waitForSignal('receive.evt', expr="Msg", condition="==1")
def startAnalyticsBuilderCorrelator(self,
blockSourceDir=None,
Xclock=True,
numWorkers=4,
**kwargs):
"""
Start a correlator with the EPL for Analytics Builder loaded.
:param blockSourceDir: A location of blocks to include.
:param Xclock: Externally clock correlator (on by default).
:param numWorkers: Number of workers for Analytics Builder runtime (4 by default).
"""
# Build and extract the block extension:
if blockSourceDir == None: blockSourceDir = self.input
blockOutput = self.output + '/block-output.zip'
self.runAnalyticsBuilderScript([
'build', 'extension', '--input', blockSourceDir, '--output',
blockOutput
])
with zipfile.ZipFile(blockOutput, 'r') as zf:
blockOutput = Path(self.output + '/block-output/')
os.mkdir(blockOutput)
zf.extractall(blockOutput)
# Start the correlator:
corr = CorrelatorHelper(self)
arguments = kwargs.get('arguments', [])
arguments.append(f'-DanalyticsBuilder.numWorkerThreads={numWorkers}')
kwargs['arguments'] = arguments
logfile = kwargs.get('logfile', 'correlator.log')
kwargs['logfile'] = logfile
corr.start(Xclock=Xclock, **kwargs)
corr.logfile = logfile
corr.injectEPL([
self.project.APAMA_HOME + '/monitors/' + i + '.mon' for i in [
'ScenarioService', 'data_storage/MemoryStore', 'JSONPlugin',
'AnyExtractor', 'ManagementImpl', 'Management',
'ConnectivityPluginsControl', 'ConnectivityPlugins',
'HTTPClientEvents', 'AutomaticOnApplicationInitialized'
]
])
corr.injectCDP(self.project.ANALYTICS_BUILDER_SDK +
'/block-api/framework/analyticsbuilder-framework.cdp')
self.injectCumulocityEvents(corr)
corr.injectCDP(
self.project.ANALYTICS_BUILDER_SDK +
'/block-api/framework/cumulocity-inventoryLookup-events.cdp')
corr.injectEPL(self.project.ANALYTICS_BUILDER_SDK +
'/testframework/resources/TestHelpers.mon')
# inject block files:
corr.injectEPL(sorted(list(blockOutput.rglob('*.mon'))))
corr.send(sorted(list(blockOutput.rglob('*.evt'))))
# now done
corr.sendEventStrings(
'com.apama.connectivity.ApplicationInitialized()')
corr.flush(10)
self.analyticsBuilderCorrelator = corr
corr.receive('output.evt', channels=['TestOutput'])
corr.injectTestEventLogger(channels=['TestOutput'])
return corr
class CycleMonitorTest(BaseTest):
def setup(self):
'''Override BaseTest setup. '''
BaseTest.setup(self)
self.stations=[]
self.jython_classpath = os.path.join(PROJECT.JYTHON_HOME, 'jython.jar')
self.addToJythonClassPath(os.path.join(PROJECT.APAMA_HOME,'lib','ap-util.jar'))
self.addToJythonClassPath(os.path.join(PROJECT.APAMA_HOME,'lib','ap-client.jar'))
def addToJythonClassPath(self, path):
'''Path builder for for jython. '''
self.jython_classpath = '%s%s%s' % (self.jython_classpath, ENVSEPERATOR, path)
def addStation(self,id,name,lat,lng):
'''Add a station to the list of station, return the station object. '''
station = Station(id,name,lat,lng)
self.stations.append(station)
return station
def dumpStations(self, file):
'''Write the current list of stations to file in JSON. '''
with open(os.path.join(self.output, file), 'w') as fp:
json.dump([station.__dict__ for station in self.stations], fp, indent=4)
def dumpStationsAndPoll(self, file):
'''Write the current list of stations to file in JSON, and force a poll on the adapter through the correlator. '''
self.dumpStations(file)
with open(os.path.join(self.output, 'poll.mon'),'w') as fp:
fp.write('monitor Poll { action onload{ send com.jtech.source.Poll() to "CITY-BIKES"; } }')
self.correlator.injectMonitorscript('poll.mon', filedir=self.output)
def startCorrelator(self, url=None, city='London', xclock=True):
'''Start a correlator and set logging, return correlator object handle. '''
self.correlator = CorrelatorHelper(self)
self.log.info('Allocating port %s for correlator'%self.correlator.port)
arguments = []
arguments.extend(['--inputLog',os.path.join(self.output,'correlator-input.log')])
if not(url is None):
replaceDict = {'${apama.project}': PROJECT.root,
'${app.city}': city,
'${app.data.url}': url}
replace(os.path.join(PROJECT.root, 'config', 'connectors.yaml'), os.path.join(self.output, 'connectors.yaml'), replaceDict)
arguments.extend(['--connectivityConfig',os.path.join(self.output, 'connectors.yaml')])
if xclock: arguments.append('-Xclock')
self.correlator.start(logfile='correlator.log', arguments=arguments)
self.correlator.receive('correlator_input.log', channels=['com.apama.input'])
self.correlator.receive('correlator_output.log')
self.setApplicationLogFile(self.correlator, 'application.log', 'com.jtech')
self.setApplicationLogLevel(self.correlator, 'DEBUG', 'com.jtech')
def initialiseApplication(self, correlator):
'''Initialise the application. '''
parser=FileListParser(os.path.join(self.project.root, 'build-filelists.xml'), self.project.root, {'${apama.home}':PROJECT.APAMA_HOME})
self.inject(correlator, parser.getFileList('apama.scenarios'))
self.inject(correlator, parser.getFileList('apama.adbc'))
self.inject(correlator, parser.getFileList('apama.connectivity'))
self.inject(correlator, parser.getFileList('utils'))
self.inject(correlator, parser.getFileList('configuration'))
self.inject(correlator, parser.getFileList('source'))
self.inject(correlator, parser.getFileList('strategy'))
correlator.send(filenames='adbc.evt',filedir=self.output)
self.waitForSignal('correlator_output.log', expr='com.jtech.source.Start', timeout=20)
def startADBCAdapter(self, correlator, insert=None):
'''Start IAF running the ADBC adapter for sqlite, return process handle. '''
schema = os.path.join(PROJECT.root,'test','utils','scripts','schema.sql')
if not insert: insert = os.path.join(PROJECT.root,'test','utils','scripts','insert.sql')
else: insert = os.path.join(self.input, insert)
self.sqlite(schema)
self.sqlite(insert)
self.adbcAdapter = IAFHelper(self)
database_url = 'jdbc:sqlite:%s' % os.path.join(self.output, 'store.db')
replaceDict = {'${basedir}':'%s' % PROJECT.root,
'${apama.home}':'%s' % PROJECT.APAMA_HOME,
'${correlator.port}':'%d' % correlator.port,
'${adbc.database-url}': database_url.replace('\\', '\\\\'),
'${adbc.driver}': 'org.sqlite.JDBC',
'${adbc.driver-jar}':'sqlite-jdbc-3.7.2.jar',
'${adbc.username}':'',
'${adbc.password}':''}
for include in ['ADBC-static-codecs.xml', 'ADBC-static.xml', 'ADBC-application.xml', 'adbc.evt']:
replace(os.path.join(PROJECT.root, 'config', include), os.path.join(self.output, include), replaceDict)
return self.adbcAdapter.start(configdir=os.path.join(PROJECT.root, 'config'), configname='adbc.xml',
logfile='adbc_adapter.log', replace=replaceDict)
def startHTTPServer(self, dir=None):
'''Start HTTP server serving files from output directory, setting port as self.httpPort. '''
command = sys.executable
displayName = 'http-server'
if dir is None: dir=self.output
instances = self.getInstanceCount(displayName)
dstdout = os.path.join(self.output, 'http-server.out')
dstderr = os.path.join(self.output, 'http-server.err')
if instances: dstdout = '%s.%d' % (dstdout, instances)
if instances: dstderr = '%s.%d' % (dstderr, instances)
self.httpPort = allocateTCPPort()
self.log.info('Allocating port %s for HTTP server'%self.httpPort)
arguments = []
arguments.append(os.path.join(PROJECT.root,'python','com','jtech','httpserver.py'))
arguments.append('%s'%dir)
arguments.append('%d'%self.httpPort)
return self.startProcess(command, arguments, os.environ, self.output, BACKGROUND, 300, dstdout, dstderr, displayName)
def startScenarioPrinter(self, correlator, filedir=None):
if filedir is None: filedir=os.path.join(PROJECT.root,'test','utils','scripts')
self.startJython(script='printer.py', filedir=filedir, scriptArgs=['%d'%correlator.port])
def startScenarioPrinter(self, correlator, dvname='DV_StationAlert', fields=None):
if fields is None: fields=['id','ratio','type']
filedir=os.path.join(PROJECT.root,'test','utils','scripts')
self.startJython(script='printer.py', filedir=filedir, scriptArgs=['%d'%correlator.port,dvname,','.join(fields)])
def startJython(self, script, filedir=None, scriptArgs=None):
command = os.path.join(PROJECT.APAMA_COMMON_JRE, 'bin', 'java')
displayName = 'jython'
if filedir is None: filedir=self.input
# set the default stdout and stderr
instances = self.getInstanceCount(displayName)
dstdout = "%s/jython.out"%self.output
dstderr = "%s/jython.err"%self.output
if instances: dstdout = "%s.%d" % (dstdout, instances)
if instances: dstderr = "%s.%d" % (dstderr, instances)
# setup args
args=[]
args.extend(['-classpath', self.jython_classpath])
args.append('-Dpython.home=%s' % PROJECT.JYTHON_HOME)
args.append('-Dpython.path=%s' % os.path.join(PROJECT.root,'python'))
args.append('org.python.util.jython')
args.append(os.path.join(self.input, os.path.join(filedir,script)))
if scriptArgs is not None: args.extend(scriptArgs)
# run the process and return the handle
return self.startProcess(command, args, os.environ, self.output, BACKGROUND, 0, dstdout, dstderr, displayName)
def setApplicationLogFile(self, correlator, logfile, package):
'''Set the correlator application log file. '''
correlator.manage(arguments=['-r', 'setApplicationLogFile %s %s'%(os.path.join(self.output,logfile),package)])
def setApplicationLogLevel(self, correlator, level, package):
'''Set the correlator application log level. '''
correlator.manage(arguments=['-r', 'setApplicationLogLevel %s %s'%(level,package)])
def inject(self, correlator, fileList, **xargs):
'''Inject a file list. '''
for entry in fileList:
filedir,filenames=entry
correlator.injectMonitorscript(filenames=filenames, filedir=filedir)
def sqlite(self, filename):
'''Run sqlite using the input script specified. '''
command = PROJECT.SQLITE3_EXE
displayName = 'sqlite3'
instances = self.getInstanceCount(displayName)
dstdout = os.path.join(self.output, 'sqlite.out')
dstderr = os.path.join(self.output, 'sqlite.err')
if instances: dstdout = '%s.%d' % (dstdout, instances)
if instances: dstderr = '%s.%d' % (dstderr, instances)
arguments = []
arguments.append('-echo')
arguments.extend(['-init', filename])
arguments.append('store.db')
arguments.append('.quit')
self.startProcess(command, arguments, os.environ, self.output, FOREGROUND, 300, dstdout, dstderr, displayName)
def sendLiteral(self, correlator, literal):
with open(os.path.join(self.output,'sendliteral.evt'),'w') as file:
file.write(literal)
correlator.send(os.path.join(self.output,'sendliteral.evt'))