def run():
SimmTestResult = True
RunTimetoTest = 180
# commented values can be used as PASS values. Set to 0 for full test.
regAppAck_passfail = 3 # 3
regDataAck_passfail = 36 # 36
sysInit_passfail = 2 # 2
subscribe_passfail = 76 # 76
sysLogFollow = log.logfollower()
cntPassFail = 0
#testLoop = 0
#BOOT PROCESSES
# start the FPGA sim
simm_utils.startFPGAsimult()
while SimmTestResult != False:
time.sleep(1)
print("TEST STARTED!")
print("PassFail Attempts: ", cntPassFail)
#print('Test Attempt: ', testLoop)
#testLoop = testLoop + 1
sysLogFollow.start()
#if False == KeepMoving
# close TCP and UDP connections? so far, it's working without doing so ...
KeepMoving = True
if KeepMoving != False:
#UDP SETUP
UDPsock = simm_utils.UDPsetup()
sysLog = sysLogFollow.read()
sysLogFollow.start()
launch_errors = sum(1 for d in sysLog if "ERROR!" in d.get('message'))
if 0 < launch_errors:
print("UDP SETUP errors: ", launch_errors)
KeepMoving = False
if KeepMoving != False:
# TCP SETUP
TCPconn, sVal = simm_utils.TCPsetup()
sysLog = sysLogFollow.read()
sysLogFollow.start()
launch_errors = sum(1 for d in sysLog if "ERROR!" in d.get('message'))
if 0 < launch_errors:
print("TCP SETUP errors: ", launch_errors)
KeepMoving = False
if KeepMoving != False:
# REGISTER APP
#time.sleep(1)
simm_utils.registerApp(TCPconn, sVal)
print("REGISTER APP: FINISHED RECEIVING")
#time.sleep(1)
sysLog = sysLogFollow.read()
sysLogFollow.start()
launch_errors = sum(1 for d in sysLog if "ERROR!" in d.get('message'))
if 0 < launch_errors:
sVal.close()
print("REGISTER APP errors: ", launch_errors)
KeepMoving = False
if KeepMoving != False:
# REGISTER APP ACK
print("REGISTER APP ACK ATTEMPT: ", regAppAck_passfail)
simm_utils.registerAppAck(TCPconn, regAppAck_passfail)
time.sleep(1)
sysLog = sysLogFollow.read()
sysLogFollow.start()
launch_errors = sum(1 for d in sysLog if "ERROR!" in d.get('message'))
if 0 < launch_errors:
sVal.close()
print("REGISTER APP ACK errors: ", launch_errors)
KeepMoving = False
regAppAck_passfail = regAppAck_passfail + 1
#time.sleep(1)
if KeepMoving != False:
# REGISTER DATA
simm_utils.registerData(TCPconn)
#time.sleep(1)
sysLog = sysLogFollow.read()
sysLogFollow.start()
launch_errors = sum(1 for d in sysLog if "ERROR!" in d.get('message'))
if 0 < launch_errors:
sVal.close()
print("REGISTER DATA errors: ", launch_errors)
KeepMoving = False
if KeepMoving != False:
# REGISTER DATA ACK
print("REGISTER DATA ACK ATTEMPT: ", regDataAck_passfail)
simm_utils.registerDataAck(TCPconn, regDataAck_passfail)
time.sleep(1)
sysLog = sysLogFollow.read()
sysLogFollow.start()
launch_errors = sum(1 for d in sysLog if "ERROR!" in d.get('message'))
if 0 < launch_errors:
sVal.close()
print("REGISTER DATA ACK errors:",launch_errors)
KeepMoving = False
regDataAck_passfail = regDataAck_passfail + 1
#time.sleep(1)
if KeepMoving != False:
# UDP OPEN
# If I don't put a large delay in my .c, this fails. If the .c stuff executes before this starts, it fails.
print("UDP OPEN")
simm_utils.udpOpen(UDPsock)
time.sleep(1)
sysLog = sysLogFollow.read()
sysLogFollow.start()
launch_errors = sum(1 for d in sysLog if "ERROR!" in d.get('message'))
if 0 < launch_errors:
sVal.close()
print("UDP OPEN errors: ", launch_errors)
KeepMoving = False
if KeepMoving != False:
# SYS INIT
print("SYS INIT ATTEMPT: ", sysInit_passfail)
time.sleep(1)
simm_utils.sysInit(UDPsock, sysInit_passfail)
time.sleep(1)
sysLog = sysLogFollow.read()
sysLogFollow.start()
launch_errors = sum(1 for d in sysLog if "ERROR!" in d.get('message'))
if 0 < launch_errors:
sVal.close()
print("SYS INIT errors: ", launch_errors)
KeepMoving = False
sysInit_passfail = sysInit_passfail + 1
if KeepMoving != False:
# SUBSCRIBE
print("SUBSCRIBE ATTEMPT: ", subscribe_passfail)
# cosnider sending some random parameters as well ...
if 2 == subscribe_passfail:
simm_utils.subscribe(TCPconn, 0)
elif 3 == subscribe_passfail:
simm_utils.subscribe(TCPconn, 0)
elif 4 == subscribe_passfail:
simm_utils.subscribe(TCPconn, 0)
elif 6 == subscribe_passfail:
simm_utils.subscribe(TCPconn, 0)
else:
simm_utils.subscribe(TCPconn, subscribe_passfail)
#time.sleep(1)
#sysLog = sysLogFollow.read()
#launch_errors = sum(1 for d in sysLog if "ERROR!" in d.get('message'))
#if 0 < launch_errors:
#print("SUBSCRIBE errors: ", launch_errors)
#KeepMoving = False
#if KeepMoving != False:
# SUBSCRIBE ACK
#sysLogFollow.start
simm_utils.subscribeAck(TCPconn, subscribe_passfail)
sysLog = sysLogFollow.read()
sysLogFollow.start
launch_errors = sum(1 for d in sysLog if "ERROR!" in d.get('message'))
#launch_success = sum(1 for d in sysLog if "simm_init() SUCCESS!" in d.get('message'))
if 0 < launch_errors:
sVal.close()
print("SUBSCRIBE ACK errors: ", launch_errors)
KeepMoving = False
subscribe_passfail = subscribe_passfail + 1
cntPassFail = cntPassFail + 1
# RUN-TIME PROCESSING
if KeepMoving != False:
print("Total tests peformed: ", cntPassFail)
startTime = time.time()
#time.sleep(1)
#endTime = time.time() - startTime
#print("END TIME: ", endTime)
# RUN-TIME
#cnt = 0
#time.sleep(20)
simm_utils.start_threads()
time.sleep(3)
simm_utils.subscribe(TCPconn, 100)
time.sleep(1)
simm_utils.subscribeAck(TCPconn, 100)
time.sleep(3)
simm_utils.subscribe(TCPconn, 200)
time.sleep(1)
simm_utils.subscribeAck(TCPconn, 200)
alwaysRun = True
while True == alwaysRun:
#alwaysRun = True
endTime = time.time() - startTime
sysLogFollow.start()
sysLog = sysLogFollow.read()
if 0 < launch_errors:
print("RUN-TIME errors: ", launch_errors)
alwaysRun = False
simm_utils.stop_simm()
simm_utils.stop_fpga()
return False
if endTime >= RunTimetoTest:
print("3 minutes elapsed with no errors detected. End of test.")
alwaysRun = False
simm_utils.stop_simm()
simm_utils.stop_fpga()
return True
# stop SIMM if it isn't stopped yet
simm_utils.stop_simm()
def run():
RunTimetoTest = 180
fdlTestResult = True
# commented values can be used as PASS values. Set to 0 for full test.
regAppAck_passfail = 3 # 3
regDataAck_passfail = 26 # 26
sysInit_passfail = 0 # 2
sysLogFollow = log.logfollower()
cntPassFail = 0
#BOOT PROCESSES
while fdlTestResult != False:
time.sleep(1)
print("TEST STARTED!")
print("PassFail Attempts: ", cntPassFail)
sysLogFollow.start()
#if False == KeepMoving
# close TCP and UDP connections? so far, it's working without doing so ...
KeepMoving = True
if KeepMoving != False:
#UDP SETUP
UDPsock = fdl_utils.UDPsetup()
sysLog = sysLogFollow.read()
sysLogFollow.start()
launch_errors = sum(1 for d in sysLog if "ERROR!" in d.get('message'))
if 0 < launch_errors:
print("UDP SETUP errors: ", launch_errors)
KeepMoving = False
if KeepMoving != False:
# TCP SETUP
TCPconn, sVal = fdl_utils.TCPsetup()
sysLog = sysLogFollow.read()
sysLogFollow.start()
launch_errors = sum(1 for d in sysLog if "ERROR!" in d.get('message'))
if 0 < launch_errors:
print("TCP SETUP errors: ", launch_errors)
KeepMoving = False
if KeepMoving != False:
# REGISTER APP
#time.sleep(1)
fdl_utils.registerApp(TCPconn, sVal)
print("REGISTER APP: FINISHED RECEIVING")
#time.sleep(1)
sysLog = sysLogFollow.read()
sysLogFollow.start()
launch_errors = sum(1 for d in sysLog if "ERROR!" in d.get('message'))
if 0 < launch_errors:
sVal.close()
print("REGISTER APP errors: ", launch_errors)
KeepMoving = False
if KeepMoving != False:
# REGISTER APP ACK
print("REGISTER APP ACK ATTEMPT: ", regAppAck_passfail)
fdl_utils.registerAppAck(TCPconn, regAppAck_passfail)
time.sleep(1)
sysLog = sysLogFollow.read()
sysLogFollow.start()
launch_errors = sum(1 for d in sysLog if "ERROR!" in d.get('message'))
if 0 < launch_errors:
sVal.close()
print("REGISTER APP ACK errors: ", launch_errors)
KeepMoving = False
regAppAck_passfail = regAppAck_passfail + 1
#time.sleep(1)
if KeepMoving != False:
# REGISTER DATA
fdl_utils.registerData(TCPconn)
print("READY TO GET REGISTER DATA")
#time.sleep(1)
sysLog = sysLogFollow.read()
sysLogFollow.start()
launch_errors = sum(1 for d in sysLog if "ERROR!" in d.get('message'))
if 0 < launch_errors:
sVal.close()
print("REGISTER DATA errors: ", launch_errors)
KeepMoving = False
if KeepMoving != False:
# REGISTER DATA ACK
print("REGISTER DATA ACK ATTEMPT: ", regDataAck_passfail)
fdl_utils.registerDataAck(TCPconn, regDataAck_passfail)
time.sleep(1)
sysLog = sysLogFollow.read()
sysLogFollow.start()
launch_errors = sum(1 for d in sysLog if "ERROR!" in d.get('message'))
if 0 < launch_errors:
sVal.close()
print("REGISTER DATA ACK errors:",launch_errors)
KeepMoving = False
regDataAck_passfail = regDataAck_passfail + 1
#time.sleep(1)
if KeepMoving != False:
# UDP OPEN
# If I don't put a large delay in my .c, this fails. If the .c stuff executes before this starts, it fails.
print("UDP OPEN")
fdl_utils.udpOpen(UDPsock)
time.sleep(1)
sysLog = sysLogFollow.read()
sysLogFollow.start()
launch_errors = sum(1 for d in sysLog if "ERROR!" in d.get('message'))
if 0 < launch_errors:
sVal.close()
print("UDP OPEN errors: ", launch_errors)
KeepMoving = False
if KeepMoving != False:
# SYS INIT
print("SYS INIT ATTEMPT: ", sysInit_passfail)
time.sleep(1)
fdl_utils.sysInit(UDPsock, sysInit_passfail)
time.sleep(1)
sysLog = sysLogFollow.read()
sysLogFollow.start()
launch_errors = sum(1 for d in sysLog if "ERROR!" in d.get('message'))
if 0 < launch_errors:
sVal.close()
print("SYS INIT errors: ", launch_errors)
KeepMoving = False
sysInit_passfail = sysInit_passfail + 1
if KeepMoving != False:
# GET SUBSCRIBE
print("GET SUBSCRIBE ATTEMPT")
fdl_utils.getSubscribe(TCPconn)
time.sleep(1)
sysLog = sysLogFollow.read()
sysLogFollow.start()
launch_errors = sum(1 for d in sysLog if "ERROR!" in d.get('message'))
if 0 < launch_errors:
print("SYS INIT errors: ", launch_errors)
KeepMoving = False
sysInit_passfail = sysInit_passfail + 1
if KeepMoving != False:
# SEND SUBSCRIBE ACK
print("SEND SUB ACK ATTEMPT")
fdl_utils.sendSubscribeAck(TCPconn)
time.sleep(1)
sysLog = sysLogFollow.read()
sysLogFollow.start()
launch_errors = sum(1 for d in sysLog if "ERROR!" in d.get('message'))
if 0 < launch_errors:
print("SEND SUB ACK errors:",launch_errors)
KeepMoving = False
regDataAck_passfail = regDataAck_passfail + 1
if KeepMoving != False:
# SEND PUBLISH DATA
print("SEND PUBLISH DATA")
fdl_utils.sendPublish(UDPsock)
time.sleep(1)
sysLog = sysLogFollow.read()
sysLogFollow.start()
launch_errors = sum(1 for d in sysLog if "ERROR!" in d.get('message'))
if 0 < launch_errors:
print("SEND PUBLISH DATA errors: ", launch_errors)
KeepMoving = False
sysInit_passfail = sysInit_passfail + 1
cntPassFail = cntPassFail + 1
#RUN-TIME PROCESSING
if KeepMoving != False:
print("Total tests peformed: ", cntPassFail)
print(cntPassFail, "/ 3 tests passed")
fdl_utils.stop_fdl()
return True
fdl_utils.stop_fdl()
def run():
success = True
# navigate to the end of the SYSLOG
follow = log.logfollower()
# clear out all app directories
barsm_utils.remove()
# add apps in directories
subprocess.call('gcc '+apps_loc+'/infinite_sec.c -o /opt/rc360/system/aacm', shell=True)
subprocess.call('gcc '+apps_loc+'/hundred_sec.c -o /opt/rc360/modules/GE/gemod', shell=True)
# start barsm
if success:
print('-Starting BARSM')
barsm_utils.start_barsm()
logs = follow.read('BARSM')
errors = sum(1 for d in logs if "ERROR" in d.get('message'))
if 0 < errors:
print('ERROR: SYSLOG errors while starting BARSM!')
for d in logs:
if "ERROR" in d.get('message'):
print (d.get('message'))
success = False
else:
print('--BARSM started')
# wait for AACM to start
if success:
print('-Waiting for AACM')
barsm_pid = (proc.findproc("valgrind.bin"))[0]['pid']
aacmStarted = False
while False == aacmStarted and success:
# check if AACM has started
all_child = proc.findchild(barsm_pid)
if 1 == len(all_child):
aacmStarted = True
aacm_pid = int(all_child[0]['pid'])
# check for launch errors
logs = follow.read('BARSM')
errors = sum(1 for d in logs if "ERROR" in d.get('message'))
if 0 < errors:
print('ERROR: SYSLOG errors while waiting for AACM startup!')
for d in logs:
if "ERROR" in d.get('message'):
print (d.get('message'))
success = False
if success:
print('--AACM started')
# setup TCP connection
if success:
print('-Setting up TCP connection')
TCPconn, sVal = barsm_utils.TCPsetup() # both return values are objects
logs = follow.read('BARSM')
errors = sum(1 for d in logs if "ERROR" in d.get('message'))
if 0 < errors:
print('ERROR: SYSLOG errors while initiating TCP connection!')
for d in logs:
if "ERROR" in d.get('message'):
print (d.get('message'))
success = False
else:
print('--TCP connection setup successful')
# receive BARSM to AACM init
if success:
print('-Waiting for BARSM to AACM init')
success = barsm_utils.barsmToAacmInit(TCPconn)
logs = follow.read('BARSM')
errors = sum(1 for d in logs if "ERROR" in d.get('message'))
if 0 < errors:
print('ERROR: SYSLOG errors while receiving BARSM to AACM init!')
for d in logs:
if "ERROR" in d.get('message'):
print (d.get('message'))
success = False
else:
print('--Init received')
# send BARSM to AACM init ack
if success:
print('-Sending ACK')
barsm_utils.barsmToAacmInitAck(TCPconn)
logs = follow.read('BARSM')
errors = sum(1 for d in logs if "ERROR" in d.get('message'))
if 0 < errors:
print('ERROR: SYSLOG errors while sending BARSM to AACM init ack!')
for d in logs:
if "ERROR" in d.get('message'):
print (d.get('message'))
success = False
else:
print('--ACK sent')
# wait for children to complete launching
if success:
print('-Waiting for launch completion')
children, nameList = barsm_utils.waitForLaunch(follow)
if 2 == children:
print('--Launch complete!')
else:
print('ERROR: only {} child procs (should have 2)'.format(children))
success = False
# setup UDP connection
if success:
print('-Setting up UDP connection')
UDPsock = barsm_utils.UDPsetup()
logs = follow.read('BARSM')
errors = sum(1 for d in logs if "ERROR" in d.get('message'))
if 0 < errors:
print('ERROR: SYSLOG errors while initiating UDP connection!')
for d in logs:
if "ERROR" in d.get('message'):
print (d.get('message'))
success = False
else:
print('--UDP connection setup')
# receive OPEN message
if 0:#success:
print('-Waiting to receive OPEN message from BARSM')
SenderAddr = barsm_utils.udpOpen(UDPsock)
logs = follow.read('BARSM')
errors = sum(1 for d in logs if "ERROR" in d.get('message'))
if 0 < errors:
print('ERROR: SYSLOG errors while opening UDP connection!')
for d in logs:
if "ERROR" in d.get('message'):
print (d.get('message'))
success = False
else:
print('--OPEN received')
# send sysInit message
if success:
print('-Sending SYS_INIT message to BARSM')
barsm_utils.sysInit(UDPsock)
# receive barsm to aacm processes
if success:
print('-Receiving BARSM_TO_AACM_PROCESSES')
expectedData = {}
expectedData['numMsgBytes'] = 42
expectedData['numMsgDataBytes'] = 38
# expect number of processes + 1 for BARSM + 1 for AACM
expectedData['unpackFMT'] = '=HHH' + 'I4sI' * (len(nameList) + 2)
expectedData['numProcesses'] = len(nameList) + 2
barsm_utils.barsmToAacmProcesses(TCPconn, expectedData, aacm_pid, nameList, follow)
print('--BARSM_TO_AACM_PROCESSES message received')
# Send for AACM_TO_BARSM_MSG
if success:
print('-Sending AACM_TO_BARSM_MSG')
#send AACM_TO_BARSM_MSG
appsToRestart = barsm_utils.aacmToBarsmMsg(TCPconn)
logs = follow.read('BARSM')
errors = sum(1 for d in logs if "ERROR" in d.get('message'))
if 0 < errors:
print('ERROR: SYSLOG errors while sending AACM_TO_BARSM_MSG!')
for d in logs:
if "ERROR" in d.get('message'):
print (d.get('message'))
success = False
if success:
print('--AACM_TO_BARSM_MSG sent')
# get AACM_TO_BARSM_MSG ack
if success:
print('-Receiving AACM_TO_BARSM_MSG ACK')
msgNotReceived = True
while True == msgNotReceived and success:
expectedERR = 12
msgNotReceived = barsm_utils.aacmToBarsmAck(TCPconn, expectedERR)
logs = follow.read('BARSM')
errors = sum(1 for d in logs if "ERROR" in d.get('message'))
if 0 < errors:
print('--ERROR: SYSLOG errors while receiving BARSM to AACM Msg ack!')
for d in logs:
if "ERROR" in d.get('message'):
print (d.get('message'))
success = False
if success:
print('--AACM_TO_BARSM_MSG ACK received')
# verify restart of App(s)
if success:
print('-Checking for successful app restart(s)')
success = barsm_utils.checkForRestart(appsToRestart)
logs = follow.read('BARSM')
errors = sum(1 for d in logs if "ERROR" in d.get('message'))
if 0 < errors:
print('--ERROR: SYSLOG error, App(s) not restarted!')
for d in logs:
if "ERROR" in d.get('message'):
print (d.get('message'))
success = False
if success:
print('--App(s) restarted successfully')
barsm_utils.populateChildren()
else:
print('--App(s) restart failure')
if success:
print('-Checking BARSM_TO_AACM_MSG')
msgNotReceived = True
if True == msgNotReceived and success:
print('--Receiving BARSM_TO_AACM_MSG')
expectedERR = 11
appName = log.logfollower()
names = []
expectedName = ''
while not names and expectedName == '':
names = appName.read('BARSM')
try:
expectedName = list(filter(lambda t: "DEBUG: Message sent with APP name:" in t['message'], names))[0]['message']
except:
expectedName = ''
names = []
expectedName = expectedName[-4:]
msgNotReceived, failedPid = barsm_utils.receiveBarsmToAacm(TCPconn, expectedERR, expectedName)
success = not msgNotReceived
logs = follow.read('BARSM')
errors = sum(1 for d in logs if "ERROR" in d.get('message'))
#all of the errors less the one we are expecting
errors = errors - sum(1 for d in logs if "ERROR: Process for /opt/rc360/modules/GE/gemod" in d.get('message'))
if 0 < errors:
print('--Error Receiving BARSM_TO_AACM_MSG')
if success:
print('--BARSM_TO_AACM_MSG received')
#send ack back to barsm
if success:
appsToRestart = []
restartData = {}
restartData['pid'] = failedPid
restartData['name'] = list(filter(lambda t: "NOTICE: Restarting " in t['message'], logs))[-1]['message'].split(' ')[-1][:-3]
appsToRestart.append(restartData)
barsm_utils.sendBarsmToAacmAck(TCPconn, failedPid)
logs = follow.read('BARSM')
errors = sum(1 for d in logs if "ERROR" in d.get('message'))
if 0 < errors:
print('--ERROR: Sending ACK To BARSM!')
for d in logs:
if "ERROR" in d.get('message'):
print (d.get('message'))
success = False
if success:
print('Checking for BARSM_TO_AACM_MSG process restart')
success = barsm_utils.checkForRestart(appsToRestart)
logs = follow.read('BARSM')
errors = sum(1 for d in logs if "ERROR" in d.get('message'))
if 0 < errors:
print('--ERROR: SYSLOG error, App(s) not restarted!')
for d in logs:
if "ERROR" in d.get('message'):
print (d.get('message'))
success = False
if success:
print('--BARSM_TO_AACM_MSG App(s) restarted successfully')
barsm_utils.populateChildren()
else:
print('--BARSM_TO_AACM_MSG App(s) restart failure')
# kill off all started processes
barsm_utils.kill_procs()
print('{}'.format('PASSED' if success else 'FAILED'))
return success
def run():
result = False
success = False
# clear out all app directories
barsm_utils.remove()
# compile needed applications and place in required directories
subprocess.call('gcc '+barsm_utils.apps_loc+'/infinite_sec.c -o /opt/rc360/system/aacm', shell=True)
subprocess.call('gcc '+barsm_utils.apps_loc+'/thirty_sec.c -o /opt/rc360/modules/TPA/thirty_sec', shell=True)
# navigate to the end of the SYSLOG
follow = log.logfollower()
# start barsm
print('-Starting BARSM')
barsm_utils.start_barsm()
logs = follow.read('BARSM')
errors = sum(1 for d in logs if "ERROR" in d.get('message'))
if 0 < errors:
print('ERROR: SYSLOG errors while starting BARSM!')
for d in logs:
if "ERROR" in d.get('message'):
print (d.get('message'))
success = False
else:
print('--BARSM started')
success = True
# wait for AACM to start
if success:
print('-Waiting for AACM')
barsm_pid = (proc.findproc("valgrind.bin"))[0]['pid']
aacmStarted = False
while False == aacmStarted and success:
# check if AACM has started
all_child = proc.findchild(barsm_pid)
if 1 == len(all_child):
aacmStarted = True
aacm_pid = int(all_child[0]['pid'])
# check for launch errors
logs = follow.read('BARSM')
errors = sum(1 for d in logs if "ERROR" in d.get('message'))
if 0 < errors:
print('ERROR: SYSLOG errors while waiting for AACM startup!')
for d in logs:
if "ERROR" in d.get('message'):
print (d.get('message'))
success = False
if success:
print('--AACM started')
# setup TCP connection
if success:
print('-Setting up TCP connection')
TCPconn, sVal = barsm_utils.TCPsetup() # both return values are objects
logs = follow.read('BARSM')
errors = sum(1 for d in logs if "ERROR" in d.get('message'))
if 0 < errors:
print('ERROR: SYSLOG errors while initiating TCP connection!')
for d in logs:
if "ERROR" in d.get('message'):
print (d.get('message'))
success = False
else:
print('--TCP connection setup successful')
# receive BARSM to AACM init
if success:
print('-Waiting for BARSM to AACM init')
success = barsm_utils.barsmToAacmInit(TCPconn)
logs = follow.read('BARSM')
errors = sum(1 for d in logs if "ERROR" in d.get('message'))
if 0 < errors:
print('ERROR: SYSLOG errors while receiving BARSM to AACM init!')
for d in logs:
if "ERROR" in d.get('message'):
print (d.get('message'))
success = False
else:
print('--Init received')
# send BARSM to AACM init ack
if success:
print('-Sending ACK')
barsm_utils.barsmToAacmInitAck(TCPconn)
logs = follow.read('BARSM')
errors = sum(1 for d in logs if "ERROR" in d.get('message'))
if 0 < errors:
print('ERROR: SYSLOG errors while sending BARSM to AACM init ack!')
for d in logs:
if "ERROR" in d.get('message'):
print (d.get('message'))
success = False
else:
print('--ACK sent')
# wait for children to complete launching
if success:
print('-Waiting for launch completion')
children, nameList = barsm_utils.waitForLaunch(follow)
if 2 == children:
print('--Launch complete!')
else:
print('ERROR: only {} child procs (should have 2)'.format(children))
success = False
# setup UDP connection
if success:
print('-Setting up UDP connection')
UDPsock = barsm_utils.UDPsetup()
logs = follow.read('BARSM')
errors = sum(1 for d in logs if "ERROR" in d.get('message'))
if 0 < errors:
print('ERROR: SYSLOG errors while initiating UDP connection!')
for d in logs:
if "ERROR" in d.get('message'):
print (d.get('message'))
success = False
else:
print('--UDP connection setup')
# send sysInit message
if success:
print('-Waiting to receive OPEN message from BARSM')
print('-Sending SYS_INIT message to BARSM')
barsm_utils.sysInit(UDPsock)
print('--OPEN received')
# receive barsm to aacm processes
if success:
print('-Receiving BARSM_TO_AACM_PROCESSES')
expectedData = {}
expectedData['numMsgBytes'] = 42
expectedData['numMsgDataBytes'] = 38
# expect number of processes + 1 for BARSM + 1 for AACM
expectedData['unpackFMT'] = '=HHH' + 'I4sI' * (len(nameList) + 2)
expectedData['numProcesses'] = len(nameList) + 2
barsm_utils.barsmToAacmProcesses(TCPconn, expectedData, aacm_pid, nameList, follow)
print('--BARSM_TO_AACM_PROCESSES message received')
# Default the time difference to a value that indicates failure
dif = 0
if success:
#TODO: have a maximum time limit to look for the restart messages
# capture the time of the first restart
print("Waiting for first restart...")
expectedERR = 11
appName = log.logfollower()
names = []
expectedName = ''
while not names and expectedName == '':
names = appName.read('BARSM')
try:
expectedName = list(filter(lambda t: "DEBUG: Message sent with APP name:" in t['message'], names))[0]['message']
except:
expectedName = ''
names = []
expectedName = expectedName[-4:]
barsm_utils.receiveBarsmToAacm(TCPconn, expectedERR, expectedName)
logs = follow.read('BARSM')
restart1_ts = list(filter(lambda t: "Restarting" in t['message'], logs))[0]['timestamp']
#print('restart1_ts: {}'.format(restart1_ts))
#update the children list after the restart
barsm_utils.populateChildren()
# capture the time of the second restart
print("Waiting for second restart...")
expectedERR = 11
appName = log.logfollower()
names = []
expectedName = ''
while not names and expectedName == '':
names = appName.read('BARSM')
try:
expectedName = list(filter(lambda t: "DEBUG: Message sent with APP name:" in t['message'], names))[0]['message']
except:
expectedName = ''
names = []
expectedName = expectedName[-4:]
barsm_utils.receiveBarsmToAacm(TCPconn, expectedERR, expectedName)
logs = follow.read('BARSM')
restart2_ts = list(filter(lambda t: "Restarting" in t['message'], logs))[0]['timestamp']
#print('restart2_ts: {}'.format(restart2_ts))
# compare timestamps
dif = (restart2_ts - restart1_ts).total_seconds()
#print('dif: {}'.format(dif))
barsm_utils.populateChildren()
# kill off all started processes
barsm_utils.kill_procs()
if success and 55 <= dif and 65 >= dif:
result = True
print('{}'.format('PASSED' if result else 'FAILED'))
return result
def run():
success = True
zombies = 0
# navigate to the end of the SYSLOG
follow = log.logfollower()
follow2 = log.logfollower()
# clear out all app directories
barsm_utils.remove()
# compile needed applications and place in required directories
barsm_utils.launch_bad()
# start barsm
if success:
print('-Starting BARSM')
barsm_utils.start_barsm()
logs = follow.read('BARSM')
errors = sum(1 for d in logs if "ERROR" in d.get('message'))
if 0 < errors:
print('ERROR: SYSLOG errors while starting BARSM!')
for d in logs:
if "ERROR" in d.get('message'):
print (d.get('message'))
success = False
else:
print('--BARSM started')
# wait for AACM to start
if success:
print('-Waiting for AACM')
barsm_pid = (proc.findproc("valgrind.bin"))[0]['pid']
aacmStarted = False
while False == aacmStarted and success:
# check if AACM has started
all_child = proc.findchild(barsm_pid)
if 1 == len(all_child):
aacmStarted = True
aacm_pid = int(all_child[0]['pid'])
# check for launch errors
logs = follow.read('BARSM')
errors = sum(1 for d in logs if "ERROR" in d.get('message'))
if 0 < errors:
print('ERROR: SYSLOG errors while waiting for AACM startup!')
for d in logs:
if "ERROR" in d.get('message'):
print (d.get('message'))
success = False
if success:
print('--AACM started')
# setup TCP connection
if success:
print('-Setting up TCP connection')
TCPconn, sVal = barsm_utils.TCPsetup() # both return values are objects
logs = follow.read('BARSM')
errors = sum(1 for d in logs if "ERROR" in d.get('message'))
if 0 < errors:
print('ERROR: SYSLOG errors while initiating TCP connection!')
for d in logs:
if "ERROR" in d.get('message'):
print (d.get('message'))
success = False
else:
print('--TCP connection setup successful')
# receive BARSM to AACM init
if success:
print('-Waiting for BARSM to AACM init')
success = barsm_utils.barsmToAacmInit(TCPconn)
logs = follow.read('BARSM')
errors = sum(1 for d in logs if "ERROR" in d.get('message'))
if 0 < errors:
print('ERROR: SYSLOG errors while receiving BARSM to AACM init!')
for d in logs:
if "ERROR" in d.get('message'):
print (d.get('message'))
success = False
else:
print('--Init received')
# send BARSM to AACM init ack
if success:
print('-Sending ACK')
barsm_utils.barsmToAacmInitAck(TCPconn)
logs = follow.read('BARSM')
errors = sum(1 for d in logs if "ERROR" in d.get('message'))
if 0 < errors:
print('ERROR: SYSLOG errors while sending BARSM to AACM init ack!')
for d in logs:
if "ERROR" in d.get('message'):
print (d.get('message'))
success = False
else:
print('--ACK sent')
# wait for children to complete launching
if success:
print('-Waiting for launch completion')
children, nameList = barsm_utils.waitForLaunch(follow)
if 9 == children:
print('--Launch complete!')
else:
print('ERROR: only {} child procs (should have 9)'.format(children))
success = False
# setup UDP connection
if success:
print('-Setting up UDP connection')
UDPsock = barsm_utils.UDPsetup()
logs = follow.read('BARSM')
errors = sum(1 for d in logs if "ERROR" in d.get('message'))
if 0 < errors:
print('ERROR: SYSLOG errors while initiating UDP connection!')
for d in logs:
if "ERROR" in d.get('message'):
print (d.get('message'))
success = False
else:
print('--UDP connection setup')
# send sysInit message
if success:
print('-Waiting to receive OPEN message from BARSM')
print('-Sending SYS_INIT message to BARSM')
barsm_utils.sysInit(UDPsock)
print('--OPEN received')
# receive barsm to aacm processes
if success:
print('-Receiving BARSM_TO_AACM_PROCESSES')
expectedData = {}
expectedData['numMsgBytes'] = 126
expectedData['numMsgDataBytes'] = 122
# expect number of processes + 1 for BARSM + 1 for AACM
expectedData['unpackFMT'] = '=HHH' + 'I4sI' * (len(nameList) + 2)
expectedData['numProcesses'] = len(nameList) + 2
barsm_utils.barsmToAacmProcesses(TCPconn, expectedData, aacm_pid, nameList, follow)
print('--BARSM_TO_AACM_PROCESSES message received')
# wait for barsm to aacm messgae and reply back
if success:
print('-Waiting for and responding to BARSM TO ACCM message 1')
appName = log.logfollower()
names = []
expectedName = ''
while not names and expectedName == '':
names = appName.read('BARSM')
try:
expectedName = list(filter(lambda t: "DEBUG: Message sent with APP name:" in t['message'], names))[0]['message']
except:
expectedName = ''
names = []
expectedName = expectedName[-4:]
expectedERR = 11
barsm_utils.receiveBarsmToAacm(TCPconn, expectedERR, expectedName)
print('-Waiting for and responding to BARSM TO ACCM message 2')
names = []
expectedName = ''
while not names and expectedName == '':
names = appName.read('BARSM')
try:
expectedName = list(filter(lambda t: "DEBUG: Message sent with APP name:" in t['message'], names))[0]['message']
except:
expectedName = ''
names = []
expectedName = expectedName[-4:]
expectedERR = 11
barsm_utils.receiveBarsmToAacm(TCPconn, expectedERR, expectedName)
print('-Waiting for and responding to BARSM TO ACCM message 3')
names = []
expectedName = ''
while not names and expectedName == '':
names = appName.read('BARSM')
try:
expectedName = list(filter(lambda t: "DEBUG: Message sent with APP name:" in t['message'], names))[0]['message']
except:
expectedName = ''
names = []
expectedName = expectedName[-4:]
expectedERR = 11
barsm_utils.receiveBarsmToAacm(TCPconn, expectedERR, expectedName)
print('-Waiting for and responding to BARSM TO ACCM message 4')
names = []
expectedName = ''
while not names and expectedName == '':
names = appName.read('BARSM')
try:
expectedName = list(filter(lambda t: "DEBUG: Message sent with APP name:" in t['message'], names))[0]['message']
except:
expectedName = ''
names = []
expectedName = expectedName[-4:]
expectedERR = 11
barsm_utils.receiveBarsmToAacm(TCPconn, expectedERR, expectedName)
print('-Waiting for and responding to BARSM TO ACCM message 5')
names = []
expectedName = ''
while not names and expectedName == '':
names = appName.read('BARSM')
try:
expectedName = list(filter(lambda t: "DEBUG: Message sent with APP name:" in t['message'], names))[0]['message']
except:
expectedName = ''
names = []
expectedName = expectedName[-4:]
expectedERR = 11
#barsm_utils.get_Message(TCPconn, expectedERR)
barsm_utils.receiveBarsmToAacm(TCPconn, expectedERR, expectedName)
# make sure all 5 stopped processes have logged errors to go with
# get the time of the launch finish
logs = follow2.read('BARSM')
if list(filter(lambda t: "ERROR: Process for /opt/rc360/system/thirty_aacm" in t['message'], logs)) == []:
print("SYSLOG error missing thirty_aacm")
success = False
if list(filter(lambda t: "ERROR: Process for /opt/rc360/modules/GE/thirty_gemod" in t['message'], logs)) == [] :
print("SYSLOG error missing thirty_gemod")
success = False
if list(filter(lambda t: "ERROR: Process for /opt/rc360/modules/TPA/thirty_tpamod" in t['message'], logs)) == [] :
print("SYSLOG error missing thirty_tpamod")
success = False
if list(filter(lambda t: "ERROR: Process for /opt/rc360/apps/GE/thirty_geapp" in t['message'], logs)) == [] :
print("SYSLOG error missing thirty_geapp")
success = False
if list(filter(lambda t: "ERROR: Process for /opt/rc360/apps/TPA/thirty_tpaapp" in t['message'], logs)) == [] :
print("SYSLOG error missing thirty_tpaapp")
success = False
#make sure all 5 apps that failed were restarted
# make sure all apps get started back up
child_info = barsm_utils.get_barsm_child_pids()
if list(filter(lambda t: "NOTICE: Restarting thirty_aacm..." in t['message'], logs)) == []:
print("SYSLOG error missing restart of thirty_aacm")
success = False
if list(filter(lambda t: "NOTICE: Restarting thirty_gemod..." in t['message'], logs)) == [] :
print("SYSLOG error missing restart of thirty_gemod")
success = False
if list(filter(lambda t: "NOTICE: Restarting thirty_tpamod..." in t['message'], logs)) == [] :
print("SYSLOG error missing restart of thirty_tpamod")
success = False
if list(filter(lambda t: "NOTICE: Restarting thirty_geapp..." in t['message'], logs)) == [] :
print("SYSLOG error missing restart of thirty_geapp")
success = False
if list(filter(lambda t: "NOTICE: Restarting thirty_tpaapp..." in t['message'], logs)) == [] :
print("SYSLOG error missing restart of thirty_tpaapp")
success = False
zombies = sum(1 for c in child_info if "Z" == c.get('state'))
# kill off all started processes
barsm_utils.populateChildren()
barsm_utils.kill_procs()
if not zombies and success:
print('Test Passes')
return True
else:
print('Test Failed')
return False
def run():
# navigate to the end of the SYSLOG
follow = log.logfollower()
logError = log.logfollower()
result = False
success = True
# clear out all app directories
barsm_utils.remove()
# compile needed applications and place in required directories
random.seed()
r = random.randint(1, 2)
print(r)
if 1 == r:
subprocess.call("gcc " + barsm_utils.apps_loc + "/zero_sec.c -o /opt/rc360/system/zero_sec_aacm", shell=True)
else:
subprocess.call("cp " + barsm_utils.apps_loc + "/non_exec.c /opt/rc360/system/", shell=True)
print("-Starting BARSM")
barsm_utils.start_barsm()
logs = follow.read("BARSM")
errors = sum(1 for d in logs if "ERROR" in d.get("message"))
if 0 < errors:
print("ERROR: SYSLOG errors while starting BARSM!")
for d in logs:
if "ERROR" in d.get("message"):
print(d.get("message"))
success = False
else:
print("--BARSM started")
success = True
# wait for AACM to start
if success:
print("-Waiting for AACM")
barsm_pid = (proc.findproc("valgrind.bin"))[0]["pid"]
aacmStarted = False
# time.sleep(1)
while False == aacmStarted and success:
# the c is moving faster than the python and is getting to the restarting before we can
# break out of this loop, so the check for errors needs to come before the check if done
logs = follow.read("BARSM")
errors = sum(1 for d in logs if "ERROR" in d.get("message"))
if 0 < errors:
print("ERROR: SYSLOG errors while waiting for AACM startup!")
for d in logs:
if "ERROR" in d.get("message"):
print(d.get("message"))
success = False
# check if AACM has started
all_child = proc.findchild(barsm_pid)
if 1 == len(all_child):
aacmStarted = True
# check for launch errors
if success:
print("--AACM start attempt finished")
if success:
# takes about 20 seconds to restart 5 times
time.sleep(25)
# check logs to see if five errors were logged for each launch failure
logs = logError.read("BARSM")
# print('logs : {}'.format(logs))
launch_errors = sum(
1 for d in logs if "File launch failed!" in d.get("message") and "/opt/rc360/" in d.get("message")
)
if 5 == launch_errors:
result = True
# kill off all started processes
barsm_utils.kill_procs()
print("{}".format("PASSED" if result else "FAILED"))
return result
#!/usr/bin/env python3
import sys
sys.path.insert(0, '/home/norwood/sandbox/repos/trunk/SW/test/utils')
import simm
import log
import subprocess
import time
import threading
if __name__ == '__main__':
barsmTestResult = True
sysLogFollow = log.logfollower()
lock = threading.Lock()
#BOOT PROCESSES
while barsmTestResult != False:
subprocess.Popen("clear")
time.sleep(1)
print("TEST STARTED!")
sysLogFollow.start()
KeepMoving = True
if KeepMoving != False:
# TCP SETUP
TCPconn, sVal = simm.TCPsetup()
sysLog = sysLogFollow.read()
sysLogFollow.start()
launch_errors = sum(1 for d in sysLog if "ERROR!" in d.get('message'))
if 0 < launch_errors: