def run():
result = False
success = False
# clear out all app directories
barsm_utils.remove()
# compile needed applications and place in required directories
subprocess.call('gcc '+barsm_utils.apps_loc+'/infinite_sec.c -o /opt/rc360/system/aacm', shell=True)
subprocess.call('gcc '+barsm_utils.apps_loc+'/thirty_sec.c -o /opt/rc360/modules/TPA/thirty_sec', shell=True)
# navigate to the end of the SYSLOG
follow = log.logfollower()
# start barsm
print('-Starting BARSM')
barsm_utils.start_barsm()
logs = follow.read('BARSM')
errors = sum(1 for d in logs if "ERROR" in d.get('message'))
if 0 < errors:
print('ERROR: SYSLOG errors while starting BARSM!')
for d in logs:
if "ERROR" in d.get('message'):
print (d.get('message'))
success = False
else:
print('--BARSM started')
success = True
# wait for AACM to start
if success:
print('-Waiting for AACM')
barsm_pid = (proc.findproc("valgrind.bin"))[0]['pid']
aacmStarted = False
while False == aacmStarted and success:
# check if AACM has started
all_child = proc.findchild(barsm_pid)
if 1 == len(all_child):
aacmStarted = True
aacm_pid = int(all_child[0]['pid'])
# check for launch errors
logs = follow.read('BARSM')
errors = sum(1 for d in logs if "ERROR" in d.get('message'))
if 0 < errors:
print('ERROR: SYSLOG errors while waiting for AACM startup!')
for d in logs:
if "ERROR" in d.get('message'):
print (d.get('message'))
success = False
if success:
print('--AACM started')
# setup TCP connection
if success:
print('-Setting up TCP connection')
TCPconn, sVal = barsm_utils.TCPsetup() # both return values are objects
logs = follow.read('BARSM')
errors = sum(1 for d in logs if "ERROR" in d.get('message'))
if 0 < errors:
print('ERROR: SYSLOG errors while initiating TCP connection!')
for d in logs:
if "ERROR" in d.get('message'):
print (d.get('message'))
success = False
else:
print('--TCP connection setup successful')
# receive BARSM to AACM init
if success:
print('-Waiting for BARSM to AACM init')
success = barsm_utils.barsmToAacmInit(TCPconn)
logs = follow.read('BARSM')
errors = sum(1 for d in logs if "ERROR" in d.get('message'))
if 0 < errors:
print('ERROR: SYSLOG errors while receiving BARSM to AACM init!')
for d in logs:
if "ERROR" in d.get('message'):
print (d.get('message'))
success = False
else:
print('--Init received')
# send BARSM to AACM init ack
if success:
print('-Sending ACK')
barsm_utils.barsmToAacmInitAck(TCPconn)
logs = follow.read('BARSM')
errors = sum(1 for d in logs if "ERROR" in d.get('message'))
if 0 < errors:
print('ERROR: SYSLOG errors while sending BARSM to AACM init ack!')
for d in logs:
if "ERROR" in d.get('message'):
print (d.get('message'))
success = False
else:
print('--ACK sent')
# wait for children to complete launching
if success:
print('-Waiting for launch completion')
children, nameList = barsm_utils.waitForLaunch(follow)
if 2 == children:
print('--Launch complete!')
else:
print('ERROR: only {} child procs (should have 2)'.format(children))
success = False
# setup UDP connection
if success:
print('-Setting up UDP connection')
UDPsock = barsm_utils.UDPsetup()
logs = follow.read('BARSM')
errors = sum(1 for d in logs if "ERROR" in d.get('message'))
if 0 < errors:
print('ERROR: SYSLOG errors while initiating UDP connection!')
for d in logs:
if "ERROR" in d.get('message'):
print (d.get('message'))
success = False
else:
print('--UDP connection setup')
# send sysInit message
if success:
print('-Waiting to receive OPEN message from BARSM')
print('-Sending SYS_INIT message to BARSM')
barsm_utils.sysInit(UDPsock)
print('--OPEN received')
# receive barsm to aacm processes
if success:
print('-Receiving BARSM_TO_AACM_PROCESSES')
expectedData = {}
expectedData['numMsgBytes'] = 42
expectedData['numMsgDataBytes'] = 38
# expect number of processes + 1 for BARSM + 1 for AACM
expectedData['unpackFMT'] = '=HHH' + 'I4sI' * (len(nameList) + 2)
expectedData['numProcesses'] = len(nameList) + 2
barsm_utils.barsmToAacmProcesses(TCPconn, expectedData, aacm_pid, nameList, follow)
print('--BARSM_TO_AACM_PROCESSES message received')
# Default the time difference to a value that indicates failure
dif = 0
if success:
#TODO: have a maximum time limit to look for the restart messages
# capture the time of the first restart
print("Waiting for first restart...")
expectedERR = 11
appName = log.logfollower()
names = []
expectedName = ''
while not names and expectedName == '':
names = appName.read('BARSM')
try:
expectedName = list(filter(lambda t: "DEBUG: Message sent with APP name:" in t['message'], names))[0]['message']
except:
expectedName = ''
names = []
expectedName = expectedName[-4:]
barsm_utils.receiveBarsmToAacm(TCPconn, expectedERR, expectedName)
logs = follow.read('BARSM')
restart1_ts = list(filter(lambda t: "Restarting" in t['message'], logs))[0]['timestamp']
#print('restart1_ts: {}'.format(restart1_ts))
#update the children list after the restart
barsm_utils.populateChildren()
# capture the time of the second restart
print("Waiting for second restart...")
expectedERR = 11
appName = log.logfollower()
names = []
expectedName = ''
while not names and expectedName == '':
names = appName.read('BARSM')
try:
expectedName = list(filter(lambda t: "DEBUG: Message sent with APP name:" in t['message'], names))[0]['message']
except:
expectedName = ''
names = []
expectedName = expectedName[-4:]
barsm_utils.receiveBarsmToAacm(TCPconn, expectedERR, expectedName)
logs = follow.read('BARSM')
restart2_ts = list(filter(lambda t: "Restarting" in t['message'], logs))[0]['timestamp']
#print('restart2_ts: {}'.format(restart2_ts))
# compare timestamps
dif = (restart2_ts - restart1_ts).total_seconds()
#print('dif: {}'.format(dif))
barsm_utils.populateChildren()
# kill off all started processes
barsm_utils.kill_procs()
if success and 55 <= dif and 65 >= dif:
result = True
print('{}'.format('PASSED' if result else 'FAILED'))
return result
def run():
success = True
# navigate to the end of the SYSLOG
follow = log.logfollower()
# clear out all app directories
barsm_utils.remove()
# add apps in directories
subprocess.call('gcc '+apps_loc+'/infinite_sec.c -o /opt/rc360/system/aacm', shell=True)
subprocess.call('gcc '+apps_loc+'/hundred_sec.c -o /opt/rc360/modules/GE/gemod', shell=True)
# start barsm
if success:
print('-Starting BARSM')
barsm_utils.start_barsm()
logs = follow.read('BARSM')
errors = sum(1 for d in logs if "ERROR" in d.get('message'))
if 0 < errors:
print('ERROR: SYSLOG errors while starting BARSM!')
for d in logs:
if "ERROR" in d.get('message'):
print (d.get('message'))
success = False
else:
print('--BARSM started')
# wait for AACM to start
if success:
print('-Waiting for AACM')
barsm_pid = (proc.findproc("valgrind.bin"))[0]['pid']
aacmStarted = False
while False == aacmStarted and success:
# check if AACM has started
all_child = proc.findchild(barsm_pid)
if 1 == len(all_child):
aacmStarted = True
aacm_pid = int(all_child[0]['pid'])
# check for launch errors
logs = follow.read('BARSM')
errors = sum(1 for d in logs if "ERROR" in d.get('message'))
if 0 < errors:
print('ERROR: SYSLOG errors while waiting for AACM startup!')
for d in logs:
if "ERROR" in d.get('message'):
print (d.get('message'))
success = False
if success:
print('--AACM started')
# setup TCP connection
if success:
print('-Setting up TCP connection')
TCPconn, sVal = barsm_utils.TCPsetup() # both return values are objects
logs = follow.read('BARSM')
errors = sum(1 for d in logs if "ERROR" in d.get('message'))
if 0 < errors:
print('ERROR: SYSLOG errors while initiating TCP connection!')
for d in logs:
if "ERROR" in d.get('message'):
print (d.get('message'))
success = False
else:
print('--TCP connection setup successful')
# receive BARSM to AACM init
if success:
print('-Waiting for BARSM to AACM init')
success = barsm_utils.barsmToAacmInit(TCPconn)
logs = follow.read('BARSM')
errors = sum(1 for d in logs if "ERROR" in d.get('message'))
if 0 < errors:
print('ERROR: SYSLOG errors while receiving BARSM to AACM init!')
for d in logs:
if "ERROR" in d.get('message'):
print (d.get('message'))
success = False
else:
print('--Init received')
# send BARSM to AACM init ack
if success:
print('-Sending ACK')
barsm_utils.barsmToAacmInitAck(TCPconn)
logs = follow.read('BARSM')
errors = sum(1 for d in logs if "ERROR" in d.get('message'))
if 0 < errors:
print('ERROR: SYSLOG errors while sending BARSM to AACM init ack!')
for d in logs:
if "ERROR" in d.get('message'):
print (d.get('message'))
success = False
else:
print('--ACK sent')
# wait for children to complete launching
if success:
print('-Waiting for launch completion')
children, nameList = barsm_utils.waitForLaunch(follow)
if 2 == children:
print('--Launch complete!')
else:
print('ERROR: only {} child procs (should have 2)'.format(children))
success = False
# setup UDP connection
if success:
print('-Setting up UDP connection')
UDPsock = barsm_utils.UDPsetup()
logs = follow.read('BARSM')
errors = sum(1 for d in logs if "ERROR" in d.get('message'))
if 0 < errors:
print('ERROR: SYSLOG errors while initiating UDP connection!')
for d in logs:
if "ERROR" in d.get('message'):
print (d.get('message'))
success = False
else:
print('--UDP connection setup')
# receive OPEN message
if 0:#success:
print('-Waiting to receive OPEN message from BARSM')
SenderAddr = barsm_utils.udpOpen(UDPsock)
logs = follow.read('BARSM')
errors = sum(1 for d in logs if "ERROR" in d.get('message'))
if 0 < errors:
print('ERROR: SYSLOG errors while opening UDP connection!')
for d in logs:
if "ERROR" in d.get('message'):
print (d.get('message'))
success = False
else:
print('--OPEN received')
# send sysInit message
if success:
print('-Sending SYS_INIT message to BARSM')
barsm_utils.sysInit(UDPsock)
# receive barsm to aacm processes
if success:
print('-Receiving BARSM_TO_AACM_PROCESSES')
expectedData = {}
expectedData['numMsgBytes'] = 42
expectedData['numMsgDataBytes'] = 38
# expect number of processes + 1 for BARSM + 1 for AACM
expectedData['unpackFMT'] = '=HHH' + 'I4sI' * (len(nameList) + 2)
expectedData['numProcesses'] = len(nameList) + 2
barsm_utils.barsmToAacmProcesses(TCPconn, expectedData, aacm_pid, nameList, follow)
print('--BARSM_TO_AACM_PROCESSES message received')
# Send for AACM_TO_BARSM_MSG
if success:
print('-Sending AACM_TO_BARSM_MSG')
#send AACM_TO_BARSM_MSG
appsToRestart = barsm_utils.aacmToBarsmMsg(TCPconn)
logs = follow.read('BARSM')
errors = sum(1 for d in logs if "ERROR" in d.get('message'))
if 0 < errors:
print('ERROR: SYSLOG errors while sending AACM_TO_BARSM_MSG!')
for d in logs:
if "ERROR" in d.get('message'):
print (d.get('message'))
success = False
if success:
print('--AACM_TO_BARSM_MSG sent')
# get AACM_TO_BARSM_MSG ack
if success:
print('-Receiving AACM_TO_BARSM_MSG ACK')
msgNotReceived = True
while True == msgNotReceived and success:
expectedERR = 12
msgNotReceived = barsm_utils.aacmToBarsmAck(TCPconn, expectedERR)
logs = follow.read('BARSM')
errors = sum(1 for d in logs if "ERROR" in d.get('message'))
if 0 < errors:
print('--ERROR: SYSLOG errors while receiving BARSM to AACM Msg ack!')
for d in logs:
if "ERROR" in d.get('message'):
print (d.get('message'))
success = False
if success:
print('--AACM_TO_BARSM_MSG ACK received')
# verify restart of App(s)
if success:
print('-Checking for successful app restart(s)')
success = barsm_utils.checkForRestart(appsToRestart)
logs = follow.read('BARSM')
errors = sum(1 for d in logs if "ERROR" in d.get('message'))
if 0 < errors:
print('--ERROR: SYSLOG error, App(s) not restarted!')
for d in logs:
if "ERROR" in d.get('message'):
print (d.get('message'))
success = False
if success:
print('--App(s) restarted successfully')
barsm_utils.populateChildren()
else:
print('--App(s) restart failure')
if success:
print('-Checking BARSM_TO_AACM_MSG')
msgNotReceived = True
if True == msgNotReceived and success:
print('--Receiving BARSM_TO_AACM_MSG')
expectedERR = 11
appName = log.logfollower()
names = []
expectedName = ''
while not names and expectedName == '':
names = appName.read('BARSM')
try:
expectedName = list(filter(lambda t: "DEBUG: Message sent with APP name:" in t['message'], names))[0]['message']
except:
expectedName = ''
names = []
expectedName = expectedName[-4:]
msgNotReceived, failedPid = barsm_utils.receiveBarsmToAacm(TCPconn, expectedERR, expectedName)
success = not msgNotReceived
logs = follow.read('BARSM')
errors = sum(1 for d in logs if "ERROR" in d.get('message'))
#all of the errors less the one we are expecting
errors = errors - sum(1 for d in logs if "ERROR: Process for /opt/rc360/modules/GE/gemod" in d.get('message'))
if 0 < errors:
print('--Error Receiving BARSM_TO_AACM_MSG')
if success:
print('--BARSM_TO_AACM_MSG received')
#send ack back to barsm
if success:
appsToRestart = []
restartData = {}
restartData['pid'] = failedPid
restartData['name'] = list(filter(lambda t: "NOTICE: Restarting " in t['message'], logs))[-1]['message'].split(' ')[-1][:-3]
appsToRestart.append(restartData)
barsm_utils.sendBarsmToAacmAck(TCPconn, failedPid)
logs = follow.read('BARSM')
errors = sum(1 for d in logs if "ERROR" in d.get('message'))
if 0 < errors:
print('--ERROR: Sending ACK To BARSM!')
for d in logs:
if "ERROR" in d.get('message'):
print (d.get('message'))
success = False
if success:
print('Checking for BARSM_TO_AACM_MSG process restart')
success = barsm_utils.checkForRestart(appsToRestart)
logs = follow.read('BARSM')
errors = sum(1 for d in logs if "ERROR" in d.get('message'))
if 0 < errors:
print('--ERROR: SYSLOG error, App(s) not restarted!')
for d in logs:
if "ERROR" in d.get('message'):
print (d.get('message'))
success = False
if success:
print('--BARSM_TO_AACM_MSG App(s) restarted successfully')
barsm_utils.populateChildren()
else:
print('--BARSM_TO_AACM_MSG App(s) restart failure')
# kill off all started processes
barsm_utils.kill_procs()
print('{}'.format('PASSED' if success else 'FAILED'))
return success
def run():
success = True
zombies = 0
# navigate to the end of the SYSLOG
follow = log.logfollower()
follow2 = log.logfollower()
# clear out all app directories
barsm_utils.remove()
# compile needed applications and place in required directories
barsm_utils.launch_bad()
# start barsm
if success:
print('-Starting BARSM')
barsm_utils.start_barsm()
logs = follow.read('BARSM')
errors = sum(1 for d in logs if "ERROR" in d.get('message'))
if 0 < errors:
print('ERROR: SYSLOG errors while starting BARSM!')
for d in logs:
if "ERROR" in d.get('message'):
print (d.get('message'))
success = False
else:
print('--BARSM started')
# wait for AACM to start
if success:
print('-Waiting for AACM')
barsm_pid = (proc.findproc("valgrind.bin"))[0]['pid']
aacmStarted = False
while False == aacmStarted and success:
# check if AACM has started
all_child = proc.findchild(barsm_pid)
if 1 == len(all_child):
aacmStarted = True
aacm_pid = int(all_child[0]['pid'])
# check for launch errors
logs = follow.read('BARSM')
errors = sum(1 for d in logs if "ERROR" in d.get('message'))
if 0 < errors:
print('ERROR: SYSLOG errors while waiting for AACM startup!')
for d in logs:
if "ERROR" in d.get('message'):
print (d.get('message'))
success = False
if success:
print('--AACM started')
# setup TCP connection
if success:
print('-Setting up TCP connection')
TCPconn, sVal = barsm_utils.TCPsetup() # both return values are objects
logs = follow.read('BARSM')
errors = sum(1 for d in logs if "ERROR" in d.get('message'))
if 0 < errors:
print('ERROR: SYSLOG errors while initiating TCP connection!')
for d in logs:
if "ERROR" in d.get('message'):
print (d.get('message'))
success = False
else:
print('--TCP connection setup successful')
# receive BARSM to AACM init
if success:
print('-Waiting for BARSM to AACM init')
success = barsm_utils.barsmToAacmInit(TCPconn)
logs = follow.read('BARSM')
errors = sum(1 for d in logs if "ERROR" in d.get('message'))
if 0 < errors:
print('ERROR: SYSLOG errors while receiving BARSM to AACM init!')
for d in logs:
if "ERROR" in d.get('message'):
print (d.get('message'))
success = False
else:
print('--Init received')
# send BARSM to AACM init ack
if success:
print('-Sending ACK')
barsm_utils.barsmToAacmInitAck(TCPconn)
logs = follow.read('BARSM')
errors = sum(1 for d in logs if "ERROR" in d.get('message'))
if 0 < errors:
print('ERROR: SYSLOG errors while sending BARSM to AACM init ack!')
for d in logs:
if "ERROR" in d.get('message'):
print (d.get('message'))
success = False
else:
print('--ACK sent')
# wait for children to complete launching
if success:
print('-Waiting for launch completion')
children, nameList = barsm_utils.waitForLaunch(follow)
if 9 == children:
print('--Launch complete!')
else:
print('ERROR: only {} child procs (should have 9)'.format(children))
success = False
# setup UDP connection
if success:
print('-Setting up UDP connection')
UDPsock = barsm_utils.UDPsetup()
logs = follow.read('BARSM')
errors = sum(1 for d in logs if "ERROR" in d.get('message'))
if 0 < errors:
print('ERROR: SYSLOG errors while initiating UDP connection!')
for d in logs:
if "ERROR" in d.get('message'):
print (d.get('message'))
success = False
else:
print('--UDP connection setup')
# send sysInit message
if success:
print('-Waiting to receive OPEN message from BARSM')
print('-Sending SYS_INIT message to BARSM')
barsm_utils.sysInit(UDPsock)
print('--OPEN received')
# receive barsm to aacm processes
if success:
print('-Receiving BARSM_TO_AACM_PROCESSES')
expectedData = {}
expectedData['numMsgBytes'] = 126
expectedData['numMsgDataBytes'] = 122
# expect number of processes + 1 for BARSM + 1 for AACM
expectedData['unpackFMT'] = '=HHH' + 'I4sI' * (len(nameList) + 2)
expectedData['numProcesses'] = len(nameList) + 2
barsm_utils.barsmToAacmProcesses(TCPconn, expectedData, aacm_pid, nameList, follow)
print('--BARSM_TO_AACM_PROCESSES message received')
# wait for barsm to aacm messgae and reply back
if success:
print('-Waiting for and responding to BARSM TO ACCM message 1')
appName = log.logfollower()
names = []
expectedName = ''
while not names and expectedName == '':
names = appName.read('BARSM')
try:
expectedName = list(filter(lambda t: "DEBUG: Message sent with APP name:" in t['message'], names))[0]['message']
except:
expectedName = ''
names = []
expectedName = expectedName[-4:]
expectedERR = 11
barsm_utils.receiveBarsmToAacm(TCPconn, expectedERR, expectedName)
print('-Waiting for and responding to BARSM TO ACCM message 2')
names = []
expectedName = ''
while not names and expectedName == '':
names = appName.read('BARSM')
try:
expectedName = list(filter(lambda t: "DEBUG: Message sent with APP name:" in t['message'], names))[0]['message']
except:
expectedName = ''
names = []
expectedName = expectedName[-4:]
expectedERR = 11
barsm_utils.receiveBarsmToAacm(TCPconn, expectedERR, expectedName)
print('-Waiting for and responding to BARSM TO ACCM message 3')
names = []
expectedName = ''
while not names and expectedName == '':
names = appName.read('BARSM')
try:
expectedName = list(filter(lambda t: "DEBUG: Message sent with APP name:" in t['message'], names))[0]['message']
except:
expectedName = ''
names = []
expectedName = expectedName[-4:]
expectedERR = 11
barsm_utils.receiveBarsmToAacm(TCPconn, expectedERR, expectedName)
print('-Waiting for and responding to BARSM TO ACCM message 4')
names = []
expectedName = ''
while not names and expectedName == '':
names = appName.read('BARSM')
try:
expectedName = list(filter(lambda t: "DEBUG: Message sent with APP name:" in t['message'], names))[0]['message']
except:
expectedName = ''
names = []
expectedName = expectedName[-4:]
expectedERR = 11
barsm_utils.receiveBarsmToAacm(TCPconn, expectedERR, expectedName)
print('-Waiting for and responding to BARSM TO ACCM message 5')
names = []
expectedName = ''
while not names and expectedName == '':
names = appName.read('BARSM')
try:
expectedName = list(filter(lambda t: "DEBUG: Message sent with APP name:" in t['message'], names))[0]['message']
except:
expectedName = ''
names = []
expectedName = expectedName[-4:]
expectedERR = 11
#barsm_utils.get_Message(TCPconn, expectedERR)
barsm_utils.receiveBarsmToAacm(TCPconn, expectedERR, expectedName)
# make sure all 5 stopped processes have logged errors to go with
# get the time of the launch finish
logs = follow2.read('BARSM')
if list(filter(lambda t: "ERROR: Process for /opt/rc360/system/thirty_aacm" in t['message'], logs)) == []:
print("SYSLOG error missing thirty_aacm")
success = False
if list(filter(lambda t: "ERROR: Process for /opt/rc360/modules/GE/thirty_gemod" in t['message'], logs)) == [] :
print("SYSLOG error missing thirty_gemod")
success = False
if list(filter(lambda t: "ERROR: Process for /opt/rc360/modules/TPA/thirty_tpamod" in t['message'], logs)) == [] :
print("SYSLOG error missing thirty_tpamod")
success = False
if list(filter(lambda t: "ERROR: Process for /opt/rc360/apps/GE/thirty_geapp" in t['message'], logs)) == [] :
print("SYSLOG error missing thirty_geapp")
success = False
if list(filter(lambda t: "ERROR: Process for /opt/rc360/apps/TPA/thirty_tpaapp" in t['message'], logs)) == [] :
print("SYSLOG error missing thirty_tpaapp")
success = False
#make sure all 5 apps that failed were restarted
# make sure all apps get started back up
child_info = barsm_utils.get_barsm_child_pids()
if list(filter(lambda t: "NOTICE: Restarting thirty_aacm..." in t['message'], logs)) == []:
print("SYSLOG error missing restart of thirty_aacm")
success = False
if list(filter(lambda t: "NOTICE: Restarting thirty_gemod..." in t['message'], logs)) == [] :
print("SYSLOG error missing restart of thirty_gemod")
success = False
if list(filter(lambda t: "NOTICE: Restarting thirty_tpamod..." in t['message'], logs)) == [] :
print("SYSLOG error missing restart of thirty_tpamod")
success = False
if list(filter(lambda t: "NOTICE: Restarting thirty_geapp..." in t['message'], logs)) == [] :
print("SYSLOG error missing restart of thirty_geapp")
success = False
if list(filter(lambda t: "NOTICE: Restarting thirty_tpaapp..." in t['message'], logs)) == [] :
print("SYSLOG error missing restart of thirty_tpaapp")
success = False
zombies = sum(1 for c in child_info if "Z" == c.get('state'))
# kill off all started processes
barsm_utils.populateChildren()
barsm_utils.kill_procs()
if not zombies and success:
print('Test Passes')
return True
else:
print('Test Failed')
return False