Beispiel #1
0
def run():
    result = False
    success = False

    # clear out all app directories
    barsm_utils.remove()

    # compile needed applications and place in required directories
    subprocess.call('gcc '+barsm_utils.apps_loc+'/infinite_sec.c -o /opt/rc360/system/aacm', shell=True)
    subprocess.call('gcc '+barsm_utils.apps_loc+'/thirty_sec.c -o /opt/rc360/modules/TPA/thirty_sec', shell=True)

    # navigate to the end of the SYSLOG
    follow = log.logfollower()

    # start barsm

    print('-Starting BARSM')
    barsm_utils.start_barsm()
    logs = follow.read('BARSM')
    errors = sum(1 for d in logs if "ERROR" in d.get('message'))
    if 0 < errors:
        print('ERROR: SYSLOG errors while starting BARSM!')
        for d in logs:
            if "ERROR" in d.get('message'):
                print (d.get('message'))
        success = False
    else:
        print('--BARSM started')
        success = True

    # wait for AACM to start
    if success:
        print('-Waiting for AACM')
        barsm_pid = (proc.findproc("valgrind.bin"))[0]['pid']
        aacmStarted = False
        while False == aacmStarted and success:
            # check if AACM has started
            all_child = proc.findchild(barsm_pid)
            if 1 == len(all_child):
                aacmStarted = True
                aacm_pid = int(all_child[0]['pid'])
            # check for launch errors
            logs = follow.read('BARSM')
            errors = sum(1 for d in logs if "ERROR" in d.get('message'))
            if 0 < errors:
                print('ERROR: SYSLOG errors while waiting for AACM startup!')
                for d in logs:
                    if "ERROR" in d.get('message'):
                        print (d.get('message'))
                success = False
        if success:
            print('--AACM started')

    # setup TCP connection
    if success:
        print('-Setting up TCP connection')
        TCPconn, sVal = barsm_utils.TCPsetup()  # both return values are objects
        logs = follow.read('BARSM')
        errors = sum(1 for d in logs if "ERROR" in d.get('message'))
        if 0 < errors:
            print('ERROR: SYSLOG errors while initiating TCP connection!')
            for d in logs:
                if "ERROR" in d.get('message'):
                    print (d.get('message'))
            success = False
        else:
            print('--TCP connection setup successful')

    # receive BARSM to AACM init
    if success:
        print('-Waiting for BARSM to AACM init')
        success = barsm_utils.barsmToAacmInit(TCPconn)
        logs = follow.read('BARSM')
        errors = sum(1 for d in logs if "ERROR" in d.get('message'))
        if 0 < errors:
            print('ERROR: SYSLOG errors while receiving BARSM to AACM init!')
            for d in logs:
                if "ERROR" in d.get('message'):
                    print (d.get('message'))
            success = False
        else:
            print('--Init received')

    # send BARSM to AACM init ack
    if success:
        print('-Sending ACK')
        barsm_utils.barsmToAacmInitAck(TCPconn)
        logs = follow.read('BARSM')
        errors = sum(1 for d in logs if "ERROR" in d.get('message'))
        if 0 < errors:
            print('ERROR: SYSLOG errors while sending BARSM to AACM init ack!')
            for d in logs:
                if "ERROR" in d.get('message'):
                    print (d.get('message'))
            success = False
        else:
            print('--ACK sent')

    # wait for children to complete launching
    if success:
        print('-Waiting for launch completion')
        children, nameList = barsm_utils.waitForLaunch(follow)
        if 2 == children:
            print('--Launch complete!')
        else:
            print('ERROR: only {} child procs (should have 2)'.format(children))
            success = False

    # setup UDP connection
    if success:
        print('-Setting up UDP connection')
        UDPsock = barsm_utils.UDPsetup()
        logs = follow.read('BARSM')
        errors = sum(1 for d in logs if "ERROR" in d.get('message'))
        if 0 < errors:
            print('ERROR: SYSLOG errors while initiating UDP connection!')
            for d in logs:
                if "ERROR" in d.get('message'):
                    print (d.get('message'))
            success = False
        else:
            print('--UDP connection setup')

    # send sysInit message
    if success:
        print('-Waiting to receive OPEN message from BARSM')
        print('-Sending SYS_INIT message to BARSM')
        barsm_utils.sysInit(UDPsock)
        print('--OPEN received')

    # receive barsm to aacm processes
    if success:
        print('-Receiving BARSM_TO_AACM_PROCESSES')

        expectedData = {}
        expectedData['numMsgBytes']  = 42
        expectedData['numMsgDataBytes']  = 38
        # expect number of processes + 1 for BARSM + 1 for AACM
        expectedData['unpackFMT'] = '=HHH' + 'I4sI' * (len(nameList) + 2)
        expectedData['numProcesses'] = len(nameList) + 2

        barsm_utils.barsmToAacmProcesses(TCPconn, expectedData, aacm_pid, nameList, follow)
        print('--BARSM_TO_AACM_PROCESSES message received')

    # Default the time difference to a value that indicates failure
    dif = 0

    if success:
        #TODO: have a maximum time limit to look for the restart messages
        # capture the time of the first restart
        print("Waiting for first restart...")
        expectedERR = 11

        appName = log.logfollower()
        names = []
        expectedName = ''

        while not names and expectedName == '':
            names = appName.read('BARSM')
            try:
                expectedName = list(filter(lambda t: "DEBUG: Message sent with APP name:" in t['message'], names))[0]['message']
            except:
                expectedName = ''
                names = []

        expectedName = expectedName[-4:]

        barsm_utils.receiveBarsmToAacm(TCPconn, expectedERR, expectedName)
        logs = follow.read('BARSM')
        restart1_ts = list(filter(lambda t: "Restarting" in t['message'], logs))[0]['timestamp']
        #print('restart1_ts: {}'.format(restart1_ts))

        #update the children list after the restart
        barsm_utils.populateChildren()

        # capture the time of the second restart
        print("Waiting for second restart...")
        expectedERR = 11

        appName = log.logfollower()
        names = []
        expectedName = ''

        while not names and expectedName == '':
            names = appName.read('BARSM')
            try:
                expectedName = list(filter(lambda t: "DEBUG: Message sent with APP name:" in t['message'], names))[0]['message']
            except:
                expectedName = ''
                names = []

        expectedName = expectedName[-4:]

        barsm_utils.receiveBarsmToAacm(TCPconn, expectedERR, expectedName)
        logs = follow.read('BARSM')
        restart2_ts = list(filter(lambda t: "Restarting" in t['message'], logs))[0]['timestamp']
        #print('restart2_ts: {}'.format(restart2_ts))

        # compare timestamps
        dif = (restart2_ts - restart1_ts).total_seconds()
        #print('dif: {}'.format(dif))
        barsm_utils.populateChildren()

    # kill off all started processes
    barsm_utils.kill_procs()

    if success and 55 <= dif and 65 >= dif:
        result = True

    print('{}'.format('PASSED' if result else 'FAILED'))
    return result
Beispiel #2
0
def run():
    success = True
    # navigate to the end of the SYSLOG
    follow = log.logfollower()

    # clear out all app directories
    barsm_utils.remove()

    # add apps in directories
    subprocess.call('gcc '+apps_loc+'/infinite_sec.c -o /opt/rc360/system/aacm', shell=True)
    subprocess.call('gcc '+apps_loc+'/hundred_sec.c -o /opt/rc360/modules/GE/gemod', shell=True)

    # start barsm
    if success:
        print('-Starting BARSM')
        barsm_utils.start_barsm()
        logs = follow.read('BARSM')
        errors = sum(1 for d in logs if "ERROR" in d.get('message'))
        if 0 < errors:
            print('ERROR: SYSLOG errors while starting BARSM!')
            for d in logs:
                if "ERROR" in d.get('message'):
                    print (d.get('message'))
            success = False
        else:
            print('--BARSM started')

    # wait for AACM to start
    if success:
        print('-Waiting for AACM')
        barsm_pid = (proc.findproc("valgrind.bin"))[0]['pid']
        aacmStarted = False
        while False == aacmStarted and success:
            # check if AACM has started
            all_child = proc.findchild(barsm_pid)
            if 1 == len(all_child):
                aacmStarted = True
                aacm_pid = int(all_child[0]['pid'])
            # check for launch errors
            logs = follow.read('BARSM')
            errors = sum(1 for d in logs if "ERROR" in d.get('message'))
            if 0 < errors:
                print('ERROR: SYSLOG errors while waiting for AACM startup!')
                for d in logs:
                    if "ERROR" in d.get('message'):
                        print (d.get('message'))
                success = False
        if success:
            print('--AACM started')

    # setup TCP connection
    if success:
        print('-Setting up TCP connection')
        TCPconn, sVal = barsm_utils.TCPsetup()  # both return values are objects
        logs = follow.read('BARSM')
        errors = sum(1 for d in logs if "ERROR" in d.get('message'))
        if 0 < errors:
            print('ERROR: SYSLOG errors while initiating TCP connection!')
            for d in logs:
                if "ERROR" in d.get('message'):
                    print (d.get('message'))
            success = False
        else:
            print('--TCP connection setup successful')

    # receive BARSM to AACM init
    if success:
        print('-Waiting for BARSM to AACM init')
        success = barsm_utils.barsmToAacmInit(TCPconn)
        logs = follow.read('BARSM')
        errors = sum(1 for d in logs if "ERROR" in d.get('message'))
        if 0 < errors:
            print('ERROR: SYSLOG errors while receiving BARSM to AACM init!')
            for d in logs:
                if "ERROR" in d.get('message'):
                    print (d.get('message'))
            success = False
        else:
            print('--Init received')

    # send BARSM to AACM init ack
    if success:
        print('-Sending ACK')
        barsm_utils.barsmToAacmInitAck(TCPconn)
        logs = follow.read('BARSM')
        errors = sum(1 for d in logs if "ERROR" in d.get('message'))
        if 0 < errors:
            print('ERROR: SYSLOG errors while sending BARSM to AACM init ack!')
            for d in logs:
                if "ERROR" in d.get('message'):
                    print (d.get('message'))
            success = False
        else:
            print('--ACK sent')

    # wait for children to complete launching
    if success:
        print('-Waiting for launch completion')
        children, nameList = barsm_utils.waitForLaunch(follow)
        if 2 == children:
            print('--Launch complete!')
        else:
            print('ERROR: only {} child procs (should have 2)'.format(children))
            success = False

    # setup UDP connection
    if success:
        print('-Setting up UDP connection')
        UDPsock = barsm_utils.UDPsetup()
        logs = follow.read('BARSM')
        errors = sum(1 for d in logs if "ERROR" in d.get('message'))
        if 0 < errors:
            print('ERROR: SYSLOG errors while initiating UDP connection!')
            for d in logs:
                if "ERROR" in d.get('message'):
                    print (d.get('message'))
            success = False
        else:
            print('--UDP connection setup')

    # receive OPEN message
    if 0:#success:
        print('-Waiting to receive OPEN message from BARSM')
        SenderAddr = barsm_utils.udpOpen(UDPsock)
        logs = follow.read('BARSM')
        errors = sum(1 for d in logs if "ERROR" in d.get('message'))
        if 0 < errors:
            print('ERROR: SYSLOG errors while opening UDP connection!')
            for d in logs:
                if "ERROR" in d.get('message'):
                    print (d.get('message'))
            success = False
        else:
            print('--OPEN received')

    # send sysInit message
    if success:
        print('-Sending SYS_INIT message to BARSM')
        barsm_utils.sysInit(UDPsock)

    # receive barsm to aacm processes
    if success:
        print('-Receiving BARSM_TO_AACM_PROCESSES')

        expectedData = {}
        expectedData['numMsgBytes']  = 42
        expectedData['numMsgDataBytes']  = 38

        # expect number of processes + 1 for BARSM + 1 for AACM
        expectedData['unpackFMT'] = '=HHH' + 'I4sI' * (len(nameList) + 2)
        expectedData['numProcesses'] = len(nameList) + 2

        barsm_utils.barsmToAacmProcesses(TCPconn, expectedData, aacm_pid, nameList, follow)
        print('--BARSM_TO_AACM_PROCESSES message received')

    # Send for AACM_TO_BARSM_MSG
    if success:
        print('-Sending AACM_TO_BARSM_MSG')

        #send AACM_TO_BARSM_MSG
        appsToRestart = barsm_utils.aacmToBarsmMsg(TCPconn)
        logs = follow.read('BARSM')
        errors = sum(1 for d in logs if "ERROR" in d.get('message'))
        if 0 < errors:
            print('ERROR: SYSLOG errors while sending AACM_TO_BARSM_MSG!')
            for d in logs:
                if "ERROR" in d.get('message'):
                    print (d.get('message'))
            success = False
        if success:
            print('--AACM_TO_BARSM_MSG sent')

    # get AACM_TO_BARSM_MSG ack
    if success:
        print('-Receiving AACM_TO_BARSM_MSG ACK')
        msgNotReceived = True
        while True == msgNotReceived and success:
            expectedERR = 12
            msgNotReceived = barsm_utils.aacmToBarsmAck(TCPconn, expectedERR)
            logs = follow.read('BARSM')
            errors = sum(1 for d in logs if "ERROR" in d.get('message'))
            if 0 < errors:
                print('--ERROR: SYSLOG errors while receiving BARSM to AACM Msg ack!')
                for d in logs:
                    if "ERROR" in d.get('message'):
                        print (d.get('message'))
                success = False
        if success:
            print('--AACM_TO_BARSM_MSG ACK received')

    # verify restart of App(s)
    if success:
        print('-Checking for successful app restart(s)')
        success = barsm_utils.checkForRestart(appsToRestart)
        logs = follow.read('BARSM')
        errors = sum(1 for d in logs if "ERROR" in d.get('message'))
        if 0 < errors:
            print('--ERROR: SYSLOG error, App(s) not restarted!')
            for d in logs:
                if "ERROR" in d.get('message'):
                    print (d.get('message'))
            success = False
        if success:
            print('--App(s) restarted successfully')
            barsm_utils.populateChildren()
        else:
            print('--App(s) restart failure')

    if success:
        print('-Checking BARSM_TO_AACM_MSG')
        msgNotReceived = True
        if True == msgNotReceived and success:
            print('--Receiving BARSM_TO_AACM_MSG')
            expectedERR = 11

            appName = log.logfollower()
            names = []
            expectedName = ''

            while not names and expectedName == '':
                names = appName.read('BARSM')
                try:
                    expectedName = list(filter(lambda t: "DEBUG: Message sent with APP name:" in t['message'], names))[0]['message']
                except:
                    expectedName = ''
                    names = []

            expectedName = expectedName[-4:]

            msgNotReceived, failedPid = barsm_utils.receiveBarsmToAacm(TCPconn, expectedERR, expectedName)
            success = not msgNotReceived
            logs = follow.read('BARSM')
            errors = sum(1 for d in logs if "ERROR" in d.get('message'))
            #all of the errors less the one we are expecting
            errors = errors - sum(1 for d in logs if "ERROR: Process for /opt/rc360/modules/GE/gemod" in d.get('message'))
            if 0 < errors:
                print('--Error Receiving BARSM_TO_AACM_MSG')
        if success:
            print('--BARSM_TO_AACM_MSG received')

    #send ack back to barsm
    if success:
        appsToRestart = []
        restartData   = {}
        restartData['pid']  = failedPid
        restartData['name']  = list(filter(lambda t: "NOTICE: Restarting " in t['message'], logs))[-1]['message'].split(' ')[-1][:-3]
        appsToRestart.append(restartData)

        barsm_utils.sendBarsmToAacmAck(TCPconn, failedPid)
        logs = follow.read('BARSM')
        errors = sum(1 for d in logs if "ERROR" in d.get('message'))
        if 0 < errors:
            print('--ERROR: Sending ACK To BARSM!')
            for d in logs:
                if "ERROR" in d.get('message'):
                    print (d.get('message'))
            success = False

    if success:
        print('Checking for BARSM_TO_AACM_MSG process restart')
        success = barsm_utils.checkForRestart(appsToRestart)

        logs = follow.read('BARSM')
        errors = sum(1 for d in logs if "ERROR" in d.get('message'))
        if 0 < errors:
            print('--ERROR: SYSLOG error, App(s) not restarted!')
            for d in logs:
                if "ERROR" in d.get('message'):
                    print (d.get('message'))
            success = False
        if success:
            print('--BARSM_TO_AACM_MSG App(s) restarted successfully')
            barsm_utils.populateChildren()
        else:
            print('--BARSM_TO_AACM_MSG App(s) restart failure')

    # kill off all started processes
    barsm_utils.kill_procs()

    print('{}'.format('PASSED' if success else 'FAILED'))
    return success
Beispiel #3
0
def run():
    success = True
    zombies = 0
    # navigate to the end of the SYSLOG
    follow = log.logfollower()
    follow2 = log.logfollower()

    # clear out all app directories
    barsm_utils.remove()

    # compile needed applications and place in required directories
    barsm_utils.launch_bad()

    # start barsm
    if success:
        print('-Starting BARSM')
        barsm_utils.start_barsm()
        logs = follow.read('BARSM')
        errors = sum(1 for d in logs if "ERROR" in d.get('message'))
        if 0 < errors:
            print('ERROR: SYSLOG errors while starting BARSM!')
            for d in logs:
                if "ERROR" in d.get('message'):
                    print (d.get('message'))
            success = False
        else:
            print('--BARSM started')

    # wait for AACM to start
    if success:
        print('-Waiting for AACM')
        barsm_pid = (proc.findproc("valgrind.bin"))[0]['pid']
        aacmStarted = False
        while False == aacmStarted and success:
            # check if AACM has started
            all_child = proc.findchild(barsm_pid)
            if 1 == len(all_child):
                aacmStarted = True
                aacm_pid = int(all_child[0]['pid'])
            # check for launch errors
            logs = follow.read('BARSM')
            errors = sum(1 for d in logs if "ERROR" in d.get('message'))
            if 0 < errors:
                print('ERROR: SYSLOG errors while waiting for AACM startup!')
                for d in logs:
                    if "ERROR" in d.get('message'):
                        print (d.get('message'))
                success = False
        if success:
            print('--AACM started')

    # setup TCP connection
    if success:
        print('-Setting up TCP connection')
        TCPconn, sVal = barsm_utils.TCPsetup()  # both return values are objects
        logs = follow.read('BARSM')
        errors = sum(1 for d in logs if "ERROR" in d.get('message'))
        if 0 < errors:
            print('ERROR: SYSLOG errors while initiating TCP connection!')
            for d in logs:
                if "ERROR" in d.get('message'):
                    print (d.get('message'))
            success = False
        else:
            print('--TCP connection setup successful')

    # receive BARSM to AACM init
    if success:
        print('-Waiting for BARSM to AACM init')
        success = barsm_utils.barsmToAacmInit(TCPconn)
        logs = follow.read('BARSM')
        errors = sum(1 for d in logs if "ERROR" in d.get('message'))
        if 0 < errors:
            print('ERROR: SYSLOG errors while receiving BARSM to AACM init!')
            for d in logs:
                if "ERROR" in d.get('message'):
                    print (d.get('message'))
            success = False
        else:
            print('--Init received')

    # send BARSM to AACM init ack
    if success:
        print('-Sending ACK')
        barsm_utils.barsmToAacmInitAck(TCPconn)
        logs = follow.read('BARSM')
        errors = sum(1 for d in logs if "ERROR" in d.get('message'))
        if 0 < errors:
            print('ERROR: SYSLOG errors while sending BARSM to AACM init ack!')
            for d in logs:
                if "ERROR" in d.get('message'):
                    print (d.get('message'))
            success = False
        else:
            print('--ACK sent')

    # wait for children to complete launching
    if success:
        print('-Waiting for launch completion')
        children, nameList = barsm_utils.waitForLaunch(follow)
        if 9 == children:
            print('--Launch complete!')
        else:
            print('ERROR: only {} child procs (should have 9)'.format(children))
            success = False

    # setup UDP connection
    if success:
        print('-Setting up UDP connection')
        UDPsock = barsm_utils.UDPsetup()
        logs = follow.read('BARSM')
        errors = sum(1 for d in logs if "ERROR" in d.get('message'))
        if 0 < errors:
            print('ERROR: SYSLOG errors while initiating UDP connection!')
            for d in logs:
                if "ERROR" in d.get('message'):
                    print (d.get('message'))
            success = False
        else:
            print('--UDP connection setup')

    # send sysInit message
    if success:
        print('-Waiting to receive OPEN message from BARSM')
        print('-Sending SYS_INIT message to BARSM')
        barsm_utils.sysInit(UDPsock)
        print('--OPEN received')

    # receive barsm to aacm processes
    if success:
        print('-Receiving BARSM_TO_AACM_PROCESSES')

        expectedData = {}
        expectedData['numMsgBytes']  = 126
        expectedData['numMsgDataBytes']  = 122
        # expect number of processes + 1 for BARSM + 1 for AACM
        expectedData['unpackFMT'] = '=HHH' + 'I4sI' * (len(nameList) + 2)
        expectedData['numProcesses'] = len(nameList) + 2

        barsm_utils.barsmToAacmProcesses(TCPconn, expectedData, aacm_pid, nameList, follow)
        print('--BARSM_TO_AACM_PROCESSES message received')

    # wait for barsm to aacm messgae and reply back
    if success:


        print('-Waiting for and responding to BARSM TO ACCM message 1')
        appName = log.logfollower()
        names = []
        expectedName = ''

        while not names and expectedName == '':
            names = appName.read('BARSM')
            try:
                expectedName = list(filter(lambda t: "DEBUG: Message sent with APP name:" in t['message'], names))[0]['message']
            except:
                expectedName = ''
                names = []

        expectedName = expectedName[-4:]

        expectedERR = 11
        barsm_utils.receiveBarsmToAacm(TCPconn, expectedERR, expectedName)

        print('-Waiting for and responding to BARSM TO ACCM message 2')
        names = []
        expectedName = ''

        while not names and expectedName == '':
            names = appName.read('BARSM')
            try:
                expectedName = list(filter(lambda t: "DEBUG: Message sent with APP name:" in t['message'], names))[0]['message']
            except:
                expectedName = ''
                names = []

        expectedName = expectedName[-4:]

        expectedERR = 11
        barsm_utils.receiveBarsmToAacm(TCPconn, expectedERR, expectedName)

        print('-Waiting for and responding to BARSM TO ACCM message 3')
        names = []
        expectedName = ''

        while not names and expectedName == '':
            names = appName.read('BARSM')
            try:
                expectedName = list(filter(lambda t: "DEBUG: Message sent with APP name:" in t['message'], names))[0]['message']
            except:
                expectedName = ''
                names = []

        expectedName = expectedName[-4:]

        expectedERR = 11
        barsm_utils.receiveBarsmToAacm(TCPconn, expectedERR, expectedName)

        print('-Waiting for and responding to BARSM TO ACCM message 4')
        names = []
        expectedName = ''

        while not names and expectedName == '':
            names = appName.read('BARSM')
            try:
                expectedName = list(filter(lambda t: "DEBUG: Message sent with APP name:" in t['message'], names))[0]['message']
            except:
                expectedName = ''
                names = []

        expectedName = expectedName[-4:]

        expectedERR = 11
        barsm_utils.receiveBarsmToAacm(TCPconn, expectedERR, expectedName)

        print('-Waiting for and responding to BARSM TO ACCM message 5')
        names = []
        expectedName = ''

        while not names and expectedName == '':
            names = appName.read('BARSM')
            try:
                expectedName = list(filter(lambda t: "DEBUG: Message sent with APP name:" in t['message'], names))[0]['message']
            except:
                expectedName = ''
                names = []

        expectedName = expectedName[-4:]

        expectedERR = 11
        #barsm_utils.get_Message(TCPconn, expectedERR)
        barsm_utils.receiveBarsmToAacm(TCPconn, expectedERR, expectedName)


     # make sure all 5 stopped processes have logged errors to go with
     # get the time of the launch finish
    logs = follow2.read('BARSM')
    if list(filter(lambda t: "ERROR: Process for /opt/rc360/system/thirty_aacm" in t['message'], logs)) == []:
        print("SYSLOG error missing thirty_aacm")
        success = False
    if list(filter(lambda t: "ERROR: Process for /opt/rc360/modules/GE/thirty_gemod" in t['message'], logs)) == [] :
        print("SYSLOG error missing thirty_gemod")
        success = False
    if list(filter(lambda t: "ERROR: Process for /opt/rc360/modules/TPA/thirty_tpamod" in t['message'], logs)) == [] :
        print("SYSLOG error missing thirty_tpamod")
        success = False
    if list(filter(lambda t: "ERROR: Process for /opt/rc360/apps/GE/thirty_geapp" in t['message'], logs)) == [] :
        print("SYSLOG error missing thirty_geapp")
        success = False
    if list(filter(lambda t: "ERROR: Process for /opt/rc360/apps/TPA/thirty_tpaapp" in t['message'], logs)) == [] :
        print("SYSLOG error missing thirty_tpaapp")
        success = False

    #make sure all 5 apps that failed were restarted

    # make sure all apps get started back up
    child_info = barsm_utils.get_barsm_child_pids()
    if list(filter(lambda t: "NOTICE: Restarting thirty_aacm..." in t['message'], logs)) == []:
        print("SYSLOG error missing restart of thirty_aacm")
        success = False
    if list(filter(lambda t: "NOTICE: Restarting thirty_gemod..." in t['message'], logs)) == [] :
        print("SYSLOG error missing restart of thirty_gemod")
        success = False
    if list(filter(lambda t: "NOTICE: Restarting thirty_tpamod..." in t['message'], logs)) == [] :
        print("SYSLOG error missing restart of thirty_tpamod")
        success = False
    if list(filter(lambda t: "NOTICE: Restarting thirty_geapp..." in t['message'], logs)) == [] :
        print("SYSLOG error missing restart of thirty_geapp")
        success = False
    if list(filter(lambda t: "NOTICE: Restarting thirty_tpaapp..." in t['message'], logs)) == [] :
        print("SYSLOG error missing restart of thirty_tpaapp")
        success = False

    zombies = sum(1 for c in child_info if "Z" == c.get('state'))

    # kill off all started processes
    barsm_utils.populateChildren()
    barsm_utils.kill_procs()

    if not zombies and success:
        print('Test Passes')
        return True
    else:
        print('Test Failed')
        return False