Esempio n. 1
0
def testRawIp(server, dataPort, filename):
    myUdp = rawUdpSendHandler(server, dataPort)
    if not os.path.exists(filename):
        print 'No such file %s. Exit' % filename
        sys.exit()
    myR = open(filename)
    msg = myR.read().strip()
    print 'udp data length: %i' % len(msg)
    myR.close()
    packets = generateRawIp.getRawIpPacket(msg, '1.1.1.1', server, dataPort)
    myUdp.sendoutEvent(packets)
Esempio n. 2
0
 def __init__(self, task, testConfig):
     baseTest.__init__(self, task, testConfig)
     self.deviceHandler = deviceHandler(testConfig.testServer.appServer)
     self.queryHandler = queryHandler(testConfig.testServer.appServer)
     if hasattr(testConfig, 'posix'):
         self.posix = True
         self.eventSender = rawUdpSendHandler.rawUdpSendHandler(
             testConfig.testServer.dataCollector, SYSLOG_PORT)
     else:
         self.posix = False
         self.eventSender = sendEventHandler(
             'syslog', self.testConfig.testServer.dataCollector)
Esempio n. 3
0
def sendRawPacket(srcAddr, destAddr, file):
    """This program will send a raw UDP packet with a fake source IP address
    inside the IP packet. Due to WinSock2 restriction, this function will not
    work in Windows system.
    sendRawPacket.py fake_src_ip dataCollector dataFile
    """
    myFile=open(file)
    myData=myFile.readlines()
    myFile.close()
    myUdp=rawUdpSendHandler.rawUdpSendHandler(destAddr, SYSLOG_PORT)
    for data in myData:
        myPacket=GenerateRawIPData.getRawIpPacket(data.strip(), srcAddr, destAddr, SYSLOG_PORT)
        myUdp.sendEvent(myPacket)

    myUdp.close()
Esempio n. 4
0
    def run(self, ruleType, testKey):
        fb_type = ruleType.filterOperators.type
        sleeper = 0
        if fb_type == 'FOLLOWED_BY':
            sleeper = 5
        incidentType = ruleType.incidentType.split('$')[-1]
        ruleId = ruleType.attribute['id']
        if incidentType not in self.testConfig.commonData.keys():
            print 'Incidnet Name %s Incident Type %s is NOT implemented.' % (
                ruleType.name, incidentType)
            testRet = None
        else:
            testConf = self.testConfig.commonData[incidentType]
            if testConf.reptDevIpAddr == '$localhost':
                testConf.reptDevIpAddr = self.testConfig.localhost
            elif testConf.reptDevIpAddr == '$appServer':
                testConf.reptDevIpAddr = self.appServer
            allDevices = self.testConfig.globalData['devices']
            approvedDevices = self.testConfig.globalData['applicableDevices']
            perfObj = self.testConfig.globalData['perfObj']
            #create device if needed
            if not allDevices or (
                    testConf.createDevice
                    and not testConf.reptDevIpAddr in allDevices.keys()):
                if testConf.domainController:
                    print 'create domain controller %s' % testConf.reptDevIpAddr
                    self.deviceHandler.createDevice(
                        testConf.reptDevIpAddr,
                        testConf.deviceName,
                        testConf.deviceType,
                        perfObj,
                        dataCollector=self.testConfig.testServer.dataCollector)
                else:
                    print 'create %s device %s' % (testConf.deviceType,
                                                   testConf.reptDevIpAddr)
                    self.deviceHandler.createDevice(testConf.reptDevIpAddr,
                                                    testConf.deviceName,
                                                    testConf.deviceType,
                                                    perfObj)
            else:
                print 'device %s is already exist.' % testConf.reptDevIpAddr
                #in case to make it domain controller
                if testConf.domainController:
                    print 'make domain controller %s' % testConf.reptDevIpAddr
                    self.deviceHandler.createDevice(
                        testConf.reptDevIpAddr,
                        testConf.deviceName,
                        testConf.deviceType,
                        perfObj,
                        dataCollector=self.testConfig.testServer.dataCollector)

            #get raw data
            myData = datFileHandler.getData(
                self.path + '/' + incident_data_path + '/' + incidentType +
                '.dat', incident_data_keys)
            #send raw event to trigger incident
            if not myData.dataMap:
                print 'Fail to get test data. Exit.'
                exit()
            rept = {}
            rept['$localhost'] = self.testConfig.localhost
            rept['$dataCollector'] = self.testConfig.testServer.dataCollector
            eventMsgs = myData.dataMap['default'].eventMsg
            if eventMsgs is None:
                print '%s: No eventMsg exist.' % incidentType
                exit()
            rawSend = False
            if self.posix and testConf.reptDevIpAddr != self.testConfig.localhost:
                rawSend = True
            if rawSend:
                mySendEvent = rawUdpSendHandler.rawUdpSendHandler(
                    self.testConfig.testServer.dataCollector,
                    PORTS[testConf.method])
            else:
                mySendEvent = sendEventHandler(
                    testConf.method, self.testConfig.testServer.dataCollector)
            sendSleep = 0
            randomIPs = []
            randomNums = []
            ip = ''
            num = ''
            now, sendTime, utcnow, utcsendTime = timeUtility.getTimeNow()
            for i in range(int(testConf.count)):
                for line in eventMsgs:
                    msg = generalUtility.multiReplace(line, rept)
                    if '$reporter' in line:
                        msg = msg.replace('$reporter', testConf.reptDevIpAddr)
                    if '$randomIP' in line:
                        repeat = True
                        while repeat:
                            ip = randomGen.getRandomIPAddr()
                            if ip not in randomIPs:
                                if testConf.incidentType not in ip_in_privatenet:
                                    msg = msg.replace('$randomIP', ip)
                                    randomIPs.append(ip)
                                    repeat = False
                                else:
                                    if ip.split('.')[0] in ['10']:
                                        msg = msg.replace('$randomIP', ip)
                                        randomIPs.append(ip)
                                        repeat = False
                    if '$randomNum' in line:
                        num_repeat = True
                        while num_repeat:
                            num = randomGen.getRandomNum(1, 1000)
                            if num not in randomNums:
                                msg = msg.replace('$randomNum', num)
                                randomNums.append(num)
                                num_repeat = False
                    if '$group_' in line:
                        groupName = line.split('$group_')[-1].split('@')[0]
                        groupItem = self.restApiHandler.getData(
                            group_name_key[groupName], module='namedValue')
                        value = ''
                        if groupItem:
                            rawValue = groupItem[
                                group_name_key[groupName]].namedValues[0]
                            if '-' in rawValue:
                                temp = rawValue.split('-')[0].split('.')
                                temp[-1] = '100'
                                value = '.'.join(temp)
                            else:
                                value = rawValue
                        msg = msg.replace('$group_' + groupName + '@', value)
                    temp_msg = ''
                    if testConf.method == 'netflow':
                        temp_msg = GenerateNetFlow.getNetFlowPacket(msg)
                    else:
                        temp_msg = msg.strip()
                    send_msg = ''
                    if rawSend:
                        if testConf.method == 'syslog':
                            temp_msg = temp_msg.encode('ascii', 'ignore')
                        send_msg = GenerateRawIPData.getRawIpPacket(
                            temp_msg, testConf.reptDevIpAddr,
                            self.testConfig.testServer.dataCollector,
                            PORTS[testConf.method])
                    else:
                        send_msg = msg
                    time.sleep(sleeper)
                    if not self.sendNoEvent:
                        mySendEvent.sendoutEvent(send_msg, utf_8=False)
                    else:
                        print 'No event sent being configured.'
                    self.msgList.append(msg)
            #retrieve incident
            sendSleep = 120
            time.sleep(sendSleep)
            timeout = int(ruleType.triggerWindow) + 180
            myParams = {}
            myParams['constr'] = incident_query_params['SingleEvtConstr'] % (
                testConf.reptDevIpAddr, incidentType)
            if self.sendNoEvent:
                print 'Query:,', myParams['constr']
            condition, oriRet, incidentId, failDetail, veriData, debugInfo = self.retriveIncident(
                incidentType, testConf, timeout, myParams)
            incident_id_sql = None
            if not condition:  #check CMDB for incident
                rawResult = self.psql.execute(
                    incident_sql_cmd %
                    (incidentType, testConf.reptDevIpAddr, sendTime * 1000))
                if rawResult:
                    incident_id_sql = rawResult[0][0]
                    condition = True
                    failDetail = 'Incident Id found in SQL but not from query.'
                else:
                    print 'no id in SQL'
            if not condition and not self.sendNoEvent:
                failDetail = "need debug"  #self.eventDebug(testConf.method, self.testConfig.testServer.appServer, testConf.reptDevIpAddr, ruleType, sendTime, utcsendTime, approvedDevices)
            testRet = getClassObj('TestSuiteResult', module='autoTest')
            testRet.name = testConf.name
            testRet.type = incidentType
            testRet.ruleId = ruleId
            testRet.queryString = myParams['constr']
            testRet.rawMsg = self.msgList
            testRet.testMethod = testConf.method
            testRet.reptDevIpAddr = testConf.reptDevIpAddr
            testRet.taskName = 'Incident'
            testRet.totalRun = 1
            testRet.debugInfo = debugInfo
            setattr(testRet, 'total' + oriRet.status, 1)
            testRet.caseList.append(oriRet)
            if incidentId:
                print '%s: incident triggered with id: %s' % (incidentType,
                                                              incidentId)
                testRet.info = 'incidentId: ' + incidentId
            elif incident_id_sql:
                print '%s: incident triggered with id from sql: %s' % (
                    incidentType, incident_id_sql)
                testRet.info = 'incidentId from SQL: ' + incident_id_sql
                setattr(oriRet, 'reasons', failDetail)
            else:
                print '%s(%s): no incident triggered' % (testRet.name,
                                                         incidentType)
                testRet.info = 'incidentId: None'
                setattr(oriRet, 'reasons', failDetail)
            aggmsgList = []
            if self.advance in ['aggregate', 'both']:
                if oriRet.status != 'NoReturn' and incidentId and incidentId.strip(
                ):
                    #aggregate test
                    for i in range(int(testConf.count)):
                        for m in self.msgList:
                            mySendEvent.sendoutEvent(m, utf_8=False)
                    time.sleep(60)
                    aggParams = {}
                    aggParams['constr'] = incident_query_advance[
                        'SingleEvtConstr'] % (incidentId, '0')
                    condition, aggRet, inId, failDetail, aggVeriData, aggData = self.retriveIncident(
                        incidentType,
                        testConf,
                        timeout,
                        aggParams,
                        agg=veriData)
                    if not condition:
                        failDetail = self.eventDebug(
                            testConf.method,
                            self.testConfig.testServer.appServer,
                            testConf.reptDevIpAddr, ruleType, sendTime,
                            utcsendTime, approvedDevices)
                    if failDetail:
                        setattr(aggRet, 'reasons', failDetail)
                    testRet.totalRun += 1
                    oldVal = getattr(testRet, 'total' + aggRet.status)
                    oldVal += 1
                    setattr(testRet, 'total' + aggRet.status, oldVal)
                    testRet.caseList.append(aggRet)
            if self.advance in ['clear', 'both']:  #clear test
                if oriRet.status != 'NoReturn' and hasattr(
                        ruleType, 'clearCondition'
                ) and incidentId and incidentId.strip():
                    clearnow, clearsendTime, clearutcnow, clearutcsendTime = timeUtility.getTimeNow(
                    )
                    if ruleType.clearCondition.clearOption == 'patternbased':
                        if not hasattr(myData.dataMap['default'],
                                       'clearEventMsg'):
                            print '%s: need to add clearEventMsg' % incidentType
                        else:
                            raw_clear = getattr(myData.dataMap['default'],
                                                'clearEventMsg')
                            time.sleep(600)
                            if raw_clear:
                                for line in raw_clear:
                                    clearmsg = generalUtility.multiReplace(
                                        line.strip(), rept)
                                    if '$reporter' in line:
                                        clearmsg = clearmsg.replace(
                                            '$reporter',
                                            testConf.reptDevIpAddr)
                                    if '$randomIP' in line:
                                        ip = randomGen.getRandomIPAddr()
                                        clearmsg = clearmsg.replace(
                                            '$randomIP', ip)
                                    if self.posix:
                                        send_clearmsg = GenerateRawIPData.getRawIpPacket(
                                            clearmsg.encode('ascii', 'ignore'),
                                            testConf.reptDevIpAddr,
                                            self.testConfig.testServer.
                                            dataCollector,
                                            PORTS[testConf.method])
                                    else:
                                        send_clearmsg = clearmsg
                                    self.clearMsgList.append(send_clearmsg)
                                if self.clearMsgList:
                                    for i in range(int(testConf.count)):
                                        for cl_msg in self.clearMsgList:
                                            mySendEvent.sendoutEvent(
                                                cl_msg, utf_8=False)
                                    time.sleep(120)
                    else:
                        time.sleep(
                            float(
                                int(ruleType.clearCondition.clearTimeWindow) +
                                120))
                    clearParams = {}
                    clearParams['constr'] = incident_query_advance[
                        'SingleEvtConstr'] % (incidentId, '1')
                    condition, clearRet, incidentId, failDetail, clearVari, clearData = self.retriveIncident(
                        incidentType,
                        testConf,
                        timeout,
                        clearParams,
                        clear=True)
                    if not condition:
                        failDetail = "need debug"  #self.eventDebug(testConf.method, self.testConfig.testServer.appServer, testConf.reptDevIpAddr, ruleType, clearsendTime, clearutcsendTime, approvedDevices, clearDebug=True)
                    if failDetail:
                        setattr(clearRet, 'reasons', failDetail)
                    print 'clear name: %s' % clearRet.name
                    testRet.totalRun += 1
                    oldVal = getattr(testRet, 'total' + clearRet.status)
                    oldVal += 1
                    setattr(testRet, 'total' + clearRet.status, oldVal)
                    testRet.caseList.append(clearRet)
            mySendEvent.close()

        return testRet
Esempio n. 5
0
    def run(self, etype, module):
        params = {}
        path = self.path + '/' + module
        mySendEvent = ''
        # Ironportweb events will use IronportMailparser if IronportMail events come first -- by design.
        if module in sendevent_holdon_module:
            time.sleep(480)
        for key in etype.keys():
            myVal = etype[key]
            if myVal.key:
                myKey = myVal.reptDevIpAddr + ' (' + myVal.key + ')'
            else:
                myKey = myVal.reptDevIpAddr
            fileName = myVal.eventType + '.dat'
            for rep in event_replace_symbol:
                if rep in fileName:
                    fileName = fileName.replace(rep, '$')
            myData = datFileHandler.getData(path + '/' + fileName,
                                            event_data_keys)
            if myVal.reptDevIpAddr == self.testConfig.localhost:
                mkey = myKey.replace(myVal.reptDevIpAddr, '$localhost')
                msg = getattr(myData.dataMap[mkey], 'eventMsg')
                myPars = getattr(myData.dataMap[mkey], 'params')
            else:
                msg = getattr(myData.dataMap[myKey], 'eventMsg')
                myPars = getattr(myData.dataMap[myKey], 'params')
            replaceDict = {}
            replaceDict['$reporter'] = myVal.reptDevIpAddr
            replaceDict['$localhost'] = self.testConfig.localhost
            if '$localhost' in msg or '$reporter' in msg:
                msg = generalUtility.multiReplace(msg, replaceDict)
            if hasattr(self.testConfig, 'globalData'):
                custId = custIdUtil.getCustIdbyIp(
                    self.testConfig.globalData, myVal.reptDevIpAddr,
                    self.testConfig.testServer.dataCollector)
            else:
                custId = '1'
            newPar = {}
            for pkey in myPars.keys():
                if myPars[pkey] == '$localhost':
                    myPars[pkey] = self.testConfig.localhost
                if myPars[pkey] == '$reporter':
                    myPars[pkey] = myVal.reptDevIpAddr
                if myPars[pkey] == '$sender':
                    myPars[pkey] = custId
            params[myVal.eventType + '@' + myKey] = {}
            params[myVal.eventType + '@' + myKey]['params'] = myPars
            params[myVal.eventType + '@' + myKey]['msg'] = msg
            ver = ''
            comm = ''
            if key not in no_raw_send and module not in msg_too_long and self.posix and 'snmptrap' not in myVal.method:
                import Libs.rawUdpSendHandler as rawUdpSendHandler
                import Libs.GenerateRawIPData as GenerateRawIPData
                rawmsg = GenerateRawIPData.getRawIpPacket(
                    msg.encode('ascii',
                               'xmlcharrefreplace'), myVal.reptDevIpAddr,
                    self.testConfig.testServer.dataCollector, 514)
                self.rawUdpSender = rawUdpSendHandler.rawUdpSendHandler(
                    self.testConfig.testServer.dataCollector, 514)
                now, sendTime, unow, usendTime = timeUtility.getTimeNow()
                self.rawUdpSender.sendoutEvent(rawmsg)
            else:
                if 'snmptrap@' in myVal.method:
                    sendMethod, ver, comm = myVal.method.split('@')
                else:
                    sendMethod = myVal.method
                if not mySendEvent:
                    mySendEvent = sendEventHandler(
                        sendMethod, self.testConfig.testServer.dataCollector)
                now, sendTime, unow, usendTime = timeUtility.getTimeNow()
                if ver and comm:
                    mySendEvent.sendoutEvent(msg,
                                             version=ver,
                                             community=comm,
                                             utf_8=True)
                else:
                    mySendEvent.sendoutEvent(msg, utf_8=True)
        #get query
        time.sleep(300)
        myParams = {}
        if module in parser_holdon_module:
            time.sleep(360)
        for key in event_query_params.keys():
            myParams[key] = event_query_params[key]
        eventTypes = []
        reporters = []
        for myKey in params.keys():
            eventType, reporter = myKey.split(' (')[0].split('@')
            if eventType.strip() not in eventTypes:
                eventTypes.append(eventType.strip())
            if reporter.strip() not in reporters:
                reporters.append(reporter)
        finalEventTypes = []
        if len(eventTypes) > 500:
            num = generalUtility.splitByNum(len(eventTypes), 500)
            finalEventTypes.append(eventTypes[0:500])
            for i in range(num - 1):
                s = (i + 1) * 500
                e = (i + 2) * 500
                subList = eventTypes[s:e]
                finalEventTypes.append(subList)
        else:
            finalEventTypes.append(eventTypes)
        reporterStr = ','.join(reporters)
        retData = {}
        for i in range(len(finalEventTypes)):
            eventtypeStr = '","'.join(finalEventTypes[i])
            myParams['constr'] = event_query_params['constr'] % (reporterStr,
                                                                 eventtypeStr)
            retSubData, debugInfo = self.queryHandler.getQuery(myParams)
            for key in retSubData.keys():
                newList = []
                for item in retSubData[key]:
                    rTime, zone = timeUtility.getUTimeFromString(
                        item.attributes['phRecvTime'])
                    if zone == 'UTC':
                        if rTime + 300 >= usendTime:
                            newList.append(item)
                    else:
                        if rTime + 300 >= sendTime:
                            newList.append(item)
                if newList:
                    retData[key] = newList

        finalRet = self.verifyData(etype, params, retData, debugInfo, module,
                                   sendTime)
        if finalRet.totalMissing or finalRet.totalExtra:
            self.__updateCase(finalRet.caseList, path)
        return finalRet
Esempio n. 6
0
    def run(self, ruleType, testKey):
        fb_type=ruleType.filterOperators.type
        sleeper=0
        if fb_type=='FOLLOWED_BY':
            sleeper=5
        incidentType=ruleType.incidentType.split('$')[-1]
        ruleId=ruleType.attribute['id']
        if incidentType not in self.testConfig.commonData.keys():
            print 'Incidnet Name %s Incident Type %s is NOT implemented.' % (ruleType.name, incidentType)
            testRet=None
        else:
            testConf=self.testConfig.commonData[incidentType]
            if testConf.reptDevIpAddr=='$localhost':
                testConf.reptDevIpAddr=self.testConfig.localhost
            elif testConf.reptDevIpAddr=='$appServer':
                testConf.reptDevIpAddr=self.appServer
            allDevices=self.testConfig.globalData['devices']
            approvedDevices=self.testConfig.globalData['applicableDevices']
            perfObj=self.testConfig.globalData['perfObj']
            #create device if needed
            if not allDevices or (testConf.createDevice and not testConf.reptDevIpAddr in allDevices.keys()):
                if testConf.domainController:
                    self.deviceHandler.createDevice(testConf.reptDevIpAddr, testConf.deviceName, testConf.deviceType, perfObj, dataCollector=self.testConfig.testServer.dataCollector)
                else:
                    self.deviceHandler.createDevice(testConf.reptDevIpAddr, testConf.deviceName, testConf.deviceType, perfObj)
            #get raw data
            myData=datFileHandler.getData(self.path+'/'+incident_data_path+'/'+incidentType+'.dat', incident_data_keys)
            #send raw event to trigger incident
            if not myData.dataMap:
                print 'Fail to get test data. Exit.'
                exit()
            now,sendTime,utcnow,utcsendTime=timeUtility.getTimeNow()
            rept={}
            rept['$localhost']=self.testConfig.localhost
            rept['$dataCollector']=self.testConfig.testServer.dataCollector
            eventMsgs=myData.dataMap['default'].eventMsg
            if eventMsgs is None:
               print '%s: No eventMsg exist.' % incidentType
               exit()
            rawSend=False
            if self.posix and testConf.reptDevIpAddr!=self.testConfig.localhost:
                rawSend=True
            if rawSend:
                mySendEvent=rawUdpSendHandler.rawUdpSendHandler(self.testConfig.testServer.dataCollector, PORTS[testConf.method])
            else:
                mySendEvent=sendEventHandler(testConf.method, self.testConfig.testServer.dataCollector)
            for i in range(int(testConf.count)):
                for line in eventMsgs:
                    msg=generalUtility.multiReplace(line, rept)
                    if '$reporter' in line:
                        msg=msg.replace('$reporter', testConf.reptDevIpAddr)
                    if '$randomIP' in line:
                        ip=randomGen.getRandomIPAddr()
                        msg=msg.replace('$randomIP', ip)
                    if '$randomNum' in line:
                        num=randomGen.getRandomNum(1, 100)
                        msg=msg.replace('$randomNum', num)
                    temp_msg=''
                    if testConf.method=='netflow':
                        temp_msg=GenerateNetFlow.getNetFlowPacket(msg)
                    else:
                        temp_msg=msg.strip()
                    send_msg=''
                    if rawSend:
                        send_msg=GenerateRawIPData.getRawIpPacket(temp_msg, testConf.reptDevIpAddr, self.testConfig.testServer.dataCollector, PORTS[testConf.method])
                    else:
                        send_msg=msg
                    time.sleep(sleeper)
                    mySendEvent.sendoutEvent(send_msg)
                    self.msgList.append(msg)
            #retrieve incident
            time.sleep(60)
            timeout=int(ruleType.triggerWindow)+60
            if timeout>1800:
                timeout=1800
            myParams={}
            myParams['constr']=incident_query_params['SingleEvtConstr'] % (testConf.reptDevIpAddr, incidentType)
            condition, oriRet, incidentId, failDetail, veriData, debugInfo=self.retriveIncident(incidentType, testConf, timeout, myParams)
            if not condition:
                failDetail=self.eventDebug(testConf.method, self.testConfig.testServer.appServer, testConf.reptDevIpAddr, ruleType, sendTime, utcsendTime, approvedDevices)
            testRet=getClassObj('TestSuiteResult', module='autoTest')
            testRet.name=testConf.name
            testRet.type=incidentType
            testRet.ruleId=ruleId
            testRet.queryString=myParams['constr']
            testRet.rawMsg=self.msgList
            testRet.testMethod=testConf.method
            testRet.reptDevIpAddr=testConf.reptDevIpAddr
            testRet.taskName='Incident'
            testRet.totalRun=1
            testRet.debugInfo=debugInfo
            setattr(testRet, 'total'+oriRet.status, 1)
            testRet.caseList.append(oriRet)
            if incidentId:
                print '%s: incident triggered with id: %s' % (incidentType, incidentId)
                testRet.info='incidentId: '+incidentId
            else:
                print '%s: no incident triggered' % incidentType
                testRet.info='incidentId: None'
                setattr(oriRet, 'reasons', failDetail)
            aggmsgList=[]
            if self.advance:
                if oriRet.status!='NoReturn' and incidentId and incidentId.strip():
                    #aggregate test
                    for i in range(int(testConf.count)):
                        for m in self.msgList:
                            mySendEvent.sendoutEvent(m)
                    time.sleep(60)
                    aggParams={}
                    aggParams['constr']=incident_query_advance['SingleEvtConstr'] % (incidentId, '0')
                    condition, aggRet, inId, failDetail, aggData=self.retriveIncident(incidentType, testConf, timeout, aggParams, agg=veriData)
                    if not condition:
                        failDetail=self.eventDebug(testConf.method, self.testConfig.testServer.appServer, testConf.reptDevIpAddr, ruleType, sendTime, utcsendTime, approvedDevices)
                    if failDetail:
                        setattr(aggRet, 'reasons', failDetail)
                    testRet.totalRun+=1
                    oldVal=getattr(testRet, 'total'+aggRet.status)
                    oldVal+=1
                    setattr(testRet, 'total'+aggRet.status, oldVal)
                    testRet.caseList.append(aggRet)
                    #clear test
                    if oriRet.status!='NoReturn' and hasattr(ruleType, 'clearCondition') and incidentId and incidentId.strip():
                        clearnow,clearsendTime,clearutcnow,clearutcsendTime=timeUtility.getTimeNow()
                        if ruleType.clearCondition.clearOption=='patternbased':
                            if not hasattr(myData.dataMap['default'], 'clearEventMsg'):
                                print '%s: need to add clearEventMsg' % incidentType
                            else:
                                raw_clear=getattr(myData.dataMap['default'], 'clearEventMsg')
                                if raw_clear:
                                    for line in raw_clear:
                                        clearmsg=generalUtility.multiReplace(line.strip(), rept)
                                        if '$randomIP' in line:
                                            ip=randomGen.getRandomIPAddr()
                                            clearmsg=clearmsg.replace('$randomIP', ip)
                                        self.clearMsgList.append(clearmsg)
                                        if self.posix:
                                            send_clearmsg=GenerateRawIPData.getRawIpPacket(clearmsg, testConf.reptDevIpAddr, self.testConfig.testServer.dataCollector, PORTS[testConf.method])
                                        else:
                                            send_clearmsg=clearmsg
                                    if self.clearMsgList:
                                        for i in range(int(testConf.count)):
                                            for cl_msg in self.clearMsgList:
                                                mySendEvent.sendoutEvent(cl_msg)
                                        time.sleep(120)
                        else:
                            time.sleep(float(int(ruleType.clearCondition.clearTimeWindow)+120))
                        clearParams={}
                        clearParams['constr']=incident_query_advance['SingleEvtConstr'] % (incidentId, '1')
                        condition, clearRet, incidentId, failDetail, clearData=self.retriveIncident(incidentType, testConf, timeout, clearParams, clear=aggData)
                        if not condition:
                            failDetail=self.eventDebug(testConf.method, self.testConfig.testServer.appServer, testConf.reptDevIpAddr, ruleType, clearsendTime, clearutcsendTime, approvedDevices, clearDebug=True)
                        if failDetail:
                            setattr(clearRet, 'reasons', failDetail)
                        print 'clear name: %s' % clearRet.name
                        testRet.totalRun+=1
                        oldVal=getattr(testRet, 'total'+clearRet.status)
                        oldVal+=1
                        setattr(testRet, 'total'+clearRet.status, oldVal)
                        testRet.caseList.append(clearRet)
            mySendEvent.close()

        return testRet