def testRawIp(server, dataPort, filename): myUdp = rawUdpSendHandler(server, dataPort) if not os.path.exists(filename): print 'No such file %s. Exit' % filename sys.exit() myR = open(filename) msg = myR.read().strip() print 'udp data length: %i' % len(msg) myR.close() packets = generateRawIp.getRawIpPacket(msg, '1.1.1.1', server, dataPort) myUdp.sendoutEvent(packets)
def __init__(self, task, testConfig): baseTest.__init__(self, task, testConfig) self.deviceHandler = deviceHandler(testConfig.testServer.appServer) self.queryHandler = queryHandler(testConfig.testServer.appServer) if hasattr(testConfig, 'posix'): self.posix = True self.eventSender = rawUdpSendHandler.rawUdpSendHandler( testConfig.testServer.dataCollector, SYSLOG_PORT) else: self.posix = False self.eventSender = sendEventHandler( 'syslog', self.testConfig.testServer.dataCollector)
def sendRawPacket(srcAddr, destAddr, file): """This program will send a raw UDP packet with a fake source IP address inside the IP packet. Due to WinSock2 restriction, this function will not work in Windows system. sendRawPacket.py fake_src_ip dataCollector dataFile """ myFile=open(file) myData=myFile.readlines() myFile.close() myUdp=rawUdpSendHandler.rawUdpSendHandler(destAddr, SYSLOG_PORT) for data in myData: myPacket=GenerateRawIPData.getRawIpPacket(data.strip(), srcAddr, destAddr, SYSLOG_PORT) myUdp.sendEvent(myPacket) myUdp.close()
def run(self, ruleType, testKey): fb_type = ruleType.filterOperators.type sleeper = 0 if fb_type == 'FOLLOWED_BY': sleeper = 5 incidentType = ruleType.incidentType.split('$')[-1] ruleId = ruleType.attribute['id'] if incidentType not in self.testConfig.commonData.keys(): print 'Incidnet Name %s Incident Type %s is NOT implemented.' % ( ruleType.name, incidentType) testRet = None else: testConf = self.testConfig.commonData[incidentType] if testConf.reptDevIpAddr == '$localhost': testConf.reptDevIpAddr = self.testConfig.localhost elif testConf.reptDevIpAddr == '$appServer': testConf.reptDevIpAddr = self.appServer allDevices = self.testConfig.globalData['devices'] approvedDevices = self.testConfig.globalData['applicableDevices'] perfObj = self.testConfig.globalData['perfObj'] #create device if needed if not allDevices or ( testConf.createDevice and not testConf.reptDevIpAddr in allDevices.keys()): if testConf.domainController: print 'create domain controller %s' % testConf.reptDevIpAddr self.deviceHandler.createDevice( testConf.reptDevIpAddr, testConf.deviceName, testConf.deviceType, perfObj, dataCollector=self.testConfig.testServer.dataCollector) else: print 'create %s device %s' % (testConf.deviceType, testConf.reptDevIpAddr) self.deviceHandler.createDevice(testConf.reptDevIpAddr, testConf.deviceName, testConf.deviceType, perfObj) else: print 'device %s is already exist.' % testConf.reptDevIpAddr #in case to make it domain controller if testConf.domainController: print 'make domain controller %s' % testConf.reptDevIpAddr self.deviceHandler.createDevice( testConf.reptDevIpAddr, testConf.deviceName, testConf.deviceType, perfObj, dataCollector=self.testConfig.testServer.dataCollector) #get raw data myData = datFileHandler.getData( self.path + '/' + incident_data_path + '/' + incidentType + '.dat', incident_data_keys) #send raw event to trigger incident if not myData.dataMap: print 'Fail to get test data. Exit.' exit() rept = {} rept['$localhost'] = self.testConfig.localhost rept['$dataCollector'] = self.testConfig.testServer.dataCollector eventMsgs = myData.dataMap['default'].eventMsg if eventMsgs is None: print '%s: No eventMsg exist.' % incidentType exit() rawSend = False if self.posix and testConf.reptDevIpAddr != self.testConfig.localhost: rawSend = True if rawSend: mySendEvent = rawUdpSendHandler.rawUdpSendHandler( self.testConfig.testServer.dataCollector, PORTS[testConf.method]) else: mySendEvent = sendEventHandler( testConf.method, self.testConfig.testServer.dataCollector) sendSleep = 0 randomIPs = [] randomNums = [] ip = '' num = '' now, sendTime, utcnow, utcsendTime = timeUtility.getTimeNow() for i in range(int(testConf.count)): for line in eventMsgs: msg = generalUtility.multiReplace(line, rept) if '$reporter' in line: msg = msg.replace('$reporter', testConf.reptDevIpAddr) if '$randomIP' in line: repeat = True while repeat: ip = randomGen.getRandomIPAddr() if ip not in randomIPs: if testConf.incidentType not in ip_in_privatenet: msg = msg.replace('$randomIP', ip) randomIPs.append(ip) repeat = False else: if ip.split('.')[0] in ['10']: msg = msg.replace('$randomIP', ip) randomIPs.append(ip) repeat = False if '$randomNum' in line: num_repeat = True while num_repeat: num = randomGen.getRandomNum(1, 1000) if num not in randomNums: msg = msg.replace('$randomNum', num) randomNums.append(num) num_repeat = False if '$group_' in line: groupName = line.split('$group_')[-1].split('@')[0] groupItem = self.restApiHandler.getData( group_name_key[groupName], module='namedValue') value = '' if groupItem: rawValue = groupItem[ group_name_key[groupName]].namedValues[0] if '-' in rawValue: temp = rawValue.split('-')[0].split('.') temp[-1] = '100' value = '.'.join(temp) else: value = rawValue msg = msg.replace('$group_' + groupName + '@', value) temp_msg = '' if testConf.method == 'netflow': temp_msg = GenerateNetFlow.getNetFlowPacket(msg) else: temp_msg = msg.strip() send_msg = '' if rawSend: if testConf.method == 'syslog': temp_msg = temp_msg.encode('ascii', 'ignore') send_msg = GenerateRawIPData.getRawIpPacket( temp_msg, testConf.reptDevIpAddr, self.testConfig.testServer.dataCollector, PORTS[testConf.method]) else: send_msg = msg time.sleep(sleeper) if not self.sendNoEvent: mySendEvent.sendoutEvent(send_msg, utf_8=False) else: print 'No event sent being configured.' self.msgList.append(msg) #retrieve incident sendSleep = 120 time.sleep(sendSleep) timeout = int(ruleType.triggerWindow) + 180 myParams = {} myParams['constr'] = incident_query_params['SingleEvtConstr'] % ( testConf.reptDevIpAddr, incidentType) if self.sendNoEvent: print 'Query:,', myParams['constr'] condition, oriRet, incidentId, failDetail, veriData, debugInfo = self.retriveIncident( incidentType, testConf, timeout, myParams) incident_id_sql = None if not condition: #check CMDB for incident rawResult = self.psql.execute( incident_sql_cmd % (incidentType, testConf.reptDevIpAddr, sendTime * 1000)) if rawResult: incident_id_sql = rawResult[0][0] condition = True failDetail = 'Incident Id found in SQL but not from query.' else: print 'no id in SQL' if not condition and not self.sendNoEvent: failDetail = "need debug" #self.eventDebug(testConf.method, self.testConfig.testServer.appServer, testConf.reptDevIpAddr, ruleType, sendTime, utcsendTime, approvedDevices) testRet = getClassObj('TestSuiteResult', module='autoTest') testRet.name = testConf.name testRet.type = incidentType testRet.ruleId = ruleId testRet.queryString = myParams['constr'] testRet.rawMsg = self.msgList testRet.testMethod = testConf.method testRet.reptDevIpAddr = testConf.reptDevIpAddr testRet.taskName = 'Incident' testRet.totalRun = 1 testRet.debugInfo = debugInfo setattr(testRet, 'total' + oriRet.status, 1) testRet.caseList.append(oriRet) if incidentId: print '%s: incident triggered with id: %s' % (incidentType, incidentId) testRet.info = 'incidentId: ' + incidentId elif incident_id_sql: print '%s: incident triggered with id from sql: %s' % ( incidentType, incident_id_sql) testRet.info = 'incidentId from SQL: ' + incident_id_sql setattr(oriRet, 'reasons', failDetail) else: print '%s(%s): no incident triggered' % (testRet.name, incidentType) testRet.info = 'incidentId: None' setattr(oriRet, 'reasons', failDetail) aggmsgList = [] if self.advance in ['aggregate', 'both']: if oriRet.status != 'NoReturn' and incidentId and incidentId.strip( ): #aggregate test for i in range(int(testConf.count)): for m in self.msgList: mySendEvent.sendoutEvent(m, utf_8=False) time.sleep(60) aggParams = {} aggParams['constr'] = incident_query_advance[ 'SingleEvtConstr'] % (incidentId, '0') condition, aggRet, inId, failDetail, aggVeriData, aggData = self.retriveIncident( incidentType, testConf, timeout, aggParams, agg=veriData) if not condition: failDetail = self.eventDebug( testConf.method, self.testConfig.testServer.appServer, testConf.reptDevIpAddr, ruleType, sendTime, utcsendTime, approvedDevices) if failDetail: setattr(aggRet, 'reasons', failDetail) testRet.totalRun += 1 oldVal = getattr(testRet, 'total' + aggRet.status) oldVal += 1 setattr(testRet, 'total' + aggRet.status, oldVal) testRet.caseList.append(aggRet) if self.advance in ['clear', 'both']: #clear test if oriRet.status != 'NoReturn' and hasattr( ruleType, 'clearCondition' ) and incidentId and incidentId.strip(): clearnow, clearsendTime, clearutcnow, clearutcsendTime = timeUtility.getTimeNow( ) if ruleType.clearCondition.clearOption == 'patternbased': if not hasattr(myData.dataMap['default'], 'clearEventMsg'): print '%s: need to add clearEventMsg' % incidentType else: raw_clear = getattr(myData.dataMap['default'], 'clearEventMsg') time.sleep(600) if raw_clear: for line in raw_clear: clearmsg = generalUtility.multiReplace( line.strip(), rept) if '$reporter' in line: clearmsg = clearmsg.replace( '$reporter', testConf.reptDevIpAddr) if '$randomIP' in line: ip = randomGen.getRandomIPAddr() clearmsg = clearmsg.replace( '$randomIP', ip) if self.posix: send_clearmsg = GenerateRawIPData.getRawIpPacket( clearmsg.encode('ascii', 'ignore'), testConf.reptDevIpAddr, self.testConfig.testServer. dataCollector, PORTS[testConf.method]) else: send_clearmsg = clearmsg self.clearMsgList.append(send_clearmsg) if self.clearMsgList: for i in range(int(testConf.count)): for cl_msg in self.clearMsgList: mySendEvent.sendoutEvent( cl_msg, utf_8=False) time.sleep(120) else: time.sleep( float( int(ruleType.clearCondition.clearTimeWindow) + 120)) clearParams = {} clearParams['constr'] = incident_query_advance[ 'SingleEvtConstr'] % (incidentId, '1') condition, clearRet, incidentId, failDetail, clearVari, clearData = self.retriveIncident( incidentType, testConf, timeout, clearParams, clear=True) if not condition: failDetail = "need debug" #self.eventDebug(testConf.method, self.testConfig.testServer.appServer, testConf.reptDevIpAddr, ruleType, clearsendTime, clearutcsendTime, approvedDevices, clearDebug=True) if failDetail: setattr(clearRet, 'reasons', failDetail) print 'clear name: %s' % clearRet.name testRet.totalRun += 1 oldVal = getattr(testRet, 'total' + clearRet.status) oldVal += 1 setattr(testRet, 'total' + clearRet.status, oldVal) testRet.caseList.append(clearRet) mySendEvent.close() return testRet
def run(self, etype, module): params = {} path = self.path + '/' + module mySendEvent = '' # Ironportweb events will use IronportMailparser if IronportMail events come first -- by design. if module in sendevent_holdon_module: time.sleep(480) for key in etype.keys(): myVal = etype[key] if myVal.key: myKey = myVal.reptDevIpAddr + ' (' + myVal.key + ')' else: myKey = myVal.reptDevIpAddr fileName = myVal.eventType + '.dat' for rep in event_replace_symbol: if rep in fileName: fileName = fileName.replace(rep, '$') myData = datFileHandler.getData(path + '/' + fileName, event_data_keys) if myVal.reptDevIpAddr == self.testConfig.localhost: mkey = myKey.replace(myVal.reptDevIpAddr, '$localhost') msg = getattr(myData.dataMap[mkey], 'eventMsg') myPars = getattr(myData.dataMap[mkey], 'params') else: msg = getattr(myData.dataMap[myKey], 'eventMsg') myPars = getattr(myData.dataMap[myKey], 'params') replaceDict = {} replaceDict['$reporter'] = myVal.reptDevIpAddr replaceDict['$localhost'] = self.testConfig.localhost if '$localhost' in msg or '$reporter' in msg: msg = generalUtility.multiReplace(msg, replaceDict) if hasattr(self.testConfig, 'globalData'): custId = custIdUtil.getCustIdbyIp( self.testConfig.globalData, myVal.reptDevIpAddr, self.testConfig.testServer.dataCollector) else: custId = '1' newPar = {} for pkey in myPars.keys(): if myPars[pkey] == '$localhost': myPars[pkey] = self.testConfig.localhost if myPars[pkey] == '$reporter': myPars[pkey] = myVal.reptDevIpAddr if myPars[pkey] == '$sender': myPars[pkey] = custId params[myVal.eventType + '@' + myKey] = {} params[myVal.eventType + '@' + myKey]['params'] = myPars params[myVal.eventType + '@' + myKey]['msg'] = msg ver = '' comm = '' if key not in no_raw_send and module not in msg_too_long and self.posix and 'snmptrap' not in myVal.method: import Libs.rawUdpSendHandler as rawUdpSendHandler import Libs.GenerateRawIPData as GenerateRawIPData rawmsg = GenerateRawIPData.getRawIpPacket( msg.encode('ascii', 'xmlcharrefreplace'), myVal.reptDevIpAddr, self.testConfig.testServer.dataCollector, 514) self.rawUdpSender = rawUdpSendHandler.rawUdpSendHandler( self.testConfig.testServer.dataCollector, 514) now, sendTime, unow, usendTime = timeUtility.getTimeNow() self.rawUdpSender.sendoutEvent(rawmsg) else: if 'snmptrap@' in myVal.method: sendMethod, ver, comm = myVal.method.split('@') else: sendMethod = myVal.method if not mySendEvent: mySendEvent = sendEventHandler( sendMethod, self.testConfig.testServer.dataCollector) now, sendTime, unow, usendTime = timeUtility.getTimeNow() if ver and comm: mySendEvent.sendoutEvent(msg, version=ver, community=comm, utf_8=True) else: mySendEvent.sendoutEvent(msg, utf_8=True) #get query time.sleep(300) myParams = {} if module in parser_holdon_module: time.sleep(360) for key in event_query_params.keys(): myParams[key] = event_query_params[key] eventTypes = [] reporters = [] for myKey in params.keys(): eventType, reporter = myKey.split(' (')[0].split('@') if eventType.strip() not in eventTypes: eventTypes.append(eventType.strip()) if reporter.strip() not in reporters: reporters.append(reporter) finalEventTypes = [] if len(eventTypes) > 500: num = generalUtility.splitByNum(len(eventTypes), 500) finalEventTypes.append(eventTypes[0:500]) for i in range(num - 1): s = (i + 1) * 500 e = (i + 2) * 500 subList = eventTypes[s:e] finalEventTypes.append(subList) else: finalEventTypes.append(eventTypes) reporterStr = ','.join(reporters) retData = {} for i in range(len(finalEventTypes)): eventtypeStr = '","'.join(finalEventTypes[i]) myParams['constr'] = event_query_params['constr'] % (reporterStr, eventtypeStr) retSubData, debugInfo = self.queryHandler.getQuery(myParams) for key in retSubData.keys(): newList = [] for item in retSubData[key]: rTime, zone = timeUtility.getUTimeFromString( item.attributes['phRecvTime']) if zone == 'UTC': if rTime + 300 >= usendTime: newList.append(item) else: if rTime + 300 >= sendTime: newList.append(item) if newList: retData[key] = newList finalRet = self.verifyData(etype, params, retData, debugInfo, module, sendTime) if finalRet.totalMissing or finalRet.totalExtra: self.__updateCase(finalRet.caseList, path) return finalRet
def run(self, ruleType, testKey): fb_type=ruleType.filterOperators.type sleeper=0 if fb_type=='FOLLOWED_BY': sleeper=5 incidentType=ruleType.incidentType.split('$')[-1] ruleId=ruleType.attribute['id'] if incidentType not in self.testConfig.commonData.keys(): print 'Incidnet Name %s Incident Type %s is NOT implemented.' % (ruleType.name, incidentType) testRet=None else: testConf=self.testConfig.commonData[incidentType] if testConf.reptDevIpAddr=='$localhost': testConf.reptDevIpAddr=self.testConfig.localhost elif testConf.reptDevIpAddr=='$appServer': testConf.reptDevIpAddr=self.appServer allDevices=self.testConfig.globalData['devices'] approvedDevices=self.testConfig.globalData['applicableDevices'] perfObj=self.testConfig.globalData['perfObj'] #create device if needed if not allDevices or (testConf.createDevice and not testConf.reptDevIpAddr in allDevices.keys()): if testConf.domainController: self.deviceHandler.createDevice(testConf.reptDevIpAddr, testConf.deviceName, testConf.deviceType, perfObj, dataCollector=self.testConfig.testServer.dataCollector) else: self.deviceHandler.createDevice(testConf.reptDevIpAddr, testConf.deviceName, testConf.deviceType, perfObj) #get raw data myData=datFileHandler.getData(self.path+'/'+incident_data_path+'/'+incidentType+'.dat', incident_data_keys) #send raw event to trigger incident if not myData.dataMap: print 'Fail to get test data. Exit.' exit() now,sendTime,utcnow,utcsendTime=timeUtility.getTimeNow() rept={} rept['$localhost']=self.testConfig.localhost rept['$dataCollector']=self.testConfig.testServer.dataCollector eventMsgs=myData.dataMap['default'].eventMsg if eventMsgs is None: print '%s: No eventMsg exist.' % incidentType exit() rawSend=False if self.posix and testConf.reptDevIpAddr!=self.testConfig.localhost: rawSend=True if rawSend: mySendEvent=rawUdpSendHandler.rawUdpSendHandler(self.testConfig.testServer.dataCollector, PORTS[testConf.method]) else: mySendEvent=sendEventHandler(testConf.method, self.testConfig.testServer.dataCollector) for i in range(int(testConf.count)): for line in eventMsgs: msg=generalUtility.multiReplace(line, rept) if '$reporter' in line: msg=msg.replace('$reporter', testConf.reptDevIpAddr) if '$randomIP' in line: ip=randomGen.getRandomIPAddr() msg=msg.replace('$randomIP', ip) if '$randomNum' in line: num=randomGen.getRandomNum(1, 100) msg=msg.replace('$randomNum', num) temp_msg='' if testConf.method=='netflow': temp_msg=GenerateNetFlow.getNetFlowPacket(msg) else: temp_msg=msg.strip() send_msg='' if rawSend: send_msg=GenerateRawIPData.getRawIpPacket(temp_msg, testConf.reptDevIpAddr, self.testConfig.testServer.dataCollector, PORTS[testConf.method]) else: send_msg=msg time.sleep(sleeper) mySendEvent.sendoutEvent(send_msg) self.msgList.append(msg) #retrieve incident time.sleep(60) timeout=int(ruleType.triggerWindow)+60 if timeout>1800: timeout=1800 myParams={} myParams['constr']=incident_query_params['SingleEvtConstr'] % (testConf.reptDevIpAddr, incidentType) condition, oriRet, incidentId, failDetail, veriData, debugInfo=self.retriveIncident(incidentType, testConf, timeout, myParams) if not condition: failDetail=self.eventDebug(testConf.method, self.testConfig.testServer.appServer, testConf.reptDevIpAddr, ruleType, sendTime, utcsendTime, approvedDevices) testRet=getClassObj('TestSuiteResult', module='autoTest') testRet.name=testConf.name testRet.type=incidentType testRet.ruleId=ruleId testRet.queryString=myParams['constr'] testRet.rawMsg=self.msgList testRet.testMethod=testConf.method testRet.reptDevIpAddr=testConf.reptDevIpAddr testRet.taskName='Incident' testRet.totalRun=1 testRet.debugInfo=debugInfo setattr(testRet, 'total'+oriRet.status, 1) testRet.caseList.append(oriRet) if incidentId: print '%s: incident triggered with id: %s' % (incidentType, incidentId) testRet.info='incidentId: '+incidentId else: print '%s: no incident triggered' % incidentType testRet.info='incidentId: None' setattr(oriRet, 'reasons', failDetail) aggmsgList=[] if self.advance: if oriRet.status!='NoReturn' and incidentId and incidentId.strip(): #aggregate test for i in range(int(testConf.count)): for m in self.msgList: mySendEvent.sendoutEvent(m) time.sleep(60) aggParams={} aggParams['constr']=incident_query_advance['SingleEvtConstr'] % (incidentId, '0') condition, aggRet, inId, failDetail, aggData=self.retriveIncident(incidentType, testConf, timeout, aggParams, agg=veriData) if not condition: failDetail=self.eventDebug(testConf.method, self.testConfig.testServer.appServer, testConf.reptDevIpAddr, ruleType, sendTime, utcsendTime, approvedDevices) if failDetail: setattr(aggRet, 'reasons', failDetail) testRet.totalRun+=1 oldVal=getattr(testRet, 'total'+aggRet.status) oldVal+=1 setattr(testRet, 'total'+aggRet.status, oldVal) testRet.caseList.append(aggRet) #clear test if oriRet.status!='NoReturn' and hasattr(ruleType, 'clearCondition') and incidentId and incidentId.strip(): clearnow,clearsendTime,clearutcnow,clearutcsendTime=timeUtility.getTimeNow() if ruleType.clearCondition.clearOption=='patternbased': if not hasattr(myData.dataMap['default'], 'clearEventMsg'): print '%s: need to add clearEventMsg' % incidentType else: raw_clear=getattr(myData.dataMap['default'], 'clearEventMsg') if raw_clear: for line in raw_clear: clearmsg=generalUtility.multiReplace(line.strip(), rept) if '$randomIP' in line: ip=randomGen.getRandomIPAddr() clearmsg=clearmsg.replace('$randomIP', ip) self.clearMsgList.append(clearmsg) if self.posix: send_clearmsg=GenerateRawIPData.getRawIpPacket(clearmsg, testConf.reptDevIpAddr, self.testConfig.testServer.dataCollector, PORTS[testConf.method]) else: send_clearmsg=clearmsg if self.clearMsgList: for i in range(int(testConf.count)): for cl_msg in self.clearMsgList: mySendEvent.sendoutEvent(cl_msg) time.sleep(120) else: time.sleep(float(int(ruleType.clearCondition.clearTimeWindow)+120)) clearParams={} clearParams['constr']=incident_query_advance['SingleEvtConstr'] % (incidentId, '1') condition, clearRet, incidentId, failDetail, clearData=self.retriveIncident(incidentType, testConf, timeout, clearParams, clear=aggData) if not condition: failDetail=self.eventDebug(testConf.method, self.testConfig.testServer.appServer, testConf.reptDevIpAddr, ruleType, clearsendTime, clearutcsendTime, approvedDevices, clearDebug=True) if failDetail: setattr(clearRet, 'reasons', failDetail) print 'clear name: %s' % clearRet.name testRet.totalRun+=1 oldVal=getattr(testRet, 'total'+clearRet.status) oldVal+=1 setattr(testRet, 'total'+clearRet.status, oldVal) testRet.caseList.append(clearRet) mySendEvent.close() return testRet