def __init__(self, path='/home/wxw/data/RedisData'): self.msgConvert = MessageConvert() self.path = path self.cmds = [ b'SADD', b'LLEN', b'STRLEN', b'SET', b'SMEMBERS', b'MSET', b'SELECT', b'LPUSH', b'MGET', b'GET', b'LPOP' ]
def tuningTotol(self): messages = read_datas('/home/wxw/data/httpDatas/http', 'single') messages = get_puredatas(messages) messages = MessageConvert.filterMsgs(['GET', 'POST', 'HTTP', 'HEAD'], messages, 10) clsMessages = MessageConvert.clsMessagesByRegix(['GET', 'POST', 'HEAD'], 5, messages) srcDatas = [] desDatas = [] i = 0 srcDatas.extend(clsMessages['GET']) srcDatas.extend(clsMessages['POST']) desDatas.extend(clsMessages['unkown']) return srcDatas, desDatas
def tuningHttpByregix(self): messages = read_datas('/home/wxw/data/httpDatas/http', 'single') messages = get_puredatas(messages) messages = MessageConvert.filterMsgs(['GET', 'POST', 'HTTP'], messages, 10) clsMessages = MessageConvert.clsMessagesByRegix(['GET', 'POST'], 5, messages) srcDatas = [] desDatas = [] i = 0 while (i < 252): srcDatas.append(clsMessages['GET'][i]) srcDatas.append(clsMessages['POST'][i]) i = i + 1 desDatas.extend(clsMessages['unkown']) desDatas = desDatas[0:505] return srcDatas, desDatas
def tuningHttpBydire(self): messages = read_datas('/home/wxw/data/httpDatas/http', 'single') messages = get_puredatas(messages) messages = MessageConvert.filterMsgs(['GET', 'POST', 'HTTP'], messages, 10) clsMessages = MessageConvert.clsMessagesByRegix(['GET', 'POST'], 5, messages) clsDesMessages = clsMessages['unkown'] message_parser = TextParseLogic() desDatas = message_parser.ConvertDataToMessage(clsDesMessages, b'\r\n') desNowDatas = [datanow.now() for datanow in desDatas] diff = {} for desNow in desNowDatas: if desNow not in diff: diff[desNow] = 1 else: diff[desNow] = diff[desNow] + 1 print(diff)
def tuningHttpBydire(self, tPath='/home/wxw/data/httpDatas/http'): messages = read_datas(tPath, 'single') messages = MessageConvert.filterMsgs(['GET', 'POST', 'HTTP', 'HEAD'], messages, 10) clsMessages = MessageConvert.clsMessagesByRegix(['GET', 'POST', 'HEAD'], 5, messages) clsDesMessages = clsMessages['unkown'] srcDatas = [] srcDatas.extend(clsMessages['GET']) srcDatas.extend(clsMessages['POST']) srcDatas.extend(clsMessages['HEAD']) message_parser = TextParseLogic() #desDatas = message_parser.ConvertDataToMessage(clsDesMessages, b'\r\n') desDatas = message_parser.ConvertDataToMessage(srcDatas, b'\r\n') #desNowDatas = [datanow.now() for datanow in desDatas] desNowDatas = [datanow.now().split(b' ')[0] for datanow in desDatas] diff = {} for desNow in desNowDatas: if desNow not in diff: diff[desNow] = 1 else: diff[desNow] = diff[desNow] + 1 print(diff)
def sampleData(self): datas = self.getDatas() datasSplit = MessageConvert.clsMsgsByRegix(self.cmds, 20, datas) Fdatas = [] for key in datasSplit: print(key, len(datasSplit[key])) if len(datasSplit[key]) > 1000: print('zzz') Fdatas.extend(datasSplit[key][0:500]) else: Fdatas.extend(datasSplit[key]) print(len(Fdatas)) return Fdatas
def tuningHttpByregix(self): datas = read_datas('/home/wxw/data/ftp/ftpData', 'multy') datasF = [] for data in datas: if len(data) < 100: datasF.extend(data) else: datasF.extend(data[0:500]) srcDatasF, desDatasF = MessageConvert.clsMessageByDire(datasF) srcDatasF = get_puredatas(srcDatasF) desDatasF = get_puredatas(desDatasF) print(len(desDatasF)) return srcDatasF, desDatasF
def ftpTestOne(self, DirPath): datas = read_datas(DirPath, 'multy') datasF = [] for data in datas: if len(data) < 100: datasF.extend(data) else: datasF.extend(data[0:500]) srcDatasF, desDatasF = MessageConvert.clsMessageByDire(datasF) srcdatas = get_puredatas(srcDatasF) desdatas = get_puredatas(desDatasF) message_parser = TextParseLogic() srcmessages = message_parser.ConvertDataToMessage(srcdatas, b'\r\n') desmessages = message_parser.ConvertDataToMessage(desdatas, b'\r\n') srctextcls = TextClassifyLogicTest(srcmessages, 0.2, 0.3, 0.4, 3) # srctextcls.FormatInferCirclelyTest(srcmessages) destextcls = TextClassifyLogicTest(desmessages, 0.1, 0.1, 0.1, 3) # destextcls.getHeadAndNum(desmessages) destextcls.FormatInferCirclelyTest(desmessages)
def getMsgsLen(self, clses): correLen = 0 conciouLen = 0 print(len(clses)) clsCmds = [] for cls in clses: clsCmd = MessageConvert.clsMsgsByRegix(self.cmds, 20, cls) clsCmds.append(clsCmd) for clscmd in clsCmds: if len(clscmd) == 1: correLen = correLen + 1 for cmd in self.cmds: tLo = 0 for clscmd in clsCmds: if cmd in clscmd: tLo = tLo + 1 if tLo == 1: conciouLen = conciouLen + 1 print(correLen, conciouLen)
class RedisDataTuningTest: def __init__(self, path): self.redisData = RedisDataTuning(path) self.msgConvert = MessageConvert() def getDatas(self, path): redisDatas = read_datas(path, 'single') return redisDatas def splitDatas(self, datas): srcDatas, desDatas = self.msgConvert.clsMessageByDire(datas) return srcDatas, desDatas def getDesCommondTest(self, desDatas): print(self.redisData.getDesCommond(desDatas)) def testSample(self): srcDatas, desDatas = self.redisData.getProDatas() sPSrcDatas = self.redisData.sampleSourceDatas(srcDatas, 150) for datas in sPSrcDatas: print(datas, len(sPSrcDatas[datas]))
def getTotalData(self): datas = self.getRawDatas() datasSplit = MessageConvert.clsMsgsByRegix(self.cmds, 20, datas) return datasSplit
def __init__(self, path): self.redisData = RedisDataTuning(path) self.msgConvert = MessageConvert()
class RedisDataTuning: def __init__(self, path='/home/wxw/data/RedisData'): self.msgConvert = MessageConvert() self.path = path self.cmds = [ b'SADD', b'LLEN', b'STRLEN', b'SET', b'SMEMBERS', b'MSET', b'SELECT', b'LPUSH', b'MGET', b'GET', b'LPOP' ] def readDatas(self, path='/home/wxw/data/RedisData'): redisDatas = read_datas(path, 'single') return redisDatas def sampleDatas(self): datas = self.readDatas() srcDatas, desDatas = self.splitByPureDatas(datas) splitSrcDatas = self.sampleSourceDatas(srcDatas, 2) print(len(splitSrcDatas)) return splitSrcDatas def splitByDire(self, datas): srcDatas, desDatas = self.msgConvert.clsMessageByDire(datas) return srcDatas, desDatas def splitByPureDatas(self, datas): srcDatas, desDatas = self.splitByDire(datas) return get_puredatas(srcDatas), get_puredatas(desDatas) def getProDatas(self): datas = self.readDatas(self.path) srcDatas, desDatas = self.splitByPureDatas(datas) return srcDatas, desDatas def getSouceCommond(self, srcDatas): srcCommond = {} srcOne = {} for srcData in srcDatas: srcSplitData = srcData.split(b'\r\n') srcDataCommond = srcSplitData[2] if srcDataCommond not in srcCommond: srcCommond[srcDataCommond] = 0 srcCommond[srcDataCommond] = srcCommond[srcDataCommond] + 1 srcDataOne = srcSplitData[0] if srcDataOne not in srcOne: srcOne[srcDataOne] = 0 srcOne[srcDataOne] = srcOne[srcDataOne] + 1 return srcCommond, srcOne def sampleSourceDatas(self, srcDatas, msgLen, dataLen=300): srcCDatas = {} for srcData in srcDatas: srcSplitData = srcData.split(b'\r\n') srcC = srcSplitData[msgLen] if srcC not in srcCDatas: srcCDatas[srcC] = [] if len(srcCDatas[srcC]) < dataLen: srcCDatas[srcC].append(srcData) srcDatasList = [] for key in srcCDatas: print(key) srcDatasList.extend(srcCDatas[key]) return srcDatasList def sampleDesDatas(self, desSrcDatas, dataLen=300): desDatas = {} for desData in desSrcDatas: if desData[0] not in desDatas: desDatas[desData[0]] = [] desDatas[desData[0]].append(desData) Fdatas = [] Fdatas.extend(desDatas[b'+']) Fdatas.extend(desDatas[b'-']) return Fdatas def getDesCommond(self, desDatas): desCommond = {} desCommondS = {} desCommonT = {} for desData in desDatas: desSplitData = desData.split(b'\r\n') desDataCommond = desSplitData[0] if desDataCommond not in desCommond: desCommond[desDataCommond] = 0 desCommond[desDataCommond] = desCommond[desDataCommond] + 1 if len(desSplitData) > 1: desDataCommondS = desSplitData[1] if desDataCommondS not in desCommondS: desCommondS[desDataCommondS] = 0 desCommondS[desDataCommondS] = desCommondS[desDataCommondS] + 1 if len(desSplitData) > 2: desDataCommondT = desSplitData[2] if desDataCommondT not in desCommonT: desCommonT[desDataCommondT] = 0 desCommonT[desDataCommondT] = desCommonT[desDataCommondT] + 1 return self.filterDatas(desCommond), self.filterDatas( desCommondS), self.filterDatas(desDataCommondT) def getMsgsLen(self, clses): correLen = 0 conciouLen = 0 print(len(clses)) clsCmds = [] for cls in clses: clsCmd = MessageConvert.clsMsgsByRegix(self.cmds, 20, cls) clsCmds.append(clsCmd) for clscmd in clsCmds: if len(clscmd) == 1: #print(clscmd) correLen = correLen + 1 for cmd in self.cmds: tLo = 0 for clscmd in clsCmds: if cmd in clscmd: tLo = tLo + 1 if tLo == 1: #print('ss ',cmd) conciouLen = conciouLen + 1 print(correLen, conciouLen) def filterDatas(self, datas): tNewDatas = {} for data in datas: if datas[data] > 5: tNewDatas[data] = datas[data] return tNewDatas