def getRidisData(): redisDatas = RedisDataTuning() datas = redisDatas.sampleDatas() print('aaa') print(len(datas)) freWords, deliword = getDelimiter(datas) print(freWords, deliword)
class FieldHunterTest: def __init__(self): self.fhunter = FieldHunter() self.httpdata = HttpDataTuning() self.ftpdata = FTPDataTuning() self.redisdata = RedisDataTuning() def testFordeliFind(self, dirPath): datas = read_datas(dirPath, 'multy') datasF = [] for data in datas: if len(data) < 100: datasF.extend(data) else: datasF.extend(data[0:500]) datas = get_puredatas(datasF) print(self.fhunter.findDelimiter(datas)) def testForHTTPFind(self): datas = self.httpdata.sampleDatas() print(filterFieldWords(self.fhunter.findDelimiter(datas))) def testForFTPFind(self): datas = self.ftpdata.sampleData() print(filterFieldWords(self.fhunter.findDelimiter(datas))) def testForREDISFind(self): datas = self.redisdata.sampleDatas() print(filterFieldWords(self.fhunter.findDelimiter(datas)))
def redisTest(self): redisDataTuning = RedisDataTuning('/home/wxw/data/RedisData') #srcDatas, desDatas = redisDataTuning.getProDatas() #samplesrcmsgs = redisDataTuning.sampleSourceDatas(srcDatas, 2) #print(samplesrcmsgs[0]) srcDatas = redisDataTuning.sampleDatas() message_parser = TextParseLogic() srcmessages = message_parser.ConvertDataToMessage(srcDatas, b'\r\n', h=2) #desmessages = message_parser.ConvertDataToMessage(desDatas, b'\r\n', h=1) srctextcls = TextClassifyLogicTest(srcmessages, 0.05, 0.2, 0.2, 3) srctextcls.FormatInferCirclelyTest(srcmessages)
def redisGenerateTest(self): redisDataTuning = RedisDataTuning('/home/wxw/data/RedisData') # srcDatas, desDatas = redisDataTuning.getProDatas() # samplesrcmsgs = redisDataTuning.sampleSourceDatas(srcDatas, 2) # print(samplesrcmsgs[0]) srcDatas = redisDataTuning.sampleDatas() message_parser = TextParseLogic() srcmessages = message_parser.ConvertDataToMessage(srcDatas, b'\r\n', h=2) # desmessages = message_parser.ConvertDataToMessage(desDatas, b'\r\n', h=1) srctextcls = TextClassifyLogicTest(srcmessages, 0.05, 0.2, 0.2, 3) srctextcls.FormatInferCirclelyTestTwo( srcmessages, '/home/wxw/paper/researchresult/text/formatInfer/KFCluster/redisshortOne', 'R')
class DataTuning: def __init__(self): self.ftp = FTPDataTuning() self.redis = RedisDataTuning() self.redis_deal = redis_deal() self.fConfig = FileConfig() def readDatas(self, filePath): datas = read_filedatas(filePath) datas = get_puredatas(datas) return datas def readDataSummarys(self, filePath): datas = read_filedatas(filePath) return datas def readDatasTemp(self, filePath): srcDatas, desDatas = self.ftp.tuningHttpByregix() datas = [] datas.extend(srcDatas) datas.extend(desDatas) return datas def icsReadDatasTemp(self, filePath): messages = read_datas('/home/wxw/data/ToolDatas/15895903730.10.222', 'single') messages = get_puredatas(messages) return messages def textReadDatasTemp(self, filePath): messages = self.redis.sampleDatas() return messages def readDatasByType(self, fileType): fileName = self.redis_deal.read_from_redis(fileType) filePath = os.path.join(self.fConfig.pathDir, fileName) return self.readDatas(filePath) def readSummaryByType(self, fileType): fileName = self.redis_deal.read_from_redis(fileType) filePath = os.path.join(self.fConfig.pathDir, fileName) print('aa') print(filePath) print('bb') return self.readDataSummarys(filePath)
class TextFormInferTest: def __init__(self, messages): self.tFomInfer = TextFormInfer(messages) self.httpdata = HttpDataTuning() self.ftpData = FTPDataTuning() self.redisData = RedisDataTuning() self.txtSymTree = TextSympolToTree() def ldaFormatInferTest(self, wSize, TK, wLen, Kcls, path='', infercls='H'): fNums = self.tFomInfer.ldaFormatInfer(wSize, TK, wLen, Kcls, infercls) self.txtSymTree.symbolsToTree(fNums, path) #for fnum in fNums: # nodeT = self.txtSymTree.symbolToTree(fnum) # nodeT.showTree(0) #print(fnum._str_debug()) def ladDbscanFormatInfer(self, wSize, TK, wLen, mindis, minpt, path='', infercls='H'): fNums = self.tFomInfer.ladDbscanFormatInfer(wSize, TK, wLen, mindis, minpt, infercls) self.txtSymTree.symbolsToTree(fNums, path) #for fnum in fNums: # print(fnum._str_debug()) def httpDataTest(self): srcDatas, desDatas = self.httpdata.tuningHttpByregix() self.tFomInfer = TextFormInfer(desDatas) self.ldaFormatInferTest(3, 15, 2, 4) def httpTotalTest(self): datas = self.httpdata.sampleDatas() self.tFomInfer = TextFormInfer(datas) self.ldaFormatInferTest(3, 15, 2, 5) def httpTotalDBSTest(self): datas = self.httpdata.sampleDatas() self.tFomInfer = TextFormInfer(datas) self.ladDbscanFormatInfer(3, 5, 3, 0.15, 10) def ftpTotalTest(self): datas = self.ftpData.sampleData() self.tFomInfer = TextFormInfer(datas) #self.ldaFormatInferTest(3, 15, 2, 10) self.ladDbscanFormatInfer(3, 5, 3, 0.05, 10) #self.ldaFormatInferTest(3, 15, 2, 15) #self.ldaFormatInferTest(3, 15, 2, 20) #self.ldaFormatInferTest(3, 15, 2, 10) #self.ldaFormatInferTest(3, 15, 2, 5) def ftpTotalGenerate(self): datas = self.ftpData.sampleData() self.tFomInfer = TextFormInfer(datas) #self.ldaFormatInferTest(3, 15, 2, 15, '/home/wxw/paper/researchresult/text/formatInfer/KMEANS/15 ftp one.png') #self.ladDbscanFormatInfer(3, 15, 3, 0.01, 4, '/home/wxw/paper/researchresult/text/formatInfer/DBSCAN/0.01 ftp.png') self.ladDbscanFormatInfer( 3, 15, 3, 0.05, 10, '/home/wxw/paper/researchresult/text/formatInfer/DBSCAN/0.05 ftp.png' ) self.ladDbscanFormatInfer(3, 5, 3, 0.05, 10) def httpDataTestDBS(self): httpDataTuning = HttpDataTuning() srcMsgs, desMsgs = httpDataTuning.tuningHttpByregix() self.tFomInfer = TextFormInfer(desMsgs) self.ladDbscanFormatInfer(3, 5, 3, 0.15, 10) def ftpDataTest(self): srcDatas, desDatas = self.ftpData.tuningHttpByregix() self.tFomInfer = TextFormInfer(srcDatas) self.ldaFormatInferTest(3, 15, 3, 15) def ftpDataTestDBS(self): httpDataTuning = FTPDataTuning() srcMsgs, desMsgs = httpDataTuning.tuningHttpByregix() print(len(srcMsgs) + len(desMsgs)) self.tFomInfer = TextFormInfer(desMsgs) self.ladDbscanFormatInfer(3, 15, 3, 0.01, 4) def redisTotalTest(self): datas = self.redisData.sampleDatas() self.tFomInfer = TextFormInfer(datas) #self.ladDbscanFormatInfer(3, 5, 3, 0.10, 10) #self.ladDbscanFormatInfer(3, 5, 3, 0.10, 20) #self.ladDbscanFormatInfer(3, 5, 3, 0.01, 20) #self.ladDbscanFormatInfer(3, 5, 3, 0.2, 20) #self.ldaFormatInferTest(3, 15, 2, 5) self.ldaFormatInferTest(3, 15, 2, 10) #self.ldaFormatInferTest(3, 15, 2, 15) #self.ldaFormatInferTest(3, 15, 2, 20) def redisTotalGenerate(self): datas = self.redisData.sampleDatas() self.tFomInfer = TextFormInfer(datas) self.ladDbscanFormatInfer( 3, 5, 3, 0.05, 4, '/home/wxw/paper/researchresult/text/formatInfer/DBSCAN/redis .png' ) #self.ldaFormatInferTest(3, 15, 2, 10, '/home/wxw/paper/researchresult/text/formatInfer/KMEANS/11 three redis.png') def httpTotalGenerate(self, kClus): datas = self.httpdata.sampleDatas() self.tFomInfer = TextFormInfer(datas) print('ss', kClus) #self.ladDbscanFormatInfer(3, 5, 3, 0.05, 10, '/home/wxw/paper/researchresult/text/formatInfer/DBSCAN/0.15 http.png') self.ldaFormatInferTest( 3, 15, 2, kClus, '/home/wxw/paper/researchresult/text/formatInfer/KMEANS/' + str(kClus) + 'httptwo.png', infercls='H') print('ee', kClus) def httpTotalRepeatGenerate(self, kClus, rTime): datas = self.httpdata.sampleDatas() self.tFomInfer = TextFormInfer(datas) self.ldaFormatInferTest( 3, 15, 2, kClus, '/home/wxw/paper/researchresult/text/formatInfer/KMEANS/' + str(kClus) + ' ' + str(rTime) + 'httpone.png', infercls='H') def httpDBSTotalGenerate(self, r, C, rTime=''): datas = self.httpdata.sampleDatas() self.tFomInfer = TextFormInfer(datas) print('ss', r) self.ladDbscanFormatInfer( 3, 5, 3, r, C, '/home/wxw/paper/researchresult/text/formatInfer/DBSCAN/' + str(r) + str(C) + rTime + 'httptwo.png', infercls='H') #self.ldaFormatInferTest(3, 15, 2, kClus, '/home/wxw/paper/researchresult/text/formatInfer/KMEANS/' #+ str(kClus) + 'httpone.png', infercls='H') print('ee', r) def ftpTotalGenerate(self, kClus): datas = self.ftpData.sampleData() self.tFomInfer = TextFormInfer(datas) self.ldaFormatInferTest( 3, 15, 2, kClus, '/home/wxw/paper/researchresult/text/formatInfer/KMEANS/FTP/' + str(kClus) + 'ftp.png', infercls='F') #self.ladDbscanFormatInfer(3, 15, 3, 0.01, 4, '/home/wxw/paper/researchresult/text/formatInfer/DBSCAN/0.01 ftp.png') #self.ladDbscanFormatInfer(3, 15, 3, 0.05, 10, # '/home/wxw/paper/researchresult/text/formatInfer/DBSCAN/0.05 ftp.png') #self.ladDbscanFormatInfer(3, 5, 3, 0.05, 10) def ftpTotalGenerateRepeat(self, kClus, rTime): datas = self.ftpData.sampleData() self.tFomInfer = TextFormInfer(datas) self.ldaFormatInferTest( 3, 15, 2, kClus, '/home/wxw/paper/researchresult/text/formatInfer/KMEANS/FTP/' + str(kClus) + str(rTime) + 'ftp.png', infercls='F') #self.ladDbscanFormatInfer(3, 15, 3, 0.01, 4, '/home/wxw/paper/researchresult/text/formatInfer/DBSCAN/0.01 ftp.png') #self.ladDbscanFormatInfer(3, 15, 3, 0.05, 10, # '/home/wxw/paper/researchresult/text/formatInfer/DBSCAN/0.05 ftp.png') #self.ladDbscanFormatInfer(3, 5, 3, 0.05, 10) def ftpDBSCANGenerate(self, r, C, rTime=''): datas = self.ftpData.sampleData() self.tFomInfer = TextFormInfer(datas) print('ss', r) self.ladDbscanFormatInfer( 3, 5, 3, r, C, '/home/wxw/paper/researchresult/text/formatInfer/DBSCAN/FTP/' + str(r) + str(C) + rTime + 'ftp.png', infercls='F') # self.ldaFormatInferTest(3, 15, 2, kClus, '/home/wxw/paper/researchresult/text/formatInfer/KMEANS/' # + str(kClus) + 'httpone.png', infercls='H') print('ee', r) def redisTotalGenerate(self, kClus, rTime=''): datas = self.redisData.sampleDatas() self.tFomInfer = TextFormInfer(datas) self.ldaFormatInferTest( 3, 15, 2, kClus, '/home/wxw/paper/researchresult/text/formatInfer/KMEANS/REDIS/' + str(kClus) + rTime + 'redis.png', infercls='R') #self.ladDbscanFormatInfer(3, 5, 3, 0.05, kClus, '/home/wxw/paper/researchresult/text/formatInfer/DBSCAN/redis .png') def redisDBSCANGenerate(self, r, C, rTime=''): datas = self.redisData.sampleDatas() self.tFomInfer = TextFormInfer(datas) print('ss', r) self.ladDbscanFormatInfer( 3, 5, 3, r, C, '/home/wxw/paper/researchresult/text/formatInfer/DBSCAN/REDIS/' + str(r) + str(C) + rTime + 'redis.png', infercls='R') # self.ldaFormatInferTest(3, 15, 2, kClus, '/home/wxw/paper/researchresult/text/formatInfer/KMEANS/' # + str(kClus) + 'httpone.png', infercls='H') print('ee', r)