示例#1
0
def getRidisData():
    redisDatas = RedisDataTuning()
    datas = redisDatas.sampleDatas()
    print('aaa')
    print(len(datas))
    freWords, deliword = getDelimiter(datas)
    print(freWords, deliword)
示例#2
0
class FieldHunterTest:
    def __init__(self):
        self.fhunter = FieldHunter()
        self.httpdata = HttpDataTuning()
        self.ftpdata = FTPDataTuning()
        self.redisdata = RedisDataTuning()

    def testFordeliFind(self, dirPath):
        datas = read_datas(dirPath, 'multy')
        datasF = []
        for data in datas:
            if len(data) < 100:
                datasF.extend(data)
            else:
                datasF.extend(data[0:500])
        datas = get_puredatas(datasF)
        print(self.fhunter.findDelimiter(datas))

    def testForHTTPFind(self):
        datas = self.httpdata.sampleDatas()
        print(filterFieldWords(self.fhunter.findDelimiter(datas)))

    def testForFTPFind(self):
        datas = self.ftpdata.sampleData()
        print(filterFieldWords(self.fhunter.findDelimiter(datas)))


    def testForREDISFind(self):
        datas = self.redisdata.sampleDatas()
        print(filterFieldWords(self.fhunter.findDelimiter(datas)))
示例#3
0
 def redisTest(self):
     redisDataTuning = RedisDataTuning('/home/wxw/data/RedisData')
     #srcDatas, desDatas = redisDataTuning.getProDatas()
     #samplesrcmsgs = redisDataTuning.sampleSourceDatas(srcDatas, 2)
     #print(samplesrcmsgs[0])
     srcDatas = redisDataTuning.sampleDatas()
     message_parser = TextParseLogic()
     srcmessages = message_parser.ConvertDataToMessage(srcDatas,
                                                       b'\r\n',
                                                       h=2)
     #desmessages = message_parser.ConvertDataToMessage(desDatas, b'\r\n', h=1)
     srctextcls = TextClassifyLogicTest(srcmessages, 0.05, 0.2, 0.2, 3)
     srctextcls.FormatInferCirclelyTest(srcmessages)
示例#4
0
 def redisGenerateTest(self):
     redisDataTuning = RedisDataTuning('/home/wxw/data/RedisData')
     # srcDatas, desDatas = redisDataTuning.getProDatas()
     # samplesrcmsgs = redisDataTuning.sampleSourceDatas(srcDatas, 2)
     # print(samplesrcmsgs[0])
     srcDatas = redisDataTuning.sampleDatas()
     message_parser = TextParseLogic()
     srcmessages = message_parser.ConvertDataToMessage(srcDatas,
                                                       b'\r\n',
                                                       h=2)
     # desmessages = message_parser.ConvertDataToMessage(desDatas, b'\r\n', h=1)
     srctextcls = TextClassifyLogicTest(srcmessages, 0.05, 0.2, 0.2, 3)
     srctextcls.FormatInferCirclelyTestTwo(
         srcmessages,
         '/home/wxw/paper/researchresult/text/formatInfer/KFCluster/redisshortOne',
         'R')
示例#5
0
class DataTuning:
    def __init__(self):
        self.ftp = FTPDataTuning()
        self.redis = RedisDataTuning()
        self.redis_deal = redis_deal()
        self.fConfig = FileConfig()

    def readDatas(self, filePath):
        datas = read_filedatas(filePath)
        datas = get_puredatas(datas)
        return datas

    def readDataSummarys(self, filePath):
        datas = read_filedatas(filePath)
        return datas

    def readDatasTemp(self, filePath):
        srcDatas, desDatas = self.ftp.tuningHttpByregix()
        datas = []
        datas.extend(srcDatas)
        datas.extend(desDatas)
        return datas

    def icsReadDatasTemp(self, filePath):
        messages = read_datas('/home/wxw/data/ToolDatas/15895903730.10.222',
                              'single')
        messages = get_puredatas(messages)
        return messages

    def textReadDatasTemp(self, filePath):
        messages = self.redis.sampleDatas()
        return messages

    def readDatasByType(self, fileType):
        fileName = self.redis_deal.read_from_redis(fileType)
        filePath = os.path.join(self.fConfig.pathDir, fileName)
        return self.readDatas(filePath)

    def readSummaryByType(self, fileType):
        fileName = self.redis_deal.read_from_redis(fileType)
        filePath = os.path.join(self.fConfig.pathDir, fileName)
        print('aa')
        print(filePath)
        print('bb')
        return self.readDataSummarys(filePath)
示例#6
0
class TextFormInferTest:
    def __init__(self, messages):
        self.tFomInfer = TextFormInfer(messages)
        self.httpdata = HttpDataTuning()
        self.ftpData = FTPDataTuning()
        self.redisData = RedisDataTuning()
        self.txtSymTree = TextSympolToTree()

    def ldaFormatInferTest(self, wSize, TK, wLen, Kcls, path='', infercls='H'):
        fNums = self.tFomInfer.ldaFormatInfer(wSize, TK, wLen, Kcls, infercls)
        self.txtSymTree.symbolsToTree(fNums, path)
        #for fnum in fNums:
        #    nodeT = self.txtSymTree.symbolToTree(fnum)
        #    nodeT.showTree(0)
        #print(fnum._str_debug())

    def ladDbscanFormatInfer(self,
                             wSize,
                             TK,
                             wLen,
                             mindis,
                             minpt,
                             path='',
                             infercls='H'):
        fNums = self.tFomInfer.ladDbscanFormatInfer(wSize, TK, wLen, mindis,
                                                    minpt, infercls)
        self.txtSymTree.symbolsToTree(fNums, path)
        #for fnum in fNums:
        #   print(fnum._str_debug())

    def httpDataTest(self):
        srcDatas, desDatas = self.httpdata.tuningHttpByregix()
        self.tFomInfer = TextFormInfer(desDatas)
        self.ldaFormatInferTest(3, 15, 2, 4)

    def httpTotalTest(self):
        datas = self.httpdata.sampleDatas()
        self.tFomInfer = TextFormInfer(datas)
        self.ldaFormatInferTest(3, 15, 2, 5)

    def httpTotalDBSTest(self):
        datas = self.httpdata.sampleDatas()
        self.tFomInfer = TextFormInfer(datas)
        self.ladDbscanFormatInfer(3, 5, 3, 0.15, 10)

    def ftpTotalTest(self):
        datas = self.ftpData.sampleData()
        self.tFomInfer = TextFormInfer(datas)
        #self.ldaFormatInferTest(3, 15, 2, 10)
        self.ladDbscanFormatInfer(3, 5, 3, 0.05, 10)
        #self.ldaFormatInferTest(3, 15, 2, 15)
        #self.ldaFormatInferTest(3, 15, 2, 20)
        #self.ldaFormatInferTest(3, 15, 2, 10)
        #self.ldaFormatInferTest(3, 15, 2, 5)

    def ftpTotalGenerate(self):
        datas = self.ftpData.sampleData()
        self.tFomInfer = TextFormInfer(datas)
        #self.ldaFormatInferTest(3, 15, 2, 15, '/home/wxw/paper/researchresult/text/formatInfer/KMEANS/15 ftp one.png')
        #self.ladDbscanFormatInfer(3, 15, 3, 0.01, 4, '/home/wxw/paper/researchresult/text/formatInfer/DBSCAN/0.01 ftp.png')
        self.ladDbscanFormatInfer(
            3, 15, 3, 0.05, 10,
            '/home/wxw/paper/researchresult/text/formatInfer/DBSCAN/0.05 ftp.png'
        )
        self.ladDbscanFormatInfer(3, 5, 3, 0.05, 10)

    def httpDataTestDBS(self):
        httpDataTuning = HttpDataTuning()
        srcMsgs, desMsgs = httpDataTuning.tuningHttpByregix()
        self.tFomInfer = TextFormInfer(desMsgs)
        self.ladDbscanFormatInfer(3, 5, 3, 0.15, 10)

    def ftpDataTest(self):
        srcDatas, desDatas = self.ftpData.tuningHttpByregix()
        self.tFomInfer = TextFormInfer(srcDatas)
        self.ldaFormatInferTest(3, 15, 3, 15)

    def ftpDataTestDBS(self):
        httpDataTuning = FTPDataTuning()
        srcMsgs, desMsgs = httpDataTuning.tuningHttpByregix()
        print(len(srcMsgs) + len(desMsgs))
        self.tFomInfer = TextFormInfer(desMsgs)
        self.ladDbscanFormatInfer(3, 15, 3, 0.01, 4)

    def redisTotalTest(self):
        datas = self.redisData.sampleDatas()
        self.tFomInfer = TextFormInfer(datas)
        #self.ladDbscanFormatInfer(3, 5, 3, 0.10, 10)
        #self.ladDbscanFormatInfer(3, 5, 3, 0.10, 20)
        #self.ladDbscanFormatInfer(3, 5, 3, 0.01, 20)
        #self.ladDbscanFormatInfer(3, 5, 3, 0.2, 20)
        #self.ldaFormatInferTest(3, 15, 2, 5)
        self.ldaFormatInferTest(3, 15, 2, 10)
        #self.ldaFormatInferTest(3, 15, 2, 15)
        #self.ldaFormatInferTest(3, 15, 2, 20)

    def redisTotalGenerate(self):
        datas = self.redisData.sampleDatas()
        self.tFomInfer = TextFormInfer(datas)
        self.ladDbscanFormatInfer(
            3, 5, 3, 0.05, 4,
            '/home/wxw/paper/researchresult/text/formatInfer/DBSCAN/redis .png'
        )
        #self.ldaFormatInferTest(3, 15, 2, 10, '/home/wxw/paper/researchresult/text/formatInfer/KMEANS/11 three redis.png')

    def httpTotalGenerate(self, kClus):
        datas = self.httpdata.sampleDatas()
        self.tFomInfer = TextFormInfer(datas)
        print('ss', kClus)
        #self.ladDbscanFormatInfer(3, 5, 3, 0.05, 10, '/home/wxw/paper/researchresult/text/formatInfer/DBSCAN/0.15 http.png')
        self.ldaFormatInferTest(
            3,
            15,
            2,
            kClus,
            '/home/wxw/paper/researchresult/text/formatInfer/KMEANS/' +
            str(kClus) + 'httptwo.png',
            infercls='H')
        print('ee', kClus)

    def httpTotalRepeatGenerate(self, kClus, rTime):
        datas = self.httpdata.sampleDatas()
        self.tFomInfer = TextFormInfer(datas)
        self.ldaFormatInferTest(
            3,
            15,
            2,
            kClus,
            '/home/wxw/paper/researchresult/text/formatInfer/KMEANS/' +
            str(kClus) + ' ' + str(rTime) + 'httpone.png',
            infercls='H')

    def httpDBSTotalGenerate(self, r, C, rTime=''):
        datas = self.httpdata.sampleDatas()
        self.tFomInfer = TextFormInfer(datas)
        print('ss', r)
        self.ladDbscanFormatInfer(
            3,
            5,
            3,
            r,
            C,
            '/home/wxw/paper/researchresult/text/formatInfer/DBSCAN/' +
            str(r) + str(C) + rTime + 'httptwo.png',
            infercls='H')
        #self.ldaFormatInferTest(3, 15, 2, kClus, '/home/wxw/paper/researchresult/text/formatInfer/KMEANS/'
        #+ str(kClus) + 'httpone.png', infercls='H')
        print('ee', r)

    def ftpTotalGenerate(self, kClus):
        datas = self.ftpData.sampleData()
        self.tFomInfer = TextFormInfer(datas)
        self.ldaFormatInferTest(
            3,
            15,
            2,
            kClus,
            '/home/wxw/paper/researchresult/text/formatInfer/KMEANS/FTP/' +
            str(kClus) + 'ftp.png',
            infercls='F')
        #self.ladDbscanFormatInfer(3, 15, 3, 0.01, 4, '/home/wxw/paper/researchresult/text/formatInfer/DBSCAN/0.01 ftp.png')
        #self.ladDbscanFormatInfer(3, 15, 3, 0.05, 10,
        #                          '/home/wxw/paper/researchresult/text/formatInfer/DBSCAN/0.05 ftp.png')
        #self.ladDbscanFormatInfer(3, 5, 3, 0.05, 10)

    def ftpTotalGenerateRepeat(self, kClus, rTime):
        datas = self.ftpData.sampleData()
        self.tFomInfer = TextFormInfer(datas)
        self.ldaFormatInferTest(
            3,
            15,
            2,
            kClus,
            '/home/wxw/paper/researchresult/text/formatInfer/KMEANS/FTP/' +
            str(kClus) + str(rTime) + 'ftp.png',
            infercls='F')
        #self.ladDbscanFormatInfer(3, 15, 3, 0.01, 4, '/home/wxw/paper/researchresult/text/formatInfer/DBSCAN/0.01 ftp.png')
        #self.ladDbscanFormatInfer(3, 15, 3, 0.05, 10,
        #                          '/home/wxw/paper/researchresult/text/formatInfer/DBSCAN/0.05 ftp.png')
        #self.ladDbscanFormatInfer(3, 5, 3, 0.05, 10)

    def ftpDBSCANGenerate(self, r, C, rTime=''):
        datas = self.ftpData.sampleData()
        self.tFomInfer = TextFormInfer(datas)
        print('ss', r)
        self.ladDbscanFormatInfer(
            3,
            5,
            3,
            r,
            C,
            '/home/wxw/paper/researchresult/text/formatInfer/DBSCAN/FTP/' +
            str(r) + str(C) + rTime + 'ftp.png',
            infercls='F')
        # self.ldaFormatInferTest(3, 15, 2, kClus, '/home/wxw/paper/researchresult/text/formatInfer/KMEANS/'
        # + str(kClus) + 'httpone.png', infercls='H')
        print('ee', r)

    def redisTotalGenerate(self, kClus, rTime=''):
        datas = self.redisData.sampleDatas()
        self.tFomInfer = TextFormInfer(datas)
        self.ldaFormatInferTest(
            3,
            15,
            2,
            kClus,
            '/home/wxw/paper/researchresult/text/formatInfer/KMEANS/REDIS/' +
            str(kClus) + rTime + 'redis.png',
            infercls='R')
        #self.ladDbscanFormatInfer(3, 5, 3, 0.05, kClus, '/home/wxw/paper/researchresult/text/formatInfer/DBSCAN/redis .png')

    def redisDBSCANGenerate(self, r, C, rTime=''):
        datas = self.redisData.sampleDatas()
        self.tFomInfer = TextFormInfer(datas)
        print('ss', r)
        self.ladDbscanFormatInfer(
            3,
            5,
            3,
            r,
            C,
            '/home/wxw/paper/researchresult/text/formatInfer/DBSCAN/REDIS/' +
            str(r) + str(C) + rTime + 'redis.png',
            infercls='R')
        # self.ldaFormatInferTest(3, 15, 2, kClus, '/home/wxw/paper/researchresult/text/formatInfer/KMEANS/'
        # + str(kClus) + 'httpone.png', infercls='H')
        print('ee', r)