def ftpTest(self):
ftptuning = FTPDataTuning()
datas = ftptuning.sampleData()
message_parser = TextParseLogic()
datas = message_parser.ConvertDataToMessage(datas, b'\r\n')
#srctextcls = TextClassifyLogicTest(datas, 0.1, 0.3, 0.3, 3)
srctextcls = TextClassifyLogicTest(datas, 0.1, 0.2, 0.2, 3)
srctextcls.FormatInferCirclelyTest(datas)
def httpTest(self, DirPath=''):
httptuning = HttpDataTuning()
datas = httptuning.sampleDatas()
message_parser = TextParseLogic()
datas = message_parser.ConvertDataToMessage(datas, b'\r\n')
#srctextcls = TextClassifyLogicTest(datas, 0.05, 0.2, 0.2, 3)
#srctextcls = TextClassifyLogicTest(datas, 0.02, 0.2, 0.2, 3)
srctextcls = TextClassifyLogicTest(datas, 0.02, 0.1, 0.1, 3)
srctextcls.FormatInferCirclelyTest(datas)
def __init__(self):
self.msgP = TextParseLogic()
self.cmds = {b'PWD', b'331 Please specify the password', b'RETR', b'TYPE I', b'250 Directory successfully changed',
b'227 Entering Passive Mode', b'PASS', b'150 Ok to send data',
b'226 Directory send OK', b'220', b'226 Transfer complete', b'230 Login successful',
b'NLST', b'STOR', b'LIST', b'USER',
b'550 Create directory operation', b'200 Switching to', b'150 Opening BINARY mode data connection for',
b'150 Here comes the directory listing', b'200 Switching to ASCII mode', b'MKD', b'257', b'TYPE A',
b'CWD', b'PASV'}
def httpGenerateTest(self):
httptuning = HttpDataTuning()
datas = httptuning.sampleDatas()
message_parser = TextParseLogic()
datas = message_parser.ConvertDataToMessage(datas, b'\r\n')
srctextcls = TextClassifyLogicTest(datas, 0.05, 0.2, 0.2, 3)
# srctextcls = TextClassifyLogicTest(datas, 0.02, 0.2, 0.2, 3)
# srctextcls = TextClassifyLogicTest(datas, 0.02, 0.1, 0.1, 3)
srctextcls.FormatInferCirclelyTestTwo(
datas,
'/home/wxw/paper/researchresult/text/formatInfer/KFCluster/httpOne.png',
'H')
def tuningTwoHttpBydire(self):
srcData, desData = self.tuningHttpByregix()
message_parser = TextParseLogic()
desDatas = message_parser.ConvertDataToMessage(desData, b'\r\n')
desNowDatas = [datanow.now() for datanow in desDatas]
diff = {}
for desNow in desNowDatas:
if desNow not in diff:
diff[desNow] = 1
else:
diff[desNow] = diff[desNow] + 1
print(diff)
def redisTest(self):
redisDataTuning = RedisDataTuning('/home/wxw/data/RedisData')
#srcDatas, desDatas = redisDataTuning.getProDatas()
#samplesrcmsgs = redisDataTuning.sampleSourceDatas(srcDatas, 2)
#print(samplesrcmsgs[0])
srcDatas = redisDataTuning.sampleDatas()
message_parser = TextParseLogic()
srcmessages = message_parser.ConvertDataToMessage(srcDatas,
b'\r\n',
h=2)
#desmessages = message_parser.ConvertDataToMessage(desDatas, b'\r\n', h=1)
srctextcls = TextClassifyLogicTest(srcmessages, 0.05, 0.2, 0.2, 3)
srctextcls.FormatInferCirclelyTest(srcmessages)
def formatInfer(self):
#msgs = self.dataTuning.textReadDatasTemp('')
msgs = self.dataTuning.readDatasByType('textPro')
message_parser = TextParseLogic()
srcmessages = message_parser.ConvertDataToMessage(msgs, b'\r\n', h=2)
# future
#textCls = TextClassifyLogic(srcmessages, 0.05, 0.2, 0.2, 3)
textCls = TextClassifyLogic(srcmessages, 0.1, 0.2, 0.2, 3)
formats = textCls.formatInfer(srcmessages)
nodeRoot = node(wType='root')
textSymTree = TextSympolToTree()
for f in formats:
nodeT = textSymTree.subSymbolToTree(f)
nodeRoot.children.append(nodeT)
return nodeRoot.transToDictTree()
def redisGenerateTest(self):
redisDataTuning = RedisDataTuning('/home/wxw/data/RedisData')
# srcDatas, desDatas = redisDataTuning.getProDatas()
# samplesrcmsgs = redisDataTuning.sampleSourceDatas(srcDatas, 2)
# print(samplesrcmsgs[0])
srcDatas = redisDataTuning.sampleDatas()
message_parser = TextParseLogic()
srcmessages = message_parser.ConvertDataToMessage(srcDatas,
b'\r\n',
h=2)
# desmessages = message_parser.ConvertDataToMessage(desDatas, b'\r\n', h=1)
srctextcls = TextClassifyLogicTest(srcmessages, 0.05, 0.2, 0.2, 3)
srctextcls.FormatInferCirclelyTestTwo(
srcmessages,
'/home/wxw/paper/researchresult/text/formatInfer/KFCluster/redisshortOne',
'R')
def tuningHttpBydire(self):
messages = read_datas('/home/wxw/data/httpDatas/http', 'single')
messages = get_puredatas(messages)
messages = MessageConvert.filterMsgs(['GET', 'POST', 'HTTP'], messages, 10)
clsMessages = MessageConvert.clsMessagesByRegix(['GET', 'POST'], 5, messages)
clsDesMessages = clsMessages['unkown']
message_parser = TextParseLogic()
desDatas = message_parser.ConvertDataToMessage(clsDesMessages, b'\r\n')
desNowDatas = [datanow.now() for datanow in desDatas]
diff = {}
for desNow in desNowDatas:
if desNow not in diff:
diff[desNow] = 1
else:
diff[desNow] = diff[desNow] + 1
print(diff)
def tuningHttpDatasTotal(self):
srcData, desData = self.tuningTotol()
totalDatas = []
totalDatas.extend(srcData)
totalDatas.extend(desData)
message_parser = TextParseLogic()
desDatas = message_parser.ConvertDataToMessage(desData, b'\r\n')
desNowDatas = [datanow.now() for datanow in desDatas]
#desNowDatas = [str(datanow.now()).split(' ')[0] for datanow in desDatas]
diff = {}
for desNow in desNowDatas:
if desNow not in diff:
diff[desNow] = 1
else:
diff[desNow] = diff[desNow] + 1
print(diff)
def ftpTestOne(self, DirPath):
datas = read_datas(DirPath, 'multy')
datasF = []
for data in datas:
if len(data) < 100:
datasF.extend(data)
else:
datasF.extend(data[0:500])
srcDatasF, desDatasF = MessageConvert.clsMessageByDire(datasF)
srcdatas = get_puredatas(srcDatasF)
desdatas = get_puredatas(desDatasF)
message_parser = TextParseLogic()
srcmessages = message_parser.ConvertDataToMessage(srcdatas, b'\r\n')
desmessages = message_parser.ConvertDataToMessage(desdatas, b'\r\n')
srctextcls = TextClassifyLogicTest(srcmessages, 0.2, 0.3, 0.4, 3)
# srctextcls.FormatInferCirclelyTest(srcmessages)
destextcls = TextClassifyLogicTest(desmessages, 0.1, 0.1, 0.1, 3)
# destextcls.getHeadAndNum(desmessages)
destextcls.FormatInferCirclelyTest(desmessages)
def tuningTwoHttpBydire(self):
srcData, desData = self.tuningHttpByregix()
message_parser = TextParseLogic()
desDatas = message_parser.ConvertDataToMessage(srcData, b'\r\n')
desNowDatas = [datanow.now() for datanow in desDatas]
diff = {}
for desNow in desNowDatas:
#desNow = str(desNow).split(' ')[0]
desNow = str(desNow)
lo = desNow.find('(')
if lo != -1:
desNow = desNow[0:lo]
lo = desNow.find('/')
if lo != -1:
desNow = desNow[0:lo]
if desNow not in diff:
diff[desNow] = 1
else:
diff[desNow] = diff[desNow] + 1
print(diff)
def tuningHttpBydire(self, tPath='/home/wxw/data/httpDatas/http'):
messages = read_datas(tPath, 'single')
messages = MessageConvert.filterMsgs(['GET', 'POST', 'HTTP', 'HEAD'], messages, 10)
clsMessages = MessageConvert.clsMessagesByRegix(['GET', 'POST', 'HEAD'], 5, messages)
clsDesMessages = clsMessages['unkown']
srcDatas = []
srcDatas.extend(clsMessages['GET'])
srcDatas.extend(clsMessages['POST'])
srcDatas.extend(clsMessages['HEAD'])
message_parser = TextParseLogic()
#desDatas = message_parser.ConvertDataToMessage(clsDesMessages, b'\r\n')
desDatas = message_parser.ConvertDataToMessage(srcDatas, b'\r\n')
#desNowDatas = [datanow.now() for datanow in desDatas]
desNowDatas = [datanow.now().split(b' ')[0] for datanow in desDatas]
diff = {}
for desNow in desNowDatas:
if desNow not in diff:
diff[desNow] = 1
else:
diff[desNow] = diff[desNow] + 1
print(diff)
return finalFormats
def filterSets(self, result, fResult):
cverter = Converter()
cverter.ConvertMultiListPure(result, fResult)
def classifyMessages(self, preSet, messages):
datas = self.ClassifyCircleLy(preSet, messages)
result = []
self.filterSets(datas, result)
return result
if __name__ == '__main__':
beginTime = time.time()
messages = read_datas('/home/wxw/data/httpDatas/http_measure', 'single')
messages = get_puredatas(messages)
print(len(messages))
message_parser = TextParseLogic()
messages = message_parser.ConvertDataToMessage(messages, b'\r\n')
textClassify = TextClassifyLogic(messages, 0.1, 0.4, 0.2, 3)
fFormats = textClassify.FormatInferCirclely(messages)
for f in fFormats:
print(f.getLeafFields)
#textClassify.GetFrequentWords(0.3, 3)
#print(textClassify.GetWodsRank(messages))
#msgSet = textClassify.FormatInfer(0.3, 3)
#for value in msgSet:
# print(value._str_debug())
#endTime = time.time()
#print(endTime - beginTime)
def getTotalDatas(self, messages):
message_parser = TextParseLogic()
messages = message_parser.ConvertDataToMessage(messages, b'\r\n')
self.getDiffDatas(messages)
def __init__(self):
self.dataTuning = DataTuning()
self.textParser = TextParseLogic()