def ftpTest(self):
ftptuning = FTPDataTuning()
datas = ftptuning.sampleData()
message_parser = TextParseLogic()
datas = message_parser.ConvertDataToMessage(datas, b'\r\n')
#srctextcls = TextClassifyLogicTest(datas, 0.1, 0.3, 0.3, 3)
srctextcls = TextClassifyLogicTest(datas, 0.1, 0.2, 0.2, 3)
srctextcls.FormatInferCirclelyTest(datas)
def httpTest(self, DirPath=''):
httptuning = HttpDataTuning()
datas = httptuning.sampleDatas()
message_parser = TextParseLogic()
datas = message_parser.ConvertDataToMessage(datas, b'\r\n')
#srctextcls = TextClassifyLogicTest(datas, 0.05, 0.2, 0.2, 3)
#srctextcls = TextClassifyLogicTest(datas, 0.02, 0.2, 0.2, 3)
srctextcls = TextClassifyLogicTest(datas, 0.02, 0.1, 0.1, 3)
srctextcls.FormatInferCirclelyTest(datas)
def __init__(self):
self.msgP = TextParseLogic()
self.cmds = {b'PWD', b'331 Please specify the password', b'RETR', b'TYPE I', b'250 Directory successfully changed',
b'227 Entering Passive Mode', b'PASS', b'150 Ok to send data',
b'226 Directory send OK', b'220', b'226 Transfer complete', b'230 Login successful',
b'NLST', b'STOR', b'LIST', b'USER',
b'550 Create directory operation', b'200 Switching to', b'150 Opening BINARY mode data connection for',
b'150 Here comes the directory listing', b'200 Switching to ASCII mode', b'MKD', b'257', b'TYPE A',
b'CWD', b'PASV'}
def tuningTwoHttpBydire(self):
srcData, desData = self.tuningHttpByregix()
message_parser = TextParseLogic()
desDatas = message_parser.ConvertDataToMessage(desData, b'\r\n')
desNowDatas = [datanow.now() for datanow in desDatas]
diff = {}
for desNow in desNowDatas:
if desNow not in diff:
diff[desNow] = 1
else:
diff[desNow] = diff[desNow] + 1
print(diff)
def httpGenerateTest(self):
httptuning = HttpDataTuning()
datas = httptuning.sampleDatas()
message_parser = TextParseLogic()
datas = message_parser.ConvertDataToMessage(datas, b'\r\n')
srctextcls = TextClassifyLogicTest(datas, 0.05, 0.2, 0.2, 3)
# srctextcls = TextClassifyLogicTest(datas, 0.02, 0.2, 0.2, 3)
# srctextcls = TextClassifyLogicTest(datas, 0.02, 0.1, 0.1, 3)
srctextcls.FormatInferCirclelyTestTwo(
datas,
'/home/wxw/paper/researchresult/text/formatInfer/KFCluster/httpOne.png',
'H')
def redisTest(self):
redisDataTuning = RedisDataTuning('/home/wxw/data/RedisData')
#srcDatas, desDatas = redisDataTuning.getProDatas()
#samplesrcmsgs = redisDataTuning.sampleSourceDatas(srcDatas, 2)
#print(samplesrcmsgs[0])
srcDatas = redisDataTuning.sampleDatas()
message_parser = TextParseLogic()
srcmessages = message_parser.ConvertDataToMessage(srcDatas,
b'\r\n',
h=2)
#desmessages = message_parser.ConvertDataToMessage(desDatas, b'\r\n', h=1)
srctextcls = TextClassifyLogicTest(srcmessages, 0.05, 0.2, 0.2, 3)
srctextcls.FormatInferCirclelyTest(srcmessages)
def formatInfer(self):
#msgs = self.dataTuning.textReadDatasTemp('')
msgs = self.dataTuning.readDatasByType('textPro')
message_parser = TextParseLogic()
srcmessages = message_parser.ConvertDataToMessage(msgs, b'\r\n', h=2)
# future
#textCls = TextClassifyLogic(srcmessages, 0.05, 0.2, 0.2, 3)
textCls = TextClassifyLogic(srcmessages, 0.1, 0.2, 0.2, 3)
formats = textCls.formatInfer(srcmessages)
nodeRoot = node(wType='root')
textSymTree = TextSympolToTree()
for f in formats:
nodeT = textSymTree.subSymbolToTree(f)
nodeRoot.children.append(nodeT)
return nodeRoot.transToDictTree()
def tuningHttpDatasTotal(self):
srcData, desData = self.tuningTotol()
totalDatas = []
totalDatas.extend(srcData)
totalDatas.extend(desData)
message_parser = TextParseLogic()
desDatas = message_parser.ConvertDataToMessage(desData, b'\r\n')
desNowDatas = [datanow.now() for datanow in desDatas]
#desNowDatas = [str(datanow.now()).split(' ')[0] for datanow in desDatas]
diff = {}
for desNow in desNowDatas:
if desNow not in diff:
diff[desNow] = 1
else:
diff[desNow] = diff[desNow] + 1
print(diff)
def redisGenerateTest(self):
redisDataTuning = RedisDataTuning('/home/wxw/data/RedisData')
# srcDatas, desDatas = redisDataTuning.getProDatas()
# samplesrcmsgs = redisDataTuning.sampleSourceDatas(srcDatas, 2)
# print(samplesrcmsgs[0])
srcDatas = redisDataTuning.sampleDatas()
message_parser = TextParseLogic()
srcmessages = message_parser.ConvertDataToMessage(srcDatas,
b'\r\n',
h=2)
# desmessages = message_parser.ConvertDataToMessage(desDatas, b'\r\n', h=1)
srctextcls = TextClassifyLogicTest(srcmessages, 0.05, 0.2, 0.2, 3)
srctextcls.FormatInferCirclelyTestTwo(
srcmessages,
'/home/wxw/paper/researchresult/text/formatInfer/KFCluster/redisshortOne',
'R')
def tuningHttpBydire(self):
messages = read_datas('/home/wxw/data/httpDatas/http', 'single')
messages = get_puredatas(messages)
messages = MessageConvert.filterMsgs(['GET', 'POST', 'HTTP'], messages, 10)
clsMessages = MessageConvert.clsMessagesByRegix(['GET', 'POST'], 5, messages)
clsDesMessages = clsMessages['unkown']
message_parser = TextParseLogic()
desDatas = message_parser.ConvertDataToMessage(clsDesMessages, b'\r\n')
desNowDatas = [datanow.now() for datanow in desDatas]
diff = {}
for desNow in desNowDatas:
if desNow not in diff:
diff[desNow] = 1
else:
diff[desNow] = diff[desNow] + 1
print(diff)
class TextViewLogic:
def __init__(self):
self.dataTuning = DataTuning()
self.textParser = TextParseLogic()
def spltMsgs(self, msgs, maxRange=150):
#msgs = self.dataTuning.textReadDatasTemp('')
spltResults = self.textParser.spltMsgsSimple(msgs, b'\r\n', maxRange)
return spltResults
def formatInfer(self):
#msgs = self.dataTuning.textReadDatasTemp('')
msgs = self.dataTuning.readDatasByType('textPro')
message_parser = TextParseLogic()
srcmessages = message_parser.ConvertDataToMessage(msgs, b'\r\n', h=2)
# future
#textCls = TextClassifyLogic(srcmessages, 0.05, 0.2, 0.2, 3)
textCls = TextClassifyLogic(srcmessages, 0.1, 0.2, 0.2, 3)
formats = textCls.formatInfer(srcmessages)
nodeRoot = node(wType='root')
textSymTree = TextSympolToTree()
for f in formats:
nodeT = textSymTree.subSymbolToTree(f)
nodeRoot.children.append(nodeT)
return nodeRoot.transToDictTree()
def ftpTestOne(self, DirPath):
datas = read_datas(DirPath, 'multy')
datasF = []
for data in datas:
if len(data) < 100:
datasF.extend(data)
else:
datasF.extend(data[0:500])
srcDatasF, desDatasF = MessageConvert.clsMessageByDire(datasF)
srcdatas = get_puredatas(srcDatasF)
desdatas = get_puredatas(desDatasF)
message_parser = TextParseLogic()
srcmessages = message_parser.ConvertDataToMessage(srcdatas, b'\r\n')
desmessages = message_parser.ConvertDataToMessage(desdatas, b'\r\n')
srctextcls = TextClassifyLogicTest(srcmessages, 0.2, 0.3, 0.4, 3)
# srctextcls.FormatInferCirclelyTest(srcmessages)
destextcls = TextClassifyLogicTest(desmessages, 0.1, 0.1, 0.1, 3)
# destextcls.getHeadAndNum(desmessages)
destextcls.FormatInferCirclelyTest(desmessages)
def tuningTwoHttpBydire(self):
srcData, desData = self.tuningHttpByregix()
message_parser = TextParseLogic()
desDatas = message_parser.ConvertDataToMessage(srcData, b'\r\n')
desNowDatas = [datanow.now() for datanow in desDatas]
diff = {}
for desNow in desNowDatas:
#desNow = str(desNow).split(' ')[0]
desNow = str(desNow)
lo = desNow.find('(')
if lo != -1:
desNow = desNow[0:lo]
lo = desNow.find('/')
if lo != -1:
desNow = desNow[0:lo]
if desNow not in diff:
diff[desNow] = 1
else:
diff[desNow] = diff[desNow] + 1
print(diff)
def tuningHttpBydire(self, tPath='/home/wxw/data/httpDatas/http'):
messages = read_datas(tPath, 'single')
messages = MessageConvert.filterMsgs(['GET', 'POST', 'HTTP', 'HEAD'], messages, 10)
clsMessages = MessageConvert.clsMessagesByRegix(['GET', 'POST', 'HEAD'], 5, messages)
clsDesMessages = clsMessages['unkown']
srcDatas = []
srcDatas.extend(clsMessages['GET'])
srcDatas.extend(clsMessages['POST'])
srcDatas.extend(clsMessages['HEAD'])
message_parser = TextParseLogic()
#desDatas = message_parser.ConvertDataToMessage(clsDesMessages, b'\r\n')
desDatas = message_parser.ConvertDataToMessage(srcDatas, b'\r\n')
#desNowDatas = [datanow.now() for datanow in desDatas]
desNowDatas = [datanow.now().split(b' ')[0] for datanow in desDatas]
diff = {}
for desNow in desNowDatas:
if desNow not in diff:
diff[desNow] = 1
else:
diff[desNow] = diff[desNow] + 1
print(diff)
class FTPDataTuning:
def __init__(self):
self.msgP = TextParseLogic()
self.cmds = {b'PWD', b'331 Please specify the password', b'RETR', b'TYPE I', b'250 Directory successfully changed',
b'227 Entering Passive Mode', b'PASS', b'150 Ok to send data',
b'226 Directory send OK', b'220', b'226 Transfer complete', b'230 Login successful',
b'NLST', b'STOR', b'LIST', b'USER',
b'550 Create directory operation', b'200 Switching to', b'150 Opening BINARY mode data connection for',
b'150 Here comes the directory listing', b'200 Switching to ASCII mode', b'MKD', b'257', b'TYPE A',
b'CWD', b'PASV'}
def getDatas(self):
datas = read_datas('/home/wxw/data/ftp/ftpData', 'single')
datas = get_puredatas(datas)
return datas
def getDiff(self, datas):
desDatas = self.msgP.ConvertDataToMessage(datas, b'\r\n')
desNowDatas = [datanow.now() for datanow in desDatas]
diff = {}
for desNow in desNowDatas:
# desNow = str(desNow).split(' ')[0]
desNow = str(desNow)
lo = desNow.find('(')
if lo != -1:
desNow = desNow[0:lo]
lo = desNow.find('/')
if lo != -1:
desNow = desNow[0:lo]
if desNow not in diff:
diff[desNow] = 1
else:
diff[desNow] = diff[desNow] + 1
print(diff)
def tuningTwoHttpBydire(self):
srcData, desData = self.tuningHttpByregix()
message_parser = TextParseLogic()
desDatas = message_parser.ConvertDataToMessage(srcData, b'\r\n')
desNowDatas = [datanow.now() for datanow in desDatas]
diff = {}
for desNow in desNowDatas:
#desNow = str(desNow).split(' ')[0]
desNow = str(desNow)
lo = desNow.find('(')
if lo != -1:
desNow = desNow[0:lo]
lo = desNow.find('/')
if lo != -1:
desNow = desNow[0:lo]
if desNow not in diff:
diff[desNow] = 1
else:
diff[desNow] = diff[desNow] + 1
print(diff)
def tuningHttpBydire(self):
messages = read_datas('/home/wxw/data/httpDatas/http', 'single')
messages = get_puredatas(messages)
messages = MessageConvert.filterMsgs(['GET', 'POST', 'HTTP'], messages, 10)
clsMessages = MessageConvert.clsMessagesByRegix(['GET', 'POST'], 5, messages)
clsDesMessages = clsMessages['unkown']
message_parser = TextParseLogic()
desDatas = message_parser.ConvertDataToMessage(clsDesMessages, b'\r\n')
desNowDatas = [datanow.now() for datanow in desDatas]
diff = {}
for desNow in desNowDatas:
if desNow not in diff:
diff[desNow] = 1
else:
diff[desNow] = diff[desNow] + 1
print(diff)
def tuningHttpByregix(self):
datas = read_datas('/home/wxw/data/ftp/ftpData', 'multy')
datasF = []
for data in datas:
if len(data) < 100:
datasF.extend(data)
else:
datasF.extend(data[0:500])
srcDatasF, desDatasF = MessageConvert.clsMessageByDire(datasF)
srcDatasF = get_puredatas(srcDatasF)
desDatasF = get_puredatas(desDatasF)
print(len(desDatasF))
return srcDatasF, desDatasF
def getTotalData(self):
Tdatas = []
srcD, desD = self.tuningHttpByregix()
Tdatas.extend(srcD)
Tdatas.extend(desD)
return Tdatas
def sampleData(self):
datas = self.getDatas()
datasSplit = MessageConvert.clsMsgsByRegix(self.cmds, 20, datas)
Fdatas = []
for key in datasSplit:
print(key, len(datasSplit[key]))
if len(datasSplit[key]) > 1000:
print('zzz')
Fdatas.extend(datasSplit[key][0:500])
else:
Fdatas.extend(datasSplit[key])
print(len(Fdatas))
return Fdatas
def getTotalCommond(self):
datas = self.getDatas()
self.getDiff(datas)
def getMsgsLen(self, clses):
correLen = 0
conciouLen = 0
print(len(clses))
clsCmds = []
for cls in clses:
clsCmd = MessageConvert.clsMsgsByRegix(self.cmds, 20, cls)
clsCmds.append(clsCmd)
for clscmd in clsCmds:
if len(clscmd) == 1:
correLen = correLen + 1
for cmd in self.cmds:
tLo = 0
for clscmd in clsCmds:
if cmd in clscmd:
tLo = tLo + 1
if tLo == 1:
conciouLen = conciouLen + 1
print(correLen, conciouLen)
def __init__(self):
self.dataTuning = DataTuning()
self.textParser = TextParseLogic()
def getTotalDatas(self, messages):
message_parser = TextParseLogic()
messages = message_parser.ConvertDataToMessage(messages, b'\r\n')
self.getDiffDatas(messages)
return finalFormats
def filterSets(self, result, fResult):
cverter = Converter()
cverter.ConvertMultiListPure(result, fResult)
def classifyMessages(self, preSet, messages):
datas = self.ClassifyCircleLy(preSet, messages)
result = []
self.filterSets(datas, result)
return result
if __name__ == '__main__':
beginTime = time.time()
messages = read_datas('/home/wxw/data/httpDatas/http_measure', 'single')
messages = get_puredatas(messages)
print(len(messages))
message_parser = TextParseLogic()
messages = message_parser.ConvertDataToMessage(messages, b'\r\n')
textClassify = TextClassifyLogic(messages, 0.1, 0.4, 0.2, 3)
fFormats = textClassify.FormatInferCirclely(messages)
for f in fFormats:
print(f.getLeafFields)
#textClassify.GetFrequentWords(0.3, 3)
#print(textClassify.GetWodsRank(messages))
#msgSet = textClassify.FormatInfer(0.3, 3)
#for value in msgSet:
# print(value._str_debug())
#endTime = time.time()
#print(endTime - beginTime)