class NetZobFormatInferTest:
def __init__(self):
self.nz = NetZobFormatInfer()
self.md = ModBusDataTuning()
self.ftpdata = FTPDataTuning()
def clsModbus(self):
datas = self.md.getModDatas()
dic = set()
for data in datas:
item = data[7:8]
dic.add(item)
print(len(dic))
sys.exit()
self.nz.clsMessages(datas)
def clsFTP(self):
datas = self.ftpdata.getTotalData()
self.nz.clsMessages(datas)
def clsMix(self):
Tdatas = []
Tdatas.extend(self.md.getModDatas())
Tdatas.extend(self.ftpdata.getTotalData())
self.nz.clsMessages(Tdatas, 60)
def ftpTest(self):
ftptuning = FTPDataTuning()
datas = ftptuning.sampleData()
message_parser = TextParseLogic()
datas = message_parser.ConvertDataToMessage(datas, b'\r\n')
#srctextcls = TextClassifyLogicTest(datas, 0.1, 0.3, 0.3, 3)
srctextcls = TextClassifyLogicTest(datas, 0.1, 0.2, 0.2, 3)
srctextcls.FormatInferCirclelyTest(datas)
def __init__(self, messages, tRate, sRate, wRate, wHeight):
self.tRate = tRate
self.srate = sRate
self.wRate = self.srate
self.wHeight = wHeight
self.messages = messages
self.httpData = HttpDataTuning()
self.ftpData = FTPDataTuning()
self.redisData = RedisDataTuning()
def __init__(self):
self.msgLogic = MegSplitLogic()
self.modbus = ModBusDataTuning()
self.md = modbus()
self.anlzer = base_analyzer()
self.ftp = FTPDataTuning()
self.ftpPaser = FTPParser()
self.cmPaser = ComPaser()
self.cvt = Converter()
self.mtool = MessageSplitMeasure()
self.rds = redis_deal()
def testFTPDataOne(self):
# datas = self.geneDatas()
srcDatas, desDatas = FTPDataTuning().tuningHttpByregix()
datas = []
datas.extend(srcDatas)
datas.extend(desDatas)
vePara = {'diffMeasure': 'abs', 'vWayFre': 'loose', 'T': 0, 'r': 0.3}
borders = self.msg.getFreBorders(vePara, datas)
#borders = self.msg.getOrderBorders(vePara, datas)
i = 0
primDatas = []
print(datas[0])
while (i < len(datas)):
primDatas.append(primeData(datas[i], borders[i], i))
i = i + 1
tree_builder = treeFormat(primDatas, 10, 0.2)
#t_result = tree_builder.generateNT()
t_result = tree_builder.generateSplitNT(5)
Nodes = []
Edges = []
t_result.showTree(0)
t_result.getGraph(0, Nodes, Edges, [0])
self.TGraph.graph_buildTwo(
Nodes, Edges,
'/home/wxw/paper/researchresult/BinaryFormat/treeshow/Fre 5 20.png'
)
class FieldHunterTest:
def __init__(self):
self.fhunter = FieldHunter()
self.httpdata = HttpDataTuning()
self.ftpdata = FTPDataTuning()
self.redisdata = RedisDataTuning()
def testFordeliFind(self, dirPath):
datas = read_datas(dirPath, 'multy')
datasF = []
for data in datas:
if len(data) < 100:
datasF.extend(data)
else:
datasF.extend(data[0:500])
datas = get_puredatas(datasF)
print(self.fhunter.findDelimiter(datas))
def testForHTTPFind(self):
datas = self.httpdata.sampleDatas()
print(filterFieldWords(self.fhunter.findDelimiter(datas)))
def testForFTPFind(self):
datas = self.ftpdata.sampleData()
print(filterFieldWords(self.fhunter.findDelimiter(datas)))
def testForREDISFind(self):
datas = self.redisdata.sampleDatas()
print(filterFieldWords(self.fhunter.findDelimiter(datas)))
class TextFormInfer:
def __init__(self, messages):
self.clser = TextClassify(messages)
self.httptuning = HttpDataTuning()
self.ftptuning = FTPDataTuning()
self.redistuning = RedisDataTuning()
def ldaFormatInfer(self, wSize, TK, wLen, Kcls, infercls='H'):
clsDatas = self.clser.clsMessages(wSize, TK, wLen, Kcls)
clsFormats = []
formatInfer = Format()
clusters = [cluster for cluster in clsDatas.values()]
if infercls == 'H':
self.httptuning.getMsgsLen(clusters)
elif infercls == 'F':
self.ftptuning.getMsgsLen(clusters)
else:
self.redistuning.getMsgsLen(clusters)
for clsData in clsDatas.values():
tMessages = [RawMessage(message) for message in clsData]
tempFormat = Symbol(messages=tMessages)
formatInfer.splitAligned(tempFormat, doInternalSlick=True)
clsFormats.append(tempFormat)
return clsFormats
def ladDbscanFormatInfer(self, wSize, TK, wLen, mindis, minpt, infercls):
clsDatas = self.clser.clsByDbscan(wSize, TK, wLen, mindis, minpt)
clusters = [cluster for cluster in clsDatas.values()]
if infercls == 'H':
self.httptuning.getMsgsLen(clusters)
elif infercls == 'F':
self.ftptuning.getMsgsLen(clusters)
else:
self.redistuning.getMsgsLen(clusters)
clsFormats = []
formatInfer = Format()
for clsData in clsDatas.values():
tMessages = [RawMessage(message) for message in clsData]
tempFormat = Symbol(messages=tMessages)
formatInfer.splitAligned(tempFormat, doInternalSlick=True)
clsFormats.append(tempFormat)
return clsFormats
class DataTuning:
def __init__(self):
self.ftp = FTPDataTuning()
self.redis = RedisDataTuning()
self.redis_deal = redis_deal()
self.fConfig = FileConfig()
def readDatas(self, filePath):
datas = read_filedatas(filePath)
datas = get_puredatas(datas)
return datas
def readDataSummarys(self, filePath):
datas = read_filedatas(filePath)
return datas
def readDatasTemp(self, filePath):
srcDatas, desDatas = self.ftp.tuningHttpByregix()
datas = []
datas.extend(srcDatas)
datas.extend(desDatas)
return datas
def icsReadDatasTemp(self, filePath):
messages = read_datas('/home/wxw/data/ToolDatas/15895903730.10.222',
'single')
messages = get_puredatas(messages)
return messages
def textReadDatasTemp(self, filePath):
messages = self.redis.sampleDatas()
return messages
def readDatasByType(self, fileType):
fileName = self.redis_deal.read_from_redis(fileType)
filePath = os.path.join(self.fConfig.pathDir, fileName)
return self.readDatas(filePath)
def readSummaryByType(self, fileType):
fileName = self.redis_deal.read_from_redis(fileType)
filePath = os.path.join(self.fConfig.pathDir, fileName)
print('aa')
print(filePath)
print('bb')
return self.readDataSummarys(filePath)
def testFTPData(self):
#datas = self.geneDatas()
srcDatas, desDatas = FTPDataTuning().tuningHttpByregix()
datas = []
datas.extend(srcDatas)
datas.extend(desDatas)
vePara = {'diffMeasure': 'abs', 'vWayFre': 'loose', 'T': 0, 'r': 0.3}
borders = self.msg.getFreBorders(vePara, datas)
i = 0
primDatas = []
print(datas[0])
while (i < len(datas)):
primDatas.append(primeData(datas[i], borders[i], i))
i = i + 1
tree_builder = treeFormat(primDatas, 20, 0.2)
t_result = tree_builder.generateNT()
t_result.depth_traverse()
for f in t_result.result:
print("format start")
for node_i in f:
print(node_i.loc, node_i.word_type, node_i.value)
for data in resultMs[cls]:
print(str(data))
print('mmm')
def clsDbscanTest(self, wSize, TK, wLen, minDis, minPt):
resultMs = self.textClassify.clsByDbscan(wSize, TK, wLen, minDis,
minPt)
for cls in resultMs:
print(cls)
for data in resultMs[cls]:
print(str(data))
print('mmm')
if __name__ == '__main__':
ftpDATAtuning = FTPDataTuning()
srcMsgs, desMsgs = ftpDATAtuning.tuningHttpByregix()
textCls = TextClassifyTest(srcMsgs)
textCls.clsDatasTest(3, 15, 3, 15)
"""
httpDataTuning = HttpDataTuning()
srcMsgs, desMsgs = httpDataTuning.tuningHttpByregix()
textCls = TextClassifyTest(srcMsgs)
textCls.clsDbscanTest(3, 5, 5, 0.05, 10)
"""
"""
dataF = textCls.TestFeatureGene(3, 5, 5)
dbscan = DbScanClasser()
disLists = dbscan.caculateEuras(dataF)
disLists = list(chain.from_iterable(disLists))
minData = min(disLists)
def TSumFTP():
ftpdatas = FTPDataTuning()
totalDatas = ftpdatas.getTotalData()
def __init__(self):
self.nz = NetZobFormatInfer()
self.md = ModBusDataTuning()
self.ftpdata = FTPDataTuning()
def parseMsgs(self):
ftpDatas = FTPDataTuning().getTotalData()
return self.ff.parseMsgs(ftpDatas)
def __init__(self):
self.ff = FTPParser()
self.ftpData = FTPDataTuning()
class FormatViewTest:
def __init__(self):
self.msg = MegSplitLogic()
self.TGraph = tree_graph('a', 'B')
self.moddatas = ModBusDataTuning()
self.ftp = FTPDataTuning()
def geneDatas(self):
messages = read_datas('/home/wxw/data/modbusdata', 'single')
messages = get_puredatas(messages)[0:1000]
return messages
def getMixDatas(self):
MergeDatas = []
modDatas = self.moddatas.getModDatas()
ftpDatas = self.ftp.getTotalData()
MergeDatas.extend(modDatas)
MergeDatas.extend(ftpDatas)
return MergeDatas
def testModData(self):
datas = self.geneDatas()
vePara = {'diffMeasure': 'abs', 'vWayFre': 'loose', 'T': 0, 'r': 0.3}
#borders = self.msg.getFreBorders(vePara, datas)
borders = self.msg.getOrderBorders(vePara, datas)
i = 0
primDatas = []
print(datas[0])
while (i < len(datas)):
primDatas.append(primeData(datas[i], borders[i], i))
i = i + 1
tree_builder = treeFormat(primDatas, 20, 0.2)
# t_result = tree_builder.generateNT()
t_result = tree_builder.generateSplitNT(5)
Nodes = []
Edges = []
t_result.showTree(0)
t_result.getGraph(0, Nodes, Edges, [0])
self.TGraph.graph_buildTwo(
Nodes, Edges,
'/home/wxw/paper/researchresult/BinaryFormat/treeshow/Modbus Order 10 20.png'
)
def testFTPData(self):
#datas = self.geneDatas()
srcDatas, desDatas = FTPDataTuning().tuningHttpByregix()
datas = []
datas.extend(srcDatas)
datas.extend(desDatas)
vePara = {'diffMeasure': 'abs', 'vWayFre': 'loose', 'T': 0, 'r': 0.3}
borders = self.msg.getFreBorders(vePara, datas)
i = 0
primDatas = []
print(datas[0])
while (i < len(datas)):
primDatas.append(primeData(datas[i], borders[i], i))
i = i + 1
tree_builder = treeFormat(primDatas, 20, 0.2)
t_result = tree_builder.generateNT()
t_result.depth_traverse()
for f in t_result.result:
print("format start")
for node_i in f:
print(node_i.loc, node_i.word_type, node_i.value)
#print(node_i.getNodeData())
def testFTPDataOne(self):
# datas = self.geneDatas()
srcDatas, desDatas = FTPDataTuning().tuningHttpByregix()
datas = []
datas.extend(srcDatas)
datas.extend(desDatas)
vePara = {'diffMeasure': 'abs', 'vWayFre': 'loose', 'T': 0, 'r': 0.3}
borders = self.msg.getFreBorders(vePara, datas)
#borders = self.msg.getOrderBorders(vePara, datas)
i = 0
primDatas = []
print(datas[0])
while (i < len(datas)):
primDatas.append(primeData(datas[i], borders[i], i))
i = i + 1
tree_builder = treeFormat(primDatas, 10, 0.2)
#t_result = tree_builder.generateNT()
t_result = tree_builder.generateSplitNT(5)
Nodes = []
Edges = []
t_result.showTree(0)
t_result.getGraph(0, Nodes, Edges, [0])
self.TGraph.graph_buildTwo(
Nodes, Edges,
'/home/wxw/paper/researchresult/BinaryFormat/treeshow/Fre 5 20.png'
)
#t_result.showTree(0)
#tree_builder.layyerTree()
def testMixData(self):
datas = self.getMixDatas()
vePara = {'diffMeasure': 'abs', 'vWayFre': 'loose', 'T': 0, 'r': 0.3}
#borders = self.msg.getFreBorders(vePara, datas)
borders = self.msg.getOrderBorders(vePara, datas)
i = 0
primDatas = []
print(datas[0])
while (i < len(datas)):
primDatas.append(primeData(datas[i], borders[i], i))
i = i + 1
tree_builder = treeFormat(primDatas, 10, 0.2)
# t_result = tree_builder.generateNT()
t_result = tree_builder.generateSplitNT(5)
Nodes = []
Edges = []
t_result.showTree(0)
t_result.getGraph(0, Nodes, Edges, [0])
self.TGraph.graph_buildTwo(
Nodes, Edges,
'/home/wxw/paper/researchresult/BinaryFormat/treeshow/Order 5 20 10 MixData.png'
)
def testMixTree(self):
datas = self.getMixDatas()
vePara = {'diffMeasure': 'abs', 'vWayFre': 'loose', 'T': 0, 'r': 0.3}
# borders = self.msg.getFreBorders(vePara, datas)
borders = self.msg.getOrderBorders(vePara, datas)
i = 0
primDatas = []
print(datas[0])
while (i < len(datas)):
primDatas.append(primeData(datas[i], borders[i], i))
i = i + 1
tree_builder = treeFormat(primDatas, 10, 0.2)
# t_result = tree_builder.generateNT()
t_result = tree_builder.generateSplitNT(5)
print(t_result.transToDictTree())
def __init__(self):
self.msg = MegSplitLogic()
self.TGraph = tree_graph('a', 'B')
self.moddatas = ModBusDataTuning()
self.ftp = FTPDataTuning()
def __init__(self):
self.fhunter = FieldHunter()
self.httpdata = HttpDataTuning()
self.ftpdata = FTPDataTuning()
self.redisdata = RedisDataTuning()
def __init__(self, messages):
self.clser = TextClassify(messages)
self.httptuning = HttpDataTuning()
self.ftptuning = FTPDataTuning()
self.redistuning = RedisDataTuning()
def __init__(self):
self.ftp = FTPDataTuning()
self.redis = RedisDataTuning()
self.redis_deal = redis_deal()
self.fConfig = FileConfig()
class TextClassifyLogic:
def __init__(self, messages, tRate, sRate, wRate, wHeight):
self.tRate = tRate
self.srate = sRate
self.wRate = self.srate
self.wHeight = wHeight
self.messages = messages
self.httpData = HttpDataTuning()
self.ftpData = FTPDataTuning()
self.redisData = RedisDataTuning()
def GetLocData(self, datas):
nowLocData = []
for data in datas:
nowLocData.append(data.now())
return nowLocData
def filterShort(self, freWords, h):
newFreWords = set()
for value in freWords:
if (len(value) >= h):
newFreWords.add(value)
return newFreWords
def GetFrequentWords(self, rate, h, datas):
nowLocDatas = self.GetLocData(datas)
Datas = [str(data) for data in nowLocDatas]
freWords = ApriorFreAnalyZer(Datas, rate).getApriorFre()
freWords = self.filterShort(freWords, h)
return freWords
def RankWord(self, word, datas):
nowLocDatas = self.GetLocData(datas)
Datas = [str(data) for data in nowLocDatas]
cnt = 0
loc = 0
for data in Datas:
tempLoc = data.find(word)
if tempLoc != -1:
cnt = cnt + 1
loc = loc + tempLoc
print(word, cnt, loc)
return (cnt, (loc + 1) / cnt)
def RankWords(self, freWords, datas):
words = []
for freWord in freWords:
nums = self.RankWord(freWord, datas)
words.append((freWord, nums[0], nums[1]))
words = BaseRankModel.sortList(words)
return words
def ConvertFreWords(self, data):
freSet = {}
for freWord in self.freWords:
lo = data.find(freWord)
if lo != -1:
freSet[freWord] = lo
frePattern = sorted(freSet.items(), key=lambda key: key[1])
finalPattern = ''.join([item[0] for item in frePattern])
return finalPattern
def GetWodsRank(self, datas):
freWords = self.GetFrequentWords(self.wRate, self.wHeight, datas)
rankWords = self.RankWords(freWords, datas)
return rankWords
def ClassifyMessages(self, messages):
msgSet = {}
for message in messages:
freWord = self.ConvertFreWords(str(message.message))
if freWord not in msgSet:
msgSet[freWord] = []
msgSet[freWord].append(message)
return msgSet
def ClassifyCircleLy(self, preWords, messages):
rankWords = self.GetWodsRank(messages)
funCode = None
for word in rankWords:
if word not in preWords and word[1] / len(
self.messages) > self.tRate and word[1] != len(messages):
funCode = word
break
fResult = []
print(funCode, len(messages))
#if funCode is not None and funCode[1] / len(messages) > self.trate:
if funCode is not None:
clsOne, clsTwo = self.ClassifyByCodes(funCode[0], messages)
print(len(clsOne), len(clsTwo))
if len(clsTwo) / len(self.messages) > self.tRate:
fResult.append(self.ClassifyCircleLy(preWords, clsTwo))
else:
if len(clsTwo) > 0:
fResult.append(clsTwo)
if len(clsOne) / len(self.messages) > self.tRate:
preWords.add(funCode)
fResult.append(self.ClassifyCircleLy(preWords, clsOne))
preWords.remove(funCode)
else:
fResult.append(clsOne)
else:
fResult = messages
return fResult
def ClassifyByCodes(self, codes, messages):
clsTwo = []
clsOne = []
for message in messages:
value = str(message.now())
if value.find(codes) != -1:
clsOne.append(message)
else:
clsTwo.append(message)
return (clsOne, clsTwo)
def FormatInfer(self, rate, h):
self.GetFrequentWords(rate, h)
messageClassify = self.ClassifyMessages(self.datas)
finalFormats = []
formatInfer = Format()
for key, value in messageClassify.items():
tMessages = []
for message in value:
singleMessage = RawMessage(message.message)
tMessages.append(singleMessage)
tempFormat = Symbol(messages=tMessages)
formatInfer.splitAligned(tempFormat, doInternalSlick=True)
finalFormats.append(tempFormat)
return finalFormats
def FormatInferCirclely(self, messages, Mtype):
preFre = set()
#result = textClassify.classifyMessages(preFre, messages)
result = self.classifyMessages(preFre, messages)
clsResult = []
for res in result:
clsr = []
for msg in res:
clsr.append(msg.message)
clsResult.append(clsr)
if Mtype == 'H':
self.httpData.getMsgsLen(clsResult)
elif Mtype == 'F':
self.ftpData.getMsgsLen(clsResult)
else:
self.redisData.getMsgsLen(clsResult)
#httpTuning = HttpDataTuning()
#print(httpTuning.getMsgsLen(clsResult))
#ftpTuning = FTPDataTuning()
#print(ftpTuning.getMsgsLen(clsResult))
#redisTuning = RedisDataTuning()
#redisTuning.getMsgsLen(clsResult)
finalFormats = []
formatInfer = Format()
for dataList in result:
tMessages = []
for data in dataList:
singMessage = RawMessage(data.message)
tMessages.append(singMessage)
tempFormat = Symbol(messages=tMessages)
formatInfer.splitAligned(tempFormat, doInternalSlick=True)
finalFormats.append(tempFormat)
return finalFormats
def formatInfer(self, messages):
preFre = set()
result = self.classifyMessages(preFre, messages)
finalFormats = []
formatInfer = Format()
for dataList in result:
tMessages = []
for data in dataList:
singMessage = RawMessage(data.message)
tMessages.append(singMessage)
tempFormat = Symbol(messages=tMessages)
formatInfer.splitAligned(tempFormat, doInternalSlick=True)
finalFormats.append(tempFormat)
return finalFormats
def filterSets(self, result, fResult):
cverter = Converter()
cverter.ConvertMultiListPure(result, fResult)
def classifyMessages(self, preSet, messages):
datas = self.ClassifyCircleLy(preSet, messages)
result = []
self.filterSets(datas, result)
return result
def ftpDataTestDBS(self):
httpDataTuning = FTPDataTuning()
srcMsgs, desMsgs = httpDataTuning.tuningHttpByregix()
print(len(srcMsgs) + len(desMsgs))
self.tFomInfer = TextFormInfer(desMsgs)
self.ladDbscanFormatInfer(3, 15, 3, 0.01, 4)
def __init__(self, messages):
self.tFomInfer = TextFormInfer(messages)
self.httpdata = HttpDataTuning()
self.ftpData = FTPDataTuning()
self.redisData = RedisDataTuning()
self.txtSymTree = TextSympolToTree()
class TextFormInferTest:
def __init__(self, messages):
self.tFomInfer = TextFormInfer(messages)
self.httpdata = HttpDataTuning()
self.ftpData = FTPDataTuning()
self.redisData = RedisDataTuning()
self.txtSymTree = TextSympolToTree()
def ldaFormatInferTest(self, wSize, TK, wLen, Kcls, path='', infercls='H'):
fNums = self.tFomInfer.ldaFormatInfer(wSize, TK, wLen, Kcls, infercls)
self.txtSymTree.symbolsToTree(fNums, path)
#for fnum in fNums:
# nodeT = self.txtSymTree.symbolToTree(fnum)
# nodeT.showTree(0)
#print(fnum._str_debug())
def ladDbscanFormatInfer(self,
wSize,
TK,
wLen,
mindis,
minpt,
path='',
infercls='H'):
fNums = self.tFomInfer.ladDbscanFormatInfer(wSize, TK, wLen, mindis,
minpt, infercls)
self.txtSymTree.symbolsToTree(fNums, path)
#for fnum in fNums:
# print(fnum._str_debug())
def httpDataTest(self):
srcDatas, desDatas = self.httpdata.tuningHttpByregix()
self.tFomInfer = TextFormInfer(desDatas)
self.ldaFormatInferTest(3, 15, 2, 4)
def httpTotalTest(self):
datas = self.httpdata.sampleDatas()
self.tFomInfer = TextFormInfer(datas)
self.ldaFormatInferTest(3, 15, 2, 5)
def httpTotalDBSTest(self):
datas = self.httpdata.sampleDatas()
self.tFomInfer = TextFormInfer(datas)
self.ladDbscanFormatInfer(3, 5, 3, 0.15, 10)
def ftpTotalTest(self):
datas = self.ftpData.sampleData()
self.tFomInfer = TextFormInfer(datas)
#self.ldaFormatInferTest(3, 15, 2, 10)
self.ladDbscanFormatInfer(3, 5, 3, 0.05, 10)
#self.ldaFormatInferTest(3, 15, 2, 15)
#self.ldaFormatInferTest(3, 15, 2, 20)
#self.ldaFormatInferTest(3, 15, 2, 10)
#self.ldaFormatInferTest(3, 15, 2, 5)
def ftpTotalGenerate(self):
datas = self.ftpData.sampleData()
self.tFomInfer = TextFormInfer(datas)
#self.ldaFormatInferTest(3, 15, 2, 15, '/home/wxw/paper/researchresult/text/formatInfer/KMEANS/15 ftp one.png')
#self.ladDbscanFormatInfer(3, 15, 3, 0.01, 4, '/home/wxw/paper/researchresult/text/formatInfer/DBSCAN/0.01 ftp.png')
self.ladDbscanFormatInfer(
3, 15, 3, 0.05, 10,
'/home/wxw/paper/researchresult/text/formatInfer/DBSCAN/0.05 ftp.png'
)
self.ladDbscanFormatInfer(3, 5, 3, 0.05, 10)
def httpDataTestDBS(self):
httpDataTuning = HttpDataTuning()
srcMsgs, desMsgs = httpDataTuning.tuningHttpByregix()
self.tFomInfer = TextFormInfer(desMsgs)
self.ladDbscanFormatInfer(3, 5, 3, 0.15, 10)
def ftpDataTest(self):
srcDatas, desDatas = self.ftpData.tuningHttpByregix()
self.tFomInfer = TextFormInfer(srcDatas)
self.ldaFormatInferTest(3, 15, 3, 15)
def ftpDataTestDBS(self):
httpDataTuning = FTPDataTuning()
srcMsgs, desMsgs = httpDataTuning.tuningHttpByregix()
print(len(srcMsgs) + len(desMsgs))
self.tFomInfer = TextFormInfer(desMsgs)
self.ladDbscanFormatInfer(3, 15, 3, 0.01, 4)
def redisTotalTest(self):
datas = self.redisData.sampleDatas()
self.tFomInfer = TextFormInfer(datas)
#self.ladDbscanFormatInfer(3, 5, 3, 0.10, 10)
#self.ladDbscanFormatInfer(3, 5, 3, 0.10, 20)
#self.ladDbscanFormatInfer(3, 5, 3, 0.01, 20)
#self.ladDbscanFormatInfer(3, 5, 3, 0.2, 20)
#self.ldaFormatInferTest(3, 15, 2, 5)
self.ldaFormatInferTest(3, 15, 2, 10)
#self.ldaFormatInferTest(3, 15, 2, 15)
#self.ldaFormatInferTest(3, 15, 2, 20)
def redisTotalGenerate(self):
datas = self.redisData.sampleDatas()
self.tFomInfer = TextFormInfer(datas)
self.ladDbscanFormatInfer(
3, 5, 3, 0.05, 4,
'/home/wxw/paper/researchresult/text/formatInfer/DBSCAN/redis .png'
)
#self.ldaFormatInferTest(3, 15, 2, 10, '/home/wxw/paper/researchresult/text/formatInfer/KMEANS/11 three redis.png')
def httpTotalGenerate(self, kClus):
datas = self.httpdata.sampleDatas()
self.tFomInfer = TextFormInfer(datas)
print('ss', kClus)
#self.ladDbscanFormatInfer(3, 5, 3, 0.05, 10, '/home/wxw/paper/researchresult/text/formatInfer/DBSCAN/0.15 http.png')
self.ldaFormatInferTest(
3,
15,
2,
kClus,
'/home/wxw/paper/researchresult/text/formatInfer/KMEANS/' +
str(kClus) + 'httptwo.png',
infercls='H')
print('ee', kClus)
def httpTotalRepeatGenerate(self, kClus, rTime):
datas = self.httpdata.sampleDatas()
self.tFomInfer = TextFormInfer(datas)
self.ldaFormatInferTest(
3,
15,
2,
kClus,
'/home/wxw/paper/researchresult/text/formatInfer/KMEANS/' +
str(kClus) + ' ' + str(rTime) + 'httpone.png',
infercls='H')
def httpDBSTotalGenerate(self, r, C, rTime=''):
datas = self.httpdata.sampleDatas()
self.tFomInfer = TextFormInfer(datas)
print('ss', r)
self.ladDbscanFormatInfer(
3,
5,
3,
r,
C,
'/home/wxw/paper/researchresult/text/formatInfer/DBSCAN/' +
str(r) + str(C) + rTime + 'httptwo.png',
infercls='H')
#self.ldaFormatInferTest(3, 15, 2, kClus, '/home/wxw/paper/researchresult/text/formatInfer/KMEANS/'
#+ str(kClus) + 'httpone.png', infercls='H')
print('ee', r)
def ftpTotalGenerate(self, kClus):
datas = self.ftpData.sampleData()
self.tFomInfer = TextFormInfer(datas)
self.ldaFormatInferTest(
3,
15,
2,
kClus,
'/home/wxw/paper/researchresult/text/formatInfer/KMEANS/FTP/' +
str(kClus) + 'ftp.png',
infercls='F')
#self.ladDbscanFormatInfer(3, 15, 3, 0.01, 4, '/home/wxw/paper/researchresult/text/formatInfer/DBSCAN/0.01 ftp.png')
#self.ladDbscanFormatInfer(3, 15, 3, 0.05, 10,
# '/home/wxw/paper/researchresult/text/formatInfer/DBSCAN/0.05 ftp.png')
#self.ladDbscanFormatInfer(3, 5, 3, 0.05, 10)
def ftpTotalGenerateRepeat(self, kClus, rTime):
datas = self.ftpData.sampleData()
self.tFomInfer = TextFormInfer(datas)
self.ldaFormatInferTest(
3,
15,
2,
kClus,
'/home/wxw/paper/researchresult/text/formatInfer/KMEANS/FTP/' +
str(kClus) + str(rTime) + 'ftp.png',
infercls='F')
#self.ladDbscanFormatInfer(3, 15, 3, 0.01, 4, '/home/wxw/paper/researchresult/text/formatInfer/DBSCAN/0.01 ftp.png')
#self.ladDbscanFormatInfer(3, 15, 3, 0.05, 10,
# '/home/wxw/paper/researchresult/text/formatInfer/DBSCAN/0.05 ftp.png')
#self.ladDbscanFormatInfer(3, 5, 3, 0.05, 10)
def ftpDBSCANGenerate(self, r, C, rTime=''):
datas = self.ftpData.sampleData()
self.tFomInfer = TextFormInfer(datas)
print('ss', r)
self.ladDbscanFormatInfer(
3,
5,
3,
r,
C,
'/home/wxw/paper/researchresult/text/formatInfer/DBSCAN/FTP/' +
str(r) + str(C) + rTime + 'ftp.png',
infercls='F')
# self.ldaFormatInferTest(3, 15, 2, kClus, '/home/wxw/paper/researchresult/text/formatInfer/KMEANS/'
# + str(kClus) + 'httpone.png', infercls='H')
print('ee', r)
def redisTotalGenerate(self, kClus, rTime=''):
datas = self.redisData.sampleDatas()
self.tFomInfer = TextFormInfer(datas)
self.ldaFormatInferTest(
3,
15,
2,
kClus,
'/home/wxw/paper/researchresult/text/formatInfer/KMEANS/REDIS/' +
str(kClus) + rTime + 'redis.png',
infercls='R')
#self.ladDbscanFormatInfer(3, 5, 3, 0.05, kClus, '/home/wxw/paper/researchresult/text/formatInfer/DBSCAN/redis .png')
def redisDBSCANGenerate(self, r, C, rTime=''):
datas = self.redisData.sampleDatas()
self.tFomInfer = TextFormInfer(datas)
print('ss', r)
self.ladDbscanFormatInfer(
3,
5,
3,
r,
C,
'/home/wxw/paper/researchresult/text/formatInfer/DBSCAN/REDIS/' +
str(r) + str(C) + rTime + 'redis.png',
infercls='R')
# self.ldaFormatInferTest(3, 15, 2, kClus, '/home/wxw/paper/researchresult/text/formatInfer/KMEANS/'
# + str(kClus) + 'httpone.png', infercls='H')
print('ee', r)
def getFTPdelimiter():
ftpDatas = FTPDataTuning()
datas = ftpDatas.sampleData()
freWords, deliword = getDelimiter(datas)
print(freWords, deliword)
class BorderTest:
def __init__(self):
self.msgLogic = MegSplitLogic()
self.modbus = ModBusDataTuning()
self.md = modbus()
self.anlzer = base_analyzer()
self.ftp = FTPDataTuning()
self.ftpPaser = FTPParser()
self.cmPaser = ComPaser()
self.cvt = Converter()
self.mtool = MessageSplitMeasure()
self.rds = redis_deal()
def getModHPfone(self):
self.msgs = self.modbus.getModDatas()
#vePara = {'diffMeasure': 'abs', 'vWayFre': 'loose', 'T': 0, 'r': 0.3}
vePara = {'diffMeasure': 'abs', 'vWayFre': 'normal', 'T': 0, 'r': 0.3}
msgs = self.modbus.getModDatas()
#Inferborders = self.msgLogic.getFreBorders(vePara, msgs)
#Inferborders = self.msgLogic.getOrderBorders(vePara, msgs)
#Inferborders = self.msgLogic.getEntryBorders(vePara, msgs)
Inferborders = self.msgLogic.getMbourders(vePara, msgs)
rightBorders = [self.md.GetMessageBorder(msg) for msg in self.msgs]
i = 0
print(Inferborders[0], rightBorders[0], msgs[0])
scores = []
while (i < len(Inferborders)):
scores.append(self.anlzer.get_f1(Inferborders[i], rightBorders[i]))
i = i + 1
print(sum([score[0] for score in scores]) / len(Inferborders))
print(sum([score[1] for score in scores]) / len(Inferborders))
print(sum([score[2] for score in scores]) / len(Inferborders))
def getModfoneMix(self):
self.msgs = self.modbus.getModDatas()
vePara = {'diffMeasure': 'abs', 'vWayFre': 'loose', 'T': 0, 'r': 0.3}
msgs = self.modbus.getModDatas()
Inferborders = self.msgLogic.getFreBorders(vePara, msgs)
InferbordersOne = self.msgLogic.getOrderBorders(vePara, msgs)
#InferbordersOne = self.msgLogic.getEntryBorders(vePara, msgs)
Inferborders = self.cvt.MergeListGroup(Inferborders, InferbordersOne)
rightBorders = [self.md.GetMessageBorder(msg) for msg in self.msgs]
i = 0
scores = []
while (i < len(Inferborders)):
scores.append(self.anlzer.get_f1(Inferborders[i], rightBorders[i]))
i = i + 1
print(sum([score[0] for score in scores]) / len(Inferborders))
print(sum([score[1] for score in scores]) / len(Inferborders))
print(sum([score[2] for score in scores]) / len(Inferborders))
def getFtpMixfone(self):
msgs = self.ftp.getTotalData()
vePara = {'diffMeasure': 'abs', 'vWayFre': 'loose', 'T': 0, 'r': 0.3}
msgs = self.ftpPaser.parseMsgs(msgs)
Inferborders = self.msgLogic.getFreBorders(vePara, msgs)
InferbordersOne = self.msgLogic.getOrderBorders(vePara, msgs)
# InferbordersOne = self.msgLogic.getEntryBorders(vePara, msgs)
Inferborders = self.cvt.MergeListGroup(Inferborders, InferbordersOne)
rightBorders = msgs
i = 0
scores = []
while (i < len(Inferborders)):
scores.append(self.anlzer.get_f1(Inferborders[i], rightBorders[i]))
i = i + 1
print(sum([score[0] for score in scores]) / len(Inferborders))
print(sum([score[1] for score in scores]) / len(Inferborders))
print(sum([score[2] for score in scores]) / len(Inferborders))
def getHPFtpfone(self):
msgs = self.ftp.getTotalData()
#vePara = {'diffMeasure': 'abs', 'vWayFre': 'loose', 'T': 0, 'r': 0.3}
vePara = {'diffMeasure': 'abs', 'vWayFre': 'normal', 'T': 0, 'r': 0.3}
#Inferborders = self.msgLogic.getFreBorders(vePara, msgs)
Inferborders = self.msgLogic.getOrderBorders(vePara, msgs)
#Inferborders = self.msgLogic.getEntryBorders(vePara, msgs)
#Inferborders = self.msgLogic.getMbourders(vePara, msgs)
#self.rds.insert_to_redis('nowborder', Inferborders)
#InferbordersOne = self.rds.read_from_redis('nowborder')
#print(self.mtool.MLs(Inferborders, InferbordersOne))
rightBorders = self.ftpPaser.parseMsgs(msgs)
print(Inferborders[0], rightBorders[0], msgs[0])
i = 0
scores = []
while (i < len(Inferborders)):
scores.append(self.anlzer.get_f1(Inferborders[i], rightBorders[i]))
i = i + 1
print(sum([score[0] for score in scores]) / len(Inferborders))
print(sum([score[1] for score in scores]) / len(Inferborders))
print(sum([score[2] for score in scores]) / len(Inferborders))
def getMixFone(self):
mixMsgs = []
ftpData = self.ftp.getTotalData()
modData = self.modbus.getModDatas()
mixMsgs.extend(ftpData)
mixMsgs.extend(modData)
#vePara = {'diffMeasure': 'abs', 'vWayFre': 'loose', 'T': 0, 'r': 0.3}
vePara = {'diffMeasure': 'abs', 'vWayFre': 'normal', 'T': 0, 'r': 0.3}
#Inferborders = self.msgLogic.getFreBorders(vePara, mixMsgs)
Inferborders = self.msgLogic.getOrderBorders(vePara, mixMsgs)
#Inferborders = self.msgLogic.getEntryBorders(vePara, mixMsgs)
#Inferborders = self.msgLogic.getMbourders(vePara, mixMsgs)
rightBorders = self.cmPaser.parseMsgs(mixMsgs)
print(Inferborders[0], rightBorders[0], str(mixMsgs[0]))
print(Inferborders[-1], rightBorders[-1], str(mixMsgs[-1]))
i = 0
scores = []
while (i < len(Inferborders)):
scores.append(self.anlzer.get_f1(Inferborders[i], rightBorders[i]))
i = i + 1
print(sum([score[0] for score in scores]) / len(Inferborders))
print(sum([score[1] for score in scores]) / len(Inferborders))
print(sum([score[2] for score in scores]) / len(Inferborders))
def getMixFoneScores(self, vePara, parserType):
mixMsgs = []
ftpData = self.ftp.getTotalData()
modData = self.modbus.getModDatas()
mixMsgs.extend(ftpData)
mixMsgs.extend(modData)
Inferborders = None
if parserType == 'HP':
Inferborders = self.msgLogic.getFreBorders(vePara, mixMsgs)
elif parserType == 'HE':
Inferborders = self.msgLogic.getEntryBorders(vePara, mixMsgs)
elif parserType == 'HO':
Inferborders = self.msgLogic.getOrderBorders(vePara, mixMsgs)
else:
Inferborders = self.msgLogic.getMbourders(vePara, mixMsgs)
rightBorders = self.cmPaser.parseMsgs(mixMsgs)
i = 0
scores = []
while (i < len(Inferborders)):
scores.append(self.anlzer.get_f1(Inferborders[i], rightBorders[i]))
i = i + 1
print(parserType)
print(sum([score[2] for score in scores]) / len(Inferborders))
return [score[2] for score in scores]
def getMixMixFone(self):
mixMsgs = []
ftpData = self.ftp.getTotalData()
modData = self.modbus.getModDatas()
mixMsgs.extend(ftpData)
mixMsgs.extend(ftpData)
vePara = {'diffMeasure': 'abs', 'vWayFre': 'loose', 'T': 0, 'r': 0.3}
msgs = self.ftpPaser.parseMsgs(mixMsgs)
InferbordersOne = self.msgLogic.getFreBorders(vePara, msgs)
Inferborders = self.msgLogic.getOrderBorders(vePara, msgs)
# InferbordersOne = self.msgLogic.getEntryBorders(vePara, msgs)
Inferborders = self.cvt.MergeListGroup(Inferborders, InferbordersOne)
rightBorders = msgs
i = 0
scores = []
while (i < len(Inferborders)):
scores.append(self.anlzer.get_f1(Inferborders[i], rightBorders[i]))
i = i + 1
print(sum([score[0] for score in scores]) / len(Inferborders))
print(sum([score[1] for score in scores]) / len(Inferborders))
print(sum([score[2] for score in scores]) / len(Inferborders))
