def __init__(self, path='/home/wxw/data/RedisData'):
self.msgConvert = MessageConvert()
self.path = path
self.cmds = [
b'SADD', b'LLEN', b'STRLEN', b'SET', b'SMEMBERS', b'MSET',
b'SELECT', b'LPUSH', b'MGET', b'GET', b'LPOP'
]
def tuningTotol(self):
messages = read_datas('/home/wxw/data/httpDatas/http', 'single')
messages = get_puredatas(messages)
messages = MessageConvert.filterMsgs(['GET', 'POST', 'HTTP', 'HEAD'], messages, 10)
clsMessages = MessageConvert.clsMessagesByRegix(['GET', 'POST', 'HEAD'], 5, messages)
srcDatas = []
desDatas = []
i = 0
srcDatas.extend(clsMessages['GET'])
srcDatas.extend(clsMessages['POST'])
desDatas.extend(clsMessages['unkown'])
return srcDatas, desDatas
def tuningHttpByregix(self):
messages = read_datas('/home/wxw/data/httpDatas/http', 'single')
messages = get_puredatas(messages)
messages = MessageConvert.filterMsgs(['GET', 'POST', 'HTTP'], messages, 10)
clsMessages = MessageConvert.clsMessagesByRegix(['GET', 'POST'], 5, messages)
srcDatas = []
desDatas = []
i = 0
while (i < 252):
srcDatas.append(clsMessages['GET'][i])
srcDatas.append(clsMessages['POST'][i])
i = i + 1
desDatas.extend(clsMessages['unkown'])
desDatas = desDatas[0:505]
return srcDatas, desDatas
def tuningHttpBydire(self):
messages = read_datas('/home/wxw/data/httpDatas/http', 'single')
messages = get_puredatas(messages)
messages = MessageConvert.filterMsgs(['GET', 'POST', 'HTTP'], messages, 10)
clsMessages = MessageConvert.clsMessagesByRegix(['GET', 'POST'], 5, messages)
clsDesMessages = clsMessages['unkown']
message_parser = TextParseLogic()
desDatas = message_parser.ConvertDataToMessage(clsDesMessages, b'\r\n')
desNowDatas = [datanow.now() for datanow in desDatas]
diff = {}
for desNow in desNowDatas:
if desNow not in diff:
diff[desNow] = 1
else:
diff[desNow] = diff[desNow] + 1
print(diff)
def tuningHttpBydire(self, tPath='/home/wxw/data/httpDatas/http'):
messages = read_datas(tPath, 'single')
messages = MessageConvert.filterMsgs(['GET', 'POST', 'HTTP', 'HEAD'], messages, 10)
clsMessages = MessageConvert.clsMessagesByRegix(['GET', 'POST', 'HEAD'], 5, messages)
clsDesMessages = clsMessages['unkown']
srcDatas = []
srcDatas.extend(clsMessages['GET'])
srcDatas.extend(clsMessages['POST'])
srcDatas.extend(clsMessages['HEAD'])
message_parser = TextParseLogic()
#desDatas = message_parser.ConvertDataToMessage(clsDesMessages, b'\r\n')
desDatas = message_parser.ConvertDataToMessage(srcDatas, b'\r\n')
#desNowDatas = [datanow.now() for datanow in desDatas]
desNowDatas = [datanow.now().split(b' ')[0] for datanow in desDatas]
diff = {}
for desNow in desNowDatas:
if desNow not in diff:
diff[desNow] = 1
else:
diff[desNow] = diff[desNow] + 1
print(diff)
def sampleData(self):
datas = self.getDatas()
datasSplit = MessageConvert.clsMsgsByRegix(self.cmds, 20, datas)
Fdatas = []
for key in datasSplit:
print(key, len(datasSplit[key]))
if len(datasSplit[key]) > 1000:
print('zzz')
Fdatas.extend(datasSplit[key][0:500])
else:
Fdatas.extend(datasSplit[key])
print(len(Fdatas))
return Fdatas
def tuningHttpByregix(self):
datas = read_datas('/home/wxw/data/ftp/ftpData', 'multy')
datasF = []
for data in datas:
if len(data) < 100:
datasF.extend(data)
else:
datasF.extend(data[0:500])
srcDatasF, desDatasF = MessageConvert.clsMessageByDire(datasF)
srcDatasF = get_puredatas(srcDatasF)
desDatasF = get_puredatas(desDatasF)
print(len(desDatasF))
return srcDatasF, desDatasF
def ftpTestOne(self, DirPath):
datas = read_datas(DirPath, 'multy')
datasF = []
for data in datas:
if len(data) < 100:
datasF.extend(data)
else:
datasF.extend(data[0:500])
srcDatasF, desDatasF = MessageConvert.clsMessageByDire(datasF)
srcdatas = get_puredatas(srcDatasF)
desdatas = get_puredatas(desDatasF)
message_parser = TextParseLogic()
srcmessages = message_parser.ConvertDataToMessage(srcdatas, b'\r\n')
desmessages = message_parser.ConvertDataToMessage(desdatas, b'\r\n')
srctextcls = TextClassifyLogicTest(srcmessages, 0.2, 0.3, 0.4, 3)
# srctextcls.FormatInferCirclelyTest(srcmessages)
destextcls = TextClassifyLogicTest(desmessages, 0.1, 0.1, 0.1, 3)
# destextcls.getHeadAndNum(desmessages)
destextcls.FormatInferCirclelyTest(desmessages)
def getMsgsLen(self, clses):
correLen = 0
conciouLen = 0
print(len(clses))
clsCmds = []
for cls in clses:
clsCmd = MessageConvert.clsMsgsByRegix(self.cmds, 20, cls)
clsCmds.append(clsCmd)
for clscmd in clsCmds:
if len(clscmd) == 1:
correLen = correLen + 1
for cmd in self.cmds:
tLo = 0
for clscmd in clsCmds:
if cmd in clscmd:
tLo = tLo + 1
if tLo == 1:
conciouLen = conciouLen + 1
print(correLen, conciouLen)
class RedisDataTuningTest:
def __init__(self, path):
self.redisData = RedisDataTuning(path)
self.msgConvert = MessageConvert()
def getDatas(self, path):
redisDatas = read_datas(path, 'single')
return redisDatas
def splitDatas(self, datas):
srcDatas, desDatas = self.msgConvert.clsMessageByDire(datas)
return srcDatas, desDatas
def getDesCommondTest(self, desDatas):
print(self.redisData.getDesCommond(desDatas))
def testSample(self):
srcDatas, desDatas = self.redisData.getProDatas()
sPSrcDatas = self.redisData.sampleSourceDatas(srcDatas, 150)
for datas in sPSrcDatas:
print(datas, len(sPSrcDatas[datas]))
def getTotalData(self):
datas = self.getRawDatas()
datasSplit = MessageConvert.clsMsgsByRegix(self.cmds, 20, datas)
return datasSplit
def __init__(self, path):
self.redisData = RedisDataTuning(path)
self.msgConvert = MessageConvert()
class RedisDataTuning:
def __init__(self, path='/home/wxw/data/RedisData'):
self.msgConvert = MessageConvert()
self.path = path
self.cmds = [
b'SADD', b'LLEN', b'STRLEN', b'SET', b'SMEMBERS', b'MSET',
b'SELECT', b'LPUSH', b'MGET', b'GET', b'LPOP'
]
def readDatas(self, path='/home/wxw/data/RedisData'):
redisDatas = read_datas(path, 'single')
return redisDatas
def sampleDatas(self):
datas = self.readDatas()
srcDatas, desDatas = self.splitByPureDatas(datas)
splitSrcDatas = self.sampleSourceDatas(srcDatas, 2)
print(len(splitSrcDatas))
return splitSrcDatas
def splitByDire(self, datas):
srcDatas, desDatas = self.msgConvert.clsMessageByDire(datas)
return srcDatas, desDatas
def splitByPureDatas(self, datas):
srcDatas, desDatas = self.splitByDire(datas)
return get_puredatas(srcDatas), get_puredatas(desDatas)
def getProDatas(self):
datas = self.readDatas(self.path)
srcDatas, desDatas = self.splitByPureDatas(datas)
return srcDatas, desDatas
def getSouceCommond(self, srcDatas):
srcCommond = {}
srcOne = {}
for srcData in srcDatas:
srcSplitData = srcData.split(b'\r\n')
srcDataCommond = srcSplitData[2]
if srcDataCommond not in srcCommond:
srcCommond[srcDataCommond] = 0
srcCommond[srcDataCommond] = srcCommond[srcDataCommond] + 1
srcDataOne = srcSplitData[0]
if srcDataOne not in srcOne:
srcOne[srcDataOne] = 0
srcOne[srcDataOne] = srcOne[srcDataOne] + 1
return srcCommond, srcOne
def sampleSourceDatas(self, srcDatas, msgLen, dataLen=300):
srcCDatas = {}
for srcData in srcDatas:
srcSplitData = srcData.split(b'\r\n')
srcC = srcSplitData[msgLen]
if srcC not in srcCDatas:
srcCDatas[srcC] = []
if len(srcCDatas[srcC]) < dataLen:
srcCDatas[srcC].append(srcData)
srcDatasList = []
for key in srcCDatas:
print(key)
srcDatasList.extend(srcCDatas[key])
return srcDatasList
def sampleDesDatas(self, desSrcDatas, dataLen=300):
desDatas = {}
for desData in desSrcDatas:
if desData[0] not in desDatas:
desDatas[desData[0]] = []
desDatas[desData[0]].append(desData)
Fdatas = []
Fdatas.extend(desDatas[b'+'])
Fdatas.extend(desDatas[b'-'])
return Fdatas
def getDesCommond(self, desDatas):
desCommond = {}
desCommondS = {}
desCommonT = {}
for desData in desDatas:
desSplitData = desData.split(b'\r\n')
desDataCommond = desSplitData[0]
if desDataCommond not in desCommond:
desCommond[desDataCommond] = 0
desCommond[desDataCommond] = desCommond[desDataCommond] + 1
if len(desSplitData) > 1:
desDataCommondS = desSplitData[1]
if desDataCommondS not in desCommondS:
desCommondS[desDataCommondS] = 0
desCommondS[desDataCommondS] = desCommondS[desDataCommondS] + 1
if len(desSplitData) > 2:
desDataCommondT = desSplitData[2]
if desDataCommondT not in desCommonT:
desCommonT[desDataCommondT] = 0
desCommonT[desDataCommondT] = desCommonT[desDataCommondT] + 1
return self.filterDatas(desCommond), self.filterDatas(
desCommondS), self.filterDatas(desDataCommondT)
def getMsgsLen(self, clses):
correLen = 0
conciouLen = 0
print(len(clses))
clsCmds = []
for cls in clses:
clsCmd = MessageConvert.clsMsgsByRegix(self.cmds, 20, cls)
clsCmds.append(clsCmd)
for clscmd in clsCmds:
if len(clscmd) == 1:
#print(clscmd)
correLen = correLen + 1
for cmd in self.cmds:
tLo = 0
for clscmd in clsCmds:
if cmd in clscmd:
tLo = tLo + 1
if tLo == 1:
#print('ss ',cmd)
conciouLen = conciouLen + 1
print(correLen, conciouLen)
def filterDatas(self, datas):
tNewDatas = {}
for data in datas:
if datas[data] > 5:
tNewDatas[data] = datas[data]
return tNewDatas
