def __init__(self):
super().__init__()
self.converter = Converter()
self.msgSpliter = MsgSpliter()
self.redis_dealer = redis_deal()
self.splt = splitter()
self.desiner = Desiner()
self.msAb = MeasureAb()
self.cvt = Converter()
self.dataTuning = DataTuning()
def __init__(self, messages):
self.messages = messages
self.cVerTer = Converter()
self.freWords = {}
self.mger = WordsMerger()
self.analyzer = base_analyzer()
self.features = None
def split_by_frequent(self, messages):
prefix = ve_strategy().GetWordsKeys('FrequentWords')
entry_words = None
if self.redis_read.is_exist_key(prefix):
frequent_words = self.redis_read.read_from_redis(prefix)
else:
raw_keys = ve_strategy().GetWordsKeys('RawWords')
raw_words = self.redis_read.read_from_redis(raw_keys)
frequent_words = Converter().ConvertRawToNormalFrequent(raw_words, self.parameters['height'] + 1)
self.redis_read.insert_to_redis(prefix, frequent_words)
frequent_voter = frequence_voter(frequent_words)
PrimBorders = frequent_voter.vote_for_messages(messages, self.parameters['height'])
FinalBorders = Desiner().VoteMultiM(PrimBorders, self.parameters['diff_measure'],
self.parameters['decision_type'],
self.parameters['Threshold_T'], self.parameters['Threshod_R'])
return Converter().ConvertListToOrder(FinalBorders)
def __init__(self, messages=None):
self.MaxLen = 40
self.lengthThreshold = 0.8
self.constThreshold = 0.98
self.idThreshold = 0.7
self.messages = messages
self.cverter = Converter()
def __init__(self, messages=None):
self.messages = messages
self.wordTypeInfer = WholeFieldTypeInfer(self.messages)
self.cvter = Converter()
self.wcvter = word_convert()
self.msgSplt = MsgSpliter()
self.dataTuning = DataTuning()
self.icsSymTree = IcsSymbolToTree()
def upDataByType(self):
datas = self.getNodeDataV()
if self.word_type == 'C':
self.value.append(datas[0])
elif self.word_type == 'F':
dicDatas = Converter().convert_raw_to_count(datas)
for key in dicDatas:
self.value.append(key)
def raw_to_log(file_path, r_way, protocol):
datas = read_datas(file_path, r_way)
datas = get_puredatas(datas)
raw_datas = []
converter = Converter()
logger_raw = get_logger(log_path + '/' + protocol, 'raw_message_logger')
i = 0
for data in datas:
logger_raw.error(str(i) + ':' + converter.convert_raw_to_text(data))
def raw_to_redis(file_path, r_way):
datas = read_datas(file_path, r_way)
datas = get_puredatas(datas)
raw_datas = []
converter = Converter()
for data in datas:
raw_datas.append(converter.convert_raw_to_text(data))
key = file_path
phrase_redis = redis_deal()
phrase_redis.insert_to_redis(key, raw_datas)
def __init__(self):
self.msgLogic = MegSplitLogic()
self.modbus = ModBusDataTuning()
self.md = modbus()
self.anlzer = base_analyzer()
self.ftp = FTPDataTuning()
self.ftpPaser = FTPParser()
self.cmPaser = ComPaser()
self.cvt = Converter()
self.mtool = MessageSplitMeasure()
self.rds = redis_deal()
def getDelimiter(datas):
convert = Converter()
messages = [convert.convert_raw_to_text(data) for data in datas]
t_results = []
for message in messages:
t_results.extend(get_ngram_words([message], (1, 2), 10))
words = analyzer.get_topk(t_results)[0:10]
deliWords = filterWords(words)
wordsList = [chr(int(word)) for word in deliWords.split(' ')]
deliW = ''.join(wordsList)
return deliW
def getBoundaries(self, configParas, gveConfigParas, messages):
freGVotes, entryGVotes = self.getGVotes(configParas, messages)
desiner = Desiner()
paraFre = {}
paraFre['diff_measure'] = gveConfigParas['diffMeasure']
paraFre['vWay'] = gveConfigParas['vWayFre']
paraFre['T'] = gveConfigParas['T']
paraFre['r'] = gveConfigParas['r']
freBoundaries = desiner.VoteSingleByDicParas(paraFre, freGVotes)
paraFre['vWay'] = gveConfigParas['vWayEntry']
entryBoundaries = desiner.VoteSingleByDicParas(paraFre, entryGVotes)
return Converter().MergeLists(freBoundaries, entryBoundaries)
def SplitByOrder(self, messages):
key = ve_strategy().GetWordsKeys('OrderWords')
if self.redis_read.is_exist_key(key):
OrderWords = self.redis_read.read_from_redis(key)
else:
raw_keys = ve_strategy().GetWordsKeys('RawWords')
raw_words = self.redis_read.read_from_redis(raw_keys)
OrderWords = word_convert().ConvertRawWordsToOrder(raw_words, self.parameters['height'] + 1)
self.redis_read.insert_to_redis(key, OrderWords)
orderVoter = OrderVoter(OrderWords)
PrimBorders = orderVoter.vote_for_messages(messages, self.parameters['height'])
FinalBorders = Desiner().VoteMultiM(PrimBorders, self.parameters['diff_measure'],
self.parameters['decision_type'],
self.parameters['Threshold_T'], self.parameters['Threshod_R'])
return Converter().ConvertListToOrder(FinalBorders)
def split_by_words_type(self, datas, T_max_range):
fields_set = []
w_infer = word_infer()
w_merger = base_merger()
w_convert = Converter()
b_analyzer = base_analyzer()
for i in range(T_max_range):
lo_datas = get_data_bylo(datas, i)
w_cnt = w_convert.convert_raw_to_count(lo_datas)
w_frequent = b_analyzer.convert_num_to_frequent(w_cnt)
w_type = w_infer.is_const_word(w_frequent, 0.95)
if w_type:
t_field = loc_field((i,i), 0)
else:
t_field = loc_field((i,i), 4)
fields_set.append(t_field)
words_f = w_merger.merge_words(fields_set)
candidate_borders = [w.loc[0] for w in words_f]
return words_f, candidate_borders
def split_by_entry(self, messages):
keys = ve_strategy().GetWordsKeys("EntryWords")
entry_words = None
if self.redis_read.is_exist_key(keys):
entry_words = self.redis_read.read_from_redis(keys)
else:
raw_keys = ve_strategy().GetWordsKeys("RawWords")
raw_words = self.redis_read.read_from_redis(raw_keys)
entry_words = word_convert().convert_raw_to_entry(
raw_words, self.parameters['height'] + 1)
self.redis_read.insert_to_redis(keys, entry_words)
entry_voter = Entry_voter(entry_words)
PrimBorders = entry_voter.vote_for_messages(messages,
self.parameters['height'])
FinalBorders = Desiner().VoteMultiM(PrimBorders,
self.parameters['diff_measure'],
self.parameters['decision_type'],
self.parameters['Threshold_T'],
self.parameters['Threshod_R'])
return Converter().ConvertListToOrder(FinalBorders)
def __init__(self):
self.cvt = Converter()
def TestMergeDicts():
dicOne = {'a': 111, 'b': 22, 'c': 10}
dicTwo = {'a': 11, 'd': 22, 'c': 10}
dicThree = {'a': 111, 'e': 22, 'f': 10}
L = [dicOne, dicTwo, dicThree]
print(Converter().MergeListDics(L))
def TestHexConvert():
cvt = Converter()
print(cvt.MergeListGroup([[1, 3, 4]], [[2, 5, 6]]))
def __init__(self):
self.prefix = ve_strategy().get_strategy_str()
self.redis_read = redis_deal()
self.parameters = ve_strategy().vote_parameters
self.ngram = voters()
self.cvt = Converter()
def CombineSplitBorders(self, messages, VoterA, VoterB):
BorderA = self.VoterNameToBorders(VoterA, messages)
BorderB = self.VoterNameToBorders(VoterB, messages)
return Converter().MergeListGroup(BorderA, BorderB)
def single_message_voter(self,
messages,
h,
voters="both",
diff_measure="abs",
v_way="normal",
T=0,
r=0):
h = ve_parameter['height']
voters = ve_parameter['voters']
diff_measure = ve_parameter['diff_measure']
v_way = ve_parameter['decision_type']
T = ve_parameter['Threshold_T']
r = ve_parameter['Threshod_R']
redis_raw_word_keys = redis_prefix + 'correct_raw_words'
if redis_writer.is_exist_key(redis_raw_word_keys):
t_dics = redis_writer.read_from_redis(redis_raw_word_keys)
else:
t_dics = self.get_keywords(messages, h + 1)
redis_writer.insert_to_redis(redis_prefix + 'correct_raw_words',
t_dics)
redis_normal_word_key = redis_prefix + 'normal_correct_words'
if redis_writer.is_exist_key(redis_normal_word_key):
t_fres = redis_writer.read_from_redis(redis_normal_word_key)
else:
t_fres = self.get_frequent(t_dics, h + 1)
t_fres["300"] = 0
redis_writer.insert_to_redis(redis_prefix + 'normal_correct_words',
t_fres)
self.words_fre = t_fres
t_entrys = self.get_backentry(t_dics, h + 1)
self.words_entry = t_entrys
self.words_table = t_dics
f_boundaries = []
voters = ve_parameter['voters']
raw_conv = Converter()
for i in range(len(messages)):
t_fre_r, t_entry_r = self.vote_sequence(messages[i], h, t_fres,
t_entrys)
#t_fre_r = self.filter_los(t_fre_r, int(len(messages[i]) - h)) # change
#t_entry_r = self.filter_los(t_entry_r, int(len(messages[i]) - h)) # change
if (voters == 'both'):
t_fre_votes = self.get_gvotes([t_fre_r, t_entry_r])
#voter_logger.error('raw: ' + str(t_fre_votes))
t_candidate_loc = self.vote_for_single_message(
t_fre_votes, diff_measure, v_way, T, r)
#voter_logger.error("voted: " + str(i) + " " + str(t_candidate_loc))
f_boundaries.append(t_candidate_loc)
elif voters == 'frequent_voter':
voter_logger.error(
'raw: ' + str(raw_conv.convert_raw_to_text(messages[i])))
voter_logger.error('raw + frequent: ' + str(t_fre_r))
t_candidate_loc = self.vote_for_single_message(
t_fre_r, diff_measure, v_way, T, r)
voter_logger.error("voted: " + str(i) + " " +
str(t_candidate_loc))
f_boundaries.append(t_candidate_loc)
else:
#voter_logger.error('raw + entry: ' + str(t_fre_r))
t_candidate_loc = self.vote_for_single_message(
t_entry_r, diff_measure, v_way, T, r)
#voter_logger.error("voted: " + str(i) + " " + str(t_candidate_loc))
f_boundaries.append(t_candidate_loc)
return f_boundaries
def __init__(self):
self.analyer = base_analyzer()
self.convert = Converter()
self.ranker = ranker()
def __init__(self):
super().__init__()
self.converter = Converter()
self.msgSpliter = MsgSpliter()
self.redis_dealer = redis_deal()
def filterSets(self, result, fResult):
cverter = Converter()
cverter.ConvertMultiListPure(result, fResult)
