def __init__(self): super().__init__() self.converter = Converter() self.msgSpliter = MsgSpliter() self.redis_dealer = redis_deal() self.splt = splitter() self.desiner = Desiner() self.msAb = MeasureAb() self.cvt = Converter() self.dataTuning = DataTuning()
def __init__(self, messages): self.messages = messages self.cVerTer = Converter() self.freWords = {} self.mger = WordsMerger() self.analyzer = base_analyzer() self.features = None
def split_by_frequent(self, messages): prefix = ve_strategy().GetWordsKeys('FrequentWords') entry_words = None if self.redis_read.is_exist_key(prefix): frequent_words = self.redis_read.read_from_redis(prefix) else: raw_keys = ve_strategy().GetWordsKeys('RawWords') raw_words = self.redis_read.read_from_redis(raw_keys) frequent_words = Converter().ConvertRawToNormalFrequent(raw_words, self.parameters['height'] + 1) self.redis_read.insert_to_redis(prefix, frequent_words) frequent_voter = frequence_voter(frequent_words) PrimBorders = frequent_voter.vote_for_messages(messages, self.parameters['height']) FinalBorders = Desiner().VoteMultiM(PrimBorders, self.parameters['diff_measure'], self.parameters['decision_type'], self.parameters['Threshold_T'], self.parameters['Threshod_R']) return Converter().ConvertListToOrder(FinalBorders)
def __init__(self, messages=None): self.MaxLen = 40 self.lengthThreshold = 0.8 self.constThreshold = 0.98 self.idThreshold = 0.7 self.messages = messages self.cverter = Converter()
def __init__(self, messages=None): self.messages = messages self.wordTypeInfer = WholeFieldTypeInfer(self.messages) self.cvter = Converter() self.wcvter = word_convert() self.msgSplt = MsgSpliter() self.dataTuning = DataTuning() self.icsSymTree = IcsSymbolToTree()
def upDataByType(self): datas = self.getNodeDataV() if self.word_type == 'C': self.value.append(datas[0]) elif self.word_type == 'F': dicDatas = Converter().convert_raw_to_count(datas) for key in dicDatas: self.value.append(key)
def raw_to_log(file_path, r_way, protocol): datas = read_datas(file_path, r_way) datas = get_puredatas(datas) raw_datas = [] converter = Converter() logger_raw = get_logger(log_path + '/' + protocol, 'raw_message_logger') i = 0 for data in datas: logger_raw.error(str(i) + ':' + converter.convert_raw_to_text(data))
def raw_to_redis(file_path, r_way): datas = read_datas(file_path, r_way) datas = get_puredatas(datas) raw_datas = [] converter = Converter() for data in datas: raw_datas.append(converter.convert_raw_to_text(data)) key = file_path phrase_redis = redis_deal() phrase_redis.insert_to_redis(key, raw_datas)
def __init__(self): self.msgLogic = MegSplitLogic() self.modbus = ModBusDataTuning() self.md = modbus() self.anlzer = base_analyzer() self.ftp = FTPDataTuning() self.ftpPaser = FTPParser() self.cmPaser = ComPaser() self.cvt = Converter() self.mtool = MessageSplitMeasure() self.rds = redis_deal()
def getDelimiter(datas): convert = Converter() messages = [convert.convert_raw_to_text(data) for data in datas] t_results = [] for message in messages: t_results.extend(get_ngram_words([message], (1, 2), 10)) words = analyzer.get_topk(t_results)[0:10] deliWords = filterWords(words) wordsList = [chr(int(word)) for word in deliWords.split(' ')] deliW = ''.join(wordsList) return deliW
def getBoundaries(self, configParas, gveConfigParas, messages): freGVotes, entryGVotes = self.getGVotes(configParas, messages) desiner = Desiner() paraFre = {} paraFre['diff_measure'] = gveConfigParas['diffMeasure'] paraFre['vWay'] = gveConfigParas['vWayFre'] paraFre['T'] = gveConfigParas['T'] paraFre['r'] = gveConfigParas['r'] freBoundaries = desiner.VoteSingleByDicParas(paraFre, freGVotes) paraFre['vWay'] = gveConfigParas['vWayEntry'] entryBoundaries = desiner.VoteSingleByDicParas(paraFre, entryGVotes) return Converter().MergeLists(freBoundaries, entryBoundaries)
def SplitByOrder(self, messages): key = ve_strategy().GetWordsKeys('OrderWords') if self.redis_read.is_exist_key(key): OrderWords = self.redis_read.read_from_redis(key) else: raw_keys = ve_strategy().GetWordsKeys('RawWords') raw_words = self.redis_read.read_from_redis(raw_keys) OrderWords = word_convert().ConvertRawWordsToOrder(raw_words, self.parameters['height'] + 1) self.redis_read.insert_to_redis(key, OrderWords) orderVoter = OrderVoter(OrderWords) PrimBorders = orderVoter.vote_for_messages(messages, self.parameters['height']) FinalBorders = Desiner().VoteMultiM(PrimBorders, self.parameters['diff_measure'], self.parameters['decision_type'], self.parameters['Threshold_T'], self.parameters['Threshod_R']) return Converter().ConvertListToOrder(FinalBorders)
def split_by_words_type(self, datas, T_max_range): fields_set = [] w_infer = word_infer() w_merger = base_merger() w_convert = Converter() b_analyzer = base_analyzer() for i in range(T_max_range): lo_datas = get_data_bylo(datas, i) w_cnt = w_convert.convert_raw_to_count(lo_datas) w_frequent = b_analyzer.convert_num_to_frequent(w_cnt) w_type = w_infer.is_const_word(w_frequent, 0.95) if w_type: t_field = loc_field((i,i), 0) else: t_field = loc_field((i,i), 4) fields_set.append(t_field) words_f = w_merger.merge_words(fields_set) candidate_borders = [w.loc[0] for w in words_f] return words_f, candidate_borders
def split_by_entry(self, messages): keys = ve_strategy().GetWordsKeys("EntryWords") entry_words = None if self.redis_read.is_exist_key(keys): entry_words = self.redis_read.read_from_redis(keys) else: raw_keys = ve_strategy().GetWordsKeys("RawWords") raw_words = self.redis_read.read_from_redis(raw_keys) entry_words = word_convert().convert_raw_to_entry( raw_words, self.parameters['height'] + 1) self.redis_read.insert_to_redis(keys, entry_words) entry_voter = Entry_voter(entry_words) PrimBorders = entry_voter.vote_for_messages(messages, self.parameters['height']) FinalBorders = Desiner().VoteMultiM(PrimBorders, self.parameters['diff_measure'], self.parameters['decision_type'], self.parameters['Threshold_T'], self.parameters['Threshod_R']) return Converter().ConvertListToOrder(FinalBorders)
def __init__(self): self.cvt = Converter()
def TestMergeDicts(): dicOne = {'a': 111, 'b': 22, 'c': 10} dicTwo = {'a': 11, 'd': 22, 'c': 10} dicThree = {'a': 111, 'e': 22, 'f': 10} L = [dicOne, dicTwo, dicThree] print(Converter().MergeListDics(L))
def TestHexConvert(): cvt = Converter() print(cvt.MergeListGroup([[1, 3, 4]], [[2, 5, 6]]))
def __init__(self): self.prefix = ve_strategy().get_strategy_str() self.redis_read = redis_deal() self.parameters = ve_strategy().vote_parameters self.ngram = voters() self.cvt = Converter()
def CombineSplitBorders(self, messages, VoterA, VoterB): BorderA = self.VoterNameToBorders(VoterA, messages) BorderB = self.VoterNameToBorders(VoterB, messages) return Converter().MergeListGroup(BorderA, BorderB)
def single_message_voter(self, messages, h, voters="both", diff_measure="abs", v_way="normal", T=0, r=0): h = ve_parameter['height'] voters = ve_parameter['voters'] diff_measure = ve_parameter['diff_measure'] v_way = ve_parameter['decision_type'] T = ve_parameter['Threshold_T'] r = ve_parameter['Threshod_R'] redis_raw_word_keys = redis_prefix + 'correct_raw_words' if redis_writer.is_exist_key(redis_raw_word_keys): t_dics = redis_writer.read_from_redis(redis_raw_word_keys) else: t_dics = self.get_keywords(messages, h + 1) redis_writer.insert_to_redis(redis_prefix + 'correct_raw_words', t_dics) redis_normal_word_key = redis_prefix + 'normal_correct_words' if redis_writer.is_exist_key(redis_normal_word_key): t_fres = redis_writer.read_from_redis(redis_normal_word_key) else: t_fres = self.get_frequent(t_dics, h + 1) t_fres["300"] = 0 redis_writer.insert_to_redis(redis_prefix + 'normal_correct_words', t_fres) self.words_fre = t_fres t_entrys = self.get_backentry(t_dics, h + 1) self.words_entry = t_entrys self.words_table = t_dics f_boundaries = [] voters = ve_parameter['voters'] raw_conv = Converter() for i in range(len(messages)): t_fre_r, t_entry_r = self.vote_sequence(messages[i], h, t_fres, t_entrys) #t_fre_r = self.filter_los(t_fre_r, int(len(messages[i]) - h)) # change #t_entry_r = self.filter_los(t_entry_r, int(len(messages[i]) - h)) # change if (voters == 'both'): t_fre_votes = self.get_gvotes([t_fre_r, t_entry_r]) #voter_logger.error('raw: ' + str(t_fre_votes)) t_candidate_loc = self.vote_for_single_message( t_fre_votes, diff_measure, v_way, T, r) #voter_logger.error("voted: " + str(i) + " " + str(t_candidate_loc)) f_boundaries.append(t_candidate_loc) elif voters == 'frequent_voter': voter_logger.error( 'raw: ' + str(raw_conv.convert_raw_to_text(messages[i]))) voter_logger.error('raw + frequent: ' + str(t_fre_r)) t_candidate_loc = self.vote_for_single_message( t_fre_r, diff_measure, v_way, T, r) voter_logger.error("voted: " + str(i) + " " + str(t_candidate_loc)) f_boundaries.append(t_candidate_loc) else: #voter_logger.error('raw + entry: ' + str(t_fre_r)) t_candidate_loc = self.vote_for_single_message( t_entry_r, diff_measure, v_way, T, r) #voter_logger.error("voted: " + str(i) + " " + str(t_candidate_loc)) f_boundaries.append(t_candidate_loc) return f_boundaries
def __init__(self): self.analyer = base_analyzer() self.convert = Converter() self.ranker = ranker()
def __init__(self): super().__init__() self.converter = Converter() self.msgSpliter = MsgSpliter() self.redis_dealer = redis_deal()
def filterSets(self, result, fResult): cverter = Converter() cverter.ConvertMultiListPure(result, fResult)