def ConvertRawToDict(self, messages): """ function : get frequent of each words messages: str multiple sentences t_dics: dict words and its frequent """ t_inputs = [self.ConvertRawToLengthTexts(messages)] length = ve_strategy().vote_parameters['height'] + 1 vetorizer = CountVectorizer(ngram_range=(1, length), stop_words=[' ', '.'], token_pattern='(?u)\\b\\w\\w*\\b') X = vetorizer.fit_transform(t_inputs) t_arrays = np.squeeze(X.toarray()) words = vetorizer.get_feature_names() t_len = len(words) t_dics = {} i = 0 while (i < t_len): t_dics[words[i]] = int(str(t_arrays[i])) i = i + 1 t_dics = self.filter_words(t_dics) self.words_table = t_dics prefix = ve_strategy().GetWordsKeys('RawWords') if not redis_convert.is_exist_key(prefix): redis_convert.insert_to_redis(prefix, t_dics) return t_dics
def SplitByOrder(self, messages): key = ve_strategy().GetWordsKeys('OrderWords') if self.redis_read.is_exist_key(key): OrderWords = self.redis_read.read_from_redis(key) else: raw_keys = ve_strategy().GetWordsKeys('RawWords') raw_words = self.redis_read.read_from_redis(raw_keys) OrderWords = word_convert().ConvertRawWordsToOrder(raw_words, self.parameters['height'] + 1) self.redis_read.insert_to_redis(key, OrderWords) orderVoter = OrderVoter(OrderWords) PrimBorders = orderVoter.vote_for_messages(messages, self.parameters['height']) FinalBorders = Desiner().VoteMultiM(PrimBorders, self.parameters['diff_measure'], self.parameters['decision_type'], self.parameters['Threshold_T'], self.parameters['Threshod_R']) return Converter().ConvertListToOrder(FinalBorders)
def split_by_frequent(self, messages): prefix = ve_strategy().GetWordsKeys('FrequentWords') entry_words = None if self.redis_read.is_exist_key(prefix): frequent_words = self.redis_read.read_from_redis(prefix) else: raw_keys = ve_strategy().GetWordsKeys('RawWords') raw_words = self.redis_read.read_from_redis(raw_keys) frequent_words = Converter().ConvertRawToNormalFrequent(raw_words, self.parameters['height'] + 1) self.redis_read.insert_to_redis(prefix, frequent_words) frequent_voter = frequence_voter(frequent_words) PrimBorders = frequent_voter.vote_for_messages(messages, self.parameters['height']) FinalBorders = Desiner().VoteMultiM(PrimBorders, self.parameters['diff_measure'], self.parameters['decision_type'], self.parameters['Threshold_T'], self.parameters['Threshod_R']) return Converter().ConvertListToOrder(FinalBorders)
def filter_words(self, t_dic): stop_word = ve_strategy().vote_parameters['stop_words'] t_words_new = {} for key in t_dic: if key.find(stop_word) == -1: t_words_new[key] = t_dic[key] return t_words_new
def split_by_entry(self, messages): keys = ve_strategy().GetWordsKeys("EntryWords") entry_words = None if self.redis_read.is_exist_key(keys): entry_words = self.redis_read.read_from_redis(keys) else: raw_keys = ve_strategy().GetWordsKeys("RawWords") raw_words = self.redis_read.read_from_redis(raw_keys) entry_words = word_convert().convert_raw_to_entry( raw_words, self.parameters['height'] + 1) self.redis_read.insert_to_redis(keys, entry_words) entry_voter = Entry_voter(entry_words) PrimBorders = entry_voter.vote_for_messages(messages, self.parameters['height']) FinalBorders = Desiner().VoteMultiM(PrimBorders, self.parameters['diff_measure'], self.parameters['decision_type'], self.parameters['Threshold_T'], self.parameters['Threshod_R']) return Converter().ConvertListToOrder(FinalBorders)
def ConvertRawToLengthText(self, message): """ converse a message to n-gram item message: a list o bytes return : str of n-gram items """ t_list = [] t_len = len(message) i = 0 t_flist = '' h = ve_strategy().vote_parameters['height'] while (i < t_len): if (len(t_flist) == 0): t_flist = t_flist + str(message[i]) else: t_flist = t_flist + ' ' + str(message[i]) i = i + 1 i = 0 while (i < h): t_flist = t_flist + ' ' + ve_strategy( ).vote_parameters['stop_words'] i = i + 1 return t_flist
def get_format_by_voting_expert(messages, h, combine, model, v_way, T, r): message_splitter = splitter() #message_split = message_splitter.split_by_ve(messages, h, combine, model, v_way, T, r) redis_dealer = redis_deal() strategy = ve_strategy().get_strategy_str() #redis_dealer.insert_to_redis('split' + strategy, json.dumps(message_split)) message_split = redis_dealer.read_from_redis('split' + strategy) message_prim_format = {} for key in message_split: message_prim_format[key[0]] = prime_b(key[1]) tree_builder = treefL(message_prim_format, int(0.1 * len(message_split)), 0.2) t_result = tree_builder.generate_T() t_result.depth_traverse() for f in t_result.result: print("format start") for node_i in f: print(node_i.loc)
import logging from Config.ve_strategy import ve_strategy from Config.Abtest import ABtest vote_pre = ve_strategy().get_strategy_str() def get_logger(file_path, logger_name): logger = logging.getLogger(logger_name) logger.setLevel(level = logging.INFO) handler = logging.FileHandler(file_path) handler.setLevel(logging.INFO) logger.addHandler(handler) return logger
for i in range(1, nrange + 1): t_entrymean[i] = np.mean(np.array(t_entrylist[i])) t_entrystd[i] = np.std(np.array(t_entrylist[i]), ddof=1) for key in t_entrys: if t_entrystd[len(key.split(' '))] == 0: t_entrys[key] = 0 else: t_entrys[key] = (t_entrys[key] - t_entrymean[len( key.split(' '))]) / (t_entrystd[len(key.split(' '))]) return t_entrys if __name__ == '__main__': redis_read = redis_deal() word_converter = word_convert() prefix = ve_strategy().GetWordsKeys('OrderWords') OrderWords = redis_read.read_from_redis(prefix) OrderWords = word_converter.convert_order_to_raw(OrderWords) redis_read.insert_to_redis(prefix, OrderWords) print(word_converter.splitwords_bylen(OrderWords, 4)[1]) #OrderWords = word_converter.convert_raw_word_to_order(RawWords) #OrderWords = word_converter.convert_order_to_raw(OrderWords) #OrderPrefix = ve_strategy().GetWordsKeys('OrderWords') #redis_read.insert_to_redis(OrderPrefix, OrderWords) #entry_words = word_converter.convert_raw_to_entry(raw_words, 5) #prefix = ve_strategy().get_strategy_str() #redis_read.insert_to_redis('{}_normal_entry_words'.format(prefix), entry_words) #order_words = redis_read.read_from_redis("order_raw_words") #raw_words = word_converter.convert_order_to_raw(order_words) #print(raw_words['0 6 255'], raw_words['1']) #word_converter.convert_raw_word_to_order(raw_words)
def __init__(self): self.prefix = ve_strategy().get_strategy_str() self.redis_read = redis_deal() self.parameters = ve_strategy().vote_parameters self.ngram = voters() self.cvt = Converter()
def __init__(self): self.prefix = ve_strategy().get_strategy_str() self.redis_read = redis_deal() self.parameters = ve_strategy().vote_parameters
while (i < len(dicAs)): if not self.MeasureDic(dicAs[i], dicBs[i]) or not self.MeasureDic( dicBs[i], dicAs[i]): diffDic.append((dicAs[i], dicBs[i])) i = i + 1 return diffDic def Measure(self, DataTure, DataPredict): return self.MeasureDic(DataTure, DataPredict) def MeasureTuple(self, tA, tB): if tA[0] == tB[0] and tA[1] == tB[1]: return True else: return False def MeasureTuples(self, tAs, tBs): result = [] if __name__ == '__main__': prefix = ve_strategy().GetWordsKeys('raw_words') Newprefix = ve_strategy().GetWordsKeys('RawWords') PrimModbus = redis_convert.read_from_redis(prefix) SecondModbus = redis_convert.read_from_redis(Newprefix) print(MeasureAb().MeasureDic(PrimModbus, SecondModbus)) #PrimIec104 = redis_convert.read_from_redis('correct_raw_words') #prefix = ve_strategy().GetWordsKeys('RawWords') #NewIec104 = redis_convert.read_from_redis(prefix) #print(word_convert().splitwords_bylen(NewIec104, 5)[1]) #print(MeasureAb().MeasureDic(PrimIec104, NewIec104))
from Fields_info.field_discover import words_discover from log_info.logger import get_logger, vote_pre from Fields_info.fields_measure import Fields_measure from Config.modbus import modbus from common.f_cg import transer from common.Converter.base_convert import Converter import time from Data_base.Data_redis.redis_deal import redis_deal from common.readdata import read_datas, get_puredatas from functools import cmp_to_key from common.analyzer.analyzer_common import base_analyzer from Config.log_config import log_path from Config.ve_strategy import ve_strategy ve_stra_str = ve_strategy().get_strategy_str() analyzer = base_analyzer() def cmp_word(word_one, word_two): start_one = int(word_one[0].split(' ')[0]) start_two = int(word_two[0].split(' ')[0]) if start_one < start_two: return 1 elif start_one > start_two: return -1 else: if word_one[1] >= word_two[1]: return 1 else: return 0
from log_info.logger import get_logger, vote_pre from common.readdata import * from Config.ve_strategy import ve_strategy from Config.log_config import log_path from Config.encode_types import Message_encoder from common.Converter.base_convert import Converter from Data_base.Data_redis.redis_deal import redis_deal from Config.UserConfig import UserConfig from Config.VeConfig import VeConfig from common.Converter.word_converter import word_convert import sys now_time = time.strftime("%Y-%m-%d %H:%m:%s", time.localtime(time.time())) voter_logger = get_logger(log_path + '/message_vote' + vote_pre + now_time, 'messagedetaillogger') ve_parameter = ve_strategy().vote_parameters redis_writer = redis_deal() redis_prefix = ve_strategy().get_strategy_str() class voters: def __init__(self): self.words_fre = None self.words_table = None self.words_entry = None self.glvotes = None self.svotes = None self.redisDeal = redis_deal() self.wCvert = word_convert() def query_key(self, key):
if not self.link.exists(key): return None else: return json.loads(self.link.get(key)) def is_exist_key(self, key): if self.link.exists(key): return True else: return False redis_convert = redis_deal() if __name__ == '__main__': redis_dealer = redis_deal() prefix = ve_strategy().GetWordsKeys("OrderWords") Iec104 = redis_dealer.read_from_redis(prefix) print(Iec104) #NewPrefix = ve_strategy().GetWordsKeys('RawWords') #redis_dealer.insert_to_redis(NewPrefix, ModbusData) #FrequentWordsPrim = redis_dealer.read_from_redis('modbus_one_frequent_voter_abs_normal_0_0_normal_entry_words') #rawWordsSecond = redis_dealer.read_from_redis(prefix); #redis_dealer.insert_to_redis(prefix, FrequentWordsPrim) #rawWordsSecond = redis_dealer.read_from_redis(prefix); #print(MeasureAb().Measure(FrequentWordsPrim, rawWordsSecond)) #raw_words_ones = redis_dealer.read_from_redis(raw_words_key) #redis_dealer.insert_to_redis(raw_words_key, raw_words_values)
import sys sys.path.append('../common/measure_tool/') from common.measure_tool.measure_base import Base_measure from Config.ve_strategy import ve_strategy from log_info.logger import get_logger import time from Config.const_config import log_path VE_strategy = ve_strategy().get_strategy_str() words_logger = get_logger( log_path + '/words_compare' + str(VE_strategy) + time.strftime("%Y-%m-%d %H:%m:%s", time.localtime(time.time())), 'word_compare') class Fields_measure(Base_measure): def __init__(self, words_true, words_pre): super().__init__(words_true, words_pre) words_logger.error("True: ") words_logger.error(str(words_true)) words_logger.error('\n') words_logger.error("Predict: ") words_logger.error(str(words_pre)) def measure(self, topk): print(topk) true_dic = {} for item in self.true_data: true_dic[item] = 1 i = 0 infer_cnt = 0