def ConvertRawToDict(self, messages):
"""
function : get frequent of each words
messages: str multiple sentences
t_dics: dict words and its frequent
"""
t_inputs = [self.ConvertRawToLengthTexts(messages)]
length = ve_strategy().vote_parameters['height'] + 1
vetorizer = CountVectorizer(ngram_range=(1, length),
stop_words=[' ', '.'],
token_pattern='(?u)\\b\\w\\w*\\b')
X = vetorizer.fit_transform(t_inputs)
t_arrays = np.squeeze(X.toarray())
words = vetorizer.get_feature_names()
t_len = len(words)
t_dics = {}
i = 0
while (i < t_len):
t_dics[words[i]] = int(str(t_arrays[i]))
i = i + 1
t_dics = self.filter_words(t_dics)
self.words_table = t_dics
prefix = ve_strategy().GetWordsKeys('RawWords')
if not redis_convert.is_exist_key(prefix):
redis_convert.insert_to_redis(prefix, t_dics)
return t_dics
def SplitByOrder(self, messages):
key = ve_strategy().GetWordsKeys('OrderWords')
if self.redis_read.is_exist_key(key):
OrderWords = self.redis_read.read_from_redis(key)
else:
raw_keys = ve_strategy().GetWordsKeys('RawWords')
raw_words = self.redis_read.read_from_redis(raw_keys)
OrderWords = word_convert().ConvertRawWordsToOrder(raw_words, self.parameters['height'] + 1)
self.redis_read.insert_to_redis(key, OrderWords)
orderVoter = OrderVoter(OrderWords)
PrimBorders = orderVoter.vote_for_messages(messages, self.parameters['height'])
FinalBorders = Desiner().VoteMultiM(PrimBorders, self.parameters['diff_measure'],
self.parameters['decision_type'],
self.parameters['Threshold_T'], self.parameters['Threshod_R'])
return Converter().ConvertListToOrder(FinalBorders)
def split_by_frequent(self, messages):
prefix = ve_strategy().GetWordsKeys('FrequentWords')
entry_words = None
if self.redis_read.is_exist_key(prefix):
frequent_words = self.redis_read.read_from_redis(prefix)
else:
raw_keys = ve_strategy().GetWordsKeys('RawWords')
raw_words = self.redis_read.read_from_redis(raw_keys)
frequent_words = Converter().ConvertRawToNormalFrequent(raw_words, self.parameters['height'] + 1)
self.redis_read.insert_to_redis(prefix, frequent_words)
frequent_voter = frequence_voter(frequent_words)
PrimBorders = frequent_voter.vote_for_messages(messages, self.parameters['height'])
FinalBorders = Desiner().VoteMultiM(PrimBorders, self.parameters['diff_measure'],
self.parameters['decision_type'],
self.parameters['Threshold_T'], self.parameters['Threshod_R'])
return Converter().ConvertListToOrder(FinalBorders)
def filter_words(self, t_dic):
stop_word = ve_strategy().vote_parameters['stop_words']
t_words_new = {}
for key in t_dic:
if key.find(stop_word) == -1:
t_words_new[key] = t_dic[key]
return t_words_new
def split_by_entry(self, messages):
keys = ve_strategy().GetWordsKeys("EntryWords")
entry_words = None
if self.redis_read.is_exist_key(keys):
entry_words = self.redis_read.read_from_redis(keys)
else:
raw_keys = ve_strategy().GetWordsKeys("RawWords")
raw_words = self.redis_read.read_from_redis(raw_keys)
entry_words = word_convert().convert_raw_to_entry(
raw_words, self.parameters['height'] + 1)
self.redis_read.insert_to_redis(keys, entry_words)
entry_voter = Entry_voter(entry_words)
PrimBorders = entry_voter.vote_for_messages(messages,
self.parameters['height'])
FinalBorders = Desiner().VoteMultiM(PrimBorders,
self.parameters['diff_measure'],
self.parameters['decision_type'],
self.parameters['Threshold_T'],
self.parameters['Threshod_R'])
return Converter().ConvertListToOrder(FinalBorders)
def ConvertRawToLengthText(self, message):
"""
converse a message to n-gram item
message: a list o bytes
return : str of n-gram items
"""
t_list = []
t_len = len(message)
i = 0
t_flist = ''
h = ve_strategy().vote_parameters['height']
while (i < t_len):
if (len(t_flist) == 0):
t_flist = t_flist + str(message[i])
else:
t_flist = t_flist + ' ' + str(message[i])
i = i + 1
i = 0
while (i < h):
t_flist = t_flist + ' ' + ve_strategy(
).vote_parameters['stop_words']
i = i + 1
return t_flist
def get_format_by_voting_expert(messages, h, combine, model, v_way, T, r):
message_splitter = splitter()
#message_split = message_splitter.split_by_ve(messages, h, combine, model, v_way, T, r)
redis_dealer = redis_deal()
strategy = ve_strategy().get_strategy_str()
#redis_dealer.insert_to_redis('split' + strategy, json.dumps(message_split))
message_split = redis_dealer.read_from_redis('split' + strategy)
message_prim_format = {}
for key in message_split:
message_prim_format[key[0]] = prime_b(key[1])
tree_builder = treefL(message_prim_format, int(0.1 * len(message_split)),
0.2)
t_result = tree_builder.generate_T()
t_result.depth_traverse()
for f in t_result.result:
print("format start")
for node_i in f:
print(node_i.loc)
import logging
from Config.ve_strategy import ve_strategy
from Config.Abtest import ABtest
vote_pre = ve_strategy().get_strategy_str()
def get_logger(file_path, logger_name):
logger = logging.getLogger(logger_name)
logger.setLevel(level = logging.INFO)
handler = logging.FileHandler(file_path)
handler.setLevel(logging.INFO)
logger.addHandler(handler)
return logger
for i in range(1, nrange + 1):
t_entrymean[i] = np.mean(np.array(t_entrylist[i]))
t_entrystd[i] = np.std(np.array(t_entrylist[i]), ddof=1)
for key in t_entrys:
if t_entrystd[len(key.split(' '))] == 0:
t_entrys[key] = 0
else:
t_entrys[key] = (t_entrys[key] - t_entrymean[len(
key.split(' '))]) / (t_entrystd[len(key.split(' '))])
return t_entrys
if __name__ == '__main__':
redis_read = redis_deal()
word_converter = word_convert()
prefix = ve_strategy().GetWordsKeys('OrderWords')
OrderWords = redis_read.read_from_redis(prefix)
OrderWords = word_converter.convert_order_to_raw(OrderWords)
redis_read.insert_to_redis(prefix, OrderWords)
print(word_converter.splitwords_bylen(OrderWords, 4)[1])
#OrderWords = word_converter.convert_raw_word_to_order(RawWords)
#OrderWords = word_converter.convert_order_to_raw(OrderWords)
#OrderPrefix = ve_strategy().GetWordsKeys('OrderWords')
#redis_read.insert_to_redis(OrderPrefix, OrderWords)
#entry_words = word_converter.convert_raw_to_entry(raw_words, 5)
#prefix = ve_strategy().get_strategy_str()
#redis_read.insert_to_redis('{}_normal_entry_words'.format(prefix), entry_words)
#order_words = redis_read.read_from_redis("order_raw_words")
#raw_words = word_converter.convert_order_to_raw(order_words)
#print(raw_words['0 6 255'], raw_words['1'])
#word_converter.convert_raw_word_to_order(raw_words)
def __init__(self):
self.prefix = ve_strategy().get_strategy_str()
self.redis_read = redis_deal()
self.parameters = ve_strategy().vote_parameters
self.ngram = voters()
self.cvt = Converter()
def __init__(self):
self.prefix = ve_strategy().get_strategy_str()
self.redis_read = redis_deal()
self.parameters = ve_strategy().vote_parameters
while (i < len(dicAs)):
if not self.MeasureDic(dicAs[i], dicBs[i]) or not self.MeasureDic(
dicBs[i], dicAs[i]):
diffDic.append((dicAs[i], dicBs[i]))
i = i + 1
return diffDic
def Measure(self, DataTure, DataPredict):
return self.MeasureDic(DataTure, DataPredict)
def MeasureTuple(self, tA, tB):
if tA[0] == tB[0] and tA[1] == tB[1]:
return True
else:
return False
def MeasureTuples(self, tAs, tBs):
result = []
if __name__ == '__main__':
prefix = ve_strategy().GetWordsKeys('raw_words')
Newprefix = ve_strategy().GetWordsKeys('RawWords')
PrimModbus = redis_convert.read_from_redis(prefix)
SecondModbus = redis_convert.read_from_redis(Newprefix)
print(MeasureAb().MeasureDic(PrimModbus, SecondModbus))
#PrimIec104 = redis_convert.read_from_redis('correct_raw_words')
#prefix = ve_strategy().GetWordsKeys('RawWords')
#NewIec104 = redis_convert.read_from_redis(prefix)
#print(word_convert().splitwords_bylen(NewIec104, 5)[1])
#print(MeasureAb().MeasureDic(PrimIec104, NewIec104))
from Fields_info.field_discover import words_discover
from log_info.logger import get_logger, vote_pre
from Fields_info.fields_measure import Fields_measure
from Config.modbus import modbus
from common.f_cg import transer
from common.Converter.base_convert import Converter
import time
from Data_base.Data_redis.redis_deal import redis_deal
from common.readdata import read_datas, get_puredatas
from functools import cmp_to_key
from common.analyzer.analyzer_common import base_analyzer
from Config.log_config import log_path
from Config.ve_strategy import ve_strategy
ve_stra_str = ve_strategy().get_strategy_str()
analyzer = base_analyzer()
def cmp_word(word_one, word_two):
start_one = int(word_one[0].split(' ')[0])
start_two = int(word_two[0].split(' ')[0])
if start_one < start_two:
return 1
elif start_one > start_two:
return -1
else:
if word_one[1] >= word_two[1]:
return 1
else:
return 0
from log_info.logger import get_logger, vote_pre
from common.readdata import *
from Config.ve_strategy import ve_strategy
from Config.log_config import log_path
from Config.encode_types import Message_encoder
from common.Converter.base_convert import Converter
from Data_base.Data_redis.redis_deal import redis_deal
from Config.UserConfig import UserConfig
from Config.VeConfig import VeConfig
from common.Converter.word_converter import word_convert
import sys
now_time = time.strftime("%Y-%m-%d %H:%m:%s", time.localtime(time.time()))
voter_logger = get_logger(log_path + '/message_vote' + vote_pre + now_time,
'messagedetaillogger')
ve_parameter = ve_strategy().vote_parameters
redis_writer = redis_deal()
redis_prefix = ve_strategy().get_strategy_str()
class voters:
def __init__(self):
self.words_fre = None
self.words_table = None
self.words_entry = None
self.glvotes = None
self.svotes = None
self.redisDeal = redis_deal()
self.wCvert = word_convert()
def query_key(self, key):
if not self.link.exists(key):
return None
else:
return json.loads(self.link.get(key))
def is_exist_key(self, key):
if self.link.exists(key):
return True
else:
return False
redis_convert = redis_deal()
if __name__ == '__main__':
redis_dealer = redis_deal()
prefix = ve_strategy().GetWordsKeys("OrderWords")
Iec104 = redis_dealer.read_from_redis(prefix)
print(Iec104)
#NewPrefix = ve_strategy().GetWordsKeys('RawWords')
#redis_dealer.insert_to_redis(NewPrefix, ModbusData)
#FrequentWordsPrim = redis_dealer.read_from_redis('modbus_one_frequent_voter_abs_normal_0_0_normal_entry_words')
#rawWordsSecond = redis_dealer.read_from_redis(prefix);
#redis_dealer.insert_to_redis(prefix, FrequentWordsPrim)
#rawWordsSecond = redis_dealer.read_from_redis(prefix);
#print(MeasureAb().Measure(FrequentWordsPrim, rawWordsSecond))
#raw_words_ones = redis_dealer.read_from_redis(raw_words_key)
#redis_dealer.insert_to_redis(raw_words_key, raw_words_values)
import sys
sys.path.append('../common/measure_tool/')
from common.measure_tool.measure_base import Base_measure
from Config.ve_strategy import ve_strategy
from log_info.logger import get_logger
import time
from Config.const_config import log_path
VE_strategy = ve_strategy().get_strategy_str()
words_logger = get_logger(
log_path + '/words_compare' + str(VE_strategy) +
time.strftime("%Y-%m-%d %H:%m:%s", time.localtime(time.time())),
'word_compare')
class Fields_measure(Base_measure):
def __init__(self, words_true, words_pre):
super().__init__(words_true, words_pre)
words_logger.error("True: ")
words_logger.error(str(words_true))
words_logger.error('\n')
words_logger.error("Predict: ")
words_logger.error(str(words_pre))
def measure(self, topk):
print(topk)
true_dic = {}
for item in self.true_data:
true_dic[item] = 1
i = 0
infer_cnt = 0
