def main(): ''' 构造测试集，并用restore之前的模型来进行evaluate ''' # 读取配置文件 config = Configure() # 设置logger logger = get_logger(config['path_log']) # 读取词典 vocab_util = Vocab_Util() # dict[word] = idx vocab_words = vocab_util.load_vocab(config['word_vocab_file']) # dict[char] = idx vocab_chars = vocab_util.load_vocab(config['char_vocab_file']) # dict[tag] = idx vocab_tags = vocab_util.load_vocab(config['tag_vocab_file']) # 将词典封装给模型 vocabs = [vocab_words, vocab_chars, vocab_tags] embeddings = vocab_util.get_trimmed_glove_vectors(config['trimmed_file']) # 对数据进行处理 processing_word = get_processing_word(vocab_words = vocab_words, vocab_chars = vocab_chars, lowercase = True, chars = config['use_chars'], allow_unk = True) processing_tag = get_processing_word(vocab_words = vocab_tags, lowercase = False, allow_unk = False) # 得到训练数据 test_dataset = Dataset(filename = config['test_data'], max_iter = None, processing_word = processing_word, processing_tag = processing_tag) # 构造模型进行训练 model = ner_model(config,logger,vocabs,embeddings) model.build() model.restore_session() model.evaluate(test_dataset)
def main():
# 读取配置文件
config = Configure()
# 设置logger
logger = get_logger(config['path_log'])
# 读取词典
vocab_util = Vocab_Util()
# dict[word] = idx
vocab_words = vocab_util.load_vocab(config['word_vocab_file'])
# dict[char] = idx
vocab_chars = vocab_util.load_vocab(config['char_vocab_file'])
# dict[tag] = idx
vocab_tags = vocab_util.load_vocab(config['tag_vocab_file'])
# 将词典封装给模型
vocabs = [vocab_words, vocab_chars, vocab_tags]
embeddings = vocab_util.get_trimmed_glove_vectors(config['trimmed_file'])
# 对数据进行处理
processing_word = get_processing_word(vocab_words=vocab_words,
vocab_chars=vocab_chars,
lowercase=True,
chars=config['use_chars'],
allow_unk=True)
processing_tag = get_processing_word(vocab_words=vocab_tags,
lowercase=False,
allow_unk=False)
# 得到训练数据
train_dataset = Dataset(filename=config['train_data'],
max_iter=None,
processing_word=processing_word,
processing_tag=processing_tag)
# 得到dev数据
dev_dataset = Dataset(filename=config['dev_data'],
max_iter=None,
processing_word=processing_word,
processing_tag=processing_tag)
# for data in train_dataset:
# print data
for x_batch, y_batch in train_dataset.get_minibatch(4):
print x_batch
print y_batch
# 构造模型进行训练
model = ner_model(config, logger, vocabs, embeddings)
# 构建模型图
model.build()
# 训练
model.train(train_dataset, dev_dataset)
def main():
'''
完成数据的预处理
'''
configure = Configure('./config.cfg')
processing_word = get_processing_word(lowercase=True)
# 构造dataset
train_dataset = Dataset(configure['train_data'],
processing_word=processing_word)
dev_dataset = Dataset(configure['train_data'],
processing_word=processing_word)
test_dataset = Dataset(configure['train_data'],
processing_word=processing_word)
# 构造word和tag的vocab
vocab_util = Vocab_Util()
vocab_words, vocab_tags = vocab_util.get_vocabs_from_datasets(
[train_dataset, dev_dataset, test_dataset])
# 构造词向量中的词
vocab_glove = vocab_util.get_vocabs_from_glove(configure['glove_file'])
# 取交集，同时出现在词向量词典和数据集中的词
vocab_words = vocab_words & vocab_glove
# 加入UNK和数字NUM
vocab_words.add(UNK)
vocab_words.add(NUM)
# 保存单词和tag的vocab文件
vocab_util.write_vocab(vocab_words, configure['word_vocab_file'])
vocab_util.write_vocab(vocab_tags, configure['tag_vocab_file'])
# 获取Trim Glove Vectors，并存储
vocab = vocab_util.load_vocab(configure['word_vocab_file'])
vocab_util.export_trimmed_glove_vectors(vocab, configure['glove_file'],
configure['trimmed_file'],
configure['word_embedding_dim'])
# 构造char vocab, 并且进行存储
train_dataset = Dataset(configure['train_data'])
vocab_chars = vocab_util.get_char_vocab_from_datasets(train_dataset)
vocab_util.write_vocab(vocab_chars, configure['char_vocab_file'])
result2 = result.split('\r\n')
self.child.sendline('exit')
#self.child.expect('cisuts:~ #')
print self.child.before
self.child.close()
#m = self.pReturn.search(result)
#if m != None:
# print 'return value = ', m.group('value')
if result2[1] != '0':
raise Error('05010', 'Proceeding_Test_Fail')
if __name__ == '__main__':
parser = OptionParser(usage="usage: %prog [option])")
parser.add_option("-s", "--serial_number", \
action="store", \
dest="serialNumber", \
default="RMS1003612G006R", \
help="serialNumber specifies the UUT SN")
(options, args) = parser.parse_args()
home_dir = os.environ['SATI_FT']
config = Configure(home_dir + '/BFTConfig.txt')
#config.Put('LS_Canister_SN', options.serialNumber)
config.Put('CanisterSN', options.serialNumber)
test = ShopFloorQuery(config)
result = test.Start()
print result
self.train_data.translate_sentence(self.word_embeddings)
self.dev_data.translate_sentence(self.word_embeddings)
self.test_data.translate_sentence(self.word_embeddings)
def load_init_word_embeddings(self):
self.word_embeddings = GloveEmbeddings(self.word_embeddings_file,
self.word2cnt)
def do_test(self):
self.test_data = EEData("test")
self.test_data.translate_sentence(self.word_embeddings)
print "start to decode on test set"
(p_span, r_span, f_span), (p_typed, r_typed,
f_typed) = self.decode(self.test_data,
epoch=self.epoch,
prefix="test",
do_eval=True)
print(p_span, r_span, f_span), (p_typed, r_typed, f_typed)
if __name__ == "__main__":
confs = Configure(sys.argv[1])
model = CNNModel(confs)
model.load_train_dev_data_and_embeddings()
model.create_model()
#model.save_parameters()
model.train_model()
m = p.search(reply_body)
if m:
value = m.group('Value')
return value
#def GetValue(self, name):
# return self.unitInfoData[name]
if __name__ == "__main__":
parser = OptionParser(usage="usage: %prog SerialNumber", version="0.1")
(options, args) = parser.parse_args()
if len(args) != 1:
sys.exit("Usage: FFGetUnitInfo.py SerailNumber")
config = Configure('/home/bft/SatiFT/BFTConfig.txt')
config.Put('PcbaSN', args[0])
log = Log()
log.Open('test.log')
comm = None
eventManager = EventManager()
#ffclient = FFGetUnitInfo(config, eventManager, log, comm)
ffclient = FFGetUnitInfo(config, None, None, None)
#ffclient = FFGetUnitInfo(config, log, comm)
result = ffclient.Start()
print result
print 'do info'
else:
print 'can not handle this %s' % logType
return errorDict
def deBugOutPut():
pids = psutil.pids()
systemInfo.getMemInfo(pids)
print systemInfo.processInfo_mem
if __name__ == '__main__':
# config setter
config = Configure.Configure('./config.xml')
# log file
logFileName = time.strftime('%Y-%m-%d', time.localtime(time.time()))
logger = Loger.Loger('Monitor-' + logFileName + '.log')
# system info get
systemInfo = SystemInfo.SystemInfo(pids=None, processName=None)
# alert center
alertcenter = Alert.Alert()
# process names
processNames = []
# email info
return: {word->cnt}
'''
if not word2cnt:
word2cnt = {}
for sent_key in self.word_sents_str:
for token in self.word_sents_str[sent_key]:
ch = token[0]
if ch not in word2cnt:
word2cnt[ch] = 0
word2cnt[ch] += 1
return word2cnt
if __name__ == "__main__":
confs = Configure("kbp_config.cfg")
train_data = EEData("train", confs)
test_data = EEData("test", confs)
dev_data = EEData("dev", confs)
print(train_data.size())
print(test_data.size())
print(dev_data.size())
word2cnt = train_data.get_word_to_cnt()
char2cnt = train_data.get_char_to_cnt()
word_embed = GloveEmbeddings("word_word2vec.dat", word2cnt)
char_embed = GloveEmbeddings("char_word2vec.dat", char2cnt)
print(
dest="timeout", \
default="60", \
help="timeout specifies how much time to wait for \
Badger message" )
(options, args) = parser.parse_args()
if len(args) != 1:
parser.error("wrong number of arguments")
sys.exit(1)
else:
serial_port = '' + args[0]
home_dir = os.environ['HOME']
#eventManager = EventManager()
config = Configure('Config.txt')
log = Log()
log.Open('test.log')
Comm = Comm232(config, log, serial_port)
Comm.setPort(serial_port)
Comm.setTimeout(int(options.timeout))
#time.sleep(3)
#Comm.WaitForBootPrompt()
if Comm.RecvFlag == False:
sys.exit(3)
#Comm.EnterCmdPrompt()
#log.Print( 'Comm port time out is %d' % Comm.getTimeout() )
return: {word->cnt}
'''
if not word2cnt:
word2cnt = {}
for sent_key in self.word_sents_str:
for token in self.word_sents_str[sent_key]:
ch = token[0]
if ch not in word2cnt:
word2cnt[ch] = 0
word2cnt[ch] += 1
return word2cnt
if __name__ == "__main__":
confs = Configure("config.cfg")
train_data = EEData("train", confs)
test_data = EEData("test", confs)
dev_data = EEData("dev", confs)
print train_data.size()
print test_data.size()
print dev_data.size()
word2cnt = train_data.get_word_to_cnt()
char2cnt = train_data.get_char_to_cnt()
word_embed = GloveEmbeddings("word_word2vec.dat", word2cnt)
char_embed = GloveEmbeddings("char_word2vec.dat", char2cnt)
print train_data.translate_sentence(char_embeddings=char_embed,
tpr_index = tmp_index
else:
tpr_index = tmp_index - 1
return tpr_array[tpr_index]
def eval_metric(labels, pred):
fpr, tpr, _ = metrics.roc_curve(labels, pred, pos_label=1)
tpr1 = get_tpr_from_fpr(fpr, tpr, 0.001)
tpr2 = get_tpr_from_fpr(fpr, tpr, 0.005)
tpr3 = get_tpr_from_fpr(fpr, tpr, 0.01)
return 0.4 * tpr1 + 0.3 * tpr2 + 0.3 * tpr3
if __name__ == '__main__':
config = Configure()
expert = AntExpert(model_type='lightgbm')
expert.read_train_test_data('../data/full_size/atec_anti_fraud_train.csv',
'../data/full_size/atec_anti_fraud_test_b.csv')
expert.sample_train_data(sample_type='oversample')
expert.preprocess()
expert.train_test_split(percent = 0.2)
expert.cv_train(cv=5,split_type='random')
expert.predict()
expert.write_result_to_file()
def TxTest(self):
########################################
#to to TX Test
#XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
#1,open serial and set the 38400.
#2,Pump out a window to ask operator to push the reset button.
#3,receive the radiotool@CC3220: and send out 2 to enter the Tx task options
#4,set tx interval to 10. input 7, -->10
#5,set txframe amount to 200. input 9 -->200.
#6,set txrate to 1M. 3 -->1
#7,set txpower to 6. 4 -->6
#option1 1Mbps, 6 6Mbps, 13 54Mbps
########################################
rate_list = ['1', '6', '54']
rate_index = 0
self.config = Configure('Config.txt')
for rate in ['1', '6', '13']: #(1:1Mbps, 6:6Mbps, 13:54Mbps)
#self.uut_comm = Comm232(self.config, self.log, self.uut_serial_port)
#self.uut_comm.setTimeout(2)
#self.uut_comm.SendReturn('') # Enter
#line = self.uut_comm.RecvTerminatedBy() # |
#self.uut_comm.SendReturn('2') # 2 --- to select Tx task options
#line = self.uut_comm.RecvTerminatedBy() # |
#self.uut_comm.SendReturn('7') # 7 --- set tx interval value(7) to 10
#line = self.uut_comm.RecvTerminatedBy() # |
#self.uut_comm.SendReturn('10') # 10 ---- set tx interval 10.
#line = self.uut_comm.RecvTerminatedBy() # |
#self.uut_comm.SendReturn('9') # 9 ---- set tx fram amount(9) to 200
#line = self.uut_comm.RecvTerminatedBy() # |
#self.uut_comm.SendReturn('200') # 200 ---- set fram amount 200.
#line = self.uut_comm.RecvTerminatedBy() # |
#self.uut_comm.SendReturn('3') # 3 ---- set tx rate(3) to xMbps
#line = self.uut_comm.RecvTerminatedBy() # |
#self.uut_comm.SendReturn('%s' %rate) # 1 ---- set tx rate to 1Mbps. (1:1Mbps, 6:6Mbps, 13:54Mbps)
#line = self.uut_comm.RecvTerminatedBy() # |
#self.uut_comm.SendReturn('4') # 4 --- set tx power (4) to 6
#line = self.uut_comm.RecvTerminatedBy() # |
#self.uut_comm.SendReturn('6') # 6 ---- set tx power to 6
#line = self.uut_comm.RecvTerminatedBy() # |
############################################################
# to push the reset(restart) button on N9010A signal analyzer
############################################################
############################################################
# to push the Run/Stop button and set the DC Power analyzer Time/Div to 50ms/d
############################################################
#self.uut_comm.SendReturn('5') # 5 ---- start tx task
#line = self.uut_comm.RecvTerminatedBy() # |
############################################################
# to push the Run/Stop button
############################################################
############################################################
# to fintune the DC power analyzer offset got get Current Max
############################################################
while True:
currentMax = raw_input(
"\nPlease input the DC Power Max Current for rate(%sMbps) : "
% rate_list[rate_index])
p = re.compile('^(\-|\+)?\d+(\.\d+)?$')
if p.match(currentMax):
currentMax = float(currentMax)
break
else:
print "the input Current Max [%s] is not in correct format. please try again"
lowValue = float(
eval('self.current_max_%sm_low' % rate_list[rate_index]))
highValue = float(
eval('self.current_max_%sm_high' % rate_list[rate_index]))
testName = "current_max_%sm_test" % rate_list[rate_index]
tir = TestItemResult()
tir.testName = testName
tir.lowValue = lowValue
tir.highValue = highValue
tir.testValue = currentMax
self.testItemResultList.append(tir)
if lowValue <= currentMax <= highValue:
pass
else:
self.ErrorList.append("current_max_%sm_test" %
rate_list[rate_index])
if self.fail_break == "True":
return
############################################################
# to fintune the DC power analyzer offset got get Current Avg
############################################################
while True:
currentAvg = raw_input(
"\nPlease input the DC Power Avg Current for rate(%sMbps) : "
% rate_list[rate_index])
p = re.compile('^(\-|\+)?\d+(\.\d+)?$')
if p.match(currentAvg):
currentAvg = float(currentAvg)
break
else:
print "the input Current Avg [%s] is not in correct format. please try again" % rate_list[
rate_index]
lowValue = float(
eval('self.current_avg_%sm_low' % rate_list[rate_index]))
highValue = float(
eval('self.current_avg_%sm_high' % rate_list[rate_index]))
testName = "current_avg_%sm_test" % rate_list[rate_index]
tir = TestItemResult()
tir.testName = testName
tir.lowValue = lowValue
tir.highValue = highValue
tir.testValue = currentAvg
self.testItemResultList.append(tir)
if lowValue <= currentAvg <= highValue:
pass
else:
self.ErrorList.append("current_avg_%sm_test" %
rate_list[rate_index])
if self.fail_break == "True":
return
############################################################
# to get the Power on N9010A signal analyzer
############################################################
while True:
signalPower = raw_input(
"\nPlease input the Signal Power for rate(%sMbps) : " %
rate_list[rate_index])
p = re.compile('^(\-|\+)?\d+(\.\d+)?$')
if p.match(signalPower):
signalPower = float(signalPower)
break
else:
print "the input signal Power [%s] is not in correct format. please try again"
lowValue = float(eval('self.power_%sm_low' %
rate_list[rate_index]))
highValue = float(
eval('self.power_%sm_high' % rate_list[rate_index]))
testName = "power_%sm_test" % rate_list[rate_index]
tir = TestItemResult()
tir.testName = testName
tir.lowValue = lowValue
tir.highValue = highValue
tir.testValue = signalPower
self.testItemResultList.append(tir)
if lowValue <= signalPower <= highValue:
pass
else:
self.ErrorList.append("power_%sm_test" % rate_list[rate_index])
if self.fail_break == "True":
return
rate_index = rate_index + 1
