def main(): ''' 构造测试集,并用restore之前的模型来进行evaluate ''' # 读取配置文件 config = Configure() # 设置logger logger = get_logger(config['path_log']) # 读取词典 vocab_util = Vocab_Util() # dict[word] = idx vocab_words = vocab_util.load_vocab(config['word_vocab_file']) # dict[char] = idx vocab_chars = vocab_util.load_vocab(config['char_vocab_file']) # dict[tag] = idx vocab_tags = vocab_util.load_vocab(config['tag_vocab_file']) # 将词典封装给模型 vocabs = [vocab_words, vocab_chars, vocab_tags] embeddings = vocab_util.get_trimmed_glove_vectors(config['trimmed_file']) # 对数据进行处理 processing_word = get_processing_word(vocab_words = vocab_words, vocab_chars = vocab_chars, lowercase = True, chars = config['use_chars'], allow_unk = True) processing_tag = get_processing_word(vocab_words = vocab_tags, lowercase = False, allow_unk = False) # 得到训练数据 test_dataset = Dataset(filename = config['test_data'], max_iter = None, processing_word = processing_word, processing_tag = processing_tag) # 构造模型进行训练 model = ner_model(config,logger,vocabs,embeddings) model.build() model.restore_session() model.evaluate(test_dataset)
def main(): # 读取配置文件 config = Configure() # 设置logger logger = get_logger(config['path_log']) # 读取词典 vocab_util = Vocab_Util() # dict[word] = idx vocab_words = vocab_util.load_vocab(config['word_vocab_file']) # dict[char] = idx vocab_chars = vocab_util.load_vocab(config['char_vocab_file']) # dict[tag] = idx vocab_tags = vocab_util.load_vocab(config['tag_vocab_file']) # 将词典封装给模型 vocabs = [vocab_words, vocab_chars, vocab_tags] embeddings = vocab_util.get_trimmed_glove_vectors(config['trimmed_file']) # 对数据进行处理 processing_word = get_processing_word(vocab_words=vocab_words, vocab_chars=vocab_chars, lowercase=True, chars=config['use_chars'], allow_unk=True) processing_tag = get_processing_word(vocab_words=vocab_tags, lowercase=False, allow_unk=False) # 得到训练数据 train_dataset = Dataset(filename=config['train_data'], max_iter=None, processing_word=processing_word, processing_tag=processing_tag) # 得到dev数据 dev_dataset = Dataset(filename=config['dev_data'], max_iter=None, processing_word=processing_word, processing_tag=processing_tag) # for data in train_dataset: # print data for x_batch, y_batch in train_dataset.get_minibatch(4): print x_batch print y_batch # 构造模型进行训练 model = ner_model(config, logger, vocabs, embeddings) # 构建模型图 model.build() # 训练 model.train(train_dataset, dev_dataset)
def main(): ''' 完成数据的预处理 ''' configure = Configure('./config.cfg') processing_word = get_processing_word(lowercase=True) # 构造dataset train_dataset = Dataset(configure['train_data'], processing_word=processing_word) dev_dataset = Dataset(configure['train_data'], processing_word=processing_word) test_dataset = Dataset(configure['train_data'], processing_word=processing_word) # 构造word和tag的vocab vocab_util = Vocab_Util() vocab_words, vocab_tags = vocab_util.get_vocabs_from_datasets( [train_dataset, dev_dataset, test_dataset]) # 构造词向量中的词 vocab_glove = vocab_util.get_vocabs_from_glove(configure['glove_file']) # 取交集,同时出现在词向量词典和数据集中的词 vocab_words = vocab_words & vocab_glove # 加入UNK和数字NUM vocab_words.add(UNK) vocab_words.add(NUM) # 保存单词和tag的vocab文件 vocab_util.write_vocab(vocab_words, configure['word_vocab_file']) vocab_util.write_vocab(vocab_tags, configure['tag_vocab_file']) # 获取Trim Glove Vectors,并存储 vocab = vocab_util.load_vocab(configure['word_vocab_file']) vocab_util.export_trimmed_glove_vectors(vocab, configure['glove_file'], configure['trimmed_file'], configure['word_embedding_dim']) # 构造char vocab, 并且进行存储 train_dataset = Dataset(configure['train_data']) vocab_chars = vocab_util.get_char_vocab_from_datasets(train_dataset) vocab_util.write_vocab(vocab_chars, configure['char_vocab_file'])
result2 = result.split('\r\n') self.child.sendline('exit') #self.child.expect('cisuts:~ #') print self.child.before self.child.close() #m = self.pReturn.search(result) #if m != None: # print 'return value = ', m.group('value') if result2[1] != '0': raise Error('05010', 'Proceeding_Test_Fail') if __name__ == '__main__': parser = OptionParser(usage="usage: %prog [option])") parser.add_option("-s", "--serial_number", \ action="store", \ dest="serialNumber", \ default="RMS1003612G006R", \ help="serialNumber specifies the UUT SN") (options, args) = parser.parse_args() home_dir = os.environ['SATI_FT'] config = Configure(home_dir + '/BFTConfig.txt') #config.Put('LS_Canister_SN', options.serialNumber) config.Put('CanisterSN', options.serialNumber) test = ShopFloorQuery(config) result = test.Start() print result
self.train_data.translate_sentence(self.word_embeddings) self.dev_data.translate_sentence(self.word_embeddings) self.test_data.translate_sentence(self.word_embeddings) def load_init_word_embeddings(self): self.word_embeddings = GloveEmbeddings(self.word_embeddings_file, self.word2cnt) def do_test(self): self.test_data = EEData("test") self.test_data.translate_sentence(self.word_embeddings) print "start to decode on test set" (p_span, r_span, f_span), (p_typed, r_typed, f_typed) = self.decode(self.test_data, epoch=self.epoch, prefix="test", do_eval=True) print(p_span, r_span, f_span), (p_typed, r_typed, f_typed) if __name__ == "__main__": confs = Configure(sys.argv[1]) model = CNNModel(confs) model.load_train_dev_data_and_embeddings() model.create_model() #model.save_parameters() model.train_model()
m = p.search(reply_body) if m: value = m.group('Value') return value #def GetValue(self, name): # return self.unitInfoData[name] if __name__ == "__main__": parser = OptionParser(usage="usage: %prog SerialNumber", version="0.1") (options, args) = parser.parse_args() if len(args) != 1: sys.exit("Usage: FFGetUnitInfo.py SerailNumber") config = Configure('/home/bft/SatiFT/BFTConfig.txt') config.Put('PcbaSN', args[0]) log = Log() log.Open('test.log') comm = None eventManager = EventManager() #ffclient = FFGetUnitInfo(config, eventManager, log, comm) ffclient = FFGetUnitInfo(config, None, None, None) #ffclient = FFGetUnitInfo(config, log, comm) result = ffclient.Start() print result
print 'do info' else: print 'can not handle this %s' % logType return errorDict def deBugOutPut(): pids = psutil.pids() systemInfo.getMemInfo(pids) print systemInfo.processInfo_mem if __name__ == '__main__': # config setter config = Configure.Configure('./config.xml') # log file logFileName = time.strftime('%Y-%m-%d', time.localtime(time.time())) logger = Loger.Loger('Monitor-' + logFileName + '.log') # system info get systemInfo = SystemInfo.SystemInfo(pids=None, processName=None) # alert center alertcenter = Alert.Alert() # process names processNames = [] # email info
return: {word->cnt} ''' if not word2cnt: word2cnt = {} for sent_key in self.word_sents_str: for token in self.word_sents_str[sent_key]: ch = token[0] if ch not in word2cnt: word2cnt[ch] = 0 word2cnt[ch] += 1 return word2cnt if __name__ == "__main__": confs = Configure("kbp_config.cfg") train_data = EEData("train", confs) test_data = EEData("test", confs) dev_data = EEData("dev", confs) print(train_data.size()) print(test_data.size()) print(dev_data.size()) word2cnt = train_data.get_word_to_cnt() char2cnt = train_data.get_char_to_cnt() word_embed = GloveEmbeddings("word_word2vec.dat", word2cnt) char_embed = GloveEmbeddings("char_word2vec.dat", char2cnt) print(
dest="timeout", \ default="60", \ help="timeout specifies how much time to wait for \ Badger message" ) (options, args) = parser.parse_args() if len(args) != 1: parser.error("wrong number of arguments") sys.exit(1) else: serial_port = '' + args[0] home_dir = os.environ['HOME'] #eventManager = EventManager() config = Configure('Config.txt') log = Log() log.Open('test.log') Comm = Comm232(config, log, serial_port) Comm.setPort(serial_port) Comm.setTimeout(int(options.timeout)) #time.sleep(3) #Comm.WaitForBootPrompt() if Comm.RecvFlag == False: sys.exit(3) #Comm.EnterCmdPrompt() #log.Print( 'Comm port time out is %d' % Comm.getTimeout() )
return: {word->cnt} ''' if not word2cnt: word2cnt = {} for sent_key in self.word_sents_str: for token in self.word_sents_str[sent_key]: ch = token[0] if ch not in word2cnt: word2cnt[ch] = 0 word2cnt[ch] += 1 return word2cnt if __name__ == "__main__": confs = Configure("config.cfg") train_data = EEData("train", confs) test_data = EEData("test", confs) dev_data = EEData("dev", confs) print train_data.size() print test_data.size() print dev_data.size() word2cnt = train_data.get_word_to_cnt() char2cnt = train_data.get_char_to_cnt() word_embed = GloveEmbeddings("word_word2vec.dat", word2cnt) char_embed = GloveEmbeddings("char_word2vec.dat", char2cnt) print train_data.translate_sentence(char_embeddings=char_embed,
tpr_index = tmp_index else: tpr_index = tmp_index - 1 return tpr_array[tpr_index] def eval_metric(labels, pred): fpr, tpr, _ = metrics.roc_curve(labels, pred, pos_label=1) tpr1 = get_tpr_from_fpr(fpr, tpr, 0.001) tpr2 = get_tpr_from_fpr(fpr, tpr, 0.005) tpr3 = get_tpr_from_fpr(fpr, tpr, 0.01) return 0.4 * tpr1 + 0.3 * tpr2 + 0.3 * tpr3 if __name__ == '__main__': config = Configure() expert = AntExpert(model_type='lightgbm') expert.read_train_test_data('../data/full_size/atec_anti_fraud_train.csv', '../data/full_size/atec_anti_fraud_test_b.csv') expert.sample_train_data(sample_type='oversample') expert.preprocess() expert.train_test_split(percent = 0.2) expert.cv_train(cv=5,split_type='random') expert.predict() expert.write_result_to_file()
def TxTest(self): ######################################## #to to TX Test #XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX #1,open serial and set the 38400. #2,Pump out a window to ask operator to push the reset button. #3,receive the radiotool@CC3220: and send out 2 to enter the Tx task options #4,set tx interval to 10. input 7, -->10 #5,set txframe amount to 200. input 9 -->200. #6,set txrate to 1M. 3 -->1 #7,set txpower to 6. 4 -->6 #option1 1Mbps, 6 6Mbps, 13 54Mbps ######################################## rate_list = ['1', '6', '54'] rate_index = 0 self.config = Configure('Config.txt') for rate in ['1', '6', '13']: #(1:1Mbps, 6:6Mbps, 13:54Mbps) #self.uut_comm = Comm232(self.config, self.log, self.uut_serial_port) #self.uut_comm.setTimeout(2) #self.uut_comm.SendReturn('') # Enter #line = self.uut_comm.RecvTerminatedBy() # | #self.uut_comm.SendReturn('2') # 2 --- to select Tx task options #line = self.uut_comm.RecvTerminatedBy() # | #self.uut_comm.SendReturn('7') # 7 --- set tx interval value(7) to 10 #line = self.uut_comm.RecvTerminatedBy() # | #self.uut_comm.SendReturn('10') # 10 ---- set tx interval 10. #line = self.uut_comm.RecvTerminatedBy() # | #self.uut_comm.SendReturn('9') # 9 ---- set tx fram amount(9) to 200 #line = self.uut_comm.RecvTerminatedBy() # | #self.uut_comm.SendReturn('200') # 200 ---- set fram amount 200. #line = self.uut_comm.RecvTerminatedBy() # | #self.uut_comm.SendReturn('3') # 3 ---- set tx rate(3) to xMbps #line = self.uut_comm.RecvTerminatedBy() # | #self.uut_comm.SendReturn('%s' %rate) # 1 ---- set tx rate to 1Mbps. (1:1Mbps, 6:6Mbps, 13:54Mbps) #line = self.uut_comm.RecvTerminatedBy() # | #self.uut_comm.SendReturn('4') # 4 --- set tx power (4) to 6 #line = self.uut_comm.RecvTerminatedBy() # | #self.uut_comm.SendReturn('6') # 6 ---- set tx power to 6 #line = self.uut_comm.RecvTerminatedBy() # | ############################################################ # to push the reset(restart) button on N9010A signal analyzer ############################################################ ############################################################ # to push the Run/Stop button and set the DC Power analyzer Time/Div to 50ms/d ############################################################ #self.uut_comm.SendReturn('5') # 5 ---- start tx task #line = self.uut_comm.RecvTerminatedBy() # | ############################################################ # to push the Run/Stop button ############################################################ ############################################################ # to fintune the DC power analyzer offset got get Current Max ############################################################ while True: currentMax = raw_input( "\nPlease input the DC Power Max Current for rate(%sMbps) : " % rate_list[rate_index]) p = re.compile('^(\-|\+)?\d+(\.\d+)?$') if p.match(currentMax): currentMax = float(currentMax) break else: print "the input Current Max [%s] is not in correct format. please try again" lowValue = float( eval('self.current_max_%sm_low' % rate_list[rate_index])) highValue = float( eval('self.current_max_%sm_high' % rate_list[rate_index])) testName = "current_max_%sm_test" % rate_list[rate_index] tir = TestItemResult() tir.testName = testName tir.lowValue = lowValue tir.highValue = highValue tir.testValue = currentMax self.testItemResultList.append(tir) if lowValue <= currentMax <= highValue: pass else: self.ErrorList.append("current_max_%sm_test" % rate_list[rate_index]) if self.fail_break == "True": return ############################################################ # to fintune the DC power analyzer offset got get Current Avg ############################################################ while True: currentAvg = raw_input( "\nPlease input the DC Power Avg Current for rate(%sMbps) : " % rate_list[rate_index]) p = re.compile('^(\-|\+)?\d+(\.\d+)?$') if p.match(currentAvg): currentAvg = float(currentAvg) break else: print "the input Current Avg [%s] is not in correct format. please try again" % rate_list[ rate_index] lowValue = float( eval('self.current_avg_%sm_low' % rate_list[rate_index])) highValue = float( eval('self.current_avg_%sm_high' % rate_list[rate_index])) testName = "current_avg_%sm_test" % rate_list[rate_index] tir = TestItemResult() tir.testName = testName tir.lowValue = lowValue tir.highValue = highValue tir.testValue = currentAvg self.testItemResultList.append(tir) if lowValue <= currentAvg <= highValue: pass else: self.ErrorList.append("current_avg_%sm_test" % rate_list[rate_index]) if self.fail_break == "True": return ############################################################ # to get the Power on N9010A signal analyzer ############################################################ while True: signalPower = raw_input( "\nPlease input the Signal Power for rate(%sMbps) : " % rate_list[rate_index]) p = re.compile('^(\-|\+)?\d+(\.\d+)?$') if p.match(signalPower): signalPower = float(signalPower) break else: print "the input signal Power [%s] is not in correct format. please try again" lowValue = float(eval('self.power_%sm_low' % rate_list[rate_index])) highValue = float( eval('self.power_%sm_high' % rate_list[rate_index])) testName = "power_%sm_test" % rate_list[rate_index] tir = TestItemResult() tir.testName = testName tir.lowValue = lowValue tir.highValue = highValue tir.testValue = signalPower self.testItemResultList.append(tir) if lowValue <= signalPower <= highValue: pass else: self.ErrorList.append("power_%sm_test" % rate_list[rate_index]) if self.fail_break == "True": return rate_index = rate_index + 1