class Model(object): """感知器模型 """ name="感知器" #: 模型的名字 def __init__(self,model_file,Task=None,Searcher=None, Updater=None, beam_width=8,logger=None,cmd_args={},**conf): """ 初始化 如果不设置,则读取已有模型。如果设置,就是学习新模型 """ if logger==None : logger=logging.getLogger(__name__) console=logging.StreamHandler() console.setLevel(logging.INFO) logger.addHandler(console) logger.setLevel(logging.INFO) self.result_logger=logger self.beam_width=beam_width#:搜索宽度 self.conf=conf if model_file!=None: file=gzip.open(model_file,"rb") self.task=Task(model=pickle.load(file),logger=logger) file.close() else : # new model to train self.paras=Parameters(Updater) #self.paras=Parameters(Ada_Grad) self.task=Task(logger=logger,paras=self.paras) if hasattr(self.task,'init'): self.task.init() self.searcher=Searcher(self.task,beam_width) self.step=0 def __del__(self): del self.searcher def test(self,test_file): """ 测试 """ eval=self.task.Eval() for line in open(test_file): arg=self.task.codec.decode(line.strip()) raw=arg.get('raw') Y=arg.get('Y_a',None) y=arg.get('y',None) hat_y=self(raw) eval(y,hat_y) if hasattr(eval,'get_result'): self.result_logger.info(eval.get_result()) else : eval.print_result()#打印评测结果 return eval def develop(self,dev_file): """ @brief 预测开发集 """ self.paras.final(self.step) eval=self.task.Eval() for line in open(dev_file): arg=self.task.codec.decode(line.strip()) if not arg:continue raw=arg.get('raw') y=arg.get('y',None) hat_y=self(raw) eval(y,hat_y) if hasattr(eval,'get_result'): self.result_logger.info(eval.get_result()) else : eval.print_result()#打印评测结果 self.paras.un_final() if hasattr(eval,'get_scaler'): return eval.get_scaler() def save(self,model_file=None): """ 保存模型 """ if model_file==None : model_file=self.model_file if model_file==None : return if model_file=='/dev/null' : return #self.task.average_weights(self.step) self.paras.final(self.step) file=gzip.open(model_file,'wb') data=self.task.dump_weights() pickle.dump(data,file) file.close() def search(self,raw,Y=None): """ 搜索 """ self.task.set_raw(raw,Y) #self.searcher.set_raw(raw) return self.searcher.search() def __call__(self,raw,Y=None,threshold=0): """ 解码,读入生句子,返回词的数组 """ rst_moves=self.search(raw,Y) hat_y=self.task.moves_to_result(rst_moves,raw) if threshold==0 : return hat_y else: margins=self.searcher.cal_margins() return self.task.gen_candidates(margins,threshold) def _learn_sentence(self,arg): """ 学习,根据生句子和标准分词结果 """ raw=arg.get('raw') self.raw=raw y=arg.get('y',None) Y_a=arg.get('Y_a',None) #self.logger.debug('get training example') #self.logger.debug("raw: %s"%raw) #self.logger.debug("y: %s"%y) #self.logger.debug("Y_a: %s"%Y_a) #学习步数加一 self.step+=1 #set oracle, get standard actions if hasattr(self.task,'set_oracle'): std_moves=self.task.set_oracle(raw,y) #self.logger.debug(std_moves) #get result actions #self.searcher.set_step(self.step) rst_moves=self.search(raw,Y_a)#得到解码后动作 #update if not self.task.check(std_moves,rst_moves):#check self.update(std_moves,rst_moves)#update #clean oracle if hasattr(self.task,'remove_oracle'): self.task.remove_oracle() hat_y=self.task.moves_to_result(rst_moves,raw)#得到解码后结果 return y,hat_y def update(self,std_moves,rst_moves): #self.task.cal_delta(std_moves,rst_moves,self.step) self.task.cal_delta(std_moves,rst_moves) if self.step%self.batch_size==0 : self.paras.update(self.step) def train(self,training_file, iteration=5,peek=-1, dev_files=None,keep_data=True,batch_size=1): """ 训练 """ if iteration<=0 and peek <=0 : peek=5 self.batch_size=batch_size if type(training_file)==str:training_file=[training_file] #random.seed(123) if keep_data : training_data=[] for t_file in training_file : for line in open(t_file):#迭代每个句子 rtn=self.task.codec.decode(line.strip())#得到标准输出 if not rtn:continue training_data.append(rtn) random.shuffle(training_data) def gen_data(): if keep_data : perc=0 print(perc,end='%\r') #random.shuffle(training_data) for i,e in enumerate(training_data) : p=int(i*100/len(training_data)) if p != perc : print("%i"%(p),end='%\r',file=sys.stderr) perc=p yield e else : for t_file in training_file: for line in open(t_file):#迭代每个句子 rtn=self.task.codec.decode(line.strip())#得到标准输出 if not rtn:continue yield rtn it=0 best_it=None best_scaler=None while True : if it == iteration : break self.result_logger.info("训练集第 \033[33;01m%i\033[1;m 次迭代"%(it+1)) eval=self.task.Eval()#: 测试用的对象 for rtn in gen_data(): if rtn is None : continue y,hat_y=self._learn_sentence(rtn)#根据(输入,输出)学习参数,顺便得到解码结果 eval(y,hat_y)#根据解码结果和标准输出,评价效果 if hasattr(eval,'get_result'): self.result_logger.info(eval.get_result()) else : eval.print_result()#打印评测结果 if hasattr(self.task,'report'): self.task.report() if dev_files: #self.result_logger.info("使用开发集 %s 评价当前模型效果"%(dev_file)) for dev_id,dev_file in enumerate(dev_files) : scaler=self.develop(dev_file) if dev_id==0 : if best_scaler==None or (scaler and best_scaler<scaler) : best_it=it best_scaler=scaler it+=1 if peek>=0 and it-best_it>peek : break def __del__(self): self.task.__del__() del self.task