def updateValue(self,laststate, state, lastaction,lastreward,lbda=1): if self.t < self.horizon: return qvalues = self.getValues() if qvalues is None: return qvalue = qvalues[lastaction] next_qvalues = self.getTargetValues() max_q_index = np.argmax(next_qvalues) maxnext = next_qvalues[max_q_index] if self.nn: update = (lastreward + (self.gamma * maxnext)) qvalues[lastaction] = update from pybrain.supervised import RPropMinusTrainer trainer = RPropMinusTrainer(self.nn) dataset = Sequential() trainer.trainOnDataset(dataset) else: self.Q[laststate][lastaction]=qvalue + self.alpha * lbda * (lastreward + self.gamma * maxnext - qvalue)
#code for normalize data in ds i = np.array([d[0] for d in ds]) inorm = np.array([d[0] for d in ds]) i /= np.max(np.abs(i), axis=0) o = np.array([d[1] for d in ds]) onorm = np.array([d[1] for d in ds]) o /= np.max(np.abs(o), axis=0) print routes_frommeasur_ids["11000602"] / np.max(np.abs(inorm), axis=0) #creating new object for normalized data nds = SupervisedDataSet(1, 1) for ix in range(len(ds)): nds.addSample(i[ix], o[ix]) #print routes_frommeasur #creating net net = buildNetwork(nds.indim, 3, nds.outdim, bias=True, hiddenclass=TanhLayer) #training net trainer = RPropMinusTrainer(net, verbose=True) trainer.trainOnDataset(nds, 10) #trainer.testOnData(verbose=True) p = net.activate(routes_frommeasur_ids["11000602"] / np.max(np.abs(inorm), axis=0)) print(p[0] * np.max(np.abs(onorm), axis=0)[0])
def learn(self, pathdataset=["dstc4_train"], Pathdataroot="data", numberOfHiddenUnit=20, EPOCHS_PER_CYCLE=10, CYCLES=40, weightdecayw=0.01): print "Start learning LSTM, and make dictionary file" #Construct dictionary: variable name -> corresponding index of element in i/o vector print "Star make dictionary: variable name -> corresponding index of element in i/o vector" self.dictOut = { } #"TOPIC_SLOT_VALUE" -> corresponding index of element self.dictIn = { } #"SPEAKER_{val}"or"TOPIC_{val}","WORD_{word}" "BIO_{BIO}", "CLASS_{slot,value}", ""{defined label}-> corresponding index of element #-target vector dictionary index = 0 totalNumSlot = 0 for topic in self.tagsets.keys(): for slot in self.tagsets[topic].keys(): totalNumSlot += 1 for value in self.tagsets[topic][slot]: self.dictOut[topic + "_" + slot + "_" + value] = index index += 1 print "totalNumSlot:" + str(totalNumSlot) print "outputSize:" + str(len(self.dictOut.keys())) #-input dictionry dataset = [] for pathdat in pathdataset: dataset.append( dataset_walker.dataset_walker(pathdat, dataroot=Pathdataroot, labels=False)) #--(sub input vector 1) Class features i.e., Slot and value ratio (Similar to base line) index = 0 for topic in self.tagsets.keys(): for slot in self.tagsets[topic].keys(): if ("CLASS_" + slot) not in self.dictIn: self.dictIn["CLASS_" + slot] = index index += 1 for value in self.tagsets[topic][slot]: if ("CLASS_" + value) not in self.dictIn: self.dictIn["CLASS_" + value] = index index += 1 self.TOTALSIZEOFCLASSFeature = index f = open(self.FileNameofNumClassFeature, "wb") pickle.dump(self.TOTALSIZEOFCLASSFeature, f) f.close() #--(sub input vector 2) Sentence features if not self.isUseSentenceRepresentationInsteadofBOW: index = 0 for elemDataset in dataset: for call in elemDataset: for (uttr, _) in call: #General info1 (CLASS; this feature must be rejistered at first) if ("SPEAKER_" + uttr["speaker"]) not in self.dictIn: self.dictIn["SPEAKER_" + uttr["speaker"]] = index index += 1 if ("TOPIC_" + uttr["segment_info"]["topic"] ) not in self.dictIn: self.dictIn["TOPIC_" + uttr["segment_info"]["topic"]] = index index += 1 #General info2 #-BIO if ("BIO_" + uttr['segment_info']['target_bio'] ) not in self.dictIn: self.dictIn[ "BIO_" + uttr['segment_info']['target_bio']] = index index += 1 #BOW if LSTMWithBOWTracker.isIgnoreUtterancesNotRelatedToMainTask: if not (uttr['segment_info']['target_bio'] == "O"): #-BOW splitedtrans = self.__getRegurelisedBOW( uttr["transcript"]) for word in splitedtrans: if ("WORD_" + word) not in self.dictIn: self.dictIn["WORD_" + word] = index index += 1 self.TOTALSIZEOFSENTENCEFeature = index f = open(self.FileNameofNumSentenceFeature, "wb") pickle.dump(self.TOTALSIZEOFSENTENCEFeature, f) f.close() elif self.isUseSentenceRepresentationInsteadofBOW: index = 0 for i in range(0, LSTMWithBOWTracker.D2V_VECTORSIZE): self.dictIn[str(index) + "thElemPV"] = index index += 1 index = 0 for i in range(0, LSTMWithBOWTracker.D2V_VECTORSIZE): self.dictIn[str(index) + "thAvrWord"] = index index += 1 assert self.D2V_VECTORSIZE == LSTMWithBOWTracker.D2V_VECTORSIZE, "D2V_VECTORSIZE is restrected to be same over the class" else: assert False, "Unexpected block" #--(sub input vector 3) Features M1s defined index = 0 if self.isEnableToUseM1sFeature: rejisteredFeatures = self.__rejisterM1sInputFeatureLabel( self.tagsets, dataset) for rFeature in rejisteredFeatures: assert rFeature not in self.dictIn, rFeature + " already registered in input vector. Use different label name. " self.dictIn[rFeature] = index index += 1 self.TOTALSIZEOFM1DEFINEDFeature = index f = open(self.FileNameofNumM1Feature, "wb") pickle.dump(self.TOTALSIZEOFM1DEFINEDFeature, f) f.close() print "inputSize:" + str(len(self.dictIn.keys())) assert self.dictIn[ "CLASS_INFO"] == 0, "Unexpected index CLASS_INFO should has value 0" assert self.dictIn[ "CLASS_Fort Siloso"] == 334, "Unexpected index CLASS_Fort Siloso should has value 334" assert self.dictIn[ "CLASS_Yunnan"] == 1344, "Unexpected index CLASS_Yunnan should has value 1611" #--write fileObject = open('dictInput.pic', 'w') pickle.dump(self.dictIn, fileObject) fileObject.close() fileObject = open('dictOutput.pic', 'w') pickle.dump(self.dictOut, fileObject) fileObject.close() #Build RNN frame work print "Start learning Network" #Capability of network is: (30 hidden units can represents 1048576 relations) wherease (10 hidden units can represents 1024) #Same to Henderson (http://www.aclweb.org/anthology/W13-4073)? net = buildNetwork(len(self.dictIn.keys()), numberOfHiddenUnit, len(self.dictOut.keys()), hiddenclass=LSTMLayer, outclass=SigmoidLayer, outputbias=False, recurrent=True) #Train network #-convert training data into sequence of vector convDataset = [] #[call][uttr][input,targetvec] iuttr = 0 convCall = [] for elemDataset in dataset: for call in elemDataset: for (uttr, label) in call: if self.isIgnoreUtterancesNotRelatedToMainTask: if uttr['segment_info']['target_bio'] == "O": continue #-input convInput = self._translateUtteranceIntoInputVector( uttr, call) #-output convOutput = [0.0] * len( self.dictOut.keys()) #Occured:+1, Not occured:0 if "frame_label" in label: for slot in label["frame_label"].keys(): for value in label["frame_label"][slot]: convOutput[self.dictOut[ uttr["segment_info"]["topic"] + "_" + slot + "_" + value]] = 1 #-post proccess if self.isSeparateDialogIntoSubDialog: if uttr['segment_info']['target_bio'] == "B": if len(convCall) > 0: convDataset.append(convCall) convCall = [] convCall.append([convInput, convOutput]) #print "Converted utterance" + str(iuttr) iuttr += 1 if not self.isSeparateDialogIntoSubDialog: if len(convCall) > 0: convDataset.append(convCall) convCall = [] #Online learning trainer = RPropMinusTrainer(net, weightdecay=weightdecayw) EPOCHS = EPOCHS_PER_CYCLE * CYCLES for i in xrange(CYCLES): #Shuffle order ds = SequentialDataSet(len(self.dictIn.keys()), len(self.dictOut.keys())) datInd = range(0, len(convDataset)) random.shuffle( datInd ) #Backpropergation already implemeted data shuffling, however though RpropMinus don't. for ind in datInd: ds.newSequence() for convuttr in convDataset[ind]: ds.addSample(convuttr[0], convuttr[1]) #Evaluation and Train epoch = (i + 1) * EPOCHS_PER_CYCLE print "\r epoch {}/{} Error={}".format( epoch, EPOCHS, trainer.testOnData(dataset=ds)) stdout.flush() trainer.trainOnDataset(dataset=ds, epochs=EPOCHS_PER_CYCLE) NetworkWriter.writeToFile( trainer.module, "LSTM_" + "Epoche" + str(i + 1) + ".rnnw") NetworkWriter.writeToFile(trainer.module, "LSTM.rnnw")
def learn(self,pathdataset=["dstc4_train"], Pathdataroot="data",numberOfHiddenUnit=20, EPOCHS_PER_CYCLE = 10, CYCLES = 40,weightdecayw=0.01): print "Start learning LSTM, and make dictionary file" #Construct dictionary: variable name -> corresponding index of element in i/o vector print "Star make dictionary: variable name -> corresponding index of element in i/o vector" self.dictOut={}#"TOPIC_SLOT_VALUE" -> corresponding index of element self.dictIn={}#"SPEAKER_{val}"or"TOPIC_{val}","WORD_{word}" "BIO_{BIO}", "CLASS_{slot,value}", ""{defined label}-> corresponding index of element #-target vector dictionary index=0 totalNumSlot=0 for topic in self.tagsets.keys(): for slot in self.tagsets[topic].keys(): totalNumSlot+=1 for value in self.tagsets[topic][slot]: self.dictOut[topic+"_"+slot+"_"+value]=index index+=1 print "totalNumSlot:" + str(totalNumSlot) print "outputSize:"+str(len(self.dictOut.keys())) #-input dictionry dataset=[] for pathdat in pathdataset: dataset.append(dataset_walker.dataset_walker(pathdat,dataroot=Pathdataroot,labels=False)) #--(sub input vector 1) Class features i.e., Slot and value ratio (Similar to base line) index=0 for topic in self.tagsets.keys(): for slot in self.tagsets[topic].keys(): if ("CLASS_"+ slot) not in self.dictIn: self.dictIn["CLASS_"+slot]=index index+=1 for value in self.tagsets[topic][slot]: if ("CLASS_"+ value) not in self.dictIn: self.dictIn["CLASS_"+value]=index index+=1 self.TOTALSIZEOFCLASSFeature=index f=open(self.FileNameofNumClassFeature,"wb") pickle.dump(self.TOTALSIZEOFCLASSFeature,f) f.close() #--(sub input vector 2) Sentence features if not self.isUseSentenceRepresentationInsteadofBOW: index=0 for elemDataset in dataset: for call in elemDataset: for (uttr,_) in call: #General info1 (CLASS; this feature must be rejistered at first) if ("SPEAKER_"+uttr["speaker"]) not in self.dictIn: self.dictIn["SPEAKER_"+uttr["speaker"]]=index index+=1 if ("TOPIC_"+uttr["segment_info"]["topic"]) not in self.dictIn: self.dictIn["TOPIC_"+uttr["segment_info"]["topic"]]=index index+=1 #General info2 #-BIO if ("BIO_"+uttr['segment_info']['target_bio']) not in self.dictIn: self.dictIn["BIO_"+uttr['segment_info']['target_bio']]=index index+=1 #BOW if LSTMWithBOWTracker.isIgnoreUtterancesNotRelatedToMainTask: if not (uttr['segment_info']['target_bio'] == "O"): #-BOW splitedtrans=self.__getRegurelisedBOW(uttr["transcript"]) for word in splitedtrans: if ("WORD_"+word) not in self.dictIn: self.dictIn["WORD_"+word]=index index+=1 self.TOTALSIZEOFSENTENCEFeature=index f=open(self.FileNameofNumSentenceFeature,"wb") pickle.dump(self.TOTALSIZEOFSENTENCEFeature,f) f.close() elif self.isUseSentenceRepresentationInsteadofBOW: index=0 for i in range(0,LSTMWithBOWTracker.D2V_VECTORSIZE): self.dictIn[str(index)+"thElemPV"]=index index+=1 index=0 for i in range(0,LSTMWithBOWTracker.D2V_VECTORSIZE): self.dictIn[str(index)+"thAvrWord"]=index index+=1 assert self.D2V_VECTORSIZE == LSTMWithBOWTracker.D2V_VECTORSIZE, "D2V_VECTORSIZE is restrected to be same over the class" else: assert False, "Unexpected block" #--(sub input vector 3) Features M1s defined index=0 if self.isEnableToUseM1sFeature: rejisteredFeatures=self.__rejisterM1sInputFeatureLabel(self.tagsets,dataset) for rFeature in rejisteredFeatures: assert rFeature not in self.dictIn, rFeature +" already registered in input vector. Use different label name. " self.dictIn[rFeature]=index index+=1 self.TOTALSIZEOFM1DEFINEDFeature=index f=open(self.FileNameofNumM1Feature,"wb") pickle.dump(self.TOTALSIZEOFM1DEFINEDFeature,f) f.close() print "inputSize:"+str(len(self.dictIn.keys())) assert self.dictIn["CLASS_INFO"] == 0, "Unexpected index CLASS_INFO should has value 0" assert self.dictIn["CLASS_Fort Siloso"] == 334, "Unexpected index CLASS_Fort Siloso should has value 334" assert self.dictIn["CLASS_Yunnan"] == 1344, "Unexpected index CLASS_Yunnan should has value 1611" #--write fileObject = open('dictInput.pic', 'w') pickle.dump(self.dictIn, fileObject) fileObject.close() fileObject = open('dictOutput.pic', 'w') pickle.dump(self.dictOut, fileObject) fileObject.close() #Build RNN frame work print "Start learning Network" #Capability of network is: (30 hidden units can represents 1048576 relations) wherease (10 hidden units can represents 1024) #Same to Henderson (http://www.aclweb.org/anthology/W13-4073)? net = buildNetwork(len(self.dictIn.keys()), numberOfHiddenUnit, len(self.dictOut.keys()), hiddenclass=LSTMLayer, outclass=SigmoidLayer, outputbias=False, recurrent=True) #Train network #-convert training data into sequence of vector convDataset=[]#[call][uttr][input,targetvec] iuttr=0 convCall=[] for elemDataset in dataset: for call in elemDataset: for (uttr,label) in call: if self.isIgnoreUtterancesNotRelatedToMainTask: if uttr['segment_info']['target_bio'] == "O": continue #-input convInput=self._translateUtteranceIntoInputVector(uttr,call) #-output convOutput=[0.0]*len(self.dictOut.keys())#Occured:+1, Not occured:0 if "frame_label" in label: for slot in label["frame_label"].keys(): for value in label["frame_label"][slot]: convOutput[self.dictOut[uttr["segment_info"]["topic"]+"_"+slot+"_"+value]]=1 #-post proccess if self.isSeparateDialogIntoSubDialog: if uttr['segment_info']['target_bio'] == "B": if len(convCall) > 0: convDataset.append(convCall) convCall=[] convCall.append([convInput,convOutput]) #print "Converted utterance" + str(iuttr) iuttr+=1 if not self.isSeparateDialogIntoSubDialog: if len(convCall) > 0: convDataset.append(convCall) convCall=[] #Online learning trainer = RPropMinusTrainer(net,weightdecay=weightdecayw) EPOCHS = EPOCHS_PER_CYCLE * CYCLES for i in xrange(CYCLES): #Shuffle order ds = SequentialDataSet(len(self.dictIn.keys()),len(self.dictOut.keys())) datInd=range(0,len(convDataset)) random.shuffle(datInd)#Backpropergation already implemeted data shuffling, however though RpropMinus don't. for ind in datInd: ds.newSequence() for convuttr in convDataset[ind]: ds.addSample(convuttr[0],convuttr[1]) #Evaluation and Train epoch = (i+1) * EPOCHS_PER_CYCLE print "\r epoch {}/{} Error={}".format(epoch, EPOCHS,trainer.testOnData(dataset=ds)) stdout.flush() trainer.trainOnDataset(dataset=ds,epochs=EPOCHS_PER_CYCLE) NetworkWriter.writeToFile(trainer.module, "LSTM_"+"Epoche"+str(i+1)+".rnnw") NetworkWriter.writeToFile(trainer.module, "LSTM.rnnw")