def testloadData(self): nExamples = 15 train_data = '/home/bhanu/Downloads/relationClassification/dataCamera/allDataTrain.mat' pre_trained_weights = '/home/bhanu/Downloads/relationClassification/dataCamera/pretrainedWeights.mat' rnnData = RNNDataCorpus() rnnData.load_data(load_file=train_data, nExamples=nExamples) params = Params(data=rnnData, wordSize=50, rankWo=3) # n = params.wordSize; fanIn = params.fanIn; nWords = params.nWords; nLabels = params.categories; rank=params.rankWo # theta = np.random.normal(loc=0.0, scale=np.math.pow(10,-5)**2, size = n*(2*n+1) + n*2*n + nLabels*fanIn + n*nWords + (2*n*rank+n)*nWords) # # #init respective parameters with prior values # W, WO, Wcat, Wv, Wo = unroll_theta(theta, params) # Wo[:n,:] = np.ones((n,Wo.shape[1])) #init word matrices with Identity matrices + epsilon #load pre-trained weights here mats = sio.loadmat(pre_trained_weights) Wv = mats.get('Wv') W = mats.get('W') WO = mats.get('WO') Wo = np.ndarray( shape=(2 * params.wordSize * params.rankWo + params.wordSize) * params.nWords, dtype='float64') sentencesIdx = np.arange(nExamples) [allSNum_batch, allSNN_batch, Wv_batch, Wo_batch, allWordInds] = getRelevantWords(rnnData, sentencesIdx, Wv, Wo, params) print
def test(): # load testing data print "loading test data.." rnnData = RNNDataCorpus() rnnData.load_data_srl(load_file=config.test_data_srl, nExamples=10) # rnnData.load_data(load_file=config.test_data, nExamples=-1) print 'loading trained model : ', sys.argv[1] # mats = sio.loadmat(config.model_path+str(sys.argv[1])) # Wv = mats.get('Wv') #L, as in paper # W = mats.get('W') #W, as in paper # WO = mats.get('WO') #Wm, as in paper # Wo = mats.get('Wo') # Wcat = mats.get('Wcat') # n = Wv.shape[0] # r = (Wo.shape[0] - n)/(2*n) # rnn = MVRNN(W, WO, Wcat, Wv, Wo) with open(config.model_path+sys.argv[1], 'r') as loadfile: rnn = cPickle.load(loadfile) n = rnn.Wv.shape[0] r = (rnn.Wo.shape[0] - n)/(2*n) print "initializing params.." params = Params(data=rnnData, wordSize=n, rankWo=r) print "evaluating.." predictLabels = rnn.evaluate(params, rnnData) print "creating labels file .." create_preds_file(predictLabels, rnnData.categories, rnnData.sentenceLabels, predictions_file=config.results_path+'preds_srl.txt', testKeys_file='srl_test_keys.txt')
def test_write_srl_verbIndices(self): infilenames = [config.train_data_srl, config.test_data_srl, config.dev_data_srl] outfilenames = [config.corpus_path+'srl_vids.train', config.corpus_path+'srl_vids.test', config.corpus_path+'srl_vids.dev'] for infilename, outfilename in zip(infilenames, outfilenames): rnndata = RNNDataCorpus() rnndata.load_data_srl(infilename, nExamples=100) allVerbIndices = rnndata.verbIndices with open(outfilename, 'w') as wf: for i in range(len(allVerbIndices)) : verbids = allVerbIndices[i].flatten() verbidsStr = [str(x) for x in verbids] wf.write("|".join(verbidsStr)+"\n") wf.flush()
def train(): SEED = 131742 load_model = False custom_load = True #loads model from previously saved model except Wcat np.random.seed(SEED) #get sentences, trees and labels nExamples = 5 print "loading data.." rnnData_train = RNNDataCorpus() rnnData_train.load_data_srl(load_file=config.train_data_srl, nExamples=nExamples) rnnData_dev = RNNDataCorpus() rnnData_dev.load_data_srl(load_file=config.dev_data_srl, nExamples=nExamples) print "Number of sentences loaded in training data: ", rnnData_train.ndoc() #initialize params print "initializing params" params = Params(data=rnnData_train, wordSize=52, rankWo=2) n = params.wordSize; fanIn = params.fanIn; nWords = params.nWords; nLabels = params.categories; rank=params.rankWo if(load_model): with open(config.saved_params_file+"_45", 'r') as loadfile: rnn = cPickle.load(loadfile) elif(custom_load): d = 2#extra features for wordvectors Wo = 0.01*np.random.randn(n + 2*n*rank, nWords) #Lm, as in paper Wo[:n,:] = np.ones((n,Wo.shape[1])) #Lm, as in paper Wcat = 0.005*np.random.randn(nLabels, fanIn) #Wlabel, as in paper Wv = 0.01*np.random.randn(n, nWords) WO = 0.01*np.random.randn(n, 2*n) W = 0.01*np.random.randn(n, 2*n+1) #load pre-trained weights here with open(config.saved_params_file+'_65', 'r') as loadfile: oldrnn = cPickle.load(loadfile) Wv[:-d,:] = oldrnn.Wv # WO[:-d,:] = oldrnn.WO rnn = MVRNN(W, WO, Wcat, Wv, Wo) else: Wo = 0.01*np.random.randn(n + 2*n*rank, nWords) #Lm, as in paper Wo[:n,:] = np.ones((n,Wo.shape[1])) #Lm, as in paper Wcat = 0.005*np.random.randn(nLabels, fanIn) #Wlabel, as in paper Wv = 0.01*np.random.randn(n, nWords) WO = 0.01*np.random.randn(n, 2*n) W = 0.01*np.random.randn(n, 2*n+1) rnn = MVRNN(W, WO, Wcat, Wv, Wo) [_, _, all_train_idx] = getRelevantWords(rnnData_train, rnn.Wv,rnn.Wo,params) #sets nWords_reduced, returns new arrays params.setNumReducedWords(len(all_train_idx)) theta = rnn.getTheta(params, all_train_idx) #optimize print "starting training using SGD..." nIter = 500 optimizer = StochasticGradientDescent(niter=nIter , learning_rate=0.01, learningrateFactor=1.0, printAt10Iter='.', printAt100Iter='\n+') optimizer.minimizeBatches(rnn=rnn, rnnData_train=rnnData_train, allTrainSentIdx=all_train_idx, params=params, x0=theta, func=costFn, fprime=None, rnnData_test=rnnData_dev, initialSetSize=1, niter=nIter, seed=17, modelFileName=config.saved_params_file+'SGD_SRL', printStatistics=True, modelSaveIter=1, nIterInPart=1, nFetch=-1, rnd=None, nodeid=-1) print "Finished training! "
def test_write_srl_verbIndices(self): infilenames = [ config.train_data_srl, config.test_data_srl, config.dev_data_srl ] outfilenames = [ config.corpus_path + 'srl_vids.train', config.corpus_path + 'srl_vids.test', config.corpus_path + 'srl_vids.dev' ] for infilename, outfilename in zip(infilenames, outfilenames): rnndata = RNNDataCorpus() rnndata.load_data_srl(infilename, nExamples=100) allVerbIndices = rnndata.verbIndices with open(outfilename, 'w') as wf: for i in range(len(allVerbIndices)): verbids = allVerbIndices[i].flatten() verbidsStr = [str(x) for x in verbids] wf.write("|".join(verbidsStr) + "\n") wf.flush()
def get_subset(self, rnnData, workerIndex): ''' data : complete train or test set for which a subset of docs is to be calculated for this worker ''' rnnData_mini = RNNDataCorpus() dataSize = rnnData.ndoc() sizePerNode = dataSize/(self.totalNodes-1) #exclude master node startPos = (workerIndex-1) * sizePerNode if(workerIndex != self.totalNodes-1): #last node on a machine gets all the remaining data endPos = startPos+sizePerNode else: endPos = dataSize rnnData.copy_into_minibatch(rnnData_mini, range(startPos, endPos)) return rnnData_mini
def testloadData(self): nExamples = 15 train_data = '/home/bhanu/Downloads/relationClassification/dataCamera/allDataTrain.mat' pre_trained_weights = '/home/bhanu/Downloads/relationClassification/dataCamera/pretrainedWeights.mat' rnnData = RNNDataCorpus() rnnData.load_data(load_file=train_data, nExamples=nExamples) params = Params(data=rnnData, wordSize=50, rankWo=3) # n = params.wordSize; fanIn = params.fanIn; nWords = params.nWords; nLabels = params.categories; rank=params.rankWo # theta = np.random.normal(loc=0.0, scale=np.math.pow(10,-5)**2, size = n*(2*n+1) + n*2*n + nLabels*fanIn + n*nWords + (2*n*rank+n)*nWords) # # #init respective parameters with prior values # W, WO, Wcat, Wv, Wo = unroll_theta(theta, params) # Wo[:n,:] = np.ones((n,Wo.shape[1])) #init word matrices with Identity matrices + epsilon #load pre-trained weights here mats = sio.loadmat(pre_trained_weights) Wv = mats.get('Wv'); W = mats.get('W'); WO = mats.get('WO') Wo = np.ndarray(shape=(2*params.wordSize*params.rankWo + params.wordSize)*params.nWords, dtype='float64') sentencesIdx = np.arange(nExamples) [allSNum_batch, allSNN_batch, Wv_batch, Wo_batch, allWordInds] = getRelevantWords(rnnData, sentencesIdx, Wv,Wo,params) print
def test(): # load testing data print "loading test data.." rnnData = RNNDataCorpus() rnnData.load_data_srl(load_file=config.test_data_srl, nExamples=10) # rnnData.load_data(load_file=config.test_data, nExamples=-1) print 'loading trained model : ', sys.argv[1] # mats = sio.loadmat(config.model_path+str(sys.argv[1])) # Wv = mats.get('Wv') #L, as in paper # W = mats.get('W') #W, as in paper # WO = mats.get('WO') #Wm, as in paper # Wo = mats.get('Wo') # Wcat = mats.get('Wcat') # n = Wv.shape[0] # r = (Wo.shape[0] - n)/(2*n) # rnn = MVRNN(W, WO, Wcat, Wv, Wo) with open(config.model_path + sys.argv[1], 'r') as loadfile: rnn = cPickle.load(loadfile) n = rnn.Wv.shape[0] r = (rnn.Wo.shape[0] - n) / (2 * n) print "initializing params.." params = Params(data=rnnData, wordSize=n, rankWo=r) print "evaluating.." predictLabels = rnn.evaluate(params, rnnData) print "creating labels file .." create_preds_file(predictLabels, rnnData.categories, rnnData.sentenceLabels, predictions_file=config.results_path + 'preds_srl.txt', testKeys_file='srl_test_keys.txt')
def initMVRNN(self): print "Node: ",self.index," Loading training and dev sets.." self.rnnData_train = RNNDataCorpus() self.rnnData_train.load_data_srl(config.train_data_srl, nExamples=self.nuse) self.rnnData_dev = RNNDataCorpus() self.rnnData_dev.load_data_srl(config.dev_data_srl, nExamples=self.nuse) modelfilename = config.saved_params_file+'SGD_SRLiter305' print "Node: ", self.index," Loading model: ", modelfilename # mats = sio.loadmat(config.saved_params_file+'iter120.mat') # Wv = mats.get('Wv') #L, as in paper # W = mats.get('W') #W, as in paper # WO = mats.get('WO') #Wm, as in paper # Wo = mats.get('Wo') # Wcat = mats.get('Wcat') # n = Wv.shape[0] # r = (Wo.shape[0] - n)/(2*n) with open(modelfilename, 'r') as loadfile: self.rnn = cPickle.load(loadfile)#MVRNN(W, WO, Wcat, Wv, Wo) n = self.rnn.Wv.shape[0] r = (self.rnn.Wo.shape[0] - n)/(2*n) print "Node: ",self.index, "initializing params.." self.params = Params(data=self.rnnData_train, wordSize=n, rankWo=r) # #to be removed # Wcat = 0.005*np.random.randn(self.params.categories, self.params.fanIn) # self.rnn.Wcat = Wcat if(self.index == 0): print "Master## Total trees in training set: ", self.rnnData_train.ndoc() print "Master## nFetch: ", self.nFetch if(self.index!=0): self.rnnData_train = self.get_subset(self.rnnData_train, self.index) self.rnnData_dev = None # workers don't need test set and trainTest, so to free memory release them [_, _, self.all_train_idx] = getRelevantWords(self.rnnData_train, self.rnn.Wv,self.rnn.Wo,self.params) #sets nWords_reduced, returns new arrays
def train(): SEED = 13742 load_model = False custom_load = False np.random.seed(SEED) #get sentences, trees and labels nExamples = 5 print "loading data.." rnnData_train = RNNDataCorpus() rnnData_train.load_data_srl(load_file=config.train_data_srl, nExamples=nExamples) rnnData_dev = RNNDataCorpus() rnnData_dev.load_data_srl(load_file=config.dev_data_srl, nExamples=nExamples) print "Number of sentences loaded in training data: ", rnnData_train.ndoc() #initialize params print "initializing params" params = Params(data=rnnData_train, wordSize=52, rankWo=2) n = params.wordSize fanIn = params.fanIn nWords = params.nWords nLabels = params.categories rank = params.rankWo if (load_model): modelfile = config.saved_params_file + 'SGD_SLL300' rnn = MVRNNSLL.load(modelfile) print 'loaded model : ', modelfile elif (custom_load): modelfile = config.saved_params_file + 'SGD_SLL300' print "loading customized model..", modelfile # d = 2#extra features for wordvectors # Wo = 0.01*np.random.randn(n + 2*n*rank, nWords) #Lm, as in paper # Wo[:n,:] = np.ones((n,Wo.shape[1])) #Lm, as in paper # Wcat = 0.005*np.random.randn(nLabels, fanIn) #Wlabel, as in paper # Wv = 0.01*np.random.randn(n, nWords) # WO = 0.01*np.random.randn(n, 2*n) # W = 0.01*np.random.randn(n, 2*n+1) #load pre-trained weights here oldrnn = MVRNNSLL.load(modelfile) # Wv[:-d,:] = oldrnn.Wv categories = [x.strip() for x in rnnData_train.categories] Tran = init_transitions(dict(zip(categories, range(len(categories)))), 'iob') rnn = MVRNNSLL(oldrnn.W, oldrnn.WO, oldrnn.Wcat, oldrnn.Wv, oldrnn.Wo, Tran) else: #define theta #one vector for all the parameters of mvrnn model: W, Wm, Wlabel, L, Lm # n = params.wordSize; fanIn = params.fanIn; nWords = params.nWords; nLabels = params.categories; rank=params.rankWo # Wo = 0.01*np.random.randn(n + 2*n*rank, nWords) #Lm, as in paper # Wo[:n,:] = np.ones((n,Wo.shape[1])) #Lm, as in paper # Wcat = 0.005*np.random.randn(nLabels, fanIn) #Wlabel, as in paper # #load pre-trained weights here ## mats = sio.loadmat(config.saved_params_file) # oldrnn = MVRNNSLL.load(modelfile) # Wv = oldrnn.Wv #L, as in paper # W = oldrnn..get('W') #W, as in paper # WO = mats.get('WO') #Wm, as in paper Wo = 0.01 * np.random.randn(n + 2 * n * rank, nWords) #Lm, as in paper Wo[:n, :] = np.ones((n, Wo.shape[1])) #Lm, as in paper Wcat = 0.005 * np.random.randn(nLabels, fanIn) #Wlabel, as in paper Wv = 0.01 * np.random.randn(n, nWords) WO = 0.01 * np.random.randn(n, 2 * n) W = 0.01 * np.random.randn(n, 2 * n + 1) categories = [x.strip() for x in rnnData_train.categories] Tran = init_transitions(dict(zip(categories, range(len(categories)))), 'iob') rnn = MVRNNSLL(W, WO, Wcat, Wv, Wo, Tran) # rnn = MVRNNSLL(W, WO, Wcat, Wv, Wo, Tran) [_, _, all_train_idx ] = getRelevantWords(rnnData_train, rnn.Wv, rnn.Wo, params) #sets nWords_reduced, returns new arrays params.setNumReducedWords(len(all_train_idx)) theta = rnn.getTheta(params, all_train_idx) #optimize print "starting training using SGD..." nIter = 500 optimizer = StochasticGradientDescent(niter=nIter, learning_rate=0.01, learningrateFactor=1.0, printAt10Iter='.', printAt100Iter='\n+') optimizer.minimizeBatches(rnn=rnn, rnnData_train=rnnData_train, allTrainSentIdx=all_train_idx, params=params, x0=theta, func=costFn, fprime=None, rnnData_test=rnnData_dev, initialSetSize=1, niter=nIter, seed=SEED, modelFileName=config.saved_params_file + 'SGD_SLL', printStatistics=True, modelSaveIter=100, nIterInPart=1, nFetch=-1, rnd=None, nodeid=-1)
class SGDParallel(playdoh.ParallelTask): ''' ''' def initialize(self, nuse, totalNodes ): """ This method is called on each node when playdoh.start_task method is called. Each node initializes its own model. """ self.seed = 1742 self.nuse = nuse self.totalNodes = totalNodes #including master node self.debug = True self.nIter = 1000 self.nFetch = 15 #after how many steps(minibatch processing) should a worker fetch new parameters from master self.rnd = np.random.RandomState(self.seed) def start(self): if self.index == 0: self.startMasterNode() else: self.startWorkerNode() def startWorkerNode(self): '''Receives updated parameters from the master, keeps updating local parameters using its own subset of training data, pushes local parameters to master, all asynchronously ''' self.initMVRNN() #also initializes training and test sets self.func = costFn self.fprime = None self.optimizer = StochasticGradientDescent(niter=self.nIter , learning_rate=0.01, learningrateFactor=1.0, printAt10Iter='.', printAt100Iter='\n+') genObj = None while True: #fetch action and parameters from masters action, master_rnn = self.pop('fromMaster_' + str(self.index)) if(action == "finish"): print "Node: ",self.index," Received finish action from master, exiting..." return if self.debug: print "Node: ", self.index, " Fetched new action & parameters from Master.. " + str(master_rnn.W[0,0]) self.params.resetNumReducedWords() #to unroll all the words # W, WO, Wcat, Wv, Wo = unroll_theta(theta, self.params) self.copy_into_rnn(master_rnn.getParamsList()) #copy to local rnn to global theta = self.rnn.getTheta(self.params, self.all_train_idx) self.params.setNumReducedWords(len(self.all_train_idx)) #set number of reduced words, to be used by costfn for unrolling start_time = time.clock() try: genObj.next() except (AttributeError, StopIteration) : genObj = self.optimizer.minimizeBatchesPll(rnn=self.rnn, rnnData_train=self.rnnData_train, allTrainSentIdx=self.all_train_idx, params=self.params, x0=theta, func=costFn, fprime=None, rnnData_test=self.rnnData_dev, initialSetSize=1, niter=1, seed=self.seed, modelFileName='', printStatistics=False, modelSaveIter=10, nIterInPart=1, nFetch=self.nFetch, rnd=self.rnd, nodeid=self.index) #optimize this theta and save it in self.rnn except: raise end_time = time.clock() # [W, WO, Wcat, Wv, Wo] = self.rnn # theta = np.concatenate((W.flatten(), WO.flatten(), Wcat.flatten(), Wv.flatten(), Wo.flatten())) # current local optimal theta #push local theta to master self.push('toMaster_' + str(self.index), (self.rnn, None)) #push the theta value #pushing a tuple object #playdoh bug if self.debug: print "Node:", self.index, " Execution time for ", self.nFetch, " minibatches: ", (end_time - start_time)/60, 'minutes' print "Node:", self.index, " Pushed local parameters to Master.." def async_eval(self, itr, testSet, result_file=None): if testSet != None: # if len(testSet.allSNum) > 100: # rnnData_test_mini = RNNDataCorpus() # testSet.copy_into_minibatch(rnnData_test_mini, range(100)) # else: rnnData_test_mini = testSet self.rnn.evaluate( self.params, rnnData_test_mini) def async_save(self, itr): timeStr = strftime("%Y-%m-%d_%H-%M-%S", gmtime()) # for generating names modelFileName = config.saved_params_file+"_"+str(itr)+'_'+timeStr #save weights with open(modelFileName, 'wb') as wf: cPickle.dump(self.rnn, wf, protocol=-1) # [W, WO, Wcat, Wv, Wo] = self.rnn # save_dict = {'Wv':Wv, 'Wo':Wo, 'Wcat':Wcat, 'W':W, 'WO':WO} # sio.savemat(modelFileName, mdict=save_dict) print 'Master## saved trained model to ', modelFileName def startMasterNode(self): '''Parameter server, receives gradients from workers, updates parameters of model, sends updated parameters to workers ''' nDocsTest = 150 printStatistics = True action = ["continue", "finish"] evalFreq = 10 saveFreq = 5 self.initMVRNN() #also initializes training and test sets nparts = self.rnnData_dev.ndoc()/nDocsTest if self.rnnData_dev.ndoc() > nDocsTest else 1 testSeq = range(self.rnnData_dev.ndoc()) #start optimization by pushing intial params for iNode in xrange(1, self.totalNodes): if self.debug: print "Master## Pushing parameters to " + str(iNode) + " theta[0]= " + str(self.rnn.W[0,0]) self.push('fromMaster_' + str(iNode),(action[0],self.rnn)) # send updated params to workers #start iterations and evaluate after completion of set of iterations for itr in range(self.nIter): # workerW = np.zeros(self.rnn.W.shape); workerWO = np.zeros(self.rnn.WO.shape); workerWcat = np.zeros(self.rnn.Wcat.shape); # workerWv = np.zeros(self.rnn.Wv.shape); workerWo = np.zeros(self.rnn.Wo.shape) workerParams = [] for p in self.rnn.getParamsList(): workerParams.append(np.zeros(p.shape)) self.rnd.shuffle(testSeq) (testSetPart, _) = self.rnnData_dev.getSubset(idocNext=0, ipart=0, nparts=nparts, sequenceIndex=testSeq) if self.debug: print "Master## Iteration: ",itr ,"Waiting to receive params from workers..." for iNode in xrange(1, self.totalNodes): #get gradients from all workers receivedRNN, _ = self.pop('toMaster_' + str(iNode)) # poll the workers for result # print str(receivedRNN) receivedParams = receivedRNN.getParamsList() for workerParam, receivedParam in zip(workerParams, receivedParams): workerParam += receivedParam # workerW = workerW + rW # workerWO = workerWO + rWO # workerWcat = workerWcat + rWcat # workerWv = workerWv + rWv # workerWo = workerWo + rWo if self.debug: print "Master## Iteration: ",itr ,"Received theta from: " + str(iNode) + " theta[0]="+str(receivedParams[0][0,0]) #average all the received parameters for i in range(len(workerParams)): workerParams[i] /= (self.totalNodes-1) # workerW = workerW /(self.totalNodes - 1) # workerWO = workerWO /(self.totalNodes - 1) # workerWcat = workerWcat /(self.totalNodes - 1) # workerWv = workerWv /(self.totalNodes - 1) # workerWo = workerWo /(self.totalNodes - 1) #one is master rest are workers #copy into master's rnn # workerRnn = [workerW, workerWO, workerWcat, workerWv, workerWo] self.copy_into_rnn(workerParams) if(self.debug): print "Master## Average Params : ", str(workerParams[0][0,0]) for iNode in xrange(1, self.totalNodes): self.push('fromMaster_' + str(iNode),(action[0],self.rnn)) # send updated params to workers if self.debug: print "Master## Sending new parameters to Worker ", iNode, "Param[0]: ", str(self.rnn.W[0,0]) if printStatistics and ((itr % evalFreq) == 0) : p = mp.Process(target=self.async_eval, args=(itr,self.rnnData_dev)) p.start() if((itr % saveFreq == 0)): # save mlp after every 5 iterations of nFetch cycles. p = mp.Process(target=self.async_save, args=(itr,)) p.start() #finish optimization and close workers for iNode in xrange(1, self.totalNodes): self.push('fromMaster_' + str(iNode),(action[1],None)) # send updated params to workers def get_result(self): if self.index == 0: return self.rnn def copy_into_rnn(self, workerRnnParams): for dst, src in zip(self.rnn.getParamsList(), workerRnnParams): dst[:,:] = src def initMVRNN(self): print "Node: ",self.index," Loading training and dev sets.." self.rnnData_train = RNNDataCorpus() self.rnnData_train.load_data_srl(config.train_data_srl, nExamples=self.nuse) self.rnnData_dev = RNNDataCorpus() self.rnnData_dev.load_data_srl(config.dev_data_srl, nExamples=self.nuse) modelfilename = config.saved_params_file+'SGD_SRLiter305' print "Node: ", self.index," Loading model: ", modelfilename # mats = sio.loadmat(config.saved_params_file+'iter120.mat') # Wv = mats.get('Wv') #L, as in paper # W = mats.get('W') #W, as in paper # WO = mats.get('WO') #Wm, as in paper # Wo = mats.get('Wo') # Wcat = mats.get('Wcat') # n = Wv.shape[0] # r = (Wo.shape[0] - n)/(2*n) with open(modelfilename, 'r') as loadfile: self.rnn = cPickle.load(loadfile)#MVRNN(W, WO, Wcat, Wv, Wo) n = self.rnn.Wv.shape[0] r = (self.rnn.Wo.shape[0] - n)/(2*n) print "Node: ",self.index, "initializing params.." self.params = Params(data=self.rnnData_train, wordSize=n, rankWo=r) # #to be removed # Wcat = 0.005*np.random.randn(self.params.categories, self.params.fanIn) # self.rnn.Wcat = Wcat if(self.index == 0): print "Master## Total trees in training set: ", self.rnnData_train.ndoc() print "Master## nFetch: ", self.nFetch if(self.index!=0): self.rnnData_train = self.get_subset(self.rnnData_train, self.index) self.rnnData_dev = None # workers don't need test set and trainTest, so to free memory release them [_, _, self.all_train_idx] = getRelevantWords(self.rnnData_train, self.rnn.Wv,self.rnn.Wo,self.params) #sets nWords_reduced, returns new arrays #set this to none, as unrolling of theta will take all words into account. # self.theta = np.concatenate((W.flatten(), WO.flatten(), Wcat.flatten(), Wv_trainTest.flatten(), Wo_trainTest.flatten())) def get_subset(self, rnnData, workerIndex): ''' data : complete train or test set for which a subset of docs is to be calculated for this worker ''' rnnData_mini = RNNDataCorpus() dataSize = rnnData.ndoc() sizePerNode = dataSize/(self.totalNodes-1) #exclude master node startPos = (workerIndex-1) * sizePerNode if(workerIndex != self.totalNodes-1): #last node on a machine gets all the remaining data endPos = startPos+sizePerNode else: endPos = dataSize rnnData.copy_into_minibatch(rnnData_mini, range(startPos, endPos)) return rnnData_mini
def writeVectors(): vecFileName = config.results_path+"vectors.out" vecFile = open(vecFileName, 'w') mats = sio.loadmat(config.corpus_path+'vars.normalized.100.mat') We_orig = mats.get('We') params = sio.loadmat(config.corpus_path+'params_rae.mat') W1 = params.get('W1') W2 = params.get('W2') b1 = params.get('b1') We = params.get('We') b = params.get('b') W = params.get('W') hiddenSize = 100 nExamples = 5 print "loading data.." rnnData_train = RNNDataCorpus() rnnData_train.load_data_srl(load_file=config.train_data_srl, nExamples=nExamples) print 'writing vectors to: ', vecFileName for ii in range(len(rnnData_train.allSNum)): sNum = rnnData_train.allSNum[ii] sStr = rnnData_train.allSStr[ii] sTree = rnnData_train.allSTree[ii] sKids = rnnData_train.allSKids[ii] words_indexed = np.where(sNum >= 0)[0] #L is only the part of the embedding matrix that is relevant for this sentence #L is deltaWe if We.shape[1] != 0: L = We[:, words_indexed] words_embedded = We_orig[:, words_indexed] + L; else : words_embedded = We_orig[:, words_indexed] # sl = words_embedded.shape[1] tree = Tree() tree.pp = all#np.zeros(((2*sl-1),1)) tree.nodeScores = np.zeros(len(sNum)) # tree.nodeNames = np.arange(1,(2*sl-1)) tree.kids = np.zeros((len(sNum),2)) tree.nodeFeatures = np.zeros((hiddenSize, len(sNum))) tree.nodeFeatures[:,:len(words_indexed)] = words_embedded; toMerge = np.zeros(shape=(words_indexed.shape), dtype='int32') toMerge[:] = words_indexed[:] while len(toMerge)>1 : # find unpaired bottom leaf pairs (initially words) that share parent i=-1; foundGoodPair = False while (not foundGoodPair ) : i += 1 if sTree[toMerge[i]]==sTree[toMerge[i+1]]: foundGoodPair = True newParent = sTree[toMerge[i]] kid1 = toMerge[i] kid2 = toMerge[i+1] tree.kids[newParent,:] = [kid1, kid2]; # set new parent to be possible merge candidate toMerge[i] = newParent; # delete other kid toMerge = np.delete(toMerge,i+1) c1 = tree.nodeFeatures[:,kid1] c2 = tree.nodeFeatures[:,kid2] p = np.tanh(np.dot(W1,c1) + np.dot(W2,c2) + b1.flatten()) tree.nodeFeatures[:,newParent] = p; vec = tree.nodeFeatures[-1] vecFile.write(" ".join([str(x) for x in vec])+'\n') vecFile.close() print "finished! "
def train(): np.random.seed(131742) #get sentences, trees and labels nExamples = -1 print "loading data.." rnnData = RNNDataCorpus() rnnData.load_data(load_file=config.train_data, nExamples=nExamples) #initialize params print "initializing params" params = Params(data=rnnData, wordSize=50, rankWo=2) #define theta #one vector for all the parameters of mvrnn model: W, Wm, Wlabel, L, Lm n = params.wordSize; fanIn = params.fanIn; nWords = params.nWords; nLabels = params.categories; rank=params.rankWo Wo = 0.01*np.random.randn(n + 2*n*rank, nWords) #Lm, as in paper Wo[:n,:] = np.ones((n,Wo.shape[1])) #Lm, as in paper Wcat = 0.005*np.random.randn(nLabels, fanIn) #Wlabel, as in paper # Wv = 0.01*np.random.randn(n, nWords) # WO = 0.01*np.random.randn(n, 2*n) # W = 0.01*np.random.randn(n, 2*n+1) #load pre-trained weights here mats = sio.loadmat(config.pre_trained_weights) Wv = mats.get('Wv') #L, as in paper W = mats.get('W') #W, as in paper WO = mats.get('WO') #Wm, as in paper sentencesIdx = np.arange(rnnData.ndoc()) np.random.shuffle(sentencesIdx) nTrain = 4*len(sentencesIdx)/5 trainSentIdx = sentencesIdx[0:nTrain] testSentIdx = sentencesIdx[nTrain:] batchSize = 5 nBatches = len(trainSentIdx)/batchSize evalFreq = 5 #evaluate after every 5 minibatches nTestSentEval = 50 #number of test sentences to be evaluated rnnData_train = RNNDataCorpus() rnnData.copy_into_minibatch(rnnData_train, trainSentIdx) rnnData_test = RNNDataCorpus() if(len(testSentIdx) > nTestSentEval): # np.random.shuffle(testSentIdx) #choose random test examples thisTestSentIdx = testSentIdx[:nTestSentEval] else: thisTestSentIdx = testSentIdx rnnData.copy_into_minibatch(rnnData_test, thisTestSentIdx) # [Wv_test, Wo_test, _] = getRelevantWords(rnnData_test, Wv,Wo,params) [Wv_trainTest, Wo_trainTest, all_train_idx] = getRelevantWords(rnnData, Wv,Wo,params) #sets nWords_reduced, returns new arrays theta = np.concatenate((W.flatten(), WO.flatten(), Wcat.flatten(), Wv_trainTest.flatten(), Wo_trainTest.flatten())) #optimize print "starting training..." nIter = 100 rnnData_minibatch = RNNDataCorpus() for i in range(nIter): #train in minibatches # ftrain = np.zeros(nBatches) # for ibatch in range(nBatches): # set_minibatch(rnnData, rnnData_minibatch, ibatch, nBatches, trainSentIdx) # print 'Iteration: ', i, ' minibatch: ', ibatch tunedTheta, fbatch_train, _ = lbfgsb.fmin_l_bfgs_b(func=costFn, x0=theta, fprime=None, args=(rnnData_train, params), approx_grad=0, bounds=None, m=5, factr=1000000000000000.0, pgtol=1.0000000000000001e-5, epsilon=1e-08, iprint=3, maxfun=1, disp=0) #map parameters back W[:,:], WO[:,:], Wcat[:,:], Wv_trainTest, Wo_trainTest = unroll_theta(tunedTheta, params) Wv[:,all_train_idx] = Wv_trainTest Wo[:,all_train_idx] = Wo_trainTest # ftrain[ibatch] = fbatch_train theta = tunedTheta #for next iteration print "========================================" print "XXXXXXIteration ", i, print "Average cost: ", np.average(fbatch_train) evaluate(Wv,Wo,W,WO,Wcat,params, rnnData_test) print "========================================" #save weights save_dict = {'Wv':Wv, 'Wo':Wo, 'Wcat':Wcat, 'W':W, 'WO':WO} sio.savemat(config.saved_params_file+'_lbfgs_iter'+str(i), mdict=save_dict) print "saved tuned theta. "
def train(): SEED = 131742 load_model = False custom_load = True #loads model from previously saved model except Wcat np.random.seed(SEED) #get sentences, trees and labels nExamples = 5 print "loading data.." rnnData_train = RNNDataCorpus() rnnData_train.load_data_srl(load_file=config.train_data_srl, nExamples=nExamples) rnnData_dev = RNNDataCorpus() rnnData_dev.load_data_srl(load_file=config.dev_data_srl, nExamples=nExamples) print "Number of sentences loaded in training data: ", rnnData_train.ndoc() #initialize params print "initializing params" params = Params(data=rnnData_train, wordSize=52, rankWo=2) n = params.wordSize fanIn = params.fanIn nWords = params.nWords nLabels = params.categories rank = params.rankWo if (load_model): with open(config.saved_params_file + "_45", 'r') as loadfile: rnn = cPickle.load(loadfile) elif (custom_load): d = 2 #extra features for wordvectors Wo = 0.01 * np.random.randn(n + 2 * n * rank, nWords) #Lm, as in paper Wo[:n, :] = np.ones((n, Wo.shape[1])) #Lm, as in paper Wcat = 0.005 * np.random.randn(nLabels, fanIn) #Wlabel, as in paper Wv = 0.01 * np.random.randn(n, nWords) WO = 0.01 * np.random.randn(n, 2 * n) W = 0.01 * np.random.randn(n, 2 * n + 1) #load pre-trained weights here with open(config.saved_params_file + '_65', 'r') as loadfile: oldrnn = cPickle.load(loadfile) Wv[:-d, :] = oldrnn.Wv # WO[:-d,:] = oldrnn.WO rnn = MVRNN(W, WO, Wcat, Wv, Wo) else: Wo = 0.01 * np.random.randn(n + 2 * n * rank, nWords) #Lm, as in paper Wo[:n, :] = np.ones((n, Wo.shape[1])) #Lm, as in paper Wcat = 0.005 * np.random.randn(nLabels, fanIn) #Wlabel, as in paper Wv = 0.01 * np.random.randn(n, nWords) WO = 0.01 * np.random.randn(n, 2 * n) W = 0.01 * np.random.randn(n, 2 * n + 1) rnn = MVRNN(W, WO, Wcat, Wv, Wo) [_, _, all_train_idx ] = getRelevantWords(rnnData_train, rnn.Wv, rnn.Wo, params) #sets nWords_reduced, returns new arrays params.setNumReducedWords(len(all_train_idx)) theta = rnn.getTheta(params, all_train_idx) #optimize print "starting training using SGD..." nIter = 500 optimizer = StochasticGradientDescent(niter=nIter, learning_rate=0.01, learningrateFactor=1.0, printAt10Iter='.', printAt100Iter='\n+') optimizer.minimizeBatches(rnn=rnn, rnnData_train=rnnData_train, allTrainSentIdx=all_train_idx, params=params, x0=theta, func=costFn, fprime=None, rnnData_test=rnnData_dev, initialSetSize=1, niter=nIter, seed=17, modelFileName=config.saved_params_file + 'SGD_SRL', printStatistics=True, modelSaveIter=1, nIterInPart=1, nFetch=-1, rnd=None, nodeid=-1) print "Finished training! "
def train(): SEED = 13742 load_model = False custom_load = False np.random.seed(SEED) #get sentences, trees and labels nExamples = 5 print "loading data.." rnnData_train = RNNDataCorpus() rnnData_train.load_data_srl(load_file=config.train_data_srl, nExamples=nExamples) rnnData_dev = RNNDataCorpus() rnnData_dev.load_data_srl(load_file=config.dev_data_srl, nExamples=nExamples) print "Number of sentences loaded in training data: ", rnnData_train.ndoc() #initialize params print "initializing params" params = Params(data=rnnData_train, wordSize=52, rankWo=2) n = params.wordSize; fanIn = params.fanIn; nWords = params.nWords; nLabels = params.categories; rank=params.rankWo if(load_model): modelfile = config.saved_params_file+'SGD_SLL300' rnn = MVRNNSLL.load(modelfile) print 'loaded model : ', modelfile elif(custom_load): modelfile = config.saved_params_file+'SGD_SLL300' print "loading customized model..", modelfile # d = 2#extra features for wordvectors # Wo = 0.01*np.random.randn(n + 2*n*rank, nWords) #Lm, as in paper # Wo[:n,:] = np.ones((n,Wo.shape[1])) #Lm, as in paper # Wcat = 0.005*np.random.randn(nLabels, fanIn) #Wlabel, as in paper # Wv = 0.01*np.random.randn(n, nWords) # WO = 0.01*np.random.randn(n, 2*n) # W = 0.01*np.random.randn(n, 2*n+1) #load pre-trained weights here oldrnn = MVRNNSLL.load(modelfile) # Wv[:-d,:] = oldrnn.Wv categories = [x.strip() for x in rnnData_train.categories] Tran = init_transitions(dict(zip(categories, range(len(categories)))), 'iob') rnn = MVRNNSLL(oldrnn.W, oldrnn.WO, oldrnn.Wcat, oldrnn.Wv, oldrnn.Wo, Tran) else: #define theta #one vector for all the parameters of mvrnn model: W, Wm, Wlabel, L, Lm # n = params.wordSize; fanIn = params.fanIn; nWords = params.nWords; nLabels = params.categories; rank=params.rankWo # Wo = 0.01*np.random.randn(n + 2*n*rank, nWords) #Lm, as in paper # Wo[:n,:] = np.ones((n,Wo.shape[1])) #Lm, as in paper # Wcat = 0.005*np.random.randn(nLabels, fanIn) #Wlabel, as in paper # #load pre-trained weights here ## mats = sio.loadmat(config.saved_params_file) # oldrnn = MVRNNSLL.load(modelfile) # Wv = oldrnn.Wv #L, as in paper # W = oldrnn..get('W') #W, as in paper # WO = mats.get('WO') #Wm, as in paper Wo = 0.01*np.random.randn(n + 2*n*rank, nWords) #Lm, as in paper Wo[:n,:] = np.ones((n,Wo.shape[1])) #Lm, as in paper Wcat = 0.005*np.random.randn(nLabels, fanIn) #Wlabel, as in paper Wv = 0.01*np.random.randn(n, nWords) WO = 0.01*np.random.randn(n, 2*n) W = 0.01*np.random.randn(n, 2*n+1) categories = [x.strip() for x in rnnData_train.categories] Tran = init_transitions(dict(zip(categories, range(len(categories)))), 'iob') rnn = MVRNNSLL(W, WO, Wcat, Wv, Wo, Tran) # rnn = MVRNNSLL(W, WO, Wcat, Wv, Wo, Tran) [_, _, all_train_idx] = getRelevantWords(rnnData_train, rnn.Wv, rnn.Wo, params) #sets nWords_reduced, returns new arrays params.setNumReducedWords(len(all_train_idx)) theta = rnn.getTheta(params, all_train_idx) #optimize print "starting training using SGD..." nIter = 500 optimizer = StochasticGradientDescent(niter=nIter , learning_rate=0.01, learningrateFactor=1.0, printAt10Iter='.', printAt100Iter='\n+') optimizer.minimizeBatches(rnn=rnn, rnnData_train=rnnData_train, allTrainSentIdx=all_train_idx, params=params, x0=theta, func=costFn, fprime=None, rnnData_test=rnnData_dev, initialSetSize=1, niter=nIter, seed=SEED, modelFileName=config.saved_params_file+'SGD_SLL', printStatistics=True, modelSaveIter=100, nIterInPart=1, nFetch=-1, rnd=None, nodeid=-1)
def writeVectors(): vecFileName = config.results_path + "vectors.out" vecFile = open(vecFileName, 'w') mats = sio.loadmat(config.corpus_path + 'vars.normalized.100.mat') We_orig = mats.get('We') params = sio.loadmat(config.corpus_path + 'params_rae.mat') W1 = params.get('W1') W2 = params.get('W2') b1 = params.get('b1') We = params.get('We') b = params.get('b') W = params.get('W') hiddenSize = 100 nExamples = 5 print "loading data.." rnnData_train = RNNDataCorpus() rnnData_train.load_data_srl(load_file=config.train_data_srl, nExamples=nExamples) print 'writing vectors to: ', vecFileName for ii in range(len(rnnData_train.allSNum)): sNum = rnnData_train.allSNum[ii] sStr = rnnData_train.allSStr[ii] sTree = rnnData_train.allSTree[ii] sKids = rnnData_train.allSKids[ii] words_indexed = np.where(sNum >= 0)[0] #L is only the part of the embedding matrix that is relevant for this sentence #L is deltaWe if We.shape[1] != 0: L = We[:, words_indexed] words_embedded = We_orig[:, words_indexed] + L else: words_embedded = We_orig[:, words_indexed] # sl = words_embedded.shape[1] tree = Tree() tree.pp = all #np.zeros(((2*sl-1),1)) tree.nodeScores = np.zeros(len(sNum)) # tree.nodeNames = np.arange(1,(2*sl-1)) tree.kids = np.zeros((len(sNum), 2)) tree.nodeFeatures = np.zeros((hiddenSize, len(sNum))) tree.nodeFeatures[:, :len(words_indexed)] = words_embedded toMerge = np.zeros(shape=(words_indexed.shape), dtype='int32') toMerge[:] = words_indexed[:] while len(toMerge) > 1: # find unpaired bottom leaf pairs (initially words) that share parent i = -1 foundGoodPair = False while (not foundGoodPair): i += 1 if sTree[toMerge[i]] == sTree[toMerge[i + 1]]: foundGoodPair = True newParent = sTree[toMerge[i]] kid1 = toMerge[i] kid2 = toMerge[i + 1] tree.kids[newParent, :] = [kid1, kid2] # set new parent to be possible merge candidate toMerge[i] = newParent # delete other kid toMerge = np.delete(toMerge, i + 1) c1 = tree.nodeFeatures[:, kid1] c2 = tree.nodeFeatures[:, kid2] p = np.tanh(np.dot(W1, c1) + np.dot(W2, c2) + b1.flatten()) tree.nodeFeatures[:, newParent] = p vec = tree.nodeFeatures[-1] vecFile.write(" ".join([str(x) for x in vec]) + '\n') vecFile.close() print "finished! "
def test_load_srl_data(self): rnndata = RNNDataCorpus() rnndata.load_data_srl(config.test_data_srl, nExamples=-1)
def train(): np.random.seed(131742) #get sentences, trees and labels nExamples = -1 print "loading data.." rnnData = RNNDataCorpus() rnnData.load_data(load_file=config.train_data, nExamples=nExamples) #initialize params print "initializing params" params = Params(data=rnnData, wordSize=50, rankWo=2) #define theta #one vector for all the parameters of mvrnn model: W, Wm, Wlabel, L, Lm n = params.wordSize fanIn = params.fanIn nWords = params.nWords nLabels = params.categories rank = params.rankWo Wo = 0.01 * np.random.randn(n + 2 * n * rank, nWords) #Lm, as in paper Wo[:n, :] = np.ones((n, Wo.shape[1])) #Lm, as in paper Wcat = 0.005 * np.random.randn(nLabels, fanIn) #Wlabel, as in paper # Wv = 0.01*np.random.randn(n, nWords) # WO = 0.01*np.random.randn(n, 2*n) # W = 0.01*np.random.randn(n, 2*n+1) #load pre-trained weights here mats = sio.loadmat(config.pre_trained_weights) Wv = mats.get('Wv') #L, as in paper W = mats.get('W') #W, as in paper WO = mats.get('WO') #Wm, as in paper sentencesIdx = np.arange(rnnData.ndoc()) np.random.shuffle(sentencesIdx) nTrain = 4 * len(sentencesIdx) / 5 trainSentIdx = sentencesIdx[0:nTrain] testSentIdx = sentencesIdx[nTrain:] batchSize = 5 nBatches = len(trainSentIdx) / batchSize evalFreq = 5 #evaluate after every 5 minibatches nTestSentEval = 50 #number of test sentences to be evaluated rnnData_train = RNNDataCorpus() rnnData.copy_into_minibatch(rnnData_train, trainSentIdx) rnnData_test = RNNDataCorpus() if (len(testSentIdx) > nTestSentEval): # np.random.shuffle(testSentIdx) #choose random test examples thisTestSentIdx = testSentIdx[:nTestSentEval] else: thisTestSentIdx = testSentIdx rnnData.copy_into_minibatch(rnnData_test, thisTestSentIdx) # [Wv_test, Wo_test, _] = getRelevantWords(rnnData_test, Wv,Wo,params) [Wv_trainTest, Wo_trainTest, all_train_idx ] = getRelevantWords(rnnData, Wv, Wo, params) #sets nWords_reduced, returns new arrays theta = np.concatenate((W.flatten(), WO.flatten(), Wcat.flatten(), Wv_trainTest.flatten(), Wo_trainTest.flatten())) #optimize print "starting training..." nIter = 100 rnnData_minibatch = RNNDataCorpus() for i in range(nIter): #train in minibatches # ftrain = np.zeros(nBatches) # for ibatch in range(nBatches): # set_minibatch(rnnData, rnnData_minibatch, ibatch, nBatches, trainSentIdx) # print 'Iteration: ', i, ' minibatch: ', ibatch tunedTheta, fbatch_train, _ = lbfgsb.fmin_l_bfgs_b( func=costFn, x0=theta, fprime=None, args=(rnnData_train, params), approx_grad=0, bounds=None, m=5, factr=1000000000000000.0, pgtol=1.0000000000000001e-5, epsilon=1e-08, iprint=3, maxfun=1, disp=0) #map parameters back W[:, :], WO[:, :], Wcat[:, :], Wv_trainTest, Wo_trainTest = unroll_theta( tunedTheta, params) Wv[:, all_train_idx] = Wv_trainTest Wo[:, all_train_idx] = Wo_trainTest # ftrain[ibatch] = fbatch_train theta = tunedTheta #for next iteration print "========================================" print "XXXXXXIteration ", i, print "Average cost: ", np.average(fbatch_train) evaluate(Wv, Wo, W, WO, Wcat, params, rnnData_test) print "========================================" #save weights save_dict = {'Wv': Wv, 'Wo': Wo, 'Wcat': Wcat, 'W': W, 'WO': WO} sio.savemat(config.saved_params_file + '_lbfgs_iter' + str(i), mdict=save_dict) print "saved tuned theta. "