def testloadData(self):
nExamples = 15
train_data = '/home/bhanu/Downloads/relationClassification/dataCamera/allDataTrain.mat'
pre_trained_weights = '/home/bhanu/Downloads/relationClassification/dataCamera/pretrainedWeights.mat'
rnnData = RNNDataCorpus()
rnnData.load_data(load_file=train_data, nExamples=nExamples)
params = Params(data=rnnData, wordSize=50, rankWo=3)
# n = params.wordSize; fanIn = params.fanIn; nWords = params.nWords; nLabels = params.categories; rank=params.rankWo
# theta = np.random.normal(loc=0.0, scale=np.math.pow(10,-5)**2, size = n*(2*n+1) + n*2*n + nLabels*fanIn + n*nWords + (2*n*rank+n)*nWords)
#
# #init respective parameters with prior values
# W, WO, Wcat, Wv, Wo = unroll_theta(theta, params)
# Wo[:n,:] = np.ones((n,Wo.shape[1])) #init word matrices with Identity matrices + epsilon
#load pre-trained weights here
mats = sio.loadmat(pre_trained_weights)
Wv = mats.get('Wv')
W = mats.get('W')
WO = mats.get('WO')
Wo = np.ndarray(
shape=(2 * params.wordSize * params.rankWo + params.wordSize) *
params.nWords,
dtype='float64')
sentencesIdx = np.arange(nExamples)
[allSNum_batch, allSNN_batch, Wv_batch, Wo_batch,
allWordInds] = getRelevantWords(rnnData, sentencesIdx, Wv, Wo, params)
print
def test():
# load testing data
print "loading test data.."
rnnData = RNNDataCorpus()
rnnData.load_data_srl(load_file=config.test_data_srl, nExamples=10)
# rnnData.load_data(load_file=config.test_data, nExamples=-1)
print 'loading trained model : ', sys.argv[1]
# mats = sio.loadmat(config.model_path+str(sys.argv[1]))
# Wv = mats.get('Wv') #L, as in paper
# W = mats.get('W') #W, as in paper
# WO = mats.get('WO') #Wm, as in paper
# Wo = mats.get('Wo')
# Wcat = mats.get('Wcat')
# n = Wv.shape[0]
# r = (Wo.shape[0] - n)/(2*n)
# rnn = MVRNN(W, WO, Wcat, Wv, Wo)
with open(config.model_path+sys.argv[1], 'r') as loadfile:
rnn = cPickle.load(loadfile)
n = rnn.Wv.shape[0]
r = (rnn.Wo.shape[0] - n)/(2*n)
print "initializing params.."
params = Params(data=rnnData, wordSize=n, rankWo=r)
print "evaluating.."
predictLabels = rnn.evaluate(params, rnnData)
print "creating labels file .."
create_preds_file(predictLabels, rnnData.categories, rnnData.sentenceLabels,
predictions_file=config.results_path+'preds_srl.txt',
testKeys_file='srl_test_keys.txt')
def test_write_srl_verbIndices(self):
infilenames = [config.train_data_srl, config.test_data_srl, config.dev_data_srl]
outfilenames = [config.corpus_path+'srl_vids.train', config.corpus_path+'srl_vids.test', config.corpus_path+'srl_vids.dev']
for infilename, outfilename in zip(infilenames, outfilenames):
rnndata = RNNDataCorpus()
rnndata.load_data_srl(infilename, nExamples=100)
allVerbIndices = rnndata.verbIndices
with open(outfilename, 'w') as wf:
for i in range(len(allVerbIndices)) :
verbids = allVerbIndices[i].flatten()
verbidsStr = [str(x) for x in verbids]
wf.write("|".join(verbidsStr)+"\n")
wf.flush()
def train():
SEED = 131742
load_model = False
custom_load = True #loads model from previously saved model except Wcat
np.random.seed(SEED)
#get sentences, trees and labels
nExamples = 5
print "loading data.."
rnnData_train = RNNDataCorpus()
rnnData_train.load_data_srl(load_file=config.train_data_srl, nExamples=nExamples)
rnnData_dev = RNNDataCorpus()
rnnData_dev.load_data_srl(load_file=config.dev_data_srl, nExamples=nExamples)
print "Number of sentences loaded in training data: ", rnnData_train.ndoc()
#initialize params
print "initializing params"
params = Params(data=rnnData_train, wordSize=52, rankWo=2)
n = params.wordSize; fanIn = params.fanIn; nWords = params.nWords; nLabels = params.categories; rank=params.rankWo
if(load_model):
with open(config.saved_params_file+"_45", 'r') as loadfile:
rnn = cPickle.load(loadfile)
elif(custom_load):
d = 2#extra features for wordvectors
Wo = 0.01*np.random.randn(n + 2*n*rank, nWords) #Lm, as in paper
Wo[:n,:] = np.ones((n,Wo.shape[1])) #Lm, as in paper
Wcat = 0.005*np.random.randn(nLabels, fanIn) #Wlabel, as in paper
Wv = 0.01*np.random.randn(n, nWords)
WO = 0.01*np.random.randn(n, 2*n)
W = 0.01*np.random.randn(n, 2*n+1)
#load pre-trained weights here
with open(config.saved_params_file+'_65', 'r') as loadfile:
oldrnn = cPickle.load(loadfile)
Wv[:-d,:] = oldrnn.Wv
# WO[:-d,:] = oldrnn.WO
rnn = MVRNN(W, WO, Wcat, Wv, Wo)
else:
Wo = 0.01*np.random.randn(n + 2*n*rank, nWords) #Lm, as in paper
Wo[:n,:] = np.ones((n,Wo.shape[1])) #Lm, as in paper
Wcat = 0.005*np.random.randn(nLabels, fanIn) #Wlabel, as in paper
Wv = 0.01*np.random.randn(n, nWords)
WO = 0.01*np.random.randn(n, 2*n)
W = 0.01*np.random.randn(n, 2*n+1)
rnn = MVRNN(W, WO, Wcat, Wv, Wo)
[_, _, all_train_idx] = getRelevantWords(rnnData_train, rnn.Wv,rnn.Wo,params) #sets nWords_reduced, returns new arrays
params.setNumReducedWords(len(all_train_idx))
theta = rnn.getTheta(params, all_train_idx)
#optimize
print "starting training using SGD..."
nIter = 500
optimizer = StochasticGradientDescent(niter=nIter , learning_rate=0.01, learningrateFactor=1.0, printAt10Iter='.', printAt100Iter='\n+')
optimizer.minimizeBatches(rnn=rnn, rnnData_train=rnnData_train, allTrainSentIdx=all_train_idx, params=params, x0=theta, func=costFn, fprime=None,
rnnData_test=rnnData_dev, initialSetSize=1, niter=nIter, seed=17,
modelFileName=config.saved_params_file+'SGD_SRL', printStatistics=True, modelSaveIter=1, nIterInPart=1,
nFetch=-1, rnd=None, nodeid=-1)
print "Finished training! "
def test_write_srl_verbIndices(self):
infilenames = [
config.train_data_srl, config.test_data_srl, config.dev_data_srl
]
outfilenames = [
config.corpus_path + 'srl_vids.train',
config.corpus_path + 'srl_vids.test',
config.corpus_path + 'srl_vids.dev'
]
for infilename, outfilename in zip(infilenames, outfilenames):
rnndata = RNNDataCorpus()
rnndata.load_data_srl(infilename, nExamples=100)
allVerbIndices = rnndata.verbIndices
with open(outfilename, 'w') as wf:
for i in range(len(allVerbIndices)):
verbids = allVerbIndices[i].flatten()
verbidsStr = [str(x) for x in verbids]
wf.write("|".join(verbidsStr) + "\n")
wf.flush()
def get_subset(self, rnnData, workerIndex):
''' data : complete train or test set for which a subset of docs is to be calculated for this worker '''
rnnData_mini = RNNDataCorpus()
dataSize = rnnData.ndoc()
sizePerNode = dataSize/(self.totalNodes-1) #exclude master node
startPos = (workerIndex-1) * sizePerNode
if(workerIndex != self.totalNodes-1): #last node on a machine gets all the remaining data
endPos = startPos+sizePerNode
else:
endPos = dataSize
rnnData.copy_into_minibatch(rnnData_mini, range(startPos, endPos))
return rnnData_mini
def testloadData(self):
nExamples = 15
train_data = '/home/bhanu/Downloads/relationClassification/dataCamera/allDataTrain.mat'
pre_trained_weights = '/home/bhanu/Downloads/relationClassification/dataCamera/pretrainedWeights.mat'
rnnData = RNNDataCorpus()
rnnData.load_data(load_file=train_data, nExamples=nExamples)
params = Params(data=rnnData, wordSize=50, rankWo=3)
# n = params.wordSize; fanIn = params.fanIn; nWords = params.nWords; nLabels = params.categories; rank=params.rankWo
# theta = np.random.normal(loc=0.0, scale=np.math.pow(10,-5)**2, size = n*(2*n+1) + n*2*n + nLabels*fanIn + n*nWords + (2*n*rank+n)*nWords)
#
# #init respective parameters with prior values
# W, WO, Wcat, Wv, Wo = unroll_theta(theta, params)
# Wo[:n,:] = np.ones((n,Wo.shape[1])) #init word matrices with Identity matrices + epsilon
#load pre-trained weights here
mats = sio.loadmat(pre_trained_weights)
Wv = mats.get('Wv'); W = mats.get('W'); WO = mats.get('WO')
Wo = np.ndarray(shape=(2*params.wordSize*params.rankWo + params.wordSize)*params.nWords, dtype='float64')
sentencesIdx = np.arange(nExamples)
[allSNum_batch, allSNN_batch, Wv_batch, Wo_batch, allWordInds] = getRelevantWords(rnnData, sentencesIdx, Wv,Wo,params)
print
def test():
# load testing data
print "loading test data.."
rnnData = RNNDataCorpus()
rnnData.load_data_srl(load_file=config.test_data_srl, nExamples=10)
# rnnData.load_data(load_file=config.test_data, nExamples=-1)
print 'loading trained model : ', sys.argv[1]
# mats = sio.loadmat(config.model_path+str(sys.argv[1]))
# Wv = mats.get('Wv') #L, as in paper
# W = mats.get('W') #W, as in paper
# WO = mats.get('WO') #Wm, as in paper
# Wo = mats.get('Wo')
# Wcat = mats.get('Wcat')
# n = Wv.shape[0]
# r = (Wo.shape[0] - n)/(2*n)
# rnn = MVRNN(W, WO, Wcat, Wv, Wo)
with open(config.model_path + sys.argv[1], 'r') as loadfile:
rnn = cPickle.load(loadfile)
n = rnn.Wv.shape[0]
r = (rnn.Wo.shape[0] - n) / (2 * n)
print "initializing params.."
params = Params(data=rnnData, wordSize=n, rankWo=r)
print "evaluating.."
predictLabels = rnn.evaluate(params, rnnData)
print "creating labels file .."
create_preds_file(predictLabels,
rnnData.categories,
rnnData.sentenceLabels,
predictions_file=config.results_path + 'preds_srl.txt',
testKeys_file='srl_test_keys.txt')
def initMVRNN(self):
print "Node: ",self.index," Loading training and dev sets.."
self.rnnData_train = RNNDataCorpus()
self.rnnData_train.load_data_srl(config.train_data_srl, nExamples=self.nuse)
self.rnnData_dev = RNNDataCorpus()
self.rnnData_dev.load_data_srl(config.dev_data_srl, nExamples=self.nuse)
modelfilename = config.saved_params_file+'SGD_SRLiter305'
print "Node: ", self.index," Loading model: ", modelfilename
# mats = sio.loadmat(config.saved_params_file+'iter120.mat')
# Wv = mats.get('Wv') #L, as in paper
# W = mats.get('W') #W, as in paper
# WO = mats.get('WO') #Wm, as in paper
# Wo = mats.get('Wo')
# Wcat = mats.get('Wcat')
# n = Wv.shape[0]
# r = (Wo.shape[0] - n)/(2*n)
with open(modelfilename, 'r') as loadfile:
self.rnn = cPickle.load(loadfile)#MVRNN(W, WO, Wcat, Wv, Wo)
n = self.rnn.Wv.shape[0]
r = (self.rnn.Wo.shape[0] - n)/(2*n)
print "Node: ",self.index, "initializing params.."
self.params = Params(data=self.rnnData_train, wordSize=n, rankWo=r)
# #to be removed
# Wcat = 0.005*np.random.randn(self.params.categories, self.params.fanIn)
# self.rnn.Wcat = Wcat
if(self.index == 0):
print "Master## Total trees in training set: ", self.rnnData_train.ndoc()
print "Master## nFetch: ", self.nFetch
if(self.index!=0):
self.rnnData_train = self.get_subset(self.rnnData_train, self.index)
self.rnnData_dev = None # workers don't need test set and trainTest, so to free memory release them
[_, _, self.all_train_idx] = getRelevantWords(self.rnnData_train, self.rnn.Wv,self.rnn.Wo,self.params) #sets nWords_reduced, returns new arrays
def train():
SEED = 13742
load_model = False
custom_load = False
np.random.seed(SEED)
#get sentences, trees and labels
nExamples = 5
print "loading data.."
rnnData_train = RNNDataCorpus()
rnnData_train.load_data_srl(load_file=config.train_data_srl,
nExamples=nExamples)
rnnData_dev = RNNDataCorpus()
rnnData_dev.load_data_srl(load_file=config.dev_data_srl,
nExamples=nExamples)
print "Number of sentences loaded in training data: ", rnnData_train.ndoc()
#initialize params
print "initializing params"
params = Params(data=rnnData_train, wordSize=52, rankWo=2)
n = params.wordSize
fanIn = params.fanIn
nWords = params.nWords
nLabels = params.categories
rank = params.rankWo
if (load_model):
modelfile = config.saved_params_file + 'SGD_SLL300'
rnn = MVRNNSLL.load(modelfile)
print 'loaded model : ', modelfile
elif (custom_load):
modelfile = config.saved_params_file + 'SGD_SLL300'
print "loading customized model..", modelfile
# d = 2#extra features for wordvectors
# Wo = 0.01*np.random.randn(n + 2*n*rank, nWords) #Lm, as in paper
# Wo[:n,:] = np.ones((n,Wo.shape[1])) #Lm, as in paper
# Wcat = 0.005*np.random.randn(nLabels, fanIn) #Wlabel, as in paper
# Wv = 0.01*np.random.randn(n, nWords)
# WO = 0.01*np.random.randn(n, 2*n)
# W = 0.01*np.random.randn(n, 2*n+1)
#load pre-trained weights here
oldrnn = MVRNNSLL.load(modelfile)
# Wv[:-d,:] = oldrnn.Wv
categories = [x.strip() for x in rnnData_train.categories]
Tran = init_transitions(dict(zip(categories, range(len(categories)))),
'iob')
rnn = MVRNNSLL(oldrnn.W, oldrnn.WO, oldrnn.Wcat, oldrnn.Wv, oldrnn.Wo,
Tran)
else:
#define theta
#one vector for all the parameters of mvrnn model: W, Wm, Wlabel, L, Lm
# n = params.wordSize; fanIn = params.fanIn; nWords = params.nWords; nLabels = params.categories; rank=params.rankWo
# Wo = 0.01*np.random.randn(n + 2*n*rank, nWords) #Lm, as in paper
# Wo[:n,:] = np.ones((n,Wo.shape[1])) #Lm, as in paper
# Wcat = 0.005*np.random.randn(nLabels, fanIn) #Wlabel, as in paper
# #load pre-trained weights here
## mats = sio.loadmat(config.saved_params_file)
# oldrnn = MVRNNSLL.load(modelfile)
# Wv = oldrnn.Wv #L, as in paper
# W = oldrnn..get('W') #W, as in paper
# WO = mats.get('WO') #Wm, as in paper
Wo = 0.01 * np.random.randn(n + 2 * n * rank, nWords) #Lm, as in paper
Wo[:n, :] = np.ones((n, Wo.shape[1])) #Lm, as in paper
Wcat = 0.005 * np.random.randn(nLabels, fanIn) #Wlabel, as in paper
Wv = 0.01 * np.random.randn(n, nWords)
WO = 0.01 * np.random.randn(n, 2 * n)
W = 0.01 * np.random.randn(n, 2 * n + 1)
categories = [x.strip() for x in rnnData_train.categories]
Tran = init_transitions(dict(zip(categories, range(len(categories)))),
'iob')
rnn = MVRNNSLL(W, WO, Wcat, Wv, Wo, Tran)
# rnn = MVRNNSLL(W, WO, Wcat, Wv, Wo, Tran)
[_, _, all_train_idx
] = getRelevantWords(rnnData_train, rnn.Wv, rnn.Wo,
params) #sets nWords_reduced, returns new arrays
params.setNumReducedWords(len(all_train_idx))
theta = rnn.getTheta(params, all_train_idx)
#optimize
print "starting training using SGD..."
nIter = 500
optimizer = StochasticGradientDescent(niter=nIter,
learning_rate=0.01,
learningrateFactor=1.0,
printAt10Iter='.',
printAt100Iter='\n+')
optimizer.minimizeBatches(rnn=rnn,
rnnData_train=rnnData_train,
allTrainSentIdx=all_train_idx,
params=params,
x0=theta,
func=costFn,
fprime=None,
rnnData_test=rnnData_dev,
initialSetSize=1,
niter=nIter,
seed=SEED,
modelFileName=config.saved_params_file +
'SGD_SLL',
printStatistics=True,
modelSaveIter=100,
nIterInPart=1,
nFetch=-1,
rnd=None,
nodeid=-1)
class SGDParallel(playdoh.ParallelTask):
''' '''
def initialize(self,
nuse,
totalNodes
):
""" This method is called on each node when playdoh.start_task method is called.
Each node initializes its own model.
"""
self.seed = 1742
self.nuse = nuse
self.totalNodes = totalNodes #including master node
self.debug = True
self.nIter = 1000
self.nFetch = 15 #after how many steps(minibatch processing) should a worker fetch new parameters from master
self.rnd = np.random.RandomState(self.seed)
def start(self):
if self.index == 0:
self.startMasterNode()
else:
self.startWorkerNode()
def startWorkerNode(self):
'''Receives updated parameters from the master, keeps updating local parameters using its own subset of training data, pushes local parameters to
master, all asynchronously '''
self.initMVRNN() #also initializes training and test sets
self.func = costFn
self.fprime = None
self.optimizer = StochasticGradientDescent(niter=self.nIter , learning_rate=0.01, learningrateFactor=1.0, printAt10Iter='.', printAt100Iter='\n+')
genObj = None
while True:
#fetch action and parameters from masters
action, master_rnn = self.pop('fromMaster_' + str(self.index))
if(action == "finish"):
print "Node: ",self.index," Received finish action from master, exiting..."
return
if self.debug:
print "Node: ", self.index, " Fetched new action & parameters from Master.. " + str(master_rnn.W[0,0])
self.params.resetNumReducedWords() #to unroll all the words
# W, WO, Wcat, Wv, Wo = unroll_theta(theta, self.params)
self.copy_into_rnn(master_rnn.getParamsList()) #copy to local rnn to global
theta = self.rnn.getTheta(self.params, self.all_train_idx)
self.params.setNumReducedWords(len(self.all_train_idx)) #set number of reduced words, to be used by costfn for unrolling
start_time = time.clock()
try:
genObj.next()
except (AttributeError, StopIteration) :
genObj = self.optimizer.minimizeBatchesPll(rnn=self.rnn, rnnData_train=self.rnnData_train, allTrainSentIdx=self.all_train_idx,
params=self.params, x0=theta, func=costFn, fprime=None,
rnnData_test=self.rnnData_dev, initialSetSize=1, niter=1, seed=self.seed,
modelFileName='', printStatistics=False, modelSaveIter=10, nIterInPart=1,
nFetch=self.nFetch, rnd=self.rnd, nodeid=self.index) #optimize this theta and save it in self.rnn
except:
raise
end_time = time.clock()
# [W, WO, Wcat, Wv, Wo] = self.rnn
# theta = np.concatenate((W.flatten(), WO.flatten(), Wcat.flatten(), Wv.flatten(), Wo.flatten())) # current local optimal theta
#push local theta to master
self.push('toMaster_' + str(self.index), (self.rnn, None)) #push the theta value #pushing a tuple object #playdoh bug
if self.debug:
print "Node:", self.index, " Execution time for ", self.nFetch, " minibatches: ", (end_time - start_time)/60, 'minutes'
print "Node:", self.index, " Pushed local parameters to Master.."
def async_eval(self, itr, testSet, result_file=None):
if testSet != None:
# if len(testSet.allSNum) > 100:
# rnnData_test_mini = RNNDataCorpus()
# testSet.copy_into_minibatch(rnnData_test_mini, range(100))
# else:
rnnData_test_mini = testSet
self.rnn.evaluate( self.params, rnnData_test_mini)
def async_save(self, itr):
timeStr = strftime("%Y-%m-%d_%H-%M-%S", gmtime()) # for generating names
modelFileName = config.saved_params_file+"_"+str(itr)+'_'+timeStr
#save weights
with open(modelFileName, 'wb') as wf:
cPickle.dump(self.rnn, wf, protocol=-1)
# [W, WO, Wcat, Wv, Wo] = self.rnn
# save_dict = {'Wv':Wv, 'Wo':Wo, 'Wcat':Wcat, 'W':W, 'WO':WO}
# sio.savemat(modelFileName, mdict=save_dict)
print 'Master## saved trained model to ', modelFileName
def startMasterNode(self):
'''Parameter server, receives gradients from workers, updates parameters of model, sends updated parameters to workers '''
nDocsTest = 150
printStatistics = True
action = ["continue", "finish"]
evalFreq = 10
saveFreq = 5
self.initMVRNN() #also initializes training and test sets
nparts = self.rnnData_dev.ndoc()/nDocsTest if self.rnnData_dev.ndoc() > nDocsTest else 1
testSeq = range(self.rnnData_dev.ndoc())
#start optimization by pushing intial params
for iNode in xrange(1, self.totalNodes):
if self.debug:
print "Master## Pushing parameters to " + str(iNode) + " theta[0]= " + str(self.rnn.W[0,0])
self.push('fromMaster_' + str(iNode),(action[0],self.rnn)) # send updated params to workers
#start iterations and evaluate after completion of set of iterations
for itr in range(self.nIter):
# workerW = np.zeros(self.rnn.W.shape); workerWO = np.zeros(self.rnn.WO.shape); workerWcat = np.zeros(self.rnn.Wcat.shape);
# workerWv = np.zeros(self.rnn.Wv.shape); workerWo = np.zeros(self.rnn.Wo.shape)
workerParams = []
for p in self.rnn.getParamsList():
workerParams.append(np.zeros(p.shape))
self.rnd.shuffle(testSeq)
(testSetPart, _) = self.rnnData_dev.getSubset(idocNext=0, ipart=0, nparts=nparts, sequenceIndex=testSeq)
if self.debug:
print "Master## Iteration: ",itr ,"Waiting to receive params from workers..."
for iNode in xrange(1, self.totalNodes): #get gradients from all workers
receivedRNN, _ = self.pop('toMaster_' + str(iNode)) # poll the workers for result
# print str(receivedRNN)
receivedParams = receivedRNN.getParamsList()
for workerParam, receivedParam in zip(workerParams, receivedParams):
workerParam += receivedParam
# workerW = workerW + rW
# workerWO = workerWO + rWO
# workerWcat = workerWcat + rWcat
# workerWv = workerWv + rWv
# workerWo = workerWo + rWo
if self.debug:
print "Master## Iteration: ",itr ,"Received theta from: " + str(iNode) + " theta[0]="+str(receivedParams[0][0,0])
#average all the received parameters
for i in range(len(workerParams)):
workerParams[i] /= (self.totalNodes-1)
# workerW = workerW /(self.totalNodes - 1)
# workerWO = workerWO /(self.totalNodes - 1)
# workerWcat = workerWcat /(self.totalNodes - 1)
# workerWv = workerWv /(self.totalNodes - 1)
# workerWo = workerWo /(self.totalNodes - 1) #one is master rest are workers
#copy into master's rnn
# workerRnn = [workerW, workerWO, workerWcat, workerWv, workerWo]
self.copy_into_rnn(workerParams)
if(self.debug):
print "Master## Average Params : ", str(workerParams[0][0,0])
for iNode in xrange(1, self.totalNodes):
self.push('fromMaster_' + str(iNode),(action[0],self.rnn)) # send updated params to workers
if self.debug:
print "Master## Sending new parameters to Worker ", iNode, "Param[0]: ", str(self.rnn.W[0,0])
if printStatistics and ((itr % evalFreq) == 0) :
p = mp.Process(target=self.async_eval, args=(itr,self.rnnData_dev))
p.start()
if((itr % saveFreq == 0)): # save mlp after every 5 iterations of nFetch cycles.
p = mp.Process(target=self.async_save, args=(itr,))
p.start()
#finish optimization and close workers
for iNode in xrange(1, self.totalNodes):
self.push('fromMaster_' + str(iNode),(action[1],None)) # send updated params to workers
def get_result(self):
if self.index == 0:
return self.rnn
def copy_into_rnn(self, workerRnnParams):
for dst, src in zip(self.rnn.getParamsList(), workerRnnParams):
dst[:,:] = src
def initMVRNN(self):
print "Node: ",self.index," Loading training and dev sets.."
self.rnnData_train = RNNDataCorpus()
self.rnnData_train.load_data_srl(config.train_data_srl, nExamples=self.nuse)
self.rnnData_dev = RNNDataCorpus()
self.rnnData_dev.load_data_srl(config.dev_data_srl, nExamples=self.nuse)
modelfilename = config.saved_params_file+'SGD_SRLiter305'
print "Node: ", self.index," Loading model: ", modelfilename
# mats = sio.loadmat(config.saved_params_file+'iter120.mat')
# Wv = mats.get('Wv') #L, as in paper
# W = mats.get('W') #W, as in paper
# WO = mats.get('WO') #Wm, as in paper
# Wo = mats.get('Wo')
# Wcat = mats.get('Wcat')
# n = Wv.shape[0]
# r = (Wo.shape[0] - n)/(2*n)
with open(modelfilename, 'r') as loadfile:
self.rnn = cPickle.load(loadfile)#MVRNN(W, WO, Wcat, Wv, Wo)
n = self.rnn.Wv.shape[0]
r = (self.rnn.Wo.shape[0] - n)/(2*n)
print "Node: ",self.index, "initializing params.."
self.params = Params(data=self.rnnData_train, wordSize=n, rankWo=r)
# #to be removed
# Wcat = 0.005*np.random.randn(self.params.categories, self.params.fanIn)
# self.rnn.Wcat = Wcat
if(self.index == 0):
print "Master## Total trees in training set: ", self.rnnData_train.ndoc()
print "Master## nFetch: ", self.nFetch
if(self.index!=0):
self.rnnData_train = self.get_subset(self.rnnData_train, self.index)
self.rnnData_dev = None # workers don't need test set and trainTest, so to free memory release them
[_, _, self.all_train_idx] = getRelevantWords(self.rnnData_train, self.rnn.Wv,self.rnn.Wo,self.params) #sets nWords_reduced, returns new arrays
#set this to none, as unrolling of theta will take all words into account.
# self.theta = np.concatenate((W.flatten(), WO.flatten(), Wcat.flatten(), Wv_trainTest.flatten(), Wo_trainTest.flatten()))
def get_subset(self, rnnData, workerIndex):
''' data : complete train or test set for which a subset of docs is to be calculated for this worker '''
rnnData_mini = RNNDataCorpus()
dataSize = rnnData.ndoc()
sizePerNode = dataSize/(self.totalNodes-1) #exclude master node
startPos = (workerIndex-1) * sizePerNode
if(workerIndex != self.totalNodes-1): #last node on a machine gets all the remaining data
endPos = startPos+sizePerNode
else:
endPos = dataSize
rnnData.copy_into_minibatch(rnnData_mini, range(startPos, endPos))
return rnnData_mini
def writeVectors():
vecFileName = config.results_path+"vectors.out"
vecFile = open(vecFileName, 'w')
mats = sio.loadmat(config.corpus_path+'vars.normalized.100.mat')
We_orig = mats.get('We')
params = sio.loadmat(config.corpus_path+'params_rae.mat')
W1 = params.get('W1')
W2 = params.get('W2')
b1 = params.get('b1')
We = params.get('We')
b = params.get('b')
W = params.get('W')
hiddenSize = 100
nExamples = 5
print "loading data.."
rnnData_train = RNNDataCorpus()
rnnData_train.load_data_srl(load_file=config.train_data_srl, nExamples=nExamples)
print 'writing vectors to: ', vecFileName
for ii in range(len(rnnData_train.allSNum)):
sNum = rnnData_train.allSNum[ii]
sStr = rnnData_train.allSStr[ii]
sTree = rnnData_train.allSTree[ii]
sKids = rnnData_train.allSKids[ii]
words_indexed = np.where(sNum >= 0)[0]
#L is only the part of the embedding matrix that is relevant for this sentence
#L is deltaWe
if We.shape[1] != 0:
L = We[:, words_indexed]
words_embedded = We_orig[:, words_indexed] + L;
else :
words_embedded = We_orig[:, words_indexed]
# sl = words_embedded.shape[1]
tree = Tree()
tree.pp = all#np.zeros(((2*sl-1),1))
tree.nodeScores = np.zeros(len(sNum))
# tree.nodeNames = np.arange(1,(2*sl-1))
tree.kids = np.zeros((len(sNum),2))
tree.nodeFeatures = np.zeros((hiddenSize, len(sNum)))
tree.nodeFeatures[:,:len(words_indexed)] = words_embedded;
toMerge = np.zeros(shape=(words_indexed.shape), dtype='int32')
toMerge[:] = words_indexed[:]
while len(toMerge)>1 :
# find unpaired bottom leaf pairs (initially words) that share parent
i=-1;
foundGoodPair = False
while (not foundGoodPair ) :
i += 1
if sTree[toMerge[i]]==sTree[toMerge[i+1]]:
foundGoodPair = True
newParent = sTree[toMerge[i]]
kid1 = toMerge[i]
kid2 = toMerge[i+1]
tree.kids[newParent,:] = [kid1, kid2];
# set new parent to be possible merge candidate
toMerge[i] = newParent;
# delete other kid
toMerge = np.delete(toMerge,i+1)
c1 = tree.nodeFeatures[:,kid1]
c2 = tree.nodeFeatures[:,kid2]
p = np.tanh(np.dot(W1,c1) + np.dot(W2,c2) + b1.flatten())
tree.nodeFeatures[:,newParent] = p;
vec = tree.nodeFeatures[-1]
vecFile.write(" ".join([str(x) for x in vec])+'\n')
vecFile.close()
print "finished! "
def train():
np.random.seed(131742)
#get sentences, trees and labels
nExamples = -1
print "loading data.."
rnnData = RNNDataCorpus()
rnnData.load_data(load_file=config.train_data, nExamples=nExamples)
#initialize params
print "initializing params"
params = Params(data=rnnData, wordSize=50, rankWo=2)
#define theta
#one vector for all the parameters of mvrnn model: W, Wm, Wlabel, L, Lm
n = params.wordSize; fanIn = params.fanIn; nWords = params.nWords; nLabels = params.categories; rank=params.rankWo
Wo = 0.01*np.random.randn(n + 2*n*rank, nWords) #Lm, as in paper
Wo[:n,:] = np.ones((n,Wo.shape[1])) #Lm, as in paper
Wcat = 0.005*np.random.randn(nLabels, fanIn) #Wlabel, as in paper
# Wv = 0.01*np.random.randn(n, nWords)
# WO = 0.01*np.random.randn(n, 2*n)
# W = 0.01*np.random.randn(n, 2*n+1)
#load pre-trained weights here
mats = sio.loadmat(config.pre_trained_weights)
Wv = mats.get('Wv') #L, as in paper
W = mats.get('W') #W, as in paper
WO = mats.get('WO') #Wm, as in paper
sentencesIdx = np.arange(rnnData.ndoc())
np.random.shuffle(sentencesIdx)
nTrain = 4*len(sentencesIdx)/5
trainSentIdx = sentencesIdx[0:nTrain]
testSentIdx = sentencesIdx[nTrain:]
batchSize = 5
nBatches = len(trainSentIdx)/batchSize
evalFreq = 5 #evaluate after every 5 minibatches
nTestSentEval = 50 #number of test sentences to be evaluated
rnnData_train = RNNDataCorpus()
rnnData.copy_into_minibatch(rnnData_train, trainSentIdx)
rnnData_test = RNNDataCorpus()
if(len(testSentIdx) > nTestSentEval):
# np.random.shuffle(testSentIdx) #choose random test examples
thisTestSentIdx = testSentIdx[:nTestSentEval]
else:
thisTestSentIdx = testSentIdx
rnnData.copy_into_minibatch(rnnData_test, thisTestSentIdx)
# [Wv_test, Wo_test, _] = getRelevantWords(rnnData_test, Wv,Wo,params)
[Wv_trainTest, Wo_trainTest, all_train_idx] = getRelevantWords(rnnData, Wv,Wo,params) #sets nWords_reduced, returns new arrays
theta = np.concatenate((W.flatten(), WO.flatten(), Wcat.flatten(), Wv_trainTest.flatten(), Wo_trainTest.flatten()))
#optimize
print "starting training..."
nIter = 100
rnnData_minibatch = RNNDataCorpus()
for i in range(nIter):
#train in minibatches
# ftrain = np.zeros(nBatches)
# for ibatch in range(nBatches):
# set_minibatch(rnnData, rnnData_minibatch, ibatch, nBatches, trainSentIdx)
# print 'Iteration: ', i, ' minibatch: ', ibatch
tunedTheta, fbatch_train, _ = lbfgsb.fmin_l_bfgs_b(func=costFn, x0=theta, fprime=None, args=(rnnData_train, params), approx_grad=0, bounds=None, m=5,
factr=1000000000000000.0, pgtol=1.0000000000000001e-5, epsilon=1e-08,
iprint=3, maxfun=1, disp=0)
#map parameters back
W[:,:], WO[:,:], Wcat[:,:], Wv_trainTest, Wo_trainTest = unroll_theta(tunedTheta, params)
Wv[:,all_train_idx] = Wv_trainTest
Wo[:,all_train_idx] = Wo_trainTest
# ftrain[ibatch] = fbatch_train
theta = tunedTheta #for next iteration
print "========================================"
print "XXXXXXIteration ", i,
print "Average cost: ", np.average(fbatch_train)
evaluate(Wv,Wo,W,WO,Wcat,params, rnnData_test)
print "========================================"
#save weights
save_dict = {'Wv':Wv, 'Wo':Wo, 'Wcat':Wcat, 'W':W, 'WO':WO}
sio.savemat(config.saved_params_file+'_lbfgs_iter'+str(i), mdict=save_dict)
print "saved tuned theta. "
def train():
SEED = 131742
load_model = False
custom_load = True #loads model from previously saved model except Wcat
np.random.seed(SEED)
#get sentences, trees and labels
nExamples = 5
print "loading data.."
rnnData_train = RNNDataCorpus()
rnnData_train.load_data_srl(load_file=config.train_data_srl,
nExamples=nExamples)
rnnData_dev = RNNDataCorpus()
rnnData_dev.load_data_srl(load_file=config.dev_data_srl,
nExamples=nExamples)
print "Number of sentences loaded in training data: ", rnnData_train.ndoc()
#initialize params
print "initializing params"
params = Params(data=rnnData_train, wordSize=52, rankWo=2)
n = params.wordSize
fanIn = params.fanIn
nWords = params.nWords
nLabels = params.categories
rank = params.rankWo
if (load_model):
with open(config.saved_params_file + "_45", 'r') as loadfile:
rnn = cPickle.load(loadfile)
elif (custom_load):
d = 2 #extra features for wordvectors
Wo = 0.01 * np.random.randn(n + 2 * n * rank, nWords) #Lm, as in paper
Wo[:n, :] = np.ones((n, Wo.shape[1])) #Lm, as in paper
Wcat = 0.005 * np.random.randn(nLabels, fanIn) #Wlabel, as in paper
Wv = 0.01 * np.random.randn(n, nWords)
WO = 0.01 * np.random.randn(n, 2 * n)
W = 0.01 * np.random.randn(n, 2 * n + 1)
#load pre-trained weights here
with open(config.saved_params_file + '_65', 'r') as loadfile:
oldrnn = cPickle.load(loadfile)
Wv[:-d, :] = oldrnn.Wv
# WO[:-d,:] = oldrnn.WO
rnn = MVRNN(W, WO, Wcat, Wv, Wo)
else:
Wo = 0.01 * np.random.randn(n + 2 * n * rank, nWords) #Lm, as in paper
Wo[:n, :] = np.ones((n, Wo.shape[1])) #Lm, as in paper
Wcat = 0.005 * np.random.randn(nLabels, fanIn) #Wlabel, as in paper
Wv = 0.01 * np.random.randn(n, nWords)
WO = 0.01 * np.random.randn(n, 2 * n)
W = 0.01 * np.random.randn(n, 2 * n + 1)
rnn = MVRNN(W, WO, Wcat, Wv, Wo)
[_, _, all_train_idx
] = getRelevantWords(rnnData_train, rnn.Wv, rnn.Wo,
params) #sets nWords_reduced, returns new arrays
params.setNumReducedWords(len(all_train_idx))
theta = rnn.getTheta(params, all_train_idx)
#optimize
print "starting training using SGD..."
nIter = 500
optimizer = StochasticGradientDescent(niter=nIter,
learning_rate=0.01,
learningrateFactor=1.0,
printAt10Iter='.',
printAt100Iter='\n+')
optimizer.minimizeBatches(rnn=rnn,
rnnData_train=rnnData_train,
allTrainSentIdx=all_train_idx,
params=params,
x0=theta,
func=costFn,
fprime=None,
rnnData_test=rnnData_dev,
initialSetSize=1,
niter=nIter,
seed=17,
modelFileName=config.saved_params_file +
'SGD_SRL',
printStatistics=True,
modelSaveIter=1,
nIterInPart=1,
nFetch=-1,
rnd=None,
nodeid=-1)
print "Finished training! "
def train():
SEED = 13742
load_model = False
custom_load = False
np.random.seed(SEED)
#get sentences, trees and labels
nExamples = 5
print "loading data.."
rnnData_train = RNNDataCorpus()
rnnData_train.load_data_srl(load_file=config.train_data_srl, nExamples=nExamples)
rnnData_dev = RNNDataCorpus()
rnnData_dev.load_data_srl(load_file=config.dev_data_srl, nExamples=nExamples)
print "Number of sentences loaded in training data: ", rnnData_train.ndoc()
#initialize params
print "initializing params"
params = Params(data=rnnData_train, wordSize=52, rankWo=2)
n = params.wordSize; fanIn = params.fanIn; nWords = params.nWords; nLabels = params.categories; rank=params.rankWo
if(load_model):
modelfile = config.saved_params_file+'SGD_SLL300'
rnn = MVRNNSLL.load(modelfile)
print 'loaded model : ', modelfile
elif(custom_load):
modelfile = config.saved_params_file+'SGD_SLL300'
print "loading customized model..", modelfile
# d = 2#extra features for wordvectors
# Wo = 0.01*np.random.randn(n + 2*n*rank, nWords) #Lm, as in paper
# Wo[:n,:] = np.ones((n,Wo.shape[1])) #Lm, as in paper
# Wcat = 0.005*np.random.randn(nLabels, fanIn) #Wlabel, as in paper
# Wv = 0.01*np.random.randn(n, nWords)
# WO = 0.01*np.random.randn(n, 2*n)
# W = 0.01*np.random.randn(n, 2*n+1)
#load pre-trained weights here
oldrnn = MVRNNSLL.load(modelfile)
# Wv[:-d,:] = oldrnn.Wv
categories = [x.strip() for x in rnnData_train.categories]
Tran = init_transitions(dict(zip(categories, range(len(categories)))), 'iob')
rnn = MVRNNSLL(oldrnn.W, oldrnn.WO, oldrnn.Wcat, oldrnn.Wv, oldrnn.Wo, Tran)
else:
#define theta
#one vector for all the parameters of mvrnn model: W, Wm, Wlabel, L, Lm
# n = params.wordSize; fanIn = params.fanIn; nWords = params.nWords; nLabels = params.categories; rank=params.rankWo
# Wo = 0.01*np.random.randn(n + 2*n*rank, nWords) #Lm, as in paper
# Wo[:n,:] = np.ones((n,Wo.shape[1])) #Lm, as in paper
# Wcat = 0.005*np.random.randn(nLabels, fanIn) #Wlabel, as in paper
# #load pre-trained weights here
## mats = sio.loadmat(config.saved_params_file)
# oldrnn = MVRNNSLL.load(modelfile)
# Wv = oldrnn.Wv #L, as in paper
# W = oldrnn..get('W') #W, as in paper
# WO = mats.get('WO') #Wm, as in paper
Wo = 0.01*np.random.randn(n + 2*n*rank, nWords) #Lm, as in paper
Wo[:n,:] = np.ones((n,Wo.shape[1])) #Lm, as in paper
Wcat = 0.005*np.random.randn(nLabels, fanIn) #Wlabel, as in paper
Wv = 0.01*np.random.randn(n, nWords)
WO = 0.01*np.random.randn(n, 2*n)
W = 0.01*np.random.randn(n, 2*n+1)
categories = [x.strip() for x in rnnData_train.categories]
Tran = init_transitions(dict(zip(categories, range(len(categories)))), 'iob')
rnn = MVRNNSLL(W, WO, Wcat, Wv, Wo, Tran)
# rnn = MVRNNSLL(W, WO, Wcat, Wv, Wo, Tran)
[_, _, all_train_idx] = getRelevantWords(rnnData_train, rnn.Wv, rnn.Wo, params) #sets nWords_reduced, returns new arrays
params.setNumReducedWords(len(all_train_idx))
theta = rnn.getTheta(params, all_train_idx)
#optimize
print "starting training using SGD..."
nIter = 500
optimizer = StochasticGradientDescent(niter=nIter , learning_rate=0.01, learningrateFactor=1.0, printAt10Iter='.', printAt100Iter='\n+')
optimizer.minimizeBatches(rnn=rnn, rnnData_train=rnnData_train, allTrainSentIdx=all_train_idx, params=params, x0=theta, func=costFn, fprime=None,
rnnData_test=rnnData_dev, initialSetSize=1, niter=nIter, seed=SEED,
modelFileName=config.saved_params_file+'SGD_SLL', printStatistics=True, modelSaveIter=100, nIterInPart=1,
nFetch=-1, rnd=None, nodeid=-1)
def writeVectors():
vecFileName = config.results_path + "vectors.out"
vecFile = open(vecFileName, 'w')
mats = sio.loadmat(config.corpus_path + 'vars.normalized.100.mat')
We_orig = mats.get('We')
params = sio.loadmat(config.corpus_path + 'params_rae.mat')
W1 = params.get('W1')
W2 = params.get('W2')
b1 = params.get('b1')
We = params.get('We')
b = params.get('b')
W = params.get('W')
hiddenSize = 100
nExamples = 5
print "loading data.."
rnnData_train = RNNDataCorpus()
rnnData_train.load_data_srl(load_file=config.train_data_srl,
nExamples=nExamples)
print 'writing vectors to: ', vecFileName
for ii in range(len(rnnData_train.allSNum)):
sNum = rnnData_train.allSNum[ii]
sStr = rnnData_train.allSStr[ii]
sTree = rnnData_train.allSTree[ii]
sKids = rnnData_train.allSKids[ii]
words_indexed = np.where(sNum >= 0)[0]
#L is only the part of the embedding matrix that is relevant for this sentence
#L is deltaWe
if We.shape[1] != 0:
L = We[:, words_indexed]
words_embedded = We_orig[:, words_indexed] + L
else:
words_embedded = We_orig[:, words_indexed]
# sl = words_embedded.shape[1]
tree = Tree()
tree.pp = all #np.zeros(((2*sl-1),1))
tree.nodeScores = np.zeros(len(sNum))
# tree.nodeNames = np.arange(1,(2*sl-1))
tree.kids = np.zeros((len(sNum), 2))
tree.nodeFeatures = np.zeros((hiddenSize, len(sNum)))
tree.nodeFeatures[:, :len(words_indexed)] = words_embedded
toMerge = np.zeros(shape=(words_indexed.shape), dtype='int32')
toMerge[:] = words_indexed[:]
while len(toMerge) > 1:
# find unpaired bottom leaf pairs (initially words) that share parent
i = -1
foundGoodPair = False
while (not foundGoodPair):
i += 1
if sTree[toMerge[i]] == sTree[toMerge[i + 1]]:
foundGoodPair = True
newParent = sTree[toMerge[i]]
kid1 = toMerge[i]
kid2 = toMerge[i + 1]
tree.kids[newParent, :] = [kid1, kid2]
# set new parent to be possible merge candidate
toMerge[i] = newParent
# delete other kid
toMerge = np.delete(toMerge, i + 1)
c1 = tree.nodeFeatures[:, kid1]
c2 = tree.nodeFeatures[:, kid2]
p = np.tanh(np.dot(W1, c1) + np.dot(W2, c2) + b1.flatten())
tree.nodeFeatures[:, newParent] = p
vec = tree.nodeFeatures[-1]
vecFile.write(" ".join([str(x) for x in vec]) + '\n')
vecFile.close()
print "finished! "
def test_load_srl_data(self):
rnndata = RNNDataCorpus()
rnndata.load_data_srl(config.test_data_srl, nExamples=-1)
def train():
np.random.seed(131742)
#get sentences, trees and labels
nExamples = -1
print "loading data.."
rnnData = RNNDataCorpus()
rnnData.load_data(load_file=config.train_data, nExamples=nExamples)
#initialize params
print "initializing params"
params = Params(data=rnnData, wordSize=50, rankWo=2)
#define theta
#one vector for all the parameters of mvrnn model: W, Wm, Wlabel, L, Lm
n = params.wordSize
fanIn = params.fanIn
nWords = params.nWords
nLabels = params.categories
rank = params.rankWo
Wo = 0.01 * np.random.randn(n + 2 * n * rank, nWords) #Lm, as in paper
Wo[:n, :] = np.ones((n, Wo.shape[1])) #Lm, as in paper
Wcat = 0.005 * np.random.randn(nLabels, fanIn) #Wlabel, as in paper
# Wv = 0.01*np.random.randn(n, nWords)
# WO = 0.01*np.random.randn(n, 2*n)
# W = 0.01*np.random.randn(n, 2*n+1)
#load pre-trained weights here
mats = sio.loadmat(config.pre_trained_weights)
Wv = mats.get('Wv') #L, as in paper
W = mats.get('W') #W, as in paper
WO = mats.get('WO') #Wm, as in paper
sentencesIdx = np.arange(rnnData.ndoc())
np.random.shuffle(sentencesIdx)
nTrain = 4 * len(sentencesIdx) / 5
trainSentIdx = sentencesIdx[0:nTrain]
testSentIdx = sentencesIdx[nTrain:]
batchSize = 5
nBatches = len(trainSentIdx) / batchSize
evalFreq = 5 #evaluate after every 5 minibatches
nTestSentEval = 50 #number of test sentences to be evaluated
rnnData_train = RNNDataCorpus()
rnnData.copy_into_minibatch(rnnData_train, trainSentIdx)
rnnData_test = RNNDataCorpus()
if (len(testSentIdx) > nTestSentEval):
# np.random.shuffle(testSentIdx) #choose random test examples
thisTestSentIdx = testSentIdx[:nTestSentEval]
else:
thisTestSentIdx = testSentIdx
rnnData.copy_into_minibatch(rnnData_test, thisTestSentIdx)
# [Wv_test, Wo_test, _] = getRelevantWords(rnnData_test, Wv,Wo,params)
[Wv_trainTest, Wo_trainTest, all_train_idx
] = getRelevantWords(rnnData, Wv, Wo,
params) #sets nWords_reduced, returns new arrays
theta = np.concatenate((W.flatten(), WO.flatten(), Wcat.flatten(),
Wv_trainTest.flatten(), Wo_trainTest.flatten()))
#optimize
print "starting training..."
nIter = 100
rnnData_minibatch = RNNDataCorpus()
for i in range(nIter):
#train in minibatches
# ftrain = np.zeros(nBatches)
# for ibatch in range(nBatches):
# set_minibatch(rnnData, rnnData_minibatch, ibatch, nBatches, trainSentIdx)
# print 'Iteration: ', i, ' minibatch: ', ibatch
tunedTheta, fbatch_train, _ = lbfgsb.fmin_l_bfgs_b(
func=costFn,
x0=theta,
fprime=None,
args=(rnnData_train, params),
approx_grad=0,
bounds=None,
m=5,
factr=1000000000000000.0,
pgtol=1.0000000000000001e-5,
epsilon=1e-08,
iprint=3,
maxfun=1,
disp=0)
#map parameters back
W[:, :], WO[:, :], Wcat[:, :], Wv_trainTest, Wo_trainTest = unroll_theta(
tunedTheta, params)
Wv[:, all_train_idx] = Wv_trainTest
Wo[:, all_train_idx] = Wo_trainTest
# ftrain[ibatch] = fbatch_train
theta = tunedTheta #for next iteration
print "========================================"
print "XXXXXXIteration ", i,
print "Average cost: ", np.average(fbatch_train)
evaluate(Wv, Wo, W, WO, Wcat, params, rnnData_test)
print "========================================"
#save weights
save_dict = {'Wv': Wv, 'Wo': Wo, 'Wcat': Wcat, 'W': W, 'WO': WO}
sio.savemat(config.saved_params_file + '_lbfgs_iter' + str(i),
mdict=save_dict)
print "saved tuned theta. "
def test_load_srl_data(self):
rnndata = RNNDataCorpus()
rnndata.load_data_srl(config.test_data_srl, nExamples=-1)
