word2id, config['maxSentLen'], test_file_paths)
# test_sents, test_masks, test_labels, word2id = load_il_groundtruth_as_testset(word2id, config['maxSentLen'], test_file_path)
'''task1: combine datasets from two sub-tasks'''
train_sents = train_p1_sents + train_p2_sents
train_masks = train_p1_masks + train_p2_masks
train_labels = train_p1_labels + train_p2_labels
# all_sentences, all_masks, all_labels, word2id=load_BBN_multi_labels_dataset(maxlen=config['maxSentLen'])
label_sent, label_mask = load_SF_type_descriptions(
word2id, config['type_size'], config['describ_max_len'])
# emb_root = '/scratch/wyin3/dickens_save_dataset/LORELEI/multi-lingual-emb/Uyghur/'
# print('loading bilingual embeddings....')
# word2vec=load_fasttext_multiple_word2vec_given_file([emb_root+'vectors-en.txt',emb_root+'vectors-ug.txt'], 300)
# emb_root = '/scratch/wyin3/dickens_save_dataset/LORELEI/il11/bilingual_emb/'
# print('loading bilingual embeddings....')
word2vec = load_fasttext_multiple_word2vec_given_file(
emb_paths[emb_path_id], 300)
vocab_size = len(word2id) + 1
rand_values = np.random.RandomState(1234).normal(
0.0, 0.01,
(vocab_size,
config['emb_size'])) #generate a matrix by Gaussian distribution
rand_values[0] = np.array(np.zeros(config['emb_size']), dtype=np.float32)
id2word = {y: x for x, y in word2id.items()}
rand_values = load_word2vec_to_init(rand_values, id2word, word2vec)
embeddings = torch.Tensor(rand_values)
print("build model...")
model, loss_function, optimizer = build_model(config['emb_size'],
config['hidden_size'],
vocab_size, 12, embeddings,
def evaluate_lenet5(learning_rate=0.01, n_epochs=100, emb_size=300, batch_size=50, filter_size=[3,5], maxSentLen=100, hidden_size=[300,300]):
model_options = locals().copy()
print "model options", model_options
emb_root = '/save/wenpeng/datasets/LORELEI/multi-lingual-emb/'
seed=1234
np.random.seed(seed)
rng = np.random.RandomState(seed) #random seed, control the model generates the same results
srng = T.shared_randomstreams.RandomStreams(rng.randint(seed))
all_sentences, all_masks, all_labels, word2id=load_BBN_multi_labels_dataset(maxlen=maxSentLen) #minlen, include one label, at least one word in the sentence
train_sents=np.asarray(all_sentences[0], dtype='int32')
train_masks=np.asarray(all_masks[0], dtype=theano.config.floatX)
train_labels=np.asarray(all_labels[0], dtype='int32')
train_size=len(train_labels)
dev_sents=np.asarray(all_sentences[1], dtype='int32')
dev_masks=np.asarray(all_masks[1], dtype=theano.config.floatX)
dev_labels=np.asarray(all_labels[1], dtype='int32')
dev_size=len(dev_labels)
test_sents=np.asarray(all_sentences[2], dtype='int32')
test_masks=np.asarray(all_masks[2], dtype=theano.config.floatX)
test_labels=np.asarray(all_labels[2], dtype='int32')
test_size=len(test_labels)
comb_sents = np.concatenate([train_sents,test_sents], axis=0)
comb_masks = np.concatenate([train_masks,test_masks], axis=0)
comb_labels = np.asarray([0]*train_size+[1]*test_size, dtype='int32')
comb_size = len(comb_labels)
vocab_size= len(word2id)+1 # add one zero pad index
rand_values=rng.normal(0.0, 0.01, (vocab_size, emb_size)) #generate a matrix by Gaussian distribution
rand_values[0]=np.array(np.zeros(emb_size),dtype=theano.config.floatX)
id2word = {y:x for x,y in word2id.iteritems()}
word2vec=load_fasttext_multiple_word2vec_given_file([emb_root+'wiki.en.vec',emb_root+'mono-lingual-il5-xinli.vec'], 300)
rand_values=load_word2vec_to_init(rand_values, id2word, word2vec)
embeddings=theano.shared(value=np.array(rand_values,dtype=theano.config.floatX), borrow=True) #wrap up the python variable "rand_values" into theano variable
#now, start to build the input form of the model
sents_id_matrix=T.imatrix('sents_id_matrix')
sents_mask=T.fmatrix('sents_mask')
labels=T.imatrix('labels') #batch*12
domain_labels = T.ivector()
######################
# BUILD ACTUAL MODEL #
######################
print '... building the model'
common_input=embeddings[sents_id_matrix.flatten()].reshape((batch_size,maxSentLen, emb_size)).dimshuffle(0,2,1) #the input format can be adapted into CNN or GRU or LSTM
bow_emb = T.sum(common_input*sents_mask.dimshuffle(0,'x',1),axis=2)
# bow_mean_emb = bow_emb/T.sum(sents_mask,axis=1).dimshuffle(0,'x')
conv_W, conv_b=create_conv_para(rng, filter_shape=(hidden_size[0], 1, emb_size, filter_size[0]))
conv_W2, conv_b2=create_conv_para(rng, filter_shape=(hidden_size[0], 1, emb_size, filter_size[1]))
NN_para = [conv_W, conv_b, conv_W2, conv_b2]
conv_model = Conv_with_Mask(rng, input_tensor3=common_input,
mask_matrix = sents_mask,
image_shape=(batch_size, 1, emb_size, maxSentLen),
filter_shape=(hidden_size[0], 1, emb_size, filter_size[0]), W=conv_W, b=conv_b) #mutiple mask with the conv_out to set the features by UNK to zero
sent_embeddings=conv_model.maxpool_vec #(batch_size, hidden_size) # each sentence then have an embedding of length hidden_size
conv_model2 = Conv_with_Mask(rng, input_tensor3=common_input,
mask_matrix = sents_mask,
image_shape=(batch_size, 1, emb_size, maxSentLen),
filter_shape=(hidden_size[0], 1, emb_size, filter_size[1]), W=conv_W2, b=conv_b2) #mutiple mask with the conv_out to set the features by UNK to zero
sent_embeddings2=conv_model2.maxpool_vec #(batch_size, hidden_size) # each sentence then have an embedding of length hidden_size
'''
adversarial
'''
domain_input = T.concatenate([sent_embeddings,sent_embeddings2, bow_emb], axis=1)
domain_input_size = hidden_size[0]*2+emb_size
HL_layer_1_W, HL_layer_1_b = create_HiddenLayer_para(rng, domain_input_size, hidden_size[0])
HL_layer_2_W, HL_layer_2_b = create_HiddenLayer_para(rng, hidden_size[0], hidden_size[0])
adver_HL_para = [HL_layer_1_W, HL_layer_1_b, HL_layer_2_W, HL_layer_2_b]
HL_layer_1=HiddenLayer(rng, input=domain_input, n_in=domain_input_size, n_out=hidden_size[0], W=HL_layer_1_W, b=HL_layer_1_b, activation=T.tanh)
HL_layer_2=HiddenLayer(rng, input=HL_layer_1.output, n_in=hidden_size[0], n_out=hidden_size[0], W=HL_layer_2_W, b=HL_layer_2_b, activation=T.tanh)
disc_para_W, disc_para_b = create_LR_para(rng,hidden_size[0],2)
conf_para_W, conf_para_b = create_LR_para(rng,hidden_size[0],2)
conf_para_W2, conf_para_b2 = create_LR_para(rng,hidden_size[0],2)
adver_LR_para = [disc_para_W, disc_para_b, conf_para_W, conf_para_b,conf_para_W2, conf_para_b2]
disc_layer_LR=LogisticRegression(rng, input=HL_layer_1.output, n_in=hidden_size[0], n_out=2, W=disc_para_W, b=disc_para_b)
conf_layer_LR=LogisticRegression(rng, input=HL_layer_2.output, n_in=hidden_size[0], n_out=2, W=conf_para_W, b=conf_para_b)
conf2_layer_LR=LogisticRegression(rng, input=HL_layer_2.output, n_in=hidden_size[0], n_out=2, W=conf_para_W2, b=conf_para_b2)
disc_loss = disc_layer_LR.negative_log_likelihood(domain_labels)
conf_loss = conf_layer_LR.negative_log_likelihood_specific_label(0)
conf_loss2 = conf2_layer_LR.negative_log_likelihood_specific_label(1)
adver_loss = disc_loss+conf_loss+conf_loss2
adver_para = adver_HL_para+adver_LR_para +NN_para
adver_updates = Gradient_Cost_Para(adver_loss,adver_para, learning_rate)
'''
SF classification
'''
LR_input = HL_layer_2.output#T.concatenate([sent_embeddings,sent_embeddings2, bow_emb], axis=1)
LR_input_size = hidden_size[0]
#classification layer, it is just mapping from a feature vector of size "hidden_size" to a vector of only two values: positive, negative
U_a = create_ensemble_para(rng, 12, LR_input_size) # the weight matrix hidden_size*2
LR_b = theano.shared(value=np.zeros((12,),dtype=theano.config.floatX),name='LR_b', borrow=True) #bias for each target class
LR_para=[U_a, LR_b]
layer_LR=LogisticRegression(rng, input=LR_input, n_in=LR_input_size, n_out=12, W=U_a, b=LR_b) #basically it is a multiplication between weight matrix and input feature vector
score_matrix = T.nnet.sigmoid(layer_LR.before_softmax) #batch * 12
prob_pos = T.where( labels < 1, 1.0-score_matrix, score_matrix)
loss = -T.mean(T.log(prob_pos))
# loss=layer_LR.negative_log_likelihood(labels) #for classification task, we usually used negative log likelihood as loss, the lower the better.
params = NN_para+adver_HL_para+LR_para # put all model parameters together
cost=loss+1e-4*((conv_W**2).sum()+(conv_W2**2).sum())
updates = Gradient_Cost_Para(cost,params, learning_rate)
'''
testing
'''
binarize_prob = T.where(score_matrix > 0.3, 1, 0)
#train_model = theano.function([sents_id_matrix, sents_mask, labels], cost, updates=updates, on_unused_input='ignore')
train_model = theano.function([sents_id_matrix, sents_mask, labels], cost, updates=updates, allow_input_downcast=True, on_unused_input='ignore')
comb_model = theano.function([sents_id_matrix, sents_mask, domain_labels], adver_loss, updates=adver_updates, allow_input_downcast=True, on_unused_input='ignore')
test_model = theano.function([sents_id_matrix, sents_mask], binarize_prob, allow_input_downcast=True, on_unused_input='ignore')
###############
# TRAIN MODEL #
###############
print '... training'
# early-stopping parameters
patience = 50000000000 # look as this many examples regardless
start_time = time.time()
mid_time = start_time
past_time= mid_time
epoch = 0
done_looping = False
n_train_batches=train_size/batch_size
train_batch_start=list(np.arange(n_train_batches)*batch_size)+[train_size-batch_size]
n_comb_batches=comb_size/batch_size
comb_batch_start=list(np.arange(n_comb_batches)*batch_size)+[comb_size-batch_size]
n_test_batches=test_size/batch_size
test_batch_start=list(np.arange(n_test_batches)*batch_size)+[test_size-batch_size]
# max_acc_dev=0.0
max_meanf1_test=0.0
max_weightf1_test=0.0
train_indices = range(train_size)
comb_indices = range(comb_size)
cost_i=0.0
adver_cost_i=0.0
while epoch < n_epochs:
epoch = epoch + 1
random.Random(100).shuffle(train_indices)
random.Random(100).shuffle(comb_indices)
iter_accu=0
for batch_id in train_batch_start: #for each batch
# iter means how many batches have been run, taking into loop
iter = (epoch - 1) * n_train_batches + iter_accu +1
iter_accu+=1
train_id_batch = train_indices[batch_id:batch_id+batch_size]
cost_i+= train_model(
train_sents[train_id_batch],
train_masks[train_id_batch],
train_labels[train_id_batch])
comb_id_batch = comb_indices[batch_id:batch_id+batch_size]
adver_cost_i+= comb_model(
comb_sents[comb_id_batch],
comb_masks[comb_id_batch],
comb_labels[comb_id_batch])
#after each 1000 batches, we test the performance of the model on all test data
if iter%20==0:
print 'Epoch ', epoch, 'iter ', iter, ' average cost: ', cost_i/iter , adver_cost_i/iter , 'uses ', (time.time()-past_time)/60.0, 'min'
past_time = time.time()
error_sum=0.0
all_pred_labels = []
all_gold_labels = []
for test_batch_id in test_batch_start: # for each test batch
pred_labels=test_model(
test_sents[test_batch_id:test_batch_id+batch_size],
test_masks[test_batch_id:test_batch_id+batch_size])
gold_labels = test_labels[test_batch_id:test_batch_id+batch_size]
# print 'pred_labels:', pred_labels
# print 'gold_labels;', gold_labels
all_pred_labels.append(pred_labels)
all_gold_labels.append(gold_labels)
all_pred_labels = np.concatenate(all_pred_labels)
all_gold_labels = np.concatenate(all_gold_labels)
test_mean_f1, test_weight_f1 =average_f1_two_array_by_col(all_pred_labels, all_gold_labels)
if test_weight_f1 > max_weightf1_test:
max_weightf1_test=test_weight_f1
if test_mean_f1 > max_meanf1_test:
max_meanf1_test=test_mean_f1
print '\t\t\t\t\t\t\t\tcurrent f1s:', test_mean_f1, test_weight_f1, '\t\tmax_f1:', max_meanf1_test, max_weightf1_test
print 'Epoch ', epoch, 'uses ', (time.time()-mid_time)/60.0, 'min'
mid_time = time.time()
#print 'Batch_size: ', update_freq
end_time = time.time()
print >> sys.stderr, ('The code for file ' +
os.path.split(__file__)[1] +
' ran for %.2fm' % ((end_time - start_time) / 60.))
'''task1: combine datasets from two sub-tasks'''
train_sents = train_p1_sents + train_p2_sents
train_masks = train_p1_masks + train_p2_masks
train_labels = train_p1_labels + train_p2_labels
# all_sentences, all_masks, all_labels, word2id=load_BBN_multi_labels_dataset(maxlen=config['maxSentLen'])
label_sent, label_mask = load_SF_type_descriptions(
word2id, config['type_size'], config['describ_max_len'])
# emb_root = '/scratch/wyin3/dickens_save_dataset/LORELEI/multi-lingual-emb/Uyghur/'
# print('loading bilingual embeddings....')
# word2vec=load_fasttext_multiple_word2vec_given_file([emb_root+'vectors-en.txt',emb_root+'vectors-ug.txt'], 300)
emb_root = '/scratch/wyin3/dickens_save_dataset/LORELEI/multi-lingual-emb/2018-il9-il10/multi-emb/'
# print('loading bilingual embeddings....')
if is_train:
word2vec = load_fasttext_multiple_word2vec_given_file([
emb_root + '100k-ENG-multicca.300.ENG.txt',
emb_root + '100k-IL10-multicca.d300.IL10.txt'
], 300)
vocab_size = len(word2id) + 1
rand_values = np.random.RandomState(1234).normal(
0.0, 0.01,
(vocab_size,
config['emb_size'])) #generate a matrix by Gaussian distribution
rand_values[0] = np.array(np.zeros(config['emb_size']), dtype=np.float32)
id2word = {y: x for x, y in word2id.items()}
if is_train:
rand_values = load_word2vec_to_init(rand_values, id2word, word2vec)
embeddings = torch.Tensor(rand_values)
print("build model...")
model, loss_function, optimizer = build_model(config['emb_size'],
def evaluate_lenet5(learning_rate=0.01, n_epochs=100, emb_size=40, batch_size=50, describ_max_len=20, type_size=12,filter_size=[3,5], maxSentLen=100, hidden_size=[300,300]):
model_options = locals().copy()
print "model options", model_options
emb_root = '/save/wenpeng/datasets/LORELEI/multi-lingual-emb/'
seed=1234
np.random.seed(seed)
rng = np.random.RandomState(seed) #random seed, control the model generates the same results
srng = T.shared_randomstreams.RandomStreams(rng.randint(seed))
all_sentences, all_masks, all_labels, word2id=load_BBN_multi_labels_dataset(maxlen=maxSentLen) #minlen, include one label, at least one word in the sentence
label_sent, label_mask = load_SF_type_descriptions(word2id, type_size, describ_max_len)
label_sent=np.asarray(label_sent, dtype='int32')
label_mask=np.asarray(label_mask, dtype=theano.config.floatX)
train_sents=np.asarray(all_sentences[0], dtype='int32')
train_masks=np.asarray(all_masks[0], dtype=theano.config.floatX)
train_labels=np.asarray(all_labels[0], dtype='int32')
train_size=len(train_labels)
dev_sents=np.asarray(all_sentences[1], dtype='int32')
dev_masks=np.asarray(all_masks[1], dtype=theano.config.floatX)
dev_labels=np.asarray(all_labels[1], dtype='int32')
dev_size=len(dev_labels)
test_sents=np.asarray(all_sentences[2], dtype='int32')
test_masks=np.asarray(all_masks[2], dtype=theano.config.floatX)
test_labels=np.asarray(all_labels[2], dtype='int32')
test_size=len(test_labels)
vocab_size= len(word2id)+1 # add one zero pad index
rand_values=rng.normal(0.0, 0.01, (vocab_size, emb_size)) #generate a matrix by Gaussian distribution
rand_values[0]=np.array(np.zeros(emb_size),dtype=theano.config.floatX)
id2word = {y:x for x,y in word2id.iteritems()}
word2vec=load_fasttext_multiple_word2vec_given_file([emb_root+'IL5-cca-wiki-lorelei-d40.eng.vec',emb_root+'IL5-cca-wiki-lorelei-d40.IL5.vec'], 40)
rand_values=load_word2vec_to_init(rand_values, id2word, word2vec)
embeddings=theano.shared(value=np.array(rand_values,dtype=theano.config.floatX), borrow=True) #wrap up the python variable "rand_values" into theano variable
#now, start to build the input form of the model
sents_id_matrix=T.imatrix('sents_id_matrix')
sents_mask=T.fmatrix('sents_mask')
labels=T.imatrix('labels') #batch*12
des_id_matrix = T.imatrix()
des_mask = T.fmatrix()
######################
# BUILD ACTUAL MODEL #
######################
print '... building the model'
common_input=embeddings[sents_id_matrix.flatten()].reshape((batch_size,maxSentLen, emb_size)).dimshuffle(0,2,1) #the input format can be adapted into CNN or GRU or LSTM
repeat_common_input = T.repeat(normalize_tensor3_colwise(common_input), type_size, axis=0) #(batch_size*type_size, emb_size, maxsentlen)
des_input=embeddings[des_id_matrix.flatten()].reshape((type_size,describ_max_len, emb_size)).dimshuffle(0,2,1)
repeat_des_input = T.tile(normalize_tensor3_colwise(des_input), (batch_size,1,1))#(batch_size*type_size, emb_size, maxsentlen)
fine_grained_cosine = T.batched_dot(repeat_common_input.dimshuffle(0,2,1),repeat_des_input) #(batch_size*type_size,maxsentlen,describ_max_len)
fine_grained_cosine_to_matrix = fine_grained_cosine.reshape((batch_size*type_size,maxSentLen*describ_max_len))
sort_fine_grained_cosine_to_matrix = T.sort(fine_grained_cosine_to_matrix, axis=1)
top_5_simi = sort_fine_grained_cosine_to_matrix[:,-30:] # (batch_size*type_size, 5)
max_fine_grained_cosine = T.mean(top_5_simi, axis=1)
cosine_scores = max_fine_grained_cosine.reshape((batch_size, type_size))
# score_matrix = T.nnet.sigmoid(cosine_scores) #(batch_size, type_size)
score_matrix = T.nnet.sigmoid(cosine_scores)
prob_pos = T.where( labels < 1, 1.0-score_matrix, score_matrix)
loss = -T.mean(T.log(prob_pos))
# loss=layer_LR.negative_log_likelihood(labels) #for classification task, we usually used negative log likelihood as loss, the lower the better.
'''
do not update emb, so that can generalize from english to ils
'''
params = []#+NN_para+LR_para # put all model parameters together
cost=loss#+1e-4*((conv_W**2).sum()+(conv_W2**2).sum())
updates = Gradient_Cost_Para(cost,params, learning_rate)
'''
testing
'''
binarize_prob = T.where(cosine_scores > 0.3, 1, 0)
#train_model = theano.function([sents_id_matrix, sents_mask, labels], cost, updates=updates, on_unused_input='ignore')
train_model = theano.function([sents_id_matrix, sents_mask, labels, des_id_matrix, des_mask], cost, updates=updates, allow_input_downcast=True, on_unused_input='ignore')
# dev_model = theano.function([sents_id_matrix, sents_mask, labels], layer_LR.errors(labels), allow_input_downcast=True, on_unused_input='ignore')
test_model = theano.function([sents_id_matrix, sents_mask, des_id_matrix, des_mask], binarize_prob, allow_input_downcast=True, on_unused_input='ignore')
###############
# TRAIN MODEL #
###############
print '... training'
# early-stopping parameters
patience = 50000000000 # look as this many examples regardless
start_time = time.time()
mid_time = start_time
past_time= mid_time
epoch = 0
done_looping = False
n_train_batches=train_size/batch_size
train_batch_start=list(np.arange(n_train_batches)*batch_size)+[train_size-batch_size]
# n_dev_batches=dev_size/batch_size
# dev_batch_start=list(np.arange(n_dev_batches)*batch_size)+[dev_size-batch_size]
n_test_batches=test_size/batch_size
test_batch_start=list(np.arange(n_test_batches)*batch_size)+[test_size-batch_size]
# max_acc_dev=0.0
max_meanf1_test=0.0
max_weightf1_test=0.0
train_indices = range(train_size)
cost_i=0.0
while epoch < n_epochs:
epoch = epoch + 1
random.Random(100).shuffle(train_indices)
iter_accu=0
for batch_id in train_batch_start: #for each batch
# iter means how many batches have been run, taking into loop
iter = (epoch - 1) * n_train_batches + iter_accu +1
iter_accu+=1
train_id_batch = train_indices[batch_id:batch_id+batch_size]
cost_i+= train_model(
train_sents[train_id_batch],
train_masks[train_id_batch],
train_labels[train_id_batch],
label_sent,
label_mask)
#after each 1000 batches, we test the performance of the model on all test data
if iter%20==0:
print 'Epoch ', epoch, 'iter '+str(iter)+' average cost: '+str(cost_i/iter), 'uses ', (time.time()-past_time)/60.0, 'min'
past_time = time.time()
error_sum=0.0
all_pred_labels = []
all_gold_labels = []
for test_batch_id in test_batch_start: # for each test batch
pred_labels=test_model(
test_sents[test_batch_id:test_batch_id+batch_size],
test_masks[test_batch_id:test_batch_id+batch_size],
label_sent,
label_mask)
gold_labels = test_labels[test_batch_id:test_batch_id+batch_size]
# print 'pred_labels:', pred_labels
# print 'gold_labels;', gold_labels
all_pred_labels.append(pred_labels)
all_gold_labels.append(gold_labels)
all_pred_labels = np.concatenate(all_pred_labels)
all_gold_labels = np.concatenate(all_gold_labels)
test_mean_f1, test_weight_f1 =average_f1_two_array_by_col(all_pred_labels, all_gold_labels)
if test_weight_f1 > max_weightf1_test:
max_weightf1_test=test_weight_f1
if test_mean_f1 > max_meanf1_test:
max_meanf1_test=test_mean_f1
print '\t\t\t\t\t\t\t\tcurrent f1s:', test_mean_f1, test_weight_f1, '\t\tmax_f1:', max_meanf1_test, max_weightf1_test
print 'Epoch ', epoch, 'uses ', (time.time()-mid_time)/60.0, 'min'
mid_time = time.time()
#print 'Batch_size: ', update_freq
end_time = time.time()
print >> sys.stderr, ('The code for file ' +
os.path.split(__file__)[1] +
' ran for %.2fm' % ((end_time - start_time) / 60.))
def evaluate_lenet5(learning_rate=0.01,
n_epochs=4,
emb_size=300,
batch_size=50,
describ_max_len=20,
type_size=12,
filter_size=[3, 5],
maxSentLen=200,
hidden_size=[300, 300]):
model_options = locals().copy()
print "model options", model_options
emb_root = '/save/wenpeng/datasets/LORELEI/multi-lingual-emb/2018-il9-il10/multi-emb/'
test_file_path = '/save/wenpeng/datasets/LORELEI/il9/il9-setE-as-test-input_ner_filtered_w2.txt'
output_file_path = '/save/wenpeng/datasets/LORELEI/il9/il9_system_output_concMT_BBN_NI_epoch4.json'
seed = 1234
np.random.seed(seed)
rng = np.random.RandomState(
seed) #random seed, control the model generates the same results
srng = T.shared_randomstreams.RandomStreams(rng.randint(seed))
word2id = {}
# all_sentences, all_masks, all_labels, all_other_labels, word2id=load_BBN_il5Trans_il5_dataset(maxlen=maxSentLen) #minlen, include one label, at least one word in the sentence
train_p1_sents, train_p1_masks, train_p1_labels, word2id = load_trainingData_types(
word2id, maxSentLen)
train_p2_sents, train_p2_masks, train_p2_labels, train_p2_other_labels, word2id = load_trainingData_types_plus_others(
word2id, maxSentLen)
test_sents, test_masks, test_lines, word2id = load_official_testData_il_and_MT(
word2id, maxSentLen, test_file_path)
label_sent, label_mask = load_SF_type_descriptions(word2id, type_size,
describ_max_len)
label_sent = np.asarray(label_sent, dtype='int32')
label_mask = np.asarray(label_mask, dtype=theano.config.floatX)
train_p1_sents = np.asarray(train_p1_sents, dtype='int32')
train_p1_masks = np.asarray(train_p1_masks, dtype=theano.config.floatX)
train_p1_labels = np.asarray(train_p1_labels, dtype='int32')
train_p1_size = len(train_p1_labels)
train_p2_sents = np.asarray(train_p2_sents, dtype='int32')
train_p2_masks = np.asarray(train_p2_masks, dtype=theano.config.floatX)
train_p2_labels = np.asarray(train_p2_labels, dtype='int32')
train_p2_other_labels = np.asarray(train_p2_other_labels, dtype='int32')
train_p2_size = len(train_p2_labels)
'''
combine train_p1 and train_p2
'''
train_sents = np.concatenate([train_p1_sents, train_p2_sents], axis=0)
train_masks = np.concatenate([train_p1_masks, train_p2_masks], axis=0)
train_labels = np.concatenate([train_p1_labels, train_p2_labels], axis=0)
train_size = train_p1_size + train_p2_size
test_sents = np.asarray(test_sents, dtype='int32')
test_masks = np.asarray(test_masks, dtype=theano.config.floatX)
# test_labels=np.asarray(all_labels[2], dtype='int32')
test_size = len(test_sents)
vocab_size = len(word2id) + 1 # add one zero pad index
rand_values = rng.normal(
0.0, 0.01,
(vocab_size, emb_size)) #generate a matrix by Gaussian distribution
rand_values[0] = np.array(np.zeros(emb_size), dtype=theano.config.floatX)
id2word = {y: x for x, y in word2id.iteritems()}
word2vec = load_fasttext_multiple_word2vec_given_file([
emb_root + '100k-ENG-multicca.300.ENG.txt',
emb_root + '100k-IL9-multicca.d300.IL9.txt'
], 300)
rand_values = load_word2vec_to_init(rand_values, id2word, word2vec)
embeddings = theano.shared(
value=np.array(rand_values, dtype=theano.config.floatX), borrow=True
) #wrap up the python variable "rand_values" into theano variable
#now, start to build the input form of the model
sents_id_matrix = T.imatrix('sents_id_matrix')
sents_mask = T.fmatrix('sents_mask')
labels = T.imatrix('labels') #batch*12
other_labels = T.imatrix() #batch*4
des_id_matrix = T.imatrix()
des_mask = T.fmatrix()
######################
# BUILD ACTUAL MODEL #
######################
print '... building the model'
common_input = embeddings[sents_id_matrix.flatten()].reshape(
(batch_size, maxSentLen, emb_size)).dimshuffle(
0, 2, 1) #the input format can be adapted into CNN or GRU or LSTM
bow_emb = T.sum(common_input * sents_mask.dimshuffle(0, 'x', 1), axis=2)
repeat_common_input = T.repeat(
normalize_tensor3_colwise(common_input), type_size,
axis=0) #(batch_size*type_size, emb_size, maxsentlen)
des_input = embeddings[des_id_matrix.flatten()].reshape(
(type_size, describ_max_len, emb_size)).dimshuffle(0, 2, 1)
bow_des = T.sum(des_input * des_mask.dimshuffle(0, 'x', 1),
axis=2) #(tyope_size, emb_size)
repeat_des_input = T.tile(
normalize_tensor3_colwise(des_input),
(batch_size, 1, 1)) #(batch_size*type_size, emb_size, maxsentlen)
conv_W, conv_b = create_conv_para(rng,
filter_shape=(hidden_size[0], 1,
emb_size, filter_size[0]))
conv_W2, conv_b2 = create_conv_para(rng,
filter_shape=(hidden_size[0], 1,
emb_size,
filter_size[1]))
multiCNN_para = [conv_W, conv_b, conv_W2, conv_b2]
conv_att_W, conv_att_b = create_conv_para(rng,
filter_shape=(hidden_size[0], 1,
emb_size,
filter_size[0]))
conv_W_context, conv_b_context = create_conv_para(
rng, filter_shape=(hidden_size[0], 1, emb_size, 1))
conv_att_W2, conv_att_b2 = create_conv_para(rng,
filter_shape=(hidden_size[0],
1, emb_size,
filter_size[1]))
conv_W_context2, conv_b_context2 = create_conv_para(
rng, filter_shape=(hidden_size[0], 1, emb_size, 1))
ACNN_para = [
conv_att_W, conv_att_b, conv_W_context, conv_att_W2, conv_att_b2,
conv_W_context2
]
'''
multi-CNN
'''
conv_model = Conv_with_Mask(
rng,
input_tensor3=common_input,
mask_matrix=sents_mask,
image_shape=(batch_size, 1, emb_size, maxSentLen),
filter_shape=(hidden_size[0], 1, emb_size, filter_size[0]),
W=conv_W,
b=conv_b
) #mutiple mask with the conv_out to set the features by UNK to zero
sent_embeddings = conv_model.maxpool_vec #(batch_size, hidden_size) # each sentence then have an embedding of length hidden_size
conv_model2 = Conv_with_Mask(
rng,
input_tensor3=common_input,
mask_matrix=sents_mask,
image_shape=(batch_size, 1, emb_size, maxSentLen),
filter_shape=(hidden_size[0], 1, emb_size, filter_size[1]),
W=conv_W2,
b=conv_b2
) #mutiple mask with the conv_out to set the features by UNK to zero
sent_embeddings2 = conv_model2.maxpool_vec #(batch_size, hidden_size) # each sentence then have an embedding of length hidden_size
'''
GRU
'''
U1, W1, b1 = create_GRU_para(rng, emb_size, hidden_size[0])
GRU_NN_para = [
U1, W1, b1
] #U1 includes 3 matrices, W1 also includes 3 matrices b1 is bias
# gru_input = common_input.dimshuffle((0,2,1)) #gru requires input (batch_size, emb_size, maxSentLen)
gru_layer = GRU_Batch_Tensor_Input_with_Mask(common_input, sents_mask,
hidden_size[0], U1, W1, b1)
gru_sent_embeddings = gru_layer.output_sent_rep # (batch_size, hidden_size)
'''
ACNN
'''
attentive_conv_layer = Attentive_Conv_for_Pair(
rng,
origin_input_tensor3=common_input,
origin_input_tensor3_r=common_input,
input_tensor3=common_input,
input_tensor3_r=common_input,
mask_matrix=sents_mask,
mask_matrix_r=sents_mask,
image_shape=(batch_size, 1, emb_size, maxSentLen),
image_shape_r=(batch_size, 1, emb_size, maxSentLen),
filter_shape=(hidden_size[0], 1, emb_size, filter_size[0]),
filter_shape_context=(hidden_size[0], 1, emb_size, 1),
W=conv_att_W,
b=conv_att_b,
W_context=conv_W_context,
b_context=conv_b_context)
sent_att_embeddings = attentive_conv_layer.attentive_maxpool_vec_l
attentive_conv_layer2 = Attentive_Conv_for_Pair(
rng,
origin_input_tensor3=common_input,
origin_input_tensor3_r=common_input,
input_tensor3=common_input,
input_tensor3_r=common_input,
mask_matrix=sents_mask,
mask_matrix_r=sents_mask,
image_shape=(batch_size, 1, emb_size, maxSentLen),
image_shape_r=(batch_size, 1, emb_size, maxSentLen),
filter_shape=(hidden_size[0], 1, emb_size, filter_size[1]),
filter_shape_context=(hidden_size[0], 1, emb_size, 1),
W=conv_att_W2,
b=conv_att_b2,
W_context=conv_W_context2,
b_context=conv_b_context2)
sent_att_embeddings2 = attentive_conv_layer2.attentive_maxpool_vec_l
'''
cross-DNN-dataless
'''
#first map label emb into hidden space
HL_layer_1_W, HL_layer_1_b = create_HiddenLayer_para(
rng, emb_size, hidden_size[0])
HL_layer_1_params = [HL_layer_1_W, HL_layer_1_b]
HL_layer_1 = HiddenLayer(rng,
input=bow_des,
n_in=emb_size,
n_out=hidden_size[0],
W=HL_layer_1_W,
b=HL_layer_1_b,
activation=T.tanh)
des_rep_hidden = HL_layer_1.output #(type_size, hidden_size)
dot_dnn_dataless_1 = T.tanh(sent_embeddings.dot(
des_rep_hidden.T)) #(batch_size, type_size)
dot_dnn_dataless_2 = T.tanh(sent_embeddings2.dot(des_rep_hidden.T))
'''
dataless cosine
'''
cosine_scores = normalize_matrix_rowwise(bow_emb).dot(
normalize_matrix_rowwise(bow_des).T)
cosine_score_matrix = T.nnet.sigmoid(
cosine_scores) #(batch_size, type_size)
'''
dataless top-30 fine grained cosine
'''
fine_grained_cosine = T.batched_dot(
repeat_common_input.dimshuffle(0, 2, 1),
repeat_des_input) #(batch_size*type_size,maxsentlen,describ_max_len)
fine_grained_cosine_to_matrix = fine_grained_cosine.reshape(
(batch_size * type_size, maxSentLen * describ_max_len))
sort_fine_grained_cosine_to_matrix = T.sort(fine_grained_cosine_to_matrix,
axis=1)
top_k_simi = sort_fine_grained_cosine_to_matrix[:,
-30:] # (batch_size*type_size, 5)
max_fine_grained_cosine = T.mean(top_k_simi, axis=1)
top_k_cosine_scores = max_fine_grained_cosine.reshape(
(batch_size, type_size))
top_k_score_matrix = T.nnet.sigmoid(top_k_cosine_scores)
acnn_LR_input = T.concatenate([
dot_dnn_dataless_1, dot_dnn_dataless_2, cosine_score_matrix,
top_k_score_matrix, sent_embeddings, sent_embeddings2,
gru_sent_embeddings, sent_att_embeddings, sent_att_embeddings2, bow_emb
],
axis=1)
acnn_LR_input_size = hidden_size[0] * 5 + emb_size + 4 * type_size
#classification layer, it is just mapping from a feature vector of size "hidden_size" to a vector of only two values: positive, negative
acnn_U_a, acnn_LR_b = create_LR_para(rng, acnn_LR_input_size, 12)
acnn_LR_para = [acnn_U_a, acnn_LR_b]
acnn_layer_LR = LogisticRegression(
rng,
input=acnn_LR_input,
n_in=acnn_LR_input_size,
n_out=12,
W=acnn_U_a,
b=acnn_LR_b
) #basically it is a multiplication between weight matrix and input feature vector
acnn_score_matrix = T.nnet.sigmoid(
acnn_layer_LR.before_softmax) #batch * 12
acnn_prob_pos = T.where(labels < 1, 1.0 - acnn_score_matrix,
acnn_score_matrix)
acnn_loss = -T.mean(T.log(acnn_prob_pos))
acnn_other_U_a, acnn_other_LR_b = create_LR_para(rng, acnn_LR_input_size,
16)
acnn_other_LR_para = [acnn_other_U_a, acnn_other_LR_b]
acnn_other_layer_LR = LogisticRegression(rng,
input=acnn_LR_input,
n_in=acnn_LR_input_size,
n_out=16,
W=acnn_other_U_a,
b=acnn_other_LR_b)
acnn_other_prob_matrix = T.nnet.softmax(
acnn_other_layer_LR.before_softmax.reshape((batch_size * 4, 4)))
acnn_other_prob_tensor3 = acnn_other_prob_matrix.reshape(
(batch_size, 4, 4))
acnn_other_prob = acnn_other_prob_tensor3[
T.repeat(T.arange(batch_size), 4),
T.tile(T.arange(4), (batch_size)),
other_labels.flatten()]
acnn_other_field_loss = -T.mean(T.log(acnn_other_prob))
params = multiCNN_para + GRU_NN_para + ACNN_para + acnn_LR_para + HL_layer_1_params # put all model parameters together
cost = acnn_loss + 1e-4 * ((conv_W**2).sum() + (conv_W2**2).sum() +
(conv_att_W**2).sum() + (conv_att_W2**2).sum())
updates = Gradient_Cost_Para(cost, params, learning_rate)
other_paras = params + acnn_other_LR_para
cost_other = cost + acnn_other_field_loss
other_updates = Gradient_Cost_Para(cost_other, other_paras, learning_rate)
'''
testing
'''
ensemble_NN_scores = acnn_score_matrix #T.max(T.concatenate([att_score_matrix.dimshuffle('x',0,1), score_matrix.dimshuffle('x',0,1), acnn_score_matrix.dimshuffle('x',0,1)],axis=0),axis=0)
# '''
# majority voting, does not work
# '''
# binarize_NN = T.where(ensemble_NN_scores > 0.5, 1, 0)
# binarize_dataless = T.where(cosine_score_matrix > 0.5, 1, 0)
# binarize_dataless_finegrained = T.where(top_k_score_matrix > 0.5, 1, 0)
# binarize_conc = T.concatenate([binarize_NN.dimshuffle('x',0,1), binarize_dataless.dimshuffle('x',0,1),binarize_dataless_finegrained.dimshuffle('x',0,1)],axis=0)
# sum_binarize_conc = T.sum(binarize_conc,axis=0)
# binarize_prob = T.where(sum_binarize_conc > 0.0, 1, 0)
# '''
# sum up prob, works
# '''
# ensemble_scores_1 = 0.6*ensemble_NN_scores+0.4*top_k_score_matrix
# binarize_prob = T.where(ensemble_scores_1 > 0.3, 1, 0)
'''
sum up prob, works
'''
ensemble_scores = ensemble_NN_scores #0.6*ensemble_NN_scores+0.4*0.5*(cosine_score_matrix+top_k_score_matrix)
binarize_prob = T.where(ensemble_scores > 0.3, 1, 0)
'''
test for other fields
'''
sum_tensor3 = acnn_other_prob_tensor3 #(batch, 4, 3)
#train_model = theano.function([sents_id_matrix, sents_mask, labels], cost, updates=updates, on_unused_input='ignore')
train_p1_model = theano.function(
[sents_id_matrix, sents_mask, labels, des_id_matrix, des_mask],
cost,
updates=updates,
allow_input_downcast=True,
on_unused_input='ignore')
train_p2_model = theano.function([
sents_id_matrix, sents_mask, labels, des_id_matrix, des_mask,
other_labels
],
cost_other,
updates=other_updates,
allow_input_downcast=True,
on_unused_input='ignore')
test_model = theano.function(
[sents_id_matrix, sents_mask, des_id_matrix, des_mask],
[binarize_prob, ensemble_scores, sum_tensor3],
allow_input_downcast=True,
on_unused_input='ignore')
###############
# TRAIN MODEL #
###############
print '... training'
# early-stopping parameters
patience = 50000000000 # look as this many examples regardless
start_time = time.time()
mid_time = start_time
past_time = mid_time
epoch = 0
done_looping = False
n_train_batches = train_size / batch_size
train_batch_start = list(
np.arange(n_train_batches) * batch_size) + [train_size - batch_size]
n_train_p2_batches = train_p2_size / batch_size
train_p2_batch_start = list(np.arange(n_train_p2_batches) *
batch_size) + [train_p2_size - batch_size]
n_test_batches = test_size / batch_size
n_test_remain = test_size % batch_size
test_batch_start = list(
np.arange(n_test_batches) * batch_size) + [test_size - batch_size]
train_p2_batch_start_set = set(train_p2_batch_start)
# max_acc_dev=0.0
# max_meanf1_test=0.0
# max_weightf1_test=0.0
train_indices = range(train_size)
train_p2_indices = range(train_p2_size)
cost_i = 0.0
other_cost_i = 0.0
min_mean_frame = 100.0
while epoch < n_epochs:
epoch = epoch + 1
random.Random(100).shuffle(train_indices)
random.Random(100).shuffle(train_p2_indices)
iter_accu = 0
for batch_id in train_batch_start: #for each batch
# iter means how many batches have been run, taking into loop
iter = (epoch - 1) * n_train_batches + iter_accu + 1
iter_accu += 1
train_id_batch = train_indices[batch_id:batch_id + batch_size]
cost_i += train_p1_model(train_sents[train_id_batch],
train_masks[train_id_batch],
train_labels[train_id_batch], label_sent,
label_mask)
if batch_id in train_p2_batch_start_set:
train_p2_id_batch = train_p2_indices[batch_id:batch_id +
batch_size]
other_cost_i += train_p2_model(
train_p2_sents[train_p2_id_batch],
train_p2_masks[train_p2_id_batch],
train_p2_labels[train_p2_id_batch], label_sent, label_mask,
train_p2_other_labels[train_p2_id_batch])
# else:
# random_batch_id = random.choice(train_p2_batch_start)
# train_p2_id_batch = train_p2_indices[random_batch_id:random_batch_id+batch_size]
# other_cost_i+=train_p2_model(
# train_p2_sents[train_p2_id_batch],
# train_p2_masks[train_p2_id_batch],
# train_p2_labels[train_p2_id_batch],
# label_sent,
# label_mask,
# train_p2_other_labels[train_p2_id_batch]
# )
#after each 1000 batches, we test the performance of the model on all test data
if iter % 20 == 0:
print 'Epoch ', epoch, 'iter ' + str(
iter) + ' average cost: ' + str(cost_i / iter), str(
other_cost_i /
iter), 'uses ', (time.time() - past_time) / 60.0, 'min'
past_time = time.time()
pred_types = []
pred_confs = []
pred_others = []
for i, test_batch_id in enumerate(
test_batch_start): # for each test batch
pred_types_i, pred_conf_i, pred_fields_i = test_model(
test_sents[test_batch_id:test_batch_id + batch_size],
test_masks[test_batch_id:test_batch_id + batch_size],
label_sent, label_mask)
if i < len(test_batch_start) - 1:
pred_types.append(pred_types_i)
pred_confs.append(pred_conf_i)
pred_others.append(pred_fields_i)
else:
pred_types.append(pred_types_i[-n_test_remain:])
pred_confs.append(pred_conf_i[-n_test_remain:])
pred_others.append(pred_fields_i[-n_test_remain:])
pred_types = np.concatenate(pred_types, axis=0)
pred_confs = np.concatenate(pred_confs, axis=0)
pred_others = np.concatenate(pred_others, axis=0)
mean_frame = generate_2018_official_output(
test_lines, output_file_path, pred_types, pred_confs,
pred_others, min_mean_frame)
if mean_frame < min_mean_frame:
min_mean_frame = mean_frame
print '\t\t\t test over, min_mean_frame:', min_mean_frame
print 'Epoch ', epoch, 'uses ', (time.time() - mid_time) / 60.0, 'min'
mid_time = time.time()
#print 'Batch_size: ', update_freq
end_time = time.time()
print >> sys.stderr, ('The code for file ' + os.path.split(__file__)[1] +
' ran for %.2fm' % ((end_time - start_time) / 60.))
def evaluate_lenet5(learning_rate=0.01,
n_epochs=100,
emb_size=40,
batch_size=50,
filter_size=[3, 5],
maxSentLen=100,
hidden_size=[300, 300]):
model_options = locals().copy()
print "model options", model_options
emb_root = '/save/wenpeng/datasets/LORELEI/multi-lingual-emb/'
seed = 1234
np.random.seed(seed)
rng = np.random.RandomState(
seed) #random seed, control the model generates the same results
srng = T.shared_randomstreams.RandomStreams(rng.randint(seed))
all_sentences, all_masks, all_labels, word2id = load_reliefweb_il5_12_multilabel(
maxlen=maxSentLen
) #minlen, include one label, at least one word in the sentence
train_sents = np.asarray(all_sentences[0], dtype='int32')
train_masks = np.asarray(all_masks[0], dtype=theano.config.floatX)
train_labels = np.asarray(all_labels[0], dtype='int32')
train_size = len(train_labels)
# dev_sents=all_sentences[1]
# dev_masks=all_masks[1]
# dev_labels=all_labels[1]
# dev_size=len(dev_labels)
test_sents = np.asarray(all_sentences[2], dtype='int32')
test_masks = np.asarray(all_masks[2], dtype=theano.config.floatX)
test_labels = np.asarray(all_labels[2], dtype='int32')
test_size = len(test_labels)
vocab_size = len(word2id) + 1 # add one zero pad index
rand_values = rng.normal(
0.0, 0.01,
(vocab_size, emb_size)) #generate a matrix by Gaussian distribution
#here, we leave code for loading word2vec to initialize words
rand_values[0] = np.array(np.zeros(emb_size), dtype=theano.config.floatX)
id2word = {y: x for x, y in word2id.iteritems()}
word2vec = load_fasttext_multiple_word2vec_given_file([
emb_root + 'IL5-cca-wiki-lorelei-d40.eng.vec',
emb_root + 'IL5-cca-wiki-lorelei-d40.IL5.vec'
], 40)
rand_values = load_word2vec_to_init(rand_values, id2word, word2vec)
embeddings = theano.shared(
value=np.array(rand_values, dtype=theano.config.floatX), borrow=True
) #wrap up the python variable "rand_values" into theano variable
#now, start to build the input form of the model
sents_id_matrix = T.imatrix('sents_id_matrix')
sents_mask = T.fmatrix('sents_mask')
labels = T.imatrix('labels')
######################
# BUILD ACTUAL MODEL #
######################
print '... building the model'
common_input = embeddings[sents_id_matrix.flatten()].reshape(
(batch_size, maxSentLen, emb_size)).dimshuffle(
0, 2, 1) #the input format can be adapted into CNN or GRU or LSTM
bow_emb = T.sum(common_input * sents_mask.dimshuffle(0, 'x', 1), axis=2)
# conv_W, conv_b=create_conv_para(rng, filter_shape=(hidden_size[0], 1, emb_size, filter_size[0]))
# conv_W2, conv_b2=create_conv_para(rng, filter_shape=(hidden_size[0], 1, emb_size, filter_size[1]))
# NN_para = [conv_W, conv_b, conv_W2, conv_b2]
# conv_model = Conv_with_Mask(rng, input_tensor3=common_input,
# mask_matrix = sents_mask,
# image_shape=(batch_size, 1, emb_size, maxSentLen),
# filter_shape=(hidden_size[0], 1, emb_size, filter_size[0]), W=conv_W, b=conv_b) #mutiple mask with the conv_out to set the features by UNK to zero
# sent_embeddings=conv_model.maxpool_vec #(batch_size, hidden_size) # each sentence then have an embedding of length hidden_size
#
# conv_model2 = Conv_with_Mask(rng, input_tensor3=common_input,
# mask_matrix = sents_mask,
# image_shape=(batch_size, 1, emb_size, maxSentLen),
# filter_shape=(hidden_size[0], 1, emb_size, filter_size[1]), W=conv_W2, b=conv_b2) #mutiple mask with the conv_out to set the features by UNK to zero
# sent_embeddings2=conv_model2.maxpool_vec #(batch_size, hidden_size) # each sentence then have an embedding of length hidden_size
#
# LR_input = T.concatenate([sent_embeddings,sent_embeddings2, bow_emb], axis=1)
# LR_input_size = hidden_size[0]*2+emb_size
# #classification layer, it is just mapping from a feature vector of size "hidden_size" to a vector of only two values: positive, negative
# U_a = create_ensemble_para(rng, 8, LR_input_size) # the weight matrix hidden_size*2
# LR_b = theano.shared(value=np.zeros((8,),dtype=theano.config.floatX),name='LR_b', borrow=True) #bias for each target class
# LR_para=[U_a, LR_b]
# layer_LR=LogisticRegression(rng, input=LR_input, n_in=LR_input_size, n_out=8, W=U_a, b=LR_b) #basically it is a multiplication between weight matrix and input feature vector
# score_matrix = T.nnet.sigmoid(layer_LR.before_softmax) #batch * 8
# sub_labels = labels[:,:8]
# prob_pos = T.where( sub_labels < 1, 1.0-score_matrix, score_matrix)
# loss = -T.mean(T.log(prob_pos))
'''
GRU
'''
U1, W1, b1 = create_GRU_para(rng, emb_size, hidden_size[0])
GRU_NN_para = [
U1, W1, b1
] #U1 includes 3 matrices, W1 also includes 3 matrices b1 is bias
# gru_input = common_input.dimshuffle((0,2,1)) #gru requires input (batch_size, emb_size, maxSentLen)
gru_layer = GRU_Batch_Tensor_Input_with_Mask(common_input, sents_mask,
hidden_size[0], U1, W1, b1)
gru_sent_embeddings = gru_layer.maxpooling_vec #gru_layer.output_sent_rep # (batch_size, hidden_size)
LR_att_input = gru_sent_embeddings #T.concatenate([gru_sent_embeddings, bow_emb], axis=1)
LR_att_input_size = hidden_size[0]
#classification layer, it is just mapping from a feature vector of size "hidden_size" to a vector of only two values: positive, negative
U_att_a = create_ensemble_para(
rng, 8, LR_att_input_size) # the weight matrix hidden_size*2
LR_att_b = theano.shared(value=np.zeros((8, ), dtype=theano.config.floatX),
name='LR_b',
borrow=True) #bias for each target class
LR_att_para = [U_att_a, LR_att_b]
layer_att_LR = LogisticRegression(
rng,
input=LR_att_input,
n_in=LR_att_input_size,
n_out=8,
W=U_att_a,
b=LR_att_b
) #basically it is a multiplication between weight matrix and input feature vector
att_score_matrix = T.nnet.sigmoid(layer_att_LR.before_softmax) #batch * 12
sub_labels = labels[:, :8]
att_prob_pos = T.where(sub_labels < 1, 1.0 - att_score_matrix,
att_score_matrix)
att_loss = -T.mean(T.log(att_prob_pos))
params = GRU_NN_para + LR_att_para # put all model parameters together
cost = att_loss #+Div_reg*diversify_reg#+L2_weight*L2_reg
updates = Gradient_Cost_Para(cost, params, learning_rate)
'''
testing
'''
binarize_prob = T.where(att_score_matrix > 0.3, 1, 0)
#train_model = theano.function([sents_id_matrix, sents_mask, labels], cost, updates=updates, on_unused_input='ignore')
train_model = theano.function([sents_id_matrix, sents_mask, labels],
cost,
updates=updates,
allow_input_downcast=True,
on_unused_input='ignore')
# dev_model = theano.function([sents_id_matrix, sents_mask, labels], layer_LR.errors(labels), allow_input_downcast=True, on_unused_input='ignore')
test_model = theano.function([sents_id_matrix, sents_mask],
binarize_prob,
allow_input_downcast=True,
on_unused_input='ignore')
###############
# TRAIN MODEL #
###############
print '... training'
# early-stopping parameters
patience = 50000000000 # look as this many examples regardless
start_time = time.time()
mid_time = start_time
past_time = mid_time
epoch = 0
done_looping = False
n_train_batches = train_size / batch_size
train_batch_start = list(
np.arange(n_train_batches) * batch_size) + [train_size - batch_size]
# n_dev_batches=dev_size/batch_size
# dev_batch_start=list(np.arange(n_dev_batches)*batch_size)+[dev_size-batch_size]
n_test_batches = test_size / batch_size
test_batch_start = list(
np.arange(n_test_batches) * batch_size) + [test_size - batch_size]
max_meanf1_test = 0.0
max_weightf1_test = 0.0
train_indices = range(train_size)
while epoch < n_epochs:
epoch = epoch + 1
random.Random(100).shuffle(train_indices)
iter_accu = 0
cost_i = 0.0
for batch_id in train_batch_start: #for each batch
# iter means how many batches have been run, taking into loop
iter = (epoch - 1) * n_train_batches + iter_accu + 1
iter_accu += 1
train_id_batch = train_indices[batch_id:batch_id + batch_size]
# labels_matrix = []
# for lab in train_labels[train_id_batch]:
# vec = [0]*8
# vec[lab]=1
# labels_matrix.append(vec)
# labels_matrix = np.asarray(labels_matrix, dtype='int32')
cost_i += train_model(train_sents[train_id_batch],
train_masks[train_id_batch],
train_labels[train_id_batch])
#after each 1000 batches, we test the performance of the model on all test data
if iter % 20 == 0:
print 'Epoch ', epoch, 'iter ' + str(
iter) + ' average cost: ' + str(cost_i / iter), 'uses ', (
time.time() - past_time) / 60.0, 'min'
past_time = time.time()
# error_sum=0.0
# for test_batch_id in test_batch_start: # for each test batch
# error_i, pred_labels=test_model(
# test_sents[test_batch_id:test_batch_id+batch_size],
# test_masks[test_batch_id:test_batch_id+batch_size])
# pred_labels=list(pred_labels)
# error_sum+=error_i
#
# test_accuracy=1.0-error_sum/(len(test_batch_start))
# if test_accuracy > max_acc_test:
# max_acc_test=test_accuracy
# print '\t\t\t\t\t\t\t\tcurrent testbacc:', test_accuracy, '\t\tmax_acc_test:', max_acc_test
error_sum = 0.0
all_pred_labels = []
all_gold_labels = []
for test_batch_id in test_batch_start: # for each test batch
pred_labels = test_model(
test_sents[test_batch_id:test_batch_id + batch_size],
test_masks[test_batch_id:test_batch_id + batch_size])
gold_labels = test_labels[test_batch_id:test_batch_id +
batch_size]
# gold_labels_matrix = []
# for lab in gold_labels:
# vec = [0]*8
# vec[lab]=1
# gold_labels_matrix.append(vec)
# gold_labels_matrix = np.asarray(gold_labels_matrix, dtype='int32')
all_pred_labels.append(pred_labels)
all_gold_labels.append(gold_labels)
all_pred_labels = np.concatenate(all_pred_labels)
all_gold_labels = np.concatenate(all_gold_labels)
test_mean_f1, test_weight_f1 = average_f1_two_array_by_col_firstK(
all_pred_labels, all_gold_labels, 8)
if test_weight_f1 > max_weightf1_test:
max_weightf1_test = test_weight_f1
if test_mean_f1 > max_meanf1_test:
max_meanf1_test = test_mean_f1
print '\t\t\t\t\t\t\t\tcurrent f1s:', test_mean_f1, test_weight_f1, '\t\tmax_f1:', max_meanf1_test, max_weightf1_test
print 'Epoch ', epoch, 'uses ', (time.time() - mid_time) / 60.0, 'min'
mid_time = time.time()
#print 'Batch_size: ', update_freq
end_time = time.time()
print >> sys.stderr, ('The code for file ' + os.path.split(__file__)[1] +
' ran for %.2fm' % ((end_time - start_time) / 60.))
def evaluate_lenet5(learning_rate=0.01,
n_epochs=100,
emb_size=40,
batch_size=50,
describ_max_len=20,
type_size=12,
filter_size=[3, 5],
maxSentLen=100,
hidden_size=[300, 300]):
model_options = locals().copy()
print "model options", model_options
emb_root = '/save/wenpeng/datasets/LORELEI/multi-lingual-emb/'
seed = 1234
np.random.seed(seed)
rng = np.random.RandomState(
seed) #random seed, control the model generates the same results
srng = T.shared_randomstreams.RandomStreams(rng.randint(seed))
all_sentences, all_masks, all_labels, word2id = load_BBN_multi_labels_dataset(
maxlen=maxSentLen
) #minlen, include one label, at least one word in the sentence
label_sent, label_mask = load_SF_type_descriptions(word2id, type_size,
describ_max_len)
label_sent = np.asarray(label_sent, dtype='int32')
label_mask = np.asarray(label_mask, dtype=theano.config.floatX)
train_sents = np.asarray(all_sentences[0], dtype='int32')
train_masks = np.asarray(all_masks[0], dtype=theano.config.floatX)
train_labels = np.asarray(all_labels[0], dtype='int32')
train_size = len(train_labels)
dev_sents = np.asarray(all_sentences[1], dtype='int32')
dev_masks = np.asarray(all_masks[1], dtype=theano.config.floatX)
dev_labels = np.asarray(all_labels[1], dtype='int32')
dev_size = len(dev_labels)
test_sents = np.asarray(all_sentences[2], dtype='int32')
test_masks = np.asarray(all_masks[2], dtype=theano.config.floatX)
test_labels = np.asarray(all_labels[2], dtype='int32')
test_size = len(test_labels)
vocab_size = len(word2id) + 1 # add one zero pad index
rand_values = rng.normal(
0.0, 0.01,
(vocab_size, emb_size)) #generate a matrix by Gaussian distribution
rand_values[0] = np.array(np.zeros(emb_size), dtype=theano.config.floatX)
id2word = {y: x for x, y in word2id.iteritems()}
word2vec = load_fasttext_multiple_word2vec_given_file([
emb_root + 'IL5-cca-wiki-lorelei-d40.eng.vec',
emb_root + 'IL5-cca-wiki-lorelei-d40.IL5.vec'
], 40)
rand_values = load_word2vec_to_init(rand_values, id2word, word2vec)
embeddings = theano.shared(
value=np.array(rand_values, dtype=theano.config.floatX), borrow=True
) #wrap up the python variable "rand_values" into theano variable
#now, start to build the input form of the model
sents_id_matrix = T.imatrix('sents_id_matrix')
sents_mask = T.fmatrix('sents_mask')
labels = T.imatrix('labels') #batch*12
des_id_matrix = T.imatrix()
des_mask = T.fmatrix()
######################
# BUILD ACTUAL MODEL #
######################
print '... building the model'
common_input = embeddings[sents_id_matrix.flatten()].reshape(
(batch_size, maxSentLen, emb_size)).dimshuffle(
0, 2, 1) #the input format can be adapted into CNN or GRU or LSTM
bow_emb = T.sum(common_input * sents_mask.dimshuffle(0, 'x', 1), axis=2)
# bow_mean_emb = bow_emb/T.sum(sents_mask,axis=1).dimshuffle(0,'x')
des_input = embeddings[des_id_matrix.flatten()].reshape(
(type_size, describ_max_len, emb_size)).dimshuffle(0, 2, 1)
bow_des = T.sum(des_input * des_mask.dimshuffle(0, 'x', 1),
axis=2) #(tyope_size, emb_size)
conv_W, conv_b = create_conv_para(rng,
filter_shape=(hidden_size[0], 1,
emb_size, filter_size[0]))
conv_W2, conv_b2 = create_conv_para(rng,
filter_shape=(hidden_size[0], 1,
emb_size,
filter_size[1]))
multiCNN_para = [conv_W, conv_b, conv_W2, conv_b2]
conv_att_W, conv_att_b = create_conv_para(rng,
filter_shape=(hidden_size[0], 1,
emb_size,
filter_size[0]))
conv_W_context, conv_b_context = create_conv_para(
rng, filter_shape=(hidden_size[0], 1, emb_size, 1))
conv_att_W2, conv_att_b2 = create_conv_para(rng,
filter_shape=(hidden_size[0],
1, emb_size,
filter_size[1]))
conv_W_context2, conv_b_context2 = create_conv_para(
rng, filter_shape=(hidden_size[0], 1, emb_size, 1))
ACNN_para = [
conv_att_W, conv_att_b, conv_W_context, conv_att_W2, conv_att_b2,
conv_W_context2
]
# NN_para = multiCNN_para+ACNN_para
conv_model = Conv_with_Mask(
rng,
input_tensor3=common_input,
mask_matrix=sents_mask,
image_shape=(batch_size, 1, emb_size, maxSentLen),
filter_shape=(hidden_size[0], 1, emb_size, filter_size[0]),
W=conv_W,
b=conv_b
) #mutiple mask with the conv_out to set the features by UNK to zero
sent_embeddings = conv_model.maxpool_vec #(batch_size, hidden_size) # each sentence then have an embedding of length hidden_size
conv_model2 = Conv_with_Mask(
rng,
input_tensor3=common_input,
mask_matrix=sents_mask,
image_shape=(batch_size, 1, emb_size, maxSentLen),
filter_shape=(hidden_size[0], 1, emb_size, filter_size[1]),
W=conv_W2,
b=conv_b2
) #mutiple mask with the conv_out to set the features by UNK to zero
sent_embeddings2 = conv_model2.maxpool_vec #(batch_size, hidden_size) # each sentence then have an embedding of length hidden_size
LR_input = T.concatenate([sent_embeddings, sent_embeddings2, bow_emb],
axis=1)
LR_input_size = hidden_size[0] * 2 + emb_size
#classification layer, it is just mapping from a feature vector of size "hidden_size" to a vector of only two values: positive, negative
U_a = create_ensemble_para(
rng, 12, LR_input_size) # the weight matrix hidden_size*2
LR_b = theano.shared(value=np.zeros((12, ), dtype=theano.config.floatX),
name='LR_b',
borrow=True) #bias for each target class
LR_para = [U_a, LR_b]
layer_LR = LogisticRegression(
rng, input=LR_input, n_in=LR_input_size, n_out=12, W=U_a, b=LR_b
) #basically it is a multiplication between weight matrix and input feature vector
score_matrix = T.nnet.sigmoid(layer_LR.before_softmax) #batch * 12
prob_pos = T.where(labels < 1, 1.0 - score_matrix, score_matrix)
loss = -T.mean(T.log(prob_pos))
'''
GRU
'''
U1, W1, b1 = create_GRU_para(rng, emb_size, hidden_size[0])
GRU_NN_para = [
U1, W1, b1
] #U1 includes 3 matrices, W1 also includes 3 matrices b1 is bias
# gru_input = common_input.dimshuffle((0,2,1)) #gru requires input (batch_size, emb_size, maxSentLen)
gru_layer = GRU_Batch_Tensor_Input_with_Mask(common_input, sents_mask,
hidden_size[0], U1, W1, b1)
gru_sent_embeddings = gru_layer.output_sent_rep # (batch_size, hidden_size)
LR_att_input = T.concatenate([gru_sent_embeddings, bow_emb], axis=1)
LR_att_input_size = hidden_size[0] + emb_size
#classification layer, it is just mapping from a feature vector of size "hidden_size" to a vector of only two values: positive, negative
U_att_a = create_ensemble_para(
rng, 12, LR_att_input_size) # the weight matrix hidden_size*2
LR_att_b = theano.shared(value=np.zeros((12, ),
dtype=theano.config.floatX),
name='LR_b',
borrow=True) #bias for each target class
LR_att_para = [U_att_a, LR_att_b]
layer_att_LR = LogisticRegression(
rng,
input=LR_att_input,
n_in=LR_att_input_size,
n_out=12,
W=U_att_a,
b=LR_att_b
) #basically it is a multiplication between weight matrix and input feature vector
att_score_matrix = T.nnet.sigmoid(layer_att_LR.before_softmax) #batch * 12
att_prob_pos = T.where(labels < 1, 1.0 - att_score_matrix,
att_score_matrix)
att_loss = -T.mean(T.log(att_prob_pos))
'''
ACNN
'''
attentive_conv_layer = Attentive_Conv_for_Pair(
rng,
origin_input_tensor3=common_input,
origin_input_tensor3_r=common_input,
input_tensor3=common_input,
input_tensor3_r=common_input,
mask_matrix=sents_mask,
mask_matrix_r=sents_mask,
image_shape=(batch_size, 1, emb_size, maxSentLen),
image_shape_r=(batch_size, 1, emb_size, maxSentLen),
filter_shape=(hidden_size[0], 1, emb_size, filter_size[0]),
filter_shape_context=(hidden_size[0], 1, emb_size, 1),
W=conv_att_W,
b=conv_att_b,
W_context=conv_W_context,
b_context=conv_b_context)
sent_att_embeddings = attentive_conv_layer.attentive_maxpool_vec_l
attentive_conv_layer2 = Attentive_Conv_for_Pair(
rng,
origin_input_tensor3=common_input,
origin_input_tensor3_r=common_input,
input_tensor3=common_input,
input_tensor3_r=common_input,
mask_matrix=sents_mask,
mask_matrix_r=sents_mask,
image_shape=(batch_size, 1, emb_size, maxSentLen),
image_shape_r=(batch_size, 1, emb_size, maxSentLen),
filter_shape=(hidden_size[0], 1, emb_size, filter_size[1]),
filter_shape_context=(hidden_size[0], 1, emb_size, 1),
W=conv_att_W2,
b=conv_att_b2,
W_context=conv_W_context2,
b_context=conv_b_context2)
sent_att_embeddings2 = attentive_conv_layer2.attentive_maxpool_vec_l
acnn_LR_input = T.concatenate(
[sent_att_embeddings, sent_att_embeddings2, bow_emb], axis=1)
acnn_LR_input_size = hidden_size[0] * 2 + emb_size
#classification layer, it is just mapping from a feature vector of size "hidden_size" to a vector of only two values: positive, negative
acnn_U_a = create_ensemble_para(
rng, 12, acnn_LR_input_size) # the weight matrix hidden_size*2
acnn_LR_b = theano.shared(value=np.zeros((12, ),
dtype=theano.config.floatX),
name='LR_b',
borrow=True) #bias for each target class
acnn_LR_para = [acnn_U_a, acnn_LR_b]
acnn_layer_LR = LogisticRegression(
rng,
input=acnn_LR_input,
n_in=acnn_LR_input_size,
n_out=12,
W=acnn_U_a,
b=acnn_LR_b
) #basically it is a multiplication between weight matrix and input feature vector
acnn_score_matrix = T.nnet.sigmoid(
acnn_layer_LR.before_softmax) #batch * 12
acnn_prob_pos = T.where(labels < 1, 1.0 - acnn_score_matrix,
acnn_score_matrix)
acnn_loss = -T.mean(T.log(acnn_prob_pos))
'''
dataless cosine
'''
cosine_scores = normalize_matrix_rowwise(bow_emb).dot(
normalize_matrix_rowwise(bow_des).T)
cosine_score_matrix = T.nnet.sigmoid(
cosine_scores) #(batch_size, type_size)
params = multiCNN_para + LR_para + GRU_NN_para + LR_att_para + ACNN_para + acnn_LR_para # put all model parameters together
cost = loss + att_loss + acnn_loss + 1e-4 * ((conv_W**2).sum() +
(conv_W2**2).sum())
updates = Gradient_Cost_Para(cost, params, learning_rate)
'''
testing
'''
ensemble_NN_scores = T.max(T.concatenate([
att_score_matrix.dimshuffle('x', 0, 1),
score_matrix.dimshuffle('x', 0, 1),
acnn_score_matrix.dimshuffle('x', 0, 1)
],
axis=0),
axis=0)
ensemble_scores = 0.5 * ensemble_NN_scores + 0.5 * cosine_score_matrix
binarize_prob = T.where(ensemble_scores > 0.3, 1, 0)
#train_model = theano.function([sents_id_matrix, sents_mask, labels], cost, updates=updates, on_unused_input='ignore')
train_model = theano.function(
[sents_id_matrix, sents_mask, labels, des_id_matrix, des_mask],
cost,
updates=updates,
allow_input_downcast=True,
on_unused_input='ignore')
# dev_model = theano.function([sents_id_matrix, sents_mask, labels], layer_LR.errors(labels), allow_input_downcast=True, on_unused_input='ignore')
test_model = theano.function(
[sents_id_matrix, sents_mask, des_id_matrix, des_mask],
binarize_prob,
allow_input_downcast=True,
on_unused_input='ignore')
###############
# TRAIN MODEL #
###############
print '... training'
# early-stopping parameters
patience = 50000000000 # look as this many examples regardless
start_time = time.time()
mid_time = start_time
past_time = mid_time
epoch = 0
done_looping = False
n_train_batches = train_size / batch_size
train_batch_start = list(
np.arange(n_train_batches) * batch_size) + [train_size - batch_size]
# n_dev_batches=dev_size/batch_size
# dev_batch_start=list(np.arange(n_dev_batches)*batch_size)+[dev_size-batch_size]
n_test_batches = test_size / batch_size
test_batch_start = list(
np.arange(n_test_batches) * batch_size) + [test_size - batch_size]
# max_acc_dev=0.0
max_meanf1_test = 0.0
max_weightf1_test = 0.0
train_indices = range(train_size)
cost_i = 0.0
while epoch < n_epochs:
epoch = epoch + 1
random.Random(100).shuffle(train_indices)
iter_accu = 0
for batch_id in train_batch_start: #for each batch
# iter means how many batches have been run, taking into loop
iter = (epoch - 1) * n_train_batches + iter_accu + 1
iter_accu += 1
train_id_batch = train_indices[batch_id:batch_id + batch_size]
cost_i += train_model(train_sents[train_id_batch],
train_masks[train_id_batch],
train_labels[train_id_batch], label_sent,
label_mask)
#after each 1000 batches, we test the performance of the model on all test data
if iter % 20 == 0:
print 'Epoch ', epoch, 'iter ' + str(
iter) + ' average cost: ' + str(cost_i / iter), 'uses ', (
time.time() - past_time) / 60.0, 'min'
past_time = time.time()
error_sum = 0.0
all_pred_labels = []
all_gold_labels = []
for test_batch_id in test_batch_start: # for each test batch
pred_labels = test_model(
test_sents[test_batch_id:test_batch_id + batch_size],
test_masks[test_batch_id:test_batch_id + batch_size],
label_sent, label_mask)
gold_labels = test_labels[test_batch_id:test_batch_id +
batch_size]
# print 'pred_labels:', pred_labels
# print 'gold_labels;', gold_labels
all_pred_labels.append(pred_labels)
all_gold_labels.append(gold_labels)
all_pred_labels = np.concatenate(all_pred_labels)
all_gold_labels = np.concatenate(all_gold_labels)
test_mean_f1, test_weight_f1 = average_f1_two_array_by_col(
all_pred_labels, all_gold_labels)
if test_weight_f1 > max_weightf1_test:
max_weightf1_test = test_weight_f1
if test_mean_f1 > max_meanf1_test:
max_meanf1_test = test_mean_f1
print '\t\t\t\t\t\t\t\tcurrent f1s:', test_mean_f1, test_weight_f1, '\t\tmax_f1:', max_meanf1_test, max_weightf1_test
print 'Epoch ', epoch, 'uses ', (time.time() - mid_time) / 60.0, 'min'
mid_time = time.time()
#print 'Batch_size: ', update_freq
end_time = time.time()
print >> sys.stderr, ('The code for file ' + os.path.split(__file__)[1] +
' ran for %.2fm' % ((end_time - start_time) / 60.))
'''task1: combine datasets from two sub-tasks'''
train_sents = train_p1_sents + train_p2_sents
train_masks = train_p1_masks + train_p2_masks
train_labels = train_p1_labels + train_p2_labels
# all_sentences, all_masks, all_labels, word2id=load_BBN_multi_labels_dataset(maxlen=config['maxSentLen'])
label_sent, label_mask = load_SF_type_descriptions(
word2id, config['type_size'], config['describ_max_len'])
# emb_root = '/scratch/wyin3/dickens_save_dataset/LORELEI/multi-lingual-emb/Uyghur/'
# print('loading bilingual embeddings....')
# word2vec=load_fasttext_multiple_word2vec_given_file([emb_root+'vectors-en.txt',emb_root+'vectors-ug.txt'], 300)
# emb_root = '/scratch/wyin3/dickens_save_dataset/LORELEI/multi-lingual-emb/2018-il9-il10/multi-emb/'
emb_root = '/scratch/wyin3/dickens_save_dataset/LORELEI/il9/bilingual_emb/'
# print('loading bilingual embeddings....')
word2vec = load_fasttext_multiple_word2vec_given_file([
emb_root + 'cleaned-100k-il9longerw2v-multicca.d300.eng.txt',
emb_root + 'cleaned-100k-il9longerw2v-multicca.d300.il9longerw2v.txt'
], 300)
# word2vec=load_fasttext_multiple_word2vec_given_file(['100k-ENG-multicca.300.ENG.txt','100k-IL9-multicca.d300.IL9.txt'], 300)
# word2vec=load_fasttext_multiple_word2vec_given_file(['cleaned-100k-il9-multicca.d300.eng.txt','cleaned-100k-il9-multicca.d300.il9.txt'], 300)
vocab_size = len(word2id) + 1
rand_values = np.random.RandomState(1234).normal(
0.0, 0.01,
(vocab_size,
config['emb_size'])) #generate a matrix by Gaussian distribution
rand_values[0] = np.array(np.zeros(config['emb_size']), dtype=np.float32)
id2word = {y: x for x, y in word2id.items()}
rand_values = load_word2vec_to_init(rand_values, id2word, word2vec)
embeddings = torch.Tensor(rand_values)
print("build model...")
'lr': 0.0003,
'batch_size': 5,
'partial_rate': 0.5,
'maxSentLen': 100,
'describ_max_len': 20,
'type_size': 12
}
all_sentences, all_masks, all_labels, word2id = load_BBN_multi_labels_dataset(
maxlen=config['maxSentLen'])
label_sent, label_mask = load_SF_type_descriptions(
word2id, config['type_size'], config['describ_max_len'])
emb_root = '/scratch/wyin3/dickens_save_dataset/LORELEI/multi-lingual-emb/'
print('loading bilingual embeddings....')
word2vec = load_fasttext_multiple_word2vec_given_file([
emb_root + 'IL5-cca-wiki-lorelei-d40.eng.vec',
emb_root + 'IL5-cca-wiki-lorelei-d40.IL5.vec'
], 40)
vocab_size = len(word2id) + 1
rand_values = np.random.RandomState(1234).normal(
0.0, 0.01,
(vocab_size,
config['emb_size'])) #generate a matrix by Gaussian distribution
rand_values[0] = np.array(np.zeros(config['emb_size']), dtype=np.float32)
id2word = {y: x for x, y in word2id.items()}
rand_values = load_word2vec_to_init(rand_values, id2word, word2vec)
embeddings = torch.Tensor(rand_values)
print("build model...")
model, loss_function, optimizer = build_model(config['emb_size'],
config['hidden_size'],
test_sents, test_masks, test_labels, word2id = load_il_groundtruth_as_testset(
word2id, config['maxSentLen'], test_file_path)
'''task1: combine datasets from two sub-tasks'''
train_sents = train_p1_sents + train_p2_sents
train_masks = train_p1_masks + train_p2_masks
train_labels = train_p1_labels + train_p2_labels
# all_sentences, all_masks, all_labels, word2id=load_BBN_multi_labels_dataset(maxlen=config['maxSentLen'])
label_sent, label_mask = load_SF_type_descriptions(
word2id, config['type_size'], config['describ_max_len'])
# emb_root = '/scratch/wyin3/dickens_save_dataset/LORELEI/multi-lingual-emb/Uyghur/'
# print('loading bilingual embeddings....')
# word2vec=load_fasttext_multiple_word2vec_given_file([emb_root+'vectors-en.txt',emb_root+'vectors-ug.txt'], 300)
emb_root = '/home/wyin3/LORELEI/2019/dry_run_Uyghur/'
# print('loading bilingual embeddings....')
word2vec = load_fasttext_multiple_word2vec_given_file(
[emb_root + 'wiki.en.vec', emb_root + 'wiki.ug.vec'], 300)
vocab_size = len(word2id) + 1
rand_values = np.random.RandomState(1234).normal(
0.0, 0.01,
(vocab_size,
config['emb_size'])) #generate a matrix by Gaussian distribution
rand_values[0] = np.array(np.zeros(config['emb_size']), dtype=np.float32)
id2word = {y: x for x, y in word2id.items()}
rand_values = load_word2vec_to_init(rand_values, id2word, word2vec)
embeddings = torch.Tensor(rand_values)
print("build model...")
model, loss_function, optimizer = build_model(config['emb_size'],
config['hidden_size'],
vocab_size, 12, embeddings,
