def augment(model_path, input_nbest_path, vocab_path, output_nbest_path):
classifier = MLP(model_path=model_path)
evaluator = eval.Evaluator(None, classifier)
vocab = VocabManager(vocab_path)
ngram_size = classifier.ngram_size
def get_ngrams(tokens):
for i in range(ngram_size - 1):
tokens.insert(0, '<s>')
if vocab.has_end_padding:
tokens.append('</s>')
indices = vocab.get_ids_given_word_list(tokens)
return U.get_all_windows(indices, ngram_size)
input_nbest = NBestList(input_nbest_path, mode='r')
output_nbest = NBestList(output_nbest_path, mode='w')
L.info('Augmenting: ' + input_nbest_path)
start_time = time.time()
counter = 0
cache = dict()
for group in input_nbest:
ngram_list = []
for item in group:
tokens = item.hyp.split()
ngrams = get_ngrams(tokens)
for ngram in ngrams:
if not cache.has_key(str(ngram)):
ngram_list.append(ngram)
cache[str(ngram)] = 1000
if len(ngram_list) > 0:
ngram_array = np.asarray(ngram_list, dtype='int32')
ngram_log_prob_list = evaluator.get_ngram_log_prob(
ngram_array[:, 0:-1], ngram_array[:, -1])
for i in range(len(ngram_list)):
cache[str(ngram_list[i])] = ngram_log_prob_list[i]
for item in group:
tokens = item.hyp.split()
ngrams = get_ngrams(tokens)
sum_ngram_log_prob = 0
for ngram in ngrams:
sum_ngram_log_prob += cache[str(ngram)]
item.append_feature(sum_ngram_log_prob)
output_nbest.write(item)
#print counter
counter += 1
output_nbest.close()
L.info("Ran for %.2fs" % (time.time() - start_time))
# Setting the args for the classifier
args_nn.emb_dim = int(config_dict['input_embedding_dimension'])
args_nn.num_hidden = config_dict['num_hidden'] + ',' + config_dict['output_embedding_dimension']
args_nn.vocab_size = int(config_dict['input_vocab_size'])
args_nn.ngram_size = int(config_dict['ngram_size'])
args_nn.num_classes = int(config_dict['output_vocab_size'])
act_func = config_dict['activation_function']
if act_func == 'rectifier':
act_func = 'relu'
args_nn.activation_name = act_func
# Creating the classifier with the arguments read
L.info("Creating CoreLM model")
classifier = MLP(args_nn)
# Loading matrices
embeddings = np.loadtxt(model_dict['\input_embeddings'])
W1 = np.loadtxt(model_dict['\hidden_weights 1'])
W1 = np.transpose(W1)
b1 = np.loadtxt(model_dict['\hidden_biases 1'])
W2 = np.loadtxt(model_dict['\hidden_weights 2'])
W2 = np.transpose(W2)
b2 = np.loadtxt(model_dict['\hidden_biases 2'])
W3 = np.loadtxt(model_dict['\output_weights'])
W3 = np.transpose(W3)
b3 = np.loadtxt(model_dict['\output_biases'])
params_nn =[embeddings, W1, b1, W2, b2, W3, b3]
"--directory",
dest="out_dir",
help="The output directory for log file, model, etc.")
args = parser.parse_args()
U.set_theano_device('cpu', 1)
from dlm.models.mlp import MLP
if args.out_dir is None:
args.out_dir = 'primelm_convert-' + U.curr_time()
U.mkdir_p(args.out_dir)
# Loading PrimeLM model and creating classifier class
L.info("Loading PrimeLM model")
classifier = MLP(model_path=args.primelm_model)
args_nn = classifier.args
params_nn = classifier.params
U.xassert(
len(params_nn) == 7,
"PrimeLM model is not compatible with NPLM architecture. 2 hidden layers and an output linear layer is required."
)
embeddings = params_nn[0].get_value()
W1 = params_nn[1].get_value()
W1 = np.transpose(W1)
b1 = params_nn[2].get_value()
W2 = params_nn[3].get_value()
W2 = np.transpose(W2)
b2 = params_nn[4].get_value()
W3 = params_nn[5].get_value()
## Creating model
#
args.ngram_size = trainset.get_ngram_size()
args.num_classes = trainset.get_num_classes()
L.info('Building the model')
if args.feature_emb_dim is None:
args.features_info = trainset.get_features_info()
args.vocab_size = args.features_info[0][0]
#args.vocab_size = trainset.get_vocab_size()
#args.features_info = [(args.vocab_size, args.ngram_size)]
else:
args.features_info = trainset.get_features_info()
classifier = MLP(args)
L.info('Parameters: ' + str(classifier.params))
if args.base_model_path is not None:
initialization_classifier = MLP(model_path=args.base_model_path)
for param, aparam in zip(classifier.params,
initialization_classifier.params):
param.set_value(aparam.get_value())
#########################
## Training criterion
#
if args.loss_function == "nll":
from dlm.criterions.nll import NegLogLikelihood
criterion = NegLogLikelihood(classifier, args)
instance_weights_path=args.instance_weights_path)
devset = MemMapReader(args.devset)
testset = None
if args.testset:
testset = MemMapReader(args.testset)
#########################
## Creating model
#
L.info('Building the model')
args.vocab_size = trainset.get_vocab_size()
args.ngram_size = trainset.get_ngram_size()
args.num_classes = trainset.get_num_classes()
classifier = MLP(args)
L.info('Parameters: ' + str(classifier.params))
#########################
## Training criterion
#
if args.loss_function == "nll":
from dlm.criterions.weighted_nll import NegLogLikelihood
criterion = NegLogLikelihood(classifier, args)
elif args.loss_function == "nce":
from dlm.criterions.nce import NCELikelihood
noise_dist = trainset.get_unigram_model()
criterion = NCELikelihood(classifier, args, noise_dist)
else:
L.error('Invalid loss function \'' + args.loss_function + '\'')
default="gpu",
help="The computing device (cpu or gpu)")
args = parser.parse_args()
U.set_theano_device(args.device, 1)
from dlm.models.mlp import MLP
from dlm import eval
import theano
import theano.tensor as T
#########################
## Loading model
#
classifier = MLP(model_path=args.model_path)
#########################
## Loading dataset
#
from dlm.io.ngramsReader import NgramsReader
from dlm.io.vocabReader import VocabManager
testset = NgramsReader(dataset_path=args.input_path,
ngram_size=classifier.ngram_size,
vocab_path=args.vocab_path)
vocab = VocabManager(args.vocab_path)
## Loading restricted vocab
restricted_ids = []
"--directory",
dest="out_dir",
help="The output directory for log file, model, etc.")
args = parser.parse_args()
U.set_theano_device('cpu', 1)
from dlm.models.mlp import MLP
if args.out_dir is None:
args.out_dir = 'corelm_convert-' + U.curr_time()
U.mkdir_p(args.out_dir)
# Loading CoreLM model and creating classifier class
L.info("Loading CoreLM model")
classifier = MLP(model_path=args.corelm_model)
args_nn = classifier.args
params_nn = classifier.params
U.xassert(
len(params_nn) == 7,
"CoreLM model is not compatible with NPLM architecture. 2 hidden layers and an output linear layer is required."
)
embeddings = params_nn[0].get_value()
W1 = params_nn[1].get_value()
W1 = np.transpose(W1)
b1 = params_nn[2].get_value()
W2 = params_nn[3].get_value()
W2 = np.transpose(W2)
b2 = params_nn[4].get_value()
W3 = params_nn[5].get_value()