'''
ct = 0
for i in data:
ct += len(i[0].phrase.split()) + len(i[1].phrase.split())
data = []
print ct
idx = 0
ct2 = 0
while ct > 0:
dd = d[idx]
data.append(dd)
v = len(dd[0].phrase.split()) + len(dd[1].phrase.split())
ct -= v
ct2 += v
idx += 1
print ct2
'''
if params.wordfile:
(words, We) = utils.get_wordmap(params.wordfile)
model = models(We, params)
if params.loadmodel:
base_params = cPickle.load(open(params.loadmodel, 'rb'))
lasagne.layers.set_all_param_values(model.final_layer, base_params)
print " ".join(sys.argv)
print "Num examples:", len(data)
model.train(data, words, params)
if params.combination_type == "ngram-word":
model = mixed_models(saved_params[0], saved_params[1], params)
elif params.combination_type == "ngram-word-lstm":
model = mixed_models(saved_params[0],
saved_params[1],
params,
We_initial_lstm=saved_params[2])
lasagne.layers.set_all_param_values(model.final_layer, saved_params)
else:
words_3grams, We_3gram = utils.get_ngrams(data, params)
if params.random_embs:
words_words, We_word = utils.get_words(data, params)
else:
words_words, We_word = utils.get_wordmap(args.wordfile)
We_lstm = copy.deepcopy(We_word)
if params.combination_type == "ngram-word":
model = mixed_models(We_3gram, We_word, params)
elif params.combination_type == "ngram-lstm":
model = mixed_models(We_3gram,
None,
params,
We_initial_lstm=We_lstm)
elif params.combination_type == "word-lstm":
model = mixed_models(None,
We_word,
params,
We_initial_lstm=We_lstm)