max_length_en)
target_tensor = tensor_from_sentence("de", output_lang, pair[1],
max_length_de)
return input_tensor, target_tensor
language, total_data = data_generator(batch_size, 20, device)
train_data, test_data, y_train, y_test = train_test_split(
total_data, np.zeros(len(total_data, )), test_size=0.1, random_state=42)
d_model = 128
heads = 8
N = 6
src_vocab = language.n_words
trg_vocab = language.n_words
en_weight_matrix = Embedder.initial_weights_matrix(
"word_vector/glove.6B.300d.txt", language, 300)
src_vocab = language.n_words
trg_vocab = language.n_words
model = Transformer(src_vocab, trg_vocab, d_model, N, heads, device,
en_weight_matrix, en_weight_matrix)
try:
model.load_state_dict(
torch.load("model/transformer.pt", map_location=device))
model.eval()
except:
print("no weights exist")
for p in model.parameters():
if p.dim() > 1:
nn.init.xavier_uniform_(p)
max_length_de):
input_tensor = tensor_from_sentence("en", input_lang, pair[0],
max_length_en)
target_tensor = tensor_from_sentence("de", output_lang, pair[1],
max_length_de)
return input_tensor, target_tensor
input_lang, output_lang, _ = prepare_data(lang1, lang2, 40)
d_model = 128
heads = 8
N = 6
src_vocab = input_lang.n_words
trg_vocab = output_lang.n_words
en_weight_matrix = Embedder.initial_weights_matrix(
"word_vector/glove.6B.300d.txt", input_lang, 300)
de_weight_matrix = Embedder.initial_weights_matrix(
"word_vector/vn_word2vec_300d.txt", input_lang, 300)
src_vocab = input_lang.n_words
trg_vocab = output_lang.n_words
model = Transformer(src_vocab, trg_vocab, d_model, N, heads, device,
en_weight_matrix, de_weight_matrix)
model.load_state_dict(torch.load("model/transformer.pt", map_location=device))
def translate(model, sentence, lang_input, lang_output, max_len=80):
model.eval()
src = tensor_from_sentence("en", lang_input, sentence, len(sentence))