def main():
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
input_lang, output_lang, pairs = prepareData('eng',
'fra',
True,
dir='data',
filter=False)
hidden_size = 512
batch_size = 64
iters = 50000
# encoder = EncoderRNN(input_lang.n_words, hidden_size).to(device)
encoder = EncoderRNN(input_lang.n_words, hidden_size)
attn_decoder = AttnDecoderRNN(hidden_size,
output_lang.n_words,
dropout_p=0.1)
if torch.cuda.device_count() > 1:
print("Let's use", torch.cuda.device_count(), "GPUs!")
encoder = nn.DataParallel(encoder)
attn_decoder = nn.DataParallel(attn_decoder)
encoder = encoder.to(device)
attn_decoder = attn_decoder.to(device)
# attn_decoder = AttnDecoderRNN(hidden_size, output_lang.n_words, dropout_p=0.1).to(device)
trainIters(device,
pairs,
input_lang,
output_lang,
encoder,
attn_decoder,
batch_size,
iters,
print_every=250)
def main():
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
input_lang, output_lang, pairs = prepareData('eng', 'fra', True)
hidden_size = 256
encoder1 = EncoderRNN(input_lang.n_words, hidden_size, device).to(device)
attn_decoder1 = AttnDecoderRNN(hidden_size,
output_lang.n_words,
device,
dropout_p=0.1).to(device)
trainIters(device,
pairs,
input_lang,
output_lang,
encoder1,
attn_decoder1,
100000,
print_every=5000)
def main():
nIters = 50000
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
loadFilename = os.path.join('checkpoints',
'{}_{}.tar'.format(nIters, 'checkpoint'))
checkpoint = torch.load(loadFilename, map_location=device)
# input_lang, output_lang, pairs = prepareData('eng', 'fra', True, 'data', filter=False)
# If loading a model trained on GPU to CPU
encoder_sd = checkpoint['en']
encoder_sd
decoder_sd = checkpoint['de']
decoder_sd
hidden_size = 512
input_lang = Lang('fra')
output_lang = Lang('eng')
input_lang.__dict__ = checkpoint['input_lang']
output_lang.__dict__ = checkpoint['output_lang']
encoder = EncoderRNN(input_lang.n_words, hidden_size).to(device)
decoder = AttnDecoderRNN(hidden_size, output_lang.n_words,
dropout_p=0).to(device)
encoder.load_state_dict(encoder_sd)
decoder.load_state_dict(decoder_sd)
encoder.eval()
decoder.eval()
# encoder_optimizer_sd = checkpoint['en_opt']
# decoder_optimizer_sd = checkpoint['de_opt']
_, _, test_pairs = prepareData('eng',
'fra',
True,
dir='test',
filter=False)
evaluateRandomly(device, test_pairs, encoder, decoder, input_lang,
output_lang)
decode_batch(device,
test_pairs,
encoder,
decoder,
input_lang,
output_lang,
batch_size=64)
def main():
nIters = 100000
loadFilename = os.path.join('checkpoints',
'{}_{}.tar'.format(nIters, 'checkpoint'))
checkpoint = torch.load(loadFilename)
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
input_lang, output_lang, pairs = prepareData('eng', 'fra', True)
# If loading a model trained on GPU to CPU
encoder_sd = checkpoint['en']
decoder_sd = checkpoint['de']
hidden_size = 256
encoder = EncoderRNN(input_lang.n_words, hidden_size, device).to(device)
decoder = AttnDecoderRNN(hidden_size,
output_lang.n_words,
device,
dropout_p=0.1).to(device)
encoder.load_state_dict(encoder_sd)
decoder.load_state_dict(decoder_sd)
encoder_optimizer_sd = checkpoint['en_opt']
decoder_optimizer_sd = checkpoint['de_opt']
input_lang.__dict__ = checkpoint['input_lang']
output_lang.__dict__ = checkpoint['output_lang']
evaluateRandomly(device, pairs, encoder, decoder, input_lang, output_lang)
def run():
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
print("Device :", device, "\n")
# Preprocess data
input_lang, output_lang, pairs = prepareData('eng', 'fra', True)
print("Finished Preprocessing\n")
# Seq2Seq Model
hidden_size = 256
encoder1 = EncoderRNN(input_lang.n_words, hidden_size).to(device)
attn_decoder1 = AttnDecoderRNN(hidden_size, output_lang.n_words, dropout_p=0.1)
metadata = (input_lang, output_lang, pairs)
trainIters(encoder1, attn_decoder1, metadata, n_iters=500, print_every=100) # 원래는 n_iters=75000, print_every=5000
# Check
evaluateRandomly(encoder1, attn_decoder1, metadata)
# Evaluate and Visualize
output_words, attentions = evaluate(encoder1, attn_decoder1, metadata, "je suis trop froid .")
plt.matshow(attentions.numpy())
evaluateAndShowAttention(encoder1, attn_decoder1, metadata, "elle a cinq ans de moins que moi .")
evaluateAndShowAttention(encoder1, attn_decoder1, metadata, "elle est trop petit .")
evaluateAndShowAttention(encoder1, attn_decoder1, metadata, "je ne crains pas de mourir .")
evaluateAndShowAttention(encoder1, attn_decoder1, metadata, "c est un jeune directeur plein de talent .")
from keras.optimizers import Adam
from preprocess import prepareData, PATCH_SIZE
def network(layers):
# Define the model
model = Sequential()
model.add(
Conv2D(3 * layers, (3, 3),
activation='relu',
input_shape=(PATCH_SIZE, PATCH_SIZE, layers)))
model.add(Conv2D(6 * layers, (3, 3), activation='relu'))
model.add(Dropout(0.25))
model.add(Flatten())
model.add(Dense(6 * layers, activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(9, activation='softmax'))
model.summary()
return model
if __name__ == '__main__':
X_train, y_train, X_test, y_test = prepareData()
# generate the model
model = network(X_train.shape[3])
# Train the model
model.compile(Adam(lr=.0005),
loss='categorical_crossentropy',
metrics=['accuracy'])
model.fit(X_train, y_train, validation_data=(X_test, y_test), epochs=10)
import torch
import torch.nn as nn
from preprocess import prepareData
from loader import translate_dataloader
from transformer import Transformer
from config import Config
from train import train_one_epoch
from torch.optim import Adam
from loss import LabelSmoothingLoss
from optim import NoamOpt
from evaluate import evaluate_dataset, evaluateRandomly
import os
# data
input_lang, output_lang, pairs_train, pairs_dev = prepareData(
'eng', 'fra', 'eng-fra.txt', True)
train_loader = translate_dataloader(input_lang, output_lang, pairs_train)
dev_loader = translate_dataloader(input_lang, output_lang, pairs_dev)
# model
model = Transformer(input_lang.n_words, output_lang.n_words,
Config.n_enc_layer, Config.n_dec_layer, Config.hidden_size,
Config.num_head, Config.head_size, Config.feedforward_size,
Config.dropout, Config.attn_dropout, Config.layer_norm_eps)
model.cuda()
# load model if pretrained
if Config.pretrained:
model.load_state_dict(torch.load(Config.model_load))
decoder_input = topi.squeeze().detach()
return decoded_words, decoder_attentions[:di + 1]
def evaluateRandomly(encoder, decoder, n=10):
for i in range(n):
pair = random.choice(pairs)
print('>', pair[0])
print('=', pair[1])
output_words, attentions = evaluate(encoder, decoder, pair[0])
output_sentence = ' '.join(output_words)
print('<', output_sentence)
print('')
if __name__ == '__main__':
hidden_size = 64
input_lang, output_lang, pairs = prepareData(base_dir, 'eng', 'fra', True)
encoder = EncoderGRU(input_lang.n_words, hidden_size)
decoder = AttnDecoderGRU(hidden_size, output_lang.n_words, dropout_p=0.1)
print('begin train')
trainIters(encoder, decoder, 10000, print_every=500)
print('end train')
torch.save(encoder.state_dict(), '../model/encoder_param.pkl')
torch.save(decoder.state_dict(), '../model/decoder_param.pkl')
print('begin evluate')
evaluateRandomly(encoder, decoder)
print('end evaluate')
from util import load_wordvector_text
from lang import Lang
# train_file = '/home/prosa/Works/Text/korpus/chatbot_dataset/plain/preprocessed/split-augmented/combine/nontask/train-nontask.aug.shuffle.pre'
# train_file = '/home/prosa/Works/Text/korpus/chatbot_dataset/plain/preprocessed/split-augmented/combine/nontask/train.test'
# src_lang, tgt_lang, pairs = prepareData('dataset/chatbot/input-output.txt', reverse=False)
# train_file = '/home/prosa/Works/Text/mt/dataset/filter-en-id/lenlim80/sorted/train.dummy'
# train_file = '/home/prosa/Works/Text/mt/dataset/filter-en-id/lenlim80/sorted/limit-en-id.sorted.01.txt'
# train_file = '/home/prosa/Works/Text/korpus/asr_dataset/dataset_pruned/word/dummy'
# train_file = '/home/prosa/Works/Text/korpus/dialogue/dataset_filtered/gabung.shuffle'
# train_file = '/home/prosa/Works/Text/korpus/dialogue/misc.txt'
train_file = '/home/prosa/Works/Text/seq2seq/dataset/en-id-10k-v2.txt'
src_lang, tgt_lang, pairs = prepareData(train_file, reverse=False)
# Word vector
# word_vectors = KeyedVectors.load_word2vec_format(params.WORD_VECTORS_FILE, binary=True)
# word_vectors = KeyedVectors.load(params.WORD_VECTORS_FILE)
word_vectors = load_wordvector_text(params.WORD_VECTORS_FILE)
############
# folder_model = 'model/dialogue/fix/oovchar_rnn/'
# folder_model = 'model/dialogue/dummy/wordchar_cnn/'
# folder_model = 'model/dialogue/dummy/oovchar_rnn/'
folder_model = 'model/dialogue/dummy/wordchar_rnn/'
# folder_model = 'model/dialogue/dummy/word/'
# folder_model = 'model/dialogue/fix/oovchar_rnn/'
import torch
from model import EncoderRNN, EncoderEmbeddingInputRNN, AttnDecoderRNN
from preprocess import prepareData
from train_wordembed2 import Trainer
from preprocess import buildPairs
from gensim.models import KeyedVectors
use_cuda = torch.cuda.is_available()
src_lang, tgt_lang, pairs = prepareData('dataset/input-output.txt',
reverse=False)
# Word vector
word_vector = KeyedVectors.load_word2vec_format("word_vector/koran.vec",
binary=True)
hidden_size = 64
max_len = 50
encoder = EncoderEmbeddingInputRNN(src_lang.n_words, hidden_size, word_vector)
attn_decoder = AttnDecoderRNN(hidden_size,
tgt_lang.n_words,
dropout_p=0.1,
max_length=max_len)
if use_cuda:
encoder = encoder.cuda()
attn_decoder = attn_decoder.cuda()
num_iter = 100000
trainer = Trainer(src_lang, tgt_lang, pairs)
import torch
from gensim.models import FastText
from gensim.models import KeyedVectors
from preprocess import prepareData
from model_bidirectional import WordEncoderBiRNN, PreTrainedEmbeddingEncoderBiRNN, AttnDecoderRNN
from train_bidirectional import Trainer
import params
root_folder = '/home/prosa/Works/Text/'
dataset_folder = root_folder + 'korpus/chatbot_dataset/'
src_lang, tgt_lang, pairs = prepareData(dataset_folder +
'input-response.shuffle',
reverse=False)
word_vectors = KeyedVectors.load(params.WORD_VECTORS_FILE)
# word_vectors = FastText.load_fasttext_format(params.WORD_VECTORS_FILE)
hidden_size = word_vectors.vector_size
max_length = 50
k_validation = 10
n_data = len(pairs)
last_idx = 0
epoch = 5
tests = []
outputs = []
refs = []
for k in range(k_validation):
# Prepare dataset for kth-fold
def main(_):
print('preprocessing data ...')
processData(FLAGS.year, FLAGS.domain, FLAGS.embedding)
print('preparing data for model ...')
trainData, testData, validData, sampleData = prepareData()
import tensorflow as tf
import numpy as np
import pandas as pd
import config as cf
from preprocess import (processData, prepareData)
import time
print('preparing data for model ...')
trainData, testData, validData, sampleData = prepareData()
def get_batch_index(length, batch_size, is_shuffle=True):
index = list(range(length))
if is_shuffle:
np.random.shuffle(index)
return index
# hyperparameters
batch_iterations = 500
batch_size = 32
full_iterations = 50
learning_rate = 0.005
reg_eta = 0.001
dim_lstm = 300
num_AL = 3
# dimensionalities
dim_word = 300
dim_sentence = 80
dim_polarity = 3
