def train_batch_generator(batch_size,
max_enc_len=200,
max_dec_len=50,
sample_sum=None):
train_X, train_Y = load_dataset(config.train_x_path, config.train_y_path,
max_enc_len, max_dec_len, sample_sum)
valid_X, valid_Y = load_dataset(config.test_x_path, config.test_y_path,
max_enc_len, max_dec_len, sample_sum)
print(f'total samples: {len(train_Y)} for training ...')
train_dataset = tf.data.Dataset.from_tensor_slices((train_X, train_Y))
valid_dataset = tf.data.Dataset.from_tensor_slices((valid_X, valid_Y))
train_dataset = train_dataset.shuffle(len(train_X),
reshuffle_each_iteration=True).batch(
batch_size, drop_remainder=True)
valid_dataset = valid_dataset.shuffle(len(valid_X),
reshuffle_each_iteration=True).batch(
batch_size, drop_remainder=True)
train_steps_per_epoch = len(train_X) // batch_size
valid_steps_per_epoch = len(valid_X) // batch_size
return train_dataset, valid_dataset, train_steps_per_epoch, valid_steps_per_epoch
def predict_result(model, params, vocab):
test_x, _ = load_dataset(test_x_path, test_y_path, params['max_enc_len'],
params['max_dec_len'])
# 预测结果
results = greedy_decode(model, test_x, params['batch_size'], vocab, params)
# 结果保存
get_rouge(results)
def beam_test_batch_generator(beam_size, max_enc_len=200, max_dec_len=50):
test_x, _ = load_dataset(config.test_x_path, config.test_y_path,
max_enc_len, max_dec_len)
print(f'total test samples: {len(test_x)} .')
for row in tqdm(test_x, total=len(test_x), desc='beam search'):
beam_search_data = tf.convert_to_tensor(
[row for _ in range(beam_size)])
yield beam_search_data
def main(params):
logging.info("Loading the datasets...")
train_iter, dev_iter, test_iterator, DE, EN = load_dataset(
params.data_path, params.train_batch_size, params.dev_batch_size)
de_size, en_size = len(DE.vocab), len(EN.vocab)
logging.info("[DE Vocab Size]: {}, [EN Vocab Size]: {}".format(
de_size, en_size))
logging.info("- done.")
params.src_vocab_size = de_size
params.tgt_vocab_size = en_size
params.sos_index = EN.vocab.stoi["<s>"]
params.pad_token = EN.vocab.stoi["<pad>"]
params.eos_index = EN.vocab.stoi["</s>"]
params.itos = EN.vocab.itos
params.SRC = DE
params.TRG = EN
# make the Seq2Seq model
model = make_seq2seq_model(params)
# default optimizer
optimizer = optim.Adam(model.parameters(), lr=params.lr)
if params.model_type == "Transformer":
criterion = LabelSmoothingLoss(params.label_smoothing,
params.tgt_vocab_size,
params.pad_token).to(params.device)
optimizer = ScheduledOptimizer(optimizer=optimizer,
d_model=params.hidden_size,
factor=2,
n_warmup_steps=params.n_warmup_steps)
scheduler = None
else:
criterion = nn.NLLLoss(reduction="sum", ignore_index=params.pad_token)
scheduler = optim.lr_scheduler.ReduceLROnPlateau(
optimizer, patience=params.patience, factor=.1, verbose=True)
# intialize the Trainer
trainer = Trainer(model, optimizer, scheduler, criterion, train_iter,
dev_iter, params)
if params.restore_file:
restore_path = os.path.join(params.model_dir + "/checkpoints/",
params.restore_file)
logging.info("Restoring parameters from {}".format(restore_path))
Trainer.load_checkpoint(model, restore_path, optimizer)
# train the model
trainer.train()
def run_siamnet_train(dataset_zip_path: str, train_file_path , test_file_path, batch_size=2):
random.seed(42)
input_image_shape = (105, 105, 1)
print('images zip path: {}'.format(dataset_zip_path))
print('train file: {} , test file: {}'.format(train_file_path, test_file_path))
train_dataset, y_array_train, first_image_title_train_list, second_image_title_train_list, \
test_dataset, y_array_test, first_image_title_test_list, second_image_title_test_list = load_dataset(
dataset_zip_path,
train_file_path,
test_file_path)
print('train dataset has {} samples'.format(len(train_dataset[0])))
print('test dataset has {} samples'.format(len(test_dataset[0])))
train_dataset, y_array_train, val_dataset, y_array_val = split_train_val(train_dataset, y_array_train)
print('train (after val split) dataset has {} samples'.format(len(train_dataset[0])))
print('val dataset has {} samples'.format(len(val_dataset[0])))
num_train_samples = len(train_dataset[0])
curr_script_dir = pathlib.Path(__file__).parent.absolute()
saved_model_dir = os.path.join(curr_script_dir, 'saved_models')
if not os.path.isdir(saved_model_dir):
os.mkdir(saved_model_dir)
siamnet = SiameseNet(input_image_shape, num_train_samples, batch_size)
model_img_file = saved_model_dir+'/siam_model_architecture.png'
plot_model(siamnet.siam_model, to_file=model_img_file, show_shapes=True)
siamnet.train(train_dataset, y_array_train, val_dataset, y_array_val, save_model_path=saved_model_dir,
batch_size=batch_size)
import sys
sys.path.insert(1, '.')
import paddle
import paddle.fluid as fluid
from utils.data_loader import load_dataset
from utils.wv_loader import load_vocab
X, Y, _ = load_dataset()
x_time_steps = X.shape[1]
y_time_steps = Y.shape[1]
word2id, id2word = load_vocab()
def train_gen():
for row in Dataset:
yield row
# 定义loss函数
def loss_function(real, pred):
# 判断logit为1和0的数量
real = fluid.layers.cast(real, dtype=np.int64)
pad_array = fluid.layers.ones_like(real) * word2id['<PAD>']
mask = fluid.layers.logical_not(fluid.layers.equal(real, pad_array))
# 计算decoder的长度
dec_lens = fluid.layers.reduce_sum(fluid.layers.cast(mask,
dtype=np.float32),
dim=-1)
# 计算loss值
loss_ = fluid.layers.cross_entropy(input=pred, label=real)
# 转换mask的格式
if (epoch + 1) % 2 == 0:
checkpoint.save(file_prefix=checkpoint_prefix)
print('Epoch {} Loss {:.4f}'.format(epoch + 1,
total_loss / steps_per_epoch))
print('Time taken for 1 epoch {} sec\n'.format(time.time() - start))
# 读取vocab训练
vocab, reverse_vocab = get_vocab()
if __name__ == '__main__':
# 计算vocab size
vocab_size = len(vocab)
# 加载数据集
train_X, train_Y, test_X = load_dataset()
# 使用GenSim训练好的embedding matrix
embedding_matrix = load_embedding_matrix()
input_sequence_len = 300
EPOCHS = 10
BATCH_SIZE = 64
embedding_dim = 300
units = 1024
encoder = Encoder(vocab_size, embedding_dim, embedding_matrix, units,
BATCH_SIZE)
decoder = Decoder(vocab_size, embedding_dim, embedding_matrix, units,
BATCH_SIZE)
optimizer = tf.keras.optimizers.Adam()
loss_object = tf.keras.losses.SparseCategoricalCrossentropy(
from utils.data_loader import load_dataset
from odtw import KnnDTW
X_train, y_train, X_test, y_test = load_dataset('adiac',
normalize_timeseries=True)
print()
# parameters
num_neighbours = 1
model = KnnDTW(num_neighbours)
# fit to the dataset
model.fit(X_train, y_train)
# Predict / Evaluate the score
accuracy = model.evaluate(X_test, y_test)
error = 1. - accuracy
print("*" * 20, "\n")
print("Test Accuracy :", accuracy)
print("Test Error :", error)
def main(params, greedy, beam_size, test):
"""
The main function for decoding a trained MT model
Arguments:
params: parameters related to the `model` that is being decoded
greedy: whether or not to do greedy decoding
beam_size: size of beam if doing beam search
"""
print("Loading dataset...")
_, dev_iter, test_iterator, DE, EN = load_dataset(params.data_path,
params.train_batch_size,
params.dev_batch_size)
de_size, en_size = len(DE.vocab), len(EN.vocab)
print("[DE Vocab Size: ]: {}, [EN Vocab Size]: {}".format(
de_size, en_size))
params.src_vocab_size = de_size
params.tgt_vocab_size = en_size
params.sos_index = EN.vocab.stoi["<s>"]
params.pad_token = EN.vocab.stoi["<pad>"]
params.eos_index = EN.vocab.stoi["</s>"]
params.itos = EN.vocab.itos
device = torch.device('cuda' if params.cuda else 'cpu')
params.device = device
# make the Seq2Seq model
model = make_seq2seq_model(params)
# load the saved model for evaluation
if params.average > 1:
print("Averaging the last {} checkpoints".format(params.average))
checkpoint = {}
checkpoint["state_dict"] = average_checkpoints(params.model_dir,
params.average)
model = Trainer.load_checkpoint(model, checkpoint)
else:
model_path = os.path.join(params.model_dir + "checkpoints/",
params.model_file)
print("Restoring parameters from {}".format(model_path))
model = Trainer.load_checkpoint(model, model_path)
# evaluate on the test set
if test:
print("Doing Beam Search on the Test Set")
test_decoder = Translator(model, test_iterator, params, device)
test_beam_search_outputs = test_decoder.beam_decode(
beam_width=beam_size)
test_decoder.output_decoded_translations(
test_beam_search_outputs,
"beam_search_outputs_size_test={}.en".format(beam_size))
return
# instantiate a Translator object to translate SRC langauge to TRG language using Greedy/Beam Decoding
decoder = Translator(model, dev_iter, params, device)
if greedy:
print("Doing Greedy Decoding...")
greedy_outputs = decoder.greedy_decode(max_len=100)
decoder.output_decoded_translations(greedy_outputs,
"greedy_outputs.en")
print("Evaluating BLEU Score on Greedy Tranlsation...")
subprocess.call([
'./utils/eval.sh', params.model_dir + "outputs/greedy_outputs.en"
])
if beam_size:
print("Doing Beam Search...")
beam_search_outputs = decoder.beam_decode(beam_width=beam_size)
decoder.output_decoded_translations(
beam_search_outputs,
"beam_search_outputs_size={}.en".format(beam_size))
print("Evaluating BLEU Score on Beam Search Translation")
subprocess.call([
'./utils/eval.sh', params.model_dir +
"outputs/beam_search_outputs_size={}.en".format(beam_size)
])