def test_save_model(model_name: str, fake_model: Model):
fake_model.__asf_model_name = f"{model_name}:some_old_tag"
save_model(fake_model, "test_save_model")
model = load_model(f"{model_name}:test_save_model")
verify_model_layer_equality(model, fake_model)
def test_save_model_no_dir(model_name: str, tmpdir: py.path.local,
fake_model: Model):
shutil.rmtree(tmpdir.join("models"))
fake_model.__asf_model_name = f"{model_name}:some_old_tag"
save_model(fake_model, "test_save_model")
assert tmpdir.join("models").check(dir=1)
def test_save_model_with_history(model_name: str, fake_model: Model,
new_history: History):
fake_model.__asf_model_name = f"{model_name}:some_old_tag"
save_model(fake_model, "test_save_model", history=new_history)
model = load_model(f"{model_name}:test_save_model")
verify_model_layer_equality(model, fake_model)
assert model.__asf_model_history == new_history
model.to(device)
logging.info("initialized BERT-model")
num_train_steps = int(
len(train_examples) / Config.TRAIN_BATCH_SIZE) * Config.NUM_EPOCHS
optimizer = build_optimizer(model, num_train_steps, Config.LEARNING_RATE,
Config.WARMUP_PROPORTION, Config.WEIGHT_DECAY)
logging.info("Built optimizer: {}".format(optimizer))
eval_examples = processor.get_dev_examples(Config.DATA_DIR)
evaluation_data_loader = load_data(eval_examples, label_list, tokenizer,
Config.MAX_SEQ_LENGTH,
Config.EVAL_BATCH_SIZE,
DataType.EVALUATION)
evaluation = Evaluation(evaluation_data_loader, exp_name,
Config.MODEL_OUTPUT_DIR)
logging.info("loaded and initialized evaluation examples {}".format(
len(eval_examples)))
training = Training()
training_data_loader = load_data(train_examples, label_list, tokenizer,
Config.MAX_SEQ_LENGTH,
Config.TRAIN_BATCH_SIZE,
DataType.TRAINING)
training.fit(device, training_data_loader, model, optimizer, evaluation,
Config.NUM_EPOCHS)
save_model(model, exp_name, Config.MODEL_OUTPUT_DIR)
def main():
torch.manual_seed(317)
torch.backends.cudnn.benckmark = True
train_logger = Logger(opt, "train")
val_logger = Logger(opt, "val")
start_epoch = 0
print('Creating model...')
model = get_model(opt.arch, opt.heads).to(opt.device)
optimizer = torch.optim.Adam(model.parameters(), opt.lr)
criterion = CtdetLoss(opt)
print('Loading model...')
if opt.load_model != '':
model, optimizer, start_epoch = load_model(
model, opt.load_model, optimizer, opt.lr, opt.lr_step)
model = torch.nn.DataParallel(model)
# amp
scaler = GradScaler()
print('Setting up data...')
train_dataset = Dataset(opt, 'train')
val_dataset = Dataset(opt, 'val')
train_loader = torch.utils.data.DataLoader(
train_dataset,
batch_size=opt.batch_size,
shuffle=True,
num_workers=16,
pin_memory=True,
drop_last=True
)
val_loader = torch.utils.data.DataLoader(
val_dataset,
batch_size=1,
shuffle=False,
num_workers=1,
pin_memory=True
)
# cal left time
time_stats = TimeMeter(opt.num_epochs, len(train_loader))
for epoch in range(start_epoch + 1, opt.num_epochs + 1):
print('train...')
train(model, train_loader, criterion, optimizer,
train_logger, opt, epoch, scaler, time_stats)
if epoch % opt.val_intervals == 0:
print('val...')
val(model, val_loader, criterion, val_logger, opt, epoch)
save_model(os.path.join(opt.save_dir, f'model_{epoch}.pth'),
epoch, model, optimizer)
# update learning rate
if epoch in opt.lr_step:
lr = opt.lr * (0.1 ** (opt.lr_step.index(epoch) + 1))
print('Drop LR to', lr)
for param_group in optimizer.param_groups:
param_group['lr'] = lr
# without optimizer
save_model(os.path.join(opt.save_dir, 'model_final.pth'), epoch, model)
args.adam_eps,
args.warmup_steps,
args.weight_decay)
logging.info("Built optimizer: {}".format(optimizer))
eval_examples = processor.get_test_examples(args.data_path)
evaluation_data_loader = load_data(eval_examples,
label_list,
tokenizer,
args.max_seq_length,
args.eval_batch_size,
DataType.EVALUATION, args.model_type)
evaluation = Evaluation(evaluation_data_loader, exp_name, args.model_output_dir, len(label_list), args.model_type)
logging.info("loaded and initialized evaluation examples {}".format(len(eval_examples)))
train(device,
training_data_loader,
model,
optimizer,
scheduler,
evaluation,
args.num_epochs,
args.max_grad_norm,
args.save_model_after_epoch,
experiment_name=exp_name,
output_dir=args.model_output_dir,
model_type=args.model_type)
save_model(model, exp_name, args.model_output_dir, tokenizer=tokenizer)
def train_model(config_filename, train_data, vocabulary, char_set, targets,
additionals, output_sizes):
config = Config()
config.load(config_filename)
use_cuda = torch.cuda.is_available()
print("Use cuda: ", use_cuda)
random.seed(config.seed)
torch.manual_seed(config.seed)
if use_cuda:
torch.cuda.manual_seed_all(config.seed)
torch.backends.cudnn.deterministic = True
task_key = config.competition + "-" + config.task_type
config.model_config.word_vocabulary_size = vocabulary.size()
config.model_config.char_count = len(char_set)
if config.model_config.use_pos:
gram_vector_size = len(train_data.reviews[0].sentences[0][0].vector)
config.model_config.gram_vector_size = gram_vector_size
config.model_config.output_size = output_sizes[task_key]
config.save(config_filename)
model = RemotionRNN(config.model_config)
if config.model_config.use_word_embeddings and config.use_pretrained_embeddings:
embeddings = get_embeddings(vocabulary, config.embeddings_filename,
config.model_config.word_embedding_dim)
model.embedding.weight = torch.nn.Parameter(embeddings, requires_grad=config.train_embeddings)
model = model.cuda() if use_cuda else model
print(model)
if config.optimizer == "adam":
optimizer = optim.Adam(filter(lambda p: p.requires_grad, model.parameters()), lr=config.opt_lr)
elif config.optimizer == "adadelta":
optimizer = optim.Adadelta(filter(lambda p: p.requires_grad, model.parameters()),
lr=config.opt_lr, rho=config.opt_rho, eps=config.opt_eps)
else:
assert False
n = len(train_data.reviews)
train_size = 1 - config.val_size
border = int(n*train_size)
reviews = random.Random(config.seed).sample(train_data.reviews, n)
train_reviews, val_reviews = reviews[:border], reviews[border:]
target_function = targets[task_key]
additional_function = additionals[task_key]
prev_val_loss = float("inf")
bad_count = 0
for epoch in range(config.epochs):
train_loss = 0
train_count = 0
train_batches = get_batches(train_reviews, vocabulary, char_set, config.batch_size,
config.model_config.word_max_count, config.model_config.char_max_word_length,
target_function, additional_function, config.model_config.use_pos)
for batch in train_batches:
model.train()
loss = process_batch(model, batch, optimizer)
train_loss += loss
train_count += 1
val_loss = 0
val_count = 0
val_batches = get_batches(val_reviews, vocabulary, char_set, config.batch_size,
config.model_config.word_max_count, config.model_config.char_max_word_length,
target_function, additional_function, config.model_config.use_pos)
for batch in val_batches:
model.eval()
loss = process_batch(model, batch, None)
val_loss += loss
val_count += 1
print("Epoch: {epoch}, train loss: {train_loss}, val loss: {val_loss}".format(
epoch=epoch,
train_loss=train_loss/train_count,
val_loss=val_loss/val_count
))
if prev_val_loss < val_loss:
bad_count += 1
else:
save_model(model, optimizer, config.model_filename)
config.save(config_filename)
bad_count = 0
prev_val_loss = val_loss
if bad_count == config.patience+1:
break
return model
def main():
num_train_imgs = len(train_dataset)
for epoch in range(N_EPOCHS):
epoch += LOAD_MODEL_EPOCH
now_time = datetime.datetime.now(pytz.timezone('Asia/Tokyo'))
start_time = time.time()
GF_scheduler.step()
Da_scheduler.step()
Db_scheduler.step()
print('-----------------------------------------------------------')
print(f'Epoch {epoch + 1} / {N_EPOCHS + LOAD_MODEL_EPOCH}')
print(f'Time: {now_time}')
print('-----------------------------------------------------------\n')
g_loss, d_loss = train(train_loader, G, F, Da, Db, GF_optimizer,
Da_optimizer, Db_optimizer, criterion_gan_g,
criterion_gan_d_real, criterion_gan_d_fake,
criterion_cycle, criterion_identity, DEVICE,
LAMBDA_CYCLE, BATCH_SIZE)
if (epoch + 1) % MODEL_IMG_SAVE_EPOCH == 0 or epoch == 0:
valid(valid_loader, G, F, PRED_VAL_A_TO_B_DIR, PRED_VAL_B_TO_A_DIR,
DEVICE, epoch)
g_loss /= num_train_imgs
d_loss /= num_train_imgs
end_time = time.time()
print(f'epoch: {epoch+1}')
print(f'time: {(end_time - start_time):.3f}sec.\n')
print(f'train_G_loss: {g_loss:.5f}')
print(f'train_D_loss: {d_loss:.5f}\n')
g_lr = GF_scheduler.get_lr()[0]
print(f'G lr: {g_lr:.9f}')
d_lr = Da_scheduler.get_lr()[0]
print(f'D lr: {d_lr:.9f}\n\n')
log_epoch = {
'epoch': epoch + 1,
'train_G_loss': g_loss,
'train_D_loss': d_loss,
'g_lr': g_lr,
'd_lr': d_lr,
}
logs.append(log_epoch)
df = pd.DataFrame(logs)
df.to_csv(f'{LOG_DIR}/log.csv', index=False)
if (epoch + 1) % MODEL_IMG_SAVE_EPOCH == 0 or epoch == 0:
save_model(epoch, G, GF_optimizer, GF_scheduler, MODEL_G_DIR)
save_model(epoch, F, GF_optimizer, GF_scheduler, MODEL_F_DIR)
save_model(epoch, Da, Da_optimizer, Da_scheduler, MODEL_DA_DIR)
save_model(epoch, Db, Db_optimizer, Db_scheduler, MODEL_DB_DIR)
df = pd.read_csv(f'{LOG_DIR}/log.csv')
plt.figure()
plt.plot(df['train_D_loss'], label='Discriminator Loss', color='blue')
plt.plot(df['train_G_loss'], label='Generator Loss', color='red')
plt.legend(bbox_to_anchor=(0, -0.1), loc='upper left', borderaxespad=0)
plt.savefig(f'{FIGURE_DIR}/loss.png', bbox_inches='tight')
plt.close()
plt.figure()
plt.plot(df['d_lr'], label='Discriminator LR', color='blue')
plt.plot(df['g_lr'], label='Generator LR', color='red')
plt.legend(bbox_to_anchor=(0, -0.1), loc='upper left', borderaxespad=0)
plt.savefig(f'{FIGURE_DIR}/lr.png', bbox_inches='tight')
plt.close()
print('Creating the model graph...')
model = siamese_network((
Common.MAX_LEN,
Common.EMBEDDING_SHAPE[0],
))
model.summary()
print('Compiling the model...')
# Compile the model
lr = 0.001
opt = tf.keras.optimizers.Adam(learning_rate=lr)
model.compile(loss='categorical_crossentropy',
optimizer='RMSprop',
metrics=['accuracy'])
print('About to train...')
# Train the model
model.fit(x=[X_train1, X_train2],
y=Y_train,
batch_size=64,
epochs=50,
validation_data=([X_val[0], X_val[1]], Y_val))
# Test the model
results = model.evaluate([X_test1, X_test2], Y_test, batch_size=16)
print('test loss, test acc: ', results)
# Save the model
model_name = 'Softmax-LSTM-_epochs_loss'
save_model(model, model_name)