def test_save_model(model_name: str, fake_model: Model): fake_model.__asf_model_name = f"{model_name}:some_old_tag" save_model(fake_model, "test_save_model") model = load_model(f"{model_name}:test_save_model") verify_model_layer_equality(model, fake_model)
def test_save_model_no_dir(model_name: str, tmpdir: py.path.local, fake_model: Model): shutil.rmtree(tmpdir.join("models")) fake_model.__asf_model_name = f"{model_name}:some_old_tag" save_model(fake_model, "test_save_model") assert tmpdir.join("models").check(dir=1)
def test_save_model_with_history(model_name: str, fake_model: Model, new_history: History): fake_model.__asf_model_name = f"{model_name}:some_old_tag" save_model(fake_model, "test_save_model", history=new_history) model = load_model(f"{model_name}:test_save_model") verify_model_layer_equality(model, fake_model) assert model.__asf_model_history == new_history
model.to(device) logging.info("initialized BERT-model") num_train_steps = int( len(train_examples) / Config.TRAIN_BATCH_SIZE) * Config.NUM_EPOCHS optimizer = build_optimizer(model, num_train_steps, Config.LEARNING_RATE, Config.WARMUP_PROPORTION, Config.WEIGHT_DECAY) logging.info("Built optimizer: {}".format(optimizer)) eval_examples = processor.get_dev_examples(Config.DATA_DIR) evaluation_data_loader = load_data(eval_examples, label_list, tokenizer, Config.MAX_SEQ_LENGTH, Config.EVAL_BATCH_SIZE, DataType.EVALUATION) evaluation = Evaluation(evaluation_data_loader, exp_name, Config.MODEL_OUTPUT_DIR) logging.info("loaded and initialized evaluation examples {}".format( len(eval_examples))) training = Training() training_data_loader = load_data(train_examples, label_list, tokenizer, Config.MAX_SEQ_LENGTH, Config.TRAIN_BATCH_SIZE, DataType.TRAINING) training.fit(device, training_data_loader, model, optimizer, evaluation, Config.NUM_EPOCHS) save_model(model, exp_name, Config.MODEL_OUTPUT_DIR)
def main(): torch.manual_seed(317) torch.backends.cudnn.benckmark = True train_logger = Logger(opt, "train") val_logger = Logger(opt, "val") start_epoch = 0 print('Creating model...') model = get_model(opt.arch, opt.heads).to(opt.device) optimizer = torch.optim.Adam(model.parameters(), opt.lr) criterion = CtdetLoss(opt) print('Loading model...') if opt.load_model != '': model, optimizer, start_epoch = load_model( model, opt.load_model, optimizer, opt.lr, opt.lr_step) model = torch.nn.DataParallel(model) # amp scaler = GradScaler() print('Setting up data...') train_dataset = Dataset(opt, 'train') val_dataset = Dataset(opt, 'val') train_loader = torch.utils.data.DataLoader( train_dataset, batch_size=opt.batch_size, shuffle=True, num_workers=16, pin_memory=True, drop_last=True ) val_loader = torch.utils.data.DataLoader( val_dataset, batch_size=1, shuffle=False, num_workers=1, pin_memory=True ) # cal left time time_stats = TimeMeter(opt.num_epochs, len(train_loader)) for epoch in range(start_epoch + 1, opt.num_epochs + 1): print('train...') train(model, train_loader, criterion, optimizer, train_logger, opt, epoch, scaler, time_stats) if epoch % opt.val_intervals == 0: print('val...') val(model, val_loader, criterion, val_logger, opt, epoch) save_model(os.path.join(opt.save_dir, f'model_{epoch}.pth'), epoch, model, optimizer) # update learning rate if epoch in opt.lr_step: lr = opt.lr * (0.1 ** (opt.lr_step.index(epoch) + 1)) print('Drop LR to', lr) for param_group in optimizer.param_groups: param_group['lr'] = lr # without optimizer save_model(os.path.join(opt.save_dir, 'model_final.pth'), epoch, model)
args.adam_eps, args.warmup_steps, args.weight_decay) logging.info("Built optimizer: {}".format(optimizer)) eval_examples = processor.get_test_examples(args.data_path) evaluation_data_loader = load_data(eval_examples, label_list, tokenizer, args.max_seq_length, args.eval_batch_size, DataType.EVALUATION, args.model_type) evaluation = Evaluation(evaluation_data_loader, exp_name, args.model_output_dir, len(label_list), args.model_type) logging.info("loaded and initialized evaluation examples {}".format(len(eval_examples))) train(device, training_data_loader, model, optimizer, scheduler, evaluation, args.num_epochs, args.max_grad_norm, args.save_model_after_epoch, experiment_name=exp_name, output_dir=args.model_output_dir, model_type=args.model_type) save_model(model, exp_name, args.model_output_dir, tokenizer=tokenizer)
def train_model(config_filename, train_data, vocabulary, char_set, targets, additionals, output_sizes): config = Config() config.load(config_filename) use_cuda = torch.cuda.is_available() print("Use cuda: ", use_cuda) random.seed(config.seed) torch.manual_seed(config.seed) if use_cuda: torch.cuda.manual_seed_all(config.seed) torch.backends.cudnn.deterministic = True task_key = config.competition + "-" + config.task_type config.model_config.word_vocabulary_size = vocabulary.size() config.model_config.char_count = len(char_set) if config.model_config.use_pos: gram_vector_size = len(train_data.reviews[0].sentences[0][0].vector) config.model_config.gram_vector_size = gram_vector_size config.model_config.output_size = output_sizes[task_key] config.save(config_filename) model = RemotionRNN(config.model_config) if config.model_config.use_word_embeddings and config.use_pretrained_embeddings: embeddings = get_embeddings(vocabulary, config.embeddings_filename, config.model_config.word_embedding_dim) model.embedding.weight = torch.nn.Parameter(embeddings, requires_grad=config.train_embeddings) model = model.cuda() if use_cuda else model print(model) if config.optimizer == "adam": optimizer = optim.Adam(filter(lambda p: p.requires_grad, model.parameters()), lr=config.opt_lr) elif config.optimizer == "adadelta": optimizer = optim.Adadelta(filter(lambda p: p.requires_grad, model.parameters()), lr=config.opt_lr, rho=config.opt_rho, eps=config.opt_eps) else: assert False n = len(train_data.reviews) train_size = 1 - config.val_size border = int(n*train_size) reviews = random.Random(config.seed).sample(train_data.reviews, n) train_reviews, val_reviews = reviews[:border], reviews[border:] target_function = targets[task_key] additional_function = additionals[task_key] prev_val_loss = float("inf") bad_count = 0 for epoch in range(config.epochs): train_loss = 0 train_count = 0 train_batches = get_batches(train_reviews, vocabulary, char_set, config.batch_size, config.model_config.word_max_count, config.model_config.char_max_word_length, target_function, additional_function, config.model_config.use_pos) for batch in train_batches: model.train() loss = process_batch(model, batch, optimizer) train_loss += loss train_count += 1 val_loss = 0 val_count = 0 val_batches = get_batches(val_reviews, vocabulary, char_set, config.batch_size, config.model_config.word_max_count, config.model_config.char_max_word_length, target_function, additional_function, config.model_config.use_pos) for batch in val_batches: model.eval() loss = process_batch(model, batch, None) val_loss += loss val_count += 1 print("Epoch: {epoch}, train loss: {train_loss}, val loss: {val_loss}".format( epoch=epoch, train_loss=train_loss/train_count, val_loss=val_loss/val_count )) if prev_val_loss < val_loss: bad_count += 1 else: save_model(model, optimizer, config.model_filename) config.save(config_filename) bad_count = 0 prev_val_loss = val_loss if bad_count == config.patience+1: break return model
def main(): num_train_imgs = len(train_dataset) for epoch in range(N_EPOCHS): epoch += LOAD_MODEL_EPOCH now_time = datetime.datetime.now(pytz.timezone('Asia/Tokyo')) start_time = time.time() GF_scheduler.step() Da_scheduler.step() Db_scheduler.step() print('-----------------------------------------------------------') print(f'Epoch {epoch + 1} / {N_EPOCHS + LOAD_MODEL_EPOCH}') print(f'Time: {now_time}') print('-----------------------------------------------------------\n') g_loss, d_loss = train(train_loader, G, F, Da, Db, GF_optimizer, Da_optimizer, Db_optimizer, criterion_gan_g, criterion_gan_d_real, criterion_gan_d_fake, criterion_cycle, criterion_identity, DEVICE, LAMBDA_CYCLE, BATCH_SIZE) if (epoch + 1) % MODEL_IMG_SAVE_EPOCH == 0 or epoch == 0: valid(valid_loader, G, F, PRED_VAL_A_TO_B_DIR, PRED_VAL_B_TO_A_DIR, DEVICE, epoch) g_loss /= num_train_imgs d_loss /= num_train_imgs end_time = time.time() print(f'epoch: {epoch+1}') print(f'time: {(end_time - start_time):.3f}sec.\n') print(f'train_G_loss: {g_loss:.5f}') print(f'train_D_loss: {d_loss:.5f}\n') g_lr = GF_scheduler.get_lr()[0] print(f'G lr: {g_lr:.9f}') d_lr = Da_scheduler.get_lr()[0] print(f'D lr: {d_lr:.9f}\n\n') log_epoch = { 'epoch': epoch + 1, 'train_G_loss': g_loss, 'train_D_loss': d_loss, 'g_lr': g_lr, 'd_lr': d_lr, } logs.append(log_epoch) df = pd.DataFrame(logs) df.to_csv(f'{LOG_DIR}/log.csv', index=False) if (epoch + 1) % MODEL_IMG_SAVE_EPOCH == 0 or epoch == 0: save_model(epoch, G, GF_optimizer, GF_scheduler, MODEL_G_DIR) save_model(epoch, F, GF_optimizer, GF_scheduler, MODEL_F_DIR) save_model(epoch, Da, Da_optimizer, Da_scheduler, MODEL_DA_DIR) save_model(epoch, Db, Db_optimizer, Db_scheduler, MODEL_DB_DIR) df = pd.read_csv(f'{LOG_DIR}/log.csv') plt.figure() plt.plot(df['train_D_loss'], label='Discriminator Loss', color='blue') plt.plot(df['train_G_loss'], label='Generator Loss', color='red') plt.legend(bbox_to_anchor=(0, -0.1), loc='upper left', borderaxespad=0) plt.savefig(f'{FIGURE_DIR}/loss.png', bbox_inches='tight') plt.close() plt.figure() plt.plot(df['d_lr'], label='Discriminator LR', color='blue') plt.plot(df['g_lr'], label='Generator LR', color='red') plt.legend(bbox_to_anchor=(0, -0.1), loc='upper left', borderaxespad=0) plt.savefig(f'{FIGURE_DIR}/lr.png', bbox_inches='tight') plt.close()
print('Creating the model graph...') model = siamese_network(( Common.MAX_LEN, Common.EMBEDDING_SHAPE[0], )) model.summary() print('Compiling the model...') # Compile the model lr = 0.001 opt = tf.keras.optimizers.Adam(learning_rate=lr) model.compile(loss='categorical_crossentropy', optimizer='RMSprop', metrics=['accuracy']) print('About to train...') # Train the model model.fit(x=[X_train1, X_train2], y=Y_train, batch_size=64, epochs=50, validation_data=([X_val[0], X_val[1]], Y_val)) # Test the model results = model.evaluate([X_test1, X_test2], Y_test, batch_size=16) print('test loss, test acc: ', results) # Save the model model_name = 'Softmax-LSTM-_epochs_loss' save_model(model, model_name)