def train(): ds = get_ds(BATCH_SIZE) gan = InfoGAN(RAND_DIM, STYLE_DIM, LABEL_DIM, IMG_SHAPE, FIX_STD, STYLE_SCALE) t0 = time.time() for ep in range(EPOCH): for t, (real_img, real_img_label) in enumerate(ds): g_img, d_loss, d_bool_acc, g_loss, g_bool_acc, g_img_label, d_class_acc = gan.step( real_img, real_img_label) if t % 400 == 0: t1 = time.time() print( "ep={} | time={:.1f}|t={}|d_acc={:.2f}|d_classacc={:.2f}|g_acc={:.2f}|d_loss={:.2f}|g_loss={:.2f}" .format( ep, t1 - t0, t, d_bool_acc.numpy(), g_bool_acc.numpy(), d_class_acc.numpy(), d_loss.numpy(), g_loss.numpy(), )) t0 = t1 save_gan(gan, ep) save_weights(gan)
def test(G, D, G_ema, z_, y_, state_dict, config, sample, get_inception_metrics, experiment_name, test_log): print('Gathering inception metrics...', ) if config['accumulate_stats']: utils.accumulate_standing_stats(G_ema if config['ema'] and config['use_ema'] else G, z_, y_, config['n_classes'], config['num_standing_accumulations']) IS_mean, IS_std, FID = get_inception_metrics(sample, step=state_dict['shown_images'], num_inception_images=config['num_inception_images'], num_splits=10) print('shown_images %d: Inception Score is %3.3f +/- %3.3f, FID is %5.4f' % (state_dict['shown_images'], IS_mean, IS_std, FID), ) # If improved over previous best metric, save approrpiate copy if not math.isnan(IS_mean) and not math.isnan(FID)\ and (IS_mean > state_dict['best_IS']): print('%s improved over previous best, saving checkpoint...' % 'IS', ) utils.save_weights(G, D, state_dict, config['weights_root'], experiment_name, 'best_IS%d' % state_dict['save_best_num'], G_ema if config['ema'] else None) state_dict['save_best_num'] = (state_dict['save_best_num'] + 1 ) % config['num_best_copies'] if not math.isnan(IS_mean) and not math.isnan(FID)\ and (FID < state_dict['best_FID']): print('%s improved over previous best, saving checkpoint...' % 'FID', ) utils.save_weights(G, D, state_dict, config['weights_root'], experiment_name, 'best_FID%d' % state_dict['save_best_num'], G_ema if config['ema'] else None) state_dict['save_best_num'] = (state_dict['save_best_num'] + 1) % config['num_best_copies'] state_dict['best_IS'] = max(state_dict['best_IS'], IS_mean) state_dict['best_FID'] = min(state_dict['best_FID'], FID) # Log results to file test_log.log(itr=int(state_dict['itr']), IS_mean=float(IS_mean), IS_std=float(IS_std), FID=float(FID)) return IS_mean, IS_std, FID
def main(args): sentences, index2word = utils.load_corpus(corpus_file=args.corpus) vocab_size = len(index2word) # create input couples, labels = utils.skip_grams(sentences, args.window_size, vocab_size) print('Shape of couples: ' + str(couples.shape)) print('Shape of labels: ' + str(labels.shape)) # metrics nb_batch = len(labels) // args.batch_size samples_per_epoch = args.batch_size * nb_batch model = build_model(vocab_size, args.vec_dim, args.batch_size) if (args.multi_gpu): model = multi_gpu_model(model) opt = RMSprop(lr=5e-4, decay=5e-6) checkpoint = ModelCheckpoint(os.path.join(args.ckpt_path, 'Word2Vec_{epoch:03d}.h5'), period=args.ckpt_period, save_weights_only=True) early_stop = EarlyStopping(monitor='loss', patience=10) model.compile(optimizer=opt, loss='mse', metrics=['accuracy']) model.fit_generator(generator=batch_generator(couples, labels, args.batch_size, nb_batch), steps_per_epoch=samples_per_epoch, epochs=args.epochs, callbacks=[checkpoint, early_stop], verbose=1) # save weights utils.save_weights(model, index2word, args.vec_dim)
def main(args): # 数据加载 (x_train, y_train), (x_test, y_test) = load_cifar(args.cifar_root) # 随机选择训练样本 train_num = x_train.shape[0] def next_batch(batch_size): idx = np.random.choice(train_num, batch_size) return x_train[idx], y_train[idx] # 网络 vgg = VGG(image_size=32, name='vgg11') opt = RmsProp(vgg.weights, lr=args.lr, decay=1e-3) # 加载权重 if args.checkpoint: weights = load_weights(args.checkpoint) vgg.load_weights(weights) print("load weights done") # 评估 if args.eval_only: indices = np.random.choice(len(x_test), args.eval_num, replace=False) print('{} start evaluate'.format( time.asctime(time.localtime(time.time())))) acc = get_accuracy(vgg, x_test[indices], ys=y_test[indices]) print('{} acc on test dataset is :{:.3f}'.format( time.asctime(time.localtime(time.time())), acc)) return # 训练 num_steps = args.steps for step in range(num_steps): x, y_true = next_batch(args.batch_size) # 前向传播 y_predict = vgg.forward(x.astype(np.float)) # print('y_pred: min{},max{},mean:{}'.format(np.min(y_predict, axis=-1), # np.max(y_predict, axis=-1), # np.mean(y_predict, axis=-1))) # print('y_pred: {}'.format(y_predict)) acc = np.mean( np.argmax(y_predict, axis=1) == np.argmax(y_true, axis=1)) # 计算loss loss, gradient = cross_entropy_loss(y_predict, y_true) # 反向传播 vgg.backward(gradient) # 更新梯度 opt.iterate(vgg) # 打印信息 print('{} step:{},loss:{:.4f},acc:{:.4f}'.format( time.asctime(time.localtime(time.time())), step, loss, acc)) # 保存权重 if step % 100 == 0: save_weights( os.path.join(args.save_dir, 'weights-{:03d}.pkl'.format(step)), vgg.weights)
def test(G, D, G_ema, z_, y_, state_dict, config, sample, get_inception_metrics, experiment_name, test_log): print('Gathering inception metrics...') if config['accumulate_stats']: utils.accumulate_standing_stats( G_ema if config['ema'] and config['use_ema'] else G, z_, y_, config['n_classes'], config['num_standing_accumulations']) IS_mean, IS_std, FID = get_inception_metrics( sample, config['num_inception_images'], num_splits=10) print( 'Itr %d: PYTORCH UNOFFICIAL Inception Score is %3.3f +/- %3.3f, PYTORCH UNOFFICIAL FID is %5.4f' % (state_dict['itr'], IS_mean, IS_std, FID)) # If improved over previous best metric, save approrpiate copy if ((config['which_best'] == 'IS' and IS_mean > state_dict['best_IS']) or (config['which_best'] == 'FID' and FID < state_dict['best_FID'])): print('%s improved over previous best, saving checkpoint...' % config['which_best']) utils.save_weights(G, D, state_dict, config['weights_root'], experiment_name, 'best%d' % state_dict['save_best_num'], G_ema if config['ema'] else None) state_dict['save_best_num'] = (state_dict['save_best_num'] + 1) % config['num_best_copies'] state_dict['best_IS'] = max(state_dict['best_IS'], IS_mean) state_dict['best_FID'] = min(state_dict['best_FID'], FID) # Log results to file test_log.log(itr=int(state_dict['itr']), IS_mean=float(IS_mean), IS_std=float(IS_std), FID=float(FID))
def eval_model(engine): nonlocal best_val_loss, nb_epoch_no_improvement def log_progress(mode, metrics_values): metrics_str = ', '.join([ f'{metric_name} {metrics_values[metric_name]:.3f}' for metric_name, _ in metrics ]) print(f'{mode}: {metrics_str}', end=' | ') # evaluator.run(data_loader_train) # metrics_train = evaluator.state.metrics.copy() evaluator.run(data_loader_dev) metrics_dev = evaluator.state.metrics.copy() print(f'Epoch {engine.state.epoch:>3}', end=' | ') # log_progress('train', metrics_train) log_progress('dev', metrics_dev) if best_val_loss > metrics_dev[ 'loss'] or engine.state.epoch <= cfg.min_epochs: best_val_loss = metrics_dev['loss'] nb_epoch_no_improvement = 0 save_weights(model, os.path.join(CACHE_DIR, f'model_{cfg.model}.pt')) print('Model saved', end=' ') else: nb_epoch_no_improvement += 1 if cfg.early_stopping_patience != 0 and nb_epoch_no_improvement > cfg.early_stopping_patience: trainer.terminate() print()
def main(): args = parse_args() path_to_x_train = args.x_train_dir path_to_y_train = args.y_train_dir path_to_model = args.model_output_dir verbosity = args.verbosity iter_num = args.iter_num batch_size = args.batch_size X_original = read_mnist(path_to_x_train) y_original = read_mnist(path_to_y_train) X, image_shape = preprocessing_data(X_original) y = y_original print( f'\nbatch_size: {batch_size}, iter_num: {iter_num}, kernel: {args.kernel}\n' ) X_train, X_val, y_train, y_val = X[:50000], X[50000:], y[:50000], y[50000:] clf = MySvm(args.kernel, image_shape=image_shape) clf.fit(X_train, y_train, iter_num=iter_num, batch_size=batch_size, verbosity=verbosity) prediction_labels = clf.predict(X_val) print(classification_report(y_val, prediction_labels, digits=4)) optimal_weights = clf.get_weights() print(f'Saving model to {path_to_model}') save_weights(path_to_model, optimal_weights)
def train(name, gen, disc, gen_train_ratio=5, epochs=-1): device = utils.get_device() dataloader = data.get_dataloader() loss_func = nn.BCELoss() g_optimizer = to.Adam(gen.parameters(), lr=.0003, betas=(.5, .9)) d_optimizer = to.Adam(disc.parameters(), lr=.0003, betas=(.5, .9)) iter_axis = [] g_loss_axis = [] d_loss_axis = [] fixed_noise = torch.randn(64, gen.nz, device=device) epoch_iterator = range(epochs) if epochs >= 0 else count(0) iters = 0 for epoch in epoch_iterator: for i, batch in enumerate(dataloader): loss_disc, acc_real, acc_fake = train_discriminator( gen, disc, batch, loss_func, d_optimizer) for _ in range(gen_train_ratio): loss_gen, acc_gen = train_generator(gen, disc, loss_func, g_optimizer) # Training stats if i % 50 == 0: print( f'[{epoch:2d}/{epochs}][{i:3d}/{len(dataloader)}]\t' +\ f'Loss_D: {loss_disc:3.4f}\tLoss_G: {loss_gen:3.4f}\t' +\ f'D(x): {acc_real:3.4f}\tD(G(z)): {acc_fake:3.4f} / {acc_gen:3.4f}' ) if (iters % 10 == 0) or ((epoch == epochs - 1) and (i == len(dataloader) - 1)): iter_axis.append(iters) g_loss_axis.append(loss_gen) d_loss_axis.append(loss_disc) # Save output if (iters % 500 == 0) or ((epoch == epochs - 1) and (i == len(dataloader) - 1)): print('Saving images...') with torch.no_grad(): gen.mixing = False fake = gen(fixed_noise).detach().cpu() gen.mixing = True utils.save_image(utils.make_torch_grid(fake), f'{name}_{iters}.png') # Save weights if (iters % 1000 == 0) or ((epoch == epochs - 1) and (i == len(dataloader) - 1)): print('Saving weights...') utils.save_weights(gen, f'{name}_gen.pt') utils.save_weights(disc, f'{name}_disc.pt') iters += 1 utils.plot_losses(iter_axis, g_loss_axis, d_loss_axis)
def save_and_sample(G, D, G_ema, z_, y_, fixed_z, fixed_y, state_dict, config, experiment_name): utils.save_weights(G, D, state_dict, config['weights_root'], experiment_name, None, G_ema if config['ema'] else None) # Save an additional copy to mitigate accidental corruption if process # is killed during a save (it's happened to me before -.-) if config['num_save_copies'] > 0: utils.save_weights(G, D, state_dict, config['weights_root'], experiment_name, 'copy%d' % state_dict['save_num'], G_ema if config['ema'] else None) state_dict['save_num'] = ( state_dict['save_num'] + 1) % config['num_save_copies'] # Use EMA G for samples or non-EMA? which_G = G_ema if config['ema'] and config['use_ema'] else G # Accumulate standing statistics? if config['accumulate_stats']: utils.accumulate_standing_stats(G_ema if config['ema'] and config['use_ema'] else G, z_, y_, config['n_classes'], config['num_standing_accumulations']) # Save a random sample sheet with fixed z and y with torch.no_grad(): if config['parallel']: fixed_Gz = nn.parallel.data_parallel( which_G, (fixed_z, which_G.shared(fixed_y))) else: fixed_Gz = which_G(fixed_z, which_G.shared(fixed_y)) if not os.path.isdir('%s/%s' % (config['samples_root'], experiment_name)): os.mkdir('%s/%s' % (config['samples_root'], experiment_name)) image_filename = '%s/%s/fixed_samples%d.jpg' % (config['samples_root'], experiment_name, state_dict['itr']) torchvision.utils.save_image(fixed_Gz.float().cpu(), image_filename, nrow=int(fixed_Gz.shape[0] ** 0.5), normalize=True) # For now, every time we save, also save sample sheets utils.sample_sheet(which_G, classes_per_sheet=utils.classes_per_sheet_dict[config['dataset']], num_classes=config['n_classes'], samples_per_class=10, parallel=config['parallel'], samples_root=config['samples_root'], experiment_name=experiment_name, folder_number=state_dict['itr'], z_=z_) # Also save interp sheets for fix_z, fix_y in zip([False, False, True], [False, True, False]): utils.interp_sheet(which_G, num_per_sheet=16, num_midpoints=8, num_classes=config['n_classes'], parallel=config['parallel'], samples_root=config['samples_root'], experiment_name=experiment_name, folder_number=state_dict['itr'], sheet_number=0, fix_z=fix_z, fix_y=fix_y, device='cuda')
def train(models, it_train, it_val, params): """ Train the model. Parameters: - models: a dictionary with all the models. - atob: a model that goes from A to B. - d: the discriminator model. - p2p: a Pix2Pix model. - it_train: the iterator of the training data. - it_val: the iterator of the validation data. - params: parameters of the training procedure. - dout_size: the size of the output of the discriminator model. """ # Create the experiment folder and save the parameters create_expt_dir(params) # Get the output shape of the discriminator dout_size = models.d.output_shape[-3:-1] # Define the data generators generators = generators_creation(it_train, it_val, models, dout_size) # Define the number of samples to use on each training epoch train_samples = params.train_samples if params.train_samples == -1: train_samples = 8 batches_per_epoch = train_samples // params.samples_per_batch # Define the number of samples to use for validation val_samples = params.val_samples if val_samples == -1: val_samples = 8 losses = {'p2p': [], 'd': [], 'p2p_val': [], 'd_val': []} if params.continue_train: losses = load_losses(log_dir=params.log_dir, expt_name=params.expt_name) for e in tqdm(range(params.epochs)): for b in range(batches_per_epoch): train_iteration(models, generators, losses, params) # Evaluate how the models is doing on the validation set. evaluate(models, generators, losses, val_samples=val_samples) if (e + 1) % params.save_every == 0: save_weights(models, log_dir=params.log_dir, expt_name=params.expt_name) log(losses, models.atob, it_val, log_dir=params.log_dir, expt_name=params.expt_name, is_a_binary=params.is_a_binary, is_b_binary=params.is_b_binary)
def test(path): (x_train, y_train), (x_test, y_test) = load_cifar(path) print(x_train[0][0]) print(y_train[0]) vgg = VGG(name='vgg11') import utils utils.save_weights('./w.pkl', vgg.weights) w = utils.load_weights('./w.pkl') print(type(w)) print(w.keys())
def train_model(model, criterion, optimizer, lr_scheduler, max_num=2, init_lr=0.001, num_epochs=100): since = time.time() best_model = model best_acc = 0.0 print(model.name) for epoch in range(num_epochs): epoch_since = time.time() print('Epoch {}/{}'.format(epoch, num_epochs - 1)) print('-' * 10) for phase in ['train', 'valid']: if phase == 'train': optimizer = lr_scheduler(optimizer, epoch, init_lr=init_lr) model.train(True) else: model.train(False) running_loss = 0.0 running_corrects = 0 for data in dset_loaders[phase]: inputs, labels = data inputs, labels = Variable(inputs.cuda()), Variable( labels.cuda()) optimizer.zero_grad() outputs = model(inputs) _, preds = torch.max(outputs.data, 1) loss = criterion(outputs, labels) if phase == 'train': loss.backward() optimizer.step() running_loss += loss.data[0] running_corrects += torch.sum(preds == labels.data) epoch_loss = running_loss / dset_sizes[phase] epoch_acc = running_corrects / dset_sizes[phase] print('{} Loss: {:.4f} Acc: {:.4f}'.format(phase, epoch_loss, epoch_acc)) if phase == 'valid': save_weights(epoch_acc, model, epoch, max_num=max_num) if phase == 'valid' and epoch_acc > best_acc: best_acc = epoch_acc #best_model = copy.deepcopy(model) #torch.save(best_model.state_dict(), w_file) print('epoch {}: {:.0f}s'.format(epoch, time.time() - epoch_since)) time_elapsed = time.time() - since print('Training complete in {:.0f}m {:.0f}s'.format( time_elapsed // 60, time_elapsed % 60)) print('Best val Acc: {:4f}'.format(best_acc)) print(w_files_training) return best_model
def save_and_sample(G, D, G_ema, z_, y_, fixed_z, fixed_y, state_dict, config, experiment_name): utils.save_weights(G, D, state_dict, config['weights_root'], experiment_name, None, G_ema if config['ema'] else None) if config['num_save_copies'] > 0: utils.save_weights(G, D, state_dict, config['weights_root'], experiment_name, 'copy%d' % state_dict['save_num'], G_ema if config['ema'] else None) state_dict['save_num'] = (state_dict['save_num'] + 1) % config['num_save_copies'] which_G = G_ema if config['ema'] and config['use_ema'] else G if config['accumulate_stats']: if not config['conditional']: y_.zero_() utils.accumulate_standing_stats( G_ema if config['ema'] and config['use_ema'] else G, z_, y_, config['n_classes'], config['num_standing_accumulations']) with torch.no_grad(): if config['parallel']: fixed_Gz = nn.parallel.data_parallel( which_G, (fixed_z, which_G.shared(fixed_y))) else: fixed_Gz = which_G(fixed_z, which_G.shared(fixed_y)) if not os.path.isdir('%s/%s' % (config['samples_root'], experiment_name)): os.mkdir('%s/%s' % (config['samples_root'], experiment_name)) image_filename = '%s/%s/fixed_samples%d.jpg' % ( config['samples_root'], experiment_name, state_dict['itr']) torchvision.utils.save_image(fixed_Gz.float().cpu(), image_filename, nrow=int(fixed_Gz.shape[0]**0.5), normalize=True) utils.sample_sheet( which_G, classes_per_sheet=utils.classes_per_sheet_dict[config['dataset']], num_classes=config['n_classes'], samples_per_class=10, parallel=config['parallel'], samples_root=config['samples_root'], experiment_name=experiment_name, folder_number=state_dict['itr'], z_=z_) for fix_z, fix_y in zip([False, False, True], [False, True, False]): utils.interp_sheet(which_G, num_per_sheet=16, num_midpoints=8, num_classes=config['n_classes'], parallel=config['parallel'], samples_root=config['samples_root'], experiment_name=experiment_name, folder_number=state_dict['itr'], sheet_number=0, fix_z=fix_z, fix_y=fix_y, device='cuda')
def test(G, D, GD, G_ema, z_, y_, state_dict, config, sample, get_inception_metrics, experiment_name, test_log, acc_metrics, acc_itrs): print('Calculating validation accuracy...') D.eval() D_accuracy = [] loader = utils.get_data_loaders( **{**config, 'train': False, 'use_multiepoch_sampler': False, 'load_in_mem': False})[0] with torch.no_grad(): for x, y in loader: D_real = GD(None, None, x=x, dy=y, policy=config['DiffAugment']) D_accuracy.append((D_real > 0).float().mean().item()) D.train() D_acc_val = np.mean(D_accuracy) print('Calculating training accuracy...') D.eval() D_accuracy = [] loader = utils.get_data_loaders( **{**config, 'train': True, 'use_multiepoch_sampler': False, 'load_in_mem': False})[0] with torch.no_grad(): for x, y in loader: D_real = GD(None, None, x=x, dy=y, policy=config['DiffAugment']) D_accuracy.append((D_real > 0).float().mean().item()) D.train() D_acc_train = np.mean(D_accuracy) print('Gathering inception metrics...') if config['accumulate_stats']: utils.accumulate_standing_stats(G_ema if config['ema'] and config['use_ema'] else G, z_, y_, config['n_classes'], config['num_standing_accumulations']) IS_mean, IS_std, FID = get_inception_metrics(sample, config['num_inception_images'], num_splits=10) print('Itr %d: PYTORCH UNOFFICIAL Inception Score is %3.3f +/- %3.3f, PYTORCH UNOFFICIAL FID is %5.4f' % (state_dict['itr'], IS_mean, IS_std, FID)) # If improved over previous best metric, save approrpiate copy if ((config['which_best'] == 'IS' and IS_mean > state_dict['best_IS']) or (config['which_best'] == 'FID' and FID < state_dict['best_FID'])): print('%s improved over previous best, saving checkpoint...' % config['which_best']) utils.save_weights(G, D, state_dict, config['weights_root'], experiment_name, 'best%d' % state_dict['save_best_num'] if config['num_best_copies'] > 1 else 'best', G_ema if config['ema'] else None) state_dict['save_best_num'] = ( state_dict['save_best_num'] + 1) % config['num_best_copies'] state_dict['best_IS'] = max(state_dict['best_IS'], IS_mean) state_dict['best_FID'] = min(state_dict['best_FID'], FID) # Log results to file test_log.log(itr=int(state_dict['itr']), IS_mean=float(IS_mean), IS_std=float(IS_std), FID=float(FID), D_acc_val=D_acc_val, D_acc_train=D_acc_train, **{k: v / acc_itrs for k, v in acc_metrics.items()})
def save(G, D, G_ema, state_dict, config, experiment_name): utils.save_weights(G, D, state_dict, config['weights_root'], experiment_name, None, G_ema if config['ema'] else None) # Save an additional copy to mitigate accidental corruption if process # is killed during a save (it's happened to me before -.-) if config['num_save_copies'] > 0: utils.save_weights(G, D, state_dict, config['weights_root'], experiment_name, 'copy%d' % state_dict['save_num'], G_ema if config['ema'] else None) state_dict['save_num'] = (state_dict['save_num'] + 1 ) % config['num_save_copies']
def save_and_sample(G, D, M, G_ema, z_, y_, fixed_z, fixed_y, state_dict, config, experiment_name): utils.save_weights(G, D, M, state_dict, config['weights_root'], experiment_name, None, G_ema if config['ema'] else None) # Save an additional copy to mitigate accidental corruption if process # is killed during a save (it's happened to me before -.-) # if config['num_save_copies'] > 0: # utils.save_weights(G, D, M, state_dict, config['weights_root'], # experiment_name, # 'copy%d' % state_dict['save_num'], # G_ema if config['ema'] else None) # state_dict['save_num'] = (state_dict['save_num'] + 1 ) % config['num_save_copies'] # # Use EMA G for samples or non-EMA? which_G = G_ema if config['ema'] and config['use_ema'] else G # Accumulate standing statistics? #if config['accumulate_stats']: # utils.accumulate_standing_stats(G_ema if config['ema'] and config['use_ema'] else G, # z_, y_, config['n_classes'], # config['num_standing_accumulations']) # Save a random sample sheet with fixed z and y with torch.no_grad(): if config['parallel']: z_from_M, y_from_M = nn.parallel.data_parallel(M, (fixed_z, )) fixed_Gz = nn.parallel.data_parallel(which_G, (z_from_M, y_from_M)) #fixed_Gz = nn.parallel.data_parallel(which_G, (fixed_z, which_G.shared(fixed_y))) else: z_from_M, y_from_M = M(fixed_z) fixed_Gz = which_G(z_from_M, y_from_M) fixed_Gz = which_G(z_from_M, y_from_M) #fixed_Gz = which_G(fixed_z, which_G.shared(fixed_y)) if not os.path.isdir('%s/%s' % (config['samples_root'], experiment_name)): os.mkdir('%s/%s' % (config['samples_root'], experiment_name)) image_filename = '%s/%s/fixed_samples%d.jpg' % ( config['samples_root'], experiment_name, state_dict['itr']) torchvision.utils.save_image(fixed_Gz.float().cpu(), image_filename, nrow=int(fixed_Gz.shape[0]**0.5), normalize=True) # For now, every time we save, also save sample sheets utils.sample_sheet( which_G, M, classes_per_sheet=utils.classes_per_sheet_dict[config['dataset']], num_classes=config['n_classes'], samples_per_class=10, parallel=config['parallel'], samples_root=config['samples_root'], experiment_name=experiment_name, folder_number=state_dict['itr'], z_=z_)
def update_FID(G, D, G_ema, state_dict, config, FID, experiment_name, test_log): if ((config['which_best'] == 'IS' and IS_mean > state_dict['best_IS']) or (config['which_best'] == 'FID' and FID < state_dict['best_FID'])): utils.save_weights(G, D, state_dict, config['weights_root'], experiment_name, 'best%d' % state_dict['save_best_num'], G_ema if config['ema'] else None) state_dict['save_best_num'] = (state_dict['save_best_num'] + 1) % config['num_best_copies'] state_dict['best_FID'] = min(state_dict['best_FID'], FID) test_log.log(itr=int(state_dict['itr']), IS_mean=float(0), IS_std=float(0), FID=float(FID))
def log_training_results(engine): nonlocal best_loss # evaluator.run(data_loader_train) # losses_train = evaluator.state.metrics['loss'] evaluator.run(data_loader_val) losses_val = evaluator.state.metrics['loss'] # log_progress(trainer.state.epoch, trainer.state.iteration, losses_train, 'train', tensorboard_writer) log_progress(trainer.state.epoch, trainer.state.iteration, losses_val, 'val', tensorboard_writer) if losses_val[exp.config.best_loss] < best_loss: best_loss = losses_val[exp.config.best_loss] save_weights(model, exp.experiment_dir.joinpath('best.th'))
def main(): sentences, index2word = utils.load_sentences_brown(800) # params nb_epoch = 3 # learn `batch_size words` at a time batch_size = 128 vec_dim = 64 negsampling_num = 0.2 # half of window window_size = 6 vocab_size = len(index2word) print 'vocabulary length: ', vocab_size # create input couples, labels = utils.skip_grams(sentences, window_size, vocab_size, negsampling_num) print 'counter of positive samples and negative samples: ', Counter(labels) print 'shape of couples: ', couples.shape print 'shape of labels: ', labels.shape # metrics nb_batch = len(labels) // batch_size samples_per_epoch = batch_size * nb_batch # fit model model = make_word2vec_model(vec_dim, vocab_size) model.fit_generator(generator=utils.batch_generator( couples, labels, batch_size, nb_batch), steps_per_epoch=samples_per_epoch, epochs=nb_epoch) # save weights utils.save_weights(model, index2word, vec_dim) # eval using gensim print 'the....' utils.most_similar(positive=['the']) print 'all....' utils.most_similar(positive=['all']) print 'baby....' utils.most_similar(positive=['baby']) print 'first....' utils.most_similar(positive=['first'])
def industry_embedding_model_fit(model, embedding_model, nb_epoch, batch_size, df, dict_sequence, industry2idx, output_file): X_train, X_test, y_train, y_test, y_train_2digit, y_test_2digit = utils.get_train_test( df, batch_size, 0.1) target_grid_col = 'naics_4_digit' context_grid_col = 'context' target_col = 'target_desc' context_col = 'context_desc' model.fit(x=[ X_train[target_grid_col], X_train[context_grid_col], utils.rebuild_array(X_train[target_col]), utils.rebuild_array(X_train[context_col]) ], y=[y_train, y_train_2digit], epochs=nb_epoch, shuffle=True, batch_size=batch_size, verbose=1, validation_data=([ X_test[target_grid_col], X_test[context_grid_col], utils.rebuild_array(X_test[target_col]), utils.rebuild_array(X_test[context_col]) ], [y_test, y_test_2digit]), callbacks=[ EarlyStopping(monitor='val_loss', min_delta=0.0001, patience=3, verbose=1, mode='auto') ]) sequence_df = pd.DataFrame( {k: v for k, v in dict_sequence.items() if len(k) == SELECT_DIGIT}).T naics_idx_l = np.array([ industry2idx.get(int(naics), len(industry2idx) + 1) for naics in sequence_df.index ]) in_data = [naics_idx_l, sequence_df.values] embedding_vec = embedding_model.predict(x=in_data, verbose=1, batch_size=1) utils.save_weights(output_file, embedding_vec, naics_idx_l, idx2industry) return embedding_vec
def save_and_sample(G, D, G_ema, z_, y_, fixed_z, fixed_y, state_dict, config, experiment_name): utils.save_weights(G, D, state_dict, config['weights_root'], experiment_name, None, G_ema if config['ema'] else None) # Save an additional copy to mitigate accidental corruption if process # is killed during a save (it's happened to me before -.-) if config['num_save_copies'] > 0: utils.save_weights(G, D, state_dict, config['weights_root'], experiment_name, 'copy%d' % state_dict['save_num'], G_ema if config['ema'] else None) state_dict['save_num'] = (state_dict['save_num'] + 1) % config['num_save_copies'] # Use EMA G for samples or non-EMA? which_G = G_ema if config['ema'] and config['use_ema'] else G # Accumulate standing statistics? if config['accumulate_stats']: utils.accumulate_standing_stats( G_ema if config['ema'] and config['use_ema'] else G, z_, y_, config['n_classes'], config['num_standing_accumulations']) # Save a random sample sheet with fixed z and y with torch.no_grad(): # Get Generator output given noise if config['use_dog_cnt'] and config['G_shared']: ye0 = G.shared(fixed_y[:, 0]) ye1 = G.dog_cnt_shared(fixed_y[:, 1]) gyc = torch.cat([ye0, ye1], 1) else: gyc = G.shared(fixed_y) if config['parallel']: fixed_Gz = nn.parallel.data_parallel(which_G, (fixed_z, gyc)) else: fixed_Gz = which_G(fixed_z, gyc) if not os.path.isdir('%s/%s' % (config['samples_root'], experiment_name)): os.mkdir('%s/%s' % (config['samples_root'], experiment_name)) image_filename = '%s/%s/fixed_samples%d.jpg' % ( config['samples_root'], experiment_name, state_dict['itr']) torchvision.utils.save_image(fixed_Gz.float().cpu(), image_filename, nrow=int(fixed_Gz.shape[0]**0.5), normalize=True) # # For now, every time we save, also save sample sheets num_classes = int(np.minimum(120, config['n_classes']))
def test(G, D, G_ema, z_, y_, state_dict, config, sample, get_inception_metrics, experiment_name, test_log): print('Gathering inception metrics...') if config['accumulate_stats']: utils.accumulate_standing_stats( G_ema if config['ema'] and config['use_ema'] else G, z_, y_, config['n_classes'], config['num_standing_accumulations']) IS_mean, IS_std, FID = get_inception_metrics( sample, config['num_inception_images'], num_splits=10) if np.isnan(float(FID)): FID = 99 print( 'Itr %d: PYTORCH UNOFFICIAL Inception Score is %3.3f +/- %3.3f, PYTORCH UNOFFICIAL FID is %5.4f' % (state_dict['itr'], IS_mean, IS_std, FID)) # If improved over previous best metric, save approrpiate copy if ((config['which_best'] == 'IS' and IS_mean > state_dict['best_IS']) or (config['which_best'] == 'FID' and FID < state_dict['best_FID'])): print('%s improved over previous best, saving checkpoint...' % config['which_best']) utils.save_weights(G, D, state_dict, config['weights_root'], experiment_name, 'best%d' % state_dict['save_best_num'], G_ema if config['ema'] else None) utils.save_weights(G, D, state_dict, config['weights_root'], experiment_name, 'currentbest', G_ema if config['ema'] else None) ###early exit when reaching the goal # if IS_mean>8.22 and FID<13.7: # print("reach goal:IS:%.4f , FID:%.4f"%(IS_mean,FID)) # sys.exit(0) state_dict['save_best_num'] = (state_dict['save_best_num'] + 1) % config['num_best_copies'] state_dict['best_IS'] = max(state_dict['best_IS'], IS_mean) state_dict['best_FID'] = min(state_dict['best_FID'], FID) # Log results to file #test_log.log(itr=int(state_dict['itr']), IS_mean=float(IS_mean), # IS_std=float(IS_std), FID=float(FID)) test_log.add_scalar(step=int(state_dict['itr']), tag="IS_mean", value=float(IS_mean)) test_log.add_scalar(step=int(state_dict['itr']), tag="IS_std", value=float(IS_std)) test_log.add_scalar(step=int(state_dict['itr']), tag="FID", value=float(FID))
def run_training(opts): inputs, labels, angle_trig, opts = inputs_helper.make_inputs(opts) rnn = model.RNN(opts) train_iter, next_element = rnn.batch_inputs(inputs, labels, opts) t = time.perf_counter() with tf.Session() as sess: sess.run(tf.global_variables_initializer()) n_epoch = opts.n_epoch n_batch = int(opts.n_examples / opts.batch_size) losses = [] lr = np.logspace(-3, -4, n_epoch) # learning rate for epoch in range(n_epoch): sess.run(train_iter.initializer) for batch in range(n_batch): x, y = sess.run(next_element) init_state = np.zeros((y.shape[0], opts.network_state_size)) feed_dict = { rnn.inputs: x, rnn.labels: y, rnn.init_state: init_state, rnn.lr: lr[epoch] } pred, loss, _ = sess.run( [rnn.predictions, rnn.total_loss, rnn.train_op], feed_dict=feed_dict) losses.append(loss) if (epoch + 1) % 100 == 0: print('Epoch {0}, time elapsed: {1:.2f}s'.format( epoch, time.perf_counter() - t)) W_ah, W_sh, W_hh, W_out, bias = sess.run([ rnn.W_ah_trained, rnn.W_sh_trained, rnn.W_hh_masked, rnn.W_out, rnn.bias ]) # save weights opts.weights_name = opts.get_path() + "_weights" utils.save_weights([W_ah, W_sh, W_hh, W_out, bias], opts.folder, opts.weights_name) tup = [('Losses', np.array(losses))] fig_name = opts.folder + '/' + opts.get_path() + '_loss_fig' utils.pretty_plot(tup, 1, 1, fig_name) return opts
def update_FID(G, D, G_ema, state_dict, config, FID, experiment_name, test_log): print('Itr %d: The FID is %5.4f' % (state_dict['itr'], FID)) if ((config['which_best'] == 'IS' and IS_mean > state_dict['best_IS']) or (config['which_best'] == 'FID' and FID < state_dict['best_FID'])): print('%s improved over previous best, saving checkpoint...' % config['which_best']) utils.save_weights(G, D, state_dict, config['weights_root'], experiment_name, 'best%d' % state_dict['save_best_num'], G_ema if config['ema'] else None) state_dict['save_best_num'] = (state_dict['save_best_num'] + 1) % config['num_best_copies'] state_dict['best_FID'] = min(state_dict['best_FID'], FID) test_log.log(itr=int(state_dict['itr']), IS_mean=float(0), IS_std=float(0), FID=float(FID))
def train_aug_CNN(model, nb_epochs): batch_size = 32 # Number of sample used simultaneously on a layer # Build validation directory for training build_train_validation_data(DATA_DIR + 'train/') train_datagen = ImageDataGenerator(rotation_range=40, width_shift_range=0.2, height_shift_range=0.2, shear_range=0.2, zoom_range=0.2, rescale=1. / 255, horizontal_flip=True, fill_mode='nearest') val_datagen = ImageDataGenerator(rescale=1. / 255) train_generator = train_datagen.flow_from_directory(DATA_DIR + 'train/train', target_size=(128, 128), batch_size=batch_size, class_mode='binary', shuffle=True) validation_generator = val_datagen.flow_from_directory( DATA_DIR + 'train/validation', target_size=(128, 128), batch_size=batch_size, class_mode='binary', shuffle=True) history = model.fit_generator(train_generator, samples_per_epoch=train_generator.samples, epochs=nb_epochs, validation_data=validation_generator, verbose=1) # Clear validation directory undo_train_validation_data(DATA_DIR + 'train/') save_weights(model) return model, history
def train_CNN(model, nb_epochs): x_train, y_train = load_data(DATA_DIR + 'train') x_train = x_train.astype('float64') x_train /= 255 batch_size = 32 # Number of sample used simultaneously on a layer val_split = 0.1 # Percentage of validation data in training data history = model.fit(x_train, y_train, batch_size=batch_size, epochs=nb_epochs, verbose=1, validation_split=val_split, shuffle=False) save_weights(model) return model, history
def save_and_sample(G, D, Dv, G_ema, z_, y_, fixed_z, fixed_y, state_dict, config, experiment_name): utils.save_weights(G, D, Dv, state_dict, config['weights_root'], experiment_name, None, G_ema if config['ema'] else None) # Save an additional copy to mitigate accidental corruption if process # is killed during a save (it's happened to me before -.-) if config['num_save_copies'] > 0: utils.save_weights(G, D, Dv, state_dict, config['weights_root'], experiment_name, 'copy%d' % state_dict['save_num'], G_ema if config['ema'] else None) state_dict['save_num'] = (state_dict['save_num'] + 1) % config['num_save_copies'] # Use EMA G for samples or non-EMA? which_G = G_ema if config['ema'] and config['use_ema'] else G # Accumulate standing statistics? if config['accumulate_stats']: utils.accumulate_standing_stats( G_ema if config['ema'] and config['use_ema'] else G, z_, y_, config['n_classes'], config['num_standing_accumulations'])
def train(gan, ds, epoch): t0 = time.time() for ep in range(epoch): for t, (img, _) in enumerate(ds): d_loss, d_acc, g_loss, g_acc = gan.step(img) if t % 400 == 0: t1 = time.time() print( "ep={} | time={:.1f} | t={} | d_acc={:.2f} | g_acc={:.2f} | d_loss={:.2f} | g_loss={:.2f}" .format( ep, t1 - t0, t, d_acc.numpy(), g_acc.numpy(), d_loss.numpy(), g_loss.numpy(), )) t0 = t1 save_gan(gan, ep) save_weights(gan) cvt_gif(gan)
def save_and_sample(G, D, G_ema, z_, y_, fixed_z, fixed_y, state_dict, config, experiment_name, save_weight): if save_weight: save_weights(G, D, state_dict, config['weights_root'], experiment_name, None, G_ema if config['ema'] else None) # Save an additional copy to mitigate accidental corruption if process # is killed during a save (it's happened to me before -.-) if config['num_save_copies'] > 0: save_weights(G, D, state_dict, config['weights_root'], experiment_name, 'copy%d' % state_dict['save_num'], G_ema if config['ema'] else None) state_dict['save_num'] = (state_dict['save_num'] + 1) % config['num_save_copies'] # Use EMA G for samples or non-EMA? which_G = G_ema if config['ema'] and config['use_ema'] else G # Save a random sample sheet with fixed z and y with torch.no_grad(): if config['parallel']: fixed_Gz = nn.parallel.data_parallel(which_G, (fixed_z, which_G.shared(fixed_y))) else: fixed_Gz = which_G(fixed_z, which_G.module.shared(fixed_y)) image_filename = '%s/%s/fixed_samples%d.jpg' % (config['samples_root'], experiment_name, state_dict['itr']) os.makedirs(os.path.dirname(image_filename), exist_ok=True) torchvision.utils.save_image(fixed_Gz.float().cpu(), image_filename, nrow=int(fixed_Gz.shape[0] ** 0.5), normalize=True) # For now, every time we save, also save sample sheets utils.sample_sheet(which_G, classes_per_sheet=50, num_classes=config['n_classes'], samples_per_class=10, parallel=config['parallel'], samples_root=config['samples_root'], experiment_name=experiment_name, folder_number=state_dict['itr'], z_=z_)
def train(): ds = get_ds(BATCH_SIZE) gan = GAN(LATENT_DIM, IMG_SHAPE) t0 = time.time() for ep in range(EPOCH): for t, (img, _) in enumerate(ds): g_img, d_loss, d_acc, g_loss, g_acc = gan.step(img) if t % 400 == 0: t1 = time.time() print( "ep={} | time={:.1f} | t={} | d_acc={:.2f} | g_acc={:.2f} | d_loss={:.2f} | g_loss={:.2f}" .format( ep, t1 - t0, t, d_acc.numpy(), g_acc.numpy(), d_loss.numpy(), g_loss.numpy(), )) t0 = t1 save_gan(gan, ep) save_weights(gan)
embedded_pvt = Embedding(input_dim=vocab_size, output_dim=vec_dim, input_length=1)(input_pvt) embedded_ctx = Embedding(input_dim=vocab_size, output_dim=vec_dim, input_length=1)(input_ctx) merged = merge(inputs=[embedded_pvt, embedded_ctx], mode=lambda x: (x[0] * x[1]).sum(-1), output_shape=(batch_size, 1)) predictions = Activation('sigmoid')(merged) # build and train the model model = Model(input=[input_pvt, input_ctx], output=predictions) model.compile(optimizer='rmsprop', loss='mse', metrics=['accuracy']) model.fit_generator(generator=batch_generator(couples, labels), samples_per_epoch=samples_per_epoch, nb_epoch=nb_epoch, verbose=1) # save weights utils.save_weights(model, index2word, vec_dim) # eval using gensim print 'the....' utils.most_similar(positive=['the']) print 'she - he + him....' utils.most_similar(positive=['she', 'him'], negative=['he'])