def train():
ds = get_ds(BATCH_SIZE)
gan = InfoGAN(RAND_DIM, STYLE_DIM, LABEL_DIM, IMG_SHAPE, FIX_STD,
STYLE_SCALE)
t0 = time.time()
for ep in range(EPOCH):
for t, (real_img, real_img_label) in enumerate(ds):
g_img, d_loss, d_bool_acc, g_loss, g_bool_acc, g_img_label, d_class_acc = gan.step(
real_img, real_img_label)
if t % 400 == 0:
t1 = time.time()
print(
"ep={} | time={:.1f}|t={}|d_acc={:.2f}|d_classacc={:.2f}|g_acc={:.2f}|d_loss={:.2f}|g_loss={:.2f}"
.format(
ep,
t1 - t0,
t,
d_bool_acc.numpy(),
g_bool_acc.numpy(),
d_class_acc.numpy(),
d_loss.numpy(),
g_loss.numpy(),
))
t0 = t1
save_gan(gan, ep)
save_weights(gan)
def test(G, D, G_ema, z_, y_, state_dict, config, sample, get_inception_metrics,
experiment_name, test_log):
print('Gathering inception metrics...', )
if config['accumulate_stats']:
utils.accumulate_standing_stats(G_ema if config['ema'] and config['use_ema'] else G,
z_, y_, config['n_classes'],
config['num_standing_accumulations'])
IS_mean, IS_std, FID = get_inception_metrics(sample, step=state_dict['shown_images'],
num_inception_images=config['num_inception_images'],
num_splits=10)
print('shown_images %d: Inception Score is %3.3f +/- %3.3f, FID is %5.4f' %
(state_dict['shown_images'], IS_mean, IS_std, FID), )
# If improved over previous best metric, save approrpiate copy
if not math.isnan(IS_mean) and not math.isnan(FID)\
and (IS_mean > state_dict['best_IS']):
print('%s improved over previous best, saving checkpoint...' % 'IS', )
utils.save_weights(G, D, state_dict, config['weights_root'],
experiment_name, 'best_IS%d' % state_dict['save_best_num'],
G_ema if config['ema'] else None)
state_dict['save_best_num'] = (state_dict['save_best_num'] + 1 ) % config['num_best_copies']
if not math.isnan(IS_mean) and not math.isnan(FID)\
and (FID < state_dict['best_FID']):
print('%s improved over previous best, saving checkpoint...' % 'FID', )
utils.save_weights(G, D, state_dict, config['weights_root'],
experiment_name, 'best_FID%d' % state_dict['save_best_num'],
G_ema if config['ema'] else None)
state_dict['save_best_num'] = (state_dict['save_best_num'] + 1) % config['num_best_copies']
state_dict['best_IS'] = max(state_dict['best_IS'], IS_mean)
state_dict['best_FID'] = min(state_dict['best_FID'], FID)
# Log results to file
test_log.log(itr=int(state_dict['itr']), IS_mean=float(IS_mean),
IS_std=float(IS_std), FID=float(FID))
return IS_mean, IS_std, FID
def main(args):
sentences, index2word = utils.load_corpus(corpus_file=args.corpus)
vocab_size = len(index2word)
# create input
couples, labels = utils.skip_grams(sentences, args.window_size, vocab_size)
print('Shape of couples: ' + str(couples.shape))
print('Shape of labels: ' + str(labels.shape))
# metrics
nb_batch = len(labels) // args.batch_size
samples_per_epoch = args.batch_size * nb_batch
model = build_model(vocab_size, args.vec_dim, args.batch_size)
if (args.multi_gpu):
model = multi_gpu_model(model)
opt = RMSprop(lr=5e-4, decay=5e-6)
checkpoint = ModelCheckpoint(os.path.join(args.ckpt_path,
'Word2Vec_{epoch:03d}.h5'),
period=args.ckpt_period,
save_weights_only=True)
early_stop = EarlyStopping(monitor='loss', patience=10)
model.compile(optimizer=opt, loss='mse', metrics=['accuracy'])
model.fit_generator(generator=batch_generator(couples, labels,
args.batch_size, nb_batch),
steps_per_epoch=samples_per_epoch,
epochs=args.epochs,
callbacks=[checkpoint, early_stop],
verbose=1)
# save weights
utils.save_weights(model, index2word, args.vec_dim)
def main(args):
# 数据加载
(x_train, y_train), (x_test, y_test) = load_cifar(args.cifar_root)
# 随机选择训练样本
train_num = x_train.shape[0]
def next_batch(batch_size):
idx = np.random.choice(train_num, batch_size)
return x_train[idx], y_train[idx]
# 网络
vgg = VGG(image_size=32, name='vgg11')
opt = RmsProp(vgg.weights, lr=args.lr, decay=1e-3)
# 加载权重
if args.checkpoint:
weights = load_weights(args.checkpoint)
vgg.load_weights(weights)
print("load weights done")
# 评估
if args.eval_only:
indices = np.random.choice(len(x_test), args.eval_num, replace=False)
print('{} start evaluate'.format(
time.asctime(time.localtime(time.time()))))
acc = get_accuracy(vgg, x_test[indices], ys=y_test[indices])
print('{} acc on test dataset is :{:.3f}'.format(
time.asctime(time.localtime(time.time())), acc))
return
# 训练
num_steps = args.steps
for step in range(num_steps):
x, y_true = next_batch(args.batch_size)
# 前向传播
y_predict = vgg.forward(x.astype(np.float))
# print('y_pred: min{},max{},mean:{}'.format(np.min(y_predict, axis=-1),
# np.max(y_predict, axis=-1),
# np.mean(y_predict, axis=-1)))
# print('y_pred: {}'.format(y_predict))
acc = np.mean(
np.argmax(y_predict, axis=1) == np.argmax(y_true, axis=1))
# 计算loss
loss, gradient = cross_entropy_loss(y_predict, y_true)
# 反向传播
vgg.backward(gradient)
# 更新梯度
opt.iterate(vgg)
# 打印信息
print('{} step:{},loss:{:.4f},acc:{:.4f}'.format(
time.asctime(time.localtime(time.time())), step, loss, acc))
# 保存权重
if step % 100 == 0:
save_weights(
os.path.join(args.save_dir, 'weights-{:03d}.pkl'.format(step)),
vgg.weights)
def test(G, D, G_ema, z_, y_, state_dict, config, sample,
get_inception_metrics, experiment_name, test_log):
print('Gathering inception metrics...')
if config['accumulate_stats']:
utils.accumulate_standing_stats(
G_ema if config['ema'] and config['use_ema'] else G, z_, y_,
config['n_classes'], config['num_standing_accumulations'])
IS_mean, IS_std, FID = get_inception_metrics(
sample, config['num_inception_images'], num_splits=10)
print(
'Itr %d: PYTORCH UNOFFICIAL Inception Score is %3.3f +/- %3.3f, PYTORCH UNOFFICIAL FID is %5.4f'
% (state_dict['itr'], IS_mean, IS_std, FID))
# If improved over previous best metric, save approrpiate copy
if ((config['which_best'] == 'IS' and IS_mean > state_dict['best_IS']) or
(config['which_best'] == 'FID' and FID < state_dict['best_FID'])):
print('%s improved over previous best, saving checkpoint...' %
config['which_best'])
utils.save_weights(G, D, state_dict, config['weights_root'],
experiment_name,
'best%d' % state_dict['save_best_num'],
G_ema if config['ema'] else None)
state_dict['save_best_num'] = (state_dict['save_best_num'] +
1) % config['num_best_copies']
state_dict['best_IS'] = max(state_dict['best_IS'], IS_mean)
state_dict['best_FID'] = min(state_dict['best_FID'], FID)
# Log results to file
test_log.log(itr=int(state_dict['itr']),
IS_mean=float(IS_mean),
IS_std=float(IS_std),
FID=float(FID))
def eval_model(engine):
nonlocal best_val_loss, nb_epoch_no_improvement
def log_progress(mode, metrics_values):
metrics_str = ', '.join([
f'{metric_name} {metrics_values[metric_name]:.3f}'
for metric_name, _ in metrics
])
print(f'{mode}: {metrics_str}', end=' | ')
# evaluator.run(data_loader_train)
# metrics_train = evaluator.state.metrics.copy()
evaluator.run(data_loader_dev)
metrics_dev = evaluator.state.metrics.copy()
print(f'Epoch {engine.state.epoch:>3}', end=' | ')
# log_progress('train', metrics_train)
log_progress('dev', metrics_dev)
if best_val_loss > metrics_dev[
'loss'] or engine.state.epoch <= cfg.min_epochs:
best_val_loss = metrics_dev['loss']
nb_epoch_no_improvement = 0
save_weights(model, os.path.join(CACHE_DIR,
f'model_{cfg.model}.pt'))
print('Model saved', end=' ')
else:
nb_epoch_no_improvement += 1
if cfg.early_stopping_patience != 0 and nb_epoch_no_improvement > cfg.early_stopping_patience:
trainer.terminate()
print()
def main():
args = parse_args()
path_to_x_train = args.x_train_dir
path_to_y_train = args.y_train_dir
path_to_model = args.model_output_dir
verbosity = args.verbosity
iter_num = args.iter_num
batch_size = args.batch_size
X_original = read_mnist(path_to_x_train)
y_original = read_mnist(path_to_y_train)
X, image_shape = preprocessing_data(X_original)
y = y_original
print(
f'\nbatch_size: {batch_size}, iter_num: {iter_num}, kernel: {args.kernel}\n'
)
X_train, X_val, y_train, y_val = X[:50000], X[50000:], y[:50000], y[50000:]
clf = MySvm(args.kernel, image_shape=image_shape)
clf.fit(X_train,
y_train,
iter_num=iter_num,
batch_size=batch_size,
verbosity=verbosity)
prediction_labels = clf.predict(X_val)
print(classification_report(y_val, prediction_labels, digits=4))
optimal_weights = clf.get_weights()
print(f'Saving model to {path_to_model}')
save_weights(path_to_model, optimal_weights)
def train(name, gen, disc, gen_train_ratio=5, epochs=-1):
device = utils.get_device()
dataloader = data.get_dataloader()
loss_func = nn.BCELoss()
g_optimizer = to.Adam(gen.parameters(), lr=.0003, betas=(.5, .9))
d_optimizer = to.Adam(disc.parameters(), lr=.0003, betas=(.5, .9))
iter_axis = []
g_loss_axis = []
d_loss_axis = []
fixed_noise = torch.randn(64, gen.nz, device=device)
epoch_iterator = range(epochs) if epochs >= 0 else count(0)
iters = 0
for epoch in epoch_iterator:
for i, batch in enumerate(dataloader):
loss_disc, acc_real, acc_fake = train_discriminator(
gen, disc, batch, loss_func, d_optimizer)
for _ in range(gen_train_ratio):
loss_gen, acc_gen = train_generator(gen, disc, loss_func,
g_optimizer)
# Training stats
if i % 50 == 0:
print(
f'[{epoch:2d}/{epochs}][{i:3d}/{len(dataloader)}]\t' +\
f'Loss_D: {loss_disc:3.4f}\tLoss_G: {loss_gen:3.4f}\t' +\
f'D(x): {acc_real:3.4f}\tD(G(z)): {acc_fake:3.4f} / {acc_gen:3.4f}'
)
if (iters % 10 == 0) or ((epoch == epochs - 1) and
(i == len(dataloader) - 1)):
iter_axis.append(iters)
g_loss_axis.append(loss_gen)
d_loss_axis.append(loss_disc)
# Save output
if (iters % 500 == 0) or ((epoch == epochs - 1) and
(i == len(dataloader) - 1)):
print('Saving images...')
with torch.no_grad():
gen.mixing = False
fake = gen(fixed_noise).detach().cpu()
gen.mixing = True
utils.save_image(utils.make_torch_grid(fake),
f'{name}_{iters}.png')
# Save weights
if (iters % 1000 == 0) or ((epoch == epochs - 1) and
(i == len(dataloader) - 1)):
print('Saving weights...')
utils.save_weights(gen, f'{name}_gen.pt')
utils.save_weights(disc, f'{name}_disc.pt')
iters += 1
utils.plot_losses(iter_axis, g_loss_axis, d_loss_axis)
def save_and_sample(G, D, G_ema, z_, y_, fixed_z, fixed_y,
state_dict, config, experiment_name):
utils.save_weights(G, D, state_dict, config['weights_root'],
experiment_name, None, G_ema if config['ema'] else None)
# Save an additional copy to mitigate accidental corruption if process
# is killed during a save (it's happened to me before -.-)
if config['num_save_copies'] > 0:
utils.save_weights(G, D, state_dict, config['weights_root'],
experiment_name,
'copy%d' % state_dict['save_num'],
G_ema if config['ema'] else None)
state_dict['save_num'] = (
state_dict['save_num'] + 1) % config['num_save_copies']
# Use EMA G for samples or non-EMA?
which_G = G_ema if config['ema'] and config['use_ema'] else G
# Accumulate standing statistics?
if config['accumulate_stats']:
utils.accumulate_standing_stats(G_ema if config['ema'] and config['use_ema'] else G,
z_, y_, config['n_classes'],
config['num_standing_accumulations'])
# Save a random sample sheet with fixed z and y
with torch.no_grad():
if config['parallel']:
fixed_Gz = nn.parallel.data_parallel(
which_G, (fixed_z, which_G.shared(fixed_y)))
else:
fixed_Gz = which_G(fixed_z, which_G.shared(fixed_y))
if not os.path.isdir('%s/%s' % (config['samples_root'], experiment_name)):
os.mkdir('%s/%s' % (config['samples_root'], experiment_name))
image_filename = '%s/%s/fixed_samples%d.jpg' % (config['samples_root'],
experiment_name,
state_dict['itr'])
torchvision.utils.save_image(fixed_Gz.float().cpu(), image_filename,
nrow=int(fixed_Gz.shape[0] ** 0.5), normalize=True)
# For now, every time we save, also save sample sheets
utils.sample_sheet(which_G,
classes_per_sheet=utils.classes_per_sheet_dict[config['dataset']],
num_classes=config['n_classes'],
samples_per_class=10, parallel=config['parallel'],
samples_root=config['samples_root'],
experiment_name=experiment_name,
folder_number=state_dict['itr'],
z_=z_)
# Also save interp sheets
for fix_z, fix_y in zip([False, False, True], [False, True, False]):
utils.interp_sheet(which_G,
num_per_sheet=16,
num_midpoints=8,
num_classes=config['n_classes'],
parallel=config['parallel'],
samples_root=config['samples_root'],
experiment_name=experiment_name,
folder_number=state_dict['itr'],
sheet_number=0,
fix_z=fix_z, fix_y=fix_y, device='cuda')
def train(models, it_train, it_val, params):
"""
Train the model.
Parameters:
- models: a dictionary with all the models.
- atob: a model that goes from A to B.
- d: the discriminator model.
- p2p: a Pix2Pix model.
- it_train: the iterator of the training data.
- it_val: the iterator of the validation data.
- params: parameters of the training procedure.
- dout_size: the size of the output of the discriminator model.
"""
# Create the experiment folder and save the parameters
create_expt_dir(params)
# Get the output shape of the discriminator
dout_size = models.d.output_shape[-3:-1]
# Define the data generators
generators = generators_creation(it_train, it_val, models, dout_size)
# Define the number of samples to use on each training epoch
train_samples = params.train_samples
if params.train_samples == -1:
train_samples = 8
batches_per_epoch = train_samples // params.samples_per_batch
# Define the number of samples to use for validation
val_samples = params.val_samples
if val_samples == -1:
val_samples = 8
losses = {'p2p': [], 'd': [], 'p2p_val': [], 'd_val': []}
if params.continue_train:
losses = load_losses(log_dir=params.log_dir,
expt_name=params.expt_name)
for e in tqdm(range(params.epochs)):
for b in range(batches_per_epoch):
train_iteration(models, generators, losses, params)
# Evaluate how the models is doing on the validation set.
evaluate(models, generators, losses, val_samples=val_samples)
if (e + 1) % params.save_every == 0:
save_weights(models,
log_dir=params.log_dir,
expt_name=params.expt_name)
log(losses,
models.atob,
it_val,
log_dir=params.log_dir,
expt_name=params.expt_name,
is_a_binary=params.is_a_binary,
is_b_binary=params.is_b_binary)
def test(path):
(x_train, y_train), (x_test, y_test) = load_cifar(path)
print(x_train[0][0])
print(y_train[0])
vgg = VGG(name='vgg11')
import utils
utils.save_weights('./w.pkl', vgg.weights)
w = utils.load_weights('./w.pkl')
print(type(w))
print(w.keys())
def train_model(model,
criterion,
optimizer,
lr_scheduler,
max_num=2,
init_lr=0.001,
num_epochs=100):
since = time.time()
best_model = model
best_acc = 0.0
print(model.name)
for epoch in range(num_epochs):
epoch_since = time.time()
print('Epoch {}/{}'.format(epoch, num_epochs - 1))
print('-' * 10)
for phase in ['train', 'valid']:
if phase == 'train':
optimizer = lr_scheduler(optimizer, epoch, init_lr=init_lr)
model.train(True)
else:
model.train(False)
running_loss = 0.0
running_corrects = 0
for data in dset_loaders[phase]:
inputs, labels = data
inputs, labels = Variable(inputs.cuda()), Variable(
labels.cuda())
optimizer.zero_grad()
outputs = model(inputs)
_, preds = torch.max(outputs.data, 1)
loss = criterion(outputs, labels)
if phase == 'train':
loss.backward()
optimizer.step()
running_loss += loss.data[0]
running_corrects += torch.sum(preds == labels.data)
epoch_loss = running_loss / dset_sizes[phase]
epoch_acc = running_corrects / dset_sizes[phase]
print('{} Loss: {:.4f} Acc: {:.4f}'.format(phase, epoch_loss,
epoch_acc))
if phase == 'valid':
save_weights(epoch_acc, model, epoch, max_num=max_num)
if phase == 'valid' and epoch_acc > best_acc:
best_acc = epoch_acc
#best_model = copy.deepcopy(model)
#torch.save(best_model.state_dict(), w_file)
print('epoch {}: {:.0f}s'.format(epoch, time.time() - epoch_since))
time_elapsed = time.time() - since
print('Training complete in {:.0f}m {:.0f}s'.format(
time_elapsed // 60, time_elapsed % 60))
print('Best val Acc: {:4f}'.format(best_acc))
print(w_files_training)
return best_model
def save_and_sample(G, D, G_ema, z_, y_, fixed_z, fixed_y, state_dict, config,
experiment_name):
utils.save_weights(G, D, state_dict, config['weights_root'],
experiment_name, None, G_ema if config['ema'] else None)
if config['num_save_copies'] > 0:
utils.save_weights(G, D, state_dict, config['weights_root'],
experiment_name, 'copy%d' % state_dict['save_num'],
G_ema if config['ema'] else None)
state_dict['save_num'] = (state_dict['save_num'] +
1) % config['num_save_copies']
which_G = G_ema if config['ema'] and config['use_ema'] else G
if config['accumulate_stats']:
if not config['conditional']:
y_.zero_()
utils.accumulate_standing_stats(
G_ema if config['ema'] and config['use_ema'] else G, z_, y_,
config['n_classes'], config['num_standing_accumulations'])
with torch.no_grad():
if config['parallel']:
fixed_Gz = nn.parallel.data_parallel(
which_G, (fixed_z, which_G.shared(fixed_y)))
else:
fixed_Gz = which_G(fixed_z, which_G.shared(fixed_y))
if not os.path.isdir('%s/%s' % (config['samples_root'], experiment_name)):
os.mkdir('%s/%s' % (config['samples_root'], experiment_name))
image_filename = '%s/%s/fixed_samples%d.jpg' % (
config['samples_root'], experiment_name, state_dict['itr'])
torchvision.utils.save_image(fixed_Gz.float().cpu(),
image_filename,
nrow=int(fixed_Gz.shape[0]**0.5),
normalize=True)
utils.sample_sheet(
which_G,
classes_per_sheet=utils.classes_per_sheet_dict[config['dataset']],
num_classes=config['n_classes'],
samples_per_class=10,
parallel=config['parallel'],
samples_root=config['samples_root'],
experiment_name=experiment_name,
folder_number=state_dict['itr'],
z_=z_)
for fix_z, fix_y in zip([False, False, True], [False, True, False]):
utils.interp_sheet(which_G,
num_per_sheet=16,
num_midpoints=8,
num_classes=config['n_classes'],
parallel=config['parallel'],
samples_root=config['samples_root'],
experiment_name=experiment_name,
folder_number=state_dict['itr'],
sheet_number=0,
fix_z=fix_z,
fix_y=fix_y,
device='cuda')
def test(G, D, GD, G_ema, z_, y_, state_dict, config, sample, get_inception_metrics,
experiment_name, test_log, acc_metrics, acc_itrs):
print('Calculating validation accuracy...')
D.eval()
D_accuracy = []
loader = utils.get_data_loaders(
**{**config, 'train': False, 'use_multiepoch_sampler': False, 'load_in_mem': False})[0]
with torch.no_grad():
for x, y in loader:
D_real = GD(None, None, x=x, dy=y, policy=config['DiffAugment'])
D_accuracy.append((D_real > 0).float().mean().item())
D.train()
D_acc_val = np.mean(D_accuracy)
print('Calculating training accuracy...')
D.eval()
D_accuracy = []
loader = utils.get_data_loaders(
**{**config, 'train': True, 'use_multiepoch_sampler': False, 'load_in_mem': False})[0]
with torch.no_grad():
for x, y in loader:
D_real = GD(None, None, x=x, dy=y, policy=config['DiffAugment'])
D_accuracy.append((D_real > 0).float().mean().item())
D.train()
D_acc_train = np.mean(D_accuracy)
print('Gathering inception metrics...')
if config['accumulate_stats']:
utils.accumulate_standing_stats(G_ema if config['ema'] and config['use_ema'] else G,
z_, y_, config['n_classes'],
config['num_standing_accumulations'])
IS_mean, IS_std, FID = get_inception_metrics(sample,
config['num_inception_images'],
num_splits=10)
print('Itr %d: PYTORCH UNOFFICIAL Inception Score is %3.3f +/- %3.3f, PYTORCH UNOFFICIAL FID is %5.4f' %
(state_dict['itr'], IS_mean, IS_std, FID))
# If improved over previous best metric, save approrpiate copy
if ((config['which_best'] == 'IS' and IS_mean > state_dict['best_IS'])
or (config['which_best'] == 'FID' and FID < state_dict['best_FID'])):
print('%s improved over previous best, saving checkpoint...' %
config['which_best'])
utils.save_weights(G, D, state_dict, config['weights_root'],
experiment_name, 'best%d' % state_dict['save_best_num'] if config['num_best_copies'] > 1 else 'best',
G_ema if config['ema'] else None)
state_dict['save_best_num'] = (
state_dict['save_best_num'] + 1) % config['num_best_copies']
state_dict['best_IS'] = max(state_dict['best_IS'], IS_mean)
state_dict['best_FID'] = min(state_dict['best_FID'], FID)
# Log results to file
test_log.log(itr=int(state_dict['itr']), IS_mean=float(IS_mean),
IS_std=float(IS_std), FID=float(FID), D_acc_val=D_acc_val, D_acc_train=D_acc_train,
**{k: v / acc_itrs for k, v in acc_metrics.items()})
def save(G, D, G_ema, state_dict, config, experiment_name):
utils.save_weights(G, D, state_dict, config['weights_root'],
experiment_name, None, G_ema if config['ema'] else None)
# Save an additional copy to mitigate accidental corruption if process
# is killed during a save (it's happened to me before -.-)
if config['num_save_copies'] > 0:
utils.save_weights(G, D, state_dict, config['weights_root'],
experiment_name,
'copy%d' % state_dict['save_num'],
G_ema if config['ema'] else None)
state_dict['save_num'] = (state_dict['save_num'] + 1 ) % config['num_save_copies']
def save_and_sample(G, D, M, G_ema, z_, y_, fixed_z, fixed_y, state_dict,
config, experiment_name):
utils.save_weights(G, D, M, state_dict, config['weights_root'],
experiment_name, None, G_ema if config['ema'] else None)
# Save an additional copy to mitigate accidental corruption if process
# is killed during a save (it's happened to me before -.-)
# if config['num_save_copies'] > 0:
# utils.save_weights(G, D, M, state_dict, config['weights_root'],
# experiment_name,
# 'copy%d' % state_dict['save_num'],
# G_ema if config['ema'] else None)
# state_dict['save_num'] = (state_dict['save_num'] + 1 ) % config['num_save_copies']
# # Use EMA G for samples or non-EMA?
which_G = G_ema if config['ema'] and config['use_ema'] else G
# Accumulate standing statistics?
#if config['accumulate_stats']:
# utils.accumulate_standing_stats(G_ema if config['ema'] and config['use_ema'] else G,
# z_, y_, config['n_classes'],
# config['num_standing_accumulations'])
# Save a random sample sheet with fixed z and y
with torch.no_grad():
if config['parallel']:
z_from_M, y_from_M = nn.parallel.data_parallel(M, (fixed_z, ))
fixed_Gz = nn.parallel.data_parallel(which_G, (z_from_M, y_from_M))
#fixed_Gz = nn.parallel.data_parallel(which_G, (fixed_z, which_G.shared(fixed_y)))
else:
z_from_M, y_from_M = M(fixed_z)
fixed_Gz = which_G(z_from_M, y_from_M)
fixed_Gz = which_G(z_from_M, y_from_M)
#fixed_Gz = which_G(fixed_z, which_G.shared(fixed_y))
if not os.path.isdir('%s/%s' % (config['samples_root'], experiment_name)):
os.mkdir('%s/%s' % (config['samples_root'], experiment_name))
image_filename = '%s/%s/fixed_samples%d.jpg' % (
config['samples_root'], experiment_name, state_dict['itr'])
torchvision.utils.save_image(fixed_Gz.float().cpu(),
image_filename,
nrow=int(fixed_Gz.shape[0]**0.5),
normalize=True)
# For now, every time we save, also save sample sheets
utils.sample_sheet(
which_G,
M,
classes_per_sheet=utils.classes_per_sheet_dict[config['dataset']],
num_classes=config['n_classes'],
samples_per_class=10,
parallel=config['parallel'],
samples_root=config['samples_root'],
experiment_name=experiment_name,
folder_number=state_dict['itr'],
z_=z_)
def update_FID(G, D, G_ema, state_dict, config, FID, experiment_name,
test_log):
if ((config['which_best'] == 'IS' and IS_mean > state_dict['best_IS']) or
(config['which_best'] == 'FID' and FID < state_dict['best_FID'])):
utils.save_weights(G, D, state_dict, config['weights_root'],
experiment_name,
'best%d' % state_dict['save_best_num'],
G_ema if config['ema'] else None)
state_dict['save_best_num'] = (state_dict['save_best_num'] +
1) % config['num_best_copies']
state_dict['best_FID'] = min(state_dict['best_FID'], FID)
test_log.log(itr=int(state_dict['itr']),
IS_mean=float(0),
IS_std=float(0),
FID=float(FID))
def log_training_results(engine):
nonlocal best_loss
# evaluator.run(data_loader_train)
# losses_train = evaluator.state.metrics['loss']
evaluator.run(data_loader_val)
losses_val = evaluator.state.metrics['loss']
# log_progress(trainer.state.epoch, trainer.state.iteration, losses_train, 'train', tensorboard_writer)
log_progress(trainer.state.epoch, trainer.state.iteration,
losses_val, 'val', tensorboard_writer)
if losses_val[exp.config.best_loss] < best_loss:
best_loss = losses_val[exp.config.best_loss]
save_weights(model, exp.experiment_dir.joinpath('best.th'))
def main():
sentences, index2word = utils.load_sentences_brown(800)
# params
nb_epoch = 3
# learn `batch_size words` at a time
batch_size = 128
vec_dim = 64
negsampling_num = 0.2
# half of window
window_size = 6
vocab_size = len(index2word)
print 'vocabulary length: ', vocab_size
# create input
couples, labels = utils.skip_grams(sentences, window_size, vocab_size,
negsampling_num)
print 'counter of positive samples and negative samples: ', Counter(labels)
print 'shape of couples: ', couples.shape
print 'shape of labels: ', labels.shape
# metrics
nb_batch = len(labels) // batch_size
samples_per_epoch = batch_size * nb_batch
# fit model
model = make_word2vec_model(vec_dim, vocab_size)
model.fit_generator(generator=utils.batch_generator(
couples, labels, batch_size, nb_batch),
steps_per_epoch=samples_per_epoch,
epochs=nb_epoch)
# save weights
utils.save_weights(model, index2word, vec_dim)
# eval using gensim
print 'the....'
utils.most_similar(positive=['the'])
print 'all....'
utils.most_similar(positive=['all'])
print 'baby....'
utils.most_similar(positive=['baby'])
print 'first....'
utils.most_similar(positive=['first'])
def industry_embedding_model_fit(model, embedding_model, nb_epoch, batch_size,
df, dict_sequence, industry2idx, output_file):
X_train, X_test, y_train, y_test, y_train_2digit, y_test_2digit = utils.get_train_test(
df, batch_size, 0.1)
target_grid_col = 'naics_4_digit'
context_grid_col = 'context'
target_col = 'target_desc'
context_col = 'context_desc'
model.fit(x=[
X_train[target_grid_col], X_train[context_grid_col],
utils.rebuild_array(X_train[target_col]),
utils.rebuild_array(X_train[context_col])
],
y=[y_train, y_train_2digit],
epochs=nb_epoch,
shuffle=True,
batch_size=batch_size,
verbose=1,
validation_data=([
X_test[target_grid_col], X_test[context_grid_col],
utils.rebuild_array(X_test[target_col]),
utils.rebuild_array(X_test[context_col])
], [y_test, y_test_2digit]),
callbacks=[
EarlyStopping(monitor='val_loss',
min_delta=0.0001,
patience=3,
verbose=1,
mode='auto')
])
sequence_df = pd.DataFrame(
{k: v
for k, v in dict_sequence.items() if len(k) == SELECT_DIGIT}).T
naics_idx_l = np.array([
industry2idx.get(int(naics),
len(industry2idx) + 1) for naics in sequence_df.index
])
in_data = [naics_idx_l, sequence_df.values]
embedding_vec = embedding_model.predict(x=in_data, verbose=1, batch_size=1)
utils.save_weights(output_file, embedding_vec, naics_idx_l, idx2industry)
return embedding_vec
def save_and_sample(G, D, G_ema, z_, y_, fixed_z, fixed_y, state_dict, config,
experiment_name):
utils.save_weights(G, D, state_dict, config['weights_root'],
experiment_name, None, G_ema if config['ema'] else None)
# Save an additional copy to mitigate accidental corruption if process
# is killed during a save (it's happened to me before -.-)
if config['num_save_copies'] > 0:
utils.save_weights(G, D, state_dict, config['weights_root'],
experiment_name, 'copy%d' % state_dict['save_num'],
G_ema if config['ema'] else None)
state_dict['save_num'] = (state_dict['save_num'] +
1) % config['num_save_copies']
# Use EMA G for samples or non-EMA?
which_G = G_ema if config['ema'] and config['use_ema'] else G
# Accumulate standing statistics?
if config['accumulate_stats']:
utils.accumulate_standing_stats(
G_ema if config['ema'] and config['use_ema'] else G, z_, y_,
config['n_classes'], config['num_standing_accumulations'])
# Save a random sample sheet with fixed z and y
with torch.no_grad():
# Get Generator output given noise
if config['use_dog_cnt'] and config['G_shared']:
ye0 = G.shared(fixed_y[:, 0])
ye1 = G.dog_cnt_shared(fixed_y[:, 1])
gyc = torch.cat([ye0, ye1], 1)
else:
gyc = G.shared(fixed_y)
if config['parallel']:
fixed_Gz = nn.parallel.data_parallel(which_G, (fixed_z, gyc))
else:
fixed_Gz = which_G(fixed_z, gyc)
if not os.path.isdir('%s/%s' % (config['samples_root'], experiment_name)):
os.mkdir('%s/%s' % (config['samples_root'], experiment_name))
image_filename = '%s/%s/fixed_samples%d.jpg' % (
config['samples_root'], experiment_name, state_dict['itr'])
torchvision.utils.save_image(fixed_Gz.float().cpu(),
image_filename,
nrow=int(fixed_Gz.shape[0]**0.5),
normalize=True)
# # For now, every time we save, also save sample sheets
num_classes = int(np.minimum(120, config['n_classes']))
def test(G, D, G_ema, z_, y_, state_dict, config, sample,
get_inception_metrics, experiment_name, test_log):
print('Gathering inception metrics...')
if config['accumulate_stats']:
utils.accumulate_standing_stats(
G_ema if config['ema'] and config['use_ema'] else G, z_, y_,
config['n_classes'], config['num_standing_accumulations'])
IS_mean, IS_std, FID = get_inception_metrics(
sample, config['num_inception_images'], num_splits=10)
if np.isnan(float(FID)):
FID = 99
print(
'Itr %d: PYTORCH UNOFFICIAL Inception Score is %3.3f +/- %3.3f, PYTORCH UNOFFICIAL FID is %5.4f'
% (state_dict['itr'], IS_mean, IS_std, FID))
# If improved over previous best metric, save approrpiate copy
if ((config['which_best'] == 'IS' and IS_mean > state_dict['best_IS']) or
(config['which_best'] == 'FID' and FID < state_dict['best_FID'])):
print('%s improved over previous best, saving checkpoint...' %
config['which_best'])
utils.save_weights(G, D, state_dict, config['weights_root'],
experiment_name,
'best%d' % state_dict['save_best_num'],
G_ema if config['ema'] else None)
utils.save_weights(G, D, state_dict, config['weights_root'],
experiment_name, 'currentbest',
G_ema if config['ema'] else None)
###early exit when reaching the goal
# if IS_mean>8.22 and FID<13.7:
# print("reach goal:IS:%.4f , FID:%.4f"%(IS_mean,FID))
# sys.exit(0)
state_dict['save_best_num'] = (state_dict['save_best_num'] +
1) % config['num_best_copies']
state_dict['best_IS'] = max(state_dict['best_IS'], IS_mean)
state_dict['best_FID'] = min(state_dict['best_FID'], FID)
# Log results to file
#test_log.log(itr=int(state_dict['itr']), IS_mean=float(IS_mean),
# IS_std=float(IS_std), FID=float(FID))
test_log.add_scalar(step=int(state_dict['itr']),
tag="IS_mean",
value=float(IS_mean))
test_log.add_scalar(step=int(state_dict['itr']),
tag="IS_std",
value=float(IS_std))
test_log.add_scalar(step=int(state_dict['itr']),
tag="FID",
value=float(FID))
def run_training(opts):
inputs, labels, angle_trig, opts = inputs_helper.make_inputs(opts)
rnn = model.RNN(opts)
train_iter, next_element = rnn.batch_inputs(inputs, labels, opts)
t = time.perf_counter()
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
n_epoch = opts.n_epoch
n_batch = int(opts.n_examples / opts.batch_size)
losses = []
lr = np.logspace(-3, -4, n_epoch) # learning rate
for epoch in range(n_epoch):
sess.run(train_iter.initializer)
for batch in range(n_batch):
x, y = sess.run(next_element)
init_state = np.zeros((y.shape[0], opts.network_state_size))
feed_dict = {
rnn.inputs: x,
rnn.labels: y,
rnn.init_state: init_state,
rnn.lr: lr[epoch]
}
pred, loss, _ = sess.run(
[rnn.predictions, rnn.total_loss, rnn.train_op],
feed_dict=feed_dict)
losses.append(loss)
if (epoch + 1) % 100 == 0:
print('Epoch {0}, time elapsed: {1:.2f}s'.format(
epoch,
time.perf_counter() - t))
W_ah, W_sh, W_hh, W_out, bias = sess.run([
rnn.W_ah_trained, rnn.W_sh_trained, rnn.W_hh_masked, rnn.W_out,
rnn.bias
])
# save weights
opts.weights_name = opts.get_path() + "_weights"
utils.save_weights([W_ah, W_sh, W_hh, W_out, bias], opts.folder,
opts.weights_name)
tup = [('Losses', np.array(losses))]
fig_name = opts.folder + '/' + opts.get_path() + '_loss_fig'
utils.pretty_plot(tup, 1, 1, fig_name)
return opts
def update_FID(G, D, G_ema, state_dict, config, FID, experiment_name,
test_log):
print('Itr %d: The FID is %5.4f' % (state_dict['itr'], FID))
if ((config['which_best'] == 'IS' and IS_mean > state_dict['best_IS']) or
(config['which_best'] == 'FID' and FID < state_dict['best_FID'])):
print('%s improved over previous best, saving checkpoint...' %
config['which_best'])
utils.save_weights(G, D, state_dict, config['weights_root'],
experiment_name,
'best%d' % state_dict['save_best_num'],
G_ema if config['ema'] else None)
state_dict['save_best_num'] = (state_dict['save_best_num'] +
1) % config['num_best_copies']
state_dict['best_FID'] = min(state_dict['best_FID'], FID)
test_log.log(itr=int(state_dict['itr']),
IS_mean=float(0),
IS_std=float(0),
FID=float(FID))
def train_aug_CNN(model, nb_epochs):
batch_size = 32 # Number of sample used simultaneously on a layer
# Build validation directory for training
build_train_validation_data(DATA_DIR + 'train/')
train_datagen = ImageDataGenerator(rotation_range=40,
width_shift_range=0.2,
height_shift_range=0.2,
shear_range=0.2,
zoom_range=0.2,
rescale=1. / 255,
horizontal_flip=True,
fill_mode='nearest')
val_datagen = ImageDataGenerator(rescale=1. / 255)
train_generator = train_datagen.flow_from_directory(DATA_DIR +
'train/train',
target_size=(128, 128),
batch_size=batch_size,
class_mode='binary',
shuffle=True)
validation_generator = val_datagen.flow_from_directory(
DATA_DIR + 'train/validation',
target_size=(128, 128),
batch_size=batch_size,
class_mode='binary',
shuffle=True)
history = model.fit_generator(train_generator,
samples_per_epoch=train_generator.samples,
epochs=nb_epochs,
validation_data=validation_generator,
verbose=1)
# Clear validation directory
undo_train_validation_data(DATA_DIR + 'train/')
save_weights(model)
return model, history
def train_CNN(model, nb_epochs):
x_train, y_train = load_data(DATA_DIR + 'train')
x_train = x_train.astype('float64')
x_train /= 255
batch_size = 32 # Number of sample used simultaneously on a layer
val_split = 0.1 # Percentage of validation data in training data
history = model.fit(x_train,
y_train,
batch_size=batch_size,
epochs=nb_epochs,
verbose=1,
validation_split=val_split,
shuffle=False)
save_weights(model)
return model, history
def save_and_sample(G, D, Dv, G_ema, z_, y_, fixed_z, fixed_y, state_dict,
config, experiment_name):
utils.save_weights(G, D, Dv, state_dict, config['weights_root'],
experiment_name, None, G_ema if config['ema'] else None)
# Save an additional copy to mitigate accidental corruption if process
# is killed during a save (it's happened to me before -.-)
if config['num_save_copies'] > 0:
utils.save_weights(G, D, Dv, state_dict, config['weights_root'],
experiment_name, 'copy%d' % state_dict['save_num'],
G_ema if config['ema'] else None)
state_dict['save_num'] = (state_dict['save_num'] +
1) % config['num_save_copies']
# Use EMA G for samples or non-EMA?
which_G = G_ema if config['ema'] and config['use_ema'] else G
# Accumulate standing statistics?
if config['accumulate_stats']:
utils.accumulate_standing_stats(
G_ema if config['ema'] and config['use_ema'] else G, z_, y_,
config['n_classes'], config['num_standing_accumulations'])
def train(gan, ds, epoch):
t0 = time.time()
for ep in range(epoch):
for t, (img, _) in enumerate(ds):
d_loss, d_acc, g_loss, g_acc = gan.step(img)
if t % 400 == 0:
t1 = time.time()
print(
"ep={} | time={:.1f} | t={} | d_acc={:.2f} | g_acc={:.2f} | d_loss={:.2f} | g_loss={:.2f}"
.format(
ep,
t1 - t0,
t,
d_acc.numpy(),
g_acc.numpy(),
d_loss.numpy(),
g_loss.numpy(),
))
t0 = t1
save_gan(gan, ep)
save_weights(gan)
cvt_gif(gan)
def save_and_sample(G, D, G_ema, z_, y_, fixed_z, fixed_y,
state_dict, config, experiment_name, save_weight):
if save_weight:
save_weights(G, D, state_dict, config['weights_root'],
experiment_name, None, G_ema if config['ema'] else None)
# Save an additional copy to mitigate accidental corruption if process
# is killed during a save (it's happened to me before -.-)
if config['num_save_copies'] > 0:
save_weights(G, D, state_dict, config['weights_root'],
experiment_name,
'copy%d' % state_dict['save_num'],
G_ema if config['ema'] else None)
state_dict['save_num'] = (state_dict['save_num'] + 1) % config['num_save_copies']
# Use EMA G for samples or non-EMA?
which_G = G_ema if config['ema'] and config['use_ema'] else G
# Save a random sample sheet with fixed z and y
with torch.no_grad():
if config['parallel']:
fixed_Gz = nn.parallel.data_parallel(which_G, (fixed_z, which_G.shared(fixed_y)))
else:
fixed_Gz = which_G(fixed_z, which_G.module.shared(fixed_y))
image_filename = '%s/%s/fixed_samples%d.jpg' % (config['samples_root'],
experiment_name,
state_dict['itr'])
os.makedirs(os.path.dirname(image_filename), exist_ok=True)
torchvision.utils.save_image(fixed_Gz.float().cpu(), image_filename,
nrow=int(fixed_Gz.shape[0] ** 0.5), normalize=True)
# For now, every time we save, also save sample sheets
utils.sample_sheet(which_G,
classes_per_sheet=50,
num_classes=config['n_classes'],
samples_per_class=10, parallel=config['parallel'],
samples_root=config['samples_root'],
experiment_name=experiment_name,
folder_number=state_dict['itr'],
z_=z_)
def train():
ds = get_ds(BATCH_SIZE)
gan = GAN(LATENT_DIM, IMG_SHAPE)
t0 = time.time()
for ep in range(EPOCH):
for t, (img, _) in enumerate(ds):
g_img, d_loss, d_acc, g_loss, g_acc = gan.step(img)
if t % 400 == 0:
t1 = time.time()
print(
"ep={} | time={:.1f} | t={} | d_acc={:.2f} | g_acc={:.2f} | d_loss={:.2f} | g_loss={:.2f}"
.format(
ep,
t1 - t0,
t,
d_acc.numpy(),
g_acc.numpy(),
d_loss.numpy(),
g_loss.numpy(),
))
t0 = t1
save_gan(gan, ep)
save_weights(gan)
embedded_pvt = Embedding(input_dim=vocab_size,
output_dim=vec_dim,
input_length=1)(input_pvt)
embedded_ctx = Embedding(input_dim=vocab_size,
output_dim=vec_dim,
input_length=1)(input_ctx)
merged = merge(inputs=[embedded_pvt, embedded_ctx],
mode=lambda x: (x[0] * x[1]).sum(-1),
output_shape=(batch_size, 1))
predictions = Activation('sigmoid')(merged)
# build and train the model
model = Model(input=[input_pvt, input_ctx], output=predictions)
model.compile(optimizer='rmsprop', loss='mse', metrics=['accuracy'])
model.fit_generator(generator=batch_generator(couples, labels),
samples_per_epoch=samples_per_epoch,
nb_epoch=nb_epoch, verbose=1)
# save weights
utils.save_weights(model, index2word, vec_dim)
# eval using gensim
print 'the....'
utils.most_similar(positive=['the'])
print 'she - he + him....'
utils.most_similar(positive=['she', 'him'], negative=['he'])
