def feeds(img_size,
batch_size,
epoch_size,
n_augment=int(6e5),
with_attributes=False,
split='val'):
dataset = dp.dataset.CelebA()
if split == 'val':
train_idxs = dataset.train_idxs
test_idxs = dataset.val_idxs
elif split == 'test':
train_idxs = np.hstack((dataset.train_idxs, dataset.val_idxs))
test_idxs = dataset.test_idxs
x_train = celeba_imgs(img_size, img_idxs=train_idxs, n_augment=n_augment)
x_train = np.transpose(x_train, (0, 3, 1, 2))
x_test = celeba_imgs(img_size, img_idxs=test_idxs)
x_test = img_transform(x_test, to_bc01=True)
attributes = dataset.attributes.astype(dp.float_)
y_train = attributes[train_idxs]
y_test = attributes[test_idxs]
if n_augment > 0:
y_train = y_train[np.arange(n_augment) % len(y_train)]
if with_attributes:
train_feed = SupervisedAugmentedFeed(x_train,
y_train,
batch_size=batch_size,
epoch_size=epoch_size)
test_feed = dp.SupervisedFeed(x_test, y_test, batch_size=batch_size)
else:
train_feed = AugmentedFeed(x_train, batch_size, epoch_size)
test_feed = dp.Feed(x_test, batch_size)
return train_feed, test_feed
def test_classification():
# Make dataset
n_classes = 2
n_samples = 1000
n_features = 48
x, y = make_classification(n_samples=n_samples,
n_features=n_features,
n_classes=n_classes,
n_informative=n_classes * 2,
random_state=1)
x = x.astype(dp.float_)
y = y.astype(dp.int_)
n_train = int(0.8 * n_samples)
x_train = x[:n_train]
y_train = y[:n_train]
x_test = x[n_train:]
y_test = y[n_train:]
scaler = dp.StandardScaler()
x_train = scaler.fit_transform(x_train)
x_test = scaler.transform(x_test)
# Setup feeds
batch_size = 16
train_feed = dp.SupervisedFeed(x_train, y_train, batch_size=batch_size)
test_feed = dp.Feed(x_test)
# Setup neural network
weight_decay = 1e-03
net = dp.NeuralNetwork(
layers=[
dp.Affine(
n_out=32,
weights=dp.Parameter(dp.AutoFiller(),
weight_decay=weight_decay),
),
dp.ReLU(),
dp.Affine(
n_out=64,
weights=dp.Parameter(dp.AutoFiller(),
weight_decay=weight_decay),
),
dp.ReLU(),
dp.Affine(
n_out=n_classes,
weights=dp.Parameter(dp.AutoFiller()),
),
],
loss=dp.SoftmaxCrossEntropy(),
)
# Train neural network
learn_rule = dp.Momentum(learn_rate=0.01 / batch_size, momentum=0.9)
trainer = dp.GradientDescent(net, train_feed, learn_rule)
trainer.train_epochs(n_epochs=10)
# Evaluate on test data
error = np.mean(net.predict(test_feed) != y_test)
print('Test error rate: %.4f' % error)
assert error < 0.2
def feeds(split='test',
batch_size=128,
epoch_size=None,
preprocessing='',
augmentation='',
supervised=False):
x_train, y_train, x_val, y_val, x_test, y_test = arrays(split)
if supervised:
train_feed = ShuffledSupervisedFeed(x_train,
y_train,
batch_size=batch_size,
epoch_size=epoch_size)
val_feed = dp.SupervisedFeed(x_val, y_val, batch_size=batch_size)
test_feed = dp.SupervisedFeed(x_test, y_test, batch_size=batch_size)
else:
train_feed = ShuffledFeed(x_train,
batch_size=batch_size,
epoch_size=epoch_size)
val_feed = dp.Feed(x_val, batch_size=batch_size)
test_feed = dp.Feed(x_test, batch_size=batch_size)
return train_feed, val_feed, test_feed
def unlabeled_feed(img_size, batch_size=128, epoch_size=250,
n_augment=0):
x_unlabeled, _ = arrays('unlabeled')
x_unlabeled = resize_imgs(x_unlabeled, img_size, n_augment)
if n_augment == 0:
x_unlabeled = img_transform(x_unlabeled, to_bc01=True)
unlabeled_feed = dp.Feed(x_unlabeled, batch_size=batch_size,
epoch_size=epoch_size)
else:
x_unlabeled = np.transpose(x_unlabeled, (0, 3, 1, 2))
unlabeled_feed = AugmentedFeed(x_unlabeled, batch_size=batch_size,
epoch_size=epoch_size)
return unlabeled_feed
def feeds(alignment,
crop_size,
rescale_size,
batch_size,
epoch_size,
n_augment=int(1e5),
with_attributes=False,
split='val'):
if split == 'val':
train_split = 'valtrain'
test_split = 'val'
elif split == 'test':
train_split = 'testtrain'
test_split = 'test'
x_train, y_train = lfw_imgs_split(alignment, train_split)
# Shuffle training images
idxs = np.random.permutation(len(x_train))
x_train = x_train[idxs]
y_train = y_train[idxs]
if n_augment > 0:
y_train = y_train[np.arange(n_augment) % len(x_train)]
x_train = resize_imgs(x_train, crop_size, rescale_size, n_augment)
x_train = np.transpose(x_train, (0, 3, 1, 2))
x_test, y_test = lfw_imgs_split(alignment, test_split)
x_test = resize_imgs(x_test, crop_size, rescale_size)
x_test = img_transform(x_test, to_bc01=True)
if with_attributes:
train_feed = SupervisedAugmentedFeed(x_train,
y_train,
batch_size=batch_size,
epoch_size=epoch_size)
test_feed = dp.SupervisedFeed(x_test, y_test, batch_size=batch_size)
else:
train_feed = AugmentedFeed(x_train, batch_size, epoch_size)
test_feed = dp.Feed(x_test, batch_size)
return train_feed, test_feed
dp.Affine(
n_out=2,
weights=dp.Parameter(dp.AutoFiller(w_gain)),
),
],
loss=dp.ContrastiveLoss(margin=1.0),
)
# Train network
learn_rate = 0.01 / batch_size
learn_rule = dp.RMSProp(learn_rate)
trainer = dp.GradientDescent(net, train_feed, learn_rule)
trainer.train_epochs(n_epochs=15)
# Plot 2D embedding
test_feed = dp.Feed(x_test)
x_test = np.reshape(x_test, (-1, ) + dataset.img_shape)
embedding = net.embed(test_feed)
embedding -= np.min(embedding, 0)
embedding /= np.max(embedding, 0)
plt.figure()
ax = plt.subplot(111)
shown_images = np.array([[1., 1.]])
for i in range(embedding.shape[0]):
dist = np.sum((embedding[i] - shown_images)**2, 1)
if np.min(dist) < 6e-4:
# don't show points that are too close
continue
shown_images = np.r_[shown_images, [embedding[i]]]
imagebox = offsetbox.AnnotationBbox(offsetbox.OffsetImage(
import deeppy as dp
# Fetch CIFAR10 data
dataset = dp.dataset.CIFAR10()
x_train, y_train, x_test, y_test = dataset.arrays(dp_dtypes=True)
# Normalize pixel intensities
scaler = dp.StandardScaler()
scaler.fit(x_train)
x_train = scaler.transform(x_train)
x_test = scaler.transform(x_test)
# Prepare network feeds
batch_size = 128
train_feed = dp.SupervisedFeed(x_train, y_train, batch_size=batch_size)
test_feed = dp.Feed(x_test, batch_size=batch_size)
# Setup network
def conv_layer(n_filters):
return dp.Convolution(
n_filters=32,
filter_shape=(5, 5),
border_mode='full',
weights=dp.Parameter(dp.AutoFiller(gain=1.25), weight_decay=0.003),
)
def pool_layer():
return dp.Pool(
win_shape=(3, 3),
import numpy as np
import matplotlib.pyplot as plt
import deeppy as dp
# Fetch MNIST data
dataset = dp.dataset.MNIST()
x_train, y_train, x_test, y_test = dataset.arrays(flat=True, dp_dtypes=True)
# Normalize pixel intensities
scaler = dp.UniformScaler()
x_train = scaler.fit_transform(x_train)
x_test = scaler.transform(x_test)
# Prepare autoencoder feed
batch_size = 128
train_feed = dp.Feed(x_train, batch_size=batch_size)
# Setup autoencoders
sae = dp.StackedAutoencoder(layers=[
dp.DenoisingAutoencoder(
n_out=1000,
weights=dp.Parameter(dp.AutoFiller()),
activation='sigmoid',
corruption=0.25,
),
dp.DenoisingAutoencoder(
n_out=1000,
weights=dp.Parameter(dp.AutoFiller()),
activation='sigmoid',
corruption=0.25,
),
def run():
n_hidden = 128
real_vs_gen_weight = 0.75
gan_margin = 0.3
lr_start = 0.04
lr_stop = 0.0001
lr_gamma = 0.75
n_epochs = 150
epoch_size = 250
batch_size = 64
experiment_name = 'mnist_gan'
experiment_name += '_nhidden%i' % n_hidden
out_dir = os.path.join('out', experiment_name)
arch_path = os.path.join(out_dir, 'arch.pickle')
start_arch_path = arch_path
start_arch_path = None
print('experiment_name', experiment_name)
print('start_arch_path', start_arch_path)
print('arch_path', arch_path)
# Setup network
if start_arch_path is None:
print('Creating new model')
_, decoder, discriminator = architectures.mnist()
else:
print('Starting from %s' % start_arch_path)
with open(start_arch_path, 'rb') as f:
decoder, discriminator = pickle.load(f)
model = gan.GAN(
n_hidden=n_hidden,
generator=decoder,
discriminator=discriminator,
real_vs_gen_weight=real_vs_gen_weight,
)
# Fetch dataset
dataset = dp.dataset.MNIST()
x_train, y_train, x_test, y_test = dataset.arrays()
x_train = mnist_transform(x_train)
x_test = mnist_transform(x_test)
# Prepare network feeds
train_feed = dp.Feed(x_train, batch_size, epoch_size)
test_feed = dp.Feed(x_test, batch_size)
# Plotting
n_examples = 64
original_x, = test_feed.batches().next()
original_x = np.array(original_x)[:n_examples]
samples_z = np.random.normal(size=(n_examples, n_hidden))
samples_z = (samples_z).astype(dp.float_)
# Train network
learn_rule = dp.RMSProp()
trainer = gan.GradientDescent(model,
train_feed,
learn_rule,
margin=gan_margin)
annealer = dp.GammaAnnealer(lr_start, lr_stop, n_epochs, gamma=lr_gamma)
try:
sample_video = Video(os.path.join(out_dir, 'convergence_samples.mp4'))
sp.misc.imsave(os.path.join(out_dir, 'examples.png'),
dp.misc.img_tile(mnist_inverse_transform(original_x)))
for e in range(n_epochs):
model.phase = 'train'
model.setup(*train_feed.shapes)
learn_rule.learn_rate = annealer.value(e) / batch_size
trainer.train_epoch()
model.phase = 'test'
samples_x = model.decode(samples_z)
samples_x = mnist_inverse_transform(model.decode(samples_z))
sample_video.append(dp.misc.img_tile(samples_x))
except KeyboardInterrupt:
pass
print('Saving model to disk')
with open(arch_path, 'wb') as f:
pickle.dump((decoder, discriminator), f)
model.phase = 'test'
n_examples = 100
samples_z = np.random.normal(size=(n_examples, n_hidden)).astype(dp.float_)
output.samples(model, samples_z, out_dir, mnist_inverse_transform)
output.walk(model, samples_z, out_dir, mnist_inverse_transform)
def run():
n_hidden = 64
ae_kind = 'variational'
lr_start = 0.01
lr_stop = 0.0001
lr_gamma = 0.75
n_epochs = 150
epoch_size = 250
batch_size = 64
experiment_name = 'mnist_ae'
experiment_name += '_nhidden%i' % n_hidden
out_dir = os.path.join('out', experiment_name)
arch_path = os.path.join(out_dir, 'arch.pickle')
start_arch_path = arch_path
start_arch_path = None
print('experiment_name', experiment_name)
print('start_arch_path', start_arch_path)
print('arch_path', arch_path)
# Setup network
if start_arch_path is None:
print('Creating new model')
encoder, decoder, _ = architectures.mnist()
if ae_kind == 'variational':
latent_encoder = architectures.vae_latent_encoder(n_hidden)
elif ae_kind == 'adversarial':
latent_encoder = architectures.aae_latent_encoder(n_hidden)
else:
print('Starting from %s' % start_arch_path)
with open(start_arch_path, 'rb') as f:
decoder, discriminator = pickle.load(f)
model = ae.Autoencoder(
encoder=encoder,
latent_encoder=latent_encoder,
decoder=decoder,
)
model.recon_error = ae.NLLNormal()
# Fetch dataset
dataset = dp.dataset.MNIST()
x_train, y_train, x_test, y_test = dataset.arrays()
x_train = mnist_transform(x_train)
x_test = mnist_transform(x_test)
# Prepare network feeds
train_feed = dp.Feed(x_train, batch_size, epoch_size)
test_feed = dp.Feed(x_test, batch_size)
# Plotting
n_examples = 64
original_x, = test_feed.batches().next()
original_x = np.array(original_x)[:n_examples]
samples_z = np.random.normal(size=(n_examples, n_hidden))
samples_z = (samples_z).astype(dp.float_)
# Train network
learn_rule = dp.RMSProp()
trainer = dp.GradientDescent(model, train_feed, learn_rule)
annealer = dp.GammaAnnealer(lr_start, lr_stop, n_epochs, gamma=lr_gamma)
try:
recon_video = Video(os.path.join(out_dir, 'convergence_recon.mp4'))
sample_video = Video(os.path.join(out_dir, 'convergence_samples.mp4'))
sp.misc.imsave(os.path.join(out_dir, 'examples.png'),
dp.misc.img_tile(mnist_inverse_transform(original_x)))
for e in range(n_epochs):
model.phase = 'train'
model.setup(*train_feed.shapes)
learn_rule.learn_rate = annealer.value(e) / batch_size
loss = trainer.train_epoch()
model.phase = 'test'
original_z = model.encode(original_x)
recon_x = model.decode(original_z)
samples_x = model.decode(samples_z)
recon_x = mnist_inverse_transform(recon_x)
samples_x = mnist_inverse_transform(model.decode(samples_z))
recon_video.append(dp.misc.img_tile(recon_x))
sample_video.append(dp.misc.img_tile(samples_x))
likelihood = model.likelihood(test_feed)
print('epoch %i Train loss:%.4f Test likelihood:%.4f' %
(e, np.mean(loss), np.mean(likelihood)))
except KeyboardInterrupt:
pass
print('Saving model to disk')
with open(arch_path, 'wb') as f:
pickle.dump((decoder, discriminator), f)
model.phase = 'test'
n_examples = 100
samples_z = np.random.normal(size=(n_examples, n_hidden)).astype(dp.float_)
output.samples(model, samples_z, out_dir, mnist_inverse_transform)
output.walk(model, samples_z, out_dir, mnist_inverse_transform)
