def D_mlp():
w_init = Normal(0.0, 0.02)
return Net([
Dense(300, w_init=w_init),
LeakyReLU(),
Dense(100, w_init=w_init),
LeakyReLU(),
Dense(1, w_init=w_init)])
def G_mlp():
w_init = Normal(0.0, 0.02)
return Net([
Dense(100, w_init=w_init),
LeakyReLU(),
Dense(300, w_init=w_init),
LeakyReLU(),
Dense(784, w_init=w_init),
Sigmoid()])
def get_model(lr):
net = Net([Dense(200),
ReLU(),
Dense(100),
ReLU(),
Dense(70),
ReLU(),
Dense(30),
ReLU(),
Dense(10)])
model = Model(net=net, loss=SoftmaxCrossEntropy(),
optimizer=Adam(lr=lr))
model.net.init_params(input_shape=(784,))
return model
def D_cnn():
return Net([
Conv2D(kernel=[5, 5, 1, 6], stride=[1, 1], padding="SAME"),
LeakyReLU(),
MaxPool2D(pool_size=[2, 2], stride=[2, 2]),
Conv2D(kernel=[5, 5, 6, 16], stride=[1, 1], padding="SAME"),
LeakyReLU(),
MaxPool2D(pool_size=[2, 2], stride=[2, 2]),
Flatten(),
Dense(120),
LeakyReLU(),
Dense(84),
LeakyReLU(),
Dense(1)])
def G_cnn():
return Net([
Dense(7 * 7 * 16),
Reshape(7, 7, 16),
ConvTranspose2D(kernel=[5, 5, 16, 6], stride=[2, 2], padding="SAME"),
LeakyReLU(),
ConvTranspose2D(kernel=[5, 5, 6, 1], stride=[2, 2], padding="SAME"),
Sigmoid()])
def main(args):
if args.seed >= 0:
random_seed(args.seed)
# data preparing
train_x, train_y, img_shape = prepare_dataset(args.img)
net = Net([
Dense(30),
ReLU(),
Dense(100),
ReLU(),
Dense(100),
ReLU(),
Dense(30),
ReLU(),
Dense(3),
Sigmoid()
])
model = Model(net=net, loss=MSE(), optimizer=Adam())
iterator = BatchIterator(batch_size=args.batch_size)
for epoch in range(args.num_ep):
for batch in iterator(train_x, train_y):
preds = model.forward(batch.inputs)
loss, grads = model.backward(preds, batch.targets)
model.apply_grads(grads)
# evaluate
preds = net.forward(train_x)
mse = mean_square_error(preds, train_y)
print("Epoch %d %s" % (epoch, mse))
# generate painting
if epoch % 5 == 0:
preds = preds.reshape(img_shape[0], img_shape[1], -1)
preds = (preds * 255.0).astype("uint8")
name, ext = os.path.splitext(args.img)
filename = os.path.basename(name)
out_filename = filename + "-paint-epoch" + str(epoch) + ext
if not os.path.exists(args.output_dir):
os.makedirs(args.output_dir)
out_path = os.path.join(args.output_dir, out_filename)
Image.fromarray(preds).save(out_path)
print("save painting to %s" % out_path)
def cnn_model():
net = Net([
Conv2D(kernel=[3, 3, 1, 2]),
MaxPool2D(pool_size=[2, 2], stride=[2, 2]),
Conv2D(kernel=[3, 3, 2, 4]),
MaxPool2D(pool_size=[2, 2], stride=[2, 2]),
Flatten(),
Dense(1)
])
return Model(net, loss=MSE(), optimizer=SGD())
def test_parameters_change(fake_dataset):
# make sure the parameters does change after apply gradients
# fake dataset
X, y = fake_dataset
# simple model
net = Net([Dense(10), Dense(1)])
loss = MSE()
opt = SGD(lr=1.0)
model = Model(net, loss, opt)
# forward and backward
pred = model.forward(X)
loss, grads = model.backward(pred, y)
# parameters change test
params_before = model.net.params.values
model.apply_grads(grads)
params_after = model.net.params.values
for p1, p2 in zip(params_before, params_after):
assert np.all(p1 != p2)
def get_model(out_dim, lr):
q_net = Net([Dense(100), ReLU(), Dense(out_dim)])
model = Model(net=q_net, loss=MSE(), optimizer=RMSProp(lr))
return model
def fc_model():
net = Net([Dense(10), Dense(1)])
loss = MSE()
opt = SGD()
return Model(net, loss, opt)
def main(args):
if args.seed >= 0:
random_seed(args.seed)
train_set, _, test_set = mnist(args.data_dir, one_hot=True)
train_x, train_y = train_set
test_x, test_y = test_set
if args.model_type == "mlp":
# A multilayer perceptron model
net = Net([
Dense(200),
ReLU(),
Dense(100),
ReLU(),
Dense(70),
ReLU(),
Dense(30),
ReLU(),
Dense(10)
])
elif args.model_type == "cnn":
# A LeNet-5 model with activation function changed to ReLU
train_x = train_x.reshape((-1, 28, 28, 1))
test_x = test_x.reshape((-1, 28, 28, 1))
net = Net([
Conv2D(kernel=[5, 5, 1, 6], stride=[1, 1]),
ReLU(),
MaxPool2D(pool_size=[2, 2], stride=[2, 2]),
Conv2D(kernel=[5, 5, 6, 16], stride=[1, 1]),
ReLU(),
MaxPool2D(pool_size=[2, 2], stride=[2, 2]),
Flatten(),
Dense(120),
ReLU(),
Dense(84),
ReLU(),
Dense(10)
])
elif args.model_type == "rnn":
# A simple recurrent neural net to classify images.
train_x = train_x.reshape((-1, 28, 28))
test_x = test_x.reshape((-1, 28, 28))
net = Net([RNN(num_hidden=50, activation=Tanh()), Dense(10)])
else:
raise ValueError("Invalid argument: model_type")
model = Model(net=net,
loss=SoftmaxCrossEntropy(),
optimizer=Adam(lr=args.lr))
iterator = BatchIterator(batch_size=args.batch_size)
loss_list = list()
for epoch in range(args.num_ep):
t_start = time.time()
for batch in iterator(train_x, train_y):
pred = model.forward(batch.inputs)
loss, grads = model.backward(pred, batch.targets)
model.apply_grads(grads)
loss_list.append(loss)
print("Epoch %d time cost: %.4f" % (epoch, time.time() - t_start))
# evaluate
model.set_phase("TEST")
test_pred = model.forward(test_x)
test_pred_idx = np.argmax(test_pred, axis=1)
test_y_idx = np.argmax(test_y, axis=1)
res = accuracy(test_pred_idx, test_y_idx)
print(res)
model.set_phase("TRAIN")
# save model
if not os.path.isdir(args.model_dir):
os.makedirs(args.model_dir)
model_name = "mnist-%s-epoch%d.pkl" % (args.model_type, args.num_ep)
model_path = os.path.join(args.model_dir, model_name)
model.save(model_path)
print("model saved in %s" % model_path)
def conv_bn_relu(kernel):
return [Conv2D(kernel=kernel, stride=(1, 1), padding="SAME"), ReLU()]
def max_pool():
return MaxPool2D(pool_size=(2, 2), stride=(2, 2), padding="SAME")
teacher_net = Net([
*conv_bn_relu((3, 3, 1, 32)), *conv_bn_relu((3, 3, 32, 32)),
max_pool(), *conv_bn_relu((3, 3, 32, 64)), *conv_bn_relu((3, 3, 64, 64)),
max_pool(), *conv_bn_relu((3, 3, 64, 128)), *conv_bn_relu(
(3, 3, 128, 128)),
max_pool(),
Flatten(),
Dense(512),
ReLU(),
Dense(10)
])
student_net = Net([
Conv2D(kernel=[5, 5, 1, 6], stride=[1, 1]),
ReLU(),
Conv2D(kernel=[5, 5, 6, 12], stride=[1, 1]),
ReLU(),
Flatten(),
Dense(10)
])
def main(args):
if args.seed >= 0:
random_seed(args.seed)
# create output directory for saving result images
if not os.path.exists(args.output_dir):
os.makedirs(args.output_dir)
# prepare and read dataset
train_set, _, test_set = mnist(args.data_dir)
train_x, train_y = train_set
test_x, test_y = test_set
# specify the encoder and decoder net structure
encoder_net = Net([
Dense(256),
ReLU(),
Dense(64)
])
decoder_net = Net([
ReLU(),
Dense(256),
Tanh(),
Dense(784),
Tanh()
])
nets = (encoder_net, decoder_net)
optimizers = (Adam(args.lr), Adam(args.lr))
model = AutoEncoder(nets, loss=MSE(), optimizer=optimizers)
# for pre-trained model, test generated images from latent space
if args.load_model is not None:
# load pre-trained model
model.load(os.path.join(args.output_dir, args.load_model))
print("Loaded model fom %s" % args.load_model)
# transition from test[from_idx] to test[to_idx] in n steps
idx_arr, n = [2, 4, 32, 12, 82], 160
print("Transition in numbers", [test_y[i] for i in idx_arr],
"in %d steps ..." % n)
stops = [model.en_net.forward(test_x[i]) for i in idx_arr]
k = int(n / (len(idx_arr) - 1)) # number of code per transition
# generate all transition codes
code_arr = []
for i in range(len(stops) - 1):
t = [c.copy() for c in transition(stops[i], stops[i+1], k)]
code_arr += t
# apply decoding all n "code" from latent space...
batch = None
for code in code_arr:
# translate latent space to image
genn = model.de_net.forward(code)
# save decoded results in a batch
if batch is None:
batch = np.array(genn)
else:
batch = np.concatenate((batch, genn))
output_path = os.path.join(args.output_dir, "genn-latent.png")
save_batch_as_images(output_path, batch)
quit()
# train the auto-encoder
iterator = BatchIterator(batch_size=args.batch_size)
for epoch in range(args.num_ep):
for batch in iterator(train_x, train_y):
origin_in = batch.inputs
# make noisy inputs
m = origin_in.shape[0] # batch size
mu = args.gaussian_mean # mean
sigma = args.gaussian_std # standard deviation
noises = np.random.normal(mu, sigma, (m, 784))
noises_in = origin_in + noises # noisy inputs
# forward
genn = model.forward(noises_in)
# back-propagate
loss, grads = model.backward(genn, origin_in)
# apply gradients
model.apply_grads(grads)
print("Epoch: %d Loss: %.3f" % (epoch, loss))
# save all the generated images and original inputs for this batch
noises_in_path = os.path.join(
args.output_dir, "ep%d-input.png" % epoch)
genn_path = os.path.join(
args.output_dir, "ep%d-genn.png" % epoch)
save_batch_as_images(noises_in_path, noises_in, titles=batch.targets)
save_batch_as_images(genn_path, genn, titles=batch.targets)
# save the model after training
model.save(os.path.join(args.output_dir, args.save_model))