def generator(images, options, reuse=False, name='gen'):
# reuse or not
with tf.variable_scope(name):
if reuse:
tf.get_variable_scope().reuse_variables()
else:
assert tf.get_variable_scope().reuse is False
# down sampling
x = relu(instance_norm(conv2d(images, options.nf, ks=7, s=1, name='gen_ds_conv1'), 'in1_1'))
x = relu(instance_norm(conv2d(x, 2*options.nf, ks=4, s=2, name='gen_ds_conv2'), 'in1_2'))
x = relu(instance_norm(conv2d(x, 4*options.nf, ks=4, s=2, name='gen_ds_conv3'), 'in1_3'))
# bottleneck
x = relu(instance_norm(conv2d(x, 4*options.nf, ks=3, s=1, name='gen_bn_conv1'), 'in2_1'))
x = relu(instance_norm(conv2d(x, 4*options.nf, ks=3, s=1, name='gen_bn_conv2'), 'in2_2'))
x = relu(instance_norm(conv2d(x, 4*options.nf, ks=3, s=1, name='gen_bn_conv3'), 'in2_3'))
x = relu(instance_norm(conv2d(x, 4*options.nf, ks=3, s=1, name='gen_bn_conv4'), 'in2_4'))
x = relu(instance_norm(conv2d(x, 4*options.nf, ks=3, s=1, name='gen_bn_conv5'), 'in2_5'))
x = relu(instance_norm(conv2d(x, 4*options.nf, ks=3, s=1, name='gen_bn_conv6'), 'in2_6'))
# up sampling
x = relu(instance_norm(deconv2d(x, 2*options.nf, ks=4, s=2, name='gen_us_deconv1'), 'in3_1'))
x = relu(instance_norm(deconv2d(x, options.nf, ks=4, s=2, name='gen_us_deconv2'), 'in3_2'))
x = tanh(deconv2d(x, 3, ks=7, s=1, name='gen_us_dwconv3'))
return x
def generator(self, inputs, scope='generator', reuse=None):
with tf.variable_scope(scope, reuse=reuse):
# generator will upscale a tiny image with layers of convolution
# until it reaches the final output image dimensions. These vars
# are the starting tiny image dimensions.
# This is why this arch needs images that are divisible by 32
minirows = self.img_shape[0] // 32
minicols = self.img_shape[1] // 32
# batch normalization, which needs to know whether this is training or
# application
bn = BN(self.is_training)
# dense (i.e. fully connected) layer followed by reshaping into the tiny
# image. The tiny image has a Z dim of 512 that gradually gets reduced
# to 3 channels (r, g, b)
t = dense(inputs, minirows * minicols * 512)
t = lrelu(bn(reshape(t,
(tf.shape(t)[0], minirows, minicols, 512))))
t = lrelu(bn(conv2dtr(t, 512)))
t = lrelu(bn(conv2dtr(t, 256)))
t = lrelu(bn(conv2dtr(t, 128)))
t = lrelu(bn(conv2dtr(t, 64)))
# final conv2d transpose to get to filter depth of 3, for rgb channels
logits = conv2dtr(t, self.img_shape[2])
return tanh(logits) # common final activation in GANs
def test_tanh():
batch = np.array([[1.0, 2.0], [3.0, 4.0]])
edugrad_batch = Tensor(batch)
torch_batch = torch.tensor(batch, requires_grad=True)
# forward
outputs = ops.tanh(edugrad_batch)
torch_outputs = torch.tanh(torch_batch)
np.testing.assert_allclose(outputs.value, torch_outputs.detach().numpy())
# backward
reduce_mean(outputs).backward()
torch_outputs.mean().backward()
np.testing.assert_allclose(edugrad_batch.grad, torch_batch.grad.numpy())
def generator(self, x, c, reuse=False):
print("Generator ...........")
with tf.variable_scope('generator') as scope:
if (reuse):
scope.reuse_variables()
c = tf.reshape(c, [-1, 1, 1, self.feature_length])
c = tf.tile(c, [1, 64, 64, 1])
inputs = tf.concat([x, c], axis=3)
# print(inputs)
# Iinitial concat conv layer
# pad 3 => https://arxiv.org/pdf/1711.09020.pdf
outputs = tf.pad(inputs, [[0, 0], [3, 3], [3, 3], [0, 0]])
outputs = ops.conv2d(outputs, 64, 7, 1, 'VALID', scope="g_init")
# Downsize twice
outputs = ops.conv2d(outputs, 128, 4, 2, scope="g_down_sample_1")
outputs = ops.conv2d(outputs, 256, 4, 2, scope="g_down_sample_2")
# Resblock CONV-(N256, K3x3, S1, P1), IN, ReLU
outputs = residule_block(outputs, 256, 3, 1, scope="g_resblock_1")
outputs = residule_block(outputs, 256, 3, 1, scope="g_resblock_2")
outputs = residule_block(outputs, 256, 3, 1, scope="g_resblock_3")
outputs = residule_block(outputs, 256, 3, 1, scope="g_resblock_4")
outputs = residule_block(outputs, 256, 3, 1, scope="g_resblock_5")
outputs = residule_block(outputs, 256, 3, 1, scope="g_resblock_6")
# Upsampling twice
outputs = ops.instance_norm(
ops.relu(
ops.deconv2d(outputs, 128, 4, 2, scope='g_upsampling_1')),
'g_in_1')
outputs = ops.instance_norm(
ops.relu(
ops.deconv2d(outputs, 64, 4, 2, scope='g_upsampling_2')),
'g_in_2')
# pad 3 => https://arxiv.org/pdf/1711.09020.pdf
outputs = tf.pad(outputs, [[0, 0], [3, 3], [3, 3], [0, 0]])
outputs = ops.tanh(
ops.conv2d(outputs, 3, 7, 1, 'VALID', scope='g_out'))
return outputs
def generator(images, options, reuse=False, name='gen'):
# down sampling
x = relu(instance_norm(conv2d(images, options.nf, ks=7, s=1, name='gen_ds_conv1'), 'in1_1'))
x = relu(instance_norm(conv2d(x, 2*options.nf, ks=4, s=2, name='gen_ds_conv2'), 'in1_2'))
x = relu(instance_norm(conv2d(x, 4*options.nf, ks=4, s=2, name='gen_ds_conv3'), 'in1_3'))
# bottleneck
x = relu(instance_norm(conv2d(x, 4*options.nf, ks=3, s=1, name='gen_bn_conv1'), 'in2_1'))
x = relu(instance_norm(conv2d(x, 4*options.nf, ks=3, s=1, name='gen_bn_conv2'), 'in2_2'))
x = relu(instance_norm(conv2d(x, 4*options.nf, ks=3, s=1, name='gen_bn_conv3'), 'in2_3'))
x = relu(instance_norm(conv2d(x, 4*options.nf, ks=3, s=1, name='gen_bn_conv4'), 'in2_4'))
x = relu(instance_norm(conv2d(x, 4*options.nf, ks=3, s=1, name='gen_bn_conv5'), 'in2_5'))
x = relu(instance_norm(conv2d(x, 4*options.nf, ks=3, s=1, name='gen_bn_conv6'), 'in2_6'))
# up sampling
x = relu(instance_norm(deconv2d(x, 2*options.nf, ks=4, s=2, name='gen_us_deconv1'), 'in3_1'))
x = relu(instance_norm(deconv2d(x, options.nf, ks=4, s=2, name='gen_us_deconv2'), 'in3_2'))
x = tanh(deconv2d(x, 3, ks=7, s=1, name='gen_us_dwconv3'))
return x
def forward(self, word_indices: list[Tensor]) -> Tensor:
"""Executes the forward pass of a FeedForwardLanguageModel.
Args:
word_indices: list of [batch_size] tensors
length = number of previous characters / n-gram length
each one contains indices of chars at that position
Returns:
[batch_size, vocab_size] Tensor
containing logits (not full probabilities, i.e. pre-softmax)
over the vocab for each example in the batch
"""
# TODO: (~7 lines) implement the forward pass of FFNN LM here
# HINT: use ops.concat to concatenate word embeddings together
# It takes a variable-length list of Tensors as its input, so you can
# call it using as ops.concat(*embeddings), where embeddings is a list
# of Tensors, corresponding to the relevant embeddings
# [batch_size, num_words * embedding_size]
embs = ops.concat(*[self.embedding(index) for index in word_indices])
return self.output(ops.tanh(self.fc(embs)))
def fprop(self):
logger.debug('%s prop: %s' % (str(self), str(self.pred.out.shape)))
self.out = tanh(self.pred.out)
def fprop(self):
logger.debug("%s prop: %s" % (str(self), str(self.pred.out.shape)))
self.out = tanh(self.pred.out)