def test_two_layer_conv(self): bs = 10 r = 2 n_channels = 3 n_filters = 5 w, h = 32, 32 # define a simple model x = nn.Tensor((bs, n_channels, h, w)) W1 = nn.Constant( np.random.randn(n_filters, n_channels, 2 * r + 1, 2 * r + 1).astype(np.float32)) W2 = nn.Constant( np.random.randn(n_channels, n_filters, 2 * r + 1, 2 * r + 1).astype(np.float32)) y = nn.conv(x, W1) self.assertEqual(y.output.shape, (bs, n_filters, h, w)) y = nn.conv(y, W2) self.assertEqual(y.output.shape, (bs, n_channels, h, w)) # compile model mdl = nn.compile_model(x, y) # emulate model with numpy npx = np.random.randn(bs, n_channels, h, w).astype(np.float32) npy = np_pad(npx, r) npy = np_conv(npy, W1.value) npy = np_pad(npy, r) npy = np_conv(npy, W2.value) # results must be very similar res = mdl(npx) self.assertLess(np.max(np.abs(npy - res)), 1e-3)
def main(): tot_images = 50000 bs = 200 n_filters = 1024 x = nn.Tensor((bs, 3, 32, 32)) w_1 = nn.Constant(np.random.randn(n_filters, 3, 5, 5).astype(np.float32)) y = nn.conv(x, w_1) # y = nn.activation(y, lambda x: "exp(-({x}))") mdl = nn.compile_model(x, y) # s = time.time() npx = np.random.randn(bs, 3, 32, 32).astype(np.float32) # npy = np_pad(npx, 2) # npy = np_conv(npy, w_1.value) # e = time.time() # print("Numpy time", e - s) s = time.time() res = mdl(npx) e = time.time() print("OpenCL time", e - s) print( f"Expected time for {tot_images} is {(tot_images / bs) * (e - s)} seconds ({(tot_images / (bs * 60)) * (e - s)} minutes)" )
def network(self, input, keep_prob=0.5, reuse=None): with tf.variable_scope('network', reuse=reuse): pool_ = lambda x: nn.max_pool(x, 2, 2) max_out_ = lambda x: nn.max_out(x, 16) conv_ = lambda x, output_depth, name, trainable=True: nn.conv( x, 3, output_depth, 1, self.weight_decay, name=name, trainable=trainable) fc_ = lambda x, features, name, relu=True: nn.fc( x, features, self.weight_decay, name, relu=relu) VGG_MEAN = [103.939, 116.779, 123.68] # Convert RGB to BGR and subtract mean # red, green, blue = tf.split(input, 3, axis=3) input = tf.concat([ input - 24, input - 24, input - 24, ], axis=3) conv_1_1 = conv_(input, 64, 'conv1_1', trainable=False) conv_1_2 = conv_(conv_1_1, 64, 'conv1_2', trainable=False) pool_1 = pool_(conv_1_2) conv_2_1 = conv_(pool_1, 128, 'conv2_1', trainable=False) conv_2_2 = conv_(conv_2_1, 128, 'conv2_2', trainable=False) pool_2 = pool_(conv_2_2) conv_3_1 = conv_(pool_2, 256, 'conv3_1') conv_3_2 = conv_(conv_3_1, 256, 'conv3_2') conv_3_3 = conv_(conv_3_2, 256, 'conv3_3') pool_3 = pool_(conv_3_3) conv_4_1 = conv_(pool_3, 512, 'conv4_1') conv_4_2 = conv_(conv_4_1, 512, 'conv4_2') conv_4_3 = conv_(conv_4_2, 512, 'conv4_3') pool_4 = pool_(conv_4_3) conv_5_1 = conv_(pool_4, 512, 'conv5_1') conv_5_2 = conv_(conv_5_1, 512, 'conv5_2') conv_5_3 = conv_(conv_5_2, 512, 'conv5_3') pool_5 = pool_(conv_5_3) if self.maxout: max_5 = max_out_(pool_5) flattened = tf.contrib.layers.flatten(max_5) else: flattened = tf.contrib.layers.flatten(pool_5) fc_6 = nn.dropout(fc_(flattened, 4096, 'fc6'), keep_prob) fc_7 = nn.dropout(fc_(fc_6, 4096, 'fc7'), keep_prob) fc_8 = fc_(fc_7, self.label_dim, 'fc8', relu=False) return fc_8
def __init__(self): self.train_rate = .1 self.chess = tf.placeholder(shape=[rules.N, rules.N], dtype=np.float32, name='chess') self.tar_choise = tf.placeholder(shape=[rules.N, rules.N], dtype=np.float32, name='choise') with tf.variable_scope('model'): x = tf.pad(self.chess, [[0, 1], [0, 1]]) x = tf.reshape(x, [1, rules.N + 1, rules.N + 1, 1]) with tf.variable_scope('layer1'): x = nn.conv(x, [3, 3], 4, name='conv1') x = nn.conv(x, [3, 3], 8, name='conv2') x = nn.pool(x, [2, 2]) x = nn.relu(x) with tf.variable_scope('layer2'): x = nn.conv(x, [3, 3], 16, name='conv1') x = nn.conv(x, [3, 3], 32, name='conv2') x = nn.pool(x, [2, 2]) x = nn.relu(x) with tf.variable_scope('layer3'): x = nn.conv(x, [3, 3], 64, name='conv1') x = nn.conv(x, [3, 3], 128, name='conv2') x = nn.pool(x, [2, 2]) x = nn.relu(x) with tf.variable_scope('layer4'): x = nn.conv(x, [3, 3], 256, name='conv1') x = nn.conv(x, [3, 3], 512, name='conv2') x = nn.pool(x, [2, 2]) x = nn.relu(x) with tf.variable_scope('layer-4'): x = nn.unpool(x) x = nn.conv(x, [3, 3], 256, name='conv1') x = nn.conv(x, [3, 3], 128, name='conv2') x = nn.relu(x) with tf.variable_scope('layer-3'): x = nn.unpool(x) x = nn.conv(x, [3, 3], 64, name='conv1') x = nn.conv(x, [3, 3], 32, name='conv2') x = nn.relu(x) with tf.variable_scope('layer-2'): x = nn.unpool(x) x = nn.conv(x, [3, 3], 16, name='conv1') x = nn.conv(x, [3, 3], 8, name='conv2') x = nn.relu(x) with tf.variable_scope('layer-1'): x = nn.unpool(x) x = nn.conv(x, [3, 3], 4, name='conv1') x = nn.conv(x, [3, 3], 1, name='conv2') x = nn.relu(x) x = tf.reshape(x, [rules.N + 1, rules.N + 1]) x = x[0:15,0:15] self.choise = x self.loss = tf.nn.l2_loss(self.choise - self.tar_choise) self.opt = tf.train.AdadeltaOptimizer(self.train_rate).minimize(self.loss) self.sess = tf.InteractiveSession() self.saver = tf.train.Saver(max_to_keep=25) self.initer = tf.global_variables_initializer()
def build(self, batch_size=1, train_rate=1e-3): global SHAPE self.x = tf.placeholder(dtype=tf.float32, shape=[batch_size] + SHAPE) self.y = tf.placeholder(dtype=tf.float32, shape=[batch_size] + SHAPE) with tf.variable_scope('model0'): c1 = nn.conv(self.x, 4, 'conv1') p1 = nn.pool(c1, 'pool1') c2 = nn.conv(p1, 8, 'conv2') p2 = nn.pool(c2, 'pool2') c3 = nn.conv(p2, 16, 'conv3') p3 = nn.pool(c3, 'pool3') c4 = nn.conv(p3, 32, 'conv4') p4 = nn.pool(c4, 'pool4') c5_1 = nn.conv(p4, 64, 'conv5_1') c5_2 = nn.conv(c5_1, 64, 'conv5_2') p5 = nn.pool(c5_2, 'pool5') c6_1 = nn.conv(p5, 128, 'conv6_1') c6_2 = nn.conv(c6_1, 96, 'conv6_2') dc6_1 = nn.conv(c6_2, 96, 'deconv6_1') dc6_2 = nn.conv(dc6_1, 64, 'deconv6_2') up5 = nn.layer_add( nn.unpool(dc6_2, 'unpool5_tmp', shape=c5_2.get_shape().as_list()), c5_2, 'unpool5') dc5_1 = nn.conv(up5, 32, 'deconv5_1') dc5_2 = nn.conv(dc5_1, 32, 'deconv5_2') up4 = nn.layer_add( nn.unpool(dc5_2, 'unpool4_tmp', shape=c4.get_shape().as_list()), c4, 'unpool4') dc4 = nn.conv(up4, 16, 'deconv4') up3 = nn.layer_add( nn.unpool(dc4, 'unpool3_tmp', shape=c3.get_shape().as_list()), c3, 'unpool3') dc3 = nn.conv(up3, 8, 'deconv3') up2 = nn.layer_add( nn.unpool(dc3, 'unpool2_tmp', shape=c2.get_shape().as_list()), c2, 'unpool2') dc2 = nn.conv(up2, 4, 'deconv2') up1 = nn.layer_add( nn.unpool(dc2, 'unpool1_tmp', shape=c1.get_shape().as_list()), c1, 'unpool1') dc1 = nn.conv(up1, 3, 'deconv1') self.out = dc1 self.loss = nn.LOSS + tf.nn.l2_loss(self.out - self.y) self.opt = tf.train.AdamOptimizer(train_rate).minimize(self.loss) # for tensorboard tf.summary.scalar('loss', self.loss)
def EmbeddingImagenet(inp): """Returns the Imagenet-specific grayscale embedding for the given input.""" with tf.name_scope("embedding"): channels_cond = 64 leak = nn.conv(inp, "conv_leak", filter_size=(3, 3), stride=1, out_channels=channels_cond) with tf.name_scope("down_pass"): leak = nn.gated_resnet(leak, "down_leak_%d" % 1, a=None, conv=nn.conv) leak = nn.gated_resnet(leak, "down_leak_%d" % 2, a=None, conv=nn.conv) channels_cond *= 2 leak = nn.conv(leak, "downscale_leak_1", filter_size=(3, 3), stride=2, out_channels=channels_cond) leak = nn.gated_resnet(leak, "down_leak_%d" % 3, a=None, conv=nn.conv) leak = nn.gated_resnet(leak, "down_leak_%d" % 4, a=None, conv=nn.conv) channels_cond *= 2 leak = nn.conv(leak, "downscale_leak_2", filter_size=(3, 3), stride=2, out_channels=channels_cond) leak = nn.gated_resnet(leak, "down_leak_%d" % 5, a=None, conv=nn.conv) leak = nn.gated_resnet(leak, "down_leak_%d" % 6, a=None, conv=nn.conv) channels_cond *= 2 leak = nn.conv(leak, "downscale_leak_3", filter_size=(3, 3), stride=1, out_channels=channels_cond) leak = nn.gated_resnet(leak, "down_leak_%d" % 7, a=None, conv=nn.conv, dilation=2) leak = nn.gated_resnet(leak, "down_leak_%d" % 8, a=None, conv=nn.conv, dilation=2) leak = nn.gated_resnet(leak, "down_leak_%d" % 9, a=None, conv=nn.conv, dilation=2) leak = nn.conv(leak, "downscale_leak_4", filter_size=(3, 3), stride=1, out_channels=channels_cond) leak = nn.gated_resnet(leak, "down_leak_%d" % 10, a=None, conv=nn.conv, dilation=4) leak = nn.gated_resnet(leak, "down_leak_%d" % 11, a=None, conv=nn.conv, dilation=4) leak = nn.gated_resnet(leak, "down_leak_%d" % 12, a=None, conv=nn.conv, dilation=4) # Minor bug: wrong number of channels (TODO: retrian the model and fix the code) embedding = nn.conv(leak, "downscale_leak_5", filter_size=(3, 3), stride=1, out_channels=160) return embedding
def EmbeddingCIFAR(inp): """Returns the CIFAR-specific grayscale embedding for the given input.""" with tf.name_scope("embedding"): channels_cond = 32 leak = nn.conv(inp, "conv_leak", filter_size=(3, 3), stride=1, out_channels=channels_cond) with tf.name_scope("down_pass"): leak = nn.gated_resnet(leak, "down_leak_%d" % 1, a=None, conv=nn.conv) leak = nn.gated_resnet(leak, "down_leak_%d" % 2, a=None, conv=nn.conv) channels_cond *= 2 leak = nn.conv(leak, "downscale_leak_1", filter_size=(3, 3), stride=2, out_channels=channels_cond) leak = nn.gated_resnet(leak, "down_leak_%d" % 3, a=None, conv=nn.conv) leak = nn.gated_resnet(leak, "down_leak_%d" % 4, a=None, conv=nn.conv) channels_cond *= 2 leak = nn.conv(leak, "downscale_leak_2", filter_size=(3, 3), stride=1, out_channels=channels_cond) leak = nn.gated_resnet(leak, "down_leak_%d" % 5, a=None, conv=nn.conv) leak = nn.gated_resnet(leak, "down_leak_%d" % 6, a=None, conv=nn.conv) channels_cond *= 2 leak = nn.conv(leak, "downscale_leak_3", filter_size=(3, 3), stride=1, out_channels=channels_cond) leak = nn.gated_resnet(leak, "down_leak_%d" % 7, a=None, conv=nn.conv, dilation=2) leak = nn.gated_resnet(leak, "down_leak_%d" % 8, a=None, conv=nn.conv, dilation=2) leak = nn.gated_resnet(leak, "down_leak_%d" % 9, a=None, conv=nn.conv, dilation=2) embedding = nn.conv(leak, "downscale_leak_4", filter_size=(3, 3), stride=1, out_channels=channels_cond) return embedding
def PIColorization(x, x_gray, channels, l, num_outputs, dataset, return_embedding=False): """Define the auto-regressive network. Args: x: input x_gray: grayscale embedding channels: network width l (int): number of residual layers in the embedding network num_outputs (int): number of coeffs (ie logistic mixtures * n_coeffs per mixture) dataset (str): dataset return_embedding (bool, optional): if True, also return the embedding. Defaults to False """ # PIC with tf.name_scope("pic"): with tf.name_scope("pad"): x_pad = tf.concat([x, tf.ones(nn.int_shape(x)[:-1] + [1])], 3, name="x_pad") x_gray = tf.concat( [x_gray, tf.ones(nn.int_shape(x_gray)[:-1] + [1])], 3, name="gray_pad") # Embedding assert (dataset in ['cifar', 'imagenet']) if dataset == 'cifar': embedding = EmbeddingCIFAR(x_gray) elif dataset == 'imagenet': embedding = EmbeddingImagenet(x_gray) # PixelCNN++ with tf.name_scope("pcnn"): u = nn.down_shift( nn.down_shifted_conv2d(x_pad, "conv_down", filter_size=(2, 3), out_channels=channels)) ul = nn.down_shift(nn.down_shifted_conv2d(x_pad, "conv_down_2", filter_size=(1, 3), out_channels=channels)) + \ nn.right_shift(nn.down_right_shifted_conv2d(x_pad, "conv_down_right", filter_size=(2, 1), out_channels=channels)) for rep in range(l): u = nn.gated_resnet(u, "shortrange_down_%d" % rep, a=embedding, conv=nn.down_shifted_conv2d) ul = nn.gated_resnet(ul, "shortrange_down_right_%d" % rep, a=tf.concat([u, embedding], 3), conv=nn.down_right_shifted_conv2d) x_out = nn.conv(tf.nn.elu(ul), "conv_last", (1, 1), num_outputs) if return_embedding: return x_out, embedding else: return x_out
def unet(self, input, mean, keep_prob=0.9, reuse=None): with tf.variable_scope('vgg', reuse=reuse): input = input - mean # Demean pool_ = lambda x: nn.max_pool(x, 2, 2) conv_ = lambda x, output_depth, name, padding='SAME', relu=True, filter_size=3: nn.conv( x, filter_size, output_depth, 1, self.weight_decay, name=name, padding=padding, relu=relu) deconv_ = lambda x, output_depth, name: nn.deconv( x, 2, output_depth, 2, self.weight_decay, name=name) conv_1_1 = conv_(input, 64, 'conv1_1') conv_1_2 = conv_(conv_1_1, 64, 'conv1_2') pool_1 = pool_(conv_1_2) conv_2_1 = conv_(pool_1, 128, 'conv2_1') conv_2_2 = conv_(conv_2_1, 128, 'conv2_2') pool_2 = pool_(conv_2_2) conv_3_1 = conv_(pool_2, 256, 'conv3_1') conv_3_2 = conv_(conv_3_1, 256, 'conv3_2') pool_3 = pool_(conv_3_2) conv_4_1 = conv_(pool_3, 512, 'conv4_1') conv_4_2 = conv_(conv_4_1, 512, 'conv4_2') pool_4 = pool_(conv_4_2) conv_5_1 = conv_(pool_4, 1024, 'conv5_1') conv_5_2 = conv_(conv_5_1, 1024, 'conv5_2') pool_5 = pool_(conv_5_2) conv_6_1 = tf.nn.dropout(conv_(pool_5, 2048, 'conv6_1'), keep_prob) conv_6_2 = tf.nn.dropout(conv_(conv_6_1, 2048, 'conv6_2'), keep_prob) up_7 = tf.concat([deconv_(conv_6_2, 1024, 'up7'), conv_5_2], 3) # Error here rn conv_7_1 = conv_(up_7, 1024, 'conv7_1') conv_7_2 = conv_(conv_7_1, 1024, 'conv7_2') up_8 = tf.concat([deconv_(conv_7_2, 512, 'up8'), conv_4_2], 3) conv_8_1 = conv_(up_8, 512, 'conv8_1') conv_8_2 = conv_(conv_8_1, 512, 'conv8_2') up_9 = tf.concat([deconv_(conv_8_2, 256, 'up9'), conv_3_2], 3) conv_9_1 = conv_(up_9, 256, 'conv9_1') conv_9_2 = conv_(conv_9_1, 256, 'conv9_2') up_10 = tf.concat([deconv_(conv_9_2, 128, 'up10'), conv_2_2], 3) conv_10_1 = conv_(up_10, 128, 'conv10_1') conv_10_2 = conv_(conv_10_1, 128, 'conv10_2') up_11 = tf.concat([deconv_(conv_10_2, 64, 'up11'), conv_1_2], 3) conv_11_1 = conv_(up_11, 64, 'conv11_1') conv_11_2 = conv_(conv_11_1, 64, 'conv11_2') conv_12 = conv_(conv_11_2, 9, 'conv12_2', filter_size=1, relu=False) return conv_12