def test_forward_invariance_90(): """Test invariance to 90 rotations of the input""" tf.reset_default_graph() x = tf.placeholder(tf.float32, [4,3,3,1,1,1]) y = hl.conv2d(x, 1, 3, name='conv_forward_invariance_90') inv = hl.stack_magnitudes(y) X1 = np.random.randn(1,3,3,1,1,1) X2 = np.transpose(X1, (0,2,1,3,4,5))[:,:,::-1,:,:,:] X3 = np.transpose(X2, (0,2,1,3,4,5))[:,:,::-1,:,:,:] X4 = np.transpose(X3, (0,2,1,3,4,5))[:,:,::-1,:,:,:] X = np.concatenate((X1,X2,X3,X4), axis=0) with tf.Session() as sess: init_op = tf.global_variables_initializer() sess.run(init_op) Y, Inv = sess.run([y, inv], feed_dict={x: X}) Y = np.squeeze(Y) Inv = np.squeeze(Inv) # Look at the difference in order 0 vectors...should be EXACTLY 0 for i in range(4): for j in range(i): assert np.amax(np.abs(Y[j,0,:] - Y[i,0,:])) < 1e-5 print # Look at the difference in magnitudes for i in range(4): for j in range(i): assert np.amax(np.abs(Inv[j,:] - Inv[i,:])) < 1e-5
def test_forward_invariance_90(): """Test invariance to 90 rotations of the input""" tf.reset_default_graph() x = tf.placeholder(tf.float32, [4,3,3,1,1,1]) y = hl.conv2d(x, 1, 3, name='conv_forward_invariance_90') inv = hl.stack_magnitudes(y) X1 = np.random.randn(1,3,3,1,1,1) X2 = np.transpose(X1, (0,2,1,3,4,5))[:,:,::-1,:,:,:] X3 = np.transpose(X2, (0,2,1,3,4,5))[:,:,::-1,:,:,:] X4 = np.transpose(X3, (0,2,1,3,4,5))[:,:,::-1,:,:,:] X = np.concatenate((X1,X2,X3,X4), axis=0) with tf.Session() as sess: init_op = tf.global_variables_initializer() sess.run(init_op) Y, Inv = sess.run([y, inv], feed_dict={x: X}) Y = np.squeeze(Y) Inv = np.squeeze(Inv) # Look at the difference in order 0 vectors...should be EXACTLY 0 for i in xrange(4): for j in xrange(i): assert np.amax(np.abs(Y[j,0,:] - Y[i,0,:])) < 1e-5 print # Look at the difference in magnitudes for i in xrange(4): for j in xrange(i): assert np.amax(np.abs(Inv[j,:] - Inv[i,:])) < 1e-5
def test_backward_pass_shape(): """Make sure that a gradient is created""" tf.reset_default_graph() x = tf.placeholder(tf.float32, [2,3,3,1,1,1]) y = hl.conv2d(x, 1, 3, name='conv_backward_pass') g = tf.gradients(y, x) X = np.random.randn(2,3,3,1,1,1) with tf.Session() as sess: init_op = tf.global_variables_initializer() sess.run(init_op) G = sess.run(g, feed_dict={x: X}) assert G[0].shape == (2,3,3,1,1,1)
def test_forward_pass_shape(): """Convolve with random noise""" for i in range(3,13): tf.reset_default_graph() x = tf.placeholder(tf.float32, [6,i,i,1,1,4]) y = hl.conv2d(x, 5, 3, name='conv_forward_pass_shape') X = np.random.randn(6,i,i,1,1,4) with tf.Session() as sess: init_op = tf.global_variables_initializer() sess.run(init_op) Y = sess.run(y, feed_dict={x: X}) assert Y.shape == (6,i-2,i-2,2,2,5)
def test_backward_pass_shape(): """Make sure that a gradient is created""" tf.reset_default_graph() x = tf.placeholder(tf.float32, [2,3,3,1,1,1]) y = hl.conv2d(x, 1, 3, name='conv_backward_pass') g = tf.gradients(y, x) X = np.random.randn(2,3,3,1,1,1) with tf.Session() as sess: init_op = tf.global_variables_initializer() sess.run(init_op) G = sess.run(g, feed_dict={x: X}) assert G[0].shape == (2,3,3,1,1,1)
def test_forward_pass_shape(): """Convolve with random noise""" for i in xrange(3,13): tf.reset_default_graph() x = tf.placeholder(tf.float32, [6,i,i,1,1,4]) y = hl.conv2d(x, 5, 3, name='conv_forward_pass_shape') X = np.random.randn(6,i,i,1,1,4) with tf.Session() as sess: init_op = tf.global_variables_initializer() sess.run(init_op) Y = sess.run(y, feed_dict={x: X}) assert Y.shape == (6,i-2,i-2,2,2,5)
def deep_mnist(args, x, train_phase): """The MNIST-rot model similar to the one in Cohen & Welling, 2016""" # Sure layers weight & bias order = 1 # Number of Filters nf = args.n_filters nf2 = int(nf*args.filter_gain) nf3 = int(nf*(args.filter_gain**2.)) bs = args.batch_size fs = args.filter_size ncl = args.n_classes sm = args.std_mult nr = args.n_rings # Create bias for final layer bias = tf.get_variable('b7', shape=[args.n_classes], initializer=tf.constant_initializer(1e-2)) x = tf.reshape(x, shape=[bs,args.dim,args.dim,1,1,1]) # Convolutional Layers with pooling with tf.name_scope('block1') as scope: cv1 = hn_lite.conv2d(x, nf, fs, padding='SAME', n_rings=nr, name='1') cv1 = hn_lite.non_linearity(cv1, tf.nn.relu, name='1') cv2 = hn_lite.conv2d(cv1, nf, fs, padding='SAME', n_rings=nr, name='2') cv2 = hn_lite.batch_norm(cv2, train_phase, name='bn1') with tf.name_scope('block2') as scope: cv2 = hn_lite.mean_pool(cv2, ksize=(1,2,2,1), strides=(1,2,2,1)) cv3 = hn_lite.conv2d(cv2, nf2, fs, padding='SAME', n_rings=nr, name='3') cv3 = hn_lite.non_linearity(cv3, tf.nn.relu, name='3') cv4 = hn_lite.conv2d(cv3, nf2, fs, padding='SAME', n_rings=nr, name='4') cv4 = hn_lite.batch_norm(cv4, train_phase, name='bn2') with tf.name_scope('block3') as scope: cv4 = hn_lite.mean_pool(cv4, ksize=(1,2,2,1), strides=(1,2,2,1)) cv5 = hn_lite.conv2d(cv4, nf3, fs, padding='SAME', n_rings=nr, name='5') cv5 = hn_lite.non_linearity(cv5, tf.nn.relu, name='5') cv6 = hn_lite.conv2d(cv5, nf3, fs, padding='SAME', n_rings=nr, name='6') cv6 = hn_lite.batch_norm(cv6, train_phase, name='bn3') # Final Layer with tf.name_scope('block4') as scope: cv7 = hn_lite.conv2d(cv6, ncl, fs, padding='SAME', n_rings=nr, phase=False, name='7') real = hn_lite.sum_magnitudes(cv7) cv7 = tf.reduce_mean(real, axis=[1,2,3,4]) return tf.nn.bias_add(cv7, bias)
def deep_cifar(opt, x, train_phase, device='/cpu:0'): """High frequency convolutions are unstable, so get rid of them""" # Abbreviations nf = opt['n_filters'] fg = opt['filter_gain'] bs = opt['batch_size'] fs = opt['filter_size'] N = opt['resnet_block_multiplicity'] with tf.device(device): initializer = tf.contrib.layers.variance_scaling_initializer() Wgap = tf.get_variable('Wfc', shape=[fg * fg * nf, opt['n_classes']], initializer=initializer) bgap = tf.get_variable('bfc', shape=[opt['n_classes']], initializer=tf.constant_initializer(1e-2)) x = tf.reshape( x, shape=[bs, opt['dim'], opt['dim'], 1, 1, opt['n_channels']]) # Convolutional Layers res1 = hn_lite.conv2d(x, nf, fs, padding='SAME', name='in', device=device) for i in xrange(N): name = 'r1_' + str(i) res1 = hn_lite.residual_block(res1, nf, fs, 2, train_phase, name=name, device=device) res2 = hn_lite.mean_pool(res1, ksize=(1, 2, 2, 1), strides=(1, 2, 2, 1), name='mp1') for i in xrange(N): name = 'r2_' + str(i) res2 = hn_lite.residual_block(res2, fg * nf, fs, 2, train_phase, name=name, device=device) res3 = hn_lite.mean_pool(res2, ksize=(1, 2, 2, 1), strides=(1, 2, 2, 1), name='mp2') for i in xrange(N): name = 'r3_' + str(i) res3 = hn_lite.residual_block(res3, fg * fg * nf, fs, 2, train_phase, name=name, device=device) res4 = hn_lite.mean_pool(res3, ksize=(1, 2, 2, 1), strides=(1, 2, 2, 1), name='mp3') with tf.name_scope('gap') as scope: gap = tf.reduce_mean(hn_lite.sum_magnitudes(res4), reduction_indices=[1, 2, 3, 4]) return tf.nn.bias_add(tf.matmul(gap, Wgap), bgap)
def deep_mnist(opt, x, train_phase, device='/cpu:0'): """High frequency convolutions are unstable, so get rid of them""" # Sure layers weight & bias order = 1 # Number of Filters nf = opt['n_filters'] nf2 = int(nf * opt['filter_gain']) nf3 = int(nf * (opt['filter_gain']**2.)) bs = opt['batch_size'] fs = opt['filter_size'] nch = opt['n_channels'] ncl = opt['n_classes'] sm = opt['std_mult'] # Create bias for final layer with tf.device(device): bias = tf.get_variable('b7', shape=[opt['n_classes']], initializer=tf.constant_initializer(1e-2)) x = tf.reshape(x, shape=[bs, opt['dim'], opt['dim'], 1, 1, nch]) with arg_scope([hn_lite.conv2d, hn_lite.non_linearity, hn_lite.batch_norm], device=device): # Convolutional Layers with pooling with tf.name_scope('block1') as scope: cv1 = hn_lite.conv2d(x, nf, fs, padding='SAME', name='1') cv1 = hn_lite.non_linearity(cv1, tf.nn.relu, name='1') cv2 = hn_lite.conv2d(cv1, nf, fs, padding='SAME', name='2') cv2 = hn_lite.batch_norm(cv2, train_phase, name='bn1') with tf.name_scope('block2') as scope: cv2 = hn_lite.mean_pool(cv2, ksize=(1, 2, 2, 1), strides=(1, 2, 2, 1)) cv3 = hn_lite.conv2d(cv2, nf2, fs, padding='SAME', name='3') cv3 = hn_lite.non_linearity(cv3, tf.nn.relu, name='3') cv4 = hn_lite.conv2d(cv3, nf2, fs, padding='SAME', name='4') cv4 = hn_lite.batch_norm(cv4, train_phase, name='bn2') with tf.name_scope('block3') as scope: cv4 = hn_lite.mean_pool(cv4, ksize=(1, 2, 2, 1), strides=(1, 2, 2, 1)) cv5 = hn_lite.conv2d(cv4, nf3, fs, padding='SAME', name='5') cv5 = hn_lite.non_linearity(cv5, tf.nn.relu, name='5') cv6 = hn_lite.conv2d(cv5, nf3, fs, padding='SAME', name='6') cv6 = hn_lite.batch_norm(cv6, train_phase, name='bn3') # Final Layer with tf.name_scope('block4') as scope: cv7 = hn_lite.conv2d(cv6, ncl, fs, padding='SAME', phase=False, name='7') real = hn_lite.sum_magnitudes(cv7) cv7 = tf.reduce_mean(real, reduction_indices=[1, 2, 3, 4]) return tf.nn.bias_add(cv7, bias)
def deep_mnist(args, x, train_phase): """The MNIST-rot model similar to the one in Cohen & Welling, 2016""" # Sure layers weight & bias order = 1 # Number of Filters nf = args.n_filters nf2 = int(nf * args.filter_gain) nf3 = int(nf * (args.filter_gain**2.)) bs = args.batch_size fs = args.filter_size ncl = args.n_classes sm = args.std_mult nr = args.n_rings # Create bias for final layer bias = tf.get_variable('b7', shape=[args.n_classes], initializer=tf.constant_initializer(1e-2)) x = tf.reshape(x, shape=[bs, args.dim, args.dim, 1, 1, 1]) # Convolutional Layers with pooling with tf.name_scope('block1') as scope: cv1 = hn_lite.conv2d(x, nf, fs, padding='SAME', n_rings=nr, name='1') cv1 = hn_lite.non_linearity(cv1, tf.nn.relu, name='1') cv2 = hn_lite.conv2d(cv1, nf, fs, padding='SAME', n_rings=nr, name='2') cv2 = hn_lite.batch_norm(cv2, train_phase, name='bn1') with tf.name_scope('block2') as scope: cv2 = hn_lite.mean_pool(cv2, ksize=(1, 2, 2, 1), strides=(1, 2, 2, 1)) cv3 = hn_lite.conv2d(cv2, nf2, fs, padding='SAME', n_rings=nr, name='3') cv3 = hn_lite.non_linearity(cv3, tf.nn.relu, name='3') cv4 = hn_lite.conv2d(cv3, nf2, fs, padding='SAME', n_rings=nr, name='4') cv4 = hn_lite.batch_norm(cv4, train_phase, name='bn2') with tf.name_scope('block3') as scope: cv4 = hn_lite.mean_pool(cv4, ksize=(1, 2, 2, 1), strides=(1, 2, 2, 1)) cv5 = hn_lite.conv2d(cv4, nf3, fs, padding='SAME', n_rings=nr, name='5') cv5 = hn_lite.non_linearity(cv5, tf.nn.relu, name='5') cv6 = hn_lite.conv2d(cv5, nf3, fs, padding='SAME', n_rings=nr, name='6') cv6 = hn_lite.batch_norm(cv6, train_phase, name='bn3') # Final Layer with tf.name_scope('block4') as scope: cv7 = hn_lite.conv2d(cv6, ncl, fs, padding='SAME', n_rings=nr, phase=False, name='7') real = hn_lite.sum_magnitudes(cv7) cv7 = tf.reduce_mean(real, axis=[1, 2, 3, 4]) return tf.nn.bias_add(cv7, bias)
def hnet_bsd(args, x, train_phase): """High frequency convolutions are unstable, so get rid of them""" # Sure layers weight & bias order = 1 nf = int(args.n_filters) nf2 = int((args.filter_gain)*nf) nf3 = int((args.filter_gain**2)*nf) nf4 = int((args.filter_gain**3)*nf) bs = args.batch_size fs = args.filter_size nch = args.n_channels nr = args.n_rings tp = train_phase std = args.std_mult x = tf.reshape(x, shape=[bs,args.height,args.width,1,1,3]) fm = {} # Convolutional Layers with tf.name_scope('stage1') as scope: cv1 = hl.conv2d(x, nf, fs, stddev=std, padding='SAME', n_rings=nr, name='1_1') cv1 = hl.non_linearity(cv1, name='1_1') cv2 = hl.conv2d(cv1, nf, fs, stddev=std, padding='SAME', n_rings=nr, name='1_2') cv2 = hl.batch_norm(cv2, tp, name='bn1') mags = to_4d(hl.stack_magnitudes(cv2)) fm[1] = linear(mags, 1, 1, name='sw1') with tf.name_scope('stage2') as scope: cv3 = hl.mean_pooling(cv2, ksize=(1,2,2,1), strides=(1,2,2,1)) cv3 = hl.conv2d(cv3, nf2, fs, stddev=std, padding='SAME', n_rings=nr, name='2_1') cv3 = hl.non_linearity(cv3, name='2_1') cv4 = hl.conv2d(cv3, nf2, fs, stddev=std, padding='SAME', n_rings=nr, name='2_2') cv4 = hl.batch_norm(cv4, train_phase, name='bn2') mags = to_4d(hl.stack_magnitudes(cv4)) fm[2] = linear(mags, 1, 1, name='sw2') with tf.name_scope('stage3') as scope: cv5 = hl.mean_pooling(cv4, ksize=(1,2,2,1), strides=(1,2,2,1)) cv5 = hl.conv2d(cv5, nf3, fs, stddev=std, padding='SAME', n_rings=nr, name='3_1') cv5 = hl.non_linearity(cv5, name='3_1') cv6 = hl.conv2d(cv5, nf3, fs, stddev=std, padding='SAME', n_rings=nr, name='3_2') cv6 = hl.batch_norm(cv6, train_phase, name='bn3') mags = to_4d(hl.stack_magnitudes(cv6)) fm[3] = linear(mags, 1, 1, name='sw3') with tf.name_scope('stage4') as scope: cv7 = hl.mean_pooling(cv6, ksize=(1,2,2,1), strides=(1,2,2,1)) cv7 = hl.conv2d(cv7, nf4, fs, stddev=std, padding='SAME', n_rings=nr, name='4_1') cv7 = hl.non_linearity(cv7, name='4_1') cv8 = hl.conv2d(cv7, nf4, fs, stddev=std, padding='SAME', n_rings=nr, name='4_2') cv8 = hl.batch_norm(cv8, train_phase, name='bn4') mags = to_4d(hl.stack_magnitudes(cv8)) fm[4] = linear(mags, 1, 1, name='sw4') with tf.name_scope('stage5') as scope: cv9 = hl.mean_pooling(cv8, ksize=(1,2,2,1), strides=(1,2,2,1)) cv9 = hl.conv2d(cv9, nf4, fs, stddev=std, padding='SAME', n_rings=nr, name='5_1') cv9 = hl.non_linearity(cv9, name='5_1') cv10 = hl.conv2d(cv9, nf4, fs, stddev=std, padding='SAME', n_rings=nr, name='5_2') cv10 = hl.batch_norm(cv10, train_phase, name='bn5') mags = to_4d(hl.stack_magnitudes(cv10)) fm[5] = linear(mags, 1, 1, name='sw5') fms = {} side_preds = [] xsh = tf.shape(x) with tf.name_scope('fusion') as scope: for key in fm.keys(): fms[key] = tf.image.resize_images(fm[key], tf.stack([xsh[1], xsh[2]])) side_preds.append(fms[key]) side_preds = tf.concat(axis=3, values=side_preds) fms['fuse'] = linear(side_preds, 1, 1, bias_init=0.01, name='side_preds') return fms
KERNEL_WIDTH = 4 #assert KERNEL_WIDTH % 2 == 1 MAX_ORDER = 1 # Unit test -- Convolution tf.reset_default_graph() input_images = tf.placeholder(tf.float32, shape=INPUT_SHAPE) reshaped_input = tf.reshape(input_images, [BATCH_SIZE, WIDTH, WIDTH, 1, 1, NCHANNELS]) hconv = reshaped_input for layer_index in range(LAYER_COUNT): #if layer_index != 0: # hconv = hn_lite.mean_pool(hconv, ksize=(1,2,2,1), strides=(1,2,2,1)) hconv = hn_lite.conv2d(hconv, 1, KERNEL_WIDTH, padding='SAME', strides=(1,STRIDE,STRIDE,1), max_order=MAX_ORDER, name='hc'+str(layer_index)) real = hn_lite.sum_magnitudes(hconv) output_images = tf.nn.tanh(tf.reduce_mean(real, reduction_indices=[3,4])) with tf.Session() as sess: global_init_op = tf.global_variables_initializer() local_init_op = tf.local_variables_initializer() sess.run([global_init_op, local_init_op]) for i in xrange(360): input_image = rotate(EMBEDDED, i, axes=(1,0), reshape=False, order=3)[np.newaxis,:,:,np.newaxis] real_output = sess.run(output_images, feed_dict={input_images: input_image})[0,:,:,0] real_output = rotate(real_output, -i, axes=(1,0), reshape=False, order=3) / 1.2 print np.amin(real_output), np.amax(real_output) imsave('./nate_experiments/activations/nate_{:d}_/{:04d}.jpg'.format(LAYER_COUNT,i), real_output)