def deep_mnist(args, x, train_phase): """The MNIST-rot model similar to the one in Cohen & Welling, 2016""" # Sure layers weight & bias order = 1 # Number of Filters nf = args.n_filters nf2 = int(nf*args.filter_gain) nf3 = int(nf*(args.filter_gain**2.)) bs = args.batch_size fs = args.filter_size ncl = args.n_classes sm = args.std_mult nr = args.n_rings # Create bias for final layer bias = tf.get_variable('b7', shape=[args.n_classes], initializer=tf.constant_initializer(1e-2)) x = tf.reshape(x, shape=[bs,args.dim,args.dim,1,1,1]) # Convolutional Layers with pooling with tf.name_scope('block1') as scope: cv1 = hn_lite.conv2d(x, nf, fs, padding='SAME', n_rings=nr, name='1') cv1 = hn_lite.non_linearity(cv1, tf.nn.relu, name='1') cv2 = hn_lite.conv2d(cv1, nf, fs, padding='SAME', n_rings=nr, name='2') cv2 = hn_lite.batch_norm(cv2, train_phase, name='bn1') with tf.name_scope('block2') as scope: cv2 = hn_lite.mean_pool(cv2, ksize=(1,2,2,1), strides=(1,2,2,1)) cv3 = hn_lite.conv2d(cv2, nf2, fs, padding='SAME', n_rings=nr, name='3') cv3 = hn_lite.non_linearity(cv3, tf.nn.relu, name='3') cv4 = hn_lite.conv2d(cv3, nf2, fs, padding='SAME', n_rings=nr, name='4') cv4 = hn_lite.batch_norm(cv4, train_phase, name='bn2') with tf.name_scope('block3') as scope: cv4 = hn_lite.mean_pool(cv4, ksize=(1,2,2,1), strides=(1,2,2,1)) cv5 = hn_lite.conv2d(cv4, nf3, fs, padding='SAME', n_rings=nr, name='5') cv5 = hn_lite.non_linearity(cv5, tf.nn.relu, name='5') cv6 = hn_lite.conv2d(cv5, nf3, fs, padding='SAME', n_rings=nr, name='6') cv6 = hn_lite.batch_norm(cv6, train_phase, name='bn3') # Final Layer with tf.name_scope('block4') as scope: cv7 = hn_lite.conv2d(cv6, ncl, fs, padding='SAME', n_rings=nr, phase=False, name='7') real = hn_lite.sum_magnitudes(cv7) cv7 = tf.reduce_mean(real, axis=[1,2,3,4]) return tf.nn.bias_add(cv7, bias)
def deep_cifar(opt, x, train_phase, device='/cpu:0'): """High frequency convolutions are unstable, so get rid of them""" # Abbreviations nf = opt['n_filters'] fg = opt['filter_gain'] bs = opt['batch_size'] fs = opt['filter_size'] N = opt['resnet_block_multiplicity'] with tf.device(device): initializer = tf.contrib.layers.variance_scaling_initializer() Wgap = tf.get_variable('Wfc', shape=[fg * fg * nf, opt['n_classes']], initializer=initializer) bgap = tf.get_variable('bfc', shape=[opt['n_classes']], initializer=tf.constant_initializer(1e-2)) x = tf.reshape( x, shape=[bs, opt['dim'], opt['dim'], 1, 1, opt['n_channels']]) # Convolutional Layers res1 = hn_lite.conv2d(x, nf, fs, padding='SAME', name='in', device=device) for i in xrange(N): name = 'r1_' + str(i) res1 = hn_lite.residual_block(res1, nf, fs, 2, train_phase, name=name, device=device) res2 = hn_lite.mean_pool(res1, ksize=(1, 2, 2, 1), strides=(1, 2, 2, 1), name='mp1') for i in xrange(N): name = 'r2_' + str(i) res2 = hn_lite.residual_block(res2, fg * nf, fs, 2, train_phase, name=name, device=device) res3 = hn_lite.mean_pool(res2, ksize=(1, 2, 2, 1), strides=(1, 2, 2, 1), name='mp2') for i in xrange(N): name = 'r3_' + str(i) res3 = hn_lite.residual_block(res3, fg * fg * nf, fs, 2, train_phase, name=name, device=device) res4 = hn_lite.mean_pool(res3, ksize=(1, 2, 2, 1), strides=(1, 2, 2, 1), name='mp3') with tf.name_scope('gap') as scope: gap = tf.reduce_mean(hn_lite.sum_magnitudes(res4), reduction_indices=[1, 2, 3, 4]) return tf.nn.bias_add(tf.matmul(gap, Wgap), bgap)
def deep_mnist(opt, x, train_phase, device='/cpu:0'): """High frequency convolutions are unstable, so get rid of them""" # Sure layers weight & bias order = 1 # Number of Filters nf = opt['n_filters'] nf2 = int(nf * opt['filter_gain']) nf3 = int(nf * (opt['filter_gain']**2.)) bs = opt['batch_size'] fs = opt['filter_size'] nch = opt['n_channels'] ncl = opt['n_classes'] sm = opt['std_mult'] # Create bias for final layer with tf.device(device): bias = tf.get_variable('b7', shape=[opt['n_classes']], initializer=tf.constant_initializer(1e-2)) x = tf.reshape(x, shape=[bs, opt['dim'], opt['dim'], 1, 1, nch]) with arg_scope([hn_lite.conv2d, hn_lite.non_linearity, hn_lite.batch_norm], device=device): # Convolutional Layers with pooling with tf.name_scope('block1') as scope: cv1 = hn_lite.conv2d(x, nf, fs, padding='SAME', name='1') cv1 = hn_lite.non_linearity(cv1, tf.nn.relu, name='1') cv2 = hn_lite.conv2d(cv1, nf, fs, padding='SAME', name='2') cv2 = hn_lite.batch_norm(cv2, train_phase, name='bn1') with tf.name_scope('block2') as scope: cv2 = hn_lite.mean_pool(cv2, ksize=(1, 2, 2, 1), strides=(1, 2, 2, 1)) cv3 = hn_lite.conv2d(cv2, nf2, fs, padding='SAME', name='3') cv3 = hn_lite.non_linearity(cv3, tf.nn.relu, name='3') cv4 = hn_lite.conv2d(cv3, nf2, fs, padding='SAME', name='4') cv4 = hn_lite.batch_norm(cv4, train_phase, name='bn2') with tf.name_scope('block3') as scope: cv4 = hn_lite.mean_pool(cv4, ksize=(1, 2, 2, 1), strides=(1, 2, 2, 1)) cv5 = hn_lite.conv2d(cv4, nf3, fs, padding='SAME', name='5') cv5 = hn_lite.non_linearity(cv5, tf.nn.relu, name='5') cv6 = hn_lite.conv2d(cv5, nf3, fs, padding='SAME', name='6') cv6 = hn_lite.batch_norm(cv6, train_phase, name='bn3') # Final Layer with tf.name_scope('block4') as scope: cv7 = hn_lite.conv2d(cv6, ncl, fs, padding='SAME', phase=False, name='7') real = hn_lite.sum_magnitudes(cv7) cv7 = tf.reduce_mean(real, reduction_indices=[1, 2, 3, 4]) return tf.nn.bias_add(cv7, bias)
def deep_mnist(args, x, train_phase): """The MNIST-rot model similar to the one in Cohen & Welling, 2016""" # Sure layers weight & bias order = 1 # Number of Filters nf = args.n_filters nf2 = int(nf * args.filter_gain) nf3 = int(nf * (args.filter_gain**2.)) bs = args.batch_size fs = args.filter_size ncl = args.n_classes sm = args.std_mult nr = args.n_rings # Create bias for final layer bias = tf.get_variable('b7', shape=[args.n_classes], initializer=tf.constant_initializer(1e-2)) x = tf.reshape(x, shape=[bs, args.dim, args.dim, 1, 1, 1]) # Convolutional Layers with pooling with tf.name_scope('block1') as scope: cv1 = hn_lite.conv2d(x, nf, fs, padding='SAME', n_rings=nr, name='1') cv1 = hn_lite.non_linearity(cv1, tf.nn.relu, name='1') cv2 = hn_lite.conv2d(cv1, nf, fs, padding='SAME', n_rings=nr, name='2') cv2 = hn_lite.batch_norm(cv2, train_phase, name='bn1') with tf.name_scope('block2') as scope: cv2 = hn_lite.mean_pool(cv2, ksize=(1, 2, 2, 1), strides=(1, 2, 2, 1)) cv3 = hn_lite.conv2d(cv2, nf2, fs, padding='SAME', n_rings=nr, name='3') cv3 = hn_lite.non_linearity(cv3, tf.nn.relu, name='3') cv4 = hn_lite.conv2d(cv3, nf2, fs, padding='SAME', n_rings=nr, name='4') cv4 = hn_lite.batch_norm(cv4, train_phase, name='bn2') with tf.name_scope('block3') as scope: cv4 = hn_lite.mean_pool(cv4, ksize=(1, 2, 2, 1), strides=(1, 2, 2, 1)) cv5 = hn_lite.conv2d(cv4, nf3, fs, padding='SAME', n_rings=nr, name='5') cv5 = hn_lite.non_linearity(cv5, tf.nn.relu, name='5') cv6 = hn_lite.conv2d(cv5, nf3, fs, padding='SAME', n_rings=nr, name='6') cv6 = hn_lite.batch_norm(cv6, train_phase, name='bn3') # Final Layer with tf.name_scope('block4') as scope: cv7 = hn_lite.conv2d(cv6, ncl, fs, padding='SAME', n_rings=nr, phase=False, name='7') real = hn_lite.sum_magnitudes(cv7) cv7 = tf.reduce_mean(real, axis=[1, 2, 3, 4]) return tf.nn.bias_add(cv7, bias)