def deep_mnist(args, x, train_phase):
"""The MNIST-rot model similar to the one in Cohen & Welling, 2016"""
# Sure layers weight & bias
order = 1
# Number of Filters
nf = args.n_filters
nf2 = int(nf*args.filter_gain)
nf3 = int(nf*(args.filter_gain**2.))
bs = args.batch_size
fs = args.filter_size
ncl = args.n_classes
sm = args.std_mult
nr = args.n_rings
# Create bias for final layer
bias = tf.get_variable('b7', shape=[args.n_classes],
initializer=tf.constant_initializer(1e-2))
x = tf.reshape(x, shape=[bs,args.dim,args.dim,1,1,1])
# Convolutional Layers with pooling
with tf.name_scope('block1') as scope:
cv1 = hn_lite.conv2d(x, nf, fs, padding='SAME', n_rings=nr, name='1')
cv1 = hn_lite.non_linearity(cv1, tf.nn.relu, name='1')
cv2 = hn_lite.conv2d(cv1, nf, fs, padding='SAME', n_rings=nr, name='2')
cv2 = hn_lite.batch_norm(cv2, train_phase, name='bn1')
with tf.name_scope('block2') as scope:
cv2 = hn_lite.mean_pool(cv2, ksize=(1,2,2,1), strides=(1,2,2,1))
cv3 = hn_lite.conv2d(cv2, nf2, fs, padding='SAME', n_rings=nr, name='3')
cv3 = hn_lite.non_linearity(cv3, tf.nn.relu, name='3')
cv4 = hn_lite.conv2d(cv3, nf2, fs, padding='SAME', n_rings=nr, name='4')
cv4 = hn_lite.batch_norm(cv4, train_phase, name='bn2')
with tf.name_scope('block3') as scope:
cv4 = hn_lite.mean_pool(cv4, ksize=(1,2,2,1), strides=(1,2,2,1))
cv5 = hn_lite.conv2d(cv4, nf3, fs, padding='SAME', n_rings=nr, name='5')
cv5 = hn_lite.non_linearity(cv5, tf.nn.relu, name='5')
cv6 = hn_lite.conv2d(cv5, nf3, fs, padding='SAME', n_rings=nr, name='6')
cv6 = hn_lite.batch_norm(cv6, train_phase, name='bn3')
# Final Layer
with tf.name_scope('block4') as scope:
cv7 = hn_lite.conv2d(cv6, ncl, fs, padding='SAME', n_rings=nr, phase=False,
name='7')
real = hn_lite.sum_magnitudes(cv7)
cv7 = tf.reduce_mean(real, axis=[1,2,3,4])
return tf.nn.bias_add(cv7, bias)
def deep_mnist(opt, x, train_phase, device='/cpu:0'):
"""High frequency convolutions are unstable, so get rid of them"""
# Sure layers weight & bias
order = 1
# Number of Filters
nf = opt['n_filters']
nf2 = int(nf * opt['filter_gain'])
nf3 = int(nf * (opt['filter_gain']**2.))
bs = opt['batch_size']
fs = opt['filter_size']
nch = opt['n_channels']
ncl = opt['n_classes']
sm = opt['std_mult']
# Create bias for final layer
with tf.device(device):
bias = tf.get_variable('b7',
shape=[opt['n_classes']],
initializer=tf.constant_initializer(1e-2))
x = tf.reshape(x, shape=[bs, opt['dim'], opt['dim'], 1, 1, nch])
with arg_scope([hn_lite.conv2d, hn_lite.non_linearity, hn_lite.batch_norm],
device=device):
# Convolutional Layers with pooling
with tf.name_scope('block1') as scope:
cv1 = hn_lite.conv2d(x, nf, fs, padding='SAME', name='1')
cv1 = hn_lite.non_linearity(cv1, tf.nn.relu, name='1')
cv2 = hn_lite.conv2d(cv1, nf, fs, padding='SAME', name='2')
cv2 = hn_lite.batch_norm(cv2, train_phase, name='bn1')
with tf.name_scope('block2') as scope:
cv2 = hn_lite.mean_pool(cv2,
ksize=(1, 2, 2, 1),
strides=(1, 2, 2, 1))
cv3 = hn_lite.conv2d(cv2, nf2, fs, padding='SAME', name='3')
cv3 = hn_lite.non_linearity(cv3, tf.nn.relu, name='3')
cv4 = hn_lite.conv2d(cv3, nf2, fs, padding='SAME', name='4')
cv4 = hn_lite.batch_norm(cv4, train_phase, name='bn2')
with tf.name_scope('block3') as scope:
cv4 = hn_lite.mean_pool(cv4,
ksize=(1, 2, 2, 1),
strides=(1, 2, 2, 1))
cv5 = hn_lite.conv2d(cv4, nf3, fs, padding='SAME', name='5')
cv5 = hn_lite.non_linearity(cv5, tf.nn.relu, name='5')
cv6 = hn_lite.conv2d(cv5, nf3, fs, padding='SAME', name='6')
cv6 = hn_lite.batch_norm(cv6, train_phase, name='bn3')
# Final Layer
with tf.name_scope('block4') as scope:
cv7 = hn_lite.conv2d(cv6,
ncl,
fs,
padding='SAME',
phase=False,
name='7')
real = hn_lite.sum_magnitudes(cv7)
cv7 = tf.reduce_mean(real, reduction_indices=[1, 2, 3, 4])
return tf.nn.bias_add(cv7, bias)
def deep_mnist(args, x, train_phase):
"""The MNIST-rot model similar to the one in Cohen & Welling, 2016"""
# Sure layers weight & bias
order = 1
# Number of Filters
nf = args.n_filters
nf2 = int(nf * args.filter_gain)
nf3 = int(nf * (args.filter_gain**2.))
bs = args.batch_size
fs = args.filter_size
ncl = args.n_classes
sm = args.std_mult
nr = args.n_rings
# Create bias for final layer
bias = tf.get_variable('b7',
shape=[args.n_classes],
initializer=tf.constant_initializer(1e-2))
x = tf.reshape(x, shape=[bs, args.dim, args.dim, 1, 1, 1])
# Convolutional Layers with pooling
with tf.name_scope('block1') as scope:
cv1 = hn_lite.conv2d(x, nf, fs, padding='SAME', n_rings=nr, name='1')
cv1 = hn_lite.non_linearity(cv1, tf.nn.relu, name='1')
cv2 = hn_lite.conv2d(cv1, nf, fs, padding='SAME', n_rings=nr, name='2')
cv2 = hn_lite.batch_norm(cv2, train_phase, name='bn1')
with tf.name_scope('block2') as scope:
cv2 = hn_lite.mean_pool(cv2, ksize=(1, 2, 2, 1), strides=(1, 2, 2, 1))
cv3 = hn_lite.conv2d(cv2,
nf2,
fs,
padding='SAME',
n_rings=nr,
name='3')
cv3 = hn_lite.non_linearity(cv3, tf.nn.relu, name='3')
cv4 = hn_lite.conv2d(cv3,
nf2,
fs,
padding='SAME',
n_rings=nr,
name='4')
cv4 = hn_lite.batch_norm(cv4, train_phase, name='bn2')
with tf.name_scope('block3') as scope:
cv4 = hn_lite.mean_pool(cv4, ksize=(1, 2, 2, 1), strides=(1, 2, 2, 1))
cv5 = hn_lite.conv2d(cv4,
nf3,
fs,
padding='SAME',
n_rings=nr,
name='5')
cv5 = hn_lite.non_linearity(cv5, tf.nn.relu, name='5')
cv6 = hn_lite.conv2d(cv5,
nf3,
fs,
padding='SAME',
n_rings=nr,
name='6')
cv6 = hn_lite.batch_norm(cv6, train_phase, name='bn3')
# Final Layer
with tf.name_scope('block4') as scope:
cv7 = hn_lite.conv2d(cv6,
ncl,
fs,
padding='SAME',
n_rings=nr,
phase=False,
name='7')
real = hn_lite.sum_magnitudes(cv7)
cv7 = tf.reduce_mean(real, axis=[1, 2, 3, 4])
return tf.nn.bias_add(cv7, bias)
def hnet_bsd(args, x, train_phase):
"""High frequency convolutions are unstable, so get rid of them"""
# Sure layers weight & bias
order = 1
nf = int(args.n_filters)
nf2 = int((args.filter_gain)*nf)
nf3 = int((args.filter_gain**2)*nf)
nf4 = int((args.filter_gain**3)*nf)
bs = args.batch_size
fs = args.filter_size
nch = args.n_channels
nr = args.n_rings
tp = train_phase
std = args.std_mult
x = tf.reshape(x, shape=[bs,args.height,args.width,1,1,3])
fm = {}
# Convolutional Layers
with tf.name_scope('stage1') as scope:
cv1 = hl.conv2d(x, nf, fs, stddev=std, padding='SAME', n_rings=nr, name='1_1')
cv1 = hl.non_linearity(cv1, name='1_1')
cv2 = hl.conv2d(cv1, nf, fs, stddev=std, padding='SAME', n_rings=nr, name='1_2')
cv2 = hl.batch_norm(cv2, tp, name='bn1')
mags = to_4d(hl.stack_magnitudes(cv2))
fm[1] = linear(mags, 1, 1, name='sw1')
with tf.name_scope('stage2') as scope:
cv3 = hl.mean_pooling(cv2, ksize=(1,2,2,1), strides=(1,2,2,1))
cv3 = hl.conv2d(cv3, nf2, fs, stddev=std, padding='SAME', n_rings=nr, name='2_1')
cv3 = hl.non_linearity(cv3, name='2_1')
cv4 = hl.conv2d(cv3, nf2, fs, stddev=std, padding='SAME', n_rings=nr, name='2_2')
cv4 = hl.batch_norm(cv4, train_phase, name='bn2')
mags = to_4d(hl.stack_magnitudes(cv4))
fm[2] = linear(mags, 1, 1, name='sw2')
with tf.name_scope('stage3') as scope:
cv5 = hl.mean_pooling(cv4, ksize=(1,2,2,1), strides=(1,2,2,1))
cv5 = hl.conv2d(cv5, nf3, fs, stddev=std, padding='SAME', n_rings=nr, name='3_1')
cv5 = hl.non_linearity(cv5, name='3_1')
cv6 = hl.conv2d(cv5, nf3, fs, stddev=std, padding='SAME', n_rings=nr, name='3_2')
cv6 = hl.batch_norm(cv6, train_phase, name='bn3')
mags = to_4d(hl.stack_magnitudes(cv6))
fm[3] = linear(mags, 1, 1, name='sw3')
with tf.name_scope('stage4') as scope:
cv7 = hl.mean_pooling(cv6, ksize=(1,2,2,1), strides=(1,2,2,1))
cv7 = hl.conv2d(cv7, nf4, fs, stddev=std, padding='SAME', n_rings=nr, name='4_1')
cv7 = hl.non_linearity(cv7, name='4_1')
cv8 = hl.conv2d(cv7, nf4, fs, stddev=std, padding='SAME', n_rings=nr, name='4_2')
cv8 = hl.batch_norm(cv8, train_phase, name='bn4')
mags = to_4d(hl.stack_magnitudes(cv8))
fm[4] = linear(mags, 1, 1, name='sw4')
with tf.name_scope('stage5') as scope:
cv9 = hl.mean_pooling(cv8, ksize=(1,2,2,1), strides=(1,2,2,1))
cv9 = hl.conv2d(cv9, nf4, fs, stddev=std, padding='SAME', n_rings=nr, name='5_1')
cv9 = hl.non_linearity(cv9, name='5_1')
cv10 = hl.conv2d(cv9, nf4, fs, stddev=std, padding='SAME', n_rings=nr, name='5_2')
cv10 = hl.batch_norm(cv10, train_phase, name='bn5')
mags = to_4d(hl.stack_magnitudes(cv10))
fm[5] = linear(mags, 1, 1, name='sw5')
fms = {}
side_preds = []
xsh = tf.shape(x)
with tf.name_scope('fusion') as scope:
for key in fm.keys():
fms[key] = tf.image.resize_images(fm[key], tf.stack([xsh[1], xsh[2]]))
side_preds.append(fms[key])
side_preds = tf.concat(axis=3, values=side_preds)
fms['fuse'] = linear(side_preds, 1, 1, bias_init=0.01, name='side_preds')
return fms