def weight_variable(shape, initializer=None, init_val=None, wd=None, name=None, trainable=True):
"""Initialize weights.
Args:
shape: shape of the weights, list of int
wd: weight decay
"""
log = logger.get()
if initializer is None:
# initializer = tf.truncated_normal(shape, stddev=0.01)
initializer = tf.truncated_normal_initializer(stddev=0.01)
if init_val is None:
var = tf.Variable(initializer(shape), name=name, trainable=trainable)
else:
var = tf.Variable(init_val, name=name, trainable=trainable)
# log.info(var.name)
# if init_val is not None:
# if hasattr(init_val, 'shape'):
# log.info('Initialized with array shape {}'.format(init_val.shape))
# else:
# log.info('Initialized with {}'.format(init_val))
if wd:
weight_decay = tf.mul(tf.nn.l2_loss(var), wd, name='weight_loss')
tf.add_to_collection('losses', weight_decay)
return var
def __init__(self, sess=None, model=None):
super(Experiment, self).__init__()
self.log = logger.get()
self.log.log_args()
self._sess = sess
self._model = model
pass
def weight_variable(shape,
initializer=None,
init_val=None,
wd=None,
name=None,
trainable=True):
"""Initialize weights.
Args:
shape: shape of the weights, list of int
wd: weight decay
"""
log = logger.get()
if initializer is None:
# initializer = tf.truncated_normal(shape, stddev=0.01)
initializer = tf.truncated_normal_initializer(stddev=0.01)
if init_val is None:
var = tf.Variable(initializer(shape), name=name, trainable=trainable)
else:
var = tf.Variable(init_val, name=name, trainable=trainable)
# log.info(var.name)
# if init_val is not None:
# if hasattr(init_val, 'shape'):
# log.info('Initialized with array shape {}'.format(init_val.shape))
# else:
# log.info('Initialized with {}'.format(init_val))
if wd:
weight_decay = tf.mul(tf.nn.l2_loss(var), wd, name='weight_loss')
tf.add_to_collection('losses', weight_decay)
return var
def __init__(self, sess=None, model=None):
super(Experiment, self).__init__()
self.log = logger.get()
self.log.log_args()
self._sess = sess
self._model = model
pass
def __init__(self):
super(BasicRunner, self).__init__()
self._step = 0
self._data_provider = None
self._phase_train = True
self._outputs = []
self._current_batch = {}
self._log = logger.get()
self._preprocessor = lambda x: x
self._listeners = []
pass
def __init__(self):
super(BasicRunner, self).__init__()
self._step = 0
self._data_provider = None
self._phase_train = True
self._outputs = []
self._current_batch = {}
self._log = logger.get()
self._preprocessor = lambda x: x
self._listeners = []
pass
def maybe_download(filename, work_directory):
"""Download the data from Yann's website, unless it's already here."""
if not os.path.exists(work_directory):
os.mkdir(work_directory)
filepath = os.path.join(work_directory, filename)
if not os.path.exists(filepath):
filepath, _ = urllib.urlretrieve(SOURCE_URL + filename, filepath)
statinfo = os.stat(filepath)
log = logger.get()
log.info('Succesfully downloaded {} {} bytes.'.format(
filename, statinfo.st_size))
return filepath
def maybe_download(filename, work_directory):
"""Download the data from Yann's website, unless it's already here."""
if not os.path.exists(work_directory):
os.mkdir(work_directory)
filepath = os.path.join(work_directory, filename)
if not os.path.exists(filepath):
filepath, _ = urllib.urlretrieve(SOURCE_URL + filename, filepath)
statinfo = os.stat(filepath)
log = logger.get()
log.info('Succesfully downloaded {} {} bytes.'.format(
filename, statinfo.st_size))
return filepath
def __init__(self, top_k=1, filename=None, label=None):
self.top_k = top_k
self.correct = 0
self.count = 0
self.log = logger.get()
self.filename = filename
self.label = label
self.registered = False
if not os.path.exists(filename):
with open(filename, 'w') as f:
f.write('step,time,{}\n'.format(label))
pass
def __init__(self, top_k=1, filename=None, label=None):
self.top_k = top_k
self.correct = 0
self.count = 0
self.log = logger.get()
self.filename = filename
self.label = label
self.registered = False
if not os.path.exists(filename):
with open(filename, 'w') as f:
f.write('step,time,{}\n'.format(label))
pass
def __init__(self, split='train', filename=None):
super(MNISTDataProvider, self).__init__()
self.log = logger.get()
if split is None:
self.split = 'train'
else:
self.split = split
self.log.info('Data split: {}'.format(self.split))
self.filename = filename
self._images = None
self._labels = None
self.register_option('mnist:dataset_folder')
pass
def __init__(self, split='train', filename=None):
super(MNISTDataProvider, self).__init__()
self.log = logger.get()
if split is None:
self.split = 'train'
else:
self.split = split
self.log.info('Data split: {}'.format(self.split))
self.filename = filename
self._images = None
self._labels = None
self.register_option('mnist:dataset_folder')
pass
def extract_labels(filename, one_hot=False):
"""Extract the labels into a 1D uint8 numpy array [index]."""
log = logger.get()
log.info('Extracting {}'.format(filename))
with gzip.open(filename) as bytestream:
magic = _read32(bytestream)
if magic != 2049:
raise ValueError(
'Invalid magic number %d in MNIST label file: %s' %
(magic, filename))
num_items = _read32(bytestream)
buf = bytestream.read(num_items)
labels = numpy.frombuffer(buf, dtype=numpy.uint8)
if one_hot:
return dense_to_one_hot(labels)
return labels
def extract_labels(filename, one_hot=False):
"""Extract the labels into a 1D uint8 numpy array [index]."""
log = logger.get()
log.info('Extracting {}'.format(filename))
with gzip.open(filename) as bytestream:
magic = _read32(bytestream)
if magic != 2049:
raise ValueError(
'Invalid magic number %d in MNIST label file: %s' %
(magic, filename))
num_items = _read32(bytestream)
buf = bytestream.read(num_items)
labels = numpy.frombuffer(buf, dtype=numpy.uint8)
if one_hot:
return dense_to_one_hot(labels)
return labels
def extract_images(filename):
"""Extract the images into a 4D uint8 numpy array [index, y, x, depth]."""
log = logger.get()
log.info('Extracting {}'.format(filename))
with gzip.open(filename) as bytestream:
magic = _read32(bytestream)
if magic != 2051:
raise ValueError(
'Invalid magic number %d in MNIST image file: %s' %
(magic, filename))
num_images = _read32(bytestream)
rows = _read32(bytestream)
cols = _read32(bytestream)
buf = bytestream.read(rows * cols * num_images)
data = numpy.frombuffer(buf, dtype=numpy.uint8)
data = data.reshape(num_images, rows, cols, 1)
return data
def extract_images(filename):
"""Extract the images into a 4D uint8 numpy array [index, y, x, depth]."""
log = logger.get()
log.info('Extracting {}'.format(filename))
with gzip.open(filename) as bytestream:
magic = _read32(bytestream)
if magic != 2051:
raise ValueError(
'Invalid magic number %d in MNIST image file: %s' %
(magic, filename))
num_images = _read32(bytestream)
rows = _read32(bytestream)
cols = _read32(bytestream)
buf = bytestream.read(rows * cols * num_images)
data = numpy.frombuffer(buf, dtype=numpy.uint8)
data = data.reshape(num_images, rows, cols, 1)
return data
def gru(inp_dim, hid_dim, wd=None, scope='gru'):
"""Adds a GRU component.
Args:
inp_dim: Input data dim
hid_dim: Hidden state dim
wd: Weight decay
scope: Prefix
"""
log = logger.get()
log.info('GRU: {}'.format(scope))
log.info('Input dim: {}'.format(inp_dim))
log.info('Hidden dim: {}'.format(hid_dim))
with tf.variable_scope(scope):
w_xi = weight_variable([inp_dim, hid_dim], wd=wd, name='w_xi')
w_hi = weight_variable([hid_dim, hid_dim], wd=wd, name='w_hi')
b_i = weight_variable([hid_dim], name='b_i')
w_xu = weight_variable([inp_dim, hid_dim], wd=wd, name='w_xu')
w_hu = weight_variable([hid_dim, hid_dim], wd=wd, name='w_hu')
b_u = weight_variable([hid_dim], name='b_u')
w_xr = weight_variable([inp_dim, hid_dim], wd=wd, name='w_xr')
w_hr = weight_variable([hid_dim, hid_dim], wd=wd, name='w_hr')
b_r = weight_variable([hid_dim], name='b_r')
def unroll(inp, state):
g_i = tf.sigmoid(tf.matmul(inp, w_xi) + tf.matmul(state, w_hi) + b_i)
g_r = tf.sigmoid(tf.matmul(inp, w_xr) + tf.matmul(state, w_hr) + b_r)
u = tf.tanh(tf.matmul(inp, w_xu) + g_r * tf.matmul(state, w_hu) + b_u)
state = state * (1 - g_i) + u * g_i
return state
return unroll
def gru(inp_dim, hid_dim, wd=None, scope='gru'):
"""Adds a GRU component.
Args:
inp_dim: Input data dim
hid_dim: Hidden state dim
wd: Weight decay
scope: Prefix
"""
log = logger.get()
log.info('GRU: {}'.format(scope))
log.info('Input dim: {}'.format(inp_dim))
log.info('Hidden dim: {}'.format(hid_dim))
with tf.variable_scope(scope):
w_xi = weight_variable([inp_dim, hid_dim], wd=wd, name='w_xi')
w_hi = weight_variable([hid_dim, hid_dim], wd=wd, name='w_hi')
b_i = weight_variable([hid_dim], name='b_i')
w_xu = weight_variable([inp_dim, hid_dim], wd=wd, name='w_xu')
w_hu = weight_variable([hid_dim, hid_dim], wd=wd, name='w_hu')
b_u = weight_variable([hid_dim], name='b_u')
w_xr = weight_variable([inp_dim, hid_dim], wd=wd, name='w_xr')
w_hr = weight_variable([hid_dim, hid_dim], wd=wd, name='w_hr')
b_r = weight_variable([hid_dim], name='b_r')
def unroll(inp, state):
g_i = tf.sigmoid(tf.matmul(inp, w_xi) + tf.matmul(state, w_hi) + b_i)
g_r = tf.sigmoid(tf.matmul(inp, w_xr) + tf.matmul(state, w_hr) + b_r)
u = tf.tanh(tf.matmul(inp, w_xu) + g_r * tf.matmul(state, w_hu) + b_u)
state = state * (1 - g_i) + u * g_i
return state
return unroll
def mlp(dims, act, add_bias=True, dropout_keep=None, phase_train=None, wd=None, scope='mlp', model=None, init_weights=None, frozen=None):
"""Add MLP. N = number of layers.
Args:
dims: layer-wise dimensions, list of N int
act: activation function, list of N function
dropout_keep: keep prob of dropout, list of N float
phase_train: whether in training phase, tf bool variable
wd: weight decay
"""
log = logger.get()
nlayers = len(dims) - 1
w = [None] * nlayers
b = [None] * nlayers
log.info('MLP: {}'.format(scope))
log.info('Dimensions: {}'.format(dims))
log.info('Activation: {}'.format(act))
log.info('Dropout: {}'.format(dropout_keep))
log.info('Add bias: {}'.format(add_bias))
with tf.variable_scope(scope):
for ii in xrange(nlayers):
with tf.variable_scope('layer_{}'.format(ii)):
nin = dims[ii]
nout = dims[ii + 1]
if init_weights is not None and init_weights[ii] is not None:
init_val_w = init_weights[ii]['w']
init_val_b = init_weights[ii]['b']
else:
init_val_w = None
init_val_b = None
if frozen is not None and frozen[ii]:
trainable = False
else:
trainable = True
w[ii] = weight_variable([nin, nout], init_val=init_val_w, wd=wd,
name='w',
trainable=trainable)
log.info('Weights: {} Trainable: {}'.format(
[nin, nout], trainable))
if add_bias:
b[ii] = weight_variable([nout], init_val=init_val_b,
name='b',
trainable=trainable)
log.info('Bias: {} Trainable: {}'.format(
[nout], trainable))
if model is not None:
model['{}_w_{}'.format(scope, ii)] = w[ii]
if add_bias:
model['{}_b_{}'.format(scope, ii)] = b[ii]
def run_mlp(x):
h = [None] * nlayers
with tf.variable_scope(scope):
for ii in xrange(nlayers):
with tf.variable_scope('layer_{}'.format(ii)):
if ii == 0:
prev_inp = x
else:
prev_inp = h[ii - 1]
if dropout_keep is not None:
if dropout_keep[ii] is not None:
prev_inp = dropout(
prev_inp, dropout_keep[ii], phase_train)
h[ii] = tf.matmul(prev_inp, w[ii])
if add_bias:
h[ii] += b[ii]
if act[ii]:
h[ii] = act[ii](h[ii])
return h
return run_mlp
def dcnn(f, ch, pool, act, use_bn, skip_ch=None, phase_train=None, wd=None, scope='dcnn', model=None, init_weights=None, frozen=None):
"""Add DCNN. N = number of layers.
Args:
f: filter size, list of size N int
ch: number of channels, list of (N + 1) int
pool: pooling ratio, list of N int
act: activation function, list of N function
use_bn: whether to use batch normalization, list of N bool
skip_ch: skip connection, list of N int
phase_train: whether in training phase, tf bool variable
wd: weight decay
Returns:
run_dcnn: a function that runs the DCNN
"""
log = logger.get()
nlayers = len(f)
w = [None] * nlayers
b = [None] * nlayers
bn = [None] * nlayers
log.info('DCNN: {}'.format(scope))
log.info('Channels: {}'.format(ch))
log.info('Activation: {}'.format(act))
log.info('Unpool: {}'.format(pool))
log.info('Skip channels: {}'.format(skip_ch))
log.info('BN: {}'.format(use_bn))
with tf.variable_scope(scope):
in_ch = ch[0]
for ii in xrange(nlayers):
with tf.variable_scope('layer_{}'.format(ii)):
out_ch = ch[ii + 1]
if skip_ch is not None:
if skip_ch[ii] is not None:
in_ch += skip_ch[ii]
if init_weights is not None and init_weights[ii] is not None:
init_val_w = init_weights[ii]['w']
init_val_b = init_weights[ii]['b']
else:
init_val_w = None
init_val_b = None
if frozen is not None and frozen[ii]:
trainable = False
else:
trainable = True
w[ii] = weight_variable([f[ii], f[ii], out_ch, in_ch],
name='w',
init_val=init_val_w, wd=wd,
trainable=trainable)
b[ii] = weight_variable([out_ch], init_val=init_val_b,
name='b',
trainable=trainable)
log.info('Filter: {}, Trainable: {}'.format(
[f[ii], f[ii], out_ch, in_ch], trainable))
in_ch = out_ch
if model is not None:
model['{}_w_{}'.format(scope, ii)] = w[ii]
model['{}_b_{}'.format(scope, ii)] = b[ii]
copy = [0]
def run_dcnn(x, skip=None):
"""Run DCNN on an input.
Args:
x: input image, [B, H, W, D]
skip: skip connection activation map, list of 4-D tensor
"""
with tf.variable_scope(scope):
h = [None] * nlayers
out_shape = [None] * nlayers
batch = tf.shape(x)[0: 1]
inp_size = tf.shape(x)[1: 3]
cum_pool = 1
for ii in xrange(nlayers):
with tf.variable_scope('layer_{}'.format(ii)):
cum_pool *= pool[ii]
out_ch = ch[ii + 1]
if ii == 0:
prev_inp = x
else:
prev_inp = h[ii - 1]
if skip is not None:
if skip[ii] is not None:
if ii == 0:
prev_inp = tf.concat(3, [prev_inp, skip[ii]])
else:
prev_inp = tf.concat(3, [prev_inp, skip[ii]])
out_shape[ii] = tf.concat(
0, [batch, inp_size * cum_pool, tf.constant([out_ch])])
h[ii] = tf.nn.conv2d_transpose(
prev_inp, w[ii], out_shape[ii],
strides=[1, pool[ii], pool[ii], 1]) + b[ii]
if use_bn[ii]:
if frozen is not None and frozen[ii]:
bn_frozen = True
else:
bn_frozen = False
if init_weights is not None and \
init_weights[ii] is not None:
init_beta = init_weights[ii][
'beta_{}'.format(copy[0])]
init_gamma = init_weights[ii][
'gamma_{}'.format(copy[0])]
else:
init_beta = None
init_gamma = None
# with tf.variable_scope('layer_{}'.format(ii)):
# with tf.variable_scope('copy_{}'.format(copy[0])):
h[ii] = batch_norm(
h[ii], out_ch, phase_train,
scope2='{}_{}_{}'.format(scope, ii, copy[0]),
init_beta=init_beta,
init_gamma=init_gamma,
model=model)
if act[ii] is not None:
h[ii] = act[ii](h[ii])
copy[0] += 1
return h
return run_dcnn
def cnn(f, ch, pool, act, use_bn, phase_train=None, wd=None, scope='cnn', model=None, init_weights=None, frozen=None, shared_weights=None):
"""Add CNN. N = number of layers.
Args:
f: filter size, list of N int
ch: number of channels, list of (N + 1) int
pool: pooling ratio, list of N int
act: activation function, list of N function
use_bn: whether to use batch normalization, list of N bool
phase_train: whether in training phase, tf bool variable
wd: weight decay
Returns:
run_cnn: a function that runs the CNN
"""
log = logger.get()
nlayers = len(f)
w = [None] * nlayers
b = [None] * nlayers
log.info('CNN: {}'.format(scope))
log.info('Channels: {}'.format(ch))
log.info('Activation: {}'.format(act))
log.info('Pool: {}'.format(pool))
log.info('BN: {}'.format(use_bn))
log.info('Shared weights: {}'.format(shared_weights))
net_scope = None
layer_scope = [None] * nlayers
with tf.variable_scope(scope):
for ii in xrange(nlayers):
with tf.variable_scope('layer_{}'.format(ii)):
if init_weights:
init = tf.constant_initializer
else:
init = None
if init_weights is not None and init_weights[ii] is not None:
init_val_w = init_weights[ii]['w']
init_val_b = init_weights[ii]['b']
else:
init_val_w = None
init_val_b = None
if frozen is not None and frozen[ii]:
trainable = False
else:
trainable = True
if shared_weights:
w[ii] = shared_weights[ii]['w']
b[ii] = shared_weights[ii]['b']
else:
w[ii] = weight_variable([f[ii], f[ii], ch[ii], ch[ii + 1]],
name='w',
init_val=init_val_w, wd=wd,
trainable=trainable)
b[ii] = weight_variable([ch[ii + 1]], init_val=init_val_b,
name='b',
trainable=trainable)
log.info('Filter: {}, Trainable: {}'.format(
[f[ii], f[ii], ch[ii], ch[ii + 1]], trainable))
if model is not None:
for name, param in zip(['w', 'b'], [w[ii], b[ii]]):
key = '{}_{}_{}'.format(scope, name, ii)
if key in model:
raise Exception('Key exists: {}'.format(key))
model[key] = param
copy = [0]
def run_cnn(x):
"""
Run CNN on an input.
Args:
x: input image, [B, H, W, D]
"""
h = [None] * nlayers
with tf.variable_scope(scope):
for ii in xrange(nlayers):
with tf.variable_scope('layer_{}'.format(ii)):
out_ch = ch[ii + 1]
if ii == 0:
prev_inp = x
else:
prev_inp = h[ii - 1]
h[ii] = conv2d(prev_inp, w[ii]) + b[ii]
if use_bn[ii]:
if frozen is not None and frozen[ii]:
bn_frozen = True
else:
bn_frozen = False
if init_weights is not None and \
init_weights[ii] is not None:
init_beta = init_weights[ii][
'beta_{}'.format(copy[0])]
init_gamma = init_weights[ii][
'gamma_{}'.format(copy[0])]
else:
init_beta = None
init_gamma = None
# with tf.variable_scope('layer_{}'.format(ii)):
# with tf.variable_scope('copy_{}'.format(copy[0])):
h[ii] = batch_norm(
h[ii], out_ch, phase_train,
scope2='{}_{}_{}'.format(scope, ii, copy[0]),
init_beta=init_beta,
init_gamma=init_gamma,
model=model)
if act[ii] is not None:
h[ii] = act[ii](h[ii])
if pool[ii] > 1:
h[ii] = max_pool(h[ii], pool[ii])
copy[0] += 1
return h
return run_cnn
def mlp(dims,
act,
add_bias=True,
dropout_keep=None,
phase_train=None,
wd=None,
scope='mlp',
model=None,
init_weights=None,
frozen=None):
"""Add MLP. N = number of layers.
Args:
dims: layer-wise dimensions, list of N int
act: activation function, list of N function
dropout_keep: keep prob of dropout, list of N float
phase_train: whether in training phase, tf bool variable
wd: weight decay
"""
log = logger.get()
nlayers = len(dims) - 1
w = [None] * nlayers
b = [None] * nlayers
log.info('MLP: {}'.format(scope))
log.info('Dimensions: {}'.format(dims))
log.info('Activation: {}'.format(act))
log.info('Dropout: {}'.format(dropout_keep))
log.info('Add bias: {}'.format(add_bias))
with tf.variable_scope(scope):
for ii in xrange(nlayers):
with tf.variable_scope('layer_{}'.format(ii)):
nin = dims[ii]
nout = dims[ii + 1]
if init_weights is not None and init_weights[ii] is not None:
init_val_w = init_weights[ii]['w']
init_val_b = init_weights[ii]['b']
else:
init_val_w = None
init_val_b = None
if frozen is not None and frozen[ii]:
trainable = False
else:
trainable = True
w[ii] = weight_variable([nin, nout],
init_val=init_val_w,
wd=wd,
name='w',
trainable=trainable)
log.info('Weights: {} Trainable: {}'.format([nin, nout],
trainable))
if add_bias:
b[ii] = weight_variable([nout],
init_val=init_val_b,
name='b',
trainable=trainable)
log.info('Bias: {} Trainable: {}'.format([nout],
trainable))
if model is not None:
model['{}_w_{}'.format(scope, ii)] = w[ii]
if add_bias:
model['{}_b_{}'.format(scope, ii)] = b[ii]
def run_mlp(x):
h = [None] * nlayers
with tf.variable_scope(scope):
for ii in xrange(nlayers):
with tf.variable_scope('layer_{}'.format(ii)):
if ii == 0:
prev_inp = x
else:
prev_inp = h[ii - 1]
if dropout_keep is not None:
if dropout_keep[ii] is not None:
prev_inp = dropout(prev_inp, dropout_keep[ii],
phase_train)
h[ii] = tf.matmul(prev_inp, w[ii])
if add_bias:
h[ii] += b[ii]
if act[ii]:
h[ii] = act[ii](h[ii])
return h
return run_mlp
def __init__(self):
super(SaverRunner, self).__init__()
self._log = logger.get()
self._step = 0
self.register_option('save_ckpt')
pass
def __init__(self, folder=None):
super(RestorerRunner, self).__init__()
self._log = logger.get()
self.folder = folder
pass
def __init__(self):
super(SaverRunner, self).__init__()
self._log = logger.get()
self._step = 0
self.register_option('save_ckpt')
pass
def lstm(inp_dim,
hid_dim,
wd=None,
scope='lstm',
model=None,
init_weights=None,
frozen=False):
"""Adds an LSTM component.
Args:
inp_dim: Input data dim
hid_dim: Hidden state dim
wd: Weight decay
scope: Prefix
"""
log = logger.get()
log.info('LSTM: {}'.format(scope))
log.info('Input dim: {}'.format(inp_dim))
log.info('Hidden dim: {}'.format(hid_dim))
if init_weights is None:
init_weights = {}
for w in [
'w_xi', 'w_hi', 'b_i', 'w_xf', 'w_hf', 'b_f', 'w_xu', 'w_hu',
'b_u', 'w_xo', 'w_ho', 'b_o'
]:
init_weights[w] = None
trainable = not frozen
log.info('Trainable: {}'.format(trainable))
with tf.variable_scope(scope):
# Input gate
w_xi = weight_variable([inp_dim, hid_dim],
init_val=init_weights['w_xi'],
wd=wd,
name='w_xi',
trainable=trainable)
w_hi = weight_variable([hid_dim, hid_dim],
init_val=init_weights['w_hi'],
wd=wd,
name='w_hi',
trainable=trainable)
b_i = weight_variable([hid_dim],
init_val=init_weights['b_i'],
initializer=tf.constant_initializer(0.0),
name='b_i',
trainable=trainable)
# Forget gate
w_xf = weight_variable([inp_dim, hid_dim],
init_val=init_weights['w_xf'],
wd=wd,
name='w_xf',
trainable=trainable)
w_hf = weight_variable([hid_dim, hid_dim],
init_val=init_weights['w_hf'],
wd=wd,
name='w_hf',
trainable=trainable)
b_f = weight_variable([hid_dim],
init_val=init_weights['b_f'],
initializer=tf.constant_initializer(1.0),
name='b_f',
trainable=trainable)
# Input activation
w_xu = weight_variable([inp_dim, hid_dim],
init_val=init_weights['w_xu'],
wd=wd,
name='w_xu',
trainable=trainable)
w_hu = weight_variable([hid_dim, hid_dim],
init_val=init_weights['w_hu'],
wd=wd,
name='w_hu',
trainable=trainable)
b_u = weight_variable([hid_dim],
init_val=init_weights['b_u'],
initializer=tf.constant_initializer(0.0),
name='b_u',
trainable=trainable)
# Output gate
w_xo = weight_variable([inp_dim, hid_dim],
init_val=init_weights['w_xo'],
wd=wd,
name='w_xo',
trainable=trainable)
w_ho = weight_variable([hid_dim, hid_dim],
init_val=init_weights['w_ho'],
wd=wd,
name='w_ho',
trainable=trainable)
b_o = weight_variable([hid_dim],
init_val=init_weights['b_o'],
initializer=tf.constant_initializer(0.0),
name='b_o',
trainable=trainable)
if model is not None:
model['{}_w_xi'.format(scope)] = w_xi
model['{}_w_hi'.format(scope)] = w_hi
model['{}_b_i'.format(scope)] = b_i
model['{}_w_xf'.format(scope)] = w_xf
model['{}_w_hf'.format(scope)] = w_hf
model['{}_b_f'.format(scope)] = b_f
model['{}_w_xu'.format(scope)] = w_xu
model['{}_w_hu'.format(scope)] = w_hu
model['{}_b_u'.format(scope)] = b_u
model['{}_w_xo'.format(scope)] = w_xo
model['{}_w_ho'.format(scope)] = w_ho
model['{}_b_o'.format(scope)] = b_o
model['{}_w_x_mean'.format(scope)] = (
tf.reduce_sum(tf.abs(w_xi)) + tf.reduce_sum(tf.abs(w_xf)) +
tf.reduce_sum(tf.abs(w_xu)) +
tf.reduce_sum(tf.abs(w_xo))) / inp_dim / hid_dim / 4
model['{}_w_h_mean'.format(scope)] = (
tf.reduce_sum(tf.abs(w_hi)) + tf.reduce_sum(tf.abs(w_hf)) +
tf.reduce_sum(tf.abs(w_hu)) +
tf.reduce_sum(tf.abs(w_ho))) / hid_dim / hid_dim / 4
model['{}_b_mean'.format(scope)] = (tf.reduce_sum(
tf.abs(b_i)) + tf.reduce_sum(tf.abs(b_f)) + tf.reduce_sum(
tf.abs(b_u)) + tf.reduce_sum(tf.abs(b_o))) / hid_dim / 4
def unroll(inp, state):
with tf.variable_scope(scope):
c = tf.slice(state, [0, 0], [-1, hid_dim])
h = tf.slice(state, [0, hid_dim], [-1, hid_dim])
g_i = tf.sigmoid(tf.matmul(inp, w_xi) + tf.matmul(h, w_hi) + b_i)
g_f = tf.sigmoid(tf.matmul(inp, w_xf) + tf.matmul(h, w_hf) + b_f)
g_o = tf.sigmoid(tf.matmul(inp, w_xo) + tf.matmul(h, w_ho) + b_o)
u = tf.tanh(tf.matmul(inp, w_xu) + tf.matmul(h, w_hu) + b_u)
c = g_f * c + g_i * u
h = g_o * tf.tanh(c)
state = tf.concat(1, [c, h])
return state, g_i, g_f, g_o
return unroll
def conv_lstm(inp_depth,
hid_depth,
filter_size,
wd=None,
scope='conv_lstm',
model=None,
init_weights=None,
frozen=False):
"""Adds a Conv-LSTM component.
Args:
inp_depth: Input image depth
filter_size: Conv gate filter size
hid_depth: Hidden state depth
wd: Weight decay
name: Prefix
"""
log = logger.get()
log.info('ConvLSTM: {}'.format(scope))
log.info('Input depth: {}'.format(inp_depth))
log.info('Hidden depth: {}'.format(hid_depth))
if init_weights is None:
init_weights = {}
for w in [
'w_xi', 'w_hi', 'b_i', 'w_xf', 'w_hf', 'b_f', 'w_xu', 'w_hu',
'b_u', 'w_xo', 'w_ho', 'b_o'
]:
init_weights[w] = None
trainable = not frozen
log.info('Trainable: {}'.format(trainable))
with tf.variable_scope(scope):
# Input gate
w_xi = weight_variable(
[filter_size, filter_size, inp_depth, hid_depth],
init_val=init_weights['w_xi'],
wd=wd,
name='w_xi',
trainable=trainable)
w_hi = weight_variable(
[filter_size, filter_size, hid_depth, hid_depth],
init_val=init_weights['w_hi'],
wd=wd,
name='w_hi',
trainable=trainable)
b_i = weight_variable([hid_depth],
init_val=init_weights['b_i'],
name='b_i',
trainable=trainable)
# Forget gate
w_xf = weight_variable(
[filter_size, filter_size, inp_depth, hid_depth],
wd=wd,
init_val=init_weights['w_xf'],
name='w_xf',
trainable=trainable)
w_hf = weight_variable(
[filter_size, filter_size, hid_depth, hid_depth],
init_val=init_weights['w_hf'],
wd=wd,
name='w_hf',
trainable=trainable)
b_f = weight_variable([hid_depth],
init_val=init_weights['b_f'],
name='b_f',
trainable=trainable)
# Input activation
w_xu = weight_variable(
[filter_size, filter_size, inp_depth, hid_depth],
init_val=init_weights['w_xu'],
wd=wd,
name='w_xu',
trainable=trainable)
w_hu = weight_variable(
[filter_size, filter_size, hid_depth, hid_depth],
init_val=init_weights['w_hu'],
wd=wd,
name='w_hu',
trainable=trainable)
b_u = weight_variable([hid_depth],
init_val=init_weights['b_u'],
name='b_u',
trainable=trainable)
# Output gate
w_xo = weight_variable(
[filter_size, filter_size, inp_depth, hid_depth],
init_val=init_weights['w_xo'],
wd=wd,
name='w_xo',
trainable=trainable)
w_ho = weight_variable(
[filter_size, filter_size, hid_depth, hid_depth],
init_val=init_weights['w_ho'],
wd=wd,
name='w_ho',
trainable=trainable)
b_o = weight_variable([hid_depth],
init_val=init_weights['b_o'],
name='b_o',
trainable=trainable)
if model is not None:
model['{}_w_xi'.format(scope)] = w_xi
model['{}_w_hi'.format(scope)] = w_hi
model['{}_b_i'.format(scope)] = b_i
model['{}_w_xf'.format(scope)] = w_xf
model['{}_w_hf'.format(scope)] = w_hf
model['{}_b_f'.format(scope)] = b_f
model['{}_w_xu'.format(scope)] = w_xu
model['{}_w_hu'.format(scope)] = w_hu
model['{}_b_u'.format(scope)] = b_u
model['{}_w_xo'.format(scope)] = w_xo
model['{}_w_ho'.format(scope)] = w_ho
model['{}_b_o'.format(scope)] = b_o
def unroll(inp, state):
with tf.variable_scope(scope):
c = tf.slice(state, [0, 0, 0, 0], [-1, -1, -1, hid_depth])
h = tf.slice(state, [0, 0, 0, hid_depth], [-1, -1, -1, hid_depth])
g_i = tf.sigmoid(conv2d(inp, w_xi) + conv2d(h, w_hi) + b_i)
g_f = tf.sigmoid(conv2d(inp, w_xf) + conv2d(h, w_hf) + b_f)
g_o = tf.sigmoid(conv2d(inp, w_xo) + conv2d(h, w_ho) + b_o)
u = tf.tanh(conv2d(inp, w_xu) + conv2d(h, w_hu) + b_u)
c = g_f * c + g_i * u
h = g_o * tf.tanh(c)
state = tf.concat(3, [c, h])
return state
return unroll
def conv_lstm(inp_depth, hid_depth, filter_size, wd=None, scope='conv_lstm', model=None, init_weights=None, frozen=False):
"""Adds a Conv-LSTM component.
Args:
inp_depth: Input image depth
filter_size: Conv gate filter size
hid_depth: Hidden state depth
wd: Weight decay
name: Prefix
"""
log = logger.get()
log.info('ConvLSTM: {}'.format(scope))
log.info('Input depth: {}'.format(inp_depth))
log.info('Hidden depth: {}'.format(hid_depth))
if init_weights is None:
init_weights = {}
for w in ['w_xi', 'w_hi', 'b_i', 'w_xf', 'w_hf', 'b_f', 'w_xu',
'w_hu', 'b_u', 'w_xo', 'w_ho', 'b_o']:
init_weights[w] = None
trainable = not frozen
log.info('Trainable: {}'.format(trainable))
with tf.variable_scope(scope):
# Input gate
w_xi = weight_variable([filter_size, filter_size, inp_depth, hid_depth],
init_val=init_weights['w_xi'],
wd=wd, name='w_xi', trainable=trainable)
w_hi = weight_variable([filter_size, filter_size, hid_depth, hid_depth],
init_val=init_weights['w_hi'],
wd=wd, name='w_hi', trainable=trainable)
b_i = weight_variable([hid_depth], init_val=init_weights['b_i'],
name='b_i', trainable=trainable)
# Forget gate
w_xf = weight_variable([filter_size, filter_size, inp_depth, hid_depth],
wd=wd,
init_val=init_weights['w_xf'], name='w_xf',
trainable=trainable)
w_hf = weight_variable([filter_size, filter_size, hid_depth, hid_depth],
init_val=init_weights['w_hf'],
wd=wd, name='w_hf', trainable=trainable)
b_f = weight_variable([hid_depth], init_val=init_weights['b_f'],
name='b_f', trainable=trainable)
# Input activation
w_xu = weight_variable([filter_size, filter_size, inp_depth, hid_depth],
init_val=init_weights['w_xu'],
wd=wd, name='w_xu', trainable=trainable)
w_hu = weight_variable([filter_size, filter_size, hid_depth, hid_depth],
init_val=init_weights['w_hu'],
wd=wd, name='w_hu', trainable=trainable)
b_u = weight_variable([hid_depth], init_val=init_weights['b_u'],
name='b_u', trainable=trainable)
# Output gate
w_xo = weight_variable([filter_size, filter_size, inp_depth, hid_depth],
init_val=init_weights['w_xo'],
wd=wd, name='w_xo', trainable=trainable)
w_ho = weight_variable([filter_size, filter_size, hid_depth, hid_depth],
init_val=init_weights['w_ho'],
wd=wd, name='w_ho', trainable=trainable)
b_o = weight_variable([hid_depth], init_val=init_weights['b_o'],
name='b_o', trainable=trainable)
if model is not None:
model['{}_w_xi'.format(scope)] = w_xi
model['{}_w_hi'.format(scope)] = w_hi
model['{}_b_i'.format(scope)] = b_i
model['{}_w_xf'.format(scope)] = w_xf
model['{}_w_hf'.format(scope)] = w_hf
model['{}_b_f'.format(scope)] = b_f
model['{}_w_xu'.format(scope)] = w_xu
model['{}_w_hu'.format(scope)] = w_hu
model['{}_b_u'.format(scope)] = b_u
model['{}_w_xo'.format(scope)] = w_xo
model['{}_w_ho'.format(scope)] = w_ho
model['{}_b_o'.format(scope)] = b_o
def unroll(inp, state):
with tf.variable_scope(scope):
c = tf.slice(state, [0, 0, 0, 0], [-1, -1, -1, hid_depth])
h = tf.slice(state, [0, 0, 0, hid_depth], [-1, -1, -1, hid_depth])
g_i = tf.sigmoid(conv2d(inp, w_xi) + conv2d(h, w_hi) + b_i)
g_f = tf.sigmoid(conv2d(inp, w_xf) + conv2d(h, w_hf) + b_f)
g_o = tf.sigmoid(conv2d(inp, w_xo) + conv2d(h, w_ho) + b_o)
u = tf.tanh(conv2d(inp, w_xu) + conv2d(h, w_hu) + b_u)
c = g_f * c + g_i * u
h = g_o * tf.tanh(c)
state = tf.concat(3, [c, h])
return state
return unroll
def cnn(f,
ch,
pool,
act,
use_bn,
phase_train=None,
wd=None,
scope='cnn',
model=None,
init_weights=None,
frozen=None,
shared_weights=None):
"""Add CNN. N = number of layers.
Args:
f: filter size, list of N int
ch: number of channels, list of (N + 1) int
pool: pooling ratio, list of N int
act: activation function, list of N function
use_bn: whether to use batch normalization, list of N bool
phase_train: whether in training phase, tf bool variable
wd: weight decay
Returns:
run_cnn: a function that runs the CNN
"""
log = logger.get()
nlayers = len(f)
w = [None] * nlayers
b = [None] * nlayers
log.info('CNN: {}'.format(scope))
log.info('Channels: {}'.format(ch))
log.info('Activation: {}'.format(act))
log.info('Pool: {}'.format(pool))
log.info('BN: {}'.format(use_bn))
log.info('Shared weights: {}'.format(shared_weights))
net_scope = None
layer_scope = [None] * nlayers
with tf.variable_scope(scope):
for ii in xrange(nlayers):
with tf.variable_scope('layer_{}'.format(ii)):
if init_weights:
init = tf.constant_initializer
else:
init = None
if init_weights is not None and init_weights[ii] is not None:
init_val_w = init_weights[ii]['w']
init_val_b = init_weights[ii]['b']
else:
init_val_w = None
init_val_b = None
if frozen is not None and frozen[ii]:
trainable = False
else:
trainable = True
if shared_weights:
w[ii] = shared_weights[ii]['w']
b[ii] = shared_weights[ii]['b']
else:
w[ii] = weight_variable([f[ii], f[ii], ch[ii], ch[ii + 1]],
name='w',
init_val=init_val_w,
wd=wd,
trainable=trainable)
b[ii] = weight_variable([ch[ii + 1]],
init_val=init_val_b,
name='b',
trainable=trainable)
log.info('Filter: {}, Trainable: {}'.format(
[f[ii], f[ii], ch[ii], ch[ii + 1]], trainable))
if model is not None:
for name, param in zip(['w', 'b'], [w[ii], b[ii]]):
key = '{}_{}_{}'.format(scope, name, ii)
if key in model:
raise Exception('Key exists: {}'.format(key))
model[key] = param
copy = [0]
def run_cnn(x):
"""
Run CNN on an input.
Args:
x: input image, [B, H, W, D]
"""
h = [None] * nlayers
with tf.variable_scope(scope):
for ii in xrange(nlayers):
with tf.variable_scope('layer_{}'.format(ii)):
out_ch = ch[ii + 1]
if ii == 0:
prev_inp = x
else:
prev_inp = h[ii - 1]
h[ii] = conv2d(prev_inp, w[ii]) + b[ii]
if use_bn[ii]:
if frozen is not None and frozen[ii]:
bn_frozen = True
else:
bn_frozen = False
if init_weights is not None and \
init_weights[ii] is not None:
init_beta = init_weights[ii]['beta_{}'.format(
copy[0])]
init_gamma = init_weights[ii]['gamma_{}'.format(
copy[0])]
else:
init_beta = None
init_gamma = None
# with tf.variable_scope('layer_{}'.format(ii)):
# with tf.variable_scope('copy_{}'.format(copy[0])):
h[ii] = batch_norm(h[ii],
out_ch,
phase_train,
scope2='{}_{}_{}'.format(
scope, ii, copy[0]),
init_beta=init_beta,
init_gamma=init_gamma,
model=model)
if act[ii] is not None:
h[ii] = act[ii](h[ii])
if pool[ii] > 1:
h[ii] = max_pool(h[ii], pool[ii])
copy[0] += 1
return h
return run_cnn
def lstm(inp_dim, hid_dim, wd=None, scope='lstm', model=None, init_weights=None, frozen=False):
"""Adds an LSTM component.
Args:
inp_dim: Input data dim
hid_dim: Hidden state dim
wd: Weight decay
scope: Prefix
"""
log = logger.get()
log.info('LSTM: {}'.format(scope))
log.info('Input dim: {}'.format(inp_dim))
log.info('Hidden dim: {}'.format(hid_dim))
if init_weights is None:
init_weights = {}
for w in ['w_xi', 'w_hi', 'b_i', 'w_xf', 'w_hf', 'b_f', 'w_xu',
'w_hu', 'b_u', 'w_xo', 'w_ho', 'b_o']:
init_weights[w] = None
trainable = not frozen
log.info('Trainable: {}'.format(trainable))
with tf.variable_scope(scope):
# Input gate
w_xi = weight_variable(
[inp_dim, hid_dim], init_val=init_weights['w_xi'], wd=wd,
name='w_xi', trainable=trainable)
w_hi = weight_variable(
[hid_dim, hid_dim], init_val=init_weights['w_hi'], wd=wd,
name='w_hi', trainable=trainable)
b_i = weight_variable(
[hid_dim], init_val=init_weights['b_i'],
initializer=tf.constant_initializer(0.0),
name='b_i', trainable=trainable)
# Forget gate
w_xf = weight_variable(
[inp_dim, hid_dim], init_val=init_weights['w_xf'], wd=wd,
name='w_xf', trainable=trainable)
w_hf = weight_variable(
[hid_dim, hid_dim], init_val=init_weights['w_hf'], wd=wd,
name='w_hf', trainable=trainable)
b_f = weight_variable(
[hid_dim], init_val=init_weights['b_f'],
initializer=tf.constant_initializer(1.0),
name='b_f', trainable=trainable)
# Input activation
w_xu = weight_variable(
[inp_dim, hid_dim], init_val=init_weights['w_xu'], wd=wd,
name='w_xu', trainable=trainable)
w_hu = weight_variable(
[hid_dim, hid_dim], init_val=init_weights['w_hu'], wd=wd,
name='w_hu', trainable=trainable)
b_u = weight_variable(
[hid_dim], init_val=init_weights['b_u'],
initializer=tf.constant_initializer(0.0),
name='b_u', trainable=trainable)
# Output gate
w_xo = weight_variable(
[inp_dim, hid_dim], init_val=init_weights['w_xo'], wd=wd,
name='w_xo', trainable=trainable)
w_ho = weight_variable(
[hid_dim, hid_dim], init_val=init_weights['w_ho'], wd=wd,
name='w_ho', trainable=trainable)
b_o = weight_variable(
[hid_dim], init_val=init_weights['b_o'],
initializer=tf.constant_initializer(0.0),
name='b_o', trainable=trainable)
if model is not None:
model['{}_w_xi'.format(scope)] = w_xi
model['{}_w_hi'.format(scope)] = w_hi
model['{}_b_i'.format(scope)] = b_i
model['{}_w_xf'.format(scope)] = w_xf
model['{}_w_hf'.format(scope)] = w_hf
model['{}_b_f'.format(scope)] = b_f
model['{}_w_xu'.format(scope)] = w_xu
model['{}_w_hu'.format(scope)] = w_hu
model['{}_b_u'.format(scope)] = b_u
model['{}_w_xo'.format(scope)] = w_xo
model['{}_w_ho'.format(scope)] = w_ho
model['{}_b_o'.format(scope)] = b_o
model['{}_w_x_mean'.format(scope)] = (tf.reduce_sum(
tf.abs(w_xi)) + tf.reduce_sum(tf.abs(w_xf)) +
tf.reduce_sum(tf.abs(w_xu)) +
tf.reduce_sum(tf.abs(w_xo))) / inp_dim / hid_dim / 4
model['{}_w_h_mean'.format(scope)] = (tf.reduce_sum(
tf.abs(w_hi)) + tf.reduce_sum(tf.abs(w_hf)) +
tf.reduce_sum(tf.abs(w_hu)) +
tf.reduce_sum(tf.abs(w_ho))) / hid_dim / hid_dim / 4
model['{}_b_mean'.format(scope)] = (tf.reduce_sum(
tf.abs(b_i)) + tf.reduce_sum(tf.abs(b_f)) +
tf.reduce_sum(tf.abs(b_u)) +
tf.reduce_sum(tf.abs(b_o))) / hid_dim / 4
def unroll(inp, state):
with tf.variable_scope(scope):
c = tf.slice(state, [0, 0], [-1, hid_dim])
h = tf.slice(state, [0, hid_dim], [-1, hid_dim])
g_i = tf.sigmoid(tf.matmul(inp, w_xi) + tf.matmul(h, w_hi) + b_i)
g_f = tf.sigmoid(tf.matmul(inp, w_xf) + tf.matmul(h, w_hf) + b_f)
g_o = tf.sigmoid(tf.matmul(inp, w_xo) + tf.matmul(h, w_ho) + b_o)
u = tf.tanh(tf.matmul(inp, w_xu) + tf.matmul(h, w_hu) + b_u)
c = g_f * c + g_i * u
h = g_o * tf.tanh(c)
state = tf.concat(1, [c, h])
return state, g_i, g_f, g_o
return unroll
def __init__(self):
self.log = logger.get()
self._var_dict = {}
self._has_init_var = False
self._variable_sharing = False
pass
def res_cnn(f, ch, strides, phase_train, wd=None, scope='res_cnn'):
"""
Args:
f: Filter size for every basic block.
ch: Channel size for every basic block.
pool: Pool ratio for every basic block.
"""
log = logger.get()
nlayers = len(f)
w = [None] * nlayers
b = [None] * nlayers
log.info('Residual CNN: {}'.format(scope))
log.info('Channels: {}'.format(ch))
log.info('Strides: {}'.format(strides))
with tf.variable_scope(scope):
for ii in xrange(nlayers):
with tf.variable_scope('layer_{}'.format(ii)):
w[ii] = [None] * 2
b[ii] = [None] * 2
# init = tf.truncated_normal_initializer(stddev=0.01)
log.info('Filter: {}'.format(
[f[ii], f[ii], ch[ii], ch[ii + 1]]))
w[ii][0] = weight_variable([f[ii], f[ii], ch[ii], ch[ii + 1]],
name='w_0',
wd=wd)
b[ii][0] = weight_variable([ch[ii + 1]], name='b_0', wd=wd)
log.info('Filter: {}'.format(
[f[ii], f[ii], ch[ii + 1], ch[ii + 1]]))
w[ii][1] = weight_variable(
[f[ii], f[ii], ch[ii + 1], ch[ii + 1]], name='w_1', wd=wd)
b[ii][1] = weight_variable([ch[ii + 1]], name='b_1', wd=wd)
def shortcut(x, prev_x, stride, in_ch, out_ch):
if stride > 1:
prev_x = avg_pool(prev_x, stride)
if in_ch != out_ch:
xshape = tf.shape(x)
zeros = tf.zeros(
tf.pack([xshape[0], xshape[1], xshape[2], out_ch - in_ch]))
prev_x = tf.concat(3, [prev_x, zeros])
return prev_x + x
def run_res_cnn(x):
h = [None] * nlayers
with tf.variable_scope(scope):
for ii in xrange(nlayers):
with tf.variable_scope('layer_{}'.format(ii)):
if ii == 0:
prev_inp = x
else:
prev_inp = h[ii - 1]
h_tmp = conv2d(prev_inp, w[ii][0], strides[ii]) + b[ii][0]
h_tmp = batch_norm(h_tmp, ch[ii + 1], phase_train)
h_tmp = tf.nn.relu(h_tmp)
h_tmp = conv2d(h_tmp, w[ii][1]) + b[ii][1]
h_tmp = batch_norm(h_tmp, ch[ii + 1], phase_train)
if ii > 0:
h[ii] = shortcut(h_tmp, h[ii - 1], strides[ii], ch[ii],
ch[ii + 1])
else:
h[ii] = h_tmp
log.info(h[ii].get_shape())
pass
return h
return run_res_cnn
def res_cnn(f, ch, strides, phase_train, wd=None, scope='res_cnn'):
"""
Args:
f: Filter size for every basic block.
ch: Channel size for every basic block.
pool: Pool ratio for every basic block.
"""
log = logger.get()
nlayers = len(f)
w = [None] * nlayers
b = [None] * nlayers
log.info('Residual CNN: {}'.format(scope))
log.info('Channels: {}'.format(ch))
log.info('Strides: {}'.format(strides))
with tf.variable_scope(scope):
for ii in xrange(nlayers):
with tf.variable_scope('layer_{}'.format(ii)):
w[ii] = [None] * 2
b[ii] = [None] * 2
# init = tf.truncated_normal_initializer(stddev=0.01)
log.info('Filter: {}'.format(
[f[ii], f[ii], ch[ii], ch[ii + 1]]))
w[ii][0] = weight_variable(
[f[ii], f[ii], ch[ii], ch[ii + 1]], name='w_0', wd=wd)
b[ii][0] = weight_variable([ch[ii + 1]], name='b_0', wd=wd)
log.info('Filter: {}'.format(
[f[ii], f[ii], ch[ii + 1], ch[ii + 1]]))
w[ii][1] = weight_variable(
[f[ii], f[ii], ch[ii + 1], ch[ii + 1]], name='w_1', wd=wd)
b[ii][1] = weight_variable([ch[ii + 1]], name='b_1', wd=wd)
def shortcut(x, prev_x, stride, in_ch, out_ch):
if stride > 1:
prev_x = avg_pool(prev_x, stride)
if in_ch != out_ch:
xshape = tf.shape(x)
zeros = tf.zeros(tf.pack([xshape[0], xshape[1], xshape[2], out_ch -
in_ch]))
prev_x = tf.concat(3, [prev_x, zeros])
return prev_x + x
def run_res_cnn(x):
h = [None] * nlayers
with tf.variable_scope(scope):
for ii in xrange(nlayers):
with tf.variable_scope('layer_{}'.format(ii)):
if ii == 0:
prev_inp = x
else:
prev_inp = h[ii - 1]
h_tmp = conv2d(prev_inp, w[ii][0], strides[ii]) + b[ii][0]
h_tmp = batch_norm(h_tmp, ch[ii + 1], phase_train)
h_tmp = tf.nn.relu(h_tmp)
h_tmp = conv2d(h_tmp, w[ii][1]) + b[ii][1]
h_tmp = batch_norm(h_tmp, ch[ii + 1], phase_train)
if ii > 0:
h[ii] = shortcut(
h_tmp, h[ii - 1], strides[ii], ch[ii], ch[ii + 1])
else:
h[ii] = h_tmp
log.info(h[ii].get_shape())
pass
return h
return run_res_cnn
def __init__(self, folder=None):
super(RestorerRunner, self).__init__()
self._log = logger.get()
self.folder = folder
pass
def dcnn(f,
ch,
pool,
act,
use_bn,
skip_ch=None,
phase_train=None,
wd=None,
scope='dcnn',
model=None,
init_weights=None,
frozen=None):
"""Add DCNN. N = number of layers.
Args:
f: filter size, list of size N int
ch: number of channels, list of (N + 1) int
pool: pooling ratio, list of N int
act: activation function, list of N function
use_bn: whether to use batch normalization, list of N bool
skip_ch: skip connection, list of N int
phase_train: whether in training phase, tf bool variable
wd: weight decay
Returns:
run_dcnn: a function that runs the DCNN
"""
log = logger.get()
nlayers = len(f)
w = [None] * nlayers
b = [None] * nlayers
bn = [None] * nlayers
log.info('DCNN: {}'.format(scope))
log.info('Channels: {}'.format(ch))
log.info('Activation: {}'.format(act))
log.info('Unpool: {}'.format(pool))
log.info('Skip channels: {}'.format(skip_ch))
log.info('BN: {}'.format(use_bn))
with tf.variable_scope(scope):
in_ch = ch[0]
for ii in xrange(nlayers):
with tf.variable_scope('layer_{}'.format(ii)):
out_ch = ch[ii + 1]
if skip_ch is not None:
if skip_ch[ii] is not None:
in_ch += skip_ch[ii]
if init_weights is not None and init_weights[ii] is not None:
init_val_w = init_weights[ii]['w']
init_val_b = init_weights[ii]['b']
else:
init_val_w = None
init_val_b = None
if frozen is not None and frozen[ii]:
trainable = False
else:
trainable = True
w[ii] = weight_variable([f[ii], f[ii], out_ch, in_ch],
name='w',
init_val=init_val_w,
wd=wd,
trainable=trainable)
b[ii] = weight_variable([out_ch],
init_val=init_val_b,
name='b',
trainable=trainable)
log.info('Filter: {}, Trainable: {}'.format(
[f[ii], f[ii], out_ch, in_ch], trainable))
in_ch = out_ch
if model is not None:
model['{}_w_{}'.format(scope, ii)] = w[ii]
model['{}_b_{}'.format(scope, ii)] = b[ii]
copy = [0]
def run_dcnn(x, skip=None):
"""Run DCNN on an input.
Args:
x: input image, [B, H, W, D]
skip: skip connection activation map, list of 4-D tensor
"""
with tf.variable_scope(scope):
h = [None] * nlayers
out_shape = [None] * nlayers
batch = tf.shape(x)[0:1]
inp_size = tf.shape(x)[1:3]
cum_pool = 1
for ii in xrange(nlayers):
with tf.variable_scope('layer_{}'.format(ii)):
cum_pool *= pool[ii]
out_ch = ch[ii + 1]
if ii == 0:
prev_inp = x
else:
prev_inp = h[ii - 1]
if skip is not None:
if skip[ii] is not None:
if ii == 0:
prev_inp = tf.concat(3, [prev_inp, skip[ii]])
else:
prev_inp = tf.concat(3, [prev_inp, skip[ii]])
out_shape[ii] = tf.concat(
0, [batch, inp_size * cum_pool,
tf.constant([out_ch])])
h[ii] = tf.nn.conv2d_transpose(
prev_inp,
w[ii],
out_shape[ii],
strides=[1, pool[ii], pool[ii], 1]) + b[ii]
if use_bn[ii]:
if frozen is not None and frozen[ii]:
bn_frozen = True
else:
bn_frozen = False
if init_weights is not None and \
init_weights[ii] is not None:
init_beta = init_weights[ii]['beta_{}'.format(
copy[0])]
init_gamma = init_weights[ii]['gamma_{}'.format(
copy[0])]
else:
init_beta = None
init_gamma = None
# with tf.variable_scope('layer_{}'.format(ii)):
# with tf.variable_scope('copy_{}'.format(copy[0])):
h[ii] = batch_norm(h[ii],
out_ch,
phase_train,
scope2='{}_{}_{}'.format(
scope, ii, copy[0]),
init_beta=init_beta,
init_gamma=init_gamma,
model=model)
if act[ii] is not None:
h[ii] = act[ii](h[ii])
copy[0] += 1
return h
return run_dcnn
def __init__(self):
self.log = logger.get()
self._var_dict = {}
self._has_init_var = False
self._variable_sharing = False
pass
import cslab_environ
from tensorflow.python.framework import ops
import tfplus.utils.logger as logger
import numpy as np
import tensorflow as tf
hungarian_module = None
log = logger.get()
# Register gradient for Hungarian algorithm.
ops.NoGradient("Hungarian")
def get_device_fn(device):
"""Choose device for different ops."""
OPS_ON_CPU = set(['ResizeBilinear', 'ResizeBilinearGrad', 'Mod', 'CumMin',
'CumMinGrad', 'Hungarian', 'Reverse', 'SparseToDense',
'BatchMatMul', 'Gather', 'Print'])
def _device_fn(op):
if op.type in OPS_ON_CPU:
return "/cpu:0"
else:
# Other ops will be placed on GPU if available, otherwise CPU.
return device
return _device_fn