def weight_variable(shape, initializer=None, init_val=None, wd=None, name=None, trainable=True): """Initialize weights. Args: shape: shape of the weights, list of int wd: weight decay """ log = logger.get() if initializer is None: # initializer = tf.truncated_normal(shape, stddev=0.01) initializer = tf.truncated_normal_initializer(stddev=0.01) if init_val is None: var = tf.Variable(initializer(shape), name=name, trainable=trainable) else: var = tf.Variable(init_val, name=name, trainable=trainable) # log.info(var.name) # if init_val is not None: # if hasattr(init_val, 'shape'): # log.info('Initialized with array shape {}'.format(init_val.shape)) # else: # log.info('Initialized with {}'.format(init_val)) if wd: weight_decay = tf.mul(tf.nn.l2_loss(var), wd, name='weight_loss') tf.add_to_collection('losses', weight_decay) return var
def __init__(self, sess=None, model=None): super(Experiment, self).__init__() self.log = logger.get() self.log.log_args() self._sess = sess self._model = model pass
def __init__(self): super(BasicRunner, self).__init__() self._step = 0 self._data_provider = None self._phase_train = True self._outputs = [] self._current_batch = {} self._log = logger.get() self._preprocessor = lambda x: x self._listeners = [] pass
def maybe_download(filename, work_directory): """Download the data from Yann's website, unless it's already here.""" if not os.path.exists(work_directory): os.mkdir(work_directory) filepath = os.path.join(work_directory, filename) if not os.path.exists(filepath): filepath, _ = urllib.urlretrieve(SOURCE_URL + filename, filepath) statinfo = os.stat(filepath) log = logger.get() log.info('Succesfully downloaded {} {} bytes.'.format( filename, statinfo.st_size)) return filepath
def __init__(self, top_k=1, filename=None, label=None): self.top_k = top_k self.correct = 0 self.count = 0 self.log = logger.get() self.filename = filename self.label = label self.registered = False if not os.path.exists(filename): with open(filename, 'w') as f: f.write('step,time,{}\n'.format(label)) pass
def __init__(self, split='train', filename=None): super(MNISTDataProvider, self).__init__() self.log = logger.get() if split is None: self.split = 'train' else: self.split = split self.log.info('Data split: {}'.format(self.split)) self.filename = filename self._images = None self._labels = None self.register_option('mnist:dataset_folder') pass
def extract_labels(filename, one_hot=False): """Extract the labels into a 1D uint8 numpy array [index].""" log = logger.get() log.info('Extracting {}'.format(filename)) with gzip.open(filename) as bytestream: magic = _read32(bytestream) if magic != 2049: raise ValueError( 'Invalid magic number %d in MNIST label file: %s' % (magic, filename)) num_items = _read32(bytestream) buf = bytestream.read(num_items) labels = numpy.frombuffer(buf, dtype=numpy.uint8) if one_hot: return dense_to_one_hot(labels) return labels
def extract_images(filename): """Extract the images into a 4D uint8 numpy array [index, y, x, depth].""" log = logger.get() log.info('Extracting {}'.format(filename)) with gzip.open(filename) as bytestream: magic = _read32(bytestream) if magic != 2051: raise ValueError( 'Invalid magic number %d in MNIST image file: %s' % (magic, filename)) num_images = _read32(bytestream) rows = _read32(bytestream) cols = _read32(bytestream) buf = bytestream.read(rows * cols * num_images) data = numpy.frombuffer(buf, dtype=numpy.uint8) data = data.reshape(num_images, rows, cols, 1) return data
def gru(inp_dim, hid_dim, wd=None, scope='gru'): """Adds a GRU component. Args: inp_dim: Input data dim hid_dim: Hidden state dim wd: Weight decay scope: Prefix """ log = logger.get() log.info('GRU: {}'.format(scope)) log.info('Input dim: {}'.format(inp_dim)) log.info('Hidden dim: {}'.format(hid_dim)) with tf.variable_scope(scope): w_xi = weight_variable([inp_dim, hid_dim], wd=wd, name='w_xi') w_hi = weight_variable([hid_dim, hid_dim], wd=wd, name='w_hi') b_i = weight_variable([hid_dim], name='b_i') w_xu = weight_variable([inp_dim, hid_dim], wd=wd, name='w_xu') w_hu = weight_variable([hid_dim, hid_dim], wd=wd, name='w_hu') b_u = weight_variable([hid_dim], name='b_u') w_xr = weight_variable([inp_dim, hid_dim], wd=wd, name='w_xr') w_hr = weight_variable([hid_dim, hid_dim], wd=wd, name='w_hr') b_r = weight_variable([hid_dim], name='b_r') def unroll(inp, state): g_i = tf.sigmoid(tf.matmul(inp, w_xi) + tf.matmul(state, w_hi) + b_i) g_r = tf.sigmoid(tf.matmul(inp, w_xr) + tf.matmul(state, w_hr) + b_r) u = tf.tanh(tf.matmul(inp, w_xu) + g_r * tf.matmul(state, w_hu) + b_u) state = state * (1 - g_i) + u * g_i return state return unroll
def mlp(dims, act, add_bias=True, dropout_keep=None, phase_train=None, wd=None, scope='mlp', model=None, init_weights=None, frozen=None): """Add MLP. N = number of layers. Args: dims: layer-wise dimensions, list of N int act: activation function, list of N function dropout_keep: keep prob of dropout, list of N float phase_train: whether in training phase, tf bool variable wd: weight decay """ log = logger.get() nlayers = len(dims) - 1 w = [None] * nlayers b = [None] * nlayers log.info('MLP: {}'.format(scope)) log.info('Dimensions: {}'.format(dims)) log.info('Activation: {}'.format(act)) log.info('Dropout: {}'.format(dropout_keep)) log.info('Add bias: {}'.format(add_bias)) with tf.variable_scope(scope): for ii in xrange(nlayers): with tf.variable_scope('layer_{}'.format(ii)): nin = dims[ii] nout = dims[ii + 1] if init_weights is not None and init_weights[ii] is not None: init_val_w = init_weights[ii]['w'] init_val_b = init_weights[ii]['b'] else: init_val_w = None init_val_b = None if frozen is not None and frozen[ii]: trainable = False else: trainable = True w[ii] = weight_variable([nin, nout], init_val=init_val_w, wd=wd, name='w', trainable=trainable) log.info('Weights: {} Trainable: {}'.format( [nin, nout], trainable)) if add_bias: b[ii] = weight_variable([nout], init_val=init_val_b, name='b', trainable=trainable) log.info('Bias: {} Trainable: {}'.format( [nout], trainable)) if model is not None: model['{}_w_{}'.format(scope, ii)] = w[ii] if add_bias: model['{}_b_{}'.format(scope, ii)] = b[ii] def run_mlp(x): h = [None] * nlayers with tf.variable_scope(scope): for ii in xrange(nlayers): with tf.variable_scope('layer_{}'.format(ii)): if ii == 0: prev_inp = x else: prev_inp = h[ii - 1] if dropout_keep is not None: if dropout_keep[ii] is not None: prev_inp = dropout( prev_inp, dropout_keep[ii], phase_train) h[ii] = tf.matmul(prev_inp, w[ii]) if add_bias: h[ii] += b[ii] if act[ii]: h[ii] = act[ii](h[ii]) return h return run_mlp
def dcnn(f, ch, pool, act, use_bn, skip_ch=None, phase_train=None, wd=None, scope='dcnn', model=None, init_weights=None, frozen=None): """Add DCNN. N = number of layers. Args: f: filter size, list of size N int ch: number of channels, list of (N + 1) int pool: pooling ratio, list of N int act: activation function, list of N function use_bn: whether to use batch normalization, list of N bool skip_ch: skip connection, list of N int phase_train: whether in training phase, tf bool variable wd: weight decay Returns: run_dcnn: a function that runs the DCNN """ log = logger.get() nlayers = len(f) w = [None] * nlayers b = [None] * nlayers bn = [None] * nlayers log.info('DCNN: {}'.format(scope)) log.info('Channels: {}'.format(ch)) log.info('Activation: {}'.format(act)) log.info('Unpool: {}'.format(pool)) log.info('Skip channels: {}'.format(skip_ch)) log.info('BN: {}'.format(use_bn)) with tf.variable_scope(scope): in_ch = ch[0] for ii in xrange(nlayers): with tf.variable_scope('layer_{}'.format(ii)): out_ch = ch[ii + 1] if skip_ch is not None: if skip_ch[ii] is not None: in_ch += skip_ch[ii] if init_weights is not None and init_weights[ii] is not None: init_val_w = init_weights[ii]['w'] init_val_b = init_weights[ii]['b'] else: init_val_w = None init_val_b = None if frozen is not None and frozen[ii]: trainable = False else: trainable = True w[ii] = weight_variable([f[ii], f[ii], out_ch, in_ch], name='w', init_val=init_val_w, wd=wd, trainable=trainable) b[ii] = weight_variable([out_ch], init_val=init_val_b, name='b', trainable=trainable) log.info('Filter: {}, Trainable: {}'.format( [f[ii], f[ii], out_ch, in_ch], trainable)) in_ch = out_ch if model is not None: model['{}_w_{}'.format(scope, ii)] = w[ii] model['{}_b_{}'.format(scope, ii)] = b[ii] copy = [0] def run_dcnn(x, skip=None): """Run DCNN on an input. Args: x: input image, [B, H, W, D] skip: skip connection activation map, list of 4-D tensor """ with tf.variable_scope(scope): h = [None] * nlayers out_shape = [None] * nlayers batch = tf.shape(x)[0: 1] inp_size = tf.shape(x)[1: 3] cum_pool = 1 for ii in xrange(nlayers): with tf.variable_scope('layer_{}'.format(ii)): cum_pool *= pool[ii] out_ch = ch[ii + 1] if ii == 0: prev_inp = x else: prev_inp = h[ii - 1] if skip is not None: if skip[ii] is not None: if ii == 0: prev_inp = tf.concat(3, [prev_inp, skip[ii]]) else: prev_inp = tf.concat(3, [prev_inp, skip[ii]]) out_shape[ii] = tf.concat( 0, [batch, inp_size * cum_pool, tf.constant([out_ch])]) h[ii] = tf.nn.conv2d_transpose( prev_inp, w[ii], out_shape[ii], strides=[1, pool[ii], pool[ii], 1]) + b[ii] if use_bn[ii]: if frozen is not None and frozen[ii]: bn_frozen = True else: bn_frozen = False if init_weights is not None and \ init_weights[ii] is not None: init_beta = init_weights[ii][ 'beta_{}'.format(copy[0])] init_gamma = init_weights[ii][ 'gamma_{}'.format(copy[0])] else: init_beta = None init_gamma = None # with tf.variable_scope('layer_{}'.format(ii)): # with tf.variable_scope('copy_{}'.format(copy[0])): h[ii] = batch_norm( h[ii], out_ch, phase_train, scope2='{}_{}_{}'.format(scope, ii, copy[0]), init_beta=init_beta, init_gamma=init_gamma, model=model) if act[ii] is not None: h[ii] = act[ii](h[ii]) copy[0] += 1 return h return run_dcnn
def cnn(f, ch, pool, act, use_bn, phase_train=None, wd=None, scope='cnn', model=None, init_weights=None, frozen=None, shared_weights=None): """Add CNN. N = number of layers. Args: f: filter size, list of N int ch: number of channels, list of (N + 1) int pool: pooling ratio, list of N int act: activation function, list of N function use_bn: whether to use batch normalization, list of N bool phase_train: whether in training phase, tf bool variable wd: weight decay Returns: run_cnn: a function that runs the CNN """ log = logger.get() nlayers = len(f) w = [None] * nlayers b = [None] * nlayers log.info('CNN: {}'.format(scope)) log.info('Channels: {}'.format(ch)) log.info('Activation: {}'.format(act)) log.info('Pool: {}'.format(pool)) log.info('BN: {}'.format(use_bn)) log.info('Shared weights: {}'.format(shared_weights)) net_scope = None layer_scope = [None] * nlayers with tf.variable_scope(scope): for ii in xrange(nlayers): with tf.variable_scope('layer_{}'.format(ii)): if init_weights: init = tf.constant_initializer else: init = None if init_weights is not None and init_weights[ii] is not None: init_val_w = init_weights[ii]['w'] init_val_b = init_weights[ii]['b'] else: init_val_w = None init_val_b = None if frozen is not None and frozen[ii]: trainable = False else: trainable = True if shared_weights: w[ii] = shared_weights[ii]['w'] b[ii] = shared_weights[ii]['b'] else: w[ii] = weight_variable([f[ii], f[ii], ch[ii], ch[ii + 1]], name='w', init_val=init_val_w, wd=wd, trainable=trainable) b[ii] = weight_variable([ch[ii + 1]], init_val=init_val_b, name='b', trainable=trainable) log.info('Filter: {}, Trainable: {}'.format( [f[ii], f[ii], ch[ii], ch[ii + 1]], trainable)) if model is not None: for name, param in zip(['w', 'b'], [w[ii], b[ii]]): key = '{}_{}_{}'.format(scope, name, ii) if key in model: raise Exception('Key exists: {}'.format(key)) model[key] = param copy = [0] def run_cnn(x): """ Run CNN on an input. Args: x: input image, [B, H, W, D] """ h = [None] * nlayers with tf.variable_scope(scope): for ii in xrange(nlayers): with tf.variable_scope('layer_{}'.format(ii)): out_ch = ch[ii + 1] if ii == 0: prev_inp = x else: prev_inp = h[ii - 1] h[ii] = conv2d(prev_inp, w[ii]) + b[ii] if use_bn[ii]: if frozen is not None and frozen[ii]: bn_frozen = True else: bn_frozen = False if init_weights is not None and \ init_weights[ii] is not None: init_beta = init_weights[ii][ 'beta_{}'.format(copy[0])] init_gamma = init_weights[ii][ 'gamma_{}'.format(copy[0])] else: init_beta = None init_gamma = None # with tf.variable_scope('layer_{}'.format(ii)): # with tf.variable_scope('copy_{}'.format(copy[0])): h[ii] = batch_norm( h[ii], out_ch, phase_train, scope2='{}_{}_{}'.format(scope, ii, copy[0]), init_beta=init_beta, init_gamma=init_gamma, model=model) if act[ii] is not None: h[ii] = act[ii](h[ii]) if pool[ii] > 1: h[ii] = max_pool(h[ii], pool[ii]) copy[0] += 1 return h return run_cnn
def mlp(dims, act, add_bias=True, dropout_keep=None, phase_train=None, wd=None, scope='mlp', model=None, init_weights=None, frozen=None): """Add MLP. N = number of layers. Args: dims: layer-wise dimensions, list of N int act: activation function, list of N function dropout_keep: keep prob of dropout, list of N float phase_train: whether in training phase, tf bool variable wd: weight decay """ log = logger.get() nlayers = len(dims) - 1 w = [None] * nlayers b = [None] * nlayers log.info('MLP: {}'.format(scope)) log.info('Dimensions: {}'.format(dims)) log.info('Activation: {}'.format(act)) log.info('Dropout: {}'.format(dropout_keep)) log.info('Add bias: {}'.format(add_bias)) with tf.variable_scope(scope): for ii in xrange(nlayers): with tf.variable_scope('layer_{}'.format(ii)): nin = dims[ii] nout = dims[ii + 1] if init_weights is not None and init_weights[ii] is not None: init_val_w = init_weights[ii]['w'] init_val_b = init_weights[ii]['b'] else: init_val_w = None init_val_b = None if frozen is not None and frozen[ii]: trainable = False else: trainable = True w[ii] = weight_variable([nin, nout], init_val=init_val_w, wd=wd, name='w', trainable=trainable) log.info('Weights: {} Trainable: {}'.format([nin, nout], trainable)) if add_bias: b[ii] = weight_variable([nout], init_val=init_val_b, name='b', trainable=trainable) log.info('Bias: {} Trainable: {}'.format([nout], trainable)) if model is not None: model['{}_w_{}'.format(scope, ii)] = w[ii] if add_bias: model['{}_b_{}'.format(scope, ii)] = b[ii] def run_mlp(x): h = [None] * nlayers with tf.variable_scope(scope): for ii in xrange(nlayers): with tf.variable_scope('layer_{}'.format(ii)): if ii == 0: prev_inp = x else: prev_inp = h[ii - 1] if dropout_keep is not None: if dropout_keep[ii] is not None: prev_inp = dropout(prev_inp, dropout_keep[ii], phase_train) h[ii] = tf.matmul(prev_inp, w[ii]) if add_bias: h[ii] += b[ii] if act[ii]: h[ii] = act[ii](h[ii]) return h return run_mlp
def __init__(self): super(SaverRunner, self).__init__() self._log = logger.get() self._step = 0 self.register_option('save_ckpt') pass
def __init__(self, folder=None): super(RestorerRunner, self).__init__() self._log = logger.get() self.folder = folder pass
def lstm(inp_dim, hid_dim, wd=None, scope='lstm', model=None, init_weights=None, frozen=False): """Adds an LSTM component. Args: inp_dim: Input data dim hid_dim: Hidden state dim wd: Weight decay scope: Prefix """ log = logger.get() log.info('LSTM: {}'.format(scope)) log.info('Input dim: {}'.format(inp_dim)) log.info('Hidden dim: {}'.format(hid_dim)) if init_weights is None: init_weights = {} for w in [ 'w_xi', 'w_hi', 'b_i', 'w_xf', 'w_hf', 'b_f', 'w_xu', 'w_hu', 'b_u', 'w_xo', 'w_ho', 'b_o' ]: init_weights[w] = None trainable = not frozen log.info('Trainable: {}'.format(trainable)) with tf.variable_scope(scope): # Input gate w_xi = weight_variable([inp_dim, hid_dim], init_val=init_weights['w_xi'], wd=wd, name='w_xi', trainable=trainable) w_hi = weight_variable([hid_dim, hid_dim], init_val=init_weights['w_hi'], wd=wd, name='w_hi', trainable=trainable) b_i = weight_variable([hid_dim], init_val=init_weights['b_i'], initializer=tf.constant_initializer(0.0), name='b_i', trainable=trainable) # Forget gate w_xf = weight_variable([inp_dim, hid_dim], init_val=init_weights['w_xf'], wd=wd, name='w_xf', trainable=trainable) w_hf = weight_variable([hid_dim, hid_dim], init_val=init_weights['w_hf'], wd=wd, name='w_hf', trainable=trainable) b_f = weight_variable([hid_dim], init_val=init_weights['b_f'], initializer=tf.constant_initializer(1.0), name='b_f', trainable=trainable) # Input activation w_xu = weight_variable([inp_dim, hid_dim], init_val=init_weights['w_xu'], wd=wd, name='w_xu', trainable=trainable) w_hu = weight_variable([hid_dim, hid_dim], init_val=init_weights['w_hu'], wd=wd, name='w_hu', trainable=trainable) b_u = weight_variable([hid_dim], init_val=init_weights['b_u'], initializer=tf.constant_initializer(0.0), name='b_u', trainable=trainable) # Output gate w_xo = weight_variable([inp_dim, hid_dim], init_val=init_weights['w_xo'], wd=wd, name='w_xo', trainable=trainable) w_ho = weight_variable([hid_dim, hid_dim], init_val=init_weights['w_ho'], wd=wd, name='w_ho', trainable=trainable) b_o = weight_variable([hid_dim], init_val=init_weights['b_o'], initializer=tf.constant_initializer(0.0), name='b_o', trainable=trainable) if model is not None: model['{}_w_xi'.format(scope)] = w_xi model['{}_w_hi'.format(scope)] = w_hi model['{}_b_i'.format(scope)] = b_i model['{}_w_xf'.format(scope)] = w_xf model['{}_w_hf'.format(scope)] = w_hf model['{}_b_f'.format(scope)] = b_f model['{}_w_xu'.format(scope)] = w_xu model['{}_w_hu'.format(scope)] = w_hu model['{}_b_u'.format(scope)] = b_u model['{}_w_xo'.format(scope)] = w_xo model['{}_w_ho'.format(scope)] = w_ho model['{}_b_o'.format(scope)] = b_o model['{}_w_x_mean'.format(scope)] = ( tf.reduce_sum(tf.abs(w_xi)) + tf.reduce_sum(tf.abs(w_xf)) + tf.reduce_sum(tf.abs(w_xu)) + tf.reduce_sum(tf.abs(w_xo))) / inp_dim / hid_dim / 4 model['{}_w_h_mean'.format(scope)] = ( tf.reduce_sum(tf.abs(w_hi)) + tf.reduce_sum(tf.abs(w_hf)) + tf.reduce_sum(tf.abs(w_hu)) + tf.reduce_sum(tf.abs(w_ho))) / hid_dim / hid_dim / 4 model['{}_b_mean'.format(scope)] = (tf.reduce_sum( tf.abs(b_i)) + tf.reduce_sum(tf.abs(b_f)) + tf.reduce_sum( tf.abs(b_u)) + tf.reduce_sum(tf.abs(b_o))) / hid_dim / 4 def unroll(inp, state): with tf.variable_scope(scope): c = tf.slice(state, [0, 0], [-1, hid_dim]) h = tf.slice(state, [0, hid_dim], [-1, hid_dim]) g_i = tf.sigmoid(tf.matmul(inp, w_xi) + tf.matmul(h, w_hi) + b_i) g_f = tf.sigmoid(tf.matmul(inp, w_xf) + tf.matmul(h, w_hf) + b_f) g_o = tf.sigmoid(tf.matmul(inp, w_xo) + tf.matmul(h, w_ho) + b_o) u = tf.tanh(tf.matmul(inp, w_xu) + tf.matmul(h, w_hu) + b_u) c = g_f * c + g_i * u h = g_o * tf.tanh(c) state = tf.concat(1, [c, h]) return state, g_i, g_f, g_o return unroll
def conv_lstm(inp_depth, hid_depth, filter_size, wd=None, scope='conv_lstm', model=None, init_weights=None, frozen=False): """Adds a Conv-LSTM component. Args: inp_depth: Input image depth filter_size: Conv gate filter size hid_depth: Hidden state depth wd: Weight decay name: Prefix """ log = logger.get() log.info('ConvLSTM: {}'.format(scope)) log.info('Input depth: {}'.format(inp_depth)) log.info('Hidden depth: {}'.format(hid_depth)) if init_weights is None: init_weights = {} for w in [ 'w_xi', 'w_hi', 'b_i', 'w_xf', 'w_hf', 'b_f', 'w_xu', 'w_hu', 'b_u', 'w_xo', 'w_ho', 'b_o' ]: init_weights[w] = None trainable = not frozen log.info('Trainable: {}'.format(trainable)) with tf.variable_scope(scope): # Input gate w_xi = weight_variable( [filter_size, filter_size, inp_depth, hid_depth], init_val=init_weights['w_xi'], wd=wd, name='w_xi', trainable=trainable) w_hi = weight_variable( [filter_size, filter_size, hid_depth, hid_depth], init_val=init_weights['w_hi'], wd=wd, name='w_hi', trainable=trainable) b_i = weight_variable([hid_depth], init_val=init_weights['b_i'], name='b_i', trainable=trainable) # Forget gate w_xf = weight_variable( [filter_size, filter_size, inp_depth, hid_depth], wd=wd, init_val=init_weights['w_xf'], name='w_xf', trainable=trainable) w_hf = weight_variable( [filter_size, filter_size, hid_depth, hid_depth], init_val=init_weights['w_hf'], wd=wd, name='w_hf', trainable=trainable) b_f = weight_variable([hid_depth], init_val=init_weights['b_f'], name='b_f', trainable=trainable) # Input activation w_xu = weight_variable( [filter_size, filter_size, inp_depth, hid_depth], init_val=init_weights['w_xu'], wd=wd, name='w_xu', trainable=trainable) w_hu = weight_variable( [filter_size, filter_size, hid_depth, hid_depth], init_val=init_weights['w_hu'], wd=wd, name='w_hu', trainable=trainable) b_u = weight_variable([hid_depth], init_val=init_weights['b_u'], name='b_u', trainable=trainable) # Output gate w_xo = weight_variable( [filter_size, filter_size, inp_depth, hid_depth], init_val=init_weights['w_xo'], wd=wd, name='w_xo', trainable=trainable) w_ho = weight_variable( [filter_size, filter_size, hid_depth, hid_depth], init_val=init_weights['w_ho'], wd=wd, name='w_ho', trainable=trainable) b_o = weight_variable([hid_depth], init_val=init_weights['b_o'], name='b_o', trainable=trainable) if model is not None: model['{}_w_xi'.format(scope)] = w_xi model['{}_w_hi'.format(scope)] = w_hi model['{}_b_i'.format(scope)] = b_i model['{}_w_xf'.format(scope)] = w_xf model['{}_w_hf'.format(scope)] = w_hf model['{}_b_f'.format(scope)] = b_f model['{}_w_xu'.format(scope)] = w_xu model['{}_w_hu'.format(scope)] = w_hu model['{}_b_u'.format(scope)] = b_u model['{}_w_xo'.format(scope)] = w_xo model['{}_w_ho'.format(scope)] = w_ho model['{}_b_o'.format(scope)] = b_o def unroll(inp, state): with tf.variable_scope(scope): c = tf.slice(state, [0, 0, 0, 0], [-1, -1, -1, hid_depth]) h = tf.slice(state, [0, 0, 0, hid_depth], [-1, -1, -1, hid_depth]) g_i = tf.sigmoid(conv2d(inp, w_xi) + conv2d(h, w_hi) + b_i) g_f = tf.sigmoid(conv2d(inp, w_xf) + conv2d(h, w_hf) + b_f) g_o = tf.sigmoid(conv2d(inp, w_xo) + conv2d(h, w_ho) + b_o) u = tf.tanh(conv2d(inp, w_xu) + conv2d(h, w_hu) + b_u) c = g_f * c + g_i * u h = g_o * tf.tanh(c) state = tf.concat(3, [c, h]) return state return unroll
def conv_lstm(inp_depth, hid_depth, filter_size, wd=None, scope='conv_lstm', model=None, init_weights=None, frozen=False): """Adds a Conv-LSTM component. Args: inp_depth: Input image depth filter_size: Conv gate filter size hid_depth: Hidden state depth wd: Weight decay name: Prefix """ log = logger.get() log.info('ConvLSTM: {}'.format(scope)) log.info('Input depth: {}'.format(inp_depth)) log.info('Hidden depth: {}'.format(hid_depth)) if init_weights is None: init_weights = {} for w in ['w_xi', 'w_hi', 'b_i', 'w_xf', 'w_hf', 'b_f', 'w_xu', 'w_hu', 'b_u', 'w_xo', 'w_ho', 'b_o']: init_weights[w] = None trainable = not frozen log.info('Trainable: {}'.format(trainable)) with tf.variable_scope(scope): # Input gate w_xi = weight_variable([filter_size, filter_size, inp_depth, hid_depth], init_val=init_weights['w_xi'], wd=wd, name='w_xi', trainable=trainable) w_hi = weight_variable([filter_size, filter_size, hid_depth, hid_depth], init_val=init_weights['w_hi'], wd=wd, name='w_hi', trainable=trainable) b_i = weight_variable([hid_depth], init_val=init_weights['b_i'], name='b_i', trainable=trainable) # Forget gate w_xf = weight_variable([filter_size, filter_size, inp_depth, hid_depth], wd=wd, init_val=init_weights['w_xf'], name='w_xf', trainable=trainable) w_hf = weight_variable([filter_size, filter_size, hid_depth, hid_depth], init_val=init_weights['w_hf'], wd=wd, name='w_hf', trainable=trainable) b_f = weight_variable([hid_depth], init_val=init_weights['b_f'], name='b_f', trainable=trainable) # Input activation w_xu = weight_variable([filter_size, filter_size, inp_depth, hid_depth], init_val=init_weights['w_xu'], wd=wd, name='w_xu', trainable=trainable) w_hu = weight_variable([filter_size, filter_size, hid_depth, hid_depth], init_val=init_weights['w_hu'], wd=wd, name='w_hu', trainable=trainable) b_u = weight_variable([hid_depth], init_val=init_weights['b_u'], name='b_u', trainable=trainable) # Output gate w_xo = weight_variable([filter_size, filter_size, inp_depth, hid_depth], init_val=init_weights['w_xo'], wd=wd, name='w_xo', trainable=trainable) w_ho = weight_variable([filter_size, filter_size, hid_depth, hid_depth], init_val=init_weights['w_ho'], wd=wd, name='w_ho', trainable=trainable) b_o = weight_variable([hid_depth], init_val=init_weights['b_o'], name='b_o', trainable=trainable) if model is not None: model['{}_w_xi'.format(scope)] = w_xi model['{}_w_hi'.format(scope)] = w_hi model['{}_b_i'.format(scope)] = b_i model['{}_w_xf'.format(scope)] = w_xf model['{}_w_hf'.format(scope)] = w_hf model['{}_b_f'.format(scope)] = b_f model['{}_w_xu'.format(scope)] = w_xu model['{}_w_hu'.format(scope)] = w_hu model['{}_b_u'.format(scope)] = b_u model['{}_w_xo'.format(scope)] = w_xo model['{}_w_ho'.format(scope)] = w_ho model['{}_b_o'.format(scope)] = b_o def unroll(inp, state): with tf.variable_scope(scope): c = tf.slice(state, [0, 0, 0, 0], [-1, -1, -1, hid_depth]) h = tf.slice(state, [0, 0, 0, hid_depth], [-1, -1, -1, hid_depth]) g_i = tf.sigmoid(conv2d(inp, w_xi) + conv2d(h, w_hi) + b_i) g_f = tf.sigmoid(conv2d(inp, w_xf) + conv2d(h, w_hf) + b_f) g_o = tf.sigmoid(conv2d(inp, w_xo) + conv2d(h, w_ho) + b_o) u = tf.tanh(conv2d(inp, w_xu) + conv2d(h, w_hu) + b_u) c = g_f * c + g_i * u h = g_o * tf.tanh(c) state = tf.concat(3, [c, h]) return state return unroll
def cnn(f, ch, pool, act, use_bn, phase_train=None, wd=None, scope='cnn', model=None, init_weights=None, frozen=None, shared_weights=None): """Add CNN. N = number of layers. Args: f: filter size, list of N int ch: number of channels, list of (N + 1) int pool: pooling ratio, list of N int act: activation function, list of N function use_bn: whether to use batch normalization, list of N bool phase_train: whether in training phase, tf bool variable wd: weight decay Returns: run_cnn: a function that runs the CNN """ log = logger.get() nlayers = len(f) w = [None] * nlayers b = [None] * nlayers log.info('CNN: {}'.format(scope)) log.info('Channels: {}'.format(ch)) log.info('Activation: {}'.format(act)) log.info('Pool: {}'.format(pool)) log.info('BN: {}'.format(use_bn)) log.info('Shared weights: {}'.format(shared_weights)) net_scope = None layer_scope = [None] * nlayers with tf.variable_scope(scope): for ii in xrange(nlayers): with tf.variable_scope('layer_{}'.format(ii)): if init_weights: init = tf.constant_initializer else: init = None if init_weights is not None and init_weights[ii] is not None: init_val_w = init_weights[ii]['w'] init_val_b = init_weights[ii]['b'] else: init_val_w = None init_val_b = None if frozen is not None and frozen[ii]: trainable = False else: trainable = True if shared_weights: w[ii] = shared_weights[ii]['w'] b[ii] = shared_weights[ii]['b'] else: w[ii] = weight_variable([f[ii], f[ii], ch[ii], ch[ii + 1]], name='w', init_val=init_val_w, wd=wd, trainable=trainable) b[ii] = weight_variable([ch[ii + 1]], init_val=init_val_b, name='b', trainable=trainable) log.info('Filter: {}, Trainable: {}'.format( [f[ii], f[ii], ch[ii], ch[ii + 1]], trainable)) if model is not None: for name, param in zip(['w', 'b'], [w[ii], b[ii]]): key = '{}_{}_{}'.format(scope, name, ii) if key in model: raise Exception('Key exists: {}'.format(key)) model[key] = param copy = [0] def run_cnn(x): """ Run CNN on an input. Args: x: input image, [B, H, W, D] """ h = [None] * nlayers with tf.variable_scope(scope): for ii in xrange(nlayers): with tf.variable_scope('layer_{}'.format(ii)): out_ch = ch[ii + 1] if ii == 0: prev_inp = x else: prev_inp = h[ii - 1] h[ii] = conv2d(prev_inp, w[ii]) + b[ii] if use_bn[ii]: if frozen is not None and frozen[ii]: bn_frozen = True else: bn_frozen = False if init_weights is not None and \ init_weights[ii] is not None: init_beta = init_weights[ii]['beta_{}'.format( copy[0])] init_gamma = init_weights[ii]['gamma_{}'.format( copy[0])] else: init_beta = None init_gamma = None # with tf.variable_scope('layer_{}'.format(ii)): # with tf.variable_scope('copy_{}'.format(copy[0])): h[ii] = batch_norm(h[ii], out_ch, phase_train, scope2='{}_{}_{}'.format( scope, ii, copy[0]), init_beta=init_beta, init_gamma=init_gamma, model=model) if act[ii] is not None: h[ii] = act[ii](h[ii]) if pool[ii] > 1: h[ii] = max_pool(h[ii], pool[ii]) copy[0] += 1 return h return run_cnn
def lstm(inp_dim, hid_dim, wd=None, scope='lstm', model=None, init_weights=None, frozen=False): """Adds an LSTM component. Args: inp_dim: Input data dim hid_dim: Hidden state dim wd: Weight decay scope: Prefix """ log = logger.get() log.info('LSTM: {}'.format(scope)) log.info('Input dim: {}'.format(inp_dim)) log.info('Hidden dim: {}'.format(hid_dim)) if init_weights is None: init_weights = {} for w in ['w_xi', 'w_hi', 'b_i', 'w_xf', 'w_hf', 'b_f', 'w_xu', 'w_hu', 'b_u', 'w_xo', 'w_ho', 'b_o']: init_weights[w] = None trainable = not frozen log.info('Trainable: {}'.format(trainable)) with tf.variable_scope(scope): # Input gate w_xi = weight_variable( [inp_dim, hid_dim], init_val=init_weights['w_xi'], wd=wd, name='w_xi', trainable=trainable) w_hi = weight_variable( [hid_dim, hid_dim], init_val=init_weights['w_hi'], wd=wd, name='w_hi', trainable=trainable) b_i = weight_variable( [hid_dim], init_val=init_weights['b_i'], initializer=tf.constant_initializer(0.0), name='b_i', trainable=trainable) # Forget gate w_xf = weight_variable( [inp_dim, hid_dim], init_val=init_weights['w_xf'], wd=wd, name='w_xf', trainable=trainable) w_hf = weight_variable( [hid_dim, hid_dim], init_val=init_weights['w_hf'], wd=wd, name='w_hf', trainable=trainable) b_f = weight_variable( [hid_dim], init_val=init_weights['b_f'], initializer=tf.constant_initializer(1.0), name='b_f', trainable=trainable) # Input activation w_xu = weight_variable( [inp_dim, hid_dim], init_val=init_weights['w_xu'], wd=wd, name='w_xu', trainable=trainable) w_hu = weight_variable( [hid_dim, hid_dim], init_val=init_weights['w_hu'], wd=wd, name='w_hu', trainable=trainable) b_u = weight_variable( [hid_dim], init_val=init_weights['b_u'], initializer=tf.constant_initializer(0.0), name='b_u', trainable=trainable) # Output gate w_xo = weight_variable( [inp_dim, hid_dim], init_val=init_weights['w_xo'], wd=wd, name='w_xo', trainable=trainable) w_ho = weight_variable( [hid_dim, hid_dim], init_val=init_weights['w_ho'], wd=wd, name='w_ho', trainable=trainable) b_o = weight_variable( [hid_dim], init_val=init_weights['b_o'], initializer=tf.constant_initializer(0.0), name='b_o', trainable=trainable) if model is not None: model['{}_w_xi'.format(scope)] = w_xi model['{}_w_hi'.format(scope)] = w_hi model['{}_b_i'.format(scope)] = b_i model['{}_w_xf'.format(scope)] = w_xf model['{}_w_hf'.format(scope)] = w_hf model['{}_b_f'.format(scope)] = b_f model['{}_w_xu'.format(scope)] = w_xu model['{}_w_hu'.format(scope)] = w_hu model['{}_b_u'.format(scope)] = b_u model['{}_w_xo'.format(scope)] = w_xo model['{}_w_ho'.format(scope)] = w_ho model['{}_b_o'.format(scope)] = b_o model['{}_w_x_mean'.format(scope)] = (tf.reduce_sum( tf.abs(w_xi)) + tf.reduce_sum(tf.abs(w_xf)) + tf.reduce_sum(tf.abs(w_xu)) + tf.reduce_sum(tf.abs(w_xo))) / inp_dim / hid_dim / 4 model['{}_w_h_mean'.format(scope)] = (tf.reduce_sum( tf.abs(w_hi)) + tf.reduce_sum(tf.abs(w_hf)) + tf.reduce_sum(tf.abs(w_hu)) + tf.reduce_sum(tf.abs(w_ho))) / hid_dim / hid_dim / 4 model['{}_b_mean'.format(scope)] = (tf.reduce_sum( tf.abs(b_i)) + tf.reduce_sum(tf.abs(b_f)) + tf.reduce_sum(tf.abs(b_u)) + tf.reduce_sum(tf.abs(b_o))) / hid_dim / 4 def unroll(inp, state): with tf.variable_scope(scope): c = tf.slice(state, [0, 0], [-1, hid_dim]) h = tf.slice(state, [0, hid_dim], [-1, hid_dim]) g_i = tf.sigmoid(tf.matmul(inp, w_xi) + tf.matmul(h, w_hi) + b_i) g_f = tf.sigmoid(tf.matmul(inp, w_xf) + tf.matmul(h, w_hf) + b_f) g_o = tf.sigmoid(tf.matmul(inp, w_xo) + tf.matmul(h, w_ho) + b_o) u = tf.tanh(tf.matmul(inp, w_xu) + tf.matmul(h, w_hu) + b_u) c = g_f * c + g_i * u h = g_o * tf.tanh(c) state = tf.concat(1, [c, h]) return state, g_i, g_f, g_o return unroll
def __init__(self): self.log = logger.get() self._var_dict = {} self._has_init_var = False self._variable_sharing = False pass
def res_cnn(f, ch, strides, phase_train, wd=None, scope='res_cnn'): """ Args: f: Filter size for every basic block. ch: Channel size for every basic block. pool: Pool ratio for every basic block. """ log = logger.get() nlayers = len(f) w = [None] * nlayers b = [None] * nlayers log.info('Residual CNN: {}'.format(scope)) log.info('Channels: {}'.format(ch)) log.info('Strides: {}'.format(strides)) with tf.variable_scope(scope): for ii in xrange(nlayers): with tf.variable_scope('layer_{}'.format(ii)): w[ii] = [None] * 2 b[ii] = [None] * 2 # init = tf.truncated_normal_initializer(stddev=0.01) log.info('Filter: {}'.format( [f[ii], f[ii], ch[ii], ch[ii + 1]])) w[ii][0] = weight_variable([f[ii], f[ii], ch[ii], ch[ii + 1]], name='w_0', wd=wd) b[ii][0] = weight_variable([ch[ii + 1]], name='b_0', wd=wd) log.info('Filter: {}'.format( [f[ii], f[ii], ch[ii + 1], ch[ii + 1]])) w[ii][1] = weight_variable( [f[ii], f[ii], ch[ii + 1], ch[ii + 1]], name='w_1', wd=wd) b[ii][1] = weight_variable([ch[ii + 1]], name='b_1', wd=wd) def shortcut(x, prev_x, stride, in_ch, out_ch): if stride > 1: prev_x = avg_pool(prev_x, stride) if in_ch != out_ch: xshape = tf.shape(x) zeros = tf.zeros( tf.pack([xshape[0], xshape[1], xshape[2], out_ch - in_ch])) prev_x = tf.concat(3, [prev_x, zeros]) return prev_x + x def run_res_cnn(x): h = [None] * nlayers with tf.variable_scope(scope): for ii in xrange(nlayers): with tf.variable_scope('layer_{}'.format(ii)): if ii == 0: prev_inp = x else: prev_inp = h[ii - 1] h_tmp = conv2d(prev_inp, w[ii][0], strides[ii]) + b[ii][0] h_tmp = batch_norm(h_tmp, ch[ii + 1], phase_train) h_tmp = tf.nn.relu(h_tmp) h_tmp = conv2d(h_tmp, w[ii][1]) + b[ii][1] h_tmp = batch_norm(h_tmp, ch[ii + 1], phase_train) if ii > 0: h[ii] = shortcut(h_tmp, h[ii - 1], strides[ii], ch[ii], ch[ii + 1]) else: h[ii] = h_tmp log.info(h[ii].get_shape()) pass return h return run_res_cnn
def res_cnn(f, ch, strides, phase_train, wd=None, scope='res_cnn'): """ Args: f: Filter size for every basic block. ch: Channel size for every basic block. pool: Pool ratio for every basic block. """ log = logger.get() nlayers = len(f) w = [None] * nlayers b = [None] * nlayers log.info('Residual CNN: {}'.format(scope)) log.info('Channels: {}'.format(ch)) log.info('Strides: {}'.format(strides)) with tf.variable_scope(scope): for ii in xrange(nlayers): with tf.variable_scope('layer_{}'.format(ii)): w[ii] = [None] * 2 b[ii] = [None] * 2 # init = tf.truncated_normal_initializer(stddev=0.01) log.info('Filter: {}'.format( [f[ii], f[ii], ch[ii], ch[ii + 1]])) w[ii][0] = weight_variable( [f[ii], f[ii], ch[ii], ch[ii + 1]], name='w_0', wd=wd) b[ii][0] = weight_variable([ch[ii + 1]], name='b_0', wd=wd) log.info('Filter: {}'.format( [f[ii], f[ii], ch[ii + 1], ch[ii + 1]])) w[ii][1] = weight_variable( [f[ii], f[ii], ch[ii + 1], ch[ii + 1]], name='w_1', wd=wd) b[ii][1] = weight_variable([ch[ii + 1]], name='b_1', wd=wd) def shortcut(x, prev_x, stride, in_ch, out_ch): if stride > 1: prev_x = avg_pool(prev_x, stride) if in_ch != out_ch: xshape = tf.shape(x) zeros = tf.zeros(tf.pack([xshape[0], xshape[1], xshape[2], out_ch - in_ch])) prev_x = tf.concat(3, [prev_x, zeros]) return prev_x + x def run_res_cnn(x): h = [None] * nlayers with tf.variable_scope(scope): for ii in xrange(nlayers): with tf.variable_scope('layer_{}'.format(ii)): if ii == 0: prev_inp = x else: prev_inp = h[ii - 1] h_tmp = conv2d(prev_inp, w[ii][0], strides[ii]) + b[ii][0] h_tmp = batch_norm(h_tmp, ch[ii + 1], phase_train) h_tmp = tf.nn.relu(h_tmp) h_tmp = conv2d(h_tmp, w[ii][1]) + b[ii][1] h_tmp = batch_norm(h_tmp, ch[ii + 1], phase_train) if ii > 0: h[ii] = shortcut( h_tmp, h[ii - 1], strides[ii], ch[ii], ch[ii + 1]) else: h[ii] = h_tmp log.info(h[ii].get_shape()) pass return h return run_res_cnn
def dcnn(f, ch, pool, act, use_bn, skip_ch=None, phase_train=None, wd=None, scope='dcnn', model=None, init_weights=None, frozen=None): """Add DCNN. N = number of layers. Args: f: filter size, list of size N int ch: number of channels, list of (N + 1) int pool: pooling ratio, list of N int act: activation function, list of N function use_bn: whether to use batch normalization, list of N bool skip_ch: skip connection, list of N int phase_train: whether in training phase, tf bool variable wd: weight decay Returns: run_dcnn: a function that runs the DCNN """ log = logger.get() nlayers = len(f) w = [None] * nlayers b = [None] * nlayers bn = [None] * nlayers log.info('DCNN: {}'.format(scope)) log.info('Channels: {}'.format(ch)) log.info('Activation: {}'.format(act)) log.info('Unpool: {}'.format(pool)) log.info('Skip channels: {}'.format(skip_ch)) log.info('BN: {}'.format(use_bn)) with tf.variable_scope(scope): in_ch = ch[0] for ii in xrange(nlayers): with tf.variable_scope('layer_{}'.format(ii)): out_ch = ch[ii + 1] if skip_ch is not None: if skip_ch[ii] is not None: in_ch += skip_ch[ii] if init_weights is not None and init_weights[ii] is not None: init_val_w = init_weights[ii]['w'] init_val_b = init_weights[ii]['b'] else: init_val_w = None init_val_b = None if frozen is not None and frozen[ii]: trainable = False else: trainable = True w[ii] = weight_variable([f[ii], f[ii], out_ch, in_ch], name='w', init_val=init_val_w, wd=wd, trainable=trainable) b[ii] = weight_variable([out_ch], init_val=init_val_b, name='b', trainable=trainable) log.info('Filter: {}, Trainable: {}'.format( [f[ii], f[ii], out_ch, in_ch], trainable)) in_ch = out_ch if model is not None: model['{}_w_{}'.format(scope, ii)] = w[ii] model['{}_b_{}'.format(scope, ii)] = b[ii] copy = [0] def run_dcnn(x, skip=None): """Run DCNN on an input. Args: x: input image, [B, H, W, D] skip: skip connection activation map, list of 4-D tensor """ with tf.variable_scope(scope): h = [None] * nlayers out_shape = [None] * nlayers batch = tf.shape(x)[0:1] inp_size = tf.shape(x)[1:3] cum_pool = 1 for ii in xrange(nlayers): with tf.variable_scope('layer_{}'.format(ii)): cum_pool *= pool[ii] out_ch = ch[ii + 1] if ii == 0: prev_inp = x else: prev_inp = h[ii - 1] if skip is not None: if skip[ii] is not None: if ii == 0: prev_inp = tf.concat(3, [prev_inp, skip[ii]]) else: prev_inp = tf.concat(3, [prev_inp, skip[ii]]) out_shape[ii] = tf.concat( 0, [batch, inp_size * cum_pool, tf.constant([out_ch])]) h[ii] = tf.nn.conv2d_transpose( prev_inp, w[ii], out_shape[ii], strides=[1, pool[ii], pool[ii], 1]) + b[ii] if use_bn[ii]: if frozen is not None and frozen[ii]: bn_frozen = True else: bn_frozen = False if init_weights is not None and \ init_weights[ii] is not None: init_beta = init_weights[ii]['beta_{}'.format( copy[0])] init_gamma = init_weights[ii]['gamma_{}'.format( copy[0])] else: init_beta = None init_gamma = None # with tf.variable_scope('layer_{}'.format(ii)): # with tf.variable_scope('copy_{}'.format(copy[0])): h[ii] = batch_norm(h[ii], out_ch, phase_train, scope2='{}_{}_{}'.format( scope, ii, copy[0]), init_beta=init_beta, init_gamma=init_gamma, model=model) if act[ii] is not None: h[ii] = act[ii](h[ii]) copy[0] += 1 return h return run_dcnn
import cslab_environ from tensorflow.python.framework import ops import tfplus.utils.logger as logger import numpy as np import tensorflow as tf hungarian_module = None log = logger.get() # Register gradient for Hungarian algorithm. ops.NoGradient("Hungarian") def get_device_fn(device): """Choose device for different ops.""" OPS_ON_CPU = set(['ResizeBilinear', 'ResizeBilinearGrad', 'Mod', 'CumMin', 'CumMinGrad', 'Hungarian', 'Reverse', 'SparseToDense', 'BatchMatMul', 'Gather', 'Print']) def _device_fn(op): if op.type in OPS_ON_CPU: return "/cpu:0" else: # Other ops will be placed on GPU if available, otherwise CPU. return device return _device_fn