def __init__( self, layer=None, n_units=100, act=tf.identity, binarized_weight=tf.identity, W_init=tf.truncated_normal_initializer(stddev=0.1), b_init=tf.constant_initializer(value=0.0), W_init_args={}, b_init_args={}, name='b_dense_layer', ): Layer.__init__(self, name=name) self.inputs = layer.outputs if self.inputs.get_shape().ndims != 2: raise Exception( "The input dimension must be rank 2, please reshape or flatten it" ) n_in = int(self.inputs.get_shape()[-1]) self.n_units = n_units logging.info("Binarized_DenseLayer %s: %d %s" % (self.name, self.n_units, act.__name__)) with tf.variable_scope(name) as vs: W = tf.get_variable(name='W', shape=(n_in, n_units), initializer=W_init, **W_init_args) bin_w = binarized_weight(W) if b_init is not None: try: b = tf.get_variable(name='b', shape=(n_units), initializer=b_init, **b_init_args) except: # If initializer is a constant, do not specify shape. b = tf.get_variable(name='b', initializer=b_init, **b_init_args) self.outputs = act(tf.matmul(self.inputs, bin_w) + b) else: self.outputs = act(tf.matmul(self.inputs, bin_w)) # Hint : list(), dict() is pass by value (shallow), without them, it is # pass by reference. self.all_layers = list(layer.all_layers) self.all_params = list(layer.all_params) self.all_drop = dict(layer.all_drop) self.all_layers.extend([self.outputs]) if b_init is not None: self.all_params.extend([W, b]) else: self.all_params.extend([W])
def __init__( self, layer=None, shape=(5, 1, 5), stride=1, act=tf.identity, dilation_rate=1, padding='SAME', binarize_weight=tf.identity, W_init=tf.truncated_normal_initializer(stddev=0.02), b_init=None, W_init_args={}, b_init_args={}, use_cudnn_on_gpu=True, data_format='NWC', name='cnn_layer', ): Layer.__init__(self, name=name) self.inputs = layer.outputs logging.info( "BinarizedConvolution1D %s: shape:%s strides:%s pad:%s activation:%s" % (self.name, str(shape), str(stride), padding, act.__name__)) with tf.variable_scope(name) as vs: W = tf.get_variable(name='W_conv1d', shape=shape, initializer=W_init, **W_init_args) bin_w = binarize_weight(W) self.outputs = tf.nn.convolution(self.inputs, bin_w, strides=(stride, ), padding=padding, dilation_rate=(dilation_rate, ), data_format=data_format) if b_init: b = tf.get_variable(name='b_conv1d', shape=(shape[-1]), initializer=b_init, **b_init_args) self.outputs = self.outputs + b self.outputs = act(self.outputs) self.all_layers = list(layer.all_layers) self.all_params = list(layer.all_params) self.all_drop = dict(layer.all_drop) self.all_layers.extend([self.outputs]) if b_init: self.all_params.extend([W, b]) else: self.all_params.extend([W])
def binary_outputs(net, name=None): scope_name = tf.get_variable_scope().name if scope_name: name = scope_name + '/' + name shape = net.outputs.get_shape() logging.info("BinaryOutputs {:}: shape:{:}".format(name, shape)) outputs = binary_value(net.outputs) net_new = copy.copy(net) net_new.outputs = outputs net_new.all_layers.extend([outputs]) return net_new
def global_max_net(net, name=None): scope_name = tf.get_variable_scope().name if scope_name: name = scope_name + '/' + name shape = net.outputs.get_shape() logging.info("GlobalMaxPool {:}: shape:{:}".format(name, shape)) outputs = tf.reduce_max(net.outputs, 1) net_new = copy.copy(net) net_new.outputs = outputs net_new.all_layers.extend([outputs]) return net_new
def __init__( self, prev_layer, init_scale=0.05, name='scale', ): Layer.__init__(self, prev_layer=prev_layer, name=name) self.inputs = prev_layer.outputs logging.info("ScaleLayer %s: init_scale: %f" % (self.name, init_scale)) with tf.variable_scope(name): # scale = tf.get_variable(name='scale_factor', init, trainable=True, ) scale = tf.get_variable( "scale", shape=[1], initializer=tf.constant_initializer(value=init_scale), trainable=False) self.outputs = self.inputs * scale self.all_layers.append(self.outputs) self.all_params.append(scale)
def __init__( self, prev_layer, decay=0.9, epsilon=0.00001, act=tf.identity, is_train=False, beta_init=tf.zeros_initializer, gamma_init=tf.random_normal_initializer(mean=1.0, stddev=0.002), name='batchnorm_layer', ): Layer.__init__(self, prev_layer=prev_layer, name=name) self.inputs = prev_layer.outputs logging.info( "BatchNormLayer %s: decay:%f epsilon:%f act:%s is_train:%s" % (self.name, decay, epsilon, act.__name__, is_train)) x_shape = self.inputs.get_shape() params_shape = x_shape[-1:] from tensorflow.python.training import moving_averages with tf.variable_scope(name): axis = list(range(len(x_shape) - 1)) # 1. beta, gamma variables = [] if beta_init: if tf.__version__ > '0.12.1' and beta_init == tf.zeros_initializer: beta_init = beta_init() beta = tf.get_variable('beta', shape=params_shape, initializer=beta_init, dtype=LayersConfig.tf_dtype, trainable=is_train) variables.append(beta) else: beta = None if gamma_init: gamma = tf.get_variable( 'gamma', shape=params_shape, initializer=gamma_init, dtype=LayersConfig.tf_dtype, trainable=is_train, ) variables.append(gamma) else: gamma = None # 2. if tf.__version__ > '0.12.1': moving_mean_init = tf.zeros_initializer() else: moving_mean_init = tf.zeros_initializer moving_mean = tf.get_variable('moving_mean', params_shape, initializer=moving_mean_init, dtype=LayersConfig.tf_dtype, trainable=False) moving_variance = tf.get_variable( 'moving_variance', params_shape, initializer=tf.constant_initializer(1.), dtype=LayersConfig.tf_dtype, trainable=False, ) # 3. # These ops will only be preformed when training. mean, variance = tf.nn.moments(self.inputs, axis) self.mean = mean self.variance = variance try: # TF12 update_moving_mean = moving_averages.assign_moving_average( moving_mean, mean, decay, zero_debias=False) # if zero_debias=True, has bias update_moving_variance = moving_averages.assign_moving_average( moving_variance, variance, decay, zero_debias=False) # if zero_debias=True, has bias # logging.info("TF12 moving") except Exception: # TF11 update_moving_mean = moving_averages.assign_moving_average( moving_mean, mean, decay) update_moving_variance = moving_averages.assign_moving_average( moving_variance, variance, decay) # logging.info("TF11 moving") def mean_var_with_update(): with tf.control_dependencies( [update_moving_mean, update_moving_variance]): return tf.identity(mean), tf.identity(variance) if is_train: mean, var = mean_var_with_update() self.outputs = act( tf.nn.batch_normalization(self.inputs, mean, var, beta, gamma, epsilon)) else: self.outputs = act( tf.nn.batch_normalization(self.inputs, moving_mean, moving_variance, beta, gamma, epsilon)) variables.extend([moving_mean, moving_variance]) self.all_layers.append(self.outputs) self.all_params.extend(variables)