def _setup_prediction(self): self.batch_size = self.inputs['data'].get_shape().as_list()[0] # LAYER 0: INPUT layer: 2D tensor. Three rows (channels) x window size current_layer = self.inputs['data'] d1, d2, d3 = current_layer.get_shape().as_list() print ("[\033[94mINFO\033[0m] LAYER input shape = "+str(d1)+"x"+str(d2)+"x"+str(d3)) # LAYER 1-8: 8 CONVOLUTINAL layers, 32 1D kernels of size 3 each, stride 2, zero padding c = 32 # number of channels per conv layer ksize = self.config.ksize # size of the convolution kernel depth = self.config.num_conv_layers for i in range(depth): current_layer = layers.conv1(current_layer, c, ksize, stride=2, scope='conv{}'.format(i+1), padding='SAME') d1, d2, d3 = current_layer.get_shape().as_list() print ("[\033[94mINFO\033[0m] LAYER "+'conv{}'.format(i+1)+" shape = "+str(d1)+"x"+str(d2)+"x"+str(d3)) tf.add_to_collection(tf.GraphKeys.ACTIVATIONS, current_layer) self.layers['conv{}'.format(i+1)] = current_layer if self.config.pooling: # Max Pooling (down-sampling) with strides of 2 and kernel size of 2 #https://www.tensorflow.org/api_docs/python/tf/nn/pool current_layer = tf.nn.pool(current_layer, window_shape=[self.config.pooling_window], pooling_type='MAX', padding='SAME', strides=[self.config.pooling_stride]) d1, d2, d3 = current_layer.get_shape().as_list() print ("[\033[94mINFO\033[0m] LAYER "+'max_pool{}'.format(i+1)+" shape = "+str(d1)+"x"+str(d2)+"x"+str(d3)) tf.add_to_collection(tf.GraphKeys.ACTIVATIONS, current_layer) self.layers['pool{}'.format(i+1)] = current_layer bs, width, _ = current_layer.get_shape().as_list() current_layer = tf.reshape(current_layer, [bs, width*c], name="reshape") # FULLY CONNECTED LAYERS for i in range(0, self.config.num_fc_layers-1): # LAYER 10: FULLY CONNECTED current_layer = layers.fc(current_layer, self.config.fc_size, scope='fc{}'.format(i+1), activation_fn=None) d1, d2 = current_layer.get_shape().as_list() print ("[\033[94mINFO\033[0m] FC LAYER "+'{}'.format(i+1)+" shape = "+str(d1)+"x"+str(d2)) self.layers['fc{}'.format(i+1)] = current_layer tf.add_to_collection(tf.GraphKeys.ACTIVATIONS, current_layer) # LAYER: LAST FULLY CONNECTED (LOGITS) current_layer = layers.fc(current_layer, self.config.n_clusters, scope='logits', activation_fn=None) d1, d2 = current_layer.get_shape().as_list() print ("[\033[94mINFO\033[0m] FC LAYER LOGITS shape = "+str(d1)+"x"+str(d2)) self.layers['logits'] = current_layer tf.add_to_collection(tf.GraphKeys.ACTIVATIONS, current_layer) self.layers['class_prob'] = tf.nn.softmax(current_layer, name='class_prob') self.layers['class_prediction'] = tf.argmax(self.layers['class_prob'], 1, name='class_pred') tf.contrib.layers.apply_regularization( tf.contrib.layers.l2_regularizer(self.config.regularization), weights_list=tf.get_collection(tf.GraphKeys.WEIGHTS))
def _setup_prediction(self): # config model cfg = config.Config() self.config = cfg self.ckpt_step = cfg.ckpt_step self.summary_step = cfg.summary_step # model structure num_units = cfg.num_units num_layers = cfg.num_rnn_layers # training params self.config.lrate = cfg.rnn_lrate self.ckpt_step = cfg.ckpt_step self.summary_step = cfg.summary_step # input data current_layer = self.data # model prediction bsize, num_step, _, _ = current_layer.get_shape().as_list() current_layer = tf.reshape(current_layer, [bsize, num_step, -1]) output0, state = layers.bi_gru(current_layer, num_layers = num_layers, num_units = num_units, scope='bi_gru') output = tf.concat([output0[0], output0[1]], axis=2) output = tf.reshape(output, [-1, 2*num_units]) # flatten bi-rnn logits = layers.fc(output, 3, scope='ppk_logits', activation_fn=None) # [0 1 2] for [N, P, S] self.layers['logits'] = logits flat_prob = tf.nn.softmax(logits, name='pred_prob') self.layers['pred_prob'] = tf.reshape(flat_prob, [-1, num_step, 3]) # [bsize, num_step, 3] self.layers['pred_class'] = tf.argmax(self.layers['pred_prob'], 2, name='pred_class')
def _setup_prediction(self): # config model cfg = config.Config() self.config = cfg # model structure num_layers = cfg.num_cnn_layers num_filters = cfg.num_cnn_filters filter_width = cfg.filter_width # train params self.config.lrate = cfg.cnn_lrate lamb = cfg.cnn_l2reg self.ckpt_step = cfg.ckpt_step self.summary_step = cfg.summary_step # inputs current_layer = self.data current_layer = tf.squeeze(current_layer, [1]) bsize, _, _ = current_layer.get_shape().as_list() # model prediction for i in range(num_layers): current_layer = layers.conv1d(current_layer, num_filters, filter_width, scope='conv{}'.format(i + 1)) current_layer = layers.batch_norm(current_layer, scope='batch_norm{}'.format(i + 1)) current_layer = tf.nn.relu(current_layer) current_layer = layers.max_pooling(current_layer) self.layers['conv{}'.format(i + 1)] = current_layer # fully connected layer current_layer = tf.reshape(current_layer, [bsize, -1], name='reshape') current_layer = layers.fc(current_layer, 2, scope='logits') self.layers['logits'] = current_layer # softmax regression current_layer = tf.nn.softmax(current_layer) self.layers['pred_prob'] = current_layer current_layer = tf.argmax(current_layer, 1) self.layers['pred_class'] = current_layer # L2 regularization tf.contrib.layers.apply_regularization( tf.contrib.layers.l2_regularizer(lamb), weights_list=tf.get_collection(tf.GraphKeys.WEIGHTS))
def _setup_prediction(self): # RNN params self.num_units = 64 # number of gru cells per layer self.num_layers = 2 # number of gru layers # input data self.batch_size = self.inputs['data'].get_shape().as_list()[0] current_layer = self.inputs['data'] bs, self.num_step, step_len, chn = current_layer.get_shape().as_list() current_layer = tf.reshape(current_layer, [bs, self.num_step, -1]) # flatten the chns # RNN PpkNet # gru model # output, state = layers.gru(current_layer, num_layers=self.num_layers, num_units=self.num_units, # batch_size=self.batch_size, scope='multi_gru') output0, state = layers.bi_gru(current_layer, num_layers=self.num_layers, num_units =self.num_units, batch_size=self.batch_size, scope='bi_gru') output = tf.concat([output0[0], output0[1]], axis=2) """ bi_gru = GRU.RNN(current_layer, self.num_units) output0 = bi_gru.bi_rnn() output = tf.concat([output0['state_fw'][1], output0['state_bw'][1]], axis=2) tf.add_to_collection(tf.GraphKeys.ACTIVATIONS, output) self.layers['state_bw'] = output0['state_bw'] self.layers['reset_bw'] = output0['reset_bw'] self.layers['update_bw']= output0['update_bw'] self.layers['state_fw'] = output0['state_fw'] self.layers['reset_fw'] = output0['reset_fw'] self.layers['update_fw']= output0['update_fw'] """ output = tf.reshape(output, [-1, 2*self.num_units]) # flatten bi-rnn # output = tf.reshape(output, [-1, self.num_units]) # flatten rnn logits = layers.fc(output, 3, scope='ppk_logits', activation_fn=None) # [0 1 2] for [N, P, S] self.layers['logits'] = logits tf.add_to_collection(tf.GraphKeys.ACTIVATIONS, logits) flat_prob = tf.nn.softmax(logits, name='ppk_class_prob') self.layers['class_prob'] = tf.reshape(flat_prob, [-1, self.num_step, 3]) # shape=(32, 59, 3) self.layers['class_prediction'] = tf.argmax(self.layers['class_prob'], 2, name='ppk_class_pred')
def _setup_prediction(self): self.batch_size = self.inputs['data'].get_shape().as_list()[0] current_layer = self.inputs['data'] d1, d2, d3 = current_layer.get_shape().as_list() print ("[\033[94mINFO\033[0m] INPUT LAYER shape = "+str(d1)+"x"+str(d2)+"x"+str(d3)) c = 32 # number of channels per conv layer ksize = 3 # size of the convolution kernel depth = 8 for i in range(depth): current_layer = layers.conv1(current_layer, c, ksize, stride=2, scope='conv{}'.format(i+1), padding='SAME') d1, d2, d3 = current_layer.get_shape().as_list() print ("[\033[94mINFO\033[0m] LAYER "+'conv{}'.format(i+1)+" shape = "+str(d1)+"x"+str(d2)+"x"+str(d3)) tf.add_to_collection(tf.GraphKeys.ACTIVATIONS, current_layer) self.layers['conv{}'.format(i+1)] = current_layer bs, width, _ = current_layer.get_shape().as_list() #print ("[\033[94mINFO\033[0m] LAST LAYER CONV shape = "+str(bs)+"x"+str(width)+"x"+str(_)) current_layer = tf.reshape(current_layer, [bs, width*c], name="reshape") d1, d2 = current_layer.get_shape().as_list() print ("[\033[94mINFO\033[0m] LAST LAYER CONV RESHAPED shape = "+str(d1)+"x"+str(d2)) current_layer = layers.fc(current_layer, self.config.n_clusters, scope='logits', activation_fn=None) d1, d2 = current_layer.get_shape().as_list() print ("[\033[94mINFO\033[0m] FC LAYER shape = "+str(d1)+"x"+str(d2)) self.layers['logits'] = current_layer tf.add_to_collection(tf.GraphKeys.ACTIVATIONS, current_layer) self.layers['class_prob'] = tf.nn.softmax(current_layer, name='class_prob') self.layers['class_prediction'] = tf.argmax(self.layers['class_prob'], 1, name='class_pred') tf.contrib.layers.apply_regularization( tf.contrib.layers.l2_regularizer(self.config.regularization), weights_list=tf.get_collection(tf.GraphKeys.WEIGHTS))
def _setup_prediction(self): self.batch_size = self.inputs['data'].get_shape().as_list()[0] current_layer = self.inputs['data'] print current_layer c = 64 # number of channels per conv layer ksize = 3 # size of the convolution kernel depth = 6 for i in range(depth): current_layer = layers.conv1(current_layer, c, ksize, stride=1, scope='conv{}'.format(i + 1), padding='SAME') current_layer = layers.max_pooling(current_layer, 2) tf.add_to_collection(tf.GraphKeys.ACTIVATIONS, current_layer) self.layers['conv{}'.format(i + 1)] = current_layer bs, width, _ = current_layer.get_shape().as_list() current_layer = tf.reshape(current_layer, [bs, width * c], name="reshape") current_layer = layers.fc(current_layer, 3, scope='det_logits', activation_fn=None) self.layers['logits'] = current_layer tf.add_to_collection(tf.GraphKeys.ACTIVATIONS, current_layer) self.layers['class_prob'] = tf.nn.softmax(current_layer, name='det_class_prob') self.layers['class_prediction'] = tf.argmax(self.layers['class_prob'], 1, name='det_class_pred') tf.contrib.layers.apply_regularization( tf.contrib.layers.l2_regularizer(self.config.regularization), weights_list=tf.get_collection(tf.GraphKeys.WEIGHTS))
def _setup_prediction(self): # def inputs self.batch_size = self.inputs['data'].get_shape().as_list()[0] current_layer = self.inputs['data'] current_layer = tf.squeeze(current_layer, [1]) # def hypo-params c = 32 # number of channels per conv layer ksize = 3 # size of the convolution kernel depth = 8 lamb = 1e-4 # l2 reg # def model structure for i in range(depth): current_layer = layers.conv1(current_layer, c, ksize, stride=1, activation_fn=None, scope='conv{}'.format(i+1)) current_layer = layers.batch_norm(current_layer, 'conv{}_batch_norm'.format(i+1), self.is_training) current_layer = tf.nn.relu(current_layer) self.layers['conv0_{}'.format(i+1)] = current_layer current_layer = layers.max_pooling(current_layer, 2) tf.add_to_collection(tf.GraphKeys.ACTIVATIONS, current_layer) self.layers['conv{}'.format(i+1)] = current_layer # softmax regression bs, width, _ = current_layer.get_shape().as_list() current_layer = tf.reshape(current_layer, [bs, width*c], name="reshape") current_layer = layers.fc(current_layer, 2, scope='det_logits', activation_fn=None) self.layers['logits'] = current_layer tf.add_to_collection(tf.GraphKeys.ACTIVATIONS, current_layer) # output prediction self.layers['class_prob'] = tf.nn.softmax(current_layer, name='det_class_prob') self.layers['class_prediction'] = tf.argmax(self.layers['class_prob'], 1, name='det_class_pred') # add L2 regularization tf.contrib.layers.apply_regularization( tf.contrib.layers.l2_regularizer(lamb), weights_list=tf.get_collection(tf.GraphKeys.WEIGHTS))
def setup_layer(self): # LSTM_units_num = self.config.cblstm_l lstm_layers_num = self.config.dl_tradition_model_config.cblstm_lstm_layer_num class_num = self.config.dl_tradition_model_config.cblstm_class_num # batch_size = self.config.cblstm_batch_size layer = dict() input_ = tf.placeholder(tf.float32, shape=[None, None, 3], name='input') sequence_length = tf.placeholder(tf.int32, shape=[None], name='seq_len') targets = tf.placeholder(tf.int32, shape=[None, None], name='targets') keep_prob = tf.placeholder(tf.float32, name='keep_prob') batch_size = tf.shape(input_)[0] layer['input'] = input_ layer['seq_len'] = sequence_length layer['targets'] = targets layer['keep_prob'] = keep_prob layer['conv1'] = layers.conv1d(layer['input'], filter=[4, 3, 8], strides=1, padding='SAME', wd=5e-5, bias=0.0, name='conv1') layer['pooling1'] = layers.pool1d(layer['conv1'], ksize=[2], strides=[2], padding='SAME', name='pooling1') layer['conv2'] = layers.conv1d(layer['pooling1'], filter=[4, 8, 16], strides=1, padding='SAME', wd=5e-5, bias=0.0, name='conv2') layer['pooling2'] = layers.pool1d(layer['conv2'], ksize=[2], strides=[2], padding='SAME', name='pooling2') layer['unfold'] = tf.reshape(layer['pooling2'], [batch_size, -1, 400]) layer['unfold'] = tf.reshape(layer['unfold'], [-1, 400]) layer['unfold'] = tf.nn.dropout(layer['unfold'], keep_prob) layer['dim_red'] = layers.fc(layer['unfold'], output_dim=100, wd=5e-5, name='dim_red') layer['dim_red'] = tf.reshape(layer['dim_red'], [batch_size, -1, 100]) lstm_cell_fw1 = tf.nn.rnn_cell.LSTMCell(num_units=100, forget_bias=1.0, state_is_tuple=True, reuse=tf.get_variable_scope().reuse) lstm_cell_fw2 = tf.nn.rnn_cell.LSTMCell(num_units=100, num_proj=50, forget_bias=1.0, state_is_tuple=True, reuse=tf.get_variable_scope().reuse) lstm_cell_bw1 = tf.nn.rnn_cell.LSTMCell(num_units=100, forget_bias=1.0, state_is_tuple=True, reuse=tf.get_variable_scope().reuse) lstm_cell_bw2 = tf.nn.rnn_cell.LSTMCell(num_units=100, num_proj=50, forget_bias=1.0, state_is_tuple=True, reuse=tf.get_variable_scope().reuse) lstm_cell_fw1 = tf.nn.rnn_cell.DropoutWrapper(lstm_cell_fw1, output_keep_prob=keep_prob) lstm_cell_fw2 = tf.nn.rnn_cell.DropoutWrapper(lstm_cell_fw2, output_keep_prob=keep_prob) lstm_cell_bw1 = tf.nn.rnn_cell.DropoutWrapper(lstm_cell_bw1, output_keep_prob=keep_prob) lstm_cell_bw2 = tf.nn.rnn_cell.DropoutWrapper(lstm_cell_bw2, output_keep_prob=keep_prob) cell_fw = tf.nn.rnn_cell.MultiRNNCell( [lstm_cell_fw1, lstm_cell_fw2], state_is_tuple=True) cell_bw = tf.nn.rnn_cell.MultiRNNCell( [lstm_cell_bw1, lstm_cell_bw2], state_is_tuple=True) layer['dim_red'] = tf.nn.dropout(layer['dim_red'], keep_prob) with tf.variable_scope('bi_rnn'): (outputs, _) = tf.nn.bidirectional_dynamic_rnn(cell_fw=cell_fw, cell_bw=cell_bw, inputs=layer['dim_red'], sequence_length=sequence_length, dtype=tf.float32) output = tf.concat(outputs, 2) layer['birnn'] = tf.reshape(output, [-1, 50 * 2]) layer['birnn'] = tf.nn.dropout(layer['birnn'], keep_prob) layer['fc'] = layers.fc(layer['birnn'], output_dim=50, wd=5e-5, name='fc') with tf.variable_scope('softmax'): softmax_w = tf.get_variable(name='softmax_w', shape=[50, class_num], initializer=tf.truncated_normal_initializer(stddev=0.05), dtype=tf.float32) weight_decay = tf.multiply(tf.nn.l2_loss(softmax_w), 5e-5, name='weight_loss') tf.add_to_collection('losses', weight_decay) softmax_b = tf.get_variable(name='softmax_b', shape=[class_num], initializer=tf.constant_initializer(value=0), dtype=tf.float32) xw_plus_b = tf.nn.xw_plus_b(layer['fc'], softmax_w, softmax_b) logits = tf.reshape(xw_plus_b, [batch_size, -1, class_num]) layer['logits'] = logits class_prob = tf.nn.softmax(logits) layer['class_prob'] = class_prob layer['pred_seq'] = tf.cast(tf.argmax(class_prob, axis=2), tf.int32) return layer
def setup_layer(self): layer = dict() layer['target'] = tf.placeholder(tf.int32, shape=[None], name='target') layer['input'] = tf.placeholder( tf.float32, shape=[None, 1, self.config.winsize, 3], name='input') layer['conv1'] = layers.conv( layer['input'], filter=[1, 6, 3, 8], # strides=[1, 1, 4, 1], strides=[1, 1, 1, 1], padding='SAME', wd=1e-3, bias=0.0, name='conv1') layer['pooling1'] = layers.pool(layer['conv1'], ksize=[1, 1, 3, 1], strides=[1, 1, 3, 1], padding='SAME', pool_func=tf.nn.max_pool, name='pooling1') layer['conv2'] = layers.conv( layer['pooling1'], filter=[1, 6, 8, 16], # strides=[1, 1, 4, 1], strides=[1, 1, 1, 1], padding='SAME', wd=1e-3, bias=0.0, name='conv2') layer['pooling2'] = layers.pool(layer['conv2'], ksize=[1, 1, 3, 1], strides=[1, 1, 3, 1], padding='SAME', pool_func=tf.nn.max_pool, name='pooling2') layer['conv3'] = layers.conv( layer['pooling2'], filter=[1, 6, 16, 32], # strides=[1, 1, 4, 1], strides=[1, 1, 1, 1], padding='SAME', wd=1e-3, bias=0.0, name='conv3') layer['pooling3'] = layers.pool(layer['conv3'], ksize=[1, 1, 3, 1], strides=[1, 1, 3, 1], padding='SAME', pool_func=tf.nn.max_pool, name='pooling3') layer['conv4'] = layers.conv( layer['pooling3'], filter=[1, 6, 32, 32], # strides=[1, 1, 4, 1], strides=[1, 1, 1, 1], padding='SAME', wd=1e-3, bias=0.0, name='conv4') layer['pooling4'] = layers.pool(layer['conv4'], ksize=[1, 1, 3, 1], strides=[1, 1, 3, 1], padding='SAME', pool_func=tf.nn.max_pool, name='pooling4') layer['conv5'] = layers.conv( layer['pooling4'], filter=[1, 6, 32, 32], # strides=[1, 1, 4, 1], strides=[1, 1, 1, 1], padding='SAME', wd=1e-3, bias=0.0, name='conv5') layer['pooling5'] = layers.pool(layer['conv5'], ksize=[1, 1, 3, 1], strides=[1, 1, 3, 1], padding='SAME', pool_func=tf.nn.max_pool, name='pooling5') layer['conv6'] = layers.conv( layer['pooling5'], filter=[1, 6, 32, 32], # strides=[1, 1, 4, 1], strides=[1, 1, 1, 1], padding='SAME', wd=1e-3, bias=0.0, name='conv6') layer['pooling6'] = layers.pool(layer['conv6'], ksize=[1, 1, 3, 1], strides=[1, 1, 3, 1], padding='SAME', pool_func=tf.nn.max_pool, name='pooling6') layer['unfold'] = layers.Unfold(layer['pooling6'], name='unfold') layer['logits'] = layers.fc(layer['unfold'], 2, wd=1e-3, name='logits') layer['class_prob'] = tf.nn.softmax(layer['logits'], name='class_prob') layer['class_prediction'] = tf.argmax(layer['class_prob'], 1, name='class_pred') return layer
def _setup_prediction(self): self.batch_size = self.inputs['data'].get_shape().as_list()[0] current_layer = self.inputs['data'] #n_channels = 32 # number of channels per conv layer ksize = 3 # size of the convolution kernel for i in range(self.n_conv_layers): current_layer = layers.conv1(current_layer, self.n_channels, ksize, stride=2, scope='conv{}'.format(i + 1), padding='SAME') tf.add_to_collection(tf.GraphKeys.ACTIVATIONS, current_layer) self.layers['conv{}'.format(i + 1)] = current_layer bs, width, _ = current_layer.get_shape().as_list() n_fc_nodes = width * self.n_channels current_layer = tf.reshape(current_layer, [bs, n_fc_nodes], name="reshape") # 20180916 AJL - include stram_max in fc layers stream_max_tensor = tf.expand_dims(self.inputs['stream_max'], 1) current_layer = tf.concat([current_layer, stream_max_tensor], 1) n_fc_nodes += 1 for i in range(self.n_fc_layers - 1): current_layer = layers.fc(current_layer, n_fc_nodes, scope='fc{}'.format(i + 1), activation_fn=tf.nn.relu) current_layer = layers.fc(current_layer, self.cfg.n_distances + self.cfg.n_magnitudes + self.cfg.n_depths + self.cfg.n_azimuths, \ scope='logits', activation_fn=None) istart = 0 iend = self.cfg.n_distances self.layers['distance_logits'] = current_layer[ : , istart : iend] self.layers['distance_prob'] = tf.nn.softmax(self.layers['distance_logits'], name='distance_prob') self.layers['distance_prediction'] = tf.argmax(self.layers['distance_prob'], 1, name='distance_pred') istart = iend self.layers['magnitude_logits'] = tf.constant(-1) self.layers['magnitude_prob'] = tf.constant(-1) self.layers['magnitude_prediction'] = tf.constant(-1) if self.cfg.n_magnitudes > 0: iend += self.cfg.n_magnitudes self.layers['magnitude_logits'] = current_layer[ : , istart : iend] self.layers['magnitude_prob'] = tf.nn.softmax(self.layers['magnitude_logits'], name='magnitude_prob') self.layers['magnitude_prediction'] = tf.argmax(self.layers['magnitude_prob'], 1, name='magnitude_pred') istart = iend self.layers['depth_logits'] = tf.constant(-1) self.layers['depth_prob'] = tf.constant(-1) self.layers['depth_prediction'] = tf.constant(-1) if self.cfg.n_depths > 0: iend += self.cfg.n_depths self.layers['depth_logits'] = current_layer[ : , istart : iend] self.layers['depth_prob'] = tf.nn.softmax(self.layers['depth_logits'], name='depth_prob') self.layers['depth_prediction'] = tf.argmax(self.layers['depth_prob'], 1, name='depth_pred') istart = iend self.layers['azimuth_logits'] = tf.constant(-1) self.layers['azimuth_prob'] = tf.constant(-1) self.layers['azimuth_prediction'] = tf.constant(-1) if self.cfg.n_azimuths > 0: iend += self.cfg.n_azimuths self.layers['azimuth_logits'] = current_layer[ : , istart : iend] self.layers['azimuth_prob'] = tf.nn.softmax(self.layers['azimuth_logits'], name='azimuth_prob') self.layers['azimuth_prediction'] = tf.argmax(self.layers['azimuth_prob'], 1, name='azimuth_pred') istart = iend tf.add_to_collection(tf.GraphKeys.ACTIVATIONS, current_layer) tf.contrib.layers.apply_regularization( tf.contrib.layers.l2_regularizer(self.cfg.regularization), weights_list=tf.get_collection(tf.GraphKeys.WEIGHTS))