def _recognition_network(self, x, network_architecture): # Generate probabilistic encoder (recognition network), which # maps inputs onto a normal distribution in latent space. # The transformation is parametrized and can be learned. scope, hidden_conv, n_hidden_recog_1, n_hidden_recog_2, n_hidden_gener_1, n_hidden_gener_2, n_input, n_z = self._unpack_network_arguments(**network_architecture) with tf.variable_scope(scope): if hidden_conv: #convolution layer filtering over 2d 28x28 images input_size = int(np.sqrt(n_input)) x_2d = tf.reshape(x, [-1, input_size, input_size, 1]) #convolution layers use their own initialization of weight variables layer_0_5 = conv_2d( x_2d, filter_set=[5, 1, n_hidden_recog_1], stride=2, padding='SAME', transfer_fct=None) #no nonlinear transformation here layer_1 = conv_2d( layer_0_5, filter_set=[5, n_hidden_recog_1, n_hidden_recog_1*2], stride=2, padding='SAME', transfer_fct=None) else: weights_hidden_1 = tf.Variable(xavier_init(n_input, n_hidden_recog_1)) biases_hidden_1 = tf.Variable(tf.zeros([n_hidden_recog_1], dtype=tf.float32)) layer_1 = self.transfer_fct(tf.add(tf.matmul(x, weights_hidden_1), biases_hidden_1)) if hidden_conv: # layer_1_size = (input_size - 5) / 1 layer_1_size = input_size / 2 / 2 #convolution layers use their own initialization of weight variables layer_2_2d = conv_2d( layer_1, filter_set=[5, n_hidden_recog_1*2, n_hidden_recog_2], stride=1, padding='VALID') #no nonlinear transformation here layer_2_size = (layer_1_size - 5) / 1 + 1 layer_2 = tf.reshape(layer_2_2d, (-1, layer_2_size*layer_2_size*n_hidden_recog_2)) else: weights_hidden_2 = tf.Variable(xavier_init(n_hidden_recog_1, n_hidden_recog_2)) biases_hidden_2 = tf.Variable(tf.zeros([n_hidden_recog_2], dtype=tf.float32)) layer_2 = self.transfer_fct(tf.add(tf.matmul(layer_1, weights_hidden_2), biases_hidden_2)) if hidden_conv: weights_out_mean = tf.Variable(xavier_init(layer_2_size * layer_2_size * n_hidden_recog_2, n_z)) biases_out_mean = tf.Variable(tf.zeros([n_z], dtype=tf.float32)) weights_out_log_sigma = tf.Variable(xavier_init(layer_2_size * layer_2_size * n_hidden_recog_2, n_z)) biases_out_log_sigma = tf.Variable(tf.zeros([n_z], dtype=tf.float32)) else: weights_out_mean = tf.Variable(xavier_init(n_hidden_recog_2, n_z)) biases_out_mean = tf.Variable(tf.zeros([n_z], dtype=tf.float32)) weights_out_log_sigma = tf.Variable(xavier_init(n_hidden_recog_2, n_z)) biases_out_log_sigma = tf.Variable(tf.zeros([n_z], dtype=tf.float32)) z_mean = tf.add(tf.matmul(layer_2, weights_out_mean), biases_out_mean) z_log_sigma_sq = \ tf.add(tf.matmul(layer_2, weights_out_log_sigma), biases_out_log_sigma) return (z_mean, z_log_sigma_sq)
def loc_net_fc(images, batch_size): images -= 128 images /= 128. images = tf.image.resize_images(images, 150, 150) images_flat = tf.reshape(images, [batch_size, -1]) hidden_size = 100 with tf.name_scope('fc1') as scope: weights = tf.Variable(tf.truncated_normal([150**2*3, hidden_size], dtype=tf.float32, stddev=1e-3), name='weights') biases = tf.Variable(tf.constant(0.0, shape=[hidden_size], dtype=tf.float32), name='biases') hidden = tf.add(tf.matmul(images_flat, weights), biases, name=scope) hidden = tf.nn.relu(hidden) with tf.name_scope('fc2') as scope: weights = tf.Variable(tf.truncated_normal([hidden_size, 3], dtype=tf.float32, stddev=1e-3), name='weights') biases = tf.Variable(tf.constant(0.0, shape=[3], dtype=tf.float32), name='biases') theta = tf.add(tf.matmul(hidden, weights), biases, name=scope) theta = tf.nn.tanh(theta) return theta
def convert_to_one(bbox, width, height, S): x, y, w, h = bbox x = tf.cast(x, tf.float32) y = tf.cast(y, tf.float32) w = tf.cast(w, tf.float32) h = tf.cast(h, tf.float32) global_center_x = tf.mul(tf.add(tf.mul(x, 2), w), 0.5) global_center_y = tf.mul(tf.add(tf.mul(y, 2), h), 0.5) w = tf.div(w, width) h = tf.div(h, height) cell_w = tf.cast(tf.div(tf.cast(width, tf.int32), S), tf.float32) cell_h = tf.cast(tf.div(tf.cast(height, tf.int32), S), tf.float32) cell_coord_x = tf.cast(tf.cast(tf.div(global_center_x, cell_w), tf.int32), tf.float32) cell_coord_y = tf.cast(tf.cast(tf.div(global_center_y, cell_h), tf.int32), tf.float32) offset_x = tf.div(tf.sub(global_center_x, tf.mul(cell_coord_x, cell_w)), cell_w) offset_y = tf.div(tf.sub(global_center_y, tf.mul(cell_coord_y, cell_h)), cell_h) assert offset_x.dtype == tf.float32 and \ offset_y.dtype == tf.float32 and \ w.dtype == tf.float32 and \ h.dtype == tf.float32 bbox = [offset_x, offset_y, w, h] return bbox
def neural_network_model(data): # input_data * weights + biases hidden_1_layer = {'weights':tf.Variable(tf.random_normal([784, n_nodes_hl1])), 'biases':tf.Variable(tf.random_normal([n_nodes_hl1]))} hidden_2_layer = {'weights':tf.Variable(tf.random_normal([n_nodes_hl1, n_nodes_hl2])), 'biases':tf.Variable(tf.random_normal([n_nodes_hl2]))} hidden_3_layer = {'weights':tf.Variable(tf.random_normal([n_nodes_hl2, n_nodes_hl3])), 'biases':tf.Variable(tf.random_normal([n_nodes_hl3]))} output_layer = {'weights':tf.Variable(tf.random_normal([n_nodes_hl3, n_classes])), 'biases':tf.Variable(tf.random_normal([n_classes]))} # input_data * weights + biases l1 = tf.add(tf.matmul(data, hidden_1_layer['weights']), hidden_1_layer['biases']) l1 = tf.nn.relu(l1) l2 = tf.add(tf.matmul(l1, hidden_2_layer['weights']), hidden_2_layer['biases']) l2 = tf.nn.relu(l2) l3 = tf.add(tf.matmul(l2, hidden_3_layer['weights']), hidden_3_layer['biases']) l3 = tf.nn.relu(l3) output = tf.matmul(l3, output_layer['weights']) + output_layer['biases'] return output
def layers(vgg_layer3_out, vgg_layer4_out, vgg_layer7_out, num_classes): """ Create the layers for a fully convolutional network. Build skip-layers using the vgg layers. :param vgg_layer3_out: TF Tensor for VGG Layer 3 output :param vgg_layer4_out: TF Tensor for VGG Layer 4 output :param vgg_layer7_out: TF Tensor for VGG Layer 7 output :param num_classes: Number of classes to classify :return: The Tensor for the last layer of output """ # upsampling on layer7 by 2 input = tf.layers.conv2d(vgg_layer7_out, num_classes, 1, strides=(1,1), padding='same', kernel_initializer=tf.truncated_normal_initializer(stddev=0.01), kernel_regularizer=tf.contrib.layers.l2_regularizer(1e-3)) output = tf.layers.conv2d_transpose(input, num_classes, 4, strides = (2, 2), padding= 'same', kernel_regularizer=tf.contrib.layers.l2_regularizer(1e-3)) #skip connection followed by upsampling on layer4 by 2 input = tf.layers.conv2d(vgg_layer4_out, num_classes, 1, strides=(1,1), padding='same', kernel_initializer=tf.truncated_normal_initializer(stddev=0.01), kernel_regularizer=tf.contrib.layers.l2_regularizer(1e-3)) input = tf.add(input, output) output = tf.layers.conv2d_transpose(input, num_classes, 4, strides = (2, 2), padding= 'same', kernel_regularizer=tf.contrib.layers.l2_regularizer(1e-3)) #skip connection followed by upsampling on layer3 by 8 input = tf.layers.conv2d(vgg_layer3_out, num_classes, 1, strides=(1,1), padding='same', kernel_initializer=tf.truncated_normal_initializer(stddev=0.01), kernel_regularizer=tf.contrib.layers.l2_regularizer(1e-3)) input = tf.add(input, output) nn_last_layer = tf.layers.conv2d_transpose(input, num_classes, 32, strides = (8, 8), padding= 'same', kernel_regularizer=tf.contrib.layers.l2_regularizer(1e-3)) return nn_last_layer
def forward_propagation(X, parameters): """ Implements the forward propagation for the model: LINEAR -> RELU -> LINEAR -> RELU -> LINEAR -> SOFTMAX Arguments: X -- input dataset placeholder, of shape (input size, number of examples) parameters -- python dictionary containing your parameters "W1", "b1", "W2", "b2", "W3", "b3" the shapes are given in initialize_parameters Returns: Z3 -- the output of the last LINEAR unit """ # Retrieve the parameters from the dictionary "parameters" W1 = parameters['W1'] b1 = parameters['b1'] W2 = parameters['W2'] b2 = parameters['b2'] W3 = parameters['W3'] b3 = parameters['b3'] print(W3.shape) ### START CODE HERE ### (approx. 5 lines) # Numpy Equivalents: Z1 = tf.add(tf.matmul(W1,X),b1) # Z1 = np.dot(W1, X) + b1 A1 = tf.nn.relu(Z1) # A1 = relu(Z1) Z2 = tf.add(tf.matmul(W2,A1),b2) # Z2 = np.dot(W2, a1) + b2 A2 = tf.nn.relu(Z2) # A2 = relu(Z2) print(A2.shape) Z3 = tf.add(tf.matmul(W3,A2),b3) # Z3 = np.dot(W3,Z2) + b3 ### END CODE HERE ### return Z3
def cell_locate(size, bbox, S): """ locate the center of ground truth in which grid cell """ x = tf.cast(tf.slice(bbox, [0,0], [-1,1]), tf.float32) y = tf.cast(tf.slice(bbox, [0,1], [-1,1]), tf.float32) w = tf.cast(tf.slice(bbox, [0,2], [-1,1]), tf.float32) h = tf.cast(tf.slice(bbox, [0,3], [-1,1]), tf.float32) height, width = size cell_w = width / S cell_h = height / S center_y = tf.add(y, tf.mul(h, 0.5)) center_x = tf.add(x, tf.mul(w, 0.5)) cell_coord_x = tf.cast(tf.div(center_x, cell_w), tf.int32) cell_coord_y = tf.cast(tf.div(center_y, cell_h), tf.int32) cell_num = tf.add(tf.mul(cell_coord_y, S), cell_coord_x) return cell_num
def conv_net(_X, _weights, _biases, _dropout): # Reshape input picture _X = tf.reshape(_X, shape=[-1, 28, 28, 1]) # Convolution Layer conv1 = conv2d(_X, _weights['wc1'], _biases['bc1']) # Max Pooling (down-sampling) conv1 = max_pool(conv1, k=2) # Apply Dropout conv1 = tf.nn.dropout(conv1, _dropout) # Convolution Layer conv2 = conv2d(conv1, _weights['wc2'], _biases['bc2']) # Max Pooling (down-sampling) conv2 = max_pool(conv2, k=2) # Apply Dropout conv2 = tf.nn.dropout(conv2, _dropout) # Fully connected layer dense1 = tf.reshape(conv2, [-1, _weights['wd1'].get_shape().as_list()[0]]) # Reshape conv2 output to fit dense layer input dense1 = tf.nn.relu(tf.add(tf.matmul(dense1, _weights['wd1']), _biases['bd1'])) # Relu activation dense1 = tf.nn.dropout(dense1, _dropout) # Apply Dropout # Output, class prediction out = tf.add(tf.matmul(dense1, _weights['out']), _biases['out']) return out
def conv_net(x, weights, biases, dropout): # Reshape input picture x = tf.reshape(x, shape=[-1, 28, 28, 1]) # Convolution Layer conv1 = conv2d(x, weights['wc1'], biases['bc1']) # Max Pooling (down-sampling) conv1 = maxpool2d(conv1, k=2) # Convolution Layer conv2 = conv2d(conv1, weights['wc2'], biases['bc2']) # Max Pooling (down-sampling) conv2 = maxpool2d(conv2, k=2) # Fully connected layer # Reshape conv2 output to fit fully connected layer input fc1 = tf.reshape(conv2, [-1, weights['wd1'].get_shape().as_list()[0]]) fc1 = tf.add(tf.matmul(fc1, weights['wd1']), biases['bd1']) fc1 = tf.nn.relu(fc1) # Apply Dropout fc1 = tf.nn.dropout(fc1, dropout) # Output, class prediction out = tf.add(tf.matmul(fc1, weights['out']), biases['out']) return out
def testTensors(self): for v1_first in [True, False]: with self.test_session(): v1 = tf.Variable([1.0]) add1 = tf.add( control_flow_ops.with_dependencies([v1.initializer], v1.ref()), 2.0) v2 = tf.Variable([10.0]) add2 = tf.add( control_flow_ops.with_dependencies([v2.initializer], v2.ref()), 20.0) t1, _, t2 = control_flow_ops.tuple([add1, None, add2]) # v1 is not initialized. with self.assertRaisesOpError("Attempting to use uninitialized value"): v1.eval() # v2 is not initialized. with self.assertRaisesOpError("Attempting to use uninitialized value"): v2.eval() if v1_first: # Getting t1 initializes v2. self.assertAllClose([3.0], t1.eval()) self.assertAllClose([10.0], v2.eval()) else: # Getting t2 initializes v1. self.assertAllClose([30.0], t2.eval()) self.assertAllClose([1.0], v1.eval())
def observation_net(self, input_): #decoder # input:[B,Z] with tf.name_scope('observation_net'): n_layers = len(self.network_architecture['decoder_net']) # weights = self.network_weights['decoder_weights'] # biases = self.network_weights['decoder_biases'] for layer_i in range(n_layers): # input_ = tf.contrib.layers.layer_norm(input_) # input_ = self.transfer_fct(tf.add(tf.matmul(input_, self.params_dict['decoder_weights_l'+str(layer_i)]), self.params_dict['decoder_biases_l'+str(layer_i)])) input_ = self.transfer_fct(tf.contrib.layers.layer_norm(tf.add(tf.matmul(input_, self.params_dict['decoder_weights_l'+str(layer_i)]), self.params_dict['decoder_biases_l'+str(layer_i)]))) #add batch norm here x_mean = tf.add(tf.matmul(input_, self.params_dict['decoder_weights_out_mean']), self.params_dict['decoder_biases_out_mean']) x_log_var = tf.add(tf.matmul(input_, self.params_dict['decoder_weights_out_log_var']), self.params_dict['decoder_biases_out_log_var']) reward_mean = tf.add(tf.matmul(input_, self.params_dict['decoder_weights_reward_mean']), self.params_dict['decoder_biases_reward_mean']) reward_log_var = tf.add(tf.matmul(input_, self.params_dict['decoder_weights_reward_log_var']), self.params_dict['decoder_biases_reward_log_var']) return x_mean, x_log_var, reward_mean, reward_log_var
def drawGraph(self, n_row, n_latent, n_col): with tf.name_scope('matDecomp'): self._p = tf.placeholder(tf.float32, shape=[None, n_col]) self._c = tf.placeholder(tf.float32, shape=[None, n_col]) self._lambda = tf.placeholder(tf.float32) self._index = tf.placeholder(tf.float32, shape=[None, n_row]) self._A = tf.Variable(tf.truncated_normal([n_row, n_latent])) self._B = tf.Variable(tf.truncated_normal([n_latent, n_col])) self._h = tf.matmul(tf.matmul(self._index, self._A), self._B) weighted_loss = tf.reduce_mean(tf.mul(self._c, tf.squared_difference(self._p, self._h))) self._weighted_loss = weighted_loss l2_A = tf.reduce_sum(tf.square(self._A)) l2_B = tf.reduce_sum(tf.square(self._B)) n_w = tf.constant(n_row * n_latent + n_latent * n_col, tf.float32) l2 = tf.truediv(tf.add(l2_A, l2_B), n_w) reg_term = tf.mul(self._lambda, l2) self._loss = tf.add(weighted_loss, reg_term) self._mask = tf.placeholder(tf.float32, shape=[n_row, n_col]) one = tf.constant(1, tf.float32) pred = tf.cast(tf.greater_equal(tf.matmul(self._A, self._B), one), tf.float32) cor = tf.mul(tf.cast(tf.equal(pred, self._p), tf.float32), self._c) self._vali_err = tf.reduce_sum(tf.mul(cor, self._mask)) self._saver = tf.train.Saver([v for v in tf.all_variables() if v.name.find('matDecomp') != -1]) tf.scalar_summary('training_weighted_loss_l2', self._loss) tf.scalar_summary('validation_weighted_loss', self._weighted_loss) merged = tf.merge_all_summaries()
def __init__(self, n_layers, transfer_function=tf.nn.softplus, optimizer=tf.train.AdamOptimizer()): self.n_layers = n_layers self.transfer = transfer_function network_weights = self._initialize_weights() self.weights = network_weights # model self.x = tf.placeholder(tf.float32, [None, self.n_layers[0]]) self.hidden_encode = [] h = self.x for layer in range(len(self.n_layers)-1): h = self.transfer( tf.add(tf.matmul(h, self.weights['encode'][layer]['w']), self.weights['encode'][layer]['b'])) self.hidden_encode.append(h) self.hidden_recon = [] for layer in range(len(self.n_layers)-1): h = self.transfer( tf.add(tf.matmul(h, self.weights['recon'][layer]['w']), self.weights['recon'][layer]['b'])) self.hidden_recon.append(h) self.reconstruction = self.hidden_recon[-1] # cost self.cost = 0.5 * tf.reduce_sum(tf.pow(tf.subtract(self.reconstruction, self.x), 2.0)) self.optimizer = optimizer.minimize(self.cost) init = tf.global_variables_initializer() self.sess = tf.Session() self.sess.run(init)
def conv_basic(_input, _w, _b, _keepratio): # INPUT _input_r = tf.reshape(_input, shape=[-1, imgsize[0], imgsize[1], 1]) # CONVOLUTION LAYER 1 _conv1 = tf.nn.relu(tf.nn.bias_add(tf.nn.conv2d(_input_r , _w['wc1'], strides=[1, 1, 1, 1], padding='SAME'), _b['bc1'])) _pool1 = tf.nn.max_pool(_conv1, ksize=[1, 2, 2, 1] , strides=[1, 2, 2, 1], padding='SAME') _pool_dr1 = tf.nn.dropout(_pool1, _keepratio) # CONVOLUTION LAYER 2 _conv2 = tf.nn.relu(tf.nn.bias_add(tf.nn.conv2d(_pool_dr1 , _w['wc2'], strides=[1, 1, 1, 1], padding='SAME'), _b['bc2'])) _pool2 = tf.nn.max_pool(_conv2, ksize=[1, 2, 2, 1] , strides=[1, 2, 2, 1], padding='SAME') _pool_dr2 = tf.nn.dropout(_pool2, _keepratio) # VECTORIZE _dense1 = tf.reshape(_pool_dr2 , [-1, _w['wd1'].get_shape().as_list()[0]]) # FULLY CONNECTED LAYER 1 _fc1 = tf.nn.relu(tf.add(tf.matmul(_dense1, _w['wd1']), _b['bd1'])) _fc_dr1 = tf.nn.dropout(_fc1, _keepratio) # FULLY CONNECTED LAYER 2 _out = tf.add(tf.matmul(_fc_dr1, _w['wd2']), _b['bd2']) # RETURN out = { 'input_r': _input_r, 'conv1': _conv1, 'pool1': _pool1 , 'pool1_dr1': _pool_dr1, 'conv2': _conv2, 'pool2': _pool2 , 'pool_dr2': _pool_dr2, 'dense1': _dense1, 'fc1': _fc1 , 'fc_dr1': _fc_dr1, 'out': _out } return out
def _create_dilation_layer(self, input_batch, layer_index, dilation, in_channels, dilation_channels, skip_channels): '''Creates a single causal dilated convolution layer. The layer contains a gated filter that connects to dense output and to a skip connection: |-> [gate] -| |-> 1x1 conv -> skip output | |-> (*) -| input -|-> [filter] -| |-> 1x1 conv -| | |-> (+) -> dense output |------------------------------------| Where `[gate]` and `[filter]` are causal convolutions with a non-linear activation at the output. ''' variables = self.variables['dilated_stack'][layer_index] weights_filter = variables['filter'] weights_gate = variables['gate'] conv_filter = causal_conv(input_batch, weights_filter, dilation) conv_gate = causal_conv(input_batch, weights_gate, dilation) if self.use_biases: filter_bias = variables['filter_bias'] gate_bias = variables['gate_bias'] conv_filter = tf.add(conv_filter, filter_bias) conv_gate = tf.add(conv_gate, gate_bias) out = tf.tanh(conv_filter) * tf.sigmoid(conv_gate) # The 1x1 conv to produce the residual output weights_dense = variables['dense'] transformed = tf.nn.conv1d( out, weights_dense, stride=1, padding="SAME", name="dense") # The 1x1 conv to produce the skip output weights_skip = variables['skip'] skip_contribution = tf.nn.conv1d( out, weights_skip, stride=1, padding="SAME", name="skip") if self.use_biases: dense_bias = variables['dense_bias'] skip_bias = variables['skip_bias'] transformed = transformed + dense_bias skip_contribution = skip_contribution + skip_bias layer = 'layer{}'.format(layer_index) tf.histogram_summary(layer + '_filter', weights_filter) tf.histogram_summary(layer + '_gate', weights_gate) tf.histogram_summary(layer + '_dense', weights_dense) tf.histogram_summary(layer + '_skip', weights_skip) if self.use_biases: tf.histogram_summary(layer + '_biases_filter', filter_bias) tf.histogram_summary(layer + '_biases_gate', gate_bias) tf.histogram_summary(layer + '_biases_dense', dense_bias) tf.histogram_summary(layer + '_biases_skip', skip_bias) return skip_contribution, input_batch + transformed
def inception_net(image, _weights, _biases, _dropout): # Reshape input img_size = 50 image = tf.reshape(image, shape=[-1,img_size,img_size,1]) # Convolution layer 1 conv1 = conv2d(image, _weights['wc1'], _biases['bc1'],padding = 'VALID') conv1 = max_pool(conv1, k=2) #conv1 = tf.nn.dropout(conv1, _dropout) # Inception module 1 i_weights = _weights['wi1'] i_biases = _biases['bi1'] h_1x1 = conv2d(conv1, i_weights['direct1x1'], i_biases['direct1x1']) h_3x3 = conv2d(conv1, i_weights['1x1pre3x3'], i_biases['1x1pre3x3']) h_3x3 = conv2d(h_3x3, i_weights['3x3'], i_biases['3x3']) h_5x5 = conv2d(conv1, i_weights['1x1pre5x5'], i_biases['1x1pre5x5']) h_5x5 = conv2d(h_5x5, i_weights['5x5'], i_biases['5x5']) incep1 = tf.concat(3, [h_1x1, h_3x3, h_5x5]) incep1 = max_pool(incep1, k=2) #incep1 = tf.nn.dropout(incep1, _dropout) # Fully connected layer dense1 = tf.reshape(incep1, [-1,_weights['wd1'].get_shape().as_list()[0]]) dense1 = tf.nn.relu(tf.add(tf.matmul(dense1, _weights['wd1']), _biases['bd1'])) dense1 = tf.nn.dropout(dense1, _dropout) # Output layer (not softmax??) out = tf.add(tf.matmul(dense1, _weights['out']), _biases['out']) return out
def my_conv_net(input_data): # First Conv-ReLU-MaxPool Layer conv1 = tf.nn.conv2d(input_data, conv1_weight, strides=[1, 1, 1, 1], padding='SAME') relu1 = tf.nn.relu(tf.nn.bias_add(conv1, conv1_bias)) max_pool1 = tf.nn.max_pool(relu1, ksize=[1, max_pool_size1, max_pool_size1, 1], strides=[1, max_pool_size1, max_pool_size1, 1], padding='SAME') # Second Conv-ReLU-MaxPool Layer conv2 = tf.nn.conv2d(max_pool1, conv2_weight, strides=[1, 1, 1, 1], padding='SAME') relu2 = tf.nn.relu(tf.nn.bias_add(conv2, conv2_bias)) max_pool2 = tf.nn.max_pool(relu2, ksize=[1, max_pool_size2, max_pool_size2, 1], strides=[1, max_pool_size2, max_pool_size2, 1], padding='SAME') # Transform Output into a 1xN layer for next fully connected layer final_conv_shape = max_pool2.get_shape().as_list() final_shape = final_conv_shape[1] * final_conv_shape[2] * final_conv_shape[3] flat_output = tf.reshape(max_pool2, [final_conv_shape[0], final_shape]) # First Fully Connected Layer fully_connected1 = tf.nn.relu(tf.add(tf.matmul(flat_output, full1_weight), full1_bias)) # Second Fully Connected Layer final_model_output = tf.add(tf.matmul(fully_connected1, full2_weight), full2_bias) return(final_model_output)
def testStrippedOpListDef(self): with self.test_session(): # Creates a graph. v0 = tf.Variable(0.0) var = tf.Variable(10.0) tf.add(v0, var) @function.Defun(x=tf.float32) def minus_one(x): return x - 1 minus_one(tf.identity(v0)) save = tf.train.Saver({"v0": v0}) tf.initialize_all_variables() # Generates MetaGraphDef. meta_graph_def = save.export_meta_graph() ops = [o.name for o in meta_graph_def.meta_info_def.stripped_op_list.op] self.assertEqual(ops, ["Add", "Assign", "Const", "Identity", "NoOp", "RestoreSlice", "SaveSlices", "Sub", "Variable"]) # Test calling stripped_op_list_for_graph directly op_list = tf.contrib.util.stripped_op_list_for_graph( meta_graph_def.graph_def) self.assertEqual(ops, [o.name for o in op_list.op]) for o in op_list.op: self.assertEqual(o.summary, "") self.assertEqual(o.description, "")
def forward_propogation(self): x = tf.placeholder("float") z2 = tf.add(tf.matmul(x,self.W1),self.b1) a2 = tf.sigmoid(z2, name="Hidden Activation") z3 = tf.add(tf.matmul(a2,self.W2),self.b2) a3 = tf.sigmoid(z3, name="Output Activation") return a3
def cross_entropy(u, label_u, alpha=0.5, normed=False): label_ip = tf.cast( tf.matmul(label_u, tf.transpose(label_u)), tf.float32) s = tf.clip_by_value(label_ip, 0.0, 1.0) # compute balance param # s_t \in {-1, 1} s_t = tf.multiply(tf.add(s, tf.constant(-0.5)), tf.constant(2.0)) sum_1 = tf.reduce_sum(s) sum_all = tf.reduce_sum(tf.abs(s_t)) balance_param = tf.add(tf.abs(tf.add(s, tf.constant(-1.0))), tf.multiply(tf.div(sum_all, sum_1), s)) if normed: # ip = tf.clip_by_value(tf.matmul(u, tf.transpose(u)), -1.5e1, 1.5e1) ip_1 = tf.matmul(u, tf.transpose(u)) def reduce_shaper(t): return tf.reshape(tf.reduce_sum(t, 1), [tf.shape(t)[0], 1]) mod_1 = tf.sqrt(tf.matmul(reduce_shaper(tf.square(u)), reduce_shaper(tf.square(u)), transpose_b=True)) ip = tf.div(ip_1, mod_1) else: ip = tf.clip_by_value(tf.matmul(u, tf.transpose(u)), -1.5e1, 1.5e1) ones = tf.ones([tf.shape(u)[0], tf.shape(u)[0]]) return tf.reduce_mean(tf.multiply(tf.log(ones + tf.exp(alpha * ip)) - s * alpha * ip, balance_param))
def dnn(x): with tf.variable_scope('Layer1'): # Creating variable using TFLearn W1 = va.variable(name='W', shape=[len(X_train[0]), first_layer], initializer='uniform_scaling', regularizer='L2') b1 = va.variable(name='b', shape=[first_layer]) x = tf.nn.tanh(tf.add(tf.matmul(x, W1), b1)) with tf.variable_scope('Layer2'): W2 = va.variable(name='W', shape=[first_layer, second_layer], initializer='uniform_scaling', regularizer='L2') b2 = va.variable(name='b', shape=[second_layer]) x = tf.nn.tanh(tf.add(tf.matmul(x, W2), b2)) with tf.variable_scope('Layer3'): W3 = va.variable(name='W', shape=[second_layer, third_layer], initializer='uniform_scaling',regularizer='L2') b3 = va.variable(name='b', shape=[third_layer]) x = tf.add(tf.matmul(x, W3), b3) with tf.variable_scope('Layer4'): W4 = va.variable(name='W', shape=[third_layer, len(Y_train[0])], initializer='uniform_scaling',regularizer='L2') b4 = va.variable(name='b', shape=[len(Y_train[0])]) x = tf.add(tf.matmul(x, W4), b4) return x,W4,b4
def predict_action(self, prev_z): ''' prev_z: [B, Z] return: [B, A] needs to be one hot because model has only seen one hot actions ''' with tf.name_scope('predict_action'): n_layers = len(self.network_architecture['policy_net']) # weights = self.params_dict['policy_weights'] # biases = self.params_dict['policy_biases'] input_ = prev_z for layer_i in range(n_layers): input_ = self.transfer_fct(tf.add(tf.matmul(input_, self.params_dict['policy_weights_l'+str(layer_i)]), self.params_dict['policy_biases_l'+str(layer_i)])) action = tf.add(tf.matmul(input_, self.params_dict['policy_weights_out_mean']), self.params_dict['policy_biases_out_mean']) action = tf.nn.softmax(action) # action = tf.argmax(action, axis=1) # action = tf.one_hot(indices=action, depth=self.action_size, axis=None) return action
def neural_network_model(data): hidden_1_layer = {'weights': tf.Variable(tf.random_normal([784, n_nodes_hl1])), 'biases': tf.Variable(tf.random_normal([n_nodes_hl1]))} hidden_2_layer = {'weights': tf.Variable(tf.random_normal([n_nodes_hl1, n_nodes_hl2])), 'biases': tf.Variable(tf.random_normal([n_nodes_hl2]))} hidden_3_layer = {'weights': tf.Variable(tf.random_normal([n_nodes_hl2, n_nodes_hl3])), 'biases': tf.Variable(tf.random_normal([n_nodes_hl3]))} output_layer = {'weights': tf.Variable(tf.random_normal([n_nodes_hl3, n_classes])), 'biases': tf.Variable(tf.random_normal([n_classes]))} l1 = tf.add( tf.matmul(data, hidden_1_layer['weights']), hidden_1_layer['biases']) # rectified linear is the activation function and act like the threshold function l1 = tf.nn.relu(l1) l2 = tf.add( tf.matmul(l1, hidden_2_layer['weights']), hidden_2_layer['biases']) l2 = tf.nn.relu(l2) l3 = tf.add( tf.matmul(l2, hidden_3_layer['weights']), hidden_3_layer['biases']) l3 = tf.nn.relu(l3) output = tf.add( tf.matmul(l3, output_layer['weights']), output_layer['biases']) return output
def conv_basic(_input, _w, _b, _keepratio): _input_r = tf.reshape(_input, shape=[-1, 28, 28, 1]) # Reshape input # conv1 _conv1 = tf.nn.conv2d(_input_r, _w['wc1'], strides=[1, 1, 1, 1], padding='SAME') # Convolutioin _conv1 = tf.nn.batch_normalization(_conv1, 0.001, 1.0, 0, 1, 0.0001) _conv1 = tf.nn.relu(tf.nn.bias_add(_conv1, _b['bc1'])) # Add-bias _pool1 = tf.nn.max_pool(_conv1, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME') # Max-pooling _pool_dr1 = tf.nn.dropout(_pool1, _keepratio) # conv1 _conv2 = tf.nn.conv2d(_pool_dr1, _w['wc2'], strides=[1, 1, 1, 1], padding='SAME') # Convolutioin _conv2 = tf.nn.batch_normalization(_conv2, 0.001, 1.0, 0, 1, 0.0001) _conv2 = tf.nn.relu(tf.nn.bias_add(_conv2, _b['bc2'])) # Add-bias _pool2 = tf.nn.max_pool(_conv2, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME') # Max-pooling _pool_dr2 = tf.nn.dropout(_pool2, _keepratio) #vectorize _dense1 = tf.reshape(_pool_dr2, [-1, _w['wd1'].get_shape().as_list()[0]]) #fc1 _fc1 = tf.nn.relu(tf.add(tf.matmul(_dense1, _w['wd1']), _b['bd1'])) _fc_dr1 = tf.nn.dropout(_fc1, _keepratio) #fc2 _out = tf.add(tf.matmul(_fc_dr1, _w['wd2']), _b['bd2']) out = { 'input_r': _input_r, 'conv1': _conv1, 'pool1': _pool1, 'pool_dr1': _pool_dr1, 'conv2': _conv2, 'pool2': _pool2, 'pool_dr2': _pool_dr2, 'dense1': _dense1, 'fc1': _fc1, 'fc_dr1': _fc_dr1, 'out': _out } # Return everything return out
def build_net(_X, _weights, _biases, _dropout, _pool_dim, _mean=True, _is_dropout=True, _is_relu=True): # Reshape input picture _X = tf.reshape(_X, shape=[-1, 28, 28, 1]) # Convolution Layer conv1 = conv2d(_X, _weights['wc1'], _biases['bc1']) # Max Pooling (down-sampling) if _mean: conv1 = mean_pool(conv1, k=_pool_dim) else: conv1 = max_pool(conv1, k=_pool_dim) # Apply Dropout if _is_dropout: conv1 = tf.nn.dropout(conv1, _dropout) # Fully connected layer print _weights['wd1'] print _weights['wd1'].get_shape() print _weights['wd1'].get_shape().as_list() print _weights['wd1'].get_shape().as_list()[0] dense1 = tf.reshape(conv1, [-1, _weights['wd1'].get_shape().as_list()[0]]) # Reshape conv2 output to fit dense layer input if _is_relu: dense1 = tf.nn.relu(tf.add(tf.matmul(dense1, _weights['wd1']), _biases['bd1'])) # Relu activation else: dense1 = tf.nn.tanh(tf.add(tf.matmul(dense1, _weights['wd1']), _biases['bd1'])) # tanh activation if _is_dropout: dense1 = tf.nn.dropout(dense1, _dropout) # Apply Dropout # Output, class prediction out = tf.add(tf.matmul(dense1, _weights['out']), _biases['out']) return out
def test_tf_consistency(self): """ Should get the same graph as running pure tf """ x_val = 2702.142857 g = tf.Graph() with tf.Session(graph=g) as sess: x = tf.placeholder(tf.double, shape=[], name="x") z = tf.add(x, 3, name='z') gdef_ref = g.as_graph_def(add_shapes=True) z_ref = sess.run(z, {x: x_val}) with IsolatedSession() as issn: x = tf.placeholder(tf.double, shape=[], name="x") z = tf.add(x, 3, name='z') gfn = issn.asGraphFunction([x], [z]) z_tgt = issn.run(z, {x: x_val}) self.assertEqual(z_ref, z_tgt) # Version texts are not essential part of the graph, ignore them gdef_ref.ClearField("versions") gfn.graph_def.ClearField("versions") # The GraphDef contained in the GraphFunction object # should be the same as that in the one exported directly from TensorFlow session self.assertEqual(str(gfn.graph_def), str(gdef_ref))
def multilayer_perceptron(x, weights, biases): # Hidden layer with RELU activation layer_1 = tf.add(tf.matmul(x, weights['h1']), biases['b1']) layer_1 = tf.nn.relu(layer_1) layer_1 = tf.nn.dropout(layer_1, dropout) # Hidden layer with RELU activation layer_2 = tf.add(tf.matmul(layer_1, weights['h2']), biases['b2']) layer_2 = tf.nn.relu(layer_2) #layer_2 = tf.nn.dropout(layer_2, dropout) # Hidden layer with RELU activation layer_3 = tf.add(tf.matmul(layer_2, weights['h3']), biases['b3']) layer_3 = tf.nn.relu(layer_3) #layer_3 = tf.nn.dropout(layer_3, dropout) # Hidden layer with RELU activation layer_4 = tf.add(tf.matmul(layer_3, weights['h4']), biases['b4']) layer_4 = tf.nn.relu(layer_4) #layer_4 = tf.nn.dropout(layer_4, dropout) # Hidden layer with RELU activation layer_5 = tf.add(tf.matmul(layer_4, weights['h5']), biases['b5']) layer_5 = tf.nn.relu(layer_5) #layer_5 = tf.nn.dropout(layer_5, dropout) # Output layer with linear activation out_layer = tf.matmul(layer_5, weights['out']) + biases['out'] #out_layer = tf.nn.dropout(out_layer, 0.5) return out_layer
def lenet3_traffic(features, keep_prob): """ Define simple Lenet-like model with one convolution layer and three fully connected layers. """ # Convolutional layer 1 l1_strides = (1, 1, 1, 1) l1_padding = 'VALID' l1_conv = tf.nn.conv2d(features, L1_W, l1_strides, l1_padding) l1_biases = tf.nn.bias_add(l1_conv, L1_B) # Activation. l1_relu = tf.nn.relu(l1_biases) # Pooling. Input = 28x28xL1_DEPTH. Output = 14x14xL1_DEPTH. l1_pool = tf.nn.max_pool(l1_relu, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], \ padding='VALID') # Flatten. Input = 14x14xL1_DEPTH. Output = L1_SIZE. flat = flatten(l1_pool) print("Flatten dimensions:", flat.get_shape()) # Layer 2: Fully Connected. Input = L1_SIZE. Output = L2_SIZE. l2_linear = tf.add(tf.matmul(flat, L2_W), L2_B) # Activation. l2_relu = tf.nn.relu(l2_linear) l2_drop = tf.nn.dropout(l2_relu, keep_prob) # Layer 3: Fully Connected. Input = 500. Output = 43. return tf.add(tf.matmul(l2_drop, L3_W), L3_B)
def fcn(image,weights=None): ### use bilinear net = vgg_net(image,weights=weights) conv_final_layer = net["conv5_3"] pool5 = tf.nn.max_pool(conv_final_layer, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding="SAME") conv6 = tf.layers.conv2d(pool5,4096,7,padding='SAME',name='conv6') relu6 = tf.nn.relu(conv6, name="relu6") relu_dropout6 = tf.nn.dropout(relu6, keep_prob=keep_prob) conv7 = tf.layers.conv2d(relu_dropout6,4096,1,name="conv7") relu7 = tf.nn.relu(conv7, name="relu7") relu_dropout7 = tf.nn.dropout(relu7, keep_prob=keep_prob) # now to upscale to actual image size deconv_shape1 = net["pool4"].get_shape() conv8 = tf.layers.conv2d(relu_dropout7,NUM_OF_CLASSESS,1,padding="SAME") conv_t1 = tf.image.resize_bilinear(conv8,deconv_shape1[1:3]) tmp1 = tf.layers.conv2d(net["pool4"],NUM_OF_CLASSESS, 1 ,padding='SAME') fuse_1 = tf.add(conv_t1, tmp1, name="fuse_1") deconv_shape2 = net["pool3"].get_shape() conv_t2 = tf.image.resize_bilinear(fuse_1,deconv_shape2[1:3]) tmp2 = tf.layers.conv2d(net["pool3"],NUM_OF_CLASSESS,1,padding="SAME") fuse_2 = tf.add(conv_t2, tmp2, name="fuse_2") shape = tf.shape(image) conv_t3 = tf.image.resize_bilinear(fuse_2,shape[1:3]) return conv_t3
def feed_forward(_X,_weights,_biases,_dropout): """ Args: _X: batch of images placeholders _weights: weight variables _biases: biases variables _dropout: keep probability Return: out: Predictions of the forward pass """ _X = tf.reshape(_X,[-1,28,28,1]) #Convolutional Layer 1 conv1 = conv2d(_X,_weights['wc1'],_biases['bc1']) conv1 = max_pool(conv1,k=2) conv1 = tf.nn.dropout(conv1,_dropout) #Convolutional Layer 2 conv2 = conv2d(conv1,_weights['wc2'],_biases['bc2']) conv2 = max_pool(conv2,k=2) conv2 = tf.nn.dropout(conv2,_dropout) #Fully Connected Layer 1 dense1 = tf.reshape(conv2,[-1,_weights['wd1'].get_shape().as_list()[0]]) dense1 = tf.nn.relu(tf.add(tf.matmul(dense1,_weights['wd1']),_biases['bd1'])) dense1 = tf.nn.dropout(dense1,_dropout) #Prediction Layer out = tf.add(tf.matmul(dense1,_weights['out']),_biases['out']) return out
def deepnn(x): # Reshape to use within a convolutional neural net. # Last dimension is for "features" - there is only one here, since images are # grayscale -- it would be 3 for an RGB image, 4 for RGBA, etc. with tf.name_scope('reshape'): x_image = tf.reshape(x, [-1, 28, 28, 1]) # First convolutional layer - maps one grayscale image to 32 feature maps. with tf.name_scope('conv1'): # W_conv1 = weight_variable([5, 5, 1, 32]) # w_1 = tf.Variable(initial_value=tf.random_normal(shape=[5, 5, IMG_CHANNEL, 32], dtype=tf.float32), name='w_1') w_1 = tf.Variable(initial_value=tf.truncated_normal( shape=[5, 5, IMG_CHANNEL, 32], stddev=0.1, dtype=tf.float32), name='w_1') b_1 = tf.Variable(initial_value=tf.constant(value=0.1, shape=[32], dtype=tf.float32), name='b_1') # h_conv1 = tf.nn.relu(conv2d(x_image, w_1) + b_1) result_1_conv = tf.nn.conv2d(input=x_image, filter=w_1, strides=[1, 1, 1, 1], padding='SAME', name='conv_1') result_1_relu = tf.nn.relu(tf.nn.bias_add(result_1_conv, b_1), name='relu_1') print(result_1_relu) # Pooling layer - downsamples by 2X. with tf.name_scope('pool1'): # h_pool1 = max_pool_2x2(result_1_relu) result_1_maxpool = tf.nn.max_pool(value=result_1_relu, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME', name='maxpool_1') print(result_1_maxpool) # Second convolutional layer -- maps 32 feature maps to 64. with tf.name_scope('conv2'): # W_conv2 = weight_variable([5, 5, 32, 64]) # w_2 = tf.Variable(initial_value=tf.random_normal(shape=[5, 5, 32, 64], dtype=tf.float32), name='w_2') # w_2 = weight_variable([5, 5, 32, 64]) w_2 = tf.Variable(initial_value=tf.truncated_normal( shape=[5, 5, 32, 64], stddev=0.1, dtype=tf.float32), name='w_2') b_2 = tf.Variable(initial_value=tf.constant(value=0.1, shape=[64], dtype=tf.float32), name='b_2') result_2_conv = tf.nn.conv2d(input=result_1_maxpool, filter=w_2, strides=[1, 1, 1, 1], padding='SAME', name='conv_2') result_2_relu = tf.nn.relu(tf.add(result_2_conv, b_2), name='relu_2') # b_conv2 = bias_variable([64]) # h_conv2 = tf.nn.relu(conv2d(result_1_maxpool, w_2) + b_2) print(result_2_relu) # Second pooling layer. with tf.name_scope('pool2'): # h_pool2 = max_pool_2x2(result_2_relu) result_2_maxpool = tf.nn.max_pool(value=result_2_relu, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME', name='maxpool_2') print(result_2_maxpool) # Fully connected layer 1 -- after 2 round of downsampling, our 28x28 image # is down to 7x7x64 feature maps -- maps this to 1024 features. with tf.name_scope('fc1'): w_3 = weight_variable([7 * 7 * 64, 1024]) # w_3 = tf.Variable(initial_value=tf.random_normal(shape=[3136, 1024], dtype=tf.float32), name='w_4') b_3 = tf.Variable(initial_value=tf.random_normal(shape=[1024], dtype=tf.float32), name='b_4') # b_3 = bias_variable([1024]) '''问题所在,变量初始化有问题,他的效果比较好''' # h_pool2_flat = tf.reshape(result_2_maxpool, [-1, 7*7*64]) result_expand = tf.reshape(result_2_maxpool, [-1, 7 * 7 * 64]) # h_fc1 = tf.nn.relu(tf.matmul(h_pool2_flat, W_fc1) + b_fc1) result_3_fc = tf.matmul(result_expand, w_3, name='result_4_multiply') result_3_relu = tf.nn.relu(tf.add(result_3_fc, b_3), name='relu_4') # Dropout - controls the complexity of the model, prevents co-adaptation of # features. with tf.name_scope('dropout'): keep_prob = tf.placeholder(tf.float32) # h_fc1_drop = tf.nn.dropout(result_3_relu, keep_prob) result_3_dropout = tf.nn.dropout(x=result_3_relu, keep_prob=keep_prob, name='dropout_4') # Map the 1024 features to 10 classes, one for each digit with tf.name_scope('fc2'): w_4 = weight_variable([1024, 10]) b_4 = bias_variable([10]) # y_conv = tf.matmul(result_3_dropout, w_4) + b_2 # w_4 = tf.Variable(initial_value=tf.truncated_normal(shape=[1024, 10], stddev=0.1, dtype=tf.float32)) # b_4 = tf.Variable(initial_value=tf.random_normal(shape=[10], dtype=tf.float32), name='b_4') result_4_fc = tf.nn.bias_add(tf.matmul(result_3_dropout, w_4), b_4, name='result_4_fc') # result_4_relu = tf.nn.relu(result_4_fc, name='result_4_relu') result_4_relu = result_4_fc return result_4_relu, keep_prob
with tf.name_scope("Loss"): # define loss with tf.name_scope("cross_entropy"): ce_loss = tf.reduce_mean( tf.nn.softmax_cross_entropy_with_logits(logits=net, labels=y)) # define regularizer with tf.name_scope("regularizer"): with tf.name_scope("jacobians"): jacobians = tf_jacobian(net, x, batch_size) with tf.name_scope("regularizer_cal"): regularizer = tf.reduce_sum( tf.norm(tf.gather(jacobians, ind_i, axis=2) - tf.gather(jacobians, ind_j, axis=2), axis=1) * similarities) # get final loss by adding loss and regularizer customized_loss = tf.add(ce_loss, args.lambda_reg * regularizer) # define optimizer with tf.name_scope("optimizer"): optimizer = tf.train.AdamOptimizer( learning_rate=learning_rate).minimize(customized_loss) # define accuracy with tf.name_scope("accuracy"): prediction = tf.nn.softmax(net) correct_pred = tf.equal(tf.argmax(prediction, 1), tf.argmax(y, 1)) accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32)) # create a summary for our cost tf.summary.scalar("cost", customized_loss) # create a summary for our training accuracy
# training data y train_Y = np.array([2,3,4,5,6,7,8,9,10,11]) n_samples = train_X.shape[0] # tf Graph Input X = tf.placeholder("float") Y = tf.placeholder("float") # Set model weights W = tf.Variable(np.random.rand(), name="weight") b = tf.Variable(np.random.rand(), name="bias") # Prediction: we just do one pass [x,y] -(nn)> 0 single perceptron pred = tf.add(tf.multiply(X, W), b) # Error error = tf.reduce_sum(tf.pow(pred-Y, 2))/(2*n_samples) # Define Extra Log Information for the tensorboard - we can add more later w/ merge v.1.0 tf.summary.scalar("cost", error) summary_op = tf.summary.merge_all() # Minimize Error optimizer = tf.train.GradientDescentOptimizer(learning_rate).minimize(error) # Init init = tf.global_variables_initializer() # Launch the graph
def build_graph(num_batch,max_sequence_len, hidden_units, num_classes): # [batch_size, sequence_size, feature_len ] x=tf.placeholder(tf.float32, [num_batch, None, parameters.feature_len]) y=tf.placeholder(tf.float32, [num_batch, num_classes]) seqlen = tf.placeholder(tf.int32) keep_prob = tf.constant(0.8) x_image=tf.reshape(x, [num_batch, seqlen, parameters.feature_len, 1]) W_filter1 = tf.Variable(tf.truncated_normal([5, parameters.feature_len, 1, 40], stddev=0.01)) b_filter1 = tf.Variable(tf.truncated_normal([40], stddev=0.01)) W_filter2 = tf.Variable(tf.truncated_normal([3, parameters.feature_len, 1, 40], stddev=0.01)) b_filter2 = tf.Variable(tf.truncated_normal([40], stddev=0.01)) W_filter3 = tf.Variable(tf.truncated_normal([2, parameters.feature_len, 1, 20], stddev=0.01)) b_filter3 = tf.Variable(tf.truncated_normal([20], stddev=0.01)) h_conv1=tf.nn.relu(tf.nn.conv2d(x_image, W_filter1, strides=[1, 1, 1, 1], padding='VALID')+b_filter1) h_conv1_flat=tf.reshape(h_conv1, [-1, 40,seqlen-5+1]) h_pool1=tf.reduce_max(h_conv1_flat, 2) #h_pool1=tf.nn.max_pool(h_conv1, ksize=[1,seqlen-5+1,1, 1], strides=[1, 1, 1, 1], padding='VALID') #h_pool1_flat=tf.reshape(h_pool1, [-1, 40]) h_conv2=tf.nn.relu(tf.nn.conv2d(x_image, W_filter2, strides=[1, 1, 1, 1], padding='VALID')+b_filter2) h_conv2_flat=tf.reshape(h_conv2, [-1, 40,seqlen-3+1]) h_pool2=tf.reduce_max(h_conv2_flat, 2) #h_pool2=tf.nn.max_pool(h_conv2, ksize=[1, seqlen-3+1, 1, 1], strides=[1, 1, 1, 1], padding='VALID') #h_pool2_flat=tf.reshape(h_pool2, [-1, 40]) h_conv3=tf.nn.relu(tf.nn.conv2d(x_image, W_filter3, strides=[1, 1, 1, 1], padding='VALID')+b_filter3) h_conv3_flat=tf.reshape(h_conv3, [-1, 20,seqlen-2+1]) h_pool3=tf.reduce_max(h_conv3_flat, 2) #h_pool3=tf.nn.max_pool(h_conv3, ksize=[1, seqlen-2+1,1, 1], strides=[1, 1, 1, 1], padding='VALID') #h_pool3_flat=tf.reshape(h_pool3, [-1, 20]) intermediate_rep=tf.concat(1,[h_pool1, h_pool2, h_pool3]) #print (int(intermediate_rep.get_shape()[1])) #print ([int(x) for x in intermediate_rep.get_shape()]) #WL1=tf.Variable(tf.truncated_normal([4*4*64, 1024], stddev=0.01)) WL1=tf.Variable(tf.truncated_normal([100, 1024], stddev=0.01)) BL1=tf.Variable(tf.truncated_normal([1024], stddev=0.01)) #WL2=tf.Variable(tf.truncated_normal([1024, 1024], stddev=0.01)) #BL2=tf.Variable(tf.truncated_normal([1024], stddev=0.01)) WL3=tf.Variable(tf.truncated_normal([1024, num_classes], stddev=0.01)) BL3=tf.Variable(tf.truncated_normal([num_classes], stddev=0.01)) #Model outputL1=tf.add(tf.matmul(intermediate_rep, WL1),BL1) outputL1=tf.nn.relu(outputL1) outputL1_drop = tf.nn.dropout(outputL1, keep_prob) #outputL2=tf.add(tf.matmul(outputL1_drop, WL2),BL2) #outputL2=tf.nn.relu(outputL2) #outputL2_drop = tf.nn.dropout(outputL2, keep_prob) outputL3=tf.add(tf.matmul(outputL1_drop, WL3), BL3) loss=tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(outputL3, y)) predictions=tf.nn.softmax(outputL3) output=tf.argmax(predictions, 1) optimizer=tf.train.AdamOptimizer() minimizer=optimizer.minimize(loss) saver=tf.train.Saver() return dict( x = x, y = y, seqlen = seqlen, loss = loss, output=output, minimizer = minimizer, predictions = predictions, saver=saver ) #myRnnModel=build_graph(num_batch=parameters.num_batch, max_sequence_len=parameters.max_sequence_len, hidden_units=parameters.hidden_units, num_classes=parameters.num_classes)
strides=[1, 2, 2, 1], padding='SAME') reshape = tf.reshape(pool2, [batch_size, -1]) dim = reshape.get_shape()[1].value weight3 = variable_with_weight_loss(shape=[dim, 384], stddev=0.04, wl=0.004) bias3 = tf.Variable(tf.constant(0.1, shape=[384])) local3 = tf.nn.relu(tf.matmul(reshape, weight3) + bias3) weight4 = variable_with_weight_loss(shape=[384, 192], stddev=0.04, wl=0.004) bias4 = tf.Variable(tf.constant(0.1, shape=[192])) local4 = tf.nn.relu(tf.matmul(local3, weight4) + bias4) weight5 = variable_with_weight_loss(shape=[192, 10], stddev=1 / 192.0, wl=0.0) bias5 = tf.Variable(tf.constant(0.0, shape=[10])) logits = tf.add(tf.matmul(local4, weight5), bias5) loss = loss(logits, label_holder) train_op = tf.train.AdamOptimizer(1e-3).minimize(loss) # 0.72 top_k_op = tf.nn.in_top_k(logits, label_holder, 1) sess = tf.InteractiveSession() tf.global_variables_initializer().run() tf.train.start_queue_runners() ### for step in range(max_steps): start_time = time.time() image_batch, label_batch = sess.run([images_train, labels_train])
with tf.name_scope('Flattern'): x_flatten = tf.reshape(x, [-1, nCh * 30]) if Debug: print('x_image shape: ', x_flatten.shape) firstIn = nCh * 30 # 13*30 = 480 firstOut = 1024 # ---- number of hidden units in the first layer. with tf.name_scope('ReLu-1'): #w1 = tf.Variable(tf.truncated_normal([firstIn, firstOut], stddev=0.1), name = 'W') #b1 = tf.Variable(tf.constant(0.1, shape=[firstOut]), name = 'B' ) w1 = tf.get_variable('W1', [firstIn, firstOut], initializer=tf.contrib.layers.xavier_initializer()) b1 = tf.get_variable('B1', [1, firstOut], initializer=tf.contrib.layers.xavier_initializer()) z1 = tf.add(tf.matmul(x_flatten, w1), b1) a1 = tf.nn.relu(z1) # summary tf.summary.histogram('weights', w1) tf.summary.histogram('biases', b1) tf.summary.histogram('z', z1) tf.summary.histogram('activation', a1) # dimensionality checking if Debug: print('w1 shape: ', w1.shape) print('b1 shape: ', b1.shape) print('z1 shape: ', z1.shape) print('a1 shape: ', a1.shape) secondIn = firstOut
############################################### ############## 1st dropout layer ############# keep_prob = tf.placeholder(tf.float32) # the probability to keep a neuron's output #with tf.name_scope("drop_lay"): # keep_prob = tf.placeholder(tf.float32) # the probability to keep a neuron's output #h_fc1_drop = tf.nn.dropout(h_fc2, keep_prob) # used to avoid overfitting on large networks ########################################### ############## readout layer ############# with tf.name_scope("readout_layer"): W_fc_rd = weight_variable([500 # nb of neurons in previous layer , 4],name="W_fc_rd") # nb of class b_fc_rd = bias_variable([4]) y_conv = tf.add(tf.matmul(h_fc2, W_fc_rd) , b_fc_rd, name="y_conv") # linear combination ###################################### ############ 1St TRAINING ############ ###################################### cross_entropy = tf.reduce_mean( tf.nn.softmax_cross_entropy_with_logits( # transform distance into prob labels=y_ # target , logits=y_conv+tf.constant(value=1e-07)), name="cross_entropy") # estimation #cross_entropy = -tf.reduce_sum(y_*tf.log(y_conv + 1e-10)) #cross_entropy = -tf.reduce_sum(y_*tf.log(tf.clip_by_value(y_conv,1e-8,1.0))) #cross_entropy = tf.reduce_mean( # tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y_conv)) #epsilon = tf.constant(value=0.000001)
def main(): train_X, train_y, test_X, test_y, minCPU,maxCPU = get_iris_data() train_X = train_X.reshape(train_X.shape[0],train_X.shape[1]) test_X = test_X.reshape(test_X.shape[0],test_X.shape[1] ) learning_rate = 0.001 training_epochs = 200 batch_size = 100 display_step = 1 # Layer's sizes n_input = train_X.shape[1] # Number of input nodes: 4 features and 1 bias n_hidden_1 = 3 # Number of hidden nodes n_output = train_y.shape[1] print minCPU print maxCPU print train_X[0] print test_X[0] print train_y[0] print len(train_y) # Number of outcomes (3 iris flowers) print n_input print n_output # Symbols x = tf.placeholder("float", [None, n_input]) y = tf.placeholder("float", [None, n_output]) h1 = tf.Variable(tf.random_normal([n_input, n_hidden_1], dtype=tf.float32)) h2 = tf.Variable(tf.random_normal([n_hidden_1, n_output])) bias_layer_1 = tf.Variable(tf.random_normal([n_hidden_1])) layer_1 = tf.nn.sigmoid(tf.add(tf.matmul(x,h1),bias_layer_1)) bias_output = tf.Variable(tf.random_normal([n_output])) output_layer = tf.nn.sigmoid(tf.matmul(layer_1,h2),bias_output) # Backward propagation cost = tf.reduce_mean(tf.add(tf.square(y-output_layer),) # optimizer = tf.train.GradientDescentOptimizer(0.01).minimize(cost) optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost) # Run SGD avg_set = [] epoch_set=[] init = tf.global_variables_initializer() with tf.Session() as sess: sess.run(init) print sess.run(h1) print sess.run(h2) print sess.run(bias_layer_1) # print sess.run(bias_layer_2) for epoch in range(training_epochs): # Train with each example total_batch = int(len(train_X)/batch_size) for i in range(total_batch): # sess.run(updates) avg_cost = 0 for i in range(total_batch): batch_xs, batch_ys = train_X[i*batch_size:(i+1)*batch_size], train_y[i*batch_size:(i+1)*batch_size] sess.run(optimizer, feed_dict={x: batch_xs, y: batch_ys}) avg_cost += sess.run(cost,feed_dict={x: batch_xs,y: batch_ys})/total_batch # Display logs per epoch step if epoch % display_step == 0: print "Epoch:", '%04d' % (epoch+1),"cost=", "{:.9f}".format(avg_cost) avg_set.append(avg_cost) epoch_set.append(epoch+1) print "Training phase finished" # plt.plot(epoch_set,avg_set, 'o', label='MLP Training phase') # plt.ylabel('cost') # plt.xlabel('epoch') # plt.legend() # plt.show() print 'h1 ',sess.run(h1) # tf.cast(h1, tf.float64) print 'h2 ',sess.run(h2) print 'bias layer 1 ', sess.run(bias_layer_1) # predictions = [] # for i in range(len(test_X)): layer_1_predictioni = tf.nn.sigmoid(tf.add(tf.matmul(tf.cast(test_X,'float'),h1),bias_layer_1)) predictions = tf.nn.sigmoid(tf.matmul(layer_1_predictioni,h2)) # predictions = scaler.inverse_transform(predictions) predictions = predictions * (maxCPU - minCPU) + minCPU error = tf.reduce_sum(tf.abs(tf.subtract(predictions,test_y)))/len(test_y) print 'predictions', sess.run(predictions) print 'error', sess.run(error) # correct_prediction = tf.equal(tf.argmax(output_layer, 1),tf.argmax(y, 1)) # # evaluating its accuracy # accuracy = tf.reduce_mean(tf.cast(correct_prediction, "float")) # print "Model Accuracy:", accuracy.eval({x: mnist.test.images,y: mnist.test.labels}) # train_accuracy = np.mean(np.square(train_y, axis=1) == # sess.run(predict, feed_dict={X: train_X, y: train_y})) # test_accuracy = np.mean(np.argmax(test_y, axis=1) == # sess.run(predict, feed_dict={X: test_X, y: test_y})) sess.close() if __name__ == '__main__': main()
def __init__(self, sequence_length, num_classes, vocab_size, embedding_size, hidden_size, l2_reg_lambda=0.0): # Placeholders for input, output and dropout self.input_text = tf.placeholder(tf.int32, shape=[None, sequence_length], name='input_text') self.input_position = tf.placeholder(tf.int32, shape=[None, 2], name='input_position') self.input_y = tf.placeholder(tf.int64, shape=[None], name='input_y') self.mask = tf.placeholder(tf.int32, shape=[1, sequence_length], name='position_mask') self.emb_dropout_keep_prob = tf.placeholder(tf.float32, name='emb_dropout_keep_prob') self.rnn_dropout_keep_prob = tf.placeholder(tf.float32, name='rnn_dropout_keep_prob') self.dropout_keep_prob = tf.placeholder(tf.float32, name='dropout_keep_prob') initializer = tf.keras.initializers.glorot_normal # Word Embedding Layer with tf.device('/cpu:0'), tf.variable_scope("word-embeddings"): self.W_text = tf.Variable(tf.random_uniform([vocab_size, embedding_size], -0.25, 0.25), name="W_text") self.embedded_chars = tf.nn.embedding_lookup(self.W_text, self.input_text) # Dropout for Word Embedding with tf.variable_scope('dropout-embeddings'): self.embedded_chars = tf.nn.dropout(self.embedded_chars, self.emb_dropout_keep_prob) # Position Feature with tf.variable_scope('position-features'): self.pf = self.mask[:,:,None] - self.input_position[:,None,:] # Bidirectional LSTM with tf.variable_scope("bi-lstm"): self.embedded_input = tf.concat([self.embedded_chars, tf.to_float(self.pf)], axis=2) _fw_cell = tf.nn.rnn_cell.LSTMCell(hidden_size, initializer=initializer()) fw_cell = tf.nn.rnn_cell.DropoutWrapper(_fw_cell, self.rnn_dropout_keep_prob) _bw_cell = tf.nn.rnn_cell.LSTMCell(hidden_size, initializer=initializer()) bw_cell = tf.nn.rnn_cell.DropoutWrapper(_bw_cell, self.rnn_dropout_keep_prob) self.rnn_outputs, _ = tf.nn.bidirectional_dynamic_rnn(cell_fw=fw_cell, cell_bw=bw_cell, inputs=self.embedded_input, sequence_length=self._length(self.input_text), dtype=tf.float32) self.rnn_outputs = tf.add(self.rnn_outputs[0], self.rnn_outputs[1]) # Attention with tf.variable_scope('attention'): self.attn, self.alphas = attention(self.rnn_outputs) # Dropout with tf.variable_scope('dropout'): self.h_drop = tf.nn.dropout(self.attn, self.dropout_keep_prob) # Fully connected layer with tf.variable_scope('output'): self.logits = tf.layers.dense(self.h_drop, num_classes, kernel_initializer=initializer()) self.predictions = tf.argmax(self.logits, 1, name="predictions") self.probabilities = tf.nn.softmax(self.logits, name="probabilities")[:, 1] # Calculate mean cross-entropy loss with tf.variable_scope("loss"): losses = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=self.logits, labels=self.input_y) self.l2 = tf.add_n([tf.nn.l2_loss(v) for v in tf.trainable_variables()]) self.loss = tf.reduce_mean(losses) + l2_reg_lambda * self.l2 # Accuracy with tf.variable_scope("accuracy"): correct_predictions = tf.equal(self.predictions, self.input_y) self.accuracy = tf.reduce_mean(tf.cast(correct_predictions, tf.float32), name="accuracy")
def pixel_wise_softmax(output_map): exponential_map = tf.exp(output_map) evidence = tf.add(exponential_map, tf.reverse(exponential_map, [False, False, False, True])) return tf.div(exponential_map, evidence, name="pixel_wise_softmax")
batch = mnist.train.next_batch(100) x_batch = batch[0] batch_tensor = tf.reshape(x_batch, [100, 28, 28, 1]) resized_images = tf.image.resize_images(batch_tensor, [7,7]) # Initialize Weights W1 = tf.Variable(tf.random_normal([784, n_hidden1])) W2 = tf.Variable(tf.random_normal([n_hidden1, n_hidden2])) W_output = tf.Variable(tf.random_normal([n_hidden2, 10])) B1 = tf.Variable(tf.random_normal([n_hidden1])) B2 = tf.Variable(tf.random_normal([n_hidden2])) B_output = tf.Variable(tf.random_normal([10])) # Construct model L1 = tf.nn.relu(tf.add(tf.matmul(X_train,W1),B1)) L2 = tf.nn.relu(tf.add(tf.matmul(L1,W2),B2)) # Output layer L_output = tf.add(tf.matmul(L2, W_output), B_output) # Cross_entropy and Accuracy cross_entropy = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=Y_train, logits=L_output)) # Softmax loss optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cross_entropy) # Adam Optimizer correct_pred = tf.equal(tf.argmax(L_output, 1), tf.argmax(Y_train, 1)) accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32)) init = tf.initialize_all_variables() with tf.Session() as sess:
# 加载mnist数据集 from tensorflow.examples.tutorials.mnist import input_data mnist = input_data.read_data_sets("mnistData", one_hot=True) # 这里从数据集中抽取5000个样本数据作为训练集,200个样本做为测试集,这里Ytr是一个10维ndarray Xtr, Ytr = mnist.train.next_batch(5000) Xte, Yte = mnist.test.next_batch(200) # tf Graph的输入 xtr = tf.placeholder("float", [None, 784]) xte = tf.placeholder("float", [784]) # 使用L1距离计算最近邻 # tf.neg()给xte中的每个元素取反,这样可以使用tf.add() distance = tf.reduce_sum(tf.abs(tf.add(xtr, tf.negative(xte))), reduction_indices=1) # 这里reduction_indices=1是计算每一行的和 # 从distance tensor中寻找最小的距离索引 pred = tf.argmin(distance, 0) accuracy = 0. # 初始化所有 variables init = tf.global_variables_initializer() # Launch graph with tf.Session() as sess: sess.run(init) # 对每个测试样本,计算它的分类类别 for i in range(len(Xte)):
def add_model(self): # user_input = tf.nn.embedding_lookup(self.user_item_embedding, self.user) user_input = self.user_item_embedding item_input = self.item_user_embedding self.id_input = tf.nn.embedding_lookup(self.id_matrix, self.user) self.n_id = tf.reduce_sum(self.id_input) user_input = sparse_dropout(user_input, self.drop, self.dataSet.n_train) item_input = sparse_dropout(item_input, self.drop, self.dataSet.n_train) def init_variable(shape, name): return tf.get_variable( name, shape, initializer=tf.contrib.layers.xavier_initializer(), dtype=tf.float64) with tf.name_scope("User_Layer"): user_W1 = init_variable([self.shape[1], self.userLayer[0]], "user_W1") user_b1 = tf.get_variable("user_b1", [self.userLayer[0]], initializer=tf.constant_initializer(0.0), dtype=tf.float64) self.user_out = tf.nn.sigmoid( tf.sparse_tensor_dense_matmul(user_input, user_W1) + user_b1) for i in range(0, len(self.userLayer) - 1): W = init_variable([self.userLayer[i], self.userLayer[i + 1]], "user_W" + str(i + 2)) b = tf.get_variable("user_b" + str(i + 2), [self.userLayer[i + 1]], initializer=tf.constant_initializer(0.0), dtype=tf.float64) self.user_out = tf.nn.tanh( tf.add(tf.matmul(self.user_out, W), b)) with tf.name_scope("Item_Layer"): item_W1 = init_variable([self.shape[0], self.itemLayer[0]], "item_W1") item_b1 = tf.get_variable("item_b1", [self.itemLayer[0]], initializer=tf.constant_initializer(0.0), dtype=tf.float64) self.item_preout = tf.nn.sigmoid( tf.sparse_tensor_dense_matmul(item_input, item_W1) + item_b1) for i in range(0, len(self.itemLayer) - 1): W = init_variable([self.itemLayer[i], self.itemLayer[i + 1]], "item_W" + str(i + 2)) b = tf.get_variable("item_b" + str(i + 2), [self.itemLayer[i + 1]], initializer=tf.constant_initializer(0.0), dtype=tf.float64) self.item_preout = tf.nn.tanh( tf.add(tf.matmul(self.item_preout, W), b)) self.user_tile = tf.reshape( tf.tile(self.user_out, [1, self.dataSet.maxn_item]), (-1, self.itemLayer[-1])) self.user_tile2 = tf.reshape(tf.tile(self.user_out, [1, self.negNum]), (-1, self.itemLayer[-1])) self.item_out = tf.nn.embedding_lookup(self.item_preout, self.item) self.item2_out = tf.nn.embedding_lookup(self.item_preout, self.item2) self.y = tf.reduce_sum(tf.multiply(self.user_tile, self.item_out), axis=1) self.y2 = tf.reduce_sum(tf.multiply(self.user_tile2, self.item2_out), axis=1) self.y_ = tf.reshape(tf.nn.sigmoid(self.y), (-1, self.dataSet.maxn_item)) self.y2_ = tf.reshape(tf.nn.sigmoid(self.y2), (-1, self.negNum)) self.y_sum = tf.reshape(tf.reduce_sum(self.y2_, axis=1), (-1, 1)) + self.y_
def __init__(self, sess: tf.Session, predict: Union[Callable, tf.keras.Model, 'keras.Model'], shape: tuple, kappa: float = 0., beta: float = .1, feature_range: tuple = (-1e10, 1e10), gamma: float = 0., ae_model: Union[tf.keras.Model, 'keras.Model'] = None, enc_model: Union[tf.keras.Model, 'keras.Model'] = None, theta: float = 0., use_kdtree: bool = False, learning_rate_init: float = 1e-2, max_iterations: int = 1000, c_init: float = 10., c_steps: int = 10, eps: tuple = (1e-3, 1e-3), clip: tuple = (-1000., 1000.), update_num_grad: int = 1, write_dir: str = None) -> None: """ Initialize prototypical counterfactual method. Parameters ---------- sess TensorFlow session predict Keras or TensorFlow model or any other model's prediction function returning class probabilities shape Shape of input data starting with batch size kappa Confidence parameter for the attack loss term beta Regularization constant for L1 loss term feature_range Tuple with min and max ranges to allow for perturbed instances. Min and max ranges can be floats or numpy arrays with dimension (1x nb of features) for feature-wise ranges gamma Regularization constant for optional auto-encoder loss term ae_model Optional auto-encoder model used for loss regularization enc_model Optional encoder model used to guide instance perturbations towards a class prototype theta Constant for the prototype search loss term use_kdtree Whether to use k-d trees for the prototype loss term if no encoder is available learning_rate_init Initial learning rate of optimizer max_iterations Maximum number of iterations for finding a counterfactual c_init Initial value to scale the attack loss term c_steps Number of iterations to adjust the constant scaling the attack loss term eps If numerical gradients are used to compute dL/dx = (dL/dp) * (dp/dx), then eps[0] is used to calculate dL/dp and eps[1] is used for dp/dx. eps[0] and eps[1] can be a combination of float values and numpy arrays. For eps[0], the array dimension should be (1x nb of prediction categories) and for eps[1] it should be (1x nb of features) clip Tuple with min and max clip ranges for both the numerical gradients and the gradients obtained from the TensorFlow graph update_num_grad If numerical gradients are used, they will be updated every update_num_grad iterations write_dir Directory to write tensorboard files to """ self.sess = sess self.predict = predict # check whether the model, encoder and auto-encoder are Keras or TF models try: import keras # noqa is_model = isinstance(predict, (tf.keras.Model, keras.Model)) is_ae = isinstance(ae_model, (tf.keras.Model, keras.Model)) is_enc = isinstance(enc_model, (tf.keras.Model, keras.Model)) except ImportError: is_model = isinstance(predict, (tf.keras.Model)) is_ae = isinstance(ae_model, (tf.keras.Model)) is_enc = isinstance(enc_model, (tf.keras.Model)) if is_model: self.model = True self.classes = self.sess.run( self.predict( tf.convert_to_tensor(np.zeros(shape), dtype=tf.float32))).shape[1] else: self.model = False self.classes = self.predict(np.zeros(shape)).shape[1] if is_enc: self.enc_model = True else: self.enc_model = False if is_ae: self.ae_model = True else: self.ae_model = False if use_kdtree and self.enc_model: logger.warning( 'Both an encoder and k-d trees enabled. Using the encoder for the prototype loss term.' ) if use_kdtree or self.enc_model: self.enc_or_kdtree = True else: self.enc_or_kdtree = False self.shape = shape self.kappa = kappa self.beta = beta self.gamma = gamma self.theta = theta self.ae = ae_model self.enc = enc_model self.use_kdtree = use_kdtree self.batch_size = shape[0] self.max_iterations = max_iterations self.c_init = c_init self.c_steps = c_steps self.update_num_grad = update_num_grad self.eps = eps self.clip = clip self.write_dir = write_dir # define tf variables for original and perturbed instances, and target labels self.orig = tf.Variable(np.zeros(shape), dtype=tf.float32, name='orig') self.adv = tf.Variable(np.zeros(shape), dtype=tf.float32, name='adv') self.adv_s = tf.Variable(np.zeros(shape), dtype=tf.float32, name='adv_s') self.target = tf.Variable(np.zeros((self.batch_size, self.classes)), dtype=tf.float32, name='target') # variable for target class proto if self.enc_model: self.shape_enc = self.enc.predict(np.zeros(shape)).shape else: self.shape_enc = shape self.target_proto = tf.Variable(np.zeros(self.shape_enc), dtype=tf.float32, name='target_proto') # define tf variable for constant used in FISTA optimization self.const = tf.Variable(np.zeros(self.batch_size), dtype=tf.float32, name='const') self.global_step = tf.Variable(0.0, trainable=False, name='global_step') # define placeholders that will be assigned to relevant variables self.assign_orig = tf.placeholder(tf.float32, shape, name='assign_orig') self.assign_adv = tf.placeholder(tf.float32, shape, name='assign_adv') self.assign_adv_s = tf.placeholder(tf.float32, shape, name='assign_adv_s') self.assign_target = tf.placeholder(tf.float32, (self.batch_size, self.classes), name='assign_target') self.assign_const = tf.placeholder(tf.float32, [self.batch_size], name='assign_const') self.assign_target_proto = tf.placeholder(tf.float32, self.shape_enc, name='assign_target_proto') # define conditions and values for element-wise shrinkage thresholding with tf.name_scope('shrinkage_thresholding') as scope: cond = [ tf.cast( tf.greater(tf.subtract(self.adv_s, self.orig), self.beta), tf.float32), tf.cast( tf.less_equal(tf.abs(tf.subtract(self.adv_s, self.orig)), self.beta), tf.float32), tf.cast( tf.less(tf.subtract(self.adv_s, self.orig), tf.negative(self.beta)), tf.float32) ] upper = tf.minimum(tf.subtract(self.adv_s, self.beta), tf.cast(feature_range[1], tf.float32)) lower = tf.maximum(tf.add(self.adv_s, self.beta), tf.cast(feature_range[0], tf.float32)) self.assign_adv = tf.multiply(cond[0], upper) + tf.multiply( cond[1], self.orig) + tf.multiply(cond[2], lower) # perturbation update and vector projection on correct feature range set with tf.name_scope('perturbation_y') as scope: self.zt = tf.divide(self.global_step, self.global_step + tf.cast(3, tf.float32)) self.assign_adv_s = self.assign_adv + tf.multiply( self.zt, self.assign_adv - self.adv) # map to feature space self.assign_adv_s = tf.minimum( self.assign_adv_s, tf.cast(feature_range[1], tf.float32)) self.assign_adv_s = tf.maximum( self.assign_adv_s, tf.cast(feature_range[0], tf.float32)) # assign counterfactual of step k+1 to k with tf.name_scope('update_adv') as scope: self.adv_updater = tf.assign(self.adv, self.assign_adv) self.adv_updater_s = tf.assign(self.adv_s, self.assign_adv_s) # from perturbed instance, derive deviation delta with tf.name_scope('update_delta') as scope: self.delta = self.orig - self.adv self.delta_s = self.orig - self.adv_s # define L1 and L2 loss terms; L1+L2 is later used as an optimization constraint for FISTA ax_sum = list(np.arange(1, len(shape))) with tf.name_scope('loss_l1_l2') as scope: self.l2 = tf.reduce_sum(tf.square(self.delta), axis=ax_sum) self.l2_s = tf.reduce_sum(tf.square(self.delta_s), axis=ax_sum) self.l1 = tf.reduce_sum(tf.abs(self.delta), axis=ax_sum) self.l1_s = tf.reduce_sum(tf.abs(self.delta_s), axis=ax_sum) self.l1_l2 = self.l2 + tf.multiply(self.l1, self.beta) self.l1_l2_s = self.l2_s + tf.multiply(self.l1_s, self.beta) # sum losses self.loss_l1 = tf.reduce_sum(self.l1) self.loss_l1_s = tf.reduce_sum(self.l1_s) self.loss_l2 = tf.reduce_sum(self.l2) self.loss_l2_s = tf.reduce_sum(self.l2_s) with tf.name_scope('loss_ae') as scope: # gamma * AE loss if self.ae_model: self.loss_ae = self.gamma * tf.square( tf.norm(self.ae(self.adv) - self.adv)) self.loss_ae_s = self.gamma * tf.square( tf.norm(self.ae(self.adv_s) - self.adv_s)) else: # no auto-encoder available self.loss_ae = tf.constant(0.) self.loss_ae_s = tf.constant(0.) with tf.name_scope('loss_attack') as scope: if not self.model: self.loss_attack = tf.placeholder(tf.float32) elif self.c_init == 0. and self.c_steps == 1: # prediction loss term not used # make predictions on perturbed instance self.pred_proba = self.predict(self.adv) self.pred_proba_s = self.predict(self.adv_s) self.loss_attack = tf.constant(0.) self.loss_attack_s = tf.constant(0.) else: # make predictions on perturbed instance self.pred_proba = self.predict(self.adv) self.pred_proba_s = self.predict(self.adv_s) # probability of target label prediction self.target_proba = tf.reduce_sum( self.target * self.pred_proba, 1) target_proba_s = tf.reduce_sum(self.target * self.pred_proba_s, 1) # max probability of non target label prediction self.nontarget_proba_max = tf.reduce_max( (1 - self.target) * self.pred_proba - (self.target * 10000), 1) nontarget_proba_max_s = tf.reduce_max( (1 - self.target) * self.pred_proba_s - (self.target * 10000), 1) # loss term f(x,d) loss_attack = tf.maximum( 0.0, -self.nontarget_proba_max + self.target_proba + self.kappa) loss_attack_s = tf.maximum( 0.0, -nontarget_proba_max_s + target_proba_s + self.kappa) # c * f(x,d) self.loss_attack = tf.reduce_sum(self.const * loss_attack) self.loss_attack_s = tf.reduce_sum(self.const * loss_attack_s) with tf.name_scope('loss_prototype') as scope: if self.enc_model: self.loss_proto = self.theta * tf.square( tf.norm(self.enc(self.adv) - self.target_proto)) self.loss_proto_s = self.theta * tf.square( tf.norm(self.enc(self.adv_s) - self.target_proto)) elif self.use_kdtree: self.loss_proto = self.theta * tf.square( tf.norm(self.adv - self.target_proto)) self.loss_proto_s = self.theta * tf.square( tf.norm(self.adv_s - self.target_proto)) else: # no encoder available and no k-d trees used self.loss_proto = tf.constant(0.) self.loss_proto_s = tf.constant(0.) with tf.name_scope('loss_combined') as scope: # no need for L1 term in loss to optimize when using FISTA if self.model: self.loss_opt = self.loss_attack_s + self.loss_l2_s + self.loss_ae_s + self.loss_proto_s else: # separate numerical computation of loss attack gradient self.loss_opt = self.loss_l2_s + self.loss_ae_s + self.loss_proto_s # add L1 term to overall loss; this is not the loss that will be directly optimized self.loss_total = (self.loss_attack + self.loss_l2 + self.loss_ae + tf.multiply(self.beta, self.loss_l1) + self.loss_proto) with tf.name_scope('training') as scope: self.learning_rate = tf.train.polynomial_decay(learning_rate_init, self.global_step, self.max_iterations, 0, power=0.5) optimizer = tf.train.GradientDescentOptimizer(self.learning_rate) start_vars = set(x.name for x in tf.global_variables()) # first compute, then apply grads self.compute_grads = optimizer.compute_gradients( self.loss_opt, var_list=[self.adv_s]) self.grad_ph = tf.placeholder(tf.float32, name='grad_adv_s') var = [ tvar for tvar in tf.trainable_variables() if tvar.name.startswith('adv_s') ][-1] # get the last in # case explainer is re-initialized and a new graph is created grad_and_var = [(self.grad_ph, var)] self.apply_grads = optimizer.apply_gradients( grad_and_var, global_step=self.global_step) end_vars = tf.global_variables() new_vars = [x for x in end_vars if x.name not in start_vars] # variables to initialize self.setup = [] # type: list self.setup.append(self.orig.assign(self.assign_orig)) self.setup.append(self.target.assign(self.assign_target)) self.setup.append(self.const.assign(self.assign_const)) self.setup.append(self.adv.assign(self.assign_adv)) self.setup.append(self.adv_s.assign(self.assign_adv_s)) self.setup.append(self.target_proto.assign(self.assign_target_proto)) self.init = tf.variables_initializer( var_list=[self.global_step] + [self.adv_s] + [self.adv] + new_vars) if self.write_dir is not None: self.writer = tf.summary.FileWriter(write_dir, tf.get_default_graph()) self.writer.add_graph(tf.get_default_graph()) else: self.writer = None
def calculate_loss(output): # print('Output shape: {}'.format(np.shape(output)[0])) # l_triplets = tf.Variable(0.0, dtype=tf.float32) # l_pairs = tf.Variable(0.0, dtype=tf.float32) # m = tf.Variable(0.01, dtype=tf.float32) # triplet_term = tf.Variable(0.0, dtype=tf.float32) l_triplets = 0.0 l_pairs = 0.0 m = 0.01 # triplet_term = 0.0 i = 0 while i != np.shape(output)[0]: # print('Loss Calculation: {}'.format(i)) fxa = output[i] # fxa = tf.Print(fxa, [fxa], "fxa") # print('fxa: {}'.format(np.shape(fxa))) fxp = output[i + 1] # fxp = tf.Print(fxp, [fxp], "fxp") # print('fxp: {}'.format(np.shape(fxp))) fxm = output[i + 2] # fxm = tf.Print(fxm, [fxm], "fxm") # print('fxm: {}'.format(np.shape(fxm))) term_p = tf.subtract(fxa, fxp) # fxa - fx+ # term_p = tf.Print(term_p, [term_p], "term_p") # print('term_p: {}'.format(np.shape(term_p))) term_m = tf.subtract(fxa, fxm) # fxa - fx- # term_m = tf.Print(term_m, [term_m], "term_m") # print('term_m: {}'.format(np.shape(term_m))) triplet_term0 = tf.square(tf.norm(term_p)) # triplet_term0 = tf.Print(triplet_term0, [triplet_term0], "triplet_term0") triplet_term1 = tf.add(triplet_term0, m) # \\fxa - fx+\\22 + m # triplet_term1 = tf.Print(triplet_term1, [triplet_term1], "triplet_term1") # print('triplet_term1: {}'.format(np.shape(triplet_term1))) triplet_term2 = tf.square(tf.norm(term_m)) # \\fxa - fx-\\22 # triplet_term2 = tf.Print(triplet_term2, [triplet_term2], "triplet_term2") # print('triplet_term2: {}'.format(np.shape(triplet_term2))) triplet_term3 = tf.divide(triplet_term2, triplet_term1) # (\\fxa - fx-\\22 / \\fxa - fx+\\22 + m) # triplet_term3 = tf.Print(triplet_term3, [triplet_term3], "triplet_term3") # print('triplet_term3: {}'.format(np.shape(triplet_term3))) triplet_term4 = tf.maximum(0.0, tf.subtract(1.0, triplet_term3)) # max(0, (1 - (\\fxa - fx-\\22 / \\fxa - fx+\\22 + m))) # triplet_term4 = tf.Print(triplet_term4, [triplet_term4], "triplet_term4") # print('triplet_term4: {}'.format(np.shape(triplet_term4))) l_triplets = tf.add(l_triplets, triplet_term4) # Sigma, adding losses from previous loops # l_triplets = tf.Print(l_triplets, [l_triplets], "l_triplets") # print('l_triplets: {}'.format(np.shape(l_triplets))) l_pairs = tf.add(l_pairs, tf.square(tf.norm(term_p))) # l_pairs = tf.Print(l_pairs, [l_pairs], "l_pairs") # print('l_pairs: {}'.format(np.shape(l_pairs))) i = i + 3 loss = tf.add(l_triplets, l_pairs) loss = tf.Print(loss, [loss], "loss") # print('loss: {}'.format(np.shape(loss))) return loss
def polygon_flip_up_down(polygon, vertex_count, im_res): invert_mat = np.array([[1, 0], [0, -1]], dtype=np.float16) translate_mat = np.tile(np.array([0, im_res]), (vertex_count, 1)) polygon = tf.add(tf.matmul(polygon, invert_mat), translate_mat) return polygon
def generator_model(args, inputs, istrain, reuse): """Build generator architecture.""" # inputs: tensor with shape [bn, 256,256, 1] # inputs = Input(shape=input_shape_generator) with tf.variable_scope('gen_', reuse=reuse): x = ReflectionPadding2D((3, 3))(inputs) x = Conv2D(filters=ngf, kernel_size=(7, 7), padding='valid')(x) x = batch_norm(x, "bn1", is_train=istrain) x = Activation('relu')(x) # x = MaxPooling2D((2, 2), padding='same')(x)e')(x) # x = Conv2D(filters=ngf, kernel_size=(7,7), padding='same')(x) # x = batch_norm(x, "bn2", is_train=istrain) # x = Activation('relu')(x) n_downsampling = 2 for i in range(n_downsampling): mult = 2**i # x = ReflectionPadding2D((2, 2))(x) if args.max_pooling == False: x = Conv2D(filters=ngf * mult * 2, kernel_size=(3, 3), strides=2, padding='valid')(x) else: x = Conv2D(filters=ngf * mult * 2, kernel_size=(3, 3), strides=1, padding='valid')(x) x = MaxPooling2D((2, 2), padding='valid')(x) # x = BatchNormalization()(x, training=istrain) x = batch_norm(x, "down_bn_" + str(i), is_train=istrain) tf.summary.histogram('before_active', x) x = Activation('relu')(x) tf.summary.histogram('after_activate', x) mult = 2**n_downsampling for i in range(n_blocks_gen): x = res_block(x, ngf * mult, use_dropout=True) # for i in range(n_downsampling): # mult = 2**(n_downsampling - i) # x = UpSampling2D()(x) # x = Conv2D(filters=int(ngf * mult / 2),kernel_size=(3,3),padding='same')(x) ## x = Conv2DTranspose(filters=int(ngf * mult / 2), kernel_size=(3, 3), strides=2, padding='same')(x) ## x = BatchNormalization()(x, training=istrain) # x = batch_norm(x, "up_bn_"+str(i), is_train=istrain) # x = LeakyReLU(alpha=0.3)(x) x = Conv2D(filters=2, kernel_size=(5, 5), padding='same')(x) x = batch_norm(x, "final", is_train=istrain) wrap = Activation('sigmoid')(x) wrap = tf.multiply(tf.add(wrap, -0.5), 8) # dense layer dense = tf.layers.flatten(wrap) output_size = 128 output_size = args.final_layer # we use the args value here to decide the final layer number # output_size1 = 16 dense_out = tf.layers.dense(inputs=dense, units=output_size * 2) # dense_out1 = tf.layers.dense(inputs=dense_out, units=output_size1*2) x_mean = tf.reshape(dense_out, [-1, output_size, 2]) # x_mean = Conv2D(filters=2, kernel_size=(1,256), padding='valid')(wrap) # x_layer = wrap[...,0] # x_mean = tf.reduce_max(wrap, axis=2) x_mean = tf.expand_dims(x_mean, 2) wrap = tf.tile(x_mean, multiples=[1, 1, output_size, 1]) wrap = bicubic_interp_2d(wrap, imsize) outputs = Lambda(WarpST_one, arguments={ 'inputs': inputs, 'name': str(random.random()) })(wrap) return outputs, wrap[:, :, 0, :]
import tensorflow as tf intNum1 = 10 intNum2 = 20 num1 = tf.Variable(intNum1) num2 = tf.Variable(intNum2) sum = tf.add(num1, num2) print("sum - "+str(sum))
def generate_graph(self): self.images = tf.placeholder(shape=[None, self.img_size[0], self.img_size[1], 3], dtype=tf.float32, name='input_image') arg_scope = resnet_arg_scope() with slim.arg_scope(arg_scope): net, sub_net, end_points = resnet_v2_50(self.images,1001) saver1 = tf.train.Saver(tf.global_variables()) checkpoint_path = './model/resnet_v2_50.ckpt' saver1.restore(self.sess, checkpoint_path) # sub_net 0 : 38 38 256 # sub_net 1 : 19 19 512 # sub_net 2 : 10 10 1024 # sub_net 3 : 10 10 2048 p1 = sub_net[2] p2 = sub_net[1] p3 = sub_net[0] self.c1 = p1 c2_1 = self.convolution(p2, [1, 1, 512, 256], self.conv_strides_1, 'features_c2_1') c2_2 = tf.image.resize_bilinear(p1, [19, 19], name='sub_net2-2') c2_2 = self.convolution(c2_2, [1, 1, 1024, 256], self.conv_strides_1, 'features_c2_2') self.c2 = tf.add(c2_1, c2_2, name='c2_layer') c3_1 = self.convolution(p3, [1,1,256,256], self.conv_strides_1, 'features_c3_1') c3_2 = tf.image.resize_bilinear(p2, [38,38], name='sub_net3-2') c3_2 = self.convolution(c3_2, [1,1,512, 256], self.conv_strides_1, 'features_c3_2') self.c3 = tf.add(c3_1, c3_2, name='c3_layer') self.features_1 = self.convolution(self.c1, [3, 3, 1024, self.default_box_size[0]*(self.classes_size + 4)], self.conv_strides_1, 'features_1') print('## features_1 shape: ' + str(self.features_1.get_shape().as_list())) self.features_2 = self.convolution(self.c2, [3, 3, 256, self.default_box_size[1]*(self.classes_size + 4)], self.conv_strides_1, 'features_2') print('## features_2 shape: ' + str(self.features_2.get_shape().as_list())) self.features_3 = self.convolution(self.c3, [3, 3, 256, self.default_box_size[2]*(self.classes_size + 4)], self.conv_strides_1, 'features_3') print('## features_3 shape: ' + str(self.features_3.get_shape().as_list())) #self.features_4 = self.convolution(sub_net[3], [3, 3, 2048, self.default_box_size[3]*(self.classes_size + 4)], self.conv_strides_1, 'features_3') #self.feature_maps = [sub_net[3]]##### self.feature_maps = [self.features_1, self.features_2, self.features_3] self.feature_maps_shape = [m.get_shape().as_list() for m in self.feature_maps] print("feature map shape", self.feature_maps_shape) self.tmp_all_feature = [] for i, fmap in zip(range(len(self.feature_maps)), self.feature_maps): width = self.feature_maps_shape[i][1] height = self.feature_maps_shape[i][2] print("\n i",i) print("width", width) print("height", height) print("self.default_box_size[i]",self.default_box_size[i]) print("fmap",fmap) self.tmp_all_feature.append(tf.reshape(fmap, [-1, (width * height * self.default_box_size[i]) , (self.classes_size + 4)])) self.tmp_all_feature = tf.concat(self.tmp_all_feature, axis=1) print("==========tmp_all_feature ",self.tmp_all_feature) self.feature_class = self.tmp_all_feature[:,:,:self.classes_size] print("==========self.feature_class ",self.feature_class) self.feature_location = self.tmp_all_feature[:,:,self.classes_size:] print('## feature_class shape : ' + str(self.feature_class.get_shape().as_list())) print('## feature_location shape : ' + str(self.feature_location.get_shape().as_list())) self.all_default_boxs = self.generate_all_default_boxs() self.all_default_boxs_len = len(self.all_default_boxs) print('## all default boxs : ' + str(self.all_default_boxs_len)) # self.groundtruth_class = tf.placeholder(shape=[None,self.all_default_boxs_len], dtype=tf.int32,name='groundtruth_class') self.groundtruth_location = tf.placeholder(shape=[None,self.all_default_boxs_len,4], dtype=tf.float32,name='groundtruth_location') self.groundtruth_positives = tf.placeholder(shape=[None,self.all_default_boxs_len], dtype=tf.float32,name='groundtruth_positives') self.groundtruth_negatives = tf.placeholder(shape=[None,self.all_default_boxs_len], dtype=tf.float32,name='groundtruth_negatives') # loss self.groundtruth_count = tf.add(self.groundtruth_positives , self.groundtruth_negatives) print("f**k all-----feature_class", self.feature_class) print("f**k out-----groundtruth_class", self.groundtruth_class) self.softmax_cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=self.feature_class, labels=self.groundtruth_class) print("f**k!!!!------softmax_cross_entropy", self.softmax_cross_entropy) self.loss_location = tf.div(tf.reduce_sum(tf.multiply(tf.reduce_sum(self.smooth_L1(tf.subtract(self.groundtruth_location , self.feature_location)), reduction_indices=2) , self.groundtruth_positives), reduction_indices=1) , tf.reduce_sum(self.groundtruth_positives, reduction_indices = 1)) ####this is huber loss ####same to the smooth l1 loss #delta1 = tf.constant(1) #self.loss_location = tf.multiply(tf.square(delta1), tf.sqrt(1. + tf.square((tf.subtract(self.groundtruth_location, self.feature_location))/delta1)) - 1.) self.loss_class = tf.div(tf.reduce_sum(tf.multiply(self.softmax_cross_entropy , self.groundtruth_count), reduction_indices=1) , tf.reduce_sum(self.groundtruth_count, reduction_indices = 1)) #self.loss_class = self.focal_loss(self.feature_class, self.groundtruth_class) self.loss_all = tf.reduce_sum(tf.add(self.loss_class , self.loss_location)) # self.optimizer = tf.train.AdamOptimizer(0.001) #self.optimizer = tf.train.GradientDescentOptimizer(0.001) self.train = self.optimizer.minimize(self.loss_all)
def train(): with tf.Session() as sess: with tf.name_scope('inputs'): x_data = tf.placeholder(tf.float32, [None, img_size], name="x_data") z_prior = tf.placeholder(tf.float32, [None, z_size], name="z_prior") noise = tf.placeholder(tf.float32, shape=[None, noise_size]) keep_prob = tf.placeholder(tf.float32, name="keep_prob") x_generated = build_generator(z_prior, noise) y_data = build_discriminator(x_data, noise, keep_prob) y_generated = build_discriminator(x_generated, noise, keep_prob) with tf.name_scope('loss'): g_loss = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits( logits=y_generated, labels=tf.ones_like(y_generated))) d_fake_loss = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits( logits=y_generated, labels=tf.zeros_like(y_generated))) d_real_loss = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits( logits=y_data, labels=tf.ones_like(y_data))) d_loss = tf.add(d_fake_loss, d_real_loss) tf.summary.scalar('g_loss', g_loss) tf.summary.scalar('d_fake_loss', d_fake_loss) tf.summary.scalar('d_real_loss', d_real_loss) tf.summary.scalar('d_loss', d_loss) with tf.name_scope('optimizer'): optimizer = tf.train.AdamOptimizer(0.001) d_trainer = optimizer.minimize(d_loss, var_list=d_params) g_trainer = optimizer.minimize(g_loss, var_list=g_params) saver = tf.train.Saver() # merge summary merged = tf.summary.merge_all() # choose dir writer = tf.summary.FileWriter('F:/tf_board/basic_gan_mnist', sess.graph) sess.run(tf.global_variables_initializer()) for e in range(max_epoch): for batch_i in range(mnist.train.num_examples//batch_size): batch_data, y_data = mnist.train.next_batch(batch_size) # noise y_noise = np.random.uniform(-1.0, 1.0, size=(batch_size, noise_size)) # generator noise batch_noise = np.random.uniform(-1.0, 1.0, size=(batch_size, z_size)) # Run optimizers sess.run(d_trainer, feed_dict={x_data: batch_data, z_prior: batch_noise, noise: y_noise, keep_prob: 0.7}) sess.run(g_trainer, feed_dict={z_prior: batch_noise, noise: y_noise, keep_prob: 0.7}) if ((mnist.train.num_examples//batch_size) * e + batch_i) % (mnist.train.num_examples//batch_size) == 0: train_loss_d = sess.run(d_loss, feed_dict={x_data: batch_data, z_prior: batch_noise, noise: y_noise, keep_prob: 1.0}) fake_loss_d = sess.run(d_fake_loss, feed_dict={z_prior: batch_noise, noise: y_noise, keep_prob: 1.0}) real_loss_d = sess.run(d_real_loss, feed_dict={x_data: batch_data, noise: y_noise, keep_prob: 1.0}) # generator loss train_loss_g = sess.run(g_loss, feed_dict={z_prior: batch_noise, noise: y_noise, keep_prob: 1.0}) merge_result = sess.run(merged, feed_dict={x_data: batch_data, z_prior: batch_noise, noise: y_noise, keep_prob: 1.0}) writer.add_summary(merge_result, (mnist.train.num_examples//batch_size) * e + batch_i) print("Epoch {}/{}...".format(e+1, max_epoch), "Discriminator Loss: {:.4f}(Real: {:.4f} + Fake: {:.4f})...".format( train_loss_d, real_loss_d, fake_loss_d), "Generator Loss: {:.4f}".format(train_loss_g)) if e % 10 == 0: n_sample = 16 sample_noise = np.random.uniform(-1.0, 1.0, size=(n_sample, z_size)) y_sample = np.random.uniform(-1.0, 1.0, size=(n_sample, noise_size)) check_imgs = sess.run(x_generated, feed_dict={z_prior: sample_noise, noise: y_sample} ).reshape((n_sample, 28, 28))[:2] plt.imsave('F:/tf_board/basic_gan_mnist/' + str(e) + '-' + str(0) + '.png', check_imgs[0], cmap='Greys_r') plt.imsave('F:/tf_board/basic_gan_mnist/' + str(e) + '-' + str(1) + '.png', check_imgs[1], cmap='Greys_r') print('train done') n_sample = 16 sample_noise = np.random.uniform(-1.0, 1.0, size=(n_sample, z_size)) y_sample = np.random.uniform(-1.0, 1.0, size=(n_sample, noise_size)) check_imgs = sess.run(x_generated, feed_dict={z_prior: sample_noise, noise: y_sample} ).reshape((n_sample, 28, 28))[:5] for i in range(5): plt.imsave('F:/tf_board/basic_gan_mnist/' + 'final-' + str(i) + '.png', check_imgs[i], cmap='Greys_r') # save sess saver.save(sess, '/root/basic_gan_mnist.ckpt')
import numpy as np import pprint as pp import tensorflow as tf W1 = tf.Variable(tf.random_uniform([2, 5], -1.0, 1.0)) W2 = tf.Variable(tf.random_uniform([5, 4], -1.0, 1.0)) W3 = tf.Variable(tf.random_uniform([4, 1], -1.0, 1.0)) b1 = tf.Variable(tf.zeros([5]), name='Bias1') b2 = tf.Variable(tf.zeros([4]), name='Bias2') b3 = tf.Variable(tf.zeros([1]), name='Bias3') #Our hypothesis L2 = tf.sigmoid(tf.matmul(X, W1) + b1) L3 = tf.sigmoid(tf.matmul(L2, W2) + b2) hypothesis = tf.sigmoid(tf.matmul(L3, W3) + b3) dropout_rate = tf.placeholder("float") _L1 = tf.nn.relu(tf.add(tf.matmul(X, W1), B1)) L1 = tf.nn.dropout(_L1, dropout_rate)
def add(self, input, name): """contribution by miraclebiu""" return tf.add(input[0], input[1])
def __init__(self, sess): self.sess = sess self.global_step = tf.Variable(0.0, name='global_step', dtype=tf.float32, trainable=False) self.G_para = [] self.D_para = [] #for debug self.cnt_tep = 0 self.deb_kep = 0 self.deb_kep2 = 0 #for data input self.pipline_data_train = cdata.get_pipline_data_train( img_size, batchsize) self.pipline_data_test = cdata.get_pipline_data_test( img_size, batchsize_test) #3个placeholder, img和noise,training self.imgs_pla = tf.placeholder( tf.float32, [batchsize, img_size_h, img_size_w, G_group_img_num * img_channel], name='imgs_in') self.training = tf.placeholder(tf.bool, name='training_in') self.timerates_pla = tf.placeholder(tf.float32, [batchsize], name='timerates_in') self.timerates_expand = tf.expand_dims(self.timerates_pla, -1) self.timerates_expand = tf.expand_dims(self.timerates_expand, -1) self.timerates_expand = tf.expand_dims(self.timerates_expand, -1) #12*1*1*1 print('placeholders:\n', 'img_placeholder:', self.imgs_pla, '\ntraining:', self.training) ''' placeholders: img_placeholder: Tensor("imgs_in:0", shape=(12, 180, 320, 9), dtype=float32) training: Tensor("training_in:0", dtype=bool) ''' self.frame0 = self.imgs_pla[:, :, :, :img_channel] self.frame1 = self.imgs_pla[:, :, :, img_channel:img_channel * 2] self.frame2 = self.imgs_pla[:, :, :, img_channel * 2:] #这里是为了看看第一帧和第3帧的差距,用来给evalte self.frame_0_2_L1loss = tf.reduce_mean(tf.abs(self.frame0 - self.frame2), [1, 2, 3], name='frame_0_2_L1loss') #frame0and2=tf.concat([self.frame0, self.frame2], -1) #在第三维度连接起来 #print ('after concat:',frame0and2) #!!!!!!!!!!here is differs from v1,add to Generator output the ori img will reduce the generator difficulty self.G_opticalflow = self.Generator_net( self.frame0, self.frame2) #注意这里是直接作为optical flow #下面将结果中的代表各个意义分开 #optical flow[:,:,:,0:2] is frame0->frame2(get frame2 from frame0), [2:]is 2->0 self.opticalflow_0_2 = tf.slice( self.G_opticalflow, [0, 0, 0, 0], [-1, -1, -1, 2], name='G_opticalflow_0_2') #self.G_opticalflow[:,:,:,:2] #self.prob_flow1=tf.clip_by_value(self.G_opticalflow[:,:,:,2],0,1 , name='prob_flow1_sigmoid') self.prob_flow1 = tf.nn.sigmoid( self.G_opticalflow[:, :, :, 2], name='prob_flow1_sigmoid') #这里添加了一个置信度,用来选择光流,即相信F0->1还是相信F1->0 self.opticalflow_2_0 = tf.slice( self.G_opticalflow, [0, 0, 0, 3], [-1, -1, -1, 2], name='G_opticalflow_2_0') # self.G_opticalflow[:,:,:,3:] print('original flow:', self.opticalflow_0_2, self.prob_flow1, self.opticalflow_2_0) #original flow: Tensor("G_opticalflow_0_2:0", shape=(12, 180, 320, 2), dtype=float32) #Tensor("prob_flow1_sigmoid:0", shape=(12, 180, 320), dtype=float32) #Tensor("G_opticalflow_2_0:0", shape=(12, 180, 320, 2), dtype=float32) #反向光流算中间帧 self.opticalflow_t_0=tf.add( -(1-self.timerates_expand)*self.timerates_expand*self.opticalflow_0_2 ,\ self.timerates_expand*self.timerates_expand*self.opticalflow_2_0 , name="G_opticalflow_t_0") self.opticalflow_t_2=tf.add( (1-self.timerates_expand)*(1-self.timerates_expand)*self.opticalflow_0_2 ,\ self.timerates_expand*(self.timerates_expand-1)*self.opticalflow_2_0, name="G_opticalflow_t_2") print('two optical flow:', self.opticalflow_t_0, self.opticalflow_t_2) #two optical flow: Tensor("G_opticalflow_t_0:0", shape=(12, 180, 320, 2), dtype=float32) #Tensor("G_opticalflow_t_2:0", shape=(12, 180, 320, 2), dtype=float32), #2种方法合成t时刻的帧 self.img_flow_2_t = self.warp_op(self.frame2, -self.opticalflow_t_2) #!!! self.img_flow_0_t = self.warp_op(self.frame0, -self.opticalflow_t_0) #!!! #self.G_net=self.timerates_expand*self.img_flow_2_t + (1-self.timerates_expand)*self.img_flow_0_t tep_prob_flow1 = tf.expand_dims(self.prob_flow1, -1) tep_prob_flow1 = tf.tile(tep_prob_flow1, [1, 1, 1, 3]) #self.G_net=tf.where( tf.greater_equal(tep_prob_flow1, 0.5), self.img_flow_0_t, self.img_flow_2_t, name='G_net_generate') #这里认为>0.5就是相信frame0 tep_sujm = tep_prob_flow1 * (1 - self.timerates_expand) + ( 1 - tep_prob_flow1) * self.timerates_expand self.G_net=tf.add(self.img_flow_0_t*tep_prob_flow1*(1-self.timerates_expand)/tep_sujm, \ self.img_flow_2_t*(1-tep_prob_flow1)*self.timerates_expand/tep_sujm, name='G_net_generate') print( 'self.G_net:', self.G_net ) #self.G_net: Tensor("G_net_generate:0", shape=(12, 180, 320, 3), dtype=float32) #利用光流前后帧互相合成 self.img_flow_2_0 = self.warp_op(self.frame2, self.opticalflow_2_0) #frame2->frame0 self.img_flow_0_2 = self.warp_op(self.frame0, self.opticalflow_0_2) #frame0->frame2 #D_1的输出 frame0_False_2 = tf.concat([self.frame0, self.G_net, self.frame2], -1) self.D_linear_net_F, self.D_linear_net_F_logit = self.Discriminator_net_linear( frame0_False_2) self.D_linear_net_T, self.D_linear_net_T_logit = self.Discriminator_net_linear( self.imgs_pla) #下面是loss公式 self.D_linear_net_loss_sum, self.D_linear_net_loss_T, self.D_linear_net_loss_F=self.D_loss_TandF_logits(self.D_linear_net_T_logit, \ self.D_linear_net_F_logit, "D_linear_net") print('D1 form finished..') #D_2的输出 ''' self.D_clear_net_F, self.D_clear_net_F_logit=self.Discriminator_net_clear(self.G_net) self.D_clear_net_T, self.D_clear_net_T_logit=self.Discriminator_net_clear(self.frame1) #下面是loss公式 self.D_clear_net_loss_sum, self.D_clear_net_loss_T, self.D_clear_net_loss_F=self.D_loss_TandF_logits(self.D_clear_net_T_logit, \ self.D_clear_net_F_logit, "D_clear_net") ''' #self.G_loss_mean_Square=tf.reduce_mean(tf.squared_difference(self.G_net,self.frame1), name='G_clear_square_loss') #1、contex loss print("forming conx loss:") tep_G_shape = self.G_net.get_shape().as_list()[1:] self.contex_Genera = tf.keras.applications.VGG16( include_top=False, input_tensor=self.G_net, input_shape=tep_G_shape).get_layer("block4_conv3").output self.contex_frame1 = tf.keras.applications.VGG16( include_top=False, input_tensor=self.frame1, input_shape=tep_G_shape).get_layer("block4_conv3").output self.contex_loss = tf.reduce_mean(tf.squared_difference( self.contex_frame1, self.contex_Genera), name='G_Contex_loss') print('G_loss_mean_contex form finished..') #2、L1 loss print( "forming L1 loss:生成帧与GT、frame2->frame0与frame0、frame0->frame2与frame2" ) self.L1_loss_interframe = tf.reduce_mean( tf.abs(self.G_net - self.frame1)) self.L1_loss_all =tf.reduce_mean(tf.abs( self.G_net-self.frame1 ) + \ tf.abs(self.img_flow_2_0-self.frame0) + \ tf.abs(self.img_flow_0_2-self.frame2), name='G_clear_l1_loss') #self.G_loss_mean_Square= self.contex_loss*1 + self.L1_loss_all print('G_loss_mean_l1 form finished..') #3、下面是G的loss self.G_loss_mean_D1 = self.G_loss_F_logits(self.D_linear_net_F_logit, 'G_loss_D1') #self.G_loss_mean_D2=self.G_loss_F_logits(self.D_clear_net_F_logit, 'G_loss_D2') #4 local var loss self.local_var_loss_0_2 = self.local_var_loss(self.opticalflow_0_2) self.local_var_loss_2_0 = self.local_var_loss(self.opticalflow_2_0) #print ("local _var loss:",self.local_var_loss_0_2, self.G_loss_mean_D1) #local _var loss: Tensor("mean_local_var:0", shape=(), dtype=float32) Tensor("Mean_3:0", shape=(), dtype=float32) self.local_var_loss_all = tf.add(self.local_var_loss_0_2, self.local_var_loss_2_0, name="local_var_add") #5 global var loss self.global_var_loss_0_2 = self.global_var_loss(self.opticalflow_0_2) self.global_var_loss_2_0 = self.global_var_loss(self.opticalflow_2_0) self.global_var_loss_all = tf.add(self.global_var_loss_0_2, self.global_var_loss_2_0, name="global_var_add") #训练生成器的总LOSS 这里将G的loss和contex loss与前面G的loss做一个归一化,这样当D的loss大的时候,说明这时D不可靠,需要多训练D,而相应的减小该D对G的训练影响 tep_serer_loss = ( self.G_loss_mean_D1 + self.contex_loss + self.D_linear_net_loss_sum) * 2 #后面的数限制了总的loss大小,为5时为1/5=0.2 self.G_loss_all=self.G_loss_mean_D1/tep_serer_loss + \ self.contex_loss/tep_serer_loss + \ self.L1_loss_all +\ self.global_var_loss_all *10 #self.local_var_loss_all *10 #* (1+self.global_step/G_squareloss_rate_globalstep)# self.G_loss_mean_D2 #W ./tensorflow/core/grappler/optimizers/graph_optimizer_stage.h:241] Failed to run optimizer ArithmeticOptimizer, stage HoistCommonFactor node #add_8. Error: Node ArithmeticOptimizer/HoistCommonFactor_Add_add_7 is missing output properties at position :0 (num_outputs=0) #训练判别器D的loss 这里将d的loss与前面G的loss和contex loss做一个归一化,这样当这里D的loss大的时候,说明这时D不可靠,需要多训练D,而相应的减小该D对G的训练影响 self.D_loss_all = self.D_linear_net_loss_sum / tep_serer_loss #+ self.D_clear_net_loss_sum #还是应该以tf.trainable_variables()为主 t_vars = tf.trainable_variables() print("trainable vars cnt:", len(t_vars)) self.G_para = [var for var in t_vars if var.name.startswith('G')] self.D_para = [var for var in t_vars if var.name.startswith('D')] # weight clipping self.clip_D = [ p.assign(tf.clip_by_value(p, weightclip_min, weightclip_max)) for p in self.D_para ] #训练使用 self.train_D = self.train_op_D(decay_steps, decay_rate) self.train_G = self.train_op_G(decay_steps, decay_rate) ''' print ('\nshow all trainable vars:',len(tf.trainable_variables())) for i in tf.trainable_variables(): print (i) ''' print('\nfirst show G params') for ind, i in enumerate(self.G_para): print(ind, i) print('\nnext is D:\n') for ind, i in enumerate(self.D_para): print(ind, i) print('\nnext is tf.GraphKeys.UPDATE_OPS:') print(tf.get_collection(tf.GraphKeys.UPDATE_OPS)) self.summary_all = tf.summary.merge_all() init = tf.global_variables_initializer() #初始化tf.Variable self.sess.run(init)
x_train.append(to_one_hot(word2int[data_word[0]], vocab_size)) y_train.append(to_one_hot(word2int[data_word[1]], vocab_size)) # convert them to numpy arrays x_train = np.asarray(x_train) y_train = np.asarray(y_train) print(x_train.shape, y_train.shape) # making placeholders for x_train and y_train x = tf.placeholder(tf.float32, shape=(None, vocab_size)) y_label = tf.placeholder(tf.float32, shape=(None, vocab_size)) EMBEDDING_DIM = 5 # you can choose your own number W1 = tf.Variable(tf.random_normal([vocab_size, EMBEDDING_DIM])) b1 = tf.Variable(tf.random_normal([EMBEDDING_DIM])) #bias hidden_representation = tf.add(tf.matmul(x, W1), b1) W2 = tf.Variable(tf.random_normal([EMBEDDING_DIM, vocab_size])) b2 = tf.Variable(tf.random_normal([vocab_size])) prediction = tf.nn.softmax(tf.add(tf.matmul(hidden_representation, W2), b2)) print("Session starting") sess = tf.Session() init = tf.global_variables_initializer() sess.run(init) #make sure you do this! # define the loss function: cross_entropy_loss = tf.reduce_mean( -tf.reduce_sum(y_label * tf.log(prediction), reduction_indices=[1])) # define the training step: train_step = tf.train.GradientDescentOptimizer(0.1).minimize( cross_entropy_loss)
def conv_net(x, weights, biases, dropout): # Reshape input picture x.shape:(128,128,3) x = tf.reshape(x, shape=[-1, 128, 128, 3]) # Convolution Layer conv1 = conv2d(x, weights['wc1'], biases['bc1']) conv2 = conv2d(conv1, weights['wc2'], biases['bc2']) # Max Pooling (down-sampling) pool1 = maxpool2d(conv2, k=2) print(pool1.shape) #(64,64,64) # Convolution Layer conv3 = conv2d(pool1, weights['wc3'], biases['bc3']) conv4 = conv2d(conv3, weights['wc4'], biases['bc4']) # Max Pooling (down-sampling) pool2 = maxpool2d(conv4, k=2) print(pool2.shape) #(32,32,128) # Convolution Layer conv5 = conv2d(pool2, weights['wc5'], biases['bc5']) conv6 = conv2d(conv5, weights['wc6'], biases['bc6']) conv7 = conv2d(conv6, weights['wc7'], biases['bc7']) # Max Pooling pool3 = maxpool2d(conv7, k=2) print(pool3.shape) #(16,16,256) # Convolution Layer conv8 = conv2d(pool3, weights['wc8'], biases['bc8']) conv9 = conv2d(conv8, weights['wc9'], biases['bc9']) conv10 = conv2d(conv9, weights['wc10'], biases['bc10']) # Max Pooling pool4 = maxpool2d(conv10, k=2) print(pool4.shape) #(8,8,512) conv11 = conv2d(pool4, weights['wc11'], biases['bc11']) conv12 = conv2d(conv11, weights['wc12'], biases['bc12']) conv13 = conv2d(conv12, weights['wc13'], biases['bc13']) # Max Pooling pool5 = maxpool2d(conv13, k=2) print(pool5.shape) #(4,4,512) # Fully connected layer # Reshape conv2 output to fit fully connected layer input fc1 = tf.reshape(pool5, [-1, weights['wd1'].get_shape().as_list()[0]]) fc1 = tf.add(tf.matmul(fc1, weights['wd1']), biases['bd1']) fc1 = tf.nn.relu(fc1) # Apply Dropout fc1 = tf.nn.dropout(fc1, dropout) #fc2 = tf.reshape(fc1, [-1, weights['wd2'].get_shape().as_list()[0]]) fc2 = tf.add(tf.matmul(fc1, weights['wd2']), biases['bd2']) fc2 = tf.nn.relu(fc2) # Apply Dropout fc2 = tf.nn.dropout(fc2, dropout) ''' fc3 = tf.reshape(fc2, [-1, weights['out'].get_shape().as_list()[0]]) fc3 = tf.add(tf.matmul(fc2, weights['out']), biases['bd2']) fc3 = tf.nn.relu(fc2) ''' # Output, class prediction out = tf.add(tf.matmul(fc2, weights['out']), biases['out']) out = tf.nn.softmax(out) return out
# We will import various libraries to save our model # in proper format # Name the place holder placeholder_name = 'a' # We will do a simple scaler addition operation_name = 'add' # Let's create a placeholder to store input number a = tf.placeholder(tf.int32, name=placeholder_name) # Input number will be added by a constant 10 b = tf.constant(10) # Now whenever an input comes we will add constant to that add = tf.add(a, b, name=operation_name) # So next step is to create a session which can perform # the above task. # Run a few operations to make sure our model works with tf.Session() as sess: #Let's call our model to do an addition c = sess.run(add, feed_dict={a: 2}) print('10 + 2 = {}'.format(c)) # One more time d = sess.run(add, feed_dict={a: 10}) print('10 + 10 = {}'.format(d))
def train_validate(Model=None,N_Class=0,imgh=0,imgw=0,croph=0,cropw=0,c_blc=None,c_blc_T=None,opt=None,LR=0,decay=0.0,Momentum=0.95,epsilon=0.0,beta1=0.0,epoch=100, bfSize=10,btSize=1,trainDir=None,ValDir=None,trainLogDir=None,ValLogDir=None,ckptDir=None,TxtDir=None,MX_ITR=0,MX2KEEP=0,brkpt=0,augmentation=None,crop=None): tf.reset_default_graph() # Initialize the test model model_to_test = 0 # Create the directories if does not exist if not os.path.exists(ckptDir): os.makedirs(ckptDir) if not os.path.exists(trainLogDir): os.makedirs(trainLogDir) if not os.path.exists(ValLogDir): os.makedirs(ValLogDir) print ("\n------------------------------------------------------------------ Image ImageSegmentation using TF -------------------------------------------------------------------") print ("\n------------------------------------------------------------------ Class description-----------------------------------------------------------------------------------\n") print ("Class 0: Asphalt\nClass 1: Concrete pavement/Mosaic pavement/Natural stone pavement/Loose natural surface/ Gutter, curb stone\nClass 2: Hole cover\nClass 3: Tree trunk\nClass 4: Broadleaf crown/Conifer crown\nClass 5: Bush\nClass 6: Street sign\nClass 7: Streetlight\nClass 8: Traffic light\nClass 9: Guardrail\nClass 10: Encasement of equipment\nClass 11: Advertisement panel\nClass 12: Building\nClass 13: Enclosure\nClass 14: Metal pole\nClass 15: Sky\nClass 16: Vehicle\nClass 17: People\nClass 18: Bollard\nClass 19: Undefined") print ("\n\nCollecting parsed arguments for the model...") print ("Model to work with: %s\nNumber of classes are: %d\nImage height: %d---Image width: %d---Crop height: %d---Crop width: %d\nClass balance required: %s\nClass balance type: %s\nOptimizer: %s---Learning Rate: %f---Decay: %f---Momentum: %f---Epsilon: %f---Beta1: %f\nEpoch: %d\nBuffer size: %d Batch size: %d\nMax iterations: %d\nMaximum checkpoints to keep: %d\nBreakpoints at every %d iterations\nTrain directory: %s\nValidation Directory: %s\nCheckpoint directory: %s\nTextfile Directory: %s\nTrain log directory: %s\nValidation log directory: %s\nAugmentation Required: %s\nCropping required: %s" %(Model,N_Class,imgh,imgw,croph,cropw,c_blc,c_blc_T,opt,LR,decay,Momentum,epsilon,beta1,epoch,bfSize,btSize, MX_ITR,MX2KEEP,brkpt,trainDir,ValDir,ckptDir,TxtDir,trainLogDir,ValLogDir,augmentation,crop)) print ("\n------------------------------------------------------------------ Initialize train data ---------------------------------------------------------------------------------") # Initialize empty placeholders to be fed during the session train operation if crop == 'True': X = tf.placeholder(tf.float32, shape=[btSize, croph, cropw, 3], name="X") y = tf.placeholder(tf.uint8, shape=[btSize, croph, cropw, 1], name="y") else: X = tf.placeholder(tf.float32, shape=[btSize, imgh, imgw, 3], name="X") y = tf.placeholder(tf.uint8, shape=[btSize, imgh, imgw, 1], name="y") training = tf.placeholder(tf.bool, shape=[], name="mode") v_loss = tf.placeholder(tf.float32, shape=[], name="v_loss") v_iou = tf.placeholder(tf.float32, shape=[], name="v_iou") v_acc = tf.placeholder(tf.float32, shape=[], name="v_acc") print ("\n----------------------------------------------------------------------- Loading the model --------------------------------------------------------------------------------") # Load the network to get the predicted output if Model == 'ResNet': pred_score = ResNet50(X,training,N_Class) elif Model == 'DDNET': Reshaped_map_decoder1, Reshaped_map_decoder2, pred_score = DPDB_encoder_stacking_decoder_DeepSupervision_300_threeStages_Cardinality_ResidualStack(X, training,N_Class) elif Model == 'UNet': pred_score = ResNet50(X,training,N_Class) print ("\n----------------------------------------------------- Reshaping the logits and masks (Masks are one hot encoded) ---------------------------------------------------------") # Label is one hot encoded and reshaped for cross-entropy computation loss later on segMap = tf.reshape(pred_score, (-1, N_Class)) one_hot_encoded_label = tf.one_hot(tf.squeeze(y, axis=-1), N_Class, axis=-1) Reshaped_labels = tf.reshape(one_hot_encoded_label, shape=(-1, N_Class)) print ("\nShapes of the one_hot_encoded_label and flat label are: " + str(one_hot_encoded_label.get_shape().as_list())+"\t"+str(Reshaped_labels.get_shape().as_list())) print ("Shapes of the logit and flattened logits are: " + str(pred_score.get_shape().as_list())+"\t"+str(segMap.get_shape().as_list())) print ("\n------------------------------------------------------------ Computing the cross entropy ----------------------------------------------------------------------------------") #with tf.device("/cpu:0"): # Initilaze an empty class weights list class_weights = [] # Compute the cross entropy loss ignoring undefined object label if c_blc == 'True': trainMasks = sorted(glob(trainDir+'/*.png'), key=stringSplitByNumbers) if c_blc_T == 'weigh_equal_importance': print ("\nIgnore the last label and weigh rest of the labels equally") class_weights = [1.0] * (N_Class-1) class_weights.insert((N_Class-1), 0.0) print ("The class weights are: %s" %(class_weights)) class_weights = tf.reshape(tf.convert_to_tensor(class_weights, dtype=tf.float32), [N_Class]) elif c_blc_T == 'cls_weighing': print ("\nENet class weighing is selected") class_weights = ENet_weighing(trainMasks,N_Class) print ("The class weights are: %s" %(class_weights)) class_weights = tf.reshape(tf.convert_to_tensor(class_weights, dtype=tf.float32), [N_Class]) else: print ("\nENet Frequency median is selected") class_weights = median_frequency_balancing(trainMasks,N_Class) print ("The class weights are: %s" %(class_weights)) class_weights = tf.reshape(tf.convert_to_tensor(class_weights, dtype=tf.float32), [N_Class]) else: class_weights = None print ("Class Weights shape as a tensor: " + str(class_weights.get_shape().as_list())) if Model == 'ResNet': print ("Computing loss from ResNet") Totalloss = cross_entropy_loss(segMap, Reshaped_labels, N_Class,class_weights) elif Model == 'DDNET': print ("Computing loss from DDNET") Loss_decoder1 = cross_entropy_loss(Reshaped_map_decoder1, Reshaped_labels, N_Class,class_weights) Loss_decoder2 = cross_entropy_loss(Reshaped_map_decoder2, Reshaped_labels, N_Class,class_weights) Loss_final_decoder = cross_entropy_loss(segMap, Reshaped_labels, N_Class,class_weights) Totalloss = Loss_decoder1 + Loss_decoder2 + Loss_final_decoder elif Model == 'UNet': print ("Computing loss from UNet") Totalloss = cross_entropy_loss(segMap, Reshaped_labels, N_Class,class_weights) # Create the global step to keep track of iterations global_step = tf.train.get_or_create_global_step() # optimizer.compute_gradients and optimizer.apply_gradients is equivalent to running train_step = tf.train.AdamOptimizer(learning_rate=0.0001).minimize(loss) #update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) # Use this if updation is not handled in place during batch norm i.e if updates_collections=tf.GraphKeys.UPDATE_OPS #with tf.control_dependencies(update_ops): if opt == 'Adam': print ("\n------------------------------------------------------------------- ADAM OPTIMIZER ON THE GO -------------------------------------------------------------------------------") lr = tf.train.exponential_decay(LR, global_step, epoch, decay, staircase=True) optimizer = tf.train.AdamOptimizer(learning_rate=lr).minimize(Totalloss, global_step=global_step) elif opt == 'RMSProp': lr = tf.train.exponential_decay(LR, global_step, epoch, decay, staircase=True) print ("\n------------------------------------------------------------------- RMSPROP OPTIMIZER ON THE GO -------------------------------------------------------------------------------") optimizer = tf.train.RMSPropOptimizer(lr, decay, Momentum, epsilon).minimize(Totalloss, global_step=global_step) elif opt == 'SGD': lr = tf.train.exponential_decay(LR, global_step, epoch, decay, staircase=True) print ("\n------------------------------------------------------------------- SGD OPTIMIZER ON THE GO -------------------------------------------------------------------------------") optimizer = tf.train.MomentumOptimizer(learning_rate=lr,momentum=Momentum).minimize(Totalloss, global_step=global_step) #Performance Measures print ("\n------------------------------------------------------------------- Computing respective Accuracies and IOUs ----------------------------------------------------------------") # Accuracy softmax = tf.nn.softmax(segMap) correct_prediction = tf.equal(tf.argmax(segMap,1), tf.argmax(Reshaped_labels,1)) accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) #accuracy, accuracy_update = tf.contrib.metrics.streaming_accuracy(tf.argmax(softmax,1), tf.argmax(Reshaped_labels,1)) accuracy = accuracy * 100.0 # intersection over union IOU, IOU_update = tf.contrib.metrics.streaming_mean_iou(predictions=tf.cast(tf.argmax(softmax,1), tf.int32), labels=tf.cast(tf.argmax(Reshaped_labels,1),tf.int32), num_classes=N_Class) IOU = IOU * 100.0 logits=tf.reshape(segMap, [-1]) trn_labels=tf.reshape(Reshaped_labels, [-1]) #logits=tf.argmax(segMap,1) #trn_labels=tf.argmax(Reshaped_labels,1) #logits=tf.cast(logits, tf.float32) #trn_labels=tf.cast(trn_labels, tf.float32) inter=tf.reduce_sum(tf.multiply(logits,trn_labels)) union=tf.reduce_sum(tf.subtract(tf.add(logits,trn_labels),tf.multiply(logits,trn_labels))) val_iou=tf.subtract(tf.constant(1.0, dtype=tf.float32),tf.divide(inter,union)) val_iou=val_iou*100.0 print ("\n------------------------------------------------------------------------ Feed the RGB data ----------------------------------------------------------------------------------") # Load RGB/RGBD data # Fetch one RGB train element at a time and augment it train_input_queue = parseRGB(mode='train',trainDir=trainDir) train_iterator = train_input_queue.make_initializable_iterator() _train_image, _train_label = train_iterator.get_next() if augmentation == 'True': train_image, train_label = data_Augmentation(_train_image,_train_label,crop,croph,cropw) # Fetch one RGB validate element at a time and augment it val_input_queue = parseRGB('validate',ValDir=ValDir) val_iterator = val_input_queue.make_initializable_iterator() val_image, val_label = val_iterator.get_next() print ("\n--------------------------------------------------------------- Read a single batch successfully -----------------------------------------------------------------------------") # Log the training in the respective directory print ("\n-------------------------------------------------------- Logging: training and validation logs --------------------------------------------------------------------------------") log_time = time.asctime( time.localtime(time.time())) log_time = log_time.replace(" ", "_") train_logdir = os.path.join(trainLogDir, log_time) val_logdir = os.path.join(ValLogDir, log_time) # Counter for total number of iterations performed so far. init_iterations = 0 total_loss = 0.0 total_acc = 0.0 total_iou = 0.0 tot_loss = 0.0 tot_acc = 0.0 tot_iou = 0.0 mean_acc_valSet = [] mean_iou_valSet = [] mean_CELoss_valSet = {} overfit = 0 # Accumulate summaries and merge them to display in tensorboard print ("\n---------------------------------------------- Accumulate training summaries and merge them to display in tensorboard ------------------------------------------------------------------") im_h = tf.summary.histogram("Softmax activation on predicted image", softmax) im_tr = tf.summary.image("Image to train", _train_image, max_outputs=1) mk_tr = tf.summary.image("Label to train", _train_label, max_outputs=1) if augmentation == 'True': im_tr_aug = tf.summary.image("Augmented image to train", train_image, max_outputs=1) mk_tr_aug = tf.summary.image("Augmented label to train", train_label, max_outputs=1) if crop == 'True': mk_p = tf.summary.image("Predicted mask", tf.cast(tf.reshape((tf.argmax(softmax,1)), [1,croph,cropw,1]), tf.uint8), max_outputs=1) else: mk_p = tf.summary.image("Predicted mask", tf.cast(tf.reshape((tf.argmax(softmax,1)), [1,imgh,imgw,1]), tf.uint8), max_outputs=1) else: mk_p = tf.summary.image("Predicted mask", tf.cast(tf.reshape((tf.argmax(softmax,1)), [1,imgh,imgw,1]), tf.uint8), max_outputs=1) t_l = tf.summary.scalar('Cross entropy loss', Totalloss) t_a = tf.summary.scalar('Accuracy', accuracy) t_I = tf.summary.scalar('Intersection Over Union', IOU) #summary_op = tf.summary.merge_all() print ("\n---------------------------------------------- Accumulate validation summaries and merge them to display in tensorboard ------------------------------------------------------------------") v_im = tf.summary.image("Validation Image", val_image, max_outputs=1) v_mk = tf.summary.image("Validation Label", val_label, max_outputs=1) if crop == 'True': v_p = tf.summary.image("Validated mask", tf.cast(tf.reshape((tf.argmax(softmax,1)), [1,croph,cropw,1]), tf.uint8), max_outputs=1) else: v_p = tf.summary.image("Validated mask", tf.cast(tf.reshape((tf.argmax(softmax,1)), [1,imgh,imgw,1]), tf.uint8), max_outputs=1) v_l = tf.summary.scalar('Validation Cross entropy loss', v_loss) v_a = tf.summary.scalar('Validation Accuracy', v_acc) v_I = tf.summary.scalar('Validation Intersection Over Union', v_iou) #val_op = tf.summary.merge_all() # Create the directories if doesn't exist yet print ("\n----------------------------------------------------------------- Creating directories if does not exist ----------------------------------------------------------------------") if not os.path.exists(train_logdir): os.makedirs(train_logdir) if not os.path.exists(val_logdir): os.makedirs(val_logdir) # Create savers and tensorboard print ("\n----------------------------------------------------------------- Provision for saving models and creating Tensorboard ---------------------------------------------------------") saver = tf.train.Saver(max_to_keep=MX2KEEP) init_global = tf.global_variables_initializer() init_local = tf.local_variables_initializer() # Configure the GPU options according to your requirement config = tf.ConfigProto() config.gpu_options.allow_growth = True # allocates only the required amount of memory config.log_device_placement = False # check which part of code is executed in which core config.allow_soft_placement=False # allow automatic placement of necessary operation on the respective cores #config.gpu_options.per_process_gpu_memory_fraction = 0.8 # how much percentage of memory do you want to allocate per GPU with tf.Session(config=config) as sess: try: # Time noted when actual train processess _start = time.time() #log performance training### Create the summary writer -- to write all the logs into a specified file. This file can be later read by tensorboard. train_summary_writer = tf.summary.FileWriter(train_logdir, sess.graph) val_summary_writer = tf.summary.FileWriter(val_logdir) #resume training exactly where you left from global_step = tf.train.get_global_step(sess.graph) # Initialize all the variables sess.run(init_global) sess.run(init_local) # Initialize the trainable iterator sess.run(train_iterator.initializer) sess.run(val_iterator.initializer) # Get a text file to save important validation metrics for easy readability iu_File = open(TxtDir+"/validation_test_results.txt", "a") iu_File.write("############################################ Performance measured on validation set ###############################################") iu_File.write("\n\n") # Restore checkpoint to continue training if checkpoint exist ckpt = tf.train.get_checkpoint_state(os.path.dirname(ckptDir+'/checkpoint')) if ckpt and ckpt.model_checkpoint_path: print("\n----------------------------------------------------- Restoring from latest checkpoint ------------------------------------------------------") saver.restore(sess, '/home/bharadwaj/ImageSegmentation/data/snapshots/seg_model-245000') #saver.restore(sess, ckpt.model_checkpoint_path) init_iterations = global_step.eval() # Initialize the validation set valset = sorted(glob(ValDir+'/*.jpg'), key=stringSplitByNumbers) # Start training or resume training if init_iterations == 0: print("\n============================================================================= Training started from scratch ================================================================================") print("\n============================================================================= Loss display in every 100 iterations =========================================================================\n") else : print("\n============================================================================= Resuming training from iteration %d ==========================================================================" %(init_iterations)) print("\n============================================================================= Loss display in every 100 iterations =========================================================================\n") for i in range(init_iterations, MX_ITR): # Measure time elapsed start = time.time() # Initialize local metrics c_loss=0.0 acc=0.0 iou=0.0 if augmentation == 'True': # Get dynamic augmented images and labels trainimg, trainlbl = sess.run([train_image, train_label]) # Get all details to be displayed -- summarized on tensorboard c_loss,train_step,acc,iou,_,imtraug,mktraug,imh,imtr,mktr,mkp,tl,ta,tI,global_step_value = sess.run([Totalloss,optimizer,accuracy,IOU,IOU_update,im_tr_aug,mk_tr_aug,im_h,im_tr,mk_tr,mk_p,t_l,t_a,t_I,global_step], feed_dict={X:trainimg,y:trainlbl,training:True}) # Summarize important details train_summary_writer.add_summary(imh,global_step_value) train_summary_writer.add_summary(imtr,global_step_value) train_summary_writer.add_summary(mktr,global_step_value) train_summary_writer.add_summary(imtraug,global_step_value) train_summary_writer.add_summary(mktraug,global_step_value) train_summary_writer.add_summary(mkp,global_step_value) train_summary_writer.add_summary(tl,global_step_value) train_summary_writer.add_summary(ta,global_step_value) train_summary_writer.add_summary(tI,global_step_value) else: # Get dynamic augmented images and labels trainimg, trainlbl = sess.run([_train_image, _train_label]) # Get all details to be displayed -- summarized on tensorboard c_loss,train_step,acc,iou,_,imh,imtr,mktr,mkp,tl,ta,tI,global_step_value = sess.run([Totalloss,optimizer,accuracy,IOU,IOU_update,im_h,im_tr,mk_tr,mk_p,t_l,t_a,t_I,global_step], feed_dict={X:trainimg,y:trainlbl,training:True}) # Summarize important details train_summary_writer.add_summary(imh,global_step_value) train_summary_writer.add_summary(imtr,global_step_value) train_summary_writer.add_summary(mktr,global_step_value) train_summary_writer.add_summary(mkp,global_step_value) train_summary_writer.add_summary(tl,global_step_value) train_summary_writer.add_summary(ta,global_step_value) train_summary_writer.add_summary(tI,global_step_value) #print (a) total_loss = total_loss + c_loss total_acc = total_acc + acc total_iou = total_iou + iou tot_loss = tot_loss + c_loss tot_acc = tot_acc + acc tot_iou = tot_iou + iou current_time = str(time.asctime( time.localtime(time.time()))) end = time.time() time_Elapsed = abs(end - start) # Display the loss details in every 100 iterations if i > 0 and i % 100 == 0: mean_loss = tot_loss/100.0 mean_acc = tot_acc/100.0 mean_iou = tot_iou/100.0 print("Iteration: %d \t\t Time: %s \t Per image processing time: %.2f secs \t\t Cross_entropy_loss: %.3f \t Accuracy: %.2f \t IOU: %.2f" %(i,current_time,time_Elapsed,mean_loss,mean_acc,mean_iou)) tot_loss = 0.0 tot_acc = 0.0 tot_iou = 0.0 # Save the snapshots/checkpoints in every brkpt iterations if i > 0 and i % brkpt == 0: print ("\n---------------------- Mean cross_entropy_loss and accuracy (over %d iterations) at %dth step are ----------------------------" %(brkpt,i)) mean_loss = float(total_loss/brkpt) mean_acc = float(total_acc/brkpt) mean_iou = float(total_iou/brkpt) print ("\n--------------------- Loss: %.3f \t\t Acc: %.2f \t\t IOU: %.2f -----------------------------------------------" %(mean_loss,mean_acc,mean_iou)) total_loss = 0.0 total_acc = 0.0 total_iou = 0.0 # Save the first model checkpoint print ("\nSaving model %d in snapshots directory" %(i)) saver.save(sess, ckptDir + '/seg_model',global_step=i) _end = time.time() elapsed_time = int(_end - _start) print ("\nTraining since : {:02d}:{:02d}:{:02d} hours".format(elapsed_time // 3600, (elapsed_time % 3600 // 60), elapsed_time % 60)) acc_valSet = [] iou_valSet = [] loss_valSet = [] acc_V = 0.0 iou_V = 0.0 loss_V = 0.0 # Call validate to test model performance print ("\n================================================================== Validating on validation set at %dth step ===================================================================\n"%(i)) # Validate on small patches if trained on small patches if crop == 'True': print ("\nValidating on patches of 256X256") print ("\nValidation directory has %d data\n" %((len(os.listdir(ValDir))/2))) for n in range (len(valset)): valimg, vallbl = sess.run([val_image, val_label]) # Collect height and width height = valimg.shape[1] width = valimg.shape[2] # Squeeze the first dimensions valimg = np.squeeze(valimg,axis=0) vallbl = np.squeeze(vallbl,axis=0) patch=1 p=0 while p+256 <= height: j=0 while j+256 <= width: # Crop 256X256 patch valImg = valimg[p:p+256, j:j+256] valLabel = vallbl[p:p+256, j:j+256] # Expand the dimensions back to 4 valImg = np.expand_dims(valImg,axis=0) valLabel = np.expand_dims(valLabel,axis=0) # validate model on patch valloss,valacc,valiou,v_image,v_mask,val_pred = sess.run([Totalloss,accuracy,val_iou,v_im,v_mk,v_p], feed_dict={X:valImg, y:valLabel, training: False}) print ("Validating on image %s \t patch: %d \t CE Loss: %.3f \t Accuracy: %.2f \t IOU: %.2f" %(valset[n].split('/')[len(valset[n].split('/'))-1],patch,valloss,valacc,valiou)) # Summarize important details val_summary_writer.add_summary(v_image,global_step_value) val_summary_writer.add_summary(v_mask,global_step_value) val_summary_writer.add_summary(val_pred,global_step_value) # Accumulate metrics to determine average over the entire set acc_valSet.append(valacc) iou_valSet.append(valiou) loss_valSet.append(valloss) j = j + 256 patch =patch + 1 p = p + 256 print ("\n") else: print ("\nValidating on original image resolution\n") #print ("\nValidation directory has %d data\n" %((len(os.listdir(ValDir))/2))) print ("\nValidation directory has %d data\n" %(len(valset))) for n in range (len(valset)): valimg, vallbl = sess.run([val_image, val_label]) # validate model on image valloss,valacc,valiou,v_image,v_mask,val_pred = sess.run([Totalloss,accuracy,val_iou,v_im,v_mk,v_p], feed_dict={X:valimg, y:vallbl, training: False}) print ("Validating on image %s \t CE Loss: %.3f \t Accuracy: %.2f \t IOU: %.2f" %(valset[n].split('/')[len(valset[n].split('/'))-1],valloss,valacc,valiou)) # Summarize important details val_summary_writer.add_summary(v_image,global_step_value) val_summary_writer.add_summary(v_mask,global_step_value) val_summary_writer.add_summary(val_pred,global_step_value) # Accumulate metrics to determine average over the entire set acc_valSet.append(valacc) iou_valSet.append(valiou) loss_valSet.append(valloss) acc_V = np.mean(acc_valSet) iou_V = np.mean(iou_valSet) loss_V = np.mean(loss_valSet) # Summarize important details and add it to the directory x,y,z = sess.run([v_l,v_a,v_I],feed_dict={v_loss:loss_V,v_iou:iou_V,v_acc:acc_V}) val_summary_writer.add_summary(x,global_step_value) val_summary_writer.add_summary(y,global_step_value) val_summary_writer.add_summary(z,global_step_value) print ("\n---------------------------------------------------------------------------- validation accuracy: %.3f ----------------------------------------------------------------------" %(acc_V)) print ("---------------------------------------------------------------------------- validation IOU: %.3f ---------------------------------------------------------------------------" %(iou_V)) print ("---------------------------------------------------------------------------- Cross Entropy loss: %.3f -------------------------------------------------------------------------" %(loss_V)) # Note down performance in text file print ("\n---------------------------------------------------------------------------- Saving the metrics in the file ------------------------------------------------------------------\n") iu_File.write("accuracy for model %d is = %f" %(i,acc_V)) iu_File.write("\n") iu_File.write("iou for model %d is = %f" %(i,iou_V)) iu_File.write("\n") iu_File.write("ce_loss for model %d is = %f" %(i,loss_V)) iu_File.write("\n\n") # Check when to stop training and go for testing if not mean_CELoss_valSet: print ("\nEmpty checkpoint directory, keeping the first model ###### training contd...\n") mean_CELoss_valSet[i] = loss_V else: if all(v > loss_V for k,v in mean_CELoss_valSet.items()): mean_CELoss_valSet[i] = loss_V print ("\n--------------------------------------------------------------------- Continuing training -----------------------------------------------------------------------------\n") overfit = 0 else: overfit = overfit +1 print ("\n------------------------------------------------------------------------- Validation dropping at %d step -------------------------------------------------------------------\n"%(overfit)) if overfit == 50: model_to_test = (min(mean_CELoss_valSet.items(), key=lambda x: x[1])[0]) print ("\n############################## The testing model is %d with meam cross entropy %.3f" %(model_to_test,mean_CELoss_valSet.get(model_to_test))) print ("\n############################## Stopping training as no more improvement in segmentation is possible") print ("\n############################## Proceed for testing with the model with best validation score") iu_File.close() break finally: sess.close() return model_to_test
def inference(images): """Build the CIFAR-10 model. Args: images: Images returned from distorted_inputs() or inputs(). Returns: Logits. """ # We instantiate all variables using tf.get_variable() instead of # tf.Variable() in order to share variables across multiple GPU training runs. # If we only ran this model on a single GPU, we could simplify this function # by replacing all instances of tf.get_variable() with tf.Variable(). # # conv1 with tf.variable_scope('conv1') as scope: kernel = _variable_with_weight_decay('weights', shape=[5, 5, 3, 64], stddev=5e-2, wd=None) conv = tf.nn.conv2d(images, kernel, [1, 1, 1, 1], padding='SAME') biases = _variable_on_cpu('biases', [64], tf.constant_initializer(0.0)) pre_activation = tf.nn.bias_add(conv, biases) conv1 = tf.nn.relu(pre_activation, name=scope.name) _activation_summary(conv1) # pool1 pool1 = tf.nn.max_pool(conv1, ksize=[1, 3, 3, 1], strides=[1, 2, 2, 1], padding='SAME', name='pool1') # norm1 norm1 = tf.nn.lrn(pool1, 4, bias=1.0, alpha=0.001 / 9.0, beta=0.75, name='norm1') # conv2 with tf.variable_scope('conv2') as scope: kernel = _variable_with_weight_decay('weights', shape=[5, 5, 64, 64], stddev=5e-2, wd=None) conv = tf.nn.conv2d(norm1, kernel, [1, 1, 1, 1], padding='SAME') biases = _variable_on_cpu('biases', [64], tf.constant_initializer(0.1)) pre_activation = tf.nn.bias_add(conv, biases) conv2 = tf.nn.relu(pre_activation, name=scope.name) _activation_summary(conv2) # norm2 norm2 = tf.nn.lrn(conv2, 4, bias=1.0, alpha=0.001 / 9.0, beta=0.75, name='norm2') # pool2 pool2 = tf.nn.max_pool(norm2, ksize=[1, 3, 3, 1], strides=[1, 2, 2, 1], padding='SAME', name='pool2') # local3 with tf.variable_scope('local3') as scope: # Move everything into depth so we can perform a single matrix multiply. reshape = tf.reshape(pool2, [images.get_shape().as_list()[0], -1]) dim = reshape.get_shape()[1].value weights = _variable_with_weight_decay('weights', shape=[dim, 384], stddev=0.04, wd=0.004) biases = _variable_on_cpu('biases', [384], tf.constant_initializer(0.1)) local3 = tf.nn.relu(tf.matmul(reshape, weights) + biases, name=scope.name) _activation_summary(local3) # local4 with tf.variable_scope('local4') as scope: weights = _variable_with_weight_decay('weights', shape=[384, 192], stddev=0.04, wd=0.004) biases = _variable_on_cpu('biases', [192], tf.constant_initializer(0.1)) local4 = tf.nn.relu(tf.matmul(local3, weights) + biases, name=scope.name) _activation_summary(local4) # linear layer(WX + b), # We don't apply softmax here because # tf.nn.sparse_softmax_cross_entropy_with_logits accepts the unscaled logits # and performs the softmax internally for efficiency. with tf.variable_scope('softmax_linear') as scope: weights = _variable_with_weight_decay('weights', [192, NUM_CLASSES], stddev=1 / 192.0, wd=None) biases = _variable_on_cpu('biases', [NUM_CLASSES], tf.constant_initializer(0.0)) softmax_linear = tf.add(tf.matmul(local4, weights), biases, name=scope.name) _activation_summary(softmax_linear) # tf.summary.scalar('accuracy', accuracy(softmax_linear,labels)) return softmax_linear
def non_linear_projection(x): scope = tf.get_variable_scope() w = get_scope_variable(scope, 'p_w', x.get_shape()) b = get_scope_variable(scope, 'p_b', x.get_shape()) return tf.tanh(tf.add(tf.multiply(x, w), b))
sess = tf.Session() iris = ds.load_iris() binary_target = np.array([1. if x == 0 else 0. for x in iris.target]) iris_2d = np.array([[x[2], x[3]] for x in iris.data]) BATCH_SIZE = 20 x1_data = tf.placeholder(shape=[None,1], dtype=tf.float32) x2_data = tf.placeholder(shape=[None,1], dtype=tf.float32) y_target = tf.placeholder(shape=[None,1], dtype=tf.float32) A = tf.Variable(tf.random_normal(shape=[1,1])) b = tf.Variable(tf.random_normal(shape=[1,1])) my_output = x1_data - (tf.add(tf.matmul(x2_data, A), b)) xentropy = tf.nn.sigmoid_cross_entropy_with_logits(labels=y_target, logits=my_output) train_step = tf.train.GradientDescentOptimizer(0.05).minimize(xentropy) init = tf.global_variables_initializer() sess.run(init) for i in range(1000): random_i = np.random.choice(len(iris_2d), size=BATCH_SIZE) rand_x = iris_2d[random_i] rand_x1 = np.array([[x[0]] for x in rand_x]) rand_x2 = np.array([[x[1]] for x in rand_x]) rand_y = np.array([[y] for y in binary_target[random_i]]) sess.run(train_step, feed_dict={x1_data:rand_x1, x2_data:rand_x2, y_target: rand_y}) if (i+1)%200 == 0: