def create_model_trainable(self): """ The model definition. """ outputs, reg = self.nn(self.input_placeholder) self.predictions = outputs with tf.name_scope('CostFunction'): with tf.name_scope('Q_Vals'): self.q_vals = tf.reduce_sum(tf.mul(self.predictions, self.actions_placeholder), 1) reward = tf.reduce_sum(self.q_vals) tf.scalar_summary('Q_vals', reward) with tf.name_scope('Loss'): loss_non_regularized = tf.reduce_sum(tf.square(self.labels_placeholder - self.q_vals)) self.loss = loss_non_regularized + reg tf.scalar_summary('Loss (without regularization)', loss_non_regularized) tf.scalar_summary('Regularization', reg) with tf.name_scope('Optimizer'): #optimizer = tf.train.AdamOptimizer(learning_rate=self.lr) optimizer = tf.train.GradientDescentOptimizer(learning_rate=self.lr) # optimizer = tf.train.RMSPropOptimizer(learning_rate = self.lr) self.train_op = optimizer.minimize(self.loss)
def loc_net_fc(images, batch_size): images -= 128 images /= 128. images = tf.image.resize_images(images, 150, 150) images_flat = tf.reshape(images, [batch_size, -1]) hidden_size = 100 with tf.name_scope('fc1') as scope: weights = tf.Variable(tf.truncated_normal([150**2*3, hidden_size], dtype=tf.float32, stddev=1e-3), name='weights') biases = tf.Variable(tf.constant(0.0, shape=[hidden_size], dtype=tf.float32), name='biases') hidden = tf.add(tf.matmul(images_flat, weights), biases, name=scope) hidden = tf.nn.relu(hidden) with tf.name_scope('fc2') as scope: weights = tf.Variable(tf.truncated_normal([hidden_size, 3], dtype=tf.float32, stddev=1e-3), name='weights') biases = tf.Variable(tf.constant(0.0, shape=[3], dtype=tf.float32), name='biases') theta = tf.add(tf.matmul(hidden, weights), biases, name=scope) theta = tf.nn.tanh(theta) return theta
def init_training_graph(self): with tf.name_scope('Evaluation'): # self.logits = self.conv_layer_f(self.last, self.logits_weight, strides=[1,1,1,1], scope_name="logits/") with tf.name_scope("logits/"): self.logits2 = tf.nn.conv2d(self.last, self.logits_weight, strides=[1,1,1,1], padding="VALID") self.logits = tf.nn.bias_add(self.logits2, self.logits_biases) self.predictions = self.logits #self.predictions = tf.squeeze(self.logits, [3]) #softmax = tf.nn.softmax(self.logits) #print softmax.get_shape() #self.predictions = tf.slice(softmax, [0, 0, 0, 0], [-1, -1, -1, 1]) with tf.name_scope('Loss'): self.loss = tf.reduce_mean(tf.losses.mean_squared_error(self.logits, self.train_labels_node)) #self.loss = tf.reduce_mean(tf.losses.mean_squared_error(self.predictions, self.train_labels_node)) tf.summary.scalar("mean_squared_error", self.loss) self.predictions = tf.squeeze(self.predictions, [3]) self.train_prediction = self.predictions self.test_prediction = self.predictions tf.global_variables_initializer().run() print('Computational graph initialised')
def bboxes_clip(bbox_ref, bboxes, scope=None): """Clip bounding boxes to a reference box. Batch-compatible if the first dimension of `bbox_ref` and `bboxes` can be broadcasted. Args: bbox_ref: Reference bounding box. Nx4 or 4 shaped-Tensor; bboxes: Bounding boxes to clip. Nx4 or 4 shaped-Tensor or dictionary. Return: Clipped bboxes. """ # Bboxes is dictionary. if isinstance(bboxes, dict): with tf.name_scope(scope, 'bboxes_clip_dict'): d_bboxes = {} for c in bboxes.keys(): d_bboxes[c] = bboxes_clip(bbox_ref, bboxes[c]) return d_bboxes # Tensors inputs. with tf.name_scope(scope, 'bboxes_clip'): # Easier with transposed bboxes. Especially for broadcasting. bbox_ref = tf.transpose(bbox_ref) bboxes = tf.transpose(bboxes) # Intersection bboxes and reference bbox. ymin = tf.maximum(bboxes[0], bbox_ref[0]) xmin = tf.maximum(bboxes[1], bbox_ref[1]) ymax = tf.minimum(bboxes[2], bbox_ref[2]) xmax = tf.minimum(bboxes[3], bbox_ref[3]) # Double check! Empty boxes when no-intersection. ymin = tf.minimum(ymin, ymax) xmin = tf.minimum(xmin, xmax) bboxes = tf.transpose(tf.stack([ymin, xmin, ymax, xmax], axis=0)) return bboxes
def make_HBF2_model(x,W1,S1,C1,W2,S2,C2,phase_train): with tf.name_scope("layer1") as scope: layer1 = ml.get_Gaussian_layer(x,W1,S1,C1,phase_train) with tf.name_scope("layer2") as scope: layer2 = ml.get_Gaussian_layer(layer1,W2,S2,C2,phase_train) y = layer2 return y
def build(self, FLAGS): """None Build the model graph :return: """ with tf.name_scope('G_'): self.predict_g = self.__G__() with tf.name_scope('D_'): self.predict_d, self.predict_d_logits = self.__D__(self.input_d, input_type="Real") tf.get_variable_scope().reuse_variables() self.predict_d_for_g, self.predict_d_logits_for_g = self.__D__(self.predict_g, input_type="Gen") if len(self.regularization_values_d) > 0: self.regularization_sum_d = sum(self.regularization_values_d) with tf.name_scope('loss'): # self.loss_g = self.__loss_g__(predict=self.predict_g, self.labels, reg=self.regularization_sum) self.__loss__(FLAGS) with tf.name_scope('training'): self.train_op_d, self.train_op_g = self.__training__(learning_rate=FLAGS.learning_rate) with tf.name_scope('evaluation'): # Calculate accuracy L2 norm self.evaluation = self.__evaluation__(predict=self.predict_g, labels=self.labels)
def fc_layers(self): # fc1 with tf.name_scope('fc1') as scope: shape = int(np.prod(self.pool5.get_shape()[1:])) fc1w = tf.Variable(tf.truncated_normal([shape, 4096], dtype=tf.float32, stddev=1e-1), name='weights') fc1b = tf.Variable(tf.constant(1.0, shape=[4096], dtype=tf.float32), trainable=True, name='biases') pool5_flat = tf.reshape(self.pool5, [-1, shape]) fc1l = tf.nn.bias_add(tf.matmul(pool5_flat, fc1w), fc1b) self.fc1 = tf.nn.relu(fc1l) self.parameters += [fc1w, fc1b] # fc2 with tf.name_scope('fc2') as scope: fc2w = tf.Variable(tf.truncated_normal([4096, 4096], dtype=tf.float32, stddev=1e-1), name='weights') fc2b = tf.Variable(tf.constant(1.0, shape=[4096], dtype=tf.float32), trainable=True, name='biases') fc2l = tf.nn.bias_add(tf.matmul(self.fc1, fc2w), fc2b) self.fc2 = tf.nn.relu(fc2l) self.parameters += [fc2w, fc2b] # fc3 with tf.name_scope('fc3') as scope: fc3w = tf.Variable(tf.truncated_normal([4096, 1000], dtype=tf.float32, stddev=1e-1), name='weights') fc3b = tf.Variable(tf.constant(1.0, shape=[1000], dtype=tf.float32), trainable=True, name='biases') self.fc3l = tf.nn.bias_add(tf.matmul(self.fc2, fc3w), fc3b) self.parameters += [fc3w, fc3b]
def inference(images, hidden1_units): """Build the MNIST model up to where it may be used for inference. Args: images: Images placeholder, from inputs(). hidden1_units: Size of the first hidden layer. hidden2_units: Size of the second hidden layer. Returns: softmax_linear: Output tensor with the computed logits. """ # Hidden 1 with tf.name_scope('hidden1'): weights = tf.Variable( tf.truncated_normal([IMAGE_PIXELS, hidden1_units], stddev=1.0 / math.sqrt(float(IMAGE_PIXELS))), name='weights') biases = tf.Variable(tf.zeros([hidden1_units]), name='biases') hidden1 = tf.nn.relu(tf.matmul(images, weights) + biases) # Hidden 2 # Linear with tf.name_scope('softmax_linear'): weights = tf.Variable( tf.truncated_normal([hidden1_units, NUM_CLASSES], stddev=1.0 / math.sqrt(float(hidden1_units))), name='weights') biases = tf.Variable(tf.zeros([NUM_CLASSES]), name='biases') logits = tf.matmul(hidden1, weights) + biases return logits
def nn_conv_layer(input_tensor, patch_size, num_channels,output_depth, layer_name, biases=False,act=None, pool=None): """Reusable code for making a simple neural net layer. """ # Adding a name scope ensures logical grouping of the layers in the graph. with tf.name_scope(layer_name): # This Variable will hold the state of the weights for the layer with tf.name_scope('weights'): weights = weight_variable([patch_size,patch_size,num_channels,output_depth]) # print ("weights:%s"%(weights.get_shape())) variable_summaries(weights, layer_name + '/weights') if (biases==True): with tf.name_scope('biases'): biases = bias_variable([output_depth]) # print("biases:%s" % (biases.get_shape())) variable_summaries(biases, layer_name + '/biases') with tf.name_scope('conv2d'): # print("input:%s" % (input_tensor.get_shape())) preactivate = tf.nn.conv2d(input_tensor, weights, [1, 1, 1, 1], padding='SAME') tf.histogram_summary(layer_name + '/pre_activations', preactivate) print("preactivate:%s" % (preactivate.get_shape())) if (pool!=None): max_pool=pool(preactivate,ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME',name='max_pool') if (act!=None): activations = act(max_pool+biases, 'activation') # tf.histogram_summary(layer_name + '/activations', activations) return preactivate
def variable_summaries(var, name, collection_key): """Attach a lot of summaries to a Tensor (for TensorBoard visualization). Args: - var: Tensor for variable from which we want to log. - name: Variable name. - collection_key: Collection to save the summary to, can be any key of `VAR_LOG_LEVELS`. """ if collection_key not in VAR_LOG_LEVELS.keys(): raise ValueError('"{}" not in `VAR_LOG_LEVELS`'.format(collection_key)) collections = VAR_LOG_LEVELS[collection_key] with tf.name_scope(name): mean = tf.reduce_mean(var) tf.summary.scalar('mean', mean, collections) num_params = tf.reduce_prod(tf.shape(var)) tf.summary.scalar('num_params', num_params, collections) with tf.name_scope('stddev'): stddev = tf.sqrt(tf.reduce_mean(tf.square(var - mean))) tf.summary.scalar('stddev', stddev, collections) tf.summary.scalar('max', tf.reduce_max(var), collections) tf.summary.scalar('min', tf.reduce_min(var), collections) tf.summary.histogram('histogram', var, collections) tf.summary.scalar('sparsity', tf.nn.zero_fraction(var), collections)
def ce(model, config, scope, connect, threshold = 1e-5): with tf.variable_scope(scope), tf.name_scope(scope): with tf.variable_scope('inputs'), tf.name_scope('inputs'): model['%s_in0length' %scope] = model['%s_out0length' %connect] model['%s_in1length' %scope] = model['%s_out1length' %connect] model['%s_in2length' %scope] = model['%s_out2length' %connect] model['%s_maxin2length' %scope] = model['%s_maxout2length' %connect] model['%s_inputs' %scope] = tf.clip_by_value(tf.nn.softmax(model['%s_outputs' %connect]), threshold, 1. - threshold, name = '%s_inputs' %scope) model['%s_out0length' %scope] = model['%s_in0length' %scope] model['%s_out1length' %scope] = model['%s_in1length' %scope] model['%s_out2length' %scope] = tf.placeholder(tf.int32, [model['%s_in0length' %scope]], '%s_out2length' %scope) model['%s_maxout2length' %scope] = model['%s_maxin2length' %scope] with tf.variable_scope('labels'), tf.name_scope('labels'): model['%s_labels_len' %scope] = tf.placeholder(tf.int32, [model['%s_in0length' %scope]], '%s_labels_len' %scope) model['%s_labels_ind' %scope] = tf.placeholder(tf.int64, [None, 2], '%s_labels_ind' %scope) model['%s_labels_val' %scope] = tf.placeholder(tf.int32, [None], '%s_labels_val' %scope) model['%s_labels_collapsed' %scope] = tf.sparse_to_dense(model['%s_labels_ind' %scope], [model['%s_maxin2length' %scope], model['%s_in0length' %scope]], model['%s_labels_val' %scope], -1, name = '%s_labels_collapsed' %scope) model['%s_labels' %scope] = tf.one_hot(model['%s_labels_collapsed' %scope], model['%s_out1length' %scope], name = '%s_labels' %scope) with tf.variable_scope('loss'), tf.name_scope('loss'): model['%s_loss' %scope] = tf.reduce_sum(-tf.multiply(model['%s_labels' %scope], tf.log(model['%s_inputs' %scope])), name = '%s_loss' %scope) with tf.variable_scope('outputs'), tf.name_scope('outputs'): model['%s_output' %scope] = model['%s_inputs' %scope] return model
def add_evaluation_step(result_tensor, ground_truth_tensor): """Inserts the operations we need to evaluate the accuracy of our results. Args: result_tensor: The new final node that produces results. ground_truth_tensor: The node we feed ground truth data into. Returns: Nothing. """ with tf.name_scope('accuracy'): with tf.name_scope('correct_prediction'): # tf.argmax(result_tensor, 1) = return index of maximal value (= 1 in a 1-of-N encoding vector) in each row (axis = 1) # But we have more ones (indicating multiple labels) in one row of result_tensor due to the multi-label classification # correct_prediction = tf.equal(tf.argmax(result_tensor, 1), \ # tf.argmax(ground_truth_tensor, 1)) # ground_truth is not a binary tensor, it contains the probabilities of each label = we need to tf.round() it # to acquire a binary tensor allowing comparison by tf.equal() # See: http://stackoverflow.com/questions/39219414/in-tensorflow-how-can-i-get-nonzero-values-and-their-indices-from-a-tensor-with correct_prediction = tf.equal(tf.round(result_tensor), ground_truth_tensor) with tf.name_scope('accuracy'): # Mean accuracy over all labels: # http://stackoverflow.com/questions/37746670/tensorflow-multi-label-accuracy-calculation evaluation_step = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) tf.scalar_summary('accuracy', evaluation_step) return evaluation_step
def build(self): """None Build the model graph :return: """ with tf.name_scope('G_'): self.predict_g = self.__G__() self.predict_g2 = self.__G2__() with tf.name_scope('D_'): # Create reference examples # Input d holds real&imaginary values. The discriminative decision based on reconstructed image self.reconstructed_image_reference = self.get_reconstructed_image(real=self.input_d['real'], imag=self.input_d['imag'], name='Both_gt') predict_g2_stacked = tf.stack([self.predict_g2['real'][:,0,:,:], self.predict_g2['imag'][:,0,:,:]], axis=1) self.predict, self.predict_logits = self.__D__([self.reconstructed_image_reference, predict_g2_stacked]) self.predict_d, self.predict_d_for_g = tf.split(value=self.predict, num_or_size_splits=2, axis=0) self.predict_d_logits, self.predict_d_logits_for_g = tf.split(value=self.predict_logits, num_or_size_splits=2, axis=0) self.clip_weights = self.__clip_weights__() with tf.name_scope('loss'): # self.loss_g = self.__loss_g__(predict=self.predict_g, self.labels, reg=self.regularization_sum) self.__loss__() with tf.name_scope('training'): self.train_op_d, self.train_op_g = self.__training__(learning_rate=self.FLAGS.learning_rate) with tf.name_scope('evaluation'): # Calculate accuracy L2 norm self.evaluation = self.__evaluation__(predict=self.predict_g, labels=self.labels)
def encode(self, inputs, attention_bias): """Generate continuous representation for inputs. Args: inputs: int tensor with shape [batch_size, input_length]. attention_bias: float tensor with shape [batch_size, 1, 1, input_length] Returns: float tensor with shape [batch_size, input_length, hidden_size] """ with tf.name_scope("encode"): # Prepare inputs to the layer stack by adding positional encodings and # applying dropout. embedded_inputs = self.embedding_softmax_layer(inputs) inputs_padding = model_utils.get_padding(inputs) with tf.name_scope("add_pos_encoding"): length = tf.shape(embedded_inputs)[1] pos_encoding = model_utils.get_position_encoding( length, self.params.hidden_size) encoder_inputs = embedded_inputs + pos_encoding if self.train: encoder_inputs = tf.nn.dropout( encoder_inputs, 1 - self.params.layer_postprocess_dropout) return self.encoder_stack(encoder_inputs, attention_bias, inputs_padding)
def inference(input_tensor,train,regularizer): #第一层卷积 with tf.variable_scope('layer1-conv1'): conv1_weights = tf.get_variable("weight", [CONV1_SIZE,CONV1_SIZE,NUM_CHANNELS,CONV1_DEEP], initializer=tf.truncated_normal_initializer(stddev=0.1)) conv1_biases = tf.get_variable("biases",[CONV1_DEEP], initializer=tf.constant_initializer(0.0)) conv1 = tf.nn.conv2d(input_tensor,conv1_weights, strides=[1,1,1,1],padding='SAME') relu1 = tf.nn.relu(tf.nn.bias_add(conv1,conv1_biases)) #第二层池化 with tf.name_scope('layer2-pool1'): pool1 = tf.nn.max_pool(relu1,ksize=[1,2,2,1], strides=[1,2,2,1],padding='SAME') #第三层卷积 with tf.variable_scope('layer3-conv2'): conv2_weights = tf.get_variable("weight", [CONV2_SIZE,CONV2_SIZE,CONV1_DEEP,CONV2_DEEP], initializer=tf.truncated_normal_initializer(stddev=0.1)) conv2_biases = tf.get_variable("biases",[CONV2_DEEP], initializer=tf.constant_initializer(0.0)) conv2 = tf.nn.conv2d(pool1,conv2_weights, strides=[1,1,1,1],padding='SAME') relu2 = tf.nn.relu(tf.nn.bias_add(conv2,conv2_biases)) #第四层池化 with tf.name_scope('layer4-pool2'): pool2 = tf.nn.max_pool(relu2,ksize=[1,2,2,1], strides=[1,2,2,1],padding='SAME') pool_shape = pool2.get_shape().as_list() nodes = pool_shape[1] * pool_shape[2] * pool_shape[3] reshaped = tf.reshape(pool2,[pool_shape[0],nodes]) #第五层全连接层 with tf.variable_scope('layer5-fc1'): fc1_weights = tf.get_variable("weight",[nodes,FC_SIZE], initializer=tf.truncated_normal_initializer(stddev=0.1)) #只有全连接层的权重需要加入正则化 if regularizer != None: tf.add_to_collection('losses',regularizer(fc1_weights)) fc1_biases = tf.get_variable("bias",[FC_SIZE], initializer=tf.constant_initializer(0.1)) fc1 = tf.nn.relu(tf.matmul(reshaped,fc1_weights) + fc1_biases) if train: fc1 = tf.nn.dropout(fc1,0.5) #第六层全连接层 with tf.variable_scope('layer6-fc2'): fc2_weights = tf.get_variable("weight",[FC_SIZE,NUM_LABELS], initializer=tf.truncated_normal_initializer(stddev=0.1)) #只有全连接层的权重需要加入正则化 if regularizer != None: tf.add_to_collection('losses',regularizer(fc2_weights)) fc2_biases = tf.get_variable("bias",[NUM_LABELS], initializer=tf.constant_initializer(0.1)) logit = tf.matmul(fc1,fc2_weights) + fc2_biases return logit
def __init__(self, num_features, num_output, l2_reg_lambda=0.0, neg_output=False): self.input_x = tf.placeholder(tf.float32, [None, num_features], name="input_x") self.input_y = tf.placeholder(tf.float32, [None, num_output], name="input_y") # Keeping track of l2 regularization loss (optional) l2_loss = tf.constant(0.0) with tf.name_scope("softmax"): filter_shape = [num_features, num_output] W = tf.Variable(tf.truncated_normal(filter_shape, stddev=0.1)) b = tf.Variable(tf.constant(0.1, shape=[num_output])) self.raw_scores = tf.nn.xw_plus_b(self.input_x, W, b, name="scores") if neg_output: self.scores = tf.nn.elu(self.raw_scores, name="tanh") else: self.scores = tf.nn.relu(self.raw_scores, name="relu") l2_loss += tf.nn.l2_loss(W) l2_loss += tf.nn.l2_loss(b) with tf.name_scope("loss"): self.losses = tf.square(tf.sub(self.scores, self.input_y)) self.avgloss = tf.reduce_mean(tf.abs(tf.sub(self.scores, self.input_y))) self.loss = tf.reduce_mean(self.losses) + l2_reg_lambda * l2_loss
def feature_extraction(x, Nx, Ny, Ch, conv_w_1, conv_b_1, conv_w_2, conv_b_2, ff_w_1, ff_b_1, m1s = 4, m2s = 2, No = 512): """ Creates a convolutional neural network to analyze some world tensor and return features from it. """ # Check that all the sizes are consistent assert(Nx/m1s == int(Nx/m1s) and (Nx/m1s/m2s == int(Nx/m1s/m2s))) assert(Ny/m1s == int(Ny/m1s) and (Ny/m1s/m2s == int(Ny/m1s/m2s))) # First Convolutional Layer with tf.name_scope("Convolution1"): conv1_act = tf.nn.conv2d(x, conv_w_1, strides=[1, 1, 1, 1], padding='SAME') + conv_b_1 conv1 = tf.nn.relu(conv1_act, 'relu') # First Max Pooling Layer with tf.name_scope("Max1"): max1 = tf.nn.max_pool(conv1, ksize=[1, m1s, m1s, 1], strides=[1, m1s, m1s, 1], padding='SAME') # Second Convolutional Layer with tf.name_scope("Convolution2"): conv2_act = tf.nn.conv2d(max1, conv_w_2, strides=[1, 1, 1, 1], padding='SAME') + conv_b_2 conv2 = tf.nn.relu(conv2_act, 'relu') # Second Max Pooling Layer with tf.name_scope("Max2"): max2 = tf.nn.max_pool(conv2, ksize=[1, m2s, m2s, 1], strides=[1, m2s, m2s, 1], padding='SAME') # Reshaping max2 for FF1 max2_rshp = tf.reshape(max2, [-1, 288]) # Layer shape [None, 5, 5, 64] 1600 Total # First Feed Forward Layer with tf.name_scope('FF1'): ff1_act = tf.matmul(max2_rshp, ff_w_1) + ff_b_1 ff1 = tf.nn.relu(ff1_act, 'relu') return ff1
def get_probabilities(units_value, weights, bias, is_hidden_layer = False): if is_hidden_layer: with tf.name_scope("Hidden_Probabilities"): return tf.nn.sigmoid(tf.matmul(units_value,weights) + bias) else: with tf.name_scope("Visible_Probabilities"): return tf.nn.sigmoid(tf.matmul(units_value,tf.transpose(weights)) + bias)
def nn_graph(activation): # create graph x = tf.placeholder(tf.float32, (1, 2, 2, 3), 'x_placeholder') y = tf.placeholder(tf.int32, name='y_placeholder', shape=[1, 2]) with tf.name_scope('conv1'): conv_1 = tf.layers.conv2d( inputs=x, filters=10, kernel_size=[2, 2], padding="same", activation=activation) with tf.name_scope('fc2'): flatten = tf.layers.flatten(conv_1) top = tf.layers.dense(flatten, _classes) with tf.name_scope('logits'): logits = tf.nn.softmax(top) with tf.name_scope('loss'): cross_entropy = tf.nn.softmax_cross_entropy_with_logits(labels=y, logits=logits) cross_entropy = tf.reduce_mean(cross_entropy) return x, y, cross_entropy
def ced(model, config, scope, connect, threshold = 1e-5): with tf.variable_scope(scope), tf.name_scope(scope): with tf.variable_scope('inputs'), tf.name_scope('inputs'): model['%s_in0length' %scope] = model['%s_out0length' %connect] model['%s_in1length' %scope] = model['%s_out1length' %connect] model['%s_in2length' %scope] = model['%s_out2length' %connect] model['%s_maxin2length' %scope] = model['%s_maxout2length' %connect] model['%s_inputs' %scope] = tf.clip_by_value(model['%s_outputs' %connect], threshold, 1. - threshold, name = '%s_inputs' %scope) model['%s_out0length' %scope] = model['%s_in0length' %scope] model['%s_out1length' %scope] = model['%s_in1length' %scope] model['%s_out2length' %scope] = tf.placeholder(tf.int32, [model['%s_in0length' %scope]], '%s_out2length' %scope) model['%s_maxout2length' %scope] = model['%s_maxin2length' %scope] with tf.variable_scope('labels'), tf.name_scope('labels'): model['%s_labels_len' %scope] = tf.placeholder(tf.int32, [model['%s_in0length' %scope]], '%s_labels_len' %scope) model['%s_labels_ind' %scope] = tf.placeholder(tf.int64, [None, 3], '%s_labels_ind' %scope) model['%s_labels_val' %scope] = tf.placeholder(tf.float32, [None], '%s_labels_val' %scope) model['%s_labels' %scope] = tf.sparse_to_dense(model['%s_labels_ind' %scope], [model['%s_in0length' %scope], model['%s_maxin2length' %scope], model['%s_maxin2length' %scope]], model['%s_labels_val' %scope], -1, name = '%s_labels' %scope) with tf.variable_scope('loss'), tf.name_scope('loss'): model['%s_loss' %scope] = tf.reduce_sum(tf.where(tf.less(model['%s_labels' %scope], tf.zeros([model['%s_in0length' %scope], model['%s_maxin2length' %scope], model['%s_maxin2length' %scope]], tf.float32)), tf.zeros([model['%s_in0length' %scope], model['%s_maxin2length' %scope], model['%s_maxin2length' %scope]], tf.float32), -tf.add(tf.multiply(model['%s_labels' %scope], tf.log(model['%s_inputs' %scope])), tf.multiply(tf.subtract(1., model['%s_labels' %scope]), tf.log(tf.subtract(1., model['%s_inputs' %scope]))))), name = '%s_loss' %scope) with tf.variable_scope('outputs'), tf.name_scope('outputs'): model['%s_output' %scope] = model['%s_inputs' %scope] return model
def conv_net(x, weights, biases, dropout): with tf.name_scope('input_czm'): # Reshape input picture x = tf.reshape(x, shape=[-1, 28, 28, 1]) with tf.name_scope('first_layer'): # Convolution Layer conv1 = conv2d(x, weights['wc1'], biases['bc1']) # Max Pooling (down-sampling) conv1 = maxpool2d(conv1, k=2) with tf.name_scope('sec_layer'): # Convolution Layer conv2 = conv2d(conv1, weights['wc2'], biases['bc2']) # Max Pooling (down-sampling) conv2 = maxpool2d(conv2, k=2) with tf.name_scope('full_conn'): # Fully connected layer # Reshape conv2 output to fit fully connected layer input fc1 = tf.reshape(conv2, [-1, weights['wd1'].get_shape().as_list()[0]]) fc1 = tf.add(tf.matmul(fc1, weights['wd1']), biases['bd1']) fc1 = tf.nn.relu(fc1) with tf.name_scope('dropout_ops'): # Apply Dropout fc1 = tf.nn.dropout(fc1, dropout) # Output, class prediction out = tf.add(tf.matmul(fc1, weights['out']), biases['out']) return out
def testVarOpScopeReuseParam(self): with self.test_session(): with tf.variable_scope("outer") as outer: with tf.variable_op_scope([], "tower", "default"): self.assertEqual(tf.get_variable("w", []).name, "outer/tower/w:0") with tf.name_scope("scope2") as sc2: self.assertEqual(sc2, "outer/tower/scope2/") with tf.variable_op_scope([], None, "default"): self.assertEqual(tf.get_variable("w", []).name, "outer/default/w:0") with tf.name_scope("scope2") as sc2: self.assertEqual(sc2, "outer/default/scope2/") with tf.variable_scope(outer) as outer: with tf.variable_op_scope([], "tower", "default", reuse=True): self.assertEqual(tf.get_variable("w", []).name, "outer/tower/w:0") with tf.name_scope("scope2") as sc2: self.assertEqual(sc2, "outer_1/tower/scope2/") outer.reuse_variables() with tf.variable_op_scope([], None, "default"): self.assertEqual(tf.get_variable("w", []).name, "outer/default/w:0") with tf.name_scope("scope2") as sc2: self.assertEqual(sc2, "outer_1/default/scope2/")
def loss(self, logits, labels, regularization): """Adds to the inference model the layers required to generate loss.""" with tf.name_scope('loss'): with tf.name_scope('var_loss'): labels = tf.cast(labels, tf.float32) shape = labels.get_shape() same_class = tf.boolean_mask(logits, tf.equal(labels, tf.ones(shape))) diff_class = tf.boolean_mask(logits, tf.not_equal(labels, tf.ones(shape))) same_mean, same_var = tf.nn.moments(same_class, [0]) diff_mean, diff_var = tf.nn.moments(diff_class, [0]) var_loss = same_var + diff_var with tf.name_scope('mean_loss'): mean_loss = self.lamda * tf.where(tf.greater(self.mu - (same_mean - diff_mean), 0), self.mu - (same_mean - diff_mean), 0) with tf.name_scope('regularization'): regularization *= tf.add_n(self.regularizers) loss = var_loss + mean_loss + regularization # Summaries for TensorBoard. tf.summary.scalar('loss/total', loss) with tf.name_scope('averages'): averages = tf.train.ExponentialMovingAverage(0.9) op_averages = averages.apply([var_loss, mean_loss, regularization, loss]) tf.summary.scalar('loss/avg/var_loss', averages.average(var_loss)) tf.summary.scalar('loss/avg/mean_loss', averages.average(mean_loss)) tf.summary.scalar('loss/avg/regularization', averages.average(regularization)) tf.summary.scalar('loss/avg/total', averages.average(loss)) with tf.control_dependencies([op_averages]): loss_average = tf.identity(averages.average(loss), name='control') return loss, loss_average
def generate_model(self, model, name=''): if not model: return self with tf.name_scope('state'): self.keep_prob = tf.placeholder(tf.float32) # 1 for testing! else 1 - dropout self.train_phase = tf.placeholder(tf.bool, name='train_phase') with tf.device(_cpu): self.global_step = tf.Variable( 0) # dont set, feed or increment global_step, tensorflow will do it automatically with tf.name_scope('data'): if len(self.input_shape) == 1: self.input_width = self.input_shape[0] elif self.input_shape: self.x = x = self.input = tf.placeholder(tf.float32, [None, self.input_shape[0], self.input_shape[1]]) # todo [None, self.input_shape] self.last_layer = x self.last_shape = x elif self.input_width: self.x = x = self.target = tf.placeholder(tf.float32, [None, self.input_width]) self.last_layer = x else: raise Exception("need input_shape or input_width by now") self.y = y = self.target = tf.placeholder(tf.float32, [None, self.output_width]) with tf.name_scope('model'): model(self) if (self.last_width != self.output_width): self.classifier() # 10 classes auto
def loss(self, logits, labels, regularization): """Adds to the inference model the layers required to generate loss.""" with tf.name_scope('loss'): with tf.name_scope('hinge_loss'): labels = tf.cast(labels, tf.float32) zeros = tf.zeros(labels.get_shape()) output = tf.ones(labels.get_shape()) - tf.multiply(labels, logits) hinge_loss = tf.where(tf.greater(output, zeros), output, zeros) hinge_loss = tf.reduce_mean(hinge_loss) with tf.name_scope('regularization'): regularization *= tf.add_n(self.regularizers) loss = hinge_loss + regularization # Summaries for TensorBoard. tf.summary.scalar('loss/hinge_loss', hinge_loss) tf.summary.scalar('loss/regularization', regularization) tf.summary.scalar('loss/total', loss) with tf.name_scope('averages'): averages = tf.train.ExponentialMovingAverage(0.9) op_averages = averages.apply([hinge_loss, regularization, loss]) tf.summary.scalar('loss/avg/hinge_loss', averages.average(hinge_loss)) tf.summary.scalar('loss/avg/regularization', averages.average(regularization)) tf.summary.scalar('loss/avg/total', averages.average(loss)) with tf.control_dependencies([op_averages]): loss_average = tf.identity(averages.average(loss), name='control') return loss, loss_average
def testSharingWeightsWithDifferentNamescope(self): num_units = 3 input_size = 5 batch_size = 2 num_proj = 4 with self.test_session(graph=tf.Graph()) as sess: initializer = tf.random_uniform_initializer(-1, 1, seed=self._seed) inputs = 10 * [ tf.placeholder(tf.float32, shape=(None, input_size))] cell = rnn_cell.LSTMCell( num_units, input_size, use_peepholes=True, num_proj=num_proj, initializer=initializer) with tf.name_scope("scope0"): with tf.variable_scope("share_scope"): outputs0, _ = rnn.rnn(cell, inputs, dtype=tf.float32) with tf.name_scope("scope1"): with tf.variable_scope("share_scope", reuse=True): outputs1, _ = rnn.rnn(cell, inputs, dtype=tf.float32) tf.initialize_all_variables().run() input_value = np.random.randn(batch_size, input_size) output_values = sess.run( outputs0 + outputs1, feed_dict={inputs[0]: input_value}) outputs0_values = output_values[:10] outputs1_values = output_values[10:] self.assertEqual(len(outputs0_values), len(outputs1_values)) for out0, out1 in zip(outputs0_values, outputs1_values): self.assertAllEqual(out0, out1)
def build(self): """None Build the model graph :return: """ with tf.name_scope('G_'): self.predict_g = self.__G__() with tf.name_scope('D_'): self.predict, self.predict_logits = self.__D__([self.input_d, self.predict_g], input_type="Real") self.predict_d, self.predict_d_for_g = tf.split(value=self.predict, num_or_size_splits=2, axis=0) self.predict_d_logits, self.predict_d_logits_for_g = tf.split(value=self.predict_logits, num_or_size_splits=2, axis=0) # self.predict_d, self.predict_d_logits # with tf.variable_scope(tf.get_variable_scope(), reuse=True): # self.predict_d_for_g, self.predict_d_logits_for_g = self.__D__(self.predict_g, input_type="Gen") if len(self.regularization_values_d) > 0: self.regularization_sum_d = sum(self.regularization_values_d) with tf.name_scope('loss'): # self.loss_g = self.__loss_g__(predict=self.predict_g, self.labels, reg=self.regularization_sum) self.__loss__() with tf.name_scope('training'): self.train_op_d, self.train_op_g = self.__training__(learning_rate=self.FLAGS.learning_rate) with tf.name_scope('evaluation'): # Calculate accuracy L2 norm self.evaluation = self.__evaluation__(predict=self.predict_g, labels=self.labels)
def inference(images,hidden1_units,hidden2_units): """建立前馈神经网络模型 Args: images:输入图像数据 hidden1_units:第一个隐藏层的神经元数目 hidden2_units:第二个隐藏层 的神经元数目 returns: softmax_linear:输出张量为计算后的结果 """ #隐藏层1 with tf.name_scope('hidden1'): weights = tf.Variable(tf.truncated_normal([IMAGE_PIXELS,hidden1_units],stddev=1.0/math.sqrt(float(IMAGE_PIXELS))),name='weights')#? biases = tf.Variable(tf.zeros([hidden1_units]),name='biases') hidden1 = tf.nn.relu(tf.matmul(images,weights)+biases) #隐藏层2 with tf.name_scope('hidden2'): weights = tf.Variable(tf.truncated_normal([hidden1_units,hidden2_units],stddev=1.0/math.sqrt(float(hidden1_units))),name='weights') biases = tf.Variable(tf.zeros([hidden2_units]),name='biases') hidden2 = tf.nn.relu(tf.matmul(hidden1,weights)+biases) #线性输出层 with tf.name_scope('softmax_linear'): weights = tf.Variable(tf.truncated_normal([hidden2_units,NUM_CLASSES]),name='biases') biases = tf.Variable(tf.zeros([NUM_CLASSES]),name='biases') logits = tf.matmul(hidden2,weights) + biases return logits
def bboxes_resize(bbox_ref, bboxes, name=None): """Resize bounding boxes based on a reference bounding box, assuming that the latter is [0, 0, 1, 1] after transform. Useful for updating a collection of boxes after cropping an image. """ # Bboxes is dictionary. if isinstance(bboxes, dict): with tf.name_scope(name, 'bboxes_resize_dict'): d_bboxes = {} for c in bboxes.keys(): d_bboxes[c] = bboxes_resize(bbox_ref, bboxes[c]) return d_bboxes # Tensors inputs. with tf.name_scope(name, 'bboxes_resize'): # Translate. v = tf.stack([bbox_ref[0], bbox_ref[1], bbox_ref[0], bbox_ref[1]]) bboxes = bboxes - v # Scale. s = tf.stack([bbox_ref[2] - bbox_ref[0], bbox_ref[3] - bbox_ref[1], bbox_ref[2] - bbox_ref[0], bbox_ref[3] - bbox_ref[1]]) bboxes = bboxes / s return bboxes
def testVarOpScopeOuterScope(self): with self.test_session(): with tf.variable_scope("outer") as outer: pass with tf.variable_op_scope([], outer, "default"): self.assertEqual(tf.get_variable("w", []).name, "outer/w:0") with tf.name_scope("scope2") as sc2: self.assertEqual(sc2, "outer_1/scope2/") with tf.variable_op_scope([], None, "default"): self.assertEqual(tf.get_variable("w", []).name, "outer/default/w:0") with tf.name_scope("scope2") as sc2: self.assertEqual(sc2, "outer_1/default/scope2/") with tf.variable_op_scope([], outer, "default", reuse=True): self.assertEqual(tf.get_variable("w", []).name, "outer/w:0") with tf.name_scope("scope2") as sc2: self.assertEqual(sc2, "outer_2/scope2/") outer.reuse_variables() with tf.variable_op_scope([], None, "default"): self.assertEqual(tf.get_variable("w", []).name, "outer/default/w:0") with tf.name_scope("scope2") as sc2: self.assertEqual(sc2, "outer_2/default/scope2/")
def extract_groups(predicted_boxes, predicted_scores, predicted_group_flags=None, predicted_offsets=None, mode='train', verbose=False, epsilon=1e-8, **kwargs): """ Extract crops from the outputs of intermediate stage. Args: predicted_boxes: A (batch_size, num_cells, num_cells, num_boxes, 4) array predicted_scores: A (batch_size, num_cells, num_cells, num_boxes, 1) array predicted_group_flags: A (batch_size, num_cells, num_cells, num_boxes, 1) array predicted_offsets: A (batch_size, num_cells, num_cells, num_boxes, 2) array mode: If test, the boxes are only passed to the next stage if they are worth being refined (ie groups or unprecise individual) Kwargs: {train, test}_patch_confidence_threshold: Minimum confidence threshold to qualify for refinement patch_nms_threshold: NMS threshold {train, test}_num_crops: Number of crops to extract test_patch_strong_confidence_threshold: high confidence threshold previous_batch_size: Batch size of the previous stage (for which `predicted boxes` where output). Needs to be statistically known for the NMS loop. #Returns: Extracted crops and their confidence scores """ if mode == 'train': # train time (confidence_threshold, nms_threshold, num_outputs) = get_defaults( kwargs, ['train_patch_confidence_threshold', 'train_patch_nms_threshold', 'train_num_crops'], verbose=verbose) elif mode in ['val', 'test']: # inference (confidence_threshold, nms_threshold, num_outputs) = get_defaults( kwargs, ['test_patch_confidence_threshold', 'test_patch_nms_threshold', 'test_num_crops'], verbose=verbose) else: raise ValueError('Unknown mode', mode) if verbose: print(' extracting %d crops' % num_outputs) ## Flatten # predicted_score: (batch, num_boxes, 1) # predicted_boxes: (batch, num_boxes, 4) with tf.name_scope('flat_output'): predicted_boxes = utils.flatten_percell_output(predicted_boxes) predicted_scores = utils.flatten_percell_output(predicted_scores) ## Filter kept_out_filter = tf.zeros(tf.shape(predicted_scores)) # default with tf.name_scope('filter_groups'): # At test time, we keep out individual confidences with high confidence # we save these `shortcut` boxes in the `kept_out_filter` Tensor if mode in ['test', 'val']: strong_confidence_threshold = get_defaults( kwargs, ['test_patch_strong_confidence_threshold'], verbose=verbose)[0] if isinstance(strong_confidence_threshold, tf.Tensor) or strong_confidence_threshold < 1.0: predicted_boxes, predicted_scores, kept_out_filter = filter_individuals( predicted_boxes, predicted_scores, predicted_group_flags, strong_confidence_threshold) # Additionally, we filter out boxes with confidence below the threshold if isinstance(confidence_threshold, tf.Tensor) or confidence_threshold > 0.: with tf.name_scope('filter_confidence'): predicted_boxes, predicted_scores = filter_threshold( predicted_boxes, predicted_scores, confidence_threshold) ## Rescale remaining boxes with the learned offsets with tf.name_scope('offsets_rescale_boxes'): if predicted_offsets is not None: predicted_boxes = utils.rescale_with_offsets( predicted_boxes, utils.flatten_percell_output(predicted_offsets), epsilon) ## Extract n best patches # crop_boxes: (batch, num_crops, 4) # crop_boxes_confidences: (batch, num_crops) predicted_scores = tf.squeeze(predicted_scores, axis=-1) if isinstance(num_outputs, tf.Tensor) or num_outputs > 0: # Non-Maximum Suppression: outputs the top `num_outputs` boxes after NMS if (isinstance(nms_threshold, tf.Tensor) or nms_threshold < 1.0) or (isinstance(num_outputs, tf.Tensor)): batch_size = get_defaults(kwargs, ['previous_batch_size'], verbose=verbose)[0] current_batch = tf.shape(predicted_boxes)[0] with tf.name_scope('nms'): nms_boxes = [] nms_boxes_confidences = [] for i in range(batch_size): boxes, scores = tf.cond( i < current_batch, # last batch can be smaller true_fn=lambda: utils.nms_with_pad(predicted_boxes[i, :, :], predicted_scores[i, :], num_outputs, iou_threshold=nms_threshold), false_fn=lambda: (tf.zeros((num_outputs, 4)), tf.zeros((num_outputs,))) ) nms_boxes.append(boxes) nms_boxes_confidences.append(scores) # Reshape nms boxes output predicted_boxes = tf.stack(nms_boxes, axis=0) predicted_boxes = tf.slice(predicted_boxes, (0, 0, 0), (current_batch, -1, -1)) predicted_boxes = tf.reshape(predicted_boxes, (-1, num_outputs, 4)) # Reshape nms scores output predicted_scores = tf.stack(nms_boxes_confidences, axis=0) predicted_scores = tf.slice(predicted_scores, (0, 0), (current_batch, -1)) predicted_scores = tf.reshape(predicted_scores, (-1, num_outputs)) # No NMS: Outputs `num_outputs` boxes with the best confidence scores # num_outputs need to be defined for tf.nn.top_k else: predicted_scores, top_indices = tf.nn.top_k(predicted_scores, k=num_outputs) batch_indices = tf.range(tf.shape(predicted_boxes)[0]) batch_indices = tf.tile(tf.expand_dims(batch_indices, axis=-1), (1, num_outputs)) gather_indices = tf.stack([batch_indices, top_indices], axis=-1) predicted_boxes = tf.gather_nd(predicted_boxes, gather_indices) # No filtering return predicted_boxes, predicted_scores, kept_out_filter
def convlayers(self): self.parameters = [] # zero-mean input with tf.name_scope('preprocess') as scope: mean = tf.constant([123.68, 116.779, 103.939], dtype=tf.float32, shape=[1, 1, 1, 3], name='img_mean') images = self.imgs - mean # conv1_1 with tf.name_scope('conv1_1') as scope: kernel = tf.Variable(tf.truncated_normal([3, 3, 3, 64], dtype=tf.float32, stddev=1e-1), name='weights') conv = tf.nn.conv2d(images, kernel, [1, 1, 1, 1], padding='SAME') biases = tf.Variable(tf.constant(0.0, shape=[64], dtype=tf.float32), trainable=True, name='biases') out = tf.nn.bias_add(conv, biases) self.conv1_1 = tf.nn.relu(out, name=scope) self.parameters += [kernel, biases] # conv1_2 with tf.name_scope('conv1_2') as scope: kernel = tf.Variable(tf.truncated_normal([3, 3, 64, 64], dtype=tf.float32, stddev=1e-1), name='weights') conv = tf.nn.conv2d(self.conv1_1, kernel, [1, 1, 1, 1], padding='SAME') biases = tf.Variable(tf.constant(0.0, shape=[64], dtype=tf.float32), trainable=True, name='biases') out = tf.nn.bias_add(conv, biases) self.conv1_2 = tf.nn.relu(out, name=scope) self.parameters += [kernel, biases] # pool1 self.pool1 = tf.nn.max_pool(self.conv1_2, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME', name='pool1') # conv2_1 with tf.name_scope('conv2_1') as scope: kernel = tf.Variable(tf.truncated_normal([3, 3, 64, 128], dtype=tf.float32, stddev=1e-1), name='weights') conv = tf.nn.conv2d(self.pool1, kernel, [1, 1, 1, 1], padding='SAME') biases = tf.Variable(tf.constant(0.0, shape=[128], dtype=tf.float32), trainable=True, name='biases') out = tf.nn.bias_add(conv, biases) self.conv2_1 = tf.nn.relu(out, name=scope) self.parameters += [kernel, biases] # conv2_2 with tf.name_scope('conv2_2') as scope: kernel = tf.Variable(tf.truncated_normal([3, 3, 128, 128], dtype=tf.float32, stddev=1e-1), name='weights') conv = tf.nn.conv2d(self.conv2_1, kernel, [1, 1, 1, 1], padding='SAME') biases = tf.Variable(tf.constant(0.0, shape=[128], dtype=tf.float32), trainable=True, name='biases') out = tf.nn.bias_add(conv, biases) self.conv2_2 = tf.nn.relu(out, name=scope) self.parameters += [kernel, biases] # pool2 self.pool2 = tf.nn.max_pool(self.conv2_2, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME', name='pool2') # conv3_1 with tf.name_scope('conv3_1') as scope: kernel = tf.Variable(tf.truncated_normal([3, 3, 128, 256], dtype=tf.float32, stddev=1e-1), name='weights') conv = tf.nn.conv2d(self.pool2, kernel, [1, 1, 1, 1], padding='SAME') biases = tf.Variable(tf.constant(0.0, shape=[256], dtype=tf.float32), trainable=True, name='biases') out = tf.nn.bias_add(conv, biases) self.conv3_1 = tf.nn.relu(out, name=scope) self.parameters += [kernel, biases] # conv3_2 with tf.name_scope('conv3_2') as scope: kernel = tf.Variable(tf.truncated_normal([3, 3, 256, 256], dtype=tf.float32, stddev=1e-1), name='weights') conv = tf.nn.conv2d(self.conv3_1, kernel, [1, 1, 1, 1], padding='SAME') biases = tf.Variable(tf.constant(0.0, shape=[256], dtype=tf.float32), trainable=True, name='biases') out = tf.nn.bias_add(conv, biases) self.conv3_2 = tf.nn.relu(out, name=scope) self.parameters += [kernel, biases] # conv3_3 with tf.name_scope('conv3_3') as scope: kernel = tf.Variable(tf.truncated_normal([3, 3, 256, 256], dtype=tf.float32, stddev=1e-1), name='weights') conv = tf.nn.conv2d(self.conv3_2, kernel, [1, 1, 1, 1], padding='SAME') biases = tf.Variable(tf.constant(0.0, shape=[256], dtype=tf.float32), trainable=True, name='biases') out = tf.nn.bias_add(conv, biases) self.conv3_3 = tf.nn.relu(out, name=scope) self.parameters += [kernel, biases] # pool3 self.pool3 = tf.nn.max_pool(self.conv3_3, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME', name='pool3') # conv4_1 with tf.name_scope('conv4_1') as scope: kernel = tf.Variable(tf.truncated_normal([3, 3, 256, 512], dtype=tf.float32, stddev=1e-1), name='weights') conv = tf.nn.conv2d(self.pool3, kernel, [1, 1, 1, 1], padding='SAME') biases = tf.Variable(tf.constant(0.0, shape=[512], dtype=tf.float32), trainable=True, name='biases') out = tf.nn.bias_add(conv, biases) self.conv4_1 = tf.nn.relu(out, name=scope) self.parameters += [kernel, biases] # conv4_2 with tf.name_scope('conv4_2') as scope: kernel = tf.Variable(tf.truncated_normal([3, 3, 512, 512], dtype=tf.float32, stddev=1e-1), name='weights') conv = tf.nn.conv2d(self.conv4_1, kernel, [1, 1, 1, 1], padding='SAME') biases = tf.Variable(tf.constant(0.0, shape=[512], dtype=tf.float32), trainable=True, name='biases') out = tf.nn.bias_add(conv, biases) self.conv4_2 = tf.nn.relu(out, name=scope) self.parameters += [kernel, biases] # conv4_3 with tf.name_scope('conv4_3') as scope: kernel = tf.Variable(tf.truncated_normal([3, 3, 512, 512], dtype=tf.float32, stddev=1e-1), name='weights') conv = tf.nn.conv2d(self.conv4_2, kernel, [1, 1, 1, 1], padding='SAME') biases = tf.Variable(tf.constant(0.0, shape=[512], dtype=tf.float32), trainable=True, name='biases') out = tf.nn.bias_add(conv, biases) self.conv4_3 = tf.nn.relu(out, name=scope) self.parameters += [kernel, biases] # pool4 self.pool4 = tf.nn.max_pool(self.conv4_3, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME', name='pool4') # conv5_1 with tf.name_scope('conv5_1') as scope: kernel = tf.Variable(tf.truncated_normal([3, 3, 512, 512], dtype=tf.float32, stddev=1e-1), name='weights') conv = tf.nn.conv2d(self.pool4, kernel, [1, 1, 1, 1], padding='SAME') biases = tf.Variable(tf.constant(0.0, shape=[512], dtype=tf.float32), trainable=True, name='biases') out = tf.nn.bias_add(conv, biases) self.conv5_1 = tf.nn.relu(out, name=scope) self.parameters += [kernel, biases] # conv5_2 with tf.name_scope('conv5_2') as scope: kernel = tf.Variable(tf.truncated_normal([3, 3, 512, 512], dtype=tf.float32, stddev=1e-1), name='weights') conv = tf.nn.conv2d(self.conv5_1, kernel, [1, 1, 1, 1], padding='SAME') biases = tf.Variable(tf.constant(0.0, shape=[512], dtype=tf.float32), trainable=True, name='biases') out = tf.nn.bias_add(conv, biases) self.conv5_2 = tf.nn.relu(out, name=scope) self.parameters += [kernel, biases] # conv5_3 with tf.name_scope('conv5_3') as scope: kernel = tf.Variable(tf.truncated_normal([3, 3, 512, 512], dtype=tf.float32, stddev=1e-1), name='weights') conv = tf.nn.conv2d(self.conv5_2, kernel, [1, 1, 1, 1], padding='SAME') biases = tf.Variable(tf.constant(0.0, shape=[512], dtype=tf.float32), trainable=True, name='biases') out = tf.nn.bias_add(conv, biases) self.conv5_3 = tf.nn.relu(out, name=scope) self.parameters += [kernel, biases] # pool5 self.pool5 = tf.nn.max_pool(self.conv5_3, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME', name='pool4')
def __init__(self, block_fn, num_blocks: List[int], num_frames: int, model_structure: List[Any], input_specs: layers.InputSpec = layers.InputSpec( shape=[None, None, None, None, 3]), model_edge_weights: Optional[List[Any]] = None, use_object_input: bool = False, attention_mode: str = 'peer', bn_decay: float = rf.BATCH_NORM_DECAY, bn_epsilon: float = rf.BATCH_NORM_EPSILON, use_sync_bn: bool = False, **kwargs): """Generator for AssembleNet++ models. Args: block_fn: `function` for the block to use within the model. Currently only has `bottleneck_block_interleave as its option`. num_blocks: list of 4 `int`s denoting the number of blocks to include in each of the 4 block groups. Each group consists of blocks that take inputs of the same resolution. num_frames: the number of frames in the input tensor. model_structure: AssembleNetPlus model structure in the string format. input_specs: `tf.keras.layers.InputSpec` specs of the input tensor. Dimension should be `[batch*time, height, width, channels]`. model_edge_weights: AssembleNet model structure connection weight in the string format. use_object_input : 'bool' values whether using object inputs attention_mode : 'str' , default = 'self', If we use peer attention 'peer' bn_decay: `float` batch norm decay parameter to use. bn_epsilon: `float` batch norm epsilon parameter to use. use_sync_bn: use synchronized batch norm for TPU. **kwargs: pass through arguments. Returns: Model `function` that takes in `inputs` and `is_training` and returns the output `Tensor` of the AssembleNetPlus model. """ data_format = tf.keras.backend.image_data_format() # Creation of the model graph. logging.info('model_structure=%r', model_structure) logging.info('model_structure=%r', model_structure) logging.info('model_edge_weights=%r', model_edge_weights) structure = model_structure if use_object_input: original_inputs = tf.keras.Input(shape=input_specs[0].shape[1:]) object_inputs = tf.keras.Input(shape=input_specs[1].shape[1:]) input_specs = input_specs[0] else: original_inputs = tf.keras.Input(shape=input_specs.shape[1:]) object_inputs = None original_num_frames = num_frames assert num_frames > 0, f'Invalid num_frames {num_frames}' grouping = {-3: [], -2: [], -1: [], 0: [], 1: [], 2: [], 3: []} for i in range(len(structure)): grouping[structure[i][0]].append(i) stem_count = len(grouping[-3]) + len(grouping[-2]) + len(grouping[-1]) assert stem_count != 0 stem_filters = 128 // stem_count if len(input_specs.shape) == 5: first_dim = (input_specs.shape[0] * input_specs.shape[1] if input_specs.shape[0] and input_specs.shape[1] else -1) reshape_inputs = tf.reshape(original_inputs, (first_dim, ) + input_specs.shape[2:]) elif len(input_specs.shape) == 4: reshape_inputs = original_inputs else: raise ValueError( f'Expect input spec to be 4 or 5 dimensions {input_specs.shape}' ) if grouping[-2]: # Instead of loading optical flows as inputs from data pipeline, we are # applying the "Representation Flow" to RGB frames so that we can compute # the flow within TPU/GPU on fly. It's essentially optical flow since we # do it with RGBs. axis = 3 if data_format == 'channels_last' else 1 flow_inputs = rf.RepresentationFlow( original_num_frames, depth=reshape_inputs.shape.as_list()[axis], num_iter=40, bottleneck=1)(reshape_inputs) streams = [] for i in range(len(structure)): with tf.name_scope('Node_' + str(i)): if structure[i][0] == -1: inputs = asn.rgb_conv_stem( reshape_inputs, original_num_frames, stem_filters, temporal_dilation=structure[i][1], bn_decay=bn_decay, bn_epsilon=bn_epsilon, use_sync_bn=use_sync_bn) streams.append(inputs) elif structure[i][0] == -2: inputs = asn.flow_conv_stem( flow_inputs, stem_filters, temporal_dilation=structure[i][1], bn_decay=bn_decay, bn_epsilon=bn_epsilon, use_sync_bn=use_sync_bn) streams.append(inputs) elif structure[i][0] == -3: # In order to use the object inputs, you need to feed your object # input tensor here. inputs = object_conv_stem(object_inputs) streams.append(inputs) else: block_number = structure[i][0] combined_inputs = [ streams[structure[i][1][j]] for j in range(0, len(structure[i][1])) ] logging.info(grouping) nodes_below = [] for k in range(-3, structure[i][0]): nodes_below = nodes_below + grouping[k] peers = [] if attention_mode: lg_channel = -1 # To show structures for attention we show nodes_below logging.info(nodes_below) for k in nodes_below: logging.info(streams[k].shape) lg_channel = max(streams[k].shape[3], lg_channel) for node_index in nodes_below: attn = tf.reduce_mean(streams[node_index], [1, 2]) attn = tf.keras.layers.Dense( units=lg_channel, kernel_initializer=tf. random_normal_initializer(stddev=.01))( inputs=attn) peers.append(attn) combined_inputs = fusion_with_peer_attention( combined_inputs, index=i, attention_mode=attention_mode, attention_in=peers, use_5d_mode=False) graph = asn.block_group(inputs=combined_inputs, filters=structure[i][2], block_fn=block_fn, blocks=num_blocks[block_number], strides=structure[i][4], name='block_group' + str(i), block_level=structure[i][0], num_frames=num_frames, temporal_dilation=structure[i][3]) streams.append(graph) if use_object_input: inputs = [original_inputs, object_inputs] else: inputs = original_inputs super(AssembleNetPlus, self).__init__(inputs=inputs, outputs=streams, **kwargs)
def fusion_with_peer_attention(inputs: List[tf.Tensor], index: Optional[int] = None, attention_mode: Optional[str] = None, attention_in: Optional[List[tf.Tensor]] = None, use_5d_mode: bool = False, model_edge_weights: Optional[List[Any]] = None, num_object_classes: Optional[int] = None): """Weighted summation of multiple tensors, while using peer-attention. Summation weights are to be learned. Uses spatial max pooling and 1x1 conv. to match their sizes. Before the summation, each connection (i.e., each input) itself is scaled with channel-wise peer-attention. Notice that attention is applied for each connection, conditioned based on attention_in. Args: inputs: A list of `Tensors`. Either 4D or 5D, depending of use_5d_mode. index: `int` index of the block within the AssembleNet architecture. Used for summation weight initial loading. attention_mode: `str` specifying mode. If not `peer', does self-attention. attention_in: A list of `Tensors' of size [batch*time, channels]. use_5d_mode: `bool` indicating whether the inputs are in 5D tensor or 4D. model_edge_weights: AssembleNet model structure connection weights in the string format. num_object_classes: Assemblenet++ structure used object inputs so we should use what dataset classes you might be use (e.g. ADE-20k 151 classes) Returns: The output `Tensor` after concatenation. """ if use_5d_mode: h_channel_loc = 2 conv_function = asn.conv3d_same_padding else: h_channel_loc = 1 conv_function = asn.conv2d_fixed_padding # If only 1 input. if len(inputs) == 1: inputs[0] = apply_attention(inputs[0], attention_mode, attention_in, use_5d_mode) return inputs[0] # get smallest spatial size and largest channels sm_size = [10000, 10000] lg_channel = 0 for inp in inputs: # assume batch X height x width x channels sm_size[0] = min(sm_size[0], inp.shape[h_channel_loc]) sm_size[1] = min(sm_size[1], inp.shape[h_channel_loc + 1]) # Note that, when using object inputs, object channel sizes are usually big. # Since we do not want the object channel size to increase the number of # parameters for every fusion, we exclude it when computing lg_channel. if inp.shape[-1] > lg_channel and inp.shape[-1] != num_object_classes: # pylint: disable=line-too-long lg_channel = inp.shape[3] per_channel_inps = _ApplyEdgeWeight( weights_shape=[len(inputs)], index=index, use_5d_mode=use_5d_mode, model_edge_weights=model_edge_weights)(inputs) # Implementation of connectivity with peer-attention if attention_mode: for key, channel_inps in per_channel_inps.items(): for idx in range(len(channel_inps)): with tf.name_scope('Connection_' + str(key) + '_' + str(idx)): channel_inps[idx] = apply_attention( channel_inps[idx], attention_mode, attention_in, use_5d_mode) # Adding 1x1 conv layers (to match channel size) and fusing all inputs. # We add inputs with the same channels first before applying 1x1 conv to save # memory. inps = [] for key, channel_inps in per_channel_inps.items(): if len(channel_inps) < 1: continue if len(channel_inps) == 1: if key == lg_channel: inp = channel_inps[0] else: inp = conv_function(channel_inps[0], lg_channel, kernel_size=1, strides=1) inps.append(inp) else: if key == lg_channel: inp = tf.add_n(channel_inps) else: inp = conv_function(channel_inps[0], lg_channel, kernel_size=1, strides=1) inps.append(inp) return tf.add_n(inps)
def __init__(self, sequence_length, num_classes,pos_vocab_size, pos_embedding_size, text_embedding_size,filter_sizes, num_heads, num_filters, l2_reg_lambda=0.0): # Placeholders for input, output and dropout self.text_embedded_chars = tf.placeholder(tf.float32, shape=[None, sequence_length, 768], name='text_embedded_chars') self.input_p1 = tf.placeholder(tf.int32, shape=[None, sequence_length], name='input_p1') self.input_p2 = tf.placeholder(tf.int32, shape=[None, sequence_length], name='input_p2') self.input_y = tf.placeholder(tf.float32, shape=[None, num_classes], name='input_y') #[20 19] self.dropout_keep_prob = tf.placeholder(tf.float32, name='dropout_keep_prob') self.emb_dropout_keep_prob = tf.placeholder(tf.float32, name='emb_dropout_keep_prob') initializer = tf.keras.initializers.glorot_normal # Embedding layer # with tf.device('/device:GPU:0'), tf.variable_scope("text-embedding"): # # self.W_text = tf.Variable(tf.random_uniform([text_vocab_size, text_embedding_size], -0.25, 0.25), name="W_text") # # self.text_embedded_chars = tf.nn.embedding_lookup(self.W_text, self.input_text) #[800 90 300] # # self.text_embedded_chars = server_bert.get_sentence_embedding(self.input_text) #[800 90 768] # # self.text_embedded_chars_trans = transformer.transformerencoder(self.text_embedded_chars) # self.text_embedded_chars_change = tf.layers.dense(self.text_embedded_chars, units=300,activation=tf.nn.relu,use_bias=True, trainable=True) #[800 90 300] # print("change:",self.text_embedded_chars_change.get_shape())# (?, 90, 300) # self.text_embedded_chars_expanded = tf.expand_dims(self.text_embedded_chars_change, -1) #[800 90 300 1] # print(self.text_embedded_chars_expanded.get_shape()) with tf.device('/cpu:0'), tf.variable_scope("position-embedding"): self.W_pos = tf.get_variable("W_pos", [pos_vocab_size, pos_embedding_size], initializer=initializer()) self.p1_embedded_chars = tf.nn.embedding_lookup(self.W_pos, self.input_p1) self.p2_embedded_chars = tf.nn.embedding_lookup(self.W_pos, self.input_p2) self.p1_embedded_chars_expanded = tf.expand_dims(self.p1_embedded_chars, -1) #[800 90 50 1] self.p2_embedded_chars_expanded = tf.expand_dims(self.p2_embedded_chars, -1) # self.embedded_chars_expanded = tf.concat([self.text_embedded_chars_expanded, # self.p1_embedded_chars_expanded, # self.p2_embedded_chars_expanded], 2) #[800 90 400 1] _embedding_size = text_embedding_size self.text_shape=tf.shape(self.text_embedded_chars) # self.text_expand_shape=tf.shape(self.text_embedded_chars_expanded) # self.pos_expand_shape=tf.shape(self.p1_embedded_chars_expanded) # self.embedd_shape=tf.shape(self.text_embedded_chars_change) # self.embedding_size_shape=tf.shape(_embedding_size) # Dropout for Word Embedding with tf.variable_scope('dropout-embeddings'): self.embedded_chars = tf.nn.dropout(self.text_embedded_chars, self.emb_dropout_keep_prob) # self-attention with tf.variable_scope("self-attention"): self.self_attn_output, self.self_alphas = multihead_attention(self.embedded_chars, self.embedded_chars, num_units=768, num_heads=num_heads) # print("attention shape:", self.self_attn.get_shape) #(?, 90 ,300) self.self_attn = tf.layers.dense(self.self_attn_output, units=300, activation=tf.nn.relu, use_bias=True, trainable=True) # [800 90 300] print("change:", self.self_attn.get_shape()) # (?, 90, 300) self.self_atten_change =tf.expand_dims(self.self_attn, -1) #[800 90 300 1] # Create a convolution + maxpool layer for each filter size pooled_outputs = [] for i, filter_size in enumerate(filter_sizes): with tf.variable_scope("conv-maxpool-%s" % filter_size): # Convolution Layer conv = tf.layers.conv2d(self.self_atten_change, num_filters, [filter_size, _embedding_size], kernel_initializer=initializer(), activation=tf.nn.relu, name="conv") # num_filter=128,filter_size=2,3,4,5 print(conv.get_shape()) # (?,89,1, 128);(?88,1,128)(?87,1,128)(?86 1 128) # Maxpooling over the outputs pooled = tf.nn.max_pool(conv, ksize=[1, sequence_length - filter_size + 1, 1, 1], strides=[1, 1, 1, 1], padding='VALID', name="pool") print(pooled.get_shape()) # (?, 1, 1, 128) pooled_outputs.append(pooled) # Combine all the pooled features num_filters_total = num_filters * len(filter_sizes) # print(pooled_outputs.get_shape()) print(np.array(pooled_outputs).shape) #(4,) self.h_pool = tf.concat(pooled_outputs, 3) # print(self.h_pool.get_shape()) #(?,1,1,512) self.h_pool_flat = tf.reshape(self.h_pool, [-1, num_filters_total]) # print(self.h_pool_flat.get_shape())#(?,512) # Add dropout with tf.variable_scope("dropout"): self.h_drop = tf.nn.dropout(self.h_pool_flat, self.dropout_keep_prob) # Final scores and predictions with tf.variable_scope("output"): self.logits = tf.layers.dense(self.h_drop, num_classes, kernel_initializer=initializer()) print(self.logits.get_shape()) #(?,19) self.predictions = tf.argmax(self.logits, 1, name="predictions") # Calculate mean cross-entropy loss with tf.variable_scope("loss"): losses = tf.nn.softmax_cross_entropy_with_logits_v2(logits=self.logits, labels=self.input_y) self.l2 = tf.add_n([tf.nn.l2_loss(v) for v in tf.trainable_variables()]) self.loss = tf.reduce_mean(losses) + l2_reg_lambda * self.l2 # Accuracy with tf.name_scope("accuracy"): correct_predictions = tf.equal(self.predictions, tf.argmax(self.input_y, 1)) self.accuracy = tf.reduce_mean(tf.cast(correct_predictions, tf.float32), name="accuracy")
def conv_layer(input, filter, kernel, stride=1, layer_name="conv"): with tf.name_scope(layer_name): network = tf.layers.conv2d(inputs=input, filters=filter, kernel_size=kernel, strides=stride, padding='SAME') return network
def get_next_stage_inputs(inputs, crop_boxes, batch_size=None, image_size=256, previous_batch_size=None, full_image_size=1024, image_folder=None, image_format=None, grid_offsets=None, intersection_ratio_threshold=0.25, epsilon=1e-8, use_queue=False, shuffle_buffer=1, num_threads=1, capacity=5000, verbose=False): """ Create input queue for the second - and final - stage. Args: inputs, a dictionnary of inputs crop_boxes, a (batch_size, num_crops, 4) tensor of crops image_folder: Image directory, used for reloading the full resolution images if needed batch_size: Batch size for the output of this pipeline image_size: Size of the images patches in the new dataset full_image_size: Size of the images to load before applying the croppings grid_offsets: A (num_cells, num_cells) array use_queue: Whether to use a queue or directly output the new inputs dictionary shuffle_buffer: shuffle buffer of the output queue num_threads: number of readers in the output queue capacity: Output queue capacity verbose: verbosity """ assert 0. <= intersection_ratio_threshold < 1. num_crops = tf.shape(crop_boxes)[1] new_inputs = {} # new_im_id: (batch_size * num_crops,) if 'im_id' in inputs: with tf.name_scope('im_ids'): new_inputs['im_id'] = tile_and_reshape(inputs['im_id'], num_crops) # classes: (batch_size * num_crops, num_classes) if 'class_labels' in inputs: with tf.name_scope('class_labels'): new_inputs['class_labels'] = tile_and_reshape(inputs['class_labels'], num_crops) # new_image: (num_patches, image_size, image_size, 3) with tf.name_scope('extract_image_patches'): # Extract patches and resize # crop_boxes_indices: (batch_size * num_crops,) # crop_boxes_flat: (batch_size * num_crops, 4) crop_boxes_indices = tf.ones(tf.shape(crop_boxes)[:2], dtype=tf.int32) crop_boxes_indices = tf.cumsum(crop_boxes_indices, axis=0, exclusive=True) crop_boxes_indices = tf.reshape(crop_boxes_indices, (-1,)) crop_boxes_flat = tf.gather(tf.reshape(crop_boxes, (-1, 4)), [1, 0, 3, 2], axis=-1) new_inputs['image'] = tf.image.crop_and_resize( inputs['image'], crop_boxes_flat, crop_boxes_indices, (image_size, image_size), name='extract_groups') # new_bounding_boxes: (num_patches, max_num_bbs, 4) # rescale bounding boxes coordinates to the cropped image if 'bounding_boxes' in inputs: with tf.name_scope('shift_bbs'): # bounding_boxes: (batch, num_crops, max_num_bbs, 4) # crop_boxes: (batch, num_crops, 1, 4) bounding_boxes = inputs['bounding_boxes'] max_num_bbs = bounding_boxes.get_shape()[1].value bounding_boxes = tf.expand_dims(bounding_boxes, axis=1) bounding_boxes = tf.tile(bounding_boxes, (1, num_crops, 1, 1)) crop_boxes = tf.expand_dims(crop_boxes, axis=2) # Filter out cut bbs ratios = utils.get_intersection_ratio(tf.split(bounding_boxes, 4, axis=-1), tf.split(crop_boxes, 4, axis=-1)) condition = tf.tile(ratios > intersection_ratio_threshold, (1, 1, 1, 4)) bounding_boxes *= tf.to_float(condition) # Rescale coordinates to the cropped image crop_mins, crop_maxs = tf.split(crop_boxes, 2, axis=-1) bounding_boxes -= tf.tile(crop_mins, (1, 1, 1, 2)) bounding_boxes /= tf.maximum(epsilon, tf.tile(crop_maxs - crop_mins, (1, 1, 1, 2))) bounding_boxes = tf.clip_by_value(bounding_boxes, 0., 1.) bounding_boxes = tf.reshape(bounding_boxes, (-1, max_num_bbs, 4)) new_inputs['bounding_boxes'] = bounding_boxes # number of valid boxes: (num_patches,) if 'num_boxes' in inputs: with tf.name_scope('num_boxes'): valid_boxes = ((bounding_boxes[..., 2] > bounding_boxes[..., 0]) & (bounding_boxes[..., 3] > bounding_boxes[..., 1])) num_boxes = tf.to_float(valid_boxes) new_inputs['num_boxes'] = tf.to_int32(tf.reduce_sum(num_boxes, axis=-1) ) # Compute the box presence in cell mask # obj_i_mask_bbs: (num_patches, num_cells, num_cells, 1, num_gt) if 'obj_i_mask_bbs' in inputs: with tf.name_scope('grid_offsets'): if grid_offsets is not None: num_cells = grid_offsets.shape[:2] grid_offsets_mins = grid_offsets / num_cells grid_offsets_maxs = (grid_offsets + 1.) / num_cells bounding_boxes = tf.reshape(bounding_boxes, (-1, 1, 1, max_num_bbs, 4)) mins, maxs = tf.split(bounding_boxes, 2, axis=-1) inters = tf.maximum(0., tf.minimum(maxs, grid_offsets_maxs) - tf.maximum(mins, grid_offsets_mins)) inters = tf.reduce_prod(inters, axis=-1) obj_i_mask = tf.expand_dims(tf.to_float(inters > 0.) , axis=-2) new_inputs['obj_i_mask_bbs'] = obj_i_mask # During training: enqueue the inputs if use_queue: assert batch_size is not None filter_valid = tf.logical_and(crop_boxes[..., 2] > crop_boxes[..., 0], crop_boxes[..., 3] > crop_boxes[..., 1] ) filter_valid = tf.reshape(filter_valid, (-1,)) # TODO maybe_batch is deprecated out_ = tf.train.maybe_batch( new_inputs, filter_valid, batch_size, num_threads=num_threads, enqueue_many=True, capacity=capacity) # During inference: process crops deterministically else: out_ = new_inputs if verbose == 1: print('\n'.join(" \033[32m%s\033[0m: shape=%s, dtype=%s" % (key, value.get_shape().as_list(), value.dtype) for key, value in out_.items() if key != 'batch_size')) elif verbose > 1: print('\n'.join(" *%s*: shape=%s, dtype=%s" % (key, value.get_shape().as_list(), value.dtype) for key, value in out_.items() if key != 'batch_size')) return out_
def __init__(self, config, batch, word_mat=None, filter_sizes=None, embedding_size=None,num_filters=None,trainable=True, l2_reg_lambda=0.0, keep_prob=0.9, graph=None): # Placeholders for input, output and dropout self.config = config self.graph = graph if graph is not None else tf.Graph() self.trainable = trainable if trainable == True: self.input_x, self.input_x1, self.ch, self.qh, self.input_y, self.qa_id = batch.get_next() # self.y1 is (64, 3)self.alterh batch_size is[batch,3,alternative_len,chara_len] else: self.input_x, self.input_x1, self.ch, self.qh = batch.get_next() # self.y1 is (64, 3)self.alterh batch_size is[batch,3,alternative_len,chara_len] self.dropout_keep_prob =keep_prob self.global_step = tf.get_variable('global_step', shape=[], dtype=tf.int32, initializer=tf.constant_initializer(0), trainable=False) self.dropout = tf.placeholder_with_default(0.5, (), name="dropout") # Keeping track of l2 regularization loss (optional) l2_loss = tf.constant(0.0) # Embedding layer with tf.device('/cpu:0'), tf.name_scope("embedding"): self.W = tf.get_variable("word_mat", initializer=tf.constant(word_mat, dtype=tf.float32), trainable=True) self.c_mask = tf.cast(self.input_x, tf.bool) # self.c为填充之后的长度是一致的,用0进行填充 self.q_mask = tf.cast(self.input_x1, tf.bool) if trainable: self.c_maxlen, self.q_maxlen, = config.para_limit, config.ques_limit, else: self.c_maxlen, self.q_maxlen = config.test_para_limit, config.test_ques_limit self.embedded_chars = tf.nn.embedding_lookup(self.W, self.input_x) self.embedded_chars1 = tf.nn.embedding_lookup(self.W, self.input_x1) self.embedded_chars_expanded = tf.expand_dims(self.embedded_chars, -1) self.embedded_question = tf.expand_dims(self.embedded_chars1, -1) S = optimized_trilinear_for_attention([self.embedded_chars_expanded, self.embedded_question], self.c_maxlen, self.q_maxlen, input_keep_prob=1.0 - self.dropout) print(S,"2222222222222222222") # Create a convolution + maxpool layer for each filter size pooled_outputs = [] for i, filter_size in enumerate(filter_sizes): with tf.name_scope("conv-maxpool-%s" % filter_size): # Convolution Layer filter_shape = [filter_size, embedding_size, 1, num_filters] W = tf.Variable(tf.truncated_normal(filter_shape, stddev=0.1), name="W") b = tf.Variable(tf.constant(0.1, shape=[num_filters]), name="b") conv = tf.nn.conv2d( self.embedded_chars_expanded, W, strides=[1, 1, 1, 1], padding="VALID", name="conv") # Apply nonlinearity h = tf.nn.relu(tf.nn.bias_add(conv, b), name="relu") # Maxpooling over the outputs pooled = tf.nn.max_pool( h, ksize=[1, config.para_limit - filter_size + 1, 1, 1], strides=[1, 1, 1, 1], padding='VALID', name="pool") pooled_outputs.append(pooled) # Combine all the pooled features num_filters_total = num_filters * len(filter_sizes) self.h_pool = tf.concat(pooled_outputs, 3) self.h_pool_flat = tf.reshape(self.h_pool, [-1, num_filters_total]) # Add dropout with tf.name_scope("dropout"): self.h_drop = tf.nn.dropout(self.h_pool_flat, self.dropout_keep_prob) # Final (unnormalized) scores and predictions with tf.name_scope("output"): W = tf.get_variable( "W", shape=[num_filters_total, 3], initializer=tf.contrib.layers.xavier_initializer()) b = tf.Variable(tf.constant(0.1, shape=[3]), name="b") l2_loss += tf.nn.l2_loss(W) l2_loss += tf.nn.l2_loss(b) self.scores = tf.nn.xw_plus_b(self.h_drop, W, b, name="scores") self.predictions = tf.argmax(self.scores, 1, name="predictions") # Calculate mean cross-entropy loss with tf.name_scope("loss"): losses = tf.nn.softmax_cross_entropy_with_logits(logits=self.scores, labels=self.input_y) self.loss = tf.reduce_mean(losses) + l2_reg_lambda * l2_loss # Accuracy with tf.name_scope("accuracy"): correct_predictions = tf.equal(self.predictions, tf.argmax(self.input_y, 1)) self.accuracy = tf.reduce_mean(tf.cast(correct_predictions, "float"), name="accuracy") if config.decay is not None: self.var_ema = tf.train.ExponentialMovingAverage(config.decay) ema_op = self.var_ema.apply(tf.trainable_variables()) with tf.control_dependencies([ema_op]): self.loss = tf.identity(self.loss) self.assign_vars = [] for var in tf.global_variables(): v = self.var_ema.average(var) if v: self.assign_vars.append(tf.assign(var, v)) self.lr = tf.minimum(config.init_lr, 0.001 / tf.log(999.) * tf.log(tf.cast(self.global_step, tf.float32) + 1)) self.opt = tf.train.AdamOptimizer(learning_rate=self.lr, beta1=0.8, beta2=0.999, epsilon=1e-7) grads = self.opt.compute_gradients(self.loss) gradients, variables = zip(*grads) capped_grads, _ = tf.clip_by_global_norm( gradients, config.grad_clip) self.train_op = self.opt.apply_gradients( zip(capped_grads, variables), global_step=self.global_step) self.saver = tf.train.Saver(tf.global_variables(), max_to_keep=3)
def body_sharded(self, sharded_features): # Remove dropout if not training hparams = self._hparams dp = self._data_parallelism if hparams.use_inputs: decoder_input = dp(tf.squeeze, sharded_features["inputs"], 2) decoder_self_attention_bias = None else: targets = sharded_features["targets"] targets = dp(tf.squeeze, targets, 2) (decoder_input, decoder_self_attention_bias, pad_remover) = dp( attention_lm_moe_prepare_decoder, targets, hparams) def preprocess(x): return dp(common_layers.layer_preprocess, x, hparams) def postprocess(x, y): return dp(common_layers.layer_postprocess, x, y, hparams) x = dp(tf.nn.dropout, decoder_input, 1.0 - hparams.layer_prepostprocess_dropout) extra_loss = 0.0 if not hparams.use_inputs: # As preprocess and postprocess are called with batch of size one (all # batches concatenated), we just make sure that batch_norm is not use ( # should not either way) assert hparams.norm_type != "batch" tf.logging.info("Applying Padding Remover for the attention experts") dp_remove_pad = functools.partial( dp, remove_pad, pad_remover=pad_remover, mode=hparams.mode) dp_restore_pad = functools.partial( dp, restore_pad, ref_x=x, pad_remover=pad_remover, mode=hparams.mode) else: # Using identity function: No effect dp_remove_pad = lambda x: x dp_restore_pad = lambda x: x if hparams.attention_exp_factor != 0: tf.logging.info("Expand/compress tokens before sending them to experts") dp_expand_bc = lambda x: dp( # pylint: disable=g-long-lambda expand_batch_coordinates, x, hparams.attention_exp_factor) dp_expand_x = lambda x: dp( # pylint: disable=g-long-lambda common_attention.deconv_elems_1d, x, hparams.attention_exp_factor, hparams.attention_exp_inputdim) dp_compress_x = lambda x, l: dp( # pylint: disable=g-long-lambda common_attention.conv_elems_1d, x, hparams.attention_exp_factor, l) else: dp_expand_bc = lambda x: x dp_expand_x = lambda x: x dp_compress_x = lambda x, l: x def print_shape(x, suffix, debug=False): # To help debugging, print the input/output shapes at inference and eval # Inference for long sequences can take a long time, so that's help to # see the progression of the generation if not debug and hparams.mode == ModeKeys.TRAIN: return x return tf.Print(x, [tf.shape(x)], "shape_x_{}".format(suffix)) with tf.name_scope("batch_coordinate_preprocess"): batch_coordinate = dp(get_batch_coordinate, x) batch_coordinate = dp_remove_pad(batch_coordinate) batch_coordinate = dp_expand_bc(batch_coordinate) batch_order = dp(get_batch_coordinate, x, axis=-1) batch_order = dp_remove_pad(batch_order) batch_order = dp_expand_bc(batch_order) x = dp(print_shape, x, "in") assert hparams.batch_size >= hparams.max_length num_hidden_layers = ( len(hparams.attention_layers) or hparams.num_hidden_layers) for layer in range(num_hidden_layers): with tf.variable_scope("layer_%d" % layer): # Use the layer type defined in attention_layers if hparams.attention_layers: attention_type = LAYER_SYMBOLS[hparams.attention_layers[layer]] else: attention_type = hparams.attention_type with tf.variable_scope( "attention_{}".format(attention_type)): if attention_type in [ AttentionType.MULTIHEAD, AttentionType.MULTIHEAD_FULL]: attention_dot_type = ( "local_mask_right" if hparams.attention_local else "dot_product") if attention_type == AttentionType.MULTIHEAD_FULL: attention_dot_type = "dot_product" y = dp( common_attention.multihead_attention, preprocess(x), None, decoder_self_attention_bias, hparams.attention_key_channels or hparams.hidden_size, hparams.attention_value_channels or hparams.hidden_size, hparams.hidden_size, hparams.num_heads, hparams.attention_dropout, attention_type=attention_dot_type, block_length=hparams.attention_block_length, name="decoder_self_attention") elif attention_type == AttentionType.SPARSE_MULTIHEAD: x_in = preprocess(x) x_in = dp_remove_pad(x_in) y, loss_experts = dp( common_attention.multihead_attention_sparse_dot_prod, x_in, None, None, # Bias is computed inside hparams.attention_key_channels or hparams.hidden_size, hparams.attention_value_channels or hparams.hidden_size, hparams.hidden_size, hparams.num_heads, hparams.attention_dropout, # Additional parameters bi=[common_attention.BatchInfo( coordinates=batch_coordinate[i], order=batch_order[i], # No future mask ) for i in range(dp.n)], use_map_fn=hparams.lsh_use_map_fn, experts_params=dict( nb_hyperplanes=hparams.lsh_num_hyperplanes, ), ) y = dp_restore_pad(y) # TODO(avaswani, epot, noam): Do we need to divide by num shards ? extra_loss += tf.add_n(loss_experts) / dp.n elif attention_type == AttentionType.SPARSE_MULTIHEAD_TRUNCATED: x_in = preprocess(x) y, loss_experts = dp( common_attention.multihead_attention_sparse_truncated, x_in, None, None, # Bias is computed inside hparams.attention_key_channels or hparams.hidden_size, hparams.attention_value_channels or hparams.hidden_size, hparams.hidden_size, hparams.num_heads, hparams.attention_dropout, # Additional parameters bi=[common_attention.BatchInfo( coordinates=batch_coordinate[i], order=batch_order[i], # No future mask ) for i in range(dp.n)], mask_right=True, experts_params=dict( nb_hyperplanes=hparams.lsh_num_hyperplanes, ), ) # TODO(avaswani, epot, noam): Do we need to divide by num shards ? extra_loss += tf.add_n(loss_experts) / dp.n elif attention_type == AttentionType.MEMORY_EFFICIENT: assert hparams.layer_preprocess_sequence == "n" y = dp( common_attention.multihead_self_attention_memory_efficient, x, decoder_self_attention_bias, hparams.num_heads, name="decoder_self_attention") elif attention_type == AttentionType.MULTIHEAD_REDUCED: y = dp( common_attention.multihead_self_attention_reduced, preprocess(x), factor=hparams.attention_red_factor, reduction_type=hparams.attention_reduction_type, nonlinearity=hparams.attention_nonlinearity, multihead_params=dict( total_key_depth= hparams.attention_key_channels or hparams.hidden_size, total_value_depth= hparams.attention_value_channels or hparams.hidden_size, num_heads=hparams.num_heads, dropout_rate=hparams.attention_dropout, )) elif attention_type == AttentionType.LOCAL_EXPERTS: x_in = preprocess(x) x_in = dp_remove_pad(x_in) x_in = dp_expand_x(x_in) y, loss = dp( common_attention.local_expert_attention, x_in, k=hparams.attention_moe_k, loss_coef=hparams.attention_load_balance, attention_num_experts=hparams.attention_num_experts, train=hparams.mode == ModeKeys.TRAIN, batch_coordinate=batch_coordinate, mask_right=not hparams.use_inputs, split_batch=bool(hparams.attention_split_batch), attention_num_head=hparams.attention_num_head, attention_kq_size=hparams.attention_kq_size, attention_v_size=hparams.attention_v_size) y = dp_compress_x(y, x[0].get_shape().as_list()[-1]) y = dp_restore_pad(y) # TODO(avaswani, epot, noam): Do we need to divide by num shards ? extra_loss += tf.add_n(loss) / dp.n else: raise ValueError("Only {} supported for now.".format( AttentionType.get_choices())) x = postprocess(x, y) with tf.variable_scope("ffn"): if hparams.memory_efficient_ffn: assert hparams.layer_preprocess_sequence == "n" y = dp( common_layers.conv_hidden_relu_memory_efficient, x, hparams.filter_size) else: additional_conv_params = dict() if hparams.use_sepconv: additional_conv_params = dict( padding="LEFT", # Parameters copied from the transformer model kernel_size=(3, 1), second_kernel_size=(31, 1), ) y = dp( common_layers.conv_hidden_relu, preprocess(x), hparams.filter_size, hparams.hidden_size, dropout=hparams.relu_dropout, **additional_conv_params ) x = postprocess(x, y) x = preprocess(x) decoder_output = dp(tf.expand_dims, x, 2) return decoder_output, extra_loss
def get_tf_dataset(tfrecords_file, record_keys, image_format, max_num_bbs, with_groups=True, grouping_method='intersect', grid_offsets=None, with_classes=False, num_classes=None, batch_size=1, drop_remainder=False, num_epochs=1, image_size=1024, image_folder='', data_augmentation_threshold=0.5, num_devices=1, num_threads=4, shuffle_buffer=1, prefetch_capacity=1, make_initializable_iterator=False, verbose=1): """Parse and load inputs from the given TFRecords as a tf.data.Dataset. Args: tfrecords_file: Path to the TFRecords file containing the data. record_keys: Feature keys present in the TFrecords. Loaded from the metadata file max_num_bbs: Maximum number of bounding boxes in the dataset. Used for reshaping the `bounding_boxes` records. num_classes: Number of classes in the dataset. Only used if with_classes is True with_classes: wheter to use class information with_groups: whether to pre-compute grouped instances ground-truth grid_offsets: Precomputed grid offsets batch_size: Batch size. num_epochs: Number of epochs to repeat. image_size: The square size which to resize images to. image_folder: path to the directory containing the images in the dataset. data_augmentation_threshold: Data augmentation probabilitiy (in [0, 1]) num_devices: Number of devices num_threads: Number of readers for the batch queue. shuffle_buffer: Size of the shuffling buffer. prefetch_capacity: Buffer size for prefetching. make_initializable_iterator: if True, make an initializable and add its initializer to the collection `iterator_init` verbose: Verbosity level Returns: A tf.Data.dataset iterator (and its initializer if initializable_iterator) """ assert grouping_method in ['intersect', 'intersect_with_density', 'unique_intersect'] assert not (with_classes and num_classes is None) assert len(record_keys) assert batch_size > 0 assert image_size > 0 assert 0. <= data_augmentation_threshold <= 1. if grid_offsets is not None: num_cells = grid_offsets.shape[:2] assert num_devices > 0 assert num_threads > 0 assert shuffle_buffer > 0 if verbose == 2: print(' \033[31m> load_inputs\033[0m') elif verbose == 1: print(' > load_inputs') # Normalize grid cells offsets if grid_offsets is not None: grid_offsets_mins = grid_offsets / num_cells grid_offsets_maxs = (grid_offsets + 1.) / num_cells # Create TFRecords feature features = read_tfrecords(record_keys, max_num_bbs=max_num_bbs) def parsing_function(example_proto): # Basic features parsed_features = tf.parse_single_example(example_proto, features) output = parse_basic_feature(parsed_features, image_folder, image_format, image_size=image_size) bounding_boxes = output['bounding_boxes'] # Empty/active cells mask # obj_i_mask_bbs: (num_cells, num_cells, 1, num_bbs) mins, maxs = tf.split(bounding_boxes, 2, axis=-1) # (num_bbs, 2) inters = tf.maximum(0., tf.minimum(maxs, grid_offsets_maxs) - tf.maximum(mins, grid_offsets_mins)) inters = tf.reduce_prod(inters, axis=-1) obj_i_mask = tf.expand_dims(tf.to_float(inters > 0.) , axis=-2) output["obj_i_mask_bbs"] = obj_i_mask # Grouped instances # group_bounding_boxes_per_cell: (num_cells, num_cells, 1, 4), cell bounding box after grouping # group_flags: (num_cells, num_cells, 1, 1), whether a cell contains a group or not # num_group_boxes: (), number of bounding boxes after grouping if with_groups: ## Define group_mask: (num_cells, num_cells, num_bbs, 1) ## Maps each gt bounding box to a grid cell to be merged into a group if grouping_method == 'intersect_with_density': obj_i_mask = tf.expand_dims(tf.to_float(inters > 0.) , axis=-2) obj_i_mask *= tf.expand_dims(tf.to_float(inters < 1. / (num_cells[0] * num_cells[1])) , axis=-2) group_mask = tf.transpose(obj_i_mask, (0, 1, 3, 2)) # (num_cells, num_cells, num_bbs, 1) elif grouping_method == 'unique_intersect': # weight 1: Intersection between gt boxes and cells # Upper bounded by 1 # (num_cells, num_cells, num_bbs) w1 = inters * num_cells[0] * num_cells[1] # weight 2: Opposite of How many objects coocurs in each cells # Upper bounded by 1 # (num_cells, num_cells, 1) w2 = 1. - tf.reduce_sum(obj_i_mask, axis=-1) / tf.to_float(output['num_boxes']) # Assign each ground-truth to one unique group group_mask = w1 * w2 group_mask = tf.to_float(group_mask > 0.) * tf.to_float(group_mask >= tf.reduce_max(group_mask, axis=(0, 1), keep_dims=True)) group_mask = tf.expand_dims(group_mask, axis=-1) elif grouping_method == 'intersect': group_mask = tf.transpose(obj_i_mask, (0, 1, 3, 2)) # (num_cells, num_cells, num_bbs, 1) ## Merge bbs coocurring in the same cell to form groups mins = mins + 1. - group_mask mins = tf.reduce_min(mins, axis=2, keep_dims=True) # (num_cells, num_cells, 1, 2) maxs = maxs * group_mask maxs = tf.reduce_max(maxs, axis=2, keep_dims=True) group_bounding_boxes_per_cell = tf.concat([mins, maxs], axis=-1) group_bounding_boxes_per_cell = tf.clip_by_value(group_bounding_boxes_per_cell, 0., 1.) output["group_bounding_boxes_per_cell"] = group_bounding_boxes_per_cell num_bbs_per_cell = tf.reduce_sum(group_mask, axis=2, keep_dims=True) num_group_boxes = tf.reduce_sum(tf.to_int32(num_bbs_per_cell > 0)) output["num_group_boxes"] = num_group_boxes group_flags = tf.maximum(tf.minimum(num_bbs_per_cell, 2.) - 1., 0.) output["group_flags"] = group_flags # is_flipped flag: (), indicates whether the image has been flipped during data augmentation output["is_flipped"] = tf.constant(0.) # Optional : add classes if with_classes: class_labels = tf.one_hot(parsed_features['classes'], num_classes, axis=-1, on_value=1, off_value=0, dtype=tf.int32) output['class_labels'] = class_labels # Group classes (majority vote) # (num_cells, num_cells, 1, num_classes) if with_groups: percell_class_labels = tf.expand_dims(tf.expand_dims(class_labels, axis=0), axis=0) percell_class_labels = group_mask * tf.to_float(percell_class_labels) percell_class_labels = tf.reduce_sum(percell_class_labels, axis=2, keep_dims=True) group_class_labels = tf.argmax(percell_class_labels, axis=-1) group_class_labels = tf.one_hot(group_class_labels, num_classes, axis=-1, on_value=1, off_value=0, dtype=tf.int32) group_class_labels = tf.to_int32(percell_class_labels * tf.to_float(group_class_labels)) output["group_class_labels"] = group_class_labels return output ## Create the dataset with tf.name_scope('load_dataset'): # Parse data dataset = tf.data.TFRecordDataset(tfrecords_file) # Map dataset = dataset.shuffle(buffer_size=shuffle_buffer) dataset = dataset.map(parsing_function, num_parallel_calls=num_threads) # Repeat if num_epochs > 1: dataset = dataset.repeat(num_epochs) # Batch if tf.__version__ == '1.4.0': dataset = dataset.batch(batch_size * num_devices) else: dataset = dataset.batch(batch_size * num_devices, drop_remainder=drop_remainder) # Prefetch if prefetch_capacity > 0: dataset = dataset.prefetch(prefetch_capacity) # Iterator if make_initializable_iterator: iterator = dataset.make_initializable_iterator() iterator_init = iterator.initializer tf.add_to_collection('iterator_init', iterator_init) else: iterator = dataset.make_one_shot_iterator() iterator_init = None batch = iterator.get_next() ## Apply data augmentation with tf.name_scope('data_augmentation'): if data_augmentation_threshold > 0.: batch = apply_data_augmentation(batch, data_augmentation_threshold) ## Split across device slice_dims = [0] * num_devices unpadded_batch = tf.to_int32(tf.shape(batch['im_id'])[0]) for i in range(num_devices): slice_dims[i] = tf.maximum(0, tf.minimum(batch_size, unpadded_batch)) unpadded_batch -= batch_size inputs = [{} for _ in range(num_devices)] for key, value in batch.items(): for i, split_value in enumerate(tf.split(value, slice_dims, axis=0)): inputs[i][key] = split_value ## Verbose log if verbose == 2: print('\n'.join(" \033[32m%s\033[0m: shape=%s, dtype=%s" % (key, value.get_shape().as_list(), value.dtype) for key, value in inputs[0].items())) elif verbose == 1: print('\n'.join(" *%s*: shape=%s, dtype=%s" % (key, value.get_shape().as_list(), value.dtype) for key, value in inputs[0].items())) return inputs, iterator_init
def predict(logits, labels): with tf.name_scope('Accuracy') as scope: pred = tf.nn.softmax(logits) correct_prediction = tf.equal(tf.argmax(pred, 1), labels, name='accuracy_pre_example') evaluation_step = tf.reduce_mean(tf.cast(correct_prediction, tf.float32), name='accuracy_rate') return evaluation_step
def create_model(self,in_data,one_hot=False,training=True): with tf.variable_scope('scope_model',reuse=tf.AUTO_REUSE): if one_hot: # not using embedding layer embed = in_data[0] else: # using embedding layer with tf.name_scope('embed'): if not config.PRETRAIN_EMBD_TAG: self.embedding_size = config.EMBEDDING_SIZE self.vocab_size = config.VOCAB_SIZE embed_matrix = tf.get_variable('embed_matrix', shape=[self.vocab_size, self.embedding_size], initializer=tf.random_uniform_initializer()) else: embed_matrix = tf.Variable(self.pretrain_embd, trainable=config.PRETRAIN_EMBD_TRAINABLE,name='embed_matrix') ''' #make sure the pretrain embd is load correctly with tf.Session() as sess: sess.run(tf.initialize_all_variables()) print(sess.run(embed_matrix)) ''' embed = tf.nn.embedding_lookup(embed_matrix, in_data[0], name='embedding') print(embed_matrix.name) print(embed.name) self.create_actual_model(embed,training) self.get_logits() self.logits = tf.layers.batch_normalization(self.logits,name='Batch_Normalization') self.logits = tf.nn.relu(self.logits,name='RELU') labels = tf.argmax(input=in_data[1], axis=2) predictions = tf.argmax(input=self.logits, axis=1) print(labels) print(predictions) _, self.acc_op = tf.metrics.accuracy(labels=labels, predictions=predictions,name = 'my_metrics') if training: loss = tf.nn.softmax_cross_entropy_with_logits(logits=self.logits, labels=in_data[1]) self.loss += loss self.loss = tf.reduce_sum(self.loss) params = tf.trainable_variables() optimizer = tf.train.AdamOptimizer(config.LR) grad_and_vars = tf.gradients(self.loss,params) clipped_gradients , _= tf.clip_by_global_norm(grad_and_vars,0.5) self.opt = optimizer.apply_gradients(zip(clipped_gradients,params),global_step = self.gstep)
def __init__(self, dim_a, dim_s, action_bound): self.dim_s, self.dim_a, self.action_bound = dim_s, dim_a, action_bound self.pointer = 0 self.memory = deque(maxlen=MEMORY_SIZE) self.history_loss = [] self.history_TD_error = [] self.history_reward = [] self.sess = tf.Session() # variable self.state = tf.placeholder(tf.float32, [None, dim_s], name="state") self.state_ = tf.placeholder(tf.float32, [None, dim_s], name="state_") # self.action = tf.placeholder(tf.float32, [None, 1], name="action") self.reward = tf.placeholder(tf.float32, [None, 1], name="reward") # parameters self.ae_param = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope="actor/eval") self.at_param = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope="actor/target") self.ce_param = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope="critic/eval") self.ct_param = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope="critic/target") # with tf.variable_scope("actor"): self.action = self._build_actor(self.state, scope="eval", trainable=True) self.action_ = self._build_actor(self.state_, scope="target", trainable=False) with tf.variable_scope("critic"): self.q_value = self._build_critic(self.state, self.action, scope="eval", trainable=True) self.q_value_ = self._build_critic(self.state_, self.action_, scope="target", trainable=False) # loss with tf.name_scope('loss'): q_target = self.reward + GAMMA * self.q_value_ self.TD_error = tf.losses.mean_squared_error(labels=self.q_value, predictions=q_target) self.loss = -tf.reduce_mean(self.q_value) with tf.variable_scope("summaries"): tf.summary.scalar('TD', self.TD_error) tf.summary.scalar('Loss', self.loss) tf.summary.scalar('Reward', self.history_reward) # train actor with tf.name_scope('train'): self.actor_train = tf.train.AdamOptimizer(LR_A).minimize(self.loss) self.critic_train = tf.train.AdamOptimizer(LR_C).minimize( self.TD_error) self.sess.run(tf.global_variables_initializer()) # tensorboard # merged_summary = tf.summary.merge_all() writer = tf.summary.FileWriter("./logs/", self.sess.graph) print("Wrote Tensorboard Logs") writer.close()
def __init__(self, sequence_length, num_classes, vocab_size, embedding_size, filter_sizes, num_filters, num_domains, l2_reg_lambda): self.input_x = tf.placeholder(tf.int32, [None, sequence_length], name="input_x") self.input_y = tf.placeholder(tf.int32, [None, num_classes], name="input_y") self.input_d = tf.placeholder(tf.int32, [None, num_domains], name="input_d") l2_loss = tf.constant(0.0) with tf.variable_scope("embedding"): self.emb_W = tf.get_variable( name="lookup_emb", shape=[vocab_size, embedding_size], initializer=tf.random_uniform_initializer(minval=-1.0, maxval=1.0), trainable=False) embedded_chars = tf.nn.embedding_lookup(self.emb_W, self.input_x) self.embedded_chars_expanded = tf.expand_dims(embedded_chars, -1) #cnn+pooling num_filters_total = num_filters * len(filter_sizes) #shared self.pub_h_pool = self.cnn("shared-public", self.embedded_chars_expanded, sequence_length, embedding_size, filter_sizes, num_filters) #private self.pri_h_pool = self.cnn("shared-private", self.embedded_chars_expanded, sequence_length, embedding_size, filter_sizes, num_filters) #final representation self.h_pool = tf.concat([self.pri_h_pool, self.pub_h_pool], axis=1) input_dim = num_filters_total * 2 # Add dropout with tf.name_scope("dropout"): self.dropout_keep_prob = tf.placeholder(tf.float32, name="dropout_keep_prob") self.h_drop = tf.nn.dropout(self.h_pool, self.dropout_keep_prob) self.pub_h_drop = tf.nn.dropout(self.pub_h_pool, self.dropout_keep_prob) self.pri_h_drop = tf.nn.dropout(self.pri_h_pool, self.dropout_keep_prob) hidden_size = 300 with tf.variable_scope("label"): h1 = self.wx_plus_b(scope_name="h1", x=self.h_drop, size=[input_dim, hidden_size]) self.y_scores = self.wx_plus_b(scope_name='score', x=h1, size=[hidden_size, num_classes]) # CalculateMean cross-entropy loss with tf.name_scope("loss"): losses = tf.nn.softmax_cross_entropy_with_logits( logits=self.y_scores, labels=self.input_y, ) self.y_loss = tf.reduce_mean(losses, name="task_loss") with tf.name_scope("accuracy"): self.y_pred = tf.argmax(self.y_scores, 1, name="predictions") cor_pred = tf.cast( tf.equal(self.y_pred, tf.argmax(self.input_y, 1)), "float") self.y_accuracy = tf.reduce_mean(cor_pred, name="accuracy") with tf.variable_scope("domain"): h1 = self.wx_plus_b(scope_name="h1", x=self.pub_h_drop, size=[num_filters_total, hidden_size]) self.domain_scores = self.wx_plus_b( scope_name="score", x=h1, size=[hidden_size, num_domains]) with tf.name_scope("loss"): losses = tf.nn.softmax_cross_entropy_with_logits( logits=self.domain_scores, labels=self.input_d, ) self.domain_loss = tf.reduce_mean(losses) with tf.name_scope("accuracy"): self.domain_pred = tf.argmax(self.domain_scores, 1, name="predictions") cor_pred = tf.cast( tf.equal(self.domain_pred, tf.argmax(self.input_d, 1)), "float") self.domain_accuracy = tf.reduce_mean(cor_pred, name="acc") with tf.variable_scope("gen"): h1 = self.wx_plus_b(scope_name="h1", x=self.pri_h_drop, size=[num_filters_total, hidden_size]) self.gen_scores = self.wx_plus_b(scope_name="score", x=h1, size=[hidden_size, num_domains]) with tf.name_scope("loss"): losses = tf.nn.softmax_cross_entropy_with_logits( logits=self.gen_scores, labels=self.input_d, ) self.gen_loss = tf.reduce_mean(losses) with tf.name_scope("accuracy"): self.gen_pred = tf.argmax(self.gen_scores, 1, name="predictions") cor_pred = tf.cast( tf.equal(self.gen_pred, tf.argmax(self.input_d, 1)), "float") self.gen_accuracy = tf.reduce_mean(cor_pred, name="acc")
def main(): train_log_file = misc_utils.train_log_file_dir() ckpt_dir = misc_utils.ckpt_dir() hparam_file = misc_utils.hparams_file_dir() if not train_log_file.parent.exists(): os.makedirs(str(train_log_file.parent)) if not ckpt_dir.exists(): os.mkdir(str(ckpt_dir)) misc_utils.save_hparams(str(hparam_file)) g = tf.Graph() with g.as_default(): with tf.name_scope("inputs"): noisy_trainset_wav = misc_utils.datasets_dir().joinpath(PARAM.train_noisy_set) clean_trainset_wav = misc_utils.datasets_dir().joinpath(PARAM.train_clean_set) noisy_valset_wav = misc_utils.datasets_dir().joinpath(PARAM.validation_noisy_set) clean_valset_wav = misc_utils.datasets_dir().joinpath(PARAM.validation_clean_set) train_inputs = dataloader.get_batch_inputs_from_nosiyCleanDataset(noisy_trainset_wav, clean_trainset_wav) val_inputs = dataloader.get_batch_inputs_from_nosiyCleanDataset(noisy_valset_wav, clean_valset_wav) test_noisy_ph = tf.compat.v1.placeholder(tf.float32, shape=[1, None], name='mixed_batch') ModelC, G, D = model_builder.get_model_class_and_var() generator = G() discriminator = D() train_model = ModelC(PARAM.MODEL_TRAIN_KEY, generator, discriminator, train_inputs.mixed, train_inputs.clean) # tf.compat.v1.get_variable_scope().reuse_variables() val_model = ModelC(PARAM.MODEL_VALIDATE_KEY, generator, discriminator, val_inputs.mixed,val_inputs.clean) test_model = ModelC(PARAM.MODEL_INFER_KEY, generator, discriminator, test_noisy_ph) init = tf.group(tf.compat.v1.global_variables_initializer(), tf.compat.v1.local_variables_initializer()) # misc_utils.show_variables(train_model.save_variables) # misc_utils.show_variables(val_model.save_variables) g.finalize() config = tf.compat.v1.ConfigProto() config.gpu_options.allow_growth = PARAM.GPU_RAM_ALLOW_GROWTH # config.gpu_options.per_process_gpu_memory_fraction = PARAM.GPU_PARTION config.allow_soft_placement = False sess = tf.compat.v1.Session(config=config, graph=g) sess.run(init) ckpts = tf.train.get_checkpoint_state(str(misc_utils.ckpt_dir())) if ckpts is not None: ckpt = ckpts.model_checkpoint_path train_model.saver.restore(sess, ckpt) else: # region validation before training misc_utils.print_log("\n\n", train_log_file) misc_utils.print_log("sum_losses_G: "+str(PARAM.sum_losses_G)+"\n", train_log_file) misc_utils.print_log("sum_losses_D: "+str(PARAM.sum_losses_D)+"\n", train_log_file) misc_utils.print_log("show losses: "+str(PARAM.show_losses)+"\n", train_log_file) evalOutputs_prev = eval_one_epoch(sess, val_model, val_inputs.initializer) val_msg = "PRERUN.val> sum_loss:[G %.4F, D %.4f], show_losses:%s, Time:%.2Fs.\n\n\n" % ( evalOutputs_prev.avg_sum_loss_G, evalOutputs_prev.avg_sum_loss_D, evalOutputs_prev.avg_show_losses, evalOutputs_prev.cost_time) misc_utils.print_log(val_msg, train_log_file) # endregion avg_sum_loss_G = None avg_sum_loss_D = None avg_show_losses = None save_time = time.time() init_inputs_times = 1 sess.run(train_inputs.initializer) while True: try: one_batch_time = time.time() (sum_loss_G, sum_loss_D, show_losses, _, lr, u_D, u_G ) = sess.run([train_model.losses.sum_loss_G, train_model.losses.sum_loss_D, train_model.losses.show_losses, train_model.train_op, train_model.lr, train_model.discriminator._f_u, train_model.generator._f_u, # train_model.adam_p[:2] ]) global_step = sess.run(train_model._global_step_increase) # print(adam_p) if global_step > PARAM.max_step: sess.close() misc_utils.print_log("\n", train_log_file, no_time=True) msg = '################### Training Done. ###################\n' misc_utils.print_log(msg, train_log_file) print('initial inputs %d times' % init_inputs_times) break if avg_sum_loss_G is None: avg_sum_loss_G = sum_loss_G avg_sum_loss_D = sum_loss_D avg_show_losses = show_losses else: avg_sum_loss_G += sum_loss_G avg_sum_loss_D += sum_loss_D avg_show_losses += show_losses u_str = "#u: [D %.2e, G %.2e]" % (u_D, u_G) print("\rtrain step: %d/%d, cost %.2fs, sum_loss[G %.2f, D %.2f], show_losses %s, lr %.2e, %s " % ( global_step, PARAM.max_step, time.time()-one_batch_time, sum_loss_G, sum_loss_D, str(show_losses), lr, u_str), flush=True, end="") one_batch_time = time.time() # print(global_step) if global_step % PARAM.step_to_save == 0: print("\r " " " " \r", end="") avg_sum_loss_G /= PARAM.step_to_save avg_sum_loss_D /= PARAM.step_to_save avg_show_losses /= PARAM.step_to_save misc_utils.print_log(" Save step %03d:\n" % global_step, train_log_file) misc_utils.print_log(" sum_losses_G: "+str(PARAM.sum_losses_G)+"\n", train_log_file) misc_utils.print_log(" sum_losses_D: "+str(PARAM.sum_losses_D)+"\n", train_log_file) misc_utils.print_log(" show losses : "+str(PARAM.show_losses)+"\n", train_log_file) misc_utils.print_log(" %s\n" % u_str, train_log_file) misc_utils.print_log(" Train > sum_loss:[G %.4f, D %.4f], show_losses:%s, lr:%.2e, Time:%ds. \n" % ( avg_sum_loss_G, avg_sum_loss_D, str(avg_show_losses), lr, time.time()-save_time), train_log_file) # val evalOutputs = eval_one_epoch(sess, val_model, val_inputs.initializer) misc_utils.print_log(" Validation> sum_loss:[G %.4f, D %.4f], show_losses:%s, Time:%ds. \n" % ( evalOutputs.avg_sum_loss_G, evalOutputs.avg_sum_loss_D, str(evalOutputs.avg_show_losses), evalOutputs.cost_time), train_log_file) # test if global_step > 2500: testOutputs = test_one_epoch(sess, test_model) misc_utils.print_log(" Test > Csig: %.3f, Cbak: %.3f, Covl: %.3f, pesq: %.3f," " ssnr: %.4f, lsd: %.4f, estoi: %.4f Time:%ds. \n" % ( testOutputs.csig, testOutputs.cbak, testOutputs.covl, testOutputs.pesq, testOutputs.ssnr, testOutputs.lsd, testOutputs.estoi, testOutputs.cost_time), train_log_file) # save ckpt ckpt_name = PARAM().config_name()+('_step%06d_trloss%.4f_valloss%.4f_lr%.2e_duration%ds' % ( global_step, avg_sum_loss_G, evalOutputs.avg_sum_loss_G, lr, time.time()-save_time)) train_model.saver.save(sess, str(ckpt_dir.joinpath(ckpt_name))) misc_utils.print_log("\n", train_log_file, no_time=True) # init var avg_sum_loss_G = None avg_sum_loss_D = None avg_show_losses = None save_time = time.time() except tf.errors.OutOfRangeError: sess.run(train_inputs.initializer) init_inputs_times += 1
def forward_propagation(images, mnist, nummatches, batch_size, maxlen, train=False, dropout=False): audio_network = stack_layers([ conv_layer(10, 23, 64, name='audio-conv1-layer', padding='VALID'), pool_layer(4, 1, 2, 1, name="audio-max-pool1-layer", padding='VALID'), conv_layer(25, 1, 512, name='audio-conv2-layer', padding='VALID'), mean_pool_layer(name="audio-mean-pool-layer", padding='VALID') ]) image_network = stack_layers([ conv_layer(5, 5, 32, name='image-conv1-layer'), pool_layer(2, 2, 2, 2, name="image-max-pool1-layer"), conv_layer(5, 5, 64, name='image-conv2-layer'), pool_layer(2, 2, 2, 2, name="image-max-pool2-layer"), flatten(), fully_connected_layer(512, keep_prob=0.5 if train and dropout else 1.0, name="image-local1-layer"), fully_connected_layer(512, keep_prob=1.0, name="image-local2-layer") ]) image_joint_network = stack_layers([ fully_connected_layer(512, keep_prob=1.0, name="joint-local-layer"), ]) classification_network = stack_layers([ fully_connected_layer(256, keep_prob=1.0, name="class-local2-layer"), softmax_layer(8, name="class-softmax-layer") ]) specs = tf.concat([images] * COPY, 0) mnistset = tf.unstack(mnist, axis=1) m1 = image_network(mnistset[0]) m2 = image_network(mnistset[1]) m3 = image_network(mnistset[2]) m4 = image_network(mnistset[3]) m5 = image_network(mnistset[4]) m6 = image_network(mnistset[5]) m7 = image_network(mnistset[6]) m8 = image_network(mnistset[7]) m9 = image_network(mnistset[8]) tmp = tf.concat([m1, m2, m3, m4, m5, m6, m7, m8, m9], 1) t1 = image_joint_network(tmp) # t1 = image_network(mnist)[0] tmp = audio_network(specs) t2 = tf.reshape(tmp, [batch_size * COPY, 512]) embeddings = tf.concat([t1, t2], 1) # print("embeddings",embeddings.get_shape()) _, logits, proba, prediction = classification_network(embeddings) with tf.name_scope('accuracy'): with tf.name_scope('accuracy'): actual = nummatches with tf.name_scope('num_correct'): correct = tf.reduce_sum( tf.to_int32(tf.equal(prediction, actual))) with tf.name_scope('loss'): labels_one_hot = tf.one_hot(nummatches, 8, on_value=1.0, off_value=0.0) loss = tf.reduce_mean( tf.nn.softmax_cross_entropy_with_logits(logits=logits, labels=labels_one_hot)) tf.summary.scalar('batch_loss', loss) return correct, loss, proba, prediction
def __init__(self, num_actions, num_features, num_feature_ids, embedding_sizes, hidden_layer_sizes, seed=None, gate_gradients=False, use_locking=False, embedding_init=1.0, relu_init=1e-4, bias_init=0.2, softmax_init=1e-4, averaging_decay=0.9999, use_averaging=True, check_parameters=True, check_every=1, allow_feature_weights=False, only_train='', arg_prefix=None, **unused_kwargs): """Initialize the graph builder with parameters defining the network. Args: num_actions: int size of the set of parser actions num_features: int list of dimensions of the feature vectors num_feature_ids: int list of same length as num_features corresponding to the sizes of the input feature spaces embedding_sizes: int list of same length as num_features of the desired embedding layer sizes hidden_layer_sizes: int list of desired relu layer sizes; may be empty seed: optional random initializer seed to enable reproducibility gate_gradients: if True, gradient updates are computed synchronously, ensuring consistency and reproducibility use_locking: if True, use locking to avoid read-write contention when updating Variables embedding_init: sets the std dev of normal initializer of embeddings to embedding_init / embedding_size ** .5 relu_init: sets the std dev of normal initializer of relu weights to relu_init bias_init: sets constant initializer of relu bias to bias_init softmax_init: sets the std dev of normal initializer of softmax init to softmax_init averaging_decay: decay for exponential moving average when computing averaged parameters, set to 1 to do vanilla averaging use_averaging: whether to use moving averages of parameters during evals check_parameters: whether to check for NaN/Inf parameters during training check_every: checks numerics every check_every steps. allow_feature_weights: whether feature weights are allowed. only_train: the comma separated set of parameter names to train. If empty, all model parameters will be trained. arg_prefix: prefix for context parameters. """ self._num_actions = num_actions self._num_features = num_features self._num_feature_ids = num_feature_ids self._embedding_sizes = embedding_sizes self._hidden_layer_sizes = hidden_layer_sizes self._seed = seed self._gate_gradients = gate_gradients self._use_locking = use_locking self._use_averaging = use_averaging self._check_parameters = check_parameters self._check_every = check_every self._allow_feature_weights = allow_feature_weights self._only_train = set(only_train.split(',')) if only_train else None self._feature_size = len(embedding_sizes) self._embedding_init = embedding_init self._relu_init = relu_init self._softmax_init = softmax_init self._arg_prefix = arg_prefix # Parameters of the network with respect to which training is done. self.params = {} # Other variables, with respect to which no training is done, but which we # nonetheless need to save in order to capture the state of the graph. self.variables = {} # Operations to initialize any nodes that require initialization. self.inits = {} # Training- and eval-related nodes. self.training = {} self.evaluation = {} self.saver = None # Nodes to compute moving averages of parameters, called every train step. self._averaging = {} self._averaging_decay = averaging_decay # After the following 'with' statement, we'll be able to re-enter the # 'params' scope by re-using the self._param_scope member variable. See for # instance _AddParam. self.input = tf.placeholder(dtype=tf.string) self.labels = tf.placeholder(dtype=tf.int32) self.dropout = tf.placeholder(tf.float32) self.input_type_indices = tf.placeholder(tf.int32, [None], name="input_type_indices") self.input_mention_length = tf.placeholder(tf.int32, [None], name="input_mention_length") self.input_mention_indices = tf.placeholder(tf.int32, [None, None], name="input_mention_indices") with tf.name_scope('params') as self._param_scope: self._relu_bias_init = tf.constant_initializer(bias_init) self.training.update(self._BuildNetwork(self.input, return_average=False))
features = sparse_to_tuple(features.tocoo()) num_features = features[2][1] features_nonzero = features[1].shape[0] # Create model model = None if model_str == 'gcn_ae': model = GCNModelAE(placeholders, num_features, features_nonzero) elif model_str == 'gcn_Dir': model = GCNModelDir2(placeholders, num_features, num_nodes, features_nonzero) pos_weight = float(adj.shape[0] * adj.shape[0] - adj.sum()) / adj.sum() norm = adj.shape[0] * adj.shape[0] / float((adj.shape[0] * adj.shape[0] - adj.sum()) * 2) # Optimizer with tf.name_scope('optimizer'): if model_str == 'gcn_ae': opt = OptimizerAE(preds=model.reconstructions, labels=tf.reshape(tf.sparse_tensor_to_dense(placeholders['adj_orig'], validate_indices=False), [-1]), pos_weight=pos_weight, norm=norm) elif model_str == 'gcn_Dir': opt = OptimizerDir2( model=model, label_1=placeholders['labels_1'], label_2=placeholders['labels_2'], mask=placeholders['labels_mask'], label_3=placeholders['labels_3'], ) # Initialize session sess = tf.Session()
def global_step(self): with tf.name_scope('global'): step = tf.Variable(1, name='global_step', dtype=tf.int32, trainable=False) return step
def _ReluWeightInitializer(self): with tf.name_scope(self._param_scope): return tf.random_normal_initializer(stddev=self._relu_init, seed=self._seed)
def training_loop( G_args = {}, # Options for generator network. D_args = {}, # Options for discriminator network. G_opt_args = {}, # Options for generator optimizer. D_opt_args = {}, # Options for discriminator optimizer. G_loss_args = {}, # Options for generator loss. D_loss_args = {}, # Options for discriminator loss. dataset_args = {}, # Options for dataset.load_dataset(). sched_args = {}, # Options for train.TrainingSchedule. grid_args = {}, # Options for train.setup_snapshot_image_grid(). metric_arg_list = [], # Options for MetricGroup. tf_config = {}, # Options for tflib.init_tf(). data_dir = None, # Directory to load datasets from. G_smoothing_kimg = 10.0, # Half-life of the running average of generator weights. minibatch_repeats = 4, # Number of minibatches to run before adjusting training parameters. lazy_regularization = True, # Perform regularization as a separate training step? G_reg_interval = 4, # How often the perform regularization for G? Ignored if lazy_regularization=False. D_reg_interval = 16, # How often the perform regularization for D? Ignored if lazy_regularization=False. reset_opt_for_new_lod = True, # Reset optimizer internal state (e.g. Adam moments) when new layers are introduced? total_kimg = 25000, # Total length of the training, measured in thousands of real images. mirror_augment = False, # Enable mirror augment? drange_net = [0,1], # Dynamic range used when feeding image data to the networks. image_snapshot_ticks = 50, # How often to save image snapshots? None = only save 'reals.png' and 'fakes-init.png'. network_snapshot_ticks = 50, # How often to save network snapshots? None = only save 'networks-final.pkl'. save_tf_graph = False, # Include full TensorFlow computation graph in the tfevents file? save_weight_histograms = False, # Include weight histograms in the tfevents file? resume_pkl = None, # Network pickle to resume training from, None = train from scratch. resume_kimg = 0.0, # Assumed training progress at the beginning. Affects reporting and training schedule. resume_time = 0.0, # Assumed wallclock time at the beginning. Affects reporting. resume_with_new_nets = False): # Construct new networks according to G_args and D_args before resuming training? # Initialize dnnlib and TensorFlow. tflib.init_tf(tf_config) num_gpus = dnnlib.submit_config.num_gpus # Load training set. training_set = dataset.load_dataset(data_dir=dnnlib.convert_path(data_dir), verbose=True, **dataset_args) grid_size, grid_reals, grid_labels = misc.setup_snapshot_image_grid(training_set, **grid_args) misc.save_image_grid(grid_reals, dnnlib.make_run_dir_path('reals.png'), drange=training_set.dynamic_range, grid_size=grid_size) # Construct or load networks. training_set.configure(minibatch_size=sched_args.batch_size) with tf.device('/gpu:0'): if resume_pkl is None or resume_with_new_nets: print('Constructing networks...') G = tflib.Network('G', num_channels=training_set.shape[0], resolution=training_set.shape[1], label_size=training_set.label_size, **G_args) D = tflib.Network('D', num_channels=training_set.shape[0], resolution=training_set.shape[1], label_size=training_set.label_size, **D_args) Gs = G.clone('Gs') start = 0 if resume_pkl is not None: print('Loading networks from "%s"...' % resume_pkl) rG, rD, rGs = misc.load_pkl(resume_pkl) if resume_with_new_nets: G.copy_vars_from(rG); D.copy_vars_from(rD); Gs.copy_vars_from(rGs) else: G = rG; D = rD; Gs = rGs start = int(resume_pkl.split('-')[-1].split('.')[0]) // sched_args.batch_size # Print layers and generate initial image snapshot. G.print_layers(); D.print_layers() grid_latents = np.random.randn(np.prod(grid_size), *G.input_shape[1:]) grid_fakes = G.run(grid_latents, grid_labels, is_validation=True, minibatch_size=sched_args.batch_size) misc.save_image_grid(grid_fakes, dnnlib.make_run_dir_path('fakes_init.png'), drange=drange_net, grid_size=grid_size) global_step = tf.Variable(start, trainable=False, name='learning_rate_step') learning_rate = tf.train.exponential_decay(sched_args.lr, global_step, sched_args.decay_step, sched_args.decay_rate, staircase=sched_args.stair) add_global = global_step.assign_add(1) D_opt = tflib.Optimizer(name='TrainD', learning_rate=learning_rate, **D_opt_args) G_opt = tflib.Optimizer(name='TrainG', learning_rate=learning_rate, **G_opt_args) for gpu in range(num_gpus): print('build graph on gpu %s' % str(gpu)) with tf.name_scope('GPU%d' % gpu), tf.device('/gpu:%d' % gpu): # Create GPU-specific shadow copies of G and D. G_gpu = G if gpu == 0 else G.clone(G.name + '_shadow') D_gpu = D if gpu == 0 else D.clone(D.name + '_shadow') with tf.name_scope('DataFetch'): reals_read, labels_read = training_set.get_minibatch_tf() reals_read, labels_read = process_reals(reals_read, labels_read, mirror_augment, training_set.dynamic_range, drange_net) with tf.name_scope('Loss'), tf.control_dependencies(None): loss, reg = dnnlib.util.call_func_by_name(G=G_gpu, D=D_gpu, opt=D_opt, training_set=training_set, minibatch_size=sched_args.batch_size, reals=reals_read, labels=labels_read, **D_loss_args) with tf.control_dependencies([add_global]): G_opt.register_gradients(loss, G_gpu.trainables) D_opt.register_gradients(loss, D_gpu.trainables) G_train_op = G_opt.apply_updates() D_train_op = D_opt.apply_updates() # Finalize graph. with tf.device('/gpu:0'): try: peak_gpu_mem_op = tf.contrib.memory_stats.MaxBytesInUse() except tf.errors.NotFoundError: peak_gpu_mem_op = tf.constant(0) tflib.init_uninitialized_vars() print('Initializing logs...') summary_log = tf.summary.FileWriter(dnnlib.make_run_dir_path()) if save_tf_graph: summary_log.add_graph(tf.get_default_graph()) if save_weight_histograms: G.setup_weight_histograms(); D.setup_weight_histograms() metrics = metric_base.MetricGroup(metric_arg_list) print('Training for %d kimg...\n' % total_kimg) dnnlib.RunContext.get().update('', cur_epoch=resume_kimg, max_epoch=total_kimg) maintenance_time = dnnlib.RunContext.get().get_last_update_interval() cur_nimg = int(resume_kimg * 1000) cur_tick = -1 tick_start_nimg = cur_nimg prev_lod = -1.0 running_mb_counter = 0 while cur_nimg < total_kimg * 1000: if dnnlib.RunContext.get().should_stop(): break loss_, _, _, lr_ = tflib.run([loss, G_train_op, D_train_op, learning_rate]) cur_nimg += sched_args.batch_size * num_gpus done = (cur_nimg >= total_kimg * 1000) if cur_tick < 0 or cur_nimg >= tick_start_nimg + sched_args.tick_kimg * 1000 or done: cur_tick += 1 tick_kimg = (cur_nimg - tick_start_nimg) / 1000.0 tick_start_nimg = cur_nimg tick_time = dnnlib.RunContext.get().get_time_since_last_update() total_time = dnnlib.RunContext.get().get_time_since_start() + resume_time # Report progress. print( 'tick %-5d kimg %-8.1f minibatch %-4d time %-12s sec/tick %-7.1f ' 'sec/kimg %-7.2f maintenance %-6.1f gpumem %.1f loss %-8.1f lr %-2.5f' % ( autosummary('Progress/tick', cur_tick), autosummary('Progress/kimg', cur_nimg / 1000.0), autosummary('Progress/minibatch', sched_args.batch_size), dnnlib.util.format_time(autosummary('Timing/total_sec', total_time)), autosummary('Timing/sec_per_tick', tick_time), autosummary('Timing/sec_per_kimg', tick_time / tick_kimg), autosummary('Timing/maintenance_sec', maintenance_time), autosummary('Resources/peak_gpu_mem_gb', peak_gpu_mem_op.eval() / 2 ** 30), autosummary('loss', loss_), autosummary('lr', lr_))) autosummary('Timing/total_hours', total_time / (60.0 * 60.0)) autosummary('Timing/total_days', total_time / (24.0 * 60.0 * 60.0)) # Save snapshots. if image_snapshot_ticks is not None and (cur_tick % image_snapshot_ticks == 0 or done): grid_fakes = G.run(grid_latents, grid_labels, is_validation=True, minibatch_size=sched_args.batch_size) misc.save_image_grid(grid_fakes, dnnlib.make_run_dir_path('fakes%06d.png' % (cur_nimg // 1000)), drange=drange_net, grid_size=grid_size) if network_snapshot_ticks is not None and (cur_tick % network_snapshot_ticks == 0 or done): pkl = dnnlib.make_run_dir_path('network-snapshot-%06d.pkl' % (cur_nimg // 1000)) misc.save_pkl((G, D, Gs), pkl) metrics.run(pkl, run_dir=dnnlib.make_run_dir_path(), data_dir=dnnlib.convert_path(data_dir), num_gpus=num_gpus, tf_config=tf_config) # Update summaries and RunContext. metrics.update_autosummaries() tflib.autosummary.save_summaries(summary_log, cur_nimg) dnnlib.RunContext.get().update('%.2f' % 0.0, cur_epoch=cur_nimg // 1000, max_epoch=total_kimg) maintenance_time = dnnlib.RunContext.get().get_last_update_interval() - tick_time # Save final snapshot. misc.save_pkl((G, D, Gs), dnnlib.make_run_dir_path('network-final.pkl')) # All done. summary_log.close() training_set.close()
def __compute_stress(self, damage, effective_stress): with tf.name_scope("Apply1MinusD"): stress_vector = tf.multiply( tf.subtract(1.0, tf.expand_dims(damage, 1)), effective_stress) return stress_vector
cfg = Config graph = tf.Graph() bsize = cfg.bsize with graph.as_default(): # tf Graph input tf.set_random_seed(1) x = tf.placeholder(tf.int32, shape=(cfg.bsize, cfg.max_seq_len), name="x") y = tf.placeholder(tf.float32, shape=(cfg.bsize, 6), name="y") em = tf.placeholder(tf.float32, shape=(embedding_matrix.shape[0], embedding_matrix.shape[1]), name="em") keep_prob = tf.placeholder(dtype=tf.float32, name="keep_prob") with tf.name_scope("Embedding"): #embedding = tf.get_variable("embedding", [embedding_matrix.shape[0], embedding_matrix.shape[1]], # dtype=tf.float32, initializer=tf.constant_initializer(embedding_matrix), # trainable=False) embedded_input = tf.nn.embedding_lookup(em, x, name="embedded_input") with tf.variable_scope('forward'): fw_cell1 = tf.contrib.rnn.UGRNNCell(64, activation=tf.nn.elu) fw_cell1 = tf.nn.rnn_cell.DropoutWrapper(fw_cell1, output_keep_prob=keep_prob) fw_cell2 = tf.contrib.rnn.UGRNNCell(64, activation=tf.nn.elu) # fw_cell2 = tf.nn.rnn_cell.DropoutWrapper(fw_cell2, output_keep_prob=keep_prob) stacked_fw_rnn = [fw_cell1, fw_cell2] fw_multi_cell = tf.contrib.rnn.MultiRNNCell(cells=stacked_fw_rnn, state_is_tuple=True) with tf.variable_scope('backward'): bw_cell1 = tf.contrib.rnn.UGRNNCell(64, activation=tf.nn.elu)
def _shadownet_fun(features, labels, mode, params): is_training = (mode == tf.estimator.ModeKeys.TRAIN) tower_features = features tower_labels = labels tower_losses = [] tower_gradvars = [] tower_preds = [] tower_tensor_dict = [] tower_seq_len = [] num_devices = num_gpus device_type = 'gpu' tower_batch_size = int(params.batch_size / num_devices) for i in range(num_devices): worker_device = '/{}:{}'.format(device_type, i) device_setter = local_device_setter(worker_device=worker_device) with tf.variable_scope('shadownet', reuse=bool(i != 0)): with tf.name_scope('tower_%d' % i) as name_scope: with tf.device(device_setter): loss, gradvars, preds, tensor_dict, seq_len = _tower_fn( is_training, tower_features[i], tower_labels[i], tower_batch_size, params.l_size) tower_losses.append(loss) tower_gradvars.append(gradvars) tower_preds.append(preds) tower_tensor_dict.append(tensor_dict) tower_seq_len.append(seq_len) if i == 0: # Only trigger batch_norm moving mean and variance update from # the 1st tower. Ideally, we should grab the updates from all # towers but these stats accumulate extremely fast so we can # ignore the other stats from the other towers without # significant detriment. update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS, name_scope) # Now compute global loss and gradients. gradvars = [] with tf.name_scope('gradient_averaging'): all_grads = {} for grad, var in itertools.chain(*tower_gradvars): if grad is not None: all_grads.setdefault(var, []).append(grad) for var, grads in six.iteritems(all_grads): # Average gradients on the same device as the variables with tf.device(var.device): if len(grads) == 1: avg_grad = grads[0] else: avg_grad = tf.multiply(tf.add_n(grads), 1. / len(grads)) gradvars.append((avg_grad, var)) # Device that runs the ops to apply global gradient updates. consolidation_device = '/gpu:0' if variable_strategy == 'GPU' else '/cpu:0' with tf.device(consolidation_device): global_step = tf.train.get_global_step() starter_learning_rate = params.learning_rate learning_rate = tf.train.exponential_decay(starter_learning_rate, global_step, params.decay_steps, params.decay_rate, staircase=True) loss = tf.reduce_mean(tower_losses, name='loss') decoded, log_prob = tf.nn.ctc_beam_search_decoder(tower_preds[0], tower_seq_len[0] * np.ones(tower_batch_size), merge_repeated=False) sequence_dist = tf.reduce_mean(tf.edit_distance(tf.cast(decoded[0], tf.int32), tower_labels[0])) sequence_lengths_pred = tf.bincount(tf.cast(decoded[0].indices[:, 0], tf.int32), minlength=tf.shape(tower_labels[0])[1]) label_lengths_pred = tf.bincount(tf.cast(labels[0].indices[:, 0], tf.int32), minlength=tf.shape(tower_labels[0])[1]) tensors_to_log = {'global_step': global_step, 'learning_rate': learning_rate, 'loss': loss} ''' dist_to_log = {'global_step': global_step, 'learning_rate': learning_rate, 'loss': loss, 'train_seq_dist': sequence_dist, 'sequence_lengths_pred': sequence_lengths_pred, 'label_lengths_pred': label_lengths_pred} ''' logging_hook = tf.train.LoggingTensorHook( tensors=tensors_to_log, every_n_iter=10) train_hooks = [logging_hook] # dist_hook = tf.train.LoggingTensorHook( # tensors=dist_to_log, every_n_iter=2000) # train_hooks = [logging_hook, dist_hook] optimizer = tf.train.AdadeltaOptimizer(learning_rate=learning_rate) # optimizer = tf.train.RMSPropOptimizer(learning_rate=learning_rate) # optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate) if params.sync: optimizer = tf.train.SyncReplicasOptimizer( optimizer, replicas_to_aggregate=num_workers) sync_replicas_hook = optimizer.make_session_run_hook(params.is_chief) train_hooks.append(sync_replicas_hook) # Create single grouped train op train_op = [ optimizer.apply_gradients( gradvars, global_step=tf.train.get_global_step()) ] train_op.extend(update_ops) train_op = tf.group(*train_op) return tf.estimator.EstimatorSpec( mode=mode, loss=loss, train_op=train_op, training_hooks=train_hooks)
def ComputePredictions(self, theta, source_encs, source_paddings, targets, src_segment_id): """Decodes `targets` given encoded source. Args: theta: A `.NestedMap` object containing weights' values of this layer and its children layers. source_encs: source encoding, of shape [time, batch, depth]. source_paddings: source encoding's padding, of shape [time, batch]. targets: A dict of string to tensors representing the targets one try to predict. Each tensor in targets is of shape [batch, time]. src_segment_id: source segment id, of shape [time, batch]. Returns: A Tensor with shape [time, batch, params.softmax.input_dim]. """ p = self.params time, batch = py_utils.GetShape(source_paddings, 2) source_encs = py_utils.HasShape(source_encs, [time, batch, p.source_dim]) with tf.name_scope(p.name): target_ids = tf.transpose(targets.ids) target_paddings = py_utils.HasRank(targets.paddings, 2) target_paddings = tf.expand_dims(tf.transpose(target_paddings), 2) if p.packed_input: target_segment_id = tf.expand_dims(tf.transpose(targets.segment_ids), 2) else: target_segment_id = tf.zeros_like(target_paddings) if py_utils.use_tpu(): emb_device = cluster_factory.Current().WorkerDeviceInModelSplit(0) else: emb_device = '' with tf.device(emb_device): inputs = self.emb.EmbLookup(theta.emb, target_ids) inputs = self.ApplyClipping(theta, inputs) summary_utils.histogram(p, 'input_emb', inputs) inputs = self.ApplyDropout(inputs) self._emb_out = inputs # Layer 0 interwines with attention. (atten_ctxs, xs, atten_probs, _) = self.frnn_with_atten.FProp( theta.frnn_with_atten, source_encs, source_paddings, inputs, target_paddings, src_segment_id=src_segment_id, segment_id=target_segment_id) if p.add_summary: self._AddAttenProbsSummary(source_paddings, targets, [atten_probs]) atten_ctxs = self.ApplyClipping(theta, atten_ctxs) summary_utils.histogram(p, 'atten_ctxs', atten_ctxs) for i, (layer, layer_theta) in enumerate(zip(self.frnn, theta.frnn)): # Forward through Layer-(i + 1) because Layer-0 handled before. ys, _ = layer.FProp( layer_theta, tf.concat([xs, atten_ctxs], 2), target_paddings, segment_id=target_segment_id) ys = self.ApplyDropout(ys) if 1 + i >= p.residual_start: xs += ys # Residual skip xs = self.ApplyClipping(theta, xs) else: xs = ys summary_utils.histogram(p, 'layer_out_%s' % i, xs) if p.feed_attention_context_vec_to_softmax: xs = tf.concat([xs, atten_ctxs], 2) return xs
def _make_graph(self): self.logger.info("Generating training graph on {} GPUs ...".format(self.cfg.nr_gpus)) weights_initializer = slim.xavier_initializer() biases_initializer = tf.constant_initializer(0.) biases_regularizer = tf.no_regularizer weights_regularizer = tf.contrib.layers.l2_regularizer(self.cfg.weight_decay) tower_grads = [] with tf.variable_scope(tf.get_variable_scope()): for i in range(self.cfg.nr_gpus): with tf.device('/gpu:%d' % i): with tf.name_scope('tower_%d' % i) as name_scope: # Force all Variables to reside on the CPU. with slim.arg_scope([slim.model_variable, slim.variable], device='/device:CPU:0'): with slim.arg_scope([slim.conv2d, slim.conv2d_in_plane, \ slim.conv2d_transpose, slim.separable_conv2d, slim.fully_connected], weights_regularizer=weights_regularizer, biases_regularizer=biases_regularizer, weights_initializer=weights_initializer, biases_initializer=biases_initializer): # loss over single GPU self.net.make_network(is_train=True) if i == self.cfg.nr_gpus - 1: loss = self.net.get_loss(include_wd=True) else: loss = self.net.get_loss() self._input_list.append( self.net.get_inputs() ) tf.get_variable_scope().reuse_variables() if i == 0: if self.cfg.nr_gpus > 1 and self.cfg.bn_train is True: self.logger.warning("BN is calculated only on single GPU.") extra_update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS, name_scope) with tf.control_dependencies(extra_update_ops): grads = self._optimizer.compute_gradients(loss) else: grads = self._optimizer.compute_gradients(loss) final_grads = [] with tf.variable_scope('Gradient_Mult') as scope: for grad, var in grads: scale = 1. if self.cfg.double_bias and '/biases:' in var.name: scale *= 2. if not np.allclose(scale, 1.): grad = tf.multiply(grad, scale) final_grads.append((grad, var)) tower_grads.append(final_grads) if len(tower_grads) > 1: grads = sum_gradients(tower_grads) else: grads = tower_grads[0] if False: variable_averages = tf.train.ExponentialMovingAverage(0.9999) variables_to_average = (tf.trainable_variables() + tf.moving_average_variables()) variables_averages_op = variable_averages.apply(variables_to_average) apply_gradient_op = self._optimizer.apply_gradients(grads) train_op = tf.group(apply_gradient_op, variables_averages_op, *extra_update_ops) else: apply_gradient_op = self._optimizer.apply_gradients(grads) train_op = tf.group(apply_gradient_op, *extra_update_ops) return train_op
def getembedding(wordsetsize, embedding_size): with tf.name_scope("embedding"): embedding_dict = tf.Variable(tf.random.uniform( [wordsetsize + 1, embedding_size], -1.0, 1.0), name="dict") return embedding_dict
def _FProp(self, theta, source_encs, source_paddings, targets, src_segment_id): """Decodes `targets` given encoded source. Args: theta: A `.NestedMap` object containing weights' values of this layer and its children layers. source_encs: source encoding. When `p.is_transparent` is False, it is a tensor of shape [time, batch, depth]. When `p.is_transparent` is True, it is a tensor of shape [time, batch, depth, num_trans_layers] if `p.is_eval` is True, and a list of `num_trans_layers` tensors of shape [time, batch, depth] if `p.is_eval` is False. source_paddings: source encoding's padding, of shape [time, batch]. targets: A dict of string to tensors representing the targets one try to predict. Each tensor in targets is of shape [batch, time]. src_segment_id: source segment id, of shape [time, batch]. Returns: Output of last decoder layer, [target_time, target_batch, source_dim]. """ p = self.params time, batch = py_utils.GetShape(source_paddings, 2) if p.is_transparent: if p.is_eval: source_encs = py_utils.HasShape( source_encs, [time, batch, p.source_dim, p.num_trans_layers]) source_encs = tf.unstack(source_encs, axis=3) else: assert isinstance(source_encs, list) assert len(source_encs) == p.num_trans_layers for i in range(p.num_trans_layers): source_encs[i] = py_utils.HasShape(source_encs[i], [time, batch, p.source_dim]) else: source_encs = py_utils.HasShape(source_encs, [time, batch, p.source_dim]) source_encs = [source_encs] * p.num_trans_layers with tf.name_scope(p.name): # [batch, time] target_ids = targets.ids # [time, batch] target_paddings = tf.transpose(targets.paddings) target_segment_pos = None target_segment_id = None if p.packed_input: target_segment_id = tf.transpose(targets.segment_ids) target_segment_pos = targets.segment_pos assert src_segment_id is not None, ('Need to provide src_segment_id ' 'for packed input.') # Embedding layer # [batch, time, model_dim] token_embs = self.token_emb.EmbLookup(theta.token_emb, target_ids) target_time = py_utils.GetShape(target_ids)[1] # [1, time, model_dim] if p.packed_input: posit_embs = self.position_emb.FPropWithPosition( theta.position_emb, target_segment_pos) else: posit_embs = tf.expand_dims( self.position_emb.FProp(theta.position_emb, target_time), 0) # [time, batch, model_dim] input_embs = token_embs + posit_embs if p.model_dim != p.token_emb.embedding_dim: input_embs = self.emb_proj.FProp(theta.emb_proj, input_embs) input_embs = tf.transpose(input_embs, [1, 0, 2]) input_embs = self.input_dropout.FProp(theta.input_dropout, input_embs) atten_probs = [] layer_in = input_embs for i, (layer, layer_theta) in enumerate(zip(self.trans, theta.trans)): # [time, batch, model_dim] layer_out, probs = layer.FProp( layer_theta, layer_in, target_paddings, source_encs[i], source_paddings, source_segment_id=target_segment_id, aux_segment_id=src_segment_id) layer_in = layer_out atten_probs.append(probs) if p.add_summary: self._AddAttenProbsSummary(source_paddings, targets, atten_probs) return layer_out
def _scope_all(scope, default_scope=None): with tf.variable_scope(scope, default_name=default_scope) as s,\ tf.name_scope(s.original_name_scope): yield s
def _initialize(self): input_var = tf.compat.v1.placeholder(tf.float32, shape=(None, ) + self._input_shape) ys_var = tf.compat.v1.placeholder(dtype=tf.float32, name='ys', shape=(None, self._output_dim)) old_means_var = tf.compat.v1.placeholder(dtype=tf.float32, name='old_means', shape=(None, self._output_dim)) old_log_stds_var = tf.compat.v1.placeholder(dtype=tf.float32, name='old_log_stds', shape=(None, self._output_dim)) (_, means, log_stds, _, norm_means, norm_log_stds, self._x_mean, self._x_std, self._y_mean, self._y_std, dist) = self.build(input_var).outputs normalized_ys_var = (ys_var - self._y_mean) / self._y_std normalized_old_means_var = (old_means_var - self._y_mean) / self._y_std normalized_old_log_stds_var = (old_log_stds_var - tf.math.log(self._y_std)) normalized_dist_info_vars = dict(mean=norm_means, log_std=norm_log_stds) mean_kl = tf.reduce_mean( dist.kl_sym( dict(mean=normalized_old_means_var, log_std=normalized_old_log_stds_var), normalized_dist_info_vars, )) loss = -tf.reduce_mean( dist.log_likelihood_sym(normalized_ys_var, normalized_dist_info_vars)) self._f_predict = tensor_utils.compile_function([input_var], means) self._f_pdists = tensor_utils.compile_function([input_var], [means, log_stds]) optimizer_args = dict( loss=loss, target=self, network_outputs=[norm_means, norm_log_stds], ) if self._use_trust_region: optimizer_args['leq_constraint'] = (mean_kl, self._max_kl_step) optimizer_args['inputs'] = [ input_var, ys_var, old_means_var, old_log_stds_var ] else: optimizer_args['inputs'] = [input_var, ys_var] with tf.name_scope('update_opt'): self._optimizer.update_opt(**optimizer_args)