def task_metalearn(inp, reuse=True): """ Perform gradient descent for one task in the meta-batch. """ inputa, inputb, labela, labelb = inp task_outputbs, task_lossesb = [], [] if self.classification: task_accuraciesb = [] task_outputa = self.forward( inputa, weights, reuse=reuse) # only reuse on the first iter task_lossa = self.loss_func(task_outputa, labela) grads = tf.gradients(task_lossa, list(weights.values())) if FLAGS.stop_grad: grads = [tf.stop_gradient(grad) for grad in grads] gradients = dict(zip(weights.keys(), grads)) fast_weights = dict( zip(weights.keys(), [ weights[key] - self.update_lr * gradients[key] for key in weights.keys() ])) output = self.forward(inputb, fast_weights, reuse=True) task_outputbs.append(output) task_lossesb.append(self.loss_func(output, labelb)) for j in range(num_updates - 1): loss = self.loss_func( self.forward(inputa, fast_weights, reuse=True), labela) grads = tf.gradients(loss, list(fast_weights.values())) if FLAGS.stop_grad: grads = [tf.stop_gradient(grad) for grad in grads] gradients = dict(zip(fast_weights.keys(), grads)) fast_weights = dict( zip(fast_weights.keys(), [ fast_weights[key] - self.update_lr * gradients[key] for key in fast_weights.keys() ])) output = self.forward(inputb, fast_weights, reuse=True) task_outputbs.append(output) task_lossesb.append(self.loss_func(output, labelb)) task_output = [ task_outputa, task_outputbs, task_lossa, task_lossesb ] if self.classification: task_accuracya = contrib_metrics.accuracy( tf.argmax(tf.nn.softmax(task_outputa), 1), tf.argmax(labela, 1)) for j in range(num_updates): task_accuraciesb.append( contrib_metrics.accuracy( tf.argmax(tf.nn.softmax(task_outputbs[j]), 1), tf.argmax(labelb, 1))) task_output.extend([task_accuracya, task_accuraciesb]) return task_output
def build_net(self): # 数据 image_placeholder = tf.placeholder(dtype=tf.float32, shape=(None, self.input_size[0], self.input_size[1], 4)) label_segment_placeholder = tf.placeholder(dtype=tf.int32, shape=(None, self.input_size[0] // self.ratio, self.input_size[1] // self.ratio, 1)) label_classes_placeholder = tf.placeholder(dtype=tf.int32, shape=(None,)) # 网络 net = PSPNet({'data': image_placeholder}, is_training=True, num_classes=self.num_classes, num_segment=self.num_segment, last_pool_size=self.last_pool_size, filter_number=self.filter_number) raw_output_segment = net.layers['conv6_n_4'] raw_output_classes = net.layers['class_attention_fc'] # Predictions prediction = tf.reshape(raw_output_segment, [-1, self.num_segment]) pred_segment = tf.cast(tf.expand_dims(tf.argmax(raw_output_segment, axis=-1), axis=-1), tf.int32) pred_classes = tf.cast(tf.argmax(raw_output_classes, axis=-1), tf.int32) # label label_batch = tf.image.resize_nearest_neighbor(label_segment_placeholder, tf.stack(raw_output_segment.get_shape()[1:3])) label_batch = tf.reshape(label_batch, [-1, ]) # 当前批次的准确率:accuracy accuracy_segment = tcm.accuracy(pred_segment, label_segment_placeholder) accuracy_classes = tcm.accuracy(pred_classes, label_classes_placeholder) # loss loss_segment = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(labels=label_batch, logits=prediction)) # 分类损失 loss_classes = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(labels=label_classes_placeholder, logits=raw_output_classes)) # 总损失 loss = tf.add_n([loss_segment, 0.1 * loss_classes]) # 学习率策略 step_ph = tf.placeholder(dtype=tf.float32, shape=()) learning_rate = tf.scalar_mul(tf.constant(self.learning_rate), tf.pow((1 - step_ph / self.num_steps), 0.9)) train_op = tf.train.GradientDescentOptimizer(learning_rate).minimize(loss) # 单独训练最后的分类 classes_trainable = [v for v in tf.trainable_variables() if 'class_attention' in v.name] train_classes_op = tf.train.GradientDescentOptimizer(learning_rate).minimize(loss, var_list=classes_trainable) return (image_placeholder, label_segment_placeholder, label_classes_placeholder, raw_output_segment, raw_output_classes, pred_segment, pred_classes, loss_segment, loss_classes, loss, accuracy_segment, accuracy_classes, step_ph, train_op, train_classes_op, learning_rate)
def accuracy_(score, labels): ''' assumes score will be logits (ie unbounded real numbers) ''' pred = tf.cast(tf.round(tf.sigmoid(score)), 'int8') _labels = tf.cast(labels, 'int8') return accuracy(pred, _labels)
def build_model(inputs, labels, num_classes): weight_decay = 1e-3 conv1sz = 16 conv2sz = 32 fc3sz = 512 with tf.contrib.framework.arg_scope( [layers.convolution2d], kernel_size=5, stride=1, padding='SAME', activation_fn=tf.nn.relu, weights_initializer=layers.variance_scaling_initializer(), weights_regularizer=layers.l2_regularizer(weight_decay) ): net = layers.convolution2d(inputs, conv1sz, scope='conv1') # ostatak konvolucijskih i pooling slojeva pool1 = layers.max_pool2d(inputs=net, kernel_size=[2, 2], stride=2) conv2 = layers.convolution2d(pool1, conv2sz, scope="conv2") pool2 = layers.max_pool2d(inputs=conv2, kernel_size=2, stride=2) with tf.contrib.framework.arg_scope( [layers.fully_connected], activation_fn=tf.nn.relu, weights_initializer=layers.variance_scaling_initializer(), weights_regularizer=layers.l2_regularizer(weight_decay) ): # sada definiramo potpuno povezane slojeve # ali najprije prebacimo 4D tenzor u matricu net = layers.flatten(pool2) # originally inputs...? net = layers.fully_connected(net, fc3sz, scope='fc3') logits = layers.fully_connected(net, num_classes, activation_fn=None, scope='logits') loss = tf.reduce_mean(tf.losses.softmax_cross_entropy(labels, logits)) accuracy = metrics.accuracy(tf.argmax(logits, 1), tf.argmax(labels, 1)) return logits, loss, accuracy
def build_trainer(self, timestamps, mmsis): labels = self.multihot_labels(mmsis) with tf.variable_scope("custom-loss"): # Normal args are the totals for each correct value (as used in standard cross entropy) normal_args = tf.reduce_sum(tf.to_float(labels) * self.prediction, reduction_indices=[1]) # To encourage self consistency in the face of noise we also use a component where the args # is the largest prediction. consistent_args = tf.reduce_max(self.prediction, axis=[1]) # positives = (1 - self.epsilon) * tf.log( normal_args) + self.epsilon * tf.log(consistent_args) raw_loss = -tf.reduce_mean(positives) mask = tf.to_float(tf.equal(tf.reduce_sum(labels, 1), 1)) int_labels = tf.to_int32(tf.argmax(labels, 1)) int_predictions = tf.to_int32(tf.argmax(self.prediction, 1)) accuracy = metrics.accuracy(int_labels, int_predictions, weights=mask) loss = raw_loss * self.loss_weight update_ops = [] update_ops.append( tf.summary.scalar('%s/Training-loss' % self.name, raw_loss)) update_ops.append( tf.summary.scalar('%s/Training-accuracy' % self.name, accuracy)) return Trainer(loss, update_ops)
def predict_loss_acc(logits, real): predict = tf.to_int32(tf.argmax(logits, real.shape.ndims)) loss = tf.losses.sparse_softmax_cross_entropy(logits = logits, labels = real, scope="loss") #予測がラベルに一致する頻度を計算する acc = tf_metrics.accuracy(predict, real) return predict, loss, acc
def _compute_accuracy(logits, targets, weights=None): if self._n_classes > 2: _, predictions = nn.top_k(logits, 1) else: predictions = array_ops.reshape(logits, [-1]) predictions = math_ops.greater(predictions, array_ops.zeros_like(predictions)) targets = array_ops.reshape(targets, [-1]) return metrics_lib.accuracy( math_ops.to_int32(predictions), math_ops.to_int32(targets), weights)
def _compute_accuracy(logits, targets, weights=None): if self._n_classes > 2: _, predictions = nn.top_k(logits, 1) else: predictions = array_ops.reshape(logits, [-1]) predictions = math_ops.greater( predictions, array_ops.zeros_like(predictions)) targets = array_ops.reshape(targets, [-1]) return metrics_lib.accuracy(math_ops.to_int32(predictions), math_ops.to_int32(targets), weights)
def accuracy(self, params, data, labels): """Computes the accuracy (fraction of correct classifications). Args: params: List of parameter tensors or variables data: Batch of features with samples along the first dimension labels: Vector of labels with the same number of samples as the data Returns: accuracy: Fraction of correct classifications across the batch """ predictions = self.argmax(self.inference(params, data)) return contrib_metrics.accuracy(predictions, tf.cast(labels, tf.int32))
def accuracy_top_1(labels, logits=None, probs=None, decay=0.01): """ calculate moving accuracy for classification Arguments: labels : [batch_size, nb_classes], must be one-hot logits or probs : [batch_size, nb_classes] decay : float in range [0, 1] """ if probs is not None: acc = tcm.accuracy(predictions=tf.argmax(probs, axis=-1), labels=tf.argmax(labels, axis=-1)) elif logits is not None: acc = tcm.accuracy(predictions=tf.argmax(logits, axis=-1), labels=tf.argmax(labels, axis=-1)) else: raise Exception( 'in metric accuracy, the probability vector cannot be None') if decay == 1.0: return acc else: var = tf.Variable(0.0, name='acc_top_1') return _assign_moving_average(var, acc, decay)
def construct(self, hidden_layer_size): with self.session.graph.as_default(): with tf.name_scope("inputs"): self.images = tf.placeholder( tf.float32, [None, self.WIDTH, self.HEIGHT, 1], name="images") self.labels = tf.placeholder(tf.int64, [None], name="labels") flattened_images = tf_layers.flatten(self.images, scope="preprocessing") hidden_layer = tf_layers.fully_connected( flattened_images, num_outputs=hidden_layer_size, activation_fn=tf.nn.relu, scope="hidden_layer") output_layer = tf_layers.fully_connected(hidden_layer, num_outputs=self.LABELS, activation_fn=None, scope="output_layer") self.predictions = tf.argmax(output_layer, 1) loss = tf_losses.sparse_softmax_cross_entropy(output_layer, self.labels, scope="loss") self.global_step = tf.Variable(0, dtype=tf.int64, trainable=False, name="global_step") self.training = tf.train.AdamOptimizer().minimize( loss, global_step=self.global_step) self.accuracy = tf_metrics.accuracy(self.predictions, self.labels) # Summaries self.summaries = { "training": tf.merge_summary([ tf.scalar_summary("train/loss", loss), tf.scalar_summary("train/accuracy", self.accuracy) ]) } for dataset in ["dev", "test"]: self.summaries[dataset] = tf.scalar_summary( dataset + "/accuracy", self.accuracy) # Initialize variables self.session.run(tf.initialize_all_variables()) # Finalize graph and log it if requested self.session.graph.finalize() if self.summary_writer: self.summary_writer.add_graph(self.session.graph)
def build_model(num_classes): weight_decay = 1e-3 conv1sz = 16 conv2sz = 32 fc3sz = 256 fc4sz = 128 inputs = tf.placeholder(tf.float32, [None, 32, 32, 3]) labels = tf.placeholder(tf.int32, [None]) one_hot = tf.one_hot(labels, num_classes) with tf.contrib.framework.arg_scope( [layers.convolution2d], kernel_size=5, stride=1, padding='SAME', activation_fn=tf.nn.relu, weights_initializer=layers.variance_scaling_initializer(), weights_regularizer=layers.l2_regularizer(weight_decay) ): net = layers.convolution2d(inputs, conv1sz, scope='conv1') # ostatak konvolucijskih i pooling slojeva pool1 = layers.max_pool2d(inputs=net, kernel_size=3, stride=2) conv2 = layers.convolution2d(pool1, conv2sz, scope="conv2") pool2 = layers.max_pool2d(inputs=conv2, kernel_size=3, stride=2) with tf.contrib.framework.arg_scope( [layers.fully_connected], activation_fn=tf.nn.relu, weights_initializer=layers.variance_scaling_initializer(), weights_regularizer=layers.l2_regularizer(weight_decay) ): # sada definiramo potpuno povezane slojeve # ali najprije prebacimo 4D tenzor u matricu net = layers.flatten(pool2) # originally inputs...? net = layers.fully_connected(net, fc3sz, scope='fc3') net = layers.fully_connected(net, fc4sz, scope='fc4') logits = layers.fully_connected(net, num_classes, activation_fn=None, scope='logits') loss = tf.reduce_mean(tf.losses.softmax_cross_entropy(one_hot, logits)) accuracy = metrics.accuracy(tf.argmax(logits, 1), tf.argmax(one_hot, 1)) optimizer = tf.train.GradientDescentOptimizer(learning_rate=1e-2) train_op = optimizer.minimize(loss=loss, global_step=tf.train.get_global_step()) return logits, loss, train_op, accuracy, inputs, labels
def build_trainer(self, timestamps, mmsis): labels = self.multihot_labels(mmsis) with tf.variable_scope("custom-loss"): positives = tf.reduce_sum(tf.to_float(labels) * self.prediction, reduction_indices=[1]) raw_loss = -tf.reduce_mean(tf.log(positives)) mask = tf.to_float(tf.equal(tf.reduce_sum(labels, 1), 1)) int_labels = tf.to_int32(tf.argmax(labels, 1)) int_predictions = tf.to_int32(tf.argmax(self.prediction, 1)) accuracy = metrics.accuracy(int_labels, int_predictions, weights=mask) loss = raw_loss * self.loss_weight update_ops = [] update_ops.append( tf.summary.scalar('%s/Training-loss' % self.name, raw_loss)) update_ops.append( tf.summary.scalar('%s/Training-accuracy' % self.name, accuracy)) return Trainer(loss, update_ops)
def __init__(self, batch_size, last_pool_size, input_size, log_dir, data_root_path, train_list, data_path, annotation_path, class_path, model_name="model.ckpt", is_test=False): # 和保存模型相关的参数 self.log_dir = Tools.new_dir(log_dir) self.model_name = model_name self.checkpoint_path = os.path.join(self.log_dir, self.model_name) # 和数据相关的参数 self.input_size = input_size self.batch_size = batch_size self.num_classes = 21 self.num_segment = 4 # 解码通道数:其他对象、attention、边界、背景 self.segment_attention = 1 # 当解码的通道数是4时,attention所在的位置 self.attention_module_num = 2 # attention模块中,解码通道数是2(背景、attention)的模块个数 # 和模型相关的参数:必须保证input_size大于8倍的last_pool_size self.ratio = 8 self.last_pool_size = last_pool_size self.filter_number = 32 # 和模型训练相关的参数 self.learning_rate = 5e-3 self.num_steps = 500001 self.print_step = 5 if is_test else 25 # 读取数据 self.data_reader = Data(data_root_path=data_root_path, data_list=train_list, data_path=data_path, annotation_path=annotation_path, class_path=class_path, batch_size=self.batch_size, image_size=self.input_size, is_test=is_test, has_255=True) # 数据 self.image_placeholder = tf.placeholder(dtype=tf.float32, shape=(None, self.input_size[0], self.input_size[1], 4)) self.label_segment_placeholder = tf.placeholder( dtype=tf.int32, shape=(None, self.input_size[0] // self.ratio, self.input_size[1] // self.ratio, 1)) self.label_attention_placeholder = tf.placeholder( dtype=tf.int32, shape=(None, self.input_size[0] // self.ratio, self.input_size[1] // self.ratio, 1)) self.label_classes_placeholder = tf.placeholder(dtype=tf.int32, shape=(None, )) # 网络 self.net = BAISNet(self.image_placeholder, is_training=True, num_classes=self.num_classes, num_segment=self.num_segment, segment_attention=self.segment_attention, last_pool_size=self.last_pool_size, filter_number=self.filter_number, attention_module_num=self.attention_module_num) self.segments, self.attentions, self.classes = self.net.build() self.final_segment_logit = self.segments[0] self.final_class_logit = self.classes[0] # Predictions self.pred_segment = tf.cast( tf.expand_dims(tf.argmax(self.final_segment_logit, axis=-1), axis=-1), tf.int32) self.pred_classes = tf.cast(tf.argmax(self.final_class_logit, axis=-1), tf.int32) # loss self.label_batch = tf.image.resize_nearest_neighbor( self.label_segment_placeholder, tf.stack(self.final_segment_logit.get_shape()[1:3])) self.label_attention_batch = tf.image.resize_nearest_neighbor( self.label_attention_placeholder, tf.stack(self.final_segment_logit.get_shape()[1:3])) self.loss, self.loss_segment_all, self.loss_class_all, self.loss_segments, self.loss_classes = self.cal_loss( self.segments, self.classes, self.label_batch, self.label_attention_batch, self.label_classes_placeholder, self.num_segment, attention_module_num=self.attention_module_num) # 当前批次的准确率:accuracy self.accuracy_segment = tcm.accuracy(self.pred_segment, self.label_segment_placeholder) self.accuracy_classes = tcm.accuracy(self.pred_classes, self.label_classes_placeholder) with tf.name_scope("train"): # 学习率策略 self.step_ph = tf.placeholder(dtype=tf.float32, shape=()) self.learning_rate = tf.scalar_mul( tf.constant(self.learning_rate), tf.pow((1 - self.step_ph / self.num_steps), 0.9)) self.train_op = tf.train.GradientDescentOptimizer( self.learning_rate).minimize(self.loss) # 单独训练最后的attention attention_trainable = [ v for v in tf.trainable_variables() if 'attention' in v.name or "class_attention" in v.name ] print(len(attention_trainable)) self.train_attention_op = tf.train.GradientDescentOptimizer( self.learning_rate).minimize(self.loss, var_list=attention_trainable) pass # summary 1 with tf.name_scope("loss"): tf.summary.scalar("loss", self.loss) tf.summary.scalar("loss_segment", self.loss_segment_all) tf.summary.scalar("loss_class", self.loss_class_all) for loss_segment_index, loss_segment in enumerate( self.loss_segments): tf.summary.scalar("loss_segment_{}".format(loss_segment_index), loss_segment) for loss_class_index, loss_class in enumerate(self.loss_classes): tf.summary.scalar("loss_class_{}".format(loss_class_index), loss_class) pass with tf.name_scope("accuracy"): tf.summary.scalar("accuracy_segment", self.accuracy_segment) tf.summary.scalar("accuracy_classes", self.accuracy_classes) pass with tf.name_scope("label"): split = tf.split(self.image_placeholder, num_or_size_splits=4, axis=3) tf.summary.image("0-mask", split[3]) tf.summary.image("1-image", tf.concat(split[0:3], axis=3)) tf.summary.image( "2-label", tf.cast(self.label_segment_placeholder * 85, dtype=tf.uint8)) tf.summary.image( "3-attention", tf.cast(self.label_attention_placeholder * 255, dtype=tf.uint8)) pass with tf.name_scope("predict"): tf.summary.image("predict", tf.cast(self.pred_segment * 85, dtype=tf.uint8)) pass with tf.name_scope("attention"): # attention for attention_index, attention in enumerate(self.attentions): tf.summary.image("{}-attention".format(attention_index), attention) pass pass with tf.name_scope("sigmoid"): for segment_index, segment in enumerate(self.segments): if segment_index < self.attention_module_num: split = tf.split(segment, num_or_size_splits=self.num_segment, axis=3) tf.summary.image("{}-other".format(segment_index), split[0]) tf.summary.image("{}-attention".format(segment_index), split[1]) tf.summary.image("{}-border".format(segment_index), split[2]) tf.summary.image("{}-background".format(segment_index), split[-1]) else: split = tf.split(segment, num_or_size_splits=2, axis=3) tf.summary.image("{}-background".format(segment_index), split[0]) tf.summary.image("{}-attention".format(segment_index), split[1]) pass pass pass self.summary_op = tf.summary.merge_all() # sess 和 saver self.sess = tf.Session(config=tf.ConfigProto(gpu_options=tf.GPUOptions( allow_growth=True))) self.sess.run(tf.global_variables_initializer()) self.saver = tf.train.Saver(var_list=tf.global_variables(), max_to_keep=10) # summary 2 self.summary_writer = tf.summary.FileWriter(self.log_dir, self.sess.graph) pass
def conv_model_eval_metrics(classes, Y_, mode): return {'accuracy': metrics.accuracy(classes, Y_)} \ if mode == learn.ModeKeys.TRAIN or mode == learn.ModeKeys.EVAL else None
# In[3]: outputs, v_list = model_func(x_train_batch) for v in v_list: tf.add_to_collection('vars', v) tf.add_to_collection('vars', outputs) loss = tf.losses.sparse_softmax_cross_entropy(y_train_batch , outputs) optimizer = tf.train.AdamOptimizer(learning_rate=0.01) train_op = slim.learning.create_train_op(loss, optimizer) # In[4]: acc = accuracy(tf.cast(tf.arg_max(outputs,1),tf.int32) , y_train_batch) # In[5]: sess = tf.Session() sess.run(tf.global_variables_initializer()) # In[6]: _ = tf.train.start_queue_runners(sess=sess) # ## Train
def _build_network(self, input_width): with self.session.graph.as_default(): if self.rnn_cell_type == "LSTM": self.rnn_cell = tf.contrib.rnn.LSTMCell(self.rnn_cell_dim) elif self.rnn_cell_type == "GRU": self.rnn_cell = tf.contrib.rnn.GRUCell(self.rnn_cell_dim) else: raise ValueError("Unknown rnn_cell {}".format(rnn_cell)) self.global_step = tf.Variable(0, dtype=tf.int64, trainable=False, name='global_step') self.tokens = tf.placeholder(tf.int32, [None, None, None], name="tokens") self.token_lens = tf.placeholder(tf.int32, [None, None], name="token_lens") self.features = tf.placeholder(tf.float32, [None, None], name="features") self.labels = tf.placeholder(tf.int64, [None], name="labels") self.alphabet_size = len(self.char_vocabulary.classes_) self.dropout_keep = tf.placeholder(tf.float32) self.input_width = input_width char_embedding_matrix = tf.get_variable( "char_embeddings", [self.alphabet_size, self.EMBEDDING_SIZE], initializer=tf.random_normal_initializer(stddev=0.01), dtype=tf.float32) with tf.variable_scope("token_encoder"): tokens_flat = tf.reshape(self.tokens, [-1, tf.shape(self.tokens)[-1]]) token_lens_flat = tf.reshape(self.token_lens, [-1]) char_embeddings = tf.nn.embedding_lookup( char_embedding_matrix, tokens_flat) hidden_states, final_states = tf.nn.bidirectional_dynamic_rnn( cell_fw=self.rnn_cell, cell_bw=self.rnn_cell, inputs=char_embeddings, sequence_length=token_lens_flat, dtype=tf.float32, scope="char_BiRNN") tokens_encoded = tf_layers.linear(tf.concat(final_states, 1), self.EMBEDDING_SIZE, scope="tokens_encoded") tokens_encoded = tf.reshape(tokens_encoded, [tf.shape(self.features)[0], -1]) self.input_layer = tf.concat((tokens_encoded, self.features), 1) self.input_layer = tf.reshape(self.input_layer, [-1, self.input_width]) # input transform self.hidden_layer = tf.nn.dropout( tf_layers.fully_connected(self.input_layer, num_outputs=self.h_width, activation_fn=None, scope="input_layer"), self.dropout_keep) # hidden layers for i in range(self.h_depth): if self.layer_type == "FeedForward": self.hidden_layer = tf.nn.dropout( tf_layers.fully_connected( self.hidden_layer, num_outputs=self.h_width, activation_fn=tf.nn.relu, scope="ff_layer_{}".format(i)), self.dropout_keep) elif self.layer_type == "Highway": self.hidden_layer = tf.nn.dropout( highway_layer(self.hidden_layer, num_outputs=self.h_width, activation_fn=tf.nn.relu, scope="highway_layer_{}".format(i)), self.dropout_keep) else: raise ValueError("Unknown hidden layer type.") self.output_layer = tf_layers.fully_connected( self.hidden_layer, num_outputs=len(self.target_encoder.classes_), activation_fn=None, scope="output_layer") self.predictions = tf.argmax(self.output_layer, 1) self.loss = tf.reduce_mean( tf.nn.sparse_softmax_cross_entropy_with_logits( logits=self.output_layer, labels=self.labels), name="loss") self.training = tf.train.AdamOptimizer().minimize( self.loss, global_step=self.global_step) self.accuracy = tf_metrics.accuracy(self.predictions, self.labels) self.summary = tf.summary.merge([ tf.summary.scalar("train/loss", self.loss), tf.summary.scalar("train/accuracy", self.accuracy) ]) self._initialize_variables()
def conv_model_eval_metrics(classes, Y_, mode): # You can name the fields of your metrics dictionary as you like. return {'accuracy': metrics.accuracy(classes, Y_)} \ if mode == learn.ModeKeys.TRAIN or mode == learn.ModeKeys.EVAL else None
def __init__(self, rnn_cell, rnn_cell_dim, method, words, logdir, expname, threads=1, seed=42, character_count=50, embedding_matrix=None, max_sentence_length=None, embedding_dim=100, distinct_tags=2): # Create an empty graph and a session graph = tf.Graph() graph.seed = seed self.session = tf.Session(graph=graph, config=tf.ConfigProto( inter_op_parallelism_threads=threads, intra_op_parallelism_threads=threads)) timestamp = datetime.datetime.now().strftime("%Y-%m-%d_%H%M%S") # Construct the graph with self.session.graph.as_default(): if rnn_cell == "LSTM": rnn_cell = tf.contrib.rnn.LSTMCell(rnn_cell_dim) elif rnn_cell == "GRU": rnn_cell = tf.contrib.rnn.GRUCell(rnn_cell_dim) else: raise ValueError("Unknown rnn_cell {}".format(rnn_cell)) self.embedding_matrix = embedding_matrix self.global_step = tf.Variable(0, dtype=tf.int64, trainable=False, name="global_step") self.sentence_lens = tf.placeholder(tf.int32, [None]) self.forms = tf.placeholder(tf.int32, [None, None]) self.tags = tf.placeholder(tf.int64, [None]) self.char_ids = tf.placeholder(tf.int32, [None, None]) self.charseq_lens = tf.placeholder(tf.int32, [None]) self.charseqs = tf.placeholder(tf.int32, [None, max_sentence_length]) self.is_first = tf.placeholder(tf.bool) self.embedding_placeholder = tf.placeholder_with_default( tf.random_normal([words, embedding_dim]), [words, embedding_dim]) alpha = .3 if method == "learned_we": embeddings = tf.Variable(initial_value=tf.random_normal( [words, embedding_dim]), dtype=tf.float32) encoded_inputs = tf.nn.embedding_lookup(embeddings, ids=self.forms) elif method == "updated_pretrained_we": emb_W = tf.Variable(tf.constant(0.0, shape=[words, embedding_dim]), trainable=True, name="W") embeddings = cond(self.is_first, lambda: emb_W.assign(self.embedding_matrix), lambda: emb_W) encoded_inputs = tf.nn.embedding_lookup(embeddings, ids=self.forms) elif method == "only_pretrained_we": emb_W = tf.Variable(tf.constant(0.0, shape=[words, embedding_dim]), trainable=False, name="W") embeddings = cond(self.is_first, lambda: emb_W.assign(self.embedding_matrix), lambda: emb_W) encoded_inputs = tf.nn.embedding_lookup(embeddings, ids=self.forms) elif method == "char_rnn": embeddings = self.compute_embedding_rnn( self.charseqs, self.charseq_lens, character_count) encoded_inputs = tf.nn.embedding_lookup( embeddings, self.char_ids) elif method == "char_conv": embeddings = self.compute_embedding_conv( self.charseqs, character_count) encoded_inputs = tf.nn.embedding_lookup( embeddings, self.char_ids) with tf.variable_scope("RNN"): (outputs, states) = tf.nn.bidirectional_dynamic_rnn( rnn_cell, rnn_cell, encoded_inputs, sequence_length=self.sentence_lens, time_major=False, dtype=tf.float32) self.logits = tf_layers.linear(tf.concat(states, 1), distinct_tags) # self.logits = tf.Print(self.logits, [self.logits], message="This is a: ") self.labels = tf.one_hot(self.tags, distinct_tags, dtype=tf.int64) mask = tf.sequence_mask(self.sentence_lens, tf.reduce_max(self.sentence_lens)) logits = alpha * outputs[0] + (1 - alpha) * outputs[1] self.loss = tf.nn.sparse_softmax_cross_entropy_with_logits( logits=self.logits, labels=self.tags) self.optimizer = tf.train.AdamOptimizer(learning_rate=0.001) self.training = self.optimizer.minimize(self.loss) self.predictions = tf.argmax(self.logits, 1) self.accuracy = tf_metrics.accuracy(self.predictions, tf.argmax(self.labels, 1)) self.dataset_name = tf.placeholder(tf.string, []) # Initialize variables self.session.run(tf.initialize_all_variables())
def __init__(self, encoder, decoder, rnn_cell, rnn_cell_dim, chars_size, words_size, tags_size, bow_char, eow_char, logdir, expname, threads=1, seed=42): # Create an empty graph and a session graph = tf.Graph() graph.seed = seed self.session = tf.Session( graph=graph, config=tf.ConfigProto( inter_op_parallelism_threads=threads, intra_op_parallelism_threads=threads)) timestamp = datetime.datetime.now().strftime("%Y-%m-%d_%H%M%S") self.summary_writer = tf.summary.FileWriter("{}/{}-{}".format(logdir, timestamp, expname), flush_secs=10) # Construct the graph with self.session.graph.as_default(): if rnn_cell == "LSTM": rnn_cell = tf.contrib.rnn.LSTMCell(rnn_cell_dim) elif rnn_cell == "GRU": rnn_cell = tf.contrib.rnn.GRUCell(rnn_cell_dim) else: raise ValueError("Unknown rnn_cell {}".format(rnn_cell)) self.global_step = tf.Variable(0, dtype=tf.int64, trainable=False, name="global_step") self.sentence_lens = tf.placeholder(tf.int32, [None], name="sent_lens") self.lemma_ids = tf.placeholder(tf.int32, [None, None], name="lemma_ids") self.lemmas = tf.placeholder(tf.int64, [None, None], name="lemmas") self.lemma_lens = tf.placeholder(tf.int32, [None], name="lemma_lens") self.tag_ids = tf.placeholder(tf.int32, [None, None], name="tag_ids") self.tags = tf.placeholder(tf.int64, [None, None], name="tags") self.tag_lens = tf.placeholder(tf.int32, [None], name="tag_lens") self.form_ids = tf.placeholder(tf.int32, [None, None], name="form_ids") self.forms = tf.placeholder(tf.int64, [None, None], name="forms") self.form_lens = tf.placeholder(tf.int32, [None], name="form_lens") self.alphabet_len = chars_size self.word_vocab_len = words_size self.tag_vocab_len = tags_size self.dummy_inputs = tf.zeros([tf.shape(self.sentence_lens)[0], self.MAX_GEN_LEN], name="inference_shape") self.char_embedding_matrix = tf.get_variable( "char_embeddings", [self.alphabet_len, self.EMBEDDING_SIZE], initializer=tf.random_normal_initializer(stddev=0.01), dtype=tf.float32) self.we_lookup_matrix = tf.get_variable( "we_lookup_matrix", [self.word_vocab_len, self.EMBEDDING_SIZE], initializer=tf.random_normal_initializer(stddev=0.01), dtype=tf.float32, trainable=True) self.tag_lookup_matrix = tf.get_variable( "tag_lookup_matrix", [self.tag_vocab_len, self.EMBEDDING_SIZE], initializer=tf.random_normal_initializer(stddev=0.01), dtype=tf.float32, trainable=True) # Encode words with tf.variable_scope("encoder"): self.char_embeddings = tf.nn.embedding_lookup(self.char_embedding_matrix, self.lemmas) ch_rnn_cell = tf.contrib.rnn.GRUCell(rnn_cell_dim) hidden_states, final_states = tf.nn.bidirectional_dynamic_rnn( cell_fw=ch_rnn_cell, cell_bw=ch_rnn_cell, inputs=self.char_embeddings, sequence_length=self.lemma_lens, dtype=tf.float32, scope="char_BiRNN") self.sentence_mask = tf.sequence_mask(self.sentence_lens) # Create decoder input self.we_encoder_matrix = tf_layers.linear( tf.concat(axis=1, values=final_states), self.EMBEDDING_SIZE, scope="we_encoder_matrix") self.encoder_output = tf.nn.embedding_lookup(self.we_encoder_matrix, self.lemma_ids) self.encoder_output = tf.reshape( tf.boolean_mask(self.encoder_output, self.sentence_mask), [-1, self.EMBEDDING_SIZE], name="encoder_output_flat") # Encode tags self.tags_embedded = tf.nn.embedding_lookup(self.tag_lookup_matrix, self.tag_ids) self.tags_embedded = tf.reshape( tf.boolean_mask(self.tags_embedded, self.sentence_mask), [-1, self.EMBEDDING_SIZE], name="tag_embeddings_flat") # Combine encoder_output with tag embedding self.encoder_output = tf_layers.linear( tf.concat(axis=1, values=[self.encoder_output, self.tags_embedded]), self.EMBEDDING_SIZE, scope="encoder_output_with_tags") # Create annotations for attention self.annot_matrix = tf_layers.linear( tf.concat(axis=2, values=hidden_states), self.EMBEDDING_SIZE, scope="annot_matrix") self.annotations = tf.nn.embedding_lookup(self.annot_matrix, self.lemma_ids) self.annotations = tf.reshape( tf.boolean_mask(self.annotations, self.sentence_mask), [-1, tf.shape(self.annot_matrix)[1], self.EMBEDDING_SIZE], name="annotations_flat") # Reshape form values self.forms_flat = tf.nn.embedding_lookup(self.forms, self.form_ids) self.forms_flat = tf.reshape( tf.boolean_mask(self.forms_flat, self.sentence_mask), [-1, tf.shape(self.forms)[1]], name="forms_flat") self.forms_flat_lens = tf.nn.embedding_lookup(self.form_lens, self.form_ids) self.forms_flat_lens = tf.reshape( tf.boolean_mask(self.forms_flat_lens, self.sentence_mask), [-1], name="lemmas_flat_lens") self.attention_fn = None if decoder in ["individual", "individual_attention", "combined_attention", "combined_attention_birnn"]: if decoder in ["individual_attention", "combined_attention", "combined_attention_birnn"]: #self.attention_fn = self.attention_fn_builder(self.annotations) if decoder == "combined_attention": word_embeddings = tf.nn.embedding_lookup(self.we_lookup_matrix, self.lemma_ids) word_embeddings = tf.reshape( tf.boolean_mask(word_embeddings, self.sentence_mask), [-1, self.EMBEDDING_SIZE], name="word_embeddings_flat") self.encoder_output = tf_layers.linear( tf.concat(axis=1, values=[self.encoder_output, word_embeddings]), self.EMBEDDING_SIZE, scope="combined_encoder_output") if decoder == "combined_attention_rnn": else: raise ValueError("Unknown decoder ({}).".format(decoder)) # Decoder training with tf.variable_scope("decoder"): if decoder == "individual": self.training_logits, states = tf_seq2seq.rnn_decoder( decoder_inputs=self.forms_flat, initial_state=self.encoder_output, cell=rnn_cell) else: self.training_logits, states = tf_seq2seq.attention_decoder( decoder_inputs=self.forms_flat, initial_state=self.encoder_output, attention_states=self.annotations, cell=rnn_cell) #self.training_logits, states = tf_seq2seq.dynamic_rnn_decoder( #cell=rnn_cell, #decoder_fn=self.decoder_fn_train( # self.encoder_output, # self.output_fn_builder(), # self.input_fn_builder(self.char_embedding_matrix, self.attention_fn)), #inputs=tf.expand_dims(self.forms_flat, -1), #sequence_length=self.forms_flat_lens) # Decoder inference with tf.variable_scope("decoder", reuse=True): if decoder == "individual": self.training_logits, states = tf_seq2seq.rnn_decoder( decoder_inputs=self.dummy_inputs, initial_state=self.encoder_output, cell=rnn_cell, loop_function=decoder_fn) else: self.training_logits, states = tf_seq2seq.attention_decoder( decoder_inputs=self.dummy_inputs, initial_state=self.encoder_output, attention_states=self.annotations, cell=rnn_cell, loop_function=decoder_fn) #self.inference_logits, states = tf_seq2seq.dynamic_rnn_decoder( #cell=rnn_cell, #decoder_fn=self.decoder_fn_inference( # self.encoder_output, # self.output_fn_builder(), # self.input_fn_builder(self.char_embedding_matrix, self.attention_fn), #bow_char, #eow_char, #self.MAX_GEN_LEN)) self.predictions = tf.argmax(self.inference_logits, 2) loss = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(logits=self.training_logits, labels=self.forms_flat[:,1:])) self.training = tf.train.AdamOptimizer().minimize(loss, global_step=self.global_step) self.forms_flat = tf.cond( tf.reduce_max(self.forms_flat_lens) > self.MAX_GEN_LEN, lambda: tf.slice(self.forms_flat, [0, 0], [-1, self.MAX_GEN_LEN]), lambda: self.forms_flat) self.pred_padded = tf.pad( self.predictions, [[0,0],[0, self.MAX_GEN_LEN - tf.shape(self.predictions)[1]]], mode="CONSTANT") self.forms_padded = tf.pad( self.forms_flat, [[0,0],[0, self.MAX_GEN_LEN - tf.shape(self.forms_flat)[1] + 1]], mode="CONSTANT") self.char_accuracy = tf_metrics.accuracy(self.pred_padded, self.forms_padded[:,1:]) self.word_accuracy = tf.reduce_mean(tf.reduce_min(tf.cast(tf.equal(self.pred_padded, self.forms_padded[:,1:]), tf.float32), axis=1)) self.summary = {} for dataset_name in ["train", "dev"]: self.summary[dataset_name] = tf.summary.merge([tf.summary.scalar(dataset_name+"/loss", loss), tf.summary.scalar(dataset_name+"/char_accuracy", self.char_accuracy), tf.summary.scalar(dataset_name+"/word_accuracy", self.word_accuracy)]) # Initialize variables self.session.run(tf.global_variables_initializer()) if self.summary_writer: self.summary_writer.add_graph(self.session.graph) # Simple decoder for training def decoder_fn_train(self, encoder_state, output_fn, input_fn, name=None): def decoder_fn(time, cell_state, next_id, cell_output, context_state): cell_output = output_fn(cell_output) reuse = True if cell_state is None: # first call, return encoder_state cell_state = encoder_state reuse = None next_input = input_fn(tf.squeeze(next_id, [1]), cell_state, reuse) return (None, cell_state, next_input, cell_output, context_state) return decoder_fn # TODO: Beam search # Simple decoder for inference def decoder_fn_inference(self, encoder_state, output_fn, input_fn, beginning_of_word="<bow>", end_of_word="<eow>", maximum_length=MAX_GEN_LEN): batch_size = tf.shape(encoder_state)[0] def decoder_fn(time, cell_state, cell_input, cell_output, context_state): cell_output = output_fn(cell_output) if cell_state is None: cell_state = encoder_state next_id = tf.tile([beginning_of_word], [batch_size]) done = tf.zeros([batch_size], dtype=tf.bool) else: next_id = tf.argmax(cell_output, 1) done = tf.equal(next_id, end_of_word) done = tf.cond( tf.greater_equal(time, maximum_length), # return true if time >= maxlen lambda: tf.ones([batch_size], dtype=tf.bool), lambda: done) next_input = input_fn(next_id, cell_state, True) return (done, cell_state, next_input, cell_output, context_state) return decoder_fn def decoder_fn_builder(self, encoder_state, output_fn, input_fn, beginning_of_word="<bow>", end_of_word="<eow>", maximum_length=MAX_GEN_LEN): def decoder_fn(cell_output, i): cell_output = output_fn(cell_output) next_input = tf.argmax(cell_output, 1) next_input = input_fn(next_input) return decoder_fn # TODO: dropout def attention_fn_builder(self, annotations): def attention_fn(state): batch_size = tf.shape(state)[0] annot_len = tf.shape(annotations)[1] annot_dim = annotations.get_shape().as_list()[2] state_dim = state.get_shape().as_list()[1] e_dim = self.ALIGNMENT_SIZE a = tf.reshape(annotations, [-1, annot_dim]) U = tf.get_variable( "annot_weight", shape=[annot_dim, e_dim], initializer=tf.random_normal_initializer(stddev=0.1), trainable=True) U_b = tf.get_variable( "annot_bias", shape=[e_dim], initializer=tf.constant_initializer(0.1)) W = tf.get_variable( "state_weight", shape=[state_dim, e_dim], initializer=tf.random_normal_initializer(stddev=0.1), trainable=True) W_b = tf.get_variable( "state_bias", shape=[e_dim], initializer=tf.constant_initializer(0.1)) v = tf.get_variable( "lin_combo", shape=[e_dim, 1], initializer=tf.random_normal_initializer(stddev=0.1), trainable=True) w_res = tf.matmul(state, W) + W_b w_res = tf.tile(tf.reshape(w_res, [-1, 1]), [1, annot_len]) u_res = tf.matmul(a, U) + U_b u_res = tf.reshape(u_res, [-1, annot_len]) e = tf.matmul(tf.tanh(tf.reshape(w_res + u_res, [-1, e_dim])), v) e = tf.reshape(e, [batch_size, -1]) alpha = tf.nn.softmax(e) alpha = tf.tile(tf.reshape(alpha, [-1, 1]), [1, annot_dim]) c = tf.multiply(alpha, a) c = tf.reduce_sum(tf.reshape(c, [batch_size, -1, annot_dim]), 1) C = tf.get_variable( "attention_weight", shape=[state_dim, state_dim], initializer=tf.random_normal_initializer(stddev=0.1), trainable=True) C_b = tf.get_variable( "attention_bias", shape=[state_dim], initializer=tf.constant_initializer(0.1)) return tf.add(tf.matmul(c, C), C_b) return attention_fn # Output function builder (makes logits out of rnn outputs) def output_fn_builder(self): def output_fn(cell_output): if cell_output is None: return tf.zeros([self.alphabet_len], tf.float32) # only used for shape inference else: return tf_layers.linear( cell_output, num_outputs=self.alphabet_len, scope="decoder_output") return output_fn # Input function builder (makes rnn input from word id and cell state) def input_fn_builder(self, embeddings): def input_fn(next_id): return tf.nn.embedding_lookup(embeddings, next_id) return input_fn # Input function builder (makes rnn input from word id and cell state) #def input_fn_builder(self, embeddings, attention_fn=None): # def input_fn(next_id, cell_state, reuse=True): # if attention_fn is not None: # with tf.variable_scope("attention", reuse=reuse): # return tf.add( # tf.nn.embedding_lookup(embeddings, next_id), # attention_fn(cell_state)) # else: # return tf.nn.embedding_lookup(embeddings, next_id) # # return input_fn @property def training_step(self): return self.session.run(self.global_step) def train(self, sentence_lens, forms, form_ids, form_lens, tags, tag_ids, tag_lens, lemmas, lemma_ids, lemma_lens): try: _, summary, pred = self.session.run([self.training, self.summary, self.predictions], {self.sentence_lens: sentence_lens, self.forms: forms, self.form_ids: form_ids, self.form_lens: form_lens, self.tags: tags, self.tag_ids: tag_ids, self.tag_lens: tag_lens, self.lemmas: lemmas, self.lemma_ids: lemma_ids, self.lemma_lens: lemma_lens}) except Exception as e: import pdb; pdb.set_trace() raise e self.summary_writer.add_summary(summary["train"], self.training_step) def evaluate(self, sentence_lens, forms, form_ids, form_lens, tags, tag_ids, tag_lens, lemmas, lemma_ids, lemma_lens): try: ch_acc, w_acc, summary, pred = self.session.run([self.char_accuracy, self.word_accuracy, self.summary, self.predictions], {self.sentence_lens: sentence_lens, self.forms: forms, self.form_ids: form_ids, self.form_lens: form_lens, self.tags: tags, self.tag_ids: tag_ids, self.tag_lens: tag_lens, self.lemmas: lemmas, self.lemma_ids: lemma_ids, self.lemma_lens: lemma_lens}) except Exception as e: import pdb; pdb.set_trace() raise e self.summary_writer.add_summary(summary["dev"], self.training_step) return ch_acc, w_acc def predict(self, sentence_lens, lemmas, lemma_ids, lemma_lens, tags, tag_ids, tag_lens): predictions = self.session.run([self.predictions], {self.sentence_lens: sentence_lens, self.lemmas: lemmas, self.lemma_ids: lemma_ids, self.lemma_lens: lemma_lens, self.tags: tags, self.tag_ids: tag_ids, self.tag_lens: tag_lens}) return predictions
def __init__(self, batch_size, input_size, log_dir, data_root_path, train_list, data_path, annotation_path, class_path, model_name="model.ckpt", pretrain=None, is_test=False): # 和保存模型相关的参数 self.log_dir = Tools.new_dir(log_dir) self.model_name = model_name self.checkpoint_path = os.path.join(self.log_dir, self.model_name) self.pretrain = pretrain # 和数据相关的参数 self.input_size = input_size self.batch_size = batch_size self.num_classes = 21 # 和模型训练相关的参数 self.learning_rate = 5e-4 self.num_steps = 500001 self.print_step = 10 if is_test else 100 self.cal_step = 100 if is_test else 1000 # 读取数据 self.data_reader = Data(data_root_path=data_root_path, data_list=train_list, data_path=data_path, annotation_path=annotation_path, class_path=class_path, batch_size=self.batch_size, image_size=self.input_size, is_test=is_test) # 数据 self.image_placeholder = tf.placeholder(tf.float32, shape=(None, self.input_size[0], self.input_size[1], 3)) self.mask_placeholder = tf.placeholder(tf.float32, shape=(None, self.input_size[0], self.input_size[1], 1)) self.label_seg_placeholder = tf.placeholder(tf.int32, shape=(None, self.input_size[0], self.input_size[1], 1)) self.label_cls_placeholder = tf.placeholder(tf.int32, shape=(None,)) # 网络 self.net = BAISNet(self.image_placeholder, self.mask_placeholder, True, num_classes=self.num_classes) self.segments, self.attentions, self.classes = self.net.build() # loss self.loss, self.loss_segment_all, self.loss_class_all, self.loss_segments, self.loss_classes = self.cal_loss( self.segments, self.attentions, self.classes, self.label_seg_placeholder, self.label_cls_placeholder) # 当前批次的准确率:accuracy self.pred_classes = tf.cast(tf.argmax(self.classes[0], axis=-1), tf.int32) self.accuracy_classes = tcm.accuracy(self.pred_classes, self.label_cls_placeholder) with tf.name_scope("train"): # 学习率策略 self.step_ph = tf.placeholder(dtype=tf.float32, shape=()) self.learning_rate = tf.scalar_mul(tf.constant(self.learning_rate), tf.pow((1 - self.step_ph / self.num_steps), 0.9)) self.train_op = tf.train.GradientDescentOptimizer(self.learning_rate).minimize(self.loss) # 单独训练最后的attention attention_trainable = [v for v in tf.trainable_variables() if 'attention' in v.name or "class_attention" in v.name] print(len(attention_trainable)) self.train_attention_op = tf.train.GradientDescentOptimizer( self.learning_rate).minimize(self.loss, var_list=attention_trainable) pass # summary 1 with tf.name_scope("loss"): tf.summary.scalar("loss", self.loss) tf.summary.scalar("loss_segment", self.loss_segment_all) tf.summary.scalar("loss_class", self.loss_class_all) for loss_segment_index, loss_segment in enumerate(self.loss_segments): tf.summary.scalar("loss_segment_{}".format(loss_segment_index), loss_segment) for loss_class_index, loss_class in enumerate(self.loss_classes): tf.summary.scalar("loss_class_{}".format(loss_class_index), loss_class) pass with tf.name_scope("accuracy"): tf.summary.scalar("accuracy_classes", self.accuracy_classes) pass with tf.name_scope("label"): tf.summary.image("1-image", self.image_placeholder) tf.summary.image("2-segment", tf.cast(self.label_seg_placeholder * 255, dtype=tf.uint8)) tf.summary.image("3-mask", tf.cast(self.mask_placeholder * 255, dtype=tf.uint8)) pass with tf.name_scope("result"): # segment for segment_index, segment in enumerate(self.segments): segment = tf.split(segment, num_or_size_splits=2, axis=3) # tf.summary.image("predict-{}-0".format(segment_index), tf.nn.sigmoid(segment[0])) tf.summary.image("predict-{}-1".format(segment_index), tf.nn.sigmoid(segment[1])) # attention for attention_index, attention in enumerate(self.attentions): attention = tf.split(attention, num_or_size_splits=2, axis=3) # tf.summary.image("attention-{}-0".format(attention_index), tf.nn.sigmoid(attention[0])) tf.summary.image("attention-{}-1".format(attention_index), tf.nn.sigmoid(attention[1])) pass pass self.summary_op = tf.summary.merge_all() # sess 和 saver self.sess = tf.Session(config=tf.ConfigProto(gpu_options=tf.GPUOptions(allow_growth=True))) self.sess.run(tf.global_variables_initializer()) self.saver = tf.train.Saver(var_list=tf.global_variables(), max_to_keep=10) # summary 2 self.summary_writer = tf.summary.FileWriter(self.log_dir, self.sess.graph) pass
def __init__(self, rnn_cell, rnn_cell_dim, method, data_train, logdir, expname, threads, restore_path, seed=42): n_words = len(data_train.factors[data_train.FORMS]['words']) n_tags = len(data_train.factors[data_train.TAGS]['words']) n_lemmas = len(data_train.factors[data_train.LEMMAS]['words']) n_mwe = len(data_train.factors[data_train.MWE]['words']) # Create an empty graph and a session graph = tf.Graph() graph.seed = seed self.session = tf.Session(graph=graph, config=tf.ConfigProto( inter_op_parallelism_threads=threads, intra_op_parallelism_threads=threads)) timestamp = datetime.datetime.now().strftime("%Y-%m-%d_%H%M%S") self.summary_writer = tf.train.SummaryWriter("{}/{}-{}".format( logdir, timestamp, expname), flush_secs=10) # Construct the graph with self.session.graph.as_default(): if rnn_cell == "LSTM": rnn_cell = tf.nn.rnn_cell.LSTMCell(rnn_cell_dim) elif rnn_cell == "GRU": rnn_cell = tf.nn.rnn_cell.GRUCell(rnn_cell_dim) else: raise ValueError("Unknown rnn_cell {}".format(rnn_cell)) self.global_step = tf.Variable(0, dtype=tf.int64, trainable=False, name="global_step") self.sentence_lens = tf.placeholder(tf.int32, [None]) self.forms = tf.placeholder(tf.int32, [None, None]) self.tags = tf.placeholder(tf.int32, [None, None]) self.lemmas = tf.placeholder(tf.int32, [None, None]) self.mwe = tf.placeholder(tf.int32, [None, None]) embeddings_forms = tf.get_variable("word_embedding_matrix", [n_words, rnn_cell_dim], dtype=tf.float32) embeddings_lemmas = tf.get_variable("lemma_embedding_matrix", [n_lemmas, rnn_cell_dim], dtype=tf.float32) embeddings_tags = tf.get_variable("tag_embedding_matrix", [n_tags, rnn_cell_dim], dtype=tf.float32) embedding_in_forms = tf.nn.embedding_lookup( embeddings_forms, self.forms) embedding_in_lemmas = tf.nn.embedding_lookup( embeddings_lemmas, self.lemmas) embedding_in_tags = tf.nn.embedding_lookup(embeddings_tags, self.tags) #TODO concatenata tf.cvoncat emb_form emb_lemma emb_tags (zkusit udelat one-hot na tag - nebo zapomenout na to) embedding_in = tf.concat( 2, [embedding_in_forms, embedding_in_lemmas, embedding_in_tags ]) #mail Milanovi - je to tak??? self.input_keep_dropout = tf.placeholder_with_default( 1.0, None, name="input_keep") self.hidden_keep_dropout = tf.placeholder_with_default( 1.0, None, name="hidden_keep") hidden_layer = tf.nn.dropout(embedding_in, self.input_keep_dropout) bi_out, bi_out_states = tf.nn.bidirectional_dynamic_rnn( rnn_cell, rnn_cell, embedding_in, self.sentence_lens, dtype=tf.float32) # concatenate embedding layer1 = tf.concat(2, bi_out) layer = tf_layers.fully_connected(layer1, n_mwe, activation_fn=None, scope="output_layer") mask = tf.sequence_mask(self.sentence_lens) masked_mwe = tf.boolean_mask(self.mwe, mask) self.predictions = tf.cast(tf.argmax(layer, 2, name="predictions"), dtype=tf.int32) masked_predictions = tf.boolean_mask(self.predictions, mask) loss = tf_losses.sparse_softmax_cross_entropy(tf.boolean_mask( layer, mask), masked_mwe, scope="loss") self.training = tf.train.AdamOptimizer().minimize( loss, global_step=self.global_step) self.accuracy = tf_metrics.accuracy( masked_predictions, masked_mwe ) #vysledni scotre bude jiny !!! cely score na tech MWE self.dataset_name = tf.placeholder(tf.string, []) self.summary = tf.merge_summary([ tf.scalar_summary(self.dataset_name + "/loss", loss), tf.scalar_summary(self.dataset_name + "/accuracy", self.accuracy) ]) self.saver = tf.train.Saver(max_to_keep=None) # Initialize variables self.session.run(tf.initialize_all_variables()) if restore_path is not None: self.saver.restore(self.session, restore_path)