def polyphonic_rate(tensor, threshold=2):
"""Return the ratio of the number of time steps where the number of pitches
being played is larger than `threshold` to the total number of time steps"""
if tensor.get_shape().ndims != 5:
raise ValueError("Input tensor must have 5 dimensions.")
n_poly = tf.count_nonzero((tf.count_nonzero(tensor, 3) > threshold), 2)
return tf.reduce_mean((n_poly / tensor.get_shape()[2]), [0, 1])
def init_training_graph(self):
with tf.name_scope('Evaluation'):
logits = self.last
prob_b = tf.squeeze(logits, squeeze_dims=[1,2])
self.predictions = tf.argmax(prob_b, axis=1)
with tf.name_scope('Loss'):
self.loss = tf.reduce_mean((tf.nn.sparse_softmax_cross_entropy_with_logits(logits=prob_b,
labels=tf.cast(self.train_labels_node, tf.int32),
name="entropy")))
tf.summary.scalar("entropy", self.loss)
with tf.name_scope('Accuracy'):
LabelInt = tf.cast(self.train_labels_node, tf.int64)
CorrectPrediction = tf.equal(self.predictions, LabelInt)
self.accuracy = tf.reduce_mean(tf.cast(CorrectPrediction, tf.float32))
tf.summary.scalar("accuracy", self.accuracy)
with tf.name_scope('Prediction'):
self.TP = tf.count_nonzero(self.predictions * LabelInt)
self.TN = tf.count_nonzero((self.predictions - 1) * (LabelInt - 1))
self.FP = tf.count_nonzero(self.predictions * (LabelInt - 1))
self.FN = tf.count_nonzero((self.predictions - 1) * LabelInt)
with tf.name_scope('Precision'):
self.precision = tf.divide(self.TP, tf.add(self.TP, self.FP))
tf.summary.scalar('Precision', self.precision)
with tf.name_scope('Recall'):
self.recall = tf.divide(self.TP, tf.add(self.TP, self.FN))
tf.summary.scalar('Recall', self.recall)
with tf.name_scope('F1'):
num = tf.multiply(self.precision, self.recall)
dem = tf.add(self.precision, self.recall)
self.F1 = tf.scalar_mul(2, tf.divide(num, dem))
tf.summary.scalar('F1', self.F1)
with tf.name_scope('MeanAccuracy'):
Nprecision = tf.divide(self.TN, tf.add(self.TN, self.FN))
self.MeanAcc = tf.divide(tf.add(self.precision, Nprecision) ,2)
#self.batch = tf.Variable(0, name = "batch_iterator")
self.train_prediction = tf.nn.softmax(logits)
self.test_prediction = tf.nn.softmax(logits)
tf.global_variables_initializer().run()
print('Computational graph initialised')
def rpn_losses(anchor_labels, anchor_boxes, label_logits, box_logits):
"""
Args:
anchor_labels: fHxfWxNA
anchor_boxes: fHxfWxNAx4, encoded
label_logits: fHxfWxNA
box_logits: fHxfWxNAx4
Returns:
label_loss, box_loss
"""
with tf.device('/cpu:0'):
valid_mask = tf.stop_gradient(tf.not_equal(anchor_labels, -1))
pos_mask = tf.stop_gradient(tf.equal(anchor_labels, 1))
nr_valid = tf.stop_gradient(tf.count_nonzero(valid_mask, dtype=tf.int32), name='num_valid_anchor')
nr_pos = tf.count_nonzero(pos_mask, dtype=tf.int32, name='num_pos_anchor')
valid_anchor_labels = tf.boolean_mask(anchor_labels, valid_mask)
valid_label_logits = tf.boolean_mask(label_logits, valid_mask)
with tf.name_scope('label_metrics'):
valid_label_prob = tf.nn.sigmoid(valid_label_logits)
summaries = []
with tf.device('/cpu:0'):
for th in [0.5, 0.2, 0.1]:
valid_prediction = tf.cast(valid_label_prob > th, tf.int32)
nr_pos_prediction = tf.reduce_sum(valid_prediction, name='num_pos_prediction')
pos_prediction_corr = tf.count_nonzero(
tf.logical_and(
valid_label_prob > th,
tf.equal(valid_prediction, valid_anchor_labels)),
dtype=tf.int32)
summaries.append(tf.truediv(
pos_prediction_corr,
nr_pos, name='recall_th{}'.format(th)))
precision = tf.to_float(tf.truediv(pos_prediction_corr, nr_pos_prediction))
precision = tf.where(tf.equal(nr_pos_prediction, 0), 0.0, precision, name='precision_th{}'.format(th))
summaries.append(precision)
add_moving_summary(*summaries)
label_loss = tf.nn.sigmoid_cross_entropy_with_logits(
labels=tf.to_float(valid_anchor_labels), logits=valid_label_logits)
label_loss = tf.reduce_mean(label_loss, name='label_loss')
pos_anchor_boxes = tf.boolean_mask(anchor_boxes, pos_mask)
pos_box_logits = tf.boolean_mask(box_logits, pos_mask)
delta = 1.0 / 9
box_loss = tf.losses.huber_loss(
pos_anchor_boxes, pos_box_logits, delta=delta,
reduction=tf.losses.Reduction.SUM) / delta
box_loss = tf.div(
box_loss,
tf.cast(nr_valid, tf.float32), name='box_loss')
add_moving_summary(label_loss, box_loss, nr_valid, nr_pos)
return label_loss, box_loss
def build_graph(self):
file_pattern = os.path.join(self.params['data_dir'],
self.params['file_pattern'])
self.batched_dataset = _read_and_batch_from_files(
file_pattern=file_pattern,
batch_size=self.params['batch_size'],
max_length=self.params['max_length'],
num_cpu_cores=self.params.get('num_cpu_cores', 2),
shuffle=self.params['shuffle'],
repeat=self.params['repeat'],
num_workers=self._num_workers,
worker_id=self._worker_id)
self._iterator = self.batched_dataset.make_initializable_iterator()
x, y = self.iterator.get_next()
if self.params.get('m_padding', False):
# MAGIC PADDING
x = tf.cond(tf.equal(tf.shape(x)[1] % 8, 0),
true_fn = lambda: x,
false_fn = lambda: tf.pad(x,
paddings=[[0, 0],
[0, 8 - tf.shape(x)[1] % 8]]))
y = tf.cond(tf.equal(tf.shape(y)[1] % 8, 0),
true_fn = lambda: y,
false_fn = lambda: tf.pad(y,
paddings=[[0, 0],
[0, 8 - tf.shape(y)[1] % 8]]))
x = tf.cond(tf.equal(tf.shape(x)[0] % 8, 0),
true_fn = lambda: x,
false_fn = lambda: tf.pad(x,
paddings=[[0, 8 - tf.shape(x)[0] % 8],
[0, 0]]))
y = tf.cond(tf.equal(tf.shape(y)[0] % 8, 0),
true_fn=lambda: y,
false_fn=lambda: tf.pad(y,
paddings=[[0, 8 - tf.shape(y)[0] % 8],
[0, 0]]))
# ENDOF MAGIC PADDING
len_x = tf.count_nonzero(x, axis=1, dtype=tf.int32)
len_y = tf.count_nonzero(y, axis=1, dtype=tf.int32)
if self.params['mode'] == 'train' or self.params['mode'] == 'eval':
self._input_tensors['source_tensors'] = [x, len_x]
self._input_tensors['target_tensors'] = [y, len_y]
else:
self._input_tensors['source_tensors'] = [x, len_x]
def hard_negative_mining():
bboxes_per_batch = tf.unstack(bboxes)
classification_loss_per_batch = tf.unstack(classification_loss)
num_positives_per_batch = tf.unstack(tf.reduce_sum(positives, axis=-1))
neg_class_loss_per_batch = tf.unstack(neg_class_loss_all)
neg_class_losses = []
total_negatives = []
for bboxes_per_image, classification_loss_per_image, num_positives_per_image, neg_class_loss_per_image in \
zip(bboxes_per_batch, classification_loss_per_batch, num_positives_per_batch, neg_class_loss_per_batch):
min_negatives_keep = tf.maximum(self.neg_pos_ratio * num_positives_per_image, 3)
num_negatives_keep = tf.minimum(min_negatives_keep,
tf.count_nonzero(neg_class_loss_per_image, dtype=tf.float32))
indices = tf.image.non_max_suppression(bboxes_per_image, classification_loss_per_image,
tf.to_int32(num_negatives_keep), iou_threshold=0.99)
num_negatives = tf.size(indices)
total_negatives.append(num_negatives)
expanded_indexes = tf.expand_dims(indices, axis=1) # shape: (num_negatives, 1)
negatives_keep = tf.scatter_nd(expanded_indexes, updates=tf.ones_like(indices, dtype=tf.int32),
shape=tf.shape(classification_loss_per_image)) # shape: (num_priors,)
negatives_keep = tf.to_float(tf.reshape(negatives_keep, [num_priors])) # shape: (batch_size, num_priors)
neg_class_losses.append(tf.reduce_sum(classification_loss_per_image * negatives_keep, axis=-1)) # shape: (1,)
return tf.stack(neg_class_losses), tf.reduce_sum(tf.stack(total_negatives))
def _get_testing(rnn_logits,sequence_length,label,label_length):
"""Create ops for testing (all scalars):
loss: CTC loss function value,
label_error: Batch-normalized edit distance on beam search max
sequence_error: Batch-normalized sequence error rate
"""
with tf.name_scope("train"):
loss = model.ctc_loss_layer(rnn_logits,label,sequence_length)
with tf.name_scope("test"):
predictions,_ = tf.nn.ctc_beam_search_decoder(rnn_logits,
sequence_length,
beam_width=128,
top_paths=1,
merge_repeated=True)
hypothesis = tf.cast(predictions[0], tf.int32) # for edit_distance
label_errors = tf.edit_distance(hypothesis, label, normalize=False)
sequence_errors = tf.count_nonzero(label_errors,axis=0)
total_label_error = tf.reduce_sum( label_errors )
total_labels = tf.reduce_sum( label_length )
label_error = tf.truediv( total_label_error,
tf.cast(total_labels, tf.float32 ),
name='label_error')
sequence_error = tf.truediv( tf.cast( sequence_errors, tf.int32 ),
tf.shape(label_length)[0],
name='sequence_error')
tf.summary.scalar( 'loss', loss )
tf.summary.scalar( 'label_error', label_error )
tf.summary.scalar( 'sequence_error', sequence_error )
return loss, label_error, sequence_error
def buildGraph(self):
self.graph = tf.Graph()
with self.graph.as_default():
# train_input , [batch_size * embed_size] 一个batch有多条
self.train_input = tf.placeholder(tf.float32,shape=[self.batch_size,self.embed_size],name='train_input')
self.train_label = tf.placeholder(tf.int32,shape=[self.batch_size],name='train_label')
label_float = tf.cast(self.train_label,tf.float32)
# label_matrix = tf.Variable(tf.diag(tf.ones(self.label_size)),trainable=False)
label_matrix = tf.diag(tf.ones(self.label_size))
embed_label = tf.nn.embedding_lookup(label_matrix,self.train_label)
hidden_unit = 50
self.weight = tf.Variable(tf.truncated_normal(shape=[hidden_unit,self.embed_size],stddev=1.0/math.sqrt(self.embed_size)))
self.biase = tf.Variable(tf.zeros([hidden_unit]))
y1 = tf.matmul(self.train_input,self.weight,transpose_b=True) + self.biase
g1 = tf.nn.sigmoid(y1) # batch_size * label_size
weight2 = tf.Variable(tf.truncated_normal(shape=[self.label_size,hidden_unit],stddev=1.0/math.sqrt(hidden_unit)))
biase2 = tf.Variable(tf.zeros([self.label_size]))
y2 = tf.matmul(g1,weight2,transpose_b=True) + biase2
g2 = tf.nn.sigmoid(y2)
self.predict = tf.cast(tf.argmax(g2,axis=1),tf.float32)
self.error_num = tf.count_nonzero(label_float-self.predict)
self.loss = tf.reduce_mean(-tf.reduce_sum(embed_label*tf.log(g2+0.0001)+(1-embed_label)*tf.log(1+0.0001-g2),axis=1))
# self.train_op = tf.train.GradientDescentOptimizer(learning_rate=0.1).minimize(self.loss)
self.train_op = tf.train.AdagradOptimizer(learning_rate=1).minimize(self.loss)
self.init_op = tf.global_variables_initializer()
def _decode_and_resize(image_tensor):
"""Decodes jpeg string, resizes it and returns a uint8 tensor."""
# These constants are set by Inception v3's expectations.
height = 299
width = 299
channels = 3
image_tensor = tf.where(tf.equal(image_tensor, ''), IMAGE_DEFAULT_STRING, image_tensor)
# Fork by whether image_tensor value is a file path, or a base64 encoded string.
slash_positions = tf.equal(tf.string_split([image_tensor], delimiter="").values, '/')
is_file_path = tf.cast(tf.count_nonzero(slash_positions), tf.bool)
# The following two functions are required for tf.cond. Note that we can not replace them
# with lambda. According to TF docs, if using inline lambda, both branches of condition
# will be executed. The workaround is to use a function call.
def _read_file():
return tf.read_file(image_tensor)
def _decode_base64():
return tf.decode_base64(image_tensor)
image = tf.cond(is_file_path, lambda: _read_file(), lambda: _decode_base64())
image = tf.image.decode_jpeg(image, channels=channels)
image = tf.expand_dims(image, 0)
image = tf.image.resize_bilinear(image, [height, width], align_corners=False)
image = tf.squeeze(image, squeeze_dims=[0])
image = tf.cast(image, dtype=tf.uint8)
return image
def testSparseConstraint(self):
expected = [float(round(N * WEIGHT_SPARSITY))] * BATCH_SIZE
constraint = htm.constraints.Sparse(sparsity=WEIGHT_SPARSITY)
with self.test_session(config=CONFIG):
actual = constraint(tf.ones([BATCH_SIZE, N]))
tf.global_variables_initializer().run()
self.assertAllEqual(tf.count_nonzero(actual, axis=1).eval(), expected)
def calculate_reshape(original_shape, new_shape, validate=False, name=None):
"""Calculates the reshaped dimensions (replacing up to one -1 in reshape)."""
batch_shape_static = tensor_util.constant_value_as_shape(new_shape)
if batch_shape_static.is_fully_defined():
return np.int32(batch_shape_static.as_list()), batch_shape_static, []
with tf.name_scope(name, "calculate_reshape", [original_shape, new_shape]):
original_size = tf.reduce_prod(original_shape)
implicit_dim = tf.equal(new_shape, -1)
size_implicit_dim = (
original_size // tf.maximum(1, -tf.reduce_prod(new_shape)))
new_ndims = tf.shape(new_shape)
expanded_new_shape = tf.where( # Assumes exactly one `-1`.
implicit_dim, tf.fill(new_ndims, size_implicit_dim), new_shape)
validations = [] if not validate else [
tf.assert_rank(
original_shape, 1, message="Original shape must be a vector."),
tf.assert_rank(new_shape, 1, message="New shape must be a vector."),
tf.assert_less_equal(
tf.count_nonzero(implicit_dim, dtype=tf.int32),
1,
message="At most one dimension can be unknown."),
tf.assert_positive(
expanded_new_shape, message="Shape elements must be >=-1."),
tf.assert_equal(
tf.reduce_prod(expanded_new_shape),
original_size,
message="Shape sizes do not match."),
]
return expanded_new_shape, batch_shape_static, validations
def testDegenerate(self):
for use_gpu in False, True:
with self.test_session(use_gpu=use_gpu):
for dtype in (tf.bool,):
# A large number is needed to get Eigen to die
x = tf.zeros((0, 9938), dtype=dtype)
y = tf.count_nonzero(x, [0])
self.assertAllEqual(y.eval(), np.zeros(9938))
def buildGraph(self):
self.graph = tf.Graph()
with self.graph.as_default():
self.inputs = tf.placeholder(tf.float32,[self.batch_size,self.max_depth,self.embed_size]) # inputs: num_step * embed_size
self.seq_len = tf.placeholder(tf.int32)
self.label = tf.placeholder(tf.int32,[1])
label_float = tf.cast(self.label,tf.float32)
label_matrix = tf.diag(tf.ones(self.label_size))
embed_label = tf.nn.embedding_lookup(label_matrix,self.label)
print('pin2.1')
# input_list = list(tf.split(0,self.max_depth,expand_inputs))
input_list = tf.unpack(self.inputs,axis=1) # [[1,embed_size,]...,[1,embed_size]]
print('pin2.2')
# BasicRNNCell: [num_units, input_size, ...]
# self.rnn_cell = tf.nn.rnn_cell.BasicRNNCell(self.hidden_size,self.embed_size)
# self.rnn_cell = tf.nn.rnn_cell.LSTMCell(self.hidden_size,self.embed_size,state_is_tuple=True)
self.rnn_cell = LTMCell(self.hidden_size,self.embed_size,state_is_tuple=True)
self.rnn_cell = tf.nn.rnn_cell.DropoutWrapper(self.rnn_cell,output_keep_prob=0.9)
print('pin2.3')
init_stat = self.rnn_cell.zero_state(1,tf.float32)
output_embedding,states = tf.nn.rnn(self.rnn_cell,input_list,
initial_state=init_stat,
sequence_length=self.seq_len)
# state = init_stat
# states = []
# with tf.variable_scope('RNN'):
# for time_step in range(max_depth):
# if tf.equal(time_step,self.seq_len):
# break
# if time_step>0:
# tf.get_variable_scope().reuse_variables()
# m,state = self.rnn_cell(input_list[time_step,:],state)
# states.append(state)
# final_output = states[-1][0]
print('pin2.4')
final_output = states[-1] # final_output : [1,hidden_size]
print(final_output.get_shape())
weight = tf.Variable(tf.truncated_normal([self.label_size,self.hidden_size],
stddev=1.0/math.sqrt(self.hidden_size)))
biase = tf.Variable(tf.zeros([self.label_size]))
tmp_y = tf.matmul(final_output,weight,transpose_b=True) + biase
tmp_g = tf.sigmoid(tmp_y)
self.predict = tf.cast(tf.argmax(tmp_g,axis=1),tf.float32)
self.error_num = tf.count_nonzero(label_float-self.predict)
tiny_v = 0.0001
self.loss = -tf.reduce_mean(embed_label*tf.log(tmp_g+tiny_v) + (1-embed_label)*tf.log(1+tiny_v-tmp_g))
self.train_op = tf.train.AdagradOptimizer(learning_rate=1).minimize(self.loss)
self.init_op = tf.global_variables_initializer()
def contrastive_loss(left, right, y, margin, extra=False, scope="constrastive_loss"):
r"""Loss for Siamese networks as described in the paper:
`Learning a Similarity Metric Discriminatively, with Application to Face
Verification <http://yann.lecun.com/exdb/publis/pdf/chopra-05.pdf>`_ by Chopra et al.
.. math::
\frac{1}{2} [y \cdot d^2 + (1-y) \cdot \max(0, m - d)^2], d = \Vert l - r \Vert_2
Args:
left (tf.Tensor): left feature vectors of shape [Batch, N].
right (tf.Tensor): right feature vectors of shape [Batch, N].
y (tf.Tensor): binary labels of shape [Batch]. 1: similar, 0: not similar.
margin (float): horizon for negative examples (y==0).
extra (bool): also return distances for pos and neg.
Returns:
tf.Tensor: constrastive_loss (averaged over the batch), (and optionally average_pos_dist, average_neg_dist)
"""
with tf.name_scope(scope):
y = tf.cast(y, tf.float32)
delta = tf.reduce_sum(tf.square(left - right), 1)
delta_sqrt = tf.sqrt(delta + 1e-10)
match_loss = delta
missmatch_loss = tf.square(tf.nn.relu(margin - delta_sqrt))
loss = tf.reduce_mean(0.5 * (y * match_loss + (1 - y) * missmatch_loss))
if extra:
num_pos = tf.count_nonzero(y)
num_neg = tf.count_nonzero(1 - y)
pos_dist = tf.where(tf.equal(num_pos, 0), 0.,
tf.reduce_sum(y * delta_sqrt) / tf.cast(num_pos, tf.float32),
name="pos-dist")
neg_dist = tf.where(tf.equal(num_neg, 0), 0.,
tf.reduce_sum((1 - y) * delta_sqrt) / tf.cast(num_neg, tf.float32),
name="neg-dist")
return loss, pos_dist, neg_dist
else:
return loss
def _grad_sparsity(self):
"""Gradient sparsity."""
# If the sparse minibatch gradient has 10 percent of its entries
# non-zero, its sparsity is 0.1.
# The norm of dense gradient averaged from full dataset
# are roughly estimated norm of minibatch
# sparse gradient norm * sqrt(sparsity)
# An extension maybe only correct the sparse blob.
non_zero_cnt = tf.add_n([tf.count_nonzero(g) for g in self._grad])
all_entry_cnt = tf.add_n([tf.size(g) for g in self._grad])
self._sparsity = tf.cast(non_zero_cnt, self._grad[0].dtype)
self._sparsity /= tf.cast(all_entry_cnt, self._grad[0].dtype)
avg_op = self._moving_averager.apply([self._sparsity,])
with tf.control_dependencies([avg_op]):
self._sparsity_avg = self._moving_averager.average(self._sparsity)
return avg_op
def __init__(self, input_dim, lab_dim, learning_rate):
self.input_feature = tf.placeholder(tf.float32, [None, input_dim])
self.input_labels = tf.placeholder(tf.float32, [None, lab_dim])
self.w = tf.Variable(tf.random_normal([input_dim, lab_dim]), name="weight")
self.b = tf.Variable(tf.zeros([lab_dim]), name="bias")
self.output = tf.matmul(self.input_feature, self.w) + self.b
self.a1 = tf.argmax(tf.nn.softmax(self.output), axis=1)
self.b1 = tf.argmax(self.input_labels, axis=1)
self.err = tf.count_nonzero(self.a1 - self.b1)
cross_entropy = tf.nn.softmax_cross_entropy_with_logits(labels=self.input_labels, logits=self.output)
self.loss = tf.reduce_mean(cross_entropy)
optimizer = tf.train.AdamOptimizer(learning_rate)
self.train = optimizer.minimize(self.loss)
def _compare(self,
x,
reduction_axes,
keep_dims,
use_gpu=False,
feed_dict=None):
np_ans = (x != 0).astype(np.int32)
if reduction_axes is None:
np_ans = np.sum(np_ans, keepdims=keep_dims)
else:
reduction_axes = np.array(reduction_axes).astype(np.int32)
for ra in reduction_axes.ravel()[::-1]:
np_ans = np.sum(np_ans, axis=ra, keepdims=keep_dims)
with self.test_session(use_gpu=use_gpu) as sess:
tf_ans = tf.count_nonzero(x, reduction_axes, keep_dims)
out = sess.run(tf_ans, feed_dict)
self.assertAllClose(np_ans, out)
self.assertShapeEqual(np_ans, tf_ans)
def MaskedCrossEntropyLoss(outputs, targets, lengths=None, mask=None, max_len=None):
if lengths is None and mask is None:
raise RuntimeError('Please provide either lengths or mask')
#[batch_size, time_length]
if mask is None:
mask = sequence_mask(lengths, max_len, False)
#One hot encode targets (outputs.shape[-1] = hparams.quantize_channels)
targets_ = tf.one_hot(targets, depth=tf.shape(outputs)[-1])
with tf.control_dependencies([tf.assert_equal(tf.shape(outputs), tf.shape(targets_))]):
losses = tf.nn.softmax_cross_entropy_with_logits_v2(logits=outputs, labels=targets_)
with tf.control_dependencies([tf.assert_equal(tf.shape(mask), tf.shape(losses))]):
masked_loss = losses * mask
return tf.reduce_sum(masked_loss) / tf.count_nonzero(masked_loss, dtype=tf.float32)
def proposal_metrics(iou):
"""
Add summaries for RPN proposals.
Args:
iou: nxm, #proposal x #gt
"""
# find best roi for each gt, for summary only
best_iou = tf.reduce_max(iou, axis=0)
mean_best_iou = tf.reduce_mean(best_iou, name='best_iou_per_gt')
summaries = [mean_best_iou]
with tf.device('/cpu:0'):
for th in [0.3, 0.5]:
recall = tf.truediv(
tf.count_nonzero(best_iou >= th),
tf.size(best_iou, out_type=tf.int64),
name='recall_iou{}'.format(th))
summaries.append(recall)
add_moving_summary(*summaries)
def count_nonzero_wrapper(X, optype):
"""Wrapper for handling sparse and dense versions of `tf.count_nonzero`.
Parameters
----------
X : tf.Tensor (N, K)
optype : str, {'dense', 'sparse'}
Returns
-------
tf.Tensor (1,K)
"""
with tf.name_scope('count_nonzero_wrapper') as scope:
if optype == 'dense':
return tf.count_nonzero(X, axis=0, keep_dims=True)
elif optype == 'sparse':
indicator_X = tf.SparseTensor(X.indices, tf.ones_like(X.values), X.dense_shape)
return tf.sparse_reduce_sum(indicator_X, axis=0, keep_dims=True)
else:
raise NameError('Unknown input type in count_nonzero_wrapper')
def step(index, scores_sum, scores_num):
"""Single step."""
index %= hparams.epoch_length # Only needed in eval runs.
# Note - the only way to ensure making a copy of tensor is to run simple
# operation. We are waiting for tf.copy:
# https://github.com/tensorflow/tensorflow/issues/11186
obs_copy = batch_env.observ + 0
actor_critic = policy_factory(tf.expand_dims(obs_copy, 0))
policy = actor_critic.policy
action = tf.cond(eval_phase,
policy.mode,
policy.sample)
postprocessed_action = actor_critic.action_postprocessing(action)
simulate_output = batch_env.simulate(postprocessed_action[0, ...])
pdf = policy.prob(action)[0]
with tf.control_dependencies(simulate_output):
reward, done = simulate_output
done = tf.reshape(done, (len(batch_env),))
to_save = [obs_copy, reward, done, action[0, ...], pdf,
actor_critic.value[0]]
save_ops = [tf.scatter_update(memory_slot, index, value)
for memory_slot, value in zip(memory, to_save)]
cumulate_rewards_op = cumulative_rewards.assign_add(reward)
agent_indices_to_reset = tf.where(done)[:, 0]
with tf.control_dependencies([cumulate_rewards_op]):
scores_sum_delta = tf.reduce_sum(
tf.gather(cumulative_rewards, agent_indices_to_reset))
scores_num_delta = tf.count_nonzero(done, dtype=tf.int32)
with tf.control_dependencies(save_ops + [scores_sum_delta,
scores_num_delta]):
reset_env_op = batch_env.reset(agent_indices_to_reset)
reset_cumulative_rewards_op = tf.scatter_update(
cumulative_rewards, agent_indices_to_reset,
tf.zeros(tf.shape(agent_indices_to_reset)))
with tf.control_dependencies([reset_env_op,
reset_cumulative_rewards_op]):
return [index + 1, scores_sum + scores_sum_delta,
scores_num + scores_num_delta]
def MaskedSigmoidCrossEntropy(targets, outputs, targets_lengths, hparams, mask=None): '''Computes a masked SigmoidCrossEntropy with logits ''' #[batch_size, time_dimension] #example: #sequence_mask([1, 3, 2], 5) = [[1., 0., 0., 0., 0.], # [1., 1., 1., 0., 0.], # [1., 1., 0., 0., 0.]] #Note the maxlen argument that ensures mask shape is compatible with r>1 #This will by default mask the extra paddings caused by r>1 if mask is None: mask = sequence_mask(targets_lengths, hparams.outputs_per_step, False) with tf.control_dependencies([tf.assert_equal(tf.shape(targets), tf.shape(mask))]): #Use a weighted sigmoid cross entropy to measure the <stop_token> loss. Set hparams.cross_entropy_pos_weight to 1 #will have the same effect as vanilla tf.nn.sigmoid_cross_entropy_with_logits. losses = tf.nn.weighted_cross_entropy_with_logits(targets=targets, logits=outputs, pos_weight=hparams.cross_entropy_pos_weight) with tf.control_dependencies([tf.assert_equal(tf.shape(mask), tf.shape(losses))]): masked_loss = losses * mask return tf.reduce_sum(masked_loss) / tf.count_nonzero(masked_loss, dtype=tf.float32)
def build_graph(self,embedding):
self.graph = tf.Graph()
with self.graph.as_default():
self.batch_embed_input = tf.placeholder(tf.float32, shape=[self.batch_size, self.embed_size])
self.batch_label = tf.placeholder(tf.int32,shape=[self.batch_size])
batch_size = self.batch_label.get_shape()[0]
self.var_w = tf.Variable(
tf.truncated_normal([self.tag_num,self.embed_size],
stddev=1.0/math.sqrt(self.embed_size)),
dtype=tf.float32
)
self.var_bias = tf.Variable(tf.zeros([self.tag_num]),dtype=tf.float32)
# batch_embed_input : [batch_size, embed_size]
# var_w : [tag_num, embed_size]
# logits: [batch_size, tag_num]
logits = tf.matmul(self.batch_embed_input, self.var_w, transpose_b=True) + self.var_bias
logit_sigmoid = tf.sigmoid(logits)
self.predict = tf.to_int32(tf.argmax(logit_sigmoid,axis=1))
self.error_times = tf.count_nonzero(tf.subtract(self.predict,self.batch_label))
self.accuracy = (self.batch_size - self.error_times) / self.batch_size
self.loss = tf.nn.seq2seq.sequence_loss([logit_sigmoid],
[self.batch_label],
[tf.ones([batch_size])])
self.train_op = tf.train.GradientDescentOptimizer(learning_rate=5.0).minimize(self.loss)
self.init = tf.global_variables_initializer()
tf.scalar_summary('loss',self.loss)
tf.scalar_summary('accuracy',self.accuracy)
self.merge_summary_op = tf.merge_all_summaries()
def drum_in_pattern_rate(tensor):
"""Return the drum_in_pattern_rate metric value."""
if tensor.get_shape().ndims != 4:
raise ValueError("Input tensor must have 4 dimensions.")
def _drum_pattern_mask(n_timesteps, tolerance=0.1):
"""Return a drum pattern mask with the given tolerance."""
if n_timesteps not in (96, 48, 24, 72, 36, 64, 32, 16):
raise ValueError("Unsupported number of timesteps for the drum in "
"pattern metric.")
if n_timesteps == 96:
drum_pattern_mask = np.tile(
[1., tolerance, 0., 0., 0., tolerance], 16)
elif n_timesteps == 48:
drum_pattern_mask = np.tile([1., tolerance, tolerance], 16)
elif n_timesteps == 24:
drum_pattern_mask = np.tile([1., tolerance, tolerance], 8)
elif n_timesteps == 72:
drum_pattern_mask = np.tile(
[1., tolerance, 0., 0., 0., tolerance], 12)
elif n_timesteps == 36:
drum_pattern_mask = np.tile([1., tolerance, tolerance], 12)
elif n_timesteps == 64:
drum_pattern_mask = np.tile([1., tolerance, 0., tolerance], 16)
elif n_timesteps == 32:
drum_pattern_mask = np.tile([1., tolerance], 16)
elif n_timesteps == 16:
drum_pattern_mask = np.tile([1., tolerance], 8)
return drum_pattern_mask
drum_pattern_mask = _drum_pattern_mask(tensor.get_shape()[2])
drum_pattern_mask = tf.constant(
drum_pattern_mask.reshape(1, 1, tensor.get_shape()[2]), tf.float32)
n_in_pattern = tf.reduce_sum(drum_pattern_mask * tf.reduce_sum(tensor, 3))
n_notes = tf.count_nonzero(tensor, dtype=tf.float32)
return tf.cond((n_notes > 0), lambda: (n_in_pattern / n_notes), lambda: 0.)
def rpn_losses(anchor_labels, anchor_boxes, label_logits, box_logits):
"""
Args:
anchor_labels: fHxfWxNA
anchor_boxes: fHxfWxNAx4, encoded
label_logits: fHxfWxNA
box_logits: fHxfWxNAx4
Returns:
label_loss, box_loss
"""
with tf.device('/cpu:0'):
valid_mask = tf.stop_gradient(tf.not_equal(anchor_labels, -1))
pos_mask = tf.stop_gradient(tf.equal(anchor_labels, 1))
nr_valid = tf.stop_gradient(tf.count_nonzero(valid_mask, dtype=tf.int32), name='num_valid_anchor')
nr_pos = tf.identity(tf.count_nonzero(pos_mask, dtype=tf.int32), name='num_pos_anchor')
# nr_pos is guaranteed >0 in C4. But in FPN. even nr_valid could be 0.
valid_anchor_labels = tf.boolean_mask(anchor_labels, valid_mask)
valid_label_logits = tf.boolean_mask(label_logits, valid_mask)
with tf.name_scope('label_metrics'):
valid_label_prob = tf.nn.sigmoid(valid_label_logits)
summaries = []
with tf.device('/cpu:0'):
for th in [0.5, 0.2, 0.1]:
valid_prediction = tf.cast(valid_label_prob > th, tf.int32)
nr_pos_prediction = tf.reduce_sum(valid_prediction, name='num_pos_prediction')
pos_prediction_corr = tf.count_nonzero(
tf.logical_and(
valid_label_prob > th,
tf.equal(valid_prediction, valid_anchor_labels)),
dtype=tf.int32)
placeholder = 0.5 # A small value will make summaries appear lower.
recall = tf.to_float(tf.truediv(pos_prediction_corr, nr_pos))
recall = tf.where(tf.equal(nr_pos, 0), placeholder, recall, name='recall_th{}'.format(th))
precision = tf.to_float(tf.truediv(pos_prediction_corr, nr_pos_prediction))
precision = tf.where(tf.equal(nr_pos_prediction, 0),
placeholder, precision, name='precision_th{}'.format(th))
summaries.extend([precision, recall])
add_moving_summary(*summaries)
# Per-level loss summaries in FPN may appear lower due to the use of a small placeholder.
# But the total loss is still the same. TODO make the summary op smarter
placeholder = 0.
label_loss = tf.nn.sigmoid_cross_entropy_with_logits(
labels=tf.to_float(valid_anchor_labels), logits=valid_label_logits)
label_loss = tf.reduce_sum(label_loss) * (1. / cfg.RPN.BATCH_PER_IM)
label_loss = tf.where(tf.equal(nr_valid, 0), placeholder, label_loss, name='label_loss')
pos_anchor_boxes = tf.boolean_mask(anchor_boxes, pos_mask)
pos_box_logits = tf.boolean_mask(box_logits, pos_mask)
delta = 1.0 / 9
box_loss = tf.losses.huber_loss(
pos_anchor_boxes, pos_box_logits, delta=delta,
reduction=tf.losses.Reduction.SUM) / delta
box_loss = box_loss * (1. / cfg.RPN.BATCH_PER_IM)
box_loss = tf.where(tf.equal(nr_pos, 0), placeholder, box_loss, name='box_loss')
add_moving_summary(label_loss, box_loss, nr_valid, nr_pos)
return label_loss, box_loss
def loss(self, ground_truth, prediction, bboxes):
"""
Compute multibox loss.
Args:
ground_truth: Ground truth, shape: (?, #priors, 4 + #classes).
prediction: Dictionary of predicted tensors, shape: {'locs' : (?, #priors, 4), \
'confs' : (?, #priors, #classes), \
'logits': (?, #priors, #classes)}.
Returns:
Prediction loss, shape: (?,).
"""
with tf.variable_scope('loss_function'):
batch_size = tf.shape(prediction['locs'])[0]
num_priors = tf.shape(prediction['locs'])[1]
localization_loss = MultiboxLoss._localization_loss(ground_truth[:, :, :4],
prediction['locs']) # shape: (batch_size, num_priors)
classification_loss = MultiboxLoss._classification_loss(ground_truth[:, :, 4:],
prediction['logits']) # shape: (batch_size, num_priors)
ground_truth.set_shape([prediction['locs'].shape[0]] + ground_truth.shape[1:].as_list())
negatives = ground_truth[:, :, 4] # shape: (batch_size, num_priors)
positives = tf.reduce_max(ground_truth[:, :, 5:], axis=-1) # shape: (batch_size, num_priors)
pos_class_loss = tf.reduce_sum(classification_loss * positives, axis=-1) # shape: (batch_size,)
num_positives = tf.reduce_sum(positives) # shape: (1,)
neg_class_loss_all = classification_loss * negatives # shape: (batch_size, num_priors)
n_neg_losses = tf.count_nonzero(neg_class_loss_all, dtype=tf.float32) # shape: (1,)
def no_negatives():
return tf.zeros([batch_size], dtype=tf.float32), tf.constant(0, dtype=tf.int32)
def hard_negative_mining():
bboxes_per_batch = tf.unstack(bboxes)
classification_loss_per_batch = tf.unstack(classification_loss)
num_positives_per_batch = tf.unstack(tf.reduce_sum(positives, axis=-1))
neg_class_loss_per_batch = tf.unstack(neg_class_loss_all)
neg_class_losses = []
total_negatives = []
for bboxes_per_image, classification_loss_per_image, num_positives_per_image, neg_class_loss_per_image in \
zip(bboxes_per_batch, classification_loss_per_batch, num_positives_per_batch, neg_class_loss_per_batch):
min_negatives_keep = tf.maximum(self.neg_pos_ratio * num_positives_per_image, 3)
num_negatives_keep = tf.minimum(min_negatives_keep,
tf.count_nonzero(neg_class_loss_per_image, dtype=tf.float32))
indices = tf.image.non_max_suppression(bboxes_per_image, classification_loss_per_image,
tf.to_int32(num_negatives_keep), iou_threshold=0.99)
num_negatives = tf.size(indices)
total_negatives.append(num_negatives)
expanded_indexes = tf.expand_dims(indices, axis=1) # shape: (num_negatives, 1)
negatives_keep = tf.scatter_nd(expanded_indexes, updates=tf.ones_like(indices, dtype=tf.int32),
shape=tf.shape(classification_loss_per_image)) # shape: (num_priors,)
negatives_keep = tf.to_float(tf.reshape(negatives_keep, [num_priors])) # shape: (batch_size, num_priors)
neg_class_losses.append(tf.reduce_sum(classification_loss_per_image * negatives_keep, axis=-1)) # shape: (1,)
return tf.stack(neg_class_losses), tf.reduce_sum(tf.stack(total_negatives))
neg_class_loss, total_negatives = tf.cond(tf.equal(n_neg_losses, tf.constant(0.)),
no_negatives, hard_negative_mining) # shape: (batch_size,)
class_loss = pos_class_loss + neg_class_loss # shape: (batch_size,)
loc_loss = tf.reduce_sum(localization_loss * positives, axis=-1) # shape: (batch_size,)
total_loss = tf.reduce_sum(class_loss + self.loc_weight * loc_loss) / tf.maximum(1.0, num_positives)
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
if update_ops:
updates = tf.group(*update_ops)
total_loss = with_dependencies([updates], total_loss)
total_classification_loss = tf.reduce_mean(tf.reduce_sum(classification_loss, axis=-1))
total_localization_loss = tf.reduce_mean(loc_loss, axis=-1)
evaluation_tensors = {
'total_classification_loss': total_classification_loss,
'total_localization_loss': total_localization_loss,
'num_positives_per_batch': num_positives,
'num_negatives_per_batch': total_negatives
}
self.__add_evaluation(evaluation_tensors)
return total_loss
def xdet_model_fn(features, labels, mode, params):
"""Our model_fn for ResNet to be used with our Estimator."""
num_anchors_list = labels['num_anchors_list']
num_feature_layers = len(num_anchors_list)
shape = labels['targets'][-1]
if mode != tf.estimator.ModeKeys.TRAIN:
org_image = labels['targets'][-2]
isdifficult = labels['targets'][-3]
bbox_img = labels['targets'][-4]
gbboxes_raw = labels['targets'][-5]
glabels_raw = labels['targets'][-6]
glabels = labels['targets'][:num_feature_layers][0]
gtargets = labels['targets'][num_feature_layers:2 * num_feature_layers][0]
gscores = labels['targets'][2 * num_feature_layers:3 *
num_feature_layers][0]
with tf.variable_scope(params['model_scope'],
default_name=None,
values=[features],
reuse=tf.AUTO_REUSE):
backbone = xdet_body_v3.xdet_resnet_v3(params['resnet_size'],
params['data_format'])
body_cls_output, body_regress_output = backbone(
inputs=features, is_training=(mode == tf.estimator.ModeKeys.TRAIN))
cls_pred, location_pred = xdet_body_v3.xdet_head(
body_cls_output,
body_regress_output,
params['num_classes'],
num_anchors_list[0], (mode == tf.estimator.ModeKeys.TRAIN),
data_format=params['data_format'])
if params['data_format'] == 'channels_first':
cls_pred = tf.transpose(cls_pred, [0, 2, 3, 1])
location_pred = tf.transpose(location_pred, [0, 2, 3, 1])
#org_image = tf.transpose(org_image, [0, 2, 3, 1])
# batch size is 1
shape = tf.squeeze(shape, axis=0)
glabels = tf.squeeze(glabels, axis=0)
gtargets = tf.squeeze(gtargets, axis=0)
gscores = tf.squeeze(gscores, axis=0)
cls_pred = tf.squeeze(cls_pred, axis=0)
location_pred = tf.squeeze(location_pred, axis=0)
if mode != tf.estimator.ModeKeys.TRAIN:
org_image = tf.squeeze(org_image, axis=0)
isdifficult = tf.squeeze(isdifficult, axis=0)
gbboxes_raw = tf.squeeze(gbboxes_raw, axis=0)
glabels_raw = tf.squeeze(glabels_raw, axis=0)
bbox_img = tf.squeeze(bbox_img, axis=0)
bboxes_pred = labels['decode_fn'](
location_pred
) #(tf.reshape(location_pred, location_pred.get_shape().as_list()[:-1] + [-1, 4]))#(location_pred)#
eval_ops, save_image_op = bboxes_eval(org_image, shape, bbox_img, cls_pred,
bboxes_pred, glabels_raw,
gbboxes_raw, isdifficult,
params['num_classes'])
_ = tf.identity(save_image_op, name='save_image_with_bboxes_op')
cls_pred = tf.reshape(cls_pred, [-1, params['num_classes']])
location_pred = tf.reshape(location_pred, [-1, 4])
glabels = tf.reshape(glabels, [-1])
gscores = tf.reshape(gscores, [-1])
gtargets = tf.reshape(gtargets, [-1, 4])
# raw mask for positive > 0.5, and for negetive < 0.3
# each positive examples has one label
positive_mask = glabels > 0 #tf.logical_and(glabels > 0, gscores > params['match_threshold'])
fpositive_mask = tf.cast(positive_mask, tf.float32)
n_positives = tf.reduce_sum(fpositive_mask)
batch_glabels = tf.reshape(glabels, [tf.shape(features)[0], -1])
batch_n_positives = tf.count_nonzero(batch_glabels, -1)
batch_negtive_mask = tf.equal(batch_glabels, 0)
batch_n_negtives = tf.count_nonzero(batch_negtive_mask, -1)
batch_n_neg_select = tf.cast(
params['negative_ratio'] * tf.cast(batch_n_positives, tf.float32),
tf.int32)
batch_n_neg_select = tf.minimum(batch_n_neg_select,
tf.cast(batch_n_negtives, tf.int32))
# hard negative mining for classification
predictions_for_bg = tf.nn.softmax(
tf.reshape(cls_pred,
[tf.shape(features)[0], -1, params['num_classes']]))[:, :,
0]
prob_for_negtives = tf.where(
batch_negtive_mask,
0. - predictions_for_bg,
# ignore all the positives
0. - tf.ones_like(predictions_for_bg))
topk_prob_for_bg, _ = tf.nn.top_k(prob_for_negtives,
k=tf.shape(prob_for_negtives)[1])
score_at_k = tf.gather_nd(
topk_prob_for_bg,
tf.stack([tf.range(tf.shape(features)[0]), batch_n_neg_select - 1],
axis=-1))
selected_neg_mask = prob_for_negtives >= tf.expand_dims(score_at_k,
axis=-1)
negtive_mask = tf.reshape(
tf.logical_and(batch_negtive_mask, selected_neg_mask),
[-1]) #tf.logical_and(tf.equal(glabels, 0), gscores > 0.)
#negtive_mask = tf.logical_and(tf.logical_and(tf.logical_not(positive_mask), gscores < params['neg_threshold']), gscores > 0.)
#negtive_mask = tf.logical_and(gscores < params['neg_threshold'], tf.logical_not(positive_mask))
# # random select negtive examples for classification
# selected_neg_mask = tf.random_uniform(tf.shape(gscores), minval=0, maxval=1.) < tf.where(
# tf.greater(n_negtives, 0),
# tf.divide(tf.cast(n_neg_to_select, tf.float32), n_negtives),
# tf.zeros_like(tf.cast(n_neg_to_select, tf.float32)),
# name='rand_select_negtive')
# include both selected negtive and all positive examples
final_mask = tf.stop_gradient(tf.logical_or(negtive_mask, positive_mask))
total_examples = tf.reduce_sum(tf.cast(final_mask, tf.float32))
# add mask for glabels and cls_pred here
glabels = tf.boolean_mask(tf.clip_by_value(glabels, 0, FLAGS.num_classes),
tf.stop_gradient(final_mask))
cls_pred = tf.boolean_mask(cls_pred, tf.stop_gradient(final_mask))
location_pred = tf.boolean_mask(location_pred,
tf.stop_gradient(positive_mask))
gtargets = tf.boolean_mask(gtargets, tf.stop_gradient(positive_mask))
# Calculate loss, which includes softmax cross entropy and L2 regularization.
cross_entropy = tf.cond(
n_positives > 0., lambda: tf.losses.sparse_softmax_cross_entropy(
labels=glabels, logits=cls_pred), lambda: 0.)
#cross_entropy = tf.losses.sparse_softmax_cross_entropy(labels=glabels, logits=cls_pred)
# Create a tensor named cross_entropy for logging purposes.
tf.identity(cross_entropy, name='cross_entropy_loss')
tf.summary.scalar('cross_entropy_loss', cross_entropy)
loc_loss = tf.cond(
n_positives > 0., lambda: modified_smooth_l1(
location_pred, tf.stop_gradient(gtargets), sigma=1.),
lambda: tf.zeros_like(location_pred))
#loc_loss = modified_smooth_l1(location_pred, tf.stop_gradient(gtargets))
loc_loss = tf.reduce_mean(tf.reduce_sum(loc_loss, axis=-1))
loc_loss = tf.identity(loc_loss, name='location_loss')
tf.summary.scalar('location_loss', loc_loss)
tf.losses.add_loss(loc_loss)
with tf.control_dependencies([save_image_op]):
# Add weight decay to the loss. We exclude the batch norm variables because
# doing so leads to a small improvement in accuracy.
loss = cross_entropy + loc_loss + params['weight_decay'] * tf.add_n([
tf.nn.l2_loss(v) for v in tf.trainable_variables()
if 'batch_normalization' not in v.name
])
total_loss = tf.identity(loss, name='total_loss')
predictions = {
'classes':
tf.argmax(cls_pred, axis=-1),
'probabilities':
tf.reduce_max(tf.nn.softmax(cls_pred, name='softmax_tensor'), axis=-1),
'bboxes_predict':
tf.reshape(bboxes_pred, [-1, 4]),
'saved_image_index':
save_image_op
}
summary_hook = tf.train.SummarySaverHook(
save_secs=FLAGS.save_summary_steps,
output_dir=FLAGS.model_dir,
summary_op=tf.summary.merge_all())
if mode == tf.estimator.ModeKeys.EVAL:
return tf.estimator.EstimatorSpec(
mode=mode,
predictions=predictions,
evaluation_hooks=[summary_hook],
loss=loss,
eval_metric_ops=eval_ops) #=eval_ops)
else:
raise ValueError('This script only support predict mode!')
l2_distance = tf.sqrt( tf.reduce_sum(tf.square(tf.subtract(adjusted_correct_link_lengths, adjusted_predicted_link_lengths)), reduction_indices=1))
abs_diff_counts = tf.abs(correct_link_counts - predicted_link_counts)
wrong_link_counts = tf.cast(link_count_importance, tf.float32) * tf.cast(abs_diff_counts, tf.float32)
penalty = l2_distance + wrong_link_counts
total_penalty = tf.reduce_sum(penalty)
considered_correct = link_count_importance * 0.1
accurate_predictions = tf.cast(tf.less(penalty, considered_correct), tf.int32)
accuracy_ratio = tf.cast(tf.count_nonzero(accurate_predictions), tf.float32) / tf.cast(tf.shape(Y)[0], tf.float32)
with tf.Session() as sess: # create a session to evaluate the symbolic expressions
print "\ntrue label:\n", sess.run(correct_noop, feed_dict = {X: prediction, Y: epoch_y})
print "\nprediction:\n", sess.run(prediction_noop, feed_dict = {X: prediction, Y: epoch_y})
# print "\ntrue trans:\n", sess.run(correct_transpose, feed_dict = {X: prediction, Y: epoch_y})
# print "\nprediction trans:\n", sess.run(prediction_transpose, feed_dict = {X: prediction, Y: epoch_y})
print "\ntrue counts:\n", sess.run(correct_link_counts, feed_dict = {X: prediction, Y: epoch_y})
print "\nprediction counts:\n", sess.run(predicted_link_counts, feed_dict = {X: prediction, Y: epoch_y})
print "\nlink count differences:\n", sess.run(abs_diff_counts, feed_dict = {X: prediction, Y: epoch_y})
print "\nwrong links:\n", sess.run(wrong_link_counts, feed_dict = {X: prediction, Y: epoch_y})
def input_fn(params):
"""
WARNING:
This function contains many subtle hacks designed to ensure that the Unicode
decoding works and the strings are properly labelled with their country of origin.
Modify carefully, constantly checking the work with `print_data.py`
"""
import tensorflow as tf
with tf.device('/cpu:0'):
# create a trivial dataset
if params.trivial_data:
dataset = tf.data.Dataset.zip((
#tf.data.Dataset.from_tensors(tf.constant([0,0],dtype=tf.int64)),
#tf.data.Dataset.from_tensors(tf.constant([0],dtype=tf.int64)),
tf.data.Dataset.from_tensor_slices(tf.constant([[1,0],[1,1],[0,0],[0,1]],dtype=tf.int64)),
tf.data.Dataset.from_tensor_slices(tf.constant([[0],[1]],dtype=tf.int64)),
)).repeat()
dataset = dataset.batch(params.batch_size)
return dataset
# get input files
import os
files = [file for file in os.listdir(params.data) if os.path.isfile(os.path.join(params.data, file))]
filenames=[os.path.join(params.data,file) for file in files]
# get label information
import model.common
labels_names=model.common.data2labels(params.data)
labels_num=[len(axis) for axis in labels_names ]
labels_table=[tf.contrib.lookup.index_table_from_tensor(axis) for axis in labels_names]
num_axes=len(labels_names)
# load files into data pipeline
def filename2labels(filename):
basename=tf.string_split([filename],'/').values[-1]
tokens=tf.string_split([basename],'-').values
labels=[]
for i in range(num_axes):
labels.append(labels_table[i].lookup(tokens[i]))
labels=tf.stack(labels,axis=0)
return labels
dataset=tf.data.Dataset.from_tensor_slices(filenames)
dataset=dataset.interleave(
lambda x: tf.data.Dataset.zip((
tf.data.TextLineDataset([x]),
tf.data.Dataset.from_tensors(filename2labels(x)).repeat(),
)),
cycle_length=len(filenames),
block_length=1
)
dataset = dataset.shuffle(params.shuffle_lines,seed=params.shuffle_seed)
# create variables/summaries for monitoring progress
macro_input_variables(params)
tf.summary.scalar('num_lines',num_lines,family='progress')
tf.summary.scalar('num_words',num_words,family='progress')
tf.summary.scalar('num_words_filtered',num_words_filtered,family='progress')
filtered_ratio=tf.cast(num_words_filtered,tf.float32)/tf.cast(num_words,tf.float32)
tf.summary.scalar('filtered_ratio',filtered_ratio,family='progress')
for i in [1,10,100,1000,10000,100000]:
tf.summary.scalar(
'words_seen_'+str(i),
tf.count_nonzero(tf.maximum(tf.zeros([],dtype=tf.int64),word_counts-i+1)),
family='progress',
)
word_freq=tf.cast(word_counts,tf.float32)/tf.cast(1+num_words,tf.float32)
tf.summary.histogram(
'word_freq',
word_freq,
family='progress'
)
# convert string into skipgram input
SKIPGRAM_PAD=-2
#SKIPGRAM_PAD='PAD'
def line2wordpairs(line,label):
global word_counts
global word_counts_axis
global num_lines
global num_words
global num_words_axis
global num_words_filtered
# tokenize input line
tokens=tf.string_split([line],' ').values
tokens_ids=vocab_index.lookup(tokens)
# filter tokens_ids that have been seen too frequently
if params.filtering=='none' or params.word_counts=='none':
tokens_filtered=tokens_ids
else:
t=params.threshold_base/params.vocab_size
if params.filtering=='global' or params.word_counts=='fast':
numerator=tf.gather(word_counts,tokens_ids)
denominator=num_words
freq=tf.cast(numerator,tf.float32)/tf.cast(denominator,tf.float32)
elif params.filtering=='axis':
t=params.threshold_base/params.vocab_size
freqs=[]
for axis in range(num_axes):
numerator=tf.gather(word_counts_axis[axis][:,label[axis]],tokens_ids)
denominator=num_words_axis[axis][label[axis]]
freqs.append(tf.cast(numerator,tf.float32)/tf.cast(denominator,tf.float32))
freq=tf.reduce_min(freqs)
keep_prob=tf.sqrt(t/freq)
rand=tf.random_uniform([tf.size(tokens_ids)],seed=params.shuffle_seed)
if params.rm_top<0:
filter_mask=tf.greater(keep_prob,rand)
else:
filter_mask=tf.logical_and(
tf.greater(keep_prob,rand),
tf.greater(tokens_ids,params.rm_top),
)
tokens_filtered=tf.boolean_mask(tokens_ids,filter_mask)
# convert tokens into dp_type
if params.dp_type=='sentence':
words=tf.expand_dims(tokens_filtered,axis=0)
context=tf.expand_dims(tokens_filtered,axis=0)
raise ValueError('dp_type==sentence not yet implemented')
elif params.dp_type=='word_context':
words=tf.expand_dims(tokens_filtered,axis=1)
context=tf.stack([
tf.manip.roll(tokens_filtered,i,0)
for i in
list(range(-params.context_size,0))+list(range(1,params.context_size+1))
],axis=1)
elif params.dp_type=='word_pair':
tokens_padded=tf.pad(
tokens_filtered,
tf.constant([[params.context_size,params.context_size]],shape=[1,2]),
constant_values=SKIPGRAM_PAD
)
context=tf.concat([
tf.manip.roll(tokens_padded,i,0)
for i in
list(range(-params.context_size,0))+list(range(1,params.context_size+1))
],axis=0)
words=tf.concat([tokens_padded for i in range(params.context_size+params.context_size)],axis=0)
# filter wordpairs containing SKIPGRAM_PAD
ids=tf.logical_and(
tf.not_equal(context,SKIPGRAM_PAD),
tf.not_equal(words,SKIPGRAM_PAD),
)
context=tf.expand_dims(tf.boolean_mask(context,ids),axis=1)
words=tf.expand_dims(tf.boolean_mask(words,ids),axis=1)
# create negative samples
nce_samples=min(params.nce_samples,vocab_size)
def candidate_sampler(pos,pos_size):
if params.sampler=='log_uniform':
sampled_values=tf.nn.log_uniform_candidate_sampler(
true_classes=pos,
num_true=pos_size,
num_sampled=nce_samples,
unique=False,
range_max=vocab_size,
)
elif params.sampler=='fixed_unigram':
sampled_values=tf.nn.fixed_unigram_candidate_sampler(
true_classes=pos,
num_true=pos_size,
num_sampled=nce_samples,
unique=False,
range_max=vocab_size,
unigrams=[x+1 for x in vocab_counts],
)
return sampled_values
if params.reuse_samples=='true':
pos=context
pos_size=pos.get_shape()[1]
sampled_values=candidate_sampler(pos,pos_size)
samples, true_expected_count, sampled_expected_count = (
tf.stop_gradient(s) for s in sampled_values)
samples=tf.tile(tf.expand_dims(samples,dim=0),[tf.shape(words)[0],1])
sampled_expected_count=tf.tile(tf.expand_dims(sampled_expected_count,dim=0),[tf.shape(words)[0],1])
else:
words_size=tf.shape(words)[0]
context_size=context.get_shape()[1]
def body(a,b,c):
a_size=tf.shape(a)[0]
pos=context[a_size:a_size+1,:]
pos_size=pos.get_shape()[1]
sampled_values=candidate_sampler(pos,pos_size)
vocab_sampled, true_expected_count, sampled_expected_count = sampled_values
return [
tf.concat([a,tf.expand_dims(vocab_sampled,axis=0)],axis=0),
tf.concat([b,true_expected_count],axis=0),
tf.concat([c,tf.expand_dims(sampled_expected_count,axis=0)],axis=0),
]
sampled_values = tf.while_loop(
lambda a,b,c: tf.less(tf.shape(a)[0],words_size),
body,
loop_vars=[
tf.zeros([0,nce_samples],dtype=tf.int64),
tf.zeros([0,context_size],dtype=tf.float32),
tf.zeros([0,nce_samples],dtype=tf.float32),
],
shape_invariants=[
tf.TensorShape([None,nce_samples]),
tf.TensorShape([None,context_size]),
tf.TensorShape([None,nce_samples]),
]
)
samples, true_expected_count, sampled_expected_count = (
tf.stop_gradient(s) for s in sampled_values)
allsamples=tf.concat([context,samples],axis=1)
expected_count=tf.concat([true_expected_count,sampled_expected_count],axis=1)
# update all variable counters
if params.word_counts=='none':
update_ops=[]
else:
update_num_lines=tf.assign_add(num_lines,1)
update_num_words=tf.assign_add(
num_words,
tf.size(tokens,out_type=tf.int64)
)
update_num_words_filtered=tf.assign_add(
num_words_filtered,
tf.size(tokens_filtered,out_type=tf.int64)
)
update_word_counts=tf.scatter_update(
word_counts,
tokens_ids,
tf.gather(word_counts,tokens_ids)+1
)
update_axis=[]
if params.word_counts=='all':
for axis in range(num_axes):
label_tiled=tf.tile(
tf.reshape(label[axis],[1]),
[tf.size(tokens_ids)],
)
indices=tf.stack([tokens_ids,label_tiled],axis=1)
word_counts_axis_old=tf.gather_nd(word_counts_axis[axis],indices)
update_axis.append(tf.scatter_nd_update(
word_counts_axis[axis],
indices,
word_counts_axis_old+1
))
update_axis.append(tf.scatter_add(
num_words_axis[axis],
tf.expand_dims(label[axis],axis=0),
tf.size(tokens,out_type=tf.int64)
))
update_ops=[update_num_lines,update_num_words,update_word_counts]+update_axis
with tf.control_dependencies(update_ops):
labels=tf.tile(tf.expand_dims(label,axis=0),[tf.shape(words)[0],1])
features={
'words':words,
'allsamples':allsamples,
'labels':labels,
}
# adding expected_count to the features dict causes a slowdown,
# so only do it if it will be used by the estimator
if params.sample_method=='nce':
features['expected_count']=expected_count
return features
dataset=dataset.map(line2wordpairs,num_parallel_calls=params.parallel_map)
dataset=dataset.flat_map(lambda *xs: tf.data.Dataset.zip(tuple(tf.data.Dataset.from_tensor_slices(x) for x in xs)))
dataset=tf.data.Dataset.zip((dataset,tf.data.Dataset.from_tensors(1).repeat()))
# finalize dataset
dataset=dataset.shuffle(params.shuffle_words,seed=params.shuffle_seed)
dataset=dataset.batch(params.batch_size)
dataset=dataset.repeat()
dataset=dataset.prefetch(1)
return dataset
def create_low_latency_svdf_model(fingerprint_input, model_settings,
is_training, runtime_settings):
"""Builds an SVDF model with low compute requirements.
This is based in the topology presented in the 'Compressing Deep Neural
Networks using a Rank-Constrained Topology' paper:
https://static.googleusercontent.com/media/research.google.com/en//pubs/archive/43813.pdf
Here's the layout of the graph:
(fingerprint_input)
v
[SVDF]<-(weights)
v
[BiasAdd]<-(bias)
v
[Relu]
v
[MatMul]<-(weights)
v
[BiasAdd]<-(bias)
v
[MatMul]<-(weights)
v
[BiasAdd]<-(bias)
v
[MatMul]<-(weights)
v
[BiasAdd]<-(bias)
v
This model produces lower recognition accuracy than the 'conv' model above,
but requires fewer weight parameters and, significantly fewer computations.
During training, dropout nodes are introduced after the relu, controlled by a
placeholder.
Args:
fingerprint_input: TensorFlow node that will output audio feature vectors.
The node is expected to produce a 2D Tensor of shape:
[batch, model_settings['dct_coefficient_count'] *
model_settings['spectrogram_length']]
with the features corresponding to the same time slot arranged contiguously,
and the oldest slot at index [:, 0], and newest at [:, -1].
model_settings: Dictionary of information about the model.
is_training: Whether the model is going to be used for training.
runtime_settings: Dictionary of information about the runtime.
Returns:
TensorFlow node outputting logits results, and optionally a dropout
placeholder.
Raises:
ValueError: If the inputs tensor is incorrectly shaped.
"""
if is_training:
dropout_prob = tf.placeholder(tf.float32, name='dropout_prob')
input_frequency_size = model_settings['dct_coefficient_count']
input_time_size = model_settings['spectrogram_length']
# Validation.
input_shape = fingerprint_input.get_shape()
if len(input_shape) != 2:
raise ValueError('Inputs to `SVDF` should have rank == 2.')
if input_shape[-1].value is None:
raise ValueError('The last dimension of the inputs to `SVDF` '
'should be defined. Found `None`.')
if input_shape[-1].value % input_frequency_size != 0:
raise ValueError(
'Inputs feature dimension %d must be a multiple of '
'frame size %d', fingerprint_input.shape[-1].value,
input_frequency_size)
# Set number of units (i.e. nodes) and rank.
rank = 2
num_units = 1280
# Number of filters: pairs of feature and time filters.
num_filters = rank * num_units
# Create the runtime memory: [num_filters, batch, input_time_size]
batch = 1
memory = tf.Variable(tf.zeros([num_filters, batch, input_time_size]),
trainable=False,
name='runtime-memory')
# Determine the number of new frames in the input, such that we only operate
# on those. For training we do not use the memory, and thus use all frames
# provided in the input.
# new_fingerprint_input: [batch, num_new_frames*input_frequency_size]
if is_training:
num_new_frames = input_time_size
else:
window_stride_ms = int(model_settings['window_stride_samples'] * 1000 /
model_settings['sample_rate'])
num_new_frames = tf.cond(
tf.equal(tf.count_nonzero(memory), 0), lambda: input_time_size,
lambda: int(runtime_settings['clip_stride_ms'] / window_stride_ms))
new_fingerprint_input = fingerprint_input[:, -num_new_frames *
input_frequency_size:]
# Expand to add input channels dimension.
new_fingerprint_input = tf.expand_dims(new_fingerprint_input, 2)
# Create the frequency filters.
weights_frequency = tf.Variable(
tf.truncated_normal([input_frequency_size, num_filters], stddev=0.01))
# Expand to add input channels dimensions.
# weights_frequency: [input_frequency_size, 1, num_filters]
weights_frequency = tf.expand_dims(weights_frequency, 1)
# Convolve the 1D feature filters sliding over the time dimension.
# activations_time: [batch, num_new_frames, num_filters]
activations_time = tf.nn.conv1d(new_fingerprint_input, weights_frequency,
input_frequency_size, 'VALID')
# Rearrange such that we can perform the batched matmul.
# activations_time: [num_filters, batch, num_new_frames]
activations_time = tf.transpose(activations_time, perm=[2, 0, 1])
# Runtime memory optimization.
if not is_training:
# We need to drop the activations corresponding to the oldest frames, and
# then add those corresponding to the new frames.
new_memory = memory[:, :, num_new_frames:]
new_memory = tf.concat([new_memory, activations_time], 2)
tf.assign(memory, new_memory)
activations_time = new_memory
# Create the time filters.
weights_time = tf.Variable(
tf.truncated_normal([num_filters, input_time_size], stddev=0.01))
# Apply the time filter on the outputs of the feature filters.
# weights_time: [num_filters, input_time_size, 1]
# outputs: [num_filters, batch, 1]
weights_time = tf.expand_dims(weights_time, 2)
outputs = tf.matmul(activations_time, weights_time)
# Split num_units and rank into separate dimensions (the remaining
# dimension is the input_shape[0] -i.e. batch size). This also squeezes
# the last dimension, since it's not used.
# [num_filters, batch, 1] => [num_units, rank, batch]
outputs = tf.reshape(outputs, [num_units, rank, -1])
# Sum the rank outputs per unit => [num_units, batch].
units_output = tf.reduce_sum(outputs, axis=1)
# Transpose to shape [batch, num_units]
units_output = tf.transpose(units_output)
# Appy bias.
bias = tf.Variable(tf.zeros([num_units]))
first_bias = tf.nn.bias_add(units_output, bias)
# Relu.
first_relu = tf.nn.relu(first_bias)
if is_training:
first_dropout = tf.nn.dropout(first_relu, dropout_prob)
else:
first_dropout = first_relu
first_fc_output_channels = 256
first_fc_weights = tf.Variable(
tf.truncated_normal([num_units, first_fc_output_channels],
stddev=0.01))
first_fc_bias = tf.Variable(tf.zeros([first_fc_output_channels]))
first_fc = tf.matmul(first_dropout, first_fc_weights) + first_fc_bias
if is_training:
second_fc_input = tf.nn.dropout(first_fc, dropout_prob)
else:
second_fc_input = first_fc
second_fc_output_channels = 256
second_fc_weights = tf.Variable(
tf.truncated_normal(
[first_fc_output_channels, second_fc_output_channels],
stddev=0.01))
second_fc_bias = tf.Variable(tf.zeros([second_fc_output_channels]))
second_fc = tf.matmul(second_fc_input, second_fc_weights) + second_fc_bias
if is_training:
final_fc_input = tf.nn.dropout(second_fc, dropout_prob)
else:
final_fc_input = second_fc
label_count = model_settings['label_count']
final_fc_weights = tf.Variable(
tf.truncated_normal([second_fc_output_channels, label_count],
stddev=0.01))
final_fc_bias = tf.Variable(tf.zeros([label_count]))
final_fc = tf.matmul(final_fc_input, final_fc_weights) + final_fc_bias
if is_training:
return final_fc, dropout_prob
else:
return final_fc
def ssd_model_fn(features, labels, mode, params):
"""model_fn for SSD to be used with our Estimator."""
shape = labels['shape']
loc_targets = labels['loc_targets']
cls_targets = labels['cls_targets']
match_scores = labels['match_scores']
global global_anchor_info
decode_fn = global_anchor_info['decode_fn']
num_anchors_per_layer = global_anchor_info['num_anchors_per_layer']
all_num_anchors_depth = global_anchor_info['all_num_anchors_depth']
# bboxes_pred = decode_fn(loc_targets[0])
# bboxes_pred = [tf.reshape(preds, [-1, 4]) for preds in bboxes_pred]
# bboxes_pred = tf.concat(bboxes_pred, axis=0)
# save_image_op = tf.py_func(save_image_with_bbox,
# [ssd_preprocessing.unwhiten_image(features[0]),
# tf.clip_by_value(cls_targets[0], 0, tf.int64.max),
# match_scores[0],
# bboxes_pred],
# tf.int64, stateful=True)
# with tf.control_dependencies([save_image_op]):
#print(all_num_anchors_depth)
with tf.variable_scope(params['model_scope'],
default_name=None,
values=[features],
reuse=tf.AUTO_REUSE):
backbone = ssd_net.VGG16Backbone(params['data_format'])
feature_layers = backbone.forward(
features, training=(mode == tf.estimator.ModeKeys.TRAIN))
#print(feature_layers)
location_pred, cls_pred = ssd_net.multibox_head(
feature_layers,
params['num_classes'],
all_num_anchors_depth,
data_format=params['data_format'])
if params['data_format'] == 'channels_first':
cls_pred = [tf.transpose(pred, [0, 2, 3, 1]) for pred in cls_pred]
location_pred = [
tf.transpose(pred, [0, 2, 3, 1]) for pred in location_pred
]
cls_pred = [
tf.reshape(pred,
[tf.shape(features)[0], -1, params['num_classes']])
for pred in cls_pred
]
location_pred = [
tf.reshape(pred, [tf.shape(features)[0], -1, 4])
for pred in location_pred
]
cls_pred = tf.concat(cls_pred, axis=1)
location_pred = tf.concat(location_pred, axis=1)
cls_pred = tf.reshape(cls_pred, [-1, params['num_classes']])
location_pred = tf.reshape(location_pred, [-1, 4])
with tf.device('/cpu:0'):
with tf.control_dependencies([cls_pred, location_pred]):
with tf.name_scope('post_forward'):
#bboxes_pred = decode_fn(location_pred)
bboxes_pred = tf.map_fn(
lambda _preds: decode_fn(_preds),
tf.reshape(location_pred, [tf.shape(features)[0], -1, 4]),
dtype=[tf.float32] * len(num_anchors_per_layer),
back_prop=False)
#cls_targets = tf.Print(cls_targets, [tf.shape(bboxes_pred[0]),tf.shape(bboxes_pred[1]),tf.shape(bboxes_pred[2]),tf.shape(bboxes_pred[3])])
bboxes_pred = [
tf.reshape(preds, [-1, 4]) for preds in bboxes_pred
]
bboxes_pred = tf.concat(bboxes_pred, axis=0)
flaten_cls_targets = tf.reshape(cls_targets, [-1])
flaten_match_scores = tf.reshape(match_scores, [-1])
flaten_loc_targets = tf.reshape(loc_targets, [-1, 4])
# each positive examples has one label
positive_mask = flaten_cls_targets > 0
n_positives = tf.count_nonzero(positive_mask)
batch_n_positives = tf.count_nonzero(cls_targets, -1)
batch_negtive_mask = tf.equal(
cls_targets, 0
) #tf.logical_and(tf.equal(cls_targets, 0), match_scores > 0.)
batch_n_negtives = tf.count_nonzero(batch_negtive_mask, -1)
batch_n_neg_select = tf.cast(
params['negative_ratio'] *
tf.cast(batch_n_positives, tf.float32), tf.int32)
batch_n_neg_select = tf.minimum(
batch_n_neg_select, tf.cast(batch_n_negtives, tf.int32))
# hard negative mining for classification
predictions_for_bg = tf.nn.softmax(
tf.reshape(
cls_pred,
[tf.shape(features)[0], -1, params['num_classes']
]))[:, :, 0]
prob_for_negtives = tf.where(
batch_negtive_mask,
0. - predictions_for_bg,
# ignore all the positives
0. - tf.ones_like(predictions_for_bg))
topk_prob_for_bg, _ = tf.nn.top_k(
prob_for_negtives, k=tf.shape(prob_for_negtives)[1])
score_at_k = tf.gather_nd(
topk_prob_for_bg,
tf.stack([
tf.range(tf.shape(features)[0]), batch_n_neg_select - 1
],
axis=-1))
selected_neg_mask = prob_for_negtives >= tf.expand_dims(
score_at_k, axis=-1)
# include both selected negtive and all positive examples
final_mask = tf.stop_gradient(
tf.logical_or(
tf.reshape(
tf.logical_and(batch_negtive_mask,
selected_neg_mask), [-1]),
positive_mask))
total_examples = tf.count_nonzero(final_mask)
cls_pred = tf.boolean_mask(cls_pred, final_mask)
location_pred = tf.boolean_mask(
location_pred, tf.stop_gradient(positive_mask))
flaten_cls_targets = tf.boolean_mask(
tf.clip_by_value(flaten_cls_targets, 0,
params['num_classes']), final_mask)
flaten_loc_targets = tf.stop_gradient(
tf.boolean_mask(flaten_loc_targets, positive_mask))
predictions = {
'classes':
tf.argmax(cls_pred, axis=-1),
'probabilities':
tf.reduce_max(tf.nn.softmax(cls_pred,
name='softmax_tensor'),
axis=-1),
'loc_predict':
bboxes_pred
}
cls_accuracy = tf.metrics.accuracy(flaten_cls_targets,
predictions['classes'])
metrics = {'cls_accuracy': cls_accuracy}
# Create a tensor named train_accuracy for logging purposes.
tf.identity(cls_accuracy[1], name='cls_accuracy')
tf.summary.scalar('cls_accuracy', cls_accuracy[1])
if mode == tf.estimator.ModeKeys.PREDICT:
return tf.estimator.EstimatorSpec(mode=mode, predictions=predictions)
# Calculate loss, which includes softmax cross entropy and L2 regularization.
#cross_entropy = tf.cond(n_positives > 0, lambda: tf.losses.sparse_softmax_cross_entropy(labels=flaten_cls_targets, logits=cls_pred), lambda: 0.)# * (params['negative_ratio'] + 1.)
#flaten_cls_targets=tf.Print(flaten_cls_targets, [flaten_loc_targets],summarize=50000)
cross_entropy = tf.losses.sparse_softmax_cross_entropy(
labels=flaten_cls_targets,
logits=cls_pred) * (params['negative_ratio'] + 1.)
# Create a tensor named cross_entropy for logging purposes.
tf.identity(cross_entropy, name='cross_entropy_loss')
tf.summary.scalar('cross_entropy_loss', cross_entropy)
#loc_loss = tf.cond(n_positives > 0, lambda: modified_smooth_l1(location_pred, tf.stop_gradient(flaten_loc_targets), sigma=1.), lambda: tf.zeros_like(location_pred))
loc_loss = modified_smooth_l1(location_pred, flaten_loc_targets, sigma=1.)
#loc_loss = modified_smooth_l1(location_pred, tf.stop_gradient(gtargets))
loc_loss = tf.reduce_mean(tf.reduce_sum(loc_loss, axis=-1),
name='location_loss')
tf.summary.scalar('location_loss', loc_loss)
tf.losses.add_loss(loc_loss)
l2_loss_vars = []
for trainable_var in tf.trainable_variables():
if '_bn' not in trainable_var.name:
if 'conv4_3_scale' not in trainable_var.name:
l2_loss_vars.append(tf.nn.l2_loss(trainable_var))
else:
l2_loss_vars.append(tf.nn.l2_loss(trainable_var) * 0.1)
# Add weight decay to the loss. We exclude the batch norm variables because
# doing so leads to a small improvement in accuracy.
total_loss = tf.add(cross_entropy + loc_loss,
tf.multiply(params['weight_decay'],
tf.add_n(l2_loss_vars),
name='l2_loss'),
name='total_loss')
if mode == tf.estimator.ModeKeys.TRAIN:
global_step = tf.train.get_or_create_global_step()
lr_values = [
params['learning_rate'] * decay
for decay in params['lr_decay_factors']
]
learning_rate = tf.train.piecewise_constant(
tf.cast(global_step, tf.int32),
[int(_) for _ in params['decay_boundaries']], lr_values)
truncated_learning_rate = tf.maximum(learning_rate,
tf.constant(
params['end_learning_rate'],
dtype=learning_rate.dtype),
name='learning_rate')
# Create a tensor named learning_rate for logging purposes.
tf.summary.scalar('learning_rate', truncated_learning_rate)
optimizer = tf.train.MomentumOptimizer(
learning_rate=truncated_learning_rate, momentum=params['momentum'])
optimizer = tf.contrib.estimator.TowerOptimizer(optimizer)
# Batch norm requires update_ops to be added as a train_op dependency.
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
with tf.control_dependencies(update_ops):
train_op = optimizer.minimize(total_loss, global_step)
else:
train_op = None
return tf.estimator.EstimatorSpec(
mode=mode,
predictions=predictions,
loss=total_loss,
train_op=train_op,
eval_metric_ops=metrics,
scaffold=tf.train.Scaffold(init_fn=get_init_fn()))
def __init__(self, args, n_items, is_train=True):
self.is_train = is_train
self.global_step = tf.get_variable(
'global_step',
shape=[],
dtype=tf.int32,
initializer=tf.constant_initializer(1),
trainable=False)
self.dropout = tf.get_variable('dropout',
shape=[],
dtype=tf.float32,
initializer=tf.constant_initializer(
args.dropout),
trainable=False)
self.lr = tf.get_variable("lr",
shape=[],
dtype=tf.float32,
trainable=False)
self.gru_act, self.final_activation, self.loss_function = utils.initialize_func(
args)
## Placeholder
self.x = tf.placeholder(tf.int32, [args.batch_size]) # input
self.y = tf.placeholder(tf.int32, [args.batch_size]) # output
self.state = [
tf.placeholder(tf.float32, [args.batch_size, args.hidden_size])
for _ in range(args.rnn_layers)
]
print(self.state)
## For Masking
self.curr = tf.placeholder(tf.int32, [args.batch_size])
self.end = tf.placeholder(tf.int32, [args.batch_size])
self.chk = tf.placeholder(tf.int32, [args.batch_size])
self.mask = tf.math.equal(self.curr, self.end)
self.mask_num = tf.count_nonzero(self.mask)
self.chk_mask = tf.math.greater(self.curr, self.chk)
## Initializing Parameters
sigma = args.sigma if args.sigma != 0 else np.sqrt(6.0 /
args.hidden_size -
1)
if args.init_as_normal:
initializer = tf.random_normal_initializer(mean=0, stddev=sigma)
else:
initializer = tf.random_uniform_initializer(minval=-sigma,
maxval=sigma)
embedding = tf.get_variable('embeddeing', [n_items, args.hidden_size],
initializer=initializer)
softmax_W = tf.get_variable('softmax_W', [n_items, args.hidden_size],
initializer=initializer)
softmax_b = tf.get_variable('softmax_b', [n_items],
initializer=tf.constant_initializer(0.0))
cell = rnn_cell.GRUCell(args.hidden_size, activation=self.gru_act)
drop_cell = rnn_cell.DropoutWrapper(cell,
output_keep_prob=self.dropout)
stacked_cell = rnn_cell.MultiRNNCell([drop_cell] * args.rnn_layers)
inputs = tf.nn.embedding_lookup(embedding, self.x)
output, state = stacked_cell(inputs, tuple(self.state))
self.final_state = state
if self.is_train == True:
sampled_W = tf.nn.embedding_lookup(softmax_W, self.y)
sampled_b = tf.nn.embedding_lookup(softmax_b, self.y)
sampled_logits = tf.matmul(output, sampled_W,
transpose_b=True) + sampled_b
self.sampled_yhat = self.final_activation(sampled_logits)
self.sampled_cost = self.loss_function(self.sampled_yhat)
self.lr = tf.maximum(
1e-5,
tf.train.exponential_decay(args.lr,
self.global_step,
args.decay_steps,
args.decay,
staircase=True))
optimizer = tf.train.AdamOptimizer(self.lr)
tvars = tf.trainable_variables()
pprint(tvars)
grads = optimizer.compute_gradients(self.sampled_cost, tvars)
if args.grad_cap > 0:
capped_grads = [(tf.clip_by_norm(grad, self.grad_cap), var)
for grad, var in grads]
else:
capped_grads = grads
self.train_op = optimizer.apply_gradients(
capped_grads, global_step=self.global_step)
logits = tf.matmul(output, softmax_W, transpose_b=True) + softmax_b
self.yhat = self.final_activation(logits)
self.cost = self.loss_function(self.yhat)
def model():
###############################################################################
# This function defines the tensorflow graph of the WSD model for the fine-grained
# prediction
#
# Input:
# None
#
# Output:
# train: optimizer of the sum of the losses
# train_fine: optimizer of the fine-grained loss
# train_wndomain: optimizer of the wndomain loss
# train_lexname: optimizer of the lexname loss
# train_POS: optimizer of the pos loss
# loss: tensor of the total loss
# loss_fine: tensor of the finegrained loss
# loss_wndomain: tensor of the wn_domain loss
# loss_lexname: tensor of the lexname loss
# loss_POS: tensor of the Pos loss
# output_finegrained: tensor of the finegrained output
# output_wndomain: tensor of the wn_domain output
# output_lexname: tenspr of the lexname output
# input_: tensor of the input (string sentences)
# input_w_ids: tensor of the input (ids sentences)
# y_fine_grained: tensor of the finegrained labels
# y_wndomain: tensor of the wn_domain labels
# y_lexnames: tensor of the lexname labels
# y_POS: tensor of the Pos labels
# keep_prob: tensor of the dropout
###############################################################################
# define input tensors
input_ = tf.placeholder(tf.string, shape=[None, None], name='inputs')
input_w_ids = tf.placeholder(tf.int32,
shape=[None, None],
name='inputs_int')
seq_length = tf.count_nonzero(input_w_ids,
axis=-1,
name='length',
dtype=tf.int32)
y_fine_grained = tf.placeholder(tf.int32,
shape=[None, None],
name='fine_grained_label')
y_wndomain = tf.placeholder(tf.int32,
shape=[None, None],
name='wndomain_label')
y_lexnames = tf.placeholder(tf.int32,
shape=[None, None],
name='lexnames_label')
y_POS = tf.placeholder(tf.int32, shape=[None, None], name='POS_label')
keep_prob = tf.placeholder(tf.float32, shape=[], name='keep_prob')
elmo = hub.Module("../resources/elmo", trainable=False)
# define Elmo embeddings node
with tf.variable_scope("Elmo_embeddings"):
embeddings = elmo(inputs={
"tokens": input_,
"sequence_len": seq_length
},
signature="tokens",
as_dict=True)["elmo"]
# define BiLSTM node
with tf.variable_scope('BiLSTM'):
cell_fw = tf.contrib.rnn.LSTMCell(
HIDDEN_SIZE,
initializer=tf.truncated_normal_initializer(-0.1, 0.1, seed=2))
cell_fw = tf.contrib.rnn.DropoutWrapper(cell_fw,
input_keep_prob=keep_prob,
output_keep_prob=keep_prob,
state_keep_prob=keep_prob)
cell_bw = tf.contrib.rnn.LSTMCell(
HIDDEN_SIZE,
initializer=tf.truncated_normal_initializer(-0.1, 0.1, seed=2))
cell_bw = tf.contrib.rnn.DropoutWrapper(cell_bw,
input_keep_prob=keep_prob,
output_keep_prob=keep_prob,
state_keep_prob=keep_prob)
outputs, states = tf.nn.bidirectional_dynamic_rnn(
cell_fw,
cell_bw,
embeddings,
sequence_length=seq_length,
dtype=tf.float32)
# Concat the forward and backward outputs
outputs = tf.concat(outputs, 2)
def attention(i, out_bilstm, input_ids, W_att, c_mat):
###############################################################################
# This function calculates and updates the weights matrix by adding the weights
# of each sentence of the current batch
#
# Input:
# i: batch number
# out_bilstm: output of the BiLSTM
# input_ids: input matrix with ids
# W_att: parameter vector
# c_mat: weights matrix
#
# Output:
# : updated batch number
# out_bilstm: output of the BiLSTM
# input_ids: input matrix with ids
# W_att: parameter vector
# : updated weights matrix
###############################################################################
# take the output of the BiLSTM of the batch i
mask = out_bilstm[i]
# take the ids sentence of the batch i
sentence_ids = input_ids[i]
# mask to exclude the padding from the attention scores
mask_attention = tf.not_equal(sentence_ids, 0)
h_masked = tf.boolean_mask(mask, mask_attention)
u = tf.matmul(tf.tanh(h_masked), W_att)
# apply softmax to the u vector
attention_score = tf.nn.softmax(u)
attention_score = tf.math.l2_normalize(attention_score)
# calculate attention score
c = tf.reduce_sum(tf.multiply(h_masked, attention_score), 0)
# return the result of the first iteration
if c_mat == None:
return c
c = tf.expand_dims(c, 0)
return tf.add(i,
1), out_bilstm, input_ids, W_att, tf.concat([c_mat, c],
0)
# define attention node
with tf.variable_scope("attention"):
output_shape = outputs.get_shape()[2].value
batch_size = tf.shape(outputs)[0]
# define parameter vector
W = tf.get_variable("W_att",
shape=[output_shape, 1],
initializer=tf.truncated_normal_initializer(
mean=0.0, stddev=0.1, seed=0))
# calculate the attention weights for all the sentences of the current batch
c = tf.expand_dims(attention(0, outputs, input_w_ids, W, None), 0)
i = tf.constant(1)
cond = lambda i, out, inp_mask, W, c: tf.less(i, batch_size)
_, _, _, _, attention_out = tf.while_loop(
cond,
attention, [i, outputs, input_w_ids, W, c],
shape_invariants=[
i.get_shape(),
outputs.get_shape(),
input_w_ids.get_shape(),
W.get_shape(),
tf.TensorShape([None, HIDDEN_SIZE * 2])
])
attention_out = tf.expand_dims(attention_out, 1)
c_final = tf.tile(attention_out, [1, MAX_LENGTH, 1])
# concat output BiLSTM with attention weights
concat_attention = tf.concat([c_final, outputs], 2)
# define output nodes
with tf.variable_scope("dense_POS"):
logits_POS = tf.layers.dense(inputs=concat_attention, units=LEN_POS)
output_POS = tf.identity(logits_POS, name='POS_output')
with tf.variable_scope("dense_finegrained"):
logits_finegrained = tf.layers.dense(inputs=concat_attention,
units=LEN_FINE)
output_finegrained = tf.identity(logits_finegrained,
name='finegrained_output')
with tf.variable_scope("dense_lexname"):
logits_lexname = tf.layers.dense(inputs=concat_attention,
units=LEN_LEXNAME)
output_lexname = tf.identity(logits_lexname, name='lexname_output')
with tf.variable_scope("dense_wndomain"):
logits_wndomain = tf.layers.dense(inputs=concat_attention,
units=LEN_WNDOMAIN)
output_wndomain = tf.identity(logits_wndomain, name='wndomain_output')
# define loss nodes
with tf.variable_scope('loss_POS'):
loss_POS = tf.nn.sparse_softmax_cross_entropy_with_logits(
logits=output_POS, labels=y_POS)
# mask for the POS loss
mask_pos = tf.greater_equal(y_POS, 2)
losses_pos_POS = tf.boolean_mask(loss_POS, mask_pos)
loss_POS = tf.reduce_mean(losses_pos_POS, name="POS_loss")
with tf.variable_scope('loss_finegrained'):
loss_fine = tf.nn.sparse_softmax_cross_entropy_with_logits(
logits=output_finegrained, labels=y_fine_grained)
# mask for the fine-grained loss
mask_pos = tf.greater_equal(y_fine_grained, 3)
losses_pos_fine = tf.boolean_mask(loss_fine, mask_pos)
loss_fine = tf.reduce_mean(losses_pos_fine, name="fine_loss")
with tf.variable_scope('loss_wndomain'):
loss_wndomain = tf.nn.sparse_softmax_cross_entropy_with_logits(
logits=output_wndomain, labels=y_wndomain)
# mask for the wndomain loss
mask_pos = tf.greater_equal(y_wndomain, 3)
losses_pos_wndomain = tf.boolean_mask(loss_wndomain, mask_pos)
loss_wndomain = tf.reduce_mean(losses_pos_wndomain,
name="wndomain_loss")
with tf.variable_scope('loss_lexname'):
loss_lexname = tf.nn.sparse_softmax_cross_entropy_with_logits(
logits=output_lexname, labels=y_lexnames)
# mask for the lexname loss
mask_pos = tf.greater_equal(y_lexnames, 3)
losses_pos_lexname = tf.boolean_mask(loss_lexname, mask_pos)
loss_lexname = tf.reduce_mean(losses_pos_lexname, name="lexname_loss")
with tf.variable_scope("total_loss"):
# sum all the losses
loss = loss_fine + loss_wndomain + loss_lexname + loss_POS
loss = tf.identity(loss, name='loss')
# define optimizer nodes
with tf.variable_scope("train_total"):
train = tf.train.AdamOptimizer(
learning_rate=LEARNING_RATE).minimize(loss)
return (train, loss, loss_fine, loss_wndomain, loss_lexname, loss_POS,
output_finegrained, output_wndomain, output_lexname, input_,
input_w_ids, y_fine_grained, y_wndomain, y_lexnames, y_POS,
keep_prob)
def __init__(self, _word_vocab_size, _synsets, _weights, _options):
self.word_vocab_size = _word_vocab_size
self.synsets = tf.ragged.constant(_synsets)
self.weights = tf.ragged.constant(_weights)
self.options = _options
"""PARAMETER INITIALIZATION"""
opts = self.options
with tf.name_scope('embeddings'):
self.word_embs = tf.Variable(tf.random_uniform(
[self.word_vocab_size, opts.word_size], -1.0, 1.0),
name='word_embs')
with tf.name_scope('nce_weights'):
self.nce_weights = tf.Variable(tf.truncated_normal(
[self.word_vocab_size, opts.word_size],
stddev=1.0 / math.sqrt(opts.word_size)),
name='nce_weights')
with tf.name_scope('nce_biases'):
self.nce_biases = tf.Variable(tf.zeros([self.word_vocab_size]),
name='nce_biases')
"""PLACEHOLDERS"""
with tf.name_scope('placeholders'):
self.inputs = tf.placeholder(
tf.int32, shape=[opts.batch_size, opts.context_window * 2])
self.labels = tf.placeholder(tf.int32, shape=[opts.batch_size, 1])
"""SYNONYMS GENERATION"""
with tf.name_scope('syns_generation'):
# gather synonyms for labels (i.e. target words)
self.syns = tf.gather(self.synsets, tf.squeeze(self.labels))
# reshape synonyms to be compliant with tf.nn.nce_loss input
self.syn_labels = tf.reshape(self.syns.values, shape=[-1, 1])
# store the length of each synset associated to self.labels
self.syn_lens = self.syns.row_lengths()
# pre compute the normalized weights for each word in vocabulary given the following formula: W(ws|wt) = cf(ws) / sum([cf(w) for w in syns(wt)])
self.weights = tf.to_float(
self.weights /
tf.reshape(tf.reduce_sum(self.weights, axis=1), shape=[-1, 1]))
"""EMBEDDING LOOKUPS"""
with tf.name_scope('lookups'):
# word embedding lookups
self.words = tf.nn.embedding_lookup(self.word_embs, self.inputs)
"""FORWARD PASS"""
with tf.name_scope('context_pass'):
# take the mean of context words embeddings to generate the context embedding
self.contexts = tf.reduce_mean(self.words, 1)
"""LOSS OPERATION"""
with tf.name_scope('loss_ops'):
# compute the Noise Contrastive Estimation (NCE) loss for the given batch (cbow objective)
self.cbow_loss = tf.nn.nce_loss(self.nce_weights, self.nce_biases,
self.labels, self.contexts,
opts.neg_samples,
self.word_vocab_size)
# condition rule to decide whether to compute synonymy-enhanced loss - when batch does not contain any synset switch to tf.constant(0.0)
self.condition = tf.count_nonzero(self.syn_lens)
# perform lazy tf.cond and compute loss (syns objective)
true_fn = lambda: self.compute_syn_loss()
self.syn_loss = tf.cond(
self.condition > 0,
true_fn=true_fn,
false_fn=lambda: tf.constant(0.0, dtype=tf.float32))
# combine losses
self.loss = tf.reduce_sum(
self.cbow_loss) - opts.alpha * self.syn_loss
self.loss /= opts.batch_size
"""OPTIMIZATION OPERATION"""
with tf.name_scope('opt_ops'):
# optimize constained cbow
optimizer = tf.train.AdagradOptimizer(opts.lr)
self.train_op = optimizer.minimize(self.loss)
def spectral_norm(w):
return tf.cond(tf.equal(tf.count_nonzero(w), 0), lambda: w,
lambda: apply_spectral_norm(w))
def facs_model(learning_rate, scale_class_weight, use_two_fc, use_three_fc,
use_four_fc, use_five_fc, use_six_fc, use_seven_fc, hparam):
config = tf.ConfigProto(graph_options=tf.GraphOptions(
optimizer_options=tf.OptimizerOptions(
opt_level=tf.OptimizerOptions.L0)))
tf.reset_default_graph()
sess = tf.Session("", config=config)
# Setup placeholders, and reshape the data
x = tf.placeholder(tf.float32, [None, n_input], name="x")
y = tf.placeholder(tf.float32, [None, n_output], name="labels")
sw = tf.placeholder(tf.float32, [None, n_output], name='intensity_weights')
# Main function compares the number of FCs for performance.
if use_two_fc:
fc1 = fc_layer(x, n_input, n_hidden_1, "fc1")
relu = tf.nn.relu(fc1)
tf.summary.histogram("fc1/relu", relu)
logits = fc_layer(relu, n_hidden_1, n_output, "fc2")
elif use_three_fc:
fc1 = fc_layer(x, n_input, n_hidden_1, "fc1")
relu = tf.nn.relu(fc1)
tf.summary.histogram("fc3/relu", relu)
fc2 = fc_layer(relu, n_hidden_1, n_hidden_2, "fc2")
relu_2 = tf.nn.relu(fc2)
tf.summary.histogram("fc3/relu", relu_2)
logits = fc_layer(relu_2, n_hidden_2, n_output, "fc3")
elif use_four_fc:
fc1 = fc_layer(x, n_input, n_hidden_1, "fc1")
relu = tf.nn.relu(fc1)
tf.summary.histogram("fc4/relu", relu)
fc2 = fc_layer(relu, n_hidden_1, n_hidden_2, "fc2")
relu_2 = tf.nn.relu(fc2)
tf.summary.histogram("fc4/relu", relu_2)
fc3 = fc_layer(relu_2, n_hidden_2, n_hidden_3, "fc3")
relu_3 = tf.nn.relu(fc3)
tf.summary.histogram("fc4/relu", relu_3)
logits = fc_layer(relu_3, n_hidden_3, n_output, "fc4")
elif use_five_fc:
fc1 = fc_layer(x, n_input, n_hidden_1, "fc1")
relu = tf.nn.relu(fc1)
tf.summary.histogram("fc5/relu", relu)
fc2 = fc_layer(relu, n_hidden_1, n_hidden_2, "fc2")
relu_2 = tf.nn.relu(fc2)
tf.summary.histogram("fc5/relu", relu_2)
fc3 = fc_layer(relu_2, n_hidden_2, n_hidden_3, "fc3")
relu_3 = tf.nn.relu(fc3)
tf.summary.histogram("fc5/relu", relu_3)
fc4 = fc_layer(relu_3, n_hidden_3, n_hidden_4, "fc4")
relu_4 = tf.nn.relu(fc4)
tf.summary.histogram("fc5/relu", relu_4)
logits = fc_layer(relu_4, n_hidden_4, n_output, "fc5")
elif use_six_fc:
fc1 = fc_layer(x, n_input, n_hidden_1, "fc1")
relu = tf.nn.relu(fc1)
tf.summary.histogram("fc6/relu", relu)
fc2 = fc_layer(relu, n_hidden_1, n_hidden_2, "fc2")
relu_2 = tf.nn.relu(fc2)
tf.summary.histogram("fc6/relu", relu_2)
fc3 = fc_layer(relu_2, n_hidden_2, n_hidden_3, "fc3")
relu_3 = tf.nn.relu(fc3)
tf.summary.histogram("fc6/relu", relu_3)
fc4 = fc_layer(relu_3, n_hidden_3, n_hidden_4, "fc4")
relu_4 = tf.nn.relu(fc4)
tf.summary.histogram("fc6/relu", relu_4)
fc5 = fc_layer(relu_4, n_hidden_4, n_hidden_5, "fc5")
relu_5 = tf.nn.relu(fc5)
tf.summary.histogram("fc6/relu", relu_5)
logits = fc_layer(relu_5, n_hidden_5, n_output, "fc6")
elif use_seven_fc:
fc1 = fc_layer(x, n_input, n_hidden_1, "fc1")
relu = tf.nn.relu(fc1)
tf.summary.histogram("fc7/relu", relu)
fc2 = fc_layer(relu, n_hidden_1, n_hidden_2, "fc2")
relu_2 = tf.nn.relu(fc2)
tf.summary.histogram("fc7/relu", relu_2)
fc3 = fc_layer(relu_2, n_hidden_2, n_hidden_3, "fc3")
relu_3 = tf.nn.relu(fc3)
tf.summary.histogram("fc7/relu", relu_3)
fc4 = fc_layer(relu_3, n_hidden_3, n_hidden_4, "fc4")
relu_4 = tf.nn.relu(fc4)
tf.summary.histogram("fc7/relu", relu_4)
fc5 = fc_layer(relu_4, n_hidden_4, n_hidden_5, "fc5")
relu_5 = tf.nn.relu(fc5)
tf.summary.histogram("fc7/relu", relu_5)
fc6 = fc_layer(relu_5, n_hidden_5, n_hidden_6, "fc6")
relu_6 = tf.nn.relu(fc6)
tf.summary.histogram("fc7/relu", relu_6)
logits = fc_layer(relu_6, n_hidden_6, n_output, "fc7")
else:
logits = fc_layer(x, n_input, n_output, "fc")
# Loss function
with tf.name_scope("xent"):
# The positive and negative samples in the data are unbalanced.
# To push the algorithm to focus on fitting positives, I weighted the
# positive values more than the negative.
maxY = tf.reduce_sum(y, 1) * scale_class_weight
class_weights = (maxY + 1) / 6
# Some expressions are more intense than others in the CK+ database and
# and that is weighted in the loss function by sample weights, sw.
# However, I got better results with just weighting all AUs
# with equal intensity.
# mult_w = tf.multiply(y, sw)
# sum_w = tf.reduce_sum(mult_w,1)
#
# class_weights = ( sum_w + 1) / 6
print(class_weights.get_shape())
xent = tf.reduce_mean(
tf.nn.sigmoid_cross_entropy_with_logits(logits=logits,
labels=y,
name="xent"))
xent = tf.reduce_mean(xent * class_weights)
tf.summary.scalar("xent", xent)
with tf.name_scope("train"):
train_step = tf.train.AdamOptimizer(learning_rate).minimize(xent)
with tf.name_scope("accuracy"):
zero = tf.constant(0, dtype=tf.float32)
onesMat = tf.ones_like(logits)
zerosMat = tf.zeros_like(logits)
onesY = tf.ones_like(y, dtype=tf.float32)
yFloat = tf.cast(y, dtype=tf.float32)
yFlipped = onesY - yFloat
# PREDICTION - If logits >= 0, logits = 1, else logits = 0.
logitsBin = tf.cast(tf.where(logits >= zero, onesMat, zerosMat),
dtype=tf.float32,
name="op_to_restore")
tf.add_to_collection("coll", logitsBin)
tf.add_to_collection("coll", x)
print('logitsBin', logitsBin.get_shape())
print('y', y.get_shape())
print('where_logitsBin', tf.where(logitsBin)[:, 1].get_shape())
print('where_y', tf.where(y)[:, 1].get_shape())
time_steps = tf.cast(tf.shape(y)[0], dtype='int32')
print(time_steps.get_shape())
nFacs = tf.count_nonzero(y, 1, dtype=tf.float32)
onesFacs = tf.ones_like(nFacs)
nFacs_Zeros = onesFacs * numFacs - nFacs
nFacs = tf.where(tf.equal(nFacs, zero), onesFacs, nFacs)
nFacs_Zeros = tf.where(tf.equal(nFacs_Zeros, zero), onesFacs,
nFacs_Zeros)
# Find TPR, TNR, FPR, FNR.
matrix_positive = tf.cast(
tf.equal(logitsBin, y)
& tf.equal(yFloat, tf.constant(1, dtype=tf.float32)),
dtype=tf.float32)
correct_pos = tf.reduce_sum(matrix_positive) / tf.reduce_sum(yFloat)
tf.summary.scalar("TruePosRate", correct_pos)
matrix_negative = tf.cast(tf.equal(logitsBin, y)
& tf.equal(yFloat, zero),
dtype=tf.float32)
correct_neg = tf.reduce_sum(matrix_negative) / tf.reduce_sum(yFlipped)
tf.summary.scalar("TrueNegRate", correct_neg)
matrix_falsePos = tf.cast(tf.not_equal(logitsBin, y)
& tf.equal(y, zero),
dtype=tf.float32) #or yFlipped = 1
falsePos = tf.reduce_sum(matrix_falsePos) / tf.reduce_sum(yFlipped)
tf.summary.scalar("falsePosRate", falsePos)
matrix_falseNeg = tf.cast(
tf.not_equal(logitsBin, y)
& tf.equal(yFloat, tf.constant(1, dtype=tf.float32)),
dtype=tf.float32)
falseNeg = tf.reduce_sum(matrix_falseNeg) / tf.reduce_sum(yFloat)
tf.summary.scalar("falseNegRate", falseNeg)
tp_sum = tf.reduce_sum(matrix_positive, 0)
tp_sum_append = tf.concat([tf.constant([0], dtype=tf.float32), tp_sum],
0)
tf_sum = tf.reduce_sum(matrix_negative, 0)
fp_sum = tf.reduce_sum(matrix_falsePos, 0)
fn_sum = tf.reduce_sum(matrix_falseNeg, 0)
# Get Matrix of Confusion for multiclass binary classifier.
confusion = tf.Variable(initial_value=tf.zeros(
[n_output + 1, n_output + 1]),
name='confusion')
confusion1 = tf.Variable(initial_value=tf.cast(tf.diag(
np.repeat(1, n_output + 1)),
dtype=tf.float32),
name='confusion1')
confusion2 = tf.Variable(initial_value=tf.zeros(
[n_output + 1, n_output + 1]),
name='confusion2')
confusion3 = tf.Variable(initial_value=tf.zeros(
[n_output + 1, n_output + 1]),
name='confusion3')
confusion4 = tf.Variable(initial_value=tf.zeros(
[n_output + 1, n_output + 1]),
name='confusion4')
confusion1 = confusion1[0, 0].assign(5)
confusion1 = confusion1 * tp_sum_append
confusion2 = confusion2[0, 0].assign(tf.reduce_sum(tf_sum))
confusion3 = tf.assign(confusion3[0, 1:n_output + 1], fp_sum)
confusion4 = confusion4[1:n_output + 1, 0].assign(fn_sum)
confusion = confusion1 + confusion2 + confusion3 + confusion4
txtConfusion = tf.as_string(confusion,
precision=0,
name='txtConfusion')
tf.summary.text('txtConfusion', txtConfusion)
correct_prediction = tf.cast(tf.equal(logitsBin, y),
dtype=tf.float32,
name="correct_prediction")
accuracy = tf.reduce_mean(correct_prediction, name="accuracy")
tf.summary.scalar("accuracy", accuracy)
# Summary for tensorboard
summ = tf.summary.merge_all()
saver = tf.train.Saver()
init = tf.group(tf.global_variables_initializer(),
tf.local_variables_initializer())
sess.run(init)
writer = tf.summary.FileWriter(LOGDIR + hparam + '/train')
test_writer = tf.summary.FileWriter(LOGDIR + hparam + '/test')
writer.add_graph(sess.graph)
for i in range(3001):
if i % 5 == 0:
[train_accuracy, s] = sess.run([accuracy, summ],
feed_dict={
x: train_x,
y: train_y,
sw: sw_train
})
sess.run([confusion],
feed_dict={
x: test_x,
y: test_y,
sw: sw_test
})
writer.add_summary(s, i)
if i % 50 == 0:
[acc, s] = sess.run([accuracy, summ],
feed_dict={
x: test_x,
y: test_y,
sw: sw_test
})
sess.run([confusion],
feed_dict={
x: test_x,
y: test_y,
sw: sw_test
})
test_writer.add_summary(s, i)
saver.save(sess, os.path.join(savepath, hparam, "model"), i)
sess.run(train_step, feed_dict={x: train_x, y: train_y, sw: sw_train})
def pred_per_class(y_true, y_pred, cls):
return tf.count_nonzero(tf.cast(y_pred >= tf.constant(0.5), tf.int32))
def num_per_class(y_true, y_pred, cls):
"""
Returns the number of samples in a specific class.
Assumes that y_true contains 1 or 0.
"""
return tf.count_nonzero(tf.cast(y_true, tf.int32))
def rpn_losses(anchor_labels, anchor_boxes, label_logits, box_logits):
"""
Args:
anchor_labels: fHxfWxNA
anchor_boxes: fHxfWxNAx4, encoded
label_logits: fHxfWxNA
box_logits: fHxfWxNAx4
Returns:
label_loss, box_loss
"""
with tf.device('/cpu:0'):
valid_mask = tf.stop_gradient(tf.not_equal(anchor_labels, -1))
pos_mask = tf.stop_gradient(tf.equal(anchor_labels, 1))
nr_valid = tf.stop_gradient(tf.count_nonzero(valid_mask,
dtype=tf.int32),
name='num_valid_anchor')
nr_pos = tf.identity(tf.count_nonzero(pos_mask, dtype=tf.int32),
name='num_pos_anchor')
# nr_pos is guaranteed >0 in C4. But in FPN. even nr_valid could be 0.
valid_anchor_labels = tf.boolean_mask(anchor_labels, valid_mask)
valid_label_logits = tf.boolean_mask(label_logits, valid_mask)
with tf.name_scope('label_metrics'):
valid_label_prob = tf.nn.sigmoid(valid_label_logits)
summaries = []
with tf.device('/cpu:0'):
for th in [0.5, 0.2, 0.1]:
valid_prediction = tf.cast(valid_label_prob > th, tf.int32)
nr_pos_prediction = tf.reduce_sum(valid_prediction,
name='num_pos_prediction')
pos_prediction_corr = tf.count_nonzero(tf.logical_and(
valid_label_prob > th,
tf.equal(valid_prediction, valid_anchor_labels)),
dtype=tf.int32)
placeholder = 0.5 # A small value will make summaries appear lower.
recall = tf.to_float(tf.truediv(pos_prediction_corr, nr_pos))
recall = tf.where(tf.equal(nr_pos, 0),
placeholder,
recall,
name='recall_th{}'.format(th))
precision = tf.to_float(
tf.truediv(pos_prediction_corr, nr_pos_prediction))
precision = tf.where(tf.equal(nr_pos_prediction, 0),
placeholder,
precision,
name='precision_th{}'.format(th))
summaries.extend([precision, recall])
add_moving_summary(*summaries)
# Per-level loss summaries in FPN may appear lower due to the use of a small placeholder.
# But the total loss is still the same. TODO make the summary op smarter
placeholder = 0.
label_loss = tf.nn.sigmoid_cross_entropy_with_logits(
labels=tf.to_float(valid_anchor_labels), logits=valid_label_logits)
label_loss = tf.reduce_sum(label_loss) * (1. / config.RPN_BATCH_PER_IM)
label_loss = tf.where(tf.equal(nr_valid, 0),
placeholder,
label_loss,
name='label_loss')
pos_anchor_boxes = tf.boolean_mask(anchor_boxes, pos_mask)
pos_box_logits = tf.boolean_mask(box_logits, pos_mask)
delta = 1.0 / 9
box_loss = tf.losses.huber_loss(pos_anchor_boxes,
pos_box_logits,
delta=delta,
reduction=tf.losses.Reduction.SUM) / delta
box_loss = box_loss * (1. / config.RPN_BATCH_PER_IM)
box_loss = tf.where(tf.equal(nr_pos, 0),
placeholder,
box_loss,
name='box_loss')
add_moving_summary(label_loss, box_loss, nr_valid, nr_pos)
return label_loss, box_loss
# %% IN CASE OF PSORIASIS OR ECZEMA
x_de0 = dense(128, activation='relu', name='fc01')(x_de)
y_logits0 = dense(1, name='predictions0')(x_de0)
# %% IN CASE OF ACNE VULGARIS OR ROSACEA
x_de1 = dense(128, activation='relu', name='fc11')(x_de)
y_logits1 = dense(1, name='predictions1')(x_de1)
# %% IN CASE OF ECZEMA OR MYCOSIS FUNGOIDES
x_de2 = dense(128, activation='relu', name='fc21')(x_de)
y_logits2 = dense(1, name='predictions2')(x_de2)
y_logits = tf.concat([y_logits0, y_logits1, y_logits2], 1)
n_samples = tf.reduce_sum(tf.count_nonzero(m, axis=1,
dtype=tf.float32))
with tf.variable_scope('loss'):
masked_cross_entropy = masked_sigmoid_cross_entropy_with_logits(
logits=y_logits, labels=y, masks=m)
loss = tf.reduce_sum(masked_cross_entropy) / n_samples
with tf.variable_scope('performance'):
probabilities = tf.multiply(tf.sigmoid(y_logits), m)
prediction = tf.round(probabilities)
correct_prediction = tf.reduce_sum(
tf.multiply(tf.cast(tf.equal(prediction, y), dtype=tf.float32), m))
accuracy = correct_prediction / n_samples
with tf.variable_scope('tasks'):
t0 = tf.constant([1, 0, 0], dtype=tf.float32) # psoriasis
def forward(batch_queue, net, phase, scope, optimizer=None):
img_batch, label_instance_batch, label_existence_batch = batch_queue.dequeue()
inference = net.inference(img_batch, phase, 'lanenet_loss')
_ = net.loss(inference, label_instance_batch, label_existence_batch, 'lanenet_loss')
total_loss = tf.add_n(tf.get_collection('total_loss', scope))
instance_loss = tf.add_n(tf.get_collection('instance_seg_loss', scope))
existence_loss = tf.add_n(tf.get_collection('existence_pre_loss', scope))
out_logits = tf.add_n(tf.get_collection('instance_seg_logits', scope))
# calculate the accuracy
out_logits = tf.nn.softmax(logits=out_logits)
out_logits_out = tf.argmax(out_logits, axis=-1)
pred_0 = tf.count_nonzero(tf.multiply(tf.cast(tf.equal(label_instance_batch, 0), tf.int32),
tf.cast(tf.equal(out_logits_out, 0), tf.int32)),
dtype=tf.int32)
pred_1 = tf.count_nonzero(tf.multiply(tf.cast(tf.equal(label_instance_batch, 1), tf.int32),
tf.cast(tf.equal(out_logits_out, 1), tf.int32)),
dtype=tf.int32)
pred_2 = tf.count_nonzero(tf.multiply(tf.cast(tf.equal(label_instance_batch, 2), tf.int32),
tf.cast(tf.equal(out_logits_out, 2), tf.int32)),
dtype=tf.int32)
pred_3 = tf.count_nonzero(tf.multiply(tf.cast(tf.equal(label_instance_batch, 3), tf.int32),
tf.cast(tf.equal(out_logits_out, 3), tf.int32)),
dtype=tf.int32)
pred_4 = tf.count_nonzero(tf.multiply(tf.cast(tf.equal(label_instance_batch, 4), tf.int32),
tf.cast(tf.equal(out_logits_out, 4), tf.int32)),
dtype=tf.int32)
gt_all = tf.count_nonzero(tf.cast(tf.greater(label_instance_batch, 0), tf.int32), dtype=tf.int32)
gt_back = tf.count_nonzero(tf.cast(tf.equal(label_instance_batch, 0), tf.int32), dtype=tf.int32)
pred_all = tf.add(tf.add(tf.add(pred_1, pred_2), pred_3), pred_4)
accuracy = tf.divide(tf.cast(pred_all, tf.float32), tf.cast(gt_all, tf.float32))
accuracy_back = tf.divide(tf.cast(pred_0, tf.float32), tf.cast(gt_back, tf.float32))
# Compute mIoU of Lanes
overlap_1 = pred_1
union_1 = tf.add(tf.count_nonzero(tf.cast(tf.equal(label_instance_batch, 1),
tf.int32), dtype=tf.int32),
tf.count_nonzero(tf.cast(tf.equal(out_logits_out, 1),
tf.int32), dtype=tf.int32))
union_1 = tf.subtract(union_1, overlap_1)
IoU_1 = tf.divide(tf.cast(overlap_1, tf.float32), tf.cast(union_1, tf.float32))
overlap_2 = pred_2
union_2 = tf.add(tf.count_nonzero(tf.cast(tf.equal(label_instance_batch, 2),
tf.int32), dtype=tf.int32),
tf.count_nonzero(tf.cast(tf.equal(out_logits_out, 2),
tf.int32), dtype=tf.int32))
union_2 = tf.subtract(union_2, overlap_2)
IoU_2 = tf.divide(tf.cast(overlap_2, tf.float32), tf.cast(union_2, tf.float32))
overlap_3 = pred_3
union_3 = tf.add(tf.count_nonzero(tf.cast(tf.equal(label_instance_batch, 3),
tf.int32), dtype=tf.int32),
tf.count_nonzero(tf.cast(tf.equal(out_logits_out, 3),
tf.int32), dtype=tf.int32))
union_3 = tf.subtract(union_3, overlap_3)
IoU_3 = tf.divide(tf.cast(overlap_3, tf.float32), tf.cast(union_3, tf.float32))
overlap_4 = pred_4
union_4 = tf.add(tf.count_nonzero(tf.cast(tf.equal(label_instance_batch, 4),
tf.int64), dtype=tf.int32),
tf.count_nonzero(tf.cast(tf.equal(out_logits_out, 4),
tf.int64), dtype=tf.int32))
union_4 = tf.subtract(union_4, overlap_4)
IoU_4 = tf.divide(tf.cast(overlap_4, tf.float32), tf.cast(union_4, tf.float32))
IoU = tf.reduce_mean(tf.stack([IoU_1, IoU_2, IoU_3, IoU_4]))
tf.get_variable_scope().reuse_variables()
if optimizer is not None:
grads = optimizer.compute_gradients(total_loss)
else:
grads = None
return total_loss, instance_loss, existence_loss, accuracy, accuracy_back, IoU, out_logits_out, grads
def sparse_mean(x): return tf.reduce_sum(x) / tf.count_nonzero(x, dtype=x.dtype)
# net has shape (?,1,1,2), so use squeeze to reduce to (?,2)
net = tf.squeeze(net)
loss = tf.reduce_mean(
tf.nn.sigmoid_cross_entropy_with_logits(logits=net, labels=target))
train = tf.train.AdamOptimizer(learning_rate=dynamic_lr).minimize(loss)
prediction_prob = tf.nn.sigmoid(net)
# let prob1 be [0.4,0.6], prob2 be [0.6,0.8]
# let target1 be [0,1], target2 be [1,0]
# then tf.abs(prediction_prob - target) for the 1st example is 0.8, and it is 1.2 for the 2nd example
# we can then set a ture/false threshold at 1
# then cast to int32 will change the output for the 1st example to 0, and for the 2nd example to be 1
# finally count nonzero elements, then divided by batch size
tmp = tf.reduce_sum(tf.abs(prediction_prob - target), axis=1)
error_rate = tf.cast(tf.count_nonzero(tf.cast(tmp, tf.int32), dtype=tf.int32),
tf.float32) / tf.cast(tf.shape(target)[0], tf.float32)
tf.summary.scalar('error_rate_on_train_set', error_rate * 100.0)
tf.summary.scalar('loss', loss)
merged = tf.summary.merge_all()
init = tf.global_variables_initializer()
# Build Session
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
#config.gpu_options.per_process_gpu_memory_fraction = 0.33
sess = tf.Session(config=config)
writer = tf.summary.FileWriter("./sum", sess.graph)
var_to_save = [
var for var in tf.global_variables()
def train_neural_network(x):
prediction = recurrent_neural_network(x)
cost = tf.reduce_mean(
tf.nn.sigmoid_cross_entropy_with_logits(logits=prediction, labels=y))
optimizer = tf.train.RMSPropOptimizer(learning_rate=learning_rate,
momentum=momentum).minimize(cost)
with tf.Session() as sess:
llprint("Building Computational Graph ... ")
summaries = []
summaries.append(tf.summary.scalar("Loss", cost))
summarize_op = tf.summary.merge(summaries)
no_summarize = tf.no_op()
summarizer = tf.summary.FileWriter(tb_logs_dir, sess.graph)
llprint("Done!\n")
llprint("Initializing variables...")
sess.run(tf.global_variables_initializer())
llprint("Done!\n")
last_100_losses = []
start = 0 if start_step == 0 else start_step + 1
end = start_step + hm_epochs + 1
start_time_100 = time.time()
end_time_100 = None
avg_100_time = 0.
avg_counter = 0
for epoch in range(hm_epochs + 1):
llprint("\rIteration %d/%d" % (epoch, hm_epochs))
summarize = (epoch % 10 == 0)
epoch_loss = 0
epoch_x, epoch_y = np.array(train_x), np.array(train_y)
epoch_x = epoch_x.reshape((-1, 1, input_size))
_, c = sess.run([optimizer, cost],
feed_dict={
x: epoch_x,
y: epoch_y
})
epoch_loss += c
last_100_losses.append(c)
if summarize:
llprint("\n\tAvg. Loss: %.4f\n" % (np.mean(last_100_losses)))
print(last_100_losses)
end_time_100 = time.time()
elapsed_time = (end_time_100 - start_time_100) / 60
avg_counter += 1
avg_100_time += (1. / avg_counter) * (elapsed_time -
avg_100_time)
estimated_time = (avg_100_time * ((end - epoch) / 100.)) / 60.
print("\tAvg. 100 iterations time: %.2f minutes" %
(avg_100_time))
print("\tApprox. time to completion: %.2f hours" %
(estimated_time))
start_time_100 = time.time()
last_100_losses = []
# Model eval
argmax_prediction = tf.argmax(prediction, 1)
argmax_y = tf.argmax(y, 1)
incorrect = tf.not_equal(argmax_prediction, argmax_y)
misclass = tf.count_nonzero(incorrect)
hamm_loss = ((tf.reduce_sum(misclass / output_size))) / len((test_x))
print(
'Hamming loss:',
hamm_loss.eval({
x:
np.array(test_x).reshape((-1, len(test_x), input_size)),
y:
test_y
}))
def evaluate(output, input_y):
with tf.name_scope('evaluate'):
pred = tf.argmax(output, axis=1)
error_num = tf.count_nonzero(pred - input_y, name='error_num')
tf.summary.scalar('LeNet_error_num', error_num)
return error_num
def bboxes_matching(label, scores, bboxes,
glabels, gbboxes, gdifficults,
matching_threshold=0.5, scope=None):
"""Matching a collection of detected boxes with groundtruth values.
Does not accept batched-inputs.
The algorithm goes as follows: for every detected box, check
if one grountruth box is matching. If none, then considered as False Positive.
If the grountruth box is already matched with another one, it also counts
as a False Positive. We refer the Pascal VOC documentation for the details.
Args:
rclasses, rscores, rbboxes: N(x4) Tensors. Detected objects, sorted by score;
glabels, gbboxes: Groundtruth bounding boxes. May be zero padded, hence
zero-class objects are ignored.
matching_threshold: Threshold for a positive match.
Return: Tuple of:
n_gbboxes: Scalar Tensor with number of groundtruth boxes (may difer from
size because of zero padding).
tp_match: (N,)-shaped boolean Tensor containing with True Positives.
fp_match: (N,)-shaped boolean Tensor containing with False Positives.
"""
with tf.name_scope(scope, 'bboxes_matching_single',
[scores, bboxes, glabels, gbboxes]):
rsize = tf.size(scores)
rshape = tf.shape(scores)
rlabel = tf.cast(label, glabels.dtype)
# Number of groundtruth boxes.
gdifficults = tf.cast(gdifficults, tf.bool)
n_gbboxes = tf.count_nonzero(tf.logical_and(tf.equal(glabels, label),
tf.logical_not(gdifficults)))
# Grountruth matching arrays.
gmatch = tf.zeros(tf.shape(glabels), dtype=tf.bool)
grange = tf.range(tf.size(glabels), dtype=tf.int32)
# True/False positive matching TensorArrays.
sdtype = tf.bool
ta_tp_bool = tf.TensorArray(sdtype, size=rsize, dynamic_size=False, infer_shape=True)
ta_fp_bool = tf.TensorArray(sdtype, size=rsize, dynamic_size=False, infer_shape=True)
# Loop over returned objects.
def m_condition(i, ta_tp, ta_fp, gmatch):
r = tf.less(i, rsize)
return r
def m_body(i, ta_tp, ta_fp, gmatch):
# Jaccard score with groundtruth bboxes.
rbbox = bboxes[i]
jaccard = bboxes_jaccard(rbbox, gbboxes)
jaccard = jaccard * tf.cast(tf.equal(glabels, rlabel), dtype=jaccard.dtype)
# Best fit, checking it's above threshold.
idxmax = tf.cast(tf.argmax(jaccard, axis=0), tf.int32)
jcdmax = jaccard[idxmax]
match = jcdmax > matching_threshold
existing_match = gmatch[idxmax]
not_difficult = tf.logical_not(gdifficults[idxmax])
# TP: match & no previous match and FP: previous match | no match.
# If difficult: no record, i.e FP=False and TP=False.
tp = tf.logical_and(not_difficult,
tf.logical_and(match, tf.logical_not(existing_match)))
ta_tp = ta_tp.write(i, tp)
fp = tf.logical_and(not_difficult,
tf.logical_or(existing_match, tf.logical_not(match)))
ta_fp = ta_fp.write(i, fp)
# Update grountruth match.
mask = tf.logical_and(tf.equal(grange, idxmax),
tf.logical_and(not_difficult, match))
gmatch = tf.logical_or(gmatch, mask)
return [i+1, ta_tp, ta_fp, gmatch]
# Main loop definition.
i = 0
[i, ta_tp_bool, ta_fp_bool, gmatch] = \
tf.while_loop(m_condition, m_body,
[i, ta_tp_bool, ta_fp_bool, gmatch],
parallel_iterations=1,
back_prop=False)
# TensorArrays to Tensors and reshape.
tp_match = tf.reshape(ta_tp_bool.stack(), rshape)
fp_match = tf.reshape(ta_fp_bool.stack(), rshape)
# Some debugging information...
# tp_match = tf.Print(tp_match,
# [n_gbboxes,
# tf.reduce_sum(tf.cast(tp_match, tf.int64)),
# tf.reduce_sum(tf.cast(fp_match, tf.int64)),
# tf.reduce_sum(tf.cast(gmatch, tf.int64))],
# 'Matching (NG, TP, FP, GM): ')
return n_gbboxes, tp_match, fp_match
print('Learning Finished!')
test_size = len(test_data_y_refined)
predictions = np.zeros([test_size, 2])
for m_idx, m in enumerate(models):
print(m_idx, 'Precision: ',
m.get_precision(test_data_x, test_data_y_refined), 'recall: ',
m.get_recall(test_data_x, test_data_y_refined), 'Specificity: ',
m.get_specificity(test_data_x, test_data_y_refined))
p = m.predict(test_data_x)
predictions += p
# P: 유방암이라고 예측. N: 유방암이 아니라고 예측
ensemble_TP = tf.count_nonzero(
tf.argmax(predictions, 1) * tf.argmax(test_data_y_refined, 1),
dtype=tf.float32
) # 왜 이게 될까? -> element wise하게 곱하기 때문에, 예측 ,label 둘다 1인 경우만
ensemble_TN = tf.count_nonzero(
(tf.argmax(predictions, 1) - 1) * (tf.argmax(test_data_y_refined, 1) - 1),
dtype=tf.float32)
ensemble_FP = tf.count_nonzero(tf.argmax(predictions, 1) *
(tf.argmax(test_data_y_refined, 1) - 1),
dtype=tf.float32)
ensemble_FN = tf.count_nonzero(
(tf.argmax(predictions, 1) - 1) * tf.argmax(test_data_y_refined, 1),
dtype=tf.float32)
ensemble_precision = tf.divide(ensemble_TP, ensemble_TP +
ensemble_FP) # 내가 유방암이라고 예측한 사람 중에 맞춘 비율
ensemble_recall = tf.divide(ensemble_TP, ensemble_TP +
ensemble_FN) # 실제로 유방암인 사람중에 유방암이라고 예측한 비율
def init_training_graph(self):
with tf.name_scope('Evaluation'):
self.logits = self.conv_layer_f(self.last, self.logits_weight, strides=[1,1,1,1], scope_name="logits/")
self.predictions = tf.argmax(self.logits, axis=3)
with tf.name_scope('Loss'):
condition = tf.equal(self.train_labels_node, 255)
case_true = tf.ones(self.train_labels_node.get_shape())
case_false = self.train_labels_node
anno = tf.where(condition, case_true, case_false)
anno = tf.squeeze(tf.cast(anno, tf.int32), squeeze_dims=[3])
# anno = tf.divide(anno, 255.)
self.loss = tf.reduce_mean((tf.nn.sparse_softmax_cross_entropy_with_logits(logits=self.logits,
labels=anno,
name="entropy")))
tf.summary.scalar("entropy", self.loss)
with tf.name_scope('Accuracy'):
LabelInt = tf.squeeze(tf.cast(self.train_labels_node, tf.int64), squeeze_dims=[3])
CorrectPrediction = tf.equal(self.predictions, LabelInt)
self.accuracy = tf.reduce_mean(tf.cast(CorrectPrediction, tf.float32))
tf.summary.scalar("accuracy", self.accuracy)
with tf.name_scope('Prediction'):
self.TP = tf.count_nonzero(self.predictions * LabelInt)
self.TN = tf.count_nonzero((self.predictions - 1) * (LabelInt - 1))
self.FP = tf.count_nonzero(self.predictions * (LabelInt - 1))
self.FN = tf.count_nonzero((self.predictions - 1) * LabelInt)
with tf.name_scope('Precision'):
self.precision = tf.divide(self.TP, tf.add(self.TP, self.FP))
tf.summary.scalar('Precision', self.precision)
with tf.name_scope('Recall'):
self.recall = tf.divide(self.TP, tf.add(self.TP, self.FN))
tf.summary.scalar('Recall', self.recall)
with tf.name_scope('F1'):
num = tf.multiply(self.precision, self.recall)
dem = tf.add(self.precision, self.recall)
self.F1 = tf.scalar_mul(2, tf.divide(num, dem))
tf.summary.scalar('F1', self.F1)
with tf.name_scope('MeanAccuracy'):
Nprecision = tf.divide(self.TN, tf.add(self.TN, self.FN))
self.MeanAcc = tf.divide(tf.add(self.precision, Nprecision) ,2)
tf.summary.scalar('Performance', self.MeanAcc)
#self.batch = tf.Variable(0, name = "batch_iterator")
self.train_prediction = tf.nn.softmax(self.logits)
self.test_prediction = tf.nn.softmax(self.logits)
tf.global_variables_initializer().run()
print('Computational graph initialised')
def _build_net(self):
with tf.variable_scope(self.name):
self.training = tf.placeholder(tf.bool)
self.X = tf.placeholder(tf.float32, [None, 30])
x_in = tf.reshape(self.X, [-1, 30, 1])
self.Y = tf.placeholder(tf.float32, [None, 2])
conv1 = tf.layers.conv1d(inputs=x_in,
kernel_size=3,
filters=30,
padding="SAME",
activation=tf.nn.relu)
# conv1 후에 [None,30,30] 로 나뉘어짐
pool1 = tf.layers.max_pooling1d(inputs=conv1,
pool_size=2,
padding="SAME",
strides=2)
# pool1 후에 [None,15,30]
dropout1 = tf.layers.dropout(inputs=pool1,
rate=0.7,
training=self.training)
conv2 = tf.layers.conv1d(inputs=dropout1,
filters=60,
kernel_size=3,
padding="SAME",
activation=tf.nn.relu)
# conv2 후에 [None,15,60]
pool2 = tf.layers.max_pooling1d(inputs=conv2,
pool_size=2,
padding="SAME",
strides=2)
# pool2 후에 [None,8,60]
dropout2 = tf.layers.dropout(inputs=pool2,
rate=0.7,
training=self.training)
conv3 = tf.layers.conv1d(inputs=dropout2,
filters=60,
kernel_size=3,
padding="SAME",
activation=tf.nn.relu)
# conv2 후에 [None,8,60]
pool3 = tf.layers.max_pooling1d(inputs=conv3,
pool_size=2,
padding="SAME",
strides=2)
# pool2 후에 [None,4,60]
dropout3 = tf.layers.dropout(inputs=pool3,
rate=0.7,
training=self.training)
flat = tf.reshape(dropout3, [-1, 4 * 60])
dense4 = tf.layers.dense(inputs=flat,
units=120,
activation=tf.nn.relu)
dropout4 = tf.layers.dropout(inputs=dense4,
rate=0.7,
training=self.training)
self.logits = tf.layers.dense(inputs=dropout4, units=2)
self.cost = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(logits=self.logits,
labels=self.Y))
self.optimizer = tf.train.AdamOptimizer(
learning_rate=learning_rate).minimize(self.cost)
correct_prediction = tf.equal(tf.argmax(self.logits, 1),
tf.argmax(self.Y, 1))
self.accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
# P: 유방암이라고 예측. N: 유방암이 아니라고 예측
TP = tf.count_nonzero(
tf.argmax(self.logits, 1) * tf.argmax(self.Y, 1), dtype=tf.float32
) # 왜 이게 될까? -> element wise하게 곱하기 때문에, 예측 ,label 둘다 1인 경우만
TN = tf.count_nonzero(
(tf.argmax(self.logits, 1) - 1) * (tf.argmax(self.Y, 1) - 1),
dtype=tf.float32)
FP = tf.count_nonzero(tf.argmax(self.logits, 1) *
(tf.argmax(self.Y, 1) - 1),
dtype=tf.float32)
FN = tf.count_nonzero(
(tf.argmax(self.logits, 1) - 1) * tf.argmax(self.Y, 1),
dtype=tf.float32)
self.precision = tf.divide(TP, TP + FP) # 내가 유방암이라고 예측한 사람 중에 맞춘 비율
self.recall = tf.divide(TP, TP + FN) # 실제로 유방암인 사람중에 유방암이라고 예측한 비율
self.specificity = tf.divide(TN,
TN + FP) # 실제로 정상인 사람중에 내가 정상이라고 예측한 비율
def get_accuracy(self, y_pred, y_true):
y_pred_maxs =(tf.argmax(y_pred,1))
y_true_maxs =(tf.argmax(y_true,1))
num = tf.count_nonzero((y_true_maxs-y_pred_maxs))
result = 1-(num/self.batch_size)
return result
def rpn_losses(anchor_labels, anchor_boxes, label_logits, box_logits):
"""
Args:
anchor_labels: fHxfWxNA
anchor_boxes: fHxfWxNAx4, encoded
label_logits: fHxfWxNA
box_logits: fHxfWxNAx4
Returns:
label_loss, box_loss
"""
with tf.device('/cpu:0'):
valid_mask = tf.stop_gradient(tf.not_equal(anchor_labels, -1))
pos_mask = tf.stop_gradient(tf.equal(anchor_labels, 1))
nr_valid = tf.stop_gradient(tf.count_nonzero(valid_mask,
dtype=tf.int32),
name='num_valid_anchor')
nr_pos = tf.count_nonzero(pos_mask,
dtype=tf.int32,
name='num_pos_anchor')
valid_anchor_labels = tf.boolean_mask(anchor_labels, valid_mask)
valid_label_logits = tf.boolean_mask(label_logits, valid_mask)
with tf.name_scope('label_metrics'):
valid_label_prob = tf.nn.sigmoid(valid_label_logits)
summaries = []
with tf.device('/cpu:0'):
for th in [0.5, 0.2, 0.1]:
valid_prediction = tf.cast(valid_label_prob > th, tf.int32)
nr_pos_prediction = tf.reduce_sum(valid_prediction,
name='num_pos_prediction')
pos_prediction_corr = tf.count_nonzero(tf.logical_and(
valid_label_prob > th,
tf.equal(valid_prediction, valid_anchor_labels)),
dtype=tf.int32)
summaries.append(
tf.truediv(pos_prediction_corr,
nr_pos,
name='recall_th{}'.format(th)))
precision = tf.to_float(
tf.truediv(pos_prediction_corr, nr_pos_prediction))
precision = tf.where(tf.equal(nr_pos_prediction, 0),
0.0,
precision,
name='precision_th{}'.format(th))
summaries.append(precision)
add_moving_summary(*summaries)
label_loss = tf.nn.sigmoid_cross_entropy_with_logits(
labels=tf.to_float(valid_anchor_labels), logits=valid_label_logits)
label_loss = tf.reduce_mean(label_loss, name='label_loss')
pos_anchor_boxes = tf.boolean_mask(anchor_boxes, pos_mask)
pos_box_logits = tf.boolean_mask(box_logits, pos_mask)
delta = 1.0 / 9
box_loss = tf.losses.huber_loss(pos_anchor_boxes,
pos_box_logits,
delta=delta,
reduction=tf.losses.Reduction.SUM) / delta
box_loss = tf.div(box_loss, tf.cast(nr_valid, tf.float32), name='box_loss')
add_moving_summary(label_loss, box_loss, nr_valid, nr_pos)
return label_loss, box_loss
def __build_graph(self, n_inputs, n_outputs):
"""
Build tensorflow computional graph
"""
if self.random_state:
tf.set_random_seed(self.random_state)
np.random.seed(self.random_state)
X = tf.placeholder(tf.float32, shape=(None, n_inputs), name="X")
y = tf.placeholder(tf.int32, shape=(None, n_outputs), name="y")
training = tf.placeholder_with_default(False,
shape=(),
name="is_training")
logits = self.__build_model(X, y, training=training)
y_proba = tf.nn.softmax(logits, name="Y_proba")
with tf.name_scope("loss"):
xentropy = tf.nn.softmax_cross_entropy_with_logits_v2(
labels=y, logits=logits, name="cross_entropy")
loss = tf.reduce_mean(xentropy, name="loss")
with tf.name_scope("training"):
learning_rate = tf.placeholder(dtype=tf.float32,
shape=None,
name="learning_rate")
if self.optimizer != tf.train.MomentumOptimizer:
optimizer = self.optimizer(learning_rate=learning_rate)
else:
optimizer = self.optimizer(learning_rate=learning_rate,
momentum=0.9)
training_op = optimizer.minimize(loss)
with tf.name_scope("performance"):
#loss summary
loss_summary_ph = tf.placeholder(dtype=tf.float32,
shape=None,
name="loss_summary")
loss_summary = tf.summary.scalar("Loss", loss_summary_ph)
#accurancy summary
accuracy_summary_ph = tf.placeholder(tf.float32,
shape=None,
name='accuracy_summary')
accuracy_summary = tf.summary.scalar('accuracy',
accuracy_summary_ph)
#val loss
val_loss_summary_ph = tf.placeholder(dtype=tf.float32,
shape=None,
name="val_loss_summary")
val_loss_summary = tf.summary.scalar("val_Loss",
val_loss_summary_ph)
#val accurancy summary
val_accuracy_summary_ph = tf.placeholder(
tf.float32, shape=None, name='val_accuracy_summary')
val_accuracy_summary = tf.summary.scalar('val_accuracy',
val_accuracy_summary_ph)
#recall summary
recall_summary_ph = tf.placeholder(dtype=tf.float32,
shape=None,
name="recall_summary")
recall_summary = tf.summary.scalar('recall', recall_summary_ph)
#precision symmary
precision_summary_ph = tf.placeholder(dtype=tf.float32,
shape=None,
name="recall_summary")
precision_summary = tf.summary.scalar('precision',
recall_summary_ph)
#merged_summaries = tf.summary.merge([loss_summary, accuracy_summary, recall_summary, precision_summary])
merged_summaries = tf.summary.merge_all()
with tf.name_scope("accurancy_metrics"):
argmax_prediction = tf.argmax(y_proba, 1)
argmax_y = tf.argmax(y, 1)
#needed values for recall and precison calc
TP = tf.count_nonzero(argmax_prediction * argmax_y,
dtype=tf.float32)
TN = tf.count_nonzero((argmax_prediction - 1) * (argmax_y - 1),
dtype=tf.float32)
FP = tf.count_nonzero(argmax_prediction * (argmax_y - 1),
dtype=tf.float32)
FN = tf.count_nonzero((argmax_prediction - 1) * argmax_y,
dtype=tf.float32)
"""TP = tf.count_nonzero(y_proba * y, dtype=tf.float32)
TN = tf.count_nonzero((y_proba - 1) * (y - 1), dtype=tf.float32)
FP = tf.count_nonzero(y_proba * (y - 1), dtype=tf.float32)
FN = tf.count_nonzero((y_proba - 1) * y, dtype=tf.float32)"""
##calcs:
precision = TP / (TP + FP)
recall = TP / (TP + FN)
acc = tf.reduce_mean(
tf.cast(tf.equal(argmax_prediction, argmax_y), tf.float32))
with tf.name_scope("initialization"):
init = tf.global_variables_initializer()
saver = tf.train.Saver()
with tf.name_scope("extra_operations"):
extra_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
if self.kernel_regulizer == max_norm_regularizer():
self._clip_weights = tf.get_collection("max_norm")
## variable sharing for graph computation periods
self._X, self._y, self._training = X, y, training
self._learning_rate = learning_rate
self._y_proba, self._loss = y_proba, loss
#self._training_op, self._accuracy = training_op, accuracy
self._training_op = training_op
self._init, self._saver = init, saver
self._extra_ops = extra_ops
self._loss_summary_ph, self._loss_summary = loss_summary_ph, loss_summary
self._accuracy_summary_ph, self._accuracy_summary = accuracy_summary_ph, accuracy_summary
self._recall_summary_ph, self._recall_summary = recall_summary_ph, recall_summary
self._precision_summary_ph, self._precision_summary = precision_summary_ph, precision_summary
self._val_loss_summary_ph, self._val_loss_summary = val_loss_summary_ph, val_loss_summary
self._val_accuracy_summary_ph, self._val_accuracy_summary = val_accuracy_summary_ph, val_accuracy_summary
self._merged_summaries = merged_summaries
##eval metrics
self._rec, self._prec, self._acc_formula = precision, recall, acc
k = 2
num_sim = 2
import sys
import numpy as np
np.set_printoptions(threshold=sys.maxsize)
# tf_a1 = np.load('testnew.npy')
# print tf_a1
# tf_a1 = tfidf_matrix.A
sim_topics = cosine_score(tf_a1)
threshold = tf.reduce_mean(sim_topics)
# print sim_topics
masked_t_bad = tf.greater(
tf.count_nonzero(tf.greater(sim_topics, threshold), axis=1), num_sim)
# print masked_t_bad
ent_p = rev_entropy(tf_a1)
p = (tf_a1 + tf.abs(tf_a1)) / 2
ent_threshold = tf.reduce_mean(ent_p)
ent_mask_p = tf.greater(ent_p, ent_threshold)
# print ent_mask_p
topk_bad_rows = tf.where(tf.logical_not(ent_mask_p), tf_a1,
tf.zeros_like(tf_a1))
topk_bad_rows_value, ind = tf.nn.top_k(topk_bad_rows, 2)
my_range = tf.expand_dims(tf.range(0,
tf.shape(ind)[0]), 1) # will be [[0], [1]]
my_range_repeated = tf.tile(my_range, [1, 4 / 2]) # will be [[0, 0], [1, 1]]
full_indices = tf.stack(
plt.ylabel('Tumor size(in cm')
plt.show()
lab_dim=num_classes
input_features=tf.placeholder(tf.float32,[None,input_dim])
input_label=tf.placeholder(tf.float32,[None,lab_dim])
W=tf.Variable(tf.random_normal([input_dim,lab_dim]),name='weight')
b=tf.Variable(tf.zeros([lab_dim]),name='bias')
output=tf.matmul(input_features,W)+b
z=tf.nn.softmax(output)
a1=tf.argmax(tf.nn.softmax(output),axis=1)
b1=tf.argmax(input_label,axis=1)
err=tf.count_nonzero(a1-b1)
cross_entropy=tf.nn.softmax_cross_entropy_with_logits(labels=input_label,logits=output)
loss=tf.reduce_mean(cross_entropy)
optimizer=tf.train.AdamOptimizer(0.04)
train=optimizer.minimize(loss)
maxEpochs=50
minibatchSize=25
#setup session
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
def xdet_model_fn(features, labels, mode, params):
"""Our model_fn for ResNet to be used with our Estimator."""
num_anchors_list = labels['num_anchors_list']
num_feature_layers = len(num_anchors_list)
shape = labels['targets'][-1]
glabels = labels['targets'][:num_feature_layers][0]
gtargets = labels['targets'][num_feature_layers : 2 * num_feature_layers][0]
gscores = labels['targets'][2 * num_feature_layers : 3 * num_feature_layers][0]
with tf.variable_scope(params['model_scope'], default_name = None, values = [features], reuse=tf.AUTO_REUSE):
backbone = xdet_body_v3.xdet_resnet_v3(params['resnet_size'], params['data_format'])
body_cls_output, body_regress_output = backbone(inputs=features, is_training=(mode == tf.estimator.ModeKeys.TRAIN))
cls_pred, location_pred = xdet_body_v3.xdet_head(body_cls_output, body_regress_output, params['num_classes'], num_anchors_list[0], (mode == tf.estimator.ModeKeys.TRAIN), data_format=params['data_format'])
if params['data_format'] == 'channels_first':
cls_pred = tf.transpose(cls_pred, [0, 2, 3, 1])
location_pred = tf.transpose(location_pred, [0, 2, 3, 1])
bboxes_pred = labels['decode_fn'](location_pred)#(tf.reshape(location_pred, tf.shape(location_pred).as_list()[0:-1] + [-1, 4]))
cls_pred = tf.reshape(cls_pred, [-1, params['num_classes']])
location_pred = tf.reshape(location_pred, [-1, 4])
glabels = tf.reshape(glabels, [-1])
gscores = tf.reshape(gscores, [-1])
gtargets = tf.reshape(gtargets, [-1, 4])
# raw mask for positive > 0.5, and for negetive < 0.3
# each positive examples has one label
positive_mask = glabels > 0#tf.logical_and(glabels > 0, gscores > params['match_threshold'])
fpositive_mask = tf.cast(positive_mask, tf.float32)
n_positives = tf.reduce_sum(fpositive_mask)
batch_glabels = tf.reshape(glabels, [tf.shape(features)[0], -1])
batch_n_positives = tf.count_nonzero(batch_glabels, -1)
batch_negtive_mask = tf.equal(batch_glabels, 0)
batch_n_negtives = tf.count_nonzero(batch_negtive_mask, -1)
batch_n_neg_select = tf.cast(params['negative_ratio'] * tf.cast(batch_n_positives, tf.float32), tf.int32)
batch_n_neg_select = tf.minimum(batch_n_neg_select, tf.cast(batch_n_negtives, tf.int32))
# hard negative mining for classification
predictions_for_bg = tf.nn.softmax(tf.reshape(cls_pred, [tf.shape(features)[0], -1, params['num_classes']]))[:, :, 0]
prob_for_negtives = tf.where(batch_negtive_mask,
0. - predictions_for_bg,
# ignore all the positives
0. - tf.ones_like(predictions_for_bg))
topk_prob_for_bg, _ = tf.nn.top_k(prob_for_negtives, k=tf.shape(prob_for_negtives)[1])
score_at_k = tf.gather_nd(topk_prob_for_bg, tf.stack([tf.range(tf.shape(features)[0]), batch_n_neg_select - 1], axis=-1))
selected_neg_mask = prob_for_negtives >= tf.expand_dims(score_at_k, axis=-1)
negtive_mask = tf.reshape(tf.logical_and(batch_negtive_mask, selected_neg_mask), [-1])#tf.logical_and(tf.equal(glabels, 0), gscores > 0.)
#negtive_mask = tf.logical_and(tf.logical_and(tf.logical_not(positive_mask), gscores < params['neg_threshold']), gscores > 0.)
#negtive_mask = tf.logical_and(gscores < params['neg_threshold'], tf.logical_not(positive_mask))
# # random select negtive examples for classification
# selected_neg_mask = tf.random_uniform(tf.shape(gscores), minval=0, maxval=1.) < tf.where(
# tf.greater(n_negtives, 0),
# tf.divide(tf.cast(n_neg_to_select, tf.float32), n_negtives),
# tf.zeros_like(tf.cast(n_neg_to_select, tf.float32)),
# name='rand_select_negtive')
# include both selected negtive and all positive examples
final_mask = tf.stop_gradient(tf.logical_or(negtive_mask, positive_mask))
total_examples = tf.reduce_sum(tf.cast(final_mask, tf.float32))
# add mask for glabels and cls_pred here
glabels = tf.boolean_mask(tf.clip_by_value(glabels, 0, FLAGS.num_classes), tf.stop_gradient(final_mask))
cls_pred = tf.boolean_mask(cls_pred, tf.stop_gradient(final_mask))
location_pred = tf.boolean_mask(location_pred, tf.stop_gradient(positive_mask))
gtargets = tf.boolean_mask(gtargets, tf.stop_gradient(positive_mask))
predictions = {
'classes': tf.argmax(cls_pred, axis=-1),
'probabilities': tf.reduce_max(tf.nn.softmax(cls_pred, name='softmax_tensor'), axis=-1),
'bboxes_predict': tf.reshape(bboxes_pred, [-1, 4]) }
if mode == tf.estimator.ModeKeys.PREDICT:
return tf.estimator.EstimatorSpec(mode=mode, predictions=predictions)
# Calculate loss, which includes softmax cross entropy and L2 regularization.
cross_entropy = tf.cond(n_positives > 0., lambda: tf.losses.sparse_softmax_cross_entropy(labels=glabels, logits=cls_pred), lambda: 0.)
#cross_entropy = tf.losses.sparse_softmax_cross_entropy(labels=glabels, logits=cls_pred)
# Create a tensor named cross_entropy for logging purposes.
tf.identity(cross_entropy, name='cross_entropy_loss')
tf.summary.scalar('cross_entropy_loss', cross_entropy)
loc_loss = tf.cond(n_positives > 0., lambda: modified_smooth_l1(location_pred, tf.stop_gradient(gtargets), sigma=1.), lambda: tf.zeros_like(location_pred))
#loc_loss = modified_smooth_l1(location_pred, tf.stop_gradient(gtargets))
loc_loss = tf.reduce_mean(tf.reduce_sum(loc_loss, axis=-1))
loc_loss = tf.identity(loc_loss, name='location_loss')
tf.summary.scalar('location_loss', loc_loss)
tf.losses.add_loss(loc_loss)
# Add weight decay to the loss. We exclude the batch norm variables because
# doing so leads to a small improvement in accuracy.
loss = cross_entropy + loc_loss + params['weight_decay'] * tf.add_n(
[tf.nn.l2_loss(v) for v in tf.trainable_variables()
if 'batch_normalization' not in v.name])
total_loss = tf.identity(loss, name='total_loss')
if mode == tf.estimator.ModeKeys.TRAIN:
global_step = tf.train.get_or_create_global_step()
lr_values = [params['learning_rate'] * decay for decay in params['lr_decay_factors']]
learning_rate = tf.train.piecewise_constant(tf.cast(global_step, tf.int32),
[int(_) for _ in params['decay_boundaries']],
lr_values)
truncated_learning_rate = tf.maximum(learning_rate, tf.constant(params['end_learning_rate'], dtype=learning_rate.dtype))
# Create a tensor named learning_rate for logging purposes.
tf.identity(truncated_learning_rate, name='learning_rate')
tf.summary.scalar('learning_rate', truncated_learning_rate)
optimizer = tf.train.MomentumOptimizer(learning_rate=truncated_learning_rate,
momentum=params['momentum'])
# Batch norm requires update_ops to be added as a train_op dependency.
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
with tf.control_dependencies(update_ops):
train_op = optimizer.minimize(loss, global_step)
else:
train_op = None
cls_accuracy = tf.metrics.accuracy(glabels, predictions['classes'])
metrics = {'cls_accuracy': cls_accuracy}
# Create a tensor named train_accuracy for logging purposes.
tf.identity(cls_accuracy[1], name='cls_accuracy')
tf.summary.scalar('cls_accuracy', cls_accuracy[1])
return tf.estimator.EstimatorSpec(
mode=mode,
predictions=predictions,
loss=loss,
train_op=train_op,
eval_metric_ops=metrics,
scaffold = tf.train.Scaffold(init_fn=train_helper.get_init_fn_for_scaffold(FLAGS)))
def bboxes_matching(label, scores, bboxes,
glabels, gbboxes, gdifficults,
matching_threshold=0.5, scope=None):
"""Matching a collection of detected boxes with groundtruth values.
Does not accept batched-inputs.
The algorithm goes as follows: for every detected box, check
if one grountruth box is matching. If none, then considered as False Positive.
If the grountruth box is already matched with another one, it also counts
as a False Positive. We refer the Pascal VOC documentation for the details.
Args:
rclasses, rscores, rbboxes: N(x4) Tensors. Detected objects, sorted by score;
glabels, gbboxes: Groundtruth bounding boxes. May be zero padded, hence
zero-class objects are ignored.
matching_threshold: Threshold for a positive match.
Return: Tuple of:
n_gbboxes: Scalar Tensor with number of groundtruth boxes (may difer from
size because of zero padding).
tp_match: (N,)-shaped boolean Tensor containing with True Positives.
fp_match: (N,)-shaped boolean Tensor containing with False Positives.
"""
with tf.name_scope(scope, 'bboxes_matching_single',
[scores, bboxes, glabels, gbboxes]):
rsize = tf.size(scores)
rshape = tf.shape(scores)
rlabel = tf.cast(label, glabels.dtype)
# Number of groundtruth boxes.
gdifficults = tf.cast(gdifficults, tf.bool)
n_gbboxes = tf.count_nonzero(tf.logical_and(tf.equal(glabels, label),
tf.logical_not(gdifficults)))
# Grountruth matching arrays.
gmatch = tf.zeros(tf.shape(glabels), dtype=tf.bool)
grange = tf.range(tf.size(glabels), dtype=tf.int32)
# True/False positive matching TensorArrays.
sdtype = tf.bool
ta_tp_bool = tf.TensorArray(sdtype, size=rsize, dynamic_size=False, infer_shape=True)
ta_fp_bool = tf.TensorArray(sdtype, size=rsize, dynamic_size=False, infer_shape=True)
# Loop over returned objects.
def m_condition(i, ta_tp, ta_fp, gmatch):
r = tf.less(i, rsize)
return r
def m_body(i, ta_tp, ta_fp, gmatch):
# Jaccard score with groundtruth bboxes.
rbbox = bboxes[i]
jaccard = bboxes_jaccard(rbbox, gbboxes)
jaccard = jaccard * tf.cast(tf.equal(glabels, rlabel), dtype=jaccard.dtype)
# Best fit, checking it's above threshold.
idxmax = tf.cast(tf.argmax(jaccard, axis=0), tf.int32)
jcdmax = jaccard[idxmax]
match = jcdmax > matching_threshold
existing_match = gmatch[idxmax]
not_difficult = tf.logical_not(gdifficults[idxmax])
# TP: match & no previous match and FP: previous match | no match.
# If difficult: no record, i.e FP=False and TP=False.
tp = tf.logical_and(not_difficult,
tf.logical_and(match, tf.logical_not(existing_match)))
ta_tp = ta_tp.write(i, tp)
fp = tf.logical_and(not_difficult,
tf.logical_or(existing_match, tf.logical_not(match)))
ta_fp = ta_fp.write(i, fp)
# Update grountruth match.
mask = tf.logical_and(tf.equal(grange, idxmax),
tf.logical_and(not_difficult, match))
gmatch = tf.logical_or(gmatch, mask)
return [i+1, ta_tp, ta_fp, gmatch]
# Main loop definition.
i = 0
[i, ta_tp_bool, ta_fp_bool, gmatch] = \
tf.while_loop(m_condition, m_body,
[i, ta_tp_bool, ta_fp_bool, gmatch],
parallel_iterations=1,
back_prop=False)
# TensorArrays to Tensors and reshape.
tp_match = tf.reshape(ta_tp_bool.stack(), rshape)
fp_match = tf.reshape(ta_fp_bool.stack(), rshape)
# Some debugging information...
# tp_match = tf.Print(tp_match,
# [n_gbboxes,
# tf.reduce_sum(tf.cast(tp_match, tf.int64)),
# tf.reduce_sum(tf.cast(fp_match, tf.int64)),
# tf.reduce_sum(tf.cast(gmatch, tf.int64))],
# 'Matching (NG, TP, FP, GM): ')
return n_gbboxes, tp_match, fp_match
def step(index, scores_sum, scores_num):
"""Single step."""
index %= epoch_length # Only needed in eval runs.
# Note - the only way to ensure making a copy of tensor is to run simple
# operation. We are waiting for tf.copy:
# https://github.com/tensorflow/tensorflow/issues/11186
obs_copy = batch_env.observ + 0
def env_step(arg1, arg2, arg3): # pylint: disable=unused-argument
"""Step of the environment."""
actor_critic = get_policy(tf.expand_dims(obs_copy, 0),
ppo_hparams, batch_env.action_space)
policy = actor_critic.policy
action = policy_to_actions_lambda(policy)
postprocessed_action = actor_critic.action_postprocessing(
action)
reward, done = batch_env.simulate(postprocessed_action[0, ...])
pdf = policy.prob(action)[0]
value_function = actor_critic.value[0]
pdf = tf.reshape(pdf, shape=(num_agents, ))
value_function = tf.reshape(value_function,
shape=(num_agents, ))
done = tf.reshape(done, shape=(num_agents, ))
with tf.control_dependencies([reward, done]):
return tf.identity(pdf), tf.identity(value_function), \
tf.identity(done)
# TODO(piotrmilos): while_body is executed at most once,
# thus should be replaced with tf.cond
pdf, value_function, top_level_done = tf.while_loop(
lambda _1, _2, _3: tf.equal(speculum.size(), 0),
env_step,
[
tf.constant(0.0, shape=(num_agents, )),
tf.constant(0.0, shape=(num_agents, )),
tf.constant(False, shape=(num_agents, ))
],
parallel_iterations=1,
back_prop=False,
)
with tf.control_dependencies([pdf, value_function]):
obs, reward, done, action = speculum.dequeue()
to_save = [obs, reward, done, action, pdf, value_function]
save_ops = [
tf.scatter_update(memory_slot, index, value)
for memory_slot, value in zip(memory, to_save)
]
cumulate_rewards_op = cumulative_rewards.assign_add(reward)
agent_indices_to_reset = tf.where(top_level_done)[:, 0]
with tf.control_dependencies([cumulate_rewards_op]):
# TODO(piotrmilos): possibly we need cumulative_rewards.read_value()
scores_sum_delta = tf.reduce_sum(
tf.gather(cumulative_rewards.read_value(),
agent_indices_to_reset))
scores_num_delta = tf.count_nonzero(done, dtype=tf.int32)
with tf.control_dependencies(save_ops +
[scores_sum_delta, scores_num_delta]):
reset_env_op = batch_env.reset(agent_indices_to_reset)
reset_cumulative_rewards_op = tf.scatter_update(
cumulative_rewards, agent_indices_to_reset,
tf.gather(zeros_tensor, agent_indices_to_reset))
with tf.control_dependencies(
[reset_env_op, reset_cumulative_rewards_op]):
return [
index + 1, scores_sum + scores_sum_delta,
scores_num + scores_num_delta
]
def build_model(self,):
self.input_net['d'] = tf.placeholder(tf.int32,[None,self.d_max_sent,self.d_max_length])
self.input_net['q'] = tf.placeholder(tf.int32,[None,self.q_max_length])
#self.input_net['a'] = tf.placeholder(tf.int32,[None,self.a_max_length])
self.input_net['d_mask'] = tf.placeholder(tf.int32,[None,self.d_max_sent])
self.input_net['q_mask'] = tf.placeholder(tf.int32,[None])
#self.input_net['a_mask'] = tf.placeholder(tf.int32,[None])
self.input_net['labels'] = tf.placeholder(tf.float32,[None,self.num_class])
self.input_net['d_sent_mask'] = tf.placeholder(tf.int32,[None])
self.W = tf.Variable(tf.random_uniform([self.num_symbol,self.embedding_size],-3**0.5,3**0.5),name="embedding")
self.l2_loss += tf.nn.l2_loss(self.W)
inner = self.rnn_size
w1 = tf.get_variable("w1",[self.rnn_size*4,inner])#,initializer=tf.contrib.layers.xavier_initializer())
#self.l2_loss += tf.nn.l2_loss(w1)
b1 = tf.get_variable("b1",[1,inner])#,initializer=tf.contrib.layers.xavier_initializer())
w2 = tf.get_variable("w2",[inner,1])#,initializer=tf.contrib.layers.xavier_initializer())
#self.l2_loss += tf.nn.l2_loss(w2)
b2 = tf.get_variable("b2",[1,1])#,initializer=tf.contrib.layers.xavier_initializer())
classifer_w = tf.get_variable("classifer_w",[inner*2,2])#,initializer=tf.contrib.layers.xavier_initializer())
self.l2_loss += tf.nn.l2_loss(classifer_w)
classifer_b = tf.get_variable("classifer_b",[2])#,initializer=tf.contrib.layers.xavier_initializer())
#w4 = tf.get_variable("w4",[33,self.num_class],initializer=tf.contrib.layers.xavier_initializer())
#b4 = tf.get_variable("b4",[self.num_class],initializer=tf.contrib.layers.xavier_initializer())
#document
reader_out = []
#1
'''
# East to overfit?
for i in range(self.d_max_sent):
with tf.variable_scope('reader') as vs:
if i>0: vs.reuse_variables()
temp, _ = tf.nn.dynamic_rnn(self.cell,
tf.nn.embedding_lookup(self.W,self.input_net['d'][:,i]),
sequence_length=self.input_net['d_mask'][:,i],
dtype=tf.float32)
reader_out.append(last_relevant(temp,self.input_net['d_mask'][:,i]))
'''
#2 Position Encoding
ps = self.positional_encoding(self.embedding_size, self.d_max_length)
input_embed = [tf.nn.embedding_lookup(self.W,sent) for sent in tf.unpack(self.input_net['d'],axis=1)]
#len = self.d_max_sent
d_mask = [tf.sequence_mask(mask,self.d_max_length,dtype=tf.float32) for mask in tf.unpack(self.input_net['d_mask'],axis=1)]
reader_out = [tf.reduce_sum(ps * input_embed[i] * tf.expand_dims(d_mask[i],axis=2) ,axis=1) for i in range(self.d_max_sent)]
# reader_out is a d_max_sent list of 2D Tensors, shape(None, embedding_size)
#question
#1
with tf.variable_scope('reader2') as vs:
#vs.reuse_variables()
temp, _ = tf.nn.dynamic_rnn(self.cell,
tf.nn.embedding_lookup(self.W,self.input_net['q']),
sequence_length=self.input_net['q_mask'],
dtype=tf.float32)
last_q = last_relevant(temp,self.input_net['q_mask'])
##2
#ps = self.positional_encoding(self.q_max_length,self.embedding_size)
#q_embed = tf.nn.embedding_lookup(self.W, self.input_net['q'])
#q_mask = tf.sequence_mask(self.input_net['q_mask'], self.q_max_length, dtype=tf.float32)
#last_q = tf.reduce_sum(ps * q_embed * tf.expand_dims(q_mask,axis=2) ,axis=1)
# paragraph vector
# 1. Using RNN
if self.encode_type == "rnn":
with tf.variable_scope('paragraph'):
temp, _ = tf.nn.bidirectional_dynamic_rnn(
self.fw_cell,
self.bw_cell,
tf.pack(reader_out,axis=1),
sequence_length=self.input_net['d_sent_mask'],
dtype=tf.float32)
reader_out = tf.reduce_sum(tf.stack(temp),axis=0)
input_feature = last_relevant(reader_out, self.input_net['d_sent_mask'])
elif self.encode_type == "cnn":
# 2. Using CNN with MaxPooling
reader_out = tf.expand_dims(tf.pack(reader_out, axis=1), -1)
filter_sizes = [3]
pooled_output = []
for i, filter_size in enumerate(filter_sizes):
with tf.name_scope('conv-maxpool-%s' % filter_size):
filter_shape = [filter_size, self.embedding_size, 1, self.rnn_size] # rnn_size => # of filter
conv_W = tf.Variable(tf.truncated_normal(filter_shape, stddev = 0.1), name="conv_W")
conv_b = tf.Variable(tf.constant(0.1, shape=[self.rnn_size], name='conv_b'))
conv = tf.nn.conv2d(reader_out,
conv_W,
strides=[1, 1, 1, 1],
padding="VALID",
name="conv")
h = tf.nn.relu(conv + conv_b ,name="relu")
pooled = tf.nn.max_pool(
h,
ksize=[1, self.d_max_sent - filter_size + 1, 1, 1],
strides=[1, 1, 1, 1],
padding="VALID",
name="pooled")
pooled_output.append(pooled)
num_filters_total = self.rnn_size * len(filter_sizes)
h_pool = tf.concat(3, pooled_output)
input_feature = tf.reshape(h_pool, [-1, num_filters_total])
#input_feature = output[:,0,:]#tf.concat(1, [reader_out, last_q] )
self.prob = tf.matmul(tf.concat(1,[input_feature,last_q]), classifer_w) + classifer_b#),w4 )+ b4
self.output_net['loss'] = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits = self.prob,
labels = self.input_net['labels']) )
#pos_weight=5))
self.predict = tf.argmax(self.prob, 1)
actual = tf.argmax(self.input_net['labels'],axis=1)
self.tp = tf.count_nonzero(self.predict * actual)
self.tn = tf.count_nonzero((self.predict-1)*(actual-1))
self.fp = tf.count_nonzero(self.predict*(actual-1))
self.fn = tf.count_nonzero((self.predict-1)*actual)
#self.acc = tf.reduce_sum(tf.cast(tf.equal(self.predict,actual),tf.float32))
# Update
#self.opti = tf.train.GradientDescentOptimizer(0.01)#.minimize(self.output_net['loss'])
self.update = tf.train.AdamOptimizer(0.005).minimize(self.output_net['loss'])
#grads_and_vars = self.opti.compute_gradients(self.output_net['loss']+0.0005*l2_loss)
#capped_grads_and_vars = [ (tf.clip_by_value(gv[0], -0.1, 0.1), gv[1]) for gv in grads_and_vars ]
#self.update = self.opti.apply_gradients(capped_grads_and_vars)
init = tf.global_variables_initializer()#
self.sess.run(init)
def __init__(self, size_layer, num_layers, learning_rate, vocab_file,
bert_config, is_training):
def cells(reuse=False):
return tf.nn.rnn_cell.BasicRNNCell(size_layer, reuse=reuse)
self.bert_config = modeling.BertConfig.from_json_file(bert_config)
self.tokenizer = tokenization.FullTokenizer(vocab_file=vocab_file, )
self.is_training = is_training
self.input_ids = tf.placeholder(dtype=tf.int32,
shape=[None, None],
name="input_ids")
self.input_mask = tf.placeholder(dtype=tf.int32,
shape=[None, None],
name="input_mask")
self.segment_ids = tf.placeholder(dtype=tf.int32,
shape=[None, None],
name="segment_ids")
self.dropout = tf.placeholder(dtype=tf.float32,
shape=None,
name="dropout")
model = modeling.BertModel(config=self.bert_config,
is_training=self.is_training,
input_ids=self.input_ids,
input_mask=self.input_mask,
token_type_ids=self.segment_ids,
use_one_hot_embeddings=False)
self.embedded = model.get_sequence_output()
self.model_inputs = tf.nn.dropout(self.embedded, self.dropout)
# self.X = tf.placeholder(tf.int32, [None, None],name = "X")
self.Y = tf.placeholder(tf.int32, [None, None], name="Y")
self.X_seq_len = tf.count_nonzero(self.input_ids, 1, dtype=tf.int32)
self.Y_seq_len = tf.count_nonzero(self.Y, 1, dtype=tf.int32)
self.global_step = tf.Variable(1, name="global_step", trainable=False)
batch_size = tf.shape(self.input_ids)[0]
# batch_size = tf.shape(self.input_ids)[0]
# batch_size = tf.sign(tf.abs(self.input_ids))
# batch_size = tf.reduce_sum(batch_size, reduction_indices=1)
with tf.variable_scope("encoder"):
# self.encoder_embedding = tf.Variable(tf.random_uniform([from_dict_size, embedded_size], -1, 1),name ="encoder_embedding")
self.embedded = model.get_sequence_output()
self.model_inputs = tf.nn.dropout(self.embedded, self.dropout)
_, encoder_state = tf.nn.dynamic_rnn(
cell=tf.nn.rnn_cell.MultiRNNCell(
[cells() for _ in range(num_layers)]),
inputs=self.model_inputs,
sequence_length=self.X_seq_len,
dtype=tf.float32)
with tf.variable_scope("decoder"):
# self.decoder_embedding = tf.Variable(tf.random_uniform([to_dict_size, embedded_size], -1, 1),
# name="decoder_embedding")
self.decoder_embedding = model.get_embedding_table()
main = tf.strided_slice(self.Y, [0, 0], [batch_size, -1], [1, 1])
decoder_input = tf.concat([
tf.fill([batch_size, 1], self.tokenizer.vocab["[SEP]"]), main
], 1)
dense = tf.layers.Dense(len(self.tokenizer.vocab))
decoder_cells = tf.nn.rnn_cell.MultiRNNCell(
[cells() for _ in range(num_layers)])
training_helper = tf.contrib.seq2seq.TrainingHelper(
inputs=tf.nn.embedding_lookup(self.decoder_embedding,
decoder_input),
sequence_length=self.Y_seq_len,
time_major=False)
training_decoder = tf.contrib.seq2seq.BasicDecoder(
cell=decoder_cells,
helper=training_helper,
initial_state=encoder_state,
output_layer=dense)
training_decoder_output, _, _ = tf.contrib.seq2seq.dynamic_decode(
decoder=training_decoder,
impute_finished=True,
maximum_iterations=tf.reduce_max(self.Y_seq_len))
self.training_logits = training_decoder_output.rnn_output
predicting_helper = tf.contrib.seq2seq.GreedyEmbeddingHelper(
embedding=self.decoder_embedding,
start_tokens=tf.tile(
tf.constant([self.tokenizer.vocab["[SEP]"]],
dtype=tf.int32), [batch_size]),
end_token=self.tokenizer.vocab["[CLS]"])
predicting_decoder = tf.contrib.seq2seq.BasicDecoder(
cell=decoder_cells,
helper=predicting_helper,
initial_state=encoder_state,
output_layer=dense)
predicting_decoder_output, _, _ = tf.contrib.seq2seq.dynamic_decode(
decoder=predicting_decoder,
impute_finished=True,
maximum_iterations=128
# maximum_iterations=2 * tf.reduce_max(self.X_seq_len)
)
self.predicting_ids = predicting_decoder_output.sample_id
self.masks = tf.sequence_mask(self.Y_seq_len,
tf.reduce_max(self.Y_seq_len),
dtype=tf.float32)
self.cost = tf.contrib.seq2seq.sequence_loss(
logits=self.training_logits, targets=self.Y, weights=self.masks)
self.optimizer = tf.train.AdamOptimizer(
learning_rate=learning_rate).minimize(self.cost,
global_step=self.global_step,
name="optimizer")
y_t = tf.argmax(self.training_logits, axis=2)
y_t = tf.cast(y_t, tf.int32)
self.prediction = tf.boolean_mask(y_t, self.masks, name="prediction")
self.mask_label = tf.boolean_mask(self.Y,
self.masks,
name="mask_label")
self.correct_pred = tf.equal(self.prediction, self.mask_label)
self.correct_index = tf.cast(self.correct_pred, tf.float32)
self.accuracy = tf.reduce_mean(tf.cast(self.correct_pred, tf.float32))
def __init__(self,
input_data,
targets,
difficulty,
target_mask,
sequence_length,
params,
is_training=False):
"""
input_data: If non-sequence, then batch_size x feature_size
otherwise batch_size x max_sequence_length x feature_size
targets: If non-sequence, then batch_size x num_classes
otherwise batch_size x max_sequence_length x num_classes
sequence_length: If non-sequence, then None else tensor with shape batch_size
"""
self.targets = targets
self.params = params
self.batch_size = params.batch_size
self.hidden_size = params.hidden_size
self.clip_grad_norm = params.clip_grad_norm
self.use_lstm = params.use_lstm
self.difficulty = difficulty
self.max_difficulty = params.max_difficulty
self.target_mask = target_mask
# self.input_data has to be a (length max_sequence_length) list of tensors
# with shape batch_size x feature_size
self.input_data = tf.cast(input_data, tf.float32)
if self.input_data.shape.ndims == 2:
self.input_data = [self.input_data]
assert sequence_length is None, 'Non-sequential inputs should leave sequence_length=None'
sequence_length = tf.constant([1] * self.batch_size)
elif self.input_data.shape.ndims == 3:
self.input_data = tf.split(
self.input_data, num_or_size_splits=self.input_data.shape[1], axis=1)
self.input_data = [tf.squeeze(t, axis=1) for t in self.input_data]
else:
raise Exception('Input has to be of rank 2 or 3')
# Set up ACT cell and inner rnn-type cell for use inside the ACT cell.
with tf.variable_scope("rnn"):
if self.use_lstm:
inner_cell = BasicLSTMCell(self.hidden_size, state_is_tuple=False)
else:
inner_cell = GRUCell(self.hidden_size)
with tf.variable_scope("ACT"):
act = ACTCell(
self.hidden_size,
inner_cell,
params.epsilon,
use_new_ponder_cost=params.use_new_ponder_cost,
max_computation=params.max_computation,
batch_size=self.batch_size,
difficulty=difficulty)
self.outputs, _ = tf.nn.static_rnn(
cell=act, inputs=self.input_data, dtype=tf.float32)
output = tf.stack(self.outputs, axis=1)
self.logits = tf.layers.dense(output, params.num_classes)
if params.data == "addition":
# reshape logits and labels to (batch size, sequence, digits, one hot)
self.logits = tf.reshape(
self.logits,
shape=(params.batch_size, params.max_difficulty,
params.num_digits + 1, 10))
self.targets = tf.reshape(
self.targets,
shape=(params.batch_size, params.max_difficulty,
params.num_digits + 1, 10))
self.predictions = tf.nn.softmax(self.logits)
self.target_mask = tf.cast(self.target_mask, tf.float32)
ce = tf.nn.softmax_cross_entropy_with_logits_v2(
labels=self.targets, logits=self.logits, dim=-1)
masked_ce = self.target_mask * ce
masked_reduced_ce = sparse_mean(masked_ce)
# Compute the cross entropy based pondering cost multiplier
avg_ce = tf.Variable(initial_value=0.7, trainable=False)
avg_ce_decay = 0.85
avg_ce_update_op = tf.assign(
avg_ce,
avg_ce_decay * avg_ce + (1.0 - avg_ce_decay) * masked_reduced_ce)
with tf.control_dependencies([avg_ce_update_op]):
inverse_difficulty = safe_div(avg_ce, masked_ce)
inverse_difficulty /= sparse_mean(inverse_difficulty)
# ponder_v2 has NaN problem in its backward pass without this
inverse_difficulty = tf.stop_gradient(inverse_difficulty)
# Add up loss and retrieve batch-normalised ponder cost: sum N + sum
# Remainder
ponder_cost = act.calculate_ponder_cost(
time_penalty=self.params.ponder_time_penalty,
inverse_difficulty=inverse_difficulty)
masked_reduced_ponder_cost = sparse_mean(self.target_mask * ponder_cost)
self.cost = masked_reduced_ce + masked_reduced_ponder_cost
if is_training:
tvars = tf.trainable_variables()
grads, _ = tf.clip_by_global_norm(
tf.gradients(self.cost, tvars), self.clip_grad_norm)
optimizer = tf.contrib.estimator.TowerOptimizer(
tf.train.AdamOptimizer(self.params.learning_rate))
apply_gradients = optimizer.apply_gradients(zip(grads, tvars))
gs = tf.train.get_global_step()
self.train_op = tf.group(apply_gradients, tf.assign_add(gs, 1))
# Cost metrics
tf.summary.scalar("ce", masked_reduced_ce)
tf.summary.scalar("average_inverse_difficulty",
sparse_mean(inverse_difficulty * self.target_mask))
# Pondering metrics
pondering = tf.stack(act.iterations, axis=-1) + 1
if params.data == "addition" and pondering.shape.ndims == 2:
# expand pondering to 3 dimension with repeated last dimension
pondering = tf.tile(
tf.expand_dims(pondering, -1), [1, 1, self.target_mask.shape[-1]])
masked_pondering = self.target_mask * pondering
dense_pondering = tf.gather_nd(
masked_pondering, indices=tf.where(tf.not_equal(masked_pondering, 0)))
tf.summary.scalar("average_pondering", tf.reduce_mean(dense_pondering))
tf.summary.histogram("pondering", dense_pondering)
if params.data == "addition":
avg_pondering = tf.reduce_sum(masked_pondering, axis=[-1, -2]) / \
tf.count_nonzero(masked_pondering, axis=[-1, -2],
dtype=tf.float32)
else:
avg_pondering = tf.reduce_sum(masked_pondering, axis=-1) / \
tf.count_nonzero(masked_pondering, axis=-1,
dtype=tf.float32)
summary_ponder_metadata = histogram_metadata.create_summary_metadata(
"difficulty/pondering", "ponder_steps_difficulty")
summary_ce_metadata = histogram_metadata.create_summary_metadata(
"difficulty/ce", "ce_steps_difficulty")
input_difficulty_steps = tf.cast(self.difficulty, tf.float32)
ponder_steps = tf.cast(avg_pondering, tf.float32)
ce_steps = tf.cast(masked_reduced_ce, tf.float32)
ponder_heights = []
ce_heights = []
for i in range(self.max_difficulty):
mask = tf.to_float(tf.equal(self.difficulty, i))
ponder_avg_steps = tf.cond(
tf.equal(tf.reduce_sum(mask), 0), lambda: 0.0,
lambda: tf.reduce_sum(mask * ponder_steps) / tf.reduce_sum(mask))
ce_avg_steps = tf.cond(
tf.equal(tf.reduce_sum(mask), 0), lambda: 0.0,
lambda: tf.reduce_sum(mask * ce_steps) / tf.reduce_sum(mask))
ponder_heights.append(ponder_avg_steps)
ce_heights.append(ce_avg_steps)
ponder_difficulty_steps = tf.transpose(
tf.stack([
tf.range(self.max_difficulty, dtype=tf.float32),
tf.range(self.max_difficulty, dtype=tf.float32) + 1, ponder_heights
]))
ce_difficulty_steps = tf.transpose(
tf.stack([
tf.range(self.max_difficulty, dtype=tf.float32),
tf.range(self.max_difficulty, dtype=tf.float32) + 1, ce_heights
]))
tf.summary.tensor_summary(
name='ponder_steps_difficulty',
tensor=ponder_difficulty_steps,
collections=None,
summary_metadata=summary_ponder_metadata)
tf.summary.tensor_summary(
name='ce_steps_difficulty',
tensor=ce_difficulty_steps,
collections=None,
summary_metadata=summary_ce_metadata)
def __init__(self,
num_classes,
vocab_size,
embedding_size,
hidden_units,
context_size,
max_sequence_length,
l2_reg_lambda=0.0):
self.X = tf.compat.v1.placeholder(tf.int32,
shape=[None, None],
name='input_X')
self.sequence_length = tf.compat.v1.placeholder(
tf.int32, shape=[None], name='input_sequence_length')
self.y = tf.compat.v1.placeholder(tf.float32,
shape=[None, num_classes],
name='input_y')
self.dropout_keep_prob = tf.compat.v1.placeholder(
tf.float32, name="dropout_keep_prob")
# self.max_sequence_length = tf.placeholder(
# tf.int32, shape=None, name='max_sequence_length')
# if max_sequence_length == 0:
self.max_sequence_length = tf.reduce_max(self.sequence_length)
# else:
# self.max_sequence_length = max_sequence_length
# self.dropout_keep_prob = tf.placeholder(
# tf.float32, name='dropout_keep_prob')
# l2_loss = tf.constant(0.0)
with tf.name_scope('embedding'):
W = tf.Variable(
tf.random.uniform([vocab_size, embedding_size], -1.0, 1.0))
self.embedded_chars = tf.nn.embedding_lookup(W, self.X)
with tf.name_scope('recurrent'):
clw1 = tf.Variable(tf.random.normal(shape=[1, context_size]),
dtype=tf.float32,
name='left_context')
clw1 = tf.tile(clw1, [tf.size(self.sequence_length), 1])
# Wl = tf.Variable(tf.random_normal(
# shape=[context_size, context_size]), dtype=tf.float32)
# Wsl = tf.Variable(tf.random_normal(
# shape=[embedding_size, context_size]), dtype=tf.float32)
crwn = tf.Variable(tf.random.normal(shape=[1, context_size]),
dtype=tf.float32,
name='right_context')
crwn = tf.tile(crwn, [tf.size(self.sequence_length), 1])
# Wr = tf.Variable(tf.random_normal(
# shape=[context_size, context_size]), dtype=tf.float32)
# Wsr = tf.Variable(tf.random_normal(
# shape=[embedding_size, context_size]), dtype=tf.float32)
# print(clw1, crwn)
lstm_cell = tf.contrib.rnn.BasicRNNCell(num_units=context_size,
reuse=False)
outputs, states = tf.nn.bidirectional_dynamic_rnn(
lstm_cell,
lstm_cell,
self.embedded_chars,
sequence_length=self.sequence_length,
initial_state_fw=clw1,
initial_state_bw=crwn,
dtype=tf.float32)
left_context = outputs[0][:, :self.max_sequence_length - 1, :]
clw1 = tf.expand_dims(clw1, 1)
Cl = tf.concat([clw1, left_context], 1)
right_context = outputs[1][:, :self.max_sequence_length - 1, :]
crwn = tf.expand_dims(crwn, 1)
Cr = tf.concat([crwn, right_context], 1)
# print(left_context, right_context)
# clw1 = tf.tile(clw1, [1, max_sequence_length, 1])
X = tf.concat([Cl, self.embedded_chars, tf.reverse(Cr, [1])], 2)
W2 = tf.compat.v1.get_variable(
"W2",
shape=[embedding_size + 2 * context_size, hidden_units],
initializer=tf.contrib.layers.xavier_initializer(),
dtype=tf.float32)
# W2 = tf.Variable(tf.random_normal(
# shape=[hidden_units, embedding_size + 2 * context_size],
# dtype=tf.float32), name='W2')
b2 = tf.Variable(tf.random.normal(shape=[hidden_units],
dtype=tf.float32),
name='b2')
self.y2 = tf.tanh(
tf.matmul(
tf.reshape(X, [-1, embedding_size +
2 * context_size]), W2) + b2)
self.y2 = tf.reshape(self.y2,
[-1, self.max_sequence_length, hidden_units])
with tf.name_scope('max_pooling'):
self.y3 = tf.reduce_max(self.y2, 1, keepdims=False)
with tf.name_scope("dropout"):
self.h_drop = tf.nn.dropout(self.y3,
rate=1 - self.dropout_keep_prob)
with tf.name_scope('output'):
W4 = tf.compat.v1.get_variable(
"W4",
shape=[hidden_units, num_classes],
initializer=tf.contrib.layers.xavier_initializer(),
dtype=tf.float32)
# W4 = tf.Variable(tf.random_normal(
# shape=[num_classes, hidden_units]), dtype=tf.float32, name='W4')
b4 = tf.Variable(tf.random.normal(shape=[num_classes],
dtype=tf.float32),
name='b4')
y4 = tf.matmul(self.h_drop, W4) + b4
# self.output = tf.reshape(y4)
self.predictions = tf.nn.sigmoid(y4)
with tf.name_scope('loss'):
self.loss_for_1 = - \
tf.reduce_mean(tf.reduce_sum(
self.y * tf.math.log(self.predictions + 1e-9), 1))
self.loss_for_0 = - \
tf.reduce_mean(tf.reduce_sum(
(1 - self.y) * tf.math.log(1 - self.predictions + 1e-9), 1))
# losses = tf.nn.sigmoid_cross_entropy_with_logits(
# logits=y4, labels=self.y)
# self.loss = tf.reduce_mean(tf.reduce_sum(losses, 1))
self.loss = self.loss_for_1 + self.loss_for_0
with tf.name_scope('accuracy'):
mask = tf.equal(tf.constant(1.), self.y)
labels_size = tf.count_nonzero(self.y, dtype=tf.int32)
top10 = tf.nn.top_k(self.predictions, 10, sorted=False)
kth = tf.reduce_min(top10.values)
predictions = tf.cast(tf.greater_equal(self.predictions, kth),
tf.float32)
hit_10 = tf.count_nonzero(tf.boolean_mask(predictions, mask),
dtype=tf.int32)
# hit_10 = tf.size(tf.sets.set_intersection(indices, indices_10))
# print(hit_10, tf.size(self.sequence_length))
self.precise_10 = hit_10 / \
(tf.size(self.sequence_length) * 10)
self.recall_10 = hit_10 / labels_size
top5 = tf.nn.top_k(self.predictions, 5, sorted=False)
kth = tf.reduce_min(top5.values)
predictions = tf.cast(tf.greater_equal(self.predictions, kth),
tf.float32)
hit_5 = tf.count_nonzero(tf.boolean_mask(predictions, mask),
dtype=tf.int32)
# hit_5 = tf.size(tf.sets.set_intersection(indices, indices_5))
self.precise_5 = hit_5 / \
(tf.size(self.sequence_length) * 5)
self.recall_5 = hit_5 / labels_size