def compute_loss(self,emb_batch,curr_batch_size=None):
outloss=[]
prediction=[]
for idx_batch in range(self.config.batch_size):
tree_states=self.compute_states(emb_batch,idx_batch)
logits = self.create_output(tree_states)
labels1=tf.gather(self.labels,idx_batch)
labels2=tf.reduce_sum(tf.to_int32(tf.not_equal(labels1,-1)))
labels=tf.gather(labels1,tf.range(labels2))
loss = self.calc_loss(logits,labels)
pred = tf.nn.softmax(logits)
pred_root=tf.gather(pred,labels2-1)
prediction.append(pred_root)
outloss.append(loss)
batch_loss=tf.pack(outloss)
self.pred = tf.pack(prediction)
return batch_loss
def center_loss(features, label, alpha, nrof_classes):
"""Center loss based on the paper "A Discriminative Feature Learning Approach for Deep Face Recognition"
(http://ydwen.github.io/papers/WenECCV16.pdf)
"""
# 获取特征向量长度
nrof_features = features.get_shape()[1]
# 生成可以共享的变量centers
with tf.variable_scope('center', reuse=True):
centers = tf.get_variable('centers')
label = tf.reshape(label, [-1])
# 取出对应label下对应的center值，注意label里面的值可能会重复，因为一个标签下有可能会出现多个人
centers_batch = tf.gather(centers, label)
# 求特征点到中心的距离并乘以一定的系数，alfa是center的更新速度，越大代表更新的越慢
diff = centers_batch - features
# 获取一个batch中同一样本出现的次数，这里需要理解论文中的更新公式
unique_label, unique_idx, unique_count = tf.unique_with_counts(label)
appear_times = tf.gather(unique_count, unique_idx)
appear_times = tf.reshape(appear_times, [-1, 1])
diff = diff / tf.cast((1 + appear_times), tf.float32)
diff = alpha * diff
# 更新center，输出是将对应于label的centers减去对应的diff，如果同一个标签出现多次，那么就减去多次
centers = tf.scatter_sub(centers, label, diff)
# 求center loss，这里是将l2_loss里面的值进行平方相加，再除以2，并没有进行开方
loss = tf.nn.l2_loss(features - centers_batch)
return loss, centers
def fast_rcnn_minibatch(self, reference_boxes):
with tf.variable_scope('fast_rcnn_minibatch'):
reference_boxes_mattached_gtboxes, object_mask, label = \
self.fast_rcnn_find_positive_negative_samples(reference_boxes)
positive_indices = tf.reshape(tf.where(tf.not_equal(object_mask, 0.)), [-1])
num_of_positives = tf.minimum(tf.shape(positive_indices)[0],
tf.cast(self.fast_rcnn_minibatch_size*self.fast_rcnn_positives_ratio, tf.int32))
positive_indices = tf.random_shuffle(positive_indices)
positive_indices = tf.slice(positive_indices, begin=[0], size=[num_of_positives])
negative_indices = tf.reshape(tf.where(tf.equal(object_mask, 0.)), [-1])
num_of_negatives = tf.minimum(tf.shape(negative_indices)[0],
self.fast_rcnn_minibatch_size - num_of_positives)
negative_indices = tf.random_shuffle(negative_indices)
negative_indices = tf.slice(negative_indices, begin=[0], size=[num_of_negatives])
minibatch_indices = tf.concat([positive_indices, negative_indices], axis=0)
minibatch_indices = tf.random_shuffle(minibatch_indices)
minibatch_reference_boxes_mattached_gtboxes = tf.gather(reference_boxes_mattached_gtboxes,
minibatch_indices)
object_mask = tf.gather(object_mask, minibatch_indices)
label = tf.gather(label, minibatch_indices)
label_one_hot = tf.one_hot(label, self.num_classes + 1)
return minibatch_indices, minibatch_reference_boxes_mattached_gtboxes, object_mask, label_one_hot
def _recurrence(node_h,node_c,idx_var):
node_info=tf.gather(treestr,idx_var)
child_h=tf.gather(node_h,node_info)
child_c=tf.gather(node_c,node_info)
flat_ = tf.reshape(child_h,[-1])
tmp=tf.matmul(tf.expand_dims(flat_,0),cW)
u,o,i,fl,fr=tf.split(1,5,tmp)
i=tf.nn.sigmoid(i+bi)
o=tf.nn.sigmoid(o+bo)
u=tf.nn.tanh(u+bu)
fl=tf.nn.sigmoid(fl+bf)
fr=tf.nn.sigmoid(fr+bf)
f=tf.concat(0,[fl,fr])
c = i * u + tf.reduce_sum(f*child_c,[0])
h = o * tf.nn.tanh(c)
node_h = tf.concat(0,[node_h,h])
node_c = tf.concat(0,[node_c,c])
idx_var=tf.add(idx_var,1)
return node_h,node_c,idx_var
def modular_layer(inputs, modules: ModulePool, parallel_count: int, context: ModularContext):
with tf.variable_scope(None, 'modular_layer'):
inputs = context.begin_modular(inputs)
flat_inputs = tf.layers.flatten(inputs)
logits = tf.layers.dense(flat_inputs, modules.module_count * parallel_count)
logits = tf.reshape(logits, [-1, parallel_count, modules.module_count])
ctrl = tfd.Categorical(logits)
initializer = tf.random_uniform_initializer(maxval=modules.module_count, dtype=tf.int32)
shape = [context.dataset_size, parallel_count]
best_selection_persistent = tf.get_variable('best_selection', shape, tf.int32, initializer)
if context.mode == ModularMode.E_STEP:
# 1 x batch_size x 1
best_selection = tf.gather(best_selection_persistent, context.data_indices)[tf.newaxis]
# sample_size x batch_size x 1
sampled_selection = tf.reshape(ctrl.sample(), [context.sample_size, -1, parallel_count])
selection = tf.concat([best_selection, sampled_selection[1:]], axis=0)
selection = tf.reshape(selection, [-1, parallel_count])
elif context.mode == ModularMode.M_STEP:
selection = tf.gather(best_selection_persistent, context.data_indices)
elif context.mode == ModularMode.EVALUATION:
selection = ctrl.mode()
else:
raise ValueError('Invalid modular mode')
attrs = ModularLayerAttributes(selection, best_selection_persistent, ctrl)
context.layers.append(attrs)
return run_modules(inputs, selection, modules.module_fnc, modules.output_shape)
def proposal_top_layer(rpn_cls_prob, rpn_bbox_pred, im_info, _feat_stride, anchors, num_anchors):
"""A layer that just selects the top region proposals
without using non-maximal suppression,
For details please see the technical report
"""
rpn_top_n = cfg.TEST.RPN_TOP_N
scores = rpn_cls_prob[:, :, :, num_anchors:]
rpn_bbox_pred = tf.reshape(rpn_bbox_pred, shape=(-1, 4))
scores = tf.reshape(scores, shape=(-1,))
# Do the selection here
top_scores, top_inds = tf.nn.top_k(scores, k=rpn_top_n)
top_scores = tf.reshape(top_scores, shape=(-1, 1))
top_anchors = tf.gather(anchors, top_inds)
top_rpn_bbox = tf.gather(rpn_bbox_pred, top_inds)
proposals = bbox_transform_inv_tf(top_anchors, top_rpn_bbox)
# Clip predicted boxes to image
proposals = clip_boxes_tf(proposals, im_info[:2])
# Output rois blob
# Our RPN implementation only supports a single input image, so all
# batch inds are 0
proposals = tf.to_float(proposals)
batch_inds = tf.zeros((rpn_top_n, 1))
blob = tf.concat([batch_inds, proposals], 1)
return blob, top_scores
def f(X):
"""
prob: n probabilities
box: nx4 boxes
Returns: n boolean, the selection
"""
prob, box = X
output_shape = tf.shape(prob)
# filter by score threshold
ids = tf.reshape(tf.where(prob > cfg.TEST.RESULT_SCORE_THRESH), [-1])
prob = tf.gather(prob, ids)
box = tf.gather(box, ids)
# NMS within each class
selection = tf.image.non_max_suppression(
box, prob, cfg.TEST.RESULTS_PER_IM, cfg.TEST.FRCNN_NMS_THRESH)
selection = tf.to_int32(tf.gather(ids, selection))
# sort available in TF>1.4.0
# sorted_selection = tf.contrib.framework.sort(selection, direction='ASCENDING')
sorted_selection = -tf.nn.top_k(-selection, k=tf.size(selection))[0]
mask = tf.sparse_to_dense(
sparse_indices=sorted_selection,
output_shape=output_shape,
sparse_values=True,
default_value=False)
return mask
def _define_experience(self, agent_indices, observ, action, reward):
"""Implement the branch of experience() entered during training."""
update_filters = tf.summary.merge(
[self._observ_filter.update(observ),
self._reward_filter.update(reward)])
with tf.control_dependencies([update_filters]):
if self._config.train_on_agent_action:
# NOTE: Doesn't seem to change much.
action = self._last_action
batch = (observ, action, tf.gather(self._last_mean,
agent_indices), tf.gather(self._last_logstd,
agent_indices), reward)
append = self._episodes.append(batch, agent_indices)
with tf.control_dependencies([append]):
norm_observ = self._observ_filter.transform(observ)
norm_reward = tf.reduce_mean(self._reward_filter.transform(reward))
# pylint: disable=g-long-lambda
summary = tf.cond(
self._should_log, lambda: tf.summary.merge([
update_filters,
self._observ_filter.summary(),
self._reward_filter.summary(),
tf.summary.scalar('memory_size', self._memory_index),
tf.summary.histogram('normalized_observ', norm_observ),
tf.summary.histogram('action', self._last_action),
tf.summary.scalar('normalized_reward', norm_reward)
]), str)
return summary
def build_predict(self,Xnew,task_ind):
"""
We need to assume the task_ind starts from 0
"""
Fmean,Fvar = 0,0
for i in np.arange(self.rank):
for j in np.arange(self.num_latent_list[i]):
lat_id = np.sum(self.num_latent_list[:i],dtype = np.int64) + j
if self.whiten_list[lat_id]: # need to compute fmean and fvar by the weights
fmean, fvar = conditionals.gaussian_gp_predict_whitened(Xnew, self.Z[lat_id],
self.kern_list[i], self.q_mu_list[lat_id],
self.q_sqrt_list[lat_id], 1)
else:
fmean, fvar = conditionals.gaussian_gp_predict(Xnew, self.Z[lat_id],
self.kern_list[i], self.q_mu_list[lat_id],
self.q_sqrt_list[lat_id],1)
W_ij = tf.gather(self.W,task_ind)[lat_id]
Fmean += (fmean + self.mean_function_list[lat_id](Xnew))*W_ij
Fvar += fvar * tf.square(W_ij)
if self.tsk:
for i in np.arange(self.num_tasks):
lat_id = np.sum(self.num_latent_list,dtype = np.int64) + i
if self.whiten_list[lat_id]: # need to compute fmean and fvar by the weights
fmean, fvar = conditionals.gaussian_gp_predict_whitened(Xnew, self.Z[lat_id],
self.tskern_list[i], self.q_mu_list[lat_id],
self.q_sqrt_list[lat_id], 1)
else:
fmean, fvar = conditionals.gaussian_gp_predict(Xnew, self.Z[lat_id],
self.tskern_list[i], self.q_mu_list[lat_id],
self.q_sqrt_list[lat_id], 1)
switch = tf.cast(tf.equal(tf.to_int64(i), task_ind),tf.float64)
W_ij = tf.gather(self.Kappa,i)[0]*switch
Fmean += (fmean + self.mean_function_list[lat_id](Xnew))*W_ij
Fvar += fvar * tf.square(W_ij)
return Fmean, Fvar
def rpn_proposals(self):
with tf.variable_scope('rpn_proposals'):
rpn_decode_boxes = encode_and_decode.decode_boxes(encode_boxes=self.rpn_encode_boxes,
reference_boxes=self.anchors,
scale_factors=self.scale_factors)
rpn_softmax_scores = slim.softmax(self.rpn_scores)
rpn_object_score = rpn_softmax_scores[:, 1] # second column represent object
if self.top_k_nms:
rpn_object_score, top_k_indices = tf.nn.top_k(rpn_object_score, k=self.top_k_nms)
rpn_decode_boxes = tf.gather(rpn_decode_boxes, top_k_indices)
# NMS
valid_indices = tf_wrapper.nms_rotate_tf(boxes_list=rpn_decode_boxes,
scores=rpn_object_score,
iou_threshold=self.rpn_nms_iou_threshold,
max_output_size=self.max_proposals_num,
use_gpu=cfgs.NMS_USE_GPU)
valid_boxes = tf.gather(rpn_decode_boxes, valid_indices)
valid_scores = tf.gather(rpn_object_score, valid_indices)
# print_tensors(valid_scores, 'rpn_score')
rpn_proposals_boxes, rpn_proposals_scores = tf.cond(
tf.less(tf.shape(valid_boxes)[0], self.max_proposals_num),
lambda: boxes_utils.padd_boxes_with_zeros(valid_boxes, valid_scores,
self.max_proposals_num),
lambda: (valid_boxes, valid_scores))
return rpn_proposals_boxes, rpn_proposals_scores
def scheduled_sample_count(ground_truth_x,
generated_x,
batch_size,
scheduled_sample_var):
"""Sample batch with specified mix of groundtruth and generated data points.
Args:
ground_truth_x: tensor of ground-truth data points.
generated_x: tensor of generated data points.
batch_size: batch size
scheduled_sample_var: number of ground-truth examples to include in batch.
Returns:
New batch with num_ground_truth sampled from ground_truth_x and the rest
from generated_x.
"""
num_ground_truth = scheduled_sample_var
idx = tf.random_shuffle(tf.range(batch_size))
ground_truth_idx = tf.gather(idx, tf.range(num_ground_truth))
generated_idx = tf.gather(idx, tf.range(num_ground_truth, batch_size))
ground_truth_examps = tf.gather(ground_truth_x, ground_truth_idx)
generated_examps = tf.gather(generated_x, generated_idx)
output = tf.dynamic_stitch([ground_truth_idx, generated_idx],
[ground_truth_examps, generated_examps])
# if batch size is known set it.
if isinstance(batch_size, int):
output.set_shape([batch_size] + common_layers.shape_list(output)[1:])
return output
def _tf_linear_interp1d(x_to_interpolate, fn_x, fn_y):
"""Tensorflow implementation of 1d linear interpolation.
Args:
x_to_interpolate: tf.float32 Tensor of shape (num_examples,) over which 1d
linear interpolation is performed.
fn_x: Monotonically-increasing, non-repeating tf.float32 Tensor of shape
(length,) used as the domain to approximate a function.
fn_y: tf.float32 Tensor of shape (length,) used as the range to approximate
a function.
Returns:
tf.float32 Tensor of shape (num_examples,)
"""
x_pad = tf.concat([fn_x[:1] - 1, fn_x, fn_x[-1:] + 1], axis=0)
y_pad = tf.concat([fn_y[:1], fn_y, fn_y[-1:]], axis=0)
interval_idx = _find_interval_containing_new_value(x_pad, x_to_interpolate)
# Interpolate
alpha = (
(x_to_interpolate - tf.gather(x_pad, interval_idx)) /
(tf.gather(x_pad, interval_idx + 1) - tf.gather(x_pad, interval_idx)))
interpolation = ((1 - alpha) * tf.gather(y_pad, interval_idx) +
alpha * tf.gather(y_pad, interval_idx + 1))
return interpolation
def make_minibatch(self, valid_anchors):
with tf.variable_scope('rpn_minibatch'):
# in labels(shape is [N, ]): 1 is positive, 0 is negative, -1 is ignored
labels, anchor_matched_gtboxes, object_mask = \
self.rpn_find_positive_negative_samples(valid_anchors) # [num_of_valid_anchors, ]
positive_indices = tf.reshape(tf.where(tf.equal(labels, 1.0)), [-1]) # use labels is same as object_mask
num_of_positives = tf.minimum(tf.shape(positive_indices)[0],
tf.cast(self.rpn_mini_batch_size * self.rpn_positives_ratio, tf.int32))
# num of positives <= minibatch_size * 0.5
positive_indices = tf.random_shuffle(positive_indices)
positive_indices = tf.slice(positive_indices, begin=[0], size=[num_of_positives])
# positive_anchors = tf.gather(self.anchors, positive_indices)
negative_indices = tf.reshape(tf.where(tf.equal(labels, 0.0)), [-1])
num_of_negatives = tf.minimum(self.rpn_mini_batch_size - num_of_positives,
tf.shape(negative_indices)[0])
negative_indices = tf.random_shuffle(negative_indices)
negative_indices = tf.slice(negative_indices, begin=[0], size=[num_of_negatives])
# negative_anchors = tf.gather(self.anchors, negative_indices)
minibatch_indices = tf.concat([positive_indices, negative_indices], axis=0)
minibatch_indices = tf.random_shuffle(minibatch_indices)
minibatch_anchor_matched_gtboxes = tf.gather(anchor_matched_gtboxes, minibatch_indices)
object_mask = tf.gather(object_mask, minibatch_indices)
labels = tf.cast(tf.gather(labels, minibatch_indices), tf.int32)
labels_one_hot = tf.one_hot(labels, depth=2)
return minibatch_indices, minibatch_anchor_matched_gtboxes, object_mask, labels_one_hot
def _log_unnormalized_prob(self, x):
mean = tf.squeeze(tf.gather(x, [0], axis=-1), axis=-1)
precision = self._maybe_assert_valid_sample(
tf.squeeze(tf.gather(x, [1], axis=-1), axis=-1))
return (tf.math.xlogy(self.concentration - 0.5, precision)
- self.rate * precision
- 0.5 * self._lambda * precision * tf.square(mean - self.loc))
def append(self, transitions, rows=None):
"""Append a batch of transitions to rows of the memory.
Args:
transitions: Tuple of transition quantities with batch dimension.
rows: Episodes to append to, defaults to all.
Returns:
Operation.
"""
rows = tf.range(self._capacity) if rows is None else rows
assert rows.shape.ndims == 1
assert_capacity = tf.assert_less(
rows, self._capacity,
message='capacity exceeded')
with tf.control_dependencies([assert_capacity]):
assert_max_length = tf.assert_less(
tf.gather(self._length, rows), self._max_length,
message='max length exceeded')
append_ops = []
with tf.control_dependencies([assert_max_length]):
for buffer_, elements in zip(self._buffers, transitions):
timestep = tf.gather(self._length, rows)
indices = tf.stack([rows, timestep], 1)
append_ops.append(tf.scatter_nd_update(buffer_, indices, elements))
with tf.control_dependencies(append_ops):
episode_mask = tf.reduce_sum(tf.one_hot(
rows, self._capacity, dtype=tf.int32), 0)
return self._length.assign_add(episode_mask)
def _map_to_tfidf(x):
"""Calculates the inverse document frequency of terms in the corpus.
Args:
x : a SparseTensor of int64 representing string indices in vocab.
Returns:
The tf*idf values
"""
# Add one to the reduced term freqnencies to avoid dividing by zero.
idf = tf.log(tf.to_double(corpus_size) / (
1.0 + tf.to_double(reduced_term_freq)))
dense_doc_sizes = tf.to_double(tf.sparse_reduce_sum(tf.SparseTensor(
indices=x.indices,
values=tf.ones_like(x.values),
dense_shape=x.dense_shape), 1))
# For every term in x, divide the idf by the doc size.
# The two gathers both result in shape <sum_doc_sizes>
idf_over_doc_size = (tf.gather(idf, x.values) /
tf.gather(dense_doc_sizes, x.indices[:, 0]))
return tf.SparseTensor(
indices=x.indices,
values=idf_over_doc_size,
dense_shape=x.dense_shape)
def full_loss_op(self, logits, labels):
"""Adds loss ops to the computational graph.
Hint: Use sparse_softmax_cross_entropy_with_logits
Hint: Remember to add l2_loss (see tf.nn.l2_loss)
Args:
logits: tensor(num_nodes, output_size)
labels: python list, len = num_nodes
Returns:
loss: tensor 0-D
"""
if self.full_loss is None:
loss = None
# YOUR CODE HERE
l2_loss = self.config.l2 * tf.add_n(tf.get_collection("l2_loss"))
idx = tf.where(tf.less(self.labelholder,2))
logits = tf.gather(logits, idx)
labels = tf.gather(labels, idx)
objective_loss = tf.reduce_sum(tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits, labels=labels))
loss = objective_loss + l2_loss
tf.summary.scalar("loss_l2", l2_loss)
tf.summary.scalar("loss_objective", tf.reduce_sum(objective_loss))
tf.summary.scalar("loss_total", loss)
self.full_loss = loss
# END YOUR CODE
return self.full_loss
def _partition_and_stitch(self, args, func_name):
"""
args is a list of tensors, to be passed to self.likelihoods.<func_name>
args[-1] is the 'Y' argument, which contains the indexes to self.likelihoods.
This function splits up the args using dynamic_partition, calls the
relevant function on the likelihoods, and re-combines the result.
"""
# get the index from Y
Y = args[-1]
ind = tf.gather(tf.transpose(Y), tf.shape(Y)[1] - 1) # ind = Y[:,-1]
ind = tf.cast(ind, tf.int32)
Y = tf.transpose(tf.gather(tf.transpose(Y), tf.range(0, tf.shape(Y)[1] - 1))) # Y = Y[:,:-1]
args[-1] = Y
# split up the arguments into chunks corresponding to the relevant likelihoods
args = zip(*[tf.dynamic_partition(X, ind, self.num_likelihoods) for X in args])
# apply the likelihood-function to each section of the data
funcs = [getattr(lik, func_name) for lik in self.likelihood_list]
results = [f(*args_i) for f, args_i in zip(funcs, args)]
# stitch the results back together
partitions = tf.dynamic_partition(tf.range(0, tf.size(ind)), ind, self.num_likelihoods)
results = tf.dynamic_stitch(partitions, results)
return results
def reorder_beam(beam_size, batch_size, beam_val, output, is_first,
tensors_to_reorder):
"""Reorder to minimize beam costs."""
# beam_val is [batch_size x beam_size]; let b = batch_size * beam_size
# decided is len x b x a x b
# output is b x out_size; step is b x len x a x b;
outputs = tf.split(axis=0, num_or_size_splits=beam_size, value=tf.nn.log_softmax(output))
all_beam_vals, all_beam_idx = [], []
beam_range = 1 if is_first else beam_size
for i in xrange(beam_range):
top_out, top_out_idx = tf.nn.top_k(outputs[i], k=beam_size)
cur_beam_val = beam_val[:, i]
top_out = tf.Print(top_out, [top_out, top_out_idx, beam_val, i,
cur_beam_val], "GREPO", summarize=8)
all_beam_vals.append(top_out + tf.expand_dims(cur_beam_val, 1))
all_beam_idx.append(top_out_idx)
all_beam_idx = tf.reshape(tf.transpose(tf.concat(axis=1, values=all_beam_idx), [1, 0]),
[-1])
top_beam, top_beam_idx = tf.nn.top_k(tf.concat(axis=1, values=all_beam_vals), k=beam_size)
top_beam_idx = tf.Print(top_beam_idx, [top_beam, top_beam_idx],
"GREP", summarize=8)
reordered = [[] for _ in xrange(len(tensors_to_reorder) + 1)]
top_out_idx = []
for i in xrange(beam_size):
which_idx = top_beam_idx[:, i] * batch_size + tf.range(batch_size)
top_out_idx.append(tf.gather(all_beam_idx, which_idx))
which_beam = top_beam_idx[:, i] / beam_size # [batch]
which_beam = which_beam * batch_size + tf.range(batch_size)
reordered[0].append(tf.gather(output, which_beam))
for i, t in enumerate(tensors_to_reorder):
reordered[i + 1].append(tf.gather(t, which_beam))
new_tensors = [tf.concat(axis=0, values=t) for t in reordered]
top_out_idx = tf.concat(axis=0, values=top_out_idx)
return (top_beam, new_tensors[0], top_out_idx, new_tensors[1:])
def sparse_dot_product0(emb, tuples, use_matmul=True, output_type='real'):
"""
Compute the dot product of complex vectors.
It uses complex vectors but tensorflow does not optimize in the complex space (or there is a bug in the gradient
propagation with complex numbers...)
:param emb: embeddings
:param tuples: indices at which we compute dot products
:return: scores (dot products)
"""
n_t = tuples.get_shape()[0].value
rk = emb.get_shape()[1].value
emb_sel_a = tf.gather(emb, tuples[:, 0])
emb_sel_b = tf.gather(emb, tuples[:, 1])
if use_matmul:
pred_cplx = tf.squeeze(tf.batch_matmul(
tf.reshape(emb_sel_a, [n_t, rk, 1]),
tf.reshape(emb_sel_b, [n_t, rk, 1]), adj_x=True))
else:
pred_cplx = tf.reduce_sum(tf.mul(tf.conj(emb_sel_a), emb_sel_b), 1)
if output_type == 'complex':
return pred_cplx
elif output_type == 'real':
return tf.real(pred_cplx) + tf.imag(pred_cplx)
elif output_type == 'real':
return tf.abs(pred_cplx)
elif output_type == 'angle':
raise NotImplementedError('No argument or inverse-tanh function for complex number in Tensorflow')
else:
raise NotImplementedError()
def sparse_hermitian_product(emb, tuples):
"""
Compute the Hermitian inner product between selected complex embeddings
This corresponds to the usual dot product applied on the conjugate of the first vector: <conj(x), y>
where conj is the complex conjugate (obtained by inverting the imaginary part)
We consider that the embedding dimension is twice the rank, where the first part is in emb[:,:rk] and
the imaginary part is in emb[:,rk:].
It computes
S[i] = <conj(E[I[i,1]], E[I[i,2]]>
Usage:
S = sparse_hermitian_product(E, I):
:param emb: embedding matrix of size [n_emb, 2 * r] containing float numbers where r is the complex rank
:param tuples: tuple matrix of size [n_t, 2] containing integers that correspond to the indices of the embeddings
:return: a pair containing the real and imaginary parts of the Hermitian dot products
"""
rk = emb.get_shape()[1].value // 2
emb_re = emb[:, :rk]
emb_im = emb[:, rk:]
emb_sel_a_re = tf.gather(emb_re, tuples[:, 0])
emb_sel_a_im = tf.gather(emb_im, tuples[:, 0])
emb_sel_b_re = tf.gather(emb_re, tuples[:, 1])
emb_sel_b_im = tf.gather(emb_im, tuples[:, 1])
pred_re = tf.reduce_sum(tf.mul(emb_sel_a_re, emb_sel_b_re) + tf.mul(emb_sel_a_im, emb_sel_b_im), 1)
pred_im = tf.reduce_sum(tf.mul(emb_sel_a_re, emb_sel_b_im) - tf.mul(emb_sel_a_im, emb_sel_b_re), 1)
return pred_re, pred_im
def get_mention_emb(self, text_emb, text_outputs, mention_starts, mention_ends):
mention_emb_list = []
mention_start_emb = tf.gather(text_outputs, mention_starts) # [num_mentions, emb]
mention_emb_list.append(mention_start_emb)
mention_end_emb = tf.gather(text_outputs, mention_ends) # [num_mentions, emb]
mention_emb_list.append(mention_end_emb)
mention_width = 1 + mention_ends - mention_starts # [num_mentions]
if self.config["use_features"]:
mention_width_index = mention_width - 1 # [num_mentions]
mention_width_emb = tf.gather(tf.get_variable("mention_width_embeddings", [self.config["max_mention_width"], self.config["feature_size"]]), mention_width_index) # [num_mentions, emb]
mention_width_emb = tf.nn.dropout(mention_width_emb, self.dropout)
mention_emb_list.append(mention_width_emb)
if self.config["model_heads"]:
mention_indices = tf.expand_dims(tf.range(self.config["max_mention_width"]), 0) + tf.expand_dims(mention_starts, 1) # [num_mentions, max_mention_width]
mention_indices = tf.minimum(util.shape(text_outputs, 0) - 1, mention_indices) # [num_mentions, max_mention_width]
mention_text_emb = tf.gather(text_emb, mention_indices) # [num_mentions, max_mention_width, emb]
self.head_scores = util.projection(text_outputs, 1) # [num_words, 1]
mention_head_scores = tf.gather(self.head_scores, mention_indices) # [num_mentions, max_mention_width, 1]
mention_mask = tf.expand_dims(tf.sequence_mask(mention_width, self.config["max_mention_width"], dtype=tf.float32), 2) # [num_mentions, max_mention_width, 1]
mention_attention = tf.nn.softmax(mention_head_scores + tf.log(mention_mask), dim=1) # [num_mentions, max_mention_width, 1]
mention_head_emb = tf.reduce_sum(mention_attention * mention_text_emb, 1) # [num_mentions, emb]
mention_emb_list.append(mention_head_emb)
mention_emb = tf.concat(mention_emb_list, 1) # [num_mentions, emb]
return mention_emb
def c_body(c, pa):
# Zeroing predictions below threshold
with tf.variable_scope('bboxes_c_select', reuse=True):
c_scores = b_scores[:, c]
c_fmask = tf.cast(tf.greater(c_scores, confidence_threshold), scores.dtype)
c_scores = c_scores * c_fmask
c_bboxes = b_bboxes * tf.expand_dims(c_fmask, axis=-1)
# Apply NMS
with tf.variable_scope('bboxes_c_nms', reuse=True):
c_indices = tf.image.non_max_suppression(c_bboxes, c_scores, top_k, nms_threshold)
size = tf.size(c_indices)
c_batch_ = tf.to_float(b) * tf.ones(shape=[top_k, 1], dtype=tf.float32) # len(indices) x 1
c_labels = tf.to_float(c) * tf.ones(shape=[top_k, 1], dtype=tf.float32) # len(indices) x 1
extra_size = top_k - size
c_scores = tf.expand_dims(tf.gather(c_scores, c_indices), axis=-1) # len(indices) x 1
empty_c_scores = tf.zeros([extra_size, 1], dtype=tf.float32)
c_scores = tf.concat([c_scores, empty_c_scores], axis=0)
c_bboxes = tf.gather(c_bboxes, c_indices) # len(indices) x 4
empty_c_bboxes = tf.zeros([extra_size, 4], dtype=tf.float32)
c_bboxes = tf.concat([c_bboxes, empty_c_bboxes], axis=0)
c_predictions = tf.concat([c_batch_, c_labels, c_scores, c_bboxes], axis=1) # len(indices) x 7
return c + 1, pa.write(index=c - 1, value=c_predictions)
def build_loss(self, logits, labels, lambs):
# put a sigfunction on logits and then transpose
logits = tf.transpose(framwork.sig_func(logits))
# according to the labels, erase rows which is not in labels
labels_unique = tf.constant(range(self.image_classes), dtype=tf.int32)
labels_num = self.image_classes
logits = tf.gather(logits, indices=labels_unique)
lambs = tf.gather(lambs, indices=labels_unique)
# set the value of each row to True when it occurs in labels
template = tf.tile(tf.expand_dims(labels_unique, dim=1), [1, self.batch_size])
labels_expand = tf.tile(tf.expand_dims(labels, dim=0), [labels_num, 1])
indict_logic = tf.equal(labels_expand, template)
# split the tensor along rows
logit_list = tf.split(0, labels_num, logits)
indict_logic_list = tf.split(0, labels_num, indict_logic)
lambda_list = tf.split(0, self.image_classes, lambs)
# loss_list = list()
# for i in range(self.image_classes):
# loss_list.append(framwork.loss_func(logit_list[i], indict_logic_list[i], lambda_list[i]))
loss_list = map(framwork.loss_func, logit_list, indict_logic_list, lambda_list)
loss = tf.add_n(loss_list)
tensors_dict = {'labels_unique': labels_unique, 'template': template, 'logits_sig_trans': logits,
'loss': loss, 'indict_logic': indict_logic}
self.tensors_names.extend(tensors_dict.keys())
self.net_tensors.update(tensors_dict)
def mf_binary_likelihood(U, V, nzr, nzc, nzz, noise_prec, alpha, n, m, k, fix_entries=FIX_TRIANGLE):
with tf.name_scope("priors"):
U_prior = tf.reduce_sum(bf.dists.gaussian_log_density(U, stddev=alpha), name="U_prior")
V_prior = tf.reduce_sum(bf.dists.gaussian_log_density(V, stddev=alpha), name="V_prior")
if fix_entries == FIX_IDENTITY:
mask = np.float32(np.vstack((np.eye(k), np.ones((m-k, k)))))
V = V * mask
elif fix_entries == FIX_TRIANGLE:
mask = np.float32(np.tril(np.ones((m, k))))
V = V * mask
else:
pass
with tf.name_scope("model"):
Us = tf.gather(U, nzr, name="Us")
#tf.histogram_summary("Us", Us)
Vs = tf.gather(V, nzc, name="Vs")
#tf.histogram_summary("Vs", Vs)
Rs = tf.reduce_sum(tf.mul(Us, Vs), reduction_indices=1, name="Rs")
#tf.histogram_summary("rs", Rs)
probs, _ = bf.transforms.logit(Rs * noise_prec)
#tf.histogram_summary("probs", probs)
ll = tf.reduce_sum(bf.dists.bernoulli_log_density(nzz, probs), name="ll")
joint_logprob = U_prior + V_prior + ll
return joint_logprob
def testIndexedSlices(self):
for v1_first in [True, False]:
with self.test_session():
v1 = tf.Variable(np.array([[0.0, 1.0], [10.0, 11.0], [20.0, 21.0]]).astype(np.float32))
v1_at_1 = tf.IndexedSlices(
control_flow_ops.with_dependencies([v1.initializer], v1.ref()), tf.constant([1])
)
v2 = tf.Variable(np.array([[0.1, 1.1], [10.1, 11.1], [20.1, 21.1]]).astype(np.float32))
v2_at_1 = tf.IndexedSlices(
control_flow_ops.with_dependencies([v2.initializer], v2.ref()), tf.constant([1])
)
st1, st2 = control_flow_ops.tuple([v1_at_1, v2_at_1])
g1 = tf.gather(st1.values, st1.indices)
g2 = tf.gather(st2.values, st2.indices)
# v1 is not initialized.
with self.assertRaisesOpError("Attempting to use uninitialized value"):
v1.eval()
# v2 is not initialized.
with self.assertRaisesOpError("Attempting to use uninitialized value"):
v2.eval()
if v1_first:
# Getting g1 initializes v2.
self.assertAllClose([[10.0, 11.0]], g1.eval())
self.assertAllClose([[0.1, 1.1], [10.1, 11.1], [20.1, 21.1]], v2.eval())
else:
# Getting g2 initializes v1.
self.assertAllClose([[10.1, 11.1]], g2.eval())
self.assertAllClose([[0.0, 1.0], [10.0, 11.0], [20.0, 21.0]], v1.eval())
def build_network(self):
net_tensors = self.net_tensors
with self.net_graph.as_default(), tf.device(self.net_device):
logits = tf.placeholder(dtype=tf.float32, shape=(self.batch_size, self.image_classes))
labels = tf.placeholder(dtype=tf.int32, shape=(self.batch_size,))
lambs = tf.placeholder(dtype=tf.float32, shape=(self.image_classes,))
# put a sigfunction on logits and then transpose
logits = tf.transpose(framwork.sig_func(logits))
# according to the labels, erase rows which is not in labels
labels_unique = tf.constant(range(self.image_classes), dtype=tf.int32)
labels_num = self.image_classes
logits = tf.gather(logits, indices=labels_unique)
lambs = tf.gather(lambs, indices=labels_unique)
# set the value of each row to True when it occurs in labels
templete = tf.tile(tf.expand_dims(labels_unique, dim=1), [1, self.batch_size])
labels_expand = tf.tile(tf.expand_dims(labels, dim=0), [labels_num, 1])
indict_logic = tf.equal(labels_expand, templete)
# split the tensor along rows
logit_list = tf.split(0, labels_num, logits)
indict_logic_list = tf.split(0, labels_num, indict_logic)
lamb_list = tf.split(0, self.image_classes, lambs)
logit_list = [tf.squeeze(item) for item in logit_list]
indict_logic_list = [tf.squeeze(item) for item in indict_logic_list]
left_right_tuples = list()
for i in range(self.image_classes):
left_right_tuples.append(framwork.lamb_func(logit_list[i], indict_logic_list[i], lamb=lamb_list[i]))
# func = framwork.lamb_func()
# left_right_tuples = map(func, logit_list, indict_logic_list, lamb_list)
net_tensors.update({'left_right_tuples': left_right_tuples, 'logits': logits, 'labels': labels,
'lambs': lambs})
def _ProcessSingleScale(scale_index,
boxes,
features,
scales,
scores,
reuse=True):
"""Resize the image and run feature extraction and keypoint selection.
This function will be passed into tf.while_loop() and be called
repeatedly. The input boxes are collected from the previous iteration
[0: scale_index -1]. We get the current scale by
image_scales[scale_index], and run image resizing, feature extraction and
keypoint selection. Then we will get a new set of selected_boxes for
current scale. In the end, we concat the previous boxes with current
selected_boxes as the output.
Args:
scale_index: A valid index in the image_scales.
boxes: Box tensor with the shape of [N, 4].
features: Feature tensor with the shape of [N, depth].
scales: Scale tensor with the shape of [N].
scores: Attention score tensor with the shape of [N].
reuse: Whether or not the layer and its variables should be reused.
Returns:
scale_index: The next scale index for processing.
boxes: Concatenated box tensor with the shape of [K, 4]. K >= N.
features: Concatenated feature tensor with the shape of [K, depth].
scales: Concatenated scale tensor with the shape of [K].
scores: Concatenated attention score tensor with the shape of [K].
"""
scale = tf.gather(image_scales, scale_index)
new_image_size = tf.to_int32(tf.round(original_image_shape_float * scale))
resized_image = tf.image.resize_bilinear(image_tensor, new_image_size)
attention, feature_map = model_fn(
resized_image, normalized_image=True, reuse=reuse)
rf_boxes = CalculateReceptiveBoxes(
tf.shape(feature_map)[1],
tf.shape(feature_map)[2], rf, stride, padding)
# Re-project back to the original image space.
rf_boxes = tf.divide(rf_boxes, scale)
attention = tf.reshape(attention, [-1])
feature_map = tf.reshape(feature_map, [-1, feature_depth])
# Use attention score to select feature vectors.
indices = tf.reshape(tf.where(attention >= abs_thres), [-1])
selected_boxes = tf.gather(rf_boxes, indices)
selected_features = tf.gather(feature_map, indices)
selected_scores = tf.gather(attention, indices)
selected_scales = tf.ones_like(selected_scores, tf.float32) / scale
# Concat with the previous result from different scales.
boxes = tf.concat([boxes, selected_boxes], 0)
features = tf.concat([features, selected_features], 0)
scales = tf.concat([scales, selected_scales], 0)
scores = tf.concat([scores, selected_scores], 0)
return scale_index + 1, boxes, features, scales, scores
def proposal_layer(rpn_cls_prob, rpn_bbox_pred, im_info, cfg_key, _feat_stride, anchors, num_anchors):
if type(cfg_key) == bytes:
cfg_key = cfg_key.decode('utf-8')
pre_nms_topN = cfg[cfg_key].RPN_PRE_NMS_TOP_N
post_nms_topN = cfg[cfg_key].RPN_POST_NMS_TOP_N
nms_thresh = cfg[cfg_key].RPN_NMS_THRESH
# Get the scores and bounding boxes
scores = rpn_cls_prob[:, :, :, num_anchors:]
scores = tf.reshape(scores, shape=(-1,))
rpn_bbox_pred = tf.reshape(rpn_bbox_pred, shape=(-1, 4))
proposals = bbox_transform_inv_tf(anchors, rpn_bbox_pred)
proposals = clip_boxes_tf(proposals, im_info[:2])
# Non-maximal suppression
indices = tf.image.non_max_suppression(proposals, scores, max_output_size=post_nms_topN, iou_threshold=nms_thresh)
boxes = tf.gather(proposals, indices)
boxes = tf.to_float(boxes)
scores = tf.gather(scores, indices)
scores = tf.reshape(scores, shape=(-1, 1))
# Only support single image as input
batch_inds = tf.zeros((tf.shape(indices)[0], 1), dtype=tf.float32)
blob = tf.concat([batch_inds, boxes], 1)
return blob, scores
def build_eval_graph(self):
analogy_a = tf.placeholder(dtype=tf.int32)
analogy_b = tf.placeholder(dtype=tf.int32)
analogy_c = tf.placeholder(dtype=tf.int32)
norm_w_embed = tf.nn.l2_normalize(self._w_embed_in, 1)
a_embed = tf.gather(norm_w_embed, analogy_a)
b_embed = tf.gather(norm_w_embed, analogy_b)
c_embed = tf.gather(norm_w_embed, analogy_c)
target = c_embed + (b_embed - a_embed)
cosine_analogy_dist = tf.matmul(target, norm_w_embed, transpose_b=True)
_, analogy_indices = tf.nn.top_k(cosine_analogy_dist, word_config.top_k_analogy)
near_word = tf.placeholder(dtype=tf.int32)
near_embed = tf.gather(norm_w_embed, near_word)
cosine_near_dist = tf.matmul(near_embed, norm_w_embed, transpose_b=True)
near_val, near_ind = tf.nn.top_k(cosine_near_dist, min(1000, self._vocab_size))
self._analogy_a = analogy_a
self._analogy_b = analogy_b
self._analogy_c = analogy_c
self._analogy_indices = analogy_indices
self._near_word = near_word
self._near_val = near_val
self._near_ind = near_ind
def mil_prediction(pred, n=1):
i = tf.argsort(pred[:, 1], axis=0)
i = i[len(pred) - n:len(pred)]
return (tf.gather(pred, i), i)
def __init__(self,
field_sizes=None,
embed_size=10,
layer_sizes=None,
layer_acts=None,
drop_out=None,
embed_l2=None,
layer_l2=None,
init_path=None,
opt_algo='gd',
learning_rate=1e-2,
random_seed=None,
layer_norm=True):
Model.__init__(self)
init_vars = []
num_inputs = len(field_sizes)
for i in range(num_inputs):
init_vars.append(('embed_%d' % i, [field_sizes[i],
embed_size], 'xavier', dtype))
num_pairs = int(num_inputs * (num_inputs - 1) / 2)
node_in = num_inputs * embed_size + num_pairs
init_vars.append(('kernel', [embed_size, num_pairs,
embed_size], 'xavier', dtype))
for i in range(len(layer_sizes)):
init_vars.append(('w%d' % i, [node_in,
layer_sizes[i]], 'xavier', dtype))
init_vars.append(('b%d' % i, [layer_sizes[i]], 'zero', dtype))
node_in = layer_sizes[i]
self.graph = tf.Graph()
with self.graph.as_default():
if random_seed is not None:
tf.set_random_seed(random_seed)
self.X = [tf.sparse_placeholder(dtype) for i in range(num_inputs)]
self.y = tf.placeholder(dtype)
self.keep_prob_train = 1 - np.array(drop_out)
self.keep_prob_test = np.ones_like(drop_out)
self.layer_keeps = tf.placeholder(dtype)
self.vars = utils.init_var_map(init_vars, init_path)
w0 = [self.vars['embed_%d' % i] for i in range(num_inputs)]
xw = tf.concat([
tf.sparse_tensor_dense_matmul(self.X[i], w0[i])
for i in range(num_inputs)
], 1)
xw3d = tf.reshape(xw, [-1, num_inputs, embed_size])
row = []
col = []
for i in range(num_inputs - 1):
for j in range(i + 1, num_inputs):
row.append(i)
col.append(j)
# batch * pair * k
p = tf.transpose(
# pair * batch * k
tf.gather(
# field * batch * k
tf.transpose(xw3d, [1, 0, 2]),
row),
[1, 0, 2])
# batch * pair * k
q = tf.transpose(tf.gather(tf.transpose(xw3d, [1, 0, 2]), col),
[1, 0, 2])
# batch * pair * k
p = tf.reshape(p, [-1, num_pairs, embed_size])
# batch * pair * k
q = tf.reshape(q, [-1, num_pairs, embed_size])
# k * pair * k
k = self.vars[
'kernel'] # 外积生成二维矩阵; kernel就是用来和二维矩阵进行"卷积"（对应位置相乘相加）的。
# batch * 1 * pair * k
p = tf.expand_dims(p, 1) # 1表示在原来第一维度后面加一维
# batch * pair
kp = tf.reduce_sum(
# batch * pair * k
tf.multiply(
# batch * pair * k
tf.transpose(
# batch * k * pair
tf.reduce_sum(
# batch * k * pair * k
tf.multiply(p, k),
-1),
[0, 2, 1]),
q),
-1)
#
# if layer_norm:
# # x_mean, x_var = tf.nn.moments(xw, [1], keep_dims=True)
# # xw = (xw - x_mean) / tf.sqrt(x_var)
# # x_g = tf.Variable(tf.ones([num_inputs * embed_size]), name='x_g')
# # x_b = tf.Variable(tf.zeros([num_inputs * embed_size]), name='x_b')
# # x_g = tf.Print(x_g, [x_g[:10], x_b])
# # xw = xw * x_g + x_b
# p_mean, p_var = tf.nn.moments(op, [1], keep_dims=True)
# op = (op - p_mean) / tf.sqrt(p_var)
# p_g = tf.Variable(tf.ones([embed_size**2]), name='p_g')
# p_b = tf.Variable(tf.zeros([embed_size**2]), name='p_b')
# # p_g = tf.Print(p_g, [p_g[:10], p_b])
# op = op * p_g + p_b
l = tf.concat([xw, kp], 1)
for i in range(len(layer_sizes)):
wi = self.vars['w%d' % i]
bi = self.vars['b%d' % i]
l = tf.nn.dropout(
utils.activate(tf.matmul(l, wi) + bi, layer_acts[i]),
self.layer_keeps[i])
l = tf.squeeze(l)
self.y_prob = tf.sigmoid(l)
self.loss = tf.reduce_mean(
tf.nn.sigmoid_cross_entropy_with_logits(logits=l,
labels=self.y))
if layer_l2 is not None:
self.loss += embed_l2 * tf.nn.l2_loss(xw) #tf.concat(w0, 0))
for i in range(len(layer_sizes)):
wi = self.vars['w%d' % i]
self.loss += layer_l2[i] * tf.nn.l2_loss(wi)
self.optimizer = utils.get_optimizer(opt_algo, learning_rate,
self.loss)
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
self.sess = tf.Session(config=config)
tf.global_variables_initializer().run(session=self.sess)
def __init__(self,
field_sizes=None,
embed_size=10,
layer_sizes=None,
layer_acts=None,
drop_out=None,
embed_l2=None,
layer_l2=None,
init_path=None,
opt_algo='gd',
learning_rate=1e-2,
random_seed=None):
Model.__init__(self)
init_vars = []
num_inputs = len(field_sizes) # 26
for i in range(num_inputs): # 一个field就对应一个embedding的参数
init_vars.append(('embed_%d' % i, [field_sizes[i],
embed_size], 'xavier', dtype))
num_pairs = int(num_inputs * (num_inputs - 1) / 2)
node_in = num_inputs * embed_size + num_pairs # 第一个隐藏层的输入维度，lz大小k * pairs, lp只是pairs，也就是lp一个pair生成一个值，lz一个pair生成一个embedding大小
# node_in = num_inputs * (embed_size + num_inputs)
for i in range(len(layer_sizes)):
init_vars.append(('w%d' % i, [node_in,
layer_sizes[i]], 'xavier', dtype))
init_vars.append(('b%d' % i, [layer_sizes[i]], 'zero', dtype))
node_in = layer_sizes[i]
self.graph = tf.Graph()
with self.graph.as_default():
if random_seed is not None:
tf.set_random_seed(random_seed)
self.X = [tf.sparse_placeholder(dtype) for i in range(num_inputs)
] # num_input就是field的个数N，也就是说原始输入不用做one-hot
self.y = tf.placeholder(dtype)
self.keep_prob_train = 1 - np.array(drop_out)
self.keep_prob_test = np.ones_like(drop_out)
self.layer_keeps = tf.placeholder(dtype)
self.vars = utils.init_var_map(init_vars, init_path)
w0 = [self.vars['embed_%d' % i] for i in range(num_inputs)]
xw = tf.concat([
tf.sparse_tensor_dense_matmul(self.X[i], w0[i])
for i in range(num_inputs)
], 1) # 相乘就是在做embedding，concat就是把结果拼接起来
xw3d = tf.reshape(xw, [-1, num_inputs, embed_size
]) # [num_samples, num_field, embed_sz]
row = []
col = []
for i in range(num_inputs - 1):
for j in range(i + 1, num_inputs):
row.append(i)
col.append(j)
# batch * pair * k
p = tf.transpose(
# pair * batch * k
tf.gather(
# num * batch * k
tf.transpose(xw3d, [1, 0, 2]),
row),
[1, 0, 2])
# batch * pair * k
q = tf.transpose(tf.gather(tf.transpose(xw3d, [1, 0, 2]), col),
[1, 0, 2])
p = tf.reshape(p, [-1, num_pairs, embed_size])
q = tf.reshape(q, [-1, num_pairs, embed_size])
ip = tf.reshape(tf.reduce_sum(p * q, [-1]), [-1, num_pairs])
# simple but redundant
# batch * n * 1 * k, batch * 1 * n * k
# ip = tf.reshape(
# tf.reduce_sum(
# tf.expand_dims(xw3d, 2) *
# tf.expand_dims(xw3d, 1),
# 3),
# [-1, num_inputs**2])
l = tf.concat([xw, ip], 1)
for i in range(len(layer_sizes)):
wi = self.vars['w%d' % i]
bi = self.vars['b%d' % i]
l = tf.nn.dropout(
utils.activate(tf.matmul(l, wi) + bi, layer_acts[i]),
self.layer_keeps[i])
l = tf.squeeze(l)
self.y_prob = tf.sigmoid(l)
self.loss = tf.reduce_mean(
tf.nn.sigmoid_cross_entropy_with_logits(logits=l,
labels=self.y))
if layer_l2 is not None:
self.loss += embed_l2 * tf.nn.l2_loss(xw)
for i in range(len(layer_sizes)):
wi = self.vars['w%d' % i]
self.loss += layer_l2[i] * tf.nn.l2_loss(wi)
self.optimizer = utils.get_optimizer(opt_algo, learning_rate,
self.loss)
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
self.sess = tf.Session(config=config)
tf.global_variables_initializer().run(session=self.sess)
def train_batch(self, source_charseq_ids, source_charseqs, target_charseq_ids, target_charseqs):
#print('train batch')
# TODO(lemmatizer_noattn): Modify target_charseqs by appending EOW; only the version with appended EOW is used from now on.
target_charseqs = self._append_eow(target_charseqs)
with tf.GradientTape() as tape:
# TODO(lemmatizer_noattn): Embed source charseqs
embedded = self._model.source_embeddings(source_charseqs)
# TODO: Run self._model.source_rnn on the embedded sequences, returning outputs in `source_encoded`.
source_encoded = self._model.source_rnn(embedded)
# Copy the source_encoded to corresponding batch places, and then flatten it
source_mask = tf.not_equal(source_charseq_ids, 0)
source_encoded = tf.boolean_mask(tf.gather(source_encoded, source_charseq_ids), source_mask)
targets = tf.boolean_mask(tf.gather(target_charseqs, target_charseq_ids), source_mask)
class DecoderTraining(decoder.BaseDecoder):
@property
def batch_size(self): return tf.shape(self._source_encoded)[0] # TODO: Return the batch size of self._source_encoded, using tf.shape
@property
def output_size(self): return tf.shape(self._targets[1]) # TODO(lemmatizer_noattn): Return the number logits per each output
@property
def output_dtype(self): return tf.float32 # TODO(lemmatizer_noattn): Return the type of the logits
def _with_attention(self, inputs, states):
# TODO: Compute the attention.
# - Take self._source_encoded and pass it through the self._model.attention_source_layer.
att = self._model.attention_source_layer(self._source_encoded)
# Because self._source_encoded does not change, you should in fact do it in `initialize`.
# TODO: WHAT THE??
# - Pass `states` though self._model.attention_state_layer.
states_att = self._model.attention_state_layer(states)
# - Sum the two outputs. However, the first has shape [a, b, c] and the second [a, c]. Therefore,
# somehow expand the second to [a, b, c] first. (Hint: use broadcasting rules.)
states_att = tf.tile(tf.expand_dims(states_att, 1), [1, tf.shape(att)[1], 1])
#print('=======')
#print(att.shape)
#print(states_att.shape)
#print('=======')
sum_att = tf.add(att, states_att)
#print(sum_att.shape)
# - Pass the sum through `tf.tanh`, then self._model.attention_weight_layer.
sum_att = self._model.attention_weight_layer(tf.tanh(sum_att))
# - Then, run softmax on a suitable axis (the one corresponding to characters), generating `weights`.
weights = tf.math.softmax(sum_att, axis=1)
# - Multiply `self._source_encoded` with `weights` and sum the result in the axis
# corresponding to characters, generating `attention`. Therefore, `attention` is a a fixed-size
# representation for every batch element, independently on how many characters had
# the corresponding input forms.
attention = tf.reduce_sum(self._source_encoded * weights, axis=1)
# - Finally concatenate `inputs` and `attention` and return the result.
#print('-------')
#print(inputs.shape)
#print(attention.shape)
#print('-------')
return tf.concat([inputs, attention], axis=1)
def initialize(self, layer_inputs, initial_state=None):
self._model, self._source_encoded, self._targets = layer_inputs
# TODO(lemmatozer_noattn): Define `finished` as a vector of self.batch_size of `False` [see tf.fill].
finished = tf.fill([self.batch_size], False)
# TODO(lemmatizer_noattn): Define `inputs` as a vector of self.batch_size of MorphoDataset.Factor.BOW [see tf.fill],
# embedded using self._model.target_embedding
inputs = self._model.target_embedding(tf.fill([self.batch_size], MorphoDataset.Factor.BOW))
# TODO: Define `states` as the last words from self._source_encoded
#TODO: WHAT THE?
#print('states')
#print(self._source_encoded.shape)
states = self._source_encoded[:, -1]
#print(states.shape)
# TODO: Pass `inputs` through `self._with_attention(inputs, states)`.
#print('shapes')
#print(inputs.shape)
#print(states.shape)
inputs = self._with_attention(inputs, states)
#print(inputs.shape)
return finished, inputs, states
def step(self, time, inputs, states):
# TODO(lemmatizer_noattn): Pass `inputs` and `[states]` through self._model.target_rnn_cell, generating
# `outputs, [states]`.
outputs, [states] = self._model.target_rnn_cell(inputs, [states])
# TODO(lemmatizer_noattn): Overwrite `outputs` by passing them through self._model.target_output_layer,
outputs = self._model.target_output_layer(outputs)
# TODO(lemmatizer_noattn): Define `next_inputs` by embedding `time`-th words from `self._targets`.
next_inputs = self._model.target_embedding(self._targets[:,time])
# TODO(lemmatizer_noattn): Define `finished` as True if `time`-th word from `self._targets` is EOW, False otherwise.
finished = tf.equal(self._targets[:, time], MorphoDataset.Factor.EOW)
# Again, no == or !=.
# TODO: Pass `next_inputs` through `self._with_attention(inputs, states)`.
#print('stepping')
#print(next_inputs.shape)
#print(states.shape)
next_inputs = self._with_attention(next_inputs, states)
return outputs, states, next_inputs, finished
#print('decode')
output_layer, _, _ = DecoderTraining()([self._model, source_encoded, targets])
# TODO(lemmatizer_noattn): Compute loss. Use only nonzero `targets` as a mask.
#print('loss')
loss = self._loss(targets, output_layer, tf.not_equal(targets, 0))
#print('gradient')
gradients = tape.gradient(loss, self._model.variables)
#print('optimizer')
self._optimizer.apply_gradients(zip(gradients, self._model.variables))
#print('metrics')
tf.summary.experimental.set_step(self._optimizer.iterations)
with self._writer.as_default():
for name, metric in self._metrics_training.items():
metric.reset_states()
if name == "loss": metric(loss)
else: metric(targets, output_layer, tf.not_equal(targets, 0))
tf.summary.scalar("train/{}".format(name), metric.result())
#print('end train batch')
#print(output_layer.shape)
return tf.math.argmax(output_layer, axis=2)
def predict_batch(self, source_charseq_ids, source_charseqs):
# TODO(lemmatizer_noattn)(train_batch): Embed source charseqs
embedded = self._model.source_embeddings(source_charseqs)
# TODO(train_batch): Run self._model.source_rnn on the embedded sequences, returning outputs in `source_encoded`.
source_encoded = self._model.source_rnn(embedded)
# Copy the source_encoded to corresponding batch places, and then flatten it
source_mask = tf.not_equal(source_charseq_ids, 0)
source_encoded = tf.boolean_mask(tf.gather(source_encoded, source_charseq_ids), source_mask)
class DecoderPrediction(decoder.BaseDecoder):
@property
def batch_size(self): return tf.shape(self._source_encoded)[0] # TODO(lemmatizer_noattn)(train_batch): Return the batch size of self._source_encoded, using tf.shape
@property
def output_size(self): return 1 # TODO(lemmatizer_noattn): Return 1 because we are returning directly the predictions
@property
def output_dtype(self): return tf.int32 # TODO(lemmatizer_noattn): Return tf.int32 because the predictions are integral
def _with_attention(self, inputs, states):
# TODO: A copy of _with_attention from train_batch; you can of course
# move the definition to a place where it can be reused in both places.
# TODO: Compute the attention.
# - Take self._source_encoded and pass it through the self._model.attention_source_layer.
att = self._model.attention_source_layer(self._source_encoded)
# Because self._source_encoded does not change, you should in fact do it in `initialize`.
# TODO: WHAT THE??
# - Pass `states` though self._model.attention_state_layer.
states_att = self._model.attention_state_layer(states)
# - Sum the two outputs. However, the first has shape [a, b, c] and the second [a, c]. Therefore,
# somehow expand the second to [a, b, c] first. (Hint: use broadcasting rules.)
states_att = tf.tile(tf.expand_dims(states_att, 1), [1, tf.shape(att)[1], 1])
sum_att = tf.add(att, states_att)
# - Pass the sum through `tf.tanh`, then self._model.attention_weight_layer.
sum_att = self._model.attention_weight_layer(tf.tanh(sum_att))
# - Then, run softmax on a suitable axis (the one corresponding to characters), generating `weights`.
weights = tf.math.softmax(sum_att,axis=1)
# - Multiply `self._source_encoded` with `weights` and sum the result in the axis
# corresponding to characters, generating `attention`. Therefore, `attention` is a a fixed-size
# representation for every batch element, independently on how many characters had
# the corresponding input forms.
attention = tf.reduce_sum(self._source_encoded * weights, axis=1)
# - Finally concatenate `inputs` and `attention` and return the result.
#print(inputs.shape)
#print(attention.shape)
return tf.concat([inputs, attention], axis=1)
def initialize(self, layer_inputs, initial_state=None):
self._model, self._source_encoded = layer_inputs
# TODO(lemmatizer_noattn)(train_batch): Define `finished` as a vector of self.batch_size of `False` [see tf.fill].
finished = tf.fill([self.batch_size], False)
# TODO(lemmatizer_noattn)(train_batch): Define `inputs` as a vector of self.batch_size of MorphoDataset.Factor.BOW [see tf.fill],
# embedded using self._model.target_embedding
inputs = self._model.target_embedding(tf.fill([self.batch_size], MorphoDataset.Factor.BOW))
# TODO(train_batch): Define `states` as the last words from self._source_encoded
# TODO: WHAT THE??
states = self._source_encoded[:, -1]
# TODO(train_batch): Pass `inputs` through `self._with_attention(inputs, states)`.
inputs = self._with_attention(inputs, states)
return finished, inputs, states
def step(self, time, inputs, states):
# TODO(lemmatizer_noattn)(train_batch): Pass `inputs` and `[states]` through self._model.target_rnn_cell, generating
# `outputs, [states]`.
outputs, [states] = self._model.target_rnn_cell(inputs, [states])
# TODO(lemmatizer_noattn)(train_batch): Overwrite `outputs` by passing them through self._model.target_output_layer,
outputs = self._model.target_output_layer(outputs)
# TODO(lemmatizer_noattn): Overwirte `outputs` by passing them through `tf.argmax` on suitable axis and with
# `output_type=tf.int32` parameter.
outputs = tf.argmax(outputs, axis=1, output_type=tf.int32)
# TODO(lemmatizer_noattn): Define `next_inputs` by embedding the `outputs`
next_inputs = self._model.target_embedding(outputs)
# TODO(lemmatizer_noattn): Define `finished` as True if `outputs` are EOW, False otherwise. [No == or !=].
finished = tf.equal(outputs, MorphoDataset.Factor.EOW)
# TODO(train_batch): Pass `next_inputs` through `self._with_attention(inputs, states)`.
next_inputs = self._with_attention(next_inputs, states)
return outputs, states, next_inputs, finished
#print('predictions babyyyy')
predictions, _, _ = DecoderPrediction(maximum_iterations=tf.shape(source_charseqs)[1] + 10)([self._model, source_encoded])
return predictions
def _interpolate(im, x, y, out_size):
with tf.variable_scope('_interpolate'):
# constants
num_batch = tf.shape(im)[0]
height = tf.shape(im)[1]
width = tf.shape(im)[2]
channels = tf.shape(im)[3]
x = tf.cast(x, 'float32')
y = tf.cast(y, 'float32')
height_f = tf.cast(height, 'float32')
width_f = tf.cast(width, 'float32')
out_height = out_size[0]
out_width = out_size[1]
zero = tf.zeros([], dtype='int32')
max_y = tf.cast(tf.shape(im)[1] - 1, 'int32')
max_x = tf.cast(tf.shape(im)[2] - 1, 'int32')
# scale indices from [-1, 1] to [0, width/height]
x = (x + 1.0)*(width_f) / 2.0
y = (y + 1.0)*(height_f) / 2.0
# do sampling
x0 = tf.cast(tf.floor(x), 'int32')
x1 = x0 + 1
y0 = tf.cast(tf.floor(y), 'int32')
y1 = y0 + 1
x0 = tf.clip_by_value(x0, zero, max_x)
x1 = tf.clip_by_value(x1, zero, max_x)
y0 = tf.clip_by_value(y0, zero, max_y)
y1 = tf.clip_by_value(y1, zero, max_y)
dim2 = width
dim1 = width*height
base = _repeat(tf.range(num_batch)*dim1, out_height*out_width)
base_y0 = base + y0*dim2
base_y1 = base + y1*dim2
idx_a = base_y0 + x0
idx_b = base_y1 + x0
idx_c = base_y0 + x1
idx_d = base_y1 + x1
# use indices to lookup pixels in the flat image and restore
# channels dim
im_flat = tf.reshape(im, tf.pack([-1, channels]))
im_flat = tf.cast(im_flat, 'float32')
Ia = tf.gather(im_flat, idx_a)
Ib = tf.gather(im_flat, idx_b)
Ic = tf.gather(im_flat, idx_c)
Id = tf.gather(im_flat, idx_d)
# and finally calculate interpolated values
x0_f = tf.cast(x0, 'float32')
x1_f = tf.cast(x1, 'float32')
y0_f = tf.cast(y0, 'float32')
y1_f = tf.cast(y1, 'float32')
wa = tf.expand_dims(((x1_f-x) * (y1_f-y)), 1)
wb = tf.expand_dims(((x1_f-x) * (y-y0_f)), 1)
wc = tf.expand_dims(((x-x0_f) * (y1_f-y)), 1)
wd = tf.expand_dims(((x-x0_f) * (y-y0_f)), 1)
output = tf.add_n([wa*Ia, wb*Ib, wc*Ic, wd*Id])
return output
def task_metalearn(inp, reuse=True):
inputa, inputb, labela, labelb = inp
if FLAGS.datasource in ['2D']:
input_task_emb = tf.concat((inputa, labela), axis=-1)
with tf.variable_scope('first_embedding_sync', reuse=tf.AUTO_REUSE):
input_task_emb = tf.layers.dense(input_task_emb, units=FLAGS.sync_filters,
name='first_embedding_sync_dense')
if FLAGS.num_classes < FLAGS.update_batch_size:
with tf.variable_scope('reg_clustering', reuse=tf.AUTO_REUSE):
assign_mat = tf.nn.softmax(tf.layers.dense(input_task_emb, units=FLAGS.num_classes), dim=1)
input_task_emb_cat = tf.matmul(tf.transpose(assign_mat, perm=[1, 0]), input_task_emb)
elif FLAGS.datasource in ['plainmulti', 'artmulti']:
input_task_emb = self.image_embed.model(tf.reshape(inputa,
[-1, self.img_size, self.img_size,
self.channels]))
proto_emb = []
labela2idx = tf.argmax(labela, axis=1)
for class_idx in range(FLAGS.num_classes):
tmp_gs = tf.equal(labela2idx, class_idx)
gs = tf.where(tmp_gs)
new_vec = tf.reduce_mean(tf.gather(input_task_emb, gs), axis=0)
proto_emb.append(new_vec)
proto_emb = tf.squeeze(tf.stack(proto_emb))
label_cat = tf.eye(5)
input_task_emb_cat = tf.concat((proto_emb, label_cat), axis=-1)
if FLAGS.datasource in ['2D']:
task_embed_vec, task_emb_loss = self.lstmae.model(input_task_emb)
propagate_knowledge = self.metagraph.model(input_task_emb_cat)
elif FLAGS.datasource in ['plainmulti', 'artmulti']:
task_embed_vec, task_emb_loss = self.lstmae.model(input_task_emb_cat)
propagate_knowledge = self.metagraph.model(proto_emb)
task_embed_vec_graph, task_emb_loss_graph = self.lstmae_graph.model(propagate_knowledge)
task_enhanced_emb_vec = tf.concat([task_embed_vec, task_embed_vec_graph], axis=1)
with tf.variable_scope('task_specific_mapping', reuse=tf.AUTO_REUSE):
eta = []
for key in weights.keys():
weight_size = np.prod(weights[key].get_shape().as_list())
eta.append(tf.reshape(
tf.layers.dense(task_enhanced_emb_vec, weight_size, activation=tf.nn.sigmoid,
name='eta_{}'.format(key)), tf.shape(weights[key])))
eta = dict(zip(weights.keys(), eta))
task_weights = dict(zip(weights.keys(), [weights[key] * eta[key] for key in weights.keys()]))
task_outputbs, task_lossesb = [], []
if self.classification:
task_accuraciesb = []
task_outputa = self.forward(inputa, task_weights, reuse=reuse)
task_lossa = self.loss_func(task_outputa, labela)
grads = tf.gradients(task_lossa, list(task_weights.values()))
if FLAGS.stop_grad:
grads = [tf.stop_gradient(grad) for grad in grads]
gradients = dict(zip(task_weights.keys(), grads))
fast_weights = dict(
zip(task_weights.keys(),
[task_weights[key] - self.update_lr * gradients[key] for key in task_weights.keys()]))
output = self.forward(inputb, fast_weights, reuse=True)
task_outputbs.append(output)
task_lossesb.append(self.loss_func(output, labelb))
for j in range(num_updates - 1):
loss = self.loss_func(self.forward(inputa, fast_weights, reuse=True), labela)
grads = tf.gradients(loss, list(fast_weights.values()))
if FLAGS.stop_grad:
grads = [tf.stop_gradient(grad) for grad in grads]
gradients = dict(zip(fast_weights.keys(), grads))
fast_weights = dict(zip(fast_weights.keys(),
[fast_weights[key] - self.update_lr * gradients[key] for key in
fast_weights.keys()]))
output = self.forward(inputb, fast_weights, reuse=True)
task_outputbs.append(output)
task_lossesb.append(self.loss_func(output, labelb))
task_output = [task_emb_loss, task_emb_loss_graph, task_outputa, task_outputbs, task_lossa,
task_lossesb]
if self.classification:
task_accuracya = tf.contrib.metrics.accuracy(tf.argmax(tf.nn.softmax(task_outputa), 1),
tf.argmax(labela, 1))
for j in range(num_updates):
task_accuraciesb.append(
tf.contrib.metrics.accuracy(tf.argmax(tf.nn.softmax(task_outputbs[j]), 1),
tf.argmax(labelb, 1)))
task_output.extend([task_accuracya, task_accuraciesb])
return task_output
def _compute_logits_impl(self, context_features, example_features, labels,
mode, params, config):
# Scatter/Gather per-example scores through groupwise comparison. Each
# instance in a mini-batch will form a number of groups. Each group of
# examples are scored by `_score_fn` and scores for individual examples are
# accumulated into logits.
with tf.compat.v1.name_scope('groupwise_dnn_v2'):
batch_size, list_size, is_valid = _infer_sizes(example_features, labels)
# For each example feature, assuming the shape is [batch_size, list_size,
# feature_size], the groups are formed along the 2nd dim. Each group has a
# 'group_size' number of indices in [0, list_size). Based on these
# indices, we can gather the example feature into a sub-tensor for each
# group. The total number of groups we have for a mini-batch is batch_size
# * num_groups. Inside each group, we have a 'group_size' number of
# examples.
self._update_scatter_gather_indices(is_valid, mode)
num_groups = tf.shape(input=self._indices_mask)[1]
with tf.compat.v1.name_scope('group_features'):
# For context features, We have shape [batch_size * num_groups, ...].
large_batch_context_features = {}
for name, value in six.iteritems(context_features):
# [batch_size, 1, ...].
value = tf.expand_dims(value, axis=1)
# [batch_size, num_groups, ...].
value = tf.gather(value, tf.zeros([num_groups], tf.int32), axis=1)
# [batch_size * num_groups, ...]
large_batch_context_features[name] = utils.reshape_first_ndims(
value, 2, [batch_size * num_groups])
# For example feature, we have shape [batch_size * num_groups,
# group_size, ...].
large_batch_group_features = {}
for name, value in six.iteritems(example_features):
# [batch_size, num_groups, group_size, ...].
value = tf.gather_nd(value, self._feature_gather_indices)
# [batch_size * num_groups, group_size, ...].
large_batch_group_features[name] = utils.reshape_first_ndims(
value, 3, [batch_size * num_groups, self._group_size])
# Do the inference and get scores for the large batch of [batch_size *
# num_groups, logits_size] and reshape them to [batch_size, num_groups,
# logits_size].
with tf.compat.v1.variable_scope('group_score'):
scores = self._score_fn(large_batch_context_features,
large_batch_group_features, mode, params,
config)
scores = tf.reshape(scores, tf.shape(self._score_scatter_indices)[0:3])
with tf.compat.v1.name_scope('accumulate_scores'):
# Reset invalid scores to 0 based on mask.
scores = tf.where(
tf.gather(
tf.expand_dims(self._indices_mask, 2),
tf.zeros([tf.shape(scores)[2]], tf.int32),
axis=2), scores, tf.zeros_like(scores))
# Scatter scores from [batch_size, num_groups, logits_size] to
# [batch_size, list_size].
logits = tf.scatter_nd(self._score_scatter_indices, scores,
[batch_size, list_size])
# Use average.
logits /= tf.cast(tf.shape(scores)[2], dtype=tf.float32)
return logits
def adaptive_affinity_loss(labels,
one_hot_lab,
probs,
size,
num_classes,
kld_margin,
w_edge,
w_not_edge):
"""Adaptive affinity field (AAF) loss.
This function computes AAF loss. There are three components in the function:
1) extracts edges from the ground-truth labels.
2) extracts ignored pixels and their paired pixels (usually the eight corner
pixels).
3) extracts eight corner pixels/predictions from the center in a
(2*size+1)x(2*size+1) patch
4) computes KL-Divergence between center pixels and their paired pixels (the
eight corner).
5) imposes adaptive weightings on the loss.
Args:
labels: A tensor of size [batch_size, height_in, width_in], indicating
semantic segmentation ground-truth labels.
one_hot_lab: A tensor of size [batch_size, height_in, width_in, num_classes]
which is the ground-truth labels in the form of one-hot vector.
probs: A tensor of size [batch_size, height_in, width_in, num_classes],
indicating segmentation predictions.
size: A number indicating the half size of a patch.
num_classes: A number indicating the total number of valid classes. The
kld_margin: A number indicating the margin for KL-Divergence at edge.
w_edge: A number indicating the weighting for KL-Divergence at edge.
w_not_edge: A number indicating the weighting for KL-Divergence at non-edge.
Returns:
Two 1-D tensors value indicating the loss at edge and non-edge.
"""
# Compute ignore map (e.g, label of 255 and their paired pixels).
labels = tf.squeeze(labels, axis=-1) # NxHxW
ignore = nnx.ignores_from_label(labels, num_classes, size) # NxHxWx8
not_ignore = tf.logical_not(ignore)
not_ignore = tf.expand_dims(not_ignore, axis=3) # NxHxWx1x8
# Compute edge map.
edge = nnx.edges_from_label(one_hot_lab, size, 255) # NxHxWxCx8
# Remove ignored pixels from the edge/non-edge.
edge = tf.logical_and(edge, not_ignore)
not_edge = tf.logical_and(tf.logical_not(edge), not_ignore)
edge_indices = tf.where(tf.reshape(edge, [-1]))
not_edge_indices = tf.where(tf.reshape(not_edge, [-1]))
# Extract eight corner from the center in a patch as paired pixels.
probs_paired = nnx.eightcorner_activation(probs, size) # NxHxWxCx8
probs = tf.expand_dims(probs, axis=-1) # NxHxWxCx1
bot_epsilon = tf.constant(1e-4, name='bot_epsilon')
top_epsilon = tf.constant(1.0, name='top_epsilon')
neg_probs = tf.clip_by_value(
1-probs, bot_epsilon, top_epsilon)
neg_probs_paired = tf.clip_by_value(
1-probs_paired, bot_epsilon, top_epsilon)
probs = tf.clip_by_value(
probs, bot_epsilon, top_epsilon)
probs_paired = tf.clip_by_value(
probs_paired, bot_epsilon, top_epsilon)
# Compute KL-Divergence.
kldiv = probs_paired*tf.log(probs_paired/probs)
kldiv += neg_probs_paired*tf.log(neg_probs_paired/neg_probs)
edge_loss = tf.maximum(0.0, kld_margin-kldiv)
not_edge_loss = kldiv
# Impose weights on edge/non-edge losses.
one_hot_lab = tf.expand_dims(one_hot_lab, axis=-1)
w_edge = tf.reduce_sum(w_edge*one_hot_lab, axis=3, keep_dims=True) # NxHxWx1x1
w_not_edge = tf.reduce_sum(w_not_edge*one_hot_lab, axis=3, keep_dims=True) # NxHxWx1x1
edge_loss *= w_edge
not_edge_loss *= w_not_edge
not_edge_loss = tf.reshape(not_edge_loss, [-1])
not_edge_loss = tf.gather(not_edge_loss, not_edge_indices)
edge_loss = tf.reshape(edge_loss, [-1])
edge_loss = tf.gather(edge_loss, edge_indices)
return edge_loss, not_edge_loss
def cost(self):
if self.locate is not False:
self.W0 = tf.constant(self.w0, dtype=tf.float32)
self.W1 = tf.constant(self.w1, dtype=tf.float32)
self.qtrainPH = tf.placeholder(tf.float32,
shape=(None, self.n_steps, self.n_obs),
name="qtrain")
self.qtrain_unflat = tf.reshape(self.qtrainPH,
[-1, self.n_obs]) #b*nxp
with tf.name_scope('cost'):
if ((self.rnn is False) and (self.cnn is not 'fft')):
B = tf.argmax(self.qtrain_unflat, 1)
qtrain_OH = tf.one_hot(
B, self.n_out, on_value=1, off_value=0,
axis=-1) #need one-hot for CE calculation
self.cross = tf.add(
tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(
logits=self.logits, labels=qtrain_OH),
name="cross"), self.reg)
self.accuracy = tf.reduce_mean(tf.cast(tf.equal(self.A, B),
tf.float32),
name="accuracy")
if self.locate is not False:
with tf.name_scope('rep_outputs'):
qhat_rep = tf.matmul(self.qhat, self.W0) #b*nx3x3x3l
SE = tf.matmul(
tf.square(tf.subtract(qhat_rep,
self.qtrain_unflat)),
self.W1) #b*nx3lx3lxl
SEbn = tf.reduce_min(SE, 1) #b*n
SSE = tf.reduce_mean(SEbn) #1
self.rmse = tf.add(tf.sqrt(SSE, name="rmse"), self.reg)
else:
self.rmse = tf.add(
tf.sqrt(tf.reduce_mean(
tf.square(
tf.subtract(self.qhat, self.qtrain_unflat))),
name="rmse"), self.reg)
if self.locate is not False:
self.cost = self.rmse
else:
self.cost = self.cross
else: #yes rnn or cnn is fft
B = tf.argmax(self.qtrain_unflat, 1) #b*nx1
qtrain_OH = tf.one_hot(
B, self.n_out, on_value=1, off_value=0,
axis=-1) #need one-hot for CE calculation
qtrain_tran = tf.transpose(self.qtrainPH, [1, 0, 2]) #nxbxp
self.qtrain_last = tf.gather(qtrain_tran,
int(qtrain_tran.get_shape()[0]) -
1) #bxp
BB = tf.argmax(self.qtrain_last, 1) #bx1
self.qtrain_last_OH = tf.one_hot(
BB, self.n_out, on_value=1, off_value=0,
axis=-1) #need one-hot for CE calculation
self.cross_last = tf.add(
tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(
logits=self.logits_last, labels=self.qtrain_last_OH),
name="cross_last"), self.reg)
self.accuracy_last = tf.reduce_mean(tf.cast(
tf.equal(self.AA, BB), tf.float32),
name="accuracy_last")
if self.locate is not False:
with tf.name_scope('rep_outputs'):
qhat_rep = tf.matmul(self.qhat_last,
self.W0) #bx3x3x3l
SE = tf.matmul(
tf.square(tf.subtract(qhat_rep, self.qtrain_last)),
self.W1) #bx3lx3lxl
SEnb = tf.reduce_min(SE, axis=1) #b
SSE = tf.reduce_mean(SEnb) #1
self.rmse_last = tf.add(tf.sqrt(SSE, name="rmse"),
self.reg)
else:
self.rmse_last = tf.add(
tf.sqrt(tf.reduce_mean(
tf.square(
tf.subtract(self.qtrain_last,
self.qhat_last))),
name="rmse_last"), self.reg)
if self.locate is not False:
self.cost = self.rmse_last
else:
self.cost = self.cross_last
with tf.name_scope('summaries'):
self.train_summary = tf.summary.scalar('mean/train_cost',
self.cost)
self.valid_summary = tf.summary.scalar('mean/valid_cost',
self.cost)
if self.locate is False:
if self.rnn is True or self.cnn is 'fft':
self.train_acc_summary = tf.summary.scalar(
'mean/train_accuracy', self.accuracy_last)
self.valid_acc_summary = tf.summary.scalar(
'mean/valid_accuracy', self.accuracy_last)
else:
self.train_acc_summary = tf.summary.scalar(
'mean/train_accuracy', self.accuracy)
self.valid_acc_summary = tf.summary.scalar(
'mean/valid_accuracy', self.accuracy)
def main(ckpt_weights, image_size, output_size, model_def, class_num, depth_multiplier, obj_thresh, iou_thresh, train_set, test_image):
h = Helper(None, class_num, f'data/{train_set}_anchor.npy', np.reshape(np.array(image_size), (-1, 2)), np.reshape(np.array(output_size), (-1, 2)))
network = eval(model_def) # type :yolo_mobilev2
yolo_model, yolo_model_warpper = network([image_size[0], image_size[1], 3], len(h.anchors[0]), class_num, alpha=depth_multiplier)
yolo_model_warpper.load_weights(str(ckpt_weights))
print(INFO, f' Load CKPT {str(ckpt_weights)}')
orig_img = h._read_img(str(test_image))
image_shape = orig_img.shape[0:2]
img, _ = h._process_img(orig_img, true_box=None, is_training=False, is_resize=True)
""" load images """
img = tf.expand_dims(img, 0)
y_pred = yolo_model_warpper.predict(img)
""" box list """
_yxyx_box = []
_yxyx_box_scores = []
""" preprocess label """
for l, pred_label in enumerate(y_pred):
""" split the label """
pred_xy = pred_label[..., 0:2]
pred_wh = pred_label[..., 2:4]
pred_confidence = pred_label[..., 4:5]
pred_cls = pred_label[..., 5:]
# box_scores = obj_score * class_score
box_scores = tf.sigmoid(pred_cls) * tf.sigmoid(pred_confidence)
# obj_mask = pred_confidence_score[..., 0] > obj_thresh
""" reshape box """
# NOTE tf_xywh_to_all will auto use sigmoid function
pred_xy_A, pred_wh_A = tf_xywh_to_all(pred_xy, pred_wh, l, h)
boxes = correct_box(pred_xy_A, pred_wh_A, image_size, image_shape)
boxes = tf.reshape(boxes, (-1, 4))
box_scores = tf.reshape(box_scores, (-1, class_num))
""" append box and scores to global list """
_yxyx_box.append(boxes)
_yxyx_box_scores.append(box_scores)
yxyx_box = tf.concat(_yxyx_box, axis=0)
yxyx_box_scores = tf.concat(_yxyx_box_scores, axis=0)
mask = yxyx_box_scores >= obj_thresh
""" do nms for every classes"""
_boxes = []
_scores = []
_classes = []
for c in range(class_num):
class_boxes = tf.boolean_mask(yxyx_box, mask[:, c])
class_box_scores = tf.boolean_mask(yxyx_box_scores[:, c], mask[:, c])
select = tf.image.non_max_suppression(
class_boxes, scores=class_box_scores, max_output_size=30, iou_threshold=iou_thresh)
class_boxes = tf.gather(class_boxes, select)
class_box_scores = tf.gather(class_box_scores, select)
_boxes.append(class_boxes)
_scores.append(class_box_scores)
_classes.append(tf.ones_like(class_box_scores) * c)
boxes = tf.concat(_boxes, axis=0)
classes = tf.concat(_classes, axis=0)
scores = tf.concat(_scores, axis=0)
""" draw box """
font = ImageFont.truetype(font='asset/FiraMono-Medium.otf',
size=tf.cast(tf.floor(3e-2 * image_shape[0] + 0.5), tf.int32).numpy())
thickness = (image_shape[0] + image_shape[1]) // 300
""" show result """
if len(classes) > 0:
pil_img = Image.fromarray(orig_img)
print(f'[top\tleft\tbottom\tright\tscore\tclass]')
for i, c in enumerate(classes):
box = boxes[i]
score = scores[i]
label = '{:2d} {:.2f}'.format(int(c.numpy()), score.numpy())
draw = ImageDraw.Draw(pil_img)
label_size = draw.textsize(label, font)
top, left, bottom, right = box
print(f'[{top:.1f}\t{left:.1f}\t{bottom:.1f}\t{right:.1f}\t{score:.2f}\t{int(c):2d}]')
top = max(0, tf.cast(tf.floor(top + 0.5), tf.int32))
left = max(0, tf.cast(tf.floor(left + 0.5), tf.int32))
bottom = min(image_shape[0], tf.cast(tf.floor(bottom + 0.5), tf.int32))
right = min(image_shape[1], tf.cast(tf.floor(right + 0.5), tf.int32))
if top - image_shape[0] >= 0:
text_origin = tf.convert_to_tensor([left, top - label_size[1]])
else:
text_origin = tf.convert_to_tensor([left, top + 1])
for j in range(thickness):
draw.rectangle(
[left + j, top + j, right - j, bottom - j],
outline=h.colormap[c])
draw.rectangle(
[tuple(text_origin), tuple(text_origin + label_size)],
fill=h.colormap[c])
draw.text(text_origin, label, fill=(0, 0, 0), font=font)
del draw
pil_img.show()
else:
print(NOTE, ' no boxes detected')
def main():
"""Create the model and start the training."""
args = get_arguments()
logs_dir = os.path.join(LOGS_ROOT, args.exper_name, LOGS_DIR_SUFFIX)
snap_dir = os.path.join(LOGS_ROOT, args.exper_name, SNAPSHOT_DIR_SUFFIX)
h, w = map(int, args.input_size.split(','))
input_size = (h, w)
c_h, c_w = map(int, args.crop_size.split(','))
crop_size = (c_h, c_w)
tf.set_random_seed(args.random_seed)
# Coordinator for threads
coord = tf.train.Coordinator()
with tf.name_scope("create_inputs"):
reader = DataLoader(args.data_dir, input_size, crop_size, True, False, False, coord)
image_batch, label_batch = reader.dequeue(args.batch_size)
with tf.name_scope("validating_input"):
validate_reader = DataLoader(args.data_dir, input_size, crop_size, False, True, False, coord)
image_validate_batch, label_validate_batch = validate_reader.dequeue(args.batch_size)
# for training
with tf.variable_scope('', reuse=False):
net = UNetResidual_Ext2({'data': image_batch}, is_training=True, num_classes=args.num_classes)
for layer in net.layers:
print(layer)
print(net.layers[layer].shape)
total_params = 0
for variable in tf.trainable_variables():
shape = variable.get_shape()
variable_parameters = 1
for dim in shape:
variable_parameters *= dim.value
total_params += variable_parameters
print("Number of trainable parameters: %d" % (total_params))
# for validation
with tf.variable_scope(tf.get_variable_scope(), reuse=True):
net_val = UNetResidual_Ext2({'data': image_validate_batch}, is_training=False, num_classes=args.num_classes)
with tf.variable_scope('training_output'):
# output training
logits = net.getOutput()
with tf.variable_scope('validation_output'):
# output validation
logits_validation = net_val.getOutput()
restore_var = [v for v in tf.global_variables()]
with tf.variable_scope('training_loss'):
train_weights = tf.gather(CLASS_WEIGHTS, label_batch)
# loss for training
ce_loss = tf.losses.sparse_softmax_cross_entropy(label_batch, logits, train_weights)
ce_reduced_loss = tf.reduce_mean(ce_loss)
dice_loss = generalised_dice_loss(tf.nn.softmax(logits), label_batch)
train_loss = (ce_reduced_loss + dice_loss)
with tf.variable_scope('validation_loss'):
valid_weights = tf.gather(CLASS_WEIGHTS, label_validate_batch)
# loss for validation
ce_loss_validation = tf.losses.sparse_softmax_cross_entropy(label_validate_batch, logits_validation, valid_weights)
ce_reduced_loss_validation = tf.reduce_mean(ce_loss_validation)
dice_loss_validation = generalised_dice_loss(tf.nn.softmax(logits_validation),label_validate_batch)
valid_loss = (ce_reduced_loss_validation + dice_loss_validation)
with tf.variable_scope('accuracy'):
# accuracy
preds = tf.argmax(logits_validation, axis=-1)
acc = tf.reduce_mean(tf.cast(tf.equal(preds, label_validate_batch), tf.double))
with tf.variable_scope('train'):
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
with tf.control_dependencies(update_ops):
# training optimizer
optimizer = tf.train.AdamOptimizer(learning_rate = args.learning_rate)
train_op = optimizer.minimize(train_loss, global_step=tf.train.get_global_step())
init = tf.global_variables_initializer()
sess = tf.Session()
sess.run(init)
train_logger = Logger(logs_dir + '/train')
valid_logger = Logger(logs_dir + '/valid')
saver = tf.train.Saver(var_list=tf.global_variables(), max_to_keep=1)
# initialize weight
ckpt = tf.train.get_checkpoint_state(snap_dir)
if ckpt and ckpt.model_checkpoint_path:
loader = tf.train.Saver(var_list=restore_var)
load_step = int(os.path.basename(ckpt.model_checkpoint_path).split('-')[1])
load(loader, sess, ckpt.model_checkpoint_path)
else:
print('No checkpoint file found.')
load_step = 0
threads = tf.train.start_queue_runners(coord=coord, sess=sess)
min_val_loss = 1e9
best_acc = 0
for step in range(args.num_steps):
realStep = step + load_step
start_time = time.time()
loss_value, _ = sess.run([train_loss, train_op])
train_logger.log_scalar('loss', loss_value, realStep)
if step > 0 and realStep % args.save_pred_every == 0:
#save(saver, sess, snap_dir, realStep)
print('Validation')
numValidateImg = len(os.listdir(os.path.join(args.data_dir, 'valid', 'img')))
numStep = int(numValidateImg / args.batch_size)
loss_validation_test = 0
accuracy = 0
for step_val in range(numStep):
predicted, gt, l, ac = sess.run([logits_validation, label_validate_batch, valid_loss, acc])
loss_validation_test = loss_validation_test + l / numStep
# accuracy = accuracy + ac / numStep
accuracy = accuracy + cal_AJI_batch(predicted, gt, args.batch_size) / numStep
print('Validation result: loss = %.5f, acc = %.5f' % (loss_validation_test, accuracy))
if accuracy > best_acc:
print('Update model: previous loss: %.4lf, new loss: %.4lf, step: %d' % (min_val_loss, loss_validation_test, realStep))
min_val_loss = loss_validation_test
best_acc = accuracy
save(saver, sess, snap_dir, realStep)
valid_logger.log_scalar('loss', loss_validation_test, realStep)
valid_logger.log_scalar('acc', accuracy, realStep)
duration = time.time() - start_time
print('step %d \t loss = %.5lf, (%.3lf sec/step)' % (realStep, loss_value, duration))
coord.request_stop()
coord.join(threads)
def propagate_one_time(
iter_idx,
h_e, h_v, h_u,
h_e_history, h_v_history, h_u_history,
atom_in_mol=atom_in_mol, # (n_atoms, n_mols)
bond_in_mol=bond_in_mol # (n_bonds, n_mols)
):
# update $ e'_k $
# $$
# e'_k = \phi^e (e_k, v_{rk}, v_{sk}, u)
# $$
h_left = tf.gather(
h_v,
left_idxs)
h_right = tf.gather(
h_v,
right_idxs)
# (n_bonds, d_e)
h_e = self.phi_e(h_e, h_e_0, h_left + h_right,
tf.reduce_sum(
tf.boolean_mask(
tf.tile(
tf.expand_dims(
h_u, # (n_mols, d_u)
0), # (1, n_mols, d_u)
[tf.shape(h_e)[0], 1, 1]),
bond_in_mol),
axis=1,
keepdims=True))
h_e_history = tf.concat(
[
h_e_history,
tf.expand_dims(
h_e,
1)
],
axis=1)
# aggregate $ \bar{e_i'} $
# $$
# \bar{e_i'} = \rho^{e \rightarrow v} (E'_i)
# $$
# (n_atoms, d_e)
h_e_bar_i = self.rho_e_v(h_e, atom_is_connected_to_bonds)
# update $ v'_i $
# $$
# v'_i = phi^v (\bar{e_i}, v_i, u)
# $$
# (n_atoms, d_v)
h_v = self.phi_v(
h_v, # (n_atoms, d_v)
h_v_0, # (n_atoms, d_v)
h_e_bar_i, # (n_atoms, d_v)
tf.reduce_sum(
tf.where(
tf.tile(
tf.expand_dims(
atom_in_mol,
2),
[1, 1, tf.shape(h_u)[1]]),
tf.tile(
tf.expand_dims(
h_u,
0),
[n_atoms, 1, 1]),
tf.zeros_like(
tf.tile(
tf.expand_dims(
h_u,
0),
[n_atoms, 1, 1]))),
axis=1))
h_v_history = tf.concat(
[
h_v_history,
tf.expand_dims(
h_v,
1)
],
axis=1)
# aggregate $ \bar{e'} $
# $$
# \bar{e'} = \rhp^{e \rightarrow u} (E')
# $$
# (n_mols, d_e)
h_e_bar = self.rho_e_u(h_e, bond_in_mol)
# aggregate $ \bar{v'} $
# $$
# \bar{v'} = \rho^{v \rightarrow u} (V')
# $$
# (n_mols, d_v)
h_v_bar = self.rho_v_u(h_v, atom_in_mol)
# update $ u' $
# $$
# u' = \phi^u (\bar{e'}, \bar{v'}, u)
# $$
# (n_mols, d_u)
h_u = self.phi_u(
h_u,
h_u_0,
h_e_bar,
h_v_bar)
h_u_history = tf.concat(
[
h_u_history,
tf.expand_dims(
h_u,
1)
],
axis=1)
return (
iter_idx + 1,
h_e, h_v, h_u,
h_e_history, h_v_history, h_u_history)
def affinity_loss(labels,
probs,
num_classes,
kld_margin):
"""Affinity Field (AFF) loss.
This function computes AFF loss. There are several components in the
function:
1) extracts edges from the ground-truth labels.
2) extracts ignored pixels and their paired pixels (the neighboring
pixels on the eight corners).
3) extracts neighboring pixels on the eight corners from a 3x3 patch.
4) computes KL-Divergence between center pixels and their neighboring
pixels from the eight corners.
Args:
labels: A tensor of size [batch_size, height_in, width_in], indicating
semantic segmentation ground-truth labels.
probs: A tensor of size [batch_size, height_in, width_in, num_classes],
indicating segmentation predictions.
num_classes: A number indicating the total number of valid classes.
kld_margin: A number indicating the margin for KL-Divergence at edge.
Returns:
Two 1-D tensors value indicating the loss at edge and non-edge.
"""
# Compute ignore map (e.g, label of 255 and their paired pixels).
labels = tf.squeeze(labels, axis=-1) # NxHxW
ignore = nnx.ignores_from_label(labels, num_classes, 1) # NxHxWx8
not_ignore = tf.logical_not(ignore)
not_ignore = tf.expand_dims(not_ignore, axis=3) # NxHxWx1x8
# Compute edge map.
one_hot_lab = tf.one_hot(labels, depth=num_classes)
edge = nnx.edges_from_label(one_hot_lab, 1, 255) # NxHxWxCx8
# Remove ignored pixels from the edge/non-edge.
edge = tf.logical_and(edge, not_ignore)
not_edge = tf.logical_and(tf.logical_not(edge), not_ignore)
edge_indices = tf.where(tf.reshape(edge, [-1]))
not_edge_indices = tf.where(tf.reshape(not_edge, [-1]))
# Extract eight corner from the center in a patch as paired pixels.
probs_paired = nnx.eightcorner_activation(probs, 1) # NxHxWxCx8
probs = tf.expand_dims(probs, axis=-1) # NxHxWxCx1
bot_epsilon = tf.constant(1e-4, name='bot_epsilon')
top_epsilon = tf.constant(1.0, name='top_epsilon')
neg_probs = tf.clip_by_value(
1-probs, bot_epsilon, top_epsilon)
probs = tf.clip_by_value(
probs, bot_epsilon, top_epsilon)
neg_probs_paired= tf.clip_by_value(
1-probs_paired, bot_epsilon, top_epsilon)
probs_paired = tf.clip_by_value(
probs_paired, bot_epsilon, top_epsilon)
# Compute KL-Divergence.
kldiv = probs_paired*tf.log(probs_paired/probs)
kldiv += neg_probs_paired*tf.log(neg_probs_paired/neg_probs)
not_edge_loss = kldiv
edge_loss = tf.maximum(0.0, kld_margin-kldiv)
not_edge_loss = tf.reshape(not_edge_loss, [-1])
not_edge_loss = tf.gather(not_edge_loss, not_edge_indices)
edge_loss = tf.reshape(edge_loss, [-1])
edge_loss = tf.gather(edge_loss, edge_indices)
return edge_loss, not_edge_loss
initial_state = lstmCell.zero_state(batchSize, tf.float32)
value, _ = tf.nn.dynamic_rnn(lstmCell,
data,
initial_state=initial_state,
dtype=tf.float32)
weight = tf.Variable(tf.truncated_normal([lstmUnits, numClasses]))
tf.summary.scalar('weights1', weight[1, 1])
tf.summary.scalar('weights2', weight[3, 1])
tf.summary.histogram("weights", weight)
bias = tf.Variable(tf.constant(0.1, shape=[numClasses]))
tf.summary.histogram("biases", bias)
value = tf.transpose(value, [1, 0, 2])
last = tf.gather(value, int(value.get_shape()[0]) - 1)
prediction = (tf.matmul(last, weight) + bias)
correctPred = tf.equal(tf.argmax(prediction, 1), tf.argmax(labels, 1))
accuracy = tf.reduce_mean(tf.cast(correctPred, tf.float32))
loss = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(logits=prediction, labels=labels))
optimizer = tf.train.GradientDescentOptimizer(0.0005).minimize(loss)
import datetime
sess = tf.InteractiveSession()
tf.summary.scalar('Loss', loss)
tf.summary.scalar('Accuracy', accuracy)
merged = tf.summary.merge_all()
# logdir = "tensorboard/" + datetime.datetime.now().strftime("%Y%m%d-%H%M%S") + "/"
def build_model(self):
"""Builds the TensorFlow Computational Graph"""
lr = 1e-3 # Learning rate
lambda_reg = 1e-1
lambda_sl = 1
num_layers = 1
regularizer = tf.contrib.layers.l2_regularizer(scale=lambda_reg)
self.X = tf.placeholder(name='X',
dtype=tf.float32,
shape=[None, self.d_in])
self.Y = tf.placeholder(name='Y', dtype=tf.float32, shape=[None]) #
self.A = tf.placeholder(name='A', dtype=tf.int32, shape=[None])
self.sl_actions = tf.placeholder(name='SL_A',
dtype=tf.int32,
shape=[None])
self.margin_loss = tf.placeholder(name='L',
dtype=tf.float32,
shape=[None, self.d_out])
self.w = tf.placeholder(name='W', dtype=tf.float32, shape=[None])
self.lr = tf.placeholder(name='lr', dtype=tf.float32, shape=[])
batch_size = tf.shape(self.X)[0]
demo_size = tf.shape(self.sl_actions)[0]
hidden = self.X
for _ in range(num_layers):
hidden = tf.layers.dense(hidden,
self.h,
activation=tf.nn.relu,
kernel_regularizer=regularizer)
self.predictions = tf.layers.dense(hidden,
self.d_out,
name='output',
kernel_regularizer=regularizer)
q_rl = self.predictions[:batch_size, :]
q_sl = self.predictions[batch_size:, :]
gather_indices = tf.range(0, batch_size) * self.d_out + self.A
Y_pred = tf.gather(tf.reshape(q_rl, [-1]), gather_indices)
q_pred_sl = tf.reduce_max(q_sl + self.margin_loss, axis=1)
gather_indices = tf.range(0, demo_size) * self.d_out + self.sl_actions
q_opt_sl = tf.gather(tf.reshape(q_sl, [-1]), gather_indices)
self.td_error = tf.abs(self.Y - Y_pred)
self.loss_rl = tf.reduce_mean(tf.multiply(self.td_error**2, self.w))
self.loss_sl = tf.reduce_mean(q_pred_sl - q_opt_sl)
self.loss = self.loss_rl + lambda_sl * self.loss_sl
self.optimizer = tf.train.AdamOptimizer(learning_rate=self.lr)
self.train_op = self.optimizer.minimize(
self.loss, global_step=self.global_step_tensor)
self.summaries = tf.summary.merge([
tf.summary.scalar("loss", self.loss),
tf.summary.histogram("q_values_hist", self.predictions),
tf.summary.scalar("max_q_value", tf.reduce_max(self.predictions))
])
def sample_fast_rcnn_targets(boxes, gt_boxes, gt_labels):
"""
Sample some boxes from all proposals for training.
#fg is guaranteed to be > 0, because ground truth boxes will be added as proposals.
Args:
boxes: nx4 region proposals, floatbox
gt_boxes: mx4, floatbox
gt_labels: m, int32
Returns:
A BoxProposals instance.
sampled_boxes: tx4 floatbox, the rois
sampled_labels: t int64 labels, in [0, #class). Positive means foreground.
fg_inds_wrt_gt: #fg indices, each in range [0, m-1].
It contains the matching GT of each foreground roi.
"""
iou = pairwise_iou(boxes, gt_boxes) # nxm
proposal_metrics(iou)
# add ground truth as proposals as well
boxes = tf.concat([boxes, gt_boxes], axis=0) # (n+m) x 4
iou = tf.concat([iou, tf.eye(tf.shape(gt_boxes)[0])], axis=0) # (n+m) x m
# #proposal=n+m from now on
def sample_fg_bg(iou):
fg_mask = tf.reduce_max(iou, axis=1) >= cfg.FRCNN.FG_THRESH
fg_inds = tf.reshape(tf.where(fg_mask), [-1])
num_fg = tf.minimum(int(cfg.FRCNN.BATCH_PER_IM * cfg.FRCNN.FG_RATIO),
tf.size(fg_inds),
name='num_fg')
fg_inds = tf.random_shuffle(fg_inds)[:num_fg]
bg_inds = tf.reshape(tf.where(tf.logical_not(fg_mask)), [-1])
num_bg = tf.minimum(cfg.FRCNN.BATCH_PER_IM - num_fg,
tf.size(bg_inds),
name='num_bg')
bg_inds = tf.random_shuffle(bg_inds)[:num_bg]
add_moving_summary(num_fg, num_bg)
return fg_inds, bg_inds
fg_inds, bg_inds = sample_fg_bg(iou)
# fg,bg indices w.r.t proposals
best_iou_ind = tf.argmax(iou, axis=1) # #proposal, each in 0~m-1
fg_inds_wrt_gt = tf.gather(best_iou_ind, fg_inds) # num_fg
all_indices = tf.concat([fg_inds, bg_inds],
axis=0) # indices w.r.t all n+m proposal boxes
ret_boxes = tf.gather(boxes, all_indices)
ret_labels = tf.concat([
tf.gather(gt_labels, fg_inds_wrt_gt),
tf.zeros_like(bg_inds, dtype=tf.int64)
],
axis=0)
# stop the gradient -- they are meant to be training targets
return BoxProposals(
tf.stop_gradient(ret_boxes, name='sampled_proposal_boxes'),
tf.stop_gradient(ret_labels, name='sampled_labels'),
tf.stop_gradient(fg_inds_wrt_gt))
def add_image_summaries(images: tf.Tensor,
labels: tf.Tensor,
preds: tf.Tensor,
locs: tf.Tensor,
k: int = 1) -> tf.Tensor:
'''Adds image summaries for the k best and k worst images in each batch.
Each image is overlayed with (lat, lon), label, and prediction.
Args
- images: tf.Tensor, shape [batch_size, H, W, C], type float32
- C must be either 3 (RGB order), or 1 (grayscale)
- already standardized (relative to entire dataset) with mean 0, std 1
- labels: tf.Tensor, shape [batch_size]
- preds: tf.Tensor, shape [batch_size]
- locs: tf.Tensor, shape [batch_size, 2], each row is [lat, lon]
- k: int, number of best and worst images to show per batch
Returns: tf.summary, merged summaries
'''
# For float tensors, tf.summary.image automatically scales min/max to 0/255.
# Set +/- 3 std. dev. to 0/255.
# We want to display images with our own scaling -> cast to tf.uint8
images = tf.clip_by_value((images / 6.0 + 0.5) * 255,
clip_value_min=0,
clip_value_max=255)
images = tf.cast(images, tf.uint8)
def write_on_imgs(imgs: np.ndarray, locs: np.ndarray, labels: np.ndarray,
preds: np.ndarray) -> np.ndarray:
'''Writes white text w/ black background onto images.
Args
- imgs: np.array, shape [num_imgs, H, W, C], type uint8
C must be either 1 or 3
- locs: np.array, shape [num_imgs, 2]
- labels: np.array, shape [num_imgs]
- preds: np.array, shape [num_imgs]
Returns
- new_imgs: np.array, shape [num_imgs, H, W, C]
'''
C = imgs.shape[3]
new_imgs = np.empty_like(imgs)
for i, img in enumerate(imgs):
if C == 1:
img = img[:, :, 0] # remove C dim. new shape: [H, W]
img = PIL.Image.fromarray(img)
# write white text on black background
draw = PIL.ImageDraw.Draw(img)
text = 'loc: ({:.6f}, {:.6f})\nlabel: {:.4f}, pred: {:.4f}'.format(
locs[i][0], locs[i][1], labels[i], preds[i])
size = draw.textsize(text) # (w, h) of text
draw.rectangle(xy=[(0, 0), size], fill='black')
draw.text(xy=(0, 0), text=text, fill='white')
if C == 1:
new_imgs[i, :, :, 0] = np.asarray(img)
else:
new_imgs[i] = np.asarray(img)
return new_imgs
diff = tf.abs(preds - labels)
_, worst_indices = tf.nn.top_k(diff, k=k)
_, best_indices = tf.nn.top_k(-1 * diff, k=k)
worst_inputs = [
tf.gather(x, worst_indices) for x in [images, locs, labels, preds]
]
worst_img_sum = tf.summary.image('worst_images_in_batch',
tf.py_func(func=write_on_imgs,
inp=worst_inputs,
Tout=tf.uint8,
stateful=False,
name='write_on_worst_imgs'),
max_outputs=k)
best_inputs = [
tf.gather(x, best_indices) for x in [images, locs, labels, preds]
]
best_img_sum = tf.summary.image('best_images_in_batch',
tf.py_func(func=write_on_imgs,
inp=best_inputs,
Tout=tf.uint8,
stateful=False,
name='write_on_best_imgs'),
max_outputs=k)
return tf.summary.merge([worst_img_sum, best_img_sum])
def fg_labels(self):
""" Returns: #fg"""
return tf.gather(self.labels, self.fg_inds(), name='fg_labels')
def losses(self):
encoded_fg_gt_boxes = encode_bbox_target(
tf.gather(self.gt_boxes, self.proposals.fg_inds_wrt_gt),
self.proposals.fg_boxes()) * self.bbox_regression_weights
return fastrcnn_losses(self.proposals.labels, self.label_logits,
encoded_fg_gt_boxes, self.fg_box_logits())
def fastrcnn_predictions(boxes, scores):
"""
Generate final results from predictions of all proposals.
Args:
boxes: n#classx4 floatbox in float32
scores: nx#class
Returns:
boxes: Kx4
scores: K
labels: K
"""
assert boxes.shape[1] == cfg.DATA.NUM_CLASS
assert scores.shape[1] == cfg.DATA.NUM_CLASS
boxes = tf.transpose(boxes, [1, 0, 2])[1:, :, :] # #catxnx4
boxes.set_shape([None, cfg.DATA.NUM_CATEGORY, None])
scores = tf.transpose(scores[:, 1:], [1, 0]) # #catxn
def f(X):
"""
prob: n probabilities
box: nx4 boxes
Returns: n boolean, the selection
"""
prob, box = X
output_shape = tf.shape(prob, out_type=tf.int64)
# filter by score threshold
ids = tf.reshape(tf.where(prob > cfg.TEST.RESULT_SCORE_THRESH), [-1])
prob = tf.gather(prob, ids)
box = tf.gather(box, ids)
# NMS within each class
selection = tf.image.non_max_suppression(box, prob,
cfg.TEST.RESULTS_PER_IM,
cfg.TEST.FRCNN_NMS_THRESH)
selection = tf.gather(ids, selection)
# sort available in TF>1.4.0
# sorted_selection = tf.contrib.framework.sort(selection, direction='ASCENDING')
sorted_selection = -tf.nn.top_k(-selection, k=tf.size(selection))[0]
if get_tf_version_tuple() >= (1, 13):
mask = tf.sparse.SparseTensor(indices=tf.expand_dims(
sorted_selection, 1),
values=tf.ones_like(sorted_selection,
dtype=tf.bool),
dense_shape=output_shape)
mask = tf.sparse.to_dense(mask, default_value=False)
else:
# this function is deprecated by TF
mask = tf.sparse_to_dense(sparse_indices=sorted_selection,
output_shape=output_shape,
sparse_values=True,
default_value=False)
return mask
# TF bug in version 1.11, 1.12: https://github.com/tensorflow/tensorflow/issues/22750
buggy_tf = get_tf_version_tuple() in [(1, 11), (1, 12)]
masks = tf.map_fn(f, (scores, boxes),
dtype=tf.bool,
parallel_iterations=1 if buggy_tf else 10) # #cat x N
selected_indices = tf.where(
masks) # #selection x 2, each is (cat_id, box_id)
scores = tf.boolean_mask(scores, masks)
# filter again by sorting scores
topk_scores, topk_indices = tf.nn.top_k(scores,
tf.minimum(cfg.TEST.RESULTS_PER_IM,
tf.size(scores)),
sorted=False)
filtered_selection = tf.gather(selected_indices, topk_indices)
cat_ids, box_ids = tf.unstack(filtered_selection, axis=1)
final_scores = tf.identity(topk_scores, name='scores')
final_labels = tf.add(cat_ids, 1, name='labels')
final_ids = tf.stack([cat_ids, box_ids], axis=1, name='all_ids')
final_boxes = tf.gather_nd(boxes, final_ids, name='boxes')
return final_boxes, final_scores, final_labels
def fg_box_logits(self):
""" Returns: #fg x ? x 4 """
return tf.gather(self.box_logits,
self.proposals.fg_inds(),
name='fg_box_logits')
print(vectors.get_shape()) # (2000, 2)
# D0 차원의 크기가 2000개이고 D1 차원은 크기가 2(각 점의 x, y 좌표)
print(centroids.get_shape()) # (4, 2)
# 센트로이드는 D0 차원의 크기가 4개이고 D1 차원은 vectors 와 동일한 2인 행렬
expanded_vectors = tf.expand_dims(vectors, 0) # 두 텐서에 차원을 추가
expanded_centroids = tf.expand_dims(centroids,
1) # 두 텐서를 2차원에서 3차원으로 만들어 뺄셈 가능하도록
# 크기를 맞추는 작업
# 각 점에 대한 유클리드 제곱거리 알고리즘 무한 반복
assignments = tf.argmin(
tf.reduce_sum(tf.square(tf.subtract(expanded_vectors, expanded_centroids)),
2), 0)
# 새로운 중심 계산하기
# 매 반복마다 새롭게 그룹화를 하면서 각 그룹에 해당하는 새로운 중심을 다시 계산
means = tf.concat([
tf.reduce_mean(tf.gather(
vectors, tf.reshape(tf.where(tf.equal(assignments, c)), [1, -1])),
reduction_indices=[1]) for c in range(k)
], 0)
update_centroids = tf.assign(centroids, means)
init_op = tf.global_variables_initializer()
sess = tf.Session()
sess.run(init_op)
for i in range(100):
_, centroid_values, assignment_values = sess.run(
[update_centroids, centroids, assignments])
# assignment_values 텐서의 결과를 확인하기 위한 결과 그림 그리기
data = {"x": [], "y": [], "cluster": []}
def fg_boxes(self):
""" Returns: #fg x4"""
return tf.gather(self.boxes, self.fg_inds(), name='fg_boxes')
# Get the indices of elements in x whose values are greater than 30.
# Hint: Use tf.where().
# Then extract elements whose values are greater than 30.
# Hint: Use tf.gather().
###############################################################################
# YOUR CODE
x = tf.constant([
29.05088806, 27.61298943, 31.19073486, 29.35532951, 30.97266006,
26.67541885, 38.08450317, 20.74983215, 34.94445419, 34.45999146,
29.06485367, 36.01657104, 27.88236427, 20.56035233, 30.20379066,
29.51215172, 33.71149445, 28.59134293, 36.05556488, 28.66994858
])
indicies = tf.where(tf.less(30.0, x))
over_30 = tf.gather(x, indicies)
with tf.Session() as sess:
print "Problem 1d:"
print "Should only print elements from tensor over 30"
sess.run(over_30)
print over_30.eval()
print "*" * 10
###############################################################################
# 1e: Create a diagnoal 2-d tensor of size 6 x 6 with the diagonal values of 1,
# 2, ..., 6
# Hint: Use tf.range() and tf.diag().
###############################################################################
# YOUR CODE
def main():
net = ResNet
depth = 50
loader = VOCLoader('07', 'test')
net = net(config=net_config, depth=depth, training=False)
num_classes = 21
batch_size = args.batch_size
img_size = args.image_size
image_ph = tf.placeholder(shape=[1, img_size, img_size, 3],
dtype=tf.float32,
name='img_ph')
net.create_trunk(image_ph)
bboxer = PriorBoxGrid(net_config)
net.create_multibox_head(num_classes)
confidence = tf.nn.softmax(tf.squeeze(net.outputs['confidence']))
location = tf.squeeze(net.outputs['location'])
good_bboxes = decode_bboxes(location, bboxer.tiling)
detection_list = []
score_list = []
for i in range(1, num_classes):
class_mask = tf.greater(confidence[:, i], args.conf_thresh)
class_scores = tf.boolean_mask(confidence[:, i], class_mask)
class_bboxes = tf.boolean_mask(good_bboxes, class_mask)
K = tf.minimum(tf.size(class_scores), args.top_k_nms)
_, top_k_inds = tf.nn.top_k(class_scores, K)
top_class_scores = tf.gather(class_scores, top_k_inds)
top_class_bboxes = tf.gather(class_bboxes, top_k_inds)
final_inds = tf.image.non_max_suppression(
top_class_bboxes,
top_class_scores,
max_output_size=50,
iou_threshold=args.nms_thresh)
final_class_bboxes = tf.gather(top_class_bboxes, final_inds)
final_scores = tf.gather(top_class_scores, final_inds)
detection_list.append(final_class_bboxes)
score_list.append(final_scores)
net.create_segmentation_head(num_classes)
segmentation = tf.cast(
tf.argmax(tf.squeeze(net.outputs['segmentation']), axis=-1), tf.int32)
times = []
# im = cv2.imread("/home/lear/kshmelko/testImage.jpg")
# im = cv2.cvtColor(im, cv2.COLOR_BGR2RGB)/255.0
# im = im.astype(np.float32)
with tf.Session(config=tf.ConfigProto(allow_soft_placement=True,
log_device_placement=False)) as sess:
sess.run(tf.global_variables_initializer())
ckpt_path = train_dir + '/model.ckpt-%i000' % args.ckpt
log.debug("Restoring checkpoint %s" % ckpt_path)
saver = tf.train.Saver(tf.global_variables())
saver.restore(sess, ckpt_path)
for i in range(200):
im = loader.load_image(loader.get_filenames()[i])
im = cv2.resize(im, (img_size, img_size))
im = im.reshape((1, img_size, img_size, 3))
st = time.time()
sess.run([detection_list, score_list, segmentation],
feed_dict={image_ph: im})
et = time.time()
if i > 10:
times.append(et - st)
m = np.mean(times)
s = np.std(times)
fps = 1 / m
log.info("Mean={0:.2f}ms; Std={1:.2f}ms; FPS={2:.1f}".format(
m * 1000, s * 1000, fps))
'index': tf.VarLenFeature(dtype=tf.int64),
'value': tf.VarLenFeature(dtype=tf.float32),
})
test_label = test_features['label']
test_index = test_features['index']
test_value = test_features['value']
test_label = tf.to_float(test_label, name="test_label")
test_ind_vals = test_index.values
test_ind_inds = test_index.indices
test_val_vals = test_value.values
test_val_vals_2d = tf.expand_dims(test_val_vals, 1)
test_w_sparse = tf.gather(w, test_index.values, name="test_w_sparse")
test_w_transp = tf.transpose(test_w_sparse)
test_wtranspx = tf.matmul(test_w_transp,
test_val_vals_2d,
name="test_wTransX")
test_pred = tf.sign(test_wtranspx)
test_correctness = tf.equal(test_label,
test_pred,
name="test_correctness")
# Test End
# Train start
with tf.device("/job:worker/task:%d" % FLAGS.task_index):
filename_queue = createFileNameQueues(FLAGS.task_index)
def _build_graph(self):
config = self.config
x_size = config.dim_input_ctrl
h_size = config.dim_hidden_ctrl
a_size = config.dim_output_ctrl
lr = self.lr_plh
with self.graph.as_default():
model_name = config.controller_model_name
initializer = tf.contrib.layers.xavier_initializer(uniform=True)
if model_name == '2layer':
hidden = slim.fully_connected(self.state_plh, h_size,
weights_initializer=initializer,
activation_fn=tf.nn.leaky_relu)
self.logits = slim.fully_connected(hidden, a_size,
weights_initializer=initializer,
activation_fn=None)
self.output = tf.nn.softmax(self.logits)
elif model_name == '2layer_logits_clipping':
hidden = slim.fully_connected(self.state_plh, h_size,
weights_initializer=initializer,
activation_fn=tf.nn.leaky_relu)
self.logits = slim.fully_connected(hidden, a_size,
weights_initializer=initializer,
activation_fn=None)
self.output = tf.nn.softmax(self.logits /
config.logit_clipping_c)
elif model_name == 'linear':
self.logits = slim.fully_connected(self.state_plh, a_size,
weights_initializer=initializer,
activation_fn=None)
self.output = tf.nn.softmax(self.logits)
elif model_name == 'linear_logits_clipping':
#self.logits = slim.fully_connected(self.state_plh, a_size,
# weights_initializer=initializer,
# activation_fn=None)
# ----Old version----
w = tf.get_variable('w', shape=[x_size, a_size], dtype=tf.float32,
initializer=initializer)
b = tf.get_variable('b', shape=[a_size], dtype=tf.float32,
initializer=tf.zeros_initializer())
self.logits = tf.matmul(self.state_plh, w) + b
self.output = tf.nn.softmax(self.logits /
config.logit_clipping_c)
else:
raise Exception('Invalid controller_model_name')
self.chosen_action = tf.argmax(self.output, 1)
self.action = tf.cast(tf.argmax(self.action_plh, 1), tf.int32)
self.indexes = tf.range(0, tf.shape(self.output)[0])\
* tf.shape(self.output)[1] + self.action
self.responsible_outputs = tf.gather(tf.reshape(self.output, [-1]),
self.indexes)
self.loss = -tf.reduce_mean(tf.log(self.responsible_outputs)
* self.reward_plh)
# ----Restore gradients and update them after several iterals.----
optimizer = tf.train.AdamOptimizer(learning_rate=lr)
self.tvars = tf.trainable_variables()
tvars = self.tvars
self.gradient_plhs = []
for idx, var in enumerate(tvars):
placeholder = tf.placeholder(tf.float32, name=str(idx) + '_plh')
self.gradient_plhs.append(placeholder)
gvs = optimizer.compute_gradients(self.loss, tvars)
self.grads = [grad for grad, _ in gvs]
self.train_op = optimizer.apply_gradients(zip(self.gradient_plhs, tvars))
#self.train_op = optimizer.apply_gradients(gvs)
self.init = tf.global_variables_initializer()
self.saver = tf.train.Saver()
def call(self, inputs):
inputs = tf.cast(inputs, tf.int32)
outputs = tf.gather(self.embeddings, inputs)
return outputs
def main(argv):
del argv # unused arg
tf.io.gfile.makedirs(FLAGS.output_dir)
logging.info('Saving checkpoints at %s', FLAGS.output_dir)
tf.random.set_seed(FLAGS.seed)
if FLAGS.use_gpu:
logging.info('Use GPU')
strategy = tf.distribute.MirroredStrategy()
else:
logging.info('Use TPU at %s',
FLAGS.tpu if FLAGS.tpu is not None else 'local')
resolver = tf.distribute.cluster_resolver.TPUClusterResolver(
tpu=FLAGS.tpu)
tf.config.experimental_connect_to_cluster(resolver)
tf.tpu.experimental.initialize_tpu_system(resolver)
strategy = tf.distribute.TPUStrategy(resolver)
aug_params = {
'augmix': FLAGS.augmix,
'aug_count': FLAGS.aug_count,
'augmix_depth': FLAGS.augmix_depth,
'augmix_prob_coeff': FLAGS.augmix_prob_coeff,
'augmix_width': FLAGS.augmix_width,
'ensemble_size': 1,
'mixup_alpha': FLAGS.mixup_alpha,
'adaptive_mixup': FLAGS.adaptive_mixup,
'random_augment': FLAGS.random_augment,
'forget_mixup': FLAGS.forget_mixup,
'num_cores': FLAGS.num_cores,
'threshold': FLAGS.forget_threshold,
}
batch_size = (FLAGS.per_core_batch_size * FLAGS.num_cores //
FLAGS.num_dropout_samples_training)
train_input_fn = data_utils.load_input_fn(
split=tfds.Split.TRAIN,
name=FLAGS.dataset,
batch_size=batch_size,
use_bfloat16=FLAGS.use_bfloat16,
proportion=FLAGS.train_proportion,
validation_set=FLAGS.validation,
aug_params=aug_params)
if FLAGS.validation:
validation_input_fn = data_utils.load_input_fn(
split=tfds.Split.VALIDATION,
name=FLAGS.dataset,
batch_size=FLAGS.per_core_batch_size,
use_bfloat16=FLAGS.use_bfloat16,
validation_set=True)
val_dataset = strategy.experimental_distribute_datasets_from_function(
validation_input_fn)
clean_test_input_fn = data_utils.load_input_fn(
split=tfds.Split.TEST,
name=FLAGS.dataset,
batch_size=FLAGS.per_core_batch_size,
use_bfloat16=FLAGS.use_bfloat16)
train_dataset = strategy.experimental_distribute_dataset(train_input_fn())
test_datasets = {
'clean':
strategy.experimental_distribute_datasets_from_function(
clean_test_input_fn),
}
if FLAGS.corruptions_interval > 0:
if FLAGS.dataset == 'cifar10':
load_c_dataset = utils.load_cifar10_c
else:
load_c_dataset = functools.partial(utils.load_cifar100_c,
path=FLAGS.cifar100_c_path)
corruption_types, max_intensity = utils.load_corrupted_test_info(
FLAGS.dataset)
for corruption in corruption_types:
for intensity in range(1, max_intensity + 1):
dataset = load_c_dataset(corruption_name=corruption,
corruption_intensity=intensity,
batch_size=FLAGS.per_core_batch_size *
FLAGS.num_cores,
use_bfloat16=FLAGS.use_bfloat16)
test_datasets['{0}_{1}'.format(corruption, intensity)] = (
strategy.experimental_distribute_dataset(dataset))
ds_info = tfds.builder(FLAGS.dataset).info
num_train_examples = ds_info.splits['train'].num_examples
# Train_proportion is a float so need to convert steps_per_epoch to int.
if FLAGS.validation:
# TODO(ywenxu): Remove hard-coding validation images.
steps_per_epoch = int(
(num_train_examples * FLAGS.train_proportion - 2500) // batch_size)
steps_per_val = 2500 // (FLAGS.per_core_batch_size * FLAGS.num_cores)
else:
steps_per_epoch = int(
num_train_examples * FLAGS.train_proportion) // batch_size
steps_per_eval = ds_info.splits['test'].num_examples // batch_size
num_classes = ds_info.features['label'].num_classes
if FLAGS.use_bfloat16:
tf.keras.mixed_precision.set_global_policy('mixed_bfloat16')
summary_writer = tf.summary.create_file_writer(
os.path.join(FLAGS.output_dir, 'summaries'))
with strategy.scope():
logging.info('Building ResNet model')
model = ub.models.wide_resnet_dropout(
input_shape=ds_info.features['image'].shape,
depth=28,
width_multiplier=10,
num_classes=num_classes,
l2=FLAGS.l2,
dropout_rate=FLAGS.dropout_rate,
residual_dropout=FLAGS.residual_dropout,
filterwise_dropout=FLAGS.filterwise_dropout)
logging.info('Model input shape: %s', model.input_shape)
logging.info('Model output shape: %s', model.output_shape)
logging.info('Model number of weights: %s', model.count_params())
# Linearly scale learning rate and the decay epochs by vanilla settings.
base_lr = FLAGS.base_learning_rate * batch_size / 128
lr_decay_epochs = [(int(start_epoch_str) * FLAGS.train_epochs) // 200
for start_epoch_str in FLAGS.lr_decay_epochs]
lr_schedule = schedules.WarmUpPiecewiseConstantSchedule(
steps_per_epoch,
base_lr,
decay_ratio=FLAGS.lr_decay_ratio,
decay_epochs=lr_decay_epochs,
warmup_epochs=FLAGS.lr_warmup_epochs)
optimizer = tf.keras.optimizers.SGD(lr_schedule,
momentum=0.9,
nesterov=True)
metrics = {
'train/negative_log_likelihood': tf.keras.metrics.Mean(),
'train/accuracy': tf.keras.metrics.SparseCategoricalAccuracy(),
'train/loss': tf.keras.metrics.Mean(),
'train/ece': um.ExpectedCalibrationError(num_bins=FLAGS.num_bins),
'test/negative_log_likelihood': tf.keras.metrics.Mean(),
'test/accuracy': tf.keras.metrics.SparseCategoricalAccuracy(),
'test/ece': um.ExpectedCalibrationError(num_bins=FLAGS.num_bins),
}
if FLAGS.corruptions_interval > 0:
corrupt_metrics = {}
for intensity in range(1, max_intensity + 1):
for corruption in corruption_types:
dataset_name = '{0}_{1}'.format(corruption, intensity)
corrupt_metrics['test/nll_{}'.format(dataset_name)] = (
tf.keras.metrics.Mean())
corrupt_metrics['test/accuracy_{}'.format(
dataset_name)] = (
tf.keras.metrics.SparseCategoricalAccuracy())
corrupt_metrics['test/ece_{}'.format(dataset_name)] = (
um.ExpectedCalibrationError(num_bins=FLAGS.num_bins))
checkpoint = tf.train.Checkpoint(model=model, optimizer=optimizer)
latest_checkpoint = tf.train.latest_checkpoint(FLAGS.output_dir)
initial_epoch = 0
if latest_checkpoint:
# checkpoint.restore must be within a strategy.scope() so that optimizer
# slot variables are mirrored.
checkpoint.restore(latest_checkpoint)
logging.info('Loaded checkpoint %s', latest_checkpoint)
initial_epoch = optimizer.iterations.numpy() // steps_per_epoch
@tf.function
def train_step(iterator):
"""Training StepFn."""
def step_fn(inputs):
"""Per-Replica StepFn."""
if FLAGS.forget_mixup:
images, labels, idx = inputs
else:
images, labels = inputs
if FLAGS.augmix and FLAGS.aug_count >= 1:
# Index 0 at augmix preprocessing is the unperturbed image.
images = images[:, 1, ...]
# This is for the case of combining AugMix and Mixup.
if FLAGS.mixup_alpha > 0:
labels = tf.split(labels, FLAGS.aug_count + 1, axis=0)[1]
images = tf.tile(images,
[FLAGS.num_dropout_samples_training, 1, 1, 1])
if FLAGS.mixup_alpha > 0:
labels = tf.tile(labels,
[FLAGS.num_dropout_samples_training, 1])
else:
labels = tf.tile(labels, [FLAGS.num_dropout_samples_training])
with tf.GradientTape() as tape:
logits = model(images, training=True)
if FLAGS.use_bfloat16:
logits = tf.cast(logits, tf.float32)
if FLAGS.mixup_alpha > 0:
negative_log_likelihood = tf.reduce_mean(
tf.keras.losses.categorical_crossentropy(
labels, logits, from_logits=True))
else:
negative_log_likelihood = tf.reduce_mean(
tf.keras.losses.sparse_categorical_crossentropy(
labels, logits, from_logits=True))
l2_loss = sum(model.losses)
loss = negative_log_likelihood + l2_loss
# Scale the loss given the TPUStrategy will reduce sum all gradients.
scaled_loss = loss / strategy.num_replicas_in_sync
grads = tape.gradient(scaled_loss, model.trainable_variables)
optimizer.apply_gradients(zip(grads, model.trainable_variables))
probs = tf.nn.softmax(logits)
if FLAGS.mixup_alpha > 0:
labels = tf.argmax(labels, axis=-1)
metrics['train/ece'].update_state(labels, probs)
metrics['train/loss'].update_state(loss)
metrics['train/negative_log_likelihood'].update_state(
negative_log_likelihood)
metrics['train/accuracy'].update_state(labels, logits)
if FLAGS.forget_mixup:
train_predictions = tf.argmax(probs, -1)
labels = tf.cast(labels, train_predictions.dtype)
# For each ensemble member (1 here), we accumulate the accuracy counts.
accuracy_counts = tf.cast(
tf.reshape((train_predictions == labels), [1, -1]),
tf.float32)
return accuracy_counts, idx
if FLAGS.forget_mixup:
return strategy.run(step_fn, args=(next(iterator), ))
else:
strategy.run(step_fn, args=(next(iterator), ))
@tf.function
def test_step(iterator, dataset_name):
"""Evaluation StepFn."""
def step_fn(inputs):
"""Per-Replica StepFn."""
images, labels = inputs
logits_list = []
for _ in range(FLAGS.num_dropout_samples):
logits = model(images, training=False)
if FLAGS.use_bfloat16:
logits = tf.cast(logits, tf.float32)
logits_list.append(logits)
# Logits dimension is (num_samples, batch_size, num_classes).
logits_list = tf.stack(logits_list, axis=0)
probs_list = tf.nn.softmax(logits_list)
probs = tf.reduce_mean(probs_list, axis=0)
labels_broadcasted = tf.broadcast_to(
labels, [FLAGS.num_dropout_samples, labels.shape[0]])
log_likelihoods = -tf.keras.losses.sparse_categorical_crossentropy(
labels_broadcasted, logits_list, from_logits=True)
negative_log_likelihood = tf.reduce_mean(
-tf.reduce_logsumexp(log_likelihoods, axis=[0]) +
tf.math.log(float(FLAGS.num_dropout_samples)))
if dataset_name == 'clean':
metrics['test/negative_log_likelihood'].update_state(
negative_log_likelihood)
metrics['test/accuracy'].update_state(labels, probs)
metrics['test/ece'].update_state(labels, probs)
elif dataset_name != 'validation':
corrupt_metrics['test/nll_{}'.format(
dataset_name)].update_state(negative_log_likelihood)
corrupt_metrics['test/accuracy_{}'.format(
dataset_name)].update_state(labels, probs)
corrupt_metrics['test/ece_{}'.format(
dataset_name)].update_state(labels, probs)
if dataset_name == 'validation':
return tf.reshape(probs, [1, -1, num_classes]), labels
if dataset_name == 'validation':
return strategy.run(step_fn, args=(next(iterator), ))
else:
strategy.run(step_fn, args=(next(iterator), ))
metrics.update({'test/ms_per_example': tf.keras.metrics.Mean()})
train_iterator = iter(train_dataset)
forget_counts_history = []
start_time = time.time()
for epoch in range(initial_epoch, FLAGS.train_epochs):
logging.info('Starting to run epoch: %s', epoch)
acc_counts_list = []
idx_list = []
for step in range(steps_per_epoch):
if FLAGS.forget_mixup:
temp_accuracy_counts, temp_idx = train_step(train_iterator)
acc_counts_list.append(temp_accuracy_counts)
idx_list.append(temp_idx)
else:
train_step(train_iterator)
current_step = epoch * steps_per_epoch + (step + 1)
max_steps = steps_per_epoch * FLAGS.train_epochs
time_elapsed = time.time() - start_time
steps_per_sec = float(current_step) / time_elapsed
eta_seconds = (max_steps - current_step) / steps_per_sec
message = ('{:.1%} completion: epoch {:d}/{:d}. {:.1f} steps/s. '
'ETA: {:.0f} min. Time elapsed: {:.0f} min'.format(
current_step / max_steps, epoch + 1,
FLAGS.train_epochs, steps_per_sec, eta_seconds / 60,
time_elapsed / 60))
if step % 20 == 0:
logging.info(message)
# Only one of the forget_mixup and adaptive_mixup can be true.
if FLAGS.forget_mixup:
current_acc = [
tf.concat(list(acc_counts_list[i].values), axis=1)
for i in range(len(acc_counts_list))
]
total_idx = [
tf.concat(list(idx_list[i].values), axis=0)
for i in range(len(idx_list))
]
current_acc = tf.cast(tf.concat(current_acc, axis=1), tf.int32)
total_idx = tf.concat(total_idx, axis=0)
current_forget_path = os.path.join(FLAGS.output_dir,
'forget_counts.npy')
last_acc_path = os.path.join(FLAGS.output_dir, 'last_acc.npy')
if epoch == 0:
forget_counts = tf.zeros([1, num_train_examples],
dtype=tf.int32)
last_acc = tf.zeros([1, num_train_examples], dtype=tf.int32)
else:
if 'last_acc' not in locals():
with tf.io.gfile.GFile(last_acc_path, 'rb') as f:
last_acc = np.load(f)
last_acc = tf.cast(tf.convert_to_tensor(last_acc),
tf.int32)
if 'forget_counts' not in locals():
with tf.io.gfile.GFile(current_forget_path, 'rb') as f:
forget_counts = np.load(f)
forget_counts = tf.cast(
tf.convert_to_tensor(forget_counts), tf.int32)
selected_last_acc = tf.gather(last_acc, total_idx, axis=1)
forget_this_epoch = tf.cast(current_acc < selected_last_acc,
tf.int32)
forget_this_epoch = tf.transpose(forget_this_epoch)
target_shape = tf.constant([num_train_examples, 1])
current_forget_counts = tf.scatter_nd(
tf.reshape(total_idx, [-1, 1]), forget_this_epoch,
target_shape)
current_forget_counts = tf.transpose(current_forget_counts)
acc_this_epoch = tf.transpose(current_acc)
last_acc = tf.scatter_nd(tf.reshape(total_idx, [-1, 1]),
acc_this_epoch, target_shape)
# This is lower bound of true acc.
last_acc = tf.transpose(last_acc)
# TODO(ywenxu): We count the dropped examples as forget. Fix this later.
forget_counts += current_forget_counts
forget_counts_history.append(forget_counts)
logging.info('forgetting counts')
logging.info(tf.stack(forget_counts_history, 0))
with tf.io.gfile.GFile(
os.path.join(FLAGS.output_dir,
'forget_counts_history.npy'), 'wb') as f:
np.save(f, tf.stack(forget_counts_history, 0).numpy())
with tf.io.gfile.GFile(current_forget_path, 'wb') as f:
np.save(f, forget_counts.numpy())
with tf.io.gfile.GFile(last_acc_path, 'wb') as f:
np.save(f, last_acc.numpy())
aug_params['forget_counts_dir'] = current_forget_path
train_input_fn = data_utils.load_input_fn(
split=tfds.Split.TRAIN,
name=FLAGS.dataset,
batch_size=batch_size,
use_bfloat16=FLAGS.use_bfloat16,
validation_set=FLAGS.validation,
aug_params=aug_params)
train_dataset = strategy.experimental_distribute_dataset(
train_input_fn())
train_iterator = iter(train_dataset)
elif FLAGS.adaptive_mixup:
val_iterator = iter(val_dataset)
logging.info('Testing on validation dataset')
predictions_list = []
labels_list = []
for step in range(steps_per_val):
temp_predictions, temp_labels = test_step(
val_iterator, 'validation')
predictions_list.append(temp_predictions)
labels_list.append(temp_labels)
predictions = [
tf.concat(list(predictions_list[i].values), axis=1)
for i in range(len(predictions_list))
]
labels = [
tf.concat(list(labels_list[i].values), axis=0)
for i in range(len(labels_list))
]
predictions = tf.concat(predictions, axis=1)
labels = tf.cast(tf.concat(labels, axis=0), tf.int64)
def compute_acc_conf(preds, label, focus_class):
class_preds = tf.boolean_mask(preds,
label == focus_class,
axis=1)
class_pred_labels = tf.argmax(class_preds, axis=-1)
confidence = tf.reduce_mean(
tf.reduce_max(class_preds, axis=-1), -1)
accuracy = tf.reduce_mean(tf.cast(
class_pred_labels == focus_class, tf.float32),
axis=-1)
return accuracy - confidence
calibration_per_class = [
compute_acc_conf(predictions, labels, i)
for i in range(num_classes)
]
calibration_per_class = tf.stack(calibration_per_class, axis=1)
logging.info('calibration per class')
logging.info(calibration_per_class)
mixup_coeff = tf.where(calibration_per_class > 0, 1.0,
FLAGS.mixup_alpha)
mixup_coeff = tf.clip_by_value(mixup_coeff, 0, 1)
logging.info('mixup coeff')
logging.info(mixup_coeff)
aug_params['mixup_coeff'] = mixup_coeff
train_input_fn = data_utils.load_input_fn(
split=tfds.Split.TRAIN,
name=FLAGS.dataset,
batch_size=batch_size,
use_bfloat16=FLAGS.use_bfloat16,
validation_set=True,
aug_params=aug_params)
train_dataset = strategy.experimental_distribute_dataset(
train_input_fn())
train_iterator = iter(train_dataset)
datasets_to_evaluate = {'clean': test_datasets['clean']}
if (FLAGS.corruptions_interval > 0
and (epoch + 1) % FLAGS.corruptions_interval == 0):
datasets_to_evaluate = test_datasets
for dataset_name, test_dataset in datasets_to_evaluate.items():
test_iterator = iter(test_dataset)
logging.info('Testing on dataset %s', dataset_name)
for step in range(steps_per_eval):
if step % 20 == 0:
logging.info('Starting to run eval step %s of epoch: %s',
step, epoch)
test_start_time = time.time()
test_step(test_iterator, dataset_name)
ms_per_example = (time.time() -
test_start_time) * 1e6 / batch_size
metrics['test/ms_per_example'].update_state(ms_per_example)
logging.info('Done with testing on %s', dataset_name)
corrupt_results = {}
if (FLAGS.corruptions_interval > 0
and (epoch + 1) % FLAGS.corruptions_interval == 0):
corrupt_results = utils.aggregate_corrupt_metrics(
corrupt_metrics, corruption_types, max_intensity)
logging.info('Train Loss: %.4f, Accuracy: %.2f%%',
metrics['train/loss'].result(),
metrics['train/accuracy'].result() * 100)
logging.info('Test NLL: %.4f, Accuracy: %.2f%%',
metrics['test/negative_log_likelihood'].result(),
metrics['test/accuracy'].result() * 100)
total_results = {
name: metric.result()
for name, metric in metrics.items()
}
total_results.update(corrupt_results)
with summary_writer.as_default():
for name, result in total_results.items():
tf.summary.scalar(name, result, step=epoch + 1)
for metric in metrics.values():
metric.reset_states()
if (FLAGS.checkpoint_interval > 0
and (epoch + 1) % FLAGS.checkpoint_interval == 0):
checkpoint_name = checkpoint.save(
os.path.join(FLAGS.output_dir, 'checkpoint'))
logging.info('Saved checkpoint to %s', checkpoint_name)
final_checkpoint_name = checkpoint.save(
os.path.join(FLAGS.output_dir, 'checkpoint'))
logging.info('Saved last checkpoint to %s', final_checkpoint_name)
