def build_graph(self, image, label): # 本函数返回总的损失
image = ImageNetModel.image_preprocess(image, bgr=True)
if self.data_format == 'NCHW':
image = tf.transpose(image, [0, 3, 1, 2])
logits = self.get_logits(image)
# ctx = get_current_tower_context()
# is_training = ctx.is_training
# if is_training:
# label = tf.concat([label, label, label[0:17]], axis = 0)
loss = ImageNetModel.compute_loss_and_error(logits, label)
if self.weight_decay > 0: # 如果权值衰减大于0
if self.weight_decay_on_bn: # 如果在bn上也进行权值衰减的话
pattern = '.*/W|.*/gamma|.*/beta' # 匹配变量名的正则表达式
else:
pattern = '.*/W'
wd_w = tf.train.exponential_decay(self.weight_decay,
get_global_step_var(), 150000,
1.2, True) # 指数衰减来解决学习率的问题
wd_loss = regularize_cost(
pattern,
tf.contrib.layers.l2_regularizer(wd_w),
name='l2_regularize_loss') #返回总的正则化后的loss 标量
add_moving_summary(loss, wd_loss)
total_cost = tf.add_n([loss, wd_loss], name='cost')
else:
total_cost = tf.identity(loss, name='cost')
add_moving_summary(total_cost)
return total_cost
def build_graph(self, image, label):
image = self.image_preprocess(image)
assert self.data_format in ['NCHW', 'NHWC']
if self.data_format == 'NCHW':
image = tf.transpose(image, [0, 3, 1, 2])
logits = self.get_logits(image)
loss = ImageNetModel.compute_loss_and_error(
logits, label, label_smoothing=self.label_smoothing)
if self.weight_decay > 0:
wd_loss = regularize_cost(self.weight_decay_pattern,
tf.contrib.layers.l2_regularizer(
self.weight_decay),
name='l2_regularize_loss')
add_moving_summary(loss, wd_loss)
total_cost = tf.add_n([loss, wd_loss], name='cost')
else:
total_cost = tf.identity(loss, name='cost')
add_moving_summary(total_cost)
if self.loss_scale != 1.:
logger.info("Scaling the total loss by {} ...".format(
self.loss_scale))
return total_cost * self.loss_scale
else:
return total_cost
def build_graph(self, *inputs):
inputs = dict(zip(self.input_names, inputs))
if "gt_masks_packed" in inputs:
gt_masks = tf.cast(unpackbits_masks(inputs.pop("gt_masks_packed")),
tf.uint8,
name="gt_masks")
inputs["gt_masks"] = gt_masks
image = self.preprocess(inputs['image']) # 1CHW
features = self.backbone(image)
anchor_inputs = {
k: v
for k, v in inputs.items() if k.startswith('anchor_')
}
proposals, rpn_losses = self.rpn(image, features,
anchor_inputs) # inputs?
targets = [
inputs[k] for k in ['gt_boxes', 'gt_labels', 'gt_masks']
if k in inputs
]
head_losses = self.roi_heads(image, features, proposals, targets)
if self.training:
wd_cost = regularize_cost('.*/W',
l2_regularizer(cfg.TRAIN.WEIGHT_DECAY),
name='wd_cost')
total_cost = tf.add_n(rpn_losses + head_losses + [wd_cost],
'total_cost')
add_moving_summary(total_cost, wd_cost)
return total_cost
def build_graph(self, image, label, indices):
"""
The default tower function.
"""
image = self.image_preprocess(image)
assert self.data_format == 'NCHW'
image = tf.transpose(image, [0, 3, 1, 2])
with tf.variable_scope(tf.get_variable_scope(), reuse=tf.AUTO_REUSE):
# BatchNorm always comes with trouble. We use the testing mode of it during attack.
with freeze_collection([tf.GraphKeys.UPDATE_OPS]), argscope(BatchNorm, training=False):
image, target_label = self.attacker.attack(image, label, self.get_logits)
image = tf.stop_gradient(image, name='adv_training_sample')
logits = self.get_logits(image)
loss = ImageNetModel.compute_loss_and_error(
logits, label, label_smoothing=self.label_smoothing)
AdvImageNetModel.compute_attack_success(logits, target_label)
if not self.training:
return
wd_loss = regularize_cost(self.weight_decay_pattern,
tf.contrib.layers.l2_regularizer(self.weight_decay),
name='l2_regularize_loss')
add_moving_summary(loss, wd_loss)
total_cost = tf.add_n([loss, wd_loss], name='cost')
if self.loss_scale != 1.:
logger.info("Scaling the total loss by {} ...".format(self.loss_scale))
return total_cost * self.loss_scale
else:
return total_cost
def _build_graph(self, inputs):
image, label = inputs
image = image_preprocess(image, bgr=True)
# transform the data_format if necessary
if self.data_format == 'NCHW':
image = tf.transpose(image, [0, 3, 1, 2])
# UNDEFINED
logits = tf.identity(self.get_logits(image), name='logits')
softmax_logits = tf.nn.softmax(logits, name='softmax-logits')
# definition of loss
# update @ 20180705: for AESTHETIC_LEVEL experiments, introduce output_dims
loss = compute_loss_and_error(logits,
label,
JensenFactor=self.JensenFactor,
output_dims=self.output_dims)
wd_loss = regularize_cost('.*/W',
tf.contrib.layers.l2_regularizer(
self.weight_decay),
name='l2_regularize_loss')
add_moving_summary(loss, wd_loss)
self.cost = tf.add_n([loss, wd_loss], name='cost')
def build_graph(self, *inputs):
inputs = dict(zip(self.input_names, inputs))
if "gt_masks_packed" in inputs:
gt_masks = tf.cast(unpackbits_masks(inputs.pop("gt_masks_packed")), tf.uint8, name="gt_masks")
inputs["gt_masks"] = gt_masks
image = self.preprocess(inputs['image']) # 1CHW
features = self.backbone(image)
anchor_inputs = {k: v for k, v in inputs.items() if k.startswith('anchor_')}
proposals, rpn_losses = self.rpn(image, features, anchor_inputs) # inputs?
targets = [inputs[k] for k in ['gt_boxes', 'gt_labels', 'gt_masks'] if k in inputs]
gt_boxes_area = tf.reduce_mean(tf_area(inputs["gt_boxes"]), name='mean_gt_box_area')
add_moving_summary(gt_boxes_area)
head_losses = self.roi_heads(image, features, proposals, targets)
if self.training:
wd_cost = regularize_cost(
'.*/W', l2_regularizer(cfg.TRAIN.WEIGHT_DECAY), name='wd_cost')
total_cost = tf.add_n(
rpn_losses + head_losses + [wd_cost], 'total_cost')
add_moving_summary(total_cost, wd_cost)
return total_cost
else:
# Check that the model defines the tensors it declares for inference
# For existing models, they are defined in "fastrcnn_predictions(name_scope='output')"
G = tf.get_default_graph()
ns = G.get_name_scope()
for name in self.get_inference_tensor_names()[1]:
try:
name = '/'.join([ns, name]) if ns else name
G.get_tensor_by_name(name + ':0')
except KeyError:
raise KeyError("Your model does not define the tensor '{}' in inference context.".format(name))
def build_graph(self, image, label):
image = ImageNetModel.image_preprocess(image, bgr=True)
if self.data_format == 'NCHW':
image = tf.transpose(image, [0, 3, 1, 2])
logits = self.get_logits(image)
# ctx = get_current_tower_context()
# is_training = ctx.is_training
# if is_training:
# label = tf.concat([label, label, label[0:17]], axis = 0)
loss = ImageNetModel.compute_loss_and_error(logits, label)
if self.weight_decay > 0:
if self.weight_decay_on_bn:
pattern = '.*/W|.*/gamma|.*/beta'
else:
pattern = '.*/W'
wd_w = self.weight_decay
wd_loss = regularize_cost(pattern, tf.contrib.layers.l2_regularizer(wd_w),
name='l2_regularize_loss')
add_moving_summary(loss, wd_loss)
total_cost = tf.add_n([loss, wd_loss], name='cost')
else:
total_cost = tf.identity(loss, name='cost')
add_moving_summary(total_cost)
return total_cost
def build_graph(self, image, label):
image = self.image_preprocess(image)
if is_channels_first(self.data_format):
image = tf.transpose(image, [0, 3, 1, 2], name="image_transpose")
# tf.summary.image('input_image_', image)
# tf.summary.tensor_summary('input_tensor_', image)
# with tf.name_scope('tmp1_summaries'):
# add_tensor_summary(image, ['histogram', 'rms', 'sparsity'], name='tmp1_tensor')
is_training = get_current_tower_context().is_training
logits = self.model_lambda(x=image, training=is_training)
loss = ImageNetModel.compute_loss_and_error(
logits=logits, label=label, label_smoothing=self.label_smoothing)
if self.weight_decay > 0:
wd_loss = regularize_cost(regex=self.weight_decay_pattern,
func=tf.contrib.layers.l2_regularizer(
self.weight_decay),
name='l2_regularize_loss')
add_moving_summary(loss, wd_loss)
total_cost = tf.add_n([loss, wd_loss], name='cost')
else:
total_cost = tf.identity(loss, name='cost')
add_moving_summary(total_cost)
if self.loss_scale != 1.0:
logger.info("Scaling the total loss by {} ...".format(
self.loss_scale))
return total_cost * self.loss_scale
else:
return total_cost
def build_graph(self, query, key):
# setup queue
queue_init = tf.math.l2_normalize(tf.random.normal(
[self.queue_size, self.feature_dim]),
axis=1)
queue = tf.get_variable('queue',
initializer=queue_init,
trainable=False)
queue_ptr = tf.get_variable('queue_ptr', [],
initializer=tf.zeros_initializer(),
dtype=tf.int64,
trainable=False)
tf.add_to_collection(tf.GraphKeys.MODEL_VARIABLES, queue)
tf.add_to_collection(tf.GraphKeys.MODEL_VARIABLES, queue_ptr)
# query encoder
q_feat = self.net.forward(query) # NxC
q_feat = tf.math.l2_normalize(q_feat, axis=1)
# key encoder
shuffled_key, shuffle_idxs = batch_shuffle(key)
shuffled_key.set_shape([self.batch_size, None, None, None])
with tf.variable_scope("momentum_encoder"), \
varreplace.freeze_variables(skip_collection=True), \
argscope(BatchNorm, ema_update='skip'): # don't maintain EMA (will not be used at all)
key_feat = xla.compile(lambda: self.net.forward(shuffled_key))[0]
# key_feat = self.net.forward(shuffled_key)
key_feat = tf.math.l2_normalize(key_feat, axis=1) # NxC
key_feat = batch_unshuffle(key_feat, shuffle_idxs)
key_feat = tf.stop_gradient(key_feat)
# loss
l_pos = tf.reshape(tf.einsum('nc,nc->n', q_feat, key_feat),
(-1, 1)) # nx1
l_neg = tf.einsum('nc,kc->nk', q_feat, queue) # nxK
logits = tf.concat([l_pos, l_neg], axis=1) # nx(1+k)
logits = logits * (1 / self.temp)
labels = tf.zeros(self.batch_size, dtype=tf.int64) # n
loss = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits,
labels=labels)
loss = tf.reduce_mean(loss, name='xentropy-loss')
acc = tf.reduce_mean(tf.cast(
tf.equal(tf.math.argmax(logits, axis=1), labels), tf.float32),
name='train-acc')
# update queue (depend on l_neg)
with tf.control_dependencies([l_neg]):
queue_push_op = self.push_queue(queue, queue_ptr, key_feat)
tf.add_to_collection(tf.GraphKeys.UPDATE_OPS, queue_push_op)
wd_loss = regularize_cost(".*",
l2_regularizer(1e-4),
name='l2_regularize_loss')
add_moving_summary(acc, loss, wd_loss)
total_cost = tf.add_n([loss, wd_loss], name='cost')
return total_cost
def build_graph(self, image, label):
is_training = get_current_tower_context().is_main_training_tower
image_origin = ImageNetModel.image_preprocess(
image, bgr=self.image_bgr) # [N, H, W, C]
loss, logit = 0, {}
scales = sorted(self.scales, reverse=True)
# sorted_scales = sorted(list(set(scales + self.scales)), reverse=True)
for scale in scales:
image = tf.image.resize_images(
image_origin, [scale, scale],
method=tf.image.ResizeMethod.NEAREST_NEIGHBOR)
if self.data_format == 'NCHW':
image = tf.transpose(image, [0, 3, 1, 2])
with tf.variable_scope('imagenet', reuse=tf.AUTO_REUSE):
logit[scale] = self.get_logits(image, scale)
loss_scale = self.compute_loss_and_error(logit[scale], label,
scale, is_training)
loss += loss_scale
if self.distill:
logit_ensemble = 0
alpha = tf.get_variable('alpha', [len(scales)],
initializer=tf.constant_initializer(1))
alpha_soft = tf.nn.softmax(alpha) # TODO: remove softmax
for i, scale in enumerate(scales):
logit_ensemble += alpha_soft[i] * tf.stop_gradient(
logit[scale])
tf.summary.scalar('alpha%03d' % scale, alpha_soft[i])
loss_ensemble = self.compute_loss_and_error(
logit_ensemble, label, 'ensemble', is_training)
loss += loss_ensemble
loss_distill = 0
soft_label = tf.stop_gradient(tf.nn.softmax(logit_ensemble))
for scale in scales:
loss_distill += self.compute_distill_loss(
logit[scale], soft_label)
if DISTILL_TYPE == 'top-down':
for i in range(len(scales) - 1):
soft_label = tf.stop_gradient(
tf.nn.softmax(logit[scales[i]]))
for j in range(i + 1, len(scales)):
loss_distill += self.compute_distill_loss(
logit[scales[j]], soft_label)
distill_num = len(scales) * (len(scales) + 1) / 2
loss += SOFTMAX_TEM**2 * loss_distill / distill_num * len(
scales)
else:
loss += SOFTMAX_TEM**2 * loss_distill
wd_loss = regularize_cost(self.weight_decay_pattern,
tf.contrib.layers.l2_regularizer(
self.weight_decay),
name='l2_regularize_loss')
add_moving_summary(loss, wd_loss)
self.cost = tf.add_n([loss, wd_loss], name='cost')
return self.cost
def build_graph(self, *inputs):
print(f"self.input_names : {self.input_names}")
print(F"inputs before : {inputs}")
inputs = dict(zip(self.input_names, inputs))
print(f"inputs : {inputs}")
if "gt_masks_packed" in inputs:
gt_masks = tf.cast(unpackbits_masks(inputs.pop("gt_masks_packed")), tf.uint8, name="gt_masks")
inputs["gt_masks"] = gt_masks
print("inputs[gt_masks] :", inputs["gt_masks"])
# inputs['image'] = tf.Print(inputs['image'], [tf.shape(inputs['image'])], message="image before preprocess : ", summarize=100)
image = self.preprocess(inputs['image']) # 1CHW
# image = tf.Print(image, [tf.shape(image)], message="image after preprocess : ", summarize=100)
features = self.backbone(image)
for i, feature in enumerate(features):
feature = tf.Print(feature, [tf.shape(feature)], message=f"feature p{i+2} : ", summarize=100)
# 여기까지 봄
anchor_inputs = {k: v for k, v in inputs.items() if k.startswith('anchor_')}
# anchor_inputs = tf.Print(anchor_inputs, [anchor_inputs], message="anchor_inputs : ", summarize=100)
proposals, rpn_losses = self.rpn(image, features, anchor_inputs) # inputs?
# proposals = tf.Print(proposals, [tf.shape(proposals)], message="proposals : ", summarize=100)
# rpn_losses = tf.Print(rpn_losses, [tf.shape(rpn_losses)], message="rpn_losses : ", summarize=100)
targets = [inputs[k] for k in ['gt_boxes', 'gt_labels', 'gt_masks'] if k in inputs]
gt_boxes_area = tf.reduce_mean(tf_area(inputs["gt_boxes"]), name='mean_gt_box_area')
add_moving_summary(gt_boxes_area)
head_losses = self.roi_heads(image, features, proposals, targets)
if self.training:
print("l2_regularizer")
print(l2_regularizer)
wd_cost = regularize_cost(
'.*/W', l2_regularizer(cfg.TRAIN.WEIGHT_DECAY), name='wd_cost')
total_cost = tf.add_n(
rpn_losses + head_losses + [wd_cost], 'total_cost')
add_moving_summary(total_cost, wd_cost)
return total_cost
else:
# Check that the model defines the tensors it declares for inference
# For existing models, they are defined in "fastrcnn_predictions(name_scope='output')"
G = tf.get_default_graph()
ns = G.get_name_scope()
for name in self.get_inference_tensor_names()[1]:
try:
name = '/'.join([ns, name]) if ns else name
G.get_tensor_by_name(name + ':0')
except KeyError:
raise KeyError("Your model does not define the tensor '{}' in inference context.".format(name))
def _build_graph(self, inputs):
image, label = inputs
image = image_preprocess(image, bgr=True)
if self.data_format == 'NCHW':
image = tf.transpose(image, [0, 3, 1, 2])
logits = self.get_logits(image)
loss = compute_loss_and_error(logits, label)
wd_loss = regularize_cost('.*/W', tf.contrib.layers.l2_regularizer(self.weight_decay),
name='l2_regularize_loss')
add_moving_summary(loss, wd_loss)
self.cost = tf.add_n([loss, wd_loss], name='cost')
def _build_graph(self, inputs):
image, label = inputs
image = ImageNetModel.image_preprocess(image, bgr=self.image_bgr)
if self.data_format == 'NCHW':
image = tf.transpose(image, [0, 3, 1, 2])
logits = self.get_logits(image)
loss = ImageNetModel.compute_loss_and_error(logits, label)
wd_loss = regularize_cost(self.weight_decay_pattern,
tf.contrib.layers.l2_regularizer(self.weight_decay),
name='l2_regularize_loss')
add_moving_summary(loss, wd_loss)
self.cost = tf.add_n([loss, wd_loss], name='cost')
def build_graph(self, *inputs):
inputs = dict(zip(self.input_names, inputs))
if "gt_masks_packed" in inputs:
gt_masks = tf.cast(unpackbits_masks(inputs.pop("gt_masks_packed")),
tf.uint8,
name="gt_masks")
inputs["gt_masks"] = gt_masks
image = self.preprocess(inputs["image"]) # 1CHW
features = self.backbone(image)
anchor_inputs = {
k: v
for k, v in inputs.items() if k.startswith("anchor_")
}
proposals, rpn_losses = self.rpn(image, features,
anchor_inputs) # inputs?
targets = [
inputs[k] for k in ["gt_boxes", "gt_labels", "gt_masks"]
if k in inputs
]
head_losses = self.roi_heads(image, features, proposals, targets)
if self.training:
wd_cost = regularize_cost(".*/W",
l2_regularizer(cfg.TRAIN.WEIGHT_DECAY),
name="wd_cost")
total_cost = tf.add_n(rpn_losses + head_losses + [wd_cost],
"total_cost")
image_print = tf.print(
"Image: ",
image,
" features: ",
features,
" anchor_inputs: ",
anchor_inputs,
" proposals: ",
proposals,
"targets: ",
targets,
"head_losses: ",
head_losses,
" total_cost: ",
total_cost,
)
# with tf.control_dependencies([image_print]):
add_moving_summary(total_cost, wd_cost)
return total_cost
def build_graph(self, image, label):
image = ImageNetModel.image_preprocess(image, bgr=True)
if self.data_format == 'NCHW':
image = tf.transpose(image, [0, 3, 1, 2])
logits = self.get_logits(image)
loss = ImageNetModel.compute_loss_and_error(logits, label)
if self.weight_decay > 0:
wd_loss = regularize_cost('.*/W', tf.contrib.layers.l2_regularizer(self.weight_decay),
name='l2_regularize_loss')
add_moving_summary(loss, wd_loss)
ret = tf.add_n([loss, wd_loss], name='cost')
else:
ret = tf.identity(loss, name='cost')
add_moving_summary(ret)
return ret
def compute_loss_and_error(self, logits, label, label_smoothing=0.):
if label_smoothing != 0.:
nclass = logits.shape[-1]
label = tf.one_hot(label,
nclass) if label.shape.ndims == 1 else label
if label.shape.ndims == 1:
loss = tf.nn.sparse_softmax_cross_entropy_with_logits(
logits=logits, labels=label)
else:
loss = tf.losses.softmax_cross_entropy(
label,
logits,
label_smoothing=label_smoothing,
reduction=tf.losses.Reduction.NONE)
loss = tf.reduce_mean(loss, name='xentropy-loss')
if self.weight_decay > 0:
wd_loss = regularize_cost(self.weight_decay_pattern,
l2_regularizer(self.weight_decay),
name='l2_regularize_loss')
add_moving_summary(loss, wd_loss)
total_cost = tf.add_n([loss, wd_loss], name='cost')
else:
total_cost = tf.identity(loss, name='cost')
add_moving_summary(total_cost)
if self.loss_scale != 1.:
logger.info("Scaling the total loss by {} ...".format(
self.loss_scale))
def prediction_incorrect(logits,
label,
topk=1,
name='incorrect_vector'):
with tf.name_scope('prediction_incorrect'):
x = tf.logical_not(tf.nn.in_top_k(logits, label, topk))
return tf.cast(x, tf.float32, name=name)
wrong = prediction_incorrect(logits, label, 1, name='wrong-top1')
add_moving_summary(tf.reduce_mean(wrong, name='train-error-top1'))
wrong = prediction_incorrect(logits, label, 5, name='wrong-top5')
add_moving_summary(tf.reduce_mean(wrong, name='train-error-top5'))
return total_cost
def _build_graph(self, inputs, attack=False, inputs_preprocessed=False):
name_scope = tf.get_default_graph().get_name_scope()
# IMG::[[[[24 47 34]]]...]
image, label = inputs
if name_scope == 'tower0' or args.eval:
self.get_logits(
tf.cast(tf.transpose(image, [0, 3, 1, 2]), tf.float32))
if self.attack_inline and attack == False and args.eps > 0: #and 'Inference' in name_scope
with tf.variable_scope('', reuse=True):
image = tf.cast(image, dtype=tf.float32) / tf.constant(
255.0, dtype=tf.float32)
image = self._adv(image, label) * 255.0
# img in [0,255]
image = self.image_preprocess(image, bgr=True, attack=attack)
assert image.shape[1] == image.shape[2]
if self.data_format == 'NCHW':
image = tf.transpose(image, [0, 3, 1, 2])
with tf.variable_scope('', reuse=(not attack)):
logits = self.get_logits(image)
loss = ImageNetModel.compute_loss_and_error(logits,
label,
attack=attack)
if not attack:
wd = tf.contrib.layers.l2_regularizer(self.weight_decay)
wd_loss = regularize_cost('.*/W', wd, name='l2_regularize_loss')
add_moving_summary(loss, wd_loss)
self.cost = tf.add_n([loss, wd_loss], name='cost')
num_labels = len(LABEL_RANGES)
zeroed = tf.sign(tf.maximum((tf.range(1000) - num_labels + 1), 0))
neg_logits = tf.cast(zeroed, dtype=tf.float32) * tf.constant(
-1000.0, dtype=tf.float32)
return logits + neg_logits, loss
def build_graph(self, image, label):
image = iNaturalistModel.image_preprocess(image, bgr=True)
if self.data_format == 'NCHW':
image = tf.transpose(image, [0, 3, 1, 2])
logits = self.get_logits(image)
loss = iNaturalistModel.compute_loss_and_error(logits, label)
if self.weight_decay > 0:
if self.weight_decay_on_bn:
pattern = '.*/W|.*/gamma|.*/beta'
else:
pattern = '.*/W'
wd_loss = regularize_cost(pattern, tf.contrib.layers.l2_regularizer(self.weight_decay),
name='l2_regularize_loss')
add_moving_summary(loss, wd_loss)
total_cost = tf.add_n([loss, wd_loss], name='cost')
else:
total_cost = tf.identity(loss, name='cost')
add_moving_summary(total_cost)
return total_cost
def build_graph(self, image, label):
image = ImageNetModel.image_preprocess(image, bgr=True)
image = tf.transpose(image, [0, 3, 1, 2])
logits = self.get_logits(image)
loss = ImageNetModel.compute_loss_and_error(logits, label)
if self.weight_decay > 0:
"""
Sec 5.1: We use a weight decay λ of 0.0001 and following [16] we do not apply
weight decay on the learnable BN coefficients
"""
wd_loss = regularize_cost('.*/W',
tf.contrib.layers.l2_regularizer(
self.weight_decay),
name='l2_regularize_loss')
add_moving_summary(loss, wd_loss)
ret = tf.add_n([loss, wd_loss], name='cost')
else:
ret = tf.identity(loss, name='cost')
add_moving_summary(ret)
return ret
def build_graph(self, image, label):
image = ImageNetModel.image_preprocess(image, bgr=self.image_bgr)
assert self.data_format in ['NCHW', 'NHWC']
if self.data_format == 'NCHW':
image = tf.transpose(image, [0, 3, 1, 2])
logits = self.get_logits(image)
loss = ImageNetModel.compute_loss_and_error(logits, label)
if self.weight_decay > 0:
wd_loss = regularize_cost(self.weight_decay_pattern,
tf.contrib.layers.l2_regularizer(self.weight_decay),
name='l2_regularize_loss')
add_moving_summary(loss, wd_loss)
total_cost = tf.add_n([loss, wd_loss], name='cost')
else:
total_cost = tf.identity(loss, name='cost')
add_moving_summary(total_cost)
if self.loss_scale != 1.:
logger.info("Scaling the total loss by {} ...".format(self.loss_scale))
return total_cost * self.loss_scale
else:
return total_cost
def build_graph(self, *inputs):
logger.info('-' * 100)
logger.info('This is the official STAC model (MODE_FPN = {})'.format(
cfg.MODE_FPN))
logger.info('-' * 100)
inputs = dict(zip(self.input_names, inputs))
image_l = self.preprocess(inputs['image']) # 1CHW
if self.training:
# Semi-supervsiedly train the model
image_u_strong = self.preprocess(inputs['image_strong']) # 1CHW
pseudo_targets = [
inputs[k] for k in ['gt_boxes_strong', 'gt_labels_strong']
if k in inputs
]
if cfg.TRAIN.NO_PRN_LOSS:
# [labeled and unlabeled]
proposals_boxes = [
None, BoxProposals(inputs['proposals_boxes_strong'])
]
else:
proposals_boxes = [None, None]
self.visualize_images(image_u_strong,
pseudo_targets[0],
name='unlabeled_strong')
self.visualize_images(image_l, inputs['gt_boxes'], name='labeled')
# get groundtruth of labeled data
targets = [
inputs[k] for k in ['gt_boxes', 'gt_labels'] if k in inputs
]
gt_boxes_area = tf.reduce_mean(tf_area(inputs['gt_boxes']),
name='mean_gt_box_area')
add_moving_summary(gt_boxes_area)
image_list = [image_l, image_u_strong]
target_list = [targets, pseudo_targets]
inputs_strong = {
k.replace('_strong', ''): v
for k, v in inputs.items() if 'strong' in k
}
inputs = {
k: v
for k, v in inputs.items()
if ('strong' not in k and 'weak' not in k)
} # for labeled data
input_list = [inputs, inputs_strong]
# The image are forwarded one by one. labeled image is the one
# we need to define which forward is the final branch in order to create specified name of outputs
head_losses = []
rpn_losses = []
for i, (im, tar, inp, pbus) in enumerate(
zip(image_list, target_list, input_list, proposals_boxes)):
hl_loss, rl_loss = self.forward(im,
tar,
inp,
pseudo_proposals=pbus)
head_losses.extend(hl_loss)
rpn_losses.extend(rl_loss)
k = len(head_losses) // len(image_list)
# normalize the loss by number of forward
head_losses = [a / float(len(image_list)) for a in head_losses]
rpn_losses = [a / float(len(image_list)) for a in rpn_losses]
# monitor supervised lossfrom pseudo labels/boxes only
head_losses_u = head_losses[k:]
rpn_losses_u = rpn_losses[k:]
head_cost_u = tf.add_n(head_losses_u, name='fxm/head_cost_u')
rpn_cost_u = tf.add_n(rpn_losses_u, name='fxm/rpn_cost_u')
add_moving_summary_no_nan(head_cost_u, name='fxm/head_cost_u')
add_moving_summary_no_nan(rpn_cost_u, name='fxm/rpn_cost_u')
# multiply wu to unsupervised loss
head_losses = head_losses[:k] + [
a * cfg.TRAIN.WU for a in head_losses_u
]
rpn_losses = rpn_losses[:k] + [
a * cfg.TRAIN.WU for a in rpn_losses_u
]
else:
targets = [
inputs[k] for k in ['gt_boxes', 'gt_labels'] if k in inputs
]
self.forward(image_l, targets, inputs)
if self.training:
regex = '.*/W'
wd_cost = regularize_cost(regex,
l2_regularizer(cfg.TRAIN.WEIGHT_DECAY),
name='wd_cost')
assert 'empty' not in wd_cost.name
total_cost = tf.add_n(rpn_losses + head_losses + [wd_cost],
'total_cost')
add_moving_summary_no_nan(total_cost, 'total_loss')
add_moving_summary_no_nan(wd_cost, 'wd_loss')
return total_cost
else:
# Check that the model defines the tensors it declares for inference
# For existing models, they are defined in "fastrcnn_predictions(name_scope='output')"
G = tf.get_default_graph()
ns = G.get_name_scope()
for name in self.get_inference_tensor_names()[1]:
try:
name = '/'.join([ns, name]) if ns else name
G.get_tensor_by_name(name + ':0')
except KeyError:
raise KeyError(
"Your model does not define the tensor '{}' in inference context."
.format(name))
def build_graph(self, image1, label1, image2, _):
image1 = self.image_preprocess(image1)
image2 = self.image_preprocess(image2)
is_training = get_current_tower_context().is_training
# Shuffle unlabeled data within batch
if is_training:
image2 = tf.random_shuffle(image2)
assert self.data_format in ['NCHW', 'NHWC']
if self.data_format == 'NCHW':
image1 = tf.transpose(image1, [0, 3, 1, 2])
image2 = tf.transpose(image2, [0, 3, 1, 2])
# Pseudo Label
logits2, _ = self.get_logits(image2)
label2 = tf.nn.softmax(logits2)
# Change this line if you modified training schedule or batchsize: 60 Epoch_num, 256 Batch_size
k = tf.cast(get_global_step_var(), tf.float32) / (60 * 1280000 / 256)
# Sample lambda
dist_beta = tf.distributions.Beta(1.0, 1.0)
lmb = dist_beta.sample(tf.shape(image1)[0])
lmb_x = tf.reshape(lmb, [-1, 1, 1, 1])
lmb_y = tf.reshape(lmb, [-1, 1])
# Interpolation
label_ori = label1
if is_training:
image = tf.to_float(image1) * lmb_x + tf.to_float(image2) * (1. - lmb_x)
label = tf.stop_gradient(tf.to_float(tf.one_hot(label1, 1000)) * lmb_y + tf.to_float(label2) * (1. - lmb_y))
else:
image = image1
label = tf.to_float(tf.one_hot(label1, 1000))
# Calculate feats and logits for interpolated samples
with tf.variable_scope(tf.get_variable_scope(), reuse=True):
logits, features = self.get_logits(image)
# Classification Loss and error
loss = ImageNetModel.compute_loss_and_error(
logits, label, label_smoothing=self.label_smoothing, lmb=lmb, label_ori=label_ori)
# Distribution Alignment
lp = 2. / (1. + tf.exp(-10. * k)) - 1
net_ = flip_gradient(features, lp)
fc1 = FullyConnected('linear_1', net_, 1024, nl=tf.nn.relu)
fc2 = FullyConnected('linear_2', fc1, 1024, nl=tf.nn.relu)
domain_logits = FullyConnected("logits_dm", fc2, 2)
label_dm = tf.concat([tf.reshape(lmb, [-1, 1]), tf.reshape(1. - lmb, [-1, 1])], axis=1)
da_cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=label_dm, logits=domain_logits))
# Final Loss
loss += da_cost
if self.weight_decay > 0:
wd_loss = regularize_cost(self.weight_decay_pattern,
tf.contrib.layers.l2_regularizer(self.weight_decay),
name='l2_regularize_loss')
add_moving_summary(loss, wd_loss)
total_cost = tf.add_n([loss, wd_loss], name='cost')
else:
total_cost = tf.identity(loss, name='cost')
add_moving_summary(total_cost)
if self.loss_scale != 1.:
logger.info("Scaling the total loss by {} ...".format(self.loss_scale))
return total_cost * self.loss_scale
else:
return total_cost
def build_graph(self, seq, tseq):
batch_size = self.bs_per_gpu
dynamic_seq_len = tf.shape(seq)[1]
labels = tf.reshape(tseq, [-1])
DROPOUT = 0.5
with argscope(
[
Conv2D, Deconv2D, GroupedConv2D, AvgPooling,
MaxPooling, BatchNorm, GlobalAvgPooling,
ResizeImages, SeparableConv2D
],
data_format=self.data_format
), \
argscope(
[Conv2D, Deconv2D, GroupedConv2D, SeparableConv2D],
activation=tf.identity,
use_bias=self.options.use_bias
), \
argscope(
[BatchNorm],
center=False,
scale=False,
decay=self.options.batch_norm_decay,
epsilon=self.options.batch_norm_epsilon
), \
argscope(
[candidate_gated_layer],
eps=self.options.candidate_gate_eps
):
is_training = get_current_tower_context().is_training
initializer = tf.random_uniform_initializer(
-self.init_range, self.init_range)
# B x seqlen x hidden
seq, embedding_w = self._embed_input_if_int(
seq, initializer=initializer)
seq = self.locked_dropout(seq, self.keep_prob_i)
hid_to_fs_params = _init_feature_select(
self.layer_info_list, 'master', self.options.feat_sel_lambda)
l_hallu_costs = []
self.basic_cells = basic_cells = [
self._basic_cell(initializer=initializer,
hid_to_fs_params=hid_to_fs_params,
l_hallu_costs=l_hallu_costs)
for _ in range(self.num_lstms)
]
cells = rnn.MultiRNNCell(basic_cells)
self.state = tuple([
basic_cells[k].get_state_var(
self.state_var_names[k], batch_size) \
for k in range(self.num_lstms)
])
self.last_state = tuple([
basic_cells[k].get_state_var(
self.last_state_var_names[k] + '_last', batch_size) \
for k in range(self.num_lstms)
])
self._update_init_state_op = self.update_init_state()
with tf.control_dependencies([self._update_init_state_op]):
with tf.variable_scope('RNN', initializer=initializer):
outputs, last_state = tf.nn.dynamic_rnn(
cells,
seq,
initial_state=self.state,
parallel_iterations=self.max_len)
# for the update op
self._update_last_state_op = self.update_last_state(
tf.stop_gradient(last_state))
with tf.control_dependencies([self._update_last_state_op]):
seqout, sum_hallu_costs = basic_cells[-1].split_outputs(
outputs)
seqout = self.locked_dropout(seqout, self.keep_prob)
flat_seqout = tf.reshape(seqout, [-1, self.num_units])
# compute logits and prediction log loss
if self.lock_embedding:
logits = self.linear_with_embedding_w(flat_seqout, embedding_w)
else:
logits = FullyConnected('linear',
flat_seqout,
self.vocab_size,
activation=tf.identity)
logloss = tf.nn.sparse_softmax_cross_entropy_with_logits(
logits=logits, labels=labels)
per_seq_logloss = tf.reduce_sum(tf.reshape(logloss,
[self.bs_per_gpu, -1]),
axis=1,
name="per_seq_sum_logloss")
cost = tf.truediv(tf.reduce_sum(logloss),
tf.cast(self.bs_per_gpu, tf.float32),
name='avg_batch_cost')
float_seq_len = tf.cast(dynamic_seq_len,
tf.float32,
name='seq_len')
# # tensorpack bullshits. Inferencer must use tensors
# # so we have to create a tensor ....
# test_time_udpate = self.update_state(
# [per_seq_logloss], name='test_time_update')
# with tf.control_dependencies([test_time_udpate]):
# self._inference_update_tensor = tf.multiply(
# cost, 1.0001, name=self._inference_update_tensor_name)
perpl = tf.identity(tf.exp(cost / float_seq_len),
name='perplexity')
add_moving_summary(perpl, cost, float_seq_len)
# regularization
if self.rnn_l2_reg:
cost += (self.rnn_l2_reg * tf.reduce_sum(seqout**2) /
tf.to_float(self.bs_per_gpu))
if self.rnn_slowness_reg:
assert self.t_dim == 1
all_h_diff = tf.reduce_sum(
(seqout[:, 1:, :] - seqout[:, :-1, :])**2)
cost += (self.rnn_slowness_reg * all_h_diff /
tf.to_float(self.bs_per_gpu))
wd_w = self.options.regularize_const
if self.params_to_regularize is not None and wd_w:
wd_cost = wd_w * regularize_cost(self.params_to_regularize,
tf.nn.l2_loss)
wd_cost = tf.identity(wd_cost, name='wd_cost')
add_moving_summary(wd_cost)
cost += wd_cost
cost = tf.identity(cost, name='rnn_reg_cost')
add_moving_summary(cost)
# hallucination costs
if l_hallu_costs:
sum_hallu_costs = tf.identity(sum_hallu_costs,
name='hallu_cost')
add_moving_summary(sum_hallu_costs)
cost += sum_hallu_costs
# this computes some gradient norms
self._build_hallu_stats_graph(cost)
# scale the loss according the to sequence length
self.cost = tf.identity(cost * float_seq_len /
np.float32(self.max_len),
name='cost')
add_moving_summary(self.cost)
return self.cost
def build_graph(self, *inputs):
# Dynamic weighting for multiple predictions
if self.options.ls_method == ADALOSS_LS_METHOD:
dynamic_weights = tf.get_variable(
DYNAMIC_WEIGHTS_NAME, (self.n_aux_preds,),
tf.float32, trainable=False,
initializer=tf.constant_initializer([1.0]*self.n_aux_preds))
for i in range(self.n_aux_preds):
weight_i = tf.identity(
dynamic_weights[i], 'weight_{:02d}'.format(i))
add_moving_summary(weight_i)
with argscope(
[
Conv2D, Deconv2D, GroupedConv2D, AvgPooling,
MaxPooling, BatchNorm, GlobalAvgPooling,
ResizeImages, SeparableConv2D
],
data_format=self.data_format
), \
argscope(
[Conv2D, Deconv2D, GroupedConv2D, SeparableConv2D],
activation=tf.identity,
use_bias=self.options.use_bias
), \
argscope(
[BatchNorm],
momentum=float(self.options.batch_norm_decay),
epsilon=float(self.options.batch_norm_epsilon)
), \
argscope(
[candidate_gated_layer],
eps=self.options.candidate_gate_eps
):
# regularization initialization
if self.options.regularize_coef == 'const':
wd_w = self.options.regularize_const
elif self.options.regularize_coef == 'decay':
wd_w = tf.train.exponential_decay(0.0002, get_global_step_var(),
480000, 0.2, True)
# Network-level objects / information
n_inputs = self.master.num_inputs()
drop_path_func = DropPath(
drop_path_keep_prob=self.options.drop_path_keep_prob,
max_train_steps=self.options.max_train_steps,
total_depth=self.n_layers - n_inputs)
l_hallu_costs = []
# cell dictionary
self.op_to_cell = dict()
for cname in self.net_info.cell_names:
hid_to_fs_params = _init_feature_select(
self.net_info[cname], cname, self.options.feat_sel_lambda)
if cname == 'master':
# since master has additional duties like down_sampling,
# aux prediction, accumulating hallu stats, etc,
# master is not built from cell
master_hid_to_fs_params = hid_to_fs_params
hallu_record = _init_hallu_record(self.compute_hallu_stats)
continue
self.op_to_cell[cname] = PetridishBaseCell(
self.net_info[cname],
self.data_format,
self.compute_hallu_stats,
drop_path_func=drop_path_func,
hid_to_fs_params=hid_to_fs_params,
l_hallu_costs=l_hallu_costs)
l_layers = [None] * self.n_layers
layer_dict = dict()
out_filters = self.out_filters
# on-GPU(device) preprocessing for mean/var, casting, embedding, init conv
layer, label = self._preprocess_data(inputs)
for layer_idx in range(n_inputs):
info = self.master[layer_idx]
layer_dict[info.id] = layer #if layer_idx + 1 == n_inputs else None
for layer_idx in range(n_inputs, self.n_layers):
info = self.master[layer_idx]
layer_id_str = "layer{:03d}".format(info.id)
strides = 1
if info.down_sampling:
out_filters *= 2
strides = 2
# preprocess all inputs to match the most recent layer
# in h/w and out_filters in ch_dim
#if not self.is_cell_based:
with tf.variable_scope('pre_'+layer_id_str):
orig_dict = dict()
for input_id in info.inputs:
in_l = layer_dict[input_id]
orig_dict[input_id] = in_l
layer_dict[input_id] = _reduce_prev_layer(
in_l, input_id, layer, out_filters,
self.data_format, hw_only=False)
layer = construct_layer(
layer_id_str, layer_dict, info, out_filters, strides,
self.data_format, info.stop_gradient,
op_to_cell=self.op_to_cell,
drop_path_func=drop_path_func,
non_input_layer_idx=layer_idx - n_inputs,
hid_to_fs_params=master_hid_to_fs_params,
l_hallu_costs=l_hallu_costs
)
# store info for future compute
layer_dict[info.id] = layer
l_layers[layer_idx] = layer
hallu_record = _update_hallu_record(
self.compute_hallu_stats,
hallu_record, layer_idx, self.master, layer_dict)
#if not self.is_cell_based and self.options.use_local_reduction:
if self.options.use_local_reduction:
# reset the reduction layers in dict. So each layer
# uses its own reduction
for input_id in orig_dict:
layer_dict[input_id] = orig_dict[input_id]
# end for layer wise feature construction.
# build aux predictions
total_cost = 0.0
wd_cost = 0.0
anytime_idx = -1
for layer_idx, layer in enumerate(l_layers):
# aux prediction
info = self.master[layer_idx]
cost_weight = info.aux_weight
if cost_weight > 0:
anytime_idx += 1
scope_name = scope_prediction(info.id)
cost, variables = feature_to_prediction_and_loss(
scope_name, layer, label,
self.num_classes, self.prediction_feature,
ch_dim=self.ch_dim,
label_smoothing=self.options.label_smoothing,
dense_dropout_keep_prob=self.options.dense_dropout_keep_prob,
is_last=(layer_idx + 1 == len(l_layers)))
# record the cost for the use of online learners.
cost_i = tf.identity(cost, name='anytime_cost_{:02d}'.format(anytime_idx))
# decide whether to use static or dynmic weights
if self.options.ls_method == ADALOSS_LS_METHOD:
cost_weight = dynamic_weights[anytime_idx]
total_cost += cost_weight * cost_i
# regularize variable in linear predictors
# (have to do this separately here because
# we need unregularized losses for cost_weights)
for var in variables:
wd_cost += cost_weight * wd_w * tf.nn.l2_loss(var)
# end if aux_weight > 0
# end for each layer
# regularization, cost
if self.params_to_regularize is not None:
wd_cost += wd_w * regularize_cost(self.params_to_regularize, tf.nn.l2_loss)
wd_cost = tf.identity(wd_cost, name='wd_cost')
total_cost = tf.identity(total_cost, name='sum_losses')
add_moving_summary(total_cost, wd_cost)
if l_hallu_costs:
hallu_total_cost = tf.add_n(l_hallu_costs, name='hallu_total_cost')
add_moving_summary(hallu_total_cost)
self.cost = tf.add_n([total_cost, wd_cost, hallu_total_cost], name='cost')
else:
self.cost = tf.add_n([total_cost, wd_cost], name='cost')
# hallu stats
for cname in self.net_info.cell_names:
if cname == 'master':
_hallu_stats_graph(
self.compute_hallu_stats, hallu_record, self.cost, scope=cname)
continue
cell = self.op_to_cell.get(cname, None)
cell_hallu_record = getattr(cell, 'hallu_record', None)
_hallu_stats_graph_merged(
self.compute_hallu_stats, cell_hallu_record,
self.cost, scope=cname, n_calls=cell.n_calls,
layer_info_list=cell.layer_info_list)
return self.cost