def add_frozen_bn_ops(model, blob_in, blob_out, dim, inplace=False):
"""
Affine transformation to replace BN in networks where BN cannot be used.
(eg. when minibatch size is too small)
The operation can be done inplace to save memory.
"""
prefix = blob_out
weight = model.create_param(param_name=prefix + '_w',
initializer=initializers.Initializer(
'ConstantFill', value=1.0),
tags=ParameterTags.WEIGHT,
shape=[
dim,
])
bias = model.create_param(param_name=prefix + '_b',
initializer=initializers.Initializer(
'ConstantFill', value=0.0),
tags=ParameterTags.BIAS,
shape=[
dim,
])
if inplace:
return model.net.AffineChannel([blob_in, weight, bias], blob_in)
else:
return model.net.AffineChannel([blob_in, weight, bias], blob_out)
def AffineChannel(self, blob_in, blob_out, dim, inplace=False):
"""Affine transformation to replace BN in networks where BN cannot be
used (e.g., because the minibatch size is too small).
The operations can be done in place to save memory.
"""
blob_out = blob_out or self.net.NextName()
param_prefix = blob_out
scale = self.create_param(
param_name=param_prefix + '_s',
initializer=initializers.Initializer("ConstantFill", value=1.),
tags=ParameterTags.WEIGHT,
shape=[
dim,
],
)
bias = self.create_param(
param_name=param_prefix + '_b',
initializer=initializers.Initializer("ConstantFill", value=0.),
tags=ParameterTags.BIAS,
shape=[
dim,
],
)
if inplace:
return self.net.AffineChannel([blob_in, scale, bias], blob_in)
else:
return self.net.AffineChannel([blob_in, scale, bias], blob_out)
def add_center_loss(label_blob, pred_blob, feature_blob, feature_dims, model):
CF = model.create_param(
param_name='center_feature',
initializer=initializers.Initializer("GaussianFill"),
tags=ParameterTags.COMPUTED_PARAM,
shape=[
model.num_classes - 1, cfg.WSL.CENTER_LOSS_NUMBER, feature_dims
],
)
dCF = model.create_param(
param_name='center_feature_g',
initializer=initializers.Initializer("ConstantFill", value=0.0),
# tags=ParameterTags.COMPUTED_PARAM,
shape=[
model.num_classes - 1, cfg.WSL.CENTER_LOSS_NUMBER, feature_dims
],
)
ndCF = model.create_param(
param_name='center_feature_n_u',
initializer=initializers.Initializer("ConstantFill", value=0.0),
# tags=ParameterTags.COMPUTED_PARAM,
shape=[model.num_classes - 1, cfg.WSL.CENTER_LOSS_NUMBER],
)
if cfg.WSL.CPG or cfg.WSL.CSC:
input_blobs = [
label_blob, pred_blob, feature_blob, CF, dCF, ndCF, 'cpg'
]
else:
input_blobs = [label_blob, pred_blob, feature_blob, CF, dCF, ndCF]
output_blobs = ['loss_center', 'D', 'S']
loss_center, D, S = model.net.CenterLoss(
input_blobs,
output_blobs,
max_iter=cfg.WSL.CSC_MAX_ITER,
top_k=cfg.WSL.CENTER_LOSS_TOP_K,
display=int(1280 / cfg.NUM_GPUS),
update=int(128 / cfg.NUM_GPUS))
loss_gradients = get_loss_gradients_weighted(model, [loss_center], 0.4096)
model.AddLosses(['loss_center'])
return loss_gradients
def add_fast_rcnn_outputs(model, blob_in, dim):
"""Add RoI classification and bounding box regression output ops."""
model.FC(blob_in,
'cls_score',
dim,
model.num_classes,
weight_init=gauss_fill(0.01),
bias_init=const_fill(0.0))
if not model.train: # == if test
# Only add softmax when testing; during training the softmax is combined
# with the label cross entropy loss for numerical stability
model.Softmax('cls_score', 'cls_prob', engine='CUDNN')
model.FC(blob_in,
'bbox_pred',
dim,
model.num_classes * 4,
weight_init=gauss_fill(0.001),
bias_init=const_fill(0.0))
if cfg.MODEL.ATTR and model.train:
in_dim = dim
if cfg.MODEL.CLS_EMBED:
# first slice the fc7 feature
model.net.SelectFG([blob_in, 'fg_idx'], 'fc7_fg')
model.create_param(param_name='class_embedding',
initializer=initializers.Initializer(
"GaussianFill", std=0.01),
shape=[model.num_classes, 256])
# op that just takes the class index and returns the corresponding row
model.net.Embed(['class_embedding', 'labels_int32_fg'], 'embed_fg')
# then do concatenation
model.net.Concat(['fc7_fg', 'embed_fg'],
['concat_attr', 'concat_split'],
axis=1)
in_dim += 256
else:
model.net.SelectFG([blob_in, 'fg_idx'], 'concat_attr')
model.FC('concat_attr',
'fc_attr',
in_dim,
512,
weight_init=gauss_fill(0.01),
bias_init=const_fill(0.0))
model.Relu('fc_attr', 'fc_attr')
model.FC('fc_attr',
'attr_score',
512,
model.num_attributes,
weight_init=gauss_fill(0.01),
bias_init=const_fill(0.0))
def DeformConvAffine(self,
blob_in,
prefix,
dim_in,
dim_out,
kernel,
stride,
pad,
group=1,
dilation=1,
weight_init=None,
bias_init=None,
suffix='_bn',
inplace=False):
"""ConAffine add Deformable Conv
"""
offset_blob = self.Conv(blob_in,
prefix + '_offset',
dim_in,
72,
kernel,
stride=stride,
pad=pad,
group=1,
dilation=dilation,
weight_init=("ConstantFill", {
'value': 0.0
}),
bias_init=("ConstantFill", {
'value': 0.0
}),
no_bias=0)
w = self.create_param(
param_name=prefix + '_b',
initializer=initializers.Initializer("XavierFill"),
tags=ParameterTags.WEIGHT,
shape=[dim_in, int(dim_out / 1), kernel, kernel],
)
deform_blob = self.net.DeformConv([blob_in, offset_blob, w], [prefix],
stride=stride,
pad=pad,
kernel=kernel,
order='NCHW',
deformable_group=4,
group=1)
blob_out = self.AffineChannel(deform_blob,
prefix + suffix,
dim=dim_out,
inplace=inplace)
return blob_out
def layer_norm(
model,
blob_in,
blob_out,
dim_in,
axis=1,
epsilon=1e-4,
initial_scale=1.0,
initial_bias=0.0,
):
'''
Layer normalizes the input, cf. https://arxiv.org/pdf/1607.06450.pdf.
Args:
blob_in: The input blob to layer normalize.
blob_out: The layer normalized output blob.
dim_in: The dimension of the scale and bias. For example, if blob_in is
a 2D design matrix and axis is 1, this would be the number of
columns.
axis: (optional) The axis to normalize. Typically the feature axis.
Defaults to 1.
epsilon: (optional) A small value used for numerical stability in
calculation. Defaults to 1e-4.
initial_scale: (optional) The initial value for the learned scale
parameter. Defaults to 1.0
initial_bias: (optional) The initial value for the learned bias
parameter of the layerwise standard deviation. Defaults to 0.0.
Returns:
A 3-tuple consisting of:
- The layer normalized input blob.
- The mean of the input blob across the given axis.
- The standard deviation of the input blob acress the given axis.
'''
# The LayerNorm operator only performs the layerwise z-shift, without
# scaling and shifting by the learned scale and bias parameters. We have
# to do that separately below.
normalized, mean, stdev = model.net.LayerNorm(
[blob_in],
[blob_out, blob_out + "_mean", blob_out + "_stdev"],
axis=axis,
epsilon=epsilon,
)
# The learned multiplicative scale or "gain".
scale = model.create_param(
param_name='{}_scale'.format(blob_out),
shape=[dim_in],
initializer=initializers.Initializer(
'ConstantFill',
value=initial_scale,
),
tags=ParameterTags.WEIGHT,
)
# The learned additive bias or "shift".
bias = model.create_param(
param_name='{}_bias'.format(blob_out),
shape=[dim_in],
initializer=initializers.Initializer(
'ConstantFill',
value=initial_bias,
),
tags=ParameterTags.BIAS,
)
scaled = model.net.Mul(
[normalized, scale],
['{}_scaled'.format(blob_out)],
broadcast=1,
axis=axis,
)
biased = model.net.Add(
[scaled, bias],
['{}_biased'.format(blob_out)],
broadcast=1,
axis=axis,
)
return biased, mean, stdev
def layer_norm(
model,
blob_in,
blob_out,
dim_in,
axis=1,
epsilon=1e-4,
initial_scale=1.0,
initial_bias=0.0,
):
'''
Layer normalizes the input, cf. https://arxiv.org/pdf/1607.06450.pdf.
Args:
blob_in: The input blob to layer normalize.
blob_out: The layer normalized output blob.
dim_in: The dimension of the scale and bias. For example, if blob_in is
a 2D design matrix and axis is 1, this would be the number of
columns.
axis: (optional) The axis to normalize. Typically the feature axis.
Defaults to 1.
epsilon: (optional) A small value used for numerical stability in
calculation. Defaults to 1e-4.
initial_scale: (optional) The initial value for the learned scale
parameter. Defaults to 1.0
initial_bias: (optional) The initial value for the learned bias
parameter of the layerwise standard deviation. Defaults to 0.0.
Returns:
A 3-tuple consisting of:
- The layer normalized input blob.
- The mean of the input blob across the given axis.
- The standard deviation of the input blob acress the given axis.
'''
# The learned multiplicative scale or "gain".
scale = model.create_param(
param_name='{}_scale'.format(blob_out),
shape=[dim_in] if isinstance(dim_in, int) else dim_in,
initializer=initializers.Initializer(
'ConstantFill',
value=initial_scale,
),
tags=ParameterTags.WEIGHT,
)
# The learned additive bias or "shift".
bias = model.create_param(
param_name='{}_bias'.format(blob_out),
shape=[dim_in] if isinstance(dim_in, int) else dim_in,
initializer=initializers.Initializer(
'ConstantFill',
value=initial_bias,
),
tags=ParameterTags.BIAS,
)
normalized, mean, std = model.net.LayerNorm(
[blob_in, scale, bias],
[blob_out, blob_out + "_mean", blob_out + "_std"],
axis=axis,
epsilon=epsilon,
elementwise_affine=True,
)
return normalized, mean, std
def init_normalizer(model):
norm = model.create_param(param_name='mem_00/spatial_normalizer',
initializer=initializers.Initializer(
"ConstantFill", value=1.),
shape=[1, cfg.MEM.INIT_H, cfg.MEM.INIT_W])
return model.ResizeNormalizerInit()
def init(model):
mem = model.create_param(param_name='mem_00/spatial',
initializer=initializers.Initializer(
"GaussianFill", std=cfg.MEM.STD),
shape=[cfg.MEM.C, cfg.MEM.INIT_H, cfg.MEM.INIT_W])
return model.ResizeMemoryInit()
