def _init_layers(self):
"""Initialize Align layers of the S2AHead."""
self.align_convs = nn.ModuleList()
for align_type in self.align_type:
assert align_type in ['Conv', 'DCN', 'AlignConv']
if align_type == 'Conv':
self.align_convs.append(
ConvModule(self.feat_channels,
self.feat_channels,
kernel_size=3,
stride=1,
padding=1))
elif align_type == 'DCN':
self.align_convs.append(
DeformConv2dPack(self.feat_channels,
self.feat_channels,
kernel_size=3,
stride=1,
padding=1,
deform_groups=1))
elif align_type == 'AlignConv':
self.align_convs.append(
AlignConv(self.feat_channels,
self.feat_channels,
kernel_size=3,
deform_groups=1))
def __init__(self,
in_channels=3,
mid_channels=64,
out_channels=3,
num_blocks_before_align=5,
num_blocks_after_align=10):
super().__init__()
self.feat_extract = nn.Sequential(
ConvModule(in_channels, mid_channels, 3, padding=1),
make_layer(ResidualBlockNoBN,
num_blocks_before_align,
mid_channels=mid_channels))
self.feat_aggregate = nn.Sequential(
nn.Conv2d(mid_channels * 2, mid_channels, 3, padding=1, bias=True),
DeformConv2dPack(mid_channels,
mid_channels,
3,
padding=1,
deform_groups=8),
DeformConv2dPack(mid_channels,
mid_channels,
3,
padding=1,
deform_groups=8))
self.align_1 = AugmentedDeformConv2dPack(mid_channels,
mid_channels,
3,
padding=1,
deform_groups=8)
self.align_2 = DeformConv2dPack(mid_channels,
mid_channels,
3,
padding=1,
deform_groups=8)
self.to_rgb = nn.Conv2d(mid_channels, 3, 3, padding=1, bias=True)
self.reconstruct = nn.Sequential(
ConvModule(in_channels * 5, mid_channels, 3, padding=1),
make_layer(ResidualBlockNoBN,
num_blocks_after_align,
mid_channels=mid_channels),
PixelShufflePack(mid_channels, mid_channels, 2, upsample_kernel=3),
PixelShufflePack(mid_channels, mid_channels, 2, upsample_kernel=3),
nn.Conv2d(mid_channels, out_channels, 3, 1, 1, bias=False))
def deformable_conv(in_channels, out_channels, kernel_size=3, bias=True):
return DeformConv2dPack(in_channels,
out_channels,
kernel_size=kernel_size,
padding=(kernel_size // 2),
stride=1,
bias=bias,
groups=in_channels)
def _test_amp_deformconv(self, input_dtype, threshold=1e-3):
"""The function to test amp released on pytorch 1.6.0.
The type of input data might be torch.float or torch.half,
so we should test deform_conv in both cases. With amp, the
data type of model will NOT be set manually.
Args:
input_dtype: torch.float or torch.half.
threshold: the same as above function.
"""
if not torch.cuda.is_available():
return
from mmcv.ops import DeformConv2dPack
c_in = 1
c_out = 1
x = torch.Tensor(input).cuda().type(input_dtype)
x.requires_grad = True
model = DeformConv2dPack(c_in, c_out, 2, stride=1, padding=0)
model.conv_offset.weight.data = torch.nn.Parameter(
torch.Tensor(offset_weight).reshape(8, 1, 2, 2))
model.conv_offset.bias.data = torch.nn.Parameter(
torch.Tensor(offset_bias).reshape(8))
model.weight.data = torch.nn.Parameter(
torch.Tensor(deform_weight).reshape(1, 1, 2, 2))
model.cuda()
out = model(x)
out.backward(torch.ones_like(out))
assert np.allclose(out.data.detach().cpu().numpy(), gt_out, threshold)
assert np.allclose(x.grad.detach().cpu().numpy(), gt_x_grad, threshold)
assert np.allclose(
model.conv_offset.weight.grad.detach().cpu().numpy(),
gt_offset_weight_grad, threshold)
assert np.allclose(model.conv_offset.bias.grad.detach().cpu().numpy(),
gt_offset_bias_grad, threshold)
assert np.allclose(model.weight.grad.detach().cpu().numpy(),
gt_deform_weight_grad, threshold)
from mmcv.ops import DeformConv2d
# test bias
model = DeformConv2d(1, 1, 2, stride=1, padding=0)
assert not hasattr(model, 'bias')
# test bias=True
with pytest.raises(AssertionError):
model = DeformConv2d(1, 1, 2, stride=1, padding=0, bias=True)
# test in_channels % group != 0
with pytest.raises(AssertionError):
model = DeformConv2d(3, 2, 3, groups=2)
# test out_channels % group != 0
with pytest.raises(AssertionError):
model = DeformConv2d(3, 4, 3, groups=3)
def _init_layers(self):
"""Initialize layers of the head."""
super()._init_layers()
#self._init_reid_convs()
self.conv_centerness = nn.Conv2d(self.feat_channels, 1, 3, padding=1)
#self.conv_reid = nn.Conv2d(self.feat_channels, self.feat_channels, 3, padding=1)
self.conv_reid = DeformConv2dPack(self.feat_channels, self.feat_channels, 3, 1, 1)
#num_person = 359
num_person = 5532
#self.classifier_reid = nn.Linear(self.feat_channels, num_person)
self.scales = nn.ModuleList([Scale(1.0) for _ in self.strides])
self.labeled_matching_layer = LabeledMatchingLayer(num_persons=num_person) # for mot17half
self.unlabeled_matching_layer = UnlabeledMatchingLayer()
def __init__(self):
super().__init__()
self.param1 = nn.Parameter(torch.ones(1))
self.conv1 = nn.Conv2d(3, 4, kernel_size=1, bias=False)
self.conv2 = nn.Conv2d(4, 2, kernel_size=1)
self.bn = nn.BatchNorm2d(2)
self.sub = SubModel()
if OPS_AVAILABLE:
from mmcv.ops import DeformConv2dPack
self.dcn = DeformConv2dPack(3,
4,
kernel_size=3,
deformable_groups=1)
def _test_deformconv(self,
dtype=torch.float,
threshold=1e-3,
device='cuda'):
if not torch.cuda.is_available() and device == 'cuda':
pytest.skip('test requires GPU')
from mmcv.ops import DeformConv2dPack
c_in = 1
c_out = 1
x = torch.tensor(input, device=device, dtype=dtype)
x.requires_grad = True
model = DeformConv2dPack(c_in, c_out, 2, stride=1, padding=0)
model.conv_offset.weight.data = torch.nn.Parameter(
torch.Tensor(offset_weight).reshape(8, 1, 2, 2))
model.conv_offset.bias.data = torch.nn.Parameter(
torch.Tensor(offset_bias).reshape(8))
model.weight.data = torch.nn.Parameter(
torch.Tensor(deform_weight).reshape(1, 1, 2, 2))
if device == 'cuda':
model.cuda()
model.type(dtype)
out = model(x)
out.backward(torch.ones_like(out))
assert np.allclose(out.data.detach().cpu().numpy(), gt_out, threshold)
assert np.allclose(x.grad.detach().cpu().numpy(), gt_x_grad, threshold)
assert np.allclose(
model.conv_offset.weight.grad.detach().cpu().numpy(),
gt_offset_weight_grad, threshold)
assert np.allclose(model.conv_offset.bias.grad.detach().cpu().numpy(),
gt_offset_bias_grad, threshold)
assert np.allclose(model.weight.grad.detach().cpu().numpy(),
gt_deform_weight_grad, threshold)
from mmcv.ops import DeformConv2d
# test bias
model = DeformConv2d(1, 1, 2, stride=1, padding=0)
assert not hasattr(model, 'bias')
# test bias=True
with pytest.raises(AssertionError):
model = DeformConv2d(1, 1, 2, stride=1, padding=0, bias=True)
# test in_channels % group != 0
with pytest.raises(AssertionError):
model = DeformConv2d(3, 2, 3, groups=2)
# test out_channels % group != 0
with pytest.raises(AssertionError):
model = DeformConv2d(3, 4, 3, groups=3)
def __init__(self, in_planes, out_planes, modulate_deform=True):
super(DeconvLayer, self).__init__()
if modulate_deform:
self.dcn = ModulatedDeformConv2dPack(in_planes,
out_planes,
kernel_size=3,
padding=1,
deform_groups=1)
else:
self.dcn = DeformConv2dPack(in_planes,
out_planes,
kernel_size=3,
padding=1,
deform_groups=1)
self.dcn_bn = nn.BatchNorm2d(out_planes)
self.up_sample = nn.UpsamplingBilinear2d(scale_factor=2)
self.relu = nn.ReLU(inplace=True)
def __init__(self,
in_channels,
out_channels,
num_outs,
start_level=0,
end_level=-1,
add_extra_convs=False,
extra_convs_on_inputs=True,
relu_before_extra_convs=False,
no_norm_on_lateral=False,
conv_cfg=None,
norm_cfg=None,
act_cfg=None,
upsample_cfg=dict(mode='nearest')):
super(FPNDcnNocat, self).__init__()
assert isinstance(in_channels, list)
self.in_channels = in_channels
self.out_channels = out_channels
self.num_ins = len(in_channels)
self.num_outs = num_outs
self.relu_before_extra_convs = relu_before_extra_convs
self.no_norm_on_lateral = no_norm_on_lateral
self.fp16_enabled = False
self.upsample_cfg = upsample_cfg.copy()
if end_level == -1:
self.backbone_end_level = self.num_ins
assert num_outs >= self.num_ins - start_level
else:
# if end_level < inputs, no extra level is allowed
self.backbone_end_level = end_level
assert end_level <= len(in_channels)
assert num_outs == end_level - start_level
self.start_level = start_level
self.end_level = end_level
self.add_extra_convs = add_extra_convs
assert isinstance(add_extra_convs, (str, bool))
if isinstance(add_extra_convs, str):
# Extra_convs_source choices: 'on_input', 'on_lateral', 'on_output'
assert add_extra_convs in ('on_input', 'on_lateral', 'on_output')
elif add_extra_convs: # True
if extra_convs_on_inputs:
# For compatibility with previous release
# TODO: deprecate `extra_convs_on_inputs`
self.add_extra_convs = 'on_input'
else:
self.add_extra_convs = 'on_output'
self.lateral_convs = nn.ModuleList()
self.fpn_convs = nn.ModuleList()
for i in range(self.start_level, self.backbone_end_level):
l_conv = ConvModule(
in_channels[i],
out_channels,
1,
conv_cfg=conv_cfg,
norm_cfg=norm_cfg if not self.no_norm_on_lateral else None,
act_cfg=act_cfg,
inplace=False)
fpn_conv = DeformConv2dPack(out_channels,
out_channels,
3,
padding=1)
'''
fpn_conv = ConvModule(
out_channels,
out_channels,
3,
padding=1,
conv_cfg=conv_cfg,
norm_cfg=norm_cfg,
act_cfg=act_cfg,
inplace=False)
'''
self.lateral_convs.append(l_conv)
self.fpn_convs.append(fpn_conv)
# add extra conv layers (e.g., RetinaNet)
extra_levels = num_outs - self.backbone_end_level + self.start_level
if self.add_extra_convs and extra_levels >= 1:
for i in range(extra_levels):
if i == 0 and self.add_extra_convs == 'on_input':
in_channels = self.in_channels[self.backbone_end_level - 1]
else:
in_channels = out_channels
extra_fpn_conv = ConvModule(in_channels,
out_channels,
3,
stride=2,
padding=1,
conv_cfg=conv_cfg,
norm_cfg=norm_cfg,
act_cfg=act_cfg,
inplace=False)
self.fpn_convs.append(extra_fpn_conv)
def test_deform_conv():
try:
from mmcv.ops import DeformConv2dPack
except (ImportError, ModuleNotFoundError):
pytest.skip('test requires compilation')
input = [[[[1., 2., 3.], [0., 1., 2.], [3., 5., 2.]]]]
offset_weight = [[[0.1, 0.4, 0.6, 0.1]], [[0.3, 0.2, 0.1, 0.3]],
[[0.5, 0.5, 0.2, 0.8]], [[0.8, 0.3, 0.9, 0.1]],
[[0.3, 0.1, 0.2, 0.5]], [[0.3, 0.7, 0.5, 0.3]],
[[0.6, 0.2, 0.5, 0.3]], [[0.4, 0.1, 0.8, 0.4]]]
offset_bias = [0.7, 0.1, 0.8, 0.5, 0.6, 0.5, 0.4, 0.7]
deform_weight = [[[0.4, 0.2, 0.1, 0.9]]]
c_in = 1
c_out = 1
x = torch.Tensor(input).cuda()
x.requires_grad = True
model = DeformConv2dPack(c_in, c_out, 2, stride=1, padding=0)
model.conv_offset.weight.data = torch.nn.Parameter(
torch.Tensor(offset_weight).reshape(8, 1, 2, 2))
model.conv_offset.bias.data = torch.nn.Parameter(
torch.Tensor(offset_bias).reshape(8))
model.weight.data = torch.nn.Parameter(
torch.Tensor(deform_weight).reshape(1, 1, 2, 2))
model.cuda().eval()
input_names = ['input']
output_names = ['output']
with torch.no_grad():
torch.onnx.export(model, (x.clone(), ),
onnx_file,
export_params=True,
keep_initializers_as_inputs=True,
input_names=input_names,
output_names=output_names,
opset_version=11)
onnx_model = onnx.load(onnx_file)
# create trt engine and wrapper
opt_shape_dict = {
'input': [list(x.shape), list(x.shape),
list(x.shape)],
}
# trt config
fp16_mode = False
max_workspace_size = 1 << 30
trt_engine = onnx2trt(onnx_model,
opt_shape_dict,
fp16_mode=fp16_mode,
max_workspace_size=max_workspace_size)
save_trt_engine(trt_engine, trt_file)
trt_model = TRTWrapper(trt_file, input_names, output_names)
with torch.no_grad():
trt_outputs = trt_model({'input': x.clone()})
trt_results = trt_outputs['output']
# compute pytorch_output
with torch.no_grad():
pytorch_results = model(x.clone())
# allclose
if os.path.exists(onnx_file):
os.remove(onnx_file)
if os.path.exists(trt_file):
os.remove(trt_file)
assert torch.allclose(pytorch_results, trt_results)
def __init__(self,
in_channels,
out_channels,
num_outs,
start_level=0,
end_level=-1,
norm_cfg=None,
act_cfg=None,
order=('conv', 'norm', 'act'),
upsample_cfg=dict(
type='carafe',
up_kernel=5,
up_group=1,
encoder_kernel=3,
encoder_dilation=1)):
super(FPN_CARAFE_LDCN1, self).__init__()
assert isinstance(in_channels, list)
self.in_channels = in_channels
self.out_channels = out_channels
self.num_ins = len(in_channels)
self.num_outs = num_outs
self.norm_cfg = norm_cfg
self.act_cfg = act_cfg
self.with_bias = norm_cfg is None
self.upsample_cfg = upsample_cfg.copy()
self.upsample = self.upsample_cfg.get('type')
self.relu = nn.ReLU(inplace=False)
self.order = order
assert order in [('conv', 'norm', 'act'), ('act', 'conv', 'norm')]
assert self.upsample in [
'nearest', 'bilinear', 'deconv', 'pixel_shuffle', 'carafe', None
]
if self.upsample in ['deconv', 'pixel_shuffle']:
assert hasattr(
self.upsample_cfg,
'upsample_kernel') and self.upsample_cfg.upsample_kernel > 0
self.upsample_kernel = self.upsample_cfg.pop('upsample_kernel')
if end_level == -1:
self.backbone_end_level = self.num_ins
assert num_outs >= self.num_ins - start_level
else:
# if end_level < inputs, no extra level is allowed
self.backbone_end_level = end_level
assert end_level <= len(in_channels)
assert num_outs == end_level - start_level
self.start_level = start_level
self.end_level = end_level
self.lateral_convs = nn.ModuleList()
self.fpn_convs = nn.ModuleList()
self.upsample_modules = nn.ModuleList()
for i in range(self.start_level, self.backbone_end_level):
# l_conv = ConvModule(
# in_channels[i],
# out_channels,
# 1,
# norm_cfg=norm_cfg,
# bias=self.with_bias,
# act_cfg=act_cfg,
# inplace=False,
# order=self.order)
l_conv = DeformConv2dPack(
in_channels[i],
out_channels,
3,
padding=1
)
# fpn_conv = ConvModule(
# out_channels,
# out_channels,
# 3,
# padding=1,
# norm_cfg=self.norm_cfg,
# bias=self.with_bias,
# act_cfg=act_cfg,
# inplace=False,
# order=self.order)
fpn_conv = DeformConv2dPack(
(self.backbone_end_level - i) * out_channels,
out_channels,
3,
padding=1)
if i != self.backbone_end_level - 1:
upsample_cfg_ = self.upsample_cfg.copy()
if self.upsample == 'deconv':
upsample_cfg_.update(
in_channels=out_channels,
out_channels=out_channels,
kernel_size=self.upsample_kernel,
stride=2,
padding=(self.upsample_kernel - 1) // 2,
output_padding=(self.upsample_kernel - 1) // 2)
elif self.upsample == 'pixel_shuffle':
upsample_cfg_.update(
in_channels=out_channels,
out_channels=out_channels,
scale_factor=2,
upsample_kernel=self.upsample_kernel)
elif self.upsample == 'carafe':
upsample_cfg_.update(channels=out_channels, scale_factor=2)
else:
# suppress warnings
align_corners = (None
if self.upsample == 'nearest' else False)
upsample_cfg_.update(
scale_factor=2,
mode=self.upsample,
align_corners=align_corners)
upsample_module = build_upsample_layer(upsample_cfg_)
self.upsample_modules.append(upsample_module)
self.lateral_convs.append(l_conv)
self.fpn_convs.append(fpn_conv)
# add extra conv layers (e.g., RetinaNet)
extra_out_levels = (
num_outs - self.backbone_end_level + self.start_level)
if extra_out_levels >= 1:
for i in range(extra_out_levels):
in_channels = (
self.in_channels[self.backbone_end_level -
1] if i == 0 else out_channels)
extra_l_conv = ConvModule(
in_channels,
out_channels,
3,
stride=2,
padding=1,
norm_cfg=norm_cfg,
bias=self.with_bias,
act_cfg=act_cfg,
inplace=False,
order=self.order)
# extra_l_conv = DeformConv2dPack(
# in_channels,
# out_channels,
# 3,
# stride=2,
# padding=1,
# bias=False)
if self.upsample == 'deconv':
upsampler_cfg_ = dict(
in_channels=out_channels,
out_channels=out_channels,
kernel_size=self.upsample_kernel,
stride=2,
padding=(self.upsample_kernel - 1) // 2,
output_padding=(self.upsample_kernel - 1) // 2)
elif self.upsample == 'pixel_shuffle':
upsampler_cfg_ = dict(
in_channels=out_channels,
out_channels=out_channels,
scale_factor=2,
upsample_kernel=self.upsample_kernel)
elif self.upsample == 'carafe':
upsampler_cfg_ = dict(
channels=out_channels,
scale_factor=2,
**self.upsample_cfg)
else:
# suppress warnings
align_corners = (None
if self.upsample == 'nearest' else False)
upsampler_cfg_ = dict(
scale_factor=2,
mode=self.upsample,
align_corners=align_corners)
upsampler_cfg_['type'] = self.upsample
upsample_module = build_upsample_layer(upsampler_cfg_)
extra_fpn_conv = ConvModule(
out_channels,
out_channels,
3,
padding=1,
norm_cfg=self.norm_cfg,
bias=self.with_bias,
act_cfg=act_cfg,
inplace=False,
order=self.order)
self.upsample_modules.append(upsample_module)
self.fpn_convs.append(extra_fpn_conv)
self.lateral_convs.append(extra_l_conv)
def _test_deformconv(self,
dtype=torch.float,
threshold=1e-3,
device='cuda',
batch_size=10,
im2col_step=2):
if not torch.cuda.is_available() and device == 'cuda':
pytest.skip('test requires GPU')
from mmcv.ops import DeformConv2dPack
c_in = 1
c_out = 1
batch_size = 10
repeated_input = np.repeat(input, batch_size, axis=0)
repeated_gt_out = np.repeat(gt_out, batch_size, axis=0)
repeated_gt_x_grad = np.repeat(gt_x_grad, batch_size, axis=0)
x = torch.tensor(repeated_input, device=device, dtype=dtype)
x.requires_grad = True
model = DeformConv2dPack(in_channels=c_in,
out_channels=c_out,
kernel_size=2,
stride=1,
padding=0,
im2col_step=im2col_step)
model.conv_offset.weight.data = torch.nn.Parameter(
torch.Tensor(offset_weight).reshape(8, 1, 2, 2))
model.conv_offset.bias.data = torch.nn.Parameter(
torch.Tensor(offset_bias).reshape(8))
model.weight.data = torch.nn.Parameter(
torch.Tensor(deform_weight).reshape(1, 1, 2, 2))
if device == 'cuda':
model.cuda()
model.type(dtype)
out = model(x)
out.backward(torch.ones_like(out))
assert np.allclose(out.data.detach().cpu().numpy(), repeated_gt_out,
threshold)
assert np.allclose(x.grad.detach().cpu().numpy(), repeated_gt_x_grad,
threshold)
# the batch size of the input is increased which results in
# a larger gradient so we need to divide by the batch_size
assert np.allclose(
model.conv_offset.weight.grad.detach().cpu().numpy() / batch_size,
gt_offset_weight_grad, threshold)
assert np.allclose(
model.conv_offset.bias.grad.detach().cpu().numpy() / batch_size,
gt_offset_bias_grad, threshold)
assert np.allclose(
model.weight.grad.detach().cpu().numpy() / batch_size,
gt_deform_weight_grad, threshold)
from mmcv.ops import DeformConv2d
# test bias
model = DeformConv2d(1, 1, 2, stride=1, padding=0)
assert not hasattr(model, 'bias')
# test bias=True
with pytest.raises(AssertionError):
model = DeformConv2d(1, 1, 2, stride=1, padding=0, bias=True)
# test in_channels % group != 0
with pytest.raises(AssertionError):
model = DeformConv2d(3, 2, 3, groups=2)
# test out_channels % group != 0
with pytest.raises(AssertionError):
model = DeformConv2d(3, 4, 3, groups=3)