def __init__(
self,
in_channels: int,
out_channels: int,
kernel_size: Union[int, Tuple],
stride: Union[int, Tuple] = 1,
dilation: Union[int, Tuple] = 1,
bias: bool = True,
) -> None:
kernel_size = _ntuple(self._ndim)(kernel_size)
stride = _ntuple(self._ndim)(stride)
dilation = _ntuple(self._ndim)(dilation)
nn.Conv2d.__init__(
self,
in_channels,
out_channels,
kernel_size,
stride=stride,
padding=_ntuple(self._ndim)(0),
dilation=dilation,
groups=1,
bias=bias,
padding_mode="zeros",
)
self.active_in_channels = in_channels
self.active_out_channels = out_channels
def _output_size(dim):
_check_size_scale_factor(dim)
if size is not None:
return size
scale_factors = _ntuple(dim)(scale_factor)
# math.floor might return float in py2.7
return [int(math.floor(input.size(i + 2) * scale_factors[i])) for i in range(dim)]
def output_shape(self):
if self.input_shape is None:
return None
scale_factor = _ntuple(len(self.input_shape) - 2)(self.scale_factor)
shape = [np.nan if s is None else s for s in self.input_shape]
return (tuple(shape[:2]) + tuple(self.size)) if self.size \
else shape[:2] + [shape[i + 2] * scale_factor[i] for i in range(len(scale_factor))]
def forward(self, x):
weight = self.weight
weight_mean = weight.mean(dim=1, keepdim=True).mean(dim=2, keepdim=True).mean(dim=3, keepdim=True)
weight = weight - weight_mean
if x.numel() > 0:
return nn.functional.conv2d(x, weight, self.bias, self.stride, self.padding, self.dilation, self.groups)
_pair = _ntuple(2)
output_shape = [
(i + 2 * p - (di * (k - 1) + 1)) // d + 1
for i, p, di, k, d in zip(
x.shape[-2:], _pair(self.padding), _pair(self.dilation), _pair(self.kernel_size), _pair(self.stride)
)
]
output_shape = [x.shape[0], self.weight.shape[0]] + output_shape
return _NewEmptyTensorOp.apply(x, output_shape)
def _output_size(dim):
_check_size_scale_factor(dim)
if size is not None:
return size
scale_factors = _ntuple(dim)(scale_factor)
# math.floor might return float in py2.7
# make scale_factor a tensor in tracing so constant doesn't get baked in
"""
if torch._C._get_tracing_state():
return [(torch.floor((input.size(i + 2) * torch.tensor(float(scale_factors[i]))).float())) for i in range(dim)]
else:
return [int(math.floor(float(input.size(i + 2)) * scale_factors[i])) for i in range(dim)]
"""
return [(torch.floor((input.size(i + 2) *
torch.tensor(float(scale_factors[i]))).float()))
for i in range(dim)]
def _output_size(dim):
_check_size_scale_factor(dim)
if size is not None:
if is_tracing:
return [torch.tensor(i) for i in size]
else:
return size
scale_factors = _ntuple(dim)(scale_factor)
# math.floor might return float in py2.7
# make scale_factor a tensor in tracing so constant doesn't get baked in
if is_tracing:
return [
(torch.floor((input.size(i + 2).float() * torch.tensor(scale_factors[i], dtype=torch.float32)).float()))
for i in range(dim)
]
else:
return [int(math.floor(float(input.size(i + 2)) * scale_factors[i])) for i in range(dim)]
def _update_long_cycle(self, runner):
"""Before every epoch, check if long cycle shape should change. If it
should, change the pipelines accordingly.
change dataloader and model's subbn3d(split_bn)
"""
base_b, base_t, base_s = self._get_schedule(runner.epoch)
# rebuild dataset
from mmaction.datasets import build_dataset
resize_list = []
for trans in self.cfg.data.train.pipeline:
if trans['type'] == 'SampleFrames':
curr_t = trans['clip_len']
trans['clip_len'] = base_t
trans['frame_interval'] = (curr_t *
trans['frame_interval']) / base_t
elif trans['type'] == 'Resize':
resize_list.append(trans)
resize_list[-1]['scale'] = _ntuple(2)(base_s)
ds = build_dataset(self.cfg.data.train)
from mmaction.datasets import build_dataloader
dataloader = build_dataloader(
ds,
self.data_cfg.videos_per_gpu * base_b,
self.data_cfg.workers_per_gpu,
dist=True,
num_gpus=len(self.cfg.gpu_ids),
drop_last=True,
seed=self.cfg.get('seed', None),
)
runner.data_loader = dataloader
self.logger.info('Rebuild runner.data_loader')
# the self._max_epochs is changed, therefore update here
runner._max_iters = runner._max_epochs * len(runner.data_loader)
# rebuild all the sub_batch_bn layers
num_modifies = modify_subbn3d_num_splits(self.logger, runner.model,
base_b)
self.logger.info(f'{num_modifies} subbns modified to {base_b}.')
def __init__(self,
depth,
num_segments=8,
is_shift=True,
non_local=(0, 0, 0, 0),
non_local_cfg=dict(),
shift_div=8,
shift_place='blockres',
temporal_pool=False,
**kwargs):
super().__init__(depth, **kwargs)
self.num_segments = num_segments
self.is_shift = is_shift
self.shift_div = shift_div
self.shift_place = shift_place
self.temporal_pool = temporal_pool
self.non_local = non_local
self.non_local_stages = _ntuple(self.num_stages)(non_local)
self.non_local_cfg = non_local_cfg
import torch.nn as nn
import math
import torch.utils.model_zoo as model_zoo
import torch.nn.functional as F
from model.dsbn import DomainSpecificBatchNorm2d
from torch.nn.modules.conv import _ConvNd
from torch.nn.modules.utils import _ntuple
from collections import OrderedDict
import operator
from itertools import islice
_pair = _ntuple(2)
__all__ = ['resnet50dsbn', 'resnet101dsbn', 'resnet152dsbn']
model_urls = {
'resnet18': 'https://download.pytorch.org/models/resnet18-5c106cde.pth',
'resnet34': 'https://download.pytorch.org/models/resnet34-333f7ec4.pth',
'resnet50': 'https://download.pytorch.org/models/resnet50-19c8e357.pth',
'resnet101': 'https://download.pytorch.org/models/resnet101-5d3b4d8f.pth',
'resnet152': 'https://download.pytorch.org/models/resnet152-b121ed2d.pth',
}
def conv3x3(in_planes, out_planes, stride=1):
"""3x3 convolution with padding"""
return nn.Conv2d(in_planes,
out_planes,
kernel_size=3,
stride=stride,
def __init__(self,
depth,
pretrained,
pretrained2d=True,
in_channels=3,
num_stages=4,
base_channels=64,
out_indices=(3, ),
spatial_strides=(1, 2, 2, 2),
temporal_strides=(1, 1, 1, 1),
dilations=(1, 1, 1, 1),
conv1_kernel=(5, 7, 7),
conv1_stride_t=2,
pool1_stride_t=2,
with_pool2=True,
style='pytorch',
frozen_stages=-1,
inflate=(1, 1, 1, 1),
inflate_style='3x1x1',
conv_cfg=dict(type='Conv3d'),
norm_cfg=dict(type='BN3d', requires_grad=True),
act_cfg=dict(type='ReLU', inplace=True),
norm_eval=False,
with_cp=False,
non_local=(0, 0, 0, 0),
non_local_cfg=dict(),
zero_init_residual=True,
**kwargs):
super().__init__()
if depth not in self.arch_settings:
raise KeyError(f'invalid depth {depth} for resnet')
self.depth = depth
self.pretrained = pretrained
self.pretrained2d = pretrained2d
self.in_channels = in_channels
self.base_channels = base_channels
self.num_stages = num_stages
assert num_stages >= 1 and num_stages <= 4
self.out_indices = out_indices
assert max(out_indices) < num_stages
self.spatial_strides = spatial_strides
self.temporal_strides = temporal_strides
self.dilations = dilations
assert len(spatial_strides) == len(temporal_strides) == len(
dilations) == num_stages
self.conv1_kernel = conv1_kernel
self.conv1_stride_t = conv1_stride_t
self.pool1_stride_t = pool1_stride_t
self.with_pool2 = with_pool2
self.style = style
self.frozen_stages = frozen_stages
self.stage_inflations = _ntuple(num_stages)(inflate)
self.non_local_stages = _ntuple(num_stages)(non_local)
self.inflate_style = inflate_style
self.conv_cfg = conv_cfg
self.norm_cfg = norm_cfg
self.act_cfg = act_cfg
self.norm_eval = norm_eval
self.with_cp = with_cp
self.zero_init_residual = zero_init_residual
self.block, stage_blocks = self.arch_settings[depth]
self.stage_blocks = stage_blocks[:num_stages]
self.inplanes = self.base_channels
self.non_local_cfg = non_local_cfg
self._make_stem_layer()
self.res_layers = []
for i, num_blocks in enumerate(self.stage_blocks):
spatial_stride = spatial_strides[i]
temporal_stride = temporal_strides[i]
dilation = dilations[i]
planes = self.base_channels * 2**i
res_layer = self.make_res_layer(self.block,
self.inplanes,
planes,
num_blocks,
spatial_stride=spatial_stride,
temporal_stride=temporal_stride,
dilation=dilation,
style=self.style,
norm_cfg=self.norm_cfg,
conv_cfg=self.conv_cfg,
act_cfg=self.act_cfg,
non_local=self.non_local_stages[i],
non_local_cfg=self.non_local_cfg,
inflate=self.stage_inflations[i],
inflate_style=self.inflate_style,
with_cp=with_cp,
**kwargs)
self.inplanes = planes * self.block.expansion
layer_name = f'layer{i + 1}'
self.add_module(layer_name, res_layer)
self.res_layers.append(layer_name)
self.feat_dim = self.block.expansion * self.base_channels * 2**(
len(self.stage_blocks) - 1)
return x
keep_prob = 1 - drop_prob
# work with diff dim tensors, not just 2D ConvNets
shape = (x.shape[0],) + (1,) * (x.ndim - 1)
random_tensor = keep_prob + torch.rand(shape, dtype=x.dtype, device=x.device)
random_tensor.floor_() # binarize
output = x.div(keep_prob) * random_tensor
return output
class DropPath(nn.Module):
"""Drop paths (Stochastic Depth) per sample (when applied in main path
of residual blocks)."""
def __init__(self, drop_prob=None):
super(DropPath, self).__init__()
self.drop_prob = drop_prob
def forward(self, x):
return drop_path(x, self.drop_prob, self.training)
def extra_repr(self) -> str:
return "p={}".format(self.drop_prob)
to_1tuple = _ntuple(1)
to_2tuple = _ntuple(2)
to_3tuple = _ntuple(3)
to_4tuple = _ntuple(4)
to_ntuple = _ntuple
def __init__(self,
depth,
pretrained,
in_channels=1,
num_stages=4,
base_channels=32,
strides=(1, 2, 2, 2),
dilations=(1, 1, 1, 1),
conv1_kernel=9,
conv1_stride=1,
frozen_stages=-1,
factorize=(1, 1, 0, 0),
norm_eval=False,
with_cp=False,
conv_cfg=dict(type='Conv'),
norm_cfg=dict(type='BN2d', requires_grad=True),
act_cfg=dict(type='ReLU', inplace=True),
zero_init_residual=True):
super().__init__()
if depth not in self.arch_settings:
raise KeyError(f'invalid depth {depth} for resnet')
self.depth = depth
self.pretrained = pretrained
self.in_channels = in_channels
self.base_channels = base_channels
self.num_stages = num_stages
assert 1 <= num_stages <= 4
self.dilations = dilations
self.conv1_kernel = conv1_kernel
self.conv1_stride = conv1_stride
self.frozen_stages = frozen_stages
self.stage_factorization = _ntuple(num_stages)(factorize)
self.norm_eval = norm_eval
self.with_cp = with_cp
self.conv_cfg = conv_cfg
self.norm_cfg = norm_cfg
self.act_cfg = act_cfg
self.zero_init_residual = zero_init_residual
self.block, stage_blocks = self.arch_settings[depth]
self.stage_blocks = stage_blocks[:num_stages]
self.inplanes = self.base_channels
self._make_stem_layer()
self.res_layers = []
for i, num_blocks in enumerate(self.stage_blocks):
stride = strides[i]
dilation = dilations[i]
planes = self.base_channels * 2**i
res_layer = self.make_res_layer(
self.block,
self.inplanes,
planes,
num_blocks,
stride=stride,
dilation=dilation,
factorize=self.stage_factorization[i],
norm_cfg=self.norm_cfg,
with_cp=with_cp)
self.inplanes = planes * self.block.expansion
layer_name = f'layer{i + 1}'
self.add_module(layer_name, res_layer)
self.res_layers.append(layer_name)
self.feat_dim = self.block.expansion * self.base_channels * 2**(
len(self.stage_blocks) - 1)
def __init__(self, padding: Union[int, Tuple[int]]):
super(ReflectionPad3d, self).__init__()
self.padding = _ntuple(6)(padding)
def __init__(self,
depth,
pretrained2d=True,
in_channels=3,
num_stages=4,
base_channels=64,
spatial_strides=(1, 2, 2, 2),
temporal_strides=(1, 1, 1, 1),
dilations=(1, 1, 1, 1),
conv1_kernel=(5, 7, 7),
conv1_stride_t=2,
pool1_stride_t=2,
with_pool2=True,
frozen_stages=-1,
inflate=(1, 1, 1, 1),
inflate_style='3x1x1',
norm_eval=True,
with_checkpoint=False,
zero_init_residual=True):
super().__init__()
if depth not in self.arch_settings:
raise KeyError(f'invalid depth {depth} for resnet')
self.depth = depth
self.pretrained2d = pretrained2d
self.in_channels = in_channels
self.base_channels = base_channels
self.num_stages = num_stages
assert num_stages >= 1 and num_stages <= 4
self.spatial_strides = spatial_strides
self.temporal_strides = temporal_strides
self.dilations = dilations
assert len(spatial_strides) == len(temporal_strides) == len(
dilations) == num_stages
self.conv1_kernel = conv1_kernel
self.conv1_stride_t = conv1_stride_t
self.pool1_stride_t = pool1_stride_t
self.with_pool2 = with_pool2
self.frozen_stages = frozen_stages
self.stage_inflations = _ntuple(num_stages)(inflate)
self.inflate_style = inflate_style
self.norm_eval = norm_eval
self.with_checkpoint = with_checkpoint
self.zero_init_residual = zero_init_residual
self.log = None
self.block, stage_blocks = self.arch_settings[depth]
self.stage_blocks = stage_blocks[:num_stages]
self.inplanes = self.base_channels
self._make_stem_layer()
self.res_layers = []
for i, num_blocks in enumerate(self.stage_blocks):
spatial_stride = spatial_strides[i]
temporal_stride = temporal_strides[i]
dilation = dilations[i]
planes = self.base_channels * 2**i
res_layer = self.make_res_layer(self.block,
self.inplanes,
planes,
num_blocks,
spatial_stride=spatial_stride,
temporal_stride=temporal_stride,
dilation=dilation,
inflate=self.stage_inflations[i],
inflate_style=self.inflate_style,
with_checkpoint=with_checkpoint)
self.inplanes = planes * self.block.expansion
layer_name = f'layer{i + 1}'
self.add_module(layer_name, res_layer)
self.res_layers.append(layer_name)
self.feat_dim = self.block.expansion * self.base_channels * 2**(
len(self.stage_blocks) - 1)
def __init__(self, padding) -> None:
super(PeriodicPad3d, self).__init__()
self.padding = _ntuple(6)(padding)