def test_classification_losses(loss_class, input_shape):
if input_shape[0] == 0 and digit_version(
torch.__version__) < digit_version('1.5.0'):
pytest.skip(
f'CELoss in PyTorch {torch.__version__} does not support empty'
f'tensor.')
pred = torch.rand(input_shape)
target = torch.randint(0, 5, (input_shape[0], ))
# Test loss forward
loss = loss_class()(pred, target)
assert isinstance(loss, torch.Tensor)
# Test loss forward with reduction_override
loss = loss_class()(pred, target, reduction_override='mean')
assert isinstance(loss, torch.Tensor)
# Test loss forward with avg_factor
loss = loss_class()(pred, target, avg_factor=10)
assert isinstance(loss, torch.Tensor)
with pytest.raises(ValueError):
# loss can evaluate with avg_factor only if
# reduction is None, 'none' or 'mean'.
reduction_override = 'sum'
loss_class()(
pred, target, avg_factor=10, reduction_override=reduction_override)
# Test loss forward with avg_factor and reduction
for reduction_override in [None, 'none', 'mean']:
loss_class()(
pred, target, avg_factor=10, reduction_override=reduction_override)
assert isinstance(loss, torch.Tensor)
def test_single_gpu(self, _):
common_cfg = dict(dataset=self.data,
samples_per_gpu=self.samples_per_gpu,
workers_per_gpu=self.workers_per_gpu,
dist=False)
# Test default config
dataloader = build_dataloader(**common_cfg)
if digit_version(torch.__version__) >= digit_version('1.8.0'):
assert dataloader.persistent_workers
elif hasattr(dataloader, 'persistent_workers'):
assert not dataloader.persistent_workers
assert dataloader.batch_size == self.samples_per_gpu
assert dataloader.num_workers == self.workers_per_gpu
assert not all(
torch.cat(list(iter(dataloader))) == torch.tensor(self.data))
# Test without shuffle
dataloader = build_dataloader(**common_cfg, shuffle=False)
assert all(
torch.cat(list(iter(dataloader))) == torch.tensor(self.data))
# Test with custom sampler_cfg
dataloader = build_dataloader(**common_cfg,
sampler_cfg=dict(type='RepeatAugSampler',
selected_round=0),
shuffle=False)
expect = [0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 3, 4, 4, 4, 5, 5, 5, 6, 6]
assert all(torch.cat(list(iter(dataloader))) == torch.tensor(expect))
def test_distributed(self, _):
common_cfg = dict(
dataset=self.data,
samples_per_gpu=self.samples_per_gpu,
workers_per_gpu=self.workers_per_gpu,
num_gpus=2, # num_gpus will be ignored in distributed environment.
dist=True)
# Test default config
dataloader = build_dataloader(**common_cfg)
if digit_version(torch.__version__) >= digit_version('1.8.0'):
assert dataloader.persistent_workers
elif hasattr(dataloader, 'persistent_workers'):
assert not dataloader.persistent_workers
assert dataloader.batch_size == self.samples_per_gpu
assert dataloader.num_workers == self.workers_per_gpu
non_expect = torch.tensor(self.data[1::2])
assert not all(torch.cat(list(iter(dataloader))) == non_expect)
# Test without shuffle
dataloader = build_dataloader(**common_cfg, shuffle=False)
expect = torch.tensor(self.data[1::2])
assert all(torch.cat(list(iter(dataloader))) == expect)
# Test with custom sampler_cfg
dataloader = build_dataloader(**common_cfg,
sampler_cfg=dict(type='RepeatAugSampler',
selected_round=0),
shuffle=False)
expect = torch.tensor(
[0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 3, 4, 4, 4, 5, 5, 5, 6, 6][1::2])
assert all(torch.cat(list(iter(dataloader))) == expect)
def __init__(self,
embed_dims,
num_heads,
num_frames,
attn_drop=0.,
proj_drop=0.,
dropout_layer=dict(type='DropPath', drop_prob=0.1),
norm_cfg=dict(type='LN'),
init_cfg=None,
**kwargs):
super().__init__(init_cfg)
self.embed_dims = embed_dims
self.num_heads = num_heads
self.num_frames = num_frames
self.norm = build_norm_layer(norm_cfg, self.embed_dims)[1]
if digit_version(torch.__version__) < digit_version('1.9.0'):
kwargs.pop('batch_first', None)
self.attn = nn.MultiheadAttention(embed_dims, num_heads, attn_drop,
**kwargs)
self.proj_drop = nn.Dropout(proj_drop)
self.dropout_layer = build_dropout(
dropout_layer) if dropout_layer else nn.Identity()
self.temporal_fc = nn.Linear(self.embed_dims, self.embed_dims)
self.init_weights()
def test_load_url():
url1 = 'https://download.openmmlab.com/mmcv/test_data/saved_in_pt1.5.pth'
url2 = 'https://download.openmmlab.com/mmcv/test_data/saved_in_pt1.6.pth'
# The 1.6 release of PyTorch switched torch.save to use a new zipfile-based
# file format. It will cause RuntimeError when a checkpoint was saved in
# torch >= 1.6.0 but loaded in torch < 1.7.0.
# More details at https://github.com/open-mmlab/mmpose/issues/904
if digit_version(TORCH_VERSION) < digit_version('1.7.0'):
model_zoo.load_url(url1)
with pytest.raises(RuntimeError):
model_zoo.load_url(url2)
else:
# high version of PyTorch can load checkpoints from url, regardless
# of which version they were saved in
model_zoo.load_url(url1)
model_zoo.load_url(url2)
load_url(url1)
# if a checkpoint was saved in torch >= 1.6.0 but loaded in torch < 1.5.0,
# it will raise a RuntimeError
if digit_version(TORCH_VERSION) < digit_version('1.5.0'):
with pytest.raises(RuntimeError):
load_url(url2)
else:
load_url(url2)
def wrap_fp16_model(model):
"""Wrap the FP32 model to FP16.
If you are using PyTorch >= 1.6, torch.cuda.amp is used as the
backend, otherwise, original mmcv implementation will be adopted.
For PyTorch >= 1.6, this function will
1. Set fp16 flag inside the model to True.
Otherwise:
1. Convert FP32 model to FP16.
2. Remain some necessary layers to be FP32, e.g., normalization layers.
3. Set `fp16_enabled` flag inside the model to True.
Args:
model (nn.Module): Model in FP32.
"""
if (TORCH_VERSION == 'parrots'
or digit_version(TORCH_VERSION) < digit_version('1.6.0')):
# convert model to fp16
model.half()
# patch the normalization layers to make it work in fp32 mode
patch_norm_fp32(model)
# set `fp16_enabled` flag
for m in model.modules():
if hasattr(m, 'fp16_enabled'):
m.fp16_enabled = True
def test_mdconv(self):
self._test_mdconv(torch.double)
self._test_mdconv(torch.float)
self._test_mdconv(torch.half)
# test amp when torch version >= '1.6.0', the type of
# input data for mdconv might be torch.float or torch.half
if (TORCH_VERSION != 'parrots'
and digit_version(TORCH_VERSION) >= digit_version('1.6.0')):
with autocast(enabled=True):
self._test_amp_mdconv(torch.float)
self._test_amp_mdconv(torch.half)
def test_deformconv(self):
self._test_deformconv(torch.double, device='cpu')
self._test_deformconv(torch.float, device='cpu', threshold=1e-1)
self._test_deformconv(torch.double)
self._test_deformconv(torch.float)
self._test_deformconv(torch.half, threshold=1e-1)
# test amp when torch version >= '1.6.0', the type of
# input data for deformconv might be torch.float or torch.half
if (TORCH_VERSION != 'parrots'
and digit_version(TORCH_VERSION) >= digit_version('1.6.0')):
with autocast(enabled=True):
self._test_amp_deformconv(torch.float, 1e-1)
self._test_amp_deformconv(torch.half, 1e-1)
def _sync_params(self):
module_states = list(self.module.state_dict().values())
if len(module_states) > 0:
self._dist_broadcast_coalesced(module_states,
self.broadcast_bucket_size)
if self.broadcast_buffers:
if (TORCH_VERSION != 'parrots'
and digit_version(TORCH_VERSION) < digit_version('1.0')):
buffers = [b.data for b in self.module._all_buffers()]
else:
buffers = [b.data for b in self.module.buffers()]
if len(buffers) > 0:
self._dist_broadcast_coalesced(buffers,
self.broadcast_bucket_size)
def is_tracing() -> bool:
if digit_version(torch.__version__) >= digit_version('1.6.0'):
on_trace = torch.jit.is_tracing()
# In PyTorch 1.6, torch.jit.is_tracing has a bug.
# Refers to https://github.com/pytorch/pytorch/issues/42448
if isinstance(on_trace, bool):
return on_trace
else:
return torch._C._is_tracing()
else:
warnings.warn(
'torch.jit.is_tracing is only supported after v1.6.0. '
'Therefore is_tracing returns False automatically. Please '
'set on_trace manually if you are using trace.', UserWarning)
return False
def get_dist_info():
if (TORCH_VERSION != 'parrots'
and digit_version(TORCH_VERSION) < digit_version('1.0')):
initialized = dist._initialized
else:
if dist.is_available():
initialized = dist.is_initialized()
else:
initialized = False
if initialized:
rank = dist.get_rank()
world_size = dist.get_world_size()
else:
rank = 0
world_size = 1
return rank, world_size
def forward(self, x):
# pre-context
avg_x = F.adaptive_avg_pool2d(x, output_size=1)
avg_x = self.pre_context(avg_x)
avg_x = avg_x.expand_as(x)
x = x + avg_x
# switch
avg_x = F.pad(x, pad=(2, 2, 2, 2), mode='reflect')
avg_x = F.avg_pool2d(avg_x, kernel_size=5, stride=1, padding=0)
switch = self.switch(avg_x)
# sac
weight = self._get_weight(self.weight)
zero_bias = torch.zeros(self.out_channels,
device=weight.device,
dtype=weight.dtype)
if self.use_deform:
offset = self.offset_s(avg_x)
out_s = deform_conv2d(x, offset, weight, self.stride, self.padding,
self.dilation, self.groups, 1)
else:
if (TORCH_VERSION == 'parrots'
or digit_version(TORCH_VERSION) < digit_version('1.5.0')):
out_s = super().conv2d_forward(x, weight)
elif digit_version(TORCH_VERSION) >= digit_version('1.8.0'):
# bias is a required argument of _conv_forward in torch 1.8.0
out_s = super()._conv_forward(x, weight, zero_bias)
else:
out_s = super()._conv_forward(x, weight)
ori_p = self.padding
ori_d = self.dilation
self.padding = tuple(3 * p for p in self.padding)
self.dilation = tuple(3 * d for d in self.dilation)
weight = weight + self.weight_diff
if self.use_deform:
offset = self.offset_l(avg_x)
out_l = deform_conv2d(x, offset, weight, self.stride, self.padding,
self.dilation, self.groups, 1)
else:
if (TORCH_VERSION == 'parrots'
or digit_version(TORCH_VERSION) < digit_version('1.5.0')):
out_l = super().conv2d_forward(x, weight)
elif digit_version(TORCH_VERSION) >= digit_version('1.8.0'):
# bias is a required argument of _conv_forward in torch 1.8.0
out_l = super()._conv_forward(x, weight, zero_bias)
else:
out_l = super()._conv_forward(x, weight)
out = switch * out_s + (1 - switch) * out_l
self.padding = ori_p
self.dilation = ori_d
# post-context
avg_x = F.adaptive_avg_pool2d(out, output_size=1)
avg_x = self.post_context(avg_x)
avg_x = avg_x.expand_as(out)
out = out + avg_x
return out
def train_step(self, *inputs, **kwargs):
"""train_step() API for module wrapped by DistributedDataParallel.
This method is basically the same as
``DistributedDataParallel.forward()``, while replacing
``self.module.forward()`` with ``self.module.train_step()``.
It is compatible with PyTorch 1.1 - 1.5.
"""
# In PyTorch >= 1.7, ``reducer._rebuild_buckets()`` is moved from the
# end of backward to the beginning of forward.
if ('parrots' not in TORCH_VERSION
and digit_version(TORCH_VERSION) >= digit_version('1.7')
and self.reducer._rebuild_buckets()):
print_log(
'Reducer buckets have been rebuilt in this iteration.',
logger='mmcv')
if ('parrots' not in TORCH_VERSION
and digit_version(TORCH_VERSION) >= digit_version('1.11.0')):
if self._check_sync_bufs_pre_fwd():
self._sync_buffers()
else:
if (getattr(self, 'require_forward_param_sync', False)
and self.require_forward_param_sync):
self._sync_params()
if self.device_ids:
inputs, kwargs = self.scatter(inputs, kwargs, self.device_ids)
if len(self.device_ids) == 1:
output = self.module.train_step(*inputs[0], **kwargs[0])
else:
outputs = self.parallel_apply(
self._module_copies[:len(inputs)], inputs, kwargs)
output = self.gather(outputs, self.output_device)
else:
output = self.module.train_step(*inputs, **kwargs)
if ('parrots' not in TORCH_VERSION
and digit_version(TORCH_VERSION) >= digit_version('1.11.0')):
if self._check_sync_bufs_post_fwd():
self._sync_buffers()
if (torch.is_grad_enabled()
and getattr(self, 'require_backward_grad_sync', False)
and self.require_backward_grad_sync):
if self.find_unused_parameters:
self.reducer.prepare_for_backward(list(_find_tensors(output)))
else:
self.reducer.prepare_for_backward([])
else:
if ('parrots' not in TORCH_VERSION
and digit_version(TORCH_VERSION) > digit_version('1.2')):
self.require_forward_param_sync = False
return output
def import_wandb(self):
try:
import wandb
from wandb import init # noqa
# Fix ResourceWarning when calling wandb.log in wandb v0.12.10.
# https://github.com/wandb/client/issues/2837
if digit_version(wandb.__version__) < digit_version('0.12.10'):
warnings.warn(
f'The current wandb {wandb.__version__} is '
f'lower than v0.12.10 will cause ResourceWarning '
f'when calling wandb.log, Please run '
f'"pip install --upgrade wandb"')
except ImportError:
raise ImportError(
'Please run "pip install "wandb>=0.12.10"" to install wandb')
self.wandb = wandb
def new_func(*args, **kwargs):
# check if the module has set the attribute `fp16_enabled`, if not,
# just fallback to the original method.
if not isinstance(args[0], torch.nn.Module):
raise TypeError('@auto_fp16 can only be used to decorate the '
'method of nn.Module')
if not (hasattr(args[0], 'fp16_enabled') and args[0].fp16_enabled):
return old_func(*args, **kwargs)
# get the arg spec of the decorated method
args_info = getfullargspec(old_func)
# get the argument names to be casted
args_to_cast = args_info.args if apply_to is None else apply_to
# convert the args that need to be processed
new_args = []
# NOTE: default args are not taken into consideration
if args:
arg_names = args_info.args[:len(args)]
for i, arg_name in enumerate(arg_names):
if arg_name in args_to_cast:
new_args.append(
cast_tensor_type(args[i], torch.float, torch.half))
else:
new_args.append(args[i])
# convert the kwargs that need to be processed
new_kwargs = {}
if kwargs:
for arg_name, arg_value in kwargs.items():
if arg_name in args_to_cast:
new_kwargs[arg_name] = cast_tensor_type(
arg_value, torch.float, torch.half)
else:
new_kwargs[arg_name] = arg_value
# apply converted arguments to the decorated method
if (TORCH_VERSION != 'parrots' and
digit_version(TORCH_VERSION) >= digit_version('1.6.0')):
with autocast(enabled=True):
output = old_func(*new_args, **new_kwargs)
else:
output = old_func(*new_args, **new_kwargs)
# cast the results back to fp32 if necessary
if out_fp32:
output = cast_tensor_type(output, torch.half, torch.float)
return output
def test_deformconv(self):
self._test_deformconv(torch.double, device='cpu')
self._test_deformconv(torch.float, device='cpu', threshold=1e-1)
self._test_deformconv(torch.double)
self._test_deformconv(torch.float)
self._test_deformconv(torch.half, threshold=1e-1)
# test batch_size < im2col_step
self._test_deformconv(torch.float, batch_size=1, im2col_step=2)
# test bach_size % im2col_step != 0
with pytest.raises(
AssertionError,
match='batch size must be divisible by im2col_step'):
self._test_deformconv(torch.float, batch_size=10, im2col_step=3)
# test amp when torch version >= '1.6.0', the type of
# input data for deformconv might be torch.float or torch.half
if (TORCH_VERSION != 'parrots'
and digit_version(TORCH_VERSION) >= digit_version('1.6.0')):
with autocast(enabled=True):
self._test_amp_deformconv(torch.float, 1e-1)
self._test_amp_deformconv(torch.half, 1e-1)
def before_run(self, runner):
super(TensorboardLoggerHook, self).before_run(runner)
if (TORCH_VERSION == 'parrots'
or digit_version(TORCH_VERSION) < digit_version('1.1')):
try:
from tensorboardX import SummaryWriter
except ImportError:
raise ImportError('Please install tensorboardX to use '
'TensorboardLoggerHook.')
else:
try:
from torch.utils.tensorboard import SummaryWriter
except ImportError:
raise ImportError(
'Please run "pip install future tensorboard" to install '
'the dependencies to use torch.utils.tensorboard '
'(applicable to PyTorch 1.1 or higher)')
if self.log_dir is None:
self.log_dir = osp.join(runner.work_dir, 'tf_logs')
self.writer = SummaryWriter(self.log_dir)
def __init__(self,
in_channels,
out_channels,
inner_channels,
deform_groups=17,
dilations=(3, 6, 12, 18, 24),
trans_conv_kernel=1,
res_blocks_cfg=None,
offsets_kernel=3,
deform_conv_kernel=3,
in_index=0,
input_transform=None,
freeze_trans_layer=True,
norm_eval=False,
im2col_step=80):
super().__init__()
self.in_channels = in_channels
self.out_channels = out_channels
self.inner_channels = inner_channels
self.deform_groups = deform_groups
self.dilations = dilations
self.trans_conv_kernel = trans_conv_kernel
self.res_blocks_cfg = res_blocks_cfg
self.offsets_kernel = offsets_kernel
self.deform_conv_kernel = deform_conv_kernel
self.in_index = in_index
self.input_transform = input_transform
self.freeze_trans_layer = freeze_trans_layer
self.norm_eval = norm_eval
self.im2col_step = im2col_step
identity_trans_layer = False
assert trans_conv_kernel in [0, 1, 3]
kernel_size = trans_conv_kernel
if kernel_size == 3:
padding = 1
elif kernel_size == 1:
padding = 0
else:
# 0 for Identity mapping.
identity_trans_layer = True
if identity_trans_layer:
self.trans_layer = nn.Identity()
else:
self.trans_layer = build_conv_layer(cfg=dict(type='Conv2d'),
in_channels=in_channels,
out_channels=out_channels,
kernel_size=kernel_size,
stride=1,
padding=padding)
# build chain of residual blocks
if res_blocks_cfg is not None and not isinstance(res_blocks_cfg, dict):
raise TypeError('res_blocks_cfg should be dict or None.')
if res_blocks_cfg is None:
block_type = 'BASIC'
num_blocks = 20
else:
block_type = res_blocks_cfg.get('block', 'BASIC')
num_blocks = res_blocks_cfg.get('num_blocks', 20)
block = self.blocks_dict[block_type]
res_layers = []
downsample = nn.Sequential(
build_conv_layer(cfg=dict(type='Conv2d'),
in_channels=out_channels,
out_channels=inner_channels,
kernel_size=1,
stride=1,
bias=False),
build_norm_layer(dict(type='BN'), inner_channels)[1])
res_layers.append(
block(in_channels=out_channels,
out_channels=inner_channels,
downsample=downsample))
for _ in range(1, num_blocks):
res_layers.append(block(inner_channels, inner_channels))
self.offset_feats = nn.Sequential(*res_layers)
# build offset layers
self.num_offset_layers = len(dilations)
assert self.num_offset_layers > 0, 'Number of offset layers ' \
'should be larger than 0.'
target_offset_channels = 2 * offsets_kernel**2 * deform_groups
offset_layers = [
build_conv_layer(
cfg=dict(type='Conv2d'),
in_channels=inner_channels,
out_channels=target_offset_channels,
kernel_size=offsets_kernel,
stride=1,
dilation=dilations[i],
padding=dilations[i],
bias=False,
) for i in range(self.num_offset_layers)
]
self.offset_layers = nn.ModuleList(offset_layers)
# build deformable conv layers
assert digit_version(mmcv.__version__) >= \
digit_version(self.minimum_mmcv_version), \
f'Current MMCV version: {mmcv.__version__}, ' \
f'but MMCV >= {self.minimum_mmcv_version} is required, see ' \
f'https://github.com/open-mmlab/mmcv/issues/1440, ' \
f'Please install the latest MMCV.'
if has_mmcv_full:
deform_conv_layers = [
DeformConv2d(
in_channels=out_channels,
out_channels=out_channels,
kernel_size=deform_conv_kernel,
stride=1,
padding=int(deform_conv_kernel / 2) * dilations[i],
dilation=dilations[i],
deform_groups=deform_groups,
im2col_step=self.im2col_step,
) for i in range(self.num_offset_layers)
]
else:
raise ImportError('Please install the full version of mmcv '
'to use `DeformConv2d`.')
self.deform_conv_layers = nn.ModuleList(deform_conv_layers)
self.freeze_layers()
# Copyright (c) OpenMMLab. All rights reserved.
import pytest
import torch
from mmcv.utils import digit_version, is_jit_tracing
@pytest.mark.skipif(digit_version(torch.__version__) < digit_version('1.6.0'),
reason='torch.jit.is_tracing is not available before 1.6.0'
)
def test_is_jit_tracing():
def foo(x):
if is_jit_tracing():
return x
else:
return x.tolist()
x = torch.rand(3)
# test without trace
assert isinstance(foo(x), list)
# test with trace
traced_foo = torch.jit.trace(foo, (torch.rand(1), ))
assert isinstance(traced_foo(x), torch.Tensor)
# Copyright (c) OpenMMLab. All rights reserved.
from functools import partial
import torch
from mmcv.utils import digit_version
TORCH_VERSION = torch.__version__
is_rocm_pytorch = False
if (TORCH_VERSION != 'parrots'
and digit_version(TORCH_VERSION) >= digit_version('1.5')):
from torch.utils.cpp_extension import ROCM_HOME
is_rocm_pytorch = True if ((torch.version.hip is not None) and
(ROCM_HOME is not None)) else False
def _get_cuda_home():
if TORCH_VERSION == 'parrots':
from parrots.utils.build_extension import CUDA_HOME
else:
if is_rocm_pytorch:
from torch.utils.cpp_extension import ROCM_HOME
CUDA_HOME = ROCM_HOME
else:
from torch.utils.cpp_extension import CUDA_HOME
return CUDA_HOME
def get_build_config():
if TORCH_VERSION == 'parrots':
def build_dataloader(dataset,
samples_per_gpu,
workers_per_gpu,
num_gpus=1,
dist=True,
shuffle=True,
seed=None,
persistent_workers=False,
**kwargs):
"""Build PyTorch DataLoader.
In distributed training, each GPU/process has a dataloader.
In non-distributed training, there is only one dataloader for all GPUs.
Args:
dataset (Dataset): A PyTorch dataset.
samples_per_gpu (int): Number of training samples on each GPU, i.e.,
batch size of each GPU.
workers_per_gpu (int): How many subprocesses to use for data loading
for each GPU.
num_gpus (int): Number of GPUs. Only used in non-distributed training.
dist (bool): Distributed training/test or not. Default: True.
shuffle (bool): Whether to shuffle the data at every epoch.
Default: True.
persistent_workers (bool, optional): If True, the data loader will
not shutdown the worker processes after a dataset has been
consumed once. This allows to maintain the workers Dataset
instances alive. The argument also has effect in PyTorch>=1.7.0.
Default: False.
kwargs: any keyword argument to be used to initialize DataLoader
Returns:
DataLoader: A PyTorch dataloader.
"""
rank, world_size = get_dist_info()
if dist:
sampler = DistributedSampler(dataset,
world_size,
rank,
shuffle=shuffle,
samples_per_gpu=samples_per_gpu,
seed=seed)
shuffle = False
batch_size = samples_per_gpu
num_workers = workers_per_gpu
else:
sampler = None
batch_size = num_gpus * samples_per_gpu
num_workers = num_gpus * workers_per_gpu
init_fn = partial(
worker_init_fn, num_workers=num_workers, rank=rank,
seed=seed) if seed is not None else None
if (digit_version(TORCH_VERSION) >= digit_version('1.7.0')
and TORCH_VERSION != 'parrots'):
kwargs['persistent_workers'] = persistent_workers
elif persistent_workers is True:
warnings.warn('persistent_workers is invalid because your pytorch '
'version is lower than 1.7.0')
data_loader = DataLoader(dataset,
batch_size=batch_size,
sampler=sampler,
num_workers=num_workers,
collate_fn=partial(
collate, samples_per_gpu=samples_per_gpu),
shuffle=shuffle,
worker_init_fn=init_fn,
**kwargs)
return data_loader
def build_dataloader(dataset,
videos_per_gpu,
workers_per_gpu,
num_gpus=1,
dist=True,
shuffle=True,
seed=None,
drop_last=False,
pin_memory=True,
persistent_workers=False,
**kwargs):
"""Build PyTorch DataLoader.
In distributed training, each GPU/process has a dataloader.
In non-distributed training, there is only one dataloader for all GPUs.
Args:
dataset (:obj:`Dataset`): A PyTorch dataset.
videos_per_gpu (int): Number of videos on each GPU, i.e.,
batch size of each GPU.
workers_per_gpu (int): How many subprocesses to use for data
loading for each GPU.
num_gpus (int): Number of GPUs. Only used in non-distributed
training. Default: 1.
dist (bool): Distributed training/test or not. Default: True.
shuffle (bool): Whether to shuffle the data at every epoch.
Default: True.
seed (int | None): Seed to be used. Default: None.
drop_last (bool): Whether to drop the last incomplete batch in epoch.
Default: False
pin_memory (bool): Whether to use pin_memory in DataLoader.
Default: True
persistent_workers (bool): If True, the data loader will not shutdown
the worker processes after a dataset has been consumed once.
This allows to maintain the workers Dataset instances alive.
The argument also has effect in PyTorch>=1.8.0.
Default: False
kwargs (dict, optional): Any keyword argument to be used to initialize
DataLoader.
Returns:
DataLoader: A PyTorch dataloader.
"""
rank, world_size = get_dist_info()
sample_by_class = getattr(dataset, 'sample_by_class', False)
if dist:
if sample_by_class:
dynamic_length = getattr(dataset, 'dynamic_length', True)
sampler = ClassSpecificDistributedSampler(
dataset,
world_size,
rank,
dynamic_length=dynamic_length,
shuffle=shuffle,
seed=seed)
else:
sampler = DistributedSampler(
dataset, world_size, rank, shuffle=shuffle, seed=seed)
shuffle = False
batch_size = videos_per_gpu
num_workers = workers_per_gpu
else:
sampler = None
batch_size = num_gpus * videos_per_gpu
num_workers = num_gpus * workers_per_gpu
init_fn = partial(
worker_init_fn, num_workers=num_workers, rank=rank,
seed=seed) if seed is not None else None
if digit_version(torch.__version__) >= digit_version('1.8.0'):
kwargs['persistent_workers'] = persistent_workers
data_loader = DataLoader(
dataset,
batch_size=batch_size,
sampler=sampler,
num_workers=num_workers,
collate_fn=partial(collate, samples_per_gpu=videos_per_gpu),
pin_memory=pin_memory,
shuffle=shuffle,
worker_init_fn=init_fn,
drop_last=drop_last,
**kwargs)
return data_loader
def test_digit_version():
assert digit_version('0.2.16') == (0, 2, 16, 0, 0, 0)
assert digit_version('1.2.3') == (1, 2, 3, 0, 0, 0)
assert digit_version('1.2.3rc0') == (1, 2, 3, 0, -1, 0)
assert digit_version('1.2.3rc1') == (1, 2, 3, 0, -1, 1)
assert digit_version('1.0rc0') == (1, 0, 0, 0, -1, 0)
assert digit_version('1.0') == digit_version('1.0.0')
assert digit_version('1.5.0+cuda90_cudnn7.6.3_lms') == digit_version('1.5')
assert digit_version('1.0.0dev') < digit_version('1.0.0a')
assert digit_version('1.0.0a') < digit_version('1.0.0a1')
assert digit_version('1.0.0a') < digit_version('1.0.0b')
assert digit_version('1.0.0b') < digit_version('1.0.0rc')
assert digit_version('1.0.0rc1') < digit_version('1.0.0')
assert digit_version('1.0.0') < digit_version('1.0.0post')
assert digit_version('1.0.0post') < digit_version('1.0.0post1')
assert digit_version('v1') == (1, 0, 0, 0, 0, 0)
assert digit_version('v1.1.5') == (1, 1, 5, 0, 0, 0)
with pytest.raises(AssertionError):
digit_version('a')
with pytest.raises(AssertionError):
digit_version('1x')
with pytest.raises(AssertionError):
digit_version('1.x')
try:
from mmcv.utils import digit_version
except ImportError:
def digit_version(version_str):
digit_ver = []
for x in version_str.split('.'):
if x.isdigit():
digit_ver.append(int(x))
elif x.find('rc') != -1:
patch_version = x.split('rc')
digit_ver.append(int(patch_version[0]) - 1)
digit_ver.append(int(patch_version[1]))
return digit_ver
MMCV_MIN = '1.3'
MMCV_MAX = '1.5'
mmcv_min_version = digit_version(MMCV_MIN)
mmcv_max_version = digit_version(MMCV_MAX)
mmcv_version = digit_version(mmcv.__version__)
assert (mmcv_min_version <= mmcv_version <= mmcv_max_version), \
f'MMCV=={mmcv.__version__} is used but incompatible. ' \
f'Please install mmcv>={mmcv_min_version}, <={mmcv_max_version}.'
__all__ = ['__version__', 'version_info']
def test_conv_module():
with pytest.raises(AssertionError):
# conv_cfg must be a dict or None
conv_cfg = 'conv'
ConvModule(3, 8, 2, conv_cfg=conv_cfg)
with pytest.raises(AssertionError):
# norm_cfg must be a dict or None
norm_cfg = 'norm'
ConvModule(3, 8, 2, norm_cfg=norm_cfg)
with pytest.raises(KeyError):
# softmax is not supported
act_cfg = dict(type='softmax')
ConvModule(3, 8, 2, act_cfg=act_cfg)
# conv + norm + act
conv = ConvModule(3, 8, 2, norm_cfg=dict(type='BN'))
assert conv.with_activation
assert hasattr(conv, 'activate')
assert conv.with_norm
assert hasattr(conv, 'norm')
x = torch.rand(1, 3, 256, 256)
output = conv(x)
assert output.shape == (1, 8, 255, 255)
# conv + act
conv = ConvModule(3, 8, 2)
assert conv.with_activation
assert hasattr(conv, 'activate')
assert not conv.with_norm
assert conv.norm is None
x = torch.rand(1, 3, 256, 256)
output = conv(x)
assert output.shape == (1, 8, 255, 255)
# conv
conv = ConvModule(3, 8, 2, act_cfg=None)
assert not conv.with_norm
assert conv.norm is None
assert not conv.with_activation
assert not hasattr(conv, 'activate')
x = torch.rand(1, 3, 256, 256)
output = conv(x)
assert output.shape == (1, 8, 255, 255)
# conv with its own `init_weights` method
conv_module = ConvModule(3,
8,
2,
conv_cfg=dict(type='ExampleConv'),
act_cfg=None)
assert torch.equal(conv_module.conv.conv0.weight, torch.zeros(8, 3, 2, 2))
# with_spectral_norm=True
conv = ConvModule(3, 8, 3, padding=1, with_spectral_norm=True)
assert hasattr(conv.conv, 'weight_orig')
output = conv(x)
assert output.shape == (1, 8, 256, 256)
# padding_mode='reflect'
conv = ConvModule(3, 8, 3, padding=1, padding_mode='reflect')
assert isinstance(conv.padding_layer, nn.ReflectionPad2d)
output = conv(x)
assert output.shape == (1, 8, 256, 256)
# non-existing padding mode
with pytest.raises(KeyError):
conv = ConvModule(3, 8, 3, padding=1, padding_mode='non_exists')
# leaky relu
conv = ConvModule(3, 8, 3, padding=1, act_cfg=dict(type='LeakyReLU'))
assert isinstance(conv.activate, nn.LeakyReLU)
output = conv(x)
assert output.shape == (1, 8, 256, 256)
# tanh
conv = ConvModule(3, 8, 3, padding=1, act_cfg=dict(type='Tanh'))
assert isinstance(conv.activate, nn.Tanh)
output = conv(x)
assert output.shape == (1, 8, 256, 256)
# Sigmoid
conv = ConvModule(3, 8, 3, padding=1, act_cfg=dict(type='Sigmoid'))
assert isinstance(conv.activate, nn.Sigmoid)
output = conv(x)
assert output.shape == (1, 8, 256, 256)
# PReLU
conv = ConvModule(3, 8, 3, padding=1, act_cfg=dict(type='PReLU'))
assert isinstance(conv.activate, nn.PReLU)
output = conv(x)
assert output.shape == (1, 8, 256, 256)
# HSwish
conv = ConvModule(3, 8, 3, padding=1, act_cfg=dict(type='HSwish'))
if (TORCH_VERSION == 'parrots'
or digit_version(TORCH_VERSION) < digit_version('1.7')):
assert isinstance(conv.activate, HSwish)
else:
assert isinstance(conv.activate, nn.Hardswish)
output = conv(x)
assert output.shape == (1, 8, 256, 256)
# HSigmoid
conv = ConvModule(3, 8, 3, padding=1, act_cfg=dict(type='HSigmoid'))
assert isinstance(conv.activate, HSigmoid)
output = conv(x)
assert output.shape == (1, 8, 256, 256)
def build_dataloader(dataset,
samples_per_gpu,
workers_per_gpu,
num_gpus=1,
dist=True,
shuffle=True,
round_up=True,
seed=None,
pin_memory=True,
persistent_workers=True,
sampler_cfg=None,
**kwargs):
"""Build PyTorch DataLoader.
In distributed training, each GPU/process has a dataloader.
In non-distributed training, there is only one dataloader for all GPUs.
Args:
dataset (Dataset): A PyTorch dataset.
samples_per_gpu (int): Number of training samples on each GPU, i.e.,
batch size of each GPU.
workers_per_gpu (int): How many subprocesses to use for data loading
for each GPU.
num_gpus (int): Number of GPUs. Only used in non-distributed training.
dist (bool): Distributed training/test or not. Default: True.
shuffle (bool): Whether to shuffle the data at every epoch.
Default: True.
round_up (bool): Whether to round up the length of dataset by adding
extra samples to make it evenly divisible. Default: True.
pin_memory (bool): Whether to use pin_memory in DataLoader.
Default: True
persistent_workers (bool): If True, the data loader will not shutdown
the worker processes after a dataset has been consumed once.
This allows to maintain the workers Dataset instances alive.
The argument also has effect in PyTorch>=1.7.0.
Default: True
sampler_cfg (dict): sampler configuration to override the default
sampler
kwargs: any keyword argument to be used to initialize DataLoader
Returns:
DataLoader: A PyTorch dataloader.
"""
rank, world_size = get_dist_info()
# Custom sampler logic
if sampler_cfg:
# shuffle=False when val and test
sampler_cfg.update(shuffle=shuffle)
sampler = build_sampler(sampler_cfg,
default_args=dict(dataset=dataset,
num_replicas=world_size,
rank=rank))
# Default sampler logic
elif dist:
sampler = build_sampler(
dict(type='DistributedSampler',
dataset=dataset,
num_replicas=world_size,
rank=rank,
shuffle=shuffle,
round_up=round_up))
else:
sampler = None
# If sampler exists, turn off dataloader shuffle
if sampler is not None:
shuffle = False
if dist:
batch_size = samples_per_gpu
num_workers = workers_per_gpu
else:
batch_size = num_gpus * samples_per_gpu
num_workers = num_gpus * workers_per_gpu
init_fn = partial(
worker_init_fn, num_workers=num_workers, rank=rank,
seed=seed) if seed is not None else None
if digit_version(torch.__version__) >= digit_version('1.8.0'):
kwargs['persistent_workers'] = persistent_workers
data_loader = DataLoader(dataset,
batch_size=batch_size,
sampler=sampler,
num_workers=num_workers,
collate_fn=partial(
collate, samples_per_gpu=samples_per_gpu),
pin_memory=pin_memory,
shuffle=shuffle,
worker_init_fn=init_fn,
**kwargs)
return data_loader
# Copyright (c) OpenMMLab. All rights reserved.
import os
import random
import numpy as np
import torch
from mmcv.runner import set_random_seed
from mmcv.utils import TORCH_VERSION, digit_version
is_rocm_pytorch = False
if digit_version(TORCH_VERSION) >= digit_version('1.5'):
from torch.utils.cpp_extension import ROCM_HOME
is_rocm_pytorch = True if ((torch.version.hip is not None) and
(ROCM_HOME is not None)) else False
def test_set_random_seed():
set_random_seed(0)
a_random = random.randint(0, 10)
a_np_random = np.random.rand(2, 2)
a_torch_random = torch.rand(2, 2)
assert torch.backends.cudnn.deterministic is False
assert torch.backends.cudnn.benchmark is False
assert os.environ['PYTHONHASHSEED'] == str(0)
set_random_seed(0, True)
b_random = random.randint(0, 10)
b_np_random = np.random.rand(2, 2)
b_torch_random = torch.rand(2, 2)
assert torch.backends.cudnn.deterministic is True
# Copyright (c) OpenMMLab. All rights reserved.
import numpy as np
import pytest
from mmcv.utils import assert_dict_has_keys, digit_version
try:
import torch
from mmaction.datasets.pipelines import PytorchVideoTrans
pytorchvideo_ok = False
if digit_version(torch.__version__) >= digit_version('1.8.0'):
pytorchvideo_ok = True
except (ImportError, ModuleNotFoundError):
pytorchvideo_ok = False
@pytest.mark.skipif(not pytorchvideo_ok, reason='torch >= 1.8.0 is required')
class TestPytorchVideoTrans:
@staticmethod
def test_pytorchvideo_trans():
with pytest.raises(AssertionError):
# transforms not supported in pytorchvideo
PytorchVideoTrans(type='BlaBla')
with pytest.raises(AssertionError):
# This trans exists in pytorchvideo but not supported in MMAction2
PytorchVideoTrans(type='MixUp')
target_keys = ['imgs']
imgs = list(np.random.randint(0, 256, (4, 32, 32, 3)).astype(np.uint8))
def __init__(self,
arch='768/32',
in_channels=3,
norm_cfg=dict(type='BN'),
act_cfg=dict(type='GELU'),
out_indices=-1,
frozen_stages=0,
init_cfg=None):
super().__init__(init_cfg=init_cfg)
if isinstance(arch, str):
assert arch in self.arch_settings, \
f'Unavailable arch, please choose from ' \
f'({set(self.arch_settings)}) or pass a dict.'
arch = self.arch_settings[arch]
elif isinstance(arch, dict):
essential_keys = {
'embed_dims', 'depth', 'patch_size', 'kernel_size'
}
assert isinstance(arch, dict) and essential_keys <= set(arch), \
f'Custom arch needs a dict with keys {essential_keys}'
self.embed_dims = arch['embed_dims']
self.depth = arch['depth']
self.patch_size = arch['patch_size']
self.kernel_size = arch['kernel_size']
self.act = build_activation_layer(act_cfg)
# check out indices and frozen stages
if isinstance(out_indices, int):
out_indices = [out_indices]
assert isinstance(out_indices, Sequence), \
f'"out_indices" must by a sequence or int, ' \
f'get {type(out_indices)} instead.'
for i, index in enumerate(out_indices):
if index < 0:
out_indices[i] = self.depth + index
assert out_indices[i] >= 0, f'Invalid out_indices {index}'
self.out_indices = out_indices
self.frozen_stages = frozen_stages
# Set stem layers
self.stem = nn.Sequential(
nn.Conv2d(in_channels,
self.embed_dims,
kernel_size=self.patch_size,
stride=self.patch_size), self.act,
build_norm_layer(norm_cfg, self.embed_dims)[1])
# Set conv2d according to torch version
convfunc = nn.Conv2d
if digit_version(torch.__version__) < digit_version('1.9.0'):
convfunc = Conv2dAdaptivePadding
# Repetitions of ConvMixer Layer
self.stages = nn.Sequential(*[
nn.Sequential(
Residual(
nn.Sequential(
convfunc(self.embed_dims,
self.embed_dims,
self.kernel_size,
groups=self.embed_dims,
padding='same'), self.act,
build_norm_layer(norm_cfg, self.embed_dims)[1])),
nn.Conv2d(self.embed_dims, self.embed_dims, kernel_size=1),
self.act,
build_norm_layer(norm_cfg, self.embed_dims)[1])
for _ in range(self.depth)
])
self._freeze_stages()
- Output: :math:`(N, *)`, same shape as the input
.. image:: scripts/activation_images/GELU.png
Examples::
>>> m = nn.GELU()
>>> input = torch.randn(2)
>>> output = m(input)
"""
def forward(self, input):
return F.gelu(input)
if (TORCH_VERSION == 'parrots'
or digit_version(TORCH_VERSION) < digit_version('1.4')):
ACTIVATION_LAYERS.register_module(module=GELU)
else:
ACTIVATION_LAYERS.register_module(module=nn.GELU)
def build_activation_layer(cfg):
"""Build activation layer.
Args:
cfg (dict): The activation layer config, which should contain:
- type (str): Layer type.
- layer args: Args needed to instantiate an activation layer.
Returns:
