def test020_speed(self, b_size=32, dim=256,
alpha_fwd=0.999, alpha_bkw=0.99, eps=1e-05, epoch=10):
"""
Speed test online norm linearized vs loop
Note: this test is for comparing speed to looped and linearized
implementations of online norm so the user can decide which
algorithm to use.
"""
input = torch.randn(b_size, dim, 32, 32)
# instantiate Linearized Online Norm class
onlin = OnlineNorm2D(dim, alpha_fwd=alpha_fwd, alpha_bkw=alpha_bkw, eps=eps, b_size=b_size)
# time lin algo
forward = 0
backward = 0
for _ in range(epoch):
start = time.time()
# fprop through lin algo
out = onlin(input)
forward += time.time() - start
start = time.time()
# bprop through lin algo
out.sum().backward()
backward += time.time() - start
self.logger.info(f'Linearized Control Normalization Speed Test: '
f'Forward {forward * 1e6/1e5:.3f} us | '
f'Backward {backward * 1e6/1e5:.3f} us | '
f'Total {(forward + backward) * 1e6/1e5:.3f} us')
# Speed test online norm
# instantiate Looping Online Norm class
onloop = OnlineNorm2D(dim, eps=eps,
ctrl_norm=ControlNorm2DLoop(dim, alpha_fwd=alpha_fwd,
alpha_bkw=alpha_bkw, eps=eps))
# time loop algo
forward = 0
backward = 0
for _ in range(epoch):
start = time.time()
# fprop through loop algo
out = onloop(input)
forward += time.time() - start
start = time.time()
# bprop through loop algo
out.sum().backward()
backward += time.time() - start
self.logger.info(f'Loop Control Normalization Speed Test: '
f'Forward {forward * 1e6/1e5:.3f} us | '
f'Backward {backward * 1e6/1e5:.3f} us | '
f'Total {(forward + backward) * 1e6/1e5:.3f} us')
self.logger.info('Make input tensors representative of size you will '
'use and then use the correct algorithm based on '
'speed of execution.')
def test010_similarity(self,
b_size=4,
dim=256,
alpha_fwd=0.999,
alpha_bkw=0.99,
eps=1e-05,
itrs=4):
""" numerical comparison of online norm linearized vs loop """
# instantiate inputs
input = torch.randn(b_size, dim, 32, 32)
input_0 = input.clone().detach().requires_grad_(True)
input_1 = input.clone().detach().requires_grad_(True)
# instantiate gradient at the output
grad_out = torch.randn(b_size, dim, 32, 32)
# instantiate Linearized Online Norm class
onlin = OnlineNorm2D(dim,
alpha_fwd=alpha_fwd,
alpha_bkw=alpha_bkw,
eps=eps,
b_size=b_size)
# instantiate Looping Online Norm class
onloop = OnlineNorm2D(dim,
eps=eps,
ctrl_norm=ControlNorm2DLoop(dim,
alpha_fwd=alpha_fwd,
alpha_bkw=alpha_bkw,
eps=eps))
for _ in range(itrs):
# fprop through Linearized Online Norm class
y_0 = onlin(input_0)
# bprop through Linearized Online Norm class
y_0.backward(grad_out)
# fprop through Looping Online Norm class
y_1 = onloop(input_1)
# bprop through Looping Online Norm class
y_1.backward(grad_out)
# numerically compare output
np.testing.assert_allclose(y_0.detach().numpy(),
y_1.detach().numpy(),
rtol=1e-4,
atol=1e-5)
# numerically grad_in
np.testing.assert_allclose(input_0.grad.detach().numpy(),
input_1.grad.detach().numpy(),
rtol=1e-4,
atol=1e-5)
self.logger.info('Algorithm implemented using linearization of ops '
'numerically matches algorithm implemented with '
'loops')
def online_norm(num_features,
afwd=0.999,
abkw=0.99,
eps=1e-05,
weight=True,
bias=True,
ctrl_norm=None,
layer_scaling=True,
b_size=args.batch_size,
**kwargs):
""" Function which instantiates Online Norm Layer """
return OnlineNorm2D(num_features,
alpha_fwd=afwd,
alpha_bkw=abkw,
eps=eps,
weight=weight,
bias=bias,
ctrl_norm=ctrl_norm,
layer_scaling=layer_scaling,
b_size=b_size,
**kwargs)
def online_norm(num_features, alpha_fwd=0.999, alpha_bkw=0.99,
eps=1e-05, affine=True, ecm='ls', **kwargs):
""" Function which instantiates Online Norm Layer """
return OnlineNorm2D(num_features, alpha_fwd=alpha_fwd, alpha_bkw=alpha_bkw,
eps=eps, affine=affine, ecm=ecm, **kwargs)
def norm(num_features,
mode='batch',
eps=1e-05,
momentum=0.1,
weight=True,
bias=True,
track_running_stats=True,
gn_num_groups=32,
**kwargs):
"""
Function which instantiates a normalization scheme based on mode
Arguments:
num_features: :math:`C` from an expected input of size
:math:`(N, C, H, W)`
mode: Option to select normalization method (Default: None)
eps: a value added to the denominator for numerical stability.
Default: 1e-5
momentum: the value used for the running_mean and running_var
computation. Can be set to ``None`` for cumulative moving average
(i.e. simple average). Default: 0.1
weight: a boolean value that when set to ``True``, this module has
learnable linear parameters. Default: ``True``
bias: a boolean value that when set to ``True``, this module has
learnable bias parameters. Default: ``True``
track_running_stats: a boolean value that when set to ``True``, this
module tracks the running mean and variance, and when set to ``False``,
this module does not track such statistics and always uses batch
statistics in both training and eval modes. Argument valid when
using batch or instance norm. Default: ``True``
Note:
1. When using BN affine = weight & bias
2. When using OnlineNorm **kwargs will hold hfwd, hbkw, ctrl_norm
layer_scaling, and b_size. See definition of OnlineNorm2D for
specifics.
"""
if mode == 'batch':
warnings.warn('Normalizer: Batch')
affine = weight and bias
if weight != bias:
warnings.warn('affine not used in batch norm')
normalizer = nn.BatchNorm2d(num_features=num_features,
eps=eps,
momentum=momentum,
affine=affine,
track_running_stats=track_running_stats)
elif mode == 'group':
warnings.warn('Normalizer: Group')
affine = weight and bias
if weight != bias:
warnings.warn('affine not used in group norm')
normalizer = nn.GroupNorm(gn_num_groups,
num_features,
eps=eps,
affine=affine)
elif mode == 'layer':
warnings.warn('Normalizer: Layer')
normalizer = LayerNorm2d(eps=eps, weight=weight, bias=bias)
elif mode == 'instance':
warnings.warn('Normalizer: Instance')
affine = weight and bias
if weight != bias:
warnings.warn('affine not used in batch norm')
normalizer = nn.InstanceNorm2d(num_features,
eps=eps,
momentum=momentum,
affine=affine,
track_running_stats=track_running_stats)
elif mode == 'online':
warnings.warn('Normalizer: Online')
normalizer = OnlineNorm2D(num_features=num_features,
eps=eps,
weight=weight,
bias=bias,
**kwargs)
elif mode == 'none' or mode is None:
warnings.warn('Normalizer: None')
normalizer = Identity()
else:
raise KeyError('mode options include: "batch" | "group" | "layer" | '
'"instance" | "online" | "none"')
return normalizer