def gen_block_params(ni, no):
return {
'conv0': utils.conv_params(ni, no, 3),
'conv1': utils.conv_params(no, no, 3),
'bn0': utils.bnparams(ni),
'bn1': utils.bnparams(no),
'convdim': utils.conv_params(ni, no, 1) if ni != no else None,
}
def gen_block_params(ni, no):
return {
'harmonic0': gen_harmonic_params(ni, no, k=3, normalize=False, level=level, linear=True),
'harmonic1': gen_harmonic_params(no, no, k=3, normalize=False, level=level, linear=True),
'bn0': utils.bnparams(ni),
'bn1': utils.bnparams(no),
'convdim': utils.conv_params(ni, no, 1) if ni != no else None,
}
def gen_block_params(ni, no, scalar):
if scalar:
return {
'bn0': bnparams(ni),
'bn1': bnparams(no),
}
return {
'conv0': conv_params(ni, no, 3),
'conv1': conv_params(no, no, 3),
'convdim': conv_params(ni, no, 1) if ni != no else None,
}
def resnet(depth, width, num_classes, dropout):
assert (depth - 4) % 6 == 0, 'depth should be 6n+4'
n = (depth - 4) // 6
widths = [int(v * width) for v in (16, 32, 64)]
def gen_block_params(ni, no):
return {
'conv0': utils.conv_params(ni, no, 3),
'conv1': utils.conv_params(no, no, 3),
'bn0': utils.bnparams(ni),
'bn1': utils.bnparams(no),
'convdim': utils.conv_params(ni, no, 1) if ni != no else None,
}
def gen_group_params(ni, no, count):
return {'block%d' % i: gen_block_params(ni if i == 0 else no, no)
for i in range(count)}
flat_params = utils.cast(utils.flatten({
'conv0': utils.conv_params(3, 16, 3),
'group0': gen_group_params(16, widths[0], n),
'group1': gen_group_params(widths[0], widths[1], n),
'group2': gen_group_params(widths[1], widths[2], n),
'bn': utils.bnparams(widths[2]),
'fc': utils.linear_params(widths[2], num_classes),
}))
utils.set_requires_grad_except_bn_(flat_params)
def block(x, params, base, mode, stride):
o1 = F.relu(utils.batch_norm(x, params, base + '.bn0', mode), inplace=True)
y = F.conv2d(o1, params[base + '.conv0'], stride=stride, padding=1)
o2 = F.relu(utils.batch_norm(y, params, base + '.bn1', mode), inplace=True)
if dropout > 0:
o2 = F.dropout(o2, p=dropout, training=mode, inplace=False)
z = F.conv2d(o2, params[base + '.conv1'], stride=1, padding=1)
if base + '.convdim' in params:
return z + F.conv2d(o1, params[base + '.convdim'], stride=stride)
else:
return z + x
def group(o, params, base, mode, stride):
for i in range(n):
o = block(o, params, '%s.block%d' % (base,i), mode, stride if i == 0 else 1)
return o
def f(input, params, mode):
x = F.conv2d(input, params['conv0'], stride=2, padding=1)
g0 = group(x, params, 'group0', mode, 1)
g1 = group(g0, params, 'group1', mode, 2)
g2 = group(g1, params, 'group2', mode, 2)
o = F.relu(utils.batch_norm(g2, params, 'bn', mode))
o = F.avg_pool2d(o, 12, 1, 0)
o = o.view(o.size(0), -1)
o = F.linear(o, params['fc.weight'], params['fc.bias'])
return o
return f, flat_params
def resnet(depth, width, num_classes):
assert (depth - 4) % 6 == 0, 'depth should be 6n+4'
n = (depth - 4) // 6
widths = [int(v * width) for v in (16, 32, 64)]
def gen_block_params(ni, no):
return {
'conv0': utils.conv_params(ni, no, 3),
'conv1': utils.conv_params(no, no, 3),
'bn0': utils.bnparams(ni),
'bn1': utils.bnparams(no),
'convdim': utils.conv_params(ni, no, 1) if ni != no else None,
}
def gen_group_params(ni, no, count):
return {'block%d' % i: gen_block_params(ni if i == 0 else no, no)
for i in range(count)}
flat_params = utils.cast(utils.flatten({
'conv0': utils.conv_params(3, 16, 3),
'group0': gen_group_params(16, widths[0], n),
'group1': gen_group_params(widths[0], widths[1], n),
'group2': gen_group_params(widths[1], widths[2], n),
'bn': utils.bnparams(widths[2]),
'fc': utils.linear_params(widths[2], num_classes),
}))
utils.set_requires_grad_except_bn_(flat_params)
def block(x, params, base, mode, stride):
o1 = F.relu(utils.batch_norm(x, params, base + '.bn0', mode), inplace=True)
y = F.conv2d(o1, params[base + '.conv0'], stride=stride, padding=1)
o2 = F.relu(utils.batch_norm(y, params, base + '.bn1', mode), inplace=True)
z = F.conv2d(o2, params[base + '.conv1'], stride=1, padding=1)
if base + '.convdim' in params:
return z + F.conv2d(o1, params[base + '.convdim'], stride=stride)
else:
return z + x
def group(o, params, base, mode, stride):
for i in range(n):
o = block(o, params, '%s.block%d' % (base,i), mode, stride if i == 0 else 1)
return o
def f(input, params, mode):
x = F.conv2d(input, params['conv0'], padding=1)
g0 = group(x, params, 'group0', mode, 1)
g1 = group(g0, params, 'group1', mode, 2)
g2 = group(g1, params, 'group2', mode, 2)
o = F.relu(utils.batch_norm(g2, params, 'bn', mode))
o = F.avg_pool2d(o, 8, 1, 0)
o = o.view(o.size(0), -1)
o = F.linear(o, params['fc.weight'], params['fc.bias'])
return o
return f, flat_params
def gen_feature_params():
in_channels = 3
dic = {}
for i in range(len(_cfg)):
if not _cfg[i] == 'M':
dic['conv{0}'.format(i)] = conv_params(in_channels, _cfg[i], 3)
dic['bn{0}'.format(i)] = bnparams(_cfg[i])
in_channels = _cfg[i]
return dic
def resnet(depth, width, num_classes, dropout_prob, activation_dropout):
assert (depth - 4) % 6 == 0, 'depth should be 6n+4'
n = (depth - 4) // 6
widths = [int(v * width) for v in (16, 32, 64)]
def gen_block_params(ni, no):
return {
'conv0': utils.conv_params(ni, no, 3),
'conv1': utils.conv_params(no, no, 3),
'bn0': utils.bnparams(ni),
'bn1': utils.bnparams(no),
'convdim': utils.conv_params(ni, no, 1) if ni != no else None,
}
def gen_group_params(ni, no, count):
return {
'block%d' % i: gen_block_params(ni if i == 0 else no, no)
for i in range(count)
}
flat_params = utils.cast(
utils.flatten({
'conv0': utils.conv_params(3, 16, 3),
'group0': gen_group_params(16, widths[0], n),
'group1': gen_group_params(widths[0], widths[1], n),
'group2': gen_group_params(widths[1], widths[2], n),
'bn': utils.bnparams(widths[2]),
'fc': utils.linear_params(widths[2], num_classes),
}))
utils.set_requires_grad_except_bn_(flat_params)
def activation_dropout(x, p_drop, training):
if training:
# the input P. is the base DROPOUT PROBABILITY
P = 1. - p_drop
# x.size() = [bs, f , h, w]
#sum over w and h to get total activation of a filter across space -> [bs, f]
# normalize feature activations to 1 for each example in the batch
bs, N, w, h = x.size()
p_act = F.normalize(x.sum(-1).sum(-1), p=1, dim=-1)
p_retain = 1. - ((1. - P) * (N - 1.) * p_act) / ((
(1. - P) * N - 1.) * p_act + P)
mask = torch.bernoulli(p_retain)
scale = mask.mean(-1)
mask = mask / torch.stack([scale for i in range(N)], -1)
mask = torch.stack([mask for i in range(w)], -1)
mask = torch.stack([mask for i in range(h)], -1)
return mask * x
else:
return x
def block(x, params, base, mode, stride):
o1 = F.relu(utils.batch_norm(x, params, base + '.bn0', mode),
inplace=True)
y = F.conv2d(o1, params[base + '.conv0'], stride=stride, padding=1)
o2 = F.relu(utils.batch_norm(y, params, base + '.bn1', mode),
inplace=True)
if activation_dropout:
o2 = activation_dropout(o2, p_drop=dropout_prob, training=mode)
elif dropout_prob:
o2 = F.dropout2d(o2, p=dropout_prob, training=mode)
z = F.conv2d(o2, params[base + '.conv1'], stride=1, padding=1)
if base + '.convdim' in params:
return z + F.conv2d(o1, params[base + '.convdim'], stride=stride)
else:
return z + x
def group(o, params, base, mode, stride):
for i in range(n):
o = block(o, params, '%s.block%d' % (base, i), mode,
stride if i == 0 else 1)
return o
def f(input, params, mode):
x = F.conv2d(input, params['conv0'], padding=1)
g0 = group(x, params, 'group0', mode, 1)
g1 = group(g0, params, 'group1', mode, 2)
g2 = group(g1, params, 'group2', mode, 2)
o = F.relu(utils.batch_norm(g2, params, 'bn', mode))
o = F.avg_pool2d(o, 8, 1, 0)
o = o.view(o.size(0), -1)
o = F.linear(o, params['fc.weight'], params['fc.bias'])
return o
return f, flat_params
def resnet(depth, width, num_classes):
assert (depth - 4) % 6 == 0, 'depth should be 6n+4'
n = (depth - 4) // 6
widths = [int(x * width) for x in [16, 32, 64]]
def gen_block_params(ni, no):
return {
'conv0': conv_params(ni, no, 3),
'conv1': conv_params(no, no, 3),
'bn0': bnparams(ni),
'bn1': bnparams(no),
'convdim': conv_params(ni, no, 1) if ni != no else None,
}
def gen_group_params(ni, no, count):
return {'block%d' % i: gen_block_params(ni if i == 0 else no, no)
for i in range(count)}
def gen_group_stats(ni, no, count):
return {'block%d' % i: {'bn0': bnstats(ni if i == 0 else no), 'bn1': bnstats(no)}
for i in range(count)}
flat_params = flatten_params({
'conv0': conv_params(3,16,3),
'group0': gen_group_params(16, widths[0], n),
'group1': gen_group_params(widths[0], widths[1], n),
'group2': gen_group_params(widths[1], widths[2], n),
'bn': bnparams(widths[2]),
'fc': linear_params(widths[2], num_classes),
})
flat_stats = flatten_stats({
'group0': gen_group_stats(16, widths[0], n),
'group1': gen_group_stats(widths[0], widths[1], n),
'group2': gen_group_stats(widths[1], widths[2], n),
'bn': bnstats(widths[2]),
})
def block(x, params, stats, base, mode, stride):
o1 = F.relu(batch_norm(x, params, stats, base + '.bn0', mode), inplace=True)
y = F.conv2d(o1, params[base + '.conv0'], stride=stride, padding=1)
o2 = F.relu(batch_norm(y, params, stats, base + '.bn1', mode), inplace=True)
z = F.conv2d(o2, params[base + '.conv1'], stride=1, padding=1)
if base + '.convdim' in params:
return z + F.conv2d(o1, params[base + '.convdim'], stride=stride)
else:
return z + x
def group(o, params, stats, base, mode, stride):
for i in range(n):
o = block(o, params, stats, '%s.block%d' % (base,i), mode, stride if i == 0 else 1)
return o
def f(input, params, stats, mode):
x = F.conv2d(input, params['conv0'], padding=1)
g0 = group(x, params, stats, 'group0', mode, 1)
g1 = group(g0, params, stats, 'group1', mode, 2)
g2 = group(g1, params, stats, 'group2', mode, 2)
o = F.relu(batch_norm(g2, params, stats, 'bn', mode))
o = F.avg_pool2d(o, 8, 1, 0)
o = o.view(o.size(0), -1)
o = F.linear(o, params['fc.weight'], params['fc.bias'])
return o
return f, flat_params, flat_stats
def resnet(depth, width, num_classes, stu_depth=0):
assert (depth - 4) % 6 == 0, 'depth should be 6n+4'
n = (depth - 4) // 6
if stu_depth != 0:
assert (stu_depth - 4) % 6 == 0, 'student depth should be 6n+4'
n_s = (stu_depth - 4) // 6
else:
n_s = 0
widths = torch.Tensor([16, 32, 64]).mul(width).int()
def gen_block_params(ni, no):
return {
'conv0': conv_params(ni, no, 3),
'conv1': conv_params(no, no, 3),
'bn0': bnparams(ni),
'bn1': bnparams(no),
'bns0': bnparams(ni),
'bns1': bnparams(no),
'convdim': conv_params(ni, no, 1) if ni != no else None,
}
def gen_group_params(ni, no, count):
return {
'block%d' % i: gen_block_params(ni if i == 0 else no, no)
for i in range(count)
}
def gen_group_stats(ni, no, count):
return {
'block%d' % i: {
'bn0': bnstats(ni if i == 0 else no),
'bn1': bnstats(no),
'bns0': bnstats(ni if i == 0 else no),
'bns1': bnstats(no)
}
for i in range(count)
}
if stu_depth != 0 and not opt.param_share:
params = {
'conv0': conv_params(3, 16, 3),
'group0': gen_group_params(16, widths[0], n),
'group1': gen_group_params(widths[0], widths[1], n),
'group2': gen_group_params(widths[1], widths[2], n),
'groups0': gen_group_params(16, widths[0], n_s),
'groups1': gen_group_params(widths[0], widths[1], n_s),
'groups2': gen_group_params(widths[1], widths[2], n_s),
'bn': bnparams(widths[2]),
'bns': bnparams(widths[2]),
'fc': linear_params(widths[2], num_classes),
'fcs': linear_params(widths[2], num_classes),
}
stats = {
'group0': gen_group_stats(16, widths[0], n),
'group1': gen_group_stats(widths[0], widths[1], n),
'group2': gen_group_stats(widths[1], widths[2], n),
'groups0': gen_group_stats(16, widths[0], n_s),
'groups1': gen_group_stats(widths[0], widths[1], n_s),
'groups2': gen_group_stats(widths[1], widths[2], n_s),
'bn': bnstats(widths[2]),
'bns': bnstats(widths[2]),
}
else:
params = {
'conv0': conv_params(3, 16, 3),
'group0': gen_group_params(16, widths[0], n),
'group1': gen_group_params(widths[0], widths[1], n),
'group2': gen_group_params(widths[1], widths[2], n),
'bn': bnparams(widths[2]),
'bns': bnparams(widths[2]),
'fc': linear_params(widths[2], num_classes),
'fcs': linear_params(widths[2], num_classes),
}
stats = {
'group0': gen_group_stats(16, widths[0], n),
'group1': gen_group_stats(widths[0], widths[1], n),
'group2': gen_group_stats(widths[1], widths[2], n),
'bn': bnstats(widths[2]),
'bns': bnstats(widths[2]),
}
flat_params = flatten_params(params)
flat_stats = flatten_stats(stats)
def block(x, params, stats, base, mode, stride, flag, drop_switch=True):
if flag == 's':
o1 = F.relu(batch_norm(x, params, stats, base + '.bns0', mode))
y = F.conv2d(o1, params[base + '.conv0'], stride=stride, padding=1)
o2 = F.relu(batch_norm(y, params, stats, base + '.bns1', mode))
z = F.conv2d(o2, params[base + '.conv1'], stride=1, padding=1)
if base + '.convdim' in params:
return z + F.conv2d(
o1, params[base + '.convdim'], stride=stride)
else:
return z + x
o1 = F.relu(batch_norm(x, params, stats, base + '.bn0', mode))
y = F.conv2d(o1, params[base + '.conv0'], stride=stride, padding=1)
o2 = F.relu(batch_norm(y, params, stats, base + '.bn1', mode))
if opt.dropout > 0 and drop_switch:
o2 = F.dropout(o2, p=opt.dropout, training=mode)
z = F.conv2d(o2, params[base + '.conv1'], stride=1, padding=1)
if base + '.convdim' in params:
return z + F.conv2d(o1, params[base + '.convdim'], stride=stride)
else:
return z + x
def group(o, params, stats, base, mode, stride):
for i in range(n):
o = block(o, params, stats, '%s.block%d' % (base, i), mode,
stride if i == 0 else 1, 't', False)
return o
def group_student(o, params, stats, base, mode, stride, n_layer):
for i in range(n_layer):
o = block(o, params, stats, '%s.block%d' % (base, i), mode,
stride if i == 0 else 1, 's', False)
return o
def f(input, params, stats, mode, prefix=''):
x = F.conv2d(input, params[prefix + 'conv0'], padding=1)
g0 = group(x, params, stats, prefix + 'group0', mode, 1)
g1 = group(g0, params, stats, prefix + 'group1', mode, 2)
g2 = group(g1, params, stats, prefix + 'group2', mode, 2)
o = F.relu(batch_norm(g2, params, stats, prefix + 'bn', mode))
o = F.avg_pool2d(o, 8, 1, 0)
o = o.view(o.size(0), -1)
o = F.linear(o, params[prefix + 'fc.weight'],
params[prefix + 'fc.bias'])
#x_s = F.conv2d(input, params[prefix+'conv0_s'], padding=1)
if stu_depth != 0:
if opt.param_share:
gs0 = group_student(x, params, stats, prefix + 'group0', mode,
1, n_s)
gs1 = group_student(gs0, params, stats, prefix + 'group1',
mode, 2, n_s)
gs2 = group_student(gs1, params, stats, prefix + 'group2',
mode, 2, n_s)
else:
gs0 = group_student(x, params, stats, prefix + 'groups0', mode,
1, n_s)
gs1 = group_student(gs0, params, stats, prefix + 'groups1',
mode, 2, n_s)
gs2 = group_student(gs1, params, stats, prefix + 'groups2',
mode, 2, n_s)
os = F.relu(batch_norm(gs2, params, stats, prefix + 'bns', mode))
os = F.avg_pool2d(os, 8, 1, 0)
os = os.view(os.size(0), -1)
os = F.linear(os, params[prefix + 'fcs.weight'],
params[prefix + 'fcs.bias'])
return os, o, [g0, g1, g2, gs0, gs1, gs2]
else:
return o, [g0, g1, g2]
return f, flat_params, flat_stats
def resnet(depth, width, num_classes):
assert (depth - 4) % 6 == 0, 'depth should be 6n+4'
n = (depth - 4) // 6
widths = torch.Tensor([16, 32, 64]).mul(width).int().numpy().tolist()
def gen_block_params(ni, no, scalar):
if scalar:
return {
'bn0': bnparams(ni),
'bn1': bnparams(no),
}
return {
'conv0': conv_params(ni, no, 3),
'conv1': conv_params(no, no, 3),
'convdim': conv_params(ni, no, 1) if ni != no else None,
}
def gen_group_params(ni, no, count, bias=False):
return {
'block%d' % i: gen_block_params(ni if i == 0 else no, no, bias)
for i in range(count)
}
def gen_group_stats(ni, no, count):
return {
'block%d' % i: {
'bn0': bnstats(ni if i == 0 else no),
'bn1': bnstats(no)
}
for i in range(count)
}
flat_vectors = flatten_params({
'conv0':
conv_params(3, 16, 3),
'group0':
gen_group_params(16, widths[0], n),
'group1':
gen_group_params(widths[0], widths[1], n),
'group2':
gen_group_params(widths[1], widths[2], n),
'conv1':
conv_params(widths[2], num_classes, 1),
})
flat_scalars = flatten_params({
'group0':
gen_group_params(16, widths[0], n, True),
'group1':
gen_group_params(widths[0], widths[1], n, True),
'group2':
gen_group_params(widths[1], widths[2], n, True),
'bn':
bnparams(widths[2]),
})
flat_stats = flatten_stats({
'group0':
gen_group_stats(16, widths[0], n),
'group1':
gen_group_stats(widths[0], widths[1], n),
'group2':
gen_group_stats(widths[1], widths[2], n),
'bn':
bnstats(widths[2]),
})
def block(x, params, stats, base, mode, stride):
o1 = F.relu(batch_norm(x, params, stats, base + '.bn0', mode, 1.),
inplace=True)
y = F.conv2d(o1, params[base + '.conv0'], stride=stride, padding=1)
o2 = F.relu(batch_norm(y, params, stats, base + '.bn1', mode, 1.),
inplace=True)
z = F.conv2d(o2, params[base + '.conv1'], stride=1, padding=1)
if base + '.convdim' in params:
return z + F.conv2d(o1, params[base + '.convdim'], stride=stride)
else:
return z + x
def group(o, params, stats, base, mode, stride):
for i in range(n):
o = block(o, params, stats, '%s.block%d' % (base, i), mode,
stride if i == 0 else 1)
return o
def f(input, params, stats, mode):
x = F.conv2d(input, params['conv0'], padding=1)
g0 = group(x, params, stats, 'group0', mode, 1)
g1 = group(g0, params, stats, 'group1', mode, 2)
g2 = group(g1, params, stats, 'group2', mode, 2)
o = F.relu(batch_norm(g2, params, stats, 'bn', mode, 1.))
o = F.conv2d(o, params['conv1'])
o = F.avg_pool2d(o, 8, 1, 0)
o = o.view(o.size(0), -1)
return o
return f, flat_vectors, flat_scalars, flat_stats
def resnet(depth, width, num_classes):
assert (depth - 4) % 6 == 0, 'depth should be 6n+4'
n = (depth - 4) // 6
widths = torch.Tensor([16, 32, 64]).mul(width).int()
def gen_block_params(ni, no):
return {
'conv0': conv_params(ni, no, 3),
'conv1': conv_params(no, no, 3),
'bn0': bnparams(ni),
'bn1': bnparams(no),
'convdim': conv_params(ni, no, 1) if ni != no else None,
}
def gen_group_params(ni, no, count):
return {
'block%d' % i: gen_block_params(ni if i == 0 else no, no)
for i in range(count)
}
def gen_group_stats(ni, no, count):
return {
'block%d' % i: {
'bn0': bnstats(ni if i == 0 else no),
'bn1': bnstats(no)
}
for i in range(count)
}
params = {
'conv0': conv_params(3, 16, 3),
'group0': gen_group_params(16, widths[0], n),
'group1': gen_group_params(widths[0], widths[1], n),
'group2': gen_group_params(widths[1], widths[2], n),
'bn': bnparams(widths[2]),
'fc': linear_params(widths[2], num_classes),
}
stats = {
'group0': gen_group_stats(16, widths[0], n),
'group1': gen_group_stats(widths[0], widths[1], n),
'group2': gen_group_stats(widths[1], widths[2], n),
'bn': bnstats(widths[2]),
}
flat_params = flatten_params(params)
flat_stats = flatten_stats(stats)
def activation(x, params, stats, base, mode):
return F.relu(F.batch_norm(x,
weight=params[base + '.weight'],
bias=params[base + '.bias'],
running_mean=stats[base + '.running_mean'],
running_var=stats[base + '.running_var'],
training=mode,
momentum=0.1,
eps=1e-5),
inplace=True)
def block(x, params, stats, base, mode, stride):
o1 = activation(x, params, stats, base + '.bn0', mode)
y = F.conv2d(o1, params[base + '.conv0'], stride=stride, padding=1)
o2 = activation(y, params, stats, base + '.bn1', mode)
z = F.conv2d(o2, params[base + '.conv1'], stride=1, padding=1)
if base + '.convdim' in params:
return z + F.conv2d(o1, params[base + '.convdim'], stride=stride)
else:
return z + x
def group(o, params, stats, base, mode, stride):
for i in range(n):
o = block(o, params, stats, '%s.block%d' % (base, i), mode,
stride if i == 0 else 1)
return o
def f(input, params, stats, mode):
assert input.get_device() == params['conv0'].get_device()
x = F.conv2d(input, params['conv0'], padding=1)
g0 = group(x, params, stats, 'group0', mode, 1)
g1 = group(g0, params, stats, 'group1', mode, 2)
g2 = group(g1, params, stats, 'group2', mode, 2)
o = activation(g2, params, stats, 'bn', mode)
o = F.avg_pool2d(o, 8, 1, 0)
o = o.view(o.size(0), -1)
o = F.linear(o, params['fc.weight'], params['fc.bias'])
return o
return f, flat_params, flat_stats
def resnet(depth, width, num_classes,activation):
assert (depth - 4) % 6 == 0, 'depth should be 6n+4'
n = (depth - 4) // 6
widths = torch.Tensor([16, 32, 64]).mul(width).int()
actfun=None
if activation=='swish':
actfun=swish
elif activation=='new':
actfun=new
elif activation=='elu':
actfun=F.elu
elif activation=='tanh':
actfun=F.tanh
elif activation=='lrelu':
actfun=F.leaky_relu
elif activation=='relu':
actfun=F.relu
def gen_block_params(ni, no):
return {
'conv0': conv_params(ni, no, 3),
'conv1': conv_params(no, no, 3),
'bn0': bnparams(ni),
'bn1': bnparams(no),
'convdim': conv_params(ni, no, 1) if ni != no else None,
}
def gen_group_params(ni, no, count):
return {'block%d' % i: gen_block_params(ni if i == 0 else no, no)
for i in range(count)}
def gen_group_stats(ni, no, count):
return {'block%d' % i: {'bn0': bnstats(ni if i == 0 else no), 'bn1': bnstats(no)}
for i in range(count)}
flat_params = flatten_params({
'conv0': conv_params(3,16,3),
'group0': gen_group_params(16, widths[0], n),
'group1': gen_group_params(widths[0], widths[1], n),
'group2': gen_group_params(widths[1], widths[2], n),
'bn': bnparams(widths[2]),
'fc': linear_params(widths[2], num_classes),
})
flat_stats = flatten_stats({
'group0': gen_group_stats(16, widths[0], n),
'group1': gen_group_stats(widths[0], widths[1], n),
'group2': gen_group_stats(widths[1], widths[2], n),
'bn': bnstats(widths[2]),
})
def block(x, params, stats, base, mode, stride):
o1 = actfun(batch_norm(x, params, stats, base + '.bn0', mode),0.2)
y = F.conv2d(o1, params[base + '.conv0'], stride=stride, padding=1)
o2 = actfun(batch_norm(y, params, stats, base + '.bn1', mode),0.2)
z = F.conv2d(o2, params[base + '.conv1'], stride=1, padding=1)
if base + '.convdim' in params:
return z + F.conv2d(o1, params[base + '.convdim'], stride=stride)
else:
return z + x
def group(o, params, stats, base, mode, stride):
for i in range(n):
o = block(o, params, stats, '%s.block%d' % (base,i), mode, stride if i == 0 else 1)
return o
def f(input, params, stats, mode):
x = F.conv2d(input, params['conv0'], padding=1)
g0 = group(x, params, stats, 'group0', mode, 1)
g1 = group(g0, params, stats, 'group1', mode, 2)
g2 = group(g1, params, stats, 'group2', mode, 2)
o = actfun(batch_norm(g2, params, stats, 'bn', mode),0.2)
o = F.avg_pool2d(o, 8, 1, 0)
o = o.view(o.size(0), -1)
o = F.linear(o, params['fc.weight'], params['fc.bias'])
return o
return f, flat_params, flat_stats
def define_student(depth, width):
definitions = {
18: [2, 2, 2, 2],
34: [3, 4, 6, 5],
}
assert depth in definitions.keys()
widths = np.floor(np.asarray([64, 128, 256, 512]) * width).astype(np.int)
blocks = definitions[depth]
def batch_norm(x, params, stats, base, mode):
return F.batch_norm(x,
weight=params[base + '.weight'],
bias=params[base + '.bias'],
running_mean=stats[base + '.running_mean'],
running_var=stats[base + '.running_var'],
training=mode,
momentum=0.1,
eps=1e-5)
def gen_block_params(ni, no):
return {
'conv0': conv_params(ni, no, 3),
'conv1': conv_params(no, no, 3),
'bn0': bnparams(no),
'bn1': bnparams(no),
'convdim': conv_params(ni, no, 1) if ni != no else None,
}
def gen_group_params(ni, no, count):
return {
'block%d' % i: gen_block_params(ni if i == 0 else no, no)
for i in range(count)
}
def gen_group_stats(no, count):
return {
'block%d' % i: {
'bn0': bnstats(no),
'bn1': bnstats(no)
}
for i in range(count)
}
params = nested_dict({
'conv0':
conv_params(3, 64, 7),
'bn0':
bnparams(64),
'group0':
gen_group_params(64, widths[0], blocks[0]),
'group1':
gen_group_params(widths[0], widths[1], blocks[1]),
'group2':
gen_group_params(widths[1], widths[2], blocks[2]),
'group3':
gen_group_params(widths[2], widths[3], blocks[3]),
'fc':
linear_params(widths[3], 1000),
})
stats = nested_dict({
'bn0': bnstats(64),
'group0': gen_group_stats(widths[0], blocks[0]),
'group1': gen_group_stats(widths[1], blocks[1]),
'group2': gen_group_stats(widths[2], blocks[2]),
'group3': gen_group_stats(widths[3], blocks[3]),
})
# flatten parameters and additional buffers
flat_params = OrderedDict()
flat_stats = OrderedDict()
for keys, v in params.iteritems_flat():
if v is not None:
flat_params['.'.join(keys)] = Variable(v, requires_grad=True)
for keys, v in stats.iteritems_flat():
flat_stats['.'.join(keys)] = v
def block(x, params, stats, base, mode, stride):
y = F.conv2d(x, params[base + '.conv0'], stride=stride, padding=1)
o1 = F.relu(batch_norm(y, params, stats, base + '.bn0', mode),
inplace=True)
z = F.conv2d(o1, params[base + '.conv1'], stride=1, padding=1)
o2 = batch_norm(z, params, stats, base + '.bn1', mode)
if base + '.convdim' in params:
return F.relu(
o2 + F.conv2d(x, params[base + '.convdim'], stride=stride),
inplace=True)
else:
return F.relu(o2 + x, inplace=True)
def group(o, params, stats, base, mode, stride, n):
for i in range(n):
o = block(o, params, stats, '%s.block%d' % (base, i), mode,
stride if i == 0 else 1)
return o
def f(input, params, stats, mode, pr=''):
o = F.conv2d(input, params[pr + 'conv0'], stride=2, padding=3)
o = F.relu(batch_norm(o, params, stats, pr + 'bn0', mode),
inplace=True)
o = F.max_pool2d(o, 3, 2, 1)
g0 = group(o, params, stats, pr + 'group0', mode, 1, blocks[0])
g1 = group(g0, params, stats, pr + 'group1', mode, 2, blocks[1])
g2 = group(g1, params, stats, pr + 'group2', mode, 2, blocks[2])
g3 = group(g2, params, stats, pr + 'group3', mode, 2, blocks[3])
o = F.avg_pool2d(g3, 7)
o = o.view(o.size(0), -1)
o = F.linear(o, params[pr + 'fc.weight'], params[pr + 'fc.bias'])
return o, [g0, g1, g2, g3]
return f, flat_params, flat_stats
def resnet(depth, width, num_classes):
assert (depth - 4) % 6 == 0, 'depth should be 6n+4'
n = (depth - 4) // 6
widths = [int(v * width) for v in (16, 32, 64)]
def gen_block_params(ni, no):
return {
'conv0': utils.conv_params(ni, no, 3),
'conv1': utils.conv_params(no, no, 3),
'bn0': utils.bnparams(ni),
'bn1': utils.bnparams(no),
'convdim': utils.conv_params(ni, no, 1) if ni != no else None,
}
def gen_group_params(ni, no, count):
return {
'block%d' % i: gen_block_params(ni if i == 0 else no, no)
for i in range(count)
}
flat_params = utils.cast(
utils.flatten({
'conv0': utils.conv_params(3, 16, 3),
'group0': gen_group_params(16, widths[0], n),
'group1': gen_group_params(widths[0], widths[1], n),
'group2': gen_group_params(widths[1], widths[2], n),
'bn': utils.bnparams(widths[2]),
'fc': utils.linear_params(widths[2], num_classes),
}))
utils.set_requires_grad_except_bn_(flat_params)
def block(x, params, base, mode, stride, out_dict):
o1 = F.relu(utils.batch_norm(x, params, base + '.bn0', mode),
inplace=True)
y = F.conv2d(o1, params[base + '.conv0'], stride=stride, padding=1)
o2 = F.relu(utils.batch_norm(y, params, base + '.bn1', mode),
inplace=True)
z = F.conv2d(o2, params[base + '.conv1'], stride=1, padding=1)
if base + '.convdim' in params:
o = z + F.conv2d(o1, params[base + '.convdim'], stride=stride)
else:
o = z + x
out_dict[base + '.relu0'] = o1
out_dict[base + '.relu1'] = o2
return o, out_dict
def group(o, params, base, mode, stride, out_dict):
for i in range(n):
o, out_dict = block(o, params, '{}.block{}'.format(base, i), mode,
stride if i == 0 else 1, out_dict)
return o, out_dict
def f(input, params, mode, base=''):
out_dict = {}
x = F.conv2d(input, params['{}conv0'.format(base)], padding=1)
g0, out_dict = group(x, params, '{}group0'.format(base), mode, 1,
out_dict)
g1, out_dict = group(g0, params, '{}group1'.format(base), mode, 2,
out_dict)
g2, out_dict = group(g1, params, '{}group2'.format(base), mode, 2,
out_dict)
o = F.relu(utils.batch_norm(g2, params, '{}bn'.format(base), mode))
out_dict[base + "last_relu"] = o
o = F.avg_pool2d(o, 8, 1, 0)
o = o.view(o.size(0), -1)
o = F.linear(o, params['{}fc.weight'.format(base)],
params['{}fc.bias'.format(base)])
return o, (g0, g1, g2), out_dict
return f, flat_params
def resnet(depth, width, num_classes):
assert (depth - 4) % 6 == 0, 'depth should be 6n+4'
#n为残差块个数
n = (depth - 4) // 6
#widths为网络每一层的filter个数
widths = [int(v * width) for v in (16, 32, 64)]
#初始化，生成残差块的参数
def gen_block_params(ni, no):
return {
'conv0': utils.conv_params(ni, no, 3),
'conv1': utils.conv_params(no, no, 3),
'bn0': utils.bnparams(ni),
'bn1': utils.bnparams(no),
'convdim': utils.conv_params(ni, no, 1) if ni != no else None,
}
#初始化，生成残差组的参数，里面有count个残差块
def gen_group_params(ni, no, count):
return {
'block%d' % i: gen_block_params(ni if i == 0 else no, no)
for i in range(count)
}
#初始化，整个网络框架的参数，卷积-组0-组1-组2-
#flatten：将所有的param平铺出来
flat_params = utils.cast(
utils.flatten({
'conv0': utils.conv_params(3, 16, 3),
'group0': gen_group_params(16, widths[0], n),
#每一层的宽度不一样
'group1': gen_group_params(widths[0], widths[1], n),
'group2': gen_group_params(widths[1], widths[2], n),
'bn': utils.bnparams(widths[2]),
'fc': utils.linear_params(widths[2], num_classes),
}))
utils.set_requires_grad_except_bn_(flat_params)
#大一个残差块
#RELU-卷积-relu-卷积
def block(x, params, base, mode, stride):
o1 = F.relu(utils.batch_norm(x, params, base + '.bn0', mode),
inplace=True)
y = F.conv2d(o1, params[base + '.conv0'], stride=stride, padding=1)
o2 = F.relu(utils.batch_norm(y, params, base + '.bn1', mode),
inplace=True)
z = F.conv2d(o2, params[base + '.conv1'], stride=1, padding=1)
#加入的是bottlneck?1*1的卷积层
if base + '.convdim' in params:
return z + F.conv2d(o1, params[base + '.convdim'], stride=stride)
else:
return z + x
#搭一个残差组
def group(o, params, base, mode, stride):
for i in range(n):
#o = block(o, params, f'{base}.block{i}',mode,stride if i == 0 else 1)
o = block(o, params, '%s.block%d' % (base, i), mode,
stride if i == 0 else 1)
return o
#搭整个网络
#卷积-第一组-第二组-第三组-relu-池化-扁平化（全连接）
def f(input, params, mode, base=''):
x = F.conv2d(input, params[base + 'conv0'], padding=1)
g0 = group(x, params, base + 'group0', mode, 1)
g1 = group(g0, params, base + 'group1', mode, 2)
g2 = group(g1, params, base + 'group2', mode, 2)
o = F.relu(utils.batch_norm(g2, params, base + 'bn', mode))
o = F.avg_pool2d(o, 8, 1, 0)
#将多行的tensor，变为一行
o = o.view(o.size(0), -1)
#o = F.linear(o, params[base+'Connection timed outfc.weight'], params[base+'fc.bias'])
o = F.linear(o, params[base + 'fc.weight'], params[base + 'fc.bias'])
return o, (g0, g1, g2)
#返回的是最后的输出，以及每个组输出的tuple
return f, flat_params
def __init__(self,
depth,
width,
ninputs=3,
num_groups=3,
num_classes=None,
dropout=0.):
super(WideResNet, self).__init__()
self.depth = depth
self.width = width
self.num_groups = num_groups
self.num_classes = num_classes
self.dropout = dropout
self.mode = True # Training
#widths = torch.Tensor([16, 32, 64]).mul(width).int()
widths = np.array([16, 32, 64]).astype(np.int) * width
def gen_block_params(ni, no):
return {
'conv0': conv_params(ni, no, 3),
'conv1': conv_params(no, no, 3),
'bn0': bnparams(ni),
'bn1': bnparams(no),
'convdim': conv_params(ni, no, 1) if ni != no else None,
}
def gen_group_params(ni, no, count):
return {
'block%d' % i: gen_block_params(ni if i == 0 else no, no)
for i in range(count)
}
def gen_group_stats(ni, no, count):
return {
'block%d' % i: {
'bn0': bnstats(ni if i == 0 else no),
'bn1': bnstats(no)
}
for i in range(count)
}
params = {'conv0': conv_params(ni=ninputs, no=widths[0], k=3)}
stats = {}
for i in range(num_groups + 1):
if i == 0:
params.update({
'group' + str(i):
gen_group_params(widths[i], widths[i], depth)
})
stats.update({
'group' + str(i):
gen_group_stats(widths[i], widths[i], depth)
})
else:
params.update({
'group' + str(i):
gen_group_params(widths[i - 1], widths[i], depth)
})
stats.update({
'group' + str(i):
gen_group_stats(widths[i - 1], widths[i], depth)
})
if num_classes is not None:
params.update({'fc': linear_params(widths[i], num_classes)})
params.update({'bn': bnparams(widths[i])})
stats.update({'bn': bnstats(widths[i])})
params = flatten_params(params)
stats = flatten_stats(stats)
self.params = nn.ParameterDict({})
self.stats = nn.ParameterDict({})
for key in params.keys():
self.params.update(
{key: nn.Parameter(params[key], requires_grad=True)})
for key in stats.keys():
self.stats.update(
{key: nn.Parameter(stats[key], requires_grad=False)})
def resnet(depth,
width,
num_classes,
is_full_wrn=True,
is_fully_convolutional=False):
#assert (depth - 4) % 6 == 0, 'depth should be 6n+4'
#n = (depth - 4) // 6
#wrn = WideResNet(depth, width, ninputs=3,useCuda=True, num_groups=3, num_classes=num_classes)
n = depth
widths = torch.Tensor([16, 32, 64]).mul(width).int()
def gen_block_params(ni, no):
return {
'conv0': conv_params(ni, no, 3),
'conv1': conv_params(no, no, 3),
'bn0': bnparams(ni),
'bn1': bnparams(no),
'convdim': conv_params(ni, no, 1) if ni != no else None,
}
def gen_group_params(ni, no, count):
return {
'block%d' % i: gen_block_params(ni if i == 0 else no, no)
for i in range(count)
}
def gen_group_stats(ni, no, count):
return {
'block%d' % i: {
'bn0': bnstats(ni if i == 0 else no),
'bn1': bnstats(no)
}
for i in range(count)
}
params = {
'conv0': conv_params(3, 16, 3),
'group0': gen_group_params(16, widths[0], n),
'group1': gen_group_params(widths[0], widths[1], n),
'group2': gen_group_params(widths[1], widths[2], n),
'bn': bnparams(widths[2]),
'fc': linear_params(widths[2], num_classes),
}
stats = {
'group0': gen_group_stats(16, widths[0], n),
'group1': gen_group_stats(widths[0], widths[1], n),
'group2': gen_group_stats(widths[1], widths[2], n),
'bn': bnstats(widths[2]),
}
if not is_full_wrn:
''' omniglot '''
params['bn'] = bnparams(widths[1])
#params['fc'] = linear_params(widths[1]*16*16, num_classes)
params['fc'] = linear_params(widths[1], num_classes)
stats['bn'] = bnstats(widths[1])
'''
# banknote
params['bn'] = bnparams(widths[2])
#params['fc'] = linear_params(widths[2]*16*16, num_classes)
params['fc'] = linear_params(widths[2], num_classes)
stats['bn'] = bnstats(widths[2])
'''
flat_params = flatten_params(params)
flat_stats = flatten_stats(stats)
def activation(x, params, stats, base, mode):
return F.relu(F.batch_norm(x,
weight=params[base + '.weight'],
bias=params[base + '.bias'],
running_mean=stats[base + '.running_mean'],
running_var=stats[base + '.running_var'],
training=mode,
momentum=0.1,
eps=1e-5),
inplace=True)
def block(x, params, stats, base, mode, stride):
o1 = activation(x, params, stats, base + '.bn0', mode)
y = F.conv2d(o1, params[base + '.conv0'], stride=stride, padding=1)
o2 = activation(y, params, stats, base + '.bn1', mode)
z = F.conv2d(o2, params[base + '.conv1'], stride=1, padding=1)
if base + '.convdim' in params:
return z + F.conv2d(o1, params[base + '.convdim'], stride=stride)
else:
return z + x
def group(o, params, stats, base, mode, stride):
for i in range(n):
o = block(o, params, stats, '%s.block%d' % (base, i), mode,
stride if i == 0 else 1)
return o
def full_wrn(input, params, stats, mode):
assert input.get_device() == params['conv0'].get_device()
x = F.conv2d(input, params['conv0'], padding=1)
g0 = group(x, params, stats, 'group0', mode, 1)
g1 = group(g0, params, stats, 'group1', mode, 2)
g2 = group(g1, params, stats, 'group2', mode, 2)
o = activation(g2, params, stats, 'bn', mode)
o = F.avg_pool2d(o, o.shape[2], 1, 0)
o = o.view(o.size(0), -1)
o = F.linear(o, params['fc.weight'], params['fc.bias'])
return o
def not_full_wrn(input, params, stats, mode):
assert input.get_device() == params['conv0'].get_device()
x = F.conv2d(input, params['conv0'], padding=1)
g0 = group(x, params, stats, 'group0', mode, 1)
g1 = group(g0, params, stats, 'group1', mode, 2)
# omniglot
o = activation(g1, params, stats, 'bn', mode)
o = F.avg_pool2d(o, o.shape[2], 1, 0)
# banknote
'''
g2 = group(g1, params, stats, 'group2', mode, 2)
o = activation(g2, params, stats, 'bn', mode)
o = F.avg_pool2d(o, 16, 1, 0)
'''
o = o.view(o.size(0), -1)
o = F.linear(o, params['fc.weight'], params['fc.bias'])
return o
def fcn_full_wrn(input, params, stats, mode):
assert input.get_device() == params['conv0'].get_device()
x = F.conv2d(input, params['conv0'], padding=1)
g0 = group(x, params, stats, 'group0', mode, 1)
g1 = group(g0, params, stats, 'group1', mode, 2)
g2 = group(g1, params, stats, 'group2', mode, 2)
o = activation(g2, params, stats, 'bn', mode)
return o
def fcn_not_full_wrn(input, params, stats, mode):
assert input.get_device() == params['conv0'].get_device()
x = F.conv2d(input, params['conv0'], padding=1)
g0 = group(x, params, stats, 'group0', mode, 1)
g1 = group(g0, params, stats, 'group1', mode, 2)
o = activation(g1, params, stats, 'bn', mode)
return o
if is_fully_convolutional:
if is_full_wrn:
return fcn_full_wrn, flat_params, flat_stats
else:
return fcn_not_full_wrn, flat_params, flat_stats
else:
if is_full_wrn:
return full_wrn, flat_params, flat_stats
else:
return not_full_wrn, flat_params, flat_stats
def gen_harmonic_params(ni, no, k, normalize=False):
paramdict = {'conv': utils.conv_params(ni * k * k, no, 1)}
if normalize:
paramdict.update({'bn': utils.bnparams(ni * k * k, affine=False)})
return paramdict