def gen_group_stats(no, count):
return {
'block%d' % i: {
'bn0': bnstats(no),
'bn1': bnstats(no)
}
for i in range(count)
}
def gen_group_stats(ni, no, count):
return {
'block%d' % i: {
'bn0': bnstats(ni if i == 0 else no),
'bn1': bnstats(no),
'bns0': bnstats(ni if i == 0 else no),
'bns1': bnstats(no)
}
for i in range(count)
}
def define_student(depth, width):
definitions = {
18: [2, 2, 2, 2],
34: [3, 4, 6, 5],
}
assert depth in list(definitions.keys())
widths = np.floor(np.asarray([64, 128, 256, 512]) * width).astype(np.int)
blocks = definitions[depth]
def gen_block_params(ni, no):
return {
'conv0': conv_params(ni, no, 3),
'conv1': conv_params(no, no, 3),
'bn0': bnparams(no),
'bn1': bnparams(no),
'convdim': conv_params(ni, no, 1) if ni != no else None,
}
def gen_group_params(ni, no, count):
return {
'block%d' % i: gen_block_params(ni if i == 0 else no, no)
for i in range(count)
}
def gen_group_stats(no, count):
return {
'block%d' % i: {
'bn0': bnstats(no),
'bn1': bnstats(no)
}
for i in range(count)
}
params = {
'conv0': conv_params(3, 64, 7),
'bn0': bnparams(64),
'group0': gen_group_params(64, widths[0], blocks[0]),
'group1': gen_group_params(widths[0], widths[1], blocks[1]),
'group2': gen_group_params(widths[1], widths[2], blocks[2]),
'group3': gen_group_params(widths[2], widths[3], blocks[3]),
'fc': linear_params(widths[3], 1000),
}
stats = {
'bn0': bnstats(64),
'group0': gen_group_stats(widths[0], blocks[0]),
'group1': gen_group_stats(widths[1], blocks[1]),
'group2': gen_group_stats(widths[2], blocks[2]),
'group3': gen_group_stats(widths[3], blocks[3]),
}
# flatten parameters and additional buffers
flat_params = flatten_params(params)
flat_stats = flatten_stats(stats)
def block(x, params, stats, base, mode, stride):
y = F.conv2d(x, params[base + '.conv0'], stride=stride, padding=1)
o1 = F.relu(batch_norm(y, params, stats, base + '.bn0', mode),
inplace=True)
z = F.conv2d(o1, params[base + '.conv1'], stride=1, padding=1)
o2 = batch_norm(z, params, stats, base + '.bn1', mode)
if base + '.convdim' in params:
return F.relu(
o2 + F.conv2d(x, params[base + '.convdim'], stride=stride),
inplace=True)
else:
return F.relu(o2 + x, inplace=True)
def group(o, params, stats, base, mode, stride, n):
for i in range(n):
o = block(o, params, stats, '%s.block%d' % (base, i), mode,
stride if i == 0 else 1)
return o
def f(input, params, stats, mode, pr=''):
o = F.conv2d(input, params[pr + 'conv0'], stride=2, padding=3)
o = F.relu(batch_norm(o, params, stats, pr + 'bn0', mode),
inplace=True)
o = F.max_pool2d(o, 3, 2, 1)
g0 = group(o, params, stats, pr + 'group0', mode, 1, blocks[0])
g1 = group(g0, params, stats, pr + 'group1', mode, 2, blocks[1])
g2 = group(g1, params, stats, pr + 'group2', mode, 2, blocks[2])
g3 = group(g2, params, stats, pr + 'group3', mode, 2, blocks[3])
o = F.avg_pool2d(g3, 7)
o = o.view(o.size(0), -1)
o = F.linear(o, params[pr + 'fc.weight'], params[pr + 'fc.bias'])
return o, [g0, g1, g2, g3]
return f, flat_params, flat_stats
def resnet(depth, width, num_classes):
assert (depth - 4) % 6 == 0, 'depth should be 6n+4'
n = int((depth - 4) / 6)
widths = np.floor(np.asarray([16., 32., 64.]) * width).astype(np.int)
def gen_block_params(ni, no):
return {
'conv0': conv_params(ni, no, 3),
'conv1': conv_params(no, no, 3),
'bn0': bnparams(ni),
'bn1': bnparams(no),
'convdim': conv_params(ni, no, 1) if ni != no else None,
}
def gen_group_params(ni, no, count):
return {
'block%d' % i: gen_block_params(ni if i == 0 else no, no)
for i in range(count)
}
def gen_group_stats(ni, no, count):
return {
'block%d' % i: {
'bn0': bnstats(ni if i == 0 else no),
'bn1': bnstats(no)
}
for i in range(count)
}
params = nested_dict({
'conv0': conv_params(3, 16, 3),
'group0': gen_group_params(16, widths[0], n),
'group1': gen_group_params(widths[0], widths[1], n),
'group2': gen_group_params(widths[1], widths[2], n),
'bn': bnparams(widths[2]),
'fc': linear_params(widths[2], num_classes),
})
stats = nested_dict({
'group0': gen_group_stats(16, widths[0], n),
'group1': gen_group_stats(widths[0], widths[1], n),
'group2': gen_group_stats(widths[1], widths[2], n),
'bn': bnstats(widths[2]),
})
flat_params = OrderedDict()
flat_stats = OrderedDict()
for keys, v in params.iteritems_flat():
if v is not None:
flat_params['.'.join(keys)] = Variable(v, requires_grad=True)
for keys, v in stats.iteritems_flat():
flat_stats['.'.join(keys)] = v
def activation(x, params, stats, base, mode):
return F.relu(
F.batch_norm(x,
weight=params[base + '.weight'],
bias=params[base + '.bias'],
running_mean=stats[base + '.running_mean'],
running_var=stats[base + '.running_var'],
training=mode,
momentum=0.1,
eps=1e-5))
def block(x, params, stats, base, mode, stride):
o1 = activation(x, params, stats, base + '.bn0', mode)
y = F.conv2d(o1, params[base + '.conv0'], stride=stride, padding=1)
o2 = activation(y, params, stats, base + '.bn1', mode)
z = F.conv2d(o2, params[base + '.conv1'], stride=1, padding=1)
if base + '.convdim' in params:
return z + F.conv2d(o1, params[base + '.convdim'], stride=stride)
else:
return z + x
def group(o, params, stats, base, mode, stride):
for i in range(n):
o = block(o, params, stats, '%s.block%d' % (base, i), mode,
stride if i == 0 else 1)
return o
def f(input, params, stats, mode, prefix=''):
x = F.conv2d(input, params[prefix + 'conv0'], padding=1)
g0 = group(x, params, stats, prefix + 'group0', mode, 1)
g1 = group(g0, params, stats, prefix + 'group1', mode, 2)
g2 = group(g1, params, stats, prefix + 'group2', mode, 2)
o = activation(g2, params, stats, prefix + 'bn', mode)
o = F.avg_pool2d(o, 8, 1, 0)
o = o.view(o.size(0), -1)
o = F.linear(o, params[prefix + 'fc.weight'],
params[prefix + 'fc.bias'])
return o, [g0, g1, g2]
return f, flat_params, flat_stats
def resnet(depth, width, num_classes):
assert (depth - 4) % 6 == 0, 'depth should be 6n+4'
n = (depth - 4) // 6
widths = [int(x * width) for x in [16, 32, 64]]
def gen_block_params(ni, no):
return {
'conv0': conv_params(ni, no, 3),
'conv1': conv_params(no, no, 3),
'bn0': bnparams(ni),
'bn1': bnparams(no),
'convdim': conv_params(ni, no, 1) if ni != no else None,
}
def gen_group_params(ni, no, count):
return {'block%d' % i: gen_block_params(ni if i == 0 else no, no)
for i in range(count)}
def gen_group_stats(ni, no, count):
return {'block%d' % i: {'bn0': bnstats(ni if i == 0 else no), 'bn1': bnstats(no)}
for i in range(count)}
flat_params = flatten_params({
'conv0': conv_params(3,16,3),
'group0': gen_group_params(16, widths[0], n),
'group1': gen_group_params(widths[0], widths[1], n),
'group2': gen_group_params(widths[1], widths[2], n),
'bn': bnparams(widths[2]),
'fc': linear_params(widths[2], num_classes),
})
flat_stats = flatten_stats({
'group0': gen_group_stats(16, widths[0], n),
'group1': gen_group_stats(widths[0], widths[1], n),
'group2': gen_group_stats(widths[1], widths[2], n),
'bn': bnstats(widths[2]),
})
def block(x, params, stats, base, mode, stride):
o1 = F.relu(batch_norm(x, params, stats, base + '.bn0', mode), inplace=True)
y = F.conv2d(o1, params[base + '.conv0'], stride=stride, padding=1)
o2 = F.relu(batch_norm(y, params, stats, base + '.bn1', mode), inplace=True)
z = F.conv2d(o2, params[base + '.conv1'], stride=1, padding=1)
if base + '.convdim' in params:
return z + F.conv2d(o1, params[base + '.convdim'], stride=stride)
else:
return z + x
def group(o, params, stats, base, mode, stride):
for i in range(n):
o = block(o, params, stats, '%s.block%d' % (base,i), mode, stride if i == 0 else 1)
return o
def f(input, params, stats, mode):
x = F.conv2d(input, params['conv0'], padding=1)
g0 = group(x, params, stats, 'group0', mode, 1)
g1 = group(g0, params, stats, 'group1', mode, 2)
g2 = group(g1, params, stats, 'group2', mode, 2)
o = F.relu(batch_norm(g2, params, stats, 'bn', mode))
o = F.avg_pool2d(o, 8, 1, 0)
o = o.view(o.size(0), -1)
o = F.linear(o, params['fc.weight'], params['fc.bias'])
return o
return f, flat_params, flat_stats
def resnet(depth, width, num_classes, stu_depth=0):
assert (depth - 4) % 6 == 0, 'depth should be 6n+4'
n = (depth - 4) // 6
if stu_depth != 0:
assert (stu_depth - 4) % 6 == 0, 'student depth should be 6n+4'
n_s = (stu_depth - 4) // 6
else:
n_s = 0
widths = torch.Tensor([16, 32, 64]).mul(width).int()
def gen_block_params(ni, no):
return {
'conv0': conv_params(ni, no, 3),
'conv1': conv_params(no, no, 3),
'bn0': bnparams(ni),
'bn1': bnparams(no),
'bns0': bnparams(ni),
'bns1': bnparams(no),
'convdim': conv_params(ni, no, 1) if ni != no else None,
}
def gen_group_params(ni, no, count):
return {
'block%d' % i: gen_block_params(ni if i == 0 else no, no)
for i in range(count)
}
def gen_group_stats(ni, no, count):
return {
'block%d' % i: {
'bn0': bnstats(ni if i == 0 else no),
'bn1': bnstats(no),
'bns0': bnstats(ni if i == 0 else no),
'bns1': bnstats(no)
}
for i in range(count)
}
if stu_depth != 0 and not opt.param_share:
params = {
'conv0': conv_params(3, 16, 3),
'group0': gen_group_params(16, widths[0], n),
'group1': gen_group_params(widths[0], widths[1], n),
'group2': gen_group_params(widths[1], widths[2], n),
'groups0': gen_group_params(16, widths[0], n_s),
'groups1': gen_group_params(widths[0], widths[1], n_s),
'groups2': gen_group_params(widths[1], widths[2], n_s),
'bn': bnparams(widths[2]),
'bns': bnparams(widths[2]),
'fc': linear_params(widths[2], num_classes),
'fcs': linear_params(widths[2], num_classes),
}
stats = {
'group0': gen_group_stats(16, widths[0], n),
'group1': gen_group_stats(widths[0], widths[1], n),
'group2': gen_group_stats(widths[1], widths[2], n),
'groups0': gen_group_stats(16, widths[0], n_s),
'groups1': gen_group_stats(widths[0], widths[1], n_s),
'groups2': gen_group_stats(widths[1], widths[2], n_s),
'bn': bnstats(widths[2]),
'bns': bnstats(widths[2]),
}
else:
params = {
'conv0': conv_params(3, 16, 3),
'group0': gen_group_params(16, widths[0], n),
'group1': gen_group_params(widths[0], widths[1], n),
'group2': gen_group_params(widths[1], widths[2], n),
'bn': bnparams(widths[2]),
'bns': bnparams(widths[2]),
'fc': linear_params(widths[2], num_classes),
'fcs': linear_params(widths[2], num_classes),
}
stats = {
'group0': gen_group_stats(16, widths[0], n),
'group1': gen_group_stats(widths[0], widths[1], n),
'group2': gen_group_stats(widths[1], widths[2], n),
'bn': bnstats(widths[2]),
'bns': bnstats(widths[2]),
}
flat_params = flatten_params(params)
flat_stats = flatten_stats(stats)
def block(x, params, stats, base, mode, stride, flag, drop_switch=True):
if flag == 's':
o1 = F.relu(batch_norm(x, params, stats, base + '.bns0', mode))
y = F.conv2d(o1, params[base + '.conv0'], stride=stride, padding=1)
o2 = F.relu(batch_norm(y, params, stats, base + '.bns1', mode))
z = F.conv2d(o2, params[base + '.conv1'], stride=1, padding=1)
if base + '.convdim' in params:
return z + F.conv2d(
o1, params[base + '.convdim'], stride=stride)
else:
return z + x
o1 = F.relu(batch_norm(x, params, stats, base + '.bn0', mode))
y = F.conv2d(o1, params[base + '.conv0'], stride=stride, padding=1)
o2 = F.relu(batch_norm(y, params, stats, base + '.bn1', mode))
if opt.dropout > 0 and drop_switch:
o2 = F.dropout(o2, p=opt.dropout, training=mode)
z = F.conv2d(o2, params[base + '.conv1'], stride=1, padding=1)
if base + '.convdim' in params:
return z + F.conv2d(o1, params[base + '.convdim'], stride=stride)
else:
return z + x
def group(o, params, stats, base, mode, stride):
for i in range(n):
o = block(o, params, stats, '%s.block%d' % (base, i), mode,
stride if i == 0 else 1, 't', False)
return o
def group_student(o, params, stats, base, mode, stride, n_layer):
for i in range(n_layer):
o = block(o, params, stats, '%s.block%d' % (base, i), mode,
stride if i == 0 else 1, 's', False)
return o
def f(input, params, stats, mode, prefix=''):
x = F.conv2d(input, params[prefix + 'conv0'], padding=1)
g0 = group(x, params, stats, prefix + 'group0', mode, 1)
g1 = group(g0, params, stats, prefix + 'group1', mode, 2)
g2 = group(g1, params, stats, prefix + 'group2', mode, 2)
o = F.relu(batch_norm(g2, params, stats, prefix + 'bn', mode))
o = F.avg_pool2d(o, 8, 1, 0)
o = o.view(o.size(0), -1)
o = F.linear(o, params[prefix + 'fc.weight'],
params[prefix + 'fc.bias'])
#x_s = F.conv2d(input, params[prefix+'conv0_s'], padding=1)
if stu_depth != 0:
if opt.param_share:
gs0 = group_student(x, params, stats, prefix + 'group0', mode,
1, n_s)
gs1 = group_student(gs0, params, stats, prefix + 'group1',
mode, 2, n_s)
gs2 = group_student(gs1, params, stats, prefix + 'group2',
mode, 2, n_s)
else:
gs0 = group_student(x, params, stats, prefix + 'groups0', mode,
1, n_s)
gs1 = group_student(gs0, params, stats, prefix + 'groups1',
mode, 2, n_s)
gs2 = group_student(gs1, params, stats, prefix + 'groups2',
mode, 2, n_s)
os = F.relu(batch_norm(gs2, params, stats, prefix + 'bns', mode))
os = F.avg_pool2d(os, 8, 1, 0)
os = os.view(os.size(0), -1)
os = F.linear(os, params[prefix + 'fcs.weight'],
params[prefix + 'fcs.bias'])
return os, o, [g0, g1, g2, gs0, gs1, gs2]
else:
return o, [g0, g1, g2]
return f, flat_params, flat_stats
def resnet(depth, width, num_classes):
assert (depth - 4) % 6 == 0, 'depth should be 6n+4'
n = (depth - 4) // 6
widths = torch.Tensor([16, 32, 64]).mul(width).int().numpy().tolist()
def gen_block_params(ni, no, scalar):
if scalar:
return {
'bn0': bnparams(ni),
'bn1': bnparams(no),
}
return {
'conv0': conv_params(ni, no, 3),
'conv1': conv_params(no, no, 3),
'convdim': conv_params(ni, no, 1) if ni != no else None,
}
def gen_group_params(ni, no, count, bias=False):
return {
'block%d' % i: gen_block_params(ni if i == 0 else no, no, bias)
for i in range(count)
}
def gen_group_stats(ni, no, count):
return {
'block%d' % i: {
'bn0': bnstats(ni if i == 0 else no),
'bn1': bnstats(no)
}
for i in range(count)
}
flat_vectors = flatten_params({
'conv0':
conv_params(3, 16, 3),
'group0':
gen_group_params(16, widths[0], n),
'group1':
gen_group_params(widths[0], widths[1], n),
'group2':
gen_group_params(widths[1], widths[2], n),
'conv1':
conv_params(widths[2], num_classes, 1),
})
flat_scalars = flatten_params({
'group0':
gen_group_params(16, widths[0], n, True),
'group1':
gen_group_params(widths[0], widths[1], n, True),
'group2':
gen_group_params(widths[1], widths[2], n, True),
'bn':
bnparams(widths[2]),
})
flat_stats = flatten_stats({
'group0':
gen_group_stats(16, widths[0], n),
'group1':
gen_group_stats(widths[0], widths[1], n),
'group2':
gen_group_stats(widths[1], widths[2], n),
'bn':
bnstats(widths[2]),
})
def block(x, params, stats, base, mode, stride):
o1 = F.relu(batch_norm(x, params, stats, base + '.bn0', mode, 1.),
inplace=True)
y = F.conv2d(o1, params[base + '.conv0'], stride=stride, padding=1)
o2 = F.relu(batch_norm(y, params, stats, base + '.bn1', mode, 1.),
inplace=True)
z = F.conv2d(o2, params[base + '.conv1'], stride=1, padding=1)
if base + '.convdim' in params:
return z + F.conv2d(o1, params[base + '.convdim'], stride=stride)
else:
return z + x
def group(o, params, stats, base, mode, stride):
for i in range(n):
o = block(o, params, stats, '%s.block%d' % (base, i), mode,
stride if i == 0 else 1)
return o
def f(input, params, stats, mode):
x = F.conv2d(input, params['conv0'], padding=1)
g0 = group(x, params, stats, 'group0', mode, 1)
g1 = group(g0, params, stats, 'group1', mode, 2)
g2 = group(g1, params, stats, 'group2', mode, 2)
o = F.relu(batch_norm(g2, params, stats, 'bn', mode, 1.))
o = F.conv2d(o, params['conv1'])
o = F.avg_pool2d(o, 8, 1, 0)
o = o.view(o.size(0), -1)
return o
return f, flat_vectors, flat_scalars, flat_stats
def resnet(depth, width, num_classes):
assert (depth - 4) % 6 == 0, 'depth should be 6n+4'
n = (depth - 4) // 6
widths = torch.Tensor([16, 32, 64]).mul(width).int()
def gen_block_params(ni, no):
return {
'conv0': conv_params(ni, no, 3),
'conv1': conv_params(no, no, 3),
'bn0': bnparams(ni),
'bn1': bnparams(no),
'convdim': conv_params(ni, no, 1) if ni != no else None,
}
def gen_group_params(ni, no, count):
return {
'block%d' % i: gen_block_params(ni if i == 0 else no, no)
for i in range(count)
}
def gen_group_stats(ni, no, count):
return {
'block%d' % i: {
'bn0': bnstats(ni if i == 0 else no),
'bn1': bnstats(no)
}
for i in range(count)
}
params = {
'conv0': conv_params(3, 16, 3),
'group0': gen_group_params(16, widths[0], n),
'group1': gen_group_params(widths[0], widths[1], n),
'group2': gen_group_params(widths[1], widths[2], n),
'bn': bnparams(widths[2]),
'fc': linear_params(widths[2], num_classes),
}
stats = {
'group0': gen_group_stats(16, widths[0], n),
'group1': gen_group_stats(widths[0], widths[1], n),
'group2': gen_group_stats(widths[1], widths[2], n),
'bn': bnstats(widths[2]),
}
flat_params = flatten_params(params)
flat_stats = flatten_stats(stats)
def activation(x, params, stats, base, mode):
return F.relu(F.batch_norm(x,
weight=params[base + '.weight'],
bias=params[base + '.bias'],
running_mean=stats[base + '.running_mean'],
running_var=stats[base + '.running_var'],
training=mode,
momentum=0.1,
eps=1e-5),
inplace=True)
def block(x, params, stats, base, mode, stride):
o1 = activation(x, params, stats, base + '.bn0', mode)
y = F.conv2d(o1, params[base + '.conv0'], stride=stride, padding=1)
o2 = activation(y, params, stats, base + '.bn1', mode)
z = F.conv2d(o2, params[base + '.conv1'], stride=1, padding=1)
if base + '.convdim' in params:
return z + F.conv2d(o1, params[base + '.convdim'], stride=stride)
else:
return z + x
def group(o, params, stats, base, mode, stride):
for i in range(n):
o = block(o, params, stats, '%s.block%d' % (base, i), mode,
stride if i == 0 else 1)
return o
def f(input, params, stats, mode):
assert input.get_device() == params['conv0'].get_device()
x = F.conv2d(input, params['conv0'], padding=1)
g0 = group(x, params, stats, 'group0', mode, 1)
g1 = group(g0, params, stats, 'group1', mode, 2)
g2 = group(g1, params, stats, 'group2', mode, 2)
o = activation(g2, params, stats, 'bn', mode)
o = F.avg_pool2d(o, 8, 1, 0)
o = o.view(o.size(0), -1)
o = F.linear(o, params['fc.weight'], params['fc.bias'])
return o
return f, flat_params, flat_stats
def resnet(depth, width, num_classes):
assert (depth - 4) % 6 == 0, 'depth should be 6n+4'
n = (depth - 4) // 6
widths = torch.Tensor([16, 32, 64]).mul(width).int()
def gen_block_params(ni, no):
return {
'conv0': conv_params(ni, no, 3),
'conv1': conv_params(no, no, 3),
'bn0': bnparams(ni),
'bn1': bnparams(no),
'convdim': conv_params(ni, no, 1) if ni != no else None,
}
def gen_group_params(ni, no, count):
return {'block%d' % i: gen_block_params(ni if i == 0 else no, no)
for i in range(count)}
def gen_group_stats(ni, no, count):
return {'block%d' % i: {'bn0': bnstats(ni if i == 0 else no), 'bn1': bnstats(no)}
for i in range(count)}
flat_params = flatten_params({
'conv0': conv_params(3,16,3),
'group0': gen_group_params(16, widths[0], n),
'group1': gen_group_params(widths[0], widths[1], n),
'group2': gen_group_params(widths[1], widths[2], n),
'bn': bnparams(widths[2]),
'fc': linear_params(widths[2], num_classes),
})
flat_stats = flatten_stats({
'group0': gen_group_stats(16, widths[0], n),
'group1': gen_group_stats(widths[0], widths[1], n),
'group2': gen_group_stats(widths[1], widths[2], n),
'bn': bnstats(widths[2]),
})
def block(x, params, stats, base, mode, stride):
o1 = F.relu(batch_norm(x, params, stats, base + '.bn0', mode), inplace=True)
y = F.conv2d(o1, params[base + '.conv0'], stride=stride, padding=1)
o2 = F.relu(batch_norm(y, params, stats, base + '.bn1', mode), inplace=True)
z = F.conv2d(o2, params[base + '.conv1'], stride=1, padding=1)
if base + '.convdim' in params:
return z + F.conv2d(o1, params[base + '.convdim'], stride=stride)
else:
return z + x
def group(o, params, stats, base, mode, stride):
for i in range(n):
o = block(o, params, stats, '%s.block%d' % (base,i), mode, stride if i == 0 else 1)
return o
def f(input, params, stats, mode):
x = F.conv2d(input, params['conv0'], padding=1)
g0 = group(x, params, stats, 'group0', mode, 1)
g1 = group(g0, params, stats, 'group1', mode, 2)
g2 = group(g1, params, stats, 'group2', mode, 2)
o = F.relu(batch_norm(g2, params, stats, 'bn', mode))
o = F.avg_pool2d(o, 8, 1, 0)
o = o.view(o.size(0), -1)
o = F.linear(o, params['fc.weight'], params['fc.bias'])
return o
return f, flat_params, flat_stats
def resnet(depth, width, num_classes,activation):
assert (depth - 4) % 6 == 0, 'depth should be 6n+4'
n = (depth - 4) // 6
widths = torch.Tensor([16, 32, 64]).mul(width).int()
actfun=None
if activation=='swish':
actfun=swish
elif activation=='new':
actfun=new
elif activation=='elu':
actfun=F.elu
elif activation=='tanh':
actfun=F.tanh
elif activation=='lrelu':
actfun=F.leaky_relu
elif activation=='relu':
actfun=F.relu
def gen_block_params(ni, no):
return {
'conv0': conv_params(ni, no, 3),
'conv1': conv_params(no, no, 3),
'bn0': bnparams(ni),
'bn1': bnparams(no),
'convdim': conv_params(ni, no, 1) if ni != no else None,
}
def gen_group_params(ni, no, count):
return {'block%d' % i: gen_block_params(ni if i == 0 else no, no)
for i in range(count)}
def gen_group_stats(ni, no, count):
return {'block%d' % i: {'bn0': bnstats(ni if i == 0 else no), 'bn1': bnstats(no)}
for i in range(count)}
flat_params = flatten_params({
'conv0': conv_params(3,16,3),
'group0': gen_group_params(16, widths[0], n),
'group1': gen_group_params(widths[0], widths[1], n),
'group2': gen_group_params(widths[1], widths[2], n),
'bn': bnparams(widths[2]),
'fc': linear_params(widths[2], num_classes),
})
flat_stats = flatten_stats({
'group0': gen_group_stats(16, widths[0], n),
'group1': gen_group_stats(widths[0], widths[1], n),
'group2': gen_group_stats(widths[1], widths[2], n),
'bn': bnstats(widths[2]),
})
def block(x, params, stats, base, mode, stride):
o1 = actfun(batch_norm(x, params, stats, base + '.bn0', mode),0.2)
y = F.conv2d(o1, params[base + '.conv0'], stride=stride, padding=1)
o2 = actfun(batch_norm(y, params, stats, base + '.bn1', mode),0.2)
z = F.conv2d(o2, params[base + '.conv1'], stride=1, padding=1)
if base + '.convdim' in params:
return z + F.conv2d(o1, params[base + '.convdim'], stride=stride)
else:
return z + x
def group(o, params, stats, base, mode, stride):
for i in range(n):
o = block(o, params, stats, '%s.block%d' % (base,i), mode, stride if i == 0 else 1)
return o
def f(input, params, stats, mode):
x = F.conv2d(input, params['conv0'], padding=1)
g0 = group(x, params, stats, 'group0', mode, 1)
g1 = group(g0, params, stats, 'group1', mode, 2)
g2 = group(g1, params, stats, 'group2', mode, 2)
o = actfun(batch_norm(g2, params, stats, 'bn', mode),0.2)
o = F.avg_pool2d(o, 8, 1, 0)
o = o.view(o.size(0), -1)
o = F.linear(o, params['fc.weight'], params['fc.bias'])
return o
return f, flat_params, flat_stats
def gen_group_stats(ni, no, count):
return {'block%d' % i: {'bn0': bnstats(ni if i == 0 else no), 'bn1': bnstats(no)}
for i in range(count)}
def __init__(self,
depth,
width,
ninputs=3,
num_groups=3,
num_classes=None,
dropout=0.):
super(WideResNet, self).__init__()
self.depth = depth
self.width = width
self.num_groups = num_groups
self.num_classes = num_classes
self.dropout = dropout
self.mode = True # Training
#widths = torch.Tensor([16, 32, 64]).mul(width).int()
widths = np.array([16, 32, 64]).astype(np.int) * width
def gen_block_params(ni, no):
return {
'conv0': conv_params(ni, no, 3),
'conv1': conv_params(no, no, 3),
'bn0': bnparams(ni),
'bn1': bnparams(no),
'convdim': conv_params(ni, no, 1) if ni != no else None,
}
def gen_group_params(ni, no, count):
return {
'block%d' % i: gen_block_params(ni if i == 0 else no, no)
for i in range(count)
}
def gen_group_stats(ni, no, count):
return {
'block%d' % i: {
'bn0': bnstats(ni if i == 0 else no),
'bn1': bnstats(no)
}
for i in range(count)
}
params = {'conv0': conv_params(ni=ninputs, no=widths[0], k=3)}
stats = {}
for i in range(num_groups + 1):
if i == 0:
params.update({
'group' + str(i):
gen_group_params(widths[i], widths[i], depth)
})
stats.update({
'group' + str(i):
gen_group_stats(widths[i], widths[i], depth)
})
else:
params.update({
'group' + str(i):
gen_group_params(widths[i - 1], widths[i], depth)
})
stats.update({
'group' + str(i):
gen_group_stats(widths[i - 1], widths[i], depth)
})
if num_classes is not None:
params.update({'fc': linear_params(widths[i], num_classes)})
params.update({'bn': bnparams(widths[i])})
stats.update({'bn': bnstats(widths[i])})
params = flatten_params(params)
stats = flatten_stats(stats)
self.params = nn.ParameterDict({})
self.stats = nn.ParameterDict({})
for key in params.keys():
self.params.update(
{key: nn.Parameter(params[key], requires_grad=True)})
for key in stats.keys():
self.stats.update(
{key: nn.Parameter(stats[key], requires_grad=False)})
def resnet(depth,
width,
num_classes,
is_full_wrn=True,
is_fully_convolutional=False):
#assert (depth - 4) % 6 == 0, 'depth should be 6n+4'
#n = (depth - 4) // 6
#wrn = WideResNet(depth, width, ninputs=3,useCuda=True, num_groups=3, num_classes=num_classes)
n = depth
widths = torch.Tensor([16, 32, 64]).mul(width).int()
def gen_block_params(ni, no):
return {
'conv0': conv_params(ni, no, 3),
'conv1': conv_params(no, no, 3),
'bn0': bnparams(ni),
'bn1': bnparams(no),
'convdim': conv_params(ni, no, 1) if ni != no else None,
}
def gen_group_params(ni, no, count):
return {
'block%d' % i: gen_block_params(ni if i == 0 else no, no)
for i in range(count)
}
def gen_group_stats(ni, no, count):
return {
'block%d' % i: {
'bn0': bnstats(ni if i == 0 else no),
'bn1': bnstats(no)
}
for i in range(count)
}
params = {
'conv0': conv_params(3, 16, 3),
'group0': gen_group_params(16, widths[0], n),
'group1': gen_group_params(widths[0], widths[1], n),
'group2': gen_group_params(widths[1], widths[2], n),
'bn': bnparams(widths[2]),
'fc': linear_params(widths[2], num_classes),
}
stats = {
'group0': gen_group_stats(16, widths[0], n),
'group1': gen_group_stats(widths[0], widths[1], n),
'group2': gen_group_stats(widths[1], widths[2], n),
'bn': bnstats(widths[2]),
}
if not is_full_wrn:
''' omniglot '''
params['bn'] = bnparams(widths[1])
#params['fc'] = linear_params(widths[1]*16*16, num_classes)
params['fc'] = linear_params(widths[1], num_classes)
stats['bn'] = bnstats(widths[1])
'''
# banknote
params['bn'] = bnparams(widths[2])
#params['fc'] = linear_params(widths[2]*16*16, num_classes)
params['fc'] = linear_params(widths[2], num_classes)
stats['bn'] = bnstats(widths[2])
'''
flat_params = flatten_params(params)
flat_stats = flatten_stats(stats)
def activation(x, params, stats, base, mode):
return F.relu(F.batch_norm(x,
weight=params[base + '.weight'],
bias=params[base + '.bias'],
running_mean=stats[base + '.running_mean'],
running_var=stats[base + '.running_var'],
training=mode,
momentum=0.1,
eps=1e-5),
inplace=True)
def block(x, params, stats, base, mode, stride):
o1 = activation(x, params, stats, base + '.bn0', mode)
y = F.conv2d(o1, params[base + '.conv0'], stride=stride, padding=1)
o2 = activation(y, params, stats, base + '.bn1', mode)
z = F.conv2d(o2, params[base + '.conv1'], stride=1, padding=1)
if base + '.convdim' in params:
return z + F.conv2d(o1, params[base + '.convdim'], stride=stride)
else:
return z + x
def group(o, params, stats, base, mode, stride):
for i in range(n):
o = block(o, params, stats, '%s.block%d' % (base, i), mode,
stride if i == 0 else 1)
return o
def full_wrn(input, params, stats, mode):
assert input.get_device() == params['conv0'].get_device()
x = F.conv2d(input, params['conv0'], padding=1)
g0 = group(x, params, stats, 'group0', mode, 1)
g1 = group(g0, params, stats, 'group1', mode, 2)
g2 = group(g1, params, stats, 'group2', mode, 2)
o = activation(g2, params, stats, 'bn', mode)
o = F.avg_pool2d(o, o.shape[2], 1, 0)
o = o.view(o.size(0), -1)
o = F.linear(o, params['fc.weight'], params['fc.bias'])
return o
def not_full_wrn(input, params, stats, mode):
assert input.get_device() == params['conv0'].get_device()
x = F.conv2d(input, params['conv0'], padding=1)
g0 = group(x, params, stats, 'group0', mode, 1)
g1 = group(g0, params, stats, 'group1', mode, 2)
# omniglot
o = activation(g1, params, stats, 'bn', mode)
o = F.avg_pool2d(o, o.shape[2], 1, 0)
# banknote
'''
g2 = group(g1, params, stats, 'group2', mode, 2)
o = activation(g2, params, stats, 'bn', mode)
o = F.avg_pool2d(o, 16, 1, 0)
'''
o = o.view(o.size(0), -1)
o = F.linear(o, params['fc.weight'], params['fc.bias'])
return o
def fcn_full_wrn(input, params, stats, mode):
assert input.get_device() == params['conv0'].get_device()
x = F.conv2d(input, params['conv0'], padding=1)
g0 = group(x, params, stats, 'group0', mode, 1)
g1 = group(g0, params, stats, 'group1', mode, 2)
g2 = group(g1, params, stats, 'group2', mode, 2)
o = activation(g2, params, stats, 'bn', mode)
return o
def fcn_not_full_wrn(input, params, stats, mode):
assert input.get_device() == params['conv0'].get_device()
x = F.conv2d(input, params['conv0'], padding=1)
g0 = group(x, params, stats, 'group0', mode, 1)
g1 = group(g0, params, stats, 'group1', mode, 2)
o = activation(g1, params, stats, 'bn', mode)
return o
if is_fully_convolutional:
if is_full_wrn:
return fcn_full_wrn, flat_params, flat_stats
else:
return fcn_not_full_wrn, flat_params, flat_stats
else:
if is_full_wrn:
return full_wrn, flat_params, flat_stats
else:
return not_full_wrn, flat_params, flat_stats
def define_student(depth, width):
definitions = {
18: [2, 2, 2, 2],
34: [3, 4, 6, 5],
}
assert depth in definitions.keys()
widths = np.floor(np.asarray([64, 128, 256, 512]) * width).astype(np.int)
blocks = definitions[depth]
def batch_norm(x, params, stats, base, mode):
return F.batch_norm(x,
weight=params[base + '.weight'],
bias=params[base + '.bias'],
running_mean=stats[base + '.running_mean'],
running_var=stats[base + '.running_var'],
training=mode,
momentum=0.1,
eps=1e-5)
def gen_block_params(ni, no):
return {
'conv0': conv_params(ni, no, 3),
'conv1': conv_params(no, no, 3),
'bn0': bnparams(no),
'bn1': bnparams(no),
'convdim': conv_params(ni, no, 1) if ni != no else None,
}
def gen_group_params(ni, no, count):
return {
'block%d' % i: gen_block_params(ni if i == 0 else no, no)
for i in range(count)
}
def gen_group_stats(no, count):
return {
'block%d' % i: {
'bn0': bnstats(no),
'bn1': bnstats(no)
}
for i in range(count)
}
params = nested_dict({
'conv0':
conv_params(3, 64, 7),
'bn0':
bnparams(64),
'group0':
gen_group_params(64, widths[0], blocks[0]),
'group1':
gen_group_params(widths[0], widths[1], blocks[1]),
'group2':
gen_group_params(widths[1], widths[2], blocks[2]),
'group3':
gen_group_params(widths[2], widths[3], blocks[3]),
'fc':
linear_params(widths[3], 1000),
})
stats = nested_dict({
'bn0': bnstats(64),
'group0': gen_group_stats(widths[0], blocks[0]),
'group1': gen_group_stats(widths[1], blocks[1]),
'group2': gen_group_stats(widths[2], blocks[2]),
'group3': gen_group_stats(widths[3], blocks[3]),
})
# flatten parameters and additional buffers
flat_params = OrderedDict()
flat_stats = OrderedDict()
for keys, v in params.iteritems_flat():
if v is not None:
flat_params['.'.join(keys)] = Variable(v, requires_grad=True)
for keys, v in stats.iteritems_flat():
flat_stats['.'.join(keys)] = v
def block(x, params, stats, base, mode, stride):
y = F.conv2d(x, params[base + '.conv0'], stride=stride, padding=1)
o1 = F.relu(batch_norm(y, params, stats, base + '.bn0', mode),
inplace=True)
z = F.conv2d(o1, params[base + '.conv1'], stride=1, padding=1)
o2 = batch_norm(z, params, stats, base + '.bn1', mode)
if base + '.convdim' in params:
return F.relu(
o2 + F.conv2d(x, params[base + '.convdim'], stride=stride),
inplace=True)
else:
return F.relu(o2 + x, inplace=True)
def group(o, params, stats, base, mode, stride, n):
for i in range(n):
o = block(o, params, stats, '%s.block%d' % (base, i), mode,
stride if i == 0 else 1)
return o
def f(input, params, stats, mode, pr=''):
o = F.conv2d(input, params[pr + 'conv0'], stride=2, padding=3)
o = F.relu(batch_norm(o, params, stats, pr + 'bn0', mode),
inplace=True)
o = F.max_pool2d(o, 3, 2, 1)
g0 = group(o, params, stats, pr + 'group0', mode, 1, blocks[0])
g1 = group(g0, params, stats, pr + 'group1', mode, 2, blocks[1])
g2 = group(g1, params, stats, pr + 'group2', mode, 2, blocks[2])
g3 = group(g2, params, stats, pr + 'group3', mode, 2, blocks[3])
o = F.avg_pool2d(g3, 7)
o = o.view(o.size(0), -1)
o = F.linear(o, params[pr + 'fc.weight'], params[pr + 'fc.bias'])
return o, [g0, g1, g2, g3]
return f, flat_params, flat_stats
def gen_feature_stats():
dic = {}
for i in range(len(_cfg)):
if not _cfg[i] == 'M':
dic['bn{0}'.format(i)] = bnstats(_cfg[i])
return dic
