def gen_group_stats(no, count): return { 'block%d' % i: { 'bn0': bnstats(no), 'bn1': bnstats(no) } for i in range(count) }
def gen_group_stats(ni, no, count): return { 'block%d' % i: { 'bn0': bnstats(ni if i == 0 else no), 'bn1': bnstats(no), 'bns0': bnstats(ni if i == 0 else no), 'bns1': bnstats(no) } for i in range(count) }
def define_student(depth, width): definitions = { 18: [2, 2, 2, 2], 34: [3, 4, 6, 5], } assert depth in list(definitions.keys()) widths = np.floor(np.asarray([64, 128, 256, 512]) * width).astype(np.int) blocks = definitions[depth] def gen_block_params(ni, no): return { 'conv0': conv_params(ni, no, 3), 'conv1': conv_params(no, no, 3), 'bn0': bnparams(no), 'bn1': bnparams(no), 'convdim': conv_params(ni, no, 1) if ni != no else None, } def gen_group_params(ni, no, count): return { 'block%d' % i: gen_block_params(ni if i == 0 else no, no) for i in range(count) } def gen_group_stats(no, count): return { 'block%d' % i: { 'bn0': bnstats(no), 'bn1': bnstats(no) } for i in range(count) } params = { 'conv0': conv_params(3, 64, 7), 'bn0': bnparams(64), 'group0': gen_group_params(64, widths[0], blocks[0]), 'group1': gen_group_params(widths[0], widths[1], blocks[1]), 'group2': gen_group_params(widths[1], widths[2], blocks[2]), 'group3': gen_group_params(widths[2], widths[3], blocks[3]), 'fc': linear_params(widths[3], 1000), } stats = { 'bn0': bnstats(64), 'group0': gen_group_stats(widths[0], blocks[0]), 'group1': gen_group_stats(widths[1], blocks[1]), 'group2': gen_group_stats(widths[2], blocks[2]), 'group3': gen_group_stats(widths[3], blocks[3]), } # flatten parameters and additional buffers flat_params = flatten_params(params) flat_stats = flatten_stats(stats) def block(x, params, stats, base, mode, stride): y = F.conv2d(x, params[base + '.conv0'], stride=stride, padding=1) o1 = F.relu(batch_norm(y, params, stats, base + '.bn0', mode), inplace=True) z = F.conv2d(o1, params[base + '.conv1'], stride=1, padding=1) o2 = batch_norm(z, params, stats, base + '.bn1', mode) if base + '.convdim' in params: return F.relu( o2 + F.conv2d(x, params[base + '.convdim'], stride=stride), inplace=True) else: return F.relu(o2 + x, inplace=True) def group(o, params, stats, base, mode, stride, n): for i in range(n): o = block(o, params, stats, '%s.block%d' % (base, i), mode, stride if i == 0 else 1) return o def f(input, params, stats, mode, pr=''): o = F.conv2d(input, params[pr + 'conv0'], stride=2, padding=3) o = F.relu(batch_norm(o, params, stats, pr + 'bn0', mode), inplace=True) o = F.max_pool2d(o, 3, 2, 1) g0 = group(o, params, stats, pr + 'group0', mode, 1, blocks[0]) g1 = group(g0, params, stats, pr + 'group1', mode, 2, blocks[1]) g2 = group(g1, params, stats, pr + 'group2', mode, 2, blocks[2]) g3 = group(g2, params, stats, pr + 'group3', mode, 2, blocks[3]) o = F.avg_pool2d(g3, 7) o = o.view(o.size(0), -1) o = F.linear(o, params[pr + 'fc.weight'], params[pr + 'fc.bias']) return o, [g0, g1, g2, g3] return f, flat_params, flat_stats
def resnet(depth, width, num_classes): assert (depth - 4) % 6 == 0, 'depth should be 6n+4' n = int((depth - 4) / 6) widths = np.floor(np.asarray([16., 32., 64.]) * width).astype(np.int) def gen_block_params(ni, no): return { 'conv0': conv_params(ni, no, 3), 'conv1': conv_params(no, no, 3), 'bn0': bnparams(ni), 'bn1': bnparams(no), 'convdim': conv_params(ni, no, 1) if ni != no else None, } def gen_group_params(ni, no, count): return { 'block%d' % i: gen_block_params(ni if i == 0 else no, no) for i in range(count) } def gen_group_stats(ni, no, count): return { 'block%d' % i: { 'bn0': bnstats(ni if i == 0 else no), 'bn1': bnstats(no) } for i in range(count) } params = nested_dict({ 'conv0': conv_params(3, 16, 3), 'group0': gen_group_params(16, widths[0], n), 'group1': gen_group_params(widths[0], widths[1], n), 'group2': gen_group_params(widths[1], widths[2], n), 'bn': bnparams(widths[2]), 'fc': linear_params(widths[2], num_classes), }) stats = nested_dict({ 'group0': gen_group_stats(16, widths[0], n), 'group1': gen_group_stats(widths[0], widths[1], n), 'group2': gen_group_stats(widths[1], widths[2], n), 'bn': bnstats(widths[2]), }) flat_params = OrderedDict() flat_stats = OrderedDict() for keys, v in params.iteritems_flat(): if v is not None: flat_params['.'.join(keys)] = Variable(v, requires_grad=True) for keys, v in stats.iteritems_flat(): flat_stats['.'.join(keys)] = v def activation(x, params, stats, base, mode): return F.relu( F.batch_norm(x, weight=params[base + '.weight'], bias=params[base + '.bias'], running_mean=stats[base + '.running_mean'], running_var=stats[base + '.running_var'], training=mode, momentum=0.1, eps=1e-5)) def block(x, params, stats, base, mode, stride): o1 = activation(x, params, stats, base + '.bn0', mode) y = F.conv2d(o1, params[base + '.conv0'], stride=stride, padding=1) o2 = activation(y, params, stats, base + '.bn1', mode) z = F.conv2d(o2, params[base + '.conv1'], stride=1, padding=1) if base + '.convdim' in params: return z + F.conv2d(o1, params[base + '.convdim'], stride=stride) else: return z + x def group(o, params, stats, base, mode, stride): for i in range(n): o = block(o, params, stats, '%s.block%d' % (base, i), mode, stride if i == 0 else 1) return o def f(input, params, stats, mode, prefix=''): x = F.conv2d(input, params[prefix + 'conv0'], padding=1) g0 = group(x, params, stats, prefix + 'group0', mode, 1) g1 = group(g0, params, stats, prefix + 'group1', mode, 2) g2 = group(g1, params, stats, prefix + 'group2', mode, 2) o = activation(g2, params, stats, prefix + 'bn', mode) o = F.avg_pool2d(o, 8, 1, 0) o = o.view(o.size(0), -1) o = F.linear(o, params[prefix + 'fc.weight'], params[prefix + 'fc.bias']) return o, [g0, g1, g2] return f, flat_params, flat_stats
def resnet(depth, width, num_classes): assert (depth - 4) % 6 == 0, 'depth should be 6n+4' n = (depth - 4) // 6 widths = [int(x * width) for x in [16, 32, 64]] def gen_block_params(ni, no): return { 'conv0': conv_params(ni, no, 3), 'conv1': conv_params(no, no, 3), 'bn0': bnparams(ni), 'bn1': bnparams(no), 'convdim': conv_params(ni, no, 1) if ni != no else None, } def gen_group_params(ni, no, count): return {'block%d' % i: gen_block_params(ni if i == 0 else no, no) for i in range(count)} def gen_group_stats(ni, no, count): return {'block%d' % i: {'bn0': bnstats(ni if i == 0 else no), 'bn1': bnstats(no)} for i in range(count)} flat_params = flatten_params({ 'conv0': conv_params(3,16,3), 'group0': gen_group_params(16, widths[0], n), 'group1': gen_group_params(widths[0], widths[1], n), 'group2': gen_group_params(widths[1], widths[2], n), 'bn': bnparams(widths[2]), 'fc': linear_params(widths[2], num_classes), }) flat_stats = flatten_stats({ 'group0': gen_group_stats(16, widths[0], n), 'group1': gen_group_stats(widths[0], widths[1], n), 'group2': gen_group_stats(widths[1], widths[2], n), 'bn': bnstats(widths[2]), }) def block(x, params, stats, base, mode, stride): o1 = F.relu(batch_norm(x, params, stats, base + '.bn0', mode), inplace=True) y = F.conv2d(o1, params[base + '.conv0'], stride=stride, padding=1) o2 = F.relu(batch_norm(y, params, stats, base + '.bn1', mode), inplace=True) z = F.conv2d(o2, params[base + '.conv1'], stride=1, padding=1) if base + '.convdim' in params: return z + F.conv2d(o1, params[base + '.convdim'], stride=stride) else: return z + x def group(o, params, stats, base, mode, stride): for i in range(n): o = block(o, params, stats, '%s.block%d' % (base,i), mode, stride if i == 0 else 1) return o def f(input, params, stats, mode): x = F.conv2d(input, params['conv0'], padding=1) g0 = group(x, params, stats, 'group0', mode, 1) g1 = group(g0, params, stats, 'group1', mode, 2) g2 = group(g1, params, stats, 'group2', mode, 2) o = F.relu(batch_norm(g2, params, stats, 'bn', mode)) o = F.avg_pool2d(o, 8, 1, 0) o = o.view(o.size(0), -1) o = F.linear(o, params['fc.weight'], params['fc.bias']) return o return f, flat_params, flat_stats
def resnet(depth, width, num_classes, stu_depth=0): assert (depth - 4) % 6 == 0, 'depth should be 6n+4' n = (depth - 4) // 6 if stu_depth != 0: assert (stu_depth - 4) % 6 == 0, 'student depth should be 6n+4' n_s = (stu_depth - 4) // 6 else: n_s = 0 widths = torch.Tensor([16, 32, 64]).mul(width).int() def gen_block_params(ni, no): return { 'conv0': conv_params(ni, no, 3), 'conv1': conv_params(no, no, 3), 'bn0': bnparams(ni), 'bn1': bnparams(no), 'bns0': bnparams(ni), 'bns1': bnparams(no), 'convdim': conv_params(ni, no, 1) if ni != no else None, } def gen_group_params(ni, no, count): return { 'block%d' % i: gen_block_params(ni if i == 0 else no, no) for i in range(count) } def gen_group_stats(ni, no, count): return { 'block%d' % i: { 'bn0': bnstats(ni if i == 0 else no), 'bn1': bnstats(no), 'bns0': bnstats(ni if i == 0 else no), 'bns1': bnstats(no) } for i in range(count) } if stu_depth != 0 and not opt.param_share: params = { 'conv0': conv_params(3, 16, 3), 'group0': gen_group_params(16, widths[0], n), 'group1': gen_group_params(widths[0], widths[1], n), 'group2': gen_group_params(widths[1], widths[2], n), 'groups0': gen_group_params(16, widths[0], n_s), 'groups1': gen_group_params(widths[0], widths[1], n_s), 'groups2': gen_group_params(widths[1], widths[2], n_s), 'bn': bnparams(widths[2]), 'bns': bnparams(widths[2]), 'fc': linear_params(widths[2], num_classes), 'fcs': linear_params(widths[2], num_classes), } stats = { 'group0': gen_group_stats(16, widths[0], n), 'group1': gen_group_stats(widths[0], widths[1], n), 'group2': gen_group_stats(widths[1], widths[2], n), 'groups0': gen_group_stats(16, widths[0], n_s), 'groups1': gen_group_stats(widths[0], widths[1], n_s), 'groups2': gen_group_stats(widths[1], widths[2], n_s), 'bn': bnstats(widths[2]), 'bns': bnstats(widths[2]), } else: params = { 'conv0': conv_params(3, 16, 3), 'group0': gen_group_params(16, widths[0], n), 'group1': gen_group_params(widths[0], widths[1], n), 'group2': gen_group_params(widths[1], widths[2], n), 'bn': bnparams(widths[2]), 'bns': bnparams(widths[2]), 'fc': linear_params(widths[2], num_classes), 'fcs': linear_params(widths[2], num_classes), } stats = { 'group0': gen_group_stats(16, widths[0], n), 'group1': gen_group_stats(widths[0], widths[1], n), 'group2': gen_group_stats(widths[1], widths[2], n), 'bn': bnstats(widths[2]), 'bns': bnstats(widths[2]), } flat_params = flatten_params(params) flat_stats = flatten_stats(stats) def block(x, params, stats, base, mode, stride, flag, drop_switch=True): if flag == 's': o1 = F.relu(batch_norm(x, params, stats, base + '.bns0', mode)) y = F.conv2d(o1, params[base + '.conv0'], stride=stride, padding=1) o2 = F.relu(batch_norm(y, params, stats, base + '.bns1', mode)) z = F.conv2d(o2, params[base + '.conv1'], stride=1, padding=1) if base + '.convdim' in params: return z + F.conv2d( o1, params[base + '.convdim'], stride=stride) else: return z + x o1 = F.relu(batch_norm(x, params, stats, base + '.bn0', mode)) y = F.conv2d(o1, params[base + '.conv0'], stride=stride, padding=1) o2 = F.relu(batch_norm(y, params, stats, base + '.bn1', mode)) if opt.dropout > 0 and drop_switch: o2 = F.dropout(o2, p=opt.dropout, training=mode) z = F.conv2d(o2, params[base + '.conv1'], stride=1, padding=1) if base + '.convdim' in params: return z + F.conv2d(o1, params[base + '.convdim'], stride=stride) else: return z + x def group(o, params, stats, base, mode, stride): for i in range(n): o = block(o, params, stats, '%s.block%d' % (base, i), mode, stride if i == 0 else 1, 't', False) return o def group_student(o, params, stats, base, mode, stride, n_layer): for i in range(n_layer): o = block(o, params, stats, '%s.block%d' % (base, i), mode, stride if i == 0 else 1, 's', False) return o def f(input, params, stats, mode, prefix=''): x = F.conv2d(input, params[prefix + 'conv0'], padding=1) g0 = group(x, params, stats, prefix + 'group0', mode, 1) g1 = group(g0, params, stats, prefix + 'group1', mode, 2) g2 = group(g1, params, stats, prefix + 'group2', mode, 2) o = F.relu(batch_norm(g2, params, stats, prefix + 'bn', mode)) o = F.avg_pool2d(o, 8, 1, 0) o = o.view(o.size(0), -1) o = F.linear(o, params[prefix + 'fc.weight'], params[prefix + 'fc.bias']) #x_s = F.conv2d(input, params[prefix+'conv0_s'], padding=1) if stu_depth != 0: if opt.param_share: gs0 = group_student(x, params, stats, prefix + 'group0', mode, 1, n_s) gs1 = group_student(gs0, params, stats, prefix + 'group1', mode, 2, n_s) gs2 = group_student(gs1, params, stats, prefix + 'group2', mode, 2, n_s) else: gs0 = group_student(x, params, stats, prefix + 'groups0', mode, 1, n_s) gs1 = group_student(gs0, params, stats, prefix + 'groups1', mode, 2, n_s) gs2 = group_student(gs1, params, stats, prefix + 'groups2', mode, 2, n_s) os = F.relu(batch_norm(gs2, params, stats, prefix + 'bns', mode)) os = F.avg_pool2d(os, 8, 1, 0) os = os.view(os.size(0), -1) os = F.linear(os, params[prefix + 'fcs.weight'], params[prefix + 'fcs.bias']) return os, o, [g0, g1, g2, gs0, gs1, gs2] else: return o, [g0, g1, g2] return f, flat_params, flat_stats
def resnet(depth, width, num_classes): assert (depth - 4) % 6 == 0, 'depth should be 6n+4' n = (depth - 4) // 6 widths = torch.Tensor([16, 32, 64]).mul(width).int().numpy().tolist() def gen_block_params(ni, no, scalar): if scalar: return { 'bn0': bnparams(ni), 'bn1': bnparams(no), } return { 'conv0': conv_params(ni, no, 3), 'conv1': conv_params(no, no, 3), 'convdim': conv_params(ni, no, 1) if ni != no else None, } def gen_group_params(ni, no, count, bias=False): return { 'block%d' % i: gen_block_params(ni if i == 0 else no, no, bias) for i in range(count) } def gen_group_stats(ni, no, count): return { 'block%d' % i: { 'bn0': bnstats(ni if i == 0 else no), 'bn1': bnstats(no) } for i in range(count) } flat_vectors = flatten_params({ 'conv0': conv_params(3, 16, 3), 'group0': gen_group_params(16, widths[0], n), 'group1': gen_group_params(widths[0], widths[1], n), 'group2': gen_group_params(widths[1], widths[2], n), 'conv1': conv_params(widths[2], num_classes, 1), }) flat_scalars = flatten_params({ 'group0': gen_group_params(16, widths[0], n, True), 'group1': gen_group_params(widths[0], widths[1], n, True), 'group2': gen_group_params(widths[1], widths[2], n, True), 'bn': bnparams(widths[2]), }) flat_stats = flatten_stats({ 'group0': gen_group_stats(16, widths[0], n), 'group1': gen_group_stats(widths[0], widths[1], n), 'group2': gen_group_stats(widths[1], widths[2], n), 'bn': bnstats(widths[2]), }) def block(x, params, stats, base, mode, stride): o1 = F.relu(batch_norm(x, params, stats, base + '.bn0', mode, 1.), inplace=True) y = F.conv2d(o1, params[base + '.conv0'], stride=stride, padding=1) o2 = F.relu(batch_norm(y, params, stats, base + '.bn1', mode, 1.), inplace=True) z = F.conv2d(o2, params[base + '.conv1'], stride=1, padding=1) if base + '.convdim' in params: return z + F.conv2d(o1, params[base + '.convdim'], stride=stride) else: return z + x def group(o, params, stats, base, mode, stride): for i in range(n): o = block(o, params, stats, '%s.block%d' % (base, i), mode, stride if i == 0 else 1) return o def f(input, params, stats, mode): x = F.conv2d(input, params['conv0'], padding=1) g0 = group(x, params, stats, 'group0', mode, 1) g1 = group(g0, params, stats, 'group1', mode, 2) g2 = group(g1, params, stats, 'group2', mode, 2) o = F.relu(batch_norm(g2, params, stats, 'bn', mode, 1.)) o = F.conv2d(o, params['conv1']) o = F.avg_pool2d(o, 8, 1, 0) o = o.view(o.size(0), -1) return o return f, flat_vectors, flat_scalars, flat_stats
def resnet(depth, width, num_classes): assert (depth - 4) % 6 == 0, 'depth should be 6n+4' n = (depth - 4) // 6 widths = torch.Tensor([16, 32, 64]).mul(width).int() def gen_block_params(ni, no): return { 'conv0': conv_params(ni, no, 3), 'conv1': conv_params(no, no, 3), 'bn0': bnparams(ni), 'bn1': bnparams(no), 'convdim': conv_params(ni, no, 1) if ni != no else None, } def gen_group_params(ni, no, count): return { 'block%d' % i: gen_block_params(ni if i == 0 else no, no) for i in range(count) } def gen_group_stats(ni, no, count): return { 'block%d' % i: { 'bn0': bnstats(ni if i == 0 else no), 'bn1': bnstats(no) } for i in range(count) } params = { 'conv0': conv_params(3, 16, 3), 'group0': gen_group_params(16, widths[0], n), 'group1': gen_group_params(widths[0], widths[1], n), 'group2': gen_group_params(widths[1], widths[2], n), 'bn': bnparams(widths[2]), 'fc': linear_params(widths[2], num_classes), } stats = { 'group0': gen_group_stats(16, widths[0], n), 'group1': gen_group_stats(widths[0], widths[1], n), 'group2': gen_group_stats(widths[1], widths[2], n), 'bn': bnstats(widths[2]), } flat_params = flatten_params(params) flat_stats = flatten_stats(stats) def activation(x, params, stats, base, mode): return F.relu(F.batch_norm(x, weight=params[base + '.weight'], bias=params[base + '.bias'], running_mean=stats[base + '.running_mean'], running_var=stats[base + '.running_var'], training=mode, momentum=0.1, eps=1e-5), inplace=True) def block(x, params, stats, base, mode, stride): o1 = activation(x, params, stats, base + '.bn0', mode) y = F.conv2d(o1, params[base + '.conv0'], stride=stride, padding=1) o2 = activation(y, params, stats, base + '.bn1', mode) z = F.conv2d(o2, params[base + '.conv1'], stride=1, padding=1) if base + '.convdim' in params: return z + F.conv2d(o1, params[base + '.convdim'], stride=stride) else: return z + x def group(o, params, stats, base, mode, stride): for i in range(n): o = block(o, params, stats, '%s.block%d' % (base, i), mode, stride if i == 0 else 1) return o def f(input, params, stats, mode): assert input.get_device() == params['conv0'].get_device() x = F.conv2d(input, params['conv0'], padding=1) g0 = group(x, params, stats, 'group0', mode, 1) g1 = group(g0, params, stats, 'group1', mode, 2) g2 = group(g1, params, stats, 'group2', mode, 2) o = activation(g2, params, stats, 'bn', mode) o = F.avg_pool2d(o, 8, 1, 0) o = o.view(o.size(0), -1) o = F.linear(o, params['fc.weight'], params['fc.bias']) return o return f, flat_params, flat_stats
def resnet(depth, width, num_classes): assert (depth - 4) % 6 == 0, 'depth should be 6n+4' n = (depth - 4) // 6 widths = torch.Tensor([16, 32, 64]).mul(width).int() def gen_block_params(ni, no): return { 'conv0': conv_params(ni, no, 3), 'conv1': conv_params(no, no, 3), 'bn0': bnparams(ni), 'bn1': bnparams(no), 'convdim': conv_params(ni, no, 1) if ni != no else None, } def gen_group_params(ni, no, count): return {'block%d' % i: gen_block_params(ni if i == 0 else no, no) for i in range(count)} def gen_group_stats(ni, no, count): return {'block%d' % i: {'bn0': bnstats(ni if i == 0 else no), 'bn1': bnstats(no)} for i in range(count)} flat_params = flatten_params({ 'conv0': conv_params(3,16,3), 'group0': gen_group_params(16, widths[0], n), 'group1': gen_group_params(widths[0], widths[1], n), 'group2': gen_group_params(widths[1], widths[2], n), 'bn': bnparams(widths[2]), 'fc': linear_params(widths[2], num_classes), }) flat_stats = flatten_stats({ 'group0': gen_group_stats(16, widths[0], n), 'group1': gen_group_stats(widths[0], widths[1], n), 'group2': gen_group_stats(widths[1], widths[2], n), 'bn': bnstats(widths[2]), }) def block(x, params, stats, base, mode, stride): o1 = F.relu(batch_norm(x, params, stats, base + '.bn0', mode), inplace=True) y = F.conv2d(o1, params[base + '.conv0'], stride=stride, padding=1) o2 = F.relu(batch_norm(y, params, stats, base + '.bn1', mode), inplace=True) z = F.conv2d(o2, params[base + '.conv1'], stride=1, padding=1) if base + '.convdim' in params: return z + F.conv2d(o1, params[base + '.convdim'], stride=stride) else: return z + x def group(o, params, stats, base, mode, stride): for i in range(n): o = block(o, params, stats, '%s.block%d' % (base,i), mode, stride if i == 0 else 1) return o def f(input, params, stats, mode): x = F.conv2d(input, params['conv0'], padding=1) g0 = group(x, params, stats, 'group0', mode, 1) g1 = group(g0, params, stats, 'group1', mode, 2) g2 = group(g1, params, stats, 'group2', mode, 2) o = F.relu(batch_norm(g2, params, stats, 'bn', mode)) o = F.avg_pool2d(o, 8, 1, 0) o = o.view(o.size(0), -1) o = F.linear(o, params['fc.weight'], params['fc.bias']) return o return f, flat_params, flat_stats
def resnet(depth, width, num_classes,activation): assert (depth - 4) % 6 == 0, 'depth should be 6n+4' n = (depth - 4) // 6 widths = torch.Tensor([16, 32, 64]).mul(width).int() actfun=None if activation=='swish': actfun=swish elif activation=='new': actfun=new elif activation=='elu': actfun=F.elu elif activation=='tanh': actfun=F.tanh elif activation=='lrelu': actfun=F.leaky_relu elif activation=='relu': actfun=F.relu def gen_block_params(ni, no): return { 'conv0': conv_params(ni, no, 3), 'conv1': conv_params(no, no, 3), 'bn0': bnparams(ni), 'bn1': bnparams(no), 'convdim': conv_params(ni, no, 1) if ni != no else None, } def gen_group_params(ni, no, count): return {'block%d' % i: gen_block_params(ni if i == 0 else no, no) for i in range(count)} def gen_group_stats(ni, no, count): return {'block%d' % i: {'bn0': bnstats(ni if i == 0 else no), 'bn1': bnstats(no)} for i in range(count)} flat_params = flatten_params({ 'conv0': conv_params(3,16,3), 'group0': gen_group_params(16, widths[0], n), 'group1': gen_group_params(widths[0], widths[1], n), 'group2': gen_group_params(widths[1], widths[2], n), 'bn': bnparams(widths[2]), 'fc': linear_params(widths[2], num_classes), }) flat_stats = flatten_stats({ 'group0': gen_group_stats(16, widths[0], n), 'group1': gen_group_stats(widths[0], widths[1], n), 'group2': gen_group_stats(widths[1], widths[2], n), 'bn': bnstats(widths[2]), }) def block(x, params, stats, base, mode, stride): o1 = actfun(batch_norm(x, params, stats, base + '.bn0', mode),0.2) y = F.conv2d(o1, params[base + '.conv0'], stride=stride, padding=1) o2 = actfun(batch_norm(y, params, stats, base + '.bn1', mode),0.2) z = F.conv2d(o2, params[base + '.conv1'], stride=1, padding=1) if base + '.convdim' in params: return z + F.conv2d(o1, params[base + '.convdim'], stride=stride) else: return z + x def group(o, params, stats, base, mode, stride): for i in range(n): o = block(o, params, stats, '%s.block%d' % (base,i), mode, stride if i == 0 else 1) return o def f(input, params, stats, mode): x = F.conv2d(input, params['conv0'], padding=1) g0 = group(x, params, stats, 'group0', mode, 1) g1 = group(g0, params, stats, 'group1', mode, 2) g2 = group(g1, params, stats, 'group2', mode, 2) o = actfun(batch_norm(g2, params, stats, 'bn', mode),0.2) o = F.avg_pool2d(o, 8, 1, 0) o = o.view(o.size(0), -1) o = F.linear(o, params['fc.weight'], params['fc.bias']) return o return f, flat_params, flat_stats
def gen_group_stats(ni, no, count): return {'block%d' % i: {'bn0': bnstats(ni if i == 0 else no), 'bn1': bnstats(no)} for i in range(count)}
def __init__(self, depth, width, ninputs=3, num_groups=3, num_classes=None, dropout=0.): super(WideResNet, self).__init__() self.depth = depth self.width = width self.num_groups = num_groups self.num_classes = num_classes self.dropout = dropout self.mode = True # Training #widths = torch.Tensor([16, 32, 64]).mul(width).int() widths = np.array([16, 32, 64]).astype(np.int) * width def gen_block_params(ni, no): return { 'conv0': conv_params(ni, no, 3), 'conv1': conv_params(no, no, 3), 'bn0': bnparams(ni), 'bn1': bnparams(no), 'convdim': conv_params(ni, no, 1) if ni != no else None, } def gen_group_params(ni, no, count): return { 'block%d' % i: gen_block_params(ni if i == 0 else no, no) for i in range(count) } def gen_group_stats(ni, no, count): return { 'block%d' % i: { 'bn0': bnstats(ni if i == 0 else no), 'bn1': bnstats(no) } for i in range(count) } params = {'conv0': conv_params(ni=ninputs, no=widths[0], k=3)} stats = {} for i in range(num_groups + 1): if i == 0: params.update({ 'group' + str(i): gen_group_params(widths[i], widths[i], depth) }) stats.update({ 'group' + str(i): gen_group_stats(widths[i], widths[i], depth) }) else: params.update({ 'group' + str(i): gen_group_params(widths[i - 1], widths[i], depth) }) stats.update({ 'group' + str(i): gen_group_stats(widths[i - 1], widths[i], depth) }) if num_classes is not None: params.update({'fc': linear_params(widths[i], num_classes)}) params.update({'bn': bnparams(widths[i])}) stats.update({'bn': bnstats(widths[i])}) params = flatten_params(params) stats = flatten_stats(stats) self.params = nn.ParameterDict({}) self.stats = nn.ParameterDict({}) for key in params.keys(): self.params.update( {key: nn.Parameter(params[key], requires_grad=True)}) for key in stats.keys(): self.stats.update( {key: nn.Parameter(stats[key], requires_grad=False)})
def resnet(depth, width, num_classes, is_full_wrn=True, is_fully_convolutional=False): #assert (depth - 4) % 6 == 0, 'depth should be 6n+4' #n = (depth - 4) // 6 #wrn = WideResNet(depth, width, ninputs=3,useCuda=True, num_groups=3, num_classes=num_classes) n = depth widths = torch.Tensor([16, 32, 64]).mul(width).int() def gen_block_params(ni, no): return { 'conv0': conv_params(ni, no, 3), 'conv1': conv_params(no, no, 3), 'bn0': bnparams(ni), 'bn1': bnparams(no), 'convdim': conv_params(ni, no, 1) if ni != no else None, } def gen_group_params(ni, no, count): return { 'block%d' % i: gen_block_params(ni if i == 0 else no, no) for i in range(count) } def gen_group_stats(ni, no, count): return { 'block%d' % i: { 'bn0': bnstats(ni if i == 0 else no), 'bn1': bnstats(no) } for i in range(count) } params = { 'conv0': conv_params(3, 16, 3), 'group0': gen_group_params(16, widths[0], n), 'group1': gen_group_params(widths[0], widths[1], n), 'group2': gen_group_params(widths[1], widths[2], n), 'bn': bnparams(widths[2]), 'fc': linear_params(widths[2], num_classes), } stats = { 'group0': gen_group_stats(16, widths[0], n), 'group1': gen_group_stats(widths[0], widths[1], n), 'group2': gen_group_stats(widths[1], widths[2], n), 'bn': bnstats(widths[2]), } if not is_full_wrn: ''' omniglot ''' params['bn'] = bnparams(widths[1]) #params['fc'] = linear_params(widths[1]*16*16, num_classes) params['fc'] = linear_params(widths[1], num_classes) stats['bn'] = bnstats(widths[1]) ''' # banknote params['bn'] = bnparams(widths[2]) #params['fc'] = linear_params(widths[2]*16*16, num_classes) params['fc'] = linear_params(widths[2], num_classes) stats['bn'] = bnstats(widths[2]) ''' flat_params = flatten_params(params) flat_stats = flatten_stats(stats) def activation(x, params, stats, base, mode): return F.relu(F.batch_norm(x, weight=params[base + '.weight'], bias=params[base + '.bias'], running_mean=stats[base + '.running_mean'], running_var=stats[base + '.running_var'], training=mode, momentum=0.1, eps=1e-5), inplace=True) def block(x, params, stats, base, mode, stride): o1 = activation(x, params, stats, base + '.bn0', mode) y = F.conv2d(o1, params[base + '.conv0'], stride=stride, padding=1) o2 = activation(y, params, stats, base + '.bn1', mode) z = F.conv2d(o2, params[base + '.conv1'], stride=1, padding=1) if base + '.convdim' in params: return z + F.conv2d(o1, params[base + '.convdim'], stride=stride) else: return z + x def group(o, params, stats, base, mode, stride): for i in range(n): o = block(o, params, stats, '%s.block%d' % (base, i), mode, stride if i == 0 else 1) return o def full_wrn(input, params, stats, mode): assert input.get_device() == params['conv0'].get_device() x = F.conv2d(input, params['conv0'], padding=1) g0 = group(x, params, stats, 'group0', mode, 1) g1 = group(g0, params, stats, 'group1', mode, 2) g2 = group(g1, params, stats, 'group2', mode, 2) o = activation(g2, params, stats, 'bn', mode) o = F.avg_pool2d(o, o.shape[2], 1, 0) o = o.view(o.size(0), -1) o = F.linear(o, params['fc.weight'], params['fc.bias']) return o def not_full_wrn(input, params, stats, mode): assert input.get_device() == params['conv0'].get_device() x = F.conv2d(input, params['conv0'], padding=1) g0 = group(x, params, stats, 'group0', mode, 1) g1 = group(g0, params, stats, 'group1', mode, 2) # omniglot o = activation(g1, params, stats, 'bn', mode) o = F.avg_pool2d(o, o.shape[2], 1, 0) # banknote ''' g2 = group(g1, params, stats, 'group2', mode, 2) o = activation(g2, params, stats, 'bn', mode) o = F.avg_pool2d(o, 16, 1, 0) ''' o = o.view(o.size(0), -1) o = F.linear(o, params['fc.weight'], params['fc.bias']) return o def fcn_full_wrn(input, params, stats, mode): assert input.get_device() == params['conv0'].get_device() x = F.conv2d(input, params['conv0'], padding=1) g0 = group(x, params, stats, 'group0', mode, 1) g1 = group(g0, params, stats, 'group1', mode, 2) g2 = group(g1, params, stats, 'group2', mode, 2) o = activation(g2, params, stats, 'bn', mode) return o def fcn_not_full_wrn(input, params, stats, mode): assert input.get_device() == params['conv0'].get_device() x = F.conv2d(input, params['conv0'], padding=1) g0 = group(x, params, stats, 'group0', mode, 1) g1 = group(g0, params, stats, 'group1', mode, 2) o = activation(g1, params, stats, 'bn', mode) return o if is_fully_convolutional: if is_full_wrn: return fcn_full_wrn, flat_params, flat_stats else: return fcn_not_full_wrn, flat_params, flat_stats else: if is_full_wrn: return full_wrn, flat_params, flat_stats else: return not_full_wrn, flat_params, flat_stats
def define_student(depth, width): definitions = { 18: [2, 2, 2, 2], 34: [3, 4, 6, 5], } assert depth in definitions.keys() widths = np.floor(np.asarray([64, 128, 256, 512]) * width).astype(np.int) blocks = definitions[depth] def batch_norm(x, params, stats, base, mode): return F.batch_norm(x, weight=params[base + '.weight'], bias=params[base + '.bias'], running_mean=stats[base + '.running_mean'], running_var=stats[base + '.running_var'], training=mode, momentum=0.1, eps=1e-5) def gen_block_params(ni, no): return { 'conv0': conv_params(ni, no, 3), 'conv1': conv_params(no, no, 3), 'bn0': bnparams(no), 'bn1': bnparams(no), 'convdim': conv_params(ni, no, 1) if ni != no else None, } def gen_group_params(ni, no, count): return { 'block%d' % i: gen_block_params(ni if i == 0 else no, no) for i in range(count) } def gen_group_stats(no, count): return { 'block%d' % i: { 'bn0': bnstats(no), 'bn1': bnstats(no) } for i in range(count) } params = nested_dict({ 'conv0': conv_params(3, 64, 7), 'bn0': bnparams(64), 'group0': gen_group_params(64, widths[0], blocks[0]), 'group1': gen_group_params(widths[0], widths[1], blocks[1]), 'group2': gen_group_params(widths[1], widths[2], blocks[2]), 'group3': gen_group_params(widths[2], widths[3], blocks[3]), 'fc': linear_params(widths[3], 1000), }) stats = nested_dict({ 'bn0': bnstats(64), 'group0': gen_group_stats(widths[0], blocks[0]), 'group1': gen_group_stats(widths[1], blocks[1]), 'group2': gen_group_stats(widths[2], blocks[2]), 'group3': gen_group_stats(widths[3], blocks[3]), }) # flatten parameters and additional buffers flat_params = OrderedDict() flat_stats = OrderedDict() for keys, v in params.iteritems_flat(): if v is not None: flat_params['.'.join(keys)] = Variable(v, requires_grad=True) for keys, v in stats.iteritems_flat(): flat_stats['.'.join(keys)] = v def block(x, params, stats, base, mode, stride): y = F.conv2d(x, params[base + '.conv0'], stride=stride, padding=1) o1 = F.relu(batch_norm(y, params, stats, base + '.bn0', mode), inplace=True) z = F.conv2d(o1, params[base + '.conv1'], stride=1, padding=1) o2 = batch_norm(z, params, stats, base + '.bn1', mode) if base + '.convdim' in params: return F.relu( o2 + F.conv2d(x, params[base + '.convdim'], stride=stride), inplace=True) else: return F.relu(o2 + x, inplace=True) def group(o, params, stats, base, mode, stride, n): for i in range(n): o = block(o, params, stats, '%s.block%d' % (base, i), mode, stride if i == 0 else 1) return o def f(input, params, stats, mode, pr=''): o = F.conv2d(input, params[pr + 'conv0'], stride=2, padding=3) o = F.relu(batch_norm(o, params, stats, pr + 'bn0', mode), inplace=True) o = F.max_pool2d(o, 3, 2, 1) g0 = group(o, params, stats, pr + 'group0', mode, 1, blocks[0]) g1 = group(g0, params, stats, pr + 'group1', mode, 2, blocks[1]) g2 = group(g1, params, stats, pr + 'group2', mode, 2, blocks[2]) g3 = group(g2, params, stats, pr + 'group3', mode, 2, blocks[3]) o = F.avg_pool2d(g3, 7) o = o.view(o.size(0), -1) o = F.linear(o, params[pr + 'fc.weight'], params[pr + 'fc.bias']) return o, [g0, g1, g2, g3] return f, flat_params, flat_stats
def gen_feature_stats(): dic = {} for i in range(len(_cfg)): if not _cfg[i] == 'M': dic['bn{0}'.format(i)] = bnstats(_cfg[i]) return dic