def __init__(self, num_layer, weight_decay, num_class):
super(Model, self).__init__(num_layer, weight_decay, num_class)
def kwargs(**kwargs):
return kwargs
setattr(
tcl.Conv2d, 'pre_defined',
kwargs(kernel_regularizer=tf.keras.regularizers.l2(weight_decay),
use_biases=False,
activation_fn=None))
setattr(
tcl.FC, 'pre_defined',
kwargs(kernel_regularizer=tf.keras.regularizers.l2(weight_decay),
activation_fn=tf.nn.softmax))
self.resnet_layers = {}
network_argments = {
18: {
'nb_resnet_layers': [2, 2, 2],
'depth': [16, 32, 64],
'strides': [1, 2, 2]
}
}
self.net_args = network_argments[num_layer]
self.net_name = 'ResNet'
with tf.name_scope(self.net_name):
self.resnet_layers[self.net_name + '/conv0'] = tcl.Conv2d(
[3, 3], self.net_args['depth'][0])
self.resnet_layers[self.net_name + '/bn0'] = tcl.BatchNorm(
activation_fn=tf.nn.relu)
for i, (nb_resnet_layers, depth, strides) in enumerate(
zip(self.net_args['nb_resnet_layers'],
self.net_args['depth'], self.net_args['strides'])):
for j in range(nb_resnet_layers):
block_name = '/BasicBlock%d.%d' % (i, j)
with tf.name_scope(block_name[1:]):
nb_name = self.net_name + block_name
if i != 0:
strides = 1
self.resnet_layers[nb_name + '/conv0'] = tcl.Conv2d(
[3, 3], depth, strides=strides)
self.resnet_layers[nb_name + '/bn0'] = tcl.BatchNorm(
activation_fn=tf.nn.relu)
self.resnet_layers[nb_name + '/conv1'] = tcl.Conv2d(
[3, 3], depth)
self.resnet_layers[nb_name + '/bn1'] = tcl.BatchNorm()
if strides > 1 or depth != self.net_args['depth'][max(
0, i - 1)]:
self.resnet_layers[nb_name +
'/conv2'] = tcl.Conv2d(
[1, 1],
depth,
strides=strides)
self.resnet_layers['FC'] = tcl.FC(num_class)
def __init__(self, args, model, teacher):
self.args = args
self.student = model
self.teacher = teacher
self.student_layers = self.sampled_layer(args.arch, self.student)
self.teacher_layers = self.sampled_layer(args.teacher_arch,
self.teacher)
def kwargs(**kwargs):
return kwargs
setattr(
tcl.Conv2d, 'pre_defined',
kwargs(kernel_initializer=tf.keras.initializers.he_normal(),
use_biases=False,
activation_fn=None,
trainable=True))
setattr(tcl.BatchNorm, 'pre_defined', kwargs(trainable=True))
self.student.aux_layers = [
tf.keras.Sequential([
tcl.Conv2d([1, 1], tl.gamma.shape[-1]),
tcl.BatchNorm(
param_initializers={
'gamma': tf.constant_initializer(1 /
tl.gamma.shape[-1])
})
]) for sl, tl in zip(self.student_layers, self.teacher_layers)
]
self.margin = 1.
self.weight = 1e-4
self.build()
def __init__(self, args, model, teacher):
self.args = args
self.student = model
self.teacher = teacher
self.student_layers = self.sampled_layer(args.arch, self.student)
self.teacher_layers = self.sampled_layer(args.teacher_arch,
self.teacher)
def kwargs(**kwargs):
return kwargs
setattr(
tcl.Conv2d, 'pre_defined',
kwargs(kernel_initializer=tf.keras.initializers.he_normal(),
use_biases=False,
activation_fn=None,
trainable=True))
setattr(tcl.BatchNorm, 'pre_defined', kwargs(trainable=True))
self.aux_layers = []
for s, t in zip(self.student_layers, self.teacher_layers):
layers = []
Ds = s.gamma.shape[-1]
for i in range(3):
layers.append(tcl.Conv2d([1, 1], Ds if i == 2 else Ds * 2))
layers.append(
tcl.BatchNorm(
activation_fn=None if i == 2 else tf.nn.relu))
self.aux_layers.append(tf.keras.Sequential(layers))
self.aux_layers[-1].alpha = self.aux_layers[-1].add_weight(
name='alpha',
shape=[1, 1, 1, Ds],
initializer=tf.constant_initializer(5.),
trainable=True)
def __init__(self, num_class, name='VGG', trainable=True, **kwargs):
super(Model, self).__init__(name=name, **kwargs)
def kwargs(**kwargs):
return kwargs
setattr(
tcl.Conv2d, 'pre_defined',
kwargs(use_biases=False, activation_fn=None, trainable=trainable))
setattr(tcl.BatchNorm, 'pre_defined',
kwargs(activation_fn=None, trainable=trainable))
setattr(tcl.FC, 'pre_defined', kwargs(trainable=trainable))
self.Layers = {}
self.filter_depth = [
64, 64, 128, 128, 256, 256, 256, 512, 512, 512, 512, 512, 512
]
for i, c in enumerate(self.filter_depth):
self.Layers['conv%d' % i] = tcl.Conv2d([3, 3],
c,
name='conv%d' % i)
self.Layers['bn%d' % i] = tcl.BatchNorm(name='bn%d' % i)
if i == 0:
self.Layers['conv%d' % i].type = 'input'
else:
self.Layers['conv%d' % i].type = 'mid'
self.max_pool = tf.keras.layers.MaxPool2D()
self.Layers['fc'] = tcl.FC(num_class, name='fc')
self.Layers['fc'].type = 'VGG_class'
def __init__(self, architecture, num_class, name = 'WResNet', trainable = True, **kwargs):
super(Model, self).__init__(name = name, **kwargs)
def kwargs(**kwargs):
return kwargs
setattr(tcl.Conv2d, 'pre_defined', kwargs(use_biases = False, activation_fn = None, trainable = trainable))
setattr(tcl.BatchNorm, 'pre_defined', kwargs(activation_fn = tf.nn.relu, trainable = trainable))
setattr(tcl.FC, 'pre_defined', kwargs(trainable = trainable))
self.Layers = {}
depth, widen_factor = architecture
self.nChannels = [16, 16*widen_factor, 32*widen_factor, 64*widen_factor]
self.stride = [1,2,2]
self.n = (depth-4)//6
self.Layers['conv'] = tcl.Conv2d([3,3], self.nChannels[0], name = 'conv')
in_planes = self.nChannels[0]
prev_conv_name = 'conv'
for i, (c, s) in enumerate(zip(self.nChannels[1:], self.stride)):
for j in range(self.n):
block_name = 'BasicBlock%d.%d/'%(i,j)
equalInOut = in_planes == c
in_planes = c
self.Layers[block_name + 'bn'] = tcl.BatchNorm(name = block_name + 'bn')
self.Layers[prev_conv_name.replace('conv','bn')] = self.Layers[block_name + 'bn']
if not(equalInOut):
self.Layers[block_name + 'conv2'] = tcl.Conv2d([1,1], c, strides = s if j == 0 else 1, name = block_name + 'conv2')
self.Layers[block_name + 'conv0'] = tcl.Conv2d([3,3], c, strides = s if j == 0 else 1, name = block_name + 'conv0')
self.Layers[block_name + 'bn0'] = tcl.BatchNorm(name = block_name + 'bn0')
self.Layers[block_name + 'conv1'] = tcl.Conv2d([3,3], c, strides = 1, name = block_name + 'conv1')
prev_conv_name = block_name + 'conv1'
if not(equalInOut):
prev_conv_name = block_name + 'conv2'
self.Layers['bn_last']= tcl.BatchNorm(name = 'bn_last')
self.Layers[prev_conv_name.replace('conv','bn')] = self.Layers['bn_last']
self.Layers['fc'] = tcl.FC(num_class, name = 'fc')
def __init__(self, target, source):
super(FitNet, self).__init__(target, source)
self.source = source
self.target = target
self.linear_map = []
self.feat_name = 'feature'
for t, s in zip(target, source):
Ds = source.shape[-1]
Dt = target.shape[-1]
if Ds != Dt:
self.linearmap.append(tcl.Conv2d([3,3], Dt, activation_fn = None))
else:
self.linear_map.append(None)
def __init__(self, target, source, weight_decay, **kwargs):
super(VID, self).__init__(**kwargs)
self.source = source
self.target = target
self.linear_map = []
self.alpha = []
self.feat_name = 'feature'
for t, s in zip(target.feature, source.feature):
Ds = s.shape[-1]
layers = []
for i in range(3):
tcl.Conv2d([1, 1], Ds if i == 2 else Ds * 2)
tcl.BatchNorm(activation_fn=None if i == 2 else tf.nn.relu)
self.linear_map.append(tf.keras.Sequential(layers))
self.alpha.append(
self.add_weight(name='alpha',
shape=[1, 1, 1, Ds],
initializer=tf.constant_initializer(5.)))
def __init__(self,
target,
source,
weight_decay,
margin=1.,
weight=3e-3,
**kwargs):
super(AB, self).__init__(**kwargs)
self.margin = margin
self.weight = weight
self.linear_map = []
self.feat_name = 'feature_noact'
def kwargs(**kwargs):
return kwargs
setattr(
tcl.Conv2d, 'pre_defined',
kwargs(kernel_regularizer=tf.keras.regularizers.l2(weight_decay),
use_biases=False,
activation_fn=None))
setattr(
tcl.BatchNorm, 'pre_defined',
kwargs(
param_regularizers={
'gamma': tf.keras.regularizers.l2(weight_decay),
'beta': tf.keras.regularizers.l2(weight_decay)
}))
for t, s in zip(target.feature_noact, source.feature_noact):
Ds = s.shape[-1]
Dt = t.shape[-1]
if Ds != Dt:
self.linear_map.append(
tf.keras.Sequential(
[tcl.Conv2d([3, 3], Ds),
tcl.BatchNorm()]))
else:
self.linear_map.append(None)
def __init__(self, args, model, teacher):
self.args = args
self.student = model
self.teacher = teacher
self.student_layers = self.sampled_layer(args.arch, self.student)
self.teacher_layers = self.sampled_layer(args.teacher_arch,
self.teacher)
def kwargs(**kwargs):
return kwargs
setattr(
tcl.Conv2d, 'pre_defined',
kwargs(kernel_initializer=tf.keras.initializers.he_normal(),
use_biases=False,
activation_fn=None,
trainable=True))
setattr(tcl.BatchNorm, 'pre_defined', kwargs(trainable=True))
self.aux_layers = [
tf.keras.Sequential(
[tcl.Conv2d([1, 1], tl.gamma.shape[-1]),
tcl.BatchNorm()])
for sl, tl in zip(self.student_layers, self.teacher_layers)
]
def get_margin(s, m):
s, m = np.abs(s.numpy()), m.numpy()
margin = np.where(
norm.cdf(-m / s) > 1e-3, -s * np.exp(-(m / s)**2 / 2) /
np.sqrt(2 * np.pi) / norm.cdf(-m / s) + m,
-3 * s).astype(np.float32)
return tf.constant(margin)
self.margins = [
get_margin(tl.gamma, tl.beta) for tl in self.teacher_layers
]
def __init__(self,
architecture,
weight_decay,
num_class,
name='WResNet',
trainable=True,
**kwargs):
super(Model, self).__init__(name=name, **kwargs)
def kwargs(**kwargs):
return kwargs
setattr(
tcl.Conv2d, 'pre_defined',
kwargs(kernel_regularizer=tf.keras.regularizers.l2(weight_decay),
use_biases=False,
activation_fn=None,
trainable=trainable))
setattr(
tcl.BatchNorm, 'pre_defined',
kwargs(param_regularizers={
'gamma': tf.keras.regularizers.l2(weight_decay),
'beta': tf.keras.regularizers.l2(weight_decay)
},
trainable=trainable))
setattr(
tcl.FC, 'pre_defined',
kwargs(kernel_regularizer=tf.keras.regularizers.l2(weight_decay),
biases_regularizer=tf.keras.regularizers.l2(weight_decay),
trainable=trainable))
self.wresnet_layers = {}
depth, widen_factor = architecture
self.nChannels = [
16, 16 * widen_factor, 32 * widen_factor, 64 * widen_factor
]
self.stride = [1, 2, 2]
self.n = (depth - 4) // 6
self.wresnet_layers['conv0'] = tcl.Conv2d([3, 3],
self.nChannels[0],
name='conv0')
self.wresnet_layers['bn0'] = tcl.BatchNorm(activation_fn=tf.nn.relu,
name='bn0')
in_planes = self.nChannels[0]
for i, (c, s) in enumerate(zip(self.nChannels[1:], self.stride)):
for j in range(self.n):
block_name = 'BasicBlock%d.%d' % (i, j)
with tf.name_scope(block_name):
equalInOut = in_planes == c
self.wresnet_layers[block_name +
'/bn0'] = tcl.BatchNorm(name='bn0')
self.wresnet_layers[block_name + '/conv1'] = tcl.Conv2d(
[3, 3], c, strides=s if j == 0 else 1, name='conv1')
self.wresnet_layers[block_name + '/bn1'] = tcl.BatchNorm(
activation_fn=tf.nn.relu, name='bn1')
self.wresnet_layers[block_name +
'/drop'] = tcl.Dropout(0.7)
self.wresnet_layers[block_name + '/conv2'] = tcl.Conv2d(
[3, 3], c, strides=1, name='conv2')
if not (equalInOut):
self.wresnet_layers[block_name +
'/conv3'] = tcl.Conv2d(
[1, 1],
c,
strides=s if j == 0 else 1,
name='conv3')
in_planes = c
self.wresnet_layers['bn1'] = tcl.BatchNorm(name='bn1')
self.wresnet_layers['fc'] = tcl.FC(num_class, name='fc')
def __init__(self,
num_layers,
num_class,
name='WResNet',
trainable=True,
**kwargs):
super(Model, self).__init__(name=name, **kwargs)
def kwargs(**kwargs):
return kwargs
setattr(
tcl.Conv2d, 'pre_defined',
kwargs(use_biases=False, activation_fn=None, trainable=trainable))
setattr(tcl.BatchNorm, 'pre_defined', kwargs(trainable=trainable))
setattr(tcl.FC, 'pre_defined', kwargs(trainable=trainable))
self.Layers = {}
network_argments = {
56: {
'blocks': [9, 9, 9],
'depth': [16, 32, 64],
'strides': [1, 2, 2]
}
}
self.net_args = network_argments[num_layers]
self.Layers['conv'] = tcl.Conv2d([3, 3],
self.net_args['depth'][0],
name='conv',
layertype='input')
self.Layers['bn'] = tcl.BatchNorm(name='bn')
in_depth = self.net_args['depth'][0]
for i, (nb_resnet_layers, depth, strides) in enumerate(
zip(self.net_args['blocks'], self.net_args['depth'],
self.net_args['strides'])):
for j in range(nb_resnet_layers):
name = '/BasicBlock%d.%d/' % (i, j)
if j != 0:
strides = 1
if strides > 1 or depth != in_depth:
self.Layers[name + 'conv2'] = tcl.Conv2d([1, 1],
depth,
strides=strides,
name=name +
'conv2')
self.Layers[name + 'bn2'] = tcl.BatchNorm(name=name +
'bn2')
self.Layers[name + 'conv0'] = tcl.Conv2d([3, 3],
depth,
strides=strides,
name=name + 'conv0')
self.Layers[name + 'bn0'] = tcl.BatchNorm(name=name + 'bn0')
self.Layers[name + 'conv1'] = tcl.Conv2d([3, 3],
depth,
name=name + 'conv1')
self.Layers[name + 'bn1'] = tcl.BatchNorm(name=name + 'bn1')
in_depth = depth
self.Layers['fc'] = tcl.FC(num_class, name='fc')
def __init__(self, architecture, weight_decay, num_class):
super(Model, self).__init__(architecture, weight_decay, num_class)
def kwargs(**kwargs):
return kwargs
setattr(
tcl.Conv2d, 'pre_defined',
kwargs(kernel_regularizer=tf.keras.regularizers.l2(weight_decay),
use_biases=False,
activation_fn=None))
setattr(
tcl.BatchNorm, 'pre_defined',
kwargs(
param_regularizers={
'gamma': tf.keras.regularizers.l2(weight_decay),
'beta': tf.keras.regularizers.l2(weight_decay)
}))
setattr(
tcl.FC, 'pre_defined',
kwargs(kernel_regularizer=tf.keras.regularizers.l2(weight_decay),
biases_regularizer=tf.keras.regularizers.l2(weight_decay)))
self.wresnet_layers = {}
depth, widen_factor = architecture
self.nChannels = [
16, 16 * widen_factor, 32 * widen_factor, 64 * widen_factor
]
self.stride = [1, 2, 2]
self.n = (depth - 4) // 6
self.net_name = 'WResNet'
self.feature = []
self.feature_noact = []
self.last_embedded = []
self.logits = []
with tf.name_scope(self.net_name):
self.wresnet_layers[self.net_name + '/conv0'] = tcl.Conv2d(
[3, 3], self.nChannels[0])
self.wresnet_layers[self.net_name + '/bn0'] = tcl.BatchNorm(
activation_fn=tf.nn.relu)
in_planes = self.nChannels[0]
for i, (c, s) in enumerate(zip(self.nChannels[1:], self.stride)):
for j in range(self.n):
block_name = '/BasicBlock%d.%d' % (i, j)
with tf.name_scope(block_name[1:]):
equalInOut = in_planes == c
nb_name = self.net_name + block_name
self.wresnet_layers[nb_name + '/bn0'] = tcl.BatchNorm()
self.wresnet_layers[nb_name + '/conv1'] = tcl.Conv2d(
[3, 3], c, strides=s if j == 0 else 1)
self.wresnet_layers[nb_name + '/bn1'] = tcl.BatchNorm(
activation_fn=tf.nn.relu)
self.wresnet_layers[nb_name +
'/drop'] = tcl.Dropout(0.7)
self.wresnet_layers[nb_name + '/conv2'] = tcl.Conv2d(
[3, 3], c, strides=1)
if not (equalInOut):
self.wresnet_layers[nb_name +
'/conv3'] = tcl.Conv2d(
[1, 1],
c,
strides=s if j == 0 else 1)
in_planes = c
self.wresnet_layers[self.net_name + '/bn1'] = tcl.BatchNorm()
self.wresnet_layers['FC'] = tcl.FC(num_class)
def __init__(self,
num_layers,
num_class,
name='ResNet',
trainable=True,
**kwargs):
super(Model, self).__init__(name=name, **kwargs)
def kwargs(**kwargs):
return kwargs
setattr(
tcl.Conv2d, 'pre_defined',
kwargs(use_biases=False, activation_fn=None, trainable=trainable))
setattr(tcl.BatchNorm, 'pre_defined', kwargs(trainable=trainable))
setattr(tcl.FC, 'pre_defined', kwargs(trainable=trainable))
self.num_layers = num_layers
self.Layers = {}
network_argments = {
## ILSVRC
18: {
'blocks': [2, 2, 2, 2],
'depth': [64, 128, 256, 512],
'strides': [1, 2, 2, 2]
},
50: {
'blocks': [3, 4, 6, 3],
'depth': [64, 128, 256, 512],
'strides': [1, 2, 2, 2]
},
## CIFAR
56: {
'blocks': [9, 9, 9],
'depth': [16, 32, 64],
'strides': [1, 2, 2]
},
}
self.net_args = network_argments[self.num_layers]
if num_class == 1000:
self.Layers['conv'] = tcl.Conv2d([7, 7],
self.net_args['depth'][0],
strides=2,
name='conv')
self.Layers['bn'] = tcl.BatchNorm(name='bn')
self.maxpool_3x3 = tf.keras.layers.MaxPool2D((3, 3),
strides=2,
padding='SAME')
else:
self.Layers['conv'] = tcl.Conv2d([3, 3],
self.net_args['depth'][0],
name='conv')
self.Layers['bn'] = tcl.BatchNorm(name='bn')
self.expansion = 1 if self.num_layers in {18, 56} else 4
in_depth = self.net_args['depth'][0]
for i, (nb_resnet_layers, depth, strides) in enumerate(
zip(self.net_args['blocks'], self.net_args['depth'],
self.net_args['strides'])):
for j in range(nb_resnet_layers):
name = 'BasicBlock%d.%d/' % (i, j)
if j != 0:
strides = 1
if strides > 1 or depth * self.expansion != in_depth:
self.Layers[name + 'conv3'] = tcl.Conv2d(
[1, 1],
depth * self.expansion,
strides=strides,
name=name + 'conv3')
self.Layers[name + 'bn3'] = tcl.BatchNorm(name=name +
'bn3')
if self.num_layers in {18, 56}:
self.Layers[name + 'conv1'] = tcl.Conv2d([3, 3],
depth,
strides=strides,
name=name +
'conv1')
self.Layers[name + 'bn1'] = tcl.BatchNorm(name=name +
'bn1')
self.Layers[name + 'conv2'] = tcl.Conv2d(
[3, 3], depth * self.expansion, name=name + 'conv2')
self.Layers[name + 'bn2'] = tcl.BatchNorm(name=name +
'bn2')
else:
self.Layers[name + 'conv0'] = tcl.Conv2d([1, 1],
depth,
name=name +
'conv0')
self.Layers[name + 'bn0'] = tcl.BatchNorm(name=name +
'bn0')
self.Layers[name + 'conv1'] = tcl.Conv2d([3, 3],
depth,
strides=strides,
name=name +
'conv1')
self.Layers[name + 'bn1'] = tcl.BatchNorm(name=name +
'bn1')
self.Layers[name + 'conv2'] = tcl.Conv2d(
[1, 1], depth * self.expansion, name=name + 'conv2')
self.Layers[name + 'bn2'] = tcl.BatchNorm(name=name +
'bn2', )
#param_initializers = {'gamma': tf.keras.initializers.Zeros()})
in_depth = depth * self.expansion
self.Layers['fc'] = tcl.FC(num_class, name='fc')
