def __init__(self, in_channels=3, num_classes=1000):
super(InceptionNet, self).__init__()
self.conv1 = conv_block(in_channels=in_channels,
out_channels=64,
kernel_size=(7, 7),
stride=(2, 2),
padding=(3, 3))
self.maxpool1 = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)
self.conv2 = conv_block(64, 192, kernel_size=3, stride=1, padding=1)
self.maxpool2 = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)
# in_channels, out_1x1, red_3x3, out_3x3, red_5x5, out_5x5, out_1x1pool
self.inception3a = InceptionBlock(192, 64, 96, 128, 16, 32, 32)
self.inception3b = InceptionBlock(256, 128, 128, 192, 32, 96, 64)
self.maxpool3 = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)
self.inception4a = InceptionBlock(480, 192, 96, 208, 16, 48, 64)
self.inception4b = InceptionBlock(512, 160, 112, 224, 24, 64, 64)
self.inception4c = InceptionBlock(512, 128, 128, 256, 24, 64, 64)
self.inception4d = InceptionBlock(512, 112, 144, 288, 32, 64, 64)
self.inception4e = InceptionBlock(528, 256, 160, 320, 32, 128, 128)
self.maxpool4 = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)
self.inception5a = InceptionBlock(832, 256, 160, 320, 32, 128, 128)
self.inception5b = InceptionBlock(832, 384, 192, 384, 48, 128, 128)
self.avgpool = nn.AvgPool2d(kernel_size=7, stride=1)
self.dropout = nn.Dropout(p=0.4)
self.fc1 = nn.Linear(1024, 1000)
def forward_conv_CNN(self, inp, weights, reuse=False, scope=''):
# reuse is for the normalization parameters.
channels = self.channels
inp = tf.reshape(inp, [-1, self.img_size, self.img_size, channels])
hidden1 = conv_block(inp, weights['conv1'], weights['b1'], reuse,
scope + '0')
hidden2 = conv_block(hidden1, weights['conv2'], weights['b2'], reuse,
scope + '1')
hidden3 = conv_block(hidden2, weights['conv3'], weights['b3'], reuse,
scope + '2')
hidden4 = conv_block(hidden3, weights['conv4'], weights['b4'], reuse,
scope + '3')
# Flattening of blocks 3 and 4
hidden3 = tf.reshape(
hidden3,
[-1, np.prod([int(dim) for dim in hidden3.get_shape()[1:]])])
hidden4 = tf.reshape(
hidden4,
[-1, np.prod([int(dim) for dim in hidden4.get_shape()[1:]])])
# Concatenate
flatconcat34 = tf.concat(
[hidden3, hidden4],
axis=1) # keep batched (axis 0), concatenate columns (axis 1)
return flatconcat34
def forward_conv(self, inp, weights, reuse=False, scope=''):
# reuse is for the normalization parameters.
channels = self.channels
if FLAGS.datasource == 'kdd':
inp = tf.reshape(inp, [-1, self.dim_input, 1, 1])
else:
inp = tf.reshape(inp, [-1, self.img_size, self.img_size, channels])
hidden1 = conv_block(inp, weights['conv1'], weights['b1'], reuse,
scope + '0')
hidden2 = conv_block(hidden1, weights['conv2'], weights['b2'], reuse,
scope + '1')
hidden3 = conv_block(hidden2, weights['conv3'], weights['b3'], reuse,
scope + '2')
hidden4 = conv_block(hidden3, weights['conv4'], weights['b4'], reuse,
scope + '3')
if FLAGS.datasource == 'miniimagenet':
# last hidden layer is 6x6x64-ish, reshape to a vector
hidden4 = tf.reshape(
hidden4,
[-1,
np.prod([int(dim) for dim in hidden4.get_shape()[1:]])])
else:
hidden4 = tf.reduce_mean(hidden4, [1, 2])
return tf.matmul(hidden4, weights['w5']) + weights['b5']
def forward_conv_Ls(self, inp, L, s, reuse=False, scope=''):
# reuse is for the normalization parameters.
channels = self.channels
inp = tf.reshape(inp, [-1, self.img_size, self.img_size, channels])
w1 = tf.tensordot(L['conv1'], s['conv1'],1)
w2 = tf.tensordot(L['conv2'], s['conv2'],1)
w3 = tf.tensordot(L['conv3'], s['conv3'],1)
w4 = tf.tensordot(L['conv4'], s['conv4'],1)
w5 = tf.tensordot(L['w5'], s['w5'],1)
hidden1 = conv_block(inp, w1, L['b1'], reuse, scope+'0')
hidden2 = conv_block(hidden1, w2, L['b2'], reuse, scope+'1')
hidden3 = conv_block(hidden2, w3, L['b3'], reuse, scope+'2')
hidden4 = conv_block(hidden3, w4, L['b4'], reuse, scope+'3')
if FLAGS.datasource == 'miniimagenet' or FLAGS.datasource == 'cifar100':
# last hidden layer is 6x6x64-ish, reshape to a vector, last hidden layer of cifar100 is 2x2x32
hidden4 = tf.reshape(hidden4, [-1, np.prod([int(dim) for dim in hidden4.get_shape()[1:]])])
else:
hidden4 = tf.reduce_mean(hidden4, [1, 2])
return tf.matmul(hidden4, w5) + L['b5']
def forward_conv(self, inp, weights, reuse=False, scope=''):
# reuse is for the normalization parameters.
channels = self.channels
# -1表示第一个维度，可能是batch_size，数量根据数据量自动变化
'''
x = [1 2 3 4 5 6]
reshape(x, [3,-1])
>> [[1 2]
[3 4]
[5 6]]
'''
inp = tf.reshape(inp, [-1, self.img_size_x, self.img_size_y, channels])
hidden1 = conv_block(inp, weights['conv1'], weights['b1'], reuse,
scope + '0')
hidden2 = conv_block(hidden1, weights['conv2'], weights['b2'], reuse,
scope + '1')
hidden3 = conv_block(hidden2, weights['conv3'], weights['b3'], reuse,
scope + '2')
hidden4 = conv_block(hidden3, weights['conv4'], weights['b4'], reuse,
scope + '3')
print('hidden4_1:{}'.format(hidden4)) # (5, 1, 2, 32),
hidden4 = tf.reshape(
hidden4,
[-1, np.prod([int(dim) for dim in hidden4.get_shape()[1:]])])
print('hidden4_2:{}'.format(hidden4)) # (5, 64)
return tf.matmul(hidden4, weights['w5']) + weights['b5']
def forward_conv(self, inp, weights, prefix, reuse=False, scope=''):
# reuse is for the normalization parameters.
channels = self.channels
inp = tf.reshape(inp, [-1, self.img_size, self.img_size, channels])
hidden1 = conv_block(inp, weights['conv1'], weights['b1'], reuse,
scope + '0')
hidden2 = conv_block(hidden1, weights['conv2'], weights['b2'], reuse,
scope + '1')
hidden3 = conv_block(hidden2, weights['conv3'], weights['b3'], reuse,
scope + '2')
hidden4 = conv_block(hidden3, weights['conv4'], weights['b4'], reuse,
scope + '3')
if FLAGS.datasource == 'miniimagenet' or FLAGS.datasource == 'celeba':
# last hidden layer is 6x6x64-ish, reshape to a vector
hidden4 = tf.reshape(
hidden4,
[-1,
np.prod([int(dim) for dim in hidden4.get_shape()[1:]])])
else:
hidden4 = tf.reduce_mean(hidden4, [1, 2])
logits = tf.matmul(hidden4, weights['w5']) + weights['b5']
if 'val' in prefix:
logits = tf.gather(logits, tf.range(self.dim_output_val), axis=1)
return logits
def forward_alexnet(self, inp, weights, reuse=False):
# reuse is for the normalization parameters.
conv1 = conv_block(inp, weights['conv1_weights'], weights['conv1_biases'], stride_y=4, stride_x=4, groups=1, reuse=reuse, scope='conv1')
norm1 = lrn(conv1, 2, 1e-05, 0.75)
pool1 = max_pool(norm1, 3, 3, 2, 2, padding='VALID')
# 2nd Layer: Conv (w ReLu) -> Lrn -> Pool with 2 groups
conv2 = conv_block(pool1, weights['conv2_weights'], weights['conv2_biases'], stride_y=1, stride_x=1, groups=2, reuse=reuse, scope='conv2')
norm2 = lrn(conv2, 2, 1e-05, 0.75)
pool2 = max_pool(norm2, 3, 3, 2, 2, padding='VALID')
# 3rd Layer: Conv (w ReLu)
conv3 = conv_block(pool2, weights['conv3_weights'], weights['conv3_biases'], stride_y=1, stride_x=1, groups=1, reuse=reuse, scope='conv3')
# 4th Layer: Conv (w ReLu) splitted into two groups
conv4 = conv_block(conv3, weights['conv4_weights'], weights['conv4_biases'], stride_y=1, stride_x=1, groups=2, reuse=reuse, scope='conv4')
# 5th Layer: Conv (w ReLu) -> Pool splitted into two groups
conv5 = conv_block(conv4, weights['conv5_weights'], weights['conv5_biases'], stride_y=1, stride_x=1, groups=2, reuse=reuse, scope='conv5')
pool5 = max_pool(conv5, 3, 3, 2, 2, padding='VALID')
# 6th Layer: Flatten -> FC (w ReLu) -> Dropout
flattened = tf.reshape(pool5, [-1, 6 * 6 * 256])
fc6 = fc(flattened, weights['fc6_weights'], weights['fc6_biases'], activation='relu')
dropout6 = dropout(fc6, self.KEEP_PROB)
# 7th Layer: FC (w ReLu) -> Dropout
fc7 = fc(dropout6, weights['fc7_weights'], weights['fc7_biases'], activation='relu')
dropout7 = dropout(fc7, self.KEEP_PROB)
# 8th Layer: FC and return unscaled activations
fc8 = fc(dropout7, weights['fc8_weights'], weights['fc8_biases'])
return fc7, fc8
def forward_conv(self, inp, weights, reuse=True):
scope = ''
with tf.name_scope("forward_conv"):
hidden1 = conv_block(inp, weights['conv1'], weights['b1'], scope+'1', reuse)
hidden2 = conv_block(hidden1, weights['conv2'], weights['b2'], scope+'2',reuse)
hidden3 = conv_block(hidden2, weights['conv3'], weights['b3'], scope+'3',reuse)
hidden4 = conv_block(hidden3, weights['conv4'], weights['b4'], scope+'4',reuse)
return hidden4
def forward_conv(self, inp, weights, reuse=False, scope=''):
channels = self.channels
inp = tf.reshape(inp, [-1, self.img_size, self.img_size, channels])
hidden1 = conv_block(inp, weights['conv1'], weights['b1'], reuse, scope + '0')
hidden2 = conv_block(hidden1, weights['conv2'], weights['b2'], reuse, scope + '1')
hidden3 = conv_block(hidden2, weights['conv3'], weights['b3'], reuse, scope + '2')
hidden4 = conv_block(hidden3, weights['conv4'], weights['b4'], reuse, scope + '3')
hidden4 = tf.reshape(hidden4, [-1, np.prod([int(dim) for dim in hidden4.get_shape()[1:]])])
return tf.matmul(hidden4, weights['w5']) + weights['b5']
def forward_conv(self, inp, weights, reuse=False, scope=''): # reuse is for the normalization parameters. channels = self.channels inp = tf.reshape(inp, [-1, self.img_size, self.img_size, channels]) hidden1 = conv_block(inp, weights['conv1'], weights['b1'], reuse, scope+'0') hidden2 = conv_block(hidden1, weights['conv2'], weights['b2'], reuse, scope+'1') hidden3 = conv_block(hidden2, weights['conv3'], weights['b3'], reuse, scope+'2') hidden4 = conv_block(hidden3, weights['conv4'], weights['b4'], reuse, scope+'3') hidden4 = tf.reduce_mean(hidden4, [1, 2]) return tf.matmul(hidden4, weights['w5']) + weights['b5']
def forward_conv(self, inp, weights, reuse=False, scope=''):
channels = self.channels
inp = tf.reshape(inp, [-1, self.img_size, self.img_size, channels])
hidden1 = conv_block(inp, weights['conv1'], weights['b1'], reuse, scope + '0')
hidden2 = conv_block(hidden1, weights['conv2'], weights['b2'], reuse, scope + '1')
hidden3 = conv_block(hidden2, weights['conv3'], weights['b3'], reuse, scope + '2')
hidden4 = conv_block(hidden3, weights['conv4'], weights['b4'], reuse, scope + '3')
if FLAGS.datasource in ['miniimagenet', 'multidataset', 'multidataset_leave_one_out']:
hidden4 = tf.reshape(hidden4, [-1, np.prod([int(dim) for dim in hidden4.get_shape()[1:]])])
else:
hidden4 = tf.reduce_mean(hidden4, [1, 2])
return tf.matmul(hidden4, weights['w5']) + weights['b5']
def forward_conv(self,
inp,
weights,
reuse=False,
meta_testing=False,
scope=''):
# reuse is for the normalization parameters.
channels = self.channels
inp = tf.reshape(inp, [-1, self.img_size, self.img_size, channels])
hidden1 = conv_block(inp, weights['conv1'], weights['b1'], reuse,
scope + '0')
hidden2 = conv_block(hidden1, weights['conv2'], weights['b2'], reuse,
scope + '1')
hidden3 = conv_block(hidden2, weights['conv3'], weights['b3'], reuse,
scope + '2')
hidden4 = conv_block(hidden3, weights['conv4'], weights['b4'], reuse,
scope + '3')
if FLAGS.datasource == 'miniimagenet':
# last hidden layer is 6x6x64-ish, reshape to a vector
hidden4 = tf.reshape(
hidden4,
[-1,
np.prod([int(dim) for dim in hidden4.get_shape()[1:]])])
else:
hidden4 = tf.reduce_mean(hidden4, [1, 2])
hidden4 = tf.matmul(hidden4, weights['w5']) + weights['b5']
if meta_testing:
return hidden4
else:
temporal_num_filters = FLAGS.temporal_num_filters
num_temporal_layers = FLAGS.num_temporal_layers
temporal_num_filters = num_temporal_layers * [temporal_num_filters]
output = tf.expand_dims(tf.nn.softmax(hidden4, dim=1), axis=0)
for j in range(len(temporal_num_filters)):
if j != len(temporal_num_filters) - 1:
output = conv_block(output,
weights['w_1d_conv_2_head_%d' % j],
weights['b_1d_conv_2_head_%d' % j],
reuse,
scope + str(j + 5),
temporal=True)
else:
output = tf.nn.conv1d(
output,
weights['w_1d_conv_2_head_%d' % j],
stride=1,
padding='SAME') + weights['b_1d_conv_2_head_%d' % j]
return tf.squeeze(output)
def forward_conv(self, inp, weights, reuse=False, scope=''):
# reuse is for the normalization parameters.
channels = self.channels
inp = tf.reshape(inp, [-1, self.img_size, self.img_size, channels])
hidden1 = conv_block(inp, weights['conv1'], weights['b1'], reuse, scope+'0')
hidden2 = conv_block(hidden1, weights['conv2'], weights['b2'], reuse, scope+'1')
hidden3 = conv_block(hidden2, weights['conv3'], weights['b3'], reuse, scope+'2')
hidden4 = conv_block(hidden3, weights['conv4'], weights['b4'], reuse, scope+'3')
if FLAGS.datasource == 'miniimagenet':
# last hidden layer is 6x6x64-ish, reshape to a vector
hidden4 = tf.reshape(hidden4, [-1, np.prod([int(dim) for dim in hidden4.get_shape()[1:]])])
else:
hidden4 = tf.reduce_mean(hidden4, [1, 2])
return tf.matmul(hidden4, weights['w5']) + weights['b5']
def forward_conv(self, inp, weights, reuse=False, scope=''):
# reuse is for the normalization parameters.
channels = self.channels
inp = tf.reshape(inp, [-1, self.img_size, self.img_size, channels])
hidden1 = conv_block(inp, weights['conv1'], weights['b1'], reuse, scope+'0')
hidden2 = conv_block(hidden1, weights['conv2'], weights['b2'], reuse, scope+'1')
hidden3 = conv_block(hidden2, weights['conv3'], weights['b3'], reuse, scope+'2')
hidden4 = conv_block(hidden3, weights['conv4'], weights['b4'], reuse, scope+'3')
# last hidden layer is 6x6x64-ish, reshape to a vector
hidden4 = tf.reshape(hidden4, [-1, np.prod([int(dim) for dim in hidden4.get_shape()[1:]])])
print(hidden4.get_shape().as_list())
print(weights['w5'].get_shape().as_list())
return tf.matmul(hidden4, weights['w5']) + weights['b5']
def forward_conv(self, inp, weights, index=-1, reuse=False, scope=''):
# reuse is for the normalization parameters.
channels = self.channels
inp = tf.reshape(inp, [-1, self.img_size, self.img_size, channels])
e = index
hidden1 = conv_block(inp,
weights['conv1_' + str(e)],
weights['b1_' + str(e)],
reuse,
scope=str(e) + '_0')
hidden2 = conv_block(hidden1,
weights['conv2_' + str(e)],
weights['b2_' + str(e)],
reuse,
scope=str(e) + '_1')
hidden3 = conv_block(hidden2,
weights['conv3_' + str(e)],
weights['b3_' + str(e)],
reuse,
scope=str(e) + '_2')
hidden4 = conv_block(hidden3,
weights['conv4_' + str(e)],
weights['b4_' + str(e)],
reuse,
scope=str(e) + '_3')
if FLAGS.datasource == 'miniimagenet':
# last hidden layer is 6x6x64-ish, reshape to a vector
hidden4 = tf.reshape(
hidden4,
[-1,
np.prod([int(dim) for dim in hidden4.get_shape()[1:]])])
else:
hidden4 = tf.reduce_mean(hidden4, [1, 2])
output = tf.matmul(hidden4,
weights['w5_' + str(e)]) + weights['b5_' + str(e)]
return output
def forward_conv(self, inp, weights, reuse=False, scope=''):
# reuse is for the normalization parameters.
channels = self.channels
inp = tf.reshape(inp, [-1, self.img_size, self.img_size, channels])
hidden1 = conv_block(inp, weights['conv1'], weights['b1'], reuse,
scope + '0')
gamma1 = tf.matmul(weights['sigma_1'], weights['film_1a'])
beta1 = tf.matmul(weights['sigma_1'], weights['film_1b'])
film1 = film_block(hidden1, gamma1, beta1)
hidden2 = conv_block(film1, weights['conv2'], weights['b2'], reuse,
scope + '1')
gamma2 = tf.matmul(weights['sigma_1'], weights['film_2a'])
beta2 = tf.matmul(weights['sigma_1'], weights['film_2b'])
film2 = film_block(hidden2, gamma2, beta2)
hidden3 = conv_block(film2, weights['conv3'], weights['b3'], reuse,
scope + '2')
gamma3 = tf.matmul(weights['sigma_1'], weights['film_3a'])
beta3 = tf.matmul(weights['sigma_1'], weights['film_3b'])
film3 = film_block(hidden3, gamma3, beta3)
hidden4 = conv_block(film3, weights['conv4'], weights['b4'], reuse,
scope + '3')
gamma4 = tf.matmul(weights['sigma_1'], weights['film_4a'])
beta4 = tf.matmul(weights['sigma_1'], weights['film_4b'])
film4 = film_block(hidden4, gamma4, beta4)
if FLAGS.datasource == 'miniimagenet':
# last hidden layer is 6x6x64-ish, reshape to a vector
hidden4 = tf.reshape(
film4,
[-1,
np.prod([int(dim) for dim in hidden4.get_shape()[1:]])])
else:
hidden4 = tf.reduce_mean(film4, [1, 2])
return tf.matmul(hidden4, weights['w5']) + weights['b5']
def forward_conv(self, inp, weights, reuse=False, scope=''):
# reuse is for the normalization parameters.
channels = self.channels
inp = tf.reshape(inp, [-1, self.img_size, self.img_size, channels])
hidden1 = conv_block(inp, weights['conv1'], weights['b1'], reuse,
scope + '0')
# tinp = deconv_block(hidden1, weights['conv10'], weights['b10'], reuse, scope+'10')
hidden2 = conv_block(hidden1, weights['conv2'], weights['b2'], reuse,
scope + '1')
# thidden1 = deconv_block(hidden2, weights['conv9'], weights['b9'], reuse, scope+'9')
hidden3 = conv_block(hidden2, weights['conv3'], weights['b3'], reuse,
scope + '2')
# thidden2 = deconv_block(hidden3, weights['conv8'], weights['b8'], reuse, scope+'8')
hidden4 = conv_block(hidden3, weights['conv4'], weights['b4'], reuse,
scope + '3')
# thidden3 = deconv_block(hidden4, weights['conv7'], weights['b7'], reuse, scope+'7')
if FLAGS.datasource == 'miniimagenet':
# last hidden layer is 6x6x64-ish, reshape to a vector
hidden5 = tf.reshape(
hidden4,
[-1,
np.prod([int(dim) for dim in hidden4.get_shape()[1:]])])
else:
hidden5 = tf.reduce_mean(hidden4, [1, 2])
# output = tf.matmul(hidden4, weights['w5']) + weights['b5']
# thidden4 = tf.matmul(output, weights['w6']) + weights['b6']
# forward_loss = [self.loss_func(hidden1, thidden1), self.loss_func(hidden2, thidden2), \
# self.loss_func(hidden3, thidden3), self.loss_func(hidden4, thidden4), output]
# backward_loss = [conv_loss(inp, tinp), conv_loss(hidden1, thidden1), conv_loss(hidden2, thidden2), \
# conv_loss(hidden3, thidden3), conv_loss(hidden4, thidden4)]
return [
inp, hidden1, hidden2, hidden3, hidden4,
tf.matmul(hidden5, weights['w5']) + weights['b5']
]
def relation(self, fea, weights, reuse=False, scope=''):
# reuse is for the normalization parameters.
fea = tf.reshape(fea, [-1, 5, 5, 32])
re1 = conv_block(fea,
weights['conv-r1'],
weights['b-r1'],
reuse,
scope + 'r0',
max_pool_pad='SAME')
re2 = conv_block(re1,
weights['conv-r2'],
weights['b-r2'],
reuse,
scope + 'r1',
max_pool_pad='SAME')
re2 = tf.reshape(
re2, [-1, np.prod([int(dim) for dim in re2.get_shape()[1:]])])
re3 = tf.matmul(re2, weights['w-r3']) + weights['b-r3']
re3 = tf.nn.relu(re3)
re4 = tf.matmul(re3, weights['w-r4']) + weights['b-r4']
return re4
def get_embeddings(self, inp, weights, reuse=False, scope=''):
# reuse is for the normalization parameters.
channels = self.channels
inp = tf.reshape(inp, [-1, self.img_size, self.img_size, channels])
hidden1 = conv_block(inp, weights['conv1'], weights['b1'], reuse,
scope + '0')
hidden2 = conv_block(hidden1, weights['conv2'], weights['b2'], reuse,
scope + '1')
hidden3 = conv_block(hidden2, weights['conv3'], weights['b3'], reuse,
scope + '2')
hidden4 = conv_block(hidden3, weights['conv4'], weights['b4'], reuse,
scope + '3')
if FLAGS.datasource == 'miniimagenet' or FLAGS.datasource == 'isic':
# last hidden layer is 5x5x64, reshape to a vector
hidden4 = tf.reshape(
hidden4,
[-1,
np.prod([int(dim) for dim in hidden4.get_shape()[1:]])])
else:
hidden4 = tf.reduce_mean(hidden4, [1, 2])
return hidden4
def forward_conv(self, inp, weights, reuse=False, scope=''):
# reuse is for the normalization parameters.
if FLAGS.baseline and 'context_vector' == FLAGS.baseline:
channels = self.channels - 3 #self.context_size[-1]
context = tf.tile(weights['context_var'], [FLAGS.update_batch_size*FLAGS.num_classes, 1]) #
context = tf.reshape(context, [FLAGS.update_batch_size*FLAGS.num_classes, self.img_size,self.img_size,3])
else:
channels = self.channels
inp = tf.reshape(inp, [-1, self.img_size, self.img_size, channels])
if FLAGS.baseline and 'context_vector' == FLAGS.baseline:
inp = tf.concat([inp, context], 3)
hidden1 = conv_block(inp, weights['conv1'], weights['b1'], reuse, scope+'0')
hidden2 = conv_block(hidden1, weights['conv2'], weights['b2'], reuse, scope+'1')
hidden3 = conv_block(hidden2, weights['conv3'], weights['b3'], reuse, scope+'2')
hidden4 = conv_block(hidden3, weights['conv4'], weights['b4'], reuse, scope+'3')
if FLAGS.datasource == 'miniimagenet':
# last hidden layer is 6x6x64-ish, reshape to a vector
hidden4 = tf.reshape(hidden4, [-1, np.prod([int(dim) for dim in hidden4.get_shape()[1:]])])
else:
hidden4 = tf.reduce_mean(hidden4, [1, 2])
return tf.matmul(hidden4, weights['w5']) + weights['b5']
def forward(self, inp, weights, reuse=False, scope=''):
if FLAGS.vgg_path:
inp = inp * 255.0 - tf.convert_to_tensor(np.array([103.939, 116.779, 123.68], np.float32))
inp = inp[:, :, :, ::-1]
conv_layer = tf.reshape(inp, [-1, self.im_height, self.im_width, self.channels])
for i in range(self.num_conv_layers):
conv_layer = conv_block(conv_layer, weights['conv%d_w' % (i+1)], weights['conv%d_b' % (i+1)], reuse, scope + 'conv%d' % (i+1))
if FLAGS.fp:
batch_size, num_rows, num_cols, num_fp = conv_layer.get_shape()
num_rows, num_cols, num_fp = [int(x) for x in [num_rows, num_cols, num_fp]]
x_map = np.empty([num_rows, num_cols], np.float32)
y_map = np.empty([num_rows, num_cols], np.float32)
for i in range(num_rows):
for j in range(num_cols):
x_map[i, j] = (i - num_rows / 2.0) / num_rows
y_map[i, j] = (j - num_cols / 2.0) / num_cols
x_map = tf.convert_to_tensor(x_map)
y_map = tf.convert_to_tensor(y_map)
x_map = tf.reshape(x_map, [num_rows * num_cols])
y_map = tf.reshape(y_map, [num_rows * num_cols])
features = tf.reshape(tf.transpose(conv_layer, [0, 3, 1, 2]), [-1, num_rows * num_cols])
softmax = tf.nn.softmax(features)
fp_x = tf.reduce_sum(tf.multiply(x_map, softmax), [1], keep_dims=True)
fp_y = tf.reduce_sum(tf.multiply(y_map, softmax), [1], keep_dims=True)
conv_out_flat = tf.reshape(tf.concat([fp_x, fp_y], 1), [-1, num_fp * 2])
else:
conv_out_flat = tf.reshape(conv_layer, [-1, self.conv_out_size])
fc_layer = tf.reshape(conv_out_flat, [-1, self.conv_out_size])
for i in range(self.num_fc_layers - 1):
fc_layer = normalize(tf.matmul(fc_layer, weights['fc%d_w' % (i+1)]) + weights['fc%d_b' % (i+1)], tf.nn.relu, reuse, scope + 'fc%d' % (i+1))
logits = tf.matmul(fc_layer, weights['fc%d_w' % (self.num_fc_layers)]) + weights['fc%d_b' % (self.num_fc_layers)]
return logits
def protonet_graph(inputs, cfg):
P3, P4, P5 = inputs
# refine
R1 = conv_block(P3, 128)
R2 = conv_block(P4, 128)
R2 = KL.UpSampling2D(size=(2, 2), interpolation='bilinear', name="protonet_refine2upsampled")(R2)
R3 = conv_block(P5, 128)
R3 = KL.UpSampling2D(size=(4, 4), interpolation='bilinear', name="protonet_refine3upsampled")(R3)
bases = KL.Add(name="protonet_add")([R1, R2, R3])
# tower
for _ in range(3):
bases = conv_block(bases, 128)
bases = KL.UpSampling2D(2, interpolation='bilinear')(bases)
bases = conv_block(bases, 128)
bases = KL.Conv2D(4, kernel_size=(1, 1), strides=(1, 1), name='bases_out')(bases)
# seg_head
sem_out = P3
for _ in range(2):
sem_out = conv_block(sem_out, 128)
sem_out = KL.Conv2D(cfg.DATA.NUM_CLASSES, kernel_size=(1, 1), strides=(1, 1))(sem_out)
return bases, sem_out
def forward_conv_CNN(self,
inp,
weights,
reuse=False,
scope='',
is_training=False):
""" R2D2 model specification, CNN part:
4 conv blocks:
- 3x3 convolutions
- batch-normalization
- 2x2 max pooling
- leaky relu (factor 0.1)
- filters: [96, 192, 384, 512]
- dropout on layer 3 and 4
- concatenate flattened outputs of layer 3 and 4
Args:
<see forward_conv()>
"""
channels = self.channels
inp = tf.reshape(inp, [-1, self.img_size, self.img_size, channels])
hidden1 = conv_block(inp,
weights['conv1'],
weights['b1'],
reuse,
scope + '0',
activation=self.activation)
hidden2 = conv_block(hidden1,
weights['conv2'],
weights['b2'],
reuse,
scope + '1',
activation=self.activation)
hidden3 = conv_block(hidden2,
weights['conv3'],
weights['b3'],
reuse,
scope + '2',
activation=self.activation)
hidden3 = tf.layers.dropout(hidden3,
rate=self.dropout,
training=is_training)
hidden4 = conv_block(hidden3,
weights['conv4'],
weights['b4'],
reuse,
scope + '3',
activation=self.activation)
hidden4 = tf.layers.dropout(hidden4,
rate=self.dropout,
training=is_training)
# Flattening of blocks 3 and 4
hidden3 = tf.reshape(
hidden3,
[-1, np.prod([int(dim) for dim in hidden3.get_shape()[1:]])])
hidden4 = tf.reshape(
hidden4,
[-1, np.prod([int(dim) for dim in hidden4.get_shape()[1:]])])
# Concatenate
flatconcat34 = tf.concat(
[hidden3, hidden4],
axis=1) # keep batched (axis 0), concatenate columns (axis 1)
return flatconcat34
def fcos_head_graph(inputs, cfg, top_layer=None):
"""return FCOS graph
Arguments:
inputs: input tensors
cfg: dict
top_layer:
p:
Returns:
output tensors of FCOS
"""
# FCOS Head
cls_logits = []
reg_logits = []
ctr_logits = []
top_feats = []
input_head1 = KL.Input(shape=[None, None, 256], name='cls_head')
x = input_head1
for _ in range(4):
x = conv_block(x, filters=256)
_cls_logits = KL.Conv2D(cfg.DATA.NUM_CLASSES,
kernel_size=3,
strides=1,
padding="same",
name="cls_logits")(x)
_ctr_logits = KL.Conv2D(1,
kernel_size=3,
strides=1,
padding="same",
name="ctr_logits")(x)
_cls_logits = KL.Reshape(target_shape=[-1, cfg.DATA.NUM_CLASSES],
name="cls_logits_reshape")(_cls_logits)
_ctr_logits = KL.Reshape(target_shape=[-1, 1],
name="ctr_logits_reshape")(_ctr_logits)
cls_head = tf.keras.Model(inputs=[input_head1],
outputs=[_cls_logits, _ctr_logits],
name='cls_head')
input_head2 = KL.Input(shape=[None, None, 256], name='reg_head')
x = input_head2
for _ in range(4):
x = conv_block(x, filters=256)
_reg_logits = KL.Conv2D(4,
kernel_size=3,
strides=1,
padding="same",
name="reg_logits")(x)
reg_head = tf.keras.Model(inputs=[input_head2],
outputs=[_reg_logits],
name='reg_head')
for feature in inputs:
cls_feature = cls_head(feature)
reg_feature = reg_head(feature)
cls_logits.append(cls_feature[0])
ctr_logits.append(cls_feature[1])
reg_logits.append(KL.Reshape(target_shape=[-1, 4])(reg_feature))
if top_layer is not None:
top_feat = top_layer(reg_feature)
top_feat = KL.Reshape([-1, 784])(top_feat)
top_feats.append(top_feat)
# cls_logits = KL.Concatenate(axis=1)(cls_logits)
# reg_logits = KL.Concatenate(axis=1)(reg_logits)
# ctr_logits = KL.Concatenate(axis=1)(ctr_logits)
# top_feats = KL.Concatenate(axis=1)(top_feats)
return cls_logits, reg_logits, ctr_logits, top_feats
def build(self):
inLayer = Input([self.latentSize], self.batchSize)
x = Reshape((1, 1, self.latentSize))(inLayer)
x = UpSampling2D((self.inputShape[0]//32, self.inputShape[1]//32))(x)
x = conv_block(1024, kernelSize = 3)(x, training = self.training)
x = conv_block(512, kernelSize = 1)(x, training = self.training)
x = conv_block(1024, kernelSize = 3)(x, training = self.training)
x = conv_block(512, kernelSize = 1)(x, training = self.training)
x = conv_block(1024, kernelSize = 3)(x, training = self.training)
x = UpSampling2D((2, 2))(x)
x = conv_block(512, kernelSize = 3)(x, training = self.training)
x = conv_block(256, kernelSize = 1)(x, training = self.training)
x = conv_block(512, kernelSize = 3)(x, training = self.training)
x = conv_block(256, kernelSize = 1)(x, training = self.training)
x = conv_block(512, kernelSize = 3)(x, training = self.training)
x = UpSampling2D((2, 2))(x)
x = conv_block(256, kernelSize = 3)(x, training = self.training)
x = conv_block(128, kernelSize = 1)(x, training = self.training)
x = conv_block(256, kernelSize = 3)(x, training = self.training)
x = UpSampling2D((2, 2))(x)
x = conv_block(128, kernelSize = 3)(x, training = self.training)
x = conv_block(64, kernelSize = 1)(x, training = self.training)
x = conv_block(128, kernelSize = 3)(x, training = self.training)
x = UpSampling2D((2, 2))(x)
x = conv_block(64, kernelSize = 3)(x, training = self.training)
x = UpSampling2D((2, 2))(x)
x = conv_block(32, kernelSize = 3)(x, training = self.training)
x = conv_block(64, kernelSize = 1)(x, training = self.training)
x = Conv2D(filters = self.inputShape[-1], kernel_size = (1, 1),
padding = 'same', activation = "tanh")(x)
return Model(inLayer, x)
def build(self):
inLayer = Input(self.inputShape, self.batchSize)
x = conv_block(32, kernelSize = 3)(inLayer, training = self.training)
x = MaxPool2D((2, 2), strides = (2, 2))(x)
x = conv_block(64, kernelSize = 3)(x, training = self.training)
x = MaxPool2D((2, 2), strides = (2, 2))(x)
x = conv_block(128, kernelSize = 3)(x, training = self.training)
x = conv_block(64, kernelSize = 1)(x, training = self.training)
x = conv_block(128, kernelSize = 3)(x, training = self.training)
x = MaxPool2D((2, 2), strides = (2, 2))(x)
x = conv_block(256, kernelSize = 3)(x, training = self.training)
x = conv_block(128, kernelSize = 1)(x, training = self.training)
x = conv_block(256, kernelSize = 3)(x, training = self.training)
x = MaxPool2D((2, 2), strides = (2, 2))(x)
x = conv_block(512, kernelSize = 3)(x, training = self.training)
x = conv_block(256, kernelSize = 1)(x, training = self.training)
x = conv_block(512, kernelSize = 3)(x, training = self.training)
x = conv_block(256, kernelSize = 1)(x, training = self.training)
x = conv_block(512, kernelSize = 3)(x, training = self.training)
x = MaxPool2D((2, 2), strides=(2, 2))(x)
x = conv_block(1024, kernelSize = 3)(x, training = self.training)
x = conv_block(512, kernelSize = 1)(x, training = self.training)
x = conv_block(1024, kernelSize = 3)(x, training = self.training)
x = conv_block(512, kernelSize = 1)(x, training = self.training)
x = conv_block(1024, kernelSize = 3)(x, training = self.training)
mean = Conv2D(filters = self.latentSize, kernel_size = (1, 1),
padding = 'same')(x)
mean = GlobalAveragePooling2D()(mean)
logvar = Conv2D(filters = self.latentSize, kernel_size = (1, 1),
padding = 'same')(x)
logvar = GlobalAveragePooling2D()(logvar)
sample = latent_vector(self.latentConstraints, self.beta)([mean, logvar], training = self.training)
return Model(inputs = inLayer, outputs = sample)
# Model Architecture
"""
"""## Step 1: Learning to DownSample"""
# Input Layer
#[1242, 378]
nb_classes = 34
input_size_height = 512
input_size_width = 512
input_layer = tf.keras.layers.Input(shape=(input_size_height, input_size_width,
3),
name='input_layer')
#input_layer = tf.keras.layers.Input(shape=(2048, 1024, 3), name = 'input_layer')
lds_layer = utils.conv_block(input_layer, 'conv', 32, (3, 3), strides=(2, 2))
lds_layer = utils.conv_block(lds_layer, 'ds', 48, (3, 3), strides=(2, 2))
lds_layer = utils.conv_block(lds_layer, 'ds', 64, (3, 3), strides=(2, 2))
"""## Step 2: Global Feature Extractor
"""
"""#### Assembling all the methods"""
gfe_layer = utils.bottleneck_block(lds_layer, 64, (3, 3), t=6, strides=2, n=3)
gfe_layer = utils.bottleneck_block(gfe_layer, 96, (3, 3), t=6, strides=2, n=3)
gfe_layer = utils.bottleneck_block(gfe_layer, 128, (3, 3), t=6, strides=1, n=3)
gfe_layer = utils.pyramid_pooling_block(gfe_layer, [2, 4, 6, 8])
"""## Step 3: Feature Fusion"""
ff_layer1 = utils.conv_block(lds_layer,
'conv',