def logits_layer(self, x):
''' Logits layer to further produce softmax. '''
with tf.variable_scope('logits'):
logits = common_layers.linear(
x, 'logits-matmul',
[x.shape[-1].value, self.taskconf['classes']['num']])
return logits
def model(self, feats, labels):
''' Build the model. '''
x = self.resnet(feats)
with tf.variable_scope("avg_pooling"):
batch_t = tf.shape(x)[0]
time_t = tf.shape(x)[1]
feat, channel = x.shape.as_list()[2:]
x = tf.reshape(x, [batch_t, time_t, feat * channel])
x = self.pooling_layer(x, pooling_type='average')
with tf.variable_scope("output_layer"):
shape = x.shape.as_list()
shape = shape[-1]
hidden_dims = self.params().embedding_size
y = x
y = common_layers.linear(y,
'dense-matmul', [shape, hidden_dims],
has_bias=True)
y = tf.layers.batch_normalization(y,
axis=-1,
momentum=0.99,
training=self.train,
name='dense-bn')
embedding = y
dense_output = y
logits = self.logits_layer(dense_output, labels)
model_outputs = {'logits': logits, 'embeddings': embedding}
return model_outputs
def linear_block(self, x):
'''
linear layer for dim reduction
x: shape [batch, time, feat, channel]
output: shape [b, t, f]
'''
with tf.variable_scope('linear'):
times, feat, channel = x.shape.as_list()[1:]
x = tf.reshape(x, [-1, feat * channel])
if self.netconf['use_dropout']:
x = tf.layers.dropout(x,
self.netconf['dropout_rate'],
training=self.train)
x = common_layers.linear(
x, 'linear1', [feat * channel, self.netconf['linear_num']])
x = tf.nn.relu(x)
if self.netconf['use_bn']:
bn_name = 'bn_linear'
x = tf.layers.batch_normalization(x,
axis=-1,
momentum=0.9,
training=self.train,
name=bn_name)
x = tf.reshape(x, [-1, times, self.netconf['linear_num']])
return x
def dense_layer(self, x):
''' Embedding layers. '''
with tf.variable_scope('dense'):
shape = x.shape[-1].value
if 'hidden_dims' in self.netconf:
hidden_dims = self.netconf['hidden_dims']
else:
hidden_dims = [self.netconf['hidden1']]
hidden_idx = 1
y = x
for hidden in hidden_dims:
y = common_layers.linear(y, 'dense-matmul-%d' % (hidden_idx),
[shape, hidden])
shape = hidden
y = tf.nn.relu(y)
if self.netconf['use_bn']:
y = tf.layers.batch_normalization(y,
axis=-1,
momentum=0.99,
training=self.train,
name='dense-bn-%d' %
(hidden_idx))
if self.netconf['use_dropout']:
y = tf.layers.dropout(y,
self.netconf['dropout_rate'],
training=self.train)
hidden_idx += 1
return y
def linear_block(self, x):
'''
linear layer for dim reduction
x: shape [batch, time, feat, channel]
output: shape [b, t, f]
'''
times_t = tf.shape(x)[1]
feat, channel = x.shape.as_list()[2:]
linear_num = self.netconf['linear_num']
if linear_num > 0:
with tf.variable_scope('linear'):
x = tf.reshape(x, [-1, feat * channel])
if self.netconf['use_dropout']:
x = tf.layers.dropout(x,
self.netconf['dropout_rate'],
training=self.train)
x = common_layers.linear(x, 'linear1',
[feat * channel, linear_num])
x = tf.nn.relu(x)
if self.netconf['use_bn']:
bn_name = 'bn_linear'
x = tf.layers.batch_normalization(x,
axis=-1,
momentum=0.9,
training=self.train,
name=bn_name)
else:
logging.info('linear_num <= 0, only apply reshape.')
x = tf.reshape(x, [-1, times_t, feat * channel])
return x
def logits_layer(self, x):
''' output layers'''
with tf.variable_scope('logits'):
logits = common_layers.linear(
x, 'logits-matmul',
[self.netconf['hidden1'], self.taskconf['classes']['num']])
return logits
def dense_layer(self, x):
''' Embedding layers. '''
with tf.variable_scope('dense'):
shape = x.shape[-1].value
hidden_dims = self.netconf['hidden_dims']
y = x
use_bn = self.netconf['use_bn']
remove_nonlin = self.netconf['remove_last_nonlinearity']
for idx, hidden in enumerate(hidden_dims):
last_layer = idx == (len(hidden_dims) - 1)
y = common_layers.linear(
y,
'dense-matmul-%d' % (idx + 1), [shape, hidden],
has_bias=not use_bn)
shape = hidden
embedding = y
if not last_layer or not remove_nonlin:
y = tf.nn.relu(y)
if use_bn:
y = tf.layers.batch_normalization(
y,
axis=-1,
momentum=0.99,
training=self.train,
name='dense-bn-%d' % (idx + 1))
if self.netconf['use_dropout'] and not remove_nonlin:
y = tf.layers.dropout(
y, self.netconf['dropout_rate'], training=self.train)
if self.netconf['embedding_after_linear']:
logging.info('Output embedding right after linear layer.')
else:
logging.info('Output embedding after non-lin, batch norm and dropout.')
embedding = y
return embedding, y
def dense_layer(self, x):
''' fc layers'''
with tf.variable_scope('dense'):
shape = x.shape[-1].value
y = common_layers.linear(x, 'dense-matmul',
[shape, self.netconf['hidden1']])
if self.netconf['use_bn']:
y = tf.layers.batch_normalization(
y, axis=-1, momentum=0.99, training=self.train, name='dense-bn')
y = tf.nn.relu6(y)
if self.netconf['use_dropout']:
y = tf.layers.dropout(
y, self.netconf['dropout_rate'], training=self.train)
return y
def linear_block(self, x):
'''
linear layer for dim reduction
x: shape [batch, time, feat, channel]
output: shape [b, t, f]
'''
with tf.variable_scope('linear'):
times, feat, channel = x.shape.as_list()[1:]
x = tf.reshape(x, [-1, feat * channel])
if self.netconf['use_dropout']:
x = tf.layers.dropout(
x, self.netconf['dropout_rate'], training=self.train)
x = common_layers.linear(x, 'linear1',
[feat * channel, self.netconf['linear_num']])
#x = tf.nn.relu6(x)
x = tf.reshape(x, [-1, times, self.netconf['linear_num']])
return x
