def get_cin_logits(features, feature_columns, shared_feature_vectors, units,
is_training, extra_options):
with tf.variable_scope('cin'):
_check_cin_args(extra_options)
use_shared_embedding = extra_options['cin_use_shared_embedding']
use_project = extra_options['cin_use_project']
project_size = extra_options['cin_project_size']
hidden_feature_maps = extra_options['cin_hidden_feature_maps']
split_half = extra_options['cin_split_half']
if not use_shared_embedding:
feature_vectors = get_feature_vectors(features, feature_columns)
else:
feature_vectors = shared_feature_vectors
if use_project:
feature_vectors = project(feature_vectors, project_size)
check_feature_dims(feature_vectors)
x = tf.stack(feature_vectors, axis=1) # [B, N, D]
y = _cin_layer(x, hidden_feature_maps, split_half,
reduce_sum=False) # [B, F]
with tf.variable_scope('logits') as logits_scope:
logits = fc(y, units=units, name=logits_scope)
add_hidden_layer_summary(logits, logits_scope.name)
return logits
def _afm(feature_vectors, hidden_unit, reduce_sum=True):
"""AFM
feature_vectors: List of shape [B, D] tensors, size N
hidden_unit: see paper, attention layer units
Return:
Tensor of shape [B, 1] if reduce_sum is True, or shape of [B, num_iters]
"""
with tf.variable_scope('afm'):
check_feature_dims(feature_vectors)
interactions = pairwise_dot(feature_vectors) # [B, num_inters, D]
tf.logging.info("interactions = {}".format(interactions))
# [B, num_inters, 1]
att_weights = _afm_attention_network(interactions, hidden_unit)
tf.summary.histogram('afm_att_weights', att_weights)
# attention pooling
y = att_weights * interactions # [B, num_inters, D]
y = tf.reduce_sum(y, axis=-1) # [B, num_inters]
if reduce_sum:
y = tf.reduce_sum(y, axis=1, keepdims=True) # [B, 1]
return y
def _fgcnn(feature_vectors,
filter_nums,
kernel_sizes,
pooling_sizes,
new_map_sizes,
name='fgcnn'):
"""FGCNN: Generate new features
feature_vectors: List of N shape [B, D] tensors
Return:
New features: List of Tensors of shape [B, D]
"""
with tf.variable_scope(name):
check_feature_dims(feature_vectors)
assert len(filter_nums) == len(kernel_sizes) == len(
pooling_sizes) == len(new_map_sizes)
x = tf.stack(feature_vectors, axis=1) # [B, N, D]
D = x.shape[2].value
y = x
y = tf.expand_dims(y, axis=-1) # [B, N, D, 1]
new_features = []
for idx in range(len(filter_nums)):
filters = filter_nums[idx]
kernel_size = kernel_sizes[idx]
pooling_size = pooling_sizes[idx]
new_map = new_map_sizes[idx]
name = 'conv{}'.format(idx)
y = tf.layers.conv2d(y,
filters=filters,
kernel_size=(kernel_size, 1),
strides=(1, 1),
padding='SAME',
activation=tf.nn.tanh,
name=name)
tf.logging.info('{} = {}'.format(name, y))
name = 'pooling{}'.format(idx)
y = tf.layers.max_pooling2d(y,
pool_size=(pooling_size, 1),
strides=(1, 1),
name=name)
tf.logging.info('{} = {}'.format(name, y))
flatten = tf.layers.flatten(y)
name = 'fc{}'.format(idx)
num_feature = new_map * y.shape[1].value
new_feature = tf.layers.dense(flatten,
units=num_feature * D,
activation=tf.nn.tanh,
name=name)
new_feature = tf.reshape(new_feature, [-1, num_feature, D])
tf.logging.info('{} = {}'.format(name, new_feature))
new_feature = tf.unstack(new_feature, axis=1)
new_features.extend(new_feature)
return new_features
def _build_ccpm_model(feature_vectors, kernel_sizes, filter_nums, hidden_units,
activation_fn, dropout, is_training, batch_norm,
layer_norm, use_resnet, use_densenet):
"""Build ccpm model
feature_vectors: List of 2D tensors
Return:
Tensor of shape [B, ?]
"""
assert len(kernel_sizes) == len(filter_nums)
check_feature_dims(feature_vectors)
with tf.variable_scope("ccpm"):
with tf.variable_scope('cnn'):
x = tf.stack(feature_vectors, axis=1) # [B, N, D]
L = len(kernel_sizes)
N = x.shape[1].value
D = x.shape[2].value
y = tf.expand_dims(x, -1) # [B, N, D, 1]
for idx, (width, num) in enumerate(zip(kernel_sizes, filter_nums)):
name = 'conv_{}_w{}_n{}'.format(idx, width, num)
y = tf.layers.conv2d(y,
filters=num,
kernel_size=(width, D),
strides=(1, 1),
padding='SAME',
activation=activation_fn,
name=name)
if idx == L - 1:
p = 3
else:
p = (1 - (1.0 * idx / L)**(L - idx)) * N
p = int(p)
p = min(p, y.shape[1].value)
name = 'k_max_pooling_{}_k{}'.format(idx, p)
y = k_max_pooling(y, k=p, axis=1, name=name)
y = tf.layers.flatten(y) # [B, -1]
# dnn
y = add_hidden_layers(y,
hidden_units,
activation_fn=activation_fn,
dropout=dropout,
is_training=is_training,
batch_norm=batch_norm,
layer_norm=layer_norm,
use_resnet=use_resnet,
use_densenet=use_densenet,
scope='dnn')
return y
def _fwfm(feature_vectors, units):
"""FwFM
feature_vectors: List of shape [B, D] tensors, size N
units: logits units
Return:
Tensor of shape [B, 1]
"""
with tf.variable_scope('fwfm'):
check_feature_dims(feature_vectors)
y = pairwise_dot(feature_vectors) # [B, N*(N-1)/2, D]
y = tf.reduce_sum(y, axis=-1) # [B, N*(N-1)/2]
y = fc(y, units) # [B, 1]
return y
def get_autoint_logits(
features,
feature_columns,
shared_feature_vectors,
units,
is_training,
extra_options):
with tf.variable_scope('autoint'):
_check_autoint_args(extra_options)
use_shared_embedding = extra_options['autoint_use_shared_embedding']
use_project = extra_options['autoint_use_project']
project_size = extra_options['autoint_project_size']
size_per_head = extra_options['autoint_size_per_head']
num_heads = extra_options['autoint_num_heads']
num_blocks = extra_options['autoint_num_blocks']
dropout = extra_options['autoint_dropout']
has_residual = extra_options['autoint_has_residual']
if not use_shared_embedding:
feature_vectors = get_feature_vectors(features, feature_columns)
else:
feature_vectors = shared_feature_vectors
if use_project:
feature_vectors = project(feature_vectors, project_size)
check_feature_dims(feature_vectors)
x = tf.stack(feature_vectors, axis=1) # [B, N, D]
y = _autoint(x,
num_blocks=num_blocks,
num_units=size_per_head*num_heads,
num_heads=num_heads,
dropout=dropout,
is_training=is_training,
has_residual=has_residual)
tf.logging.info("autoint output = {}".format(y))
with tf.variable_scope('logits') as logits_scope:
logits = fc(y, units, name=logits_scope)
add_hidden_layer_summary(logits, logits_scope.name)
return logits
def _fm(feature_vectors, reduce_sum=True):
"""FM
feature_vectors: List of shape [B, D] tensors, size N
Return:
Tensor of shape [B, 1] if reduce_sum is True, or shape of [B, D]
"""
with tf.variable_scope('fm'):
check_feature_dims(feature_vectors)
stack_fm_vectors = tf.stack(feature_vectors, axis=1) # [B, N, D]
sum_square = tf.square(tf.reduce_sum(stack_fm_vectors,
axis=1)) # [B, D]
square_sum = tf.reduce_sum(tf.square(stack_fm_vectors),
axis=1) # [B, D]
y = 0.5 * (tf.subtract(sum_square, square_sum)) # [B, D]
if reduce_sum:
y = tf.reduce_sum(y, axis=1, keepdims=True) # [B, 1]
return y
def _ifm(feature_vectors, hidden_unit, field_dim, reduce_sum=True):
"""ifm
feature_vectors: List of shape [B, D] tensors, size N
hidden_unit: see paper, attention layer units
field_dim: see paper formula (6), hyper-param K_F
Return:
Tensor of shape [B, 1] if reduce_sum is True, or shape of [B, num_iters]
"""
with tf.variable_scope('ifm'):
check_feature_dims(feature_vectors)
# [B, num_inters, D]
y = interaction_aware_pairwise_dot(feature_vectors, field_dim)
y = tf.reduce_sum(y, axis=-1) # [B, num_inters]
if reduce_sum:
y = tf.reduce_sum(y, axis=1, keepdims=True) # [B, 1]
return y
def shallow_fibinet(features,
feature_columns,
shared_feature_vectors,
se_use_shared_embedding,
use_project,
project_size,
interaction_type,
se_interaction_type,
use_se,
name='shallow_fibinet'):
"""Shallow part of FiBiNET
feature_vectors: list of 2-D tensors of shape [B, D], size N.
Return:
Tensor of shape [B, -1]
"""
with tf.variable_scope(name):
check_feature_dims(shared_feature_vectors)
y = bilinear(shared_feature_vectors, interaction_type) # [B, -1]
if use_se:
if se_use_shared_embedding:
se_feature_vectors = shared_feature_vectors
else:
se_feature_vectors = get_feature_vectors(features, feature_columns)
if use_project:
se_feature_vectors = project(se_feature_vectors, project_size)
check_feature_dims(se_feature_vectors)
x = tf.stack(se_feature_vectors, axis=1) # [B, N, D]
se_x = selayer(x) # [B, N, D]
new_se_feature_vectors = tf.unstack(se_x, axis=1) # N tensors of shape [B, D]
se_y = bilinear(new_se_feature_vectors,
se_interaction_type,
name='se_bilinear') # [B, -1]
y = tf.concat([y, se_y], axis=1) # [B, -1]
return y