def _model_fn(features, labels, mode, config):
head = head_lib._binary_logistic_or_multi_class_head( # pylint: disable=protected-access
n_classes, weight_column, label_vocabulary, loss_reduction,
loss_fn)
is_training = (mode == tf.estimator.ModeKeys.TRAIN)
net = tf.feature_column.input_layer(features, feature_columns)
inputs = _attention_layer(features, attention_columns, is_training)
tf.logging.info('attention outputs = {}'.format(inputs))
inputs.append(net)
net = tf.concat(inputs, axis=-1)
tf.logging.info("inputs: {}".format(net))
if batch_norm:
net = tf.layers.batch_normalization(net, training=is_training)
net = add_hidden_layers(net, hidden_units, activation_fn, dropout,
is_training, batch_norm, 'DNN')
with tf.variable_scope('logits') as logits_scope:
logits = fc(net, head.logits_dimension, name=logits_scope)
add_hidden_layer_summary(logits, logits_scope.name)
return head.create_estimator_spec(
features=features,
mode=mode,
labels=labels,
optimizer=optimizers.get_optimizer_instance(
optimizer, learning_rate=_LEARNING_RATE),
logits=logits)
def _ipnn(feature_vectors,
project_feature_vectors,
use_project,
units,
hidden_units,
activation_fn,
dropout,
batch_norm,
layer_norm,
use_resnet,
use_densenet,
is_training,
unordered_inner_product,
concat_project):
"""IPNN
feature_vectors: List of shape [B, ?] tensors, size N
project_feature_vectors: None or list of shape [B, D] tensors, size N
Return:
2D Tensor of shape [B, ?]
"""
with tf.variable_scope("ipnn"):
if use_project:
x = project_feature_vectors
else:
x = feature_vectors
if unordered_inner_product:
y = pairwise_dot_unorderd(x) # [B, N*N, D]
else:
y = pairwise_dot(x) # [B, N*(N-1)/2, D]
y = tf.reduce_sum(y, axis=-1) # [B, M]
inputs = []
inputs.append(y)
if use_project and concat_project:
inputs.extend(project_feature_vectors)
else:
inputs.extend(feature_vectors)
y = tf.concat(inputs, axis=1) # [B, ?]
y = add_hidden_layers(y,
hidden_units=hidden_units,
activation_fn=activation_fn,
dropout=dropout,
is_training=is_training,
batch_norm=batch_norm,
layer_norm=layer_norm,
use_resnet=use_resnet,
use_densenet=use_densenet,
scope='hidden_layers') # [B, ?]
return y
def _build_ccpm_model(feature_vectors, kernel_sizes, filter_nums, hidden_units,
activation_fn, dropout, is_training, batch_norm,
layer_norm, use_resnet, use_densenet):
"""Build ccpm model
feature_vectors: List of 2D tensors
Return:
Tensor of shape [B, ?]
"""
assert len(kernel_sizes) == len(filter_nums)
check_feature_dims(feature_vectors)
with tf.variable_scope("ccpm"):
with tf.variable_scope('cnn'):
x = tf.stack(feature_vectors, axis=1) # [B, N, D]
L = len(kernel_sizes)
N = x.shape[1].value
D = x.shape[2].value
y = tf.expand_dims(x, -1) # [B, N, D, 1]
for idx, (width, num) in enumerate(zip(kernel_sizes, filter_nums)):
name = 'conv_{}_w{}_n{}'.format(idx, width, num)
y = tf.layers.conv2d(y,
filters=num,
kernel_size=(width, D),
strides=(1, 1),
padding='SAME',
activation=activation_fn,
name=name)
if idx == L - 1:
p = 3
else:
p = (1 - (1.0 * idx / L)**(L - idx)) * N
p = int(p)
p = min(p, y.shape[1].value)
name = 'k_max_pooling_{}_k{}'.format(idx, p)
y = k_max_pooling(y, k=p, axis=1, name=name)
y = tf.layers.flatten(y) # [B, -1]
# dnn
y = add_hidden_layers(y,
hidden_units,
activation_fn=activation_fn,
dropout=dropout,
is_training=is_training,
batch_norm=batch_norm,
layer_norm=layer_norm,
use_resnet=use_resnet,
use_densenet=use_densenet,
scope='dnn')
return y
def get_nfm_logits(features, feature_columns, shared_feature_vectors, units,
is_training, extra_options):
with tf.variable_scope('nfm'):
_check_nfm_args(extra_options)
use_shared_embedding = extra_options['nfm_use_shared_embedding']
use_project = extra_options['nfm_use_project']
project_size = extra_options['nfm_project_size']
hidden_units = extra_options['nfm_hidden_units']
activation_fn = extra_options['nfm_activation_fn']
dropout = extra_options['nfm_dropout']
batch_norm = extra_options['nfm_batch_norm']
layer_norm = extra_options['nfm_layer_norm']
use_resnet = extra_options['nfm_use_resnet']
use_densenet = extra_options['nfm_use_densenet']
leaky_relu_alpha = extra_options['leaky_relu_alpha']
swish_beta = extra_options['swish_beta']
activation_fn = get_activation_fn(activation_fn=activation_fn,
leaky_relu_alpha=leaky_relu_alpha,
swish_beta=swish_beta)
if not use_shared_embedding:
feature_vectors = get_feature_vectors(features, feature_columns)
else:
feature_vectors = shared_feature_vectors
if use_project:
feature_vectors = project(feature_vectors, project_size)
# Neural FM
y = _fm(feature_vectors, reduce_sum=False)
y = add_hidden_layers(y,
hidden_units=hidden_units,
activation_fn=activation_fn,
dropout=dropout,
is_training=is_training,
batch_norm=batch_norm,
layer_norm=layer_norm,
use_resnet=use_resnet,
use_densenet=use_densenet,
scope='hidden_layers')
with tf.variable_scope('logits') as logits_scope:
logits = fc(y, units, name=logits_scope)
add_hidden_layer_summary(logits, logits_scope.name)
return logits
def _nifm(feature_vectors,
hidden_units,
activation_fn,
dropout,
is_training,
batch_norm,
layer_norm,
use_resnet,
use_densenet,
reduce_sum=True):
"""Network in FM
feature_vectors: List of shape [B, D] tensors, size N
Return:
Tensor of shape [B, 1] if reduce_sum is True, or shape of [B, N*(N-1)/2]
"""
with tf.variable_scope('nifm'):
outputs = []
N = len(feature_vectors)
for i in range(N - 1):
for j in range(i, N):
scope_name = 'subnet_{}_{}'.format(i, j)
vi = feature_vectors[i]
vj = feature_vectors[j]
v = tf.concat([vi, vj], axis=1)
y = add_hidden_layers(v,
hidden_units=hidden_units,
activation_fn=tf.nn.relu,
dropout=dropout,
is_training=is_training,
batch_norm=batch_norm,
layer_norm=layer_norm,
use_resnet=use_resnet,
use_densenet=use_densenet,
scope=scope_name)
y = fc(y, 1)
outputs.append(y)
y = tf.concat(outputs, axis=1) # [B, N*(N-1)/2]
if reduce_sum:
y = tf.reduce_sum(y, axis=-1, keepdims=True) # [B, 1]
tf.logging.info("nifm output = {}".format(y))
return y
def _pin(feature_vectors, project_feature_vectors, use_project,
subnet_hidden_units, units, hidden_units, activation_fn, dropout,
batch_norm, layer_norm, use_resnet, use_densenet, is_training,
concat_project, use_concat):
"""
feature_vectors: List of shape [B, ?] tensors, size N
project_feature_vectors: None or list of shape [B, D] tensors, size N
Return:
2D Tensor of shape [B, ?]
"""
with tf.variable_scope("pin"):
if use_project:
x = project_feature_vectors
else:
x = feature_vectors
y = _pin_layer(x, subnet_hidden_units, is_training) # [B, ?]
inputs = []
inputs.append(y)
if use_concat:
if use_project and concat_project:
inputs.extend(project_feature_vectors)
else:
inputs.extend(feature_vectors)
y = tf.concat(inputs, axis=1) # [B, ?]
y = add_hidden_layers(y,
hidden_units=hidden_units,
activation_fn=activation_fn,
dropout=dropout,
is_training=is_training,
batch_norm=batch_norm,
layer_norm=layer_norm,
use_resnet=use_resnet,
use_densenet=use_densenet,
scope='hidden_layers') # [B, ?]
return y
def _nkfm(feature_vectors, units, hidden_units, activation_fn, dropout,
batch_norm, layer_norm, use_resnet, use_densenet, is_training):
"""nkfm
feature_vectors: List of shape [B, ?] tensors, size N
Return:
2D Tensor of shape [B, ?]
"""
with tf.variable_scope("nkfm"):
y = pairwise_kernel_dot(feature_vectors) # [B, N*N]
y = add_hidden_layers(y,
hidden_units=hidden_units,
activation_fn=activation_fn,
dropout=dropout,
is_training=is_training,
batch_norm=batch_norm,
layer_norm=layer_norm,
use_resnet=use_resnet,
use_densenet=use_densenet,
scope='hidden_layers') # [B, ?]
return y
def _pin_layer(feature_vectors, subnet_hidden_units, is_training):
"""PIN Layer
feature_vectors: List of shape [B, ?] tensors, size N
Return:
2D Tensor of shape [B, ?]
"""
with tf.variable_scope('pin_layer'):
N = len(feature_vectors)
outputs = []
for i in range(N):
for j in range(N):
vi = feature_vectors[i]
vj = feature_vectors[j]
di = vi.shape[1].value
dj = vj.shape[1].value
kernel_name = 'pairwise_kernel_dot_{}_{}'.format(i, j)
U = tf.get_variable(kernel_name, [di, dj])
y = tf.matmul(feature_vectors[i], U) # [B, dj]
y = tf.concat([vi, vj, y], axis=1)
scope_name = 'subnet_{}_{}'.format(i, j)
y = add_hidden_layers(y,
hidden_units=subnet_hidden_units,
activation_fn=tf.nn.relu,
dropout=None,
is_training=is_training,
batch_norm=True,
layer_norm=False,
use_resnet=False,
use_densenet=False,
scope=scope_name,
last_layer_direct=True)
outputs.append(y)
y = tf.concat(outputs, axis=1) # [B, ?]
return y
def get_fibinet_logits(
features,
feature_columns,
shared_feature_vectors,
units,
is_training,
extra_options):
with tf.variable_scope('fibinet'):
_check_fibinet_args(extra_options)
use_shared_embedding = extra_options['fibinet_use_shared_embedding']
use_project = extra_options['fibinet_use_project']
project_size = extra_options['fibinet_project_size']
hidden_units = extra_options['fibinet_hidden_units']
activation_fn = extra_options['fibinet_activation_fn']
dropout = extra_options['fibinet_dropout']
batch_norm = extra_options['fibinet_batch_norm']
layer_norm = extra_options['fibinet_layer_norm']
use_resnet = extra_options['fibinet_use_resnet']
use_densenet = extra_options['fibinet_use_densenet']
use_se = extra_options['fibinet_use_se']
use_deep = extra_options['fibinet_use_deep']
interaction_type = extra_options['fibinet_interaction_type']
se_interaction_type = extra_options['fibinet_se_interaction_type']
se_use_shared_embedding = extra_options['fibinet_se_use_shared_embedding']
leaky_relu_alpha = extra_options['leaky_relu_alpha']
swish_beta = extra_options['swish_beta']
activation_fn = get_activation_fn(activation_fn=activation_fn,
leaky_relu_alpha=leaky_relu_alpha,
swish_beta=swish_beta)
if not use_shared_embedding:
feature_vectors = get_feature_vectors(features, feature_columns)
else:
feature_vectors = shared_feature_vectors
if use_project:
feature_vectors = project(feature_vectors, project_size)
y = shallow_fibinet(features=features,
feature_columns=feature_columns,
shared_feature_vectors=feature_vectors,
se_use_shared_embedding=se_use_shared_embedding,
use_project=use_project,
project_size=project_size,
interaction_type=interaction_type,
se_interaction_type=se_interaction_type,
use_se=use_se) # [B, -1]
if use_deep:
y = add_hidden_layers(y,
hidden_units=hidden_units,
activation_fn=activation_fn,
dropout=dropout,
is_training=is_training,
batch_norm=batch_norm,
layer_norm=layer_norm,
use_resnet=use_resnet,
use_densenet=use_densenet,
scope='hidden_layers')
with tf.variable_scope('logits') as logits_scope:
logits = fc(y, units, name=logits_scope)
add_hidden_layer_summary(logits, logits_scope.name)
else:
assert units == 1, "shallow_fibinet's units must be 1"
with tf.variable_scope('logits') as logits_scope:
logits = tf.reduce_sum(y, axis=-1, keepdims=True) # [B, 1]
add_hidden_layer_summary(logits, logits_scope.name)
return logits
def _model_fn(features, labels, mode, config):
head = head_lib._binary_logistic_or_multi_class_head( # pylint: disable=protected-access
n_classes, weight_column, label_vocabulary, loss_reduction,
loss_fn)
is_training = (mode == tf.estimator.ModeKeys.TRAIN)
net_dssm1 = tf.feature_column.input_layer(features, dssm1_columns)
net_dssm2 = tf.feature_column.input_layer(features, dssm2_columns)
tf.logging.info("net_dssm1: {}".format(net_dssm1))
tf.logging.info("net_dssm2: {}".format(net_dssm2))
real_activation_fn = get_activation_fn(
activation_fn=activation_fn,
leaky_relu_alpha=leaky_relu_alpha,
swish_beta=swish_beta)
net_dssm1 = add_hidden_layers(inputs=net_dssm1,
hidden_units=hidden_units,
activation_fn=real_activation_fn,
dropout=dropout,
is_training=is_training,
batch_norm=batch_norm,
layer_norm=layer_norm,
use_resnet=use_resnet,
use_densenet=use_densenet,
scope='dssm1')
net_dssm2 = add_hidden_layers(inputs=net_dssm2,
hidden_units=hidden_units,
activation_fn=real_activation_fn,
dropout=dropout,
is_training=is_training,
batch_norm=batch_norm,
layer_norm=layer_norm,
use_resnet=use_resnet,
use_densenet=use_densenet,
scope='dssm2')
with tf.variable_scope('logits') as logits_scope:
if dssm_mode == 'dot':
logits = tf.reduce_sum(net_dssm1 * net_dssm2,
-1,
keepdims=True)
elif dssm_mode == 'concat':
logits = tf.concat([net_dssm1, net_dssm2], axis=1)
logits = tf.layers.dense(logits, units=1, activation=None)
elif dssm_mode == 'cosine':
logits = tf.reduce_sum(net_dssm1 * net_dssm2,
-1,
keepdims=True)
norm1 = tf.norm(net_dssm1, axis=1, keepdims=True)
norm2 = tf.norm(net_dssm2, axis=1, keepdims=True)
logits = logits / (norm1 * norm2)
else:
raise ValueError(
"unknown dssm mode '{}'".format(dssm_mode))
add_hidden_layer_summary(logits, logits_scope.name)
tf.logging.info("logits = {}".format(logits))
return head.create_estimator_spec(
features=features,
mode=mode,
labels=labels,
optimizer=optimizers.get_optimizer_instance(
optimizer, learning_rate=_LEARNING_RATE),
logits=logits)
