def get_cross_logits(
features,
feature_columns,
shared_feature_vectors,
units,
is_training,
extra_options):
with tf.variable_scope('cross'):
_check_cross_args(extra_options)
use_shared_embedding = extra_options['cross_use_shared_embedding']
use_project = extra_options['cross_use_project']
project_size = extra_options['cross_project_size']
num_layers = extra_options['cross_num_layers']
if not use_shared_embedding:
feature_vectors = get_feature_vectors(features, feature_columns)
else:
feature_vectors = shared_feature_vectors
if use_project:
feature_vectors = project(feature_vectors, project_size)
x = tf.concat(feature_vectors, axis=1) # [B, T]
y = _cross_net(x, num_layers)
with tf.variable_scope('logits') as logits_scope:
logits = fc(y, units=units, name=logits_scope)
add_hidden_layer_summary(logits, logits_scope.name)
return logits
def get_cin_logits(features, feature_columns, shared_feature_vectors, units,
is_training, extra_options):
with tf.variable_scope('cin'):
_check_cin_args(extra_options)
use_shared_embedding = extra_options['cin_use_shared_embedding']
use_project = extra_options['cin_use_project']
project_size = extra_options['cin_project_size']
hidden_feature_maps = extra_options['cin_hidden_feature_maps']
split_half = extra_options['cin_split_half']
if not use_shared_embedding:
feature_vectors = get_feature_vectors(features, feature_columns)
else:
feature_vectors = shared_feature_vectors
if use_project:
feature_vectors = project(feature_vectors, project_size)
check_feature_dims(feature_vectors)
x = tf.stack(feature_vectors, axis=1) # [B, N, D]
y = _cin_layer(x, hidden_feature_maps, split_half,
reduce_sum=False) # [B, F]
with tf.variable_scope('logits') as logits_scope:
logits = fc(y, units=units, name=logits_scope)
add_hidden_layer_summary(logits, logits_scope.name)
return logits
def get_wkfm_logits(
features,
feature_columns,
shared_feature_vectors,
units,
is_training,
extra_options):
with tf.variable_scope('wkfm'):
_check_wkfm_args(extra_options)
use_shared_embedding = extra_options['wkfm_use_shared_embedding']
use_project = extra_options['wkfm_use_project']
project_size = extra_options['wkfm_project_size']
if not use_shared_embedding:
feature_vectors = get_feature_vectors(features, feature_columns)
else:
feature_vectors = shared_feature_vectors
if use_project:
feature_vectors = project(feature_vectors, project_size)
y = _wkfm(feature_vectors, reduce_sum=True) # [B, 1]
with tf.variable_scope('logits') as logits_scope:
# fc just for adding a bias
logits = fc(y, units=units, name=logits_scope)
add_hidden_layer_summary(logits, logits_scope.name)
return logits
def _model_fn(features, labels, mode, config):
head = head_lib._binary_logistic_or_multi_class_head( # pylint: disable=protected-access
n_classes, weight_column, label_vocabulary, loss_reduction,
loss_fn)
is_training = (mode == tf.estimator.ModeKeys.TRAIN)
net = tf.feature_column.input_layer(features, feature_columns)
inputs = _attention_layer(features, attention_columns, is_training)
tf.logging.info('attention outputs = {}'.format(inputs))
inputs.append(net)
net = tf.concat(inputs, axis=-1)
tf.logging.info("inputs: {}".format(net))
if batch_norm:
net = tf.layers.batch_normalization(net, training=is_training)
net = add_hidden_layers(net, hidden_units, activation_fn, dropout,
is_training, batch_norm, 'DNN')
with tf.variable_scope('logits') as logits_scope:
logits = fc(net, head.logits_dimension, name=logits_scope)
add_hidden_layer_summary(logits, logits_scope.name)
return head.create_estimator_spec(
features=features,
mode=mode,
labels=labels,
optimizer=optimizers.get_optimizer_instance(
optimizer, learning_rate=_LEARNING_RATE),
logits=logits)
def get_ipnn_logits(
features,
feature_columns,
shared_feature_vectors,
units,
is_training,
extra_options):
with tf.variable_scope('ipnn'):
_check_ipnn_args(extra_options)
use_shared_embedding = extra_options['ipnn_use_shared_embedding']
use_project = extra_options['ipnn_use_project']
project_size = extra_options['ipnn_project_size']
hidden_units = extra_options['ipnn_hidden_units']
activation_fn = extra_options['ipnn_activation_fn']
dropout = extra_options['ipnn_dropout']
batch_norm = extra_options['ipnn_batch_norm']
layer_norm = extra_options['ipnn_layer_norm']
use_resnet = extra_options['ipnn_use_resnet']
use_densenet = extra_options['ipnn_use_densenet']
unordered_inner_product = extra_options['ipnn_unordered_inner_product']
concat_project = extra_options['ipnn_concat_project']
leaky_relu_alpha = extra_options['leaky_relu_alpha']
swish_beta = extra_options['swish_beta']
activation_fn = get_activation_fn(activation_fn=activation_fn,
leaky_relu_alpha=leaky_relu_alpha,
swish_beta=swish_beta)
if not use_shared_embedding:
feature_vectors = get_feature_vectors(features, feature_columns)
else:
feature_vectors = shared_feature_vectors
project_feature_vectors = None
if use_project:
project_feature_vectors = project(feature_vectors, project_size)
y = _ipnn(feature_vectors=feature_vectors,
project_feature_vectors=project_feature_vectors,
use_project=use_project,
units=units,
hidden_units=hidden_units,
activation_fn=activation_fn,
dropout=dropout,
batch_norm=batch_norm,
layer_norm=layer_norm,
use_resnet=use_resnet,
use_densenet=use_densenet,
is_training=is_training,
unordered_inner_product=unordered_inner_product,
concat_project=concat_project)
with tf.variable_scope('logits') as logits_scope:
logits = fc(y, units=units, name=logits_scope)
add_hidden_layer_summary(logits, logits_scope.name)
return logits
def inference_nvidianet(images):
with tf.variable_scope('conv1'):
conv1 = common.activation(common.conv(images, 24, ksize=5, stride=2,
padding='VALID'))
with tf.variable_scope('conv2'):
conv2 = common.activation(common.conv(conv1, 36, ksize=5, stride=2,
padding='VALID'))
with tf.variable_scope('conv3'):
conv3 = common.activation(common.conv(conv2, 48, ksize=5, stride=2,
padding='VALID'))
with tf.variable_scope('conv4'):
conv4 = common.activation(common.conv(conv3, 64, ksize=3, stride=1,
padding='VALID'))
with tf.variable_scope('conv5'):
conv5 = common.activation(common.conv(conv4, 64, ksize=3, stride=1,
padding='VALID'))
with tf.variable_scope('conv6'):
conv6 = common.activation(common.conv(conv5, 64, ksize=3, stride=1,
padding='VALID'))
with tf.variable_scope('conv7'):
conv7 = common.activation(common.conv(conv6, 64, ksize=3, stride=1,
padding='VALID'))
conv7_flat = common.flatten(conv7)
with tf.variable_scope('fc1'):
fc1 = common.dropout(common.activation(common.fc(conv7_flat, 512)),
0.5)
with tf.variable_scope('fc2'):
fc2 = common.dropout(common.activation(common.fc(fc1, 128)),
0.625)
with tf.variable_scope('fc3'):
fc3 = common.dropout(common.activation(common.fc(fc2, 32)),
0.75)
with tf.variable_scope('fc4'):
fc4 = common.dropout(common.activation(common.fc(fc3, 8)),
0.875)
with tf.variable_scope('fc5'):
fc5 = tf.atan(common.fc(fc4, 1))
return fc5
def get_ccpm_logits(features, feature_columns, shared_feature_vectors, units,
is_training, extra_options):
with tf.variable_scope('ccpm'):
_check_ccpm_args(extra_options)
use_shared_embedding = extra_options['ccpm_use_shared_embedding']
use_project = extra_options['ccpm_use_project']
project_size = extra_options['ccpm_project_size']
hidden_units = extra_options['ccpm_hidden_units']
activation_fn = extra_options['ccpm_activation_fn']
dropout = extra_options['ccpm_dropout']
batch_norm = extra_options['ccpm_batch_norm']
layer_norm = extra_options['ccpm_layer_norm']
use_resnet = extra_options['ccpm_use_resnet']
use_densenet = extra_options['ccpm_use_densenet']
kernel_sizes = extra_options['ccpm_kernel_sizes']
filter_nums = extra_options['ccpm_filter_nums']
leaky_relu_alpha = extra_options['leaky_relu_alpha']
swish_beta = extra_options['swish_beta']
activation_fn = get_activation_fn(activation_fn=activation_fn,
leaky_relu_alpha=leaky_relu_alpha,
swish_beta=swish_beta)
if not use_shared_embedding:
feature_vectors = get_feature_vectors(features, feature_columns)
else:
feature_vectors = shared_feature_vectors
if use_project:
feature_vectors = project(feature_vectors, project_size)
y = _build_ccpm_model(feature_vectors=feature_vectors,
kernel_sizes=kernel_sizes,
filter_nums=filter_nums,
hidden_units=hidden_units,
activation_fn=activation_fn,
dropout=dropout,
is_training=is_training,
batch_norm=batch_norm,
layer_norm=layer_norm,
use_resnet=use_resnet,
use_densenet=use_densenet)
with tf.variable_scope('logits') as logits_scope:
logits = fc(y, units=units, name=logits_scope)
add_hidden_layer_summary(logits, logits_scope.name)
return logits
def get_nfm_logits(features, feature_columns, shared_feature_vectors, units,
is_training, extra_options):
with tf.variable_scope('nfm'):
_check_nfm_args(extra_options)
use_shared_embedding = extra_options['nfm_use_shared_embedding']
use_project = extra_options['nfm_use_project']
project_size = extra_options['nfm_project_size']
hidden_units = extra_options['nfm_hidden_units']
activation_fn = extra_options['nfm_activation_fn']
dropout = extra_options['nfm_dropout']
batch_norm = extra_options['nfm_batch_norm']
layer_norm = extra_options['nfm_layer_norm']
use_resnet = extra_options['nfm_use_resnet']
use_densenet = extra_options['nfm_use_densenet']
leaky_relu_alpha = extra_options['leaky_relu_alpha']
swish_beta = extra_options['swish_beta']
activation_fn = get_activation_fn(activation_fn=activation_fn,
leaky_relu_alpha=leaky_relu_alpha,
swish_beta=swish_beta)
if not use_shared_embedding:
feature_vectors = get_feature_vectors(features, feature_columns)
else:
feature_vectors = shared_feature_vectors
if use_project:
feature_vectors = project(feature_vectors, project_size)
# Neural FM
y = _fm(feature_vectors, reduce_sum=False)
y = add_hidden_layers(y,
hidden_units=hidden_units,
activation_fn=activation_fn,
dropout=dropout,
is_training=is_training,
batch_norm=batch_norm,
layer_norm=layer_norm,
use_resnet=use_resnet,
use_densenet=use_densenet,
scope='hidden_layers')
with tf.variable_scope('logits') as logits_scope:
logits = fc(y, units, name=logits_scope)
add_hidden_layer_summary(logits, logits_scope.name)
return logits
def _fwfm(feature_vectors, units):
"""FwFM
feature_vectors: List of shape [B, D] tensors, size N
units: logits units
Return:
Tensor of shape [B, 1]
"""
with tf.variable_scope('fwfm'):
check_feature_dims(feature_vectors)
y = pairwise_dot(feature_vectors) # [B, N*(N-1)/2, D]
y = tf.reduce_sum(y, axis=-1) # [B, N*(N-1)/2]
y = fc(y, units) # [B, 1]
return y
def _nifm(feature_vectors,
hidden_units,
activation_fn,
dropout,
is_training,
batch_norm,
layer_norm,
use_resnet,
use_densenet,
reduce_sum=True):
"""Network in FM
feature_vectors: List of shape [B, D] tensors, size N
Return:
Tensor of shape [B, 1] if reduce_sum is True, or shape of [B, N*(N-1)/2]
"""
with tf.variable_scope('nifm'):
outputs = []
N = len(feature_vectors)
for i in range(N - 1):
for j in range(i, N):
scope_name = 'subnet_{}_{}'.format(i, j)
vi = feature_vectors[i]
vj = feature_vectors[j]
v = tf.concat([vi, vj], axis=1)
y = add_hidden_layers(v,
hidden_units=hidden_units,
activation_fn=tf.nn.relu,
dropout=dropout,
is_training=is_training,
batch_norm=batch_norm,
layer_norm=layer_norm,
use_resnet=use_resnet,
use_densenet=use_densenet,
scope=scope_name)
y = fc(y, 1)
outputs.append(y)
y = tf.concat(outputs, axis=1) # [B, N*(N-1)/2]
if reduce_sum:
y = tf.reduce_sum(y, axis=-1, keepdims=True) # [B, 1]
tf.logging.info("nifm output = {}".format(y))
return y
def inference_resnet(images):
with tf.variable_scope('1'):
conv1 = common.conv(images, 64, ksize=7, stride=2)
conv1 = common.bn(conv1)
pool1 = common.max_pool(conv1)
with tf.variable_scope('2'):
stack2 = common.res_stack(pool1, [256, 256, 256], pool=False)
with tf.variable_scope('3'):
stack3 = common.res_stack(stack2, [512, 512, 512, 512])
with tf.variable_scope('4'):
stack4 = common.res_stack(stack3, [1024, 1024, 1024,
1024, 1024, 1024])
with tf.variable_scope('5'):
stack5 = common.res_stack(stack4, [2048, 2048, 2048])
pool5 = common.global_ave_pool(stack5)
with tf.variable_scope('fc'):
fc = common.fc(pool5, 1)
return fc
def get_autoint_logits(
features,
feature_columns,
shared_feature_vectors,
units,
is_training,
extra_options):
with tf.variable_scope('autoint'):
_check_autoint_args(extra_options)
use_shared_embedding = extra_options['autoint_use_shared_embedding']
use_project = extra_options['autoint_use_project']
project_size = extra_options['autoint_project_size']
size_per_head = extra_options['autoint_size_per_head']
num_heads = extra_options['autoint_num_heads']
num_blocks = extra_options['autoint_num_blocks']
dropout = extra_options['autoint_dropout']
has_residual = extra_options['autoint_has_residual']
if not use_shared_embedding:
feature_vectors = get_feature_vectors(features, feature_columns)
else:
feature_vectors = shared_feature_vectors
if use_project:
feature_vectors = project(feature_vectors, project_size)
check_feature_dims(feature_vectors)
x = tf.stack(feature_vectors, axis=1) # [B, N, D]
y = _autoint(x,
num_blocks=num_blocks,
num_units=size_per_head*num_heads,
num_heads=num_heads,
dropout=dropout,
is_training=is_training,
has_residual=has_residual)
tf.logging.info("autoint output = {}".format(y))
with tf.variable_scope('logits') as logits_scope:
logits = fc(y, units, name=logits_scope)
add_hidden_layer_summary(logits, logits_scope.name)
return logits
def inference_small(images):
with tf.variable_scope('1'):
conv1 = common.conv(images, 64, ksize=7, stride=2)
conv1 = common.bn(conv1)
pool1 = common.max_pool(conv1)
with tf.variable_scope('2'):
stack2 = common.stack(pool1, common.res_block, [64, 64])
pool2 = common.max_pool(stack2)
with tf.variable_scope('3'):
stack3 = common.stack(pool2, common.res_block, [128, 128])
pool3 = common.max_pool(stack3)
with tf.variable_scope('4'):
stack4 = common.stack(pool3, common.res_block, [256, 256, 256])
pool4 = common.max_pool(stack4)
with tf.variable_scope('5'):
stack5 = common.stack(pool4, common.res_block, [512, 512])
pool5 = common.global_ave_pool(stack5)
with tf.variable_scope('fc'):
fc = common.fc(pool5, 1)
return fc
def get_fibinet_logits(
features,
feature_columns,
shared_feature_vectors,
units,
is_training,
extra_options):
with tf.variable_scope('fibinet'):
_check_fibinet_args(extra_options)
use_shared_embedding = extra_options['fibinet_use_shared_embedding']
use_project = extra_options['fibinet_use_project']
project_size = extra_options['fibinet_project_size']
hidden_units = extra_options['fibinet_hidden_units']
activation_fn = extra_options['fibinet_activation_fn']
dropout = extra_options['fibinet_dropout']
batch_norm = extra_options['fibinet_batch_norm']
layer_norm = extra_options['fibinet_layer_norm']
use_resnet = extra_options['fibinet_use_resnet']
use_densenet = extra_options['fibinet_use_densenet']
use_se = extra_options['fibinet_use_se']
use_deep = extra_options['fibinet_use_deep']
interaction_type = extra_options['fibinet_interaction_type']
se_interaction_type = extra_options['fibinet_se_interaction_type']
se_use_shared_embedding = extra_options['fibinet_se_use_shared_embedding']
leaky_relu_alpha = extra_options['leaky_relu_alpha']
swish_beta = extra_options['swish_beta']
activation_fn = get_activation_fn(activation_fn=activation_fn,
leaky_relu_alpha=leaky_relu_alpha,
swish_beta=swish_beta)
if not use_shared_embedding:
feature_vectors = get_feature_vectors(features, feature_columns)
else:
feature_vectors = shared_feature_vectors
if use_project:
feature_vectors = project(feature_vectors, project_size)
y = shallow_fibinet(features=features,
feature_columns=feature_columns,
shared_feature_vectors=feature_vectors,
se_use_shared_embedding=se_use_shared_embedding,
use_project=use_project,
project_size=project_size,
interaction_type=interaction_type,
se_interaction_type=se_interaction_type,
use_se=use_se) # [B, -1]
if use_deep:
y = add_hidden_layers(y,
hidden_units=hidden_units,
activation_fn=activation_fn,
dropout=dropout,
is_training=is_training,
batch_norm=batch_norm,
layer_norm=layer_norm,
use_resnet=use_resnet,
use_densenet=use_densenet,
scope='hidden_layers')
with tf.variable_scope('logits') as logits_scope:
logits = fc(y, units, name=logits_scope)
add_hidden_layer_summary(logits, logits_scope.name)
else:
assert units == 1, "shallow_fibinet's units must be 1"
with tf.variable_scope('logits') as logits_scope:
logits = tf.reduce_sum(y, axis=-1, keepdims=True) # [B, 1]
add_hidden_layer_summary(logits, logits_scope.name)
return logits
def _deepx_model_fn(
features,
labels,
mode,
head,
linear_feature_columns,
linear_optimizer,
deep_feature_columns,
deep_optimizer,
model_slots,
extend_feature_mode,
input_layer_partitioner,
config,
linear_sparse_combiner,
use_seperated_logits,
use_weighted_logits,
extra_options):
_check_model_input(features, linear_feature_columns, deep_feature_columns)
num_ps_replicas = config.num_ps_replicas if config else 0
tf.logging.info("num_ps_replicas = {}".format(num_ps_replicas))
input_layer_partitioner = input_layer_partitioner or (
tf.min_max_variable_partitioner(
max_partitions=num_ps_replicas,
min_slice_size=64 << 20))
is_training = (mode == tf.estimator.ModeKeys.TRAIN)
# Build deepx logits
deepx_parent_scope = 'deepx'
if not deep_feature_columns:
deepx_logits_list = []
else:
deep_optimizer = optimizers.get_optimizer_instance(
deep_optimizer, learning_rate=_LEARNING_RATE)
check_no_sync_replicas_optimizer(deep_optimizer)
deep_partitioner = (
tf.min_max_variable_partitioner(
max_partitions=num_ps_replicas))
with tf.variable_scope(
deepx_parent_scope,
values=tuple(six.itervalues(features)),
partitioner=deep_partitioner) as scope:
deepx_absolute_scope = scope.name
deepx_logits_list = _build_deepx_logits(
features=features,
feature_columns=deep_feature_columns,
model_slots=model_slots,
units=head.logits_dimension,
is_training=is_training,
extra_options=extra_options)
linear_parent_scope = 'linear'
use_linear = 'linear' in model_slots
if not linear_feature_columns or not use_linear:
linear_logits = None
else:
linear_optimizer = optimizers.get_optimizer_instance(
linear_optimizer,
learning_rate=_LEARNING_RATE)
check_no_sync_replicas_optimizer(linear_optimizer)
with tf.variable_scope(
linear_parent_scope,
values=tuple(six.itervalues(features)),
partitioner=input_layer_partitioner) as scope:
linear_absolute_scope = scope.name
linear_logits = get_linear_logits(
features,
linear_feature_columns,
head.logits_dimension,
linear_sparse_combiner,
scope)
# Combine logits and build full model.
logits = []
logits.extend(deepx_logits_list)
if linear_logits is not None:
logits.append(linear_logits)
assert not (use_seperated_logits and use_weighted_logits), \
"'use_seperated_logits' and 'use_weighted_logits' should not be set " \
"at the same time."
if not use_seperated_logits:
logits = sum(logits)
if use_weighted_logits and len(logits) > 1:
logits = tf.concat(logits, axis=1)
logits = fc(logits, head.logits_dimension)
def _train_op_fn(loss):
"""Returns the op to optimize the loss."""
train_ops = []
global_step = tf.train.get_global_step()
if len(deepx_logits_list) > 0:
train_ops.append(
deep_optimizer.minimize(
loss,
var_list=tf.get_collection(
tf.GraphKeys.TRAINABLE_VARIABLES,
scope=deepx_absolute_scope)))
if linear_logits is not None:
train_ops.append(
linear_optimizer.minimize(
loss,
var_list=tf.get_collection(
tf.GraphKeys.TRAINABLE_VARIABLES,
scope=linear_absolute_scope)))
train_op = tf.group(*train_ops)
with tf.control_dependencies([train_op]):
return tf.assign_add(global_step, 1).op
return head.create_estimator_spec(
features=features,
mode=mode,
labels=labels,
train_op_fn=_train_op_fn,
logits=logits)