def get_cross_logits( features, feature_columns, shared_feature_vectors, units, is_training, extra_options): with tf.variable_scope('cross'): _check_cross_args(extra_options) use_shared_embedding = extra_options['cross_use_shared_embedding'] use_project = extra_options['cross_use_project'] project_size = extra_options['cross_project_size'] num_layers = extra_options['cross_num_layers'] if not use_shared_embedding: feature_vectors = get_feature_vectors(features, feature_columns) else: feature_vectors = shared_feature_vectors if use_project: feature_vectors = project(feature_vectors, project_size) x = tf.concat(feature_vectors, axis=1) # [B, T] y = _cross_net(x, num_layers) with tf.variable_scope('logits') as logits_scope: logits = fc(y, units=units, name=logits_scope) add_hidden_layer_summary(logits, logits_scope.name) return logits
def get_cin_logits(features, feature_columns, shared_feature_vectors, units, is_training, extra_options): with tf.variable_scope('cin'): _check_cin_args(extra_options) use_shared_embedding = extra_options['cin_use_shared_embedding'] use_project = extra_options['cin_use_project'] project_size = extra_options['cin_project_size'] hidden_feature_maps = extra_options['cin_hidden_feature_maps'] split_half = extra_options['cin_split_half'] if not use_shared_embedding: feature_vectors = get_feature_vectors(features, feature_columns) else: feature_vectors = shared_feature_vectors if use_project: feature_vectors = project(feature_vectors, project_size) check_feature_dims(feature_vectors) x = tf.stack(feature_vectors, axis=1) # [B, N, D] y = _cin_layer(x, hidden_feature_maps, split_half, reduce_sum=False) # [B, F] with tf.variable_scope('logits') as logits_scope: logits = fc(y, units=units, name=logits_scope) add_hidden_layer_summary(logits, logits_scope.name) return logits
def get_wkfm_logits( features, feature_columns, shared_feature_vectors, units, is_training, extra_options): with tf.variable_scope('wkfm'): _check_wkfm_args(extra_options) use_shared_embedding = extra_options['wkfm_use_shared_embedding'] use_project = extra_options['wkfm_use_project'] project_size = extra_options['wkfm_project_size'] if not use_shared_embedding: feature_vectors = get_feature_vectors(features, feature_columns) else: feature_vectors = shared_feature_vectors if use_project: feature_vectors = project(feature_vectors, project_size) y = _wkfm(feature_vectors, reduce_sum=True) # [B, 1] with tf.variable_scope('logits') as logits_scope: # fc just for adding a bias logits = fc(y, units=units, name=logits_scope) add_hidden_layer_summary(logits, logits_scope.name) return logits
def _model_fn(features, labels, mode, config): head = head_lib._binary_logistic_or_multi_class_head( # pylint: disable=protected-access n_classes, weight_column, label_vocabulary, loss_reduction, loss_fn) is_training = (mode == tf.estimator.ModeKeys.TRAIN) net = tf.feature_column.input_layer(features, feature_columns) inputs = _attention_layer(features, attention_columns, is_training) tf.logging.info('attention outputs = {}'.format(inputs)) inputs.append(net) net = tf.concat(inputs, axis=-1) tf.logging.info("inputs: {}".format(net)) if batch_norm: net = tf.layers.batch_normalization(net, training=is_training) net = add_hidden_layers(net, hidden_units, activation_fn, dropout, is_training, batch_norm, 'DNN') with tf.variable_scope('logits') as logits_scope: logits = fc(net, head.logits_dimension, name=logits_scope) add_hidden_layer_summary(logits, logits_scope.name) return head.create_estimator_spec( features=features, mode=mode, labels=labels, optimizer=optimizers.get_optimizer_instance( optimizer, learning_rate=_LEARNING_RATE), logits=logits)
def get_ipnn_logits( features, feature_columns, shared_feature_vectors, units, is_training, extra_options): with tf.variable_scope('ipnn'): _check_ipnn_args(extra_options) use_shared_embedding = extra_options['ipnn_use_shared_embedding'] use_project = extra_options['ipnn_use_project'] project_size = extra_options['ipnn_project_size'] hidden_units = extra_options['ipnn_hidden_units'] activation_fn = extra_options['ipnn_activation_fn'] dropout = extra_options['ipnn_dropout'] batch_norm = extra_options['ipnn_batch_norm'] layer_norm = extra_options['ipnn_layer_norm'] use_resnet = extra_options['ipnn_use_resnet'] use_densenet = extra_options['ipnn_use_densenet'] unordered_inner_product = extra_options['ipnn_unordered_inner_product'] concat_project = extra_options['ipnn_concat_project'] leaky_relu_alpha = extra_options['leaky_relu_alpha'] swish_beta = extra_options['swish_beta'] activation_fn = get_activation_fn(activation_fn=activation_fn, leaky_relu_alpha=leaky_relu_alpha, swish_beta=swish_beta) if not use_shared_embedding: feature_vectors = get_feature_vectors(features, feature_columns) else: feature_vectors = shared_feature_vectors project_feature_vectors = None if use_project: project_feature_vectors = project(feature_vectors, project_size) y = _ipnn(feature_vectors=feature_vectors, project_feature_vectors=project_feature_vectors, use_project=use_project, units=units, hidden_units=hidden_units, activation_fn=activation_fn, dropout=dropout, batch_norm=batch_norm, layer_norm=layer_norm, use_resnet=use_resnet, use_densenet=use_densenet, is_training=is_training, unordered_inner_product=unordered_inner_product, concat_project=concat_project) with tf.variable_scope('logits') as logits_scope: logits = fc(y, units=units, name=logits_scope) add_hidden_layer_summary(logits, logits_scope.name) return logits
def inference_nvidianet(images): with tf.variable_scope('conv1'): conv1 = common.activation(common.conv(images, 24, ksize=5, stride=2, padding='VALID')) with tf.variable_scope('conv2'): conv2 = common.activation(common.conv(conv1, 36, ksize=5, stride=2, padding='VALID')) with tf.variable_scope('conv3'): conv3 = common.activation(common.conv(conv2, 48, ksize=5, stride=2, padding='VALID')) with tf.variable_scope('conv4'): conv4 = common.activation(common.conv(conv3, 64, ksize=3, stride=1, padding='VALID')) with tf.variable_scope('conv5'): conv5 = common.activation(common.conv(conv4, 64, ksize=3, stride=1, padding='VALID')) with tf.variable_scope('conv6'): conv6 = common.activation(common.conv(conv5, 64, ksize=3, stride=1, padding='VALID')) with tf.variable_scope('conv7'): conv7 = common.activation(common.conv(conv6, 64, ksize=3, stride=1, padding='VALID')) conv7_flat = common.flatten(conv7) with tf.variable_scope('fc1'): fc1 = common.dropout(common.activation(common.fc(conv7_flat, 512)), 0.5) with tf.variable_scope('fc2'): fc2 = common.dropout(common.activation(common.fc(fc1, 128)), 0.625) with tf.variable_scope('fc3'): fc3 = common.dropout(common.activation(common.fc(fc2, 32)), 0.75) with tf.variable_scope('fc4'): fc4 = common.dropout(common.activation(common.fc(fc3, 8)), 0.875) with tf.variable_scope('fc5'): fc5 = tf.atan(common.fc(fc4, 1)) return fc5
def get_ccpm_logits(features, feature_columns, shared_feature_vectors, units, is_training, extra_options): with tf.variable_scope('ccpm'): _check_ccpm_args(extra_options) use_shared_embedding = extra_options['ccpm_use_shared_embedding'] use_project = extra_options['ccpm_use_project'] project_size = extra_options['ccpm_project_size'] hidden_units = extra_options['ccpm_hidden_units'] activation_fn = extra_options['ccpm_activation_fn'] dropout = extra_options['ccpm_dropout'] batch_norm = extra_options['ccpm_batch_norm'] layer_norm = extra_options['ccpm_layer_norm'] use_resnet = extra_options['ccpm_use_resnet'] use_densenet = extra_options['ccpm_use_densenet'] kernel_sizes = extra_options['ccpm_kernel_sizes'] filter_nums = extra_options['ccpm_filter_nums'] leaky_relu_alpha = extra_options['leaky_relu_alpha'] swish_beta = extra_options['swish_beta'] activation_fn = get_activation_fn(activation_fn=activation_fn, leaky_relu_alpha=leaky_relu_alpha, swish_beta=swish_beta) if not use_shared_embedding: feature_vectors = get_feature_vectors(features, feature_columns) else: feature_vectors = shared_feature_vectors if use_project: feature_vectors = project(feature_vectors, project_size) y = _build_ccpm_model(feature_vectors=feature_vectors, kernel_sizes=kernel_sizes, filter_nums=filter_nums, hidden_units=hidden_units, activation_fn=activation_fn, dropout=dropout, is_training=is_training, batch_norm=batch_norm, layer_norm=layer_norm, use_resnet=use_resnet, use_densenet=use_densenet) with tf.variable_scope('logits') as logits_scope: logits = fc(y, units=units, name=logits_scope) add_hidden_layer_summary(logits, logits_scope.name) return logits
def get_nfm_logits(features, feature_columns, shared_feature_vectors, units, is_training, extra_options): with tf.variable_scope('nfm'): _check_nfm_args(extra_options) use_shared_embedding = extra_options['nfm_use_shared_embedding'] use_project = extra_options['nfm_use_project'] project_size = extra_options['nfm_project_size'] hidden_units = extra_options['nfm_hidden_units'] activation_fn = extra_options['nfm_activation_fn'] dropout = extra_options['nfm_dropout'] batch_norm = extra_options['nfm_batch_norm'] layer_norm = extra_options['nfm_layer_norm'] use_resnet = extra_options['nfm_use_resnet'] use_densenet = extra_options['nfm_use_densenet'] leaky_relu_alpha = extra_options['leaky_relu_alpha'] swish_beta = extra_options['swish_beta'] activation_fn = get_activation_fn(activation_fn=activation_fn, leaky_relu_alpha=leaky_relu_alpha, swish_beta=swish_beta) if not use_shared_embedding: feature_vectors = get_feature_vectors(features, feature_columns) else: feature_vectors = shared_feature_vectors if use_project: feature_vectors = project(feature_vectors, project_size) # Neural FM y = _fm(feature_vectors, reduce_sum=False) y = add_hidden_layers(y, hidden_units=hidden_units, activation_fn=activation_fn, dropout=dropout, is_training=is_training, batch_norm=batch_norm, layer_norm=layer_norm, use_resnet=use_resnet, use_densenet=use_densenet, scope='hidden_layers') with tf.variable_scope('logits') as logits_scope: logits = fc(y, units, name=logits_scope) add_hidden_layer_summary(logits, logits_scope.name) return logits
def _fwfm(feature_vectors, units): """FwFM feature_vectors: List of shape [B, D] tensors, size N units: logits units Return: Tensor of shape [B, 1] """ with tf.variable_scope('fwfm'): check_feature_dims(feature_vectors) y = pairwise_dot(feature_vectors) # [B, N*(N-1)/2, D] y = tf.reduce_sum(y, axis=-1) # [B, N*(N-1)/2] y = fc(y, units) # [B, 1] return y
def _nifm(feature_vectors, hidden_units, activation_fn, dropout, is_training, batch_norm, layer_norm, use_resnet, use_densenet, reduce_sum=True): """Network in FM feature_vectors: List of shape [B, D] tensors, size N Return: Tensor of shape [B, 1] if reduce_sum is True, or shape of [B, N*(N-1)/2] """ with tf.variable_scope('nifm'): outputs = [] N = len(feature_vectors) for i in range(N - 1): for j in range(i, N): scope_name = 'subnet_{}_{}'.format(i, j) vi = feature_vectors[i] vj = feature_vectors[j] v = tf.concat([vi, vj], axis=1) y = add_hidden_layers(v, hidden_units=hidden_units, activation_fn=tf.nn.relu, dropout=dropout, is_training=is_training, batch_norm=batch_norm, layer_norm=layer_norm, use_resnet=use_resnet, use_densenet=use_densenet, scope=scope_name) y = fc(y, 1) outputs.append(y) y = tf.concat(outputs, axis=1) # [B, N*(N-1)/2] if reduce_sum: y = tf.reduce_sum(y, axis=-1, keepdims=True) # [B, 1] tf.logging.info("nifm output = {}".format(y)) return y
def inference_resnet(images): with tf.variable_scope('1'): conv1 = common.conv(images, 64, ksize=7, stride=2) conv1 = common.bn(conv1) pool1 = common.max_pool(conv1) with tf.variable_scope('2'): stack2 = common.res_stack(pool1, [256, 256, 256], pool=False) with tf.variable_scope('3'): stack3 = common.res_stack(stack2, [512, 512, 512, 512]) with tf.variable_scope('4'): stack4 = common.res_stack(stack3, [1024, 1024, 1024, 1024, 1024, 1024]) with tf.variable_scope('5'): stack5 = common.res_stack(stack4, [2048, 2048, 2048]) pool5 = common.global_ave_pool(stack5) with tf.variable_scope('fc'): fc = common.fc(pool5, 1) return fc
def get_autoint_logits( features, feature_columns, shared_feature_vectors, units, is_training, extra_options): with tf.variable_scope('autoint'): _check_autoint_args(extra_options) use_shared_embedding = extra_options['autoint_use_shared_embedding'] use_project = extra_options['autoint_use_project'] project_size = extra_options['autoint_project_size'] size_per_head = extra_options['autoint_size_per_head'] num_heads = extra_options['autoint_num_heads'] num_blocks = extra_options['autoint_num_blocks'] dropout = extra_options['autoint_dropout'] has_residual = extra_options['autoint_has_residual'] if not use_shared_embedding: feature_vectors = get_feature_vectors(features, feature_columns) else: feature_vectors = shared_feature_vectors if use_project: feature_vectors = project(feature_vectors, project_size) check_feature_dims(feature_vectors) x = tf.stack(feature_vectors, axis=1) # [B, N, D] y = _autoint(x, num_blocks=num_blocks, num_units=size_per_head*num_heads, num_heads=num_heads, dropout=dropout, is_training=is_training, has_residual=has_residual) tf.logging.info("autoint output = {}".format(y)) with tf.variable_scope('logits') as logits_scope: logits = fc(y, units, name=logits_scope) add_hidden_layer_summary(logits, logits_scope.name) return logits
def inference_small(images): with tf.variable_scope('1'): conv1 = common.conv(images, 64, ksize=7, stride=2) conv1 = common.bn(conv1) pool1 = common.max_pool(conv1) with tf.variable_scope('2'): stack2 = common.stack(pool1, common.res_block, [64, 64]) pool2 = common.max_pool(stack2) with tf.variable_scope('3'): stack3 = common.stack(pool2, common.res_block, [128, 128]) pool3 = common.max_pool(stack3) with tf.variable_scope('4'): stack4 = common.stack(pool3, common.res_block, [256, 256, 256]) pool4 = common.max_pool(stack4) with tf.variable_scope('5'): stack5 = common.stack(pool4, common.res_block, [512, 512]) pool5 = common.global_ave_pool(stack5) with tf.variable_scope('fc'): fc = common.fc(pool5, 1) return fc
def get_fibinet_logits( features, feature_columns, shared_feature_vectors, units, is_training, extra_options): with tf.variable_scope('fibinet'): _check_fibinet_args(extra_options) use_shared_embedding = extra_options['fibinet_use_shared_embedding'] use_project = extra_options['fibinet_use_project'] project_size = extra_options['fibinet_project_size'] hidden_units = extra_options['fibinet_hidden_units'] activation_fn = extra_options['fibinet_activation_fn'] dropout = extra_options['fibinet_dropout'] batch_norm = extra_options['fibinet_batch_norm'] layer_norm = extra_options['fibinet_layer_norm'] use_resnet = extra_options['fibinet_use_resnet'] use_densenet = extra_options['fibinet_use_densenet'] use_se = extra_options['fibinet_use_se'] use_deep = extra_options['fibinet_use_deep'] interaction_type = extra_options['fibinet_interaction_type'] se_interaction_type = extra_options['fibinet_se_interaction_type'] se_use_shared_embedding = extra_options['fibinet_se_use_shared_embedding'] leaky_relu_alpha = extra_options['leaky_relu_alpha'] swish_beta = extra_options['swish_beta'] activation_fn = get_activation_fn(activation_fn=activation_fn, leaky_relu_alpha=leaky_relu_alpha, swish_beta=swish_beta) if not use_shared_embedding: feature_vectors = get_feature_vectors(features, feature_columns) else: feature_vectors = shared_feature_vectors if use_project: feature_vectors = project(feature_vectors, project_size) y = shallow_fibinet(features=features, feature_columns=feature_columns, shared_feature_vectors=feature_vectors, se_use_shared_embedding=se_use_shared_embedding, use_project=use_project, project_size=project_size, interaction_type=interaction_type, se_interaction_type=se_interaction_type, use_se=use_se) # [B, -1] if use_deep: y = add_hidden_layers(y, hidden_units=hidden_units, activation_fn=activation_fn, dropout=dropout, is_training=is_training, batch_norm=batch_norm, layer_norm=layer_norm, use_resnet=use_resnet, use_densenet=use_densenet, scope='hidden_layers') with tf.variable_scope('logits') as logits_scope: logits = fc(y, units, name=logits_scope) add_hidden_layer_summary(logits, logits_scope.name) else: assert units == 1, "shallow_fibinet's units must be 1" with tf.variable_scope('logits') as logits_scope: logits = tf.reduce_sum(y, axis=-1, keepdims=True) # [B, 1] add_hidden_layer_summary(logits, logits_scope.name) return logits
def _deepx_model_fn( features, labels, mode, head, linear_feature_columns, linear_optimizer, deep_feature_columns, deep_optimizer, model_slots, extend_feature_mode, input_layer_partitioner, config, linear_sparse_combiner, use_seperated_logits, use_weighted_logits, extra_options): _check_model_input(features, linear_feature_columns, deep_feature_columns) num_ps_replicas = config.num_ps_replicas if config else 0 tf.logging.info("num_ps_replicas = {}".format(num_ps_replicas)) input_layer_partitioner = input_layer_partitioner or ( tf.min_max_variable_partitioner( max_partitions=num_ps_replicas, min_slice_size=64 << 20)) is_training = (mode == tf.estimator.ModeKeys.TRAIN) # Build deepx logits deepx_parent_scope = 'deepx' if not deep_feature_columns: deepx_logits_list = [] else: deep_optimizer = optimizers.get_optimizer_instance( deep_optimizer, learning_rate=_LEARNING_RATE) check_no_sync_replicas_optimizer(deep_optimizer) deep_partitioner = ( tf.min_max_variable_partitioner( max_partitions=num_ps_replicas)) with tf.variable_scope( deepx_parent_scope, values=tuple(six.itervalues(features)), partitioner=deep_partitioner) as scope: deepx_absolute_scope = scope.name deepx_logits_list = _build_deepx_logits( features=features, feature_columns=deep_feature_columns, model_slots=model_slots, units=head.logits_dimension, is_training=is_training, extra_options=extra_options) linear_parent_scope = 'linear' use_linear = 'linear' in model_slots if not linear_feature_columns or not use_linear: linear_logits = None else: linear_optimizer = optimizers.get_optimizer_instance( linear_optimizer, learning_rate=_LEARNING_RATE) check_no_sync_replicas_optimizer(linear_optimizer) with tf.variable_scope( linear_parent_scope, values=tuple(six.itervalues(features)), partitioner=input_layer_partitioner) as scope: linear_absolute_scope = scope.name linear_logits = get_linear_logits( features, linear_feature_columns, head.logits_dimension, linear_sparse_combiner, scope) # Combine logits and build full model. logits = [] logits.extend(deepx_logits_list) if linear_logits is not None: logits.append(linear_logits) assert not (use_seperated_logits and use_weighted_logits), \ "'use_seperated_logits' and 'use_weighted_logits' should not be set " \ "at the same time." if not use_seperated_logits: logits = sum(logits) if use_weighted_logits and len(logits) > 1: logits = tf.concat(logits, axis=1) logits = fc(logits, head.logits_dimension) def _train_op_fn(loss): """Returns the op to optimize the loss.""" train_ops = [] global_step = tf.train.get_global_step() if len(deepx_logits_list) > 0: train_ops.append( deep_optimizer.minimize( loss, var_list=tf.get_collection( tf.GraphKeys.TRAINABLE_VARIABLES, scope=deepx_absolute_scope))) if linear_logits is not None: train_ops.append( linear_optimizer.minimize( loss, var_list=tf.get_collection( tf.GraphKeys.TRAINABLE_VARIABLES, scope=linear_absolute_scope))) train_op = tf.group(*train_ops) with tf.control_dependencies([train_op]): return tf.assign_add(global_step, 1).op return head.create_estimator_spec( features=features, mode=mode, labels=labels, train_op_fn=_train_op_fn, logits=logits)