def _model_fn(features, labels, mode, config): head = head_lib._binary_logistic_or_multi_class_head( # pylint: disable=protected-access n_classes, weight_column, label_vocabulary, loss_reduction, loss_fn) is_training = (mode == tf.estimator.ModeKeys.TRAIN) net = tf.feature_column.input_layer(features, feature_columns) inputs = _attention_layer(features, attention_columns, is_training) tf.logging.info('attention outputs = {}'.format(inputs)) inputs.append(net) net = tf.concat(inputs, axis=-1) tf.logging.info("inputs: {}".format(net)) if batch_norm: net = tf.layers.batch_normalization(net, training=is_training) net = add_hidden_layers(net, hidden_units, activation_fn, dropout, is_training, batch_norm, 'DNN') with tf.variable_scope('logits') as logits_scope: logits = fc(net, head.logits_dimension, name=logits_scope) add_hidden_layer_summary(logits, logits_scope.name) return head.create_estimator_spec( features=features, mode=mode, labels=labels, optimizer=optimizers.get_optimizer_instance( optimizer, learning_rate=_LEARNING_RATE), logits=logits)
def _ipnn(feature_vectors, project_feature_vectors, use_project, units, hidden_units, activation_fn, dropout, batch_norm, layer_norm, use_resnet, use_densenet, is_training, unordered_inner_product, concat_project): """IPNN feature_vectors: List of shape [B, ?] tensors, size N project_feature_vectors: None or list of shape [B, D] tensors, size N Return: 2D Tensor of shape [B, ?] """ with tf.variable_scope("ipnn"): if use_project: x = project_feature_vectors else: x = feature_vectors if unordered_inner_product: y = pairwise_dot_unorderd(x) # [B, N*N, D] else: y = pairwise_dot(x) # [B, N*(N-1)/2, D] y = tf.reduce_sum(y, axis=-1) # [B, M] inputs = [] inputs.append(y) if use_project and concat_project: inputs.extend(project_feature_vectors) else: inputs.extend(feature_vectors) y = tf.concat(inputs, axis=1) # [B, ?] y = add_hidden_layers(y, hidden_units=hidden_units, activation_fn=activation_fn, dropout=dropout, is_training=is_training, batch_norm=batch_norm, layer_norm=layer_norm, use_resnet=use_resnet, use_densenet=use_densenet, scope='hidden_layers') # [B, ?] return y
def _build_ccpm_model(feature_vectors, kernel_sizes, filter_nums, hidden_units, activation_fn, dropout, is_training, batch_norm, layer_norm, use_resnet, use_densenet): """Build ccpm model feature_vectors: List of 2D tensors Return: Tensor of shape [B, ?] """ assert len(kernel_sizes) == len(filter_nums) check_feature_dims(feature_vectors) with tf.variable_scope("ccpm"): with tf.variable_scope('cnn'): x = tf.stack(feature_vectors, axis=1) # [B, N, D] L = len(kernel_sizes) N = x.shape[1].value D = x.shape[2].value y = tf.expand_dims(x, -1) # [B, N, D, 1] for idx, (width, num) in enumerate(zip(kernel_sizes, filter_nums)): name = 'conv_{}_w{}_n{}'.format(idx, width, num) y = tf.layers.conv2d(y, filters=num, kernel_size=(width, D), strides=(1, 1), padding='SAME', activation=activation_fn, name=name) if idx == L - 1: p = 3 else: p = (1 - (1.0 * idx / L)**(L - idx)) * N p = int(p) p = min(p, y.shape[1].value) name = 'k_max_pooling_{}_k{}'.format(idx, p) y = k_max_pooling(y, k=p, axis=1, name=name) y = tf.layers.flatten(y) # [B, -1] # dnn y = add_hidden_layers(y, hidden_units, activation_fn=activation_fn, dropout=dropout, is_training=is_training, batch_norm=batch_norm, layer_norm=layer_norm, use_resnet=use_resnet, use_densenet=use_densenet, scope='dnn') return y
def get_nfm_logits(features, feature_columns, shared_feature_vectors, units, is_training, extra_options): with tf.variable_scope('nfm'): _check_nfm_args(extra_options) use_shared_embedding = extra_options['nfm_use_shared_embedding'] use_project = extra_options['nfm_use_project'] project_size = extra_options['nfm_project_size'] hidden_units = extra_options['nfm_hidden_units'] activation_fn = extra_options['nfm_activation_fn'] dropout = extra_options['nfm_dropout'] batch_norm = extra_options['nfm_batch_norm'] layer_norm = extra_options['nfm_layer_norm'] use_resnet = extra_options['nfm_use_resnet'] use_densenet = extra_options['nfm_use_densenet'] leaky_relu_alpha = extra_options['leaky_relu_alpha'] swish_beta = extra_options['swish_beta'] activation_fn = get_activation_fn(activation_fn=activation_fn, leaky_relu_alpha=leaky_relu_alpha, swish_beta=swish_beta) if not use_shared_embedding: feature_vectors = get_feature_vectors(features, feature_columns) else: feature_vectors = shared_feature_vectors if use_project: feature_vectors = project(feature_vectors, project_size) # Neural FM y = _fm(feature_vectors, reduce_sum=False) y = add_hidden_layers(y, hidden_units=hidden_units, activation_fn=activation_fn, dropout=dropout, is_training=is_training, batch_norm=batch_norm, layer_norm=layer_norm, use_resnet=use_resnet, use_densenet=use_densenet, scope='hidden_layers') with tf.variable_scope('logits') as logits_scope: logits = fc(y, units, name=logits_scope) add_hidden_layer_summary(logits, logits_scope.name) return logits
def _nifm(feature_vectors, hidden_units, activation_fn, dropout, is_training, batch_norm, layer_norm, use_resnet, use_densenet, reduce_sum=True): """Network in FM feature_vectors: List of shape [B, D] tensors, size N Return: Tensor of shape [B, 1] if reduce_sum is True, or shape of [B, N*(N-1)/2] """ with tf.variable_scope('nifm'): outputs = [] N = len(feature_vectors) for i in range(N - 1): for j in range(i, N): scope_name = 'subnet_{}_{}'.format(i, j) vi = feature_vectors[i] vj = feature_vectors[j] v = tf.concat([vi, vj], axis=1) y = add_hidden_layers(v, hidden_units=hidden_units, activation_fn=tf.nn.relu, dropout=dropout, is_training=is_training, batch_norm=batch_norm, layer_norm=layer_norm, use_resnet=use_resnet, use_densenet=use_densenet, scope=scope_name) y = fc(y, 1) outputs.append(y) y = tf.concat(outputs, axis=1) # [B, N*(N-1)/2] if reduce_sum: y = tf.reduce_sum(y, axis=-1, keepdims=True) # [B, 1] tf.logging.info("nifm output = {}".format(y)) return y
def _pin(feature_vectors, project_feature_vectors, use_project, subnet_hidden_units, units, hidden_units, activation_fn, dropout, batch_norm, layer_norm, use_resnet, use_densenet, is_training, concat_project, use_concat): """ feature_vectors: List of shape [B, ?] tensors, size N project_feature_vectors: None or list of shape [B, D] tensors, size N Return: 2D Tensor of shape [B, ?] """ with tf.variable_scope("pin"): if use_project: x = project_feature_vectors else: x = feature_vectors y = _pin_layer(x, subnet_hidden_units, is_training) # [B, ?] inputs = [] inputs.append(y) if use_concat: if use_project and concat_project: inputs.extend(project_feature_vectors) else: inputs.extend(feature_vectors) y = tf.concat(inputs, axis=1) # [B, ?] y = add_hidden_layers(y, hidden_units=hidden_units, activation_fn=activation_fn, dropout=dropout, is_training=is_training, batch_norm=batch_norm, layer_norm=layer_norm, use_resnet=use_resnet, use_densenet=use_densenet, scope='hidden_layers') # [B, ?] return y
def _nkfm(feature_vectors, units, hidden_units, activation_fn, dropout, batch_norm, layer_norm, use_resnet, use_densenet, is_training): """nkfm feature_vectors: List of shape [B, ?] tensors, size N Return: 2D Tensor of shape [B, ?] """ with tf.variable_scope("nkfm"): y = pairwise_kernel_dot(feature_vectors) # [B, N*N] y = add_hidden_layers(y, hidden_units=hidden_units, activation_fn=activation_fn, dropout=dropout, is_training=is_training, batch_norm=batch_norm, layer_norm=layer_norm, use_resnet=use_resnet, use_densenet=use_densenet, scope='hidden_layers') # [B, ?] return y
def _pin_layer(feature_vectors, subnet_hidden_units, is_training): """PIN Layer feature_vectors: List of shape [B, ?] tensors, size N Return: 2D Tensor of shape [B, ?] """ with tf.variable_scope('pin_layer'): N = len(feature_vectors) outputs = [] for i in range(N): for j in range(N): vi = feature_vectors[i] vj = feature_vectors[j] di = vi.shape[1].value dj = vj.shape[1].value kernel_name = 'pairwise_kernel_dot_{}_{}'.format(i, j) U = tf.get_variable(kernel_name, [di, dj]) y = tf.matmul(feature_vectors[i], U) # [B, dj] y = tf.concat([vi, vj, y], axis=1) scope_name = 'subnet_{}_{}'.format(i, j) y = add_hidden_layers(y, hidden_units=subnet_hidden_units, activation_fn=tf.nn.relu, dropout=None, is_training=is_training, batch_norm=True, layer_norm=False, use_resnet=False, use_densenet=False, scope=scope_name, last_layer_direct=True) outputs.append(y) y = tf.concat(outputs, axis=1) # [B, ?] return y
def get_fibinet_logits( features, feature_columns, shared_feature_vectors, units, is_training, extra_options): with tf.variable_scope('fibinet'): _check_fibinet_args(extra_options) use_shared_embedding = extra_options['fibinet_use_shared_embedding'] use_project = extra_options['fibinet_use_project'] project_size = extra_options['fibinet_project_size'] hidden_units = extra_options['fibinet_hidden_units'] activation_fn = extra_options['fibinet_activation_fn'] dropout = extra_options['fibinet_dropout'] batch_norm = extra_options['fibinet_batch_norm'] layer_norm = extra_options['fibinet_layer_norm'] use_resnet = extra_options['fibinet_use_resnet'] use_densenet = extra_options['fibinet_use_densenet'] use_se = extra_options['fibinet_use_se'] use_deep = extra_options['fibinet_use_deep'] interaction_type = extra_options['fibinet_interaction_type'] se_interaction_type = extra_options['fibinet_se_interaction_type'] se_use_shared_embedding = extra_options['fibinet_se_use_shared_embedding'] leaky_relu_alpha = extra_options['leaky_relu_alpha'] swish_beta = extra_options['swish_beta'] activation_fn = get_activation_fn(activation_fn=activation_fn, leaky_relu_alpha=leaky_relu_alpha, swish_beta=swish_beta) if not use_shared_embedding: feature_vectors = get_feature_vectors(features, feature_columns) else: feature_vectors = shared_feature_vectors if use_project: feature_vectors = project(feature_vectors, project_size) y = shallow_fibinet(features=features, feature_columns=feature_columns, shared_feature_vectors=feature_vectors, se_use_shared_embedding=se_use_shared_embedding, use_project=use_project, project_size=project_size, interaction_type=interaction_type, se_interaction_type=se_interaction_type, use_se=use_se) # [B, -1] if use_deep: y = add_hidden_layers(y, hidden_units=hidden_units, activation_fn=activation_fn, dropout=dropout, is_training=is_training, batch_norm=batch_norm, layer_norm=layer_norm, use_resnet=use_resnet, use_densenet=use_densenet, scope='hidden_layers') with tf.variable_scope('logits') as logits_scope: logits = fc(y, units, name=logits_scope) add_hidden_layer_summary(logits, logits_scope.name) else: assert units == 1, "shallow_fibinet's units must be 1" with tf.variable_scope('logits') as logits_scope: logits = tf.reduce_sum(y, axis=-1, keepdims=True) # [B, 1] add_hidden_layer_summary(logits, logits_scope.name) return logits
def _model_fn(features, labels, mode, config): head = head_lib._binary_logistic_or_multi_class_head( # pylint: disable=protected-access n_classes, weight_column, label_vocabulary, loss_reduction, loss_fn) is_training = (mode == tf.estimator.ModeKeys.TRAIN) net_dssm1 = tf.feature_column.input_layer(features, dssm1_columns) net_dssm2 = tf.feature_column.input_layer(features, dssm2_columns) tf.logging.info("net_dssm1: {}".format(net_dssm1)) tf.logging.info("net_dssm2: {}".format(net_dssm2)) real_activation_fn = get_activation_fn( activation_fn=activation_fn, leaky_relu_alpha=leaky_relu_alpha, swish_beta=swish_beta) net_dssm1 = add_hidden_layers(inputs=net_dssm1, hidden_units=hidden_units, activation_fn=real_activation_fn, dropout=dropout, is_training=is_training, batch_norm=batch_norm, layer_norm=layer_norm, use_resnet=use_resnet, use_densenet=use_densenet, scope='dssm1') net_dssm2 = add_hidden_layers(inputs=net_dssm2, hidden_units=hidden_units, activation_fn=real_activation_fn, dropout=dropout, is_training=is_training, batch_norm=batch_norm, layer_norm=layer_norm, use_resnet=use_resnet, use_densenet=use_densenet, scope='dssm2') with tf.variable_scope('logits') as logits_scope: if dssm_mode == 'dot': logits = tf.reduce_sum(net_dssm1 * net_dssm2, -1, keepdims=True) elif dssm_mode == 'concat': logits = tf.concat([net_dssm1, net_dssm2], axis=1) logits = tf.layers.dense(logits, units=1, activation=None) elif dssm_mode == 'cosine': logits = tf.reduce_sum(net_dssm1 * net_dssm2, -1, keepdims=True) norm1 = tf.norm(net_dssm1, axis=1, keepdims=True) norm2 = tf.norm(net_dssm2, axis=1, keepdims=True) logits = logits / (norm1 * norm2) else: raise ValueError( "unknown dssm mode '{}'".format(dssm_mode)) add_hidden_layer_summary(logits, logits_scope.name) tf.logging.info("logits = {}".format(logits)) return head.create_estimator_spec( features=features, mode=mode, labels=labels, optimizer=optimizers.get_optimizer_instance( optimizer, learning_rate=_LEARNING_RATE), logits=logits)