with tf.variable_scope('Dense'): for i, unit in enumerate(params['hidden_units']): dense = tf.layers.dense(dense, units=unit, activation='relu', name='dense{}'.format(i)) dense = tf.layers.batch_normalization( dense, center=True, scale=True, trainable=True, training=(mode == tf.estimator.ModeKeys.TRAIN)) dense = tf.layers.dropout( dense, rate=params['dropout_rate'], training=(mode == tf.estimator.ModeKeys.TRAIN)) add_layer_summary(dense.name, dense) with tf.variable_scope('output'): y = tf.layers.dense(dense, units=1, name='output') add_layer_summary('output', y) return y build_estimator = build_estimator_helper({'dense': model_fn}, params={ 'dropout_rate': 0.2, 'learning_rate': 0.002, 'hidden_units': [5, 5] })
add_layer_summary(linear_term.name, linear_term) with tf.variable_scope('fm_interaction'): init = tf.truncated_normal(shape=(input_dim, params['factor_dim'])) v = tf.get_variable('v', dtype=tf.float32, initializer=init, validate_shape=False) sum_square = tf.pow(tf.matmul(input, v), 2) square_sum = tf.matmul(tf.pow(input, 2), tf.pow(v, 2)) interaction_term = 0.5 * tf.reduce_mean( sum_square - square_sum, axis=1, keep_dims=True) add_layer_summary(interaction_term.name, interaction_term) with tf.variable_scope('output'): y = tf.math.add(interaction_term, linear_term) add_layer_summary(y.name, y) return y build_estimator = build_estimator_helper( model_fn={'dense': model_fn}, params={'dense': { 'learning_rate': 0.01, 'factor_dim': 20 }})
build_estimator = build_estimator_helper( model_fn={ 'amazon': model_fn_varlen, 'movielens': model_fn_varlen, 'heybox': model_fn_varlen }, params={ 'amazon': { 'dropout_rate': 0.2, 'batch_norm': True, 'learning_rate': 0.01, 'hidden_units': [80, 40], 'attention_hidden_unit': 80, 'atten_mode': 'ln', 'num_heads': 1, 'item_count': AMAZON_ITEM_COUNT, 'cate_count': AMAZON_CATE_COUNT, 'seq_names': ['item', 'cate'], 'num_of_expert': 50, 'sparse_emb_dim': 128, 'emb_dim': 128, 'model_name': 'userperexpert', 'data_name': 'amazon', 'input_features': [ 'dense_emb', 'item_emb', 'cate_emb', 'item_att_emb', 'cate_att_emb' ] # 每个element表示一个field,对应的嵌入尺寸一致 }, 'heybox': { 'dropout_rate': 0.2, 'batch_norm': True, 'learning_rate': 0.01, 'hidden_units': [80, 40], 'attention_hidden_unit': 80, 'atten_mode': 'ln', 'num_heads': 1, 'item_count': HEYBOX_ITEM_COUNT, 'cate_count': HEYBOX_CATE_COUNT, 'seq_names': ['item', 'cate'], 'num_of_expert': 50, 'sparse_emb_dim': 128, 'emb_dim': 128, 'model_name': 'userperexpert', 'data_name': 'heybox', 'input_features': [ 'dense_emb', 'item_emb', 'cate_emb', 'item_att_emb', 'cate_att_emb' ] # 每个element表示一个field,对应的嵌入尺寸一致 }, 'movielens': { 'dropout_rate': 0.2, 'batch_norm': True, 'learning_rate': 0.01, 'hidden_units': [80, 40], 'attention_hidden_unit': 80, 'atten_mode': 'ln', 'num_heads': 1, 'item_count': ML_ITEM_COUNT, 'cate_count': ML_CATE_COUNT, 'seq_names': ['item', 'cate'], 'num_of_expert': 50, 'sparse_emb_dim': 128, 'emb_dim': 128, 'model_name': 'userperexpert', 'data_name': 'movielens', 'input_features': [ 'dense_emb', 'item_emb', 'cate_emb', 'item_att_emb', 'cate_att_emb' ] # 每个element表示一个field,对应的嵌入尺寸一致 } })
interaction_output = tf.reduce_sum(tf.multiply(elementwise_matrix, attention_weight), axis=1) # batch * k interaction_output = tf.layers.dense(interaction_output, units=1) # batch * 1 with tf.variable_scope('output'): y = interaction_output + linear_output add_layer_summary('output', y) return y build_estimator = build_estimator_helper(model_fn={ 'census': model_fn_dense, 'frappe': model_fn_sparse }, params={ 'census': { 'attention_factor': 3, 'dropout_rate': 0.2, 'learning_rate': 0.01 }, 'frappe': { 'attention_factor': 16, 'dropout_rate': 0.2, 'learning_rate': 0.01, 'hidden_units': [128, 64, 1], 'data_params': FRAPPE_PARAMS } })
numeric_input = tf.layers.batch_normalization(numeric_input, center = True, scale = True, trainable =True, training = (mode == tf.estimator.ModeKeys.TRAIN)) add_layer_summary( numeric_input.name, numeric_input ) dense = tf.concat([dense, numeric_input], axis = 1, name ='numeric_concat') add_layer_summary(dense.name, dense) with tf.variable_scope('MLP'): for i, unit in enumerate(params['hidden_units']): dense = tf.layers.dense(dense, units = unit, activation = 'relu', name = 'Dense_{}'.format(i)) if mode == tf.estimator.ModeKeys.TRAIN: add_layer_summary(dense.name, dense) dense = tf.layers.dropout(dense, rate = params['dropout_rate'], training = (mode==tf.estimator.ModeKeys.TRAIN)) with tf.variable_scope('output'): y = tf.layers.dense(dense, units=1, name = 'output') return y build_estimator = build_estimator_helper( {'dense':model_fn}, params = { 'learning_rate' :0.002, 'numeric_handle':'dense', # dense or bucketize are supported 'hidden_units': [20,10], 'embedding_dim': 4, 'dropout_rate': 0.1 } )
name='dense{}'.format(i)) dense = tf.layers.batch_normalization( dense, center=True, scale=True, trainable=True, training=(mode == tf.estimator.ModeKeys.TRAIN)) dense = tf.layers.dropout( dense, rate=params['dropout_rate'], training=(mode == tf.estimator.ModeKeys.TRAIN)) add_layer_summary(dense.name, dense) with tf.variable_scope('output'): y = tf.layers.dense(dense, units=1, name='output') add_layer_summary('output', y) return y build_estimator = build_estimator_helper( model_fn={'census': model_fn}, params={ 'census': { 'model_type': 'IPNN', # support IPNN/OPNN/PNN 'dropout_rate': 0.2, 'learning_rate': 0.01, 'hidden_units': [24, 12, 1] } })
x_stack = tf.concat([dense, xl], axis=1) with tf.variable_scope('output'): y = tf.layers.dense(x_stack, units=1) add_layer_summary('output', y) return y build_estimator = build_estimator_helper(model_fn={ 'dense': model_fn_dense, 'sparse': model_fn_sparse }, params={ 'dense': { 'dropout_rate': 0.2, 'batch_norm': True, 'learning_rate': 0.002, 'hidden_units': [10, 5], 'cross_layers': 3, 'cross_op': 'raw' }, 'sparse': { 'dropout_rate': 0.2, 'batch_norm': True, 'learning_rate': 0.01, 'hidden_units': [128, 64], 'cross_layers': 3, 'cross_op': 'better' } })
@tf_estimator_model def model_fn(features, labels, mode, params): dense_feature = build_features() dense = tf.feature_column.input_layer(features, dense_feature) # stacked residual layer with tf.variable_scope('Residual_layers'): for i, unit in enumerate(params['hidden_units']): dense = residual_layer(dense, unit, dropout_rate=params['dropout_rate'], batch_norm=params['batch_norm'], mode=mode) add_layer_summary('residual_layer{}'.format(i), dense) with tf.variable_scope('output'): y = tf.layers.dense(dense, units=1) add_layer_summary('output', y) return y build_estimator = build_estimator_helper(model_fn={'census': model_fn}, params={ 'census': { 'dropout_rate': 0.2, 'batch_norm': True, 'learning_rate': 0.01, 'hidden_units': [10, 5] } })
training=(mode == tf.estimator.ModeKeys.TRAIN)) dense = tf.layers.dropout( dense, rate=params['dropout_rate'], training=(mode == tf.estimator.ModeKeys.TRAIN)) add_layer_summary(dense.name, dense) with tf.variable_scope('output'): y = dense + fm_output + linear_output # batch * 1 add_layer_summary('output', y) return y build_estimator = build_estimator_helper(model_fn={ 'dense': model_fn_dense, 'sparse': model_fn_sparse }, params={ 'dense': { 'dropout_rate': 0.2, 'learning_rate': 0.001, 'hidden_units': [20, 10, 1] }, 'sparse': { 'dropout_rate': 0.2, 'learning_rate': 0.002, 'hidden_units': [128, 64, 1] } })
add_layer_summary(dense.name, dense) with tf.variable_scope('MLP'): for i, unit in enumerate(params['hidden_units']): dense = tf.layers.dense(dense, units = unit, activation = 'relu', name = 'Dense_{}'.format(i)) if mode == tf.estimator.ModeKeys.TRAIN: add_layer_summary(dense.name, dense) dense = tf.layers.dropout(dense, rate = params['dropout_rate'], training = (mode==tf.estimator.ModeKeys.TRAIN)) with tf.variable_scope('output'): y = tf.layers.dense(dense, units=1, name = 'output') return y build_estimator = build_estimator_helper( model_fn = { 'census':model_fn }, params = { 'census': {'learning_rate' :0.01, 'model_type':'dense', # dense or bucketize are supported 'hidden_units': [20,10], 'embedding_dim': 4, 'dropout_rate': 0.1 } } )
# concat and output with tf.variable_scope('output'): y = tf.concat([dense_output, cin_output, linear_output], axis=1) y = tf.layers.dense(y, units=1) add_layer_summary('output', y) return y build_estimator = build_estimator_helper(model_fn={ 'dense': model_fn_dense, 'sparse': model_fn_sparse }, params={ 'dense': { 'dropout_rate': 0.2, 'learning_rate': 0.001, 'hidden_units': [20, 10], 'batch_norm': True, 'cin_layer_size': [8, 4, 4] }, 'sparse': { 'dropout_rate': 0.2, 'learning_rate': 0.002, 'hidden_units': [128, 64, 32], 'batch_norm': True, 'cin_layer_size': [32, 16, 8] } })
dropout_rate=params['dropout_rate'], batch_norm=params['batch_norm'], mode=mode, add_summary=True) with tf.variable_scope('output'): y = linear_output + dense add_layer_summary('output', y) return y build_estimator = build_estimator_helper(model_fn={ 'census': model_fn_dense, 'frappe': model_fn_sparse }, params={ 'census': { 'dropout_rate': 0.2, 'batch_norm': True, 'learning_rate': 0.01, 'hidden_units': [20, 10, 1] }, 'frappe': { 'dropout_rate': 0.2, 'batch_norm': True, 'learning_rate': 0.01, 'hidden_units': [128, 64, 1], 'data_params': FRAPPE_PARAMS } })
dense, rate=params['dropout_rate'], training=(mode == tf.estimator.ModeKeys.TRAIN)) add_layer_summary(dense.name, dense) with tf.variable_scope('output'): y = tf.layers.dense(dense, units=1, name='output') tf.summary.histogram(y.name, y) return y build_estimator = build_estimator_helper(model_fn={'census': model_fn}, params={ 'census': { 'dropout_rate': 0.2, 'learning_rate': 0.01, 'hidden_units': [24, 12, 1] } }) # check name of all the tensor in the census_checkpoint if __name__ == '__main__': print( 'checking name of all the tensor in the FNN pretrain census_checkpoint' ) from tensorflow.python.tools.inspect_checkpoint import print_tensors_in_checkpoint_file latest_ckp = tf.train.latest_checkpoint('./census_checkpoint/FM') print_tensors_in_checkpoint_file(latest_ckp, all_tensors=True) print_tensors_in_checkpoint_file(latest_ckp, all_tensors=False,
with tf.variable_scope('Attention_pooling'): interaction_output = tf.reduce_sum(tf.multiply(elementwise_matrix, attention_weight), axis=1) # batch * k interaction_output = tf.layers.dense(interaction_output, units=1) # batch * 1 with tf.variable_scope('output'): y = interaction_output + linear_output add_layer_summary('output', y) return y build_estimator = build_estimator_helper(model_fn={ 'dense': model_fn_dense, 'sparse': model_fn_sparse }, params={ 'dense': { 'attention_factor': 3, 'dropout_rate': 0.2, 'learning_rate': 0.002 }, 'sparse': { 'attention_factor': 16, 'dropout_rate': 0.2, 'learning_rate': 0.002, 'hidden_units': [128, 64, 1] } })
with tf.variable_scope('output'): y = tf.layers.dense(x_stack, units=1) add_layer_summary('output', y) return y build_estimator = build_estimator_helper(model_fn={ 'census': model_fn_dense, 'frappe': model_fn_sparse }, params={ 'census': { 'dropout_rate': 0.2, 'batch_norm': True, 'learning_rate': 0.01, 'hidden_units': [10, 5], 'cross_layers': 3, 'cross_op': 'raw' }, 'frappe': { 'dropout_rate': 0.2, 'batch_norm': True, 'learning_rate': 0.01, 'hidden_units': [128, 64], 'cross_layers': 3, 'cross_op': 'better', 'data_params': FRAPPE_PARAMS } })
build_estimator = build_estimator_helper( model_fn={ 'census': model_fn_dense, 'frappe': model_fn_sparse }, params={ 'census': { 'dropout_rate': 0.2, 'learning_rate': 0.01, 'hidden_units': [20, 10, 1], 'batch_norm': True, 'cin_layer_size': [8, 4, 4], 'pool_op': 'avg', 'senet_ratio': 2, 'model_type': 'field_all' # support field_all / field_each / field_interaction }, 'frappe': { 'dropout_rate': 0.2, 'learning_rate': 0.01, 'hidden_units': [128, 64, 32, 1], 'batch_norm': True, 'cin_layer_size': [32, 16, 8], 'pool_op': 'avg', 'senet_ratio': 2, 'model_type': 'field_all', # support field_all / field_each / field_interaction 'data_params': FRAPPE_PARAMS } })