def test_model(self, architecture):
"""Test whether a model with specified architecture can be run."""
x = np.random.rand(5, 10).astype('float32')
sess = tf.InteractiveSession()
if architecture == 'exu_nam':
model = models.NAM(num_inputs=x.shape[1],
num_units=1024,
shallow=True,
activation='exu')
elif architecture == 'relu_nam':
model = models.NAM(num_inputs=x.shape[1],
num_units=64,
shallow=False,
activation='relu')
elif architecture == 'dnn':
model = models.DNN()
else:
raise ValueError('Architecture {} not found'.format(architecture))
out_op = model(x)
sess.run(tf.global_variables_initializer())
self.assertIsInstance(sess.run(out_op), np.ndarray)
sess.close()
def build_graph(x_train,
y_train,
x_test,
y_test,
learning_rate,
batch_size,
output_regularization,
dropout,
decay_rate,
shallow,
l2_regularization=0.0,
feature_dropout=0.0,
num_basis_functions=1000,
units_multiplier=2,
activation='exu',
name_scope='model',
regression=False,
use_dnn=False,
trainable=True):
"""Constructs the computation graph with specified hyperparameters."""
if regression:
ds_tensors = tf.data.Dataset.from_tensor_slices(
(x_train, y_train)).apply(
tf.data.experimental.shuffle_and_repeat(
buffer_size=len(x_train[0])))
ds_tensors = ds_tensors.batch(batch_size)
else:
# Create a balanced dataset to handle class imbalance
ds_tensors = create_balanced_dataset(x_train, y_train, batch_size)
x_batch, (train_init_op, test_init_op) = create_iterators(
(x_train, x_test), batch_size)
if use_dnn:
nn_model = models.DNN(dropout=dropout, trainable=trainable)
else:
nn_model = create_nam_model(x_train=x_train,
dropout=dropout,
feature_dropout=feature_dropout,
activation=activation,
num_basis_functions=num_basis_functions,
shallow=shallow,
units_multiplier=units_multiplier,
trainable=trainable,
name_scope=name_scope)
global_step = tf.train.get_or_create_global_step()
learning_rate = tf.Variable(learning_rate, trainable=False)
lr_decay_op = learning_rate.assign(decay_rate * learning_rate)
optimizer = tf.train.AdamOptimizer(learning_rate)
predictions = nn_model(x_batch, training=False)
tf.logging.info(nn_model.summary())
train_vars = nn_model.trainable_variables
if regression:
loss_fn, y_pred = penalized_mse_loss, predictions
else:
# Apply sigmoid transformation for binary classification
loss_fn, y_pred = penalized_cross_entropy_loss, tf.nn.sigmoid(
predictions)
loss_fn = functools.partial(loss_fn,
output_regularization=output_regularization,
l2_regularization=l2_regularization,
use_dnn=use_dnn)
iterator = ds_tensors.make_initializable_iterator()
x1, y1 = iterator.get_next()
loss_tensor, grads = grad(nn_model, x1, y1, loss_fn, train_vars)
update_step = optimizer.apply_gradients(zip(grads, train_vars),
global_step=global_step)
avg_loss, avg_loss_update_op = tf.metrics.mean(loss_tensor,
name='avg_train_loss')
tf.summary.scalar('avg_train_loss', avg_loss)
running_mean_vars = tf.get_collection(tf.GraphKeys.LOCAL_VARIABLES,
scope='avg_train_loss')
running_vars_initializer = tf.variables_initializer(
var_list=running_mean_vars)
# Use RMSE for regression and ROC AUC for classification.
evaluation_metric = rmse_loss if regression else roc_auc_score
train_metric = functools.partial(evaluation_metric,
y_true=y_train,
pred_tensor=y_pred,
dataset_init_op=train_init_op)
test_metric = functools.partial(evaluation_metric,
y_true=y_test,
pred_tensor=y_pred,
dataset_init_op=test_init_op)
summary_op = tf.summary.merge_all()
graph_tensors = {
'train_op': [update_step, avg_loss_update_op],
'lr_decay_op': lr_decay_op,
'summary_op': summary_op,
'iterator_initializer': iterator.initializer,
'running_vars_initializer': running_vars_initializer,
'nn_model': nn_model,
'global_step': global_step,
}
eval_metric_scores = {'test': test_metric, 'train': train_metric}
return graph_tensors, eval_metric_scores