def model_fn(features, labels, mode):
assert features.shape.ndims == 1
if labels is not None:
assert labels.shape.ndims == 0
features = tf.expand_dims(features, axis=0)
h1 = tf.layers.dense(features, 64, activation=tf.nn.relu)
h2 = tf.layers.dense(h1, 64, activation=tf.nn.relu)
logits = tf.layers.dense(h2, 10)
if mode == tf.estimator.ModeKeys.EVAL or mode == tf.estimator.ModeKeys.TRAIN:
labels = tf.expand_dims(labels, axis=0)
loss = tf.reduce_mean(
tf.losses.sparse_softmax_cross_entropy(logits=logits,
labels=labels))
train_op = ZooOptimizer(
tf.train.AdamOptimizer()).minimize(loss)
return tf.estimator.EstimatorSpec(mode,
train_op=train_op,
predictions=logits,
loss=loss)
else:
return tf.estimator.EstimatorSpec(mode, predictions=logits)
def _bert_squad_model_fn(features, labels, mode, params):
final_hidden = bert_model(features, labels, mode,
params).get_sequence_output()
final_hidden_shape = modeling.get_shape_list(final_hidden,
expected_rank=3)
batch_size = final_hidden_shape[0]
seq_length = final_hidden_shape[1]
hidden_size = final_hidden_shape[2]
output_weights = tf.get_variable(
"cls/squad/output_weights", [2, hidden_size],
initializer=tf.truncated_normal_initializer(stddev=0.02))
output_bias = tf.get_variable("cls/squad/output_bias", [2],
initializer=tf.zeros_initializer())
final_hidden_matrix = tf.reshape(
final_hidden, [batch_size * seq_length, hidden_size])
logits = tf.matmul(final_hidden_matrix,
output_weights,
transpose_b=True)
logits = tf.nn.bias_add(logits, output_bias)
logits = tf.reshape(logits, [batch_size, seq_length, 2])
logits = tf.transpose(logits, [2, 0, 1])
unstacked_logits = tf.unstack(logits, axis=0)
(start_logits, end_logits) = (unstacked_logits[0], unstacked_logits[1])
if mode == tf.estimator.ModeKeys.TRAIN:
def compute_loss(logits, positions):
one_hot_positions = tf.one_hot(positions,
depth=seq_length,
dtype=tf.float32)
log_probs = tf.nn.log_softmax(logits, axis=-1)
loss = -tf.reduce_mean(
tf.reduce_sum(one_hot_positions * log_probs, axis=-1))
return loss
start_positions = labels["start_positions"]
end_positions = labels["end_positions"]
start_loss = compute_loss(start_logits, start_positions)
end_loss = compute_loss(end_logits, end_positions)
total_loss = (start_loss + end_loss) / 2.0
train_op = ZooOptimizer(optimizer).minimize(total_loss)
return tf.estimator.EstimatorSpec(mode=mode,
train_op=train_op,
loss=total_loss)
elif mode == tf.estimator.ModeKeys.PREDICT:
predictions = {
"unique_ids": features["unique_ids"],
"start_logits": start_logits,
"end_logits": end_logits,
}
return tf.estimator.EstimatorSpec(mode=mode,
predictions=predictions)
else:
raise ValueError(
"Currently only TRAIN and PREDICT modes are supported. "
"SQuAD uses a separate script for EVAL")
def _bert_ner_model_fn(features, labels, mode, params):
output_layer = bert_model(features, labels, mode,
params).get_sequence_output()
if mode == tf.estimator.ModeKeys.TRAIN:
output_layer = tf.nn.dropout(output_layer, keep_prob=0.9)
logits = tf.layers.dense(output_layer, params["num_entities"])
if mode == tf.estimator.ModeKeys.TRAIN:
logits = tf.reshape(logits, [-1, params["num_entities"]])
labels = tf.reshape(labels, [-1])
mask = tf.cast(features["input_mask"], dtype=tf.float32)
one_hot_labels = tf.one_hot(labels,
depth=params["num_entities"],
dtype=tf.float32)
loss = tf.losses.softmax_cross_entropy(
logits=logits, onehot_labels=one_hot_labels)
loss *= tf.reshape(mask, [-1])
loss = tf.reduce_sum(loss)
total_size = tf.reduce_sum(mask)
total_size += 1e-12 # to avoid division by 0 for all-0 weights
loss /= total_size
train_op = ZooOptimizer(optimizer).minimize(loss)
return tf.estimator.EstimatorSpec(mode=mode,
train_op=train_op,
loss=loss)
elif mode == tf.estimator.ModeKeys.PREDICT:
probabilities = tf.nn.softmax(logits, axis=-1)
predict = tf.argmax(probabilities, axis=-1)
return tf.estimator.EstimatorSpec(mode=mode, predictions=predict)
else:
raise ValueError(
"Currently only TRAIN and PREDICT modes are supported for NER")
def __init__(self, *, inputs, outputs, labels, loss,
optimizer,
metrics,
updates, sess,
model_dir):
self.inputs = inputs
self.outputs = outputs
self.labels = labels
self.loss = loss
if optimizer is not None:
assert isinstance(optimizer, tf.train.Optimizer), \
"optimizer is of type {}, ".format(type(self.optimizer)) + \
"it should be an instance of tf.train.Optimizer"
self.optimizer = ZooOptimizer(optimizer)
self.train_op = self.optimizer.minimize(self.loss)
else:
self.optimizer = None
self.train_op = None
self.metrics = metrics
self.updates = updates
if sess is None:
self.sess = tf.Session()
self.sess.run(tf.global_variables_initializer())
else:
self.sess = sess
self.model_dir = model_dir
def __init__(self, *, inputs, outputs, labels, loss, optimizer, clip_norm,
clip_value, metrics, updates, sess, model_dir):
self.inputs = inputs
self.outputs = outputs
self.labels = labels
self.loss = loss
self.use_bigdl_optim = False
self.clip_norm = clip_norm
self.clip_value = clip_value
if optimizer is not None:
from zoo.orca.learn.optimizers import Optimizer
if isinstance(optimizer, Optimizer):
self.train_op = None
self.optimizer = optimizer.get_optimizer()
self.use_bigdl_optim = True
else:
assert isinstance(optimizer, tf.train.Optimizer), \
"optimizer is of type {}, ".format(type(optimizer)) + \
"it should be an instance of tf.train.Optimizer"
self.optimizer = ZooOptimizer(optimizer)
if clip_norm or clip_value:
gvs = self.optimizer.compute_gradients(self.loss)
if clip_norm:
gvs = [(tf.clip_by_norm(g_v[0], clip_norm), g_v[1])
for g_v in gvs]
if clip_value:
if isinstance(clip_value, tuple):
assert len(clip_value) == 2 and clip_value[0] < clip_value[1], \
"clip value should be (clip_min, clip_max)"
gvs = [(tf.clip_by_value(g_v[0], clip_value[0],
clip_value[1]), g_v[1])
for g_v in gvs]
if isinstance(clip_value, (int, float)):
assert clip_value > 0, "clip value should be larger than 0"
gvs = [(tf.clip_by_value(g_v[0], -clip_value,
clip_value), g_v[1])
for g_v in gvs]
else:
raise Exception(
"clip_value should be a tuple or one number")
self.train_op = self.optimizer.apply_gradients(gvs)
else:
self.train_op = self.optimizer.minimize(self.loss)
else:
self.optimizer = None
self.train_op = None
self.metrics = metrics
self.updates = updates
if sess is None:
self.sess = tf.Session()
self.sess.run(tf.global_variables_initializer())
else:
self.sess = sess
self.model_dir = model_dir
self.load_checkpoint = False
self.tf_optimizer = None
self.log_dir = None
self.app_name = None
def _bert_classifier_model_fn(features, labels, mode, params):
"""
Model function for BERTClassifier.
:param features: Dict of feature tensors. Must include the key "input_ids".
:param labels: Label tensor for training.
:param mode: 'train', 'eval' or 'infer'.
:param params: Must include the key "num_classes".
:return: tf.estimator.EstimatorSpec.
"""
import tensorflow as tf
from zoo.tfpark import ZooOptimizer
output_layer = bert_model(features, labels, mode,
params).get_pooled_output()
hidden_size = output_layer.shape[-1].value
output_weights = tf.get_variable(
"output_weights", [params["num_classes"], hidden_size],
initializer=tf.truncated_normal_initializer(stddev=0.02))
output_bias = tf.get_variable("output_bias", [params["num_classes"]],
initializer=tf.zeros_initializer())
with tf.variable_scope("loss"):
if mode == tf.estimator.ModeKeys.TRAIN:
output_layer = tf.nn.dropout(output_layer, keep_prob=0.9)
logits = tf.matmul(output_layer, output_weights, transpose_b=True)
logits = tf.nn.bias_add(logits, output_bias)
probabilities = tf.nn.softmax(logits, axis=-1)
if mode == tf.estimator.ModeKeys.PREDICT:
return tf.estimator.EstimatorSpec(mode=mode,
predictions=probabilities)
else:
log_probs = tf.nn.log_softmax(logits, axis=-1)
one_hot_labels = tf.one_hot(labels,
depth=params["num_classes"],
dtype=tf.float32)
per_example_loss = -tf.reduce_sum(one_hot_labels * log_probs,
axis=-1)
loss = tf.reduce_mean(per_example_loss)
if mode == tf.estimator.ModeKeys.EVAL:
return tf.estimator.EstimatorSpec(
mode=mode, predictions=probabilities, loss=loss)
else:
train_op = ZooOptimizer(optimizer).minimize(loss)
return tf.estimator.EstimatorSpec(mode=mode,
train_op=train_op,
loss=loss)
def model_fn(features, labels, mode):
features = tf.layers.flatten(features)
h1 = tf.layers.dense(features, 64, activation=tf.nn.relu)
h2 = tf.layers.dense(h1, 64, activation=tf.nn.relu)
logits = tf.layers.dense(h2, 10)
if mode == tf.estimator.ModeKeys.EVAL or mode == tf.estimator.ModeKeys.TRAIN:
loss = tf.reduce_mean(
tf.losses.sparse_softmax_cross_entropy(logits=logits,
labels=labels))
train_op = ZooOptimizer(
tf.train.AdamOptimizer()).minimize(loss)
return tf.estimator.EstimatorSpec(mode,
train_op=train_op,
predictions=logits,
loss=loss)
else:
return tf.estimator.EstimatorSpec(mode, predictions=logits)
def model_fn(features, labels, mode):
from nets import lenet
slim = tf.contrib.slim
with slim.arg_scope(lenet.lenet_arg_scope()):
logits, end_points = lenet.lenet(features,
num_classes=10,
is_training=True)
if mode == tf.estimator.ModeKeys.EVAL or mode == tf.estimator.ModeKeys.TRAIN:
loss = tf.reduce_mean(
tf.losses.sparse_softmax_cross_entropy(logits=logits,
labels=labels))
optimizer = ZooOptimizer(tf.train.AdamOptimizer())
train_op = optimizer.minimize(loss)
return tf.estimator.EstimatorSpec(mode,
loss=loss,
train_op=train_op)
else:
return tf.estimator.EstimatorSpec(mode, predictions=logits)
def model_fn(features, labels, mode, params):
from nets import inception
slim = tf.contrib.slim
labels = tf.squeeze(labels, axis=1)
with slim.arg_scope(inception.inception_v1_arg_scope()):
logits, end_points = inception.inception_v1(
features,
num_classes=int(params["num_classes"]),
is_training=True)
if mode == tf.estimator.ModeKeys.TRAIN:
loss = tf.reduce_mean(
tf.losses.sparse_softmax_cross_entropy(logits=logits,
labels=labels))
train_op = ZooOptimizer(tf.train.AdamOptimizer()).minimize(loss)
return tf.estimator.EstimatorSpec(mode,
train_op=train_op,
predictions=logits,
loss=loss)
else:
raise NotImplementedError
'embark_town', 'alone']
NUMERIC_COLUMNS = ['age', 'fare']
feature_columns = []
for feature_name in CATEGORICAL_COLUMNS:
vocabulary = dftrain[feature_name].unique()
feature_columns.append(tf.feature_column.
categorical_column_with_vocabulary_list(feature_name, vocabulary))
for feature_name in NUMERIC_COLUMNS:
feature_columns.append(tf.feature_column.numeric_column(feature_name, dtype=tf.float32))
sc = init_nncontext()
linear_est = tf.estimator.LinearClassifier(feature_columns=feature_columns,
optimizer=ZooOptimizer(tf.train.FtrlOptimizer(0.2)),
model_dir="/tmp/estimator/linear")
zoo_est = TFEstimator(linear_est)
train_input_fn = make_input_fn(dftrain, y_train,
mode=tf.estimator.ModeKeys.TRAIN,
batch_size=32)
zoo_est.train(train_input_fn, steps=200)
eval_input_fn = make_input_fn(dfeval, y_eval,
mode=tf.estimator.ModeKeys.EVAL,
batch_per_thread=8)
eval_result = zoo_est.evaluate(eval_input_fn, ["acc"])
print(eval_result)
pred_input_fn = make_input_fn(dfeval, y_eval,
mode=tf.estimator.ModeKeys.PREDICT,
def input_fn():
def map_func(data):
image = data['image']
label = data['label']
one_hot_label = tf.one_hot(label, depth=10)
noise = tf.random.normal(mean=0.0, stddev=1.0, shape=(NOISE_DIM,))
generator_inputs = (noise, one_hot_label)
discriminator_inputs = ((tf.to_float(image) / 255.0) - 0.5) * 2
return (generator_inputs, discriminator_inputs)
ds = tfds.load("mnist", split="train")
ds = ds.map(map_func)
dataset = TFDataset.from_tf_data_dataset(ds, batch_size=36)
return dataset
opt = GANEstimator(
generator_fn=conditional_generator,
discriminator_fn=conditional_discriminator,
generator_loss_fn=wasserstein_generator_loss,
discriminator_loss_fn=wasserstein_discriminator_loss,
generator_optimizer=ZooOptimizer(tf.train.AdamOptimizer(1e-5, 0.5)),
discriminator_optimizer=ZooOptimizer(tf.train.AdamOptimizer(1e-4, 0.5)),
model_dir=MODEL_DIR,
session_config=tf.ConfigProto()
)
for i in range(20):
opt.train(input_fn, MaxIteration(1000))
eval()
