def test_estimator_graph_checkpoint(self):
import zoo.orca.data.pandas
tf.reset_default_graph()
model = SimpleModel()
file_path = os.path.join(resource_path, "orca/learn/ncf.csv")
data_shard = zoo.orca.data.pandas.read_csv(file_path)
def transform(df):
result = {
"x": (df['user'].to_numpy(), df['item'].to_numpy()),
"y": df['label'].to_numpy()
}
return result
data_shard = data_shard.transform_shard(transform)
temp = tempfile.mkdtemp()
model_dir = os.path.join(temp, "test_model")
est = Estimator.from_graph(
inputs=[model.user, model.item],
labels=[model.label],
loss=model.loss,
optimizer=tf.train.AdamOptimizer(),
metrics={"loss": model.loss},
model_dir=model_dir
)
est.fit(data=data_shard,
batch_size=8,
epochs=6,
validation_data=data_shard,
checkpoint_trigger=SeveralIteration(4))
est.sess.close()
tf.reset_default_graph()
model = SimpleModel()
est = Estimator.from_graph(
inputs=[model.user, model.item],
labels=[model.label],
loss=model.loss,
optimizer=tf.train.AdamOptimizer(),
metrics={"loss": model.loss},
model_dir=model_dir
)
est.load_orca_checkpoint(model_dir)
est.fit(data=data_shard,
batch_size=8,
epochs=10,
validation_data=data_shard)
result = est.evaluate(data_shard)
assert "loss" in result
print(result)
shutil.rmtree(temp)
def test_estimator_graph_pandas_dataframe(self):
import zoo.orca.data.pandas
tf.reset_default_graph()
model = SimpleModel()
file_path = os.path.join(resource_path, "orca/learn/ncf.csv")
data_shard = zoo.orca.data.pandas.read_csv(file_path)
est = Estimator.from_graph(
inputs=[model.user, model.item],
labels=[model.label],
loss=model.loss,
optimizer=tf.train.AdamOptimizer(),
metrics={"loss": model.loss})
est.fit(data=data_shard,
batch_size=8,
epochs=10,
feature_cols=['user', 'item'],
label_cols=['label'],
validation_data=data_shard)
result = est.evaluate(data_shard, feature_cols=['user', 'item'], label_cols=['label'])
assert "loss" in result
print(result)
est = Estimator.from_graph(
inputs=[model.user, model.item],
outputs=[model.logits])
predictions = est.predict(data_shard, feature_cols=['user', 'item']).collect()
print(predictions)
def main(max_epoch, dataset_dir):
mnist_train = tfds.load(name="mnist", split="train", data_dir=dataset_dir)
mnist_test = tfds.load(name="mnist", split="test", data_dir=dataset_dir)
mnist_train = mnist_train.map(preprocess)
mnist_test = mnist_test.map(preprocess)
# tensorflow inputs
images = tf.placeholder(dtype=tf.float32, shape=(None, 28, 28, 1))
# tensorflow labels
labels = tf.placeholder(dtype=tf.int32, shape=(None,))
logits = lenet(images)
loss = tf.reduce_mean(tf.losses.sparse_softmax_cross_entropy(logits=logits, labels=labels))
acc = accuracy(logits, labels)
# create an estimator
est = Estimator.from_graph(inputs=images,
outputs=logits,
labels=labels,
loss=loss,
optimizer=tf.train.AdamOptimizer(),
metrics={"acc": acc})
est.fit(data=mnist_train,
batch_size=320,
epochs=max_epoch,
validation_data=mnist_test)
result = est.evaluate(mnist_test)
print(result)
est.save_tf_checkpoint("/tmp/lenet/model")
def test_estimator_graph_tf_dataset(self):
tf.reset_default_graph()
model = SimpleModel()
dataset = tf.data.Dataset.from_tensor_slices(
(np.random.randint(0, 200, size=(100, )),
np.random.randint(0, 50, size=(100, )),
np.ones(shape=(100, ), dtype=np.int32)))
est = Estimator.from_graph(inputs=[model.user, model.item],
labels=[model.label],
outputs=[model.logits],
loss=model.loss,
optimizer=tf.train.AdamOptimizer(),
metrics={"loss": model.loss})
est.fit(data=dataset, batch_size=8, epochs=10, validation_data=dataset)
result = est.evaluate(dataset, batch_size=4)
assert 'loss' in result
predict_dataset = tf.data.Dataset.from_tensor_slices(
(np.random.randint(0, 200, size=(20, )),
np.random.randint(0, 50, size=(20, ))))
predictions = est.predict(predict_dataset).collect()
assert predictions[0]['prediction'].shape[1] == 2
def test_estimator_graph_dataframe(self):
tf.reset_default_graph()
model = SimpleModel()
file_path = os.path.join(resource_path, "orca/learn/ncf.csv")
sc = init_nncontext()
sqlcontext = SQLContext(sc)
df = sqlcontext.read.csv(file_path, header=True, inferSchema=True)
est = Estimator.from_graph(
inputs=[model.user, model.item],
labels=[model.label],
outputs=[model.logits],
loss=model.loss,
optimizer=tf.train.AdamOptimizer(),
metrics={"loss": model.loss})
est.fit(data=df,
batch_size=8,
epochs=10,
feature_cols=['user', 'item'],
label_cols=['label'],
validation_data=df)
result = est.evaluate(df, batch_size=4, feature_cols=['user', 'item'],
label_cols=['label'])
print(result)
prediction_df = est.predict(df, batch_size=4, feature_cols=['user', 'item'])
assert 'prediction' in prediction_df.columns
predictions = prediction_df.collect()
assert len(predictions) == 48
def test_estimator_graph_fit_dataset(estimator_for_spark_fixture):
import zoo.orca.data.pandas
tf.reset_default_graph()
model = SimpleModel()
sc = estimator_for_spark_fixture
file_path = os.path.join(resource_path, "orca/learn/ncf.csv")
data_shard = zoo.orca.data.pandas.read_csv(file_path, sc)
def transform(df):
result = {
"x": (df['user'].to_numpy(), df['item'].to_numpy()),
"y": df['label'].to_numpy()
}
return result
data_shard = data_shard.transform_shard(transform)
dataset = Dataset.from_tensor_slices(data_shard)
est = Estimator.from_graph(inputs=[model.user, model.item],
labels=[model.label],
loss=model.loss,
optimizer=tf.train.AdamOptimizer(),
metrics={"loss": model.loss})
est.fit(data=dataset, batch_size=8, steps=10, validation_data=dataset)
result = est.evaluate(dataset, batch_size=4)
assert 'loss' in result
def test_estimator_graph_with_bigdl_optim_method(self):
import zoo.orca.data.pandas
tf.reset_default_graph()
model = SimpleModel()
file_path = os.path.join(resource_path, "orca/learn/ncf.csv")
data_shard = zoo.orca.data.pandas.read_csv(file_path)
def transform(df):
result = {
"x": (df['user'].to_numpy(), df['item'].to_numpy()),
"y": df['label'].to_numpy()
}
return result
data_shard = data_shard.transform_shard(transform)
from zoo.orca.learn.optimizers import SGD
from zoo.orca.learn.optimizers.schedule import Plateau
sgd = SGD(learningrate=0.1,
learningrate_schedule=Plateau("score",
factor=0.1,
patience=10,
mode="min", ))
est = Estimator.from_graph(
inputs=[model.user, model.item],
labels=[model.label],
outputs=[model.logits],
loss=model.loss,
optimizer=sgd,
metrics={"loss": model.loss})
est.fit(data=data_shard,
batch_size=8,
epochs=10,
validation_data=data_shard)
def test_estimator_graph_evaluate(estimator_for_spark_fixture):
import zoo.orca.data.pandas
tf.reset_default_graph()
model = SimpleModel()
sc = estimator_for_spark_fixture
file_path = os.path.join(resource_path, "orca/learn/ncf.csv")
data_shard = zoo.orca.data.pandas.read_csv(file_path, sc)
def transform(df):
result = {
"x": (df['user'].to_numpy(), df['item'].to_numpy()),
"y": df['label'].to_numpy()
}
return result
data_shard = data_shard.transform_shard(transform)
est = Estimator.from_graph(inputs=[model.user, model.item],
labels=[model.label],
loss=model.loss,
optimizer=tf.train.AdamOptimizer(),
metrics={"loss": model.loss})
result = est.evaluate(data_shard)
assert "loss" in result
print(result)
def test_estimator_graph_fit(self):
import zoo.orca.data.pandas
tf.reset_default_graph()
model = SimpleModel()
file_path = os.path.join(resource_path, "orca/learn/ncf.csv")
data_shard = zoo.orca.data.pandas.read_csv(file_path)
def transform(df):
result = {
"x": (df['user'].to_numpy(), df['item'].to_numpy()),
"y": df['label'].to_numpy()
}
return result
data_shard = data_shard.transform_shard(transform)
est = Estimator.from_graph(
inputs=[model.user, model.item],
labels=[model.label],
loss=model.loss,
optimizer=tf.train.AdamOptimizer(),
metrics={"loss": model.loss})
est.fit(data=data_shard,
batch_size=8,
epochs=10,
validation_data=data_shard)
def main(max_epoch):
sc = init_orca_context(cores=4, memory="2g")
# get DataSet
mnist_train = tfds.load(name="mnist", split="train")
mnist_test = tfds.load(name="mnist", split="test")
# Normalizes images
def normalize_img(data):
data['image'] = tf.cast(data["image"], tf.float32) / 255.
return data
mnist_train = mnist_train.map(
normalize_img, num_parallel_calls=tf.data.experimental.AUTOTUNE)
mnist_test = mnist_test.map(
normalize_img, num_parallel_calls=tf.data.experimental.AUTOTUNE)
# tensorflow inputs
images = tf.placeholder(dtype=tf.float32, shape=(None, 28, 28, 1))
# tensorflow labels
labels = tf.placeholder(dtype=tf.int32, shape=(None, ))
with slim.arg_scope(lenet.lenet_arg_scope()):
logits, end_points = lenet.lenet(images,
num_classes=10,
is_training=True)
loss = tf.reduce_mean(
tf.losses.sparse_softmax_cross_entropy(logits=logits, labels=labels))
acc = accuracy(logits, labels)
# create an estimator
est = Estimator.from_graph(inputs=images,
outputs=logits,
labels=labels,
loss=loss,
optimizer=tf.train.AdamOptimizer(),
metrics={"acc": acc})
est.fit(data=mnist_train,
batch_size=320,
epochs=max_epoch,
validation_data=mnist_test)
result = est.evaluate(mnist_test)
print(result)
est.save_tf_checkpoint("/tmp/lenet/model")
stop_orca_context()
def test_estimator_graph_dataframe_exception(self):
tf.reset_default_graph()
model = SimpleModel()
file_path = os.path.join(resource_path, "orca/learn/ncf.csv")
sc = init_nncontext()
sqlcontext = SQLContext(sc)
df = sqlcontext.read.csv(file_path, header=True, inferSchema=True)
est = Estimator.from_graph(inputs=[model.user, model.item],
labels=[model.label],
outputs=[model.logits],
loss=model.loss,
optimizer=tf.train.AdamOptimizer(),
metrics={"loss": model.loss})
with self.assertRaises(Exception) as context:
est.fit(data=df,
batch_size=8,
epochs=10,
feature_cols=['user', 'item'],
validation_data=df)
self.assertTrue(
'label columns is None; it should not be None in training' in str(
context.exception))
est.fit(data=df,
batch_size=8,
epochs=10,
feature_cols=['user', 'item'],
labels_cols=['label'])
with self.assertRaises(Exception) as context:
predictions = est.predict(df, batch_size=4).collect()
self.assertTrue(
'feature columns is None; it should not be None in prediction' in
str(context.exception))
with self.assertRaises(Exception) as context:
est.fit(data=df,
batch_size=8,
epochs=10,
feature_cols=['user', 'item'],
labels_cols=['label'],
validation_data=[1, 2, 3])
self.assertTrue(
'train data and validation data should be both Spark DataFrame' in
str(context.exception))
def _test_estimator_graph_tf_dataset(self, dataset_creator):
tf.reset_default_graph()
model = SimpleModel()
dataset = dataset_creator()
est = Estimator.from_graph(inputs=[model.user, model.item],
labels=[model.label],
outputs=[model.logits],
loss=model.loss,
optimizer=tf.train.AdamOptimizer(),
metrics={"loss": model.loss})
est.fit(data=dataset, batch_size=8, epochs=10, validation_data=dataset)
result = est.evaluate(dataset, batch_size=4)
assert 'loss' in result
def train(train_data, test_data, user_size, item_size):
model = NCF(opt.embedding_size, user_size, item_size)
estimator = Estimator.from_graph(inputs=[model.user, model.item],
outputs=[model.class_number],
labels=[model.label],
loss=model.loss,
optimizer=model.optim,
model_dir=opt.model_dir,
metrics={"loss": model.loss})
estimator.fit(data=train_data,
batch_size=opt.batch_size,
epochs=opt.epochs,
validation_data=test_data)
checkpoint_path = os.path.join(opt.model_dir, "NCF.ckpt")
estimator.save_tf_checkpoint(checkpoint_path)
estimator.sess.close()
def test_estimator_graph(estimator_for_spark_fixture):
import zoo.orca.data.pandas
sc = estimator_for_spark_fixture
tf.reset_default_graph()
model = SimpleModel()
file_path = os.path.join(resource_path, "orca/learn/ncf.csv")
data_shard = zoo.orca.data.pandas.read_csv(file_path, sc)
def transform(df):
result = {
"x": (df['user'].to_numpy(), df['item'].to_numpy()),
"y": df['label'].to_numpy()
}
return result
data_shard = data_shard.transform_shard(transform)
est = Estimator.from_graph(inputs=[model.user, model.item],
labels=[model.label],
outputs=[model.logits],
loss=model.loss,
optimizer=tf.train.AdamOptimizer(),
metrics={"loss": model.loss})
est.fit(data=data_shard,
batch_size=8,
steps=10,
validation_data=data_shard)
data_shard = zoo.orca.data.pandas.read_csv(file_path, sc)
def transform(df):
result = {
"x": (df['user'].to_numpy(), df['item'].to_numpy()),
}
return result
data_shard = data_shard.transform_shard(transform)
predictions = est.predict(data_shard).collect()
print(predictions)
def test_estimator_graph_predict_dataset(self):
tf.reset_default_graph()
model = SimpleModel()
file_path = os.path.join(resource_path, "orca/learn/ncf.csv")
data_shard = zoo.orca.data.pandas.read_csv(file_path)
est = Estimator.from_graph(inputs=[model.user, model.item],
outputs=[model.logits])
def transform(df):
result = {
"x": (df['user'].to_numpy(), df['item'].to_numpy()),
}
return result
data_shard = data_shard.transform_shard(transform)
dataset = Dataset.from_tensor_slices(data_shard)
predictions = est.predict(dataset).collect()
assert len(predictions) == 10
def test_estimator_graph_predict(estimator_for_spark_fixture):
import zoo.orca.data.pandas
tf.reset_default_graph()
sc = estimator_for_spark_fixture
model = SimpleModel()
file_path = os.path.join(resource_path, "orca/learn/ncf.csv")
data_shard = zoo.orca.data.pandas.read_csv(file_path, sc)
est = Estimator.from_graph(inputs=[model.user, model.item],
outputs=[model.logits])
def transform(df):
result = {
"x": (df['user'].to_numpy(), df['item'].to_numpy()),
}
return result
data_shard = data_shard.transform_shard(transform)
predictions = est.predict(data_shard).collect()
print(predictions)
def main(max_epoch):
(train_feature, train_label), (val_feature, val_label) = tf.keras.datasets.mnist.load_data()
# tf.data.Dataset.from_tensor_slices is for demo only. For production use, please use
# file-based approach (e.g. tfrecord).
train_dataset = tf.data.Dataset.from_tensor_slices((train_feature, train_label))
train_dataset = train_dataset.map(preprocess)
val_dataset = tf.data.Dataset.from_tensor_slices((val_feature, val_label))
val_dataset = val_dataset.map(preprocess)
# tensorflow inputs
images = tf.placeholder(dtype=tf.float32, shape=(None, 28, 28, 1))
# tensorflow labels
labels = tf.placeholder(dtype=tf.int32, shape=(None,))
logits = lenet(images)
loss = tf.reduce_mean(tf.losses.sparse_softmax_cross_entropy(logits=logits, labels=labels))
acc = accuracy(logits, labels)
# create an estimator
est = Estimator.from_graph(inputs=images,
outputs=logits,
labels=labels,
loss=loss,
optimizer=tf.train.AdamOptimizer(),
metrics={"acc": acc})
est.fit(data=train_dataset,
batch_size=320,
epochs=max_epoch,
validation_data=val_dataset)
result = est.evaluate(val_dataset)
print(result)
est.save_tf_checkpoint("/tmp/lenet/model")
def predict(predict_data, user_size, item_size):
def to_predict(data):
del data['y']
return data
predict_data = predict_data.transform_shard(to_predict)
tf.reset_default_graph()
with tf.Session() as sess:
model = NCF(opt.embedding_size, user_size, item_size)
saver = tf.train.Saver(tf.global_variables())
checkpoint_path = os.path.join(opt.model_dir, "NCF.ckpt")
saver.restore(sess, checkpoint_path)
estimator = Estimator.from_graph(inputs=[model.user, model.item],
outputs=[model.class_number],
sess=sess,
model_dir=opt.model_dir)
predict_result = estimator.predict(predict_data)
predictions = predict_result.collect()
assert 'prediction' in predictions[0]
print(predictions[0]['prediction'])
def test_estimator_save_load(self):
import zoo.orca.data.pandas
tf.reset_default_graph()
# save
model = SimpleModel()
file_path = os.path.join(resource_path, "orca/learn/ncf.csv")
data_shard = zoo.orca.data.pandas.read_csv(file_path)
def transform(df):
result = {
"x": (df['user'].to_numpy(), df['item'].to_numpy()),
"y": df['label'].to_numpy()
}
return result
data_shard = data_shard.transform_shard(transform)
est = Estimator.from_graph(
inputs=[model.user, model.item],
labels=[model.label],
outputs=[model.logits],
loss=model.loss,
optimizer=tf.train.AdamOptimizer(),
metrics={"loss": model.loss},
sess=None
)
est.fit(data=data_shard,
batch_size=8,
epochs=5,
validation_data=data_shard)
temp = tempfile.mkdtemp()
model_checkpoint = os.path.join(temp, 'tmp.ckpt')
est.save(model_checkpoint)
est.shutdown()
tf.reset_default_graph()
# load
with tf.Session() as sess:
model = SimpleModel()
est = Estimator.from_graph(
inputs=[model.user, model.item],
labels=[model.label],
outputs=[model.logits],
loss=model.loss,
metrics={"loss": model.loss},
sess=sess
)
est.load(model_checkpoint)
data_shard = zoo.orca.data.pandas.read_csv(file_path)
def transform(df):
result = {
"x": (df['user'].to_numpy(), df['item'].to_numpy()),
}
return result
data_shard = data_shard.transform_shard(transform)
predictions = est.predict(data_shard).collect()
assert 'prediction' in predictions[0]
print(predictions)
shutil.rmtree(temp)
def test_estimator_graph_tensorboard(self):
tf.reset_default_graph()
model = SimpleModel()
file_path = os.path.join(resource_path, "orca/learn/ncf.csv")
data_shard = zoo.orca.data.pandas.read_csv(file_path)
def transform(df):
result = {
"x": (df['user'].to_numpy(), df['item'].to_numpy()),
"y": df['label'].to_numpy()
}
return result
data_shard = data_shard.transform_shard(transform)
temp = tempfile.mkdtemp()
# only set model dir, summary generated under model dir
model_dir = os.path.join(temp, "test_model")
est = Estimator.from_graph(
inputs=[model.user, model.item],
labels=[model.label],
loss=model.loss,
optimizer=tf.train.AdamOptimizer(),
metrics={"loss": model.loss},
model_dir=model_dir
)
est.fit(data=data_shard,
batch_size=8,
epochs=5,
validation_data=data_shard)
train_tp = est.get_train_summary("Throughput")
val_scores = est.get_validation_summary("loss")
assert len(train_tp) > 0
assert len(val_scores) > 0
# set tensorboard dir to different directory
est.set_tensorboard("model", "test")
est.fit(data=data_shard,
batch_size=8,
epochs=5,
validation_data=data_shard)
train_tp = est.get_train_summary("Throughput")
val_scores = est.get_validation_summary("loss")
assert len(train_tp) > 0
assert len(val_scores) > 0
# no model dir, no tensorboard dir, no summary saved
est2 = Estimator.from_graph(
inputs=[model.user, model.item],
labels=[model.label],
loss=model.loss,
optimizer=tf.train.AdamOptimizer(),
metrics={"loss": model.loss}
)
est2.fit(data=data_shard,
batch_size=8,
epochs=5,
validation_data=data_shard)
train_tp = est2.get_train_summary("Throughput")
val_scores = est2.get_validation_summary("loss")
assert train_tp is None
assert val_scores is None
shutil.rmtree(temp)
