def input_fn(mode):
x = np.random.rand(20, 10)
y = np.random.randint(0, 10, (20, ))
if mode == tf.estimator.ModeKeys.TRAIN:
return TFDataset.from_ndarrays((x, y), batch_size=8)
elif mode == tf.estimator.ModeKeys.EVAL:
return TFDataset.from_ndarrays((x, y), batch_per_thread=1)
else:
return TFDataset.from_ndarrays(x, batch_per_thread=1)
def input_fn(mode):
if mode == tf.estimator.ModeKeys.TRAIN:
training_data = get_data("train")
dataset = TFDataset.from_ndarrays(training_data, batch_size=320)
elif mode == tf.estimator.ModeKeys.EVAL:
testing_data = get_data("test")
dataset = TFDataset.from_ndarrays(testing_data, batch_per_thread=80)
else:
images, _ = get_data("test")
dataset = TFDataset.from_ndarrays(images, batch_per_thread=80)
return dataset
def test_control_inputs(self):
features = np.random.randn(20, 10)
labels = np.random.randint(0, 10, size=[20])
with tf.Graph().as_default():
dataset = TFDataset.from_ndarrays((features, labels),
batch_size=4,
val_tensors=(features, labels))
is_training = tf.placeholder(dtype=tf.bool, shape=())
feature_tensor, label_tensor = dataset.tensors
features = tf.layers.dense(feature_tensor, 8)
features = tf.layers.dropout(features, training=is_training)
output = tf.layers.dense(features, 10)
loss = tf.reduce_mean(
tf.losses.sparse_softmax_cross_entropy(logits=output,
labels=label_tensor))
optimizer = TFOptimizer.from_loss(
loss,
Adam(),
val_outputs=[output],
val_labels=[label_tensor],
val_method=Accuracy(),
tensor_with_value={is_training: (True, False)},
metrics={"loss": loss})
optimizer.optimize(end_trigger=MaxEpoch(1))
optimizer.sess.close()
def test_tf_net_predict_dataset(self):
tfnet_path = os.path.join(TestTF.resource_path, "tfnet")
net = TFNet.from_export_folder(tfnet_path)
dataset = TFDataset.from_ndarrays((np.random.rand(16, 4),))
output = net.predict(dataset)
output = np.stack(output.collect())
assert output.shape == (16, 2)
def main(data_num):
data_path = '/tmp/mnist' if not args.data_path else args.data_path
cluster_mode = args.cluster_mode
if cluster_mode.startswith("yarn"):
hadoop_conf = os.environ.get("HADOOP_CONF_DIR")
assert hadoop_conf, "Directory path to hadoop conf not found for yarn-client mode. Please " \
"set the environment variable HADOOP_CONF_DIR"
spark_conf = create_spark_conf().set("spark.executor.memory", "5g") \
.set("spark.executor.cores", 2) \
.set("spark.executor.instances", 2) \
.set("spark.executorEnv.HTTP_PROXY", "http://child-prc.intel.com:913") \
.set("spark.executorEnv.HTTPS_PROXY", "http://child-prc.intel.com:913") \
.set("spark.driver.memory", "2g")
if cluster_mode == "yarn-client":
sc = init_nncontext(spark_conf,
cluster_mode="yarn-client",
hadoop_conf=hadoop_conf)
else:
sc = init_nncontext(spark_conf,
cluster_mode="yarn-cluster",
hadoop_conf=hadoop_conf)
else:
sc = init_nncontext()
# get data, pre-process and create TFDataset
(images_data, labels_data) = mnist.read_data_sets(data_path, "test")
images_data = (images_data[:data_num] - mnist.TRAIN_MEAN) / mnist.TRAIN_STD
labels_data = labels_data[:data_num].astype(np.int32)
dataset = TFDataset.from_ndarrays((images_data, labels_data),
batch_per_thread=20)
# construct the model from TFDataset
images, labels = dataset.tensors
labels = tf.squeeze(labels)
with slim.arg_scope(lenet.lenet_arg_scope()):
logits, end_points = lenet.lenet(images,
num_classes=10,
is_training=False)
predictions = tf.to_int32(tf.argmax(logits, axis=1))
correct = tf.expand_dims(tf.to_int32(tf.equal(predictions, labels)),
axis=1)
saver = tf.train.Saver()
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
saver.restore(sess, "/tmp/lenet/model")
predictor = TFPredictor(sess, [correct])
accuracy = predictor.predict().mean()
print("predict accuracy is %s" % accuracy)
def test_tf_optimizer_metrics(self):
features = np.random.randn(20, 10)
labels = np.random.randint(0, 10, size=[20])
with tf.Graph().as_default():
dataset = TFDataset.from_ndarrays((features, labels),
batch_size=4,
val_tensors=(features, labels))
feature_tensor, label_tensor = dataset.tensors
features = tf.layers.dense(feature_tensor, 8)
output = tf.layers.dense(features, 10)
loss = tf.reduce_mean(
tf.losses.sparse_softmax_cross_entropy(logits=output,
labels=label_tensor))
optimizer = TFOptimizer.from_loss(loss, {
"dense/": Adam(1e-3),
"dense_1/": SGD(0.0)
},
val_outputs=[output],
val_labels=[label_tensor],
val_method=Accuracy(),
metrics={"loss": loss})
initial_weights = optimizer.tf_model.training_helper_layer.get_weights(
)
optimizer.optimize(end_trigger=MaxEpoch(1))
updated_weights = optimizer.tf_model.training_helper_layer.get_weights(
)
for i in [
0, 1
]: # weights and bias combined with "dense/" should be updated
assert not np.allclose(initial_weights[i], updated_weights[i])
for i in [
2, 3
]: # weights and bias combined with "dense_1" should be unchanged
assert np.allclose(initial_weights[i], updated_weights[i])
optimizer.sess.close()
def test_checkpoint(self):
features = np.random.randn(20, 10)
labels = np.random.randint(0, 10, size=[20])
with tf.Graph().as_default():
dataset = TFDataset.from_ndarrays((features, labels),
batch_size=4,
val_tensors=(features, labels))
feature_tensor, label_tensor = dataset.tensors
features = tf.layers.dense(feature_tensor, 8)
output = tf.layers.dense(features, 10)
loss = tf.reduce_mean(
tf.losses.sparse_softmax_cross_entropy(logits=output,
labels=label_tensor))
model_dir = tempfile.mkdtemp()
try:
optimizer = TFOptimizer.from_loss(loss,
Adam(),
val_outputs=[output],
val_labels=[label_tensor],
val_method=Accuracy(),
metrics={"loss": loss},
model_dir=model_dir)
optimizer.optimize(end_trigger=MaxEpoch(1))
first_weights = optimizer.sess.run(tf.trainable_variables()[0])
import re
ckpt_path = None
versions = []
for (root, dirs, files) in os.walk(model_dir, topdown=True):
temp_versions = []
for file_name in files:
if re.match("^optimMethod-TFParkTraining\.[0-9]+$",
file_name) is not None:
version = int(file_name.split(".")[1])
temp_versions.append(version)
if temp_versions:
ckpt_path = root
versions = temp_versions
break
assert ckpt_path is not None, "Cannot fine checkpoint file"
optimizer.sess.run(
tf.global_variables_initializer()) # reset variable
optimizer_load = TFOptimizer.from_loss(
loss,
Adam(),
session=optimizer.sess,
val_outputs=[output],
val_labels=[label_tensor],
val_method=Accuracy(),
metrics={"loss": loss},
model_dir=model_dir)
optimizer_load.load_checkpoint(ckpt_path, max(versions))
loaded_first_weights_before_train = optimizer.sess.run(
tf.trainable_variables()[0])
assert np.allclose(first_weights,
loaded_first_weights_before_train)
# max epoch still 1, should not train
optimizer_load.optimize(end_trigger=MaxEpoch(1))
loaded_first_weights = optimizer.sess.run(
tf.trainable_variables()[0])
assert np.allclose(first_weights, loaded_first_weights)
# max epoch increase 1, should train 1 epoch
optimizer_load.optimize(end_trigger=MaxEpoch(2))
loaded_first_weights_2 = optimizer.sess.run(
tf.trainable_variables()[0])
assert not np.allclose(first_weights, loaded_first_weights_2)
optimizer_load.sess.close()
finally:
import shutil
shutil.rmtree(model_dir)
def main(max_epoch, data_num):
args = parser.parse_args()
cluster_mode = args.cluster_mode
if cluster_mode.startswith("yarn"):
hadoop_conf = os.environ.get("HADOOP_CONF_DIR")
assert hadoop_conf, "Directory path to hadoop conf not found for yarn-client mode. Please " \
"set the environment variable HADOOP_CONF_DIR"
spark_conf = create_spark_conf().set("spark.executor.memory", "5g") \
.set("spark.executor.cores", 2) \
.set("spark.executor.instances", 2) \
.set("spark.driver.memory", "2g")
if cluster_mode == "yarn-client":
sc = init_nncontext(spark_conf,
cluster_mode="yarn-client",
hadoop_conf=hadoop_conf)
else:
sc = init_nncontext(spark_conf,
cluster_mode="yarn-cluster",
hadoop_conf=hadoop_conf)
else:
sc = init_nncontext()
# get data, pre-process and create TFDataset
(train_images_data,
train_labels_data) = mnist.read_data_sets("/tmp/mnist", "train")
(test_images_data,
test_labels_data) = mnist.read_data_sets("/tmp/mnist", "train")
train_images_data = (train_images_data[:data_num] -
mnist.TRAIN_MEAN) / mnist.TRAIN_STD
train_labels_data = train_labels_data[:data_num].astype(np.int)
test_images_data = (test_images_data[:data_num] -
mnist.TRAIN_MEAN) / mnist.TRAIN_STD
test_labels_data = (test_labels_data[:data_num]).astype(np.int)
dataset = TFDataset.from_ndarrays(
(train_images_data, train_labels_data),
batch_size=360,
val_tensors=(test_images_data, test_labels_data))
# construct the model from TFDataset
images, labels = dataset.tensors
with slim.arg_scope(lenet.lenet_arg_scope()):
logits, end_points = lenet.lenet(images,
num_classes=10,
is_training=True)
loss = tf.reduce_mean(
tf.losses.sparse_softmax_cross_entropy(logits=logits, labels=labels))
acc = accuracy(logits, labels)
# create a optimizer
optimizer = TFOptimizer.from_loss(loss,
Adam(1e-3),
metrics={"acc": acc},
model_dir="/tmp/lenet/")
# kick off training
optimizer.optimize(end_trigger=MaxEpoch(max_epoch))
saver = tf.train.Saver()
saver.save(optimizer.sess, "/tmp/lenet/model")
