def test_ignite_dataset_with_plain_client(self):
"""Test Ignite Dataset with plain client.
"""
self._clear_env()
ds = IgniteDataset(cache_name="SQL_PUBLIC_TEST_CACHE", port=42300)
self._check_dataset(ds)
def test_ignite_dataset_with_ssl_client(self):
"""Test Ignite Dataset with ssl client.
"""
self._clear_env()
os.environ["IGNITE_DATASET_CERTFILE"] = os.path.dirname(
os.path.realpath(__file__)) + "/keystore/client.pem"
os.environ["IGNITE_DATASET_CERT_PASSWORD"] = "******"
ds = IgniteDataset(
cache_name="SQL_PUBLIC_TEST_CACHE",
port=42301,
certfile=os.environ["IGNITE_DATASET_CERTFILE"],
cert_password=os.environ["IGNITE_DATASET_CERT_PASSWORD"])
self._check_dataset(ds)
import os
import time
import sys
import tensorflow as tf
from tensorflow.contrib.ignite import IgniteDataset
print("PYTHONPATH: ", os.environ['PYTHONPATH'])
print("TF_CONFIG: ", os.environ['TF_CONFIG'])
print("TF_WORKERS: ", os.environ['TF_WORKERS'])
print("TF_CHIEF_SERVER: ", os.environ['TF_CHIEF_SERVER'])
sys.stdout.flush()
with tf.Session(os.environ['TF_CHIEF_SERVER']) as sess:
ds = IgniteDataset("MNIST_CACHE", page_size=10000, distributed=True)
it = ds.make_one_shot_iterator()
print("Next: ", sess.run(it.get_next()))
sys.stdout.flush()
while (True):
time.sleep(0.1)
import time
import tensorflow as tf
from tensorflow.contrib.ignite import IgniteDataset
tf.enable_eager_execution()
iterations = 10
page_size = 250
dataset = IgniteDataset(cache_name="TEST_CACHE", page_size=page_size)
start = time.time()
for _ in range(iterations):
for element in dataset:
pass
end = time.time()
print("Total throughput %s MB/s" % (iterations * 1024 / (end - start)))