def test_some_processing_functions(carbon_synthetic_dataset, reader_factory):
"""Try several ``tf.data.Dataset`` dataset operations on make_pycarbon_dataset"""
# reader1 will have a single row with id=1, reader2: a single row with id=2
# Using functools.partial(_eq, 1)) which is equivalent to lambda x: x==1 because standard python pickle
# can not pickle this lambda
with reader_factory(carbon_synthetic_dataset.url,
predicate=in_lambda(['id'],
functools.partial(operator.eq,
1))) as reader1:
with reader_factory(carbon_synthetic_dataset.url,
predicate=in_lambda(['id'],
functools.partial(
operator.eq,
2))) as reader2:
dataset = make_pycarbon_dataset(reader1) \
.prefetch(10) \
.concatenate(make_pycarbon_dataset(reader2)) \
.map(lambda x: x.id) \
.batch(2)
next_sample = dataset.make_one_shot_iterator().get_next()
with tf.Session() as sess:
# 'actual' is expected to be content of id column of a concatenated dataset
actual = sess.run(next_sample)
np.testing.assert_array_equal(actual, [1, 2])
def test_dataset_on_ngram_not_supported(carbon_synthetic_dataset,
reader_factory):
ngram = NGram({
0: list(_EXCLUDE_FIELDS),
1: [TestSchema.id]
}, 100, TestSchema.id)
with reader_factory(carbon_synthetic_dataset.url,
schema_fields=ngram) as reader:
with pytest.raises(NotImplementedError):
make_pycarbon_dataset(reader)
def tensorflow_hello_world(dataset_url='file:///tmp/carbon_pycarbon_dataset/'):
# Example: tf_tensors will return tensors with dataset data
with make_carbon_reader(dataset_url) as reader:
tensor = tf_tensors(reader)
with tf.Session() as sess:
sample = sess.run(tensor)
print(sample.id)
with make_reader(dataset_url, is_batch=False) as reader:
tensor = make_tensor(reader)
with tf.Session() as sess:
sample = sess.run(tensor)
print(sample.id)
# Example: use tf.data.Dataset API
with make_carbon_reader(dataset_url) as reader:
dataset = make_pycarbon_dataset(reader)
iterator = dataset.make_one_shot_iterator()
tensor = iterator.get_next()
with tf.Session() as sess:
sample = sess.run(tensor)
print(sample.id)
with make_reader(dataset_url, is_batch=False) as reader:
dataset = make_dataset(reader)
iterator = dataset.make_one_shot_iterator()
tensor = iterator.get_next()
with tf.Session() as sess:
sample = sess.run(tensor)
print(sample.id)
def tensorflow_hello_world(dataset_url='file:///tmp/carbon_external_dataset'):
# Example: tf_tensors will return tensors with dataset data
with make_batch_carbon_reader(dataset_url) as reader:
tensor = tf_tensors(reader)
with tf.Session() as sess:
# Because we are using make_batch_carbon_reader(), each read returns a batch of rows instead of a single row
batched_sample = sess.run(tensor)
print("id batch: {0}".format(batched_sample.id))
with make_reader(dataset_url) as reader:
tensor = make_tensor(reader)
with tf.Session() as sess:
# Because we are using make_batch_carbon_reader(), each read returns a batch of rows instead of a single row
batched_sample = sess.run(tensor)
print("id batch: {0}".format(batched_sample.id))
# Example: use tf.data.Dataset API
with make_batch_carbon_reader(dataset_url) as reader:
dataset = make_pycarbon_dataset(reader)
iterator = dataset.make_one_shot_iterator()
tensor = iterator.get_next()
with tf.Session() as sess:
batched_sample = sess.run(tensor)
print("id batch: {0}".format(batched_sample.id))
with make_reader(dataset_url) as reader:
dataset = make_dataset(reader)
iterator = dataset.make_one_shot_iterator()
tensor = iterator.get_next()
with tf.Session() as sess:
batched_sample = sess.run(tensor)
print("id batch: {0}".format(batched_sample.id))
def test_with_one_shot_iterator(carbon_synthetic_dataset, reader_factory):
"""Just a bunch of read and compares of all values to the expected values"""
with reader_factory(carbon_synthetic_dataset.url) as reader:
dataset = make_pycarbon_dataset(reader)
iterator = dataset.make_one_shot_iterator()
# Make sure we have static shape info for all fields
for shape in dataset.output_shapes:
# TODO(yevgeni): check that the shapes are actually correct, not just not None
assert shape.dims is not None
# Read a bunch of entries from the dataset and compare the data to reference
with tf.Session() as sess:
iterator = iterator.get_next()
for _, _ in enumerate(carbon_synthetic_dataset.data):
actual = sess.run(iterator)._asdict()
expected = next(d for d in carbon_synthetic_dataset.data
if d['id'] == actual['id'])
for key in actual.keys():
if isinstance(expected[key], str):
# Tensorflow returns all strings as bytes in python3. So we will need to decode it
actual_value = actual[key].decode()
elif isinstance(
expected[key], np.ndarray
) and expected[key].dtype.type == np.unicode_:
actual_value = np.array(
[item.decode() for item in actual[key]])
else:
actual_value = actual[key]
np.testing.assert_equal(actual_value, expected[key])
# Exhausted one full epoch. Fetching next value should trigger OutOfRangeError
with pytest.raises(tf.errors.OutOfRangeError):
sess.run(iterator)
def test_dataset_batch_carbon_reader(carbon_scalar_dataset):
with make_batch_carbon_reader(carbon_scalar_dataset.url,
num_epochs=1) as reader:
dataset = make_pycarbon_dataset(reader) \
.apply(tf.data.experimental.unbatch()) \
.batch(batch_size=1)
iterator = dataset.make_one_shot_iterator()
tensor = iterator.get_next()
with tf.Session() as sess:
sess.run([
tf.local_variables_initializer(),
tf.global_variables_initializer(),
])
i = 0
try:
while True:
sess.run(tensor)
i += 1
except tf.errors.OutOfRangeError:
print("Finish! the number is " + str(i))
assert i == _ROWS_COUNT
def test_non_unischema_with_many_colums_with_one_shot_iterator(
carbon_many_columns_non_unischema_dataset):
"""Just a bunch of read and compares of all values to the expected values"""
with make_batch_carbon_reader(
carbon_many_columns_non_unischema_dataset.url,
workers_count=1) as reader:
dataset = make_pycarbon_dataset(reader)
iterator = dataset.make_one_shot_iterator()
# Make sure we have static shape info for all fields
for shape in dataset.output_shapes:
# TODO(yevgeni): check that the shapes are actually correct, not just not None
assert shape.dims is not None
# Read a bunch of entries from the dataset and compare the data to reference
with tf.Session() as sess:
iterator = iterator.get_next()
sample = sess.run(iterator)._asdict()
assert set(sample.keys()) == set(
carbon_many_columns_non_unischema_dataset.data[0].keys())
def test_with_dataset_repeat(carbon_synthetic_dataset, reader_factory):
"""``tf.data.Dataset``'s ``repeat`` should not be used on ``make_pycarbon_dataset`` due to high costs of
``Reader initialization``. A user should use ``Reader`` built-in epochs support. Check that we raise an
error to alert of misuse."""
with reader_factory(carbon_synthetic_dataset.url) as reader:
dataset = make_pycarbon_dataset(reader)
dataset = dataset.repeat(2)
iterator = dataset.make_one_shot_iterator()
# Read a bunch of entries from the dataset and compare the data to reference
with tf.Session() as sess:
iterator = iterator.get_next()
for _, _ in enumerate(carbon_synthetic_dataset.data):
sess.run(iterator)
with pytest.raises(tf.errors.UnknownError,
match=r'.*Multiple iterations.*'):
sess.run(iterator)
def test_dynamic_batch_size_of_carbon_reader(carbon_synthetic_dataset):
with make_carbon_reader(carbon_synthetic_dataset.url,
num_epochs=None) as reader:
batch_size = tf.data.Dataset.range(
1, 10).make_one_shot_iterator().get_next()
dataset = make_pycarbon_dataset(reader) \
.batch(batch_size=batch_size)
iterator = dataset.make_initializable_iterator()
init_op = iterator.initializer
tensor = iterator.get_next()
with tf.train.MonitoredTrainingSession() as sess:
sess.run(init_op)
sample = sess.run(tensor)
assert 1 == len(sample.id)
sess.run(init_op)
sample = sess.run(tensor)
assert 2 == len(sample.id)
def train_and_test(train_dataset_url, test_dataset_url, num_epochs, batch_size,
evaluation_interval):
"""
Train a model for training iterations with a batch size batch_size, printing accuracy every log_interval.
:param dataset_url: The MNIST dataset url.
:param num_epochs: The number of epochs to train for.
:param batch_size: The batch size for training.
:param evaluation_interval: The interval used to print the accuracy.
:return:
"""
with make_batch_carbon_reader(os.path.join(train_dataset_url),
num_epochs=num_epochs) as train_reader:
with make_batch_carbon_reader(os.path.join(test_dataset_url),
num_epochs=num_epochs) as test_reader:
# Create the model
x = tf.placeholder(tf.float32, [None, 784])
w = tf.Variable(tf.zeros([784, 10]))
b = tf.Variable(tf.zeros([10]))
y = tf.matmul(x, w) + b
# Define loss and optimizer
y_ = tf.placeholder(tf.int64, [None])
# Define the loss function
cross_entropy = tf.losses.sparse_softmax_cross_entropy(labels=y_,
logits=y)
train_step = tf.train.GradientDescentOptimizer(0.5).minimize(
cross_entropy)
correct_prediction = tf.equal(tf.argmax(y, 1), y_)
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
train_dataset = make_pycarbon_dataset(train_reader) \
.apply(tf.data.experimental.unbatch()) \
.batch(batch_size) \
.map(decode)
train_iterator = train_dataset.make_one_shot_iterator()
label, image = train_iterator.get_next()
test_dataset = make_pycarbon_dataset(test_reader) \
.apply(tf.data.experimental.unbatch()) \
.batch(batch_size) \
.map(decode)
test_iterator = test_dataset.make_one_shot_iterator()
test_label, test_image = test_iterator.get_next()
# Train
print(
'Training model for {0} epoch with batch size {1} and evaluation interval {2}'
.format(num_epochs, batch_size, evaluation_interval))
i = 0
with tf.Session() as sess:
sess.run([
tf.local_variables_initializer(),
tf.global_variables_initializer(),
])
try:
while True:
cur_label, cur_image = sess.run([label, image])
sess.run([train_step],
feed_dict={
x: cur_image,
y_: cur_label
})
if i % evaluation_interval == 0:
test_cur_label, test_cur_image = sess.run(
[test_label, test_image])
print(
'After {0} training iterations, the accuracy of the model is: {1:.2f}'
.format(
i,
sess.run(accuracy,
feed_dict={
x: test_cur_image,
y_: test_cur_label
})))
i += 1
except tf.errors.OutOfRangeError:
print("Finish! the number is " + str(i))