def check_tst_ids(dataset_dir, file_pattern, num_samples):
keys_to_features = {
'band_1': tf.FixedLenFeature((75 * 75,), tf.float32, default_value=None),
'band_2': tf.FixedLenFeature((75 * 75,), tf.float32, default_value=None),
'id': tf.FixedLenFeature([1], tf.string, default_value=None),
'angle': tf.FixedLenFeature([1], tf.float32, default_value=None),
}
items_to_handlers = {
'band_1': slim.tfexample_decoder.Tensor('band_1', shape=[75, 75]),
'band_2': slim.tfexample_decoder.Tensor('band_2', shape=[75, 75]),
'id': slim.tfexample_decoder.Tensor('id', shape=[]),
'angle': slim.tfexample_decoder.Tensor('angle', shape=[])
}
dataset = mox.get_tfrecord(dataset_dir=dataset_dir,
file_pattern=file_pattern,
num_samples=num_samples,
keys_to_features=keys_to_features,
items_to_handlers=items_to_handlers,
shuffle=False,
num_epochs=1)
band_1, band_2, id, angle = dataset.get(
['band_1', 'band_2', 'id', 'angle'])
# sv = tf.train.Supervisor()
id_dict = {}
# with sv.managed_session() as sess:
with tf.train.MonitoredTrainingSession() as sess:
for i in range(num_samples):
id_eval = sess.run(id)
tf.logging.info('%s/%s' % (i, num_samples))
if id_eval not in id_dict:
id_dict[id_eval] = 1
else:
raise ValueError('id: %s has alreay been defined.')
print('%s test cases checked.' % len(id_dict))
def check_tst_ids(dataset_dir, file_pattern, num_samples):
keys_to_features = {
'band_1': tf.FixedLenFeature((75 * 75,), tf.float32, default_value=None),
'band_2': tf.FixedLenFeature((75 * 75,), tf.float32, default_value=None),
'id': tf.FixedLenFeature([1], tf.string, default_value=None),
'angle': tf.FixedLenFeature([1], tf.float32, default_value=None),
}
items_to_handlers = {
'band_1': slim.tfexample_decoder.Tensor('band_1', shape=[75, 75]),
'band_2': slim.tfexample_decoder.Tensor('band_2', shape=[75, 75]),
'id': slim.tfexample_decoder.Tensor('id', shape=[]),
'angle': slim.tfexample_decoder.Tensor('angle', shape=[])
}
dataset = mox.get_tfrecord(dataset_dir=dataset_dir,
file_pattern=file_pattern,
num_samples=num_samples,
keys_to_features=keys_to_features,
items_to_handlers=items_to_handlers,
shuffle=False,
num_epochs=1)
band_1, band_2, id, angle = dataset.get(
['band_1', 'band_2', 'id', 'angle'])
# sv = tf.train.Supervisor()
id_dict = {}
# with sv.managed_session() as sess:
with tf.train.MonitoredTrainingSession() as sess:
for i in range(num_samples):
id_eval = sess.run(id)
tf.logging.info('%s/%s' % (i, num_samples))
if id_eval not in id_dict:
id_dict[id_eval] = 1
else:
raise ValueError('id: %s has alreay been defined.')
print('%s test cases checked.' % len(id_dict))
def read_and_decode_tfrecord(dataset_dir, file_pattern, num_samples):
keys_to_features = {
'band_1': tf.FixedLenFeature((75 * 75, ),
tf.float32,
default_value=None),
'band_2': tf.FixedLenFeature((75 * 75, ),
tf.float32,
default_value=None),
'label': tf.FixedLenFeature([1], tf.int64, default_value=None),
'angle': tf.FixedLenFeature([1], tf.float32, default_value=None),
}
items_to_handlers = {
'band_1': slim.tfexample_decoder.Tensor('band_1', shape=[75, 75]),
'band_2': slim.tfexample_decoder.Tensor('band_2', shape=[75, 75]),
'label': slim.tfexample_decoder.Tensor('label', shape=[]),
'angle': slim.tfexample_decoder.Tensor('angle', shape=[])
}
dataset = mox.get_tfrecord(dataset_dir=dataset_dir,
file_pattern=file_pattern,
num_samples=num_samples,
keys_to_features=keys_to_features,
items_to_handlers=items_to_handlers,
shuffle=False,
num_epochs=1)
band_1, band_2, label, angle = dataset.get(
['band_1', 'band_2', 'label', 'angle'])
band_3 = (band_1 + band_2) / 2
image = tf.stack([band_1, band_2, band_3], axis=2)
image_max = tf.reduce_max(image)
image_min = tf.reduce_min(image)
image = (image - image_min) / (image_max - image_min)
sv = tf.train.Supervisor()
with sv.managed_session() as sess:
plt.figure()
for i in range(40):
subp = plt.subplot(5, 8, i + 1)
plt.subplots_adjust(hspace=0.6)
subp.imshow(sess.run(image))
label_eval = sess.run(label)
angle_eval = int(sess.run(angle))
subp.set_title('label=%s, angle=%s' % (label_eval, angle_eval))
plt.show()
def read_and_decode_tfrecord(dataset_dir, file_pattern, num_samples):
keys_to_features = {
'band_1': tf.FixedLenFeature((75 * 75,), tf.float32, default_value=None),
'band_2': tf.FixedLenFeature((75 * 75,), tf.float32, default_value=None),
'label': tf.FixedLenFeature([1], tf.int64, default_value=None),
'angle': tf.FixedLenFeature([1], tf.float32, default_value=None),
}
items_to_handlers = {
'band_1': slim.tfexample_decoder.Tensor('band_1', shape=[75, 75]),
'band_2': slim.tfexample_decoder.Tensor('band_2', shape=[75, 75]),
'label': slim.tfexample_decoder.Tensor('label', shape=[]),
'angle': slim.tfexample_decoder.Tensor('angle', shape=[])
}
dataset = mox.get_tfrecord(dataset_dir=dataset_dir,
file_pattern=file_pattern,
num_samples=num_samples,
keys_to_features=keys_to_features,
items_to_handlers=items_to_handlers,
shuffle=False,
num_epochs=1)
band_1, band_2, label, angle = dataset.get(['band_1', 'band_2', 'label', 'angle'])
band_3 = (band_1 + band_2) / 2
image = tf.stack([band_1, band_2, band_3], axis=2)
image_max = tf.reduce_max(image)
image_min = tf.reduce_min(image)
image = (image - image_min) / (image_max - image_min)
sv = tf.train.Supervisor()
with sv.managed_session() as sess:
plt.figure()
for i in range(40):
subp = plt.subplot(5, 8, i + 1)
plt.subplots_adjust(hspace=0.6)
subp.imshow(sess.run(image))
label_eval = sess.run(label)
angle_eval = int(sess.run(angle))
subp.set_title('label=%s, angle=%s' % (label_eval, angle_eval))
plt.show()
def input_fn(run_mode, **kwargs):
if run_mode == mox.ModeKeys.TRAIN:
num_samples = NUM_SAMPLES_TRAIN
num_epochs = None
shuffle = True
file_pattern = 'iceberg-train-*.tfrecord'
else:
num_epochs = 1
shuffle = False
if run_mode == mox.ModeKeys.EVAL:
num_samples = NUM_SAMPLES_EVAL
file_pattern = 'iceberg-eval-*.tfrecord'
else:
num_samples = NUM_SAMPLES_TEST
file_pattern = 'iceberg-test-*.tfrecord'
keys_to_features = {
'band_1':
tf.FixedLenFeature((75 * 75, ), tf.float32, default_value=None),
'band_2':
tf.FixedLenFeature((75 * 75, ), tf.float32, default_value=None),
'angle':
tf.FixedLenFeature([1], tf.float32, default_value=None),
}
items_to_handlers = {
'band_1': slim.tfexample_decoder.Tensor('band_1', shape=[75, 75]),
'band_2': slim.tfexample_decoder.Tensor('band_2', shape=[75, 75]),
'angle': slim.tfexample_decoder.Tensor('angle', shape=[])
}
if run_mode == mox.ModeKeys.PREDICT:
keys_to_features['id'] = tf.FixedLenFeature([1],
tf.string,
default_value=None)
items_to_handlers['id'] = slim.tfexample_decoder.Tensor('id',
shape=[])
else:
keys_to_features['label'] = tf.FixedLenFeature([1],
tf.int64,
default_value=None)
items_to_handlers['label'] = slim.tfexample_decoder.Tensor(
'label', shape=[])
dataset = mox.get_tfrecord(dataset_dir=flags.data_url,
file_pattern=file_pattern,
num_samples=num_samples,
keys_to_features=keys_to_features,
items_to_handlers=items_to_handlers,
num_epochs=num_epochs,
shuffle=shuffle)
if run_mode == mox.ModeKeys.PREDICT:
band_1, band_2, id_or_label, angle = dataset.get(
['band_1', 'band_2', 'id', 'angle'])
# Non-DMA safe string cannot tensor may not be copied to a GPU.
# So we encode string to a list of integer.
id_or_label = tf.py_func(
lambda str: np.array([ord(ch) for ch in str]), [id_or_label],
tf.int64)
# We know `id` is a string of 8 alphabets.
id_or_label = tf.reshape(id_or_label, shape=(8, ))
else:
band_1, band_2, id_or_label, angle = dataset.get(
['band_1', 'band_2', 'label', 'angle'])
band_3 = band_1 + band_2
# Rescale the input image to [0, 1]
def rescale(*args):
ret_images = []
for image in args:
image = tf.cast(image, tf.float32)
image_min = tf.reduce_min(image)
image_max = tf.reduce_max(image)
image = (image - image_min) / (image_max - image_min)
ret_images.append(image)
return ret_images
band_1, band_2, band_3 = rescale(band_1, band_2, band_3)
image = tf.stack([band_1, band_2, band_3], axis=2)
# Data augementation
if run_mode == mox.ModeKeys.TRAIN:
image = tf.image.random_flip_left_right(image)
image = tf.image.random_flip_up_down(image)
image = tf.image.rot90(image,
k=tf.random_uniform(shape=(),
maxval=3,
minval=0,
dtype=tf.int32))
return image, id_or_label, angle
def input_fn(run_mode, **kwargs):
if run_mode == mox.ModeKeys.TRAIN:
num_samples = NUM_SAMPLES_TRAIN
num_epochs = None
shuffle = True
file_pattern = 'iceberg-train-*.tfrecord'
else:
num_epochs = 1
shuffle = False
if run_mode == mox.ModeKeys.EVAL:
num_samples = NUM_SAMPLES_EVAL
file_pattern = 'iceberg-eval-*.tfrecord'
else:
num_samples = NUM_SAMPLES_TEST
file_pattern = 'iceberg-test-*.tfrecord'
keys_to_features = {
'band_1': tf.FixedLenFeature((75 * 75,), tf.float32, default_value=None),
'band_2': tf.FixedLenFeature((75 * 75,), tf.float32, default_value=None),
'angle': tf.FixedLenFeature([1], tf.float32, default_value=None),
}
items_to_handlers = {
'band_1': slim.tfexample_decoder.Tensor('band_1', shape=[75, 75]),
'band_2': slim.tfexample_decoder.Tensor('band_2', shape=[75, 75]),
'angle': slim.tfexample_decoder.Tensor('angle', shape=[])
}
if run_mode == mox.ModeKeys.PREDICT:
keys_to_features['id'] = tf.FixedLenFeature([1], tf.string, default_value=None)
items_to_handlers['id'] = slim.tfexample_decoder.Tensor('id', shape=[])
else:
keys_to_features['label'] = tf.FixedLenFeature([1], tf.int64, default_value=None)
items_to_handlers['label'] = slim.tfexample_decoder.Tensor('label', shape=[])
dataset = mox.get_tfrecord(dataset_dir=flags.data_url,
file_pattern=file_pattern,
num_samples=num_samples,
keys_to_features=keys_to_features,
items_to_handlers=items_to_handlers,
num_epochs=num_epochs,
shuffle=shuffle)
if run_mode == mox.ModeKeys.PREDICT:
band_1, band_2, id_or_label, angle = dataset.get(['band_1', 'band_2', 'id', 'angle'])
# Non-DMA safe string cannot tensor may not be copied to a GPU.
# So we encode string to a list of integer.
id_or_label = tf.py_func(lambda str: np.array([ord(ch) for ch in str]), [id_or_label], tf.int64)
# We know `id` is a string of 8 alphabets.
id_or_label = tf.reshape(id_or_label, shape=(8,))
else:
band_1, band_2, id_or_label, angle = dataset.get(['band_1', 'band_2', 'label', 'angle'])
band_3 = band_1 + band_2
# Rescale the input image to [0, 1]
def rescale(*args):
ret_images = []
for image in args:
image = tf.cast(image, tf.float32)
image_min = tf.reduce_min(image)
image_max = tf.reduce_max(image)
image = (image - image_min) / (image_max - image_min)
ret_images.append(image)
return ret_images
band_1, band_2, band_3 = rescale(band_1, band_2, band_3)
image = tf.stack([band_1, band_2, band_3], axis=2)
# Data augementation
if run_mode == mox.ModeKeys.TRAIN:
image = tf.image.random_flip_left_right(image)
image = tf.image.random_flip_up_down(image)
image = tf.image.rot90(image, k=tf.random_uniform(shape=(), maxval=3, minval=0, dtype=tf.int32))
return image, id_or_label, angle