def check_tst_ids(dataset_dir, file_pattern, num_samples): keys_to_features = { 'band_1': tf.FixedLenFeature((75 * 75,), tf.float32, default_value=None), 'band_2': tf.FixedLenFeature((75 * 75,), tf.float32, default_value=None), 'id': tf.FixedLenFeature([1], tf.string, default_value=None), 'angle': tf.FixedLenFeature([1], tf.float32, default_value=None), } items_to_handlers = { 'band_1': slim.tfexample_decoder.Tensor('band_1', shape=[75, 75]), 'band_2': slim.tfexample_decoder.Tensor('band_2', shape=[75, 75]), 'id': slim.tfexample_decoder.Tensor('id', shape=[]), 'angle': slim.tfexample_decoder.Tensor('angle', shape=[]) } dataset = mox.get_tfrecord(dataset_dir=dataset_dir, file_pattern=file_pattern, num_samples=num_samples, keys_to_features=keys_to_features, items_to_handlers=items_to_handlers, shuffle=False, num_epochs=1) band_1, band_2, id, angle = dataset.get( ['band_1', 'band_2', 'id', 'angle']) # sv = tf.train.Supervisor() id_dict = {} # with sv.managed_session() as sess: with tf.train.MonitoredTrainingSession() as sess: for i in range(num_samples): id_eval = sess.run(id) tf.logging.info('%s/%s' % (i, num_samples)) if id_eval not in id_dict: id_dict[id_eval] = 1 else: raise ValueError('id: %s has alreay been defined.') print('%s test cases checked.' % len(id_dict))
def read_and_decode_tfrecord(dataset_dir, file_pattern, num_samples): keys_to_features = { 'band_1': tf.FixedLenFeature((75 * 75, ), tf.float32, default_value=None), 'band_2': tf.FixedLenFeature((75 * 75, ), tf.float32, default_value=None), 'label': tf.FixedLenFeature([1], tf.int64, default_value=None), 'angle': tf.FixedLenFeature([1], tf.float32, default_value=None), } items_to_handlers = { 'band_1': slim.tfexample_decoder.Tensor('band_1', shape=[75, 75]), 'band_2': slim.tfexample_decoder.Tensor('band_2', shape=[75, 75]), 'label': slim.tfexample_decoder.Tensor('label', shape=[]), 'angle': slim.tfexample_decoder.Tensor('angle', shape=[]) } dataset = mox.get_tfrecord(dataset_dir=dataset_dir, file_pattern=file_pattern, num_samples=num_samples, keys_to_features=keys_to_features, items_to_handlers=items_to_handlers, shuffle=False, num_epochs=1) band_1, band_2, label, angle = dataset.get( ['band_1', 'band_2', 'label', 'angle']) band_3 = (band_1 + band_2) / 2 image = tf.stack([band_1, band_2, band_3], axis=2) image_max = tf.reduce_max(image) image_min = tf.reduce_min(image) image = (image - image_min) / (image_max - image_min) sv = tf.train.Supervisor() with sv.managed_session() as sess: plt.figure() for i in range(40): subp = plt.subplot(5, 8, i + 1) plt.subplots_adjust(hspace=0.6) subp.imshow(sess.run(image)) label_eval = sess.run(label) angle_eval = int(sess.run(angle)) subp.set_title('label=%s, angle=%s' % (label_eval, angle_eval)) plt.show()
def read_and_decode_tfrecord(dataset_dir, file_pattern, num_samples): keys_to_features = { 'band_1': tf.FixedLenFeature((75 * 75,), tf.float32, default_value=None), 'band_2': tf.FixedLenFeature((75 * 75,), tf.float32, default_value=None), 'label': tf.FixedLenFeature([1], tf.int64, default_value=None), 'angle': tf.FixedLenFeature([1], tf.float32, default_value=None), } items_to_handlers = { 'band_1': slim.tfexample_decoder.Tensor('band_1', shape=[75, 75]), 'band_2': slim.tfexample_decoder.Tensor('band_2', shape=[75, 75]), 'label': slim.tfexample_decoder.Tensor('label', shape=[]), 'angle': slim.tfexample_decoder.Tensor('angle', shape=[]) } dataset = mox.get_tfrecord(dataset_dir=dataset_dir, file_pattern=file_pattern, num_samples=num_samples, keys_to_features=keys_to_features, items_to_handlers=items_to_handlers, shuffle=False, num_epochs=1) band_1, band_2, label, angle = dataset.get(['band_1', 'band_2', 'label', 'angle']) band_3 = (band_1 + band_2) / 2 image = tf.stack([band_1, band_2, band_3], axis=2) image_max = tf.reduce_max(image) image_min = tf.reduce_min(image) image = (image - image_min) / (image_max - image_min) sv = tf.train.Supervisor() with sv.managed_session() as sess: plt.figure() for i in range(40): subp = plt.subplot(5, 8, i + 1) plt.subplots_adjust(hspace=0.6) subp.imshow(sess.run(image)) label_eval = sess.run(label) angle_eval = int(sess.run(angle)) subp.set_title('label=%s, angle=%s' % (label_eval, angle_eval)) plt.show()
def input_fn(run_mode, **kwargs): if run_mode == mox.ModeKeys.TRAIN: num_samples = NUM_SAMPLES_TRAIN num_epochs = None shuffle = True file_pattern = 'iceberg-train-*.tfrecord' else: num_epochs = 1 shuffle = False if run_mode == mox.ModeKeys.EVAL: num_samples = NUM_SAMPLES_EVAL file_pattern = 'iceberg-eval-*.tfrecord' else: num_samples = NUM_SAMPLES_TEST file_pattern = 'iceberg-test-*.tfrecord' keys_to_features = { 'band_1': tf.FixedLenFeature((75 * 75, ), tf.float32, default_value=None), 'band_2': tf.FixedLenFeature((75 * 75, ), tf.float32, default_value=None), 'angle': tf.FixedLenFeature([1], tf.float32, default_value=None), } items_to_handlers = { 'band_1': slim.tfexample_decoder.Tensor('band_1', shape=[75, 75]), 'band_2': slim.tfexample_decoder.Tensor('band_2', shape=[75, 75]), 'angle': slim.tfexample_decoder.Tensor('angle', shape=[]) } if run_mode == mox.ModeKeys.PREDICT: keys_to_features['id'] = tf.FixedLenFeature([1], tf.string, default_value=None) items_to_handlers['id'] = slim.tfexample_decoder.Tensor('id', shape=[]) else: keys_to_features['label'] = tf.FixedLenFeature([1], tf.int64, default_value=None) items_to_handlers['label'] = slim.tfexample_decoder.Tensor( 'label', shape=[]) dataset = mox.get_tfrecord(dataset_dir=flags.data_url, file_pattern=file_pattern, num_samples=num_samples, keys_to_features=keys_to_features, items_to_handlers=items_to_handlers, num_epochs=num_epochs, shuffle=shuffle) if run_mode == mox.ModeKeys.PREDICT: band_1, band_2, id_or_label, angle = dataset.get( ['band_1', 'band_2', 'id', 'angle']) # Non-DMA safe string cannot tensor may not be copied to a GPU. # So we encode string to a list of integer. id_or_label = tf.py_func( lambda str: np.array([ord(ch) for ch in str]), [id_or_label], tf.int64) # We know `id` is a string of 8 alphabets. id_or_label = tf.reshape(id_or_label, shape=(8, )) else: band_1, band_2, id_or_label, angle = dataset.get( ['band_1', 'band_2', 'label', 'angle']) band_3 = band_1 + band_2 # Rescale the input image to [0, 1] def rescale(*args): ret_images = [] for image in args: image = tf.cast(image, tf.float32) image_min = tf.reduce_min(image) image_max = tf.reduce_max(image) image = (image - image_min) / (image_max - image_min) ret_images.append(image) return ret_images band_1, band_2, band_3 = rescale(band_1, band_2, band_3) image = tf.stack([band_1, band_2, band_3], axis=2) # Data augementation if run_mode == mox.ModeKeys.TRAIN: image = tf.image.random_flip_left_right(image) image = tf.image.random_flip_up_down(image) image = tf.image.rot90(image, k=tf.random_uniform(shape=(), maxval=3, minval=0, dtype=tf.int32)) return image, id_or_label, angle
def input_fn(run_mode, **kwargs): if run_mode == mox.ModeKeys.TRAIN: num_samples = NUM_SAMPLES_TRAIN num_epochs = None shuffle = True file_pattern = 'iceberg-train-*.tfrecord' else: num_epochs = 1 shuffle = False if run_mode == mox.ModeKeys.EVAL: num_samples = NUM_SAMPLES_EVAL file_pattern = 'iceberg-eval-*.tfrecord' else: num_samples = NUM_SAMPLES_TEST file_pattern = 'iceberg-test-*.tfrecord' keys_to_features = { 'band_1': tf.FixedLenFeature((75 * 75,), tf.float32, default_value=None), 'band_2': tf.FixedLenFeature((75 * 75,), tf.float32, default_value=None), 'angle': tf.FixedLenFeature([1], tf.float32, default_value=None), } items_to_handlers = { 'band_1': slim.tfexample_decoder.Tensor('band_1', shape=[75, 75]), 'band_2': slim.tfexample_decoder.Tensor('band_2', shape=[75, 75]), 'angle': slim.tfexample_decoder.Tensor('angle', shape=[]) } if run_mode == mox.ModeKeys.PREDICT: keys_to_features['id'] = tf.FixedLenFeature([1], tf.string, default_value=None) items_to_handlers['id'] = slim.tfexample_decoder.Tensor('id', shape=[]) else: keys_to_features['label'] = tf.FixedLenFeature([1], tf.int64, default_value=None) items_to_handlers['label'] = slim.tfexample_decoder.Tensor('label', shape=[]) dataset = mox.get_tfrecord(dataset_dir=flags.data_url, file_pattern=file_pattern, num_samples=num_samples, keys_to_features=keys_to_features, items_to_handlers=items_to_handlers, num_epochs=num_epochs, shuffle=shuffle) if run_mode == mox.ModeKeys.PREDICT: band_1, band_2, id_or_label, angle = dataset.get(['band_1', 'band_2', 'id', 'angle']) # Non-DMA safe string cannot tensor may not be copied to a GPU. # So we encode string to a list of integer. id_or_label = tf.py_func(lambda str: np.array([ord(ch) for ch in str]), [id_or_label], tf.int64) # We know `id` is a string of 8 alphabets. id_or_label = tf.reshape(id_or_label, shape=(8,)) else: band_1, band_2, id_or_label, angle = dataset.get(['band_1', 'band_2', 'label', 'angle']) band_3 = band_1 + band_2 # Rescale the input image to [0, 1] def rescale(*args): ret_images = [] for image in args: image = tf.cast(image, tf.float32) image_min = tf.reduce_min(image) image_max = tf.reduce_max(image) image = (image - image_min) / (image_max - image_min) ret_images.append(image) return ret_images band_1, band_2, band_3 = rescale(band_1, band_2, band_3) image = tf.stack([band_1, band_2, band_3], axis=2) # Data augementation if run_mode == mox.ModeKeys.TRAIN: image = tf.image.random_flip_left_right(image) image = tf.image.random_flip_up_down(image) image = tf.image.rot90(image, k=tf.random_uniform(shape=(), maxval=3, minval=0, dtype=tf.int32)) return image, id_or_label, angle