Beispiel #1
0
def read_records(record_file):
    filename_queue = tf.train.string_input_producer([record_file],
                                                    num_epochs=None)
    reader = tf.TFRecordReader()
    key, serialized_example = reader.read(filename_queue)
    features = tf.parse_single_example(
        serialized_example,
        features={
            #'height': tf.FixedLenFeature([], tf.int64),
            #'width': tf.FixedLenFeature([], tf.int64),
            'image_shape': tf.FixedLenFeature([3], tf.int64),
            'label_shape': tf.FixedLenFeature([3], tf.int64),
            'label': tf.FixedLenFeature([], tf.string),
            'image_raw': tf.FixedLenFeature([], tf.string),
        })

    print(features['image_shape'])

    label = features['label']
    label = tf.parse_tensor(label, tf.float32, name=None)
    # label = tf.reverse(label,[False,False,True])
    label = tf.reshape(label, tf.cast(features['label_shape'], tf.int32))

    image = features['image_raw']
    image = tf.parse_tensor(image, tf.float32, name=None)
    # image = tf.reverse(image,[False,False,True])
    image = tf.reshape(image, tf.cast(features['image_shape'], tf.int32))

    return image, label
def load_tensor(file_name):
    sict = tf.read_file(file_name)
    t2m = tf.parse_tensor(sict, numpy.float32)
    t2m = tf.reshape(t2m, [79, 159, 1])
    file_name = tf.strings.regex_replace(file_name, 'air.2m', 'prmsl')
    sict = tf.read_file(file_name)
    prmsl = tf.parse_tensor(sict, numpy.float32)
    prmsl = tf.reshape(prmsl, [79, 159, 1])
    file_name = tf.strings.regex_replace(file_name, 'prmsl', 'uwnd.10m')
    sict = tf.read_file(file_name)
    uwnd = tf.parse_tensor(sict, numpy.float32)
    uwnd = tf.reshape(uwnd, [79, 159, 1])
    file_name = tf.strings.regex_replace(file_name, 'uwnd.10m', 'vwnd.10m')
    sict = tf.read_file(file_name)
    vwnd = tf.parse_tensor(sict, numpy.float32)
    vwnd = tf.reshape(vwnd, [79, 159, 1])
    file_name = tf.strings.regex_replace(file_name, 'vwnd.10m', 'insolation')
    fdte = tf.strings.substr(file_name, -17, 17)
    mnth = tf.strings.substr(fdte, 5, 2)
    dy = tf.strings.substr(fdte, 8, 2)
    dy = tf.cond(tf.math.equal(mnth + dy, '0229'), lambda: tf.constant('28'),
                 lambda: dy)
    file_name = (tf.strings.substr(file_name, 0,
                                   tf.strings.length(file_name) - 17) +
                 '1969-' + mnth + '-' + dy +
                 tf.strings.substr(fdte,
                                   tf.strings.length(fdte) - 7, 7))
    sict = tf.read_file(file_name)
    insol = tf.parse_tensor(sict, numpy.float32)
    insol = tf.reshape(insol, [79, 159, 1])
    ict = tf.concat([t2m, prmsl, uwnd, vwnd, insol], 2)  # Now [79,159,5]
    ict = tf.reshape(ict, [79, 159, 5])
    return ict
Beispiel #3
0
 def _parse_voxel_function(example_proto):
     voxel_example = tf.parse_single_example(example_proto,
                                             voxel_feature_description)
     return tf.reshape(
         tf.parse_tensor(voxel_example['partial'], out_type=tf.bool),
         (32, 32, 32, 1)), tf.reshape(
             tf.parse_tensor(voxel_example['full'], out_type=tf.bool),
             (32, 32, 32, 1))
Beispiel #4
0
 def parse_sparse_tensor(self, tensor):
     parsed = tf.parse_tensor(tensor, out_type=tf.string)
     sparse = tf.SparseTensor(indices=tf.parse_tensor(tf.gather(parsed, 0),
                                                      out_type=tf.int64),
                              values=tf.parse_tensor(tf.gather(parsed, 1),
                                                     out_type=self.dtype),
                              dense_shape=tf.parse_tensor(
                                  tf.gather(parsed, 2), out_type=tf.int64))
     return tf.sparse_tensor_to_dense(sparse)
def load_tensor(file_name):
    sict = tf.read_file(file_name)
    prmsl = tf.parse_tensor(sict, numpy.float32)
    prmsl = tf.reshape(prmsl, [79, 159, 1])
    file_name = tf.strings.regex_replace(file_name, 'prmsl', 'air.2m')
    sict = tf.read_file(file_name)
    t2m = tf.parse_tensor(sict, numpy.float32)
    t2m = tf.reshape(t2m, [79, 159, 1])
    file_name = tf.strings.regex_replace(file_name, 'air.2m', 'z500')
    sict = tf.read_file(file_name)
    prate = tf.parse_tensor(sict, numpy.float32)
    prate = tf.reshape(prate, [79, 159, 1])
    ict = tf.concat([prmsl, t2m, prate], 2)  # Now [79,159,3]
    ict = tf.reshape(ict, [79, 159, 3])
    return ict
Beispiel #6
0
    def _parse_voxel_function(example_proto):
        voxel_example = tf.parse_single_example(example_proto,
                                                voxel_feature_description)

        partial = tf.reshape(
            tf.parse_tensor(voxel_example['partial'], out_type=tf.bool),
            (32, 32, 32, 1))
        full = tf.reshape(
            tf.parse_tensor(voxel_example['full'], out_type=tf.bool),
            (32, 32, 32, 1))

        partial_float = tf.cast(partial, dtype=tf.float32)
        full_float = tf.cast(full, dtype=tf.float32)

        return partial_float, full_float
def load_ls(start, end):
    Dates = []
    current = start
    while current <= end:
        Dates.append(current)
        current += datetime.timedelta(hours=6)
    Dates = numpy.array(Dates)
    ls = numpy.empty((len(Dates), 100))
    if model_meta['random']: random = numpy.random.rand(len(Dates))
    if model_meta['annual']: annual = numpy.empty((len(Dates)))
    if model_meta['diurnal']: diurnal = numpy.empty((len(Dates)))
    for idx, Date in enumerate(Dates):
        l_file = (
            ("%s/Machine-Learning-experiments/datasets/latent_space/" +
             "%04d-%02d-%02d:%02d.tfd") %
            (os.getenv('SCRATCH'), Date.year, Date.month, Date.day, Date.hour))
        sict = tf.read_file(l_file)
        ls[idx, :] = tf.parse_tensor(sict, numpy.float32).numpy()
        if model_meta['annual']:
            annual[idx] = Date.timetuple().tm_yday
        if model_meta['diurnal']:
            diurnal[idx] = Date.hour / 24
    result = {'Date': Dates, 'ls': ls}
    if model_meta['random']: result['random'] = random
    if model_meta['annual']: result['annual'] = annual
    if model_meta['diurnal']: result['diurnal'] = diurnal
    return result
Beispiel #8
0
 def converter(tensors):
     tensors = {
         k: tf.parse_tensor(v, dtypes[k])
         for k, v in tensors.items()
     }
     [v.set_shape(shapes[k]) for k, v in tensors.items()]
     return tensors
Beispiel #9
0
    def _parse_sdf_function(example_proto):
        sdf_example = tf.parse_single_example(example_proto,
                                              sdf_feature_description)

        point_clouds = tf.parse_tensor(sdf_example['point_clouds'],
                                       out_type=tf.float32)
        xyzs = tf.parse_tensor(sdf_example['xyzs'], out_type=tf.float32)
        labels = tf.parse_tensor(sdf_example['labels'], out_type=tf.float32)

        # Downsample SDF points randomly.
        idxs = tf.range(tf.shape(xyzs)[0])
        ridxs = tf.random_shuffle(idxs)[:sdf_count]

        # Important to use same indices on each in order to align labels properly.
        xyzs = tf.gather(xyzs, ridxs, axis=0)
        labels = tf.reshape(tf.gather(labels, ridxs, axis=0), (sdf_count, 1))

        return point_clouds, xyzs, labels
Beispiel #10
0
def load_tensor(file_name):
    sict = tf.read_file(file_name)
    prmsl = tf.parse_tensor(sict, numpy.float32)
    prmsl = tf.reshape(prmsl, [79, 159, 1])
    file_name = tf.strings.regex_replace(file_name, 'prmsl', 'air.2m')
    sict = tf.read_file(file_name)
    t2m = tf.parse_tensor(sict, numpy.float32)
    t2m = tf.reshape(t2m, [79, 159, 1])
    file_name = tf.strings.regex_replace(file_name, 'air.2m', 'prate')
    sict = tf.read_file(file_name)
    prate = tf.parse_tensor(sict, numpy.float32)
    prate = tf.math.maximum(prate, 0)
    prate *= 10000
    prate = tf.math.sqrt(prate)  # normalise-ish
    prate = tf.reshape(prate, [79, 159, 1])
    ict = tf.concat([prmsl, t2m, prate], 2)  # Now [79,159,3]
    ict = tf.reshape(ict, [79, 159, 3])
    return ict
Beispiel #11
0
    def testInvalidInput(self):
        with self.test_session():
            serialized = tf.placeholder(tf.string)
            tensor = tf.parse_tensor(serialized, tf.uint16)

            with self.assertRaisesOpError("Could not parse `serialized` as TensorProto: 'bogus'"):
                tensor.eval(feed_dict={serialized: "bogus"})

            with self.assertRaisesOpError(r"Expected `serialized` to be a scalar, got shape: \[1\]"):
                tensor.eval(feed_dict={serialized: ["bogus"]})
Beispiel #12
0
    def testTypeMismatch(self):
        with self.test_session():
            expected = np.random.rand(3, 4, 5).astype(np.uint8)
            tensor_proto = tensor_util.make_tensor_proto(expected)

            serialized = tf.placeholder(tf.string)
            tensor = tf.parse_tensor(serialized, tf.uint16)

            with self.assertRaisesOpError(r"Type mismatch between parsed tensor \(uint8\) and dtype " r"\(uint16\)"):
                tensor.eval(feed_dict={serialized: tensor_proto.SerializeToString()})
def load_tensor(file_name):
    sict = tf.read_file(file_name)
    t2m = tf.parse_tensor(sict, numpy.float32)
    t2m = tf.reshape(t2m, [79, 159, 1])
    file_name = tf.strings.regex_replace(file_name, 'air.2m', 'prmsl')
    sict = tf.read_file(file_name)
    prmsl = tf.parse_tensor(sict, numpy.float32)
    prmsl = tf.reshape(prmsl, [79, 159, 1])
    file_name = tf.strings.regex_replace(file_name, 'prmsl', 'uwnd.10m')
    sict = tf.read_file(file_name)
    uwnd = tf.parse_tensor(sict, numpy.float32)
    uwnd = tf.reshape(uwnd, [79, 159, 1])
    file_name = tf.strings.regex_replace(file_name, 'uwnd.10m', 'vwnd.10m')
    sict = tf.read_file(file_name)
    vwnd = tf.parse_tensor(sict, numpy.float32)
    vwnd = tf.reshape(vwnd, [79, 159, 1])
    ict = tf.concat([t2m, prmsl, uwnd, vwnd], 2)  # Now [79,159,4]
    ict = tf.reshape(ict, [79, 159, 4])
    return ict
Beispiel #14
0
    def testToUint8(self):
        with self.test_session():
            expected = np.random.rand(3, 4, 5).astype(np.uint8)
            tensor_proto = tensor_util.make_tensor_proto(expected)

            serialized = tf.placeholder(tf.string)
            tensor = tf.parse_tensor(serialized, tf.uint8)

            result = tensor.eval(feed_dict={serialized: tensor_proto.SerializeToString()})

            self.assertAllEqual(expected, result)
Beispiel #15
0
  def testTypeMismatch(self):
    with self.test_session():
      expected = np.random.rand(3, 4, 5).astype(np.uint8)
      tensor_proto = tensor_util.make_tensor_proto(expected)

      serialized = tf.placeholder(tf.string)
      tensor = tf.parse_tensor(serialized, tf.uint16)

      with self.assertRaisesOpError(
          r"Type mismatch between parsed tensor \(uint8\) and dtype "
          r"\(uint16\)"):
        tensor.eval(feed_dict={serialized: tensor_proto.SerializeToString()})
Beispiel #16
0
    def testInvalidInput(self):
        with self.test_session():
            serialized = tf.placeholder(tf.string)
            tensor = tf.parse_tensor(serialized, tf.uint16)

            with self.assertRaisesOpError(
                    "Could not parse `serialized` as TensorProto: 'bogus'"):
                tensor.eval(feed_dict={serialized: "bogus"})

            with self.assertRaisesOpError(
                    r"Expected `serialized` to be a scalar, got shape: \[1\]"):
                tensor.eval(feed_dict={serialized: ["bogus"]})
Beispiel #17
0
    def testToUint8(self):
        with self.test_session():
            expected = np.random.rand(3, 4, 5).astype(np.uint8)
            tensor_proto = tensor_util.make_tensor_proto(expected)

            serialized = tf.placeholder(tf.string)
            tensor = tf.parse_tensor(serialized, tf.uint8)

            result = tensor.eval(
                feed_dict={serialized: tensor_proto.SerializeToString()})

            self.assertAllEqual(expected, result)
def load_observations(file_name):
    # Get the ob tensor file name from the field tensor file name
    file_name = tf.strings.regex_replace(file_name, 'datasets/',
                                         'datasets/DWR/')
    file_name = tf.strings.regex_replace(file_name, 'test/', 'training/')
    file_name = tf.strings.regex_replace(file_name, 'training/2009',
                                         'test/2009')
    sict = tf.read_file(file_name)  # serialised
    ict = tf.parse_tensor(sict, numpy.float32)
    ict = tf.reshape(ict, [
        32,
    ])
    return ict
Beispiel #19
0
    def parse_example(self, example, input_shape, output_shape):
        feature_description = {
            'bw': tf.FixedLenFeature([], tf.string),
            'color': tf.FixedLenFeature([], tf.string),
            'labels': tf.FixedLenFeature([], tf.string)
        }
        parsed = tf.parse_single_example(example, feature_description)

        black_white = tf.parse_tensor(parsed['bw'], self.dtype)
        black_white.set_shape(input_shape)
        labels = self.parse_sparse_tensor(parsed['labels'])
        labels.set_shape(output_shape)
        return black_white, labels
Beispiel #20
0
    def _input_fn():
        with tf.name_scope('input'):
            word_tensor = tf.parse_tensor(tf.read_file(word_indices_file),
                                          tf.int64)

            targets, contexts = skipgrams(word_tensor,
                                          num_skips,
                                          skip_window,
                                          batch_size // num_skips,
                                          num_epochs=num_epochs)

            contexts_matrix = tf.expand_dims(contexts, -1)
            return targets, contexts_matrix
    def read_data_(self):
        '''
        Returns Tensors for images and labels
        '''
        if not tf.gfile.Exists(self.record_file):
            raise IOError('record_file not found: {}'.format(self.record_file))

        with tf.variable_scope('InputProducer') as scope:
            filename_queue = tf.train.string_input_producer([self.record_file],
                                                            num_epochs=None)
            reader = tf.TFRecordReader()
            key, serialized_example = reader.read(filename_queue)
            features = tf.parse_single_example(
                serialized_example,
                features={
                    'image_shape': tf.FixedLenFeature([3], tf.int64),
                    'label_shape': tf.FixedLenFeature([3], tf.int64),
                    'label': tf.FixedLenFeature([], tf.string),
                    'image_raw': tf.FixedLenFeature([], tf.string),
                })

            label = features['label']
            label = tf.parse_tensor(label, tf.float32, name=None)
            #label = tf.reverse(label,[False,False,True])
            label = tf.reshape(
                label,
                self.in_lab_shape)  #tf.cast(features['label_shape'], tf.int32)

            image = features['image_raw']
            image = tf.parse_tensor(image, tf.float32, name=None)
            #image = tf.reverse(image,[False,False,True])
            image = tf.reshape(
                image,
                self.in_img_shape)  #tf.cast(features['image_shape'], tf.int32)

            return image, label
Beispiel #22
0
  def _input_fn():
    with tf.name_scope('input'):
      word_tensor = tf.parse_tensor(
          tf.read_file(word_indices_file),
          tf.int64
      )

      targets, contexts = skipgrams(
          word_tensor,
          num_skips,
          skip_window,
          batch_size // num_skips,
          num_epochs=num_epochs
      )

      contexts_matrix = tf.expand_dims(contexts, -1)
      return targets, contexts_matrix
Beispiel #23
0
def mywrite():
    # some data
    data = ('0x87', ['hello', 'world'], [1.2, 0.8])
    tf.serialize_tensor(data[0].encode('utf8'))
    _bytes_feature(data[0])
    tf.parse_tensor(tf.serialize_tensor(data[1]).numpy(), out_type=tf.string)
    tf.parse_tensor(tf.serialize_tensor(data[2]).numpy(), out_type=tf.float32)
    sess = tf.Session()
    tf.serialize_tensor(data[1]).numpy()
    _bytes_feature(tf.serialize_tensor(data[1]).numpy())
    tf.train.FeatureList(bytes_list=tf.train.BytesList(
        value=[tf.serialize_tensor(data[1])]))
    s = exp.SerializeToString()
    exp = tf.train.Example.FromString(s)
    feature_description = {
        'id': tf.FixedLenFeature([], tf.string, default_value=''),
        'mutate': tf.FixedLenFeature([], tf.string, default_value=''),
        'label': tf.FixedLenFeature([], tf.string, default_value=''),
        # 'feature3': tf.FixedLenFeature([], tf.float32, default_value=0.0),
    }
    tf.parse_single_example(exp[0], feature_description)
    tf.parse_single_example(tf.data.Dataset.from_tensor_slices(exp),
                            feature_description)

    with tf.python_io.TFRecordWriter('111.tfrec') as writer:
        writer.write(exp.SerializeToString())

    def _my_parse_function(pto):
        # Parse the input tf.Example proto using the dictionary above.
        return tf.parse_single_example(pto, feature_description)

    raw = tf.data.TFRecordDataset('111.tfrec')
    raw
    parsed_image_dataset = raw.map(_my_parse_function)
    parsed_image_dataset

    for a in parsed_image_dataset:
        print(a)
        print(a['label'])
        # shit!
        print(tf.parse_tensor(a['label'], out_type=tf.float32))

    feature['label']
    tf.parse_tensor(feature['label'], out_type=tf.float32)
Beispiel #24
0
    def _input_fn():
        with tf.name_scope('input'):
            index = tf.parse_tensor(tf.read_file(index_file), tf.string)
            filename_queue = tf.train.string_input_producer(
                filenames, num_epochs=num_epochs)
            reader = tf.TFRecordReader(options=tf.python_io.TFRecordOptions(
                compression_type=TFRecordCompressionType.GZIP))
            _, serialized_example = reader.read(filename_queue)

            words = tf.parse_single_example(
                serialized_example, {
                    'target_words': tf.FixedLenFeature([batch_size],
                                                       tf.string),
                    'context_words': tf.FixedLenFeature([batch_size],
                                                        tf.string)
                })
            return {
                'targets': tf.expand_dims(words['target_words'], 1),
                'index': index
            }, words['context_words']
def ls_load_at_timepoint(year, month, day, hour):
    ls_file = (("%s/Machine-Learning-experiments/datasets/latent_space/" +
                "%04d-%02d-%02d:%02d.tfd") %
               (os.getenv('SCRATCH'), year, month, day, hour))
    sict = tf.read_file(ls_file)
    ls = tf.reshape(tf.parse_tensor(sict, numpy.float32), [1, 100])
    result = generator.predict_on_batch(ls)
    result = tf.reshape(result, [79, 159, 4])
    t2m = dummy_cube()
    t2m.data = tf.reshape(result.numpy()[:, :, 0], [79, 159]).numpy()
    t2m = unnormalise_t2m(t2m)
    prmsl = dummy_cube()
    prmsl.data = tf.reshape(result.numpy()[:, :, 1], [79, 159]).numpy()
    prmsl = unnormalise_prmsl(prmsl)
    u10m = dummy_cube()
    u10m.data = tf.reshape(result.numpy()[:, :, 2], [79, 159]).numpy()
    u10m = unnormalise_wind(u10m)
    v10m = dummy_cube()
    v10m.data = tf.reshape(result.numpy()[:, :, 3], [79, 159]).numpy()
    v10m = unnormalise_wind(v10m)
    return (ls, t2m, prmsl, u10m, v10m)
def _parser_fn(value, parse_type=tf.float32):
    return tf.parse_tensor(value, parse_type)
Beispiel #27
0
def load_tensor(file_name):
    sict = tf.read_file(file_name)  # serialised
    ict = tf.parse_tensor(sict, numpy.float32)
    return ict
Beispiel #28
0
def load_tensor(file_name):
    sict = tf.read_file(file_name)  # serialised
    ict = tf.parse_tensor(sict, numpy.float32)
    ict = tf.reshape(ict, [91, 180, 1])
    return ict
def load_tensor(file_name):
    sict = tf.read_file(file_name)  # serialised
    ict = tf.parse_tensor(sict, numpy.float32)
    ict = tf.slice(ict, [0], [1])  # 1st value only
    ict = tf.reshape(ict, [1, 1])
    return ict
 def parse_tfrec(x):
     result = tf.parse_tensor(x, out_type=tf.float32)
     result = tf.reshape(result, img_shape)
     return result
Beispiel #31
0
import tensorflow as tf
import numpy as np
from PIL import Image

with open('mnist_200.pb', 'rb') as f:
    a = tf.parse_tensor(f.read(), tf.float32)

with tf.Session() as sess:
    b = a.eval(session=sess)

b = np.squeeze(b)*256
b = b.astype(np.uint8)

img = Image.fromarray(b, 'L')
img.show()


Beispiel #32
0
def parse_into_image(features_dict):
    serialized_image = features_dict["image"]
    image = tf.parse_tensor(serialized_image, tf.uint8)
    label = features_dict["label"]
    image = tf.reshape(image, [28, 28])
    return dict(images=image), label
Beispiel #33
0
def parse(x):
    result = tf.parse_tensor(x, out_type=tf.float32)
Beispiel #34
0
 def _my_parse2(pto):
     return {
         'id': pto['id'],
         'label': tf.parse_tensor(pto['label'], out_type=tf.float32),
         'mutate': tf.parse_tensor(pto['mutate'], out_type=tf.string)
     }