def read_records(record_file): filename_queue = tf.train.string_input_producer([record_file], num_epochs=None) reader = tf.TFRecordReader() key, serialized_example = reader.read(filename_queue) features = tf.parse_single_example( serialized_example, features={ #'height': tf.FixedLenFeature([], tf.int64), #'width': tf.FixedLenFeature([], tf.int64), 'image_shape': tf.FixedLenFeature([3], tf.int64), 'label_shape': tf.FixedLenFeature([3], tf.int64), 'label': tf.FixedLenFeature([], tf.string), 'image_raw': tf.FixedLenFeature([], tf.string), }) print(features['image_shape']) label = features['label'] label = tf.parse_tensor(label, tf.float32, name=None) # label = tf.reverse(label,[False,False,True]) label = tf.reshape(label, tf.cast(features['label_shape'], tf.int32)) image = features['image_raw'] image = tf.parse_tensor(image, tf.float32, name=None) # image = tf.reverse(image,[False,False,True]) image = tf.reshape(image, tf.cast(features['image_shape'], tf.int32)) return image, label
def load_tensor(file_name): sict = tf.read_file(file_name) t2m = tf.parse_tensor(sict, numpy.float32) t2m = tf.reshape(t2m, [79, 159, 1]) file_name = tf.strings.regex_replace(file_name, 'air.2m', 'prmsl') sict = tf.read_file(file_name) prmsl = tf.parse_tensor(sict, numpy.float32) prmsl = tf.reshape(prmsl, [79, 159, 1]) file_name = tf.strings.regex_replace(file_name, 'prmsl', 'uwnd.10m') sict = tf.read_file(file_name) uwnd = tf.parse_tensor(sict, numpy.float32) uwnd = tf.reshape(uwnd, [79, 159, 1]) file_name = tf.strings.regex_replace(file_name, 'uwnd.10m', 'vwnd.10m') sict = tf.read_file(file_name) vwnd = tf.parse_tensor(sict, numpy.float32) vwnd = tf.reshape(vwnd, [79, 159, 1]) file_name = tf.strings.regex_replace(file_name, 'vwnd.10m', 'insolation') fdte = tf.strings.substr(file_name, -17, 17) mnth = tf.strings.substr(fdte, 5, 2) dy = tf.strings.substr(fdte, 8, 2) dy = tf.cond(tf.math.equal(mnth + dy, '0229'), lambda: tf.constant('28'), lambda: dy) file_name = (tf.strings.substr(file_name, 0, tf.strings.length(file_name) - 17) + '1969-' + mnth + '-' + dy + tf.strings.substr(fdte, tf.strings.length(fdte) - 7, 7)) sict = tf.read_file(file_name) insol = tf.parse_tensor(sict, numpy.float32) insol = tf.reshape(insol, [79, 159, 1]) ict = tf.concat([t2m, prmsl, uwnd, vwnd, insol], 2) # Now [79,159,5] ict = tf.reshape(ict, [79, 159, 5]) return ict
def _parse_voxel_function(example_proto): voxel_example = tf.parse_single_example(example_proto, voxel_feature_description) return tf.reshape( tf.parse_tensor(voxel_example['partial'], out_type=tf.bool), (32, 32, 32, 1)), tf.reshape( tf.parse_tensor(voxel_example['full'], out_type=tf.bool), (32, 32, 32, 1))
def parse_sparse_tensor(self, tensor): parsed = tf.parse_tensor(tensor, out_type=tf.string) sparse = tf.SparseTensor(indices=tf.parse_tensor(tf.gather(parsed, 0), out_type=tf.int64), values=tf.parse_tensor(tf.gather(parsed, 1), out_type=self.dtype), dense_shape=tf.parse_tensor( tf.gather(parsed, 2), out_type=tf.int64)) return tf.sparse_tensor_to_dense(sparse)
def load_tensor(file_name): sict = tf.read_file(file_name) prmsl = tf.parse_tensor(sict, numpy.float32) prmsl = tf.reshape(prmsl, [79, 159, 1]) file_name = tf.strings.regex_replace(file_name, 'prmsl', 'air.2m') sict = tf.read_file(file_name) t2m = tf.parse_tensor(sict, numpy.float32) t2m = tf.reshape(t2m, [79, 159, 1]) file_name = tf.strings.regex_replace(file_name, 'air.2m', 'z500') sict = tf.read_file(file_name) prate = tf.parse_tensor(sict, numpy.float32) prate = tf.reshape(prate, [79, 159, 1]) ict = tf.concat([prmsl, t2m, prate], 2) # Now [79,159,3] ict = tf.reshape(ict, [79, 159, 3]) return ict
def _parse_voxel_function(example_proto): voxel_example = tf.parse_single_example(example_proto, voxel_feature_description) partial = tf.reshape( tf.parse_tensor(voxel_example['partial'], out_type=tf.bool), (32, 32, 32, 1)) full = tf.reshape( tf.parse_tensor(voxel_example['full'], out_type=tf.bool), (32, 32, 32, 1)) partial_float = tf.cast(partial, dtype=tf.float32) full_float = tf.cast(full, dtype=tf.float32) return partial_float, full_float
def load_ls(start, end): Dates = [] current = start while current <= end: Dates.append(current) current += datetime.timedelta(hours=6) Dates = numpy.array(Dates) ls = numpy.empty((len(Dates), 100)) if model_meta['random']: random = numpy.random.rand(len(Dates)) if model_meta['annual']: annual = numpy.empty((len(Dates))) if model_meta['diurnal']: diurnal = numpy.empty((len(Dates))) for idx, Date in enumerate(Dates): l_file = ( ("%s/Machine-Learning-experiments/datasets/latent_space/" + "%04d-%02d-%02d:%02d.tfd") % (os.getenv('SCRATCH'), Date.year, Date.month, Date.day, Date.hour)) sict = tf.read_file(l_file) ls[idx, :] = tf.parse_tensor(sict, numpy.float32).numpy() if model_meta['annual']: annual[idx] = Date.timetuple().tm_yday if model_meta['diurnal']: diurnal[idx] = Date.hour / 24 result = {'Date': Dates, 'ls': ls} if model_meta['random']: result['random'] = random if model_meta['annual']: result['annual'] = annual if model_meta['diurnal']: result['diurnal'] = diurnal return result
def converter(tensors): tensors = { k: tf.parse_tensor(v, dtypes[k]) for k, v in tensors.items() } [v.set_shape(shapes[k]) for k, v in tensors.items()] return tensors
def _parse_sdf_function(example_proto): sdf_example = tf.parse_single_example(example_proto, sdf_feature_description) point_clouds = tf.parse_tensor(sdf_example['point_clouds'], out_type=tf.float32) xyzs = tf.parse_tensor(sdf_example['xyzs'], out_type=tf.float32) labels = tf.parse_tensor(sdf_example['labels'], out_type=tf.float32) # Downsample SDF points randomly. idxs = tf.range(tf.shape(xyzs)[0]) ridxs = tf.random_shuffle(idxs)[:sdf_count] # Important to use same indices on each in order to align labels properly. xyzs = tf.gather(xyzs, ridxs, axis=0) labels = tf.reshape(tf.gather(labels, ridxs, axis=0), (sdf_count, 1)) return point_clouds, xyzs, labels
def load_tensor(file_name): sict = tf.read_file(file_name) prmsl = tf.parse_tensor(sict, numpy.float32) prmsl = tf.reshape(prmsl, [79, 159, 1]) file_name = tf.strings.regex_replace(file_name, 'prmsl', 'air.2m') sict = tf.read_file(file_name) t2m = tf.parse_tensor(sict, numpy.float32) t2m = tf.reshape(t2m, [79, 159, 1]) file_name = tf.strings.regex_replace(file_name, 'air.2m', 'prate') sict = tf.read_file(file_name) prate = tf.parse_tensor(sict, numpy.float32) prate = tf.math.maximum(prate, 0) prate *= 10000 prate = tf.math.sqrt(prate) # normalise-ish prate = tf.reshape(prate, [79, 159, 1]) ict = tf.concat([prmsl, t2m, prate], 2) # Now [79,159,3] ict = tf.reshape(ict, [79, 159, 3]) return ict
def testInvalidInput(self): with self.test_session(): serialized = tf.placeholder(tf.string) tensor = tf.parse_tensor(serialized, tf.uint16) with self.assertRaisesOpError("Could not parse `serialized` as TensorProto: 'bogus'"): tensor.eval(feed_dict={serialized: "bogus"}) with self.assertRaisesOpError(r"Expected `serialized` to be a scalar, got shape: \[1\]"): tensor.eval(feed_dict={serialized: ["bogus"]})
def testTypeMismatch(self): with self.test_session(): expected = np.random.rand(3, 4, 5).astype(np.uint8) tensor_proto = tensor_util.make_tensor_proto(expected) serialized = tf.placeholder(tf.string) tensor = tf.parse_tensor(serialized, tf.uint16) with self.assertRaisesOpError(r"Type mismatch between parsed tensor \(uint8\) and dtype " r"\(uint16\)"): tensor.eval(feed_dict={serialized: tensor_proto.SerializeToString()})
def load_tensor(file_name): sict = tf.read_file(file_name) t2m = tf.parse_tensor(sict, numpy.float32) t2m = tf.reshape(t2m, [79, 159, 1]) file_name = tf.strings.regex_replace(file_name, 'air.2m', 'prmsl') sict = tf.read_file(file_name) prmsl = tf.parse_tensor(sict, numpy.float32) prmsl = tf.reshape(prmsl, [79, 159, 1]) file_name = tf.strings.regex_replace(file_name, 'prmsl', 'uwnd.10m') sict = tf.read_file(file_name) uwnd = tf.parse_tensor(sict, numpy.float32) uwnd = tf.reshape(uwnd, [79, 159, 1]) file_name = tf.strings.regex_replace(file_name, 'uwnd.10m', 'vwnd.10m') sict = tf.read_file(file_name) vwnd = tf.parse_tensor(sict, numpy.float32) vwnd = tf.reshape(vwnd, [79, 159, 1]) ict = tf.concat([t2m, prmsl, uwnd, vwnd], 2) # Now [79,159,4] ict = tf.reshape(ict, [79, 159, 4]) return ict
def testToUint8(self): with self.test_session(): expected = np.random.rand(3, 4, 5).astype(np.uint8) tensor_proto = tensor_util.make_tensor_proto(expected) serialized = tf.placeholder(tf.string) tensor = tf.parse_tensor(serialized, tf.uint8) result = tensor.eval(feed_dict={serialized: tensor_proto.SerializeToString()}) self.assertAllEqual(expected, result)
def testTypeMismatch(self): with self.test_session(): expected = np.random.rand(3, 4, 5).astype(np.uint8) tensor_proto = tensor_util.make_tensor_proto(expected) serialized = tf.placeholder(tf.string) tensor = tf.parse_tensor(serialized, tf.uint16) with self.assertRaisesOpError( r"Type mismatch between parsed tensor \(uint8\) and dtype " r"\(uint16\)"): tensor.eval(feed_dict={serialized: tensor_proto.SerializeToString()})
def testInvalidInput(self): with self.test_session(): serialized = tf.placeholder(tf.string) tensor = tf.parse_tensor(serialized, tf.uint16) with self.assertRaisesOpError( "Could not parse `serialized` as TensorProto: 'bogus'"): tensor.eval(feed_dict={serialized: "bogus"}) with self.assertRaisesOpError( r"Expected `serialized` to be a scalar, got shape: \[1\]"): tensor.eval(feed_dict={serialized: ["bogus"]})
def testToUint8(self): with self.test_session(): expected = np.random.rand(3, 4, 5).astype(np.uint8) tensor_proto = tensor_util.make_tensor_proto(expected) serialized = tf.placeholder(tf.string) tensor = tf.parse_tensor(serialized, tf.uint8) result = tensor.eval( feed_dict={serialized: tensor_proto.SerializeToString()}) self.assertAllEqual(expected, result)
def load_observations(file_name): # Get the ob tensor file name from the field tensor file name file_name = tf.strings.regex_replace(file_name, 'datasets/', 'datasets/DWR/') file_name = tf.strings.regex_replace(file_name, 'test/', 'training/') file_name = tf.strings.regex_replace(file_name, 'training/2009', 'test/2009') sict = tf.read_file(file_name) # serialised ict = tf.parse_tensor(sict, numpy.float32) ict = tf.reshape(ict, [ 32, ]) return ict
def parse_example(self, example, input_shape, output_shape): feature_description = { 'bw': tf.FixedLenFeature([], tf.string), 'color': tf.FixedLenFeature([], tf.string), 'labels': tf.FixedLenFeature([], tf.string) } parsed = tf.parse_single_example(example, feature_description) black_white = tf.parse_tensor(parsed['bw'], self.dtype) black_white.set_shape(input_shape) labels = self.parse_sparse_tensor(parsed['labels']) labels.set_shape(output_shape) return black_white, labels
def _input_fn(): with tf.name_scope('input'): word_tensor = tf.parse_tensor(tf.read_file(word_indices_file), tf.int64) targets, contexts = skipgrams(word_tensor, num_skips, skip_window, batch_size // num_skips, num_epochs=num_epochs) contexts_matrix = tf.expand_dims(contexts, -1) return targets, contexts_matrix
def read_data_(self): ''' Returns Tensors for images and labels ''' if not tf.gfile.Exists(self.record_file): raise IOError('record_file not found: {}'.format(self.record_file)) with tf.variable_scope('InputProducer') as scope: filename_queue = tf.train.string_input_producer([self.record_file], num_epochs=None) reader = tf.TFRecordReader() key, serialized_example = reader.read(filename_queue) features = tf.parse_single_example( serialized_example, features={ 'image_shape': tf.FixedLenFeature([3], tf.int64), 'label_shape': tf.FixedLenFeature([3], tf.int64), 'label': tf.FixedLenFeature([], tf.string), 'image_raw': tf.FixedLenFeature([], tf.string), }) label = features['label'] label = tf.parse_tensor(label, tf.float32, name=None) #label = tf.reverse(label,[False,False,True]) label = tf.reshape( label, self.in_lab_shape) #tf.cast(features['label_shape'], tf.int32) image = features['image_raw'] image = tf.parse_tensor(image, tf.float32, name=None) #image = tf.reverse(image,[False,False,True]) image = tf.reshape( image, self.in_img_shape) #tf.cast(features['image_shape'], tf.int32) return image, label
def _input_fn(): with tf.name_scope('input'): word_tensor = tf.parse_tensor( tf.read_file(word_indices_file), tf.int64 ) targets, contexts = skipgrams( word_tensor, num_skips, skip_window, batch_size // num_skips, num_epochs=num_epochs ) contexts_matrix = tf.expand_dims(contexts, -1) return targets, contexts_matrix
def mywrite(): # some data data = ('0x87', ['hello', 'world'], [1.2, 0.8]) tf.serialize_tensor(data[0].encode('utf8')) _bytes_feature(data[0]) tf.parse_tensor(tf.serialize_tensor(data[1]).numpy(), out_type=tf.string) tf.parse_tensor(tf.serialize_tensor(data[2]).numpy(), out_type=tf.float32) sess = tf.Session() tf.serialize_tensor(data[1]).numpy() _bytes_feature(tf.serialize_tensor(data[1]).numpy()) tf.train.FeatureList(bytes_list=tf.train.BytesList( value=[tf.serialize_tensor(data[1])])) s = exp.SerializeToString() exp = tf.train.Example.FromString(s) feature_description = { 'id': tf.FixedLenFeature([], tf.string, default_value=''), 'mutate': tf.FixedLenFeature([], tf.string, default_value=''), 'label': tf.FixedLenFeature([], tf.string, default_value=''), # 'feature3': tf.FixedLenFeature([], tf.float32, default_value=0.0), } tf.parse_single_example(exp[0], feature_description) tf.parse_single_example(tf.data.Dataset.from_tensor_slices(exp), feature_description) with tf.python_io.TFRecordWriter('111.tfrec') as writer: writer.write(exp.SerializeToString()) def _my_parse_function(pto): # Parse the input tf.Example proto using the dictionary above. return tf.parse_single_example(pto, feature_description) raw = tf.data.TFRecordDataset('111.tfrec') raw parsed_image_dataset = raw.map(_my_parse_function) parsed_image_dataset for a in parsed_image_dataset: print(a) print(a['label']) # shit! print(tf.parse_tensor(a['label'], out_type=tf.float32)) feature['label'] tf.parse_tensor(feature['label'], out_type=tf.float32)
def _input_fn(): with tf.name_scope('input'): index = tf.parse_tensor(tf.read_file(index_file), tf.string) filename_queue = tf.train.string_input_producer( filenames, num_epochs=num_epochs) reader = tf.TFRecordReader(options=tf.python_io.TFRecordOptions( compression_type=TFRecordCompressionType.GZIP)) _, serialized_example = reader.read(filename_queue) words = tf.parse_single_example( serialized_example, { 'target_words': tf.FixedLenFeature([batch_size], tf.string), 'context_words': tf.FixedLenFeature([batch_size], tf.string) }) return { 'targets': tf.expand_dims(words['target_words'], 1), 'index': index }, words['context_words']
def ls_load_at_timepoint(year, month, day, hour): ls_file = (("%s/Machine-Learning-experiments/datasets/latent_space/" + "%04d-%02d-%02d:%02d.tfd") % (os.getenv('SCRATCH'), year, month, day, hour)) sict = tf.read_file(ls_file) ls = tf.reshape(tf.parse_tensor(sict, numpy.float32), [1, 100]) result = generator.predict_on_batch(ls) result = tf.reshape(result, [79, 159, 4]) t2m = dummy_cube() t2m.data = tf.reshape(result.numpy()[:, :, 0], [79, 159]).numpy() t2m = unnormalise_t2m(t2m) prmsl = dummy_cube() prmsl.data = tf.reshape(result.numpy()[:, :, 1], [79, 159]).numpy() prmsl = unnormalise_prmsl(prmsl) u10m = dummy_cube() u10m.data = tf.reshape(result.numpy()[:, :, 2], [79, 159]).numpy() u10m = unnormalise_wind(u10m) v10m = dummy_cube() v10m.data = tf.reshape(result.numpy()[:, :, 3], [79, 159]).numpy() v10m = unnormalise_wind(v10m) return (ls, t2m, prmsl, u10m, v10m)
def _parser_fn(value, parse_type=tf.float32): return tf.parse_tensor(value, parse_type)
def load_tensor(file_name): sict = tf.read_file(file_name) # serialised ict = tf.parse_tensor(sict, numpy.float32) return ict
def load_tensor(file_name): sict = tf.read_file(file_name) # serialised ict = tf.parse_tensor(sict, numpy.float32) ict = tf.reshape(ict, [91, 180, 1]) return ict
def load_tensor(file_name): sict = tf.read_file(file_name) # serialised ict = tf.parse_tensor(sict, numpy.float32) ict = tf.slice(ict, [0], [1]) # 1st value only ict = tf.reshape(ict, [1, 1]) return ict
def parse_tfrec(x): result = tf.parse_tensor(x, out_type=tf.float32) result = tf.reshape(result, img_shape) return result
import tensorflow as tf import numpy as np from PIL import Image with open('mnist_200.pb', 'rb') as f: a = tf.parse_tensor(f.read(), tf.float32) with tf.Session() as sess: b = a.eval(session=sess) b = np.squeeze(b)*256 b = b.astype(np.uint8) img = Image.fromarray(b, 'L') img.show()
def parse_into_image(features_dict): serialized_image = features_dict["image"] image = tf.parse_tensor(serialized_image, tf.uint8) label = features_dict["label"] image = tf.reshape(image, [28, 28]) return dict(images=image), label
def parse(x): result = tf.parse_tensor(x, out_type=tf.float32)
def _my_parse2(pto): return { 'id': pto['id'], 'label': tf.parse_tensor(pto['label'], out_type=tf.float32), 'mutate': tf.parse_tensor(pto['mutate'], out_type=tf.string) }