def read_records(record_file):
filename_queue = tf.train.string_input_producer([record_file],
num_epochs=None)
reader = tf.TFRecordReader()
key, serialized_example = reader.read(filename_queue)
features = tf.parse_single_example(
serialized_example,
features={
#'height': tf.FixedLenFeature([], tf.int64),
#'width': tf.FixedLenFeature([], tf.int64),
'image_shape': tf.FixedLenFeature([3], tf.int64),
'label_shape': tf.FixedLenFeature([3], tf.int64),
'label': tf.FixedLenFeature([], tf.string),
'image_raw': tf.FixedLenFeature([], tf.string),
})
print(features['image_shape'])
label = features['label']
label = tf.parse_tensor(label, tf.float32, name=None)
# label = tf.reverse(label,[False,False,True])
label = tf.reshape(label, tf.cast(features['label_shape'], tf.int32))
image = features['image_raw']
image = tf.parse_tensor(image, tf.float32, name=None)
# image = tf.reverse(image,[False,False,True])
image = tf.reshape(image, tf.cast(features['image_shape'], tf.int32))
return image, label
def load_tensor(file_name):
sict = tf.read_file(file_name)
t2m = tf.parse_tensor(sict, numpy.float32)
t2m = tf.reshape(t2m, [79, 159, 1])
file_name = tf.strings.regex_replace(file_name, 'air.2m', 'prmsl')
sict = tf.read_file(file_name)
prmsl = tf.parse_tensor(sict, numpy.float32)
prmsl = tf.reshape(prmsl, [79, 159, 1])
file_name = tf.strings.regex_replace(file_name, 'prmsl', 'uwnd.10m')
sict = tf.read_file(file_name)
uwnd = tf.parse_tensor(sict, numpy.float32)
uwnd = tf.reshape(uwnd, [79, 159, 1])
file_name = tf.strings.regex_replace(file_name, 'uwnd.10m', 'vwnd.10m')
sict = tf.read_file(file_name)
vwnd = tf.parse_tensor(sict, numpy.float32)
vwnd = tf.reshape(vwnd, [79, 159, 1])
file_name = tf.strings.regex_replace(file_name, 'vwnd.10m', 'insolation')
fdte = tf.strings.substr(file_name, -17, 17)
mnth = tf.strings.substr(fdte, 5, 2)
dy = tf.strings.substr(fdte, 8, 2)
dy = tf.cond(tf.math.equal(mnth + dy, '0229'), lambda: tf.constant('28'),
lambda: dy)
file_name = (tf.strings.substr(file_name, 0,
tf.strings.length(file_name) - 17) +
'1969-' + mnth + '-' + dy +
tf.strings.substr(fdte,
tf.strings.length(fdte) - 7, 7))
sict = tf.read_file(file_name)
insol = tf.parse_tensor(sict, numpy.float32)
insol = tf.reshape(insol, [79, 159, 1])
ict = tf.concat([t2m, prmsl, uwnd, vwnd, insol], 2) # Now [79,159,5]
ict = tf.reshape(ict, [79, 159, 5])
return ict
def _parse_voxel_function(example_proto):
voxel_example = tf.parse_single_example(example_proto,
voxel_feature_description)
return tf.reshape(
tf.parse_tensor(voxel_example['partial'], out_type=tf.bool),
(32, 32, 32, 1)), tf.reshape(
tf.parse_tensor(voxel_example['full'], out_type=tf.bool),
(32, 32, 32, 1))
def parse_sparse_tensor(self, tensor):
parsed = tf.parse_tensor(tensor, out_type=tf.string)
sparse = tf.SparseTensor(indices=tf.parse_tensor(tf.gather(parsed, 0),
out_type=tf.int64),
values=tf.parse_tensor(tf.gather(parsed, 1),
out_type=self.dtype),
dense_shape=tf.parse_tensor(
tf.gather(parsed, 2), out_type=tf.int64))
return tf.sparse_tensor_to_dense(sparse)
def load_tensor(file_name):
sict = tf.read_file(file_name)
prmsl = tf.parse_tensor(sict, numpy.float32)
prmsl = tf.reshape(prmsl, [79, 159, 1])
file_name = tf.strings.regex_replace(file_name, 'prmsl', 'air.2m')
sict = tf.read_file(file_name)
t2m = tf.parse_tensor(sict, numpy.float32)
t2m = tf.reshape(t2m, [79, 159, 1])
file_name = tf.strings.regex_replace(file_name, 'air.2m', 'z500')
sict = tf.read_file(file_name)
prate = tf.parse_tensor(sict, numpy.float32)
prate = tf.reshape(prate, [79, 159, 1])
ict = tf.concat([prmsl, t2m, prate], 2) # Now [79,159,3]
ict = tf.reshape(ict, [79, 159, 3])
return ict
def _parse_voxel_function(example_proto):
voxel_example = tf.parse_single_example(example_proto,
voxel_feature_description)
partial = tf.reshape(
tf.parse_tensor(voxel_example['partial'], out_type=tf.bool),
(32, 32, 32, 1))
full = tf.reshape(
tf.parse_tensor(voxel_example['full'], out_type=tf.bool),
(32, 32, 32, 1))
partial_float = tf.cast(partial, dtype=tf.float32)
full_float = tf.cast(full, dtype=tf.float32)
return partial_float, full_float
def load_ls(start, end):
Dates = []
current = start
while current <= end:
Dates.append(current)
current += datetime.timedelta(hours=6)
Dates = numpy.array(Dates)
ls = numpy.empty((len(Dates), 100))
if model_meta['random']: random = numpy.random.rand(len(Dates))
if model_meta['annual']: annual = numpy.empty((len(Dates)))
if model_meta['diurnal']: diurnal = numpy.empty((len(Dates)))
for idx, Date in enumerate(Dates):
l_file = (
("%s/Machine-Learning-experiments/datasets/latent_space/" +
"%04d-%02d-%02d:%02d.tfd") %
(os.getenv('SCRATCH'), Date.year, Date.month, Date.day, Date.hour))
sict = tf.read_file(l_file)
ls[idx, :] = tf.parse_tensor(sict, numpy.float32).numpy()
if model_meta['annual']:
annual[idx] = Date.timetuple().tm_yday
if model_meta['diurnal']:
diurnal[idx] = Date.hour / 24
result = {'Date': Dates, 'ls': ls}
if model_meta['random']: result['random'] = random
if model_meta['annual']: result['annual'] = annual
if model_meta['diurnal']: result['diurnal'] = diurnal
return result
def converter(tensors):
tensors = {
k: tf.parse_tensor(v, dtypes[k])
for k, v in tensors.items()
}
[v.set_shape(shapes[k]) for k, v in tensors.items()]
return tensors
def _parse_sdf_function(example_proto):
sdf_example = tf.parse_single_example(example_proto,
sdf_feature_description)
point_clouds = tf.parse_tensor(sdf_example['point_clouds'],
out_type=tf.float32)
xyzs = tf.parse_tensor(sdf_example['xyzs'], out_type=tf.float32)
labels = tf.parse_tensor(sdf_example['labels'], out_type=tf.float32)
# Downsample SDF points randomly.
idxs = tf.range(tf.shape(xyzs)[0])
ridxs = tf.random_shuffle(idxs)[:sdf_count]
# Important to use same indices on each in order to align labels properly.
xyzs = tf.gather(xyzs, ridxs, axis=0)
labels = tf.reshape(tf.gather(labels, ridxs, axis=0), (sdf_count, 1))
return point_clouds, xyzs, labels
def load_tensor(file_name):
sict = tf.read_file(file_name)
prmsl = tf.parse_tensor(sict, numpy.float32)
prmsl = tf.reshape(prmsl, [79, 159, 1])
file_name = tf.strings.regex_replace(file_name, 'prmsl', 'air.2m')
sict = tf.read_file(file_name)
t2m = tf.parse_tensor(sict, numpy.float32)
t2m = tf.reshape(t2m, [79, 159, 1])
file_name = tf.strings.regex_replace(file_name, 'air.2m', 'prate')
sict = tf.read_file(file_name)
prate = tf.parse_tensor(sict, numpy.float32)
prate = tf.math.maximum(prate, 0)
prate *= 10000
prate = tf.math.sqrt(prate) # normalise-ish
prate = tf.reshape(prate, [79, 159, 1])
ict = tf.concat([prmsl, t2m, prate], 2) # Now [79,159,3]
ict = tf.reshape(ict, [79, 159, 3])
return ict
def testInvalidInput(self):
with self.test_session():
serialized = tf.placeholder(tf.string)
tensor = tf.parse_tensor(serialized, tf.uint16)
with self.assertRaisesOpError("Could not parse `serialized` as TensorProto: 'bogus'"):
tensor.eval(feed_dict={serialized: "bogus"})
with self.assertRaisesOpError(r"Expected `serialized` to be a scalar, got shape: \[1\]"):
tensor.eval(feed_dict={serialized: ["bogus"]})
def testTypeMismatch(self):
with self.test_session():
expected = np.random.rand(3, 4, 5).astype(np.uint8)
tensor_proto = tensor_util.make_tensor_proto(expected)
serialized = tf.placeholder(tf.string)
tensor = tf.parse_tensor(serialized, tf.uint16)
with self.assertRaisesOpError(r"Type mismatch between parsed tensor \(uint8\) and dtype " r"\(uint16\)"):
tensor.eval(feed_dict={serialized: tensor_proto.SerializeToString()})
def load_tensor(file_name):
sict = tf.read_file(file_name)
t2m = tf.parse_tensor(sict, numpy.float32)
t2m = tf.reshape(t2m, [79, 159, 1])
file_name = tf.strings.regex_replace(file_name, 'air.2m', 'prmsl')
sict = tf.read_file(file_name)
prmsl = tf.parse_tensor(sict, numpy.float32)
prmsl = tf.reshape(prmsl, [79, 159, 1])
file_name = tf.strings.regex_replace(file_name, 'prmsl', 'uwnd.10m')
sict = tf.read_file(file_name)
uwnd = tf.parse_tensor(sict, numpy.float32)
uwnd = tf.reshape(uwnd, [79, 159, 1])
file_name = tf.strings.regex_replace(file_name, 'uwnd.10m', 'vwnd.10m')
sict = tf.read_file(file_name)
vwnd = tf.parse_tensor(sict, numpy.float32)
vwnd = tf.reshape(vwnd, [79, 159, 1])
ict = tf.concat([t2m, prmsl, uwnd, vwnd], 2) # Now [79,159,4]
ict = tf.reshape(ict, [79, 159, 4])
return ict
def testToUint8(self):
with self.test_session():
expected = np.random.rand(3, 4, 5).astype(np.uint8)
tensor_proto = tensor_util.make_tensor_proto(expected)
serialized = tf.placeholder(tf.string)
tensor = tf.parse_tensor(serialized, tf.uint8)
result = tensor.eval(feed_dict={serialized: tensor_proto.SerializeToString()})
self.assertAllEqual(expected, result)
def testTypeMismatch(self):
with self.test_session():
expected = np.random.rand(3, 4, 5).astype(np.uint8)
tensor_proto = tensor_util.make_tensor_proto(expected)
serialized = tf.placeholder(tf.string)
tensor = tf.parse_tensor(serialized, tf.uint16)
with self.assertRaisesOpError(
r"Type mismatch between parsed tensor \(uint8\) and dtype "
r"\(uint16\)"):
tensor.eval(feed_dict={serialized: tensor_proto.SerializeToString()})
def testInvalidInput(self):
with self.test_session():
serialized = tf.placeholder(tf.string)
tensor = tf.parse_tensor(serialized, tf.uint16)
with self.assertRaisesOpError(
"Could not parse `serialized` as TensorProto: 'bogus'"):
tensor.eval(feed_dict={serialized: "bogus"})
with self.assertRaisesOpError(
r"Expected `serialized` to be a scalar, got shape: \[1\]"):
tensor.eval(feed_dict={serialized: ["bogus"]})
def testToUint8(self):
with self.test_session():
expected = np.random.rand(3, 4, 5).astype(np.uint8)
tensor_proto = tensor_util.make_tensor_proto(expected)
serialized = tf.placeholder(tf.string)
tensor = tf.parse_tensor(serialized, tf.uint8)
result = tensor.eval(
feed_dict={serialized: tensor_proto.SerializeToString()})
self.assertAllEqual(expected, result)
def load_observations(file_name):
# Get the ob tensor file name from the field tensor file name
file_name = tf.strings.regex_replace(file_name, 'datasets/',
'datasets/DWR/')
file_name = tf.strings.regex_replace(file_name, 'test/', 'training/')
file_name = tf.strings.regex_replace(file_name, 'training/2009',
'test/2009')
sict = tf.read_file(file_name) # serialised
ict = tf.parse_tensor(sict, numpy.float32)
ict = tf.reshape(ict, [
32,
])
return ict
def parse_example(self, example, input_shape, output_shape):
feature_description = {
'bw': tf.FixedLenFeature([], tf.string),
'color': tf.FixedLenFeature([], tf.string),
'labels': tf.FixedLenFeature([], tf.string)
}
parsed = tf.parse_single_example(example, feature_description)
black_white = tf.parse_tensor(parsed['bw'], self.dtype)
black_white.set_shape(input_shape)
labels = self.parse_sparse_tensor(parsed['labels'])
labels.set_shape(output_shape)
return black_white, labels
def _input_fn():
with tf.name_scope('input'):
word_tensor = tf.parse_tensor(tf.read_file(word_indices_file),
tf.int64)
targets, contexts = skipgrams(word_tensor,
num_skips,
skip_window,
batch_size // num_skips,
num_epochs=num_epochs)
contexts_matrix = tf.expand_dims(contexts, -1)
return targets, contexts_matrix
def read_data_(self):
'''
Returns Tensors for images and labels
'''
if not tf.gfile.Exists(self.record_file):
raise IOError('record_file not found: {}'.format(self.record_file))
with tf.variable_scope('InputProducer') as scope:
filename_queue = tf.train.string_input_producer([self.record_file],
num_epochs=None)
reader = tf.TFRecordReader()
key, serialized_example = reader.read(filename_queue)
features = tf.parse_single_example(
serialized_example,
features={
'image_shape': tf.FixedLenFeature([3], tf.int64),
'label_shape': tf.FixedLenFeature([3], tf.int64),
'label': tf.FixedLenFeature([], tf.string),
'image_raw': tf.FixedLenFeature([], tf.string),
})
label = features['label']
label = tf.parse_tensor(label, tf.float32, name=None)
#label = tf.reverse(label,[False,False,True])
label = tf.reshape(
label,
self.in_lab_shape) #tf.cast(features['label_shape'], tf.int32)
image = features['image_raw']
image = tf.parse_tensor(image, tf.float32, name=None)
#image = tf.reverse(image,[False,False,True])
image = tf.reshape(
image,
self.in_img_shape) #tf.cast(features['image_shape'], tf.int32)
return image, label
def _input_fn():
with tf.name_scope('input'):
word_tensor = tf.parse_tensor(
tf.read_file(word_indices_file),
tf.int64
)
targets, contexts = skipgrams(
word_tensor,
num_skips,
skip_window,
batch_size // num_skips,
num_epochs=num_epochs
)
contexts_matrix = tf.expand_dims(contexts, -1)
return targets, contexts_matrix
def mywrite():
# some data
data = ('0x87', ['hello', 'world'], [1.2, 0.8])
tf.serialize_tensor(data[0].encode('utf8'))
_bytes_feature(data[0])
tf.parse_tensor(tf.serialize_tensor(data[1]).numpy(), out_type=tf.string)
tf.parse_tensor(tf.serialize_tensor(data[2]).numpy(), out_type=tf.float32)
sess = tf.Session()
tf.serialize_tensor(data[1]).numpy()
_bytes_feature(tf.serialize_tensor(data[1]).numpy())
tf.train.FeatureList(bytes_list=tf.train.BytesList(
value=[tf.serialize_tensor(data[1])]))
s = exp.SerializeToString()
exp = tf.train.Example.FromString(s)
feature_description = {
'id': tf.FixedLenFeature([], tf.string, default_value=''),
'mutate': tf.FixedLenFeature([], tf.string, default_value=''),
'label': tf.FixedLenFeature([], tf.string, default_value=''),
# 'feature3': tf.FixedLenFeature([], tf.float32, default_value=0.0),
}
tf.parse_single_example(exp[0], feature_description)
tf.parse_single_example(tf.data.Dataset.from_tensor_slices(exp),
feature_description)
with tf.python_io.TFRecordWriter('111.tfrec') as writer:
writer.write(exp.SerializeToString())
def _my_parse_function(pto):
# Parse the input tf.Example proto using the dictionary above.
return tf.parse_single_example(pto, feature_description)
raw = tf.data.TFRecordDataset('111.tfrec')
raw
parsed_image_dataset = raw.map(_my_parse_function)
parsed_image_dataset
for a in parsed_image_dataset:
print(a)
print(a['label'])
# shit!
print(tf.parse_tensor(a['label'], out_type=tf.float32))
feature['label']
tf.parse_tensor(feature['label'], out_type=tf.float32)
def _input_fn():
with tf.name_scope('input'):
index = tf.parse_tensor(tf.read_file(index_file), tf.string)
filename_queue = tf.train.string_input_producer(
filenames, num_epochs=num_epochs)
reader = tf.TFRecordReader(options=tf.python_io.TFRecordOptions(
compression_type=TFRecordCompressionType.GZIP))
_, serialized_example = reader.read(filename_queue)
words = tf.parse_single_example(
serialized_example, {
'target_words': tf.FixedLenFeature([batch_size],
tf.string),
'context_words': tf.FixedLenFeature([batch_size],
tf.string)
})
return {
'targets': tf.expand_dims(words['target_words'], 1),
'index': index
}, words['context_words']
def ls_load_at_timepoint(year, month, day, hour):
ls_file = (("%s/Machine-Learning-experiments/datasets/latent_space/" +
"%04d-%02d-%02d:%02d.tfd") %
(os.getenv('SCRATCH'), year, month, day, hour))
sict = tf.read_file(ls_file)
ls = tf.reshape(tf.parse_tensor(sict, numpy.float32), [1, 100])
result = generator.predict_on_batch(ls)
result = tf.reshape(result, [79, 159, 4])
t2m = dummy_cube()
t2m.data = tf.reshape(result.numpy()[:, :, 0], [79, 159]).numpy()
t2m = unnormalise_t2m(t2m)
prmsl = dummy_cube()
prmsl.data = tf.reshape(result.numpy()[:, :, 1], [79, 159]).numpy()
prmsl = unnormalise_prmsl(prmsl)
u10m = dummy_cube()
u10m.data = tf.reshape(result.numpy()[:, :, 2], [79, 159]).numpy()
u10m = unnormalise_wind(u10m)
v10m = dummy_cube()
v10m.data = tf.reshape(result.numpy()[:, :, 3], [79, 159]).numpy()
v10m = unnormalise_wind(v10m)
return (ls, t2m, prmsl, u10m, v10m)
def _parser_fn(value, parse_type=tf.float32):
return tf.parse_tensor(value, parse_type)
def load_tensor(file_name):
sict = tf.read_file(file_name) # serialised
ict = tf.parse_tensor(sict, numpy.float32)
return ict
def load_tensor(file_name):
sict = tf.read_file(file_name) # serialised
ict = tf.parse_tensor(sict, numpy.float32)
ict = tf.reshape(ict, [91, 180, 1])
return ict
def load_tensor(file_name):
sict = tf.read_file(file_name) # serialised
ict = tf.parse_tensor(sict, numpy.float32)
ict = tf.slice(ict, [0], [1]) # 1st value only
ict = tf.reshape(ict, [1, 1])
return ict
def parse_tfrec(x):
result = tf.parse_tensor(x, out_type=tf.float32)
result = tf.reshape(result, img_shape)
return result
import tensorflow as tf
import numpy as np
from PIL import Image
with open('mnist_200.pb', 'rb') as f:
a = tf.parse_tensor(f.read(), tf.float32)
with tf.Session() as sess:
b = a.eval(session=sess)
b = np.squeeze(b)*256
b = b.astype(np.uint8)
img = Image.fromarray(b, 'L')
img.show()
def parse_into_image(features_dict):
serialized_image = features_dict["image"]
image = tf.parse_tensor(serialized_image, tf.uint8)
label = features_dict["label"]
image = tf.reshape(image, [28, 28])
return dict(images=image), label
def parse(x):
result = tf.parse_tensor(x, out_type=tf.float32)
def _my_parse2(pto):
return {
'id': pto['id'],
'label': tf.parse_tensor(pto['label'], out_type=tf.float32),
'mutate': tf.parse_tensor(pto['mutate'], out_type=tf.string)
}