def build_batch_readout(data_path, batch_size, num_threads, image_size=224, is_training=False):
# Put tfrecord file name (or names) into a list, pass it to a queue
# Here, i don't set num_epochs to get cycled unlimited data.
filename_queue = tf.train.string_input_producer([data_path])
# Define the reader, and read the next record
# Actually, it doesn't really read the next record, just adds one read operation in the graph
reader = tf.TFRecordReader()
_, serialized_example = reader.read(filename_queue)
# parse sequence example
image_path_key = "image_path"
image_data_key = "image_data"
image_label_key = "image_label"
context, sequence = tf.parse_single_sequence_example(
serialized_example,
context_features={
image_path_key: tf.FixedLenFeature([], dtype=tf.string),
image_data_key: tf.FixedLenFeature([], dtype=tf.string),
},
sequence_features={
image_label_key: tf.FixedLenSequenceFeature([], dtype=tf.int64),
})
image_path = context[image_path_key]
encoded_image = context[image_data_key]
image_label = sequence[image_label_key]
# do some log
encoded_image = tf.Print(encoded_image, data=[image_path], message='imgpath: ')
# decode image. must add 'channels=3' to let img tensor has channel dimen info
img = tf.image.decode_jpeg(encoded_image, channels=3)
# can also to image augmentation here
img = tf.image.convert_image_dtype(img, dtype=tf.float32)
resize_height = image_size
resize_width = image_size
img = tf.image.resize_images(img, size=[resize_height, resize_width], method=tf.image.ResizeMethod.BILINEAR)
if is_training:
image_batch, label_batch = tf.train.shuffle_batch(
[img, image_label],
batch_size,
min_after_dequeue=10,
capacity=10 + (num_threads + 1) * batch_size,
num_threads=num_threads,
shapes=[(image_size, image_size, 3), (52,)])
else:
# If dynamic_pad is False, you must ensure that either:
# (i) the shapes argument is passed
# (ii) all of the tensors in tensors must have fully-defined shapes
# Since we have resize image to same size, and label info also have the save size,
# it's OK to set true. Otherwize, we have specify the label count info
image_batch, label_batch = tf.train.batch(
[img, image_label],
batch_size,
num_threads=num_threads,
capacity=batch_size * (num_threads + 1),
dynamic_pad=True)
return [image_batch, label_batch]
def get_padded_batch_v2(file_list,
batch_size,
input_size,
output_size,
num_enqueuing_threads=4,
num_epochs=1,
shuffle=True):
"""Reads batches of SequenceExamples from TFRecords and pads them.
Can deal with variable length SequenceExamples by padding each batch to the
length of the longest sequence with zeros.
Args:
file_list: A list of paths to TFRecord files containing SequenceExamples.
batch_size: The number of SequenceExamples to include in each batch.
input_size: The size of each input vector. The returned batch of inputs
will have a shape [batch_size, num_steps, input_size].
num_enqueuing_threads: The number of threads to use for enqueuing
SequenceExamples.
Returns:
inputs: A tensor of shape [batch_size, num_steps, input_size] of floats32s.
labels: A tensor of shape [batch_size, num_steps] of float32s.
lengths: A tensor of shape [batch_size] of int32s. The lengths of each
SequenceExample before padding.
"""
file_queue = tf.train.string_input_producer(file_list,
num_epochs=num_epochs,
shuffle=shuffle)
reader = tf.TFRecordReader()
_, serialized_example = reader.read(file_queue)
sequence_features = {
'inputs':
tf.FixedLenSequenceFeature(shape=[input_size], dtype=tf.float32),
'inputs_cmvn':
tf.FixedLenSequenceFeature(shape=[input_size], dtype=tf.float32),
'labels1':
tf.FixedLenSequenceFeature(shape=[output_size], dtype=tf.float32),
'labels2':
tf.FixedLenSequenceFeature(shape=[output_size], dtype=tf.float32),
}
_, sequence = tf.parse_single_sequence_example(
serialized_example, sequence_features=sequence_features)
length = tf.shape(sequence['inputs'])[0]
capacity = 1000 + (num_enqueuing_threads + 1) * batch_size
queue = tf.PaddingFIFOQueue(
capacity=capacity,
dtypes=[tf.float32, tf.float32, tf.float32, tf.float32, tf.int32],
shapes=[(None, input_size), (None, input_size), (None, output_size),
(None, output_size), ()])
enqueue_ops = [
queue.enqueue([
sequence['inputs'], sequence['inputs_cmvn'], sequence['labels1'],
sequence['labels2'], length
])
] * num_enqueuing_threads
tf.train.add_queue_runner(tf.train.QueueRunner(queue, enqueue_ops))
return queue.dequeue_many(batch_size)
def read_and_decode(filename_queue):
reader = tf.TFRecordReader()
_, serialized_example = reader.read(filename_queue)
return serialized_example
def _input_fn():
"""Input function compatible with `Experiment` object.
Returns:
A tuple of feature tensors and target tensors.
"""
# TODO(seominjoon): There is bottleneck in data feeding, slow for N >= 128.
filename_queue = tf.train.string_input_producer(filenames,
shuffle=shuffle_files,
num_epochs=num_epochs)
reader = tf.TFRecordReader()
_, se = reader.read(filename_queue)
# TODO(seominjoon): Consider moving data filtering to here.
features_op = tf.parse_single_example(se, features=features)
names = list(features_op.keys())
dtypes = [features_op[name].dtype for name in names]
shapes = [features_op[name].shape for name in names]
if shuffle_examples:
# Data shuffling.
rq = tf.RandomShuffleQueue(queue_capacity,
min_after_dequeue,
dtypes,
names=names)
else:
rq = tf.FIFOQueue(queue_capacity, dtypes, names=names)
enqueue_op = rq.enqueue(features_op)
dequeue_op = rq.dequeue()
dequeue_op = [dequeue_op[name] for name in names]
qr = tf.train.QueueRunner(rq, [enqueue_op])
tf.train.add_queue_runner(qr)
batch = tf.train.batch(dequeue_op,
batch_size,
capacity=queue_capacity,
dynamic_pad=True,
shapes=shapes,
allow_smaller_final_batch=True,
num_threads=5)
batch = {name: each for name, each in zip(names, batch)}
target_keys = [
'word_answer_starts', 'word_answer_ends', 'answers', 'num_answers'
]
# TODO(seominjoon) For cheating-safe, comment out #.
features_batch = {
key: val
for key, val in batch.items() # if key not in target_keys
}
# `metadata['emb_mat`]` contains GloVe embedding, and `xv` in
# `features_batch` index into the vectors.
features_batch['emb_mat'] = tf.constant(emb_mat)
targets_batch = {key: batch[key] for key in target_keys}
# Postprocessing for character data.
# Due to the limitation of the python wrapper for prototxt,
# the characters (by index) need to be flattened when saving on prototxt.
# The following 'unflattens' the character tensor.
actual_batch_size = tf.shape(batch['indexed_context_chars'])[0]
features_batch['indexed_context_chars'] = tf.reshape(
features_batch['indexed_context_chars'],
[actual_batch_size, -1, metadata['num_chars_per_word']])
features_batch['indexed_question_chars'] = tf.reshape(
features_batch['indexed_question_chars'],
[actual_batch_size, -1, metadata['num_chars_per_word']])
# Make sure answer start and end positions are less than sequence lengths.
# TODO(seominjoon) This will need to move to a separate test.
with tf.control_dependencies([
tf.assert_less(
tf.reduce_max(targets_batch['word_answer_starts'], 1),
features_batch['context_num_words'])
]):
targets_batch['word_answer_starts'] = tf.identity(
targets_batch['word_answer_starts'])
with tf.control_dependencies([
tf.assert_less(
tf.reduce_max(targets_batch['word_answer_ends'], 1),
features_batch['context_num_words'])
]):
targets_batch['word_answer_ends'] = tf.identity(
targets_batch['word_answer_ends'])
# Stress test to ensure no OOM for GPU occurs.
if oom_test:
features_batch['indexed_context_words'] = tf.constant(
np.ones([batch_size, exp_metadata['max_context_size']],
dtype='int64'))
features_batch['glove_indexed_context_words'] = tf.constant(
np.ones([batch_size, exp_metadata['max_context_size']],
dtype='int64'))
features_batch['indexed_context_chars'] = tf.constant(
np.ones([
batch_size, exp_metadata['max_context_size'],
exp_metadata['num_chars_per_word']
],
dtype='int64'))
features_batch['indexed_question_words'] = tf.constant(
np.ones([batch_size, exp_metadata['max_ques_size']],
dtype='int64'))
features_batch['glove_indexed_question_words'] = tf.constant(
np.ones([batch_size, exp_metadata['max_ques_size']],
dtype='int64'))
features_batch['indexed_question_chars'] = tf.constant(
np.ones([
batch_size, exp_metadata['max_ques_size'],
exp_metadata['num_chars_per_word']
],
dtype='int64'))
features_batch['question_num_words'] = tf.constant(
np.ones([batch_size], dtype='int64') *
exp_metadata['max_ques_size'])
features_batch['context_num_words'] = tf.constant(
np.ones([batch_size], dtype='int64') *
exp_metadata['max_context_size'])
return features_batch, targets_batch
def __init__(self, filename):
self.batch_size = FLAGS.batch_size
self.image_shape = [224, 224, 3]
self.sample_size = 2
self.num_threads = 1
self.inld_graph = tf.Graph()
with self.inld_graph.as_default():
self.global_step = tf.Variable(1,
name='global_step',
trainable=False)
with tf.name_scope('input'):
self.training_data_files = [filename]
filename_queue = tf.train.string_input_producer(
self.training_data_files)
reader = tf.TFRecordReader()
_, serialized_example = reader.read(filename_queue)
example = tf.parse_single_example(
serialized_example,
features={
'images': tf.FixedLenFeature([self.sample_size],
tf.string),
'gt_index': tf.FixedLenFeature([], tf.int64),
'gt_similarity': tf.FixedLenFeature([], tf.string),
'embedding_caption': tf.FixedLenFeature([], tf.string),
'caption': tf.FixedLenFeature([], tf.string),
})
images_raw = example['images']
gt_index = example['gt_index']
gt_similarity = example['gt_similarity']
query = example['embedding_caption']
caption = example['caption']
[
self.images_raw_batch, self.gt_index_batch,
self.gt_similarity_batch, self.query_batch,
self.caption_batch
] = tf.train.shuffle_batch(
[images_raw, gt_index, gt_similarity, query, caption],
batch_size=self.batch_size,
capacity=self.batch_size * 2 + 1,
num_threads=self.num_threads,
min_after_dequeue=self.batch_size)
self.images = tf.cast(
tf.decode_raw(tf.unstack(self.images_raw_batch), tf.uint8),
tf.float32)
self.gt_similarity_batch = tf.reshape(
tf.decode_raw(tf.unstack(self.gt_similarity_batch, axis=0),
tf.float32),
shape=[self.batch_size, self.sample_size])
self.q = []
for query in tf.unstack(self.query_batch, axis=0):
self.q.append(
tf.reshape(tf.decode_raw(query, tf.float32),
shape=[-1, 300]))
with tf.name_scope('vgg16'):
tf.logging.debug('Creating VGG16 representation layer')
images = tf.reshape(
self.images,
shape=[self.batch_size * self.sample_size] +
self.image_shape)
self.vgg = Vgg16(images)
convolution_features = self.vgg.fc3l
if not FLAGS.excitation_back_propagation:
convolution_features = tf.nn.dropout(
convolution_features, keep_prob=FLAGS.inld_keep_prob)
self.fully_connected_weight = tf.Variable(tf.truncated_normal(
[1000, 300], dtype=tf.float32, stddev=1e-1),
name='weights')
self.fully_connected_bias = tf.Variable(tf.constant(
1.0, shape=[300], dtype=tf.float32),
trainable=True,
name='biases')
self.visual_representations = tf.reshape(
tf.nn.relu(
tf.nn.bias_add(
tf.matmul(convolution_features,
self.fully_connected_weight),
self.fully_connected_bias)),
shape=[self.batch_size, self.sample_size, 300])
with tf.name_scope('lstm'):
tf.logging.debug('Creating caption similarity model')
self.lstm_cell = rnn.BasicLSTMCell(num_units=300,
forget_bias=1.0,
activation=tf.nn.relu)
initial_state = self.lstm_cell.zero_state(self.batch_size,
dtype=tf.float32)
# Define weights
self.lstm_out_weights = tf.Variable(tf.random_normal([300, 1]))
self.lstm_out_bias = tf.Variable(tf.random_normal([1]))
# This should be the similarities between RPN proposals and the given caption
similarity = []
for batch_id in range(self.batch_size):
q = self.q[batch_id]
batch_v = [self.visual_representations[batch_id]]
batch_similarity = []
for individual_visual_representation in tf.unstack(batch_v,
axis=1):
v = tf.concat([individual_visual_representation, q],
axis=0)
v = tf.reshape(v, shape=[1, -1, 300])
self.v = v
outputs, states = tf.nn.dynamic_rnn(self.lstm_cell,
v,
dtype=tf.float32)
self.outputs = outputs
last_output = tf.reshape(outputs[0][-1], shape=[1, -1])
result = tf.matmul(
last_output,
self.lstm_out_weights) + self.lstm_out_bias
self.result = result
batch_similarity.append(result)
similarity.append(tf.stack(batch_similarity))
# Linear activation, using rnn inner loop last output
self.similarities = tf.reshape(
tf.stack(similarity),
shape=[self.batch_size, self.sample_size])
self.sigmoid_similarity = tf.nn.sigmoid(self.similarities)
tf.summary.histogram('similarities', self.similarities)
with tf.name_scope('loss'):
tf.logging.debug('Creating loss, prediction and accuracy')
diff = tf.nn.sigmoid_cross_entropy_with_logits(
labels=self.gt_similarity_batch, logits=self.similarities)
loss = tf.reduce_mean(diff)
tf.summary.scalar('loss', loss)
learning_rate = tf.train.exponential_decay(1E-3,
self.global_step,
4096,
0.96,
staircase=True)
optimizer = tf.train.GradientDescentOptimizer(
learning_rate=learning_rate)
self.train_step = optimizer.minimize(
loss, global_step=self.global_step)
with tf.name_scope('prediction'):
correct_prediction = tf.cast(
tf.equal(tf.argmax(self.similarities, axis=1),
tf.argmax(self.gt_similarity_batch, axis=1)),
tf.float32)
with tf.name_scope('accuracy'):
self.accuracy = tf.reduce_mean(correct_prediction)
tf.summary.scalar('accuracy', self.accuracy)
self.global_initializer = tf.global_variables_initializer()
self.local_initializer = tf.local_variables_initializer()
self.summary_writer = tf.summary.FileWriter(
'summary/train_' + str(datetime.now()), self.inld_graph)
self.merged_summaries = tf.summary.merge_all()
self.saver = tf.train.Saver()
tf.logging.debug('Graph Created.')
def arbitrary_style_image_inputs(style_dataset_file,
batch_size=None,
image_size=None,
center_crop=True,
shuffle=True,
augment_style_images=False,
random_style_image_size=False,
min_rand_image_size=128,
max_rand_image_size=300):
"""Loads a batch of random style image given the path of tfrecord dataset.
This method does not return pre-compute Gram matrices for the images like
style_image_inputs. But it can provide data augmentation. If
augment_style_images is equal to True, then style images will randomly
modified (eg. changes in brightness, hue or saturation) for data
augmentation. If random_style_image_size is set to True then all images
in one batch will be resized to a random size.
Args:
style_dataset_file: str, path to the tfrecord dataset of style files.
batch_size: int. If provided, batches style images. Defaults to None.
image_size: int. The images will be resized bilinearly so that the smallest
side has size image_size. Defaults to None.
center_crop: bool. If True, center-crops to [image_size, image_size].
Defaults to False.
shuffle: bool, whether to shuffle style files at random. Defaults to False.
augment_style_images: bool. Wheather to augment style images or not.
random_style_image_size: bool. If this value is True, then all the style
images in one batch will be resized to a random size between
min_rand_image_size and max_rand_image_size.
min_rand_image_size: int. If random_style_image_size is True, this value
specifies the minimum image size.
max_rand_image_size: int. If random_style_image_size is True, this value
specifies the maximum image size.
Returns:
4-D tensor of shape [1, ?, ?, 3] with values in [0, 1] for the style
image (with random changes for data augmentation if
augment_style_image_size is set to true), and 0-D tensor for the style
label, 4-D tensor of shape [1, ?, ?, 3] with values in [0, 1] for the style
image without random changes for data augmentation.
Raises:
ValueError: if center cropping is requested but no image size is provided,
or if batch size is specified but center-cropping or
augment-style-images is not requested,
or if both augment-style-images and center-cropping are requested.
"""
if center_crop and image_size is None:
raise ValueError('center-cropping requires specifying the image size.')
if center_crop and augment_style_images:
raise ValueError(
'When augment_style_images is true images will be randomly cropped.'
)
if batch_size is not None and not center_crop and not augment_style_images:
raise ValueError(
'batching requires same image sizes (Set center-cropping or '
'augment_style_images to true)')
with tf.name_scope('style_image_processing'):
# Force all input processing onto CPU in order to reserve the GPU for the
# forward inference and back-propagation.
with tf.device('/cpu:0'):
filename_queue = tf.train.string_input_producer(
[style_dataset_file],
shuffle=False,
capacity=1,
name='filename_queue')
if shuffle:
examples_queue = tf.RandomShuffleQueue(
capacity=64,
min_after_dequeue=32,
dtypes=[tf.string],
name='random_examples_queue')
else:
examples_queue = tf.FIFOQueue(capacity=64,
dtypes=[tf.string],
name='fifo_examples_queue')
reader = tf.TFRecordReader()
_, value = reader.read(filename_queue)
enqueue_ops = [examples_queue.enqueue([value])]
tf.train.queue_runner.add_queue_runner(
tf.train.queue_runner.QueueRunner(examples_queue, enqueue_ops))
example_serialized = examples_queue.dequeue()
features = tf.parse_single_example(
example_serialized,
features={
'label': tf.FixedLenFeature([], tf.int64),
'image_raw': tf.FixedLenFeature([], tf.string)
})
image = tf.image.decode_jpeg(features['image_raw'])
image.set_shape([None, None, 3])
label = features['label']
if image_size is not None:
image_channels = image.shape[2].value
if augment_style_images:
image_orig = image
image = tf.image.random_brightness(image, max_delta=0.8)
image = tf.image.random_saturation(image,
lower=0.5,
upper=1.5)
image = tf.image.random_hue(image, max_delta=0.2)
image = tf.image.random_flip_left_right(image)
image = tf.image.random_flip_up_down(image)
random_larger_image_size = random_ops.random_uniform(
[],
minval=image_size + 2,
maxval=image_size + 200,
dtype=dtypes.int32)
image = _aspect_preserving_resize(
image, random_larger_image_size)
image = tf.random_crop(
image, size=[image_size, image_size, image_channels])
image.set_shape([image_size, image_size, image_channels])
image_orig = _aspect_preserving_resize(
image_orig, image_size + 2)
image_orig = _central_crop([image_orig], image_size,
image_size)[0]
image_orig.set_shape([image_size, image_size, 3])
elif center_crop:
image = _aspect_preserving_resize(image, image_size + 2)
image = _central_crop([image], image_size, image_size)[0]
image.set_shape([image_size, image_size, image_channels])
image_orig = image
else:
image = _aspect_preserving_resize(image, image_size)
image_orig = image
image = tf.to_float(image) / 255.0
image_orig = tf.to_float(image_orig) / 255.0
if batch_size is None:
image = tf.expand_dims(image, 0)
else:
[image, image_orig,
label] = tf.train.batch([image, image_orig, label],
batch_size=batch_size)
if random_style_image_size:
# Selects a random size for the style images and resizes all the images
# in the batch to that size.
image = _aspect_preserving_resize(
image,
random_ops.random_uniform([],
minval=min_rand_image_size,
maxval=max_rand_image_size,
dtype=dtypes.int32))
return image, label, image_orig
def read_and_decode(filename_queue=None, img_dims=[256,256,3], model_dims=[224,224,3], size_of_batch=32,\
augmentations_dic=None, num_of_threads=1, shuffle=True):
"""
Reads, decodes and applys augmentations to batch of images
Inputs: filename_queue - Input queue for either train, val or test tfrecords
img_dims - dimensions of input image
model_dims - output dimensions for model
size_of_batch - size of batch of images
augmentations_dic - dictionary with selected augmentations
num_of_threads - number of threads selected
shuffle - option to shuffle batch
Outputs: img - batch of images
t_l - batch of target labels
b_t - batch of bone types
f_p - batch of filepointers
"""
reader = tf.TFRecordReader()
_, serialized_example = reader.read(filename_queue)
features = tf.parse_single_example(serialized_example,
features={
'image_raw':
tf.FixedLenFeature([], tf.string),
'file_path':
tf.FixedLenFeature([], tf.string),
'target_label':
tf.FixedLenFeature([], tf.int64),
})
image = tf.decode_raw(features['image_raw'], tf.uint8)
target_label = tf.cast(features['target_label'], tf.int32)
file_path = tf.cast(features['file_path'], tf.string)
image = tf.reshape(image, img_dims)
image = tf.cast(image, tf.float32)
if augmentations_dic['rand_crop']:
image = tf.random_crop(image, model_dims)
else:
image = tf.image.resize_image_with_crop_or_pad(image, model_dims[0],\
model_dims[1])
if augmentations_dic['rand_flip_left_right']:
image = tf.image.random_flip_left_right(image)
if augmentations_dic['rand_flip_top_bottom']:
image = tf.image.random_flip_up_down(image)
if augmentations_dic['rand_rotate']:
elems = tf.cast(tf.convert_to_tensor(np.deg2rad(np.array(range(360)))),
dtype=tf.float32)
sample = tf.squeeze(
tf.multinomial(tf.log([(1.0 / np.repeat(360, 360)).tolist()]), 1))
random_angle = elems[tf.cast(sample, tf.int32)]
image = tf.contrib.image.rotate(image, random_angle)
if shuffle:
img, t_l, f_p = tf.train.shuffle_batch(
[image, target_label, file_path],
batch_size=size_of_batch,
capacity=1000 + 3 * size_of_batch,
min_after_dequeue=1000,
num_threads=num_of_threads)
else:
img, t_l, f_p = tf.train.batch([image, target_label, file_path],
batch_size=size_of_batch,
capacity=5000,
allow_smaller_final_batch=True,
num_threads=num_of_threads)
if augmentations_dic['warp']:
mean = 0.0
sigma = 1.0
alpha = 6.0
ksize = 128
X = tf.random_uniform([model_dims[0], model_dims[1]]) * 2 - 1
Y = tf.random_uniform([model_dims[0], model_dims[1]]) * 2 - 1
X = tf.reshape(X, [1, model_dims[0], model_dims[1], 1])
Y = tf.reshape(Y, [1, model_dims[0], model_dims[1], 1])
x = tf.linspace(-3.0, 3.0, ksize)
z = ((1.0 / (sigma * tf.sqrt(2.0 * PI))) * tf.exp(
tf.negative(tf.pow(x - mean, 2.0) / (2.0 * tf.pow(sigma, 2.0)))))
z_2d = tf.matmul(tf.reshape(z, [ksize, 1]), tf.reshape(z, [1, ksize]))
z_4d = tf.reshape(z_2d, [ksize, ksize, 1, 1])
X_convolved = tf.nn.conv2d(X,
z_4d,
strides=[1, 1, 1, 1],
padding='SAME')
Y_convolved = tf.nn.conv2d(Y,
z_4d,
strides=[1, 1, 1, 1],
padding='SAME')
X_convolved = (X_convolved / tf.reduce_max(X_convolved)) * alpha
Y_convolved = (Y_convolved / tf.reduce_max(Y_convolved)) * alpha
trans = tf.stack([X_convolved, Y_convolved], axis=-1)
trans = tf.reshape(trans, [-1])
batch_trans = tf.tile(trans, [size_of_batch])
batch_trans = tf.reshape(
batch_trans, [size_of_batch, model_dims[0], model_dims[1], 2])
img = tf.reshape(
img, [size_of_batch, model_dims[0], model_dims[1], model_dims[2]])
img = tf.contrib.image.dense_image_warp(img, batch_trans)
if augmentations_dic['grayscale']:
img = tf.image.rgb_to_grayscale(img)
return img, t_l, f_p
def reader(self):
return tf.TFRecordReader()
def read_tfrecord(path, config, shuffle=True):
"""
A Tfrecord decoder follows the following flow:
TFRecord > Example > Features > Feature > Raw Data
ARGS:
RETURN:
"""
# Tfrecord reader preparation
filename_queue = tf.train.string_input_producer([path], num_epochs=None)
reader = tf.TFRecordReader()
_, serialized_example = reader.read(filename_queue)
# Example Parser
features = tf.parse_single_example(
serialized_example, {
'image/encoded': tf.FixedLenFeature(
(), tf.string, default_value=''),
'image/format': tf.FixedLenFeature(
(), tf.string, default_value='jpg'),
'image/height': tf.FixedLenFeature([], tf.int64),
'image/width': tf.FixedLenFeature([], tf.int64),
'image/object/bbox/y0': tf.VarLenFeature(dtype=tf.float32),
'image/object/bbox/x0': tf.VarLenFeature(dtype=tf.float32),
'image/object/bbox/y1': tf.VarLenFeature(dtype=tf.float32),
'image/object/bbox/x1': tf.VarLenFeature(dtype=tf.float32),
'image/object/bbox/y2': tf.VarLenFeature(dtype=tf.float32),
'image/object/bbox/x2': tf.VarLenFeature(dtype=tf.float32),
'image/object/bbox/y3': tf.VarLenFeature(dtype=tf.float32),
'image/object/bbox/x3': tf.VarLenFeature(dtype=tf.float32),
'image/object/class/label': tf.VarLenFeature(dtype=tf.int64),
'image/object/bbox/cx': tf.VarLenFeature(dtype=tf.float32),
'image/object/bbox/cy': tf.VarLenFeature(dtype=tf.float32),
'image/object/bbox/w': tf.VarLenFeature(dtype=tf.float32),
'image/object/bbox/h': tf.VarLenFeature(dtype=tf.float32),
})
# Parse feature and cast
decode_image = tf.image.decode_jpeg(features['image/encoded'], channels=3)
decode_image = tf.cast(
tf.image.resize_images(decode_image, config.IMAGE_SHAPE), tf.int32)
y0_set = tf.cast(
tf.sparse_tensor_to_dense(features['image/object/bbox/y0']),
tf.float32)
x0_set = tf.cast(
tf.sparse_tensor_to_dense(features['image/object/bbox/x0']),
tf.float32)
y1_set = tf.cast(
tf.sparse_tensor_to_dense(features['image/object/bbox/y1']),
tf.float32)
x1_set = tf.cast(
tf.sparse_tensor_to_dense(features['image/object/bbox/x1']),
tf.float32)
y2_set = tf.cast(
tf.sparse_tensor_to_dense(features['image/object/bbox/y2']),
tf.float32)
x2_set = tf.cast(
tf.sparse_tensor_to_dense(features['image/object/bbox/x2']),
tf.float32)
y3_set = tf.cast(
tf.sparse_tensor_to_dense(features['image/object/bbox/y3']),
tf.float32)
x3_set = tf.cast(
tf.sparse_tensor_to_dense(features['image/object/bbox/x3']),
tf.float32)
cx_set = tf.cast(
tf.sparse_tensor_to_dense(features['image/object/bbox/cx']),
tf.float32)
cy_set = tf.cast(
tf.sparse_tensor_to_dense(features['image/object/bbox/cy']),
tf.float32)
w_set = tf.cast(tf.sparse_tensor_to_dense(features['image/object/bbox/w']),
tf.float32)
h_set = tf.cast(tf.sparse_tensor_to_dense(features['image/object/bbox/h']),
tf.float32)
label = tf.cast(
tf.sparse_tensor_to_dense(features['image/object/class/label']),
tf.float32)
return_value = [
decode_image, x0_set, y0_set, x1_set, y1_set, x2_set, y2_set, x3_set,
y3_set, cx_set, cy_set, w_set, h_set, label
]
if shuffle:
return_value = _shuffle(return_value,
capacity=config.CAPACITY,
min_after_dequeue=config.MIN_AFTER_DEQUEUE,
num_threads=config.NUM_THREADS)
return return_value
def read_from_tfrecord(self, batch_size, source_classes_num,
resample_classes_num, mode):
with tf.name_scope("read_image_from_tfrecord"):
if mode:
files = tf.train.match_filenames_once(
'E:\\shuqian\\resample\\tfrecord\\train_image.tfrecords*')
else:
files = tf.train.match_filenames_once(
'E:\\shuqian\\resample\\tfrecord\\test_image.tfrecords*')
# read into queue
filename_queue = tf.train.string_input_producer(files)
reader = tf.TFRecordReader()
_, serialized_example = reader.read(filename_queue)
# get images and labels
features = tf.parse_single_example(
serialized_example,
features={
'source_label': tf.FixedLenFeature([], tf.int64),
'resample_label': tf.FixedLenFeature([], tf.int64),
'img_raw': tf.FixedLenFeature([], tf.string)
})
# tf.decode_raw can transform strings to array
image = tf.decode_raw(features['img_raw'], tf.uint8)
image = tf.reshape(image, [128, 128, 3])
image = tf.cast(image, tf.float32)
image = tf.image.per_image_standardization(image)
source_label = tf.cast(features['source_label'], tf.int32)
resample_label = tf.cast(features['resample_label'], tf.int32)
if mode:
example_queue = tf.FIFOQueue(
# capacity of this queue
capacity=5 * batch_size,
dtypes=[tf.float32, tf.int32, tf.int32],
# image size and label size
shapes=[[128, 128, 3], (), ()])
# 读线程的数量
num_threads = 4
else:
example_queue = tf.FIFOQueue(
capacity=3 * batch_size,
dtypes=[tf.float32, tf.int32, tf.int32],
shapes=[[128, 128, 3], (), ()])
num_threads = 2
# enqueue data
example_enqueue_op = example_queue.enqueue(
[image, source_label, resample_label])
# queue runner
tf.train.add_queue_runner(
tf.train.queue_runner.QueueRunner(
example_queue, [example_enqueue_op] * num_threads))
# dequeue data
images, source_labels, resample_labels = example_queue.dequeue_many(
batch_size)
# transform labels from [1, batch_size] to [class_num, batch_size]
source_labels = tf.reshape(source_labels, [batch_size, 1])
source_indices = tf.reshape(tf.range(0, batch_size, 1),
[batch_size, 1])
source_labels = tf.sparse_to_dense(
tf.concat(values=[source_indices, source_labels], axis=1),
[batch_size, source_classes_num], 1.0, 0.0)
resample_labels = tf.reshape(resample_labels, [batch_size, 1])
resample_indices = tf.reshape(tf.range(0, batch_size, 1),
[batch_size, 1])
resample_labels = tf.sparse_to_dense(
tf.concat(values=[resample_indices, resample_labels], axis=1),
[batch_size, resample_classes_num], 1.0, 0.0)
# check data dim
assert len(images.get_shape()) == 4
assert images.get_shape()[0] == batch_size
assert images.get_shape()[-1] == 3
assert len(source_labels.get_shape()) == 2
assert source_labels.get_shape()[0] == batch_size
assert source_labels.get_shape()[1] == source_classes_num
self.images = images
self.source_labels = source_labels
self.resample_labels = resample_labels
def _image_batch(image_list, label_list, size_list, batch_size=1):
image_queue, label_queue, size_queue = tf.train.slice_input_producer(
[image_list, label_list, size_list])
image_batch, label_batch, size_batch = tf.train.batch()
reader = tf.TFRecordReader()
reader.read()
def build_input(data_path, batch_size, resize, mode=None):
#读取一个文件夹下匹配的文件
files = tf.train.match_filenames_once(data_path)
#把文件放入文件队列中
filename_queue = tf.train.string_input_producer(files, shuffle=True)
#创建一个reader
reader = tf.TFRecordReader()
#从文件中读取一个样例。也可以使用read_up_to函数一次性读取多个样例
_, serialized_example = reader.read(filename_queue)
#解析一个样本
features = tf.parse_single_example(serialized_example,
features={
'height':
tf.FixedLenFeature([], tf.int64),
'width':
tf.FixedLenFeature([], tf.int64),
'channel':
tf.FixedLenFeature([], tf.int64),
'label':
tf.FixedLenFeature([], tf.int64),
'image_raw':
tf.FixedLenFeature([], tf.string),
})
# 组合样例中队列最多可以存储的样例个数
capacity = 500 + 3 * batch_size
image, label = features['image_raw'], features['label']
height, width, channel = features['height'], features['width'], features[
'channel']
#原始图像数据解析出像素矩阵，并根据图像尺寸还原图像
decoded_images = tf.decode_raw(image, tf.uint8)
retyped_images = tf.cast(decoded_images, tf.float32)
retyped_height = tf.cast(height, tf.int32)
retyped_width = tf.cast(width, tf.int32)
retyped_channel = tf.cast(channel, tf.int32)
labels = tf.cast(label, tf.int32)
if mode == 'float':
#把原始像素矩阵[0,255]转化为实数类型像素矩阵[0,1]
# tf.reshape与set_shape的区别
# decoded_images.set_shape([retyped_height,retyped_width ,retyped_channel])
reshaped_images = tf.reshape(retyped_images,
[retyped_height, retyped_width, 3])
distorted_image = preprocess_for_train(reshaped_images, resize, resize,
None)
# 组合样例两种方法一种是tf.train.batch;另一种是tf.train.shuffle_batch，输入的shape一定要明确
example_batch, label_batch = tf.train.shuffle_batch(
[distorted_image, labels],
batch_size=batch_size,
capacity=capacity,
min_after_dequeue=500)
else:
# 把原始像素矩阵[0,255]
reshaped_images = tf.reshape(retyped_images,
[retyped_height, retyped_width, 3])
distorted_image = tf.image.resize_images(reshaped_images,
[resize, resize],
method=0)
# 组合样例两种方法一种是tf.train.batch;另一种是tf.train.shuffle_batch，输入的shape一定要明确
example_batch, label_batch = tf.train.shuffle_batch(
[distorted_image, labels],
batch_size=batch_size,
capacity=capacity,
min_after_dequeue=500)
return example_batch, label_batch
def main():
########################
# Parameters #
########################
# Declare argument-parser
parser = argparse.ArgumentParser()
parser.add_argument('--architecture', type=str, default='alexnet', help='Name of the network-architecture to use')
parser.add_argument('--source_task_dataset', type=str, default='ilsvrc_half', help='Name of the dataset used for the pre-trained network')
parser.add_argument('--source_task_SP', type=str, nargs='+', default=['_S', '_S_GroupingCategoricalLevels'], help='Name of the source-problem (SP) used for the pre-trained network')
parser.add_argument('--model_iter', type=int, default=500000, help='Number of iterations on which the source-task model was saved')
parser.add_argument('--target_task_dataset', type=str, nargs='+', default=['voc07', 'voc12'], help='Name of the target-datasets on which we want to extract features')
parser.add_argument('--target_task_phase', type=str, nargs='+', default=['train', 'test'], help='Name of the target-datasets on which we want to extract features')
parser.add_argument('--layer2extract', type=str, default='fc_7', help='Name of the layer of the network to extract')
parser.add_argument('--gpu', type=int, default=0, help='GPU device on which are declared the variables')
# TODO:
#source_dataset > source_task_dataset
#source_dataset_name > source_task_SP
#weights_iter > model_iter
#target_dataset > target_task_dataset
#phase > target_task_phase
#layer_to_extract > layer2extract
#gpu
args = parser.parse_args()
# Get the network-architecture from the arguments (if no argument, alexnet is used by default)
if args.architecture=="alexnet":
from architecture_alexnet_forTraining import NET
elif args.architecture=="darknet":
from architecture_darknet_forTraining import NET
print("architecture = "+str(args.architecture))
# Parameters of pre-trained nework
source_task_dataset = args.source_task_dataset #'ilsvrc_half'
source_task_SP = args.source_task_SP #'_S_GroupingCategoricalLevels' #'_S_GroupingCategoricalLevels' #'_S_GroupingHierarchicalLevels5' #'_S_Adding100' #'_S_GroupingCategoricalLevels' #'_S_Removing241' #'_S' #'_S_GroupingHierarchicalLevels7'
source_dataset_name_spec = '_RandomCropAndFlip'
training_strategy = '' #'_FS' FS means training From Scratch #'_FineTuning_FC78_2k'
weights_iter = 500000
batch_size = 1
with tf.device('/gpu:'+str(args.gpu)):
layer_to_extract = 'fc_7'
nbr_classes = 577
data_transformation = 1
# Declare network (AlexNet)
network = NET(name_layer2extract=layer_to_extract, output_size=nbr_classes)
variable_to_restore = tf.global_variables() # for finetuning on same database
saver4restoring = tf.train.Saver(variable_to_restore, max_to_keep=None) # saver for restoring variable_to_restore
global_step = tf.get_variable('global_step', [], initializer=tf.constant_initializer(0), trainable=False)
for target_dataset in ['voc07', 'voc12', 'mit67', 'ca101', 'ca256', 'cub200', 'nwOB', 'stACT', 'stCAR', 'fl102']: #['stACT', 'fl102', 'stCAR', 'voc12', 'ca256']: #['voc07', 'mit67', 'ca101', 'ca256', 'cub200', 'nwOB', 'voc12', 'stCAR']: # ilsvrc_half
for phase in ['train', 'test']:
# TODO: loop for source_task_SP: for source_task_SP in args.source_task_SP:
print ('Extracting features for '+phase+' phase of '+target_dataset+' dataset')
#######################
# Inferred Parameters #
#######################
# Paths
path_to_source_data = '/scratch_global/youssef/expes/gs_complementarity/transfer_learning/' + source_dataset +'/'
path_to_target_data = '/scratch_global/youssef/expes/gs_complementarity/transfer_learning/' + target_dataset +'/'
weights_file = path_to_source_data + 'modelss'+source_dataset_name+source_dataset_name_spec+'/model.'+source_dataset+source_dataset_name+'.config_#classes='+str(nbr_classes)+'_imgSize=227_imgSpec='+source_dataset_name_spec[1:]+training_strategy+'.ckpt-'
weights_file += str(weights_iter)
output_file_name = path_to_target_data + 'features_TF/'+target_dataset+'.'+phase+'.FS.'+layer_to_extract+'.cnn'+source_dataset_name+source_dataset_name_spec+training_strategy+'.alexnet_full.'+str(int(weights_iter/1000))+'kIter.txt'
# Amount of images in target-dataset
list_filename = path_to_target_data + target_dataset + '.' + phase + '.lst'
with open(list_filename, 'r') as list_file:
list_image = [x.strip() for x in list_file.readlines()]
nbr_total_img = int(len(list_image))
#######################
# Reading mean-values #
#######################
mean_filename = path_to_source_data+ 'mean_images/mean_image.'+ source_dataset + source_dataset_name +'.txt'
with open(mean_filename, 'r') as mean_file:
mean_image = [x.strip() for x in mean_file.readlines()]
###########################
# Reading tf-records file #
###########################
tfrecords_filename = path_to_target_data + 'tfrecord_files/tfrecord_files'+ target_dataset + '.' + phase + '.bin'
filename_queue = tf.train.string_input_producer([tfrecords_filename]) #, num_epochs=1) #shuffle=False)
feature = {
'height': tf.FixedLenFeature([], tf.int64, default_value=-1),
'width': tf.FixedLenFeature([], tf.int64, default_value=-1),
'image_raw': tf.FixedLenFeature((), tf.string, default_value='')
}
reader = tf.TFRecordReader()
_, serialized_example = reader.read(filename_queue)
features = tf.parse_single_example(serialized_example, features=feature)
image = tf.image.decode_jpeg(features['image_raw'], channels=3)
image = tf.image.resize_images (image, [256, 256])
if data_transformation == 1:
# random crop and flip
image = tf.random_crop(image, [227, 227, 3])
image = tf.image.random_flip_left_right(image)
else:
# center crop
image = tf.image.resize_image_with_crop_or_pad(image, 227, 227)
mean = tf.constant([float(mean_image[0]), float(mean_image[1]), float(mean_image[2])], dtype=tf.float32)
mean = tf.reshape(mean, [1, 1, 3])
image = image - mean
images = image #tf.train.batch([image], batch_size=1, capacity=30, num_threads=1)
with tf.device('/gpu:'+str(args.gpu)):
######################
# Declare parameters #
######################
"""
variable_to_restore = tf.global_variables() # for finetuning on same database
saver4restoring = tf.train.Saver(variable_to_restore, max_to_keep=None) # saver for restoring variable_to_restore
global_step = tf.get_variable('global_step', [], initializer=tf.constant_initializer(0), trainable=False)
"""
with tf.device('/gpu:'+str(args.gpu)):
#global_step = tf.get_variable('global_step', [], initializer=tf.constant_initializer(0), trainable=False)
sess = tf.InteractiveSession()
tf.global_variables_initializer().run()
# Restoring weights from pre-trained model
if weights_file is not None:
print('\nRestoring weights from: ' + weights_file + '\n\n')
saver4restoring.restore(sess, weights_file)
##############
# Extraction #
##############
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(coord=coord)
output_file = open(output_file_name, 'w')
print('Total images: ' + str(nbr_total_img))
nbr_batches = nbr_total_img
for step in range(nbr_batches):
# Display progress
if step%1000 == 0:
print(str(step)+'/'+str(nbr_batches))
# Extraction
test_images = sess.run([images])
feed_dict_test = {network.images: test_images, network.keep_prob: 1.0}
output_descriptor = np.asarray(sess.run(network.logits, feed_dict=feed_dict_test))
#print(output_descriptor.shape)
#####################################
# Writing descriptors in ASCII file #
#####################################
# convolutional/pooling features (TODO: flatten of matrices)
# fully-connected features (vectors)
for i in range(0, output_descriptor.shape[1]):
if str(output_descriptor[0, i]) == '0.0':
output_file.write('0 ')
else:
output_file.write(str(output_descriptor[0, i])+' ')
output_file.write('\n')
output_file.close()
# Stop the threads
coord.request_stop()
# Wait for threads to stop
coord.join(threads)
sess.close()
def main(_):
with tf.Session() as sess:
epochs = 1
batch_size = 5
reader = tf.TFRecordReader()
read_fn = lambda name : read_record(name, reader)
h, w, crop_shape, train_images = get_pipeline('2017_train_small_anys.tfrecords', batch_size, epochs, read_fn)
read_fn_sc_def = lambda name, sc : read_record_2(name, reader, sc)
read_fn_sc = lambda name : read_fn_sc_def(name, 9)
h2, w2, crop_shape2, train_images_2 = get_pipeline('2017_train_small_anys.tfrecords', batch_size, epochs, read_fn_sc)
read_fn_sc = lambda name : read_fn_sc_def(name, 8)
h3, w3, crop_shape3, train_images_3 = get_pipeline('2017_train_small_anys.tfrecords', batch_size, epochs, read_fn_sc)
read_fn_sc = lambda name : read_fn_sc_def(name, 7)
h4, w4, crop_shape4, train_images_4 = get_pipeline('2017_train_small_anys.tfrecords', batch_size, epochs, read_fn_sc)
read_fn_sc = lambda name : read_fn_sc_def(name, 6)
h5, w5, crop_shape5, train_images_5 = get_pipeline('2017_train_small_anys.tfrecords', batch_size, epochs, read_fn_sc)
# read_fn_sc = lambda name : read_fn_sc_def(name, 5)
# h6, w6, crop_shape6, train_images_6 = get_pipeline('2017_train_small_anys.tfrecords', batch_size, epochs, read_fn_sc)
# read_fn_sc = lambda name : read_fn_sc_def(name, 4)
# h7, w7, crop_shape7, train_images_7 = get_pipeline('2017_train_small_anys.tfrecords', batch_size, epochs, read_fn_sc)
# read_fn_sc = lambda name : read_fn_sc_def(name, 3)
# h8, w8, crop_shape8, train_images_8 = get_pipeline('2017_train_small_anys.tfrecords', batch_size, epochs, read_fn_sc)
# rec_loss_f1_f2 = tf.reduce_mean(tf.square(train_images - train_images_2))
# rec_loss_f1_f3 = tf.reduce_mean(tf.square(train_images - train_images_3))
# rec_loss_f1_f4 = tf.reduce_mean(tf.square(train_images - train_images_4))
# rec_loss_f1_f5 = tf.reduce_mean(tf.square(train_images - train_images_5))
I1_f_4 = encoder(train_images, batch_size)
tf.get_variable_scope().reuse_variables()
i2_f_4 = encoder(train_images_2, batch_size)
i3_f_4 = encoder(train_images_3, batch_size)
i4_f_4 = encoder(train_images_4, batch_size)
i5_f_4 = encoder(train_images_5, batch_size)
rec_loss_f4_f2 = tf.reduce_mean(tf.square(I1_f_4 - i2_f_4), 1)
rec_loss_f4_f3 = tf.reduce_mean(tf.square(I1_f_4 - i3_f_4), 1)
rec_loss_f4_f4 = tf.reduce_mean(tf.square(I1_f_4 - i4_f_4), 1)
rec_loss_f4_f5 = tf.reduce_mean(tf.square(I1_f_4 - i5_f_4), 1)
all = tf.stack(axis=0, values=[rec_loss_f4_f2, rec_loss_f4_f3, rec_loss_f4_f4, rec_loss_f4_f5])
print(all)
assert all.shape[0] == 4
assert all.shape[1] == batch_size
ind = tf.argmin(all, axis=0)
print(ind)
# assert ind.shape[0] == batch_size
all_tile4 = tf.concat([train_images_2, train_images_3, train_images_4, train_images_5], axis=0)
all_f4 = tf.concat([i2_f_4, i3_f_4, i4_f_4, i5_f_4], axis=0)
# print('all_f4: ', all_f4.shape)
for i in range(batch_size):
# choose which batch i holds the L2-closest tile (i.e. w/ lowest L2 loss), then choose that tile at position i
tile = all_tile4[ind[i] * batch_size:(ind[i] + 1) * batch_size][i]
tile = tf.expand_dims(tile, 0)
# print('tile: ', tile.shape)
J_4_tile = tile if i == 0 else tf.concat(axis=0, values=[J_4_tile, tile])
f = all_f4[ind[i] * batch_size:(ind[i] + 1) * batch_size][i]
f = tf.expand_dims(f, 0)
# print('f: ', f.shape)
J_4_f = f if i == 0 else tf.concat(axis=0, values=[J_4_f, f])
# print('J_4_f: ', J_4_f.shape)
# #square = tf.sum(tf.square(train_images - train_images_2))
# kd = tf.reduce_mean(tf.square(train_images - train_images_2), 1)
# # print(kd.shape)
# kd = tf.reduce_mean(tf.square(train_images - train_images_2), 1, keep_dims=True)
# # print(kd.shape)
# kr = tf.reduce_mean(tf.square(train_images - train_images_2), [1,2,3])
#
# all_img = tf.concat([train_images_2, train_images_3, train_images_4, train_images_5], axis=0)
# # print(all_img.shape)
# all = tf.stack([rec_loss_f1_f2, rec_loss_f1_f3, rec_loss_f1_f4, rec_loss_f1_f5])
# argmin = tf.argmin(all, axis=0)
#
# final_img = all_img[argmin * batch_size:(argmin + 1) * batch_size]
#
# s1 = tf.reduce_sum(tf.square(train_images) )
# s2 = tf.reduce_sum(tf.square(train_images_2))
# s3 = tf.reduce_sum(tf.square(train_images_3))
# s4 = tf.reduce_sum(tf.square(train_images_4))
# s5 = tf.reduce_sum(tf.square(train_images_5))
# s6 = tf.reduce_sum(tf.square(train_images_6))
# s7 = tf.reduce_sum(tf.square(train_images_7))
# s8 = tf.reduce_sum(tf.square(train_images_8))
# f = tf.reduce_sum( tf.square(final_img ))
# imgs_per_group = int(2/2)
# group = lambda i: train_images[(i * imgs_per_group):imgs_per_group + (i * imgs_per_group), :, :, :]
# images_i1 = group(0)
# images_i2 = group(1)
# for _ in range(3):
# print('images_i1.shape[1]: ' + images_i1.shape[1])
# print('images_i1.shape[2]: ' + images_i1.shape[2])
# size = tf.minimum(images_i1.shape[1], images_i1.shape[2])
# print('i1: %s -> %s' % (images_i1.shape, size.shape))
# crop_shape = tf.parallel_stack([size, size, 3])
# crop = tf.random_crop(images_i1, crop_shape)
# print(type(crop))
# print(crop.shape)
######################################################################3
sess.run(tf.global_variables_initializer())
sess.run(tf.local_variables_initializer())
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess, coord=coord)
try:
while not coord.should_stop():
#hr, wr, i1, i2 = sess.run([h, w, images_i1, images_i2])
# rs1,rs2,rs3,rs4,rs5,rs6,rs7,rs8,rf \
# , am \
# , rec_loss_f1_f2_, rec_loss_f1_f3_, rec_loss_f1_f4_, rec_loss_f1_f5_ \
# , tr, tr_2, tr_3, tr_4, tr_5, fi = sess.run([
# s1,s2,s3,s4,s5,s6,s7,s8,f \
# , argmin \
# , rec_loss_f1_f2, rec_loss_f1_f3, rec_loss_f1_f4, rec_loss_f1_f5 \
# , train_images, train_images_2, train_images_3, train_images_4, train_images_5, final_img])
# ind,r1, r2, r3, r4,r5,f= sess.run([ind,train_images, train_images_2,train_images_3,train_images_4,train_images_5, J_4_tile])
#
# print(ind)
# print('save 1')
# for i in range(batch_size):
# fn = 'train_images_%d.jpeg' % i
# scipy.misc.imsave(fn, r1[i])
# print('save 2')
# for i in range(batch_size):
# scipy.misc.imsave('train_images_2_%d.jpeg' % i, r2[i])
# print('save 3')
# for i in range(batch_size):
# scipy.misc.imsave('train_images_3_%d.jpeg' % i, r3[i])
# print('save 4')
# for i in range(batch_size):
# scipy.misc.imsave('train_images_4_%d.jpeg' % i, r4[i])
# print('save 5')
# for i in range(batch_size):
# scipy.misc.imsave('train_images_5_%d.jpeg' % i, r5[i])
# print('save 6')
# print(f.shape)
# for i in range(batch_size):
# scipy.misc.imsave('J_4_tile_%d.jpeg' % i, f[i])
print('--------------------------------------')
ind,l1,l2,l3,l4,l5,l6= sess.run([ind,I1_f_4,i2_f_4, i3_f_4,i4_f_4,i5_f_4,J_4_f])
print(ind)
print('i1_f_4:>>>>>')
print(str(np.linalg.norm(l1[0])))
print(str(np.linalg.norm(l1[1])))
print(str(np.linalg.norm(l1[2])))
print(str(np.linalg.norm(l1[3])))
print(str(np.linalg.norm(l1[4])))
print('i1_f_4.<<<<<')
print('i2_f_4:>>>>>')
print(str(np.linalg.norm(l2[0])))
print(str(np.linalg.norm(l2[1])))
print(str(np.linalg.norm(l2[2])))
print(str(np.linalg.norm(l2[3])))
print(str(np.linalg.norm(l2[4])))
print('i2_f_4.<<<<<')
print('i3_f_4:>>>>>')
print(str(np.linalg.norm(l3[0])))
print(str(np.linalg.norm(l3[1])))
print(str(np.linalg.norm(l3[2])))
print(str(np.linalg.norm(l3[3])))
print(str(np.linalg.norm(l3[4])))
print('i3_f_4.<<<<<')
print('i4_f_4:>>>>>')
print(str(np.linalg.norm(l4[0])))
print(str(np.linalg.norm(l4[1])))
print(str(np.linalg.norm(l4[2])))
print(str(np.linalg.norm(l4[3])))
print(str(np.linalg.norm(l4[4])))
print('i4_f_4.<<<<<')
print('i5_f_4:>>>>>')
print(str(np.linalg.norm(l5[0])))
print(str(np.linalg.norm(l5[1])))
print(str(np.linalg.norm(l5[2])))
print(str(np.linalg.norm(l5[3])))
print(str(np.linalg.norm(l5[4])))
print('i5_f_4.<<<<<')
print('J_4_f:>>>>>')
print(str(np.linalg.norm(l6[0])))
print(str(np.linalg.norm(l6[1])))
print(str(np.linalg.norm(l6[2])))
print(str(np.linalg.norm(l6[3])))
print(str(np.linalg.norm(l6[4])))
print('J_4_f.<<<<<')
# for i in range(batch_size):
# print(l6[i])
break
# print('22222222222222222222222222222222')
# print('hr: %s' % str(hr))
# print('wr: %s' % str(wr))
# print('c:')
# print(c)
# print('22222222222222222222222222222222')
# print('hr2: %s' % str(hr2))
# print('wr2: %s' % str(wr2))
# print('c2:')
# print(c2)
# print('22222222222222222222222222222222')
# print('hr3: %s' % str(hr3))
# print('wr3: %s' % str(wr3))
# print('c3:')
# print(c3)
# print('22222222222222222222222222222222')
# print('hr4: %s' % str(hr4))
# print('wr4: %s' % str(wr4))
# print('c4:')
# print(c4)
# print('22222222222222222222222222222222')
# print('hr5: %s' % str(hr5))
# print('wr5: %s' % str(wr5))
# print('c5:')
# print(c5)
# print('22222222222222222222222222222222')
# print('hr6: %s' % str(hr6))
# print('wr6: %s' % str(wr6))
# print('c6:')
# print(c6)
# print('22222222222222222222222222222222')
# print('hr7: %s' % str(hr7))
# print('wr7: %s' % str(wr7))
# print('c7:')
# print(c7)
# print('22222222222222222222222222222222')
# print('hr8: %s' % str(hr8))
# print('wr8: %s' % str(wr8))
# print('c8:')
# print(c8)
print('--------------------------------------------')
# print('argmin: %s, %s, %s, %s, %s' % (str(am), str(rec_loss_f1_f2_), str(rec_loss_f1_f3_), str(rec_loss_f1_f4_), str(rec_loss_f1_f5_)))
# print('img: %s, %s, %s, %s | %s %s' % (str(np.linalg.norm(tr_2 - tr)), str(np.linalg.norm(tr_3 - tr)),
# str(np.linalg.norm(tr_4 - tr)), str(np.linalg.norm(tr_5 - tr)),
# str(np.linalg.norm(fi - tr)), str(np.linalg.norm(fi - tr))))
# print('sum: %d | %d %d %d %d %d %d %d | %d' %(rs1,rs2,rs3,rs4,rs5,rs6,rs7,rs8,rf))
print('--------------------------------------------')
except Exception as e:
if hasattr(e, 'message') and 'is closed and has insufficient elements' in e.message:
print('Done training -- epoch limit reached')
else:
print('Exception here, ending training..')
print('>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>')
print(e)
print('<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<')
finally:
# When done, ask the threads to stop.
coord.request_stop()
coord.join(threads)
def _ReadBlockExample(data_filepattern, train_samples, hierarchy_level,
is_base_level, stored_dim_block, stored_height_block,
shuffle, num_epochs):
"""Deserializes train data.
Args:
data_filepattern: A list of data file patterns.
train_samples: Train on previous model predictions.
hierarchy_level: hierarchy level (1 is finest).
is_base_level: Whether there are no previous hierarchy levels.
stored_dim_block: Stored data x/z dimension (high-resolution).
stored_height_block: Stored data y dimension (high-resolution).
shuffle: Whether to shuffle.
num_epochs: Number of data epochs.
Returns:
key, value.
"""
if not isinstance(data_filepattern, list):
data_filepattern = [data_filepattern]
# Get filenames matching filespec.
tf.logging.info('data_filepattern: %s', data_filepattern)
filenames = []
for p in data_filepattern:
filenames.extend(tf.gfile.Glob(p))
tf.logging.info('filenames: %s', filenames)
# Create filename queue.
filename_queue = tf.train.string_input_producer(filenames,
num_epochs=num_epochs,
shuffle=shuffle)
# Read sequence examples.
reader = tf.TFRecordReader()
key, serialized_example = reader.read(filename_queue)
samples_lo = None
samples_sem_lo = None
if train_samples and not is_base_level:
key_samples = 'samples_' + _RESOLUTIONS[hierarchy_level]
key_samples_sem = 'sem_samples_' + _RESOLUTIONS[hierarchy_level]
spec = {
'data':
tf.FixedLenFeature((), tf.string),
key_samples:
tf.FixedLenFeature(
(stored_dim_block // 2, stored_height_block // 2,
stored_dim_block // 2, 1), tf.float32),
key_samples_sem:
tf.FixedLenFeature((), tf.string)
}
example = tf.parse_single_example(serialized_example, spec)
samples_lo = example[key_samples]
samples_sem_lo = example[key_samples_sem]
serialized_example = example['data']
# Parse sequence example.
key_input = _RESOLUTIONS[hierarchy_level - 1] + '_' + _INPUT_FEATURE
key_target = _RESOLUTIONS[hierarchy_level - 1] + '_' + _TARGET_FEATURE
key_target_sem = _RESOLUTIONS[hierarchy_level -
1] + '_' + _TARGET_SEM_FEATURE
#key_input = 'input_sdf'
#key_target = 'target_df'
#key_target_sem = 'target_sem'
sequence_features_spec = {
key_input:
tf.FixedLenFeature(
(stored_dim_block, stored_height_block, stored_dim_block, 1),
tf.float32),
key_target:
tf.FixedLenFeature(
(stored_dim_block, stored_height_block, stored_dim_block, 1),
tf.float32),
key_target_sem:
tf.FixedLenFeature((), tf.string)
}
if not is_base_level:
key_target_lo = _RESOLUTIONS[hierarchy_level] + '_' + _TARGET_FEATURE
sequence_features_spec[key_target_lo] = tf.FixedLenFeature(
(stored_dim_block // 2, stored_height_block // 2,
stored_dim_block // 2, 1), tf.float32)
key_target_sem_lo = (_RESOLUTIONS[hierarchy_level] + '_' +
_TARGET_SEM_FEATURE)
sequence_features_spec[key_target_sem_lo] = tf.FixedLenFeature(
(), tf.string)
example = tf.parse_single_example(serialized_example,
sequence_features_spec)
return key, example, samples_lo, samples_sem_lo
# batch_size = tf.placeholder(tf.int32, [])
# 每个时刻的输入特征是1维的，就是每个时刻输入sqlLength个值
input_size = sqlLength
# 时序持续长度为1
timestep_size = 1
# 每个隐含层的节点数
hidden_size = 256
# LSTM/GRU layer 的层数
layer_num = 3
############################获取数据部分
# 读取文件生成队列
filename_queue1 = tf.train.string_input_producer([sql_filepath], shuffle=False, num_epochs=None)
filename_queue2 = tf.train.string_input_producer([normal_filepath], shuffle=False, num_epochs=None)
# 生成reader流
reader1 = tf.TFRecordReader()
reader2 = tf.TFRecordReader()
_, serialized_example1 = reader1.read(filename_queue1)
_, serialized_example2 = reader2.read(filename_queue2)
# get feature from serialized example
features1 = tf.parse_single_example(serialized_example1,
features={
'data': tf.FixedLenFeature([], tf.string),
'label': tf.FixedLenFeature([], tf.int64)
}
)
features2 = tf.parse_single_example(serialized_example2,
features={
'data': tf.FixedLenFeature([], tf.string),
'label': tf.FixedLenFeature([], tf.int64)
}
def style_image_inputs(style_dataset_file,
batch_size=None,
image_size=None,
square_crop=False,
shuffle=True):
"""Loads a batch of random style image given the path of tfrecord dataset.
Args:
style_dataset_file: str, path to the tfrecord dataset of style files.
The dataset is produced via the create_style_dataset.py script and is
made of Example protobufs with the following features:
* 'image_raw': byte encoding of the JPEG string of the style image.
* 'label': integer identifier of the style image in [0, N - 1], where
N is the number of examples in the dataset.
* 'vgg_16/<LAYER_NAME>': Gram matrix at layer <LAYER_NAME> of the VGG-16
network (<LAYER_NAME> in {conv,pool}{1,2,3,4,5}) for the style
image.
batch_size: int. If provided, batches style images. Defaults to None.
image_size: int. The images will be resized bilinearly so that the smallest
side has size image_size. Defaults to None.
square_crop: bool. If True, square-crops to [image_size, image_size].
Defaults to False.
shuffle: bool, whether to shuffle style files at random. Defaults to True.
Returns:
If batch_size is defined, a 4-D tensor of shape [batch_size, ?, ?, 3] with
values in [0, 1] for the style image, and 1-D tensor for the style label.
Raises:
ValueError: if center cropping is requested but no image size is provided,
or if batch size is specified but center-cropping is not requested.
"""
vgg_layers = [
'vgg_16/conv1', 'vgg_16/pool1', 'vgg_16/conv2', 'vgg_16/pool2',
'vgg_16/conv3', 'vgg_16/pool3', 'vgg_16/conv4', 'vgg_16/pool4',
'vgg_16/conv5', 'vgg_16/pool5'
]
if square_crop and image_size is None:
raise ValueError('center-cropping requires specifying the image size.')
if batch_size is not None and not square_crop:
raise ValueError('batching requires center-cropping.')
with tf.name_scope('style_image_processing'):
filename_queue = tf.train.string_input_producer([style_dataset_file],
shuffle=False,
capacity=1,
name='filename_queue')
if shuffle:
examples_queue = tf.RandomShuffleQueue(
capacity=64,
min_after_dequeue=32,
dtypes=[tf.string],
name='random_examples_queue')
else:
examples_queue = tf.FIFOQueue(capacity=64,
dtypes=[tf.string],
name='fifo_examples_queue')
reader = tf.TFRecordReader()
_, value = reader.read(filename_queue)
enqueue_ops = [examples_queue.enqueue([value])]
tf.train.queue_runner.add_queue_runner(
tf.train.queue_runner.QueueRunner(examples_queue, enqueue_ops))
example_serialized = examples_queue.dequeue()
features = tf.parse_single_example(
example_serialized,
features={
'label': tf.FixedLenFeature([], tf.int64),
'image_raw': tf.FixedLenFeature([], tf.string),
'vgg_16/conv1': tf.FixedLenFeature([64, 64], tf.float32),
'vgg_16/pool1': tf.FixedLenFeature([64, 64], tf.float32),
'vgg_16/conv2': tf.FixedLenFeature([128, 128], tf.float32),
'vgg_16/pool2': tf.FixedLenFeature([128, 128], tf.float32),
'vgg_16/conv3': tf.FixedLenFeature([256, 256], tf.float32),
'vgg_16/pool3': tf.FixedLenFeature([256, 256], tf.float32),
'vgg_16/conv4': tf.FixedLenFeature([512, 512], tf.float32),
'vgg_16/pool4': tf.FixedLenFeature([512, 512], tf.float32),
'vgg_16/conv5': tf.FixedLenFeature([512, 512], tf.float32),
'vgg_16/pool5': tf.FixedLenFeature([512, 512], tf.float32)
})
image = tf.image.decode_jpeg(features['image_raw'])
label = features['label']
gram_matrices = [features[vgg_layer] for vgg_layer in vgg_layers]
image.set_shape([None, None, 3])
if image_size:
if square_crop:
image = _aspect_preserving_resize(image, image_size + 2)
image = _central_crop([image], image_size, image_size)[0]
image.set_shape([image_size, image_size, 3])
else:
image = _aspect_preserving_resize(image, image_size)
image = tf.to_float(image) / 255.0
if batch_size is None:
image = tf.expand_dims(image, 0)
else:
image_label_gram_matrices = tf.train.batch([image, label] +
gram_matrices,
batch_size=batch_size)
image, label = image_label_gram_matrices[:2]
gram_matrices = image_label_gram_matrices[2:]
gram_matrices = dict(
(vgg_layer, gram_matrix)
for vgg_layer, gram_matrix in zip(vgg_layers, gram_matrices))
return image, label, gram_matrices
def add_inputs(tfrecords,
feature_size,
build_windows=False,
input_length=None,
num_steps=None,
stride=None,
num_epochs=None,
batch_size=10,
num_threads=2,
capacity=100,
min_after_dequeue=50):
"""
We assume that the features are a raveled version of a patch for the entire
video sequence. At train time, we will want to limit the number of back prop steps.
So we will need to create "windows" of the videos, each of size num_steps (i.e. the
number of frames). The stride size dictates how much overlap the individual windows
will have with each other.
At test time, we don't need to do back prop, so we can just loop through all frames of
the video.
"""
with tf.name_scope('inputs'):
# Have a queue that produces the paths to the .tfrecords
filename_queue = tf.train.string_input_producer(tfrecords,
num_epochs=num_epochs)
# Construct a Reader to read examples from the .tfrecords file
reader = tf.TFRecordReader()
_, serialized_example = reader.read(filename_queue)
# Parse an Example to access the Features
example = tf.parse_single_example(serialized_example,
features={
'feature':
tf.FixedLenFeature(
[feature_size], tf.float32),
'label':
tf.FixedLenFeature([], tf.int64)
})
feature = example['feature']
#print "Feature shape"
#print feature.get_shape().as_list()
label = tf.cast(example['label'], tf.int32)
#print "Label shape"
#print label.get_shape().as_list()
if build_windows:
window_data = tf.py_func(
construct_windows,
[feature, label, input_length, num_steps, stride],
[tf.float32, tf.int32],
name="construct_windows")
window_features = window_data[0]
window_labels = window_data[1]
# Trick to get tensorflow to set the shapes so that the enqueue process works
window_features.set_shape(
tf.TensorShape([tf.Dimension(None), input_length * num_steps]))
window_features = tf.expand_dims(window_features, 0)
window_features = tf.squeeze(window_features, [0])
num_windows = window_features.get_shape().as_list()[0]
window_labels.set_shape(tf.TensorShape([tf.Dimension(num_windows)
]))
features, sparse_labels = tf.train.shuffle_batch(
[window_features, window_labels],
batch_size=batch_size,
num_threads=num_threads,
capacity=capacity, #batch_size * (num_threads + 2),
# Ensures a minimum amount of shuffling of examples.
min_after_dequeue=min_after_dequeue, # 3 * batch_size,
enqueue_many=True)
else:
features, sparse_labels = tf.train.shuffle_batch(
[feature, label],
batch_size=batch_size,
num_threads=num_threads,
capacity=capacity, #batch_size * (num_threads + 2),
# Ensures a minimum amount of shuffling of examples.
min_after_dequeue=min_after_dequeue, # 3 * batch_size,
)
return features, sparse_labels
def read_records(filename, resize_height, resize_width, type=None):
'''
解析record文件:源文件的图像数据是RGB,uint8,[0,255],一般作为训练数据时,需要归一化到[0,1]
:param filename:
:param resize_height:
:param resize_width:
:param type:选择图像数据的返回类型
None:默认将uint8-[0,255]转为float32-[0,255]
normalization:归一化float32-[0,1]
centralization:归一化float32-[0,1],再减均值中心化
:return:
'''
# 创建文件队列,不限读取的数量
filename_queue = tf.train.string_input_producer([filename])
# create a reader from file queue
reader = tf.TFRecordReader()
# reader从文件队列中读入一个序列化的样本
_, serialized_example = reader.read(filename_queue)
# get feature from serialized example
# 解析符号化的样本
features = tf.parse_single_example(serialized_example,
features={
'image_raw':
tf.FixedLenFeature([], tf.string),
'height':
tf.FixedLenFeature([], tf.int64),
'width':
tf.FixedLenFeature([], tf.int64),
'depth':
tf.FixedLenFeature([], tf.int64),
'label':
tf.FixedLenFeature([], tf.int64)
})
tf_image = tf.decode_raw(features['image_raw'], tf.uint8) #获得图像原始的数据
tf_height = features['height']
tf_width = features['width']
tf_depth = features['depth']
tf_label = tf.cast(features['label'], tf.int32)
# PS:恢复原始图像数据,reshape的大小必须与保存之前的图像shape一致,否则出错
# tf_image=tf.reshape(tf_image, [-1]) # 转换为行向量
tf_image = tf.reshape(tf_image,
[resize_height, resize_width, 3]) # 设置图像的维度
# 恢复数据后,才可以对图像进行resize_images:输入uint->输出float32
# tf_image=tf.image.resize_images(tf_image,[224, 224])
# 存储的图像类型为uint8,tensorflow训练时数据必须是tf.float32
if type is None:
tf_image = tf.cast(tf_image, tf.float32)
elif type == 'normalization': # [1]若需要归一化请使用:
# 仅当输入数据是uint8,才会归一化[0,255]
# tf_image = tf.image.convert_image_dtype(tf_image, tf.float32)
tf_image = tf.cast(tf_image, tf.float32) * (1. / 255.0) # 归一化
elif type == 'centralization':
# 若需要归一化,且中心化,假设均值为0.5,请使用:
tf_image = tf.cast(tf_image, tf.float32) * (1. / 255) - 0.5 #中心化
# 这里仅仅返回图像和标签
# return tf_image, tf_height,tf_width,tf_depth,tf_label
return tf_image, tf_label
def read_from_tfrecord(self, batch_size, source_classes_num,
resample_classes_num, mode):
with tf.name_scope("read_image_from_tfrecord"):
if mode == 'train':
files = tf.train.match_filenames_once(
'E:\\shuqian\\resample\\tfrecord\\train_image.tfrecords*')
else:
files = tf.train.match_filenames_once(
'E:\\shuqian\\resample\\tfrecord\\test_image.tfrecords*')
filename_queue = tf.train.string_input_producer(files) #读入流中
reader = tf.TFRecordReader()
_, serialized_example = reader.read(filename_queue) #返回文件名和文件
features = tf.parse_single_example(
serialized_example,
features={
'source_label': tf.FixedLenFeature([], tf.int64),
'resample_label': tf.FixedLenFeature([], tf.int64),
'img_raw': tf.FixedLenFeature([], tf.string),
}) #取出包含image和label的feature对象
#tf.decode_raw可以将字符串解析成图像对应的像素数组
image = tf.decode_raw(features['img_raw'], tf.uint8)
image = tf.reshape(image, [128, 128, 3])
image = tf.cast(image, tf.float32)
image = tf.image.per_image_standardization(image)
source_label = tf.cast(features['source_label'], tf.int32)
resample_label = tf.cast(features['resample_label'], tf.int32)
if mode == 'train':
example_queue = tf.FIFOQueue(
# 队列容量
capacity=5 * batch_size,
dtypes=[tf.float32, tf.int32, tf.int32],
# 图片数据尺寸，标签尺寸
shapes=[[128, 128, 3], (), ()])
# 读线程的数量
num_threads = 4
else:
example_queue = tf.FIFOQueue(
capacity=3 * batch_size,
dtypes=[tf.float32, tf.int32, tf.int32],
shapes=[[128, 128, 3], (), ()])
# 读线程的数量
num_threads = 2
# 数据入队操作
example_enqueue_op = example_queue.enqueue(
[image, source_label, resample_label])
# 队列执行器
tf.train.add_queue_runner(
tf.train.queue_runner.QueueRunner(
example_queue, [example_enqueue_op] * num_threads))
# 数据出队操作，从队列读取Batch数据
images, source_labels, resample_labels = example_queue.dequeue_many(
batch_size)
# 将标签数据由稀疏格式转换成稠密格式
# [ 2, [[0,1,0,0,0]
# 4, [0,0,0,1,0]
# 3, --> [0,0,1,0,0]
# 5, [0,0,0,0,1]
# 1 ] [1,0,0,0,0]]
source_labels = tf.reshape(source_labels, [batch_size, 1])
source_indices = tf.reshape(tf.range(0, batch_size, 1),
[batch_size, 1])
source_labels = tf.sparse_to_dense(
tf.concat(values=[source_indices, source_labels], axis=1),
[batch_size, source_classes_num], 1.0, 0.0)
resample_labels = tf.reshape(resample_labels, [batch_size, 1])
resample_indices = tf.reshape(tf.range(0, batch_size, 1),
[batch_size, 1])
resample_labels = tf.sparse_to_dense(
tf.concat(values=[resample_indices, resample_labels], axis=1),
[batch_size, resample_classes_num], 1.0, 0.0)
#检测数据维度
assert len(images.get_shape()) == 4
assert images.get_shape()[0] == batch_size
assert images.get_shape()[-1] == 3
assert len(source_labels.get_shape()) == 2
assert source_labels.get_shape()[0] == batch_size
assert source_labels.get_shape()[1] == source_classes_num
self.images = images
self.source_labels = source_labels
self.resample_labels = resample_labels
def batch_inputs(data_dir, batch_size, image_size, train, num_preprocess_threads=4,
num_readers=1, input_queue_memory_factor=16):
with tf.name_scope('batch_processing'):
if train:
files = data_files(data_dir, 'train')
filename_queue = tf.train.string_input_producer(files,
shuffle=True,
capacity=16)
else:
files = data_files(data_dir, 'validation')
filename_queue = tf.train.string_input_producer(files,
shuffle=False,
capacity=1)
if num_preprocess_threads % 4:
raise ValueError('Please make num_preprocess_threads a multiple '
'of 4 (%d % 4 != 0).', num_preprocess_threads)
if num_readers < 1:
raise ValueError('Please make num_readers at least 1')
# Approximate number of examples per shard.
examples_per_shard = 1024
# Size the random shuffle queue to balance between good global
# mixing (more examples) and memory use (fewer examples).
# 1 image uses 299*299*3*4 bytes = 1MB
# The default input_queue_memory_factor is 16 implying a shuffling queue
# size: examples_per_shard * 16 * 1MB = 17.6GB
min_queue_examples = examples_per_shard * input_queue_memory_factor
if train:
examples_queue = tf.RandomShuffleQueue(
capacity=min_queue_examples + 3 * batch_size,
min_after_dequeue=min_queue_examples,
dtypes=[tf.string])
else:
examples_queue = tf.FIFOQueue(
capacity=examples_per_shard + 3 * batch_size,
dtypes=[tf.string])
# Create multiple readers to populate the queue of examples.
if num_readers > 1:
enqueue_ops = []
for _ in range(num_readers):
reader = tf.TFRecordReader()
_, value = reader.read(filename_queue)
enqueue_ops.append(examples_queue.enqueue([value]))
tf.train.queue_runner.add_queue_runner(
tf.train.queue_runner.QueueRunner(examples_queue, enqueue_ops))
example_serialized = examples_queue.dequeue()
else:
reader = tf.TFRecordReader()
_, example_serialized = reader.read(filename_queue)
images_labels_fnames = []
for thread_id in range(num_preprocess_threads):
# Parse a serialized Example proto to extract the image and metadata.
image_buffer, label_index, fname = parse_example_proto(example_serialized)
image = image_preprocessing(image_buffer, image_size, train, thread_id)
images_labels_fnames.append([image, label_index, fname])
images, label_index_batch, fnames = tf.train.batch_join(
images_labels_fnames,
batch_size=batch_size,
capacity=2 * num_preprocess_threads * batch_size)
images = tf.cast(images, tf.float32)
images = tf.reshape(images, shape=[batch_size, image_size, image_size, 3])
# Display the training images in the visualizer.
tf.summary.image('images', images, 20)
return images, tf.reshape(label_index_batch, [batch_size]), fnames
def _init_reader(self):
# read all tfrecord files
self.reader = tf.TFRecordReader()
record_files = self._get_tfrecords
self.filequeue = tf.train.string_input_producer(record_files)
def load_image_data_from_tfrecords(
tfrecord_paths_expression,
preprocessing_fn=lambda x: x,
load_jpeg=False,
decode_pixels=False,
image_size=None,
batch_size=16,
num_preprocessing_threads=4):
"""Loads image data from tfrecord files.
Args:
tfrecord_paths_expression (str):
Wildcard expression for path to the
tfrecord files, e.g., "/path/to/*.tfrecord".
preprocessing_fn (lambda):
A lambda function that
is applied to each image batch
load_jpeg (bool):
True if the tfrecords contain binary JPEG data.
decode_pixels (bool):
Whether the tfrecord contains image data
in pixel space or contains normalized values.
image_size (int):
If not None, all images are resized
to a square of this size.
batch_size (int):
Size of a minibatch.
num_preprocessing_threads (int):
Number of threads that run in parallel
to pre-load the data.
Returns:
tuple: Image IDs, images and labels as batched tensors.
"""
print('reading %s' % tfrecord_paths_expression)
filename_queue = tf.train.string_input_producer(
glob.glob(tfrecord_paths_expression), num_epochs=1)
reader = tf.TFRecordReader()
_, serialized_example = reader.read(filename_queue)
features = decode_tf_example(serialized_example)
image_id = features['image/filename']
label = features['image/class/label']
height, width = tf.cast(features['image/height'], tf.int32), \
tf.cast(features['image/width'], tf.int32)
image = tf.image.decode_jpeg(features['image/encoded'], channels=3) \
if load_jpeg \
else tf.decode_raw(
features['image/encoded'],
tf.uint8 if decode_pixels else tf.float32)
image = tf.reshape(image, [height, width, 3])
if not load_jpeg and image_size is not None:
image = tf.reshape(image, [image_size, image_size, 3])
image = preprocessing_fn(image)
image_ids, images, labels = tf.train.batch(
[image_id, image, label],
batch_size=batch_size,
num_threads=num_preprocessing_threads,
capacity=5 * batch_size,
allow_smaller_final_batch=True)
return image_ids, images, labels
def train():
image_lr = tf.placeholder(dtype=tf.float32, shape=(None, 32, 32, 3), name='lr')
image_hr = tf.placeholder(dtype=tf.float32, shape=(None, 128, 128, 3), name='hr')
lamb = tf.placeholder(dtype=tf.float32, name='Kt')
net = BEGAN(gamma)
gen = net.generator(image_lr)
dis_fake = net.discrimintor(gen)
dis_real = net.discrimintor(image_hr, reuse=True)
with tf.name_scope('SR_loss'):
residual = image_hr - gen
square = tf.abs(residual)
SR_loss = tf.reduce_mean(square)
tf.summary.scalar('SR_loss', SR_loss)
print('test1')
with tf.name_scope('gan_loss'):
D_real = net.loss(image_hr, dis_real, type='L1')
D_fake = net.loss(gen, dis_fake, type='L1')
D_loss = D_real - lamb*D_fake
G_gan_loss = D_fake
W_loss = D_real + tf.abs(gamma*D_real - G_gan_loss)
G_loss = gan_ratio*G_gan_loss + SR_loss
tf.summary.scalar('Kt', lamb)
tf.summary.scalar('G_gan_loss', G_gan_loss)
tf.summary.scalar('G_loss', G_loss)
tf.summary.scalar('D_loss', D_loss)
tf.summary.scalar('W_loss', W_loss)
print('test2')
# get variable from G and D
var_g = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, 'generator')
var_d = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, 'discriminator')
with tf.name_scope('optim'):
optim_g = tf.train.AdamOptimizer(learning_rate=LEARNING_RATE)\
.minimize(G_loss, var_list=var_g)
optim_d = tf.train.AdamOptimizer(learning_rate=LEARNING_RATE) \
.minimize(D_loss, var_list=var_d)
print('test3')
# for gradient, var in var_g:
# tf.summary.histogram(var.name + '/gradient', gradient)
#
# # Add the variables we train to the summary
# for var in var_g:
# tf.summary.histogram(var.name, var)
# set up logging for tensorboard
writer = tf.summary.FileWriter(filewriter_path)
writer.add_graph(tf.get_default_graph())
summaries = tf.summary.merge_all()
# saver for storing/restoring checkpoints of the model
saver = tf.train.Saver()
data_path = 'train_espcn.tfrecords'
feature = {'train/image_small': tf.FixedLenFeature([], tf.string),
'train/image_origin': tf.FixedLenFeature([], tf.string)}
# create a list of file names
filename_queue = tf.train.string_input_producer([data_path], num_epochs=NUM_EPOCHS)
reader = tf.TFRecordReader()
_, tfrecord_serialized = reader.read(filename_queue)
features = tf.parse_single_example(tfrecord_serialized, features=feature)
# Convert the image data from string back to the numbers
image_blur = tf.decode_raw(features['train/image_small'], tf.uint8)
image_origin = tf.decode_raw(features['train/image_origin'], tf.uint8)
image_blur = tf.reshape(image_blur, [32, 32, 3])
image_origin = tf.reshape(image_origin, [128, 128, 3])
images, labels = tf.train.shuffle_batch([image_blur, image_origin],
batch_size=BATCH_SIZE, capacity=30,
num_threads=16,
min_after_dequeue=10)
print('test4')
Kt = 0
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
with tf.Session() as sess:
init_op = tf.group(tf.global_variables_initializer(), tf.local_variables_initializer())
sess.run(init_op)
steps, start_average, end_average = 0, 0, 0
start_time = time.clock()
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(coord=coord)
for ii in range(NUM_EPOCHS):
batch_average = 0
batch_num = int(np.floor(192794 / BATCH_SIZE))
for jj in range(batch_num):
steps += 1
img_lr, img_hr = sess.run([images, labels])
img_lr = img_lr.astype(np.float32) / 255
img_hr = img_hr.astype(np.float32) / 255
g_ops = [optim_g, G_loss, D_real, D_fake]
d_ops = [optim_d, D_loss, summaries]
_, loss_g, loss_real, loss_fake = sess.run(g_ops,
feed_dict={image_lr: img_lr, image_hr: img_hr, lamb: Kt})
_, loss_d, summary = sess.run(d_ops, feed_dict=
{image_lr: img_lr, image_hr: img_hr, lamb: Kt})
# update W_loss and Kt
Kt = np.maximum(np.minimum(1., Kt + l_lr * (gamma * loss_real - loss_fake)), 0.)
global_loss = loss_real + np.abs(gamma * loss_real - loss_fake)
writer.add_summary(summary, steps)
batch_average += global_loss
if (jj % 100 == 0):
print('step: {:d}, loss: {:.9f}'.format(steps, global_loss))
print('time:', time.clock())
batch_average = float(batch_average) / batch_num
duration = time.time() - start_time
print('Epoch: {}, step: {:d}, loss: {:.9f}, '
'({:.3f} sec/epoch)'.format(ii, steps, batch_average, duration))
start_time = time.time()
net.save(sess, saver, checkpoint_path, steps)
coord.request_stop()
# Wait for threads to stop
coord.join(threads)
sess.close()
def __init__(self,
config,
mode,
train_inception=False,
reuse_img_embed=False):
"""Basic setup.
Args:
config: Object containing configuration parameters.
mode: "train", "eval" or "inference".
train_inception: Whether the inception submodel variables are trainable.
"""
assert mode in ["train", "eval", "inference"]
self.config = config
self.mode = mode
self.train_inception = train_inception
self.reuse_img_embed = reuse_img_embed
# Reader for the input data.
self.reader = tf.TFRecordReader()
# To match the "Show and Tell" paper we initialize all variables with a
# random uniform initializer.
self.initializer = tf.random_uniform_initializer(
minval=-self.config.initializer_scale,
maxval=self.config.initializer_scale)
# A float32 Tensor with shape [batch_size, height, width, channels].
self.images = None
# An int32 Tensor with shape [batch_size, padded_length].
self.input_seqs = None
# An int32 Tensor with shape [batch_size, padded_length].
self.target_seqs = None
# An int32 0/1 Tensor with shape [batch_size, padded_length].
self.input_mask = None
# A float32 Tensor with shape [batch_size, embedding_size].
self.image_embeddings = None
# A float32 Tensor with shape [batch_size, padded_length, embedding_size].
self.seq_embeddings = None
self.inception_net = None
self.inception_output = None
self.lstm_outputs = None
self.lstm_logits = None
# A float32 scalar Tensor; the total loss for the trainer to optimize.
self.total_loss = None
# A float32 Tensor with shape [batch_size * padded_length].
self.target_cross_entropy_losses = None
# A float32 Tensor with shape [batch_size * padded_length].
self.target_cross_entropy_loss_weights = None
# Collection of variables from the inception submodel.
self.inception_variables = []
# Function to restore the inception submodel from checkpoint.
self.init_fn = None
# Global step Tensor.
self.global_step = None
def predict_tfrecord(filename):
'''
read label file
'''
label_map_path = "./labelmap/label.txt"
label_map_file = open(label_map_path)
label_map = {}
for line_number, label in enumerate(label_map_file.readlines()):
label_map[line_number] = label[:-1]
line_number += 1
label_map_file.close()
# read tfrecord file
num_files=get_num_of_files_in_tfrecord(filename)
filename_queue = tf.train.string_input_producer([filename])
reader = tf.TFRecordReader()
_, serialized_example = reader.read(filename_queue)
features = tf.parse_single_example(serialized_example,features = {
'image/encoded': tf.FixedLenFeature([], tf.string),
'image/class/label': tf.FixedLenFeature([], tf.string),
'image/filepath': tf.FixedLenFeature([], tf.string),})
image_data=tf.cast(features['image/encoded'], tf.string)
image_data=tf.image.decode_jpeg(image_data)
label = tf.cast(features['image/class/label'], tf.string)
filepath = tf.cast(features['image/filepath'], tf.string)
with tf.Session() as sess:
init_op = tf.initialize_all_variables()
sess.run(init_op)
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(coord=coord)
image_data_list=[]
# label_list=[]
# filepath_list=[]
res = []
for i in range(num_files):
start = datetime.datetime.now()
[input_0, _label,_filepath] = sess.run([image_data, label,filepath])
# _label=str(_label,encoding='utf-8')
_filepath=str(_filepath,encoding='utf-8')
while input_0.shape[0] < 331 or input_0.shape[1] < 331:
input_0 = cv2.pyrUp(input_0)
while input_0.shape[0] >= 662 and input_0.shape[1] >= 662:
input_0 = cv2.pyrDown(input_0)
image_height = input_0.shape[0]
print(image_height)
image_width = input_0.shape[1]
print(image_width)
image_height_center = int(image_height/2)
image_width_center = int(image_width/2)
tl_crop = input_0[0:331, 0:331]
tr_crop = input_0[0:331, image_width-331:image_width]
bl_crop = input_0[image_height-331:image_height, 0:331]
br_crop = input_0[image_height-331:image_height, image_width-331:image_width]
center_crop = input_0[image_height_center - 165: image_height_center + 166, image_width_center - 165: image_width_center + 166]
input_concat = np.asarray([tl_crop, tr_crop, bl_crop, br_crop, center_crop])
# print(input_concat.shape)
input_batch = input_concat.reshape(-1, 331, 331, 3)
predictions = diagnose_image(inference_session, input_batch)
# print(predictions)
overall_result = np.argmax(np.sum(predictions, axis=0))
# save img to the classified folder
'''
image_origin = cv2.imread(os.path.join(src_root, image))
save_path = "save/"+str(overall_result)
os.mkdir(save_path)
save_file_path = os.path.join(save_path, image)
if os.path.exists(save_file_path):
save_file_path = os.path.join(save_path, image.split('.')[0]+"_dup"+image.split('.')[:-1])
cv2.imwrite(os.path.join(save_path, image),image_origin)
print("Image saved.")
'''
end = datetime.datetime.now()
#print(image_path)
print(overall_result, label_map[overall_result])
print("Time cost: ", end - start, "\n")
res.append([_filepath,label_map[overall_result]])
return res
def build_tfrecord_input(batch_size,
data_dir,
sequence_length,
train_val_split,
use_state,
training=True):
"""Create input tfrecord tensors.
Args:
training: training or validation data.
Returns:
list of tensors corresponding to images, actions, and states. The images
tensor is 5D, batch x time x height x width x channels. The state and
action tensors are 3D, batch x time x dimension.
Raises:
RuntimeError: if no files found.
"""
filenames = gfile.Glob(os.path.join(data_dir, '*'))
if not filenames:
raise RuntimeError('No data files found.')
index = int(np.floor(train_val_split * len(filenames)))
if training:
filenames = filenames[:index]
else:
filenames = filenames[index:]
filename_queue = tf.train.string_input_producer(filenames, shuffle=True)
reader = tf.TFRecordReader()
_, serialized_example = reader.read(filename_queue)
image_seq, state_seq, action_seq = [], [], []
for i in range(6, 20, 2):
image_name = 'move/' + str(i) + '/image/encoded'
action_name = 'move/' + str(i) + '/commanded_pose/vec_pitch_yaw'
state_name = 'move/' + str(i) + '/endeffector/vec_pitch_yaw'
if use_state:
features = {image_name: tf.FixedLenFeature([1], tf.string),
action_name: tf.FixedLenFeature([STATE_DIM], tf.float32),
state_name: tf.FixedLenFeature([STATE_DIM], tf.float32)}
else:
features = {image_name: tf.FixedLenFeature([1], tf.string)}
features = tf.parse_single_example(serialized_example, features=features)
image_buffer = tf.reshape(features[image_name], shape=[])
image = tf.image.decode_jpeg(image_buffer, channels=COLOR_CHAN)
image.set_shape([ORIGINAL_HEIGHT, ORIGINAL_WIDTH, COLOR_CHAN])
if IMG_HEIGHT != IMG_WIDTH:
raise ValueError('Unequal height and width unsupported')
crop_size = min(ORIGINAL_HEIGHT, ORIGINAL_WIDTH)
image = tf.image.resize_image_with_crop_or_pad(image, crop_size, crop_size)
image = tf.reshape(image, [1, crop_size, crop_size, COLOR_CHAN])
image = tf.image.resize_area(image, [IMG_HEIGHT, IMG_WIDTH])
image = (tf.cast(image, tf.float32) / (255.0 / 2.)) - 1.
image_seq.append(image)
if use_state:
state = tf.reshape(features[state_name], shape=[1, STATE_DIM])
state_seq.append(state)
action = tf.reshape(features[action_name], shape=[1, STATE_DIM])
action_seq.append(action)
image_seq = tf.concat(axis=0, values=image_seq)
if use_state:
state_seq = tf.concat(axis=0, values=state_seq)
action_seq = tf.concat(axis=0, values=action_seq)
[image_batch, action_batch, state_batch] = tf.train.batch(
[image_seq, action_seq, state_seq],
batch_size,
num_threads=batch_size,
capacity=500 * batch_size)
action_state = tf.concat(values=[action_batch, state_batch], axis=2)
return image_batch, action_state
else:
image_batch = tf.train.batch(
[image_seq],
batch_size,
num_threads=batch_size,
capacity=500 * batch_size)
zeros_batch = tf.zeros([batch_size, sequence_length, 2 * STATE_DIM])
return image_batch, zeros_batch
def load_data(filename_queue, config):
load_h, load_w, c = config.load_size, config.load_size, config.c_dim
fine_h, fine_w = config.fine_size, config.fine_size
reader = tf.TFRecordReader()
_, serialized_example = reader.read(filename_queue)
img_features = tf.parse_single_example(serialized_example,
features={
'in_LDRs':
tf.FixedLenFeature([],
tf.string),
'ref_LDRs':
tf.FixedLenFeature([],
tf.string),
'in_exps':
tf.FixedLenFeature([],
tf.string),
'ref_exps':
tf.FixedLenFeature([],
tf.string),
'ref_HDR':
tf.FixedLenFeature([],
tf.string),
})
in_LDRs = tf.decode_raw(img_features['in_LDRs'], tf.uint8)
ref_LDRs = tf.decode_raw(img_features['ref_LDRs'], tf.uint8)
in_exps = tf.decode_raw(img_features['in_exps'], tf.float32)
ref_exps = tf.decode_raw(img_features['ref_exps'], tf.float32)
ref_HDR = tf.decode_raw(img_features['ref_HDR'], tf.float32)
in_LDRs = tf.reshape(in_LDRs, [load_h, load_w, c * config.num_shots])
ref_LDRs = tf.reshape(ref_LDRs, [load_h, load_w, c * config.num_shots])
in_exps = tf.reshape(in_exps, [config.num_shots])
ref_exps = tf.reshape(ref_exps, [config.num_shots])
ref_HDR = tf.reshape(ref_HDR, [load_h, load_w, c])
######### distortions #########
distortions = tf.random_uniform([2], 0, 1.0, dtype=tf.float32)
# flip horizontally
in_LDRs = tf.cond(tf.less(distortions[0],
0.5), lambda: tf.image.flip_left_right(in_LDRs),
lambda: in_LDRs)
ref_LDRs = tf.cond(tf.less(distortions[0], 0.5),
lambda: tf.image.flip_left_right(ref_LDRs),
lambda: ref_LDRs)
ref_HDR = tf.cond(tf.less(distortions[0],
0.5), lambda: tf.image.flip_left_right(ref_HDR),
lambda: ref_HDR)
# rotate
k = tf.cast(distortions[1] * 4 + 0.5, tf.int32)
in_LDRs = tf.image.rot90(in_LDRs, k)
ref_LDRs = tf.image.rot90(ref_LDRs, k)
ref_HDR = tf.image.rot90(ref_HDR, k)
######### distortions #########
in_LDRs = transform_LDR(in_LDRs, [fine_h, fine_w])
ref_LDRs = transform_LDR(ref_LDRs, [fine_h, fine_w])
ref_HDR = transform_HDR(ref_HDR, [fine_h, fine_w])
in_exps = 2.**in_exps
ref_exps = 2.**ref_exps
in_HDRs = LDR2HDR_batch(in_LDRs, in_exps)
in_LDRs_batch, in_HDRs_batch, ref_LDRs_batch, ref_HDR_batch, in_exps_batch, ref_exps_batch = tf.train.shuffle_batch(
[in_LDRs, in_HDRs, ref_LDRs, ref_HDR, in_exps, ref_exps],
batch_size=config.batch_size,
num_threads=2,
capacity=256,
min_after_dequeue=64)
return in_LDRs_batch, in_HDRs_batch, ref_LDRs_batch, ref_HDR_batch, in_exps_batch, ref_exps_batch
# --------------hyperParams--------------------------
batch_size = 2
capacity = 1000 + 3 * batch_size
train_rounds = 5
num_epochs = 30
img_h = 333
img_w = 500
# ---------------------------------------------------
tfrecord_files = tf.train.match_filenames_once('./tfrecords/train.tfrecords-*')
queue = tf.train.string_input_producer(tfrecord_files,
num_epochs=num_epochs,
shuffle=True,
capacity=10)
reader = tf.TFRecordReader()
# 从文件中读出一个队列， 也可以使用read_uo_to函数一次性读取多个样例
_, serialized_example = reader.read(queue)
# 读取多个对应tf.parse_example()
# 读取单个对应tf.parse_single_example()
features = tf.parse_single_example(serialized_example,
features={
'label':
tf.FixedLenFeature([], tf.int64),
'img_raw':
tf.FixedLenFeature([], tf.string),
})
image = tf.decode_raw(features['img_raw'], tf.uint8)
def prepare_reader(self,
filename_queue,
max_quantized_value=2,
min_quantized_value=-2):
"""Creates a single reader thread for YouTube8M SequenceExamples.
Args:
filename_queue: A tensorflow queue of filename locations.
max_quantized_value: the maximum of the quantized value.
min_quantized_value: the minimum of the quantized value.
Returns:
A tuple of video indexes, video features, labels, and padding data.
"""
reader = tf.TFRecordReader()
_, serialized_example = reader.read(filename_queue)
contexts, features = tf.parse_single_sequence_example(
serialized_example,
context_features={
"video_id": tf.FixedLenFeature([], tf.string),
"labels": tf.VarLenFeature(tf.int64)
},
sequence_features={
feature_name: tf.FixedLenSequenceFeature([], dtype=tf.string)
for feature_name in self.feature_names
})
# read ground truth labels
labels = (tf.cast(
tf.sparse_to_dense(contexts["labels"].values, (self.num_classes, ),
1,
validate_indices=False), tf.bool))
# loads (potentially) different types of features and concatenates them
num_features = len(self.feature_names)
assert num_features > 0, "No feature selected: feature_names is empty!"
assert len(self.feature_names) == len(self.feature_sizes), \
"length of feature_names (={}) != length of feature_sizes (={})".format( \
len(self.feature_names), len(self.feature_sizes))
num_frames = -1 # the number of frames in the video
feature_matrices = [None
] * num_features # an array of different features
for feature_index in range(num_features):
feature_matrix, num_frames_in_this_feature = self.get_video_matrix(
features[self.feature_names[feature_index]],
self.feature_sizes[feature_index], self.max_frames,
max_quantized_value, min_quantized_value)
if num_frames == -1:
num_frames = num_frames_in_this_feature
else:
tf.assert_equal(num_frames, num_frames_in_this_feature)
feature_matrices[feature_index] = feature_matrix
# cap the number of frames at self.max_frames
num_frames = tf.minimum(num_frames, self.max_frames)
# concatenate different features
video_matrix = tf.concat(feature_matrices, 1)
return contexts["video_id"], video_matrix, labels, num_frames
