def read_news(filename_queue):
"""Reads and parses examples from CIFAR10 data files.
Recommendation: if you want N-way read parallelism, call this function
N times. This will give you N independent Readers reading different
files & positions within those files, which will give better mixing of
examples.
Args:
filename_queue: A queue of strings with the filenames to read from.
"""
reader = tf.TFRecordReader()
_, serialized_example = reader.read(filename_queue)
features = tf.parse_single_example(serialized_example, features={
'hubs': tf.FixedLenFeature([3], dtype=tf.int64),
'words': tf.FixedLenFeature([6250], dtype=tf.int64)
})
def unpackbits(arr):
arr = arr.astype(np.ubyte)
unpacked_arr = np.unpackbits(arr)
if len(unpacked_arr) == 24:
unpacked_arr = unpacked_arr[:BagOfWords.NUM_CLASSES]
return unpacked_arr.astype(np.float32)
labels = features['hubs']
labels, = tf.py_func(unpackbits, [labels], [tf.float32])
labels.set_shape((BagOfWords.NUM_CLASSES,))
words = features['words']
words, = tf.py_func(unpackbits, [words], [tf.float32])
words.set_shape((BagOfWords.NUM_VOCABULARY_SIZE,))
return labels, words
def testLarge(self):
with self.test_session() as sess:
x = tf.zeros([1000000], dtype=np.float32)
y = tf.py_func(lambda x: x + 1, [x], [tf.float32])
z = tf.py_func(lambda x: x * 2, [x], [tf.float32])
for _ in xrange(100):
sess.run([y[0].op, z[0].op])
def get_dataset(data, labels=None, batch_size=None, data_shape=None,
use_random_transpose=False, num_threads=1):
"""Create and return a tensorflow dataset from an array."""
if labels is None:
dataset = tf.data.Dataset.from_generator(
lambda: _gen_data(data), tf.float32)
if use_random_transpose:
dataset = dataset.map(
lambda pianoroll: tf.py_func(
random_transpose, [pianoroll], tf.float32),
num_parallel_calls=num_threads)
dataset = dataset.map(lambda pianoroll: set_pianoroll_shape(
pianoroll, data_shape), num_parallel_calls=num_threads)
else:
assert len(data) == len(labels), (
"Lengths of `data` and `lables` do not match.")
dataset = tf.data.Dataset.from_generator(
lambda: _gen_data(data, labels), [tf.float32, tf.int32])
if use_random_transpose:
dataset = dataset.map(
lambda pianoroll, label: (
tf.py_func(random_transpose, [pianoroll], tf.float32),
label),
num_parallel_calls=num_threads)
dataset = dataset.map(
lambda pianoroll, label: (set_pianoroll_shape(
pianoroll, data_shape), set_label_shape(label)),
num_parallel_calls=num_threads)
dataset = dataset.shuffle(SHUFFLE_BUFFER_SIZE).repeat().batch(batch_size)
return dataset.prefetch(PREFETCH_SIZE)
def g_rinv(layer, x1_target, x0_activation):
with tf.variable_scope(vscope[layer], reuse=True):
V_ = tf.get_variable('V')
c_ = tf.get_variable('c')
relu_inv = tf.py_func(ops.relu().f_inv, [x1_target, x0_activation], [tf.float32], name='x3_')[0]
add_inv = tf.sub(relu_inv, b[layer], name='x2_')
return tf.py_func(ops.linear().f_inv, [add_inv, x0_activation, W[layer]], [tf.float32], name='x1_')[0]
def testCaching(self):
"""Confirm caching of control output is recacluated between calls."""
a = tf.constant(1)
b = tf.constant(2)
with tf.control_dependencies([a]):
c = tf.constant(42)
shared = {}
def sub(t):
shared[t] = shared.get(t, 0) + 1
return t
a = subscribe.subscribe(a, lambda t: tf.py_func(sub, [t], [t.dtype]))
with tf.control_dependencies([b]):
d = tf.constant(11)
# If it was using outdated cached control_outputs then
# evaling would not trigger the new subscription.
b = subscribe.subscribe(b, lambda t: tf.py_func(sub, [t], [t.dtype]))
with self.test_session() as sess:
c_out = sess.run([c])
d_out = sess.run([d])
self.assertEquals(c_out, [42])
self.assertEquals(d_out, [11])
self.assertEquals(shared, {2: 1, 1: 1})
def testBasic(self):
def my_func(x, y):
return np.sinh(x) + np.cosh(y)
# scalar
with self.test_session():
x = tf.constant(1.0, tf.float32)
y = tf.constant(2.0, tf.float32)
z = tf.py_func(my_func, [x, y], [tf.float32])
self.assertEqual(z[0].eval(), my_func(1.0, 2.0).astype(np.float32))
# array
with self.test_session():
x = tf.constant([1.0, 2.0], tf.float64)
y = tf.constant([2.0, 3.0], tf.float64)
z = tf.py_func(my_func, [x, y], [tf.float64])
self.assertAllEqual(z[0].eval(), my_func([1.0, 2.0], [2.0, 3.0]).astype(np.float64))
# a bit exotic type (complex64)
with self.test_session():
x = tf.constant(1 + 2j, tf.complex64)
y = tf.constant(3 + 4j, tf.complex64)
z, = tf.py_func(my_func, [x, y], [tf.complex64])
self.assertAllClose(z.eval(), my_func(1 + 2j, 3 + 4j))
# a bit excotic function (rfft)
with self.test_session():
x = tf.constant([1.0, 2.0, 3.0, 4.0], tf.float32)
def rfft(x):
return np.fft.rfft(x).astype(np.complex64)
y, = tf.py_func(rfft, [x], [tf.complex64])
self.assertAllClose(y.eval(), np.fft.rfft([1.0, 2.0, 3.0, 4.0]))
def evaluate(dataset_path):
"""Evaluate model on Dataset for a number of steps."""
with tf.Graph().as_default(), tf.device('/cpu:0'):
train_dir = Path(FLAGS.checkpoint_dir)
reference_shape = mio.import_pickle(train_dir / 'reference_shape.pkl')
images, gt_truth, inits, _ = data_provider.batch_inputs(
[dataset_path], reference_shape,
batch_size=FLAGS.batch_size, is_training=False)
mirrored_images, _, mirrored_inits, shapes = data_provider.batch_inputs(
[dataset_path], reference_shape,
batch_size=FLAGS.batch_size, is_training=False, mirror_image=True)
print('Loading model...')
# Build a Graph that computes the logits predictions from the
# inference model.
with tf.device(FLAGS.device):
patch_shape = (FLAGS.patch_size, FLAGS.patch_size)
pred, _, _ = mdm_model.model(images, inits, patch_shape=patch_shape)
tf.get_variable_scope().reuse_variables()
pred_mirrored, _, _ = mdm_model.model(
mirrored_images, mirrored_inits, patch_shape=patch_shape)
pred_images, = tf.py_func(utils.batch_draw_landmarks,
[images, pred], [tf.float32])
gt_images, = tf.py_func(utils.batch_draw_landmarks,
[images, gt_truth], [tf.float32])
summaries = []
summaries.append(tf.image_summary('images',
tf.concat(2, [gt_images, pred_images]), max_images=5))
avg_pred = pred + tf.py_func(flip_predictions, (pred_mirrored, shapes), (tf.float32, ))[0]
avg_pred /= 2.
# Calculate predictions.
norm_error = mdm_model.normalized_rmse(avg_pred, gt_truth)
# Restore the moving average version of the learned variables for eval.
variable_averages = tf.train.ExponentialMovingAverage(
mdm_train.MOVING_AVERAGE_DECAY)
variables_to_restore = variable_averages.variables_to_restore()
saver = tf.train.Saver(variables_to_restore)
# Build the summary operation based on the TF collection of Summaries.
summary_op = tf.merge_summary(summaries)
graph_def = tf.get_default_graph().as_graph_def()
summary_writer = tf.train.SummaryWriter(FLAGS.eval_dir,
graph_def=graph_def)
while True:
_eval_once(saver, summary_writer, norm_error, summary_op)
if FLAGS.run_once:
break
time.sleep(FLAGS.eval_interval_secs)
def testGradientFunction(self): # Input to tf.py_func is necessary, otherwise get_gradient_function() # returns None per default. a = tf.constant(0) x, = tf.py_func(lambda a: 0, [a], [tf.int64]) y, = tf.py_func(lambda a: 0, [a], [tf.int64], stateful=False) self.assertEqual(None, ops.get_gradient_function(x.op)) self.assertEqual(None, ops.get_gradient_function(y.op))
def build_graph(self):
"""Builds data processing graph using ``tf.data`` API."""
self._dataset = tf.data.Dataset.from_tensor_slices(self._files)
if self.params['shuffle']:
self._dataset = self._dataset.shuffle(self._size)
self._dataset = self._dataset.repeat()
if self.params['mode'] != 'infer':
self._dataset = self._dataset.map(
lambda line: tf.py_func(
self._parse_audio_transcript_element,
[line],
[self.params['dtype'], tf.int32, tf.int32, tf.int32],
stateful=False,
),
num_parallel_calls=8,
)
self._dataset = self._dataset.padded_batch(
self.params['batch_size'],
padded_shapes=([None, self.params['num_audio_features']], 1, [None], 1)
)
else:
self._dataset = self._dataset.map(
lambda line: tf.py_func(
self._parse_audio_element,
[line],
[self.params['dtype'], tf.int32],
stateful=False,
),
num_parallel_calls=8,
)
self._dataset = self._dataset.padded_batch(
self.params['batch_size'],
padded_shapes=([None, self.params['num_audio_features']], 1)
)
self._iterator = self._dataset.prefetch(8).make_initializable_iterator()
if self.params['mode'] != 'infer':
x, x_length, y, y_length = self._iterator.get_next()
# need to explicitly set batch size dimension
# (it is employed in the model)
y.set_shape([self.params['batch_size'], None])
y_length = tf.reshape(y_length, [self.params['batch_size']])
else:
x, x_length = self._iterator.get_next()
x.set_shape([self.params['batch_size'], None,
self.params['num_audio_features']])
x_length = tf.reshape(x_length, [self.params['batch_size']])
self._input_tensors = {}
self._input_tensors["source_tensors"] = [x, x_length]
if self.params['mode'] != 'infer':
self._input_tensors['target_tensors'] = [y, y_length]
def augment(image, cfg):
options = cfg.IMAGE_AUGMENTATIONS
if options.FLIP_LEFT_RIGHT:
image = tf.image.random_flip_left_right(image)
if options.CROP:
# We want the size to be larger, and then will crop a region out of it
target_size = tf.to_int32(cfg.INPUT_SIZE * options.CROP_UPSCALE_FACTOR)
if cfg.MAINTAIN_ASPECT_RATIO:
# Resize the image up, then pad with 0s
#image = resize_image_preserve_aspect_ratio(image, target_size, target_size)
params = [image, target_size, target_size]
output = tf.py_func(resize_image_maintain_aspect_ratio, params, [tf.float32], name="resize_maintain_aspect_ratio")
image = output[0]
else:
# Just resize it
image = tf.image.resize_images(
image,
(target_size,
target_size)
)
image = tf.random_crop(image, [cfg.INPUT_SIZE, cfg.INPUT_SIZE, 3])
else:
# Just resize it
if cfg.MAINTAIN_ASPECT_RATIO:
# Resize the image up, then pad with 0s
#image = resize_image_preserve_aspect_ratio(image, target_size, target_size)
params = [image, tf.constant(cfg.INPUT_SIZE), tf.constant(cfg.INPUT_SIZE)]
output = tf.py_func(resize_image_maintain_aspect_ratio, params, [tf.float32], name="resize_maintain_aspect_ratio")
image = output[0]
else:
image = tf.image.resize_images(
image,
(cfg.INPUT_SIZE,
cfg.INPUT_SIZE)
)
if options.BRIGHTNESS:
image = tf.image.random_brightness(image, max_delta=63)
if options.CONTRAST:
image = tf.image.random_contrast(image, lower=0.2, upper=1.8)
return image
def SigJoin(x,y,m,fixedLast=None):
rnd_name = 'PyFuncGrad' + str(np.random.randint(0, 1E+8))
if fixedLast is None:
tf.RegisterGradient(rnd_name)(_sigJoinGrad)
g=tf.get_default_graph()
with g.gradient_override_map({"PyFunc":rnd_name}):
return tf.py_func(_sigJoinImp, [x,y,m], tf.float32, name="SigJoin")
else:
tf.RegisterGradient(rnd_name)(_sigJoinGradFixed)
g=tf.get_default_graph()
with g.gradient_override_map({"PyFunc":rnd_name}):
return tf.py_func(_sigJoinFixedImp, [x,y,m,fixedLast], tf.float32, name="SigJoin")
def testStrings(self):
def read_fixed_length_numpy_strings():
return np.array([" there"])
def read_and_return_strings(x, y):
return x + y
with self.test_session():
x = tf.constant(["hello", "hi"], tf.string)
y, = tf.py_func(read_fixed_length_numpy_strings, [], [tf.string])
z, = tf.py_func(read_and_return_strings, [x, y], [tf.string])
self.assertListEqual(list(z.eval()), ["hello there", "hi there"])
def build_model(self):
if self.y_dim:
self.y= tf.placeholder(tf.float32, [None, self.y_dim], name='y')
self.ir_images = tf.placeholder(tf.float32, [self.batch_size] + self.ir_image_shape,
name='ir_images')
self.normal_images = tf.placeholder(tf.float32, [self.batch_size] + self.normal_image_shape,
name='normal_images')
self.ir_sample_images= tf.placeholder(tf.float32, [self.sample_size] + self.ir_image_shape,
name='ir_sample_images')
self.ei_images = tf.placeholder(tf.float32, [self.batch_size] + self.ir_image_shape,
name='ei_images')
self.G = self.generator(self.ir_images)
self.D = self.discriminator(self.normal_images) # real image output
self.sampler = self.sampler(self.ir_images)
self.D_ = self.discriminator(self.G, reuse=True) #fake image output
self.d_sum = tf.histogram_summary("d", self.D)
self.d__sum = tf.histogram_summary("d_", self.D_)
#self.G_sum = tf.image_summary("G", self.G)
self.d_loss_real = binary_cross_entropy_with_logits(tf.ones_like(self.D), self.D)
self.d_loss_fake = binary_cross_entropy_with_logits(tf.zeros_like(self.D_), self.D_)
self.ang_loss = tf.py_func(norm_,[self.G,self.normal_images],[tf.float64])
self.ang_loss = tf.to_float(self.ang_loss[0],name='ToFloat')
self.L2_loss = tf.reduce_sum(tf.pow(tf.sub(self.G,self.normal_images),2))/(2 * self.batch_size)
self.EI_loss = tf.py_func(compute_ei,[self.G],[tf.float64])
self.EI_loss = tf.to_float(self.EI_loss[0],name='ToFloat')
self.g_loss = binary_cross_entropy_with_logits(tf.ones_like(self.D_), self.D_)
self.gen_loss = self.g_loss * self.lambda_g + self.L2_loss * self.lambda_l2 + self.EI_loss * self.lambda_ei
self.g_loss_sum = tf.scalar_summary("g_loss", self.g_loss)
self.ang_loss_sum = tf.scalar_summary("ang_loss", self.ang_loss)
self.l2_loss_sum = tf.scalar_summary("l2_loss", self.L2_loss)
self.ei_loss_sum = tf.scalar_summary("ei_loss", self.EI_loss)
self.gen_loss_sum = tf.scalar_summary("gen_loss", self.gen_loss)
self.d_loss_real_sum = tf.scalar_summary("d_loss_real", self.d_loss_real)
self.d_loss_fake_sum = tf.scalar_summary("d_loss_fake", self.d_loss_fake)
self.d_loss = self.d_loss_real + self.d_loss_fake
self.d_loss_sum = tf.scalar_summary("d_loss", self.d_loss)
t_vars = tf.trainable_variables()
self.d_vars = [var for var in t_vars if 'd_' in var.name]
self.g_vars = [var for var in t_vars if 'g_' in var.name]
self.saver = tf.train.Saver()
def return_fn(trX, trY, teX):
with tf.device(device):
with tf.device("/cpu:0"):
if probs:
return tf.py_func(
_py_fit_predict,
[tf.identity(trX),
tf.identity(trY),
tf.identity(teX)], [tf.int64, tf.int64, tf.float32])
else:
return tf.py_func(
_py_fit_predict,
[tf.identity(trX),
tf.identity(trY),
tf.identity(teX)], [tf.int64, tf.int64])
def testCleanup(self):
for _ in xrange(1000):
g = tf.Graph()
with g.as_default():
c = tf.constant([1.], tf.float32)
_ = tf.py_func(lambda x: x + 1, [c], [tf.float32])
self.assertTrue(script_ops._py_funcs.size() < 100)
def simulate(self, action):
"""Step the environment.
The result of the step can be accessed from the variables defined below.
Args:
action: Tensor holding the action to apply.
Returns:
Operation.
"""
with tf.name_scope('environment/simulate'):
if action.dtype in (tf.float16, tf.float32, tf.float64):
action = tf.check_numerics(action, 'action')
observ_dtype = self._parse_dtype(self._env.observation_space)
observ, reward, done = tf.py_func(
lambda a: self._env.step(a)[:3], [action],
[observ_dtype, tf.float32, tf.bool], name='step')
observ = tf.check_numerics(observ, 'observ')
reward = tf.check_numerics(reward, 'reward')
return tf.group(
self._observ.assign(observ),
self._action.assign(action),
self._reward.assign(reward),
self._done.assign(done),
self._step.assign_add(1))
def log_prob(self, xs, zs):
"""
Parameters
----------
xs : any
A batch of data points, as any data type the user interfaces with
when defining this method.
zs : list or tf.Tensor
A list of tf.Tensor's if multiple varational families,
otherwise a tf.Tensor if single variational family.
Returns
-------
tf.Tensor
S-vector of type tf.float32,
[log p(xs, zs[1,:]), .., log p(xs, zs[S,:])].
Notes
-----
It wraps around a Python function. The Python function takes
as input zs of type np.ndarray, and outputs a np.ndarray.
"""
# Store data in order to later pass data to Python function.
self.xs = xs
return tf.py_func(self._py_log_prob_z, [zs], [tf.float32])[0]
def log_prob(self, xs, zs):
"""
Parameters
----------
xs : dict
Dictionary defining the observations according to the data
block of the Stan program.
zs : list or tf.Tensor
A list of tf.Tensor's if multiple varational families,
otherwise a tf.Tensor if single variational family.
Returns
-------
tf.Tensor
S-vector of type tf.float32,
[log p(xs, zs[1,:]), .., log p(xs, zs[S,:])].
Notes
-----
It wraps around a Python function. The Python function takes
as input zs of type np.ndarray, and outputs a np.ndarray.
"""
if self.flag_init is False:
self._initialize(xs)
return tf.py_func(self._py_log_prob, [zs], [tf.float32])[0]
def call(*args):
kwargs = dict(
zip(function_utils.fn_args(getattr(self._type, name))[1:], args))
specs = self._type._tensor_specs(name, kwargs, self._constructor_kwargs)
if specs is None:
raise ValueError(
'No tensor specifications were provided for: %s' % name)
flat_dtypes = nest.flatten(nest.map_structure(lambda s: s.dtype, specs))
flat_shapes = nest.flatten(nest.map_structure(lambda s: s.shape, specs))
def py_call(*args):
try:
self._out.send(args)
result = self._out.recv()
if isinstance(result, Exception):
raise result
if result is not None:
return result
except Exception as e:
if isinstance(e, IOError):
raise StopIteration() # Clean exit.
else:
raise
result = tf.py_func(py_call, (name,) + tuple(args), flat_dtypes,
name=name)
if isinstance(result, tf.Operation):
return result
for t, shape in zip(result, flat_shapes):
t.set_shape(shape)
return nest.pack_sequence_as(specs, result)
def chebyshev2(self, x, L, Fout, K):
"""
Filtering with Chebyshev interpolation
Implementation: numpy.
Data: x of size N x M x F
N: number of signals
M: number of vertices
F: number of features per signal per vertex
"""
N, M, Fin = x.get_shape()
N, M, Fin = int(N), int(M), int(Fin)
# Rescale Laplacian. Copy to not modify the shared L.
L = scipy.sparse.csr_matrix(L)
L = graph.rescale_L(L, lmax=2)
# Transform to Chebyshev basis
x = tf.transpose(x, perm=[1, 2, 0]) # M x Fin x N
x = tf.reshape(x, [M, Fin*N]) # M x Fin*N
def chebyshev(x):
return graph.chebyshev(L, x, K)
x = tf.py_func(chebyshev, [x], [tf.float32])[0] # K x M x Fin*N
x = tf.reshape(x, [K, M, Fin, N]) # K x M x Fin x N
x = tf.transpose(x, perm=[3,1,2,0]) # N x M x Fin x K
x = tf.reshape(x, [N*M, Fin*K]) # N*M x Fin*K
# Filter: Fin*Fout filters of order K, i.e. one filterbank per feature.
W = self._weight_variable([Fin*K, Fout], regularization=False)
x = tf.matmul(x, W) # N*M x Fout
return tf.reshape(x, [N, M, Fout]) # N x M x Fout
def test_random_transform(image_in, min_scale=0.5, max_scale=1.0, max_rotation=22.5):
"""
Scales the image between min_scale and max_scale
"""
img_shape = [299,299,3]
width = img_shape[0]
def _random_transformation():
im_scale = np.random.uniform(low=min_scale, high=1.0)
padding_after_scaling = (1-im_scale) * width
x_delta = np.random.uniform(-padding_after_scaling, padding_after_scaling)
y_delta = np.random.uniform(-padding_after_scaling, padding_after_scaling)
rot = np.random.uniform(-max_rotation, max_rotation)
return _transform_vector(width,
x_shift=x_delta,
y_shift=y_delta,
im_scale=im_scale,
rot_in_degrees=rot)
random_xform_vector = tf.py_func(_random_transformation, [], tf.float32)
random_xform_vector.set_shape([8])
output = tf.contrib.image.transform(image_in, random_xform_vector , "BILINEAR")
return output
def add_cdf_image_summary(values, name):
"""Adds a tf.summary.image for a CDF plot of the values.
Normalizes `values` such that they sum to 1, plots the cumulative distribution
function and creates a tf image summary.
Args:
values: a 1-D float32 tensor containing the values.
name: name for the image summary.
"""
def cdf_plot(values):
"""Numpy function to plot CDF."""
normalized_values = values / np.sum(values)
sorted_values = np.sort(normalized_values)
cumulative_values = np.cumsum(sorted_values)
fraction_of_examples = (np.arange(cumulative_values.size, dtype=np.float32)
/ cumulative_values.size)
fig = plt.figure(frameon=False)
ax = fig.add_subplot('111')
ax.plot(fraction_of_examples, cumulative_values)
ax.set_ylabel('cumulative normalized values')
ax.set_xlabel('fraction of examples')
fig.canvas.draw()
width, height = fig.get_size_inches() * fig.get_dpi()
image = np.fromstring(fig.canvas.tostring_rgb(), dtype='uint8').reshape(
1, int(height), int(width), 3)
return image
cdf_plot = tf.py_func(cdf_plot, [values], tf.uint8)
tf.summary.image(name, cdf_plot)
def read_and_augment_data(image_list, label_list, image_size, batch_size, max_nrof_epochs,
random_crop, random_flip, random_rotate, nrof_preprocess_threads, shuffle=True):
images = ops.convert_to_tensor(image_list, dtype=tf.string)
labels = ops.convert_to_tensor(label_list, dtype=tf.int32)
# Makes an input queue
input_queue = tf.train.slice_input_producer([images, labels],
num_epochs=max_nrof_epochs, shuffle=shuffle)
images_and_labels = []
for _ in range(nrof_preprocess_threads):
image, label = read_images_from_disk(input_queue)
if random_rotate:
image = tf.py_func(random_rotate_image, [image], tf.uint8)
if random_crop:
image = tf.random_crop(image, [image_size, image_size, 3])
else:
image = tf.image.resize_image_with_crop_or_pad(image, image_size, image_size)
if random_flip:
image = tf.image.random_flip_left_right(image)
#pylint: disable=no-member
image.set_shape((image_size, image_size, 3))
image = tf.image.per_image_standardization(image)
images_and_labels.append([image, label])
image_batch, label_batch = tf.train.batch_join(
images_and_labels, batch_size=batch_size,
capacity=4 * nrof_preprocess_threads * batch_size,
allow_smaller_final_batch=True)
return image_batch, label_batch
def add_hist_image_summary(values, bins, name):
"""Adds a tf.summary.image for a histogram plot of the values.
Plots the histogram of values and creates a tf image summary.
Args:
values: a 1-D float32 tensor containing the values.
bins: bin edges which will be directly passed to np.histogram.
name: name for the image summary.
"""
def hist_plot(values, bins):
"""Numpy function to plot hist."""
fig = plt.figure(frameon=False)
ax = fig.add_subplot('111')
y, x = np.histogram(values, bins=bins)
ax.plot(x[:-1], y)
ax.set_ylabel('count')
ax.set_xlabel('value')
fig.canvas.draw()
width, height = fig.get_size_inches() * fig.get_dpi()
image = np.fromstring(
fig.canvas.tostring_rgb(), dtype='uint8').reshape(
1, int(height), int(width), 3)
return image
hist_plot = tf.py_func(hist_plot, [values, bins], tf.uint8)
tf.summary.image(name, hist_plot)
def print_op(input, message):
"""
Inserts a print statement into the TensorFlow graph.
Parameters
----------
input : ``tf.Tensor``
The value of this tensor will be printed whenever it is computed
in the graph
message : str
String prepended to the value of ``input``, to help with logging
Returns
-------
op : ``tf.Tensor``
New tensor representing the print operation applied to ``input``
Notes
-----
This is what ``tf.Print`` is supposed to do, but it doesn't seem to work
consistently.
"""
def print_func(x): # pragma: no cover
print(message, str(x))
return x
with tf.device("/cpu:0"):
output = tf.py_func(print_func, [input], input.dtype)
output.set_shape(input.get_shape())
return output
def _build_features_dataset(self, features_source):
max_len = self._max_len
vocab = self._vocab
tokenizer = self._tokenizer
num_parallel_calls = self._num_parallel_calls
dataset = tf.data.TextLineDataset(features_source)
dataset = dataset.map(lambda text: tokenizer(text),
num_parallel_calls=num_parallel_calls)
dataset = dataset.map(lambda tokens: tokens[:max_len],
num_parallel_calls=num_parallel_calls)
dataset = dataset.map(lambda tokens: tf.cast(vocab.lookup(tokens), tf.int32),
num_parallel_calls=num_parallel_calls)
def pad_(x):
ids = np.zeros(max_len, dtype=np.int32)
ids[:x.shape[0]] = x
return ids
dataset = dataset.map(lambda x: tf.py_func(pad_, [x], [x.dtype]), num_parallel_calls)
dataset = dataset.map(lambda token_ids: {'ids': token_ids, 'length': tf.size(token_ids)},
num_parallel_calls=num_parallel_calls)
dataset = dataset.map(lambda x: {'ids': tf.reshape(x['ids'], [self._max_len]), 'length': x['length']})
return dataset
def testSideEffect(self):
a = tf.constant(1)
b = tf.constant(1)
c = tf.add(a, b)
with tf.control_dependencies([c]):
d = tf.constant(42)
n = tf.neg(c)
shared = []
def sub(t):
shared.append(t)
return t
c = subscribe.subscribe(c, lambda t: tf.py_func(sub, [t], [t.dtype]))
with self.test_session() as sess:
c_out = sess.run([c])
n_out = sess.run([n])
d_out = sess.run([d])
self.assertEquals(n_out, [-2])
self.assertEquals(c_out, [2])
self.assertEquals(d_out, [42])
self.assertEquals(shared, [2, 2, 2])
def cost_grad(op, grad): #print op output = op.inputs[0]; phi = op.inputs[1]; grad = tf.py_func(self.cost_func_grad, [output, phi], [tf.float32])[0]; #return [self.cost_func_grad(output, phi_in, epsilon = 0.01), np.zeros((phi_in.shape))]; return [grad, None];
def sequence_to_pianoroll_op(sequence_tensor, velocity_range_tensor, hparams):
"""Transforms a serialized NoteSequence to a pianoroll."""
def sequence_to_pianoroll_fn(sequence_tensor, velocity_range_tensor):
"""Converts sequence to pianorolls."""
velocity_range = music_pb2.VelocityRange.FromString(velocity_range_tensor)
sequence = music_pb2.NoteSequence.FromString(sequence_tensor)
sequence = sequences_lib.apply_sustain_control_changes(sequence)
roll = sequences_lib.sequence_to_pianoroll(
sequence,
frames_per_second=hparams_frames_per_second(hparams),
min_pitch=constants.MIN_MIDI_PITCH,
max_pitch=constants.MAX_MIDI_PITCH,
min_frame_occupancy_for_label=hparams.min_frame_occupancy_for_label,
onset_mode=hparams.onset_mode,
onset_length_ms=hparams.onset_length,
offset_length_ms=hparams.offset_length,
onset_delay_ms=hparams.onset_delay,
min_velocity=velocity_range.min,
max_velocity=velocity_range.max)
return (roll.active, roll.weights, roll.onsets, roll.onset_velocities,
roll.offsets)
res, weighted_res, onsets, velocities, offsets = tf.py_func(
sequence_to_pianoroll_fn, [sequence_tensor, velocity_range_tensor],
[tf.float32, tf.float32, tf.float32, tf.float32, tf.float32],
name='sequence_to_pianoroll_op')
res.set_shape([None, constants.MIDI_PITCHES])
weighted_res.set_shape([None, constants.MIDI_PITCHES])
onsets.set_shape([None, constants.MIDI_PITCHES])
offsets.set_shape([None, constants.MIDI_PITCHES])
velocities.set_shape([None, constants.MIDI_PITCHES])
return res, weighted_res, onsets, offsets, velocities
def create_metric_ops(self, _inputs, labels, predictions):
"""Creates (value, update_op) tensors
"""
with tf.variable_scope(self._name):
# Join tokens into single strings
predictions_flat = tf.reduce_join(
predictions["predicted_tokens"], 1, separator=self._separator)
labels_flat = tf.reduce_join(
labels["target_tokens"], 1, separator=self._separator)
sources_value, sources_update = accumulate_strings(
values=predictions_flat, name="sources")
targets_value, targets_update = accumulate_strings(
values=labels_flat, name="targets")
metric_value = tf.py_func(
func=self._py_func,
inp=[sources_value, targets_value],
Tout=tf.float32,
name="value")
with tf.control_dependencies([sources_update, targets_update]):
update_op = tf.identity(metric_value, name="update_op")
return metric_value, update_op
def _target_generate(self):
self._rois, self._roi_labels, self._target_regress, self._rcnn_reg_mask = \
tf.py_func(rcnn_target_proposal,
[self._rpn_boxes],
[tf.float32, tf.float32, tf.float32, tf.float32])
def train(self,
train_record,
val_record,
base_record,
novel_record,
output_path,
batch_size=16,
optimizer='adam',
lr_init=1e-4,
decay_every=250000,
clip_gradients=5.0,
lambda_x=1,
lambda_r=0.001,
lambda_g=0,
lambda_f=0,
cnn_trainable=False,
n_read_threads=5,
n_stage_threads=5,
capacity=16,
epoch_size=10000,
save_every=10000,
validate_every=100,
ckpt=None,
cnn_ckpt='pretrained/inception_v3_weights.h5',
inds_to_words_json='/home/paperspace/data/ms_coco/SOS_preproc_vocab-9413_threshold-5_length-20/inds_to_words.json',
):
''''''
with tf.variable_scope('BatchReader'):
print('creating batch reader')
# coordinator for tf queues
coord = tf.train.Coordinator()
# read and preprocess training record
x_train, y_train, cls_train, id_train = self.read_tfrecord(train_record,
batch_size=batch_size, capacity=capacity, n_threads=n_read_threads)
# read and preprocess validation record
x_val, y_val, cls_val, id_val = self.read_tfrecord(val_record,
batch_size=batch_size, capacity=1, n_threads=1)
# base classes
x_base, y_base, cls_base, id_base = self.read_tfrecord(base_record,
batch_size=batch_size, capacity=1, n_threads=1)
# novel classes
x_novel, y_novel, cls_novel, id_novel = self.read_tfrecord(novel_record,
batch_size=batch_size, capacity=1, n_threads=1)
with tf.variable_scope('StagingArea'):
print('creating staging area')
# create training queue on GPU
x_train, y_train = self.stage_data([x_train, y_train],
memory_gb=1, n_threads=n_stage_threads)
# create validation queue on GPU
x_val, y_val, x_base, y_base, x_novel, y_novel = self.stage_data(
[x_val, y_val, x_base, y_base, x_novel, y_novel],
memory_gb=0.1, n_threads=1)
# global step and learning phase flag
step = tf.Variable(0, name='step')
update_step = tf.assign_add(step, 1)
learning = backend.learning_phase()
with tf.variable_scope('Optimizer'):
print('creating optimizer')
# get variables
G_vars = self.gen.trainable_variables
if cnn_trainable:
G_vars += self.phi.trainable_variables
# generated caption given previous words
y_hat = self.G(x_train, y_train)
# generated caption given only "start" token
y_hat_cont = self.sample_recursively(x_train, continuous=True)
# xentropy between real & generated next words
L_x = self.xent(x_train, y_train,
sparse=(False if lambda_g else True))
# weight regularizer
L_r = self.weight_norm(self.gen.trainable_variables)
# feature vector regularizer
L_f = tf.reduce_mean(tf.reduce_sum(self.phi(x_train)**2, axis=-1))
# xentropy plus regularizers
L_G = lambda_x*L_x + lambda_r*L_r + lambda_f*L_f
# gradient penalty from arxiv.org/abs/1606.02819
grad_norm = self.weight_norm(tf.gradients(L_G, G_vars))
if lambda_g:
L_G += lambda_g*grad_norm
# learning rate with periodic halving
lr = lr_init*0.5**tf.floor(tf.cast(step, tf.float32)/decay_every)
# optimizer: one of {"adam", "sgd"}
if optimizer == 'adam':
G_opt = tf.train.AdamOptimizer(lr, beta1=0.5)
elif optimizer == 'sgd':
G_opt = tf.train.GradientDescentOptimizer(lr)
else:
raise ValueError, 'optimizer can be "adam" or "sgd"'
# clip by L2 norm, if specified
G_grad = G_opt.compute_gradients(L_G, var_list=G_vars)
if clip_gradients:
G_grad = self.clip_gradients_by_norm(G_grad, clip_gradients)
G_opt = G_opt.apply_gradients(G_grad)
with tf.variable_scope('Summary'):
print('creating summary')
# validation loss terms
L_x_val = self.xent(x_val, y_val)
L_x_base = self.xent(x_base, y_base)
L_x_novel = self.xent(x_novel, y_novel)
L_f_val = tf.reduce_mean(tf.reduce_sum(self.phi(x_val)**2, axis=-1))
# validation generated captions
y_hat_val = self.G(x_val, y_val)
y_hat_base = self.G(x_base, y_base)
y_hat_novel = self.G(x_novel, y_novel)
y_hat_samp_base = self.sample_recursively(x_base, continuous=False)
y_hat_samp_novel = self.sample_recursively(x_novel, continuous=False)
# decode predictions to words
table = self.create_table(inds_to_words_json)
real_base = self.postproc_caption(y_base, table)[0]
real_novel = self.postproc_caption(y_novel, table)[0]
fake_base = self.postproc_caption(y_hat_samp_base, table)[0]
fake_novel = self.postproc_caption(y_hat_samp_novel, table)[0]
txtfile = os.path.join(output_path, 'output.txt')
def write_func(lxv, rb, fb, rn, fn):
lxv = float(lxv)
rb, fb, rn, fn = [str(s) for s in [rb, fb, rn, fn]]
st = '\n'.join([
'Loss: {}',
'Real base: "{}"',
'Fake base: "{}"',
'Real novel: "{}"',
'Fake novel: "{}"\n'])\
.format(lxv, rb, fb, rn, fn)\
.replace('SOS ', '').replace(' EOS', '')
print(st, file=open(txtfile, 'a'))
return 0
printer = tf.py_func(write_func,
[L_x_val, real_base, fake_base, real_novel, fake_novel], Tout=tf.int64)
# associate scalars with display names
scalar_dict = {'L_x_train': L_x,
'L_x_val': L_x_val,
'L_x_base': L_x_base,
'L_x_novel': L_x_novel,
'L_f': L_f_val,
'L_r': L_r,
'lr': lr,
'printer': printer}
# summarize
summary, writer = self.make_summary(
output_path, scalar_dict=scalar_dict)
# start the session
with tf.Session(graph=self.graph) as sess:
print('initializing variables')
sess.run(tf.global_variables_initializer())
if save_every < np.inf:
print('saving weights')
self.save_h5(output_path, 0)
if ckpt:
print('loading weights from '+ckpt)
self.load_weights(ckpt)
elif cnn_ckpt:
print('loading cnn weights from '+cnn_ckpt)
self.phi.load_weights(cnn_ckpt)
print('starting threads')
tf.train.start_queue_runners(sess=sess, coord=coord)
# batch norm EMA updates
if cnn_trainable:
_x = layers.Input(tensor=x_train, shape=x_train.shape)
super(BaseModel, self).__init__(
[_x] + self.gen.inputs,
[self.phi(_x)] + self.gen.outputs)
G_opt = tf.group(G_opt, *self.updates)
print('finalizing graph')
self.graph.finalize()
try:
epoch = 1
while True:
print('epoch: ' + str(epoch)); epoch += 1
time.sleep(0.2) # wait for print buffer to flush
for _ in tqdm.trange(epoch_size, disable=False):
# check for dead threads
if not coord.should_stop():
# update generator
_, n = sess.run([G_opt, update_step],
{learning: True})
# validate
if n % validate_every == 1:
s = sess.run(summary, {learning: False})
writer.add_summary(s, n)
writer.flush()
# save checkpoint
if n % save_every == 0:
self.save_h5(output_path, n)
else:
raise IOError, 'queues closed'
# exit behaviour: request thread stop, then wait for
# them to recieve message before exiting session context
except:
coord.request_stop()
time.sleep(0.2)
raise
def build(self):
"""
build network architecture and loss
"""
"""
Visual features
"""
with tf.device('/cpu:0'):
def load_feature(image_idx):
selected_features = np.take(self.features, image_idx, axis=0)
return selected_features
V_ft = tf.py_func(
load_feature, inp=[self.batch['image_idx']], Tout=tf.float32,
name='sample_features')
V_ft.set_shape([None, self.max_box_num, self.vfeat_dim])
num_V_ft = tf.gather(self.num_boxes, self.batch['image_idx'],
name='gather_num_V_ft', axis=0)
self.mid_result['num_V_ft'] = num_V_ft
normal_boxes = tf.gather(self.normal_boxes, self.batch['image_idx'],
name='gather_normal_boxes', axis=0)
self.mid_result['normal_boxes'] = normal_boxes
log.warning('v_linear_v')
v_linear_v = modules.fc_layer(
V_ft, V_DIM, use_bias=True, use_bn=False, use_ln=True,
activation_fn=tf.nn.relu, is_training=self.is_train,
scope='v_linear_v')
"""
Encode question
"""
q_embed = tf.nn.embedding_lookup(self.glove_map, self.batch['q_intseq'])
# [bs, L_DIM]
q_L_ft = modules.encode_L(q_embed, self.batch['q_intseq_len'], L_DIM,
cell_type='GRU')
# [bs, V_DIM}
log.warning('q_linear_v')
q_linear_v = modules.fc_layer(
q_L_ft, V_DIM, use_bias=True, use_bn=False, use_ln=True,
activation_fn=tf.nn.relu, is_training=self.is_train,
scope='q_linear_v')
self.mid_result['q_linear_v'] = q_linear_v
"""
Perform attention
"""
v_adapt = modules.fc_layer(
V_ft, V_DIM, use_bias=True, use_bn=False, use_ln=True,
activation_fn=tf.nn.relu, is_training=self.is_train,
scope='v_adapt')
att_score = modules.hadamard_attention(
v_linear_v, num_V_ft, q_linear_v,
use_ln=False, is_train=self.is_train)
self.output['att_score'] = att_score
self.mid_result['att_score'] = att_score
pooled_V_ft = modules.attention_pooling(v_adapt, att_score)
self.mid_result['pooled_V_ft'] = pooled_V_ft
"""
Answer classification
"""
log.warning('pooled_linear_l')
pooled_linear_l = modules.fc_layer(
pooled_V_ft, L_DIM, use_bias=True, use_bn=False, use_ln=True,
activation_fn=tf.nn.relu, is_training=self.is_train,
scope='pooled_linear_l')
self.mid_result['pooled_linear_l'] = pooled_linear_l
log.warning('q_linear_l')
l_linear_l = modules.fc_layer(
q_L_ft, L_DIM, use_bias=True, use_bn=False, use_ln=True,
activation_fn=tf.nn.relu, is_training=self.is_train,
scope='q_linear_l')
self.mid_result['l_linear_l'] = l_linear_l
joint = modules.fc_layer(
pooled_linear_l * l_linear_l, L_DIM * 2,
use_bias=True, use_bn=False, use_ln=True,
activation_fn=tf.nn.relu, is_training=self.is_train, scope='joint_fc')
joint = tf.nn.dropout(joint, 0.5)
self.mid_result['joint'] = joint
logit = modules.WordWeightAnswer(
joint, self.answer_dict, self.word_weight_dir,
use_bias=True, is_training=self.is_train, scope='WordWeightAnswer')
self.output['logit'] = logit
self.mid_result['logit'] = logit
"""
Compute loss and accuracy
"""
with tf.name_scope('loss'):
answer_target = self.batch['answer_target']
loss = tf.nn.sigmoid_cross_entropy_with_logits(
labels=answer_target, logits=logit)
train_loss = tf.reduce_mean(tf.reduce_sum(
loss * self.train_answer_mask, axis=-1))
report_loss = tf.reduce_mean(tf.reduce_sum(loss, axis=-1))
pred = tf.cast(tf.argmax(logit, axis=-1), dtype=tf.int32)
one_hot_pred = tf.one_hot(pred, depth=self.num_answer,
dtype=tf.float32)
acc = tf.reduce_mean(
tf.reduce_sum(one_hot_pred * answer_target, axis=-1))
exist_acc = tf.reduce_mean(
tf.reduce_sum(one_hot_pred * answer_target * self.answer_exist_mask,
axis=-1))
test_acc = tf.reduce_mean(
tf.reduce_sum(one_hot_pred * answer_target * self.test_answer_mask,
axis=-1))
max_exist_answer_acc = tf.reduce_mean(
tf.reduce_max(answer_target * self.answer_exist_mask, axis=-1))
test_max_answer_acc = tf.reduce_mean(
tf.reduce_max(answer_target * self.test_answer_mask, axis=-1))
test_max_exist_answer_acc = tf.reduce_mean(
tf.reduce_max(answer_target * self.answer_exist_mask *
self.test_answer_mask, axis=-1))
normal_test_acc = tf.where(
tf.equal(test_max_answer_acc, 0),
test_max_answer_acc,
test_acc / test_max_answer_acc)
self.mid_result['pred'] = pred
self.losses['answer'] = train_loss
self.report['answer_train_loss'] = train_loss
self.report['answer_report_loss'] = report_loss
self.report['answer_accuracy'] = acc
self.report['exist_answer_accuracy'] = exist_acc
self.report['test_answer_accuracy'] = test_acc
self.report['normal_test_answer_accuracy'] = normal_test_acc
self.report['max_exist_answer_accuracy'] = max_exist_answer_acc
self.report['test_max_answer_accuracy'] = test_max_answer_acc
self.report['test_max_exist_answer_accuracy'] = test_max_exist_answer_acc
"""
Prepare image summary
"""
"""
with tf.name_scope('prepare_summary'):
self.vis_image['image_attention_qa'] = self.visualize_vqa_result(
self.batch['image_id'],
self.mid_result['normal_boxes'], self.mid_result['num_V_ft'],
self.mid_result['att_score'],
self.batch['q_intseq'], self.batch['q_intseq_len'],
self.batch['answer_target'], self.mid_result['pred'],
max_batch_num=20, line_width=2)
"""
self.loss = 0
for key, loss in self.losses.items():
self.loss = self.loss + loss
# scalar summary
for key, val in self.report.items():
tf.summary.scalar('train/{}'.format(key), val,
collections=['heavy_train', 'train'])
tf.summary.scalar('val/{}'.format(key), val,
collections=['heavy_val', 'val'])
tf.summary.scalar('testval/{}'.format(key), val,
collections=['heavy_testval', 'testval'])
# image summary
for key, val in self.vis_image.items():
tf.summary.image('train-{}'.format(key), val, max_outputs=10,
collections=['heavy_train'])
tf.summary.image('val-{}'.format(key), val, max_outputs=10,
collections=['heavy_val'])
tf.summary.image('testval-{}'.format(key), val, max_outputs=10,
collections=['heavy_testval'])
return self.loss
def __init__(self, mode="train"):
# Set phase
is_training=True if mode=="train" else False
# Input, Output Placeholder
# x: int text seq [batch_size, seq_len]
# y: reduced mel [batch_size, seq_len//r, n_mels*r]
# z: mag [batch_size, seq_len, 1+n_fft//2]
if mode=='train':
self.x, self.y, self.z, self.fnames, self.num_batch = get_batch(mode)
elif mode=='infer':
self.x = tf.placeholder(tf.int32, shape=(None, None))
self.y = tf.placeholder(tf.float32, shape=(None, None, hp.n_mels*hp.r))
else:
self.x = tf.placeholder(tf.int32, shape=(None, None))
self.y = tf.placeholder(tf.float32, shape=(None, None, hp.n_mels*hp.r))
self.z = tf.placeholder(tf.float32, shape=(None, None, 1+hp.n_fft//2))
self.fnames = tf.placeholder(tf.string, shape=(None,))
self.encoder_inputs = embed(self.x, len(hp.char_set), hp.embed_size)
self.decoder_inputs = tf.concat((tf.zeros_like(self.y[:, :1, :]), self.y[:, :-1, :]), 1)
self.decoder_inputs = self.decoder_inputs[:, :, -hp.n_mels:] # feed last frames only (N, Ty/r, n_mels)
# Network
with tf.variable_scope("ref_encoder"):
# ref_emb = [batch, hp.ref_enc_gru_size]
self.ref_emb = build_ref_encoder(self.y, is_training)
with tf.variable_scope("STL_layer"):
# style_emb = [batch, hp.token_emb_size]
self.style_emb, self.GST = build_STL(self.ref_emb)
with tf.variable_scope("encoder"):
# memory = [batch_size, seq_len, 2*gru_size=embed_size]
self.memory, self.encoder_final_state = build_encoder(
self.encoder_inputs,
is_training=is_training
)
# fusing style embedding into encoder outputs for decoder's attention
seq_len = tf.shape(self.x)[1]
self.memory += tf.tile(tf.expand_dims(self.style_emb, axis=1), [1, seq_len, 1])
with tf.variable_scope("decoder1"):
# y_hat = [batch_size, seq_len//r, n_mels*r]
# alignment = [batch_size, encoder_timesteps, decoder_timesteps]
self.y_hat, self.alignments = build_decoder1(
self.decoder_inputs, self.memory, self.encoder_final_state, is_training=is_training
)
with tf.variable_scope("decoder2"):
# z_hat = [batch_size, seq_len, (1+n_fft//2)]
self.z_hat = build_decoder2(self.y_hat, is_training=is_training)
'''
print(self.x.shape, self.y.shape, self.z.shape)
print(self.y_hat.shape, self.z_hat.shape)
print(self.encoder_inputs.shape, self.decoder_inputs.shape)
exit()
'''
'''
vs = [v for v in tf.trainable_variables()]
for v in vs : print(v)
exit()
'''
if mode in ("train", "eval"):
# monitor
self.audio_hat = tf.py_func(spectrogram2wav, [self.z_hat[0]], tf.float32)
self.audio_gt = tf.py_func(spectrogram2wav, [self.z[0]], tf.float32)
# Loss
## mel loss
self.loss1 = tf.reduce_mean(tf.abs(self.y_hat - self.y))
## mag loss
self.loss2 = tf.reduce_mean(tf.abs(self.z_hat - self.z))
## guided attention
batch_size, N, T = tf.shape(self.alignments)[0], tf.shape(self.alignments)[1], tf.shape(self.alignments)[2]
g = 0.2
Ns = tf.tile(tf.expand_dims(tf.range(N)/N, 1), [1, T]) # shape: [N, T]
Ts = tf.tile(tf.expand_dims(tf.range(T)/T, 0), [N, 1]) # shape: [N, T]
W = tf.ones([N, T]) - tf.exp(-1*(tf.cast(tf.square(Ns - Ts), tf.float32) / (2*tf.square(g))))
nearly_diagonal_constraint = tf.multiply(self.alignments, tf.tile(tf.expand_dims(W, 0), [batch_size, 1, 1]))
self.guided_attn_loss = tf.reduce_mean(nearly_diagonal_constraint)
## total loss
self.loss = self.loss1 + self.loss2 + self.guided_attn_loss
# Training Scheme
self.global_step = tf.Variable(0, name='global_step', trainable=False)
self.lr = learning_rate_decay(hp.lr, global_step=self.global_step)
self.optimizer = tf.train.AdamOptimizer(learning_rate=self.lr)
# gradient clipping
self.gvs = self.optimizer.compute_gradients(self.loss)
self.clipped = []
for grad, var in self.gvs:
grad = tf.clip_by_norm(grad, 5.)
self.clipped.append((grad, var))
self.train_op = self.optimizer.apply_gradients(
self.clipped,
global_step=self.global_step
)
# Summary
tf.summary.scalar('{}/loss1'.format(mode), self.loss1)
tf.summary.scalar('{}/loss2'.format(mode), self.loss2)
tf.summary.scalar('{}/guided_attn_loss'.format(mode), self.guided_attn_loss)
tf.summary.scalar('{}/loss'.format(mode), self.loss)
tf.summary.scalar('{}/lr'.format(mode), self.lr)
tf.summary.image("{}/mel_gt".format(mode),
tf.expand_dims(self.y, -1), max_outputs=1)
tf.summary.image("{}/mel_hat".format(mode),
tf.expand_dims(self.y_hat, -1), max_outputs=1)
tf.summary.image("{}/mag_gt".format(mode),
tf.expand_dims(self.z, -1), max_outputs=1)
tf.summary.image("{}/mag_hat".format(mode),
tf.expand_dims(self.z_hat, -1), max_outputs=1)
tf.summary.audio("{}/sample_hat".format(mode),
tf.expand_dims(self.audio_hat, 0), hp.sr)
tf.summary.audio("{}/sample_gt".format(mode),
tf.expand_dims(self.audio_gt, 0), hp.sr)
self.summary_op = tf.summary.merge_all()
# init
self.init_op = tf.global_variables_initializer()
def reset(self, indices=None, max_frames=None, name=None):
if indices is None:
indices = np.arange(self.batch_size)
with tf.variable_scope(name, default_name='PythonReset'):
return tf.py_func(self._reset, [indices], tf.int64).op
def prepare_for_unique_top_k(D, A):
indices_duplicated = tf.py_func(find_duplicate_columns, [A], tf.int32)
D += tf.reduce_max(D) * tf.cast(indices_duplicated, tf.float32)
def get_loss_pre_metrics(x, y, is_training, batch_size, args):
# 恢复图像
images = tf.cast(x, tf.uint8)
# 前向传播
logits = tf.cond(is_training,
true_fn=lambda: deeplab_v3.deeplab_v3(
x, args, is_training=True, reuse=False),
false_fn=lambda: deeplab_v3.deeplab_v3(
x, args, is_training=False, reuse=True))
pred_classes = tf.expand_dims(tf.argmax(logits,
axis=3,
output_type=tf.int32),
axis=3)
# 解码预测结果
pred_decoded_labels = tf.py_func(
preprocessing.decode_labels,
[pred_classes, batch_size, args.number_of_classes], tf.uint8)
# 解码标签
gt_decoded_labels = tf.py_func(preprocessing.decode_labels,
[y, batch_size, args.number_of_classes],
tf.uint8)
tf.summary.image(
'images',
tf.concat(axis=2,
values=[images, gt_decoded_labels, pred_decoded_labels]),
max_outputs=args.tensorboard_images_max_outputs)
# 求loss
labels = tf.squeeze(y, axis=3) # reduce the channel dimension.
logits_by_num_classes = tf.reshape(logits, [-1, args.number_of_classes])
labels_flat = tf.reshape(labels, [
-1,
])
cross_entropy = tf.losses.sparse_softmax_cross_entropy(
logits=logits_by_num_classes, labels=labels_flat)
if not args.freeze_batch_norm:
train_var_list = [v for v in tf.trainable_variables()]
else:
train_var_list = [
v for v in tf.trainable_variables()
if 'beta' not in v.name and 'gamma' not in v.name
]
global_step = tf.train.get_or_create_global_step()
with tf.variable_scope("total_loss"):
loss = cross_entropy + _WEIGHT_DECAY * tf.add_n(
[tf.nn.l2_loss(v) for v in train_var_list])
# affinity loss
edge_loss, not_edge_loss = affinity_loss(
labels=y,
probs=logits,
num_classes=args.number_of_classes,
kld_margin=args.kld_margin)
dec = tf.pow(10.0, tf.cast(-global_step / args.max_iter, tf.float32))
aff_loss = tf.reduce_mean(edge_loss) * args.kld_lambda_1 * dec
aff_loss += tf.reduce_mean(not_edge_loss) * args.kld_lambda_2 * dec
total_loss = loss + aff_loss
tf.summary.scalar('loss', total_loss)
# 优化函数
learning_rate = tf.train.polynomial_decay(
args.initial_learning_rate,
tf.cast(global_step, tf.int32) - args.initial_global_step,
args.max_iter,
args.end_learning_rate,
power=0.9) # args.max_iter = 30000 args.initial_global_step=0
tf.summary.scalar('learning_rate', learning_rate)
optimizer = tf.train.MomentumOptimizer(learning_rate=learning_rate,
momentum=0.9)
# Batch norm requires update ops to be added as a dependency to the train_op
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
with tf.control_dependencies(update_ops):
train_op = optimizer.minimize(total_loss,
global_step,
var_list=train_var_list)
# metrics
preds_flat = tf.reshape(pred_classes, [
-1,
])
confusion_matrix = tf.confusion_matrix(labels_flat,
preds_flat,
num_classes=args.number_of_classes)
correct_pred = tf.equal(preds_flat, labels_flat)
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))
tf.summary.scalar('accuracy', accuracy)
def compute_mean_iou(total_cm, name='mean_iou'):
"""Compute the mean intersection-over-union via the confusion matrix."""
sum_over_row = tf.to_float(tf.reduce_sum(total_cm, 0))
sum_over_col = tf.to_float(tf.reduce_sum(total_cm, 1))
cm_diag = tf.to_float(tf.diag_part(total_cm))
denominator = sum_over_row + sum_over_col - cm_diag
# The mean is only computed over classes that appear in the
# label or prediction tensor. If the denominator is 0, we need to
# ignore the class.
num_valid_entries = tf.reduce_sum(
tf.cast(tf.not_equal(denominator, 0), dtype=tf.float32))
# If the value of the denominator is 0, set it to 1 to avoid
# zero division.
denominator = tf.where(tf.greater(denominator, 0), denominator,
tf.ones_like(denominator))
iou = tf.div(cm_diag, denominator)
for i in range(args.number_of_classes):
tf.identity(iou[i], name='train_iou_class{}'.format(i))
tf.summary.scalar('train_iou_class{}'.format(i), iou[i])
# If the number of valid entries is 0 (no classes) we return 0.
result = tf.where(tf.greater(num_valid_entries, 0),
tf.reduce_sum(iou, name=name) / num_valid_entries, 0)
return result
mean_iou = compute_mean_iou(confusion_matrix)
tf.summary.scalar('mean_iou', mean_iou)
metrics = {
'px_accuracy': accuracy,
'mean_iou': mean_iou,
'confusion_matrix': confusion_matrix
}
return total_loss, train_op, metrics
def input_fn():
flat_example = tf.py_func(py_func, [], types)
_ = [t.set_shape(shape) for t, shape in zip(flat_example, shapes)]
example = tf.contrib.framework.nest.pack_sequence_as(
first_ex, flat_example)
return example
def preprocessor(self, text):
return tf.py_func(
lambda t: np.asarray([
self.word_to_idx.get(x, self.unknown_token)
for x in t.decode('utf-8').split(" ")
]), [text], tf.int64)
def testNoInput(self):
with self.test_session():
x, = tf.py_func(lambda: 42.0, [], [tf.float64])
self.assertAllClose(x.eval(), 42.0)
def adaptive_bilateral_filter_tf(imgs, ksize, sigmaSpace, maxSigmaColor=20.0):
my_func = lambda x: adaptive_bilateral_filter_py(x, ksize, sigmaSpace, maxSigmaColor)
y = tf.py_func(my_func, [imgs], tf.float32, stateful=False)
return y
def build_whole_detection_network(self,
input_img_batch,
gtboxes_batch_h,
gtboxes_batch_r,
gthead_quadrant,
gt_smooth_label,
gpu_id=0):
if self.is_training:
gtboxes_batch_h = tf.reshape(gtboxes_batch_h, [-1, 5])
gtboxes_batch_h = tf.cast(gtboxes_batch_h, tf.float32)
gtboxes_batch_r = tf.reshape(gtboxes_batch_r, [-1, 6])
gtboxes_batch_r = tf.cast(gtboxes_batch_r, tf.float32)
gthead_quadrant = tf.reshape(gthead_quadrant, [-1, 1])
gthead_quadrant = tf.cast(gthead_quadrant, tf.int32)
gt_smooth_label = tf.reshape(gt_smooth_label,
[-1, self.angle_range])
gt_smooth_label = tf.cast(gt_smooth_label, tf.float32)
img_shape = tf.shape(input_img_batch)
# 1. build base network
feature_pyramid = self.build_base_network(input_img_batch)
# 2. build rpn
rpn_box_pred, rpn_cls_score, rpn_cls_prob, rpn_head_cls, rpn_angle_cls = self.rpn_net(
feature_pyramid)
# 3. generate_anchors
anchors = self.make_anchors(feature_pyramid)
# 4. postprocess rpn proposals. such as: decode, clip, filter
if self.is_training:
with tf.variable_scope('build_loss'):
labels, target_delta, anchor_states, target_boxes, target_head_quadrant, target_smooth_label = tf.py_func(
func=anchor_target_layer,
inp=[
gtboxes_batch_h, gtboxes_batch_r, gthead_quadrant,
gt_smooth_label, anchors, gpu_id
],
Tout=[
tf.float32, tf.float32, tf.float32, tf.float32,
tf.float32, tf.float32
])
if self.method == 'H':
self.add_anchor_img_smry(input_img_batch, anchors,
anchor_states, 0)
else:
self.add_anchor_img_smry(input_img_batch, anchors,
anchor_states, 1)
cls_loss = losses.focal_loss(labels, rpn_cls_score,
anchor_states)
if cfgs.REG_LOSS_MODE == 0:
reg_loss = losses.iou_smooth_l1_loss(
target_delta, rpn_box_pred, anchor_states,
target_boxes, anchors)
elif cfgs.REG_LOSS_MODE == 1:
reg_loss = losses.smooth_l1_loss_atan(
target_delta, rpn_box_pred, anchor_states)
else:
reg_loss = losses.smooth_l1_loss(target_delta,
rpn_box_pred,
anchor_states)
if cfgs.DATASET_NAME.startswith('DOTA'):
head_cls_loss = losses.head_specific_cls_focal_loss(
target_head_quadrant,
rpn_head_cls,
anchor_states,
labels,
specific_cls=[6, 7, 8, 9, 10, 11])
else:
head_cls_loss = losses.head_focal_loss(
target_head_quadrant, rpn_head_cls, anchor_states)
angle_cls_loss = losses.angle_focal_loss(
target_smooth_label, rpn_angle_cls, anchor_states)
self.losses_dict = {
'cls_loss': cls_loss * cfgs.CLS_WEIGHT,
'reg_loss': reg_loss * cfgs.REG_WEIGHT,
'head_cls_loss': head_cls_loss * cfgs.HEAD_WEIGHT,
'angle_cls_loss': angle_cls_loss * cfgs.ANGLE_WEIGHT
}
with tf.variable_scope('postprocess_detctions'):
boxes, scores, category, boxes_head, boxes_angle = postprocess_detctions(
rpn_bbox_pred=rpn_box_pred,
rpn_cls_prob=rpn_cls_prob,
rpn_angle_prob=tf.sigmoid(rpn_angle_cls),
rpn_head_prob=tf.sigmoid(rpn_head_cls),
anchors=anchors,
is_training=self.is_training)
boxes = tf.stop_gradient(boxes)
scores = tf.stop_gradient(scores)
category = tf.stop_gradient(category)
boxes_head = tf.stop_gradient(boxes_head)
boxes_angle = tf.stop_gradient(boxes_angle)
if self.is_training:
return boxes, scores, category, boxes_head, boxes_angle, self.losses_dict
else:
return boxes, scores, category, boxes_head, boxes_angle
def bilateral_filter_tf(imgs, d, sigmaSpace, sigmaColor):
my_func = lambda x: bilateral_filter_py(x, d, sigmaSpace, sigmaColor)
y = tf.py_func(my_func, [imgs], tf.float32, stateful=False)
return y
def my_iou_metric_2(label, pred):
return tf.py_func(get_iou_vector, [label, pred > 0], tf.float64)
def cv2_resize_tf(x, h, w):
def resize256(x):
return cv2.resize(x, (h, w))
return tf.py_func(resize256, [x], tf.float32)
def tf_spatial_pyramid_pooling(tf_input_feature_maps,
tf_spatial_pyramid,
dtype=tf.float32):
return tf.py_func(np_spatial_pyramid_pooling,
[tf_input_feature_maps, tf_spatial_pyramid], dtype)
def non_local_means_bw_tf(imgs, search_window, block_size, photo_render):
my_func = lambda x: non_local_means_bw_py(x, search_window, block_size, photo_render)
y = tf.py_func(my_func, [imgs], tf.float32, stateful=False)
return y
def string_length_tf(t):
return tf.py_func(len, [t], [tf.int64])
def draw_boxes(image_and_detections):
"""Draws boxes on image."""
image_with_boxes = tf.py_func(visualize_boxes_fn, image_and_detections,
tf.uint8)
return image_with_boxes
unseen_label = np.zeros(num_unseen_classes,dtype=np.int32)
#print(len(df_img_label))
for index, row in df_img_label.iterrows():
if row['LabelName'] in seen_labelmap:
idx=seen_labelmap.index(row['LabelName'])
seen_label[idx] = 2*row['Confidence']-1
if row['LabelName'] in unseen_labelmap:
idx=unseen_labelmap.index(row['LabelName'])
unseen_label[idx] = 2*row['Confidence']-1
#print(partition_idx)
return processed_image,img_id,seen_label,unseen_label
#%%
print('loading data:done')
dataset = tf.data.Dataset.from_tensor_slices((files,partition_idxs))
dataset = dataset.map(read_img,num_parallel_calls)
dataset = dataset.map(lambda processed_images,file,partition_idx: tuple(tf.py_func(get_label, [processed_images,file,partition_idx], [tf.float32, tf.string, tf.int32, tf.int32])),num_parallel_calls)
dataset = dataset.batch(batch_size).prefetch(batch_size)#.map(PreprocessImage)
processed_images,img_ids,seen_labels,unseen_labels=dataset.make_one_shot_iterator().get_next()
#%%
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.InteractiveSession(config=config)
g = tf.get_default_graph()
#%%
with g.as_default():
if is_save:
opts = tf.python_io.TFRecordOptions(tf.python_io.TFRecordCompressionType.ZLIB)
feature_writer = tf.python_io.TFRecordWriter(feature_tfrecord_filename, options=opts)
img_input_ph = tf.placeholder(dtype=tf.float32,shape=[None,height,width,3])
with slim.arg_scope(vgg.vgg_arg_scope()):
def _anchor_target_layer(self, rpn_cls_score, name):
with tf.variable_scope(name) as scope:
rpn_labels, rpn_bbox_targets, rpn_bbox_inside_weights, rpn_bbox_outside_weights, anchor_targets = tf.py_func(
anchor_target_layer, [
rpn_cls_score, self._gt_boxes, self._im_info,
self._feat_stride, self._anchors, self._num_anchors
],
[tf.float32, tf.float32, tf.float32, tf.float32, tf.float32],
name="anchor_target")
rpn_labels.set_shape([1, 1, None, None])
rpn_bbox_targets.set_shape([1, None, None, self._num_anchors * 4])
rpn_bbox_inside_weights.set_shape(
[1, None, None, self._num_anchors * 4])
rpn_bbox_outside_weights.set_shape(
[1, None, None, self._num_anchors * 4])
rpn_labels = tf.to_int32(rpn_labels, name="to_int32")
self._anchor_targets['anchor_targets'] = anchor_targets
self._anchor_targets['rpn_labels'] = rpn_labels
self._anchor_targets['rpn_bbox_targets'] = rpn_bbox_targets
self._anchor_targets[
'rpn_bbox_inside_weights'] = rpn_bbox_inside_weights
self._anchor_targets[
'rpn_bbox_outside_weights'] = rpn_bbox_outside_weights
self._score_summaries.update(self._anchor_targets)
return rpn_labels
def _proposal_target_layer(self, rois, roi_scores, name):
with tf.variable_scope(name) as scope:
rois, roi_scores, labels, bbox_targets, bbox_inside_weights, bbox_outside_weights = tf.py_func(
proposal_target_layer,
[rois, roi_scores, self._gt_boxes, self._num_classes], [
tf.float32, tf.float32, tf.float32, tf.float32, tf.float32,
tf.float32
],
name="proposal_target")
rois.set_shape([cfg.TRAIN.BATCH_SIZE, 5])
roi_scores.set_shape([cfg.TRAIN.BATCH_SIZE])
labels.set_shape([cfg.TRAIN.BATCH_SIZE, 1])
bbox_targets.set_shape(
[cfg.TRAIN.BATCH_SIZE, self._num_classes * 4])
bbox_inside_weights.set_shape(
[cfg.TRAIN.BATCH_SIZE, self._num_classes * 4])
bbox_outside_weights.set_shape(
[cfg.TRAIN.BATCH_SIZE, self._num_classes * 4])
self._proposal_targets['rois'] = rois
self._proposal_targets['labels'] = tf.to_int32(labels,
name="to_int32")
self._proposal_targets['bbox_targets'] = bbox_targets
self._proposal_targets['bbox_inside_weights'] = bbox_inside_weights
self._proposal_targets[
'bbox_outside_weights'] = bbox_outside_weights
self._score_summaries.update(self._proposal_targets)
return rois, roi_scores
def eval_ship(img_num):
with tf.Graph().as_default():
img_name_batch, img_batch, gtboxes_and_label_batch, num_objects_batch = \
next_batch(dataset_name=cfgs.DATASET_NAME,
batch_size=cfgs.BATCH_SIZE,
shortside_len=cfgs.SHORT_SIDE_LEN,
is_training=False)
gtboxes_and_label = tf.py_func(
back_forward_convert,
inp=[tf.squeeze(gtboxes_and_label_batch, 0)],
Tout=tf.float32)
gtboxes_and_label = tf.reshape(gtboxes_and_label, [-1, 6])
gtboxes_and_label_minAreaRectangle = get_horizen_minAreaRectangle(
gtboxes_and_label)
gtboxes_and_label_minAreaRectangle = tf.reshape(
gtboxes_and_label_minAreaRectangle, [-1, 5])
# ***********************************************************************************************
# * share net *
# ***********************************************************************************************
_, share_net = get_network_byname(net_name=cfgs.NET_NAME,
inputs=img_batch,
num_classes=None,
is_training=True,
output_stride=None,
global_pool=False,
spatial_squeeze=False)
# ***********************************************************************************************
# * RPN *
# ***********************************************************************************************
rpn = build_rpn.RPN(
net_name=cfgs.NET_NAME,
inputs=img_batch,
gtboxes_and_label=None,
is_training=False,
share_head=cfgs.SHARE_HEAD,
share_net=share_net,
stride=cfgs.STRIDE,
anchor_ratios=cfgs.ANCHOR_RATIOS,
anchor_scales=cfgs.ANCHOR_SCALES,
scale_factors=cfgs.SCALE_FACTORS,
base_anchor_size_list=cfgs.
BASE_ANCHOR_SIZE_LIST, # P2, P3, P4, P5, P6
level=cfgs.LEVEL,
top_k_nms=cfgs.RPN_TOP_K_NMS,
rpn_nms_iou_threshold=cfgs.RPN_NMS_IOU_THRESHOLD,
max_proposals_num=cfgs.MAX_PROPOSAL_NUM,
rpn_iou_positive_threshold=cfgs.RPN_IOU_POSITIVE_THRESHOLD,
rpn_iou_negative_threshold=cfgs.RPN_IOU_NEGATIVE_THRESHOLD,
rpn_mini_batch_size=cfgs.RPN_MINIBATCH_SIZE,
rpn_positives_ratio=cfgs.RPN_POSITIVE_RATE,
remove_outside_anchors=False, # whether remove anchors outside
rpn_weight_decay=cfgs.WEIGHT_DECAY[cfgs.NET_NAME])
# rpn predict proposals
rpn_proposals_boxes, rpn_proposals_scores = rpn.rpn_proposals(
) # rpn_score shape: [300, ]
# ***********************************************************************************************
# * Fast RCNN *
# ***********************************************************************************************
fast_rcnn = build_fast_rcnn.FastRCNN(
feature_pyramid=rpn.feature_pyramid,
rpn_proposals_boxes=rpn_proposals_boxes,
rpn_proposals_scores=rpn_proposals_scores,
img_shape=tf.shape(img_batch),
roi_size=cfgs.ROI_SIZE,
roi_pool_kernel_size=cfgs.ROI_POOL_KERNEL_SIZE,
scale_factors=cfgs.SCALE_FACTORS,
gtboxes_and_label=None,
gtboxes_and_label_minAreaRectangle=
gtboxes_and_label_minAreaRectangle,
fast_rcnn_nms_iou_threshold=cfgs.FAST_RCNN_NMS_IOU_THRESHOLD,
fast_rcnn_maximum_boxes_per_img=100,
fast_rcnn_nms_max_boxes_per_class=cfgs.
FAST_RCNN_NMS_MAX_BOXES_PER_CLASS,
show_detections_score_threshold=cfgs.FINAL_SCORE_THRESHOLD,
# show detections which score >= 0.6
num_classes=cfgs.CLASS_NUM,
fast_rcnn_minibatch_size=cfgs.FAST_RCNN_MINIBATCH_SIZE,
fast_rcnn_positives_ratio=cfgs.FAST_RCNN_POSITIVE_RATE,
fast_rcnn_positives_iou_threshold=cfgs.
FAST_RCNN_IOU_POSITIVE_THRESHOLD,
# iou>0.5 is positive, iou<0.5 is negative
use_dropout=cfgs.USE_DROPOUT,
weight_decay=cfgs.WEIGHT_DECAY[cfgs.NET_NAME],
is_training=False,
level=cfgs.LEVEL)
fast_rcnn_decode_boxes, fast_rcnn_score, num_of_objects, detection_category = fast_rcnn.fast_rcnn_predict(
)
# train
init_op = tf.group(tf.global_variables_initializer(),
tf.local_variables_initializer())
restorer, restore_ckpt = restore_model.get_restorer()
config = tf.ConfigProto()
# config.gpu_options.per_process_gpu_memory_fraction = 0.5
config.gpu_options.allow_growth = True
with tf.Session(config=config) as sess:
sess.run(init_op)
if not restorer is None:
restorer.restore(sess, restore_ckpt)
print('restore model')
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess, coord)
gtboxes_horizontal_dict = {}
predict_horizontal_dict = {}
for i in range(img_num):
start = time.time()
_img_name_batch, _img_batch, _gtboxes_and_label, _gtboxes_and_label_minAreaRectangle, \
_fast_rcnn_decode_boxes, _fast_rcnn_score, _detection_category \
= sess.run([img_name_batch, img_batch, gtboxes_and_label, gtboxes_and_label_minAreaRectangle,
fast_rcnn_decode_boxes, fast_rcnn_score, detection_category])
end = time.time()
# gtboxes convert dict
gtboxes_horizontal_dict[str(_img_name_batch[0])] = []
predict_horizontal_dict[str(_img_name_batch[0])] = []
gtbox_horizontal_list, predict_horizontal_list = \
make_dict_packle(_gtboxes_and_label_minAreaRectangle, _fast_rcnn_decode_boxes,
_fast_rcnn_score, _detection_category)
gtboxes_horizontal_dict[str(
_img_name_batch[0])].extend(gtbox_horizontal_list)
predict_horizontal_dict[str(
_img_name_batch[0])].extend(predict_horizontal_list)
view_bar(
'{} image cost {}s'.format(str(_img_name_batch[0]),
(end - start)), i + 1, img_num)
fw1 = open('gtboxes_horizontal_dict.pkl', 'w')
fw2 = open('predict_horizontal_dict.pkl', 'w')
pickle.dump(gtboxes_horizontal_dict, fw1)
pickle.dump(predict_horizontal_dict, fw2)
fw1.close()
fw2.close()
coord.request_stop()
coord.join(threads)
def prepare_batch(ds, vocab, trans, params):
"""Input function HAN
Args:
ds: tf.data instance where each element comrrises a document, label tuple
vocab: pvocab: (tf.lookuptable)
params: (Params) contains hyperparameters of the model (ex: `params.num_epochs`)
"""
# Load all the dataset in memory for shuffling is training
#extract SHIFT-REDUCE transitions as a string from S-expression
def transition_parser(st):
tokens = st.split()
string = ""
for tok in tokens:
if tok == ")":
string += "REDUCE "
elif tok == "(":
pass
else:
string += "SHIFT "
return string
def _read_py_function(label, sentences, trees):
doc_len = len(sentences)
label = label - 1
sen_len = [len(str(sentence).split(" ")) for sentence in sentences]
trees_ = [tree.decode('utf-8') for tree in trees]
transitions = [transition_parser(tree) for tree in trees_]
return label, doc_len, sen_len, sentences, transitions
ds = ds.map(lambda label, sentences, trees: tuple(
tf.py_func(_read_py_function, [label, sentences, trees],
[tf.int32, tf.int32, tf.int32, tf.string, tf.string])),
num_parallel_calls=4)
#replace tokens with ids
def transform(doc, default_value=params.pad_word):
# Split sentence
out = tf.string_split(doc)
# Convert to Dense tensor, filling with default value
out = tf.sparse_tensor_to_dense(out, default_value=default_value)
out = vocab.lookup(out)
out = tf.cast(out, tf.int32)
return out
#replace SHIFT with 1, REDUCE with 2, all other entries are dummy paddings
def transform2(doc, default_value=params.pad_word):
# Split sentence
out = tf.string_split(doc)
# Convert to Dense tensor, filling with default value
out = tf.sparse_tensor_to_dense(out, default_value=default_value)
out = trans.lookup(out)
out = tf.cast(out, tf.int32)
out += 1
return out
ds = ds.map(lambda label, doc_len, sen_len, sentences, transitions:
(label, doc_len, sen_len, transform(sentences),
transform2(transitions)),
num_parallel_calls=4)
# Create batches and pad the sentences of different length
padded_shapes = (
tf.TensorShape([]),
tf.TensorShape([]), # doc of unknown size
tf.TensorShape([None]), # sentence lengths
tf.TensorShape([None, None]), # sentence tokens
tf.TensorShape([None, None])) # transition tokens
padding_values = (0, 0, 0, params.id_pad_word, 0)
ds = (ds.padded_batch(params.batch_size,
padded_shapes=padded_shapes,
padding_values=padding_values).prefetch(4)
) # make sure you always have one batch ready to serve
#SPINN Preprocessing step: reverse the entries in sentences and prepend a dummy token
def pad_reverse(label, doc_len, sen_len, sentences, transitions):
# Reverse sequence and pad an extra one.
sentences_ = np.reshape(
sentences,
(sentences.shape[0] * sentences.shape[1], sentences.shape[2]))
sentences_ = np.fliplr(np.array(sentences_, dtype=np.int32))
sentences_ = np.concatenate([
np.ones([sentences_.shape[0], 1], dtype=np.int32) *
params.id_pad_word, sentences_
],
axis=1)
sentences_ = sentences_.T
transitions = np.reshape(
transitions, (transitions.shape[0] * transitions.shape[1], -1)).T
return label, doc_len, sen_len, sentences_, transitions
ds = ds.map(lambda label, doc_len, sen_len, sentences, transitions: tuple(
tf.py_func(pad_reverse, [
label, doc_len, sen_len, sentences, transitions
], [tf.int32, tf.int32, tf.int32, tf.int32, tf.int32])),
num_parallel_calls=4)
"""
# Create initializable iterator from this dataset so that we can reset at each epoch
iterator = ds.make_initializable_iterator()
# Query the output of the iterator for input to the model
labels, document_sizes, sentence_lengths, sentences, transitions = iterator.get_next()
init_op = iterator.initializer
inputs = {
'labels': labels,
'document_sizes': document_sizes,
'sentence_lengths': sentence_lengths,
'sentences': sentences,
'transitions': transitions,
'iterator_init_op': init_op
}
return inputs
"""
return ds
def model_fn(features, labels, mode, params):
graph_ensemble = tf.Graph()
with tf.Session(graph=graph_ensemble) as sess:
meta_graph_path = tf.gfile.Glob(
os.path.join(params["ensemble_architecture_path"], '*.meta'))[0]
loader = tf.train.import_meta_graph(meta_graph_path,
clear_devices=True)
loader.restore(
sess,
tf.train.latest_checkpoint(params["ensemble_architecture_path"]))
graph = tf.get_default_graph()
meta_graph_1 = tf.train.export_meta_graph(graph=graph_ensemble)
meta_graph.import_scoped_meta_graph(
meta_graph_1,
input_map={"raw_imgs": features['img']},
import_scope='main_graph')
logits = graph.get_tensor_by_name('main_graph/logits_tf:0')
predicted_classes = tf.argmax(logits, 1)
category_map = tf.convert_to_tensor(params["category_map"])
##
class_label = tf.gather_nd(category_map, predicted_classes)
class_label = tf.convert_to_tensor([class_label], dtype=tf.string)
##
if mode == tf.estimator.ModeKeys.PREDICT:
predictions = {
'class_ids': predicted_classes[:, tf.newaxis],
'probabilities': tf.nn.softmax(logits),
'logits': logits,
'class_label': class_label[:, tf.newaxis],
# 'category_map': tf.convert_to_tensor([str(params["category_map"])])[:, tf.newaxis],
}
return tf.estimator.EstimatorSpec(mode, predictions=predictions)
cross_entropy = tf.nn.softmax_cross_entropy_with_logits_v2(
labels=labels, logits=logits, name='cross_entropy')
loss = tf.reduce_mean(cross_entropy, name='cost_fc')
accuracy = tf.metrics.accuracy(labels=tf.argmax(labels, 1),
predictions=predicted_classes,
name='acc_op')
"""confusion matrix"""
batch_confusion = tf.confusion_matrix(tf.argmax(labels, 1),
predicted_classes,
num_classes=params['n_output'],
name='batch_confusion')
plot_buf = tf.py_func(gen_plot, [batch_confusion, category_map], tf.string)
# Convert PNG buffer to TF image
cm_image = tf.image.decode_png(plot_buf, channels=4)
# Add the batch dimension
cm_image = tf.expand_dims(cm_image, 0)
""""""
tf.summary.scalar('accuracy', accuracy[1])
tf.summary.image('confusion_matrix', cm_image)
metrics = {'accuracy': accuracy}
if mode == tf.estimator.ModeKeys.EVAL:
return tf.estimator.EstimatorSpec(mode,
loss=loss,
eval_metric_ops=metrics)
assert mode == tf.estimator.ModeKeys.TRAIN
if params['retrain_primary_models'] != True:
trainable_variables = [
v for v in tf.trainable_variables() if 'ensemble' in v.name
]
else:
trainable_variables = [v for v in tf.trainable_variables()]
optimizer = tf.train.AdamOptimizer(learning_rate=params['learning_rate'],
name='adam_fc')
train_op = optimizer.minimize(
loss,
var_list=trainable_variables,
global_step=tf.train.get_or_create_global_step())
return tf.estimator.EstimatorSpec(mode, loss=loss, train_op=train_op)
def _map(self, example_serialized):
def _parse(line):
return np.int32(line), np.int32(line)
a, b = tf.py_func(_parse, [example_serialized], [tf.int32, tf.int32], stateful=True)
return a, b
def main():
"""Create the model and start the evaluation process."""
args = get_arguments()
# Create queue coordinator.
coord = tf.train.Coordinator()
# Load reader.
with tf.name_scope("create_inputs"):
reader = ImageReader(
args.data_dir,
args.data_list,
None, # No defined input size.
False, # No random scale.
False, # No random mirror.
args.ignore_label,
IMG_MEAN,
coord)
image, label = reader.image, reader.label
image_batch, label_batch = tf.expand_dims(image, dim=0), tf.expand_dims(
label, dim=0) # Add one batch dimension.
# Create network.
net = DeepLabResNetModel({'data': image_batch},
is_training=False,
num_classes=args.num_classes)
# Which variables to load.
restore_var = tf.global_variables()
# Predictions.
raw_output = net.layers['fc1_voc12']
raw_output = tf.image.resize_bilinear(raw_output,
tf.shape(image_batch)[1:3, ])
# CRF.
inv_image = tf.py_func(inv_preprocess, [image_batch], tf.uint8)
raw_output = tf.py_func(dense_crf, [tf.nn.softmax(raw_output), inv_image],
tf.float32)
raw_output = tf.argmax(raw_output, dimension=3)
pred = tf.expand_dims(raw_output, dim=3) # Create a 4-d tensor.
# mIoU
pred = tf.reshape(pred, [
-1,
])
gt = tf.reshape(label_batch, [
-1,
])
weights = tf.cast(tf.less_equal(gt, 20),
tf.int32) # Ignore the void label '255'.
mIoU, update_op = tf.contrib.metrics.streaming_mean_iou(pred,
gt,
num_classes=21,
weights=weights)
# Set up a TF session and initialise variables.
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
init = tf.global_variables_initializer()
sess.run(init)
sess.run(tf.local_variables_initializer())
# Load weights.
loader = tf.train.Saver(var_list=restore_var)
if args.restore_from is not None:
load(loader, sess, args.restore_from)
# Start queue threads.
threads = tf.train.start_queue_runners(coord=coord, sess=sess)
# Iterate over training steps.
for step in tqdm(range(args.num_steps)):
preds, _ = sess.run([pred, update_op])
print('Mean IoU: {:.3f}'.format(mIoU.eval(session=sess)))
coord.request_stop()
coord.join(threads)
def get_batch(config, train_form):
"""Loads training data and put them in queues"""
with tf.device('/cpu:0'):
# Load data
_texts, _texts_tests, _mels, _mags, _dones = load_data(
config, train_form)
# Calc total batch count
num_batch = len(_texts) // hp.batch_size
# Convert to string tensor
texts = tf.convert_to_tensor(_texts)
texts_tests = tf.convert_to_tensor(_texts_tests)
mels = tf.convert_to_tensor(_mels)
if hp.include_dones:
dones = tf.convert_to_tensor(_dones)
if train_form != 'Encoder':
mags = tf.convert_to_tensor(_mags)
if train_form == 'Both':
if hp.include_dones:
text, texts_test, mel, mag, done = tf.train.slice_input_producer(
[texts, texts_tests, mels, mags, dones], shuffle=True)
else:
text, texts_test, mel, mag = tf.train.slice_input_producer(
[texts, texts_tests, mels, mags], shuffle=True)
elif train_form == 'Encoder':
if hp.include_dones:
text, texts_test, mel, done = tf.train.slice_input_producer(
[texts, texts_tests, mels, dones], shuffle=True)
else:
text, texts_test, mel = tf.train.slice_input_producer(
[texts, texts_tests, mels], shuffle=True)
else:
text, texts_test, mel, mag = tf.train.slice_input_producer(
[texts, texts_tests, mels, mags], shuffle=True)
# Decoding
text = tf.decode_raw(text, tf.int32) # (None,)
texts_test = tf.decode_raw(texts_test, tf.int32) # (None,)
mel = tf.py_func(lambda x: np.load(x), [mel],
tf.float32) # (None, n_mels)
if hp.include_dones:
done = tf.py_func(lambda x: np.load(x), [done],
tf.int32) # (None,)
if train_form != 'Encoder':
mag = tf.py_func(lambda x: np.load(x), [mag], tf.float32)
# Padding
text = tf.pad(text, ((0, hp.T_x), ))[:hp.T_x] # (Tx,)
texts_test = tf.pad(texts_test, ((0, hp.T_x), ))[:hp.T_x] # (Tx,)
mel = tf.pad(mel, ((0, hp.T_y), (0, 0)))[:hp.T_y] # (Ty, n_mels)
if hp.include_dones:
done = tf.pad(done, ((0, hp.T_y), ))[:hp.T_y] # (Ty,)
if train_form != 'Encoder':
mag = tf.pad(mag,
((0, hp.T_y), (0, 0)))[:hp.T_y] # (Ty, 1+n_fft/2)
# Reduction
mel = tf.reshape(mel, (hp.T_y // hp.r, -1)) # (Ty/r, n_mels*r)
if hp.include_dones:
done = done[::hp.r] # (Ty/r,)
if train_form == 'Both':
if hp.include_dones:
texts, texts_tests, mels, mags, dones = tf.train.batch(
[text, texts_test, mel, mag, done],
shapes=[(hp.T_x, ), (hp.T_x, ),
(hp.T_y // hp.r, hp.n_mels * hp.r),
(hp.T_y, 1 + hp.n_fft // 2), (hp.T_y // hp.r, )],
num_threads=8,
batch_size=hp.batch_size,
capacity=hp.batch_size * 8,
dynamic_pad=False)
return texts_tests, texts, mels, dones, mags, num_batch
else:
texts, texts_tests, mels, mags = tf.train.batch(
[text, texts_test, mel, mag],
shapes=[(hp.T_x, ), (hp.T_x, ),
(hp.T_y // hp.r, hp.n_mels * hp.r),
(hp.T_y, 1 + hp.n_fft // 2)],
num_threads=8,
batch_size=hp.batch_size,
capacity=hp.batch_size * 8,
dynamic_pad=False)
return texts_tests, texts, mels, None, mags, num_batch
elif train_form == 'Encoder':
if hp.include_dones:
texts, texts_tests, mels, dones = tf.train.batch(
[text, texts_test, mel, done],
shapes=[(hp.T_x, ), (hp.T_x, ),
(hp.T_y // hp.r, hp.n_mels * hp.r),
(hp.T_y // hp.r, )],
num_threads=8,
batch_size=hp.batch_size,
capacity=hp.batch_size * 8,
dynamic_pad=False)
return texts_tests, texts, mels, dones, None, num_batch
else:
texts, texts_tests, mels = tf.train.batch(
[text, texts_test, mel],
shapes=[(hp.T_x, ), (hp.T_x, ),
(hp.T_y // hp.r, hp.n_mels * hp.r)],
num_threads=8,
batch_size=hp.batch_size,
capacity=hp.batch_size * 8,
dynamic_pad=False)
return texts_tests, texts, mels, None, None, num_batch
else:
texts, texts_tests, mels, mags = tf.train.batch(
[text, texts_test, mel, mag],
shapes=[(hp.T_x, ), (hp.T_x, ),
(hp.T_y // hp.r, hp.n_mels * hp.r),
(hp.T_y, 1 + hp.n_fft // 2)],
num_threads=8,
batch_size=hp.batch_size,
capacity=hp.batch_size * 8,
dynamic_pad=False)
return texts_tests, texts, mels, None, mags, num_batch
def get_estimator_eval_metric_ops(self, eval_dict):
"""Returns metric ops for use in tf.estimator.EstimatorSpec.
Args:
eval_dict: A dictionary that holds an image, groundtruth, and detections
for a batched example. Note that, we use only the first example for
visualization. See eval_util.result_dict_for_batched_example() for a
convenient method for constructing such a dictionary. The dictionary
contains
fields.InputDataFields.original_image: [batch_size, H, W, 3] image.
fields.InputDataFields.original_image_spatial_shape: [batch_size, 2]
tensor containing the size of the original image.
fields.InputDataFields.true_image_shape: [batch_size, 3]
tensor containing the spatial size of the upadded original image.
fields.InputDataFields.groundtruth_boxes - [batch_size, num_boxes, 4]
float32 tensor with groundtruth boxes in range [0.0, 1.0].
fields.InputDataFields.groundtruth_classes - [batch_size, num_boxes]
int64 tensor with 1-indexed groundtruth classes.
fields.InputDataFields.groundtruth_instance_masks - (optional)
[batch_size, num_boxes, H, W] int64 tensor with instance masks.
fields.DetectionResultFields.detection_boxes - [batch_size,
max_num_boxes, 4] float32 tensor with detection boxes in range [0.0,
1.0].
fields.DetectionResultFields.detection_classes - [batch_size,
max_num_boxes] int64 tensor with 1-indexed detection classes.
fields.DetectionResultFields.detection_scores - [batch_size,
max_num_boxes] float32 tensor with detection scores.
fields.DetectionResultFields.detection_masks - (optional) [batch_size,
max_num_boxes, H, W] float32 tensor of binarized masks.
fields.DetectionResultFields.detection_keypoints - (optional)
[batch_size, max_num_boxes, num_keypoints, 2] float32 tensor with
keypoints.
Returns:
A dictionary of image summary names to tuple of (value_op, update_op). The
`update_op` is the same for all items in the dictionary, and is
responsible for saving a single side-by-side image with detections and
groundtruth. Each `value_op` holds the tf.summary.image string for a given
image.
"""
if self._max_examples_to_draw == 0:
return {}
images = self.images_from_evaluation_dict(eval_dict)
def get_images():
"""Returns a list of images, padded to self._max_images_to_draw."""
images = self._images
while len(images) < self._max_examples_to_draw:
images.append(np.array(0, dtype=np.uint8))
self.clear()
return images
def image_summary_or_default_string(summary_name, image):
"""Returns image summaries for non-padded elements."""
return tf.cond(
tf.equal(tf.size(tf.shape(image)), 4),
lambda: tf.summary.image(summary_name, image),
lambda: tf.constant(''))
if tf.executing_eagerly():
update_op = self.add_images([[images[0]]])
image_tensors = get_images()
else:
update_op = tf.py_func(self.add_images, [[images[0]]], [])
image_tensors = tf.py_func(
get_images, [], [tf.uint8] * self._max_examples_to_draw)
eval_metric_ops = {}
for i, image in enumerate(image_tensors):
summary_name = self._summary_name_prefix + '/' + str(i)
value_op = image_summary_or_default_string(summary_name, image)
eval_metric_ops[summary_name] = (value_op, update_op)
return eval_metric_ops
def tf_get(self, index):
def get(index):
return self.images[index], self.labels[index]
image, label = tf.py_func(get, [index], [tf.float32, tf.int64])
return dict(image=image, label=label)
