def read_from_tfrecord(tfrecord_path):
with tf.Session() as sess:
# create feature to hold data
feature_dict = {
'image/height': tf.FixedLenFeature([], tf.int64),
'image/width': tf.FixedLenFeature([], tf.int64),
'image/filename': tf.VarLenFeature(tf.string),
'image/image': tf.FixedLenFeature([], tf.string),
'image/panel/bbox/xmin': tf.FixedLenFeature([], tf.float32),
'image/panel/bbox/ymin': tf.FixedLenFeature([], tf.float32),
'image/panel/bbox/xmax': tf.FixedLenFeature([], tf.float32),
'image/panel/bbox/ymax': tf.FixedLenFeature([], tf.float32),
'image/label/bbox/xmin': tf.FixedLenFeature([], tf.float32),
'image/label/bbox/ymin': tf.FixedLenFeature([], tf.float32),
'image/label/bbox/xmax': tf.FixedLenFeature([], tf.float32),
'image/label/bbox/ymax': tf.FixedLenFeature([], tf.float32),
'image/panel/label/text': tf.FixedLenFeature([], tf.string),
'image/panel/label/class': tf.FixedLenFeature([], tf.int64)
}
# create a list of tfrecord filenames and pass it to a queue
filename_queue = tf.train.string_input_producer([tfrecord_path], num_epochs=1)
# define a reader and read the next record
reader = tf.TFRecordReader()
_, serialized_example = reader.read(filename_queue)
# decode the record read by the reader
features = tf.parse_single_example(serialized_example, features=feature_dict)
image_height = tf.case(features['image/height'], tf.int64)
image_width = tf.case(features['image/width'], tf.int64)
# convert the image data from string back to the numbers
image = tf.decode_raw(features['image/image'], tf.uint8)
image = tf.reshape(image, [image_height, image_width, 3])
def _provide_data(input_tensors, truncated_length, hparams):
"""Returns tensors for reading batches from provider."""
(spec, labels, label_weights, length, onsets, filename,
note_sequence) = input_tensors
length = tf.to_int32(length)
labels = tf.reshape(labels, (-1, constants.MIDI_PITCHES))
label_weights = tf.reshape(label_weights, (-1, constants.MIDI_PITCHES))
onsets = tf.reshape(onsets, (-1, constants.MIDI_PITCHES))
spec = tf.reshape(spec, (-1, hparams_frame_size(hparams)))
truncated_length = (tf.reduce_min([truncated_length, length])
if truncated_length else length)
# Pad or slice specs and labels tensors to have the same lengths,
# truncating after truncated_length.
spec_delta = tf.shape(spec)[0] - truncated_length
spec = tf.case(
[(spec_delta < 0,
lambda: tf.pad(spec, tf.stack([(0, -spec_delta), (0, 0)]))),
(spec_delta > 0, lambda: spec[0:-spec_delta])],
default=lambda: spec)
labels_delta = tf.shape(labels)[0] - truncated_length
labels = tf.case(
[(labels_delta < 0,
lambda: tf.pad(labels, tf.stack([(0, -labels_delta), (0, 0)]))),
(labels_delta > 0, lambda: labels[0:-labels_delta])],
default=lambda: labels)
label_weights = tf.case(
[(labels_delta < 0,
lambda: tf.pad(label_weights, tf.stack([(0, -labels_delta), (0, 0)]))
), (labels_delta > 0, lambda: label_weights[0:-labels_delta])],
default=lambda: label_weights)
onsets = tf.case(
[(labels_delta < 0,
lambda: tf.pad(onsets, tf.stack([(0, -labels_delta), (0, 0)]))),
(labels_delta > 0, lambda: onsets[0:-labels_delta])],
default=lambda: onsets)
truncated_note_sequence = truncate_note_sequence_op(
note_sequence, truncated_length, hparams)
batch_tensors = {
'spec': tf.reshape(
spec, (truncated_length, hparams_frame_size(hparams), 1)),
'labels': tf.reshape(labels, (truncated_length, constants.MIDI_PITCHES)),
'label_weights': tf.reshape(
label_weights, (truncated_length, constants.MIDI_PITCHES)),
'lengths': truncated_length,
'onsets': tf.reshape(onsets, (truncated_length, constants.MIDI_PITCHES)),
'filenames': filename,
'note_sequences': truncated_note_sequence,
}
return batch_tensors
def piecewise_constant(x, boundaries, values):
""" Piecewise constant function.
Arguments:
x: A 0-D Tensor.
boundaries: A 1-D NumPy array with strictly increasing entries.
values: A 1-D NumPy array that specifies the values for the intervals
defined by `boundaries`. (It should therefore have one more entry
than `boundaries`.)
Returns: A 0-D Tensor. Its value is `values[0]` when `x <= boundaries[0]`,
`values[1]` when `x > boundaries[0]` and `x <= boundaries[1]`, ..., and
values[-1] when `x > boundaries[-1]`.
"""
pred_fn_pairs = {}
pred_fn_pairs[x <= boundaries[0]] = lambda: tf.constant(values[0])
pred_fn_pairs[x > boundaries[-1]] = lambda: tf.constant(values[-1])
for lower, upper, value in zip(boundaries[:-1],
boundaries[1:],
values[1:-1]):
# We need to bind value here; can do this with lambda value=value: ...
pred = (x > lower) & (x <= upper)
pred_fn_pairs[pred] = lambda value=value: tf.constant(value)
return tf.case(pred_fn_pairs, lambda: tf.constant(values[0]),
exclusive=True)
def piecewise_function(param, values, changepoints, name=None,
dtype=tf.float32):
"""Compute a piecewise function.
Arguments:
param: The function parameter.
values: List of function values (numbers or tensors).
changepoints: Sorted list of points where the function changes from
one value to the next. Must be one item shorter than `values`.
"""
if len(changepoints) != len(values) - 1:
raise ValueError("changepoints has length {}, expected {} (values "
"has length {})".format(len(changepoints),
len(values) - 1,
len(values)))
with tf.name_scope(name, "PiecewiseFunction",
[param, values, changepoints]) as s_name:
values = [tf.convert_to_tensor(y, dtype=dtype) for y in values]
# this is a trick to make each lambda return a different y:
lambdas = [lambda y=y: y for y in values]
predicates = [tf.less(param, x) for x in changepoints]
return tf.case(list(zip(predicates, lambdas[:-1])), lambdas[-1],
name=s_name)
def beta_schedule(schedule, global_step, final_beta, decay_start, decay_end):
"""Get KL multiplier (beta) based on the schedule."""
if decay_start > decay_end:
raise ValueError("decay_end is smaller than decay_end.")
# Since some of the TF schedules do not support incrementing a value,
# in all of the schedules, we anneal the beta from final_beta to zero
# and then reverse it at the bottom.
if schedule == "constant":
decayed_value = 0.0
elif schedule == "linear":
decayed_value = tf.train.polynomial_decay(
learning_rate=final_beta,
global_step=global_step - decay_start,
decay_steps=decay_end - decay_start,
end_learning_rate=0.0)
elif schedule == "noisy_linear_cosine_decay":
decayed_value = tf.train.noisy_linear_cosine_decay(
learning_rate=final_beta,
global_step=global_step - decay_start,
decay_steps=decay_end - decay_start)
# TODO(mechcoder): Add log_annealing schedule.
else:
raise ValueError("Unknown beta schedule.")
increased_value = final_beta - decayed_value
increased_value = tf.maximum(0.0, increased_value)
beta = tf.case(
pred_fn_pairs={
tf.less(global_step, decay_start): lambda: 0.0,
tf.greater(global_step, decay_end): lambda: final_beta},
default=lambda: increased_value)
return beta
def get_scheduled_sample_inputs(self,
done_warm_start,
groundtruth_items,
generated_items,
scheduled_sampling_func):
"""Scheduled sampling.
Args:
done_warm_start: whether we are done with warm start or not.
groundtruth_items: list of ground truth items.
generated_items: list of generated items.
scheduled_sampling_func: scheduled sampling function to choose between
groundtruth items and generated items.
Returns:
A mix list of ground truth and generated items.
"""
def sample():
"""Calculate the scheduled sampling params based on iteration number."""
with tf.variable_scope("scheduled_sampling", reuse=tf.AUTO_REUSE):
output_items = []
for item_gt, item_gen in zip(groundtruth_items, generated_items):
output_items.append(scheduled_sampling_func(item_gt, item_gen))
return output_items
cases = [
(tf.logical_not(done_warm_start), lambda: groundtruth_items),
(tf.logical_not(self.is_training), lambda: generated_items),
]
output_items = tf.case(cases, default=sample, strict=True)
return output_items
def proj_second_order(x):
s, v = x[:1,:], x[1:,:]
norm_v = norm(v)
s = tf.squeeze(s)
return tf.case(
((norm_v <= -s, lambda: tf.zeros_like(x)),
(norm_v <= s, lambda: x)),
lambda: 0.5*(1 + s/norm_v)*vstack([tf.reshape(norm_v, (1,1)), v]))
def learning_rate_schedule(): # Function which controls learning rate during training
import tensorflow as tf
step = tf.train.get_or_create_global_step()
lr = tf.case([(tf.less(step, 1000), lambda: tf.constant(0.0004)),
(tf.less(step, 10000), lambda: tf.constant(0.01)),
(tf.less(step, 40000), lambda: tf.constant(0.005)),
(tf.less(step, 55000), lambda: tf.constant(0.0005)),
(tf.less(step, 65000), lambda: tf.constant(0.00005))])
return lr
def build_update(self):
"""Perform sampling and exchange.
"""
# Sample by Metropolis-Hastings for each replica.
replica_sample = []
replica_accept = []
for i in range(self.n_replica):
sample_, accept_ = self._mh_sample(self.replica_vars[i],
self.inverse_temperatures[i])
replica_sample.append(sample_)
replica_accept.append(accept_)
accept = replica_accept[0]
# Variable to store order of replicas after exchange
new_replica_idx = tf.Variable(tf.range(self.n_replica))
new_replica_idx = tf.assign(new_replica_idx, tf.range(self.n_replica))
# Variable to store ratio of current samples
replica_ratio = tf.Variable(tf.zeros(
self.n_replica, dtype=list(self.latent_vars)[0].dtype))
replica_ratio = self._replica_ratio(replica_ratio, replica_sample)
# Exchange adjacent replicas at frequency of exchange_freq
u = tf.random_uniform([])
exchange = u < self.exchange_freq
new_replica_idx = tf.cond(
exchange, lambda: self._replica_exchange(
new_replica_idx, replica_ratio), lambda: new_replica_idx)
# New replica sorted by new_replica_idx
new_replica_sample = []
for i in range(self.n_replica):
new_replica_sample.append(
{z: tf.case({tf.equal(tf.gather(new_replica_idx, i), j):
_stateful_lambda(replica_sample[j][z])
for j in range(self.n_replica)},
default=lambda: replica_sample[0][z], exclusive=True) for z, qz in
six.iteritems(self.latent_vars)})
assign_ops = []
# Update Empirical random variables.
for z, qz in six.iteritems(self.latent_vars):
variable = qz.get_variables()[0]
assign_ops.append(tf.scatter_update(variable, self.t,
new_replica_sample[0][z]))
for i in range(self.n_replica):
for z, qz in six.iteritems(self.replica_vars[i]):
variable = qz.get_variables()[0]
assign_ops.append(tf.scatter_update(variable, self.t,
new_replica_sample[i][z]))
# Increment n_accept (if accepted).
assign_ops.append(self.n_accept.assign_add(tf.where(accept, 1, 0)))
return tf.group(*assign_ops)
def _tf_nth(fns, n):
"""Runs only the nth element of fns, where n is a scalar integer tensor."""
cases = [(tf.equal(tf.constant(i, n.dtype), n), fn)
for i, fn in enumerate(fns)]
final_pred, final_fn = cases.pop()
def default():
with tf.control_dependencies([
tf.Assert(final_pred, [n, len(fns)], name='nth_index_error')]):
return final_fn()
if len(fns) == 1: return default()
return tf.case(cases, default)
def read_data(file_queue):
reader = tf.TextLineReader(skip_header_lines=1)
key, value = reader.read(file_queue)
defaults = [[0], [0.], [0.], [0.], [0.], ['']]
Id,SepalLengthCm,SepalWidthCm,PetalLengthCm,PetalWidthCm,Species = tf.decode_csv(value, defaults)
preprocess_op = tf.case({
tf.equal(Species, tf.constant('Iris-setosa')): lambda: tf.constant(0),
tf.equal(Species, tf.constant('Iris-versicolor')): lambda: tf.constant(1),
tf.equal(Species, tf.constant('Iris-virginica')): lambda: tf.constant(2),
}, lambda: tf.constant(-1), exclusive=True)
return tf.stack([SepalLengthCm,SepalWidthCm,PetalLengthCm,PetalWidthCm]), preprocess_op
def _produce_posterior_estimate(posterior_dist, posterior_estimate_mode,
raw_var_name):
"""Create tensor representing estimate of posterior.
Args:
posterior_dist: An instance of `tfp.distributions.Distribution`.
The variational posterior from which to produce an estimate of the
variable in question.
posterior_estimate_mode: A `Tensor` of dtype `tf.string`, which
determines the inference mode.
raw_var_name: The name of the variable over which inference is done.
Returns:
`z_sample`, a `Tensor` representing an estimate derived from the
posterior distribution.
"""
conds = [
tf.equal(posterior_estimate_mode,
tf.constant(EstimatorModes.sample),
name="equal_sample_mode"),
tf.equal(posterior_estimate_mode,
tf.constant(EstimatorModes.mean),
name="equal_mean_mode"),
tf.equal(posterior_estimate_mode,
tf.constant(EstimatorModes.last_sample),
name="equal_last_sample_mode"),
]
# pylint: disable=unnecessary-lambda
results = [
lambda: posterior_dist.sample(),
lambda: posterior_dist.mean(),
lambda: posterior_dist.last_sample()
]
def default_case_branch_raising_error():
err_msg = "Invalid posterior estimate mode."
raise_err = tf.Assert(tf.constant(False), data=[tf.constant(err_msg)])
with tf.control_dependencies([raise_err]):
return posterior_dist.mean()
if hasattr(posterior_dist, "last_sample"):
cases = {conds[0]: results[0], conds[1]: results[1], conds[2]: results[2]}
else:
cases = {conds[0]: results[0], conds[1]: results[1]}
z_sample = tf.case(
cases,
exclusive=True,
default=default_case_branch_raising_error,
name="{}_posterior_estimate".format(raw_var_name))
# pylint: enable=unnecessary-lambda
return z_sample
def ConditionalDecoding(keys_to_tensors):
"""See base class."""
image_buffer = keys_to_tensors['image/encoded']
image_format = keys_to_tensors['image/format']
def DecodePng():
return tf.image.decode_png(image_buffer, 3)
def DecodeJpg():
return tf.image.decode_jpeg(image_buffer, 3)
image = tf.case({
tf.equal(image_format, 'png'): DecodePng,
}, default=DecodeJpg, exclusive=True)
image = tf.reshape(image, image_shape)
return image
def parse_record(record):
columns = tf.decode_csv(record, record_defaults=commons.HEADER_DEFAULTS)
features = dict(zip(commons.HEADERS, columns))
label = features.pop(commons.LABEL_COL)
features[commons.WEIGHT_COLUNM_NAME] = tf.case(
{
tf.equal(label, '7'): lambda: 7.5,
tf.equal(label, '5'): lambda: 5.5,
tf.equal(label, '8'): lambda: 5.0,
tf.equal(label, '10'): lambda: 4.2,
tf.equal(label, '1'): lambda: 3.5
}, default=lambda: 1.0, exclusive=True
)
return features, label
def _augment_data(self, inout, nchan=6):
"""Flip, crop and rotate samples randomly."""
with tf.name_scope('data_augmentation'):
if self.fliplr:
inout = tf.image.random_flip_left_right(inout, seed=1234)
if self.flipud:
inout = tf.image.random_flip_up_down(inout, seed=3456)
if self.rotate:
angle = tf.random_uniform((), minval=0, maxval=4, dtype=tf.int32, seed=4567)
inout = tf.case([(tf.equal(angle, 1), lambda: tf.image.rot90(inout, k=1)),
(tf.equal(angle, 2), lambda: tf.image.rot90(inout, k=2)),
(tf.equal(angle, 3), lambda: tf.image.rot90(inout, k=3))],
lambda: inout)
inout.set_shape([None, None, nchan])
with tf.name_scope('crop'):
shape = tf.shape(inout)
new_height = tf.to_int32(self.output_resolution[0])
new_width = tf.to_int32(self.output_resolution[1])
height_ok = tf.assert_less_equal(new_height, shape[0])
width_ok = tf.assert_less_equal(new_width, shape[1])
with tf.control_dependencies([height_ok, width_ok]):
if self.random_crop:
inout = tf.random_crop(
inout, tf.stack([new_height, new_width, nchan]))
else:
height_offset = tf.to_int32((shape[0]-new_height)/2)
width_offset = tf.to_int32((shape[1]-new_width)/2)
inout = tf.image.crop_to_bounding_box(
inout, height_offset, width_offset,
new_height, new_width)
inout.set_shape([None, None, nchan])
inout = tf.image.resize_images(
inout, [self.output_resolution[0], self.output_resolution[1]])
fullres = inout
with tf.name_scope('resize'):
new_size = 256
inout = tf.image.resize_images(
inout, [new_size, new_size],
method=tf.image.ResizeMethod.NEAREST_NEIGHBOR)
return fullres, inout
def consistency_costs(logits1, logits2, cons_coefficient, mask, consistency_trust, name=None):
"""Takes a softmax of the logits and returns their distance as described below
Consistency_trust determines the distance metric to use
- trust=0: MSE
- 0 < trust < 1: a scaled KL-divergence but both sides mixtured with
a uniform distribution with given trust used as the mixture weight
- trust=1: scaled KL-divergence
When trust > 0, the cost is scaled to make the gradients
the same size as MSE when trust -> 0. The scaling factor used is
2 * (1 - 1/num_classes) / num_classes**2 / consistency_trust**2 .
To have consistency match the strength of classification, use
consistency coefficient = num_classes**2 / (1 - 1/num_classes) / 2
which is 55.5555... when num_classes=10.
Two potential stumbling blokcs:
- When trust=0, this gives gradients to both logits, but when trust > 0
this gives gradients only towards the first logit.
So do not use trust > 0 with the Pi model.
- Numerics may be unstable when 0 < trust < 1.
"""
with tf.name_scope(name, "consistency_costs") as scope:
num_classes = 10
assert_shape(logits1, [None, num_classes])
assert_shape(logits2, [None, num_classes])
assert_shape(cons_coefficient, [])
softmax1 = tf.nn.softmax(logits1)
softmax2 = tf.nn.softmax(logits2)
kl_cost_multiplier = 2 * (1 - 1/num_classes) / num_classes**2 / consistency_trust**2
def pure_mse():
costs = tf.reduce_mean((softmax1 - softmax2) ** 2, -1)
return costs
def pure_kl():
cross_entropy = tf.nn.softmax_cross_entropy_with_logits(logits=logits1, labels=softmax2)
entropy = tf.nn.softmax_cross_entropy_with_logits(logits=logits2, labels=softmax2)
costs = cross_entropy - entropy
costs = costs * kl_cost_multiplier
return costs
def mixture_kl():
with tf.control_dependencies([tf.assert_greater(consistency_trust, 0.0),
tf.assert_less(consistency_trust, 1.0)]):
uniform = tf.constant(1 / num_classes, shape=[num_classes])
mixed_softmax1 = consistency_trust * softmax1 + (1 - consistency_trust) * uniform
mixed_softmax2 = consistency_trust * softmax2 + (1 - consistency_trust) * uniform
costs = tf.reduce_sum(mixed_softmax2 * tf.log(mixed_softmax2 / mixed_softmax1), axis=1)
costs = costs * kl_cost_multiplier
return costs
costs = tf.case([
(tf.equal(consistency_trust, 0.0), pure_mse),
(tf.equal(consistency_trust, 1.0), pure_kl)
], default=mixture_kl)
costs = costs * tf.to_float(mask) * cons_coefficient
mean_cost = tf.reduce_mean(costs, name=scope)
assert_shape(costs, [None])
assert_shape(mean_cost, [])
return mean_cost, costs
def __call__(self,
inputs,
seq_length,
is_training=False,
reuse=False,
scope=None):
'''
Add the DNN variables and operations to the graph
Args:
inputs: the inputs to the neural network, this is a list containing
a [batch_size, input_dim] tensor for each time step
seq_length: The sequence lengths of the input utterances, if None
the maximal sequence length will be taken
is_training: whether or not the network is in training mode
reuse: wheter or not the variables in the network should be reused
scope: the name scope
Returns:
A triple containing:
- output logits
- the output logits sequence lengths as a vector
- a saver object
- a dictionary of control operations:
-add: add a layer to the network
-init: initialise the final layer
'''
with tf.variable_scope(scope or type(self).__name__, reuse=reuse):
#input layer
layer = FFLayer(self.num_units, self.activation)
#output layer
outlayer = FFLayer(self.output_dim,
TfActivation(None, lambda (x): x), 0)
#do the forward computation
#convert the sequential data to non sequential data
nonseq_inputs = seq_convertors.seq2nonseq(inputs, seq_length)
activations = [None] * self.num_layers
#activations[0] = layer(nonseq_inputs, is_training, reuse, 'layer0')
#cnn_layer = RestNet()
#cnn_layer = CnnVd6()
cnn_layer = CnnLayer()
activations[0] = cnn_layer(nonseq_inputs, is_training, reuse,
'layer0')
for l in range(1, self.num_layers):
activations[l] = layer(activations[l - 1], is_training, reuse,
'layer' + str(l))
if self.layerwise_init:
#variable that determines how many layers are initialised
#in the neural net
initialisedlayers = tf.get_variable(
'initialisedlayers', [],
initializer=tf.constant_initializer(0),
trainable=False,
dtype=tf.int32)
#operation to increment the number of layers
add_layer_op = initialisedlayers.assign(initialisedlayers +
1).op
#compute the logits by selecting the activations at the layer
#that has last been added to the network, this is used for layer
#by layer initialisation
logits = tf.case([(tf.equal(initialisedlayers, tf.constant(l)),
Callable(activations[l]))
for l in range(len(activations))],
default=Callable(activations[-1]),
exclusive=True,
name='layerSelector')
logits.set_shape([None, self.num_units])
else:
logits = activations[-1]
logits = outlayer(logits, is_training, reuse,
'layer' + str(self.num_layers))
if self.layerwise_init:
#operation to initialise the final layer
init_last_layer_op = tf.initialize_variables(
tf.get_collection(tf.GraphKeys.VARIABLES,
scope=(tf.get_variable_scope().name +
'/layer' + str(self.num_layers))))
control_ops = {'add': add_layer_op, 'init': init_last_layer_op}
else:
control_ops = None
#convert the logits to sequence logits to match expected output
seq_logits = seq_convertors.nonseq2seq(logits, seq_length,
len(inputs))
#create a saver
saver = tf.train.Saver()
return seq_logits, seq_length, saver, control_ops
def _enas_layer(self, layer_id, prev_layers, start_idx, out_filters,
is_training):
"""
Args:
layer_id: current layer
prev_layers: cache of previous layers. for skip connections
start_idx: where to start looking at. technically, we can infer this
from layer_id, but why bother...
is_training: for batch_norm
"""
inputs = prev_layers[-1]
if self.whole_channels:
if self.data_format == "NHWC":
inp_h = inputs.get_shape()[1].value
inp_w = inputs.get_shape()[2].value
inp_c = inputs.get_shape()[3].value
elif self.data_format == "NCHW":
inp_c = inputs.get_shape()[1].value
inp_h = inputs.get_shape()[2].value
inp_w = inputs.get_shape()[3].value
count = self.sample_arc[start_idx]
branches = {}
with tf.variable_scope("branch_0"):
y = self._conv_branch(inputs,
3,
is_training,
out_filters,
out_filters,
start_idx=0)
branches[tf.equal(count, 0)] = lambda: y
with tf.variable_scope("branch_1"):
y = self._conv_branch(inputs,
3,
is_training,
out_filters,
out_filters,
start_idx=0,
separable=True)
branches[tf.equal(count, 1)] = lambda: y
with tf.variable_scope("branch_2"):
y = self._conv_branch(inputs,
5,
is_training,
out_filters,
out_filters,
start_idx=0)
branches[tf.equal(count, 2)] = lambda: y
with tf.variable_scope("branch_3"):
y = self._conv_branch(inputs,
5,
is_training,
out_filters,
out_filters,
start_idx=0,
separable=True)
branches[tf.equal(count, 3)] = lambda: y
if self.num_branches >= 5:
with tf.variable_scope("branch_4"):
y = self._pool_branch(inputs,
is_training,
out_filters,
"avg",
start_idx=0)
branches[tf.equal(count, 4)] = lambda: y
if self.num_branches >= 6:
with tf.variable_scope("branch_5"):
y = self._pool_branch(inputs,
is_training,
out_filters,
"max",
start_idx=0)
branches[tf.equal(count, 5)] = lambda: y
out = tf.case(branches,
default=lambda: tf.constant(0, tf.float32),
exclusive=True)
if self.data_format == "NHWC":
out.set_shape([None, inp_h, inp_w, out_filters])
elif self.data_format == "NCHW":
out.set_shape([None, out_filters, inp_h, inp_w])
else:
count = self.sample_arc[start_idx:start_idx +
2 * self.num_branches]
branches = []
with tf.variable_scope("branch_0"):
branches.append(
self._conv_branch(inputs,
3,
is_training,
count[1],
out_filters,
start_idx=count[0]))
with tf.variable_scope("branch_1"):
branches.append(
self._conv_branch(inputs,
3,
is_training,
count[3],
out_filters,
start_idx=count[2],
separable=True))
with tf.variable_scope("branch_2"):
branches.append(
self._conv_branch(inputs,
5,
is_training,
count[5],
out_filters,
start_idx=count[4]))
with tf.variable_scope("branch_3"):
branches.append(
self._conv_branch(inputs,
5,
is_training,
count[7],
out_filters,
start_idx=count[6],
separable=True))
if self.num_branches >= 5:
with tf.variable_scope("branch_4"):
branches.append(
self._pool_branch(inputs,
is_training,
count[9],
"avg",
start_idx=count[8]))
if self.num_branches >= 6:
with tf.variable_scope("branch_5"):
branches.append(
self._pool_branch(inputs,
is_training,
count[11],
"max",
start_idx=count[10]))
with tf.variable_scope("final_conv"):
w = create_weight(
"w", [self.num_branches * out_filters, out_filters])
w_mask = tf.constant(
[False] * (self.num_branches * out_filters), tf.bool)
new_range = tf.range(0,
self.num_branches * out_filters,
dtype=tf.int32)
for i in range(self.num_branches):
start = out_filters * i + count[2 * i]
new_mask = tf.logical_and(
start <= new_range,
new_range < start + count[2 * i + 1])
w_mask = tf.logical_or(w_mask, new_mask)
w = tf.boolean_mask(w, w_mask)
w = tf.reshape(w, [1, 1, -1, out_filters])
inp = prev_layers[-1]
if self.data_format == "NHWC":
branches = tf.concat(branches, axis=3)
elif self.data_format == "NCHW":
branches = tf.concat(branches, axis=1)
N = tf.shape(inp)[0]
H = inp.get_shape()[2].value
W = inp.get_shape()[3].value
branches = tf.reshape(branches, [N, -1, H, W])
out = tf.nn.conv2d(branches,
w, [1, 1, 1, 1],
"SAME",
data_format=self.data_format)
out = batch_norm(out,
is_training,
data_format=self.data_format)
out = tf.nn.relu(out)
if layer_id > 0:
if self.whole_channels:
skip_start = start_idx + 1
else:
skip_start = start_idx + 2 * self.num_branches
skip = self.sample_arc[skip_start:skip_start + layer_id]
with tf.variable_scope("skip"):
res_layers = []
for i in range(layer_id):
res_layers.append(
tf.cond(tf.equal(skip[i], 1), lambda: prev_layers[i],
lambda: tf.zeros_like(prev_layers[i])))
res_layers.append(out)
out = tf.add_n(res_layers)
out = batch_norm(out,
is_training,
data_format=self.data_format)
return out
def augmentation(input_patch, label_patch):
""" Data Augmentation with TensorFlow ops.
Args:
input_patch: input tensor representing an input patch or image
label_patch: label tensor representing an target patch or image
Returns:
rotated input_patch and label_patch randomly
"""
def no_trans():
return input_patch, label_patch
def vflip():
inpPatch = input_patch[::-1, :, :]
labPatch = label_patch[::-1, :, :]
return inpPatch, labPatch
def hflip():
inpPatch = input_patch[:, ::-1, :]
labPatch = label_patch[:, ::-1, :]
return inpPatch, labPatch
def hvflip():
inpPatch = input_patch[::-1, ::-1, :]
labPatch = label_patch[::-1, ::-1, :]
return inpPatch, labPatch
def trans():
inpPatch = tf.image.transpose_image(input_patch[:, :, :])
labPatch = tf.image.transpose_image(label_patch[:, :, :])
return inpPatch, labPatch
def tran_vflip():
inpPatch = tf.image.transpose_image(input_patch)[::-1, :, :]
labPatch = tf.image.transpose_image(label_patch)[::-1, :, :]
return inpPatch, labPatch
def tran_hflip():
inpPatch = tf.image.transpose_image(input_patch)[:, ::-1, :]
labPatch = tf.image.transpose_image(label_patch)[:, ::-1, :]
return inpPatch, labPatch
def tran_hvflip():
inpPatch = tf.image.transpose_image(input_patch)[::-1, ::-1, :]
labPatch = tf.image.transpose_image(label_patch)[::-1, ::-1, :]
return inpPatch, labPatch
rot = tf.random_uniform(shape=(), minval=2, maxval=9, dtype=tf.int32)
input_patch, label_patch = tf.case(
{
tf.equal(rot, 2): vflip,
tf.equal(rot, 3): hflip,
tf.equal(rot, 4): hvflip,
tf.equal(rot, 5): trans,
tf.equal(rot, 6): tran_vflip,
tf.equal(rot, 7): tran_hflip,
tf.equal(rot, 8): tran_hvflip
},
default=no_trans,
exclusive=True)
return input_patch, label_patch
y = tf.random_uniform([])
out = tf.cond(tf.greater(x, y), lambda: tf.add(x, y),
lambda: tf.subtract(x, y))
###############################################################################
# 1b: Create two 0-d tensors x and y randomly selected from the range [-1, 1).
# Return x + y if x < y, x - y if x > y, 0 otherwise.
# Hint: Look up tf.case().
###############################################################################
x = tf.random_uniform([], -1, 1, dtype=tf.float32)
y = tf.random_uniform([], -1, 1, dtype=tf.float32)
out = tf.case(
{
tf.less(x, y): lambda: tf.add(x, y),
tf.greater(x, y): lambda: tf.subtract(x, y)
},
default=lambda: tf.constant(0.0),
exclusive=True)
###############################################################################
# 1c: Create the tensor x of the value [[0, -2, -1], [0, 1, 2]]
# and y as a tensor of zeros with the same shape as x.
# Return a boolean tensor that yields Trues if x equals y element-wise.
# Hint: Look up tf.equal().
###############################################################################
x = tf.constant([[0, -2, -1], [0, 1, 2]])
y = tf.zeros_like(x)
out = tf.equal(x, y)
test_batch_size = FLAGS.batch_size / FLAGS.num_test_crops
test_iteration = sum(map(len, test_files)) / test_batch_size
class_names = tf.get_variable(
'class_names',
shape=(len(data.CLASS_NAMES),),
dtype=tf.string,
initializer=tf.constant_initializer(data.CLASS_NAMES),
trainable=False,
collections=[tf.GraphKeys.VARIABLES, NET_VARIABLES])
if FLAGS.command == 'test':
(key, image, label) = test_values
else:
(key, image, label) = tf.case([
(tf.equal(phase, TRAIN), lambda: train_values),
(tf.equal(phase, TEST), lambda: test_values)], default=lambda: train_values)
key.set_shape((None,))
image.set_shape((None, data.SIZE, data.SIZE, 3))
label.set_shape((None,))
''' Main Network
'''
image_splits = tf.split(0, FLAGS.num_gpus, image)
label_splits = tf.split(0, FLAGS.num_gpus, label)
logit_splits = []
loss_splits = []
grads_splits = []
def build_progressive_output(cls, N_stages, get_x_stabilize, get_x_fade,
get_final_progression):
"""
Function to build progressive output tensor - one that
conditionally returns the output through the appropriate
blocks of the network given the current stage and whether or
not we're fading-in a block.
Args:
N_stages - (int) number of total stages - enumeration assumed to start at zero
get_x_stabilize - (callable) function that takes one arg, 'stage', and returns the corresponding stabilizing tensor
get_x_fade - (callable) function that takes one arg, 'stage', and returns the corresponding fading tensor
get_final_progression - (callable) function that takes no args and returns a the final progression - a pass through all blocks
Return: tf.Tensor - specifically one from tf.case.
Note: All aforementioned callables must give the same type of output - see the documentation of tf.cast for specifics.
"""
#-----------------------------------------------------
# Define function to help build progressive output.
#-----------------------------------------------------
# Get tensors to help keep track of the progression
initial_steps_placeholder, stabilizing_steps_placeholder, stage_int_tensor, fade_phase_bool, alpha_tensor = cls.get_or_create_tracking_tensors(
)
def populate_progressive_pred_fn_list(pred_fn_list, stage, fade_phase):
'''
This function populates a given list with predicate-function pairs
that sequentially build up the blocks of the network
following 'stage' and 'fade_phase' inclusively.
The populated list may be used to construct a conditional tensor with tf.case.
For instance, with args:
pred_fn_list = [], stage = 0, fade_phase = False
one would have after running this function:
pred_fn_list = [
(predicate of 0th stage when stabilizing, pass through first block)
(predicate of 1st stage when fading-in , pass through first block + pass through fading-in second block)
...
(predicate of final stage when stabilizing, pass through all blocks)
]
Args:
pred_fn_list - (list) list to populate (starts empty and gets mutated)
stage - (int) stage of progression
fade_phase - (bool) whether or not to build the fade-in phase
Return: None
'''
# Define predicate of conditional statements.
predicate = (stage_int_tensor <= stage) & tf.equal(
fade_phase_bool, tf.constant(fade_phase))
# Base Case: Final stage and after the last fade-in.
if stage == N_stages and not fade_phase:
# Append pass through all blocks - to occur when all blocks except the last (i.e. 'call') have been stabilized.
pred_fn_list.append((predicate, get_final_progression))
# Other Cases: Previous stages.
elif not fade_phase:
# Append pass through all blocks up to the current one to be stabilized.
pred_fn_list.append(
(predicate, lambda: get_x_stabilize(stage)))
# Populate with remaining progressions.
populate_progressive_pred_fn_list(pred_fn_list,
stage + 1,
fade_phase=True)
else:
# Append pass through fading-in an fading-out blocks.
pred_fn_list.append((predicate, lambda: get_x_fade(stage)))
# Populate with remaining progressions.
populate_progressive_pred_fn_list(pred_fn_list,
stage,
fade_phase=False)
#-----------------------------------------------------
# Build and return progression.
#-----------------------------------------------------
# Populate progressive_pred_fn_list - a list of predicate-function pairs that sequentially build the up the blocks of the network.
progressive_pred_fn_list = []
populate_progressive_pred_fn_list(progressive_pred_fn_list,
stage=0,
fade_phase=False) # Populate list.
# Build output progressive tensor - the final case (pass through all blocks) is set as the default.
output_progression_tensor = tf.case(
pred_fn_pairs=progressive_pred_fn_list[:-1],
default=progressive_pred_fn_list[-1][1])
# Return tf.case tensor.
return output_progression_tensor
def case_graph():
return tf.case([(tf.reduce_all(x < 1), lambda: x + y),
(tf.reduce_all(y > 0), lambda: tf.square(y))],
default=lambda: x - y,
name="test_case")
def __init__(self, config):
self.config = config
self.tf_record_dir = config.tf_record_dir
dataset_str = 'd'
dataset_str += '_' + '_'.join(config.tf_record_dir.replace(
'data/preprocessed/vqa_v2/', '').split('/'))
dataset_str += '_' + config.vfeat_name.replace('.hdf5', '')
hyper_parameter_str = 'bs{}_lr{}'.format(
config.batch_size, config.learning_rate)
self.train_dir = './train_dir/std_{}_{}_{}_{}_{}'.format(
config.model_type, dataset_str, config.prefix, hyper_parameter_str,
time.strftime("%Y%m%d-%H%M%S"))
if not os.path.exists(self.train_dir): os.makedirs(self.train_dir)
log.infov("Train Dir: %s", self.train_dir)
# Input
self.batch_size = config.batch_size
with tf.name_scope('datasets'):
self.target_split = tf.placeholder(tf.string)
with tf.name_scope('datasets/batch'):
vqa_batch = {
'train': input_ops_vqa.create(
self.batch_size, self.tf_record_dir, 'train',
is_train=True, scope='train_ops', shuffle=True),
'val': input_ops_vqa.create(
self.batch_size, self.tf_record_dir, 'val',
is_train=True, scope='val_ops', shuffle=False),
}
batch_opt = {
tf.equal(self.target_split, 'train'): lambda: vqa_batch['train'],
tf.equal(self.target_split, 'val'): lambda: vqa_batch['val'],
}
self.batch = tf.case(
batch_opt, default=lambda: vqa_batch['train'], exclusive=True)
# Model
Model = self.get_model_class(config.model_type)
log.infov('using model class: {}'.format(Model))
self.model = Model(self.batch, config, is_train=True)
# Optimizer
self.global_step = tf.train.get_or_create_global_step(graph=None)
self.learning_rate = config.learning_rate
if config.lr_weight_decay:
self.learning_rate = tf.train.exponential_decay(
self.learning_rate,
global_step=self.global_step,
decay_steps=10000,
decay_rate=0.5,
staircase=True,
name='decaying_learning_rate')
# Checkpoint and monitoring
trainable_vars = tf.trainable_variables()
train_vars = self.model.filter_train_vars(trainable_vars)
log.warn('Trainable variables:')
tf.contrib.slim.model_analyzer.analyze_vars(trainable_vars, print_info=True)
log.warn('Filtered train variables:')
tf.contrib.slim.model_analyzer.analyze_vars(train_vars, print_info=True)
self.optimizer = tf.contrib.layers.optimize_loss(
loss=self.model.loss,
global_step=self.global_step,
learning_rate=self.learning_rate,
optimizer=tf.train.AdamOptimizer,
clip_gradients=0.25,
variables=train_vars,
increment_global_step=True,
name='optimizer')
self.avg_report = {
'train': {},
'val': {}
}
for split in ['train', 'val']:
for key in self.model.report.keys():
self.avg_report[split][key] = tf.placeholder(tf.float32)
tf.summary.scalar('average_{}/{}'.format(split, key),
self.avg_report[split][key],
collections=['average_{}'.format(split)])
self.summary_ops = {
'train': tf.summary.merge_all(key='train'),
'val': tf.summary.merge_all(key='val'),
'heavy_train': tf.summary.merge_all(key='heavy_train'),
'heavy_val': tf.summary.merge_all(key='heavy_val'),
'average_train': tf.summary.merge_all(key='average_train'),
'average_val': tf.summary.merge_all(key='average_val'),
'no_op': tf.no_op(),
}
self.saver = tf.train.Saver(max_to_keep=100)
self.checkpoint_loader = tf.train.Saver(max_to_keep=1)
self.summary_writer = tf.summary.FileWriter(self.train_dir)
self.train_average_iter = self.config.train_average_iter
self.val_average_iter = self.config.val_average_iter
self.heavy_summary_step = self.config.heavy_summary_step
self.validation_step = self.config.validation_step
self.checkpoint_step = self.config.checkpoint_step
self.supervisor = tf.train.Supervisor(
logdir=self.train_dir,
is_chief=True,
saver=None,
summary_op=None,
summary_writer=self.summary_writer,
save_summaries_secs=300,
save_model_secs=None,
global_step=self.global_step,
)
session_config = tf.ConfigProto(
allow_soft_placement=True,
gpu_options=tf.GPUOptions(allow_growth=True),
device_count={'GPU': 1})
self.session = self.supervisor.prepare_or_wait_for_session(
config=session_config)
self.ckpt_path = config.checkpoint
if self.ckpt_path is not None:
log.info('Checkpoint path: {}'.format(self.ckpt_path))
self.checkpoint_loader.restore(self.session, self.ckpt_path)
log.info('Loaded the checkpoint')
def build_model(self,
relu_target,
input_tensor,
style_encoded_tensor=None,
batch_size=8,
feature_weight=1,
pixel_weight=1,
tv_weight=0,
learning_rate=1e-4,
lr_decay=5e-5,
ss_patch_size=3,
ss_stride=1):
'''Build the EncoderDecoder architecture for a given relu layer.
Args:
relu_target: Layer of VGG to decode from
input_tensor: If None then a placeholder will be created, else use this tensor as the input to the encoder
style_encoded_tensor: Tensor for style image features at the same relu layer. Used only at test time.
batch_size: Batch size for training
feature_weight: Float weight for feature reconstruction loss
pixel_weight: Float weight for pixel reconstruction loss
tv_weight: Float weight for total variation loss
learning_rate: Float LR
lr_decay: Float linear decay for training
Returns:
EncoderDecoder namedtuple with input/encoding/output tensors and ops for training.
'''
with tf.name_scope('encoder_decoder_' + relu_target):
### Build encoder for reluX_1
with tf.name_scope('content_encoder_' + relu_target):
if input_tensor is None:
# This is the first level encoder that takes original content imgs
content_imgs = tf.placeholder_with_default(
tf.constant([[[[0., 0., 0.]]]]),
shape=(None, None, None, 3),
name='content_imgs')
else:
# This is an intermediate-level encoder that takes output tensor from previous level as input
content_imgs = input_tensor
# Build content layer encoding model
content_layer = self.vgg_model.get_layer(relu_target).output
content_encoder_model = Model(inputs=self.vgg_model.input,
outputs=content_layer)
# Setup content layer encodings for content images
content_encoded = content_encoder_model(content_imgs)
### Build style encoder & WCT if test mode
if self.mode != 'train':
with tf.name_scope('wct_' + relu_target):
if relu_target == 'relu5_1':
# Apply style swap on relu5_1 encodings if self.swap5 flag is set
# Use AdaIN as transfer op instead of WCT if self.use_adain is set
# Otherwise perform WCT
decoder_input = tf.case(
[(self.swap5, lambda: wct_style_swap(
content_encoded, style_encoded_tensor, self.
ss_alpha, ss_patch_size, ss_stride)),
(self.use_adain,
lambda: adain(content_encoded,
style_encoded_tensor, self.alpha))
],
default=lambda: wct_tf(content_encoded,
style_encoded_tensor, self.
alpha))
else:
decoder_input = tf.cond(
self.use_adain,
lambda: adain(content_encoded,
style_encoded_tensor, self.alpha),
lambda: wct_tf(content_encoded,
style_encoded_tensor, self.alpha))
else: # In train mode we're trying to reconstruct from the encoding, so pass along unchanged
decoder_input = content_encoded
### Build decoder
with tf.name_scope('decoder_' + relu_target):
n_channels = content_encoded.get_shape()[-1].value
decoder_model = self.build_decoder(input_shape=(None, None,
n_channels),
relu_target=relu_target)
# Wrap the decoder_input tensor so that it has the proper shape for decoder_model
decoder_input_wrapped = tf.placeholder_with_default(
decoder_input, shape=[None, None, None, n_channels])
# Reconstruct/decode from encoding
decoded = decoder_model(
Lambda(lambda x: x)(decoder_input_wrapped)
) # Lambda converts TF tensor to Keras
# Content layer encoding for stylized out
decoded_encoded = content_encoder_model(decoded)
if self.mode == 'train': # Train & summary ops only needed for training phase
### Losses
with tf.name_scope('losses_' + relu_target):
# Feature loss between encodings of original & reconstructed
feature_loss = feature_weight * mse(decoded_encoded,
content_encoded)
# Pixel reconstruction loss between decoded/reconstructed img and original
pixel_loss = pixel_weight * mse(decoded, content_imgs)
# Total Variation loss
if tv_weight > 0:
tv_loss = tv_weight * tf.reduce_mean(
tf.image.total_variation(decoded))
else:
tv_loss = tf.constant(0.)
total_loss = feature_loss + pixel_loss + tv_loss
### Training ops
with tf.name_scope('train_' + relu_target):
global_step = tf.Variable(0,
name='global_step_train',
trainable=False)
# self.learning_rate = tf.train.exponential_decay(learning_rate, self.global_step, 100, 0.96, staircase=False)
learning_rate = torch_decay(learning_rate, global_step,
lr_decay)
d_optimizer = tf.train.AdamOptimizer(learning_rate,
beta1=0.9,
beta2=0.999)
# Only train decoder vars, encoder is frozen
d_vars = [
var for var in tf.trainable_variables()
if 'decoder_' + relu_target in var.name
]
train_op = d_optimizer.minimize(total_loss,
var_list=d_vars,
global_step=global_step)
### Loss & image summaries
with tf.name_scope('summary_' + relu_target):
feature_loss_summary = tf.summary.scalar(
'feature_loss', feature_loss)
pixel_loss_summary = tf.summary.scalar('pixel_loss',
pixel_loss)
tv_loss_summary = tf.summary.scalar('tv_loss', tv_loss)
total_loss_summary = tf.summary.scalar('total_loss',
total_loss)
content_imgs_summary = tf.summary.image(
'content_imgs', content_imgs)
decoded_images_summary = tf.summary.image(
'decoded_images', clip(decoded))
for var in d_vars:
tf.summary.histogram(var.op.name, var)
summary_op = tf.summary.merge_all()
else:
# For inference set unnneeded ops to None
pixel_loss, feature_loss, tv_loss, total_loss, train_op, global_step, learning_rate, summary_op = [
None
] * 8
# Put it all together
encoder_decoder = EncoderDecoder(
content_input=content_imgs,
content_encoder_model=content_encoder_model,
content_encoded=content_encoded,
style_encoded=style_encoded_tensor,
decoder_input=decoder_input,
decoder_model=decoder_model,
decoded=decoded,
decoded_encoded=decoded_encoded,
pixel_loss=pixel_loss,
feature_loss=feature_loss,
tv_loss=tv_loss,
total_loss=total_loss,
train_op=train_op,
global_step=global_step,
learning_rate=learning_rate,
summary_op=summary_op)
return encoder_decoder
def video_preprocessing_training_op(video_op):
with tf.name_scope('Single_gesture_video_preprocessing_training'):
#define constant tensors needed for preprocessing
clips_in_video = tf.constant(CROP[0], shape=[1], dtype=tf.int32)
channels = tf.constant(CROP[3], shape=[1], dtype=tf.int32)
# Reshape for easier preprocessing
video_op = tf.cast(video_op, dtype=tf.float32)
clip_op = tf.reshape(video_op, [
constants_3dcnn.CLIPS_PER_VIDEO *
constants_3dcnn.FRAMES_PER_CLIP_PP
] + list(IMAGE_SIZE))
# +- 3frames of jittering
zero_tensor = tf.zeros(shape=[1], dtype=tf.int32)
processed_video_jittering = clip_op
#jittering = tf.random_uniform(shape=[1], minval=0, maxval=6, dtype=tf.int32)
#begin_jittering = tf.squeeze(tf.stack([jittering, zero_tensor, zero_tensor, zero_tensor]))
#processed_video_jittering = tf.slice(clip_op, begin=begin_jittering, size=constants_3dcnn.JITTERING)
#### Take random crop of dimension CROP=(CLIPS_PER_VIDEO * FRAMES_PER_CLIP, 112, 112, 3)
col_crop_idx = tf.random_uniform(shape=[1],
minval=17,
maxval=21,
dtype=tf.int32)
row_crop_idx = tf.random_uniform(shape=[1],
minval=0,
maxval=(IMAGE_SIZE[1] - CROP[2]),
dtype=tf.int32)
begin_crop = tf.squeeze(
tf.stack([zero_tensor, col_crop_idx, row_crop_idx, zero_tensor]))
processed_video = tf.slice(processed_video_jittering,
begin=begin_crop,
size=constants_3dcnn.CROP)
#### Random rotation of +- 15 deg
angle = tf.random_uniform(shape=[1],
minval=-ROT_ANGLE,
maxval=ROT_ANGLE,
dtype=tf.float32)
processed_video = tf.contrib.image.rotate(processed_video,
angles=angle)
#### Random scaling
## do this by taking a crop of random size and then resizing to the original shape
#begin_col = tf.random_uniform(shape=[1], minval=0, maxval=(int(0.2 * CROP[1])), dtype=tf.int32)
#begin_row = tf.random_uniform(shape=[1], minval=0, maxval=(int(0.2 * CROP[2])), dtype=tf.int32)
## get crop window size scaling_col, scaling_row
#crop_col = tf.constant(CROP[1], dtype=tf.int32)
#crop_row = tf.constant(CROP[2], dtype=tf.int32)
#scaling_col = tf.subtract(crop_col, begin_col)
#scaling_row = tf.subtract(crop_row, begin_row)
## do scaling by slicing and then resizing to orignal size
#begin_scaling = tf.squeeze(tf.stack([zero_tensor, begin_col, begin_row, zero_tensor]))
#size_scaling = tf.squeeze(tf.stack([clips_in_video, scaling_col, scaling_row, channels]))
#scaling_crop = tf.slice(processed_video, begin=begin_scaling, size=size_scaling)
#processed_video = tf.image.resize_images(scaling_crop, size=[CROP[1], CROP[2]])
# random mirroring
rand = tf.random_uniform(minval=0,
maxval=1,
shape=[],
dtype=tf.float32)
const_prob = tf.constant(0.5, dtype=tf.float32)
processed_video = tf.case(
[(tf.less(rand, const_prob), lambda: processed_video)],
default=lambda: tf.reverse(processed_video, axis=[2]))
# reshape to correct size for nework
processed_video = tf.reshape(processed_video, [
constants_3dcnn.CLIPS_PER_VIDEO, constants_3dcnn.FRAMES_PER_CLIP,
CROP[1], CROP[2], CROP[3]
])
# normalise per clip
processed_video_list = [
tf.nn.batch_normalization(i,
mean=tf.nn.moments(i, axes=[0, 1, 2])[0],
variance=tf.nn.moments(i, axes=[0, 1,
2])[1],
offset=None,
scale=None,
variance_epsilon=1e-10)
for i in tf.unstack(processed_video)
]
processed_video = tf.stack(processed_video_list)
return processed_video
import tensorflow as tf
x = tf.random_uniform([], -2, 2)
y = tf.random_uniform([], -2, 2)
def f1():
return tf.add(x, y)
def f2():
return tf.sub(x, y)
def f3():
return tf.constant(0, dtype=tf.float32)
val = tf.case({
tf.less(x, y): f2,
tf.greater(x, y): f1
},
default=f3,
exclusive=True)
sess = tf.InteractiveSession()
print(sess.run(tf.less(x, y)))
print(sess.run(tf.greater(x, y)))
print(sess.run(val))
sess.close()
def padding_inputs_width(image: tf.Tensor, target_shape: Tuple[int, int],
increment: int) -> Tuple[tf.Tensor, tf.Tensor]:
target_ratio = target_shape[1] / target_shape[0]
# Compute ratio to keep the same ratio in new image and get the size of padding
# necessary to have the final desired shape
shape = tf.shape(image)
ratio = tf.divide(shape[1], shape[0], name='ratio')
new_h = target_shape[0]
new_w = tf.cast(
tf.round((ratio * new_h) / increment) * increment, tf.int32)
f1 = lambda: (new_w, ratio)
f2 = lambda: (new_h, tf.constant(1.0, dtype=tf.float64))
new_w, ratio = tf.case(
{
tf.greater(new_w, 0): f1,
tf.less_equal(new_w, 0): f2
},
default=f1,
exclusive=True)
target_w = target_shape[1]
# Definitions for cases
def pad_fn():
with tf.name_scope('mirror_padding'):
pad = tf.subtract(target_w, new_w)
img_resized = tf.image.resize_images(image, [new_h, new_w])
# Padding to have the desired width
paddings = [[0, 0], [0, pad], [0, 0]]
pad_image = tf.pad(img_resized,
paddings,
mode='SYMMETRIC',
name=None)
# Set manually the shape
pad_image.set_shape(
[target_shape[0], target_shape[1],
img_resized.get_shape()[2]])
return pad_image, (new_h, new_w)
def replicate_fn():
with tf.name_scope('replication_padding'):
img_resized = tf.image.resize_images(image, [new_h, new_w])
# If one symmetry is not enough to have a full width
# Count number of replications needed
n_replication = tf.cast(tf.ceil(target_shape[1] / new_w), tf.int32)
img_replicated = tf.tile(img_resized,
tf.stack([1, n_replication, 1]))
pad_image = tf.image.crop_to_bounding_box(
image=img_replicated,
offset_height=0,
offset_width=0,
target_height=target_shape[0],
target_width=target_shape[1])
# Set manually the shape
pad_image.set_shape(
[target_shape[0], target_shape[1],
img_resized.get_shape()[2]])
return pad_image, (new_h, new_w)
def simple_resize():
with tf.name_scope('simple_resize'):
img_resized = tf.image.resize_images(image, target_shape)
img_resized.set_shape(
[target_shape[0], target_shape[1],
img_resized.get_shape()[2]])
return img_resized, target_shape
# 3 cases
pad_image, (new_h, new_w) = tf.case(
{ # case 1 : new_w >= target_w
tf.logical_and(tf.greater_equal(ratio, target_ratio),
tf.greater_equal(new_w, target_w)):
simple_resize,
# case 2 : new_w >= target_w/2 & new_w < target_w & ratio < target_ratio
tf.logical_and(
tf.less(ratio, target_ratio),
tf.logical_and(
tf.greater_equal(new_w,
tf.cast(tf.divide(target_w, 2), tf.int32)),
tf.less(new_w, target_w))):
pad_fn,
# case 3 : new_w < target_w/2 & new_w < target_w & ratio < target_ratio
tf.logical_and(
tf.less(ratio, target_ratio),
tf.logical_and(
tf.less(new_w, target_w),
tf.less(new_w, tf.cast(tf.divide(target_w, 2), tf.int32)))):
replicate_fn
},
default=simple_resize,
exclusive=True)
return pad_image, new_w # new_w = image width used for computing sequence lengths
def build_model(data_batch, data, step):
batch_size, num_steps = [
tf.shape(data_batch["x_value_text_ids"])[d] for d in range(2)
]
vocab = data.vocab('y_aux')
id2str = '<{}>'.format
bos_str, eos_str = map(id2str, (vocab.bos_token_id, vocab.eos_token_id))
def single_bleu(ref, hypo):
ref = [id2str(u if u != vocab.unk_token_id else -1) for u in ref]
hypo = [id2str(u) for u in hypo]
ref = tx.utils.strip_special_tokens(' '.join(ref),
strip_bos=bos_str,
strip_eos=eos_str)
hypo = tx.utils.strip_special_tokens(' '.join(hypo), strip_eos=eos_str)
return 0.01 * tx.evals.sentence_bleu(references=[ref], hypothesis=hypo)
# losses
losses = {}
# embedders
embedders = {
name: tx.modules.WordEmbedder(vocab_size=data.vocab(name).size,
hparams=hparams)
for name, hparams in config_model.embedders.items()
}
# encoders
y_encoder = tx.modules.TransformerEncoder(hparams=config_model.y_encoder)
x_encoder = tx.modules.TransformerEncoder(hparams=config_model.x_encoder)
def concat_encoder_outputs(outputs):
return tf.concat(outputs, -1)
def encode(ref_flag):
y_str = y_strs[ref_flag]
y_ids = data_batch['{}_text_ids'.format(y_str)]
y_embeds = embedders['y_aux'](y_ids)
y_sequence_length = data_batch['{}_length'.format(y_str)]
y_enc_outputs = y_encoder(y_embeds, sequence_length=y_sequence_length)
y_enc_outputs = concat_encoder_outputs(y_enc_outputs)
x_str = x_strs[ref_flag]
x_ids = {
field: data_batch['{}_{}_text_ids'.format(x_str, field)][:, 1:-1]
for field in x_fields
}
x_embeds = tf.concat([
embedders['x_{}'.format(field)](x_ids[field]) for field in x_fields
],
axis=-1)
x_sequence_length = data_batch['{}_{}_length'.format(
x_str, x_fields[0])] - 2
x_enc_outputs = x_encoder(x_embeds, sequence_length=x_sequence_length)
x_enc_outputs = concat_encoder_outputs(x_enc_outputs)
return y_ids, y_embeds, y_enc_outputs, y_sequence_length, \
x_ids, x_embeds, x_enc_outputs, x_sequence_length
encode_results = [encode(ref_flag) for ref_flag in range(2)]
y_ids, y_embeds, y_enc_outputs, y_sequence_length, \
x_ids, x_embeds, x_enc_outputs, x_sequence_length = \
zip(*encode_results)
# get rnn cell
# rnn_cell = tx.core.layers.get_rnn_cell(config_model.rnn_cell)
def get_decoder(y__ref_flag, x_ref_flag, tgt_ref_flag, beam_width=None):
output_layer_params = \
{'output_layer': tf.identity} if copy_flag else {'vocab_size': vocab.size}
if attn_flag: # attention
memory = tf.concat(
[y_enc_outputs[y__ref_flag], x_enc_outputs[x_ref_flag]],
axis=1)
memory_sequence_length = None
copy_memory_sequence_length = None
tgt_embedding = tf.concat([
tf.zeros(shape=[1, embedders['y_aux'].dim]),
embedders['y_aux'].embedding[1:, :]
],
axis=0)
decoder = tx.modules.TransformerCopyDecoder(
embedding=tgt_embedding, hparams=config_model.decoder)
return decoder
def get_decoder_and_outputs(y__ref_flag,
x_ref_flag,
tgt_ref_flag,
params,
beam_width=None):
decoder = get_decoder(y__ref_flag,
x_ref_flag,
tgt_ref_flag,
beam_width=beam_width)
if beam_width is None:
ret = decoder(**params)
else:
ret = decoder(beam_width=beam_width, **params)
return decoder, ret
get_decoder_and_outputs = tf.make_template('get_decoder_and_outputs',
get_decoder_and_outputs)
gamma = tf.Variable(1, dtype=tf.float32, trainable=True)
gamma = tf.exp(tf.log(gamma))
def teacher_forcing(y__ref_flag, x_ref_flag, loss_name):
tgt_flag = x_ref_flag
tgt_str = y_strs[tgt_flag]
memory_sequence_length = tf.add(y_sequence_length[y__ref_flag] - 1,
x_sequence_length[x_ref_flag])
sequence_length = data_batch['{}_length'.format(tgt_str)] - 1
memory = tf.concat(
[y_enc_outputs[y__ref_flag], x_enc_outputs[x_ref_flag]],
axis=1) # [64 61 384]
decoder, rets = get_decoder_and_outputs(
y__ref_flag,
x_ref_flag,
tgt_flag,
{
'memory': memory, #print_mem,
'memory_sequence_length': memory_sequence_length,
'copy_memory': x_enc_outputs[x_ref_flag],
'copy_memory_sequence_length': x_sequence_length[x_ref_flag],
'source_ids':
x_ids[x_ref_flag]['value'], #print_ids, # source_ids
'gamma': gamma,
'decoding_strategy': 'train_greedy',
'inputs': y_embeds[tgt_flag]
[:, :-1, :], #[:, 1:, :], #target yence embeds (ignore <BOS>)
'alpha': config_model.alpha,
'sequence_length': sequence_length,
'mode': tf.estimator.ModeKeys.TRAIN
})
tgt_y_ids = data_batch['{}_text_ids'.format(
tgt_str)][:, 1:] # ground_truth ids (ignore <BOS>)
tf_outputs = rets[0]
gens = rets[2]
loss = tx.losses.sequence_sparse_softmax_cross_entropy(
labels=tgt_y_ids,
logits=tf_outputs.logits,
sequence_length=data_batch['{}_length'.format(tgt_str)] - 1)
# average_across_timesteps=True,
# sum_over_timesteps=False)
# loss = tf.reduce_mean(loss, 0)
if copy_flag and FLAGS.exact_cover_w != 0:
# sum_copy_probs = list(map(lambda t: tf.cast(t, tf.float32), final_state.sum_copy_probs))
copy_probs = (1 - gens) * rets[1]
sum_copy_probs = tf.reduce_sum(copy_probs, 1)
# sum_copy_probs = tf.split(sum_copy_probs, tf.shape(sum_copy_probs)[0], axis=0)#list(map(lambda prob: tf.cast(prob, tf.float32), tuple(tf.reduce_sum(copy_probs, 1)))) #[batch_size, len_key]
memory_lengths = x_sequence_length[
x_ref_flag] #[len for len in sd_sequence_length[x_ref_flag]]
exact_coverage_loss = \
tf.reduce_mean(tf.reduce_sum(
tx.utils.mask_sequences(
tf.square(sum_copy_probs - 1.), memory_lengths),
1))
print_xe_loss_op = tf.print(loss_name, 'xe loss:', loss)
with tf.control_dependencies([print_xe_loss_op]):
print_op = tf.print(loss_name, 'exact coverage loss :',
exact_coverage_loss)
with tf.control_dependencies([print_op]):
loss += FLAGS.exact_cover_w * exact_coverage_loss
losses[loss_name] = loss
return decoder, rets, loss, tgt_y_ids
def beam_searching(y__ref_flag, x_ref_flag, beam_width):
start_tokens = tf.ones_like(data_batch['y_aux_length']) * \
vocab.bos_token_id
end_token = vocab.eos_token_id
memory_sequence_length = tf.add(y_sequence_length[y__ref_flag] - 1,
x_sequence_length[x_ref_flag])
sequence_length = data_batch['{}_length'.format(
y_strs[y__ref_flag])] - 1
memory = tf.concat(
[y_enc_outputs[y__ref_flag], x_enc_outputs[x_ref_flag]], axis=1)
source_ids = tf.concat(
[y_ids[y__ref_flag], x_ids[x_ref_flag]['value']], axis=1)
#decoder, (bs_outputs, seq_len)
decoder, bs_outputs = get_decoder_and_outputs(
y__ref_flag,
x_ref_flag,
None,
{
'memory': memory, #print_mem,
'memory_sequence_length': memory_sequence_length,
'copy_memory': x_enc_outputs[x_ref_flag],
'copy_memory_sequence_length': x_sequence_length[x_ref_flag],
'gamma': gamma,
'source_ids': x_ids[x_ref_flag]
['value'], # source_ids,#x_ids[x_ref_flag]['entry'], #[ batch_size, source_length]
# 'decoding_strategy': 'infer_sample', only for random sampling
'alpha': config_model.alpha,
'start_tokens': start_tokens,
'end_token': end_token,
'max_decoding_length': config_train.infer_max_decoding_length
},
beam_width=beam_width)
return decoder, bs_outputs, sequence_length, start_tokens
decoder, rets, loss, tgt_y_ids = teacher_forcing(1, 0, 'MLE')
rec_decoder, _, rec_loss, _ = teacher_forcing(1, 1, 'REC')
rec_weight = FLAGS.rec_w + FLAGS.rec_w_rate * tf.cast(step, tf.float32)
step_stage = tf.cast(step, tf.float32) / tf.constant(600.0)
rec_weight = tf.case([(tf.less_equal(step_stage, tf.constant(1.0)), lambda: tf.constant(1.0)), \
(tf.greater(step_stage, tf.constant(2.0)), lambda: FLAGS.rec_w)], \
default=lambda: tf.constant(1.0) - (step_stage - 1) * (1 - FLAGS.rec_w))
joint_loss = (1 - rec_weight) * loss + rec_weight * rec_loss
losses['joint'] = joint_loss
tiled_decoder, bs_outputs, sequence_length, start_tokens = beam_searching(
1, 0, config_train.infer_beam_width)
train_ops = {
name: get_train_op(losses[name], hparams=config_train.train[name])
for name in config_train.train
}
return train_ops, bs_outputs, rets, sequence_length, tgt_y_ids, start_tokens, gamma
def _prep(list_):
return list(
tf.case({tf.equal(next_replica_idx[i], j):
_stateful_lambda(list_[j])
for j in range(self.num_replica)}, exclusive=True)
for i in range(self.num_replica))
def DnCNN_outer_wrapper(r,
rvar,
theta,
tie,
iter,
training=False,
LayerbyLayer=True):
if tie:
with tf.variable_scope("Iter0"):
(xhat, dxdr) = DnCNN_wrapper(r, rvar, theta[0], training=training)
elif adaptive_weights:
rstd = 255. * tf.sqrt(
tf.reduce_mean(rvar)
) # To enable batch processing, I have to treat every image in the batch as if it has the same amount of effective noise
def x_nl0(a=rstd, iter=iter, r=r, rvar=rvar, theta=theta):
with tf.variable_scope("Adaptive_NL" + str(0)):
(xhat, dxdr) = DnCNN_wrapper(r,
rvar,
theta[0],
training=training)
xhat = tf.Print(xhat, [iter], "used denoiser 0")
return (xhat, dxdr)
def x_nl1(a=rstd, iter=iter, r=r, rvar=rvar, theta=theta):
with tf.variable_scope("Adaptive_NL" + str(1)):
(xhat, dxdr) = DnCNN_wrapper(r,
rvar,
theta[1],
training=training)
xhat = tf.Print(xhat, [iter], "used denoiser 1")
return (xhat, dxdr)
def x_nl2(a=rstd, iter=iter, r=r, rvar=rvar, theta=theta):
with tf.variable_scope("Adaptive_NL" + str(2)):
(xhat, dxdr) = DnCNN_wrapper(r,
rvar,
theta[2],
training=training)
xhat = tf.Print(xhat, [iter], "used denoiser 2")
return (xhat, dxdr)
def x_nl3(a=rstd, iter=iter, r=r, rvar=rvar, theta=theta):
with tf.variable_scope("Adaptive_NL" + str(3)):
(xhat, dxdr) = DnCNN_wrapper(r,
rvar,
theta[3],
training=training)
xhat = tf.Print(xhat, [iter], "used denoiser 3")
return (xhat, dxdr)
def x_nl4(a=rstd, iter=iter, r=r, rvar=rvar, theta=theta):
with tf.variable_scope("Adaptive_NL" + str(4)):
(xhat, dxdr) = DnCNN_wrapper(r,
rvar,
theta[4],
training=training)
xhat = tf.Print(xhat, [iter], "used denoiser 4")
return (xhat, dxdr)
def x_nl5(a=rstd, iter=iter, r=r, rvar=rvar, theta=theta):
with tf.variable_scope("Adaptive_NL" + str(5)):
(xhat, dxdr) = DnCNN_wrapper(r,
rvar,
theta[5],
training=training)
xhat = tf.Print(xhat, [iter], "used denoiser 5")
return (xhat, dxdr)
def x_nl6(a=rstd, iter=iter, r=r, rvar=rvar, theta=theta):
with tf.variable_scope("Adaptive_NL" + str(6)):
(xhat, dxdr) = DnCNN_wrapper(r,
rvar,
theta[6],
training=training)
xhat = tf.Print(xhat, [iter], "used denoiser 6")
return (xhat, dxdr)
def x_nl7(a=rstd, iter=iter, r=r, rvar=rvar, theta=theta):
with tf.variable_scope("Adaptive_NL" + str(7)) as scope:
(xhat, dxdr) = DnCNN_wrapper(r,
rvar,
theta[7],
training=training)
xhat = tf.Print(xhat, [iter], "used denoiser 7")
return (xhat, dxdr)
def x_nl8(a=rstd, iter=iter, r=r, rvar=rvar, theta=theta):
with tf.variable_scope("Adaptive_NL" + str(8)) as scope:
(xhat, dxdr) = DnCNN_wrapper(r,
rvar,
theta[8],
training=training)
xhat = tf.Print(xhat, [iter], "used denoiser 8")
return (xhat, dxdr)
rstd = tf.Print(rstd, [rstd], "rstd =")
NL_0 = tf.less_equal(rstd, 10.)
NL_1 = tf.logical_and(tf.less(10., rstd), tf.less_equal(rstd, 20.))
NL_2 = tf.logical_and(tf.less(20., rstd), tf.less_equal(rstd, 40.))
NL_3 = tf.logical_and(tf.less(40., rstd), tf.less_equal(rstd, 60.))
NL_4 = tf.logical_and(tf.less(60., rstd), tf.less_equal(rstd, 80.))
NL_5 = tf.logical_and(tf.less(80., rstd), tf.less_equal(rstd, 100.))
NL_6 = tf.logical_and(tf.less(100., rstd), tf.less_equal(rstd, 150.))
NL_7 = tf.logical_and(tf.less(150., rstd), tf.less_equal(rstd, 300.))
predicates = {
NL_0: x_nl0,
NL_1: x_nl1,
NL_2: x_nl2,
NL_3: x_nl3,
NL_4: x_nl4,
NL_5: x_nl5,
NL_6: x_nl6,
NL_7: x_nl7
}
default = x_nl8
(xhat, dxdr) = tf.case(predicates, default, exclusive=True)
xhat = tf.reshape(xhat, shape=[n, BATCH_SIZE])
dxdr = tf.reshape(dxdr, shape=[1, BATCH_SIZE])
else:
with tf.variable_scope("Iter" + str(iter)):
(xhat, dxdr) = DnCNN_wrapper(r,
rvar,
theta[iter],
training=training,
LayerbyLayer=LayerbyLayer)
return (xhat, dxdr)
###############################################################################
x = tf.random_uniform([]) # Empty array as shape creates a scalar.
y = tf.random_uniform([])
out = tf.cond(tf.greater(x, y), lambda: x + y, lambda: x - y)
print(sess.run(out))
###############################################################################
# 1b: Create two 0-d tensors x and y randomly selected from the range [-1, 1).
# Return x + y if x < y, x - y if x > y, 0 otherwise.
# Hint: Look up tf.case().
###############################################################################
x = tf.random_uniform([], -1, 1)
y = tf.random_uniform([], -1, 1)
out = tf.case({tf.less(x,y):lambda :x+y, tf.greater(x,y):lambda :x-y}, default=lambda :0.0, exclusive=True)
print(sess.run(out))
###############################################################################
# 1c: Create the tensor x of the value [[0, -2, -1], [0, 1, 2]]
# and y as a tensor of zeros with the same shape as x.
# Return a boolean tensor that yields Trues if x equals y element-wise.
# Hint: Look up tf.equal().
###############################################################################
x = tf.constant([[0, -2, -1], [0, 1, 2]])
y = tf.zeros_like(x)
out = tf.equal(x,y)
print(sess.run(out))
###############################################################################
# 1d: Create the tensor x of value
# [29.05088806, 27.61298943, 31.19073486, 29.35532951,
l2_regularizer = sum(tf.nn.l2_loss(Wxxx) for Wxxx in Ws)
mse_e = tf.losses.mean_squared_error(predictions = y, labels = y_)
loss = mse_e + l2_reg_*l2_regularizer
#train_step = tf.train.AdamOptimizer(learning_rate=learning_rate_).minimize(loss)
global_step = tf.Variable(0, trainable=False)
starter_learning_rate = start_learning_rate
learning_rate = tf.train.exponential_decay(starter_learning_rate, global_step,
learning_rate_schedule_epochs, learning_rate_decay, staircase=True)
lr_fun = lambda: learning_rate
min_lr = lambda: tf.constant(1e-5)
actual_lr = tf.case([(tf.less(learning_rate, tf.constant(1e-5)), min_lr)], default=lr_fun)
train_step = tf.train.AdamOptimizer(learning_rate=actual_lr).minimize(loss, global_step=global_step)
# In[ ]:
try:
sess.close()
except:
pass
init = tf.global_variables_initializer()
saver = tf.train.Saver()
sess = tf.Session()
def create_path_drop_masks(self, p_img, p_bev, random_values):
"""Determines global path drop decision based on given probabilities.
Args:
p_img: A tensor of float32, probability of keeping image branch
p_bev: A tensor of float32, probability of keeping bev branch
random_values: A tensor of float32 of shape [3], the results
of coin flips, values should range from 0.0 - 1.0.
Returns:
final_img_mask: A constant tensor mask containing either one or zero
depending on the final coin flip probability.
final_bev_mask: A constant tensor mask containing either one or zero
depending on the final coin flip probability.
"""
def keep_branch():
return tf.constant(1.0)
def kill_branch():
return tf.constant(0.0)
# The logic works as follows:
# We have flipped 3 coins, first determines the chance of keeping
# the image branch, second determines keeping bev branch, the third
# makes the final decision in the case where both branches were killed
# off, otherwise the initial img and bev chances are kept.
img_chances = tf.case(
[(tf.less(random_values[0], p_img), keep_branch)],
default=kill_branch)
bev_chances = tf.case(
[(tf.less(random_values[1], p_bev), keep_branch)],
default=kill_branch)
# Decision to determine whether both branches were killed off
third_flip = tf.logical_or(tf.cast(img_chances, dtype=tf.bool),
tf.cast(bev_chances, dtype=tf.bool))
third_flip = tf.cast(third_flip, dtype=tf.float32)
# Make a second choice, for the third case
# Here we use a 50/50 chance to keep either image or bev
# If its greater than 0.5, keep the image
img_second_flip = tf.case(
[(tf.greater(random_values[2], 0.5), keep_branch)],
default=kill_branch)
# If its less than or equal to 0.5, keep bev
bev_second_flip = tf.case(
[(tf.less_equal(random_values[2], 0.5), keep_branch)],
default=kill_branch)
# Use lambda since this returns another condition and it needs to
# be callable
final_img_mask = tf.case(
[(tf.equal(third_flip, 1), lambda: img_chances)],
default=lambda: img_second_flip)
final_bev_mask = tf.case(
[(tf.equal(third_flip, 1), lambda: bev_chances)],
default=lambda: bev_second_flip)
return final_img_mask, final_bev_mask
###############################################################################
x = tf.random_uniform([]) # Empty array as shape creates a scalar.
y = tf.random_uniform([])
out = tf.cond(tf.greater(x, y), lambda: tf.add(x, y), lambda: tf.subtract(x, y))
###############################################################################
# 1b: Create two 0-d tensors x and y randomly selected from the range [-1, 1).
# Return x + y if x < y, x - y if x > y, 0 otherwise.
# Hint: Look up tf.case().
###############################################################################
x = tf.random_uniform([], -1, 1, dtype=tf.float32)
y = tf.random_uniform([], -1, 1, dtype=tf.float32)
out = tf.case({tf.less(x, y): lambda: tf.add(x, y),
tf.greater(x, y): lambda: tf.subtract(x, y)},
default=lambda: tf.constant(0.0), exclusive=True)
###############################################################################
# 1c: Create the tensor x of the value [[0, -2, -1], [0, 1, 2]]
# and y as a tensor of zeros with the same shape as x.
# Return a boolean tensor that yields Trues if x equals y element-wise.
# Hint: Look up tf.equal().
###############################################################################
x = tf.constant([[0, -2, -1], [0, 1, 2]])
y = tf.zeros_like(x)
out = tf.equal(x, y)
###############################################################################
def build_progressive_real_input(self, x):
# Get tensors to help keep track of the progression
initial_steps_placeholder, stabilizing_steps_placeholder, stage_int_tensor, fade_phase_bool, alpha_tensor = cls.get_or_create_tracking_tensors(
)
# Get resizing layers and blocks of Discriminator.
resizings = self.resizings
blocks = self.blocks
def populate_progressive_pred_fn_list(pred_fn_list, stage):
'''
This function populates a given list with predicate-function pairs
that sequentially builds the input progression
following 'stage' and 'fade_phase' inclusively.
The populated list may be used to construct a conditional tensor with tf.case.
For instance, with args:
pred_fn_list = [], stage = 0e
one would have after running this function:
pred_fn_list = [
(predicate of 0th stage input, input resized to smallest resolution)
(predicate of 1st stage input, input resized to 2nd smallest resolution)
...
(predicate of final stage input, input kept at same size - no change)
]
Args:
pred_fn_list - (list) list to populate (starts empty and gets mutated)
stage - (int) stage of progression
Return: None
'''
# Define predicate of conditional statements.
predicate = (stage_int_tensor <= stage)
# Base Case: Final Stage - return input as is.
if stage == len(blocks):
# Append input as is without any resizing.
pred_fn_list.append((predicate, lambda: x))
# Other Cases: Previous Stages - resize input for the stage accordingly.
else:
# Append with input resized for the current stage.
pred_fn_list.append(
(predicate,
lambda: cls.apply_layers(resizings[stage:], x)))
# Populate with remaining input progressions.
populate_progressive_pred_fn_list(pred_fn_list, stage + 1)
#-----------------------------------------------------
# Build and return progression.
#-----------------------------------------------------
# Populate progressive_pred_fn_list - a list of predicate-function pairs that sequentially build the up the blocks of the network.
progressive_pred_fn_list = []
populate_progressive_pred_fn_list(progressive_pred_fn_list,
stage=0) # Populate list.
# Build input progressive tensor - the final case (the non-altered input) is set as the default.
input_progression_tensor = tf.case(
pred_fn_pairs=progressive_pred_fn_list[:-1],
default=progressive_pred_fn_list[-1][1])
# Return tf.case tensor.
return input_progression_tensor
def format_example_cnn(example, hp):
"""Function that prepares example for input into a CNN. It will save the last history_length positions.
Formats positions for a CNN and returns input with shape [game_length, history_length*2+1, board_size, board_size]:
1. channel: current players stones
2. channel: enemy players stones
3. channel: current player stones in the position before
4. channel: enemy player stones in the position before
... (until history_length*2)
n. channel: current player (ones means black, zeroes mean white)
Adds v_targets with shape [game_length] either 1 or -1:
1: current player wins
-1: enemy wins
"""
board_size = hp.board_size
history_length = hp.history_length
inputs = example["inputs"]
to_play = example['to_play']
# pad positions history_length times with moving padding at front and back
padded = []
for i in range(history_length):
paddings = tf.constant([[i, history_length-1-i], [0, 0], [0, 0]])
padded_input = tf.pad(inputs, paddings)
padded_input = padded_input[:-(history_length-1)]
padded.append(padded_input)
padded_inputs = tf.stack(padded, axis=1)
def format_input_cnn(elem):
"""Format a single position with shape [history_length, board_size, board_size] to
[history_length*2+1, board_size, board_size]."""
position, player = elem
# unstack at first dimension
positions = tf.unstack(position, num=history_length, axis=0)
ones = tf.ones([board_size, board_size], dtype=tf.int8)
zeros = tf.zeros([board_size, board_size], dtype=tf.int8)
pos_black = []
pos_white = []
# for each of the history_length positions
for pos in positions:
# find all black stones
black_mask = tf.equal(pos, 1)
black_stones = tf.where(black_mask, ones, zeros)
# find all white stones
white_mask = tf.equal(pos, -1)
white_stones = tf.where(white_mask, ones, zeros)
# extend black positions for player black
pos_black.extend([black_stones, white_stones])
# extend white positions for player white
pos_white.extend([white_stones, black_stones])
# append ones or zeros for current player
pos_black.append(ones)
pos_white.append(zeros)
def _black():
return tf.stack(pos_black, axis=0)
def _white():
return tf.stack(pos_white, axis=0)
_cases = [(tf.equal(player, 1), _black),
(tf.not_equal(player, 1), _white)]
new_pos = tf.case(_cases)
return new_pos
new_inputs = tf.map_fn(format_input_cnn, (padded_inputs, to_play), dtype=tf.int8, back_prop=False)
example["inputs"] = new_inputs
winner = example.pop('winner')
to_play = tf.cast(to_play, tf.int64)
def _draw():
return tf.zeros_like(to_play)
def _not_draw():
is_winner = tf.ones_like(to_play)
not_winner = tf.negative(is_winner)
return tf.where(tf.equal(to_play, winner), is_winner, not_winner)
cases = [(tf.equal(winner, 0), _draw),
(tf.not_equal(winner, 0), _not_draw)]
v_targets = tf.case(cases)
example["v_targets"] = v_targets
return example
def energy_kl(self, u_i, u_j, proximity, node_type1, node_type2):
def f1():
print("f1")
return tf.gather(self.embedding1, u_i), tf.gather(self.sigma1, u_i)
def f2():
print("f2")
return tf.gather(self.embedding2, u_i), tf.gather(self.sigma2, u_i)
def f3():
print("f3")
return tf.gather(self.embedding3, u_i), tf.gather(self.sigma3, u_i)
def f4():
print("f4")
return tf.gather(self.embedding4, u_i), tf.gather(self.sigma4, u_i)
def f5():
print("f5")
return tf.gather(self.ctx_mu1,
u_j), tf.gather(self.ctx_sigma1, u_j)
def f6():
print("f6")
return tf.gather(self.ctx_mu2,
u_j), tf.gather(self.ctx_sigma2, u_j)
def f7():
print("f7")
return tf.gather(self.ctx_mu3,
u_j), tf.gather(self.ctx_sigma3, u_j)
def f8():
print("f8")
return tf.gather(self.ctx_mu4,
u_j), tf.gather(self.ctx_sigma4, u_j)
def f9():
print("f9")
return tf.gather(self.embedding1, u_j), tf.gather(self.sigma1, u_j)
def f10():
print("f10")
return tf.gather(self.embedding2, u_j), tf.gather(self.sigma2, u_j)
def f11():
print("f11")
return tf.gather(self.embedding3, u_j), tf.gather(self.sigma3, u_j)
def f12():
print("f12")
return tf.gather(self.embedding4, u_j), tf.gather(self.sigma4, u_j)
mu_i, sigma_i = tf.case([(tf.equal(node_type1, 0), f1),
(tf.equal(node_type1, 1), f2),
(tf.equal(node_type1, 2), f3),
(tf.equal(node_type1, 3), f4)],
default=None,
exclusive=True)
mu_j, sigma_j = tf.case([(tf.equal(node_type2, 0), f9),
(tf.equal(node_type2, 1), f10),
(tf.equal(node_type2, 2), f11),
(tf.equal(node_type2, 3), f12)],
default=None,
exclusive=True)
sigma_ratio = sigma_j / sigma_i
trace_fac = tf.reduce_sum(sigma_ratio, 1)
log_det = tf.reduce_sum(tf.log(sigma_ratio + 1e-11), 1)
mu_diff_sq = tf.reduce_sum(tf.square(mu_i - mu_j) / sigma_i, 1)
ij_kl = 0.5 * (trace_fac + mu_diff_sq - self.L - log_det)
sigma_ratio = sigma_i / sigma_j
trace_fac = tf.reduce_sum(sigma_ratio, 1)
log_det = tf.reduce_sum(tf.log(sigma_ratio + 1e-11), 1)
mu_diff_sq = tf.reduce_sum(tf.square(mu_j - mu_i) / sigma_j, 1)
ji_kl = 0.5 * (trace_fac + mu_diff_sq - self.L - log_det)
kl_distance = 0.5 * (ij_kl + ji_kl)
return kl_distance
def _rhn_enas(self, x, prev_s, w_prev, w_skip, is_training,
x_mask=None, s_mask=None):
batch_size = prev_s.get_shape()[0].value
start_idx = self.sample_arc[0] * 2 * self.lstm_hidden_size
end_idx = start_idx + 2 * self.lstm_hidden_size
if is_training:
assert x_mask is not None, "x_mask is None"
assert s_mask is not None, "s_mask is None"
ht = tf.matmul(tf.concat([x * x_mask, prev_s * s_mask], axis=1),
w_prev[start_idx:end_idx, :])
else:
ht = tf.matmul(tf.concat([x, prev_s], axis=1),
w_prev[start_idx:end_idx, :])
with tf.variable_scope("rhn_layer_0"):
ht = batch_norm(ht, is_training)
h, t = tf.split(ht, 2, axis=1)
func_idx = self.sample_arc[0]
h = tf.case(
{
tf.equal(func_idx, 0): lambda: tf.tanh(h),
tf.equal(func_idx, 1): lambda: tf.nn.relu(h),
tf.equal(func_idx, 2): lambda: tf.identity(h),
tf.equal(func_idx, 3): lambda: tf.sigmoid(h),
},
default=lambda: tf.constant(0.0, dtype=tf.float32), exclusive=True)
t = tf.sigmoid(t)
s = prev_s + t * (h - prev_s)
layers = [s]
start_idx = 1
used = []
for rhn_layer_id in range(1, self.rhn_depth):
with tf.variable_scope("rhn_layer_{}".format(rhn_layer_id)):
prev_idx = self.sample_arc[start_idx]
func_idx = self.sample_arc[start_idx + 1]
curr_used = tf.one_hot(prev_idx, depth=self.rhn_depth, dtype=tf.int32)
used.append(curr_used)
w_start = (prev_idx * self.num_funcs + func_idx) * self.lstm_hidden_size
w_end = w_start + self.lstm_hidden_size
w = w_skip[rhn_layer_id][w_start:w_end, :]
prev_s = tf.concat(layers, axis=0)
prev_s = prev_s[prev_idx*batch_size : (prev_idx+1)*batch_size, :]
if is_training:
ht = tf.matmul(prev_s * s_mask, w)
else:
ht = tf.matmul(prev_s, w)
ht = batch_norm(ht, is_training)
h, t = tf.split(ht, 2, axis=1)
h = tf.case(
{
tf.equal(func_idx, 0): lambda: tf.tanh(h),
tf.equal(func_idx, 1): lambda: tf.nn.relu(h),
tf.equal(func_idx, 2): lambda: tf.identity(h),
tf.equal(func_idx, 3): lambda: tf.sigmoid(h),
},
default=lambda: tf.constant(0.0, dtype=tf.float32), exclusive=True)
t = tf.sigmoid(t)
s = prev_s + t * (h - prev_s)
layers.append(s)
start_idx += 2
used = tf.add_n(used)
used = tf.equal(used, 0)
with tf.control_dependencies([tf.Assert(tf.reduce_any(used), [used])]):
layers = tf.stack(layers)
layers = tf.boolean_mask(layers, used)
layers = tf.reduce_mean(layers, axis=0)
layers.set_shape([batch_size, self.lstm_hidden_size])
layers = batch_norm(layers, is_training)
return layers
def _provide_data(input_tensors, truncated_length, hparams):
"""Returns tensors for reading batches from provider."""
(spec, labels, label_weights, length, onsets, offsets, velocities,
unused_velocity_range, filename, note_sequence) = input_tensors
length = tf.to_int32(length)
labels = tf.reshape(labels, (-1, constants.MIDI_PITCHES))
label_weights = tf.reshape(label_weights, (-1, constants.MIDI_PITCHES))
onsets = tf.reshape(onsets, (-1, constants.MIDI_PITCHES))
offsets = tf.reshape(offsets, (-1, constants.MIDI_PITCHES))
velocities = tf.reshape(velocities, (-1, constants.MIDI_PITCHES))
spec = tf.reshape(spec, (-1, hparams_frame_size(hparams)))
truncated_length = (tf.reduce_min([truncated_length, length])
if truncated_length else length)
# Pad or slice specs and labels tensors to have the same lengths,
# truncating after truncated_length.
spec_delta = tf.shape(spec)[0] - truncated_length
spec = tf.case([(spec_delta < 0,
lambda: tf.pad(spec, tf.stack([(0, -spec_delta),
(0, 0)]))),
(spec_delta > 0, lambda: spec[0:-spec_delta])],
default=lambda: spec)
labels_delta = tf.shape(labels)[0] - truncated_length
labels = tf.case(
[(labels_delta < 0,
lambda: tf.pad(labels, tf.stack([(0, -labels_delta), (0, 0)]))),
(labels_delta > 0, lambda: labels[0:-labels_delta])],
default=lambda: labels)
label_weights = tf.case(
[(labels_delta < 0,
lambda: tf.pad(label_weights, tf.stack([(0, -labels_delta),
(0, 0)]))),
(labels_delta > 0, lambda: label_weights[0:-labels_delta])],
default=lambda: label_weights)
onsets = tf.case(
[(labels_delta < 0,
lambda: tf.pad(onsets, tf.stack([(0, -labels_delta), (0, 0)]))),
(labels_delta > 0, lambda: onsets[0:-labels_delta])],
default=lambda: onsets)
offsets = tf.case(
[(labels_delta < 0,
lambda: tf.pad(offsets, tf.stack([(0, -labels_delta), (0, 0)]))),
(labels_delta > 0, lambda: offsets[0:-labels_delta])],
default=lambda: offsets)
velocities = tf.case(
[(labels_delta < 0,
lambda: tf.pad(velocities, tf.stack([(0, -labels_delta), (0, 0)]))),
(labels_delta > 0, lambda: velocities[0:-labels_delta])],
default=lambda: velocities)
truncated_note_sequence = truncate_note_sequence_op(
note_sequence, truncated_length, hparams)
batch_tensors = {
'spec':
tf.reshape(spec, (truncated_length, hparams_frame_size(hparams), 1)),
'labels':
tf.reshape(labels, (truncated_length, constants.MIDI_PITCHES)),
'label_weights':
tf.reshape(label_weights, (truncated_length, constants.MIDI_PITCHES)),
'lengths':
truncated_length,
'onsets':
tf.reshape(onsets, (truncated_length, constants.MIDI_PITCHES)),
'offsets':
tf.reshape(offsets, (truncated_length, constants.MIDI_PITCHES)),
'velocities':
tf.reshape(velocities, (truncated_length, constants.MIDI_PITCHES)),
'filenames':
filename,
'note_sequences':
truncated_note_sequence,
}
return batch_tensors
def _inference(self, profile, stories, queries):
"""Generate the model's graph"""
def model_inference_helper(A, H, W):
"""Helper function to construct a End-to-end memory network"""
q_emb = tf.nn.embedding_lookup(A, queries)
u_0 = tf.reduce_sum(q_emb, 1)
u = [u_0]
u_k = u_0 # Typically if self._hops = 0
for count in range(self._hops):
m_emb = tf.nn.embedding_lookup(A, stories)
m = tf.reduce_sum(m_emb, 2)
# hack to get around no reduce_dot
u_temp = tf.transpose(tf.expand_dims(u[-1], -1), [0, 2, 1])
dotted = tf.reduce_sum(m * u_temp, 2)
# Calculate probabilities
probs = tf.nn.softmax(dotted)
# probs = tf.Print(probs, [count, tf.shape(probs), probs], summarize=200)
probs_temp = tf.transpose(tf.expand_dims(probs, -1), [0, 2, 1])
c_temp = tf.transpose(m, [0, 2, 1])
o_k = tf.reduce_sum(c_temp * probs_temp, 2)
u_k = tf.matmul(u[-1], H) + o_k
# u_k=u[-1]+tf.matmul(o_k,self.H)
# nonlinearity
if self._nonlin:
u_k = self._nonlin(u_k)
u.append(u_k)
candidates_emb = tf.nn.embedding_lookup(W, self._candidates)
candidates_emb_sum = tf.reduce_sum(candidates_emb, 1)
return tf.matmul(u_k, tf.transpose(candidates_emb_sum))
def construct_model_for_profile(p):
p_vars = self.get_variables_for_profile(p)
model = model_inference_helper(**p_vars)
if self._verbose:
model = tf.Print(model, [profile],
message="Profile {}".format(p))
return model
with tf.variable_scope(self._name, reuse=True):
with tf.variable_scope(MemN2NDialog.MODEL_NAME_SHARED, reuse=True):
model_vars = self.get_variables()
shared_result = model_inference_helper(**model_vars)
def construct_case_element(p):
return (tf.equal(profile,
p), lambda: construct_model_for_profile(p))
clean_case = [
construct_case_element(p) for p in self._profile_idx_set
]
# In tensorflow 0.12, default has to be given (and be a true `constructor`). This behavior is
# different in more recent implementation of tensorflow.
# The choice here is to simply add one arbitrary case with a print as default
def default_constructor():
first_model = clean_case[0][1]()
return tf.Print(
first_model,
data=[tf.constant([0])],
message="Called default case in switch. Not good.")
spec_result = tf.case(clean_case,
default=default_constructor,
exclusive=True,
name='dispatching_profile')
if self._alpha == 0:
return shared_result
elif self._alpha == 1:
return spec_result
else:
specific_scaled = tf.scalar_mul(self._alpha, spec_result)
shared_scaled = tf.scalar_mul(1 - self._alpha, shared_result)
return tf.add(specific_scaled, shared_scaled)
def experiment(report_every_n=100):
"""Run training operations, then validate.
Args:
report_every_n: Print loss every n training operations. 0 for no printing.
Returns:
Validation top-1 accuracy and a numpy array of training losses
"""
#Placeholders to feed hyperparameters into graph
learning_rate_ph = tf.placeholder(tf.float32, name="learning_rate")
beta1_ph = tf.placeholder(tf.float32, shape=(), name="beta1")
decay_ph = tf.placeholder(tf.float32, shape=(), name="decay")
gen_scale_ph = tf.placeholder(tf.float32, shape=(), name="gen_scale")
is_training_ph = tf.placeholder(tf.bool, name="is_training")
mode_ph = tf.placeholder(tf.int32, name="mode")
#data_dir = "//Desktop-sa1evjv/h/wavefunctions/"
data_dir = "//Desktop-sa1evjv/f/wavefunctions_single/wavefunctions/"
batch_size = 24
def load_data_subset(subset):
return load_data(dir=data_dir, subset=subset, batch_size=batch_size)
inputs, target_outputs = tf.case({
tf.equal(mode_ph, 0):
lambda: load_data_subset("train"),
tf.equal(mode_ph, 1):
lambda: load_data_subset("val"),
tf.equal(mode_ph, 2):
lambda: load_data_subset("test")
})
#Describe learning policy
grace_iter = 4_000
start_iter = 100_000 - grace_iter #0
train_iters = 500_000
val_iters = 1_000
learning_rate = 0.0002
beta1 = 0.9
#Configure operations
train_op0, loss, output = configure(inputs=inputs,
batch_size=batch_size,
target_outputs=target_outputs,
is_training=is_training_ph,
learning_rate=learning_rate_ph,
beta1=beta1_ph,
is_depthwise_sep=False,
decay=decay_ph,
gen_scale=gen_scale_ph)
clip_op = tf.get_collection("clip_weights")
#Tensors to dump as visual output
first_image = inputs[0]
first_target_output = target_outputs[0]
first_output = output[0]
#Session configuration
config = tf.ConfigProto()
config.gpu_options.allow_growth = True #Only use required GPU memory
config.gpu_options.force_gpu_compatible = True
model_dir = f"//flexo.ads.warwick.ac.uk/Shared41/Microscopy/Jeffrey-Ede/models/wavefunctions/{EXPER_NUM}/"
saver = tf.train.Saver(max_to_keep=1)
noteable_saver = tf.train.Saver(max_to_keep=10)
log_filepath = model_dir + "log.txt"
save_period = 1
save_period *= 3600
with tf.Session(config=config) as sess, open(log_filepath,
"a") as log_file:
#Initialize network parameters
feed_dict = {
is_training_ph: np.bool(True),
learning_rate_ph: np.float32(learning_rate),
beta1_ph: np.float32(beta1),
mode_ph: np.int32(0),
decay_ph: np.float32(0.),
gen_scale_ph: np.float32(0.)
}
if True:
sess.run(tf.global_variables_initializer(), feed_dict=feed_dict)
saver_mse = tf.train.Saver(max_to_keep=1,
var_list=tf.trainable_variables("gen"))
saver_mse.restore(
sess, tf.train.latest_checkpoint(model_dir + "noteable_ckpt/"))
else:
if start_iter:
saver.restore(sess,
tf.train.latest_checkpoint(model_dir + "model/"))
else:
sess.run(tf.global_variables_initializer(),
feed_dict=feed_dict)
#Finalize graph to prevent additional nodes from being added
#sess.graph.finalize()
#Training
avg_pred_fake = 0.4
beta_pred_fake = 0.9
time0 = time.time()
for iter in range(start_iter, train_iters):
if iter < 100_000 - grace_iter:
train_op = train_op0[-1]
lr = learning_rate * 0.5**(iter // (100_000 // 7))
elif iter < 100_000:
lr = learning_rate * 0.5**(iter // (100_000 // 7))
train_op = train_op0[1:]
else:
def zoom(x, bboxes, label, scale_min=0.6, scale_max=1.6):
"""
Zoom Image(影像縮放)
:param x: image inputs, 0~1
:param bboxes: bounding boxes inputs list [y1, x1, y2, x2], 0~w or h
:param scale_min: 縮放最小倍數
:param scale_max: 縮放最大倍數
:return: 返回(images, bboxes), images: scale0~1, bboxes: return list [y1, x1, y2, x2], scale 0~w, h
"""
h, w, _ = x.shape
scale = tf.random.uniform([], scale_min, scale_max) # 隨機縮放比例
# 等比例縮放
nh = tf.cast(h * scale, tf.int32) # 縮放後影像長度
nw = tf.cast(w * scale, tf.int32) # 縮放後影像寬度
# 如果將影像縮小執行以下程式
def scale_less_then_one():
resize_x = tf.image.resize(x, (nh, nw)) # 影像縮放
dy = tf.random.uniform([], 0, (h - nh), tf.int32)
dx = tf.random.uniform([], 0, (w - nw), tf.int32)
indexes = tf.meshgrid(tf.range(dy, dy + nh),
tf.range(dx, dx + nw),
indexing='ij')
indexes = tf.stack(indexes, axis=-1)
output = tf.scatter_nd(indexes, resize_x, (h, w, 3))
return output, dx, dy
# 如果將影像放大執行以下以下程式
def scale_greater_then_one():
resize_x = tf.image.resize(x, (nh, nw)) # 影像縮放
dy = tf.random.uniform([], 0, (nh - h), tf.int32)
dx = tf.random.uniform([], 0, (nw - w), tf.int32)
return resize_x[dy:dy + h, dx:dx + w], -dx, -dy
def scale_equal_zero():
return x, 0, 0
output, dx, dy = tf.case(
[(tf.logical_or(tf.less(nh - h, 0), tf.less(nw - w,
0)), scale_less_then_one),
(tf.logical_or(tf.greater(nh - h, 0), tf.greater(
nw - w, 0)), scale_greater_then_one)],
default=scale_equal_zero)
# 重新調整bounding box位置
y1 = bboxes[0] * scale + tf.cast(dy, dtype=tf.float32)
x1 = bboxes[1] * scale + tf.cast(dx, dtype=tf.float32)
y2 = bboxes[2] * scale + tf.cast(dy, dtype=tf.float32)
x2 = bboxes[3] * scale + tf.cast(dx, dtype=tf.float32)
# 如果座標超出範圍將其限制在邊界上
y1 = tf.where(y1 < 0, tf.zeros_like(y1), y1)
x1 = tf.where(x1 < 0, tf.zeros_like(x1), x1)
y2 = tf.where(y2 > h, h * tf.ones_like(y2), y2)
x2 = tf.where(x2 > w, w * tf.ones_like(x2), x2)
# 找出不存在影像上的bounding box並剔除
box_w = x2 - x1
box_h = y2 - y1
bboxes_filter = tf.logical_and(box_w > 1, box_h > 1)
y1 = y1[bboxes_filter]
x1 = x1[bboxes_filter]
y2 = y2[bboxes_filter]
x2 = x2[bboxes_filter]
label = label[bboxes_filter]
output = tf.ensure_shape(output, x.shape)
return output, [y1, x1, y2, x2], label
def tower(inputs,
is_training,
dropout_probability,
input_noise,
normalize_input,
flip_horizontally,
translate,
num_logits,
is_initialization=False,
name=None):
with tf.name_scope(name, "tower"):
default_conv_args = dict(
padding='SAME',
kernel_size=[3, 3],
activation_fn=nn.lrelu,
init=is_initialization
)
training_mode_funcs = [
nn.random_translate, nn.flip_randomly, nn.gaussian_noise, slim.dropout,
wn.fully_connected, wn.conv2d
]
training_args = dict(
is_training=is_training
)
with \
slim.arg_scope([wn.conv2d], **default_conv_args), \
slim.arg_scope(training_mode_funcs, **training_args):
#pylint: disable=no-value-for-parameter
net = inputs
assert_shape(net, [None, 32, 32, 3])
net = tf.cond(normalize_input,
lambda: slim.layer_norm(net,
scale=False,
center=False,
scope='normalize_inputs'),
lambda: net)
assert_shape(net, [None, 32, 32, 3])
net = nn.flip_randomly(net,
horizontally=flip_horizontally,
vertically=False,
name='random_flip')
net = tf.cond(translate,
lambda: nn.random_translate(net, scale=2, name='random_translate'),
lambda: net)
net = nn.gaussian_noise(net, scale=input_noise, name='gaussian_noise')
net = wn.conv2d(net, 128, scope="conv_1_1")
net = wn.conv2d(net, 128, scope="conv_1_2")
net = wn.conv2d(net, 128, scope="conv_1_3")
net = slim.max_pool2d(net, [2, 2], scope='max_pool_1')
net = slim.dropout(net, 1 - dropout_probability, scope='dropout_probability_1')
assert_shape(net, [None, 16, 16, 128])
net = wn.conv2d(net, 256, scope="conv_2_1")
net = wn.conv2d(net, 256, scope="conv_2_2")
net = wn.conv2d(net, 256, scope="conv_2_3")
net = slim.max_pool2d(net, [2, 2], scope='max_pool_2')
net = slim.dropout(net, 1 - dropout_probability, scope='dropout_probability_2')
assert_shape(net, [None, 8, 8, 256])
net = wn.conv2d(net, 512, padding='VALID', scope="conv_3_1")
assert_shape(net, [None, 6, 6, 512])
net = wn.conv2d(net, 256, kernel_size=[1, 1], scope="conv_3_2")
net = wn.conv2d(net, 128, kernel_size=[1, 1], scope="conv_3_3")
net = slim.avg_pool2d(net, [6, 6], scope='avg_pool')
assert_shape(net, [None, 1, 1, 128])
net = slim.flatten(net)
assert_shape(net, [None, 128])
primary_logits = wn.fully_connected(net, 10, init=is_initialization)
secondary_logits = wn.fully_connected(net, 10, init=is_initialization)
with tf.control_dependencies([tf.assert_greater_equal(num_logits, 1),
tf.assert_less_equal(num_logits, 2)]):
secondary_logits = tf.case([
(tf.equal(num_logits, 1), lambda: primary_logits),
(tf.equal(num_logits, 2), lambda: secondary_logits),
], exclusive=True, default=lambda: primary_logits)
assert_shape(primary_logits, [None, 10])
assert_shape(secondary_logits, [None, 10])
return primary_logits, secondary_logits
def convert_protein_and_ligand_to_image(ligand_elements, ligand_coords,
receptor_elements, receptor_coords,
side_pixels, pixel_size):
"""Take coordinates and elements of protein and ligand and convert them into an image.
Return image with one dimension so far."""
# FIXME abandon ligand when it does not fit into the box (it's kept now)
# max_num_attempts - maximum number of affine transforms for the ligand to be tried
max_num_attemts = 1000
# affine_transform_pool_size is the first(batch) dimension of tensor of transition matrices to be returned
# affine tranform pool is only generated once in the beginning of training and randomly sampled afterwards
affine_transform_pool_size = 10000
# transform center ligand around zero
ligand_center_of_mass = tf.reduce_mean(ligand_coords, reduction_indices=0)
centered_ligand_coords = ligand_coords - ligand_center_of_mass
centered_receptor_coords = receptor_coords - ligand_center_of_mass
# use TF while loop to find such an affine transform matrix that can fit the ligand so that no atoms are outside
box_size = (tf.cast(side_pixels, tf.float32) * pixel_size)
def generate_transition_matrix(attempt, transition_matrix,
batch_of_transition_matrices):
"""Takes initial coordinates of the ligand, generates a random affine transform matrix and transforms coordinates."""
transition_matrix = tf.gather(
batch_of_transition_matrices,
tf.random_uniform([],
minval=0,
maxval=affine_transform_pool_size,
dtype=tf.int32))
attempt += 1
return attempt, transition_matrix, batch_of_transition_matrices
def not_all_in_the_box(attempt,
transition_matrix,
batch_of_transition_matrices,
ligand_coords=centered_ligand_coords,
box_size=box_size,
max_num_attempts=max_num_attemts):
"""Takes affine transform matrix and box dimensions, performs the transformation, and checks if all atoms
are in the box."""
transformed_coords, transition_matrix = affine_transform(
ligand_coords, transition_matrix)
not_all = tf.cast(
tf.reduce_max(
tf.cast(
tf.square(box_size * 0.5) - tf.square(transformed_coords) <
0, tf.int32)), tf.bool)
within_iteration_limit = tf.cast(
tf.reduce_sum(tf.cast(attempt < max_num_attemts, tf.float32)),
tf.bool)
return tf.logical_and(within_iteration_limit, not_all)
attempt = tf.Variable(tf.constant(0, shape=[1]))
batch_of_transition_matrices = tf.Variable(
generate_deep_affine_transform(affine_transform_pool_size))
transition_matrix = tf.gather(
batch_of_transition_matrices,
tf.random_uniform([],
minval=0,
maxval=affine_transform_pool_size,
dtype=tf.int64))
last_attempt, final_transition_matrix, _ = tf.while_loop(
not_all_in_the_box,
generate_transition_matrix,
[attempt, transition_matrix, batch_of_transition_matrices],
parallel_iterations=1)
# rotate receptor and ligand using an affine transform matrix found
rotatated_ligand_coords, _ = affine_transform(centered_ligand_coords,
final_transition_matrix)
rotated_receptor_coords, _ = affine_transform(centered_receptor_coords,
final_transition_matrix)
# check if all of the atoms are in the box, if not set the ligand to 0, but do not raise an error
def set_elements_coords_zero():
return tf.constant([0], dtype=tf.int32), tf.constant([[0, 0, 0]],
dtype=tf.float32)
def keep_elements_coords():
return ligand_elements, rotatated_ligand_coords
not_all = tf.cast(
tf.reduce_max(
tf.cast(
tf.square(box_size * 0.5) - tf.square(rotatated_ligand_coords)
< 0, tf.int32)), tf.bool)
ligand_elements, rotatated_ligand_coords = tf.case(
{tf.equal(not_all, tf.constant(True)): set_elements_coords_zero},
keep_elements_coords)
# move coordinates of a complex to an integer number so as to put every atom on a grid
# ceiled coords is an integer number out of real coordinates that corresponds to the index on the cell
# epsilon - potentially, there might be very small rounding errors leading to additional indexes
epsilon = tf.constant(0.999, dtype=tf.float32)
ceiled_ligand_coords = tf.cast(
tf.round((-0.5 + (tf.cast(side_pixels, tf.float32) * 0.5) +
(rotatated_ligand_coords / pixel_size)) * epsilon), tf.int64)
ceiled_receptor_coords = tf.cast(
tf.round((-0.5 + (tf.cast(side_pixels, tf.float32) * 0.5) +
(rotated_receptor_coords / pixel_size)) * epsilon), tf.int64)
# crop atoms of the protein that do not fit inside the box
top_filter = tf.reduce_max(ceiled_receptor_coords,
reduction_indices=1) < side_pixels
bottom_filter = tf.reduce_min(ceiled_receptor_coords,
reduction_indices=1) > 0
retain_atoms = tf.logical_and(top_filter, bottom_filter)
cropped_receptor_coords = tf.boolean_mask(ceiled_receptor_coords,
retain_atoms)
cropped_receptor_elements = tf.boolean_mask(receptor_elements,
retain_atoms)
# merge protein and ligand together. In this case an arbitrary value of 10 is added to the ligand
complex_coords = tf.concat(0,
[ceiled_ligand_coords, cropped_receptor_coords])
complex_elements = tf.concat(
0, [ligand_elements + 7, cropped_receptor_elements])
# in coordinates of a protein rounded to the nearest integer can be represented as indices of a sparse 3D tensor
# values from the atom dictionary can be represented as values of a sparse tensor
# in this case TF's sparse_tensor_to_dense can be used to generate an image out of rounded coordinates
# move elemets to the dimension of depth
complex_coords_4d = tf.concat(1, [
complex_coords,
tf.reshape(tf.cast(complex_elements - 1, dtype=tf.int64), [-1, 1])
])
sparse_image_4d = tf.SparseTensor(
indices=complex_coords_4d,
values=tf.ones(tf.shape(complex_elements)),
shape=[side_pixels, side_pixels, side_pixels, 14])
# FIXME: try to save an image and see how it looks like
return sparse_image_4d, ligand_center_of_mass, final_transition_matrix
def main(unused_argv):
# Set up deployment (i.e., multi-GPUs and/or multi-replicas).
config = model_deploy.DeploymentConfig(
num_clones=FLAGS.num_clones,
clone_on_cpu=FLAGS.clone_on_cpu,
replica_id=FLAGS.task,
num_replicas=FLAGS.num_replicas,
num_ps_tasks=FLAGS.num_ps_tasks)
with tf.Graph().as_default():
with tf.device(config.inputs_device()):
train_crop_size = (None if 0 in FLAGS.train_crop_size else
FLAGS.train_crop_size)
assert FLAGS.dataset
assert len(FLAGS.dataset) == len(FLAGS.dataset_dir)
if len(FLAGS.first_frame_finetuning) == 1:
first_frame_finetuning = (list(FLAGS.first_frame_finetuning)
* len(FLAGS.dataset))
else:
first_frame_finetuning = FLAGS.first_frame_finetuning
if len(FLAGS.three_frame_dataset) == 1:
three_frame_dataset = (list(FLAGS.three_frame_dataset)
* len(FLAGS.dataset))
else:
three_frame_dataset = FLAGS.three_frame_dataset
assert len(FLAGS.dataset) == len(first_frame_finetuning)
assert len(FLAGS.dataset) == len(three_frame_dataset)
datasets, samples_list = zip(
*[_get_dataset_and_samples(config, train_crop_size, dataset,
dataset_dir, bool(first_frame_finetuning_),
bool(three_frame_dataset_))
for dataset, dataset_dir, first_frame_finetuning_,
three_frame_dataset_ in zip(FLAGS.dataset, FLAGS.dataset_dir,
first_frame_finetuning,
three_frame_dataset)])
# Note that this way of doing things is wasteful since it will evaluate
# all branches but just use one of them. But let's do it anyway for now,
# since it's easy and will probably be fast enough.
dataset = datasets[0]
if len(samples_list) == 1:
samples = samples_list[0]
else:
probabilities = FLAGS.dataset_sampling_probabilities
if probabilities:
assert len(probabilities) == len(samples_list)
else:
# Default to uniform probabilities.
probabilities = [1.0 / len(samples_list) for _ in samples_list]
probabilities = tf.constant(probabilities)
logits = tf.log(probabilities[tf.newaxis])
rand_idx = tf.squeeze(tf.multinomial(logits, 1, output_dtype=tf.int32),
axis=[0, 1])
def wrap(x):
def f():
return x
return f
samples = tf.case({tf.equal(rand_idx, idx): wrap(s)
for idx, s in enumerate(samples_list)},
exclusive=True)
# Prefetch_queue requires the shape to be known at graph creation time.
# So we only use it if we crop to a fixed size.
if train_crop_size is None:
inputs_queue = samples
else:
inputs_queue = prefetch_queue.prefetch_queue(
samples,
capacity=FLAGS.prefetch_queue_capacity_factor*config.num_clones,
num_threads=FLAGS.prefetch_queue_num_threads)
# Create the global step on the device storing the variables.
with tf.device(config.variables_device()):
global_step = tf.train.get_or_create_global_step()
# Define the model and create clones.
model_fn = _build_deeplab
if FLAGS.classification_loss == 'triplet':
embedding_dim = FLAGS.embedding_dimension
output_type_to_dim = {'embedding': embedding_dim}
else:
output_type_to_dim = {common.OUTPUT_TYPE: dataset.num_classes}
model_args = (inputs_queue, output_type_to_dim, dataset.ignore_label)
clones = model_deploy.create_clones(config, model_fn, args=model_args)
# Gather update_ops from the first clone. These contain, for example,
# the updates for the batch_norm variables created by model_fn.
first_clone_scope = config.clone_scope(0)
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS, first_clone_scope)
# Gather initial summaries.
summaries = set(tf.get_collection(tf.GraphKeys.SUMMARIES))
# Add summaries for model variables.
for model_var in tf.contrib.framework.get_model_variables():
summaries.add(tf.summary.histogram(model_var.op.name, model_var))
# Add summaries for losses.
for loss in tf.get_collection(tf.GraphKeys.LOSSES, first_clone_scope):
summaries.add(tf.summary.scalar('losses/%s' % loss.op.name, loss))
# Build the optimizer based on the device specification.
with tf.device(config.optimizer_device()):
learning_rate = train_utils.get_model_learning_rate(
FLAGS.learning_policy,
FLAGS.base_learning_rate,
FLAGS.learning_rate_decay_step,
FLAGS.learning_rate_decay_factor,
FLAGS.training_number_of_steps,
FLAGS.learning_power,
FLAGS.slow_start_step,
FLAGS.slow_start_learning_rate)
optimizer = tf.train.MomentumOptimizer(learning_rate, FLAGS.momentum)
summaries.add(tf.summary.scalar('learning_rate', learning_rate))
startup_delay_steps = FLAGS.task * FLAGS.startup_delay_steps
with tf.device(config.variables_device()):
total_loss, grads_and_vars = model_deploy.optimize_clones(
clones, optimizer)
total_loss = tf.check_numerics(total_loss, 'Loss is inf or nan.')
summaries.add(tf.summary.scalar('total_loss', total_loss))
# Modify the gradients for biases and last layer variables.
last_layers = model.get_extra_layer_scopes(
FLAGS.last_layers_contain_logits_only)
grad_mult = train_utils.get_model_gradient_multipliers(
last_layers, FLAGS.last_layer_gradient_multiplier)
if grad_mult:
grads_and_vars = slim.learning.multiply_gradients(grads_and_vars,
grad_mult)
with tf.name_scope('grad_clipping'):
grads_and_vars = slim.learning.clip_gradient_norms(grads_and_vars, 5.0)
# Create histogram summaries for the gradients.
# We have too many summaries for mldash, so disable this one for now.
# for grad, var in grads_and_vars:
# summaries.add(tf.summary.histogram(
# var.name.replace(':0', '_0') + '/gradient', grad))
# Create gradient update op.
grad_updates = optimizer.apply_gradients(grads_and_vars,
global_step=global_step)
update_ops.append(grad_updates)
update_op = tf.group(*update_ops)
with tf.control_dependencies([update_op]):
train_tensor = tf.identity(total_loss, name='train_op')
# Add the summaries from the first clone. These contain the summaries
# created by model_fn and either optimize_clones() or _gather_clone_loss().
summaries |= set(tf.get_collection(tf.GraphKeys.SUMMARIES,
first_clone_scope))
# Merge all summaries together.
summary_op = tf.summary.merge(list(summaries))
# Soft placement allows placing on CPU ops without GPU implementation.
session_config = tf.ConfigProto(allow_soft_placement=True,
log_device_placement=False)
# Start the training.
slim.learning.train(
train_tensor,
logdir=FLAGS.train_logdir,
log_every_n_steps=FLAGS.log_steps,
master=FLAGS.master,
number_of_steps=FLAGS.training_number_of_steps,
is_chief=(FLAGS.task == 0),
session_config=session_config,
startup_delay_steps=startup_delay_steps,
init_fn=train_utils.get_model_init_fn(FLAGS.train_logdir,
FLAGS.tf_initial_checkpoint,
FLAGS.initialize_last_layer,
last_layers,
ignore_missing_vars=True),
summary_op=summary_op,
save_summaries_secs=FLAGS.save_summaries_secs,
save_interval_secs=FLAGS.save_interval_secs)
def make_graph(self):
print('Compressing by', self.params['compression'], 'for a total of',
self.params['nneurons'], 'neurons')
#setup our graph
#tf.reset_default_graph()
mygraph = tf.Graph()
with mygraph.as_default():
#input images
with tf.name_scope('input'):
self.x = tf.placeholder(tf.float32,
shape=[
self.params["batchsize"],
self.params["framepatchsize"],
self.params["pixelpatchsize"],
self.params["pixelpatchsize"]
])
#self.xt = tf.transpose(self.x,perm=(0,3,2,1))
# self.xvec = tf.reshape(self.x,[self.params["batchsize"],
# params['clipvec_len']]
#activation function type
with tf.name_scope('activation_function'):
self.act_fun = self.params['nonlinearity']
#noises
with tf.name_scope('noises'):
self.noisexsigma = self.params['noise_x']
self.noisersigma = self.params['noise_r']
#function to add noise
with tf.name_scope("add_noise"):
def add_noise(input_layer, std):
noise = tf.random_normal(shape=tf.shape(input_layer),
mean=0.0,
stddev=std,
dtype=tf.float32)
return tf.add(input_layer, noise)
#weights
with tf.variable_scope("weights"):
# weights_kernel = tf.random_normal([self.params['frames_per_channel'],
# self.params['pixelpatchsize'],
# self.params['pixelpatchsize'],
# 1,
# self.params['nneurons']],
# dtype=tf.float32,stddev=0.1)
# weights_kernel = tf.random_normal([self.params['frames_per_channel'],
# self.params['pixelpatchsize'],
# self.params['pixelpatchsize'],
# self.params['nneurons']],
# dtype=tf.float32)
weights_kernel = tf.random_uniform([
self.params['frames_per_channel'],
self.params['pixelpatchsize'],
self.params['pixelpatchsize'], self.params['nneurons']
],
dtype=tf.float32,
minval=-1)
# weights_kernel = tf.random_normal([params['clipvec_len'],
# self.params['nneurons']]
self.win = tf.get_variable(name='weights_in',
initializer=weights_kernel)
self.wout = tf.get_variable(
name='weights_out',
initializer=tf.transpose(weights_kernel))
#[email protected](self.wout)
wnormalizer = tf.norm(tf.reshape(
self.win, (-1, self.params['nneurons'])),
ord='euclidean',
axis=0)
wnormalizer = tf.reshape(wnormalizer, (1, 1, 1, -1))
self.win = self.win * (1. / wnormalizer)
#self.wout = tf.transpose(self.win)
#self.wout = tf.get_variable('weights_out', initializer=tf.transpose(weights_kernel))
# self.wout = tf.get_variable('weights_out',[self.params['nneurons'],
# self.params['pixelpatchsize'],
# self.params['pixelpatchsize']],
# dtype=tf.float32)
#self.wout = tf.get_variable('weights_out',initializer=weights_kernel)
#self.wout = tf.get_variable('weights_out',initializer=tf.transpose(weights_kernel))
#bias
with tf.variable_scope("bias"):
self.bias = tf.zeros(
[self.params['nneurons']], dtype=tf.float32
) #tf.Variable(tf.random_normal([self.params['nneurons']],dtype=tf.float32))
#learning_rate
with tf.name_scope('learning_rate'):
self.learning_rate = self.params['learning_rate']
#nonlienarities
with tf.name_scope("nonlienarities"):
#define nonlinearities
def tanh_fun(arg):
return tf.nn.tanh(arg)
def sigmoid_fun(arg):
return tf.nn.sigmoid(arg)
def relu_fun(arg):
return tf.nn.relu(arg)
def no_fun(arg):
return arg
#encoding part of model
with tf.name_scope("encoding"):
#calculate input
noised_input = add_noise(self.x, self.params['noise_x'])
#expand dims for 1 channel: Need to change this for color channel
#linearin = tf.nn.conv3d(noised_input, self.win, strides= [1,
# 1,
# self.params['pixelpatchsize'],
# self.params['pixelpatchsize'],
# 1],
# padding='SAME') #Convolution over time, and multiply by weight
linearin = tf.einsum(
'ijkl,mijk->ml', self.win,
noised_input) #[500,5,12,12], [5,12,12,144] -> [500,144])
#linearin = self.win @ noised_input
#linearin = tf.add(linearin,self.bias)
self.activation = tf.case(
{
tf.equal(self.act_fun, 'tanh'):
(lambda: tanh_fun(linearin)),
tf.equal(self.act_fun, 'sigmoid'):
(lambda: sigmoid_fun(linearin)),
tf.equal(self.act_fun, 'relu'):
(lambda: relu_fun(linearin))
},
default=(lambda: no_fun(linearin)),
exclusive=True)
#self.yin = add_noise(self.activation,self.params['noise_r'])
self.yin = self.activation
#output part of model
with tf.name_scope("decoding"):
#calculate output (reconstruction)
# self.xp = tf.nn.conv3d_transpose(self.yin, self.wout,
# output_shape = (self.params["batchsize"],
# self.params["pixelpatchsize"],
# self.params["pixelpatchsize"],
# 1,1),
# strides=[1,
# 1,
# self.params['pixelpatchsize'],
# self.params['pixelpatchsize'],
# 1],
# padding='SAME') #Deconvolution
# self.xpvec = self.yin @ self.woutvec
# self.xp = tf.reshape(self.xpvec,[self.params["batchsize"],
# self.params["pixelpatchsize"],
# self.params["pixelpatchsize"],
self.xp = tf.einsum(
'ij,jklm->imlk', self.yin,
self.wout) # [500,144],[144,5,12,12] -> [500,5,12,12]
#self.xp = tf.matmul(self.yin,self.wout) #add noise to inner layer, and multiply by weight transpose
#self.xp = tf.case({tf.equal(self.act_fun,'tanh'): (lambda: tanh_fun(linearout)),
# tf.equal(self.act_fun,'sigmoid'): (lambda: sigmoid_fun(linearout)),
# tf.equal(self.act_fun,'relu'): (lambda: relu_fun(linearout))},
# default=(lambda: no_fun(linearout)),
# exclusive=True, name='output_nonlienarity')
#lambda activation
with tf.name_scope('lambda_activation'):
self.mean_act = tf.reduce_sum(tf.reduce_mean(self.activation,
axis=0),
axis=0)
desired_spikes_per_neuron = self.params[
"framepatchsize"] / self.params["frames_per_channel"]
self.lambda_act = tf.abs(
desired_spikes_per_neuron -
self.mean_act) #no cost if avg activation is as expectd
#self.xp @ tf.transpose(self.x)
#[email protected](self.xp)
#vectorize before calculating norm
with tf.name_scope('recon_error'):
#self.recon_err = (tf.reshape(self.x,[self.params["batchsize"],-1]) -
# tf.reshape(self.xp,[self.params["batchsize"],-1]))
#(self.x - self.xp) @ tf.transpose(self.xp)
self.recon_err = tf.reshape((self.x - self.xp),
[self.params["batchsize"], -1])
self.recon_err = tf.norm(self.recon_err,
ord='euclidean',
axis=1)
#self.recon_err = tf.reduce_mean(tf.norm(self.x-self.xp, ord='euclidean', axis=(2,3)))
#calculate cost
with tf.name_scope("cost_function"):
#self.lambda_act = tf.reduce_mean(self.activation+1e-5, axis=0)
self.cost = self.recon_err + tf.reduce_mean(self.lambda_act)
#train our model
with tf.name_scope("training_step"):
self.train_step = tf.train.AdamOptimizer(
self.learning_rate).minimize(self.cost)
# create a summary for our cost, im, reconstruction, & weights
with tf.name_scope('cost_viz'):
tf.summary.scalar("cost", self.cost)
#with tf.name_scope('image_viz'):
# x_t = tf.reshape(self.x,(self.params['batchsize'], 0, self.params['imxlen'],self.params['imylen'],1))
# tf.summary.image("image", x_t, max_outputs=self.params["batchsize"])
#with tf.name_scope('recon_viz'):
# xp_t = tf.reshape(self.xp,(self.params['batchsize'], 0,self.params['imxlen'],self.params['imylen'],1))
# print (xp_t)
# tf.summary.image("recon", xp_t,max_outputs=self.params["batchsize"])
#CHANGE
#with tf.name_scope('inweights_viz'):
# #inwin_t = tf.reshape(tf.transpose(self.win),
# #inwin_t = tf.reshape(tf.transpose(self.win, perm=[3,0,1,2]), (self.params['nneurons'], self.params['imxlen'], self.params['imylen'],1))
# inwin_t = tf.transpose(self.win, perm=[0,4,1,2,3])
# tf.summary.image("inweights", inwin_t, max_outputs=self.params['nneurons'])
#with tf.name_scope('outweights_viz'):
# #outwin_t = tf.reshape(tf.transpose(self.wout[0], perm=[3,0,1,2]), (self.params['nneurons'], self.params['imxlen'], self.params['imylen'],1))
# outwin_t = tf.transpose(self.wout, perm=[0,4,1,2,3])
# tf.summary.image("outweights", outwin_t, max_outputs=self.params['nneurons'])
#with tf.name_scope('activnonlin_viz'):
# activation = tf.transpose(self.yin, perm=[0,4,1,2,3])
# activation = tf.reshape(activation, (self.params["batchsize"], self.params["time_patchsize"], -1)) #reshape nonlinear-vector [batchsize, nneurons, time]
# activ_help_temp = activation[0,0] # temporarily take only one nneuron over time
# tf.summary.scalar("activnonlin", activ_help_temp)
# merge all summaries into a single "operation" which we can execute in a session
#self.summary_op = tf.summary.merge_all()
return (mygraph)
def get_net():
net = {}
global_step = tf.Variable(0, dtype=tf.int32, trainable=False)
phase = tf.placeholder(tf.int32)
net.update(dict(global_step=global_step, phase=phase))
train_values = train_pipeline(data.get_values(files[data._TRAIN]))
test_values = test_pipeline(data.get_values(files[data._VAL]))
(key, value, label) = tf.case([
(tf.equal(phase, data._TRAIN), lambda: train_values),
(tf.equal(phase, data._VAL), lambda: test_values)], default=lambda: train_values)
net.update(dict(key=key, value=value, label=label))
with tf.variable_scope('subsample-2x'):
value = conv_relu(value, 'conv-0', size=(3, 3), stride=(2, 2), out_channel=32)
value = conv_relu(value, 'conv-1', size=(3, 3), stride=(1, 1), out_channel=32)
value = conv_relu(value, 'conv-2', size=(3, 3), stride=(1, 1), out_channel=96)
value = conv_relu(value, 'conv-3', size=(3, 3), stride=(1, 1), out_channel=128)
with tf.variable_scope('subsample-4x-0'):
with tf.variable_scope('branch-0'):
value0 = conv_relu(value, 'conv', size=(3, 3), stride=(2, 2), out_channel=128)
with tf.variable_scope('branch-1'):
value1 = max_pool(value, 'pool', size=(3, 3), stride=(2, 2))
value = tf.concat(3, [value0, value1])
with tf.variable_scope('subsample-4x-1'):
with tf.variable_scope('branch-0'):
value0 = conv_relu(value, 'conv-0', size=(1, 1), stride=(1, 1), out_channel=64)
value0 = conv_relu(value0, 'conv-1', size=(3, 3), stride=(1, 1), out_channel=128)
with tf.variable_scope('branch-1'):
value1 = conv_relu(value, 'conv-0', size=(1, 1), stride=(1, 1), out_channel=64)
value1 = conv_relu(value1, 'conv-1', size=(1, 7), stride=(1, 1), out_channel=64)
value1 = conv_relu(value1, 'conv-2', size=(7, 1), stride=(1, 1), out_channel=64)
value1 = conv_relu(value1, 'conv-3', size=(3, 3), stride=(1, 1), out_channel=128)
value = tf.concat(3, [value0, value1])
with tf.variable_scope('subsample-8x'):
with tf.variable_scope('branch-0'):
value0 = conv_relu(value, 'conv', size=(3, 3), stride=(2, 2), out_channel=256)
with tf.variable_scope('branch-1'):
value1 = max_pool(value, 'pool', size=(3, 3), stride=(2, 2))
value = tf.concat(3, [value0, value1])
bbq_list = [(value, (1, 1))]
with tf.variable_scope('subsample-16x'):
for i in xrange(4):
stride = (2, 2) if i == 0 else (1, 1)
bbq_list = bbq_unit(bbq_list, 'bbq-%d' % i, stride=stride, out_channel_small=128, out_channel_large=512)
with tf.variable_scope('subsample-32x'):
for i in xrange(32):
stride = (2, 2) if i == 0 else (1, 1)
bbq_list = bbq_unit(bbq_list, 'bbq-%d' % i, stride=stride, out_channel_small=256, out_channel_large=1024)
with tf.variable_scope('subsample-64x'):
for i in xrange(4):
stride = (2, 2) if i == 0 else (1, 1)
bbq_list = bbq_unit(bbq_list, 'bbq-%d' % i, stride=stride, out_channel_small=512, out_channel_large=2048)
value = bbq_list[-1][0]
with tf.variable_scope('fc'):
value = avg_pool(value, 'pool', size=(4, 4), stride=(1, 1))
value = conv(value, 'fc', size=(1, 1), stride=(1, 1), out_channel=1000)[0]
value = tf.squeeze(value, squeeze_dims=(1, 2))
value = softmax(value, dim=1)
loss = - tf.reduce_mean(tf.reduce_sum(label * tf.log(value + util._EPSILON), 1), 0)
correct = tf.to_float(tf.equal(tf.argmax(label, 1), tf.argmax(value, 1)))
acc = tf.reduce_mean(correct)
learning_rate = tf.train.exponential_decay(
learning_rate=1e-2,
global_step=global_step,
decay_steps=50000,
decay_rate=0.9,
staircase=True)
train = tf.train.RMSPropOptimizer(
learning_rate=learning_rate,
decay=0.9,
epsilon=1.0).minimize(loss, global_step=global_step)
net.update(dict(loss=loss, acc=acc, train=train))
return net
def _setup_train_net_multigpu(self):
with tf.device('/cpu:0'):
# learning rate decay
with tf.name_scope('lr_decay'):
if FLAGS.lr_policy == 'staircase':
# decayed learning rate
lr_breakpoints = [int(o) for o in FLAGS.lr_breakpoints.split(',')]
lr_decays = [float(o) for o in FLAGS.lr_decays.split(',')]
assert(len(lr_breakpoints) == len(lr_decays))
pred_fn_pairs = []
for lr_decay, lr_breakpoint in zip(lr_decays, lr_breakpoints):
fn = (lambda o: lambda: tf.constant(o, tf.float32))(lr_decay)
pred_fn_pairs.append((tf.less(self.global_step, lr_breakpoint), fn))
lr_decay = tf.case(pred_fn_pairs, default=(lambda: tf.constant(1.0)))
else:
logging.error('Unkonw lr_policy: {}'.format(FLAGS.lr_policy))
sys.exit(1)
self.current_lr = lr_decay * FLAGS.base_lr
tf.summary.scalar('lr', self.current_lr, collections=['brief'])
# input data
# batch_size = int(FLAGS.train_batch_size / FLAGS.n_gpu)
with tf.name_scope('input_data'):
batch_size = FLAGS.train_batch_size
train_datasets = FLAGS.train_datasets.split(';')
train_pstreams_list = []
for i, dataset in enumerate(train_datasets):
if not os.path.exists(dataset):
logging.critical('Could not find dataset {}'.format(dataset))
sys.exit(1)
logging.info('Added training dataset #{}: {}'.format(i, dataset))
train_streams = data.input_stream(dataset)
train_pstreams = data.train_preprocess(train_streams)
train_pstreams_list.append(train_pstreams)
capacity = batch_size * 50
min_after_dequeue = batch_size * 3
train_batch = tf.train.shuffle_batch_join(train_pstreams_list,
batch_size,
capacity=capacity,
min_after_dequeue=min_after_dequeue)
logging.info('Batch size {}; capacity: {}; min_after_dequeue: {}'.format(batch_size, capacity, min_after_dequeue))
# split batch into sub-batches for each GPU
sub_batch_size = int(FLAGS.train_batch_size / FLAGS.n_gpu)
logging.info('Batch size is {} on each of the {} GPUs'.format(sub_batch_size, FLAGS.n_gpu))
sub_batches = []
for i in range(FLAGS.n_gpu):
sub_batch = {}
for k, v in train_batch.items():
sub_batch[k] = v[i*sub_batch_size : (i+1)*sub_batch_size]
sub_batches.append(sub_batch)
if FLAGS.optimizer == 'sgd':
optimizer = tf.train.MomentumOptimizer(self.current_lr, FLAGS.momentum)
logging.info('Using SGD optimizer. Momentum={}'.format(FLAGS.momentum))
elif FLAGS.optimizer == 'adam':
optimizer = tf.train.AdamOptimizer(self.current_lr)
logging.info('Using ADAM optimizer.')
elif FLAGS.optimizer == 'rmsprop':
optimizer = tf.train.RMSPropOptimizer(self.current_lr)
logging.info('Using RMSProp optimizer.')
else:
logging.critical('Unsupported optimizer {}'.format(FLAGS.optimizer))
sys.exit(1)
# construct towers
tower_gradients = []
tower_losses = []
for i in range(FLAGS.n_gpu):
logging.info('Setting up tower %d' % i)
with tf.device('/gpu:%d' % i):
# variables are shared
with tf.variable_scope(tf.get_variable_scope(), reuse=(i > 0)):
with tf.name_scope('tower_%d' % i):
loss = self._tower_loss(sub_batches[i])
# tf.get_variable_scope().reuse_variables()
gradients = optimizer.compute_gradients(loss)
tower_gradients.append(gradients)
tower_losses.append(loss)
# average loss and gradients
self.loss = tf.truediv(tf.add_n(tower_losses), float(len(tower_losses)),
name='average_tower_loss')
tf.summary.scalar('total_loss', self.loss, collections=['brief'])
with tf.name_scope('average_gradients'):
grads = self._average_gradients(tower_gradients)
# update variables
with tf.variable_scope('optimizer'):
self.train_op = optimizer.apply_gradients(grads, global_step=self.global_step)
# setup summaries
for var in tf.all_variables():
# remove the illegal ":x" part from the variable name
summary_name = 'parameters/' + var.name.split(':')[0]
tf.summary.histogram(summary_name, var, collections=['detailed'])
self.brief_summary_op = tf.summary.merge_all(key='brief')
self.detailed_summary_op = tf.summary.merge_all(key='detailed')
def _provide_data(input_tensors, truncated_length, hparams):
"""Returns tensors for reading batches from provider."""
length = tf.to_int32(input_tensors.length)
labels = tf.reshape(input_tensors.labels, (-1, constants.MIDI_PITCHES))
label_weights = tf.reshape(input_tensors.label_weights,
(-1, constants.MIDI_PITCHES))
onsets = tf.reshape(input_tensors.onsets, (-1, constants.MIDI_PITCHES))
offsets = tf.reshape(input_tensors.offsets, (-1, constants.MIDI_PITCHES))
velocities = tf.reshape(input_tensors.velocities,
(-1, constants.MIDI_PITCHES))
spec = tf.reshape(input_tensors.spec, (-1, hparams_frame_size(hparams)))
truncated_length = (
tf.reduce_min([truncated_length, length]) if truncated_length else length)
# Pad or slice specs and labels tensors to have the same lengths,
# truncating after truncated_length.
spec_delta = tf.shape(spec)[0] - truncated_length
spec = tf.case(
[(spec_delta < 0,
lambda: tf.pad(spec, tf.stack([(0, -spec_delta), (0, 0)]))),
(spec_delta > 0, lambda: spec[0:-spec_delta])],
default=lambda: spec)
labels_delta = tf.shape(labels)[0] - truncated_length
labels = tf.case(
[(labels_delta < 0,
lambda: tf.pad(labels, tf.stack([(0, -labels_delta), (0, 0)]))),
(labels_delta > 0, lambda: labels[0:-labels_delta])],
default=lambda: labels)
label_weights = tf.case(
[(labels_delta < 0,
lambda: tf.pad(label_weights, tf.stack([(0, -labels_delta), (0, 0)]))
), (labels_delta > 0, lambda: label_weights[0:-labels_delta])],
default=lambda: label_weights)
onsets = tf.case(
[(labels_delta < 0,
lambda: tf.pad(onsets, tf.stack([(0, -labels_delta), (0, 0)]))),
(labels_delta > 0, lambda: onsets[0:-labels_delta])],
default=lambda: onsets)
offsets = tf.case(
[(labels_delta < 0,
lambda: tf.pad(offsets, tf.stack([(0, -labels_delta), (0, 0)]))),
(labels_delta > 0, lambda: offsets[0:-labels_delta])],
default=lambda: offsets)
velocities = tf.case(
[(labels_delta < 0,
lambda: tf.pad(velocities, tf.stack([(0, -labels_delta), (0, 0)]))),
(labels_delta > 0, lambda: velocities[0:-labels_delta])],
default=lambda: velocities)
truncated_note_sequence = truncate_note_sequence_op(
input_tensors.note_sequence, truncated_length, hparams)
return (FeatureTensors(
spec=tf.reshape(spec, (truncated_length, hparams_frame_size(hparams), 1)),
length=truncated_length,
sequence_id=input_tensors.sequence_id),
LabelTensors(
labels=tf.reshape(labels,
(truncated_length, constants.MIDI_PITCHES)),
label_weights=tf.reshape(
label_weights, (truncated_length, constants.MIDI_PITCHES)),
onsets=tf.reshape(onsets,
(truncated_length, constants.MIDI_PITCHES)),
offsets=tf.reshape(offsets,
(truncated_length, constants.MIDI_PITCHES)),
velocities=tf.reshape(velocities,
(truncated_length, constants.MIDI_PITCHES)),
note_sequence=truncated_note_sequence))
def train(self, ):
shard_nums = self.config.gpus
base_lr = self.config.lr
print('*********************', shard_nums, base_lr,
self.config.batch_size_per_GPU)
steps = tf.Variable(0.0, name='ssd_steps', trainable=False)
x = 1
lr = tf.case(
{
steps < 40000.0 * x: lambda: base_lr,
steps < 50000.0 * x: lambda: base_lr / 10
},
default=lambda: base_lr / 100)
tower_grads = []
opt = tf.train.MomentumOptimizer(lr, 0.9)
var_reuse = False
Iter_list = []
for i in range(shard_nums):
with tf.device('/gpu:%d' % i):
loss = 0
Iter = readData(self.config.files,
self.config,
batch_size=self.config.batch_size_per_GPU,
num_threads=16,
shuffle_buffer=1024,
num_shards=shard_nums,
shard_index=i)
Iter_list.append(Iter)
weights_init = tf.truncated_normal_initializer(stddev=0.03)
with tf.variable_scope('', reuse=var_reuse):
with slim.arg_scope(
[
slim.conv2d, slim.fully_connected,
slim.separable_conv2d
],
weights_regularizer=slim.l2_regularizer(
self.config.weight_decay),
weights_initializer=weights_init,
activation_fn=tf.nn.relu6,
normalizer_fn=slim.batch_norm):
if i == 0:
pre_loss, var_pre = self.build_net(Iter)
else:
pre_loss, _ = self.build_net(Iter)
var_reuse = True
loss += pre_loss
train_vars = tf.trainable_variables()
l2_loss = tf.losses.get_regularization_losses()
l2_re_loss = tf.add_n(l2_loss)
ssd_train_loss = pre_loss + l2_re_loss
print('********', ssd_train_loss)
grads_and_vars = opt.compute_gradients(ssd_train_loss,
train_vars)
tower_grads.append(grads_and_vars)
# for v in tf.global_variables():
# print(v)
grads = average_gradients(tower_grads)
grads = list(zip(*grads))[0]
grads, norm = tf.clip_by_global_norm(grads, 20.0)
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
for v in tf.global_variables():
print(v)
with tf.control_dependencies(update_ops):
train_op = opt.apply_gradients(zip(grads, train_vars),
global_step=steps)
saver_pre = tf.train.Saver(var_pre)
saver = tf.train.Saver(max_to_keep=200)
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
config.allow_soft_placement = True
with tf.Session(config=config) as sess:
sess.run(tf.global_variables_initializer())
saver_pre.restore(sess, self.config.pre_model)
# self.init(var_pre, sess)
# saver.restore(sess, file)
for Iter in Iter_list:
sess.run(Iter.initializer)
for i in range(00000, int(60010 * x)):
if i % 20 == 0:
_, loss_, a, b, c, d = sess.run(
[train_op, ssd_train_loss, loss, l2_re_loss, norm, lr])
print(datetime.now(), 'ssd_loss:%.4f' % loss_,
'loss:%.4f' % a, 'l2_re_loss:%.4f' % b,
'norm:%.4f' % c, d, i)
else:
sess.run(train_op)
if (i + 1) % 5000 == 0 or ((i + 1) % 1000 == 0
and i < 10000) or (i + 1) == int(
60010 * x):
saver.save(sess,
os.path.join('./models/', 'SSD300_1.ckpt'),
global_step=i + 1)
pass
def prepare(self, is_training, global_step=0):
# config. parameters
self.global_step = global_step
self.scale_list = list(map(lambda x: int(x), FLAGS.scales.split(',')))
if (is_training):
for scale in self.scale_list:
if (not scale in [4]): # only x4 is supported
raise ValueError('Unsupported scale is provided.')
else:
for scale in self.scale_list:
if (not scale in [2, 4, 8]): # only x2, x4, and x8 are supported
raise ValueError('Unsupported scale is provided.')
self.multipass_upscaling = (not FLAGS.eusr_disable_multipass)
self.num_conv_features = FLAGS.eusr_conv_features
self.num_shared_blocks = FLAGS.eusr_shared_blocks
self.num_upscale_blocks = FLAGS.eusr_upscale_blocks
num_expected_residual_blocks = 0
num_expected_residual_blocks += (2 * len(self.scale_list)) # scale-specific local residual blocks
num_expected_residual_blocks += self.num_shared_blocks # shared residual module
for scale in self.scale_list:
num_expected_residual_blocks += (int(math.log(scale, 2)) * 4 * self.num_upscale_blocks) # enhanced upscaling modules
self.num_expected_residual_blocks = num_expected_residual_blocks
self.shift_mean_list = list(map(lambda x: float(x), FLAGS.eusr_rgb_mean.split(',')))
if (is_training):
self.initial_learning_rate = FLAGS.eusr_learning_rate
self.initial_learning_rate_discriminator = FLAGS.eusr_learning_rate_discriminator
self.learning_rate_decay = FLAGS.eusr_learning_rate_decay
self.learning_rate_decay_steps = FLAGS.eusr_learning_rate_decay_steps
self.aesthetic_nima_path = FLAGS.eusr_aesthetic_nima_path
self.subjective_nima_path = FLAGS.eusr_subjective_nima_path
self.loss_weight_r = FLAGS.eusr_weight_lr
self.loss_weight_g = FLAGS.eusr_weight_lg
self.loss_weight_as = FLAGS.eusr_weight_las
self.loss_weight_ar = FLAGS.eusr_weight_lar
self.loss_weight_ss = FLAGS.eusr_weight_lss
self.loss_weight_sr = FLAGS.eusr_weight_lsr
# tensorflow graph
self.tf_graph = tf.Graph()
with self.tf_graph.as_default():
self.tf_input = tf.placeholder(tf.float32, [None, None, None, 3], name=BaseModel.TF_INPUT_NAME)
self.tf_scale = tf.placeholder(tf.float32, [], name=BaseModel.TF_INPUT_SCALE_NAME)
# generator > 2x
self.tf_output_2x = self._generator(input_list=self.tf_input, scale=2, reuse=False)
self.tf_output_2x = self._generator(input_list=self.tf_output_2x, scale=2, reuse=True)
# generator > 4x
self.tf_output_4x = self._generator(input_list=self.tf_input, scale=4, reuse=True)
# generator > 8x
self.tf_output_8x = self._generator(input_list=self.tf_input, scale=8, reuse=True)
self.tf_output_8x = tf.image.resize_images(self.tf_output_8x, size=[tf.shape(self.tf_input)[1] * 4, tf.shape(self.tf_input)[2] * 4], method=tf.image.ResizeMethod.BICUBIC, align_corners=False)
# output based on tf_scale placeholder
output_fn_pairs = []
output_fn_pairs.append((tf.equal(self.tf_scale, 2), lambda: tf.identity(self.tf_output_2x)))
output_fn_pairs.append((tf.equal(self.tf_scale, 4), lambda: tf.identity(self.tf_output_4x)))
output_fn_pairs.append((tf.equal(self.tf_scale, 8), lambda: tf.identity(self.tf_output_8x)))
self.tf_output = tf.case(output_fn_pairs, exclusive=True)
if (is_training):
input_summary = tf.cast(tf.clip_by_value(self.tf_input, 0.0, 255.0), tf.uint8)
tf.summary.image('input', input_summary)
if (self.multipass_upscaling):
output_summary = tf.cast(tf.clip_by_value(self.tf_output_2x, 0.0, 255.0), tf.uint8)
tf.summary.image('output_2x', output_summary)
output_summary = tf.cast(tf.clip_by_value(self.tf_output_4x, 0.0, 255.0), tf.uint8)
tf.summary.image('output_4x', output_summary)
output_summary = tf.cast(tf.clip_by_value(self.tf_output_8x, 0.0, 255.0), tf.uint8)
tf.summary.image('output_8x', output_summary)
else:
output_summary = tf.cast(tf.clip_by_value(self.tf_output, 0.0, 255.0), tf.uint8)
tf.summary.image('output', output_summary)
self.tf_truth = tf.placeholder(tf.float32, [None, None, None, 3])
truth_summary = tf.cast(tf.clip_by_value(self.tf_truth, 0.0, 255.0), tf.uint8)
tf.summary.image('truth', truth_summary)
self.tf_global_step = tf.placeholder(tf.int64, [])
combined_output_list = []
if (self.multipass_upscaling):
combined_output_list.append(self.tf_output_2x)
combined_output_list.append(self.tf_output_4x)
combined_output_list.append(self.tf_output_8x)
else:
combined_output_list.append(self.tf_output)
self.tf_train_op, self.tf_loss = self._optimize(output_list=combined_output_list, truth_list=self.tf_truth, global_step=self.tf_global_step)
for key, loss in self.loss_dict.items():
tf.summary.scalar(('loss/%s' % (key)), loss)
self.tf_saver = tf.train.Saver(max_to_keep=FLAGS.save_max_keep)
self.tf_summary_op = tf.summary.merge_all()
self.tf_init_op = tf.group(tf.global_variables_initializer(), tf.local_variables_initializer())
# tensorflow session
self.tf_session = tf.Session(config=tf.ConfigProto(
log_device_placement=False,
allow_soft_placement=True
), graph=self.tf_graph)
self.tf_session.run(self.tf_init_op)
def random_augmentation(example, board_size, mode="rnn"):
"""Perform a random rotation/flip on the example.
example["inputs"] needs to be of shape [game_length, 3, board_size, board_size].
example["p_targets"] needs to be of shape [game_length] containing ints in [0, num_moves).
example["legal_moves"] needs to be of shape [game_length, num_moves].
1/8 chance to do on of:
* do nothing
* rotate 90° counter clockwise
* rotate 180° counter clockwise
* rotate 270° counter clockwise
* flip along vertical axis
* flip along horizontal axis
* flip along diagonal axis from the upper left
* flip along diagonal axis from the upper right
Args:
example: (dict), go game
board_size: (int), board size
mode: (str), rnn or cnn, only needed if the example doesnt have the dimension game_length
Return:
Randomly augmented example
"""
assert mode in ["rnn", "cnn"]
flat_board = board_size * board_size
num_moves = flat_board + 1
inputs = example["inputs"]
legal_moves = example["legal_moves"]
p_targets = example["p_targets"]
inputs = tf.convert_to_tensor(inputs, name='inputs')
legal_moves = tf.convert_to_tensor(legal_moves, name='legal_moves')
p_targets = tf.convert_to_tensor(p_targets, name='p_targets')
rand_k = tf.random_uniform([], int(0), int(8), tf.int64, name="rand_k")
for name, array in [["inputs", inputs], ["legal_moves", legal_moves], ["p_targets", p_targets]]:
split = name != "inputs"
split_p = name == "p_targets"
if mode == "cnn":
array = tf.expand_dims(array, axis=0)
if split:
if split_p:
# convert p_target index to one hot
array = tf.one_hot(array, num_moves)
# split array into a flat board and one int representing the pass move
array, rest = tf.split(array, [flat_board, 1], 1)
# reshape to boards
array = tf.reshape(array, [-1, board_size, board_size])
# need to transpose last 2 axes
perm = [0, 2, 1]
else:
# need to transpose last 2 axes
perm = [0, 1, 3, 2]
scope = ""
def _no_aug():
nonlocal scope
scope = "no_augmentation"
return array
def _rot90():
nonlocal scope
scope = "rot_90_counter"
return tf.transpose(tf.reverse(array, [-1]), perm)
def _rot180():
nonlocal scope
scope = "rot_180_counter"
return tf.reverse(array, [-1, -2])
def _rot270():
nonlocal scope
scope = "rot_270_counter"
return tf.reverse(tf.transpose(array, perm), [-1])
def _flip_left_right():
nonlocal scope
scope = "flip_left_right"
return tf.reverse(array, [-1])
def _flip_up_down():
nonlocal scope
scope = "flip_up_down"
return tf.reverse(array, [-2])
def _flip_diagonal_upper_left():
nonlocal scope
scope = "flip_diagonal_upper_left"
return tf.transpose(array, perm)
def _flip_diagonal_upper_right():
nonlocal scope
scope = "flip_diagonal_upper_right"
return tf.transpose(tf.reverse(array, [-1, -2]), perm)
cases = [
(tf.equal(rand_k, 0), _no_aug),
(tf.equal(rand_k, 1), _rot90),
(tf.equal(rand_k, 2), _rot180),
(tf.equal(rand_k, 3), _rot270),
(tf.equal(rand_k, 4), _flip_left_right),
(tf.equal(rand_k, 5), _flip_up_down),
(tf.equal(rand_k, 6), _flip_diagonal_upper_left),
(tf.equal(rand_k, 7), _flip_diagonal_upper_right)
]
result = tf.case(cases, name=scope)
if split:
# reassemble the original shape from combined result tensor and rest tensor
result = tf.reshape(result, [-1, flat_board])
result = tf.concat([result, rest], 1)
if split_p:
result = tf.argmax(result, 1)
if mode == "cnn":
result = tf.squeeze(result, axis=[0])
example[name] = result
return example
def build_model(self,
relu_target,
input_tensor,
style_encoded_tensor=None,
batch_size=8,
feature_weight=1,
pixel_weight=1,
tv_weight=0,
learning_rate=1e-4,
lr_decay=5e-5,
ss_patch_size=3,
ss_stride=1):
'''Build the EncoderDecoder architecture for a given relu layer.
Args:
relu_target: Layer of VGG to decode from
input_tensor: If None then a placeholder will be created, else use this tensor as the input to the encoder
style_encoded_tensor: Tensor for style image features at the same relu layer. Used only at test time.
batch_size: Batch size for training
feature_weight: Float weight for feature reconstruction loss
pixel_weight: Float weight for pixel reconstruction loss
tv_weight: Float weight for total variation loss
learning_rate: Float LR
lr_decay: Float linear decay for training
Returns:
EncoderDecoder namedtuple with input/encoding/output tensors and ops for training.
'''
with tf.name_scope('encoder_decoder_'+relu_target):
### Build encoder for reluX_1
with tf.name_scope('content_encoder_'+relu_target):
if input_tensor is None:
# This is the first level encoder that takes original content imgs
content_imgs = tf.placeholder_with_default(tf.constant([[[[0.,0.,0.]]]]), shape=(None, None, None, 3), name='content_imgs')
else:
# This is an intermediate-level encoder that takes output tensor from previous level as input
content_imgs = input_tensor
# Build content layer encoding model
content_layer = self.vgg_model.get_layer(relu_target).output
content_encoder_model = Model(inputs=self.vgg_model.input, outputs=content_layer)
# Setup content layer encodings for content images
content_encoded = content_encoder_model(content_imgs)
### Build style encoder & WCT if test mode
if self.mode != 'train':
with tf.name_scope('wct_'+relu_target):
if relu_target == 'relu5_1':
# Apply style swap on relu5_1 encodings if self.swap5 flag is set
# Use AdaIN as transfer op instead of WCT if self.use_adain is set
# Otherwise perform WCT
decoder_input = tf.case([(self.swap5, lambda: wct_style_swap(content_encoded,
style_encoded_tensor,
self.ss_alpha,
ss_patch_size,
ss_stride)),
(self.use_adain, lambda: adain(content_encoded, style_encoded_tensor, self.alpha))],
default=lambda: wct_tf(content_encoded, style_encoded_tensor, self.alpha))
else:
decoder_input = tf.cond(self.use_adain,
lambda: adain(content_encoded, style_encoded_tensor, self.alpha),
lambda: wct_tf(content_encoded, style_encoded_tensor, self.alpha))
else: # In train mode we're trying to reconstruct from the encoding, so pass along unchanged
decoder_input = content_encoded
### Build decoder
with tf.name_scope('decoder_'+relu_target):
n_channels = content_encoded.get_shape()[-1].value
decoder_model = self.build_decoder(input_shape=(None, None, n_channels), relu_target=relu_target)
# Wrap the decoder_input tensor so that it has the proper shape for decoder_model
decoder_input_wrapped = tf.placeholder_with_default(decoder_input, shape=[None,None,None,n_channels])
# Reconstruct/decode from encoding
decoded = decoder_model(Lambda(lambda x: x)(decoder_input_wrapped)) # Lambda converts TF tensor to Keras
# Content layer encoding for stylized out
decoded_encoded = content_encoder_model(decoded)
if self.mode == 'train': # Train & summary ops only needed for training phase
### Losses
with tf.name_scope('losses_'+relu_target):
# Feature loss between encodings of original & reconstructed
feature_loss = feature_weight * mse(decoded_encoded, content_encoded)
# Pixel reconstruction loss between decoded/reconstructed img and original
pixel_loss = pixel_weight * mse(decoded, content_imgs)
# Total Variation loss
if tv_weight > 0:
tv_loss = tv_weight * tf.reduce_mean(tf.image.total_variation(decoded))
else:
tv_loss = tf.constant(0.)
total_loss = feature_loss + pixel_loss + tv_loss
### Training ops
with tf.name_scope('train_'+relu_target):
global_step = tf.Variable(0, name='global_step_train', trainable=False)
# self.learning_rate = tf.train.exponential_decay(learning_rate, self.global_step, 100, 0.96, staircase=False)
learning_rate = torch_decay(learning_rate, global_step, lr_decay)
d_optimizer = tf.train.AdamOptimizer(learning_rate, beta1=0.9, beta2=0.999)
# Only train decoder vars, encoder is frozen
d_vars = [var for var in tf.trainable_variables() if 'decoder_'+relu_target in var.name]
train_op = d_optimizer.minimize(total_loss, var_list=d_vars, global_step=global_step)
### Loss & image summaries
with tf.name_scope('summary_'+relu_target):
feature_loss_summary = tf.summary.scalar('feature_loss', feature_loss)
pixel_loss_summary = tf.summary.scalar('pixel_loss', pixel_loss)
tv_loss_summary = tf.summary.scalar('tv_loss', tv_loss)
total_loss_summary = tf.summary.scalar('total_loss', total_loss)
content_imgs_summary = tf.summary.image('content_imgs', content_imgs)
decoded_images_summary = tf.summary.image('decoded_images', clip(decoded))
for var in d_vars:
tf.summary.histogram(var.op.name, var)
summary_op = tf.summary.merge_all()
else:
# For inference set unnneeded ops to None
pixel_loss, feature_loss, tv_loss, total_loss, train_op, global_step, learning_rate, summary_op = [None]*8
# Put it all together
encoder_decoder = EncoderDecoder(content_input=content_imgs,
content_encoder_model=content_encoder_model,
content_encoded=content_encoded,
style_encoded=style_encoded_tensor,
decoder_input=decoder_input,
decoder_model=decoder_model,
decoded=decoded,
decoded_encoded=decoded_encoded,
pixel_loss=pixel_loss,
feature_loss=feature_loss,
tv_loss=tv_loss,
total_loss=total_loss,
train_op=train_op,
global_step=global_step,
learning_rate=learning_rate,
summary_op=summary_op)
return encoder_decoder
def one_step(self, current_state, previous_kernel_results):
"""Takes one step of the TransitionKernel.
Args:
current_state: `Tensor` or Python `list` of `Tensor`s representing the
current state(s) of the Markov chain(s).
previous_kernel_results: A (possibly nested) `tuple`, `namedtuple` or
`list` of `Tensor`s representing internal calculations made within the
previous call to this function (or as returned by `bootstrap_results`).
Returns:
next_state: `Tensor` or Python `list` of `Tensor`s representing the
next state(s) of the Markov chain(s).
kernel_results: A (possibly nested) `tuple`, `namedtuple` or `list` of
`Tensor`s representing internal calculations made within this function.
This inculdes replica states.
"""
with tf.name_scope(
name=mcmc_util.make_name(self.name, 'remc', 'one_step'),
values=[current_state, previous_kernel_results]):
sampled_replica_states, sampled_replica_results = zip(*[
rk.one_step(previous_kernel_results.replica_states[i],
previous_kernel_results.replica_results[i])
for i, rk in enumerate(self.replica_kernels)])
sampled_replica_states, sampled_replica_results = \
list(sampled_replica_states), list(sampled_replica_results)
sampled_replica_results_modified = \
[srr._replace(target_log_prob=srr.target_log_prob /
self.inverse_temperatures[i]) if 'target_log_prob' in srr._fields
else srr._replace(accepted_results=srr.accepted_results._replace(
target_log_prob=srr.accepted_results.target_log_prob /
self.inverse_temperatures[i]
))
for i, srr in enumerate(sampled_replica_results)]
sampled_replica_ratios = \
[srr.target_log_prob if 'target_log_prob' in srr._fields
else srr.accepted_results.target_log_prob
for i, srr in enumerate(sampled_replica_results_modified)]
sampled_replica_ratios = \
tf.stack(sampled_replica_ratios, axis=-1)
next_replica_idx = tf.range(self.num_replica)
self._seed_stream = distributions_util.gen_new_seed(
self._seed_stream, salt='replica_exchange_one_step')
exchange_proposed, exchange_proposed_n = \
self.exchange_proposed_fn(self.num_replica, seed=self._seed_stream)
i = tf.constant(0)
def cond(i, next_replica_idx):
return tf.less(i, exchange_proposed_n)
def body(i, next_replica_idx):
ratio = \
sampled_replica_ratios[next_replica_idx[exchange_proposed[i, 0]]] \
- sampled_replica_ratios[next_replica_idx[exchange_proposed[i, 1]]]
ratio *= self.inverse_temperatures[exchange_proposed[i, 1]] \
- self.inverse_temperatures[exchange_proposed[i, 0]]
self._seed_stream = distributions_util.gen_new_seed(
self._seed_stream, salt='replica_exchange_one_step')
log_uniform = tf.log(tf.random_uniform(
shape=tf.shape(ratio),
dtype=ratio.dtype.base_dtype,
seed=self._seed_stream))
exchange = log_uniform < ratio
exchange_op = tf.sparse_to_dense(
[exchange_proposed[i, 0], exchange_proposed[i, 1]],
[self.num_replica],
[next_replica_idx[exchange_proposed[i, 1]] -
next_replica_idx[exchange_proposed[i, 0]],
next_replica_idx[exchange_proposed[i, 0]] -
next_replica_idx[exchange_proposed[i, 1]]])
next_replica_idx = tf.cond(exchange,
lambda: next_replica_idx + exchange_op,
lambda: next_replica_idx)
return [i + 1, next_replica_idx]
next_replica_idx = tf.while_loop(
cond, body, loop_vars=[i, next_replica_idx])[1]
next_replica_states, next_replica_results = zip(*[
(tf.case({tf.equal(next_replica_idx[i], j):
_stateful_lambda(sampled_replica_states[j])
for j in range(self.num_replica)}, exclusive=True),
tf.case({tf.equal(next_replica_idx[i], j):
_stateful_lambda(sampled_replica_results_modified[j])
for j in range(self.num_replica)}, exclusive=True))
for i in range(self.num_replica)])
next_replica_states, next_replica_results = \
list(next_replica_states), list(next_replica_results)
next_replica_results = \
[nrr._replace(target_log_prob=nrr.target_log_prob *
self.inverse_temperatures[i]) if 'target_log_prob' in nrr._fields
else nrr._replace(accepted_results=nrr.accepted_results._replace(
target_log_prob=nrr.accepted_results.target_log_prob *
self.inverse_temperatures[i]
))
for i, nrr in enumerate(next_replica_results)]
next_state = tf.identity(next_replica_states[0])
kernel_results = ReplicaExchangeMCKernelResults(
replica_states=next_replica_states,
replica_results=next_replica_results,
next_replica_idx=next_replica_idx,
exchange_proposed=exchange_proposed,
exchange_proposed_n=exchange_proposed_n,
sampled_replica_states=sampled_replica_states,
sampled_replica_results=sampled_replica_results,
)
return next_state, kernel_results
