def slice_constant(data, batch_size=32, name='constant_data', global_step=None):
"""Provide a slice based on the global_step.
This is useful when the entire data array can be stored in memory because it
allows you to feed the data very efficiently.
Args:
data: A numpy array or tensor.
batch_size: The batch size for the produced data.
name: An optional name for this data.
global_step: A global step variable that is used to read the data. If None
then the default prettytensor global_step is used.
Returns:
A tensor that produces the given data.
"""
with tf.name_scope(name):
all_data = tf.convert_to_tensor(data)
global_step = global_step or bookkeeper.global_step()
count = len(data) / batch_size
extra = len(data) - count * batch_size
if extra:
offset = tf.mod(global_step, count)
return tf.slice(all_data, offset * batch_size, batch_size)
else:
offset = tf.mod(global_step, count + 1)
return tf.slice(all_data, offset * batch_size,
tf.where(tf.equal(offset, count), extra, batch_size))
def training_control(global_step, print_span, evaluation_span, max_step, name=None):
with tf.name_scope(name, "training_control"):
return {
"step": global_step,
"time_to_print": tf.equal(tf.mod(global_step, print_span), 0),
"time_to_evaluate": tf.equal(tf.mod(global_step, evaluation_span), 0),
"time_to_stop": tf.greater_equal(global_step, max_step),
}
def unwrap(p, discont=np.pi, axis=-1):
"""Unwrap a cyclical phase tensor.
Args:
p: Phase tensor.
discont: Float, size of the cyclic discontinuity.
axis: Axis of which to unwrap.
Returns:
unwrapped: Unwrapped tensor of same size as input.
"""
dd = diff(p, axis=axis)
ddmod = tf.mod(dd + np.pi, 2.0 * np.pi) - np.pi
idx = tf.logical_and(tf.equal(ddmod, -np.pi), tf.greater(dd, 0))
ddmod = tf.where(idx, tf.ones_like(ddmod) * np.pi, ddmod)
ph_correct = ddmod - dd
idx = tf.less(tf.abs(dd), discont)
ddmod = tf.where(idx, tf.zeros_like(ddmod), dd)
ph_cumsum = tf.cumsum(ph_correct, axis=axis)
shape = p.get_shape().as_list()
shape[axis] = 1
ph_cumsum = tf.concat([tf.zeros(shape, dtype=p.dtype), ph_cumsum], axis=axis)
unwrapped = p + ph_cumsum
return unwrapped
def py(model, config, scope, connect = None):
with tf.variable_scope(scope), tf.name_scope(scope):
with tf.variable_scope('inputs'), tf.name_scope('inputs'):
if connect is None:
model['%s_in0length' %scope] = config.getint('global', 'batch_size')
model['%s_in1length' %scope] = config.getint('global', 'input_size')
model['%s_in2length' %scope] = tf.placeholder(tf.int32, [model['%s_in0length' %scope]], '%s_in2length' %scope)
model['%s_maxin2length' %scope] = config.getint('global', 'time_size')
model['%s_inputs' %scope] = tf.placeholder(tf.float32, [model['%s_maxin2length' %scope], model['%s_in0length' %scope], model['%s_in1length' %scope]], '%s_inputs' %scope)
else:
model['%s_in0length' %scope] = model['%s_out0length' %connect]
model['%s_in1length' %scope] = model['%s_out1length' %connect]
model['%s_in2length' %scope] = model['%s_out2length' %connect]
model['%s_maxin2length' %scope] = model['%s_maxout2length' %connect]
model['%s_inputs' %scope] = model['%s_outputs' %connect]
model['%s_factor' %scope] = config.getint(scope, 'factor')
model['%s_out0length' %scope] = model['%s_in0length' %scope]
model['%s_out1length' %scope] = model['%s_in1length' %scope] * model['%s_factor' %scope]
model['%s_out2length' %scope] = tf.div(tf.subtract(model['%s_in2length' %scope], tf.mod(model['%s_in2length' %scope], model['%s_factor' %scope])), model['%s_factor' %scope])
model['%s_maxout2length' %scope] = (model['%s_maxin2length' %scope] - model['%s_maxin2length' %scope] % model['%s_factor' %scope]) / model['%s_factor' %scope]
with tf.variable_scope('outputs'), tf.name_scope('outputs'):
model['%s_transpose' %scope] = tf.transpose(model['%s_inputs' %scope], [0, 2, 1], '%s_transpose' %scope)
model['%s_transform' %scope] = tf.reshape(model['%s_transpose' %scope], [model['%s_maxout2length' %scope], model['%s_out1length' %scope], model['%s_out0length' %scope]], '%s_transform' %scope)
model['%s_outputs' %scope] = tf.transpose(model['%s_transform' %scope], [0, 2, 1], '%s_outputs' %scope)
return model
def sample_from_discretized_mix_logistic(y, log_scale_min=-7.): ''' Args: y: Tensor, [batch_size, channels, time_length] Returns: Tensor: sample in range of [-1, 1] ''' with tf.control_dependencies([tf.assert_equal(tf.mod(tf.shape(y)[1], 3), 0)]): nr_mix = tf.shape(y)[1] // 3 #[batch_size, time_length, channels] y = tf.transpose(y, [0, 2, 1]) logit_probs = y[:, :, :nr_mix] #sample mixture indicator from softmax temp = tf.random_uniform(tf.shape(logit_probs), minval=1e-5, maxval=1. - 1e-5) temp = logit_probs - tf.log(-tf.log(temp)) argmax = tf.argmax(temp, -1) #[batch_size, time_length] -> [batch_size, time_length, nr_mix] one_hot = tf.one_hot(argmax, depth=nr_mix, dtype=tf.float32) #select logistic parameters means = tf.reduce_sum(y[:, :, nr_mix:2 * nr_mix] * one_hot, axis=-1) log_scales = tf.maximum(tf.reduce_sum( y[:, :, 2 * nr_mix:3 * nr_mix] * one_hot, axis=-1), log_scale_min) #sample from logistic & clip to interval #we don't actually round to the nearest 8-bit value when sampling u = tf.random_uniform(tf.shape(means), minval=1e-5, maxval=1. - 1e-5) x = means + tf.exp(log_scales) * (tf.log(u) - tf.log(1 -u)) return tf.minimum(tf.maximum(x, -1.), 1.)
def minimize_loss_single_machine(loss,
accuracy,
layer_collection,
device="/gpu:0",
session_config=None):
"""Minimize loss with K-FAC on a single machine.
A single Session is responsible for running all of K-FAC's ops. The covariance
and inverse update ops are placed on `device`. All model variables are on CPU.
Args:
loss: 0-D Tensor. Loss to be minimized.
accuracy: 0-D Tensor. Accuracy of classifier on current minibatch.
layer_collection: LayerCollection instance describing model architecture.
Used by K-FAC to construct preconditioner.
device: string, Either '/cpu:0' or '/gpu:0'. The covaraince and invserse
update ops are run on this device.
session_config: None or tf.ConfigProto. Configuration for tf.Session().
Returns:
final value for 'accuracy'.
"""
# Train with K-FAC.
g_step = tf.train.get_or_create_global_step()
optimizer = opt.KfacOptimizer(
learning_rate=0.0001,
cov_ema_decay=0.95,
damping=0.001,
layer_collection=layer_collection,
placement_strategy="round_robin",
cov_devices=[device],
inv_devices=[device],
momentum=0.9)
(cov_update_thunks,
inv_update_thunks) = optimizer.make_vars_and_create_op_thunks()
def make_update_op(update_thunks):
update_ops = [thunk() for thunk in update_thunks]
return tf.group(*update_ops)
cov_update_op = make_update_op(cov_update_thunks)
with tf.control_dependencies([cov_update_op]):
inverse_op = tf.cond(
tf.equal(tf.mod(g_step, _INVERT_EVERY), 0),
lambda: make_update_op(inv_update_thunks), tf.no_op)
with tf.control_dependencies([inverse_op]):
with tf.device(device):
train_op = optimizer.minimize(loss, global_step=g_step)
tf.logging.info("Starting training.")
with tf.train.MonitoredTrainingSession(config=session_config) as sess:
while not sess.should_stop():
global_step_, loss_, accuracy_, _ = sess.run(
[g_step, loss, accuracy, train_op])
if global_step_ % _INVERT_EVERY == 0:
tf.logging.info("global_step: %d | loss: %f | accuracy: %s",
global_step_, loss_, accuracy_)
return accuracy_
def roll_sequence(tensor, offsets):
"""Shifts sequences by an offset.
Args:
tensor: A ``tf.Tensor`` of shape ``[batch_size, time, ...]``.
offsets : The offset of each sequence.
Returns:
A ``tf.Tensor`` of the same shape as :obj:`tensor` with sequences shifted
by :obj:`offsets`.
"""
batch_size = tf.shape(tensor)[0]
time = tf.shape(tensor)[1]
cols = tf.range(time)
cols = tf.tile(cols, [batch_size])
cols = tf.reshape(cols, [batch_size, time])
cols -= tf.expand_dims(offsets, 1)
cols = tf.mod(cols, time)
rows = tf.range(batch_size)
rows = tf.tile(rows, [time])
rows = tf.reshape(rows, [time, batch_size])
rows = tf.transpose(rows, perm=[1, 0])
indices = tf.concat([tf.expand_dims(rows, -1), tf.expand_dims(cols, -1)], -1)
return tf.gather_nd(tensor, indices)
def _extract_feature(inputs, idxs):
idxs = tf.expand_dims(idxs,1)
idx_i = tf.floordiv(idxs, map_h)
idx_j = tf.mod(idxs, map_h)
# NOTE:
# calculate the center of input batches
# this depends on coarse layer's architecture
origin_i = 2*(2*idx_i+1)+3
origin_j = 2*(2*idx_j+1)+3
origin_centers = tf.concat(1,[origin_i,origin_j])
origin_centers = tf.to_float(origin_centers)
# NOTE: size also depends on the architecture
patches = tf.image.extract_glimpse(inputs, size=[14,14], offsets=origin_centers,
centered=False, normalized=False)
fine_features = fine_layers(patches)
# reuse variables
tf.get_variable_scope().reuse_variables()
src_idxs = tf.concat(1,[idx_i,idx_j])
return fine_features, src_idxs
def weights_concatenated(labels):
"""Assign weight 1.0 to the "target" part of the concatenated labels.
The labels look like:
source English I love you . ID1 target French Je t'aime . ID1 source
English the cat ID1 target French le chat ID1 source English ...
We want to assign weight 1.0 to all words in the target text (including the
ID1 end symbol), but not to the source text or the boilerplate. In the
above example, the target words that get positive weight are:
Je t'aime . ID1 le chat ID1
Args:
labels: a Tensor
Returns:
a Tensor
"""
eos_mask = tf.to_int32(tf.equal(labels, 1))
sentence_num = tf.cumsum(eos_mask, axis=1, exclusive=True)
in_target = tf.equal(tf.mod(sentence_num, 2), 1)
# first two tokens of each sentence are boilerplate.
sentence_num_plus_one = sentence_num + 1
shifted = tf.pad(sentence_num_plus_one, [[0, 0], [2, 0], [0, 0],
[0, 0]])[:, :-2, :, :]
nonboilerplate = tf.equal(sentence_num_plus_one, shifted)
ret = tf.to_float(tf.logical_and(nonboilerplate, in_target))
return ret
def __init__(self,
q_t,
q_tp1,
q_tp0,
importance_weights,
rewards,
done_mask,
twin_q_t,
twin_q_tp1,
actor_loss_coeff=0.1,
critic_loss_coeff=1.0,
gamma=0.99,
n_step=1,
use_huber=False,
huber_threshold=1.0,
twin_q=False,
policy_delay=1):
q_t_selected = tf.squeeze(q_t, axis=len(q_t.shape) - 1)
if twin_q:
twin_q_t_selected = tf.squeeze(twin_q_t, axis=len(q_t.shape) - 1)
q_tp1 = tf.minimum(q_tp1, twin_q_tp1)
q_tp1_best = tf.squeeze(input=q_tp1, axis=len(q_tp1.shape) - 1)
q_tp1_best_masked = (1.0 - done_mask) * q_tp1_best
# compute RHS of bellman equation
q_t_selected_target = rewards + gamma**n_step * q_tp1_best_masked
# compute the error (potentially clipped)
if twin_q:
td_error = q_t_selected - tf.stop_gradient(q_t_selected_target)
twin_td_error = twin_q_t_selected - tf.stop_gradient(
q_t_selected_target)
self.td_error = td_error + twin_td_error
if use_huber:
errors = _huber_loss(td_error, huber_threshold) + _huber_loss(
twin_td_error, huber_threshold)
else:
errors = 0.5 * tf.square(td_error) + 0.5 * tf.square(
twin_td_error)
else:
self.td_error = (
q_t_selected - tf.stop_gradient(q_t_selected_target))
if use_huber:
errors = _huber_loss(self.td_error, huber_threshold)
else:
errors = 0.5 * tf.square(self.td_error)
self.critic_loss = critic_loss_coeff * tf.reduce_mean(
importance_weights * errors)
# for policy gradient, update policy net one time v.s.
# update critic net `policy_delay` time(s)
global_step = tf.train.get_or_create_global_step()
policy_delay_mask = tf.to_float(
tf.equal(tf.mod(global_step, policy_delay), 0))
self.actor_loss = (-1.0 * actor_loss_coeff * policy_delay_mask *
tf.reduce_mean(q_tp0))
def manual_update_GDL(arg,learning_rate,g,mu_noise,stddev_noise):
sess = arg.sess
with tf.variable_scope(arg.mdl_scope_name,reuse=True):
W_var = tf.get_variable(name='W')
eps = tf.random_normal(tf.shape(g),mean=mu_noise,stddev=stddev_noise)
#
W_new = tf.mod( W_var - learning_rate*g + eps , 20)
sess.run( W_var.assign(W_new) )
def _add():
num_adds_inc = self._num_adds_cs.execute(_increment_num_adds)
current_pos = tf.mod(num_adds_inc - 1, self._buffer_size)
update_ops = []
for name in self._tensors.keys():
update_ops.append(
tf.scatter_update(self._tensors[name], current_pos, tensors[name]))
return tf.group(*update_ops)
def preprocess(audio, rate=48000): # pad with 1 second of silence on either side front = tf.zeros([rate,2], dtype=audio.dtype) back = tf.zeros([rate - tf.mod(tf.shape(audio)[0], rate) + rate, 2], dtype=audio.dtype) audio = tf.concat([front, audio, back], 0) audio = tf.add(audio, tf.abs(tf.reduce_min(audio))) audio = tf.multiply(audio, 1.0/ tf.reduce_max(audio)) # audio = tf.reshape(audio, [-1, int(rate * return audio
def dlstm_scan_fn(previous_output, current_input):
out, state_out = lstm(current_input, previous_output[1])
i = previous_output[2]
basis_i = tf.one_hot(i, depth=chunks)
state_out_dilated = dilate_one_time_step(tf.squeeze(state_out[0]), basis_i, chunks)
state_out = rnn.LSTMStateTuple(state_out_dilated, state_out[1])
i += tf.constant(1)
new_i = tf.mod(i, chunks)
return out, state_out, new_i
def discretized_mix_logistic_loss(y_hat, y, num_classes=256, log_scale_min=-7.0, reduce=True): '''Discretized mix of logistic distributions loss. Note that it is assumed that input is scaled to [-1, 1] Args: y_hat: Tensor [batch_size, channels, time_length], predicted output. y: Tensor [batch_size, time_length, 1], Target. Returns: Tensor loss ''' with tf.control_dependencies([tf.assert_equal(tf.mod(tf.shape(y_hat)[1], 3), 0), tf.assert_equal(tf.rank(y_hat), 3)]): nr_mix = tf.shape(y_hat)[1] // 3 #[Batch_size, time_length, channels] y_hat = tf.transpose(y_hat, [0, 2, 1]) #unpack parameters. [batch_size, time_length, num_mixtures] x 3 logit_probs = y_hat[:, :, :nr_mix] means = y_hat[:, :, nr_mix:2 * nr_mix] log_scales = tf.maximum(y_hat[:, :, 2* nr_mix: 3 * nr_mix], log_scale_min) #[batch_size, time_length, 1] -> [batch_size, time_length, num_mixtures] y = y * tf.ones(shape=[1, 1, nr_mix], dtype=tf.float32) centered_y = y - means inv_stdv = tf.exp(-log_scales) plus_in = inv_stdv * (centered_y + 1. / (num_classes - 1)) cdf_plus = tf.nn.sigmoid(plus_in) min_in = inv_stdv * (centered_y - 1. / (num_classes - 1)) cdf_min = tf.nn.sigmoid(min_in) log_cdf_plus = plus_in - tf.nn.softplus(plus_in) # log probability for edge case of 0 (before scaling) log_one_minus_cdf_min = -tf.nn.softplus(min_in) # log probability for edge case of 255 (before scaling) #probability for all other cases cdf_delta = cdf_plus - cdf_min mid_in = inv_stdv * centered_y #log probability in the center of the bin, to be used in extreme cases #(not actually used in this code) log_pdf_mid = mid_in - log_scales - 2. * tf.nn.softplus(mid_in) log_probs = tf.where(y < -0.999, log_cdf_plus, tf.where(y > 0.999, log_one_minus_cdf_min, tf.where(cdf_delta > 1e-5, tf.log(tf.maximum(cdf_delta, 1e-12)), log_pdf_mid - np.log((num_classes - 1) / 2)))) #log_probs = log_probs + tf.nn.log_softmax(logit_probs, -1) log_probs = log_probs + log_prob_from_logits(logit_probs) if reduce: return -tf.reduce_sum(log_sum_exp(log_probs)) else: return -tf.expand_dims(log_sum_exp(log_probs), [-1])
def get_bmu_loc(self, x):
expanded_x = tf.expand_dims(x, 0)
sqr_diff = tf.square(tf.subtract(expanded_x, self.nodes))
dists = tf.reduce_sum(sqr_diff, 1)
bmu_idx = tf.argmin(dists, 0)
bmu_loc = tf.pack([tf.mod(bmu_idx, self.width), tf.div(bmu_idx, self.width)])
return bmu_loc
def _round_up_tf(x, multiple): # Tf version of remainder = x % multiple remainder = tf.mod(x, multiple) # Tf version of return x if remainder == 0 else x + multiple - remainder x_round = tf.cond(tf.equal(remainder, tf.zeros(tf.shape(remainder), dtype=tf.int32)), lambda: x, lambda: x + multiple - remainder) return x_round
def top_k_in_2dim_tensor(ts, k): shape = tf.shape(ts) v, id = tf.nn.top_k(tf.reshape(ts, [-1]), k) id_row_col = tf.transpose(tf.reshape( tf.concat(0, [tf.div(id, shape[1]),tf.mod(id, shape[1])]), shape=tf.concat(0, [[2], tf.reshape(k, [1])]) )) return v, id_row_col
def get_timing_signal_1d(self, length, channels):
position = tf.to_float(tf.range(length))
num_timescales = channels // 2
log_timescale_increment = (math.log(float(self.max_timescale) / float(self.min_timescale)) / (tf.to_float(num_timescales) - 1))
inv_timescales = self.min_timescale * tf.exp(tf.to_float(tf.range(num_timescales)) * -log_timescale_increment)
scaled_time = tf.expand_dims(position, 1) * tf.expand_dims(inv_timescales, 0)
signal = tf.concat([tf.sin(scaled_time), tf.cos(scaled_time)], axis=1)
signal = tf.pad(signal, [[0, 0], [0, tf.mod(channels, 2)]])
signal = tf.reshape(signal, [1, length, channels])
return signal
def get_position(self):
"""Returns the position at which the last element was added.
Returns:
An int tensor representing the index at which the last element was added
to the buffer or -1 if no elements were added.
"""
return tf.cond(self.get_num_adds() < 1,
lambda: self.get_num_adds() - 1,
lambda: tf.mod(self.get_num_adds() - 1, self._buffer_size))
def arg_max_2d(x_in):
orig_shape = tf.shape(x_in)
reshape_t = tf.concat([orig_shape[0:1], [-1], orig_shape[3:4]], 0)
zz = tf.reshape(x_in, reshape_t)
pp = tf.to_int32(tf.argmax(zz, 1))
sz1 = tf.slice(orig_shape, [2], [1])
cc1 = tf.div(pp, tf.to_int32(sz1))
cc2 = tf.mod(pp, tf.to_int32(sz1))
return tf.stack([cc1, cc2])
def get_maximum_index(response): """Get the index of the maximum value in the response map""" response_shape = response.get_shape().as_list() response_spatial_size = response_shape[-2:] # e.g. [29, 29] length = response_spatial_size[0] * response_spatial_size[1] # Get maximum response index (note index starts from zero) ind_max = tf.argmax(tf.reshape(response, [-1, length]), 1) ind_row = tf.div(ind_max, response_spatial_size[1]) ind_col = tf.mod(ind_max, response_spatial_size[1]) return ind_row, ind_col
def compare_sample_to_flat(sample, flat):
a0_flat = flat[..., 0]
phi_flat = flat[..., 1]
a1_flat = flat[..., 2]
result0 = sample[..., 0] / flat[..., 0]
result1 = tf.mod(
sample[..., 1] - flat[..., 1] + np.pi,
2 * np.pi
) - np.pi
result2 = sample[..., 2] / flat[..., 2] / result0
return tf.stack([result0, result1, result2], axis=-1)
def argmax2d(Xin):
origShape = tf.shape(Xin)
reshape_t = tf.concat(0,[origShape[0:1],[-1],origShape[3:4]])
zz = tf.reshape(Xin,reshape_t)
pp = tf.to_int32(tf.argmax(zz,1))
sz1 = tf.slice(origShape,[2],[1])
cc1 = tf.div(pp,tf.to_int32(sz1))
cc2 = tf.mod(pp,tf.to_int32(sz1))
return tf.pack([cc1,cc2])
def dlstm_scan_fn(previous_output, current_input):
# out, state_out = lstm(current_input, previous_output[1])
state_step = previous_output[2]
sub_state = get_sub_state(previous_output[1], state_step)
out, sub_state_out = lstm(current_input, sub_state)
state_out = build_new_state(sub_state_out, previous_output[1], state_step)
state_step += tf.constant(1)
new_state_step = tf.mod(state_step, chunks)
return out, state_out, new_state_step
def noise_from_step_num():
"""Quantization noise equal to (phi * (step_num + 1)) mod 1.0.
Not using random_uniform here due to a problem on TPU in that random seeds
are not respected, which may cause the parameters on different replicas
to go out-of-sync.
Returns:
a float32 scalar
"""
step = tf.to_int32(tf.train.get_or_create_global_step()) + 1
phi = ((5 ** 0.5) - 1) / 2
# Naive computation tf.mod(phi * step, 1.0) in float32 would be disastrous
# due to loss of precision when the step number gets large.
# Computation in doubles does not work on TPU, so we use this complicated
# alternative computation which does not suffer from these roundoff errors.
ret = 0.0
for i in range(30):
ret += (((phi * (2 ** i)) % 1.0) # double-precision computation in python
* tf.to_float(tf.mod(step // (2 ** i), 2)))
return tf.mod(ret, 1.0)
def _tile_encoders_for_beamsearch(self, projected_sentinel):
sentinel_batch_size = tf.shape(projected_sentinel)[0]
encoders_batch_size = tf.shape(
self.encoder_projections_for_ctx[0])[0]
modulo = tf.mod(sentinel_batch_size, encoders_batch_size)
with tf.control_dependencies([tf.assert_equal(modulo, 0)]):
beam_size = tf.div(sentinel_batch_size,
encoders_batch_size)
return [tf.tile(proj, [beam_size, 1, 1])
for proj in self.encoder_projections_for_ctx]
def convert_to_reality(bbox, width, height, S):
relative_center_x, relative_center_y, global_w, global_h = bbox
w = tf.cast(tf.cast(tf.mul(global_w, width), tf.int32), tf.float32)
h = tf.cast(tf.cast(tf.mul(global_h, height), tf.int32), tf.float32)
index = tf.reshape(tf.range(S * S),[-1,1])
cell_coord_y = tf.cast(tf.div(index, S), tf.float32)
cell_coord_x = tf.cast(tf.mod(index, S), tf.float32)
S = tf.cast(S, tf.float32)
width = tf.cast(width, tf.float32)
height = tf.cast(height, tf.float32)
cell_w = tf.cast(width / S, tf.float32)
cell_h = tf.cast(height / S, tf.float32)
#real_x_left_up = tf.reshape((cell_coord_x + relative_center_x) * cell_w - w / 2,[-1])
#real_y_left_up = tf.reshape((cell_coord_y + relative_center_y) * cell_h - h / 2, [-1])
real_x_left_up = tf.sub(tf.add(tf.reshape(tf.mul(cell_coord_x, cell_w), [-1]), relative_center_x * cell_w), tf.cast(w * 0.5, tf.float32))
real_y_left_up = tf.sub(tf.add(tf.reshape(tf.mul(cell_coord_y, cell_h), [-1]), relative_center_y * cell_h), tf.cast(h * 0.5, tf.float32))
real_x_left_up = tf.cast(tf.nn.relu(real_x_left_up), tf.int32)
real_y_left_up = tf.cast(tf.nn.relu(real_y_left_up), tf.int32)
w = tf.cast(w, tf.int32)
h = tf.cast(h, tf.int32)
print 'real x ', relative_center_x.get_shape()
print 'real w' , w.get_shape()
"""
assert real_x_left_up.dtype == tf.int32 and \
real_y_left_up.dtype == tf.int32 and \
w.dtype == tf.int32 and \
h.dtype == tf.int32
"""
bbox = [real_x_left_up, real_y_left_up, w, h]
return bbox
def _next(self, i):
"""Return op that increments pointer to next value.
Args:
i: A tensorflow integer variable.
Returns:
Op that increments pointer.
"""
if self._scheduler == 'cycle':
inc = ('inc' in self._scheduler_params and
self._scheduler_params['inc']) or 1
return tf.assign(i, tf.mod(i+inc, self._n))
else:
raise NotImplementedError(self._scheduler)
def periodic_context_fn(contexts, timer, sampler_fn, period=1):
"""Periodically samples contexts.
Args:
contexts: a list of [num_context_dims] tensor variables representing
current contexts.
timer: a scalar integer tensor variable holding the current time step.
sampler_fn: a sampler function that samples a list of [num_context_dims]
tensors.
period: (integer) period of update.
Returns:
a list of [num_context_dims] tensors.
"""
contexts = list(contexts[:]) # create copy
return tf.cond(tf.mod(timer, period) == 0, sampler_fn, lambda: contexts)
import tensorflow as tf
node2 = tf.constant(2.0)
node3 = tf.constant(3.0)
node4 = tf.constant(4.0)
node5 = tf.constant(5.0)
# 3 + 4 + 5
add = tf.add_n([node3, node4, node5]) # 많은 양의 tesor한번에 처리
# 4 - 3
sub = tf.subtract(node4, node3)
# 3 * 4
mul = tf.multiply(node3, node4)
# mul2 = tf.matmul() # 행렬의 곱셈
# 4 / 2
div = tf.div(node4, node2)
mod = tf.mod(node4, node2) # 나눈 나머지 값
sess = tf.Session()
print('3 + 4 + 5 =', sess.run(add)) # 3 + 4 + 5 = 12.0
print('4 - 3 =', sess.run(sub)) # 4 - 3 = 1.0
print('3 * 4 =', sess.run(mul)) # 3 * 4 = 12.0
print('4 / 2 =', sess.run(div)) # 4 / 2 = 2.0
print('4 % 2 =', sess.run(mod)) # 4 % 2 = 0.0
print(sess.run(node3 + node4))
def filter_fn(elem_index, _):
mod_result = tf.mod(elem_index, num_shards)
return tf.equal(mod_result, shard_id)
def bottom(self, x):
"""Use batchnorm instead of CMVN and shorten the stft with strided convs.
Args:
x: float32 tensor with shape [batch_size, len, 1, freqs * channels]
Returns:
float32 tensor with shape [batch_size, shorter_len, 1, hidden_size]
"""
inputs = x
p = self._model_hparams
num_mel_bins = p.audio_num_mel_bins
num_channels = 3 if p.audio_add_delta_deltas else 1
with tf.variable_scope(self.name):
if p.audio_preproc_in_bottom:
# Compute filterbanks
with tf.variable_scope("fbanks"):
waveforms = tf.squeeze(inputs, [2, 3])
mel_fbanks = common_audio.compute_mel_filterbank_features(
waveforms,
sample_rate=p.audio_sample_rate,
dither=p.audio_dither,
preemphasis=p.audio_preemphasis,
frame_length=p.audio_frame_length,
frame_step=p.audio_frame_step,
lower_edge_hertz=p.audio_lower_edge_hertz,
upper_edge_hertz=p.audio_upper_edge_hertz,
num_mel_bins=p.audio_num_mel_bins,
apply_mask=True)
if p.audio_add_delta_deltas:
mel_fbanks = common_audio.add_delta_deltas(mel_fbanks)
# frame concatenation
if p.concat_frame:
fbank_size = common_layers.shape_list(mel_fbanks)
x = tf.pad(
mel_fbanks,
[[0, 0], [
0, tf.mod(-fbank_size[1], p.concat_frame)
], [0, 0], [0, 0]])
x = tf.reshape(x, [
fbank_size[0],
common_layers.shape_list(x)[1] / p.concat_frame,
num_mel_bins * p.concat_frame, num_channels
])
else:
x = tf.reshape(
mel_fbanks,
common_layers.shape_list(mel_fbanks)[:2] +
[num_mel_bins, num_channels])
nonpadding_mask = 1. - common_attention.embedding_to_padding(
x)
if p.concat_frame:
num_of_nonpadding_elements = tf.reduce_sum(
nonpadding_mask
) * num_mel_bins * num_channels / p.concat_frame
else:
num_of_nonpadding_elements = tf.reduce_sum(
nonpadding_mask) * num_mel_bins * num_channels
# This replaces CMVN estimation on data
mean = tf.reduce_sum(x, axis=[
1
], keepdims=True) / num_of_nonpadding_elements
variance = (
num_of_nonpadding_elements * mean**2. -
2. * mean * tf.reduce_sum(x, axis=[1], keepdims=True) +
tf.reduce_sum(x**2, axis=[1], keepdims=True)
) / num_of_nonpadding_elements
x = (x - mean) / variance * tf.expand_dims(
nonpadding_mask, -1)
else:
x = inputs
# The convention is that the models are flattened along the spatial,
# dimensions, thus the speech preprocessor treats frequencies and
# channels as image colors (last axis)
if p.concat_frame:
x.set_shape(
[None, None, num_mel_bins * p.concat_frame, num_channels])
else:
x.set_shape([None, None, num_mel_bins, num_channels])
# TODO(chorowski): how to specify bottom's hparams and avoid hardcoding?
x = tf.pad(x, [[0, 0], [0, 8], [0, 0], [0, 0]])
for _ in range(2):
x = tf.layers.conv2d(x, 128, (3, 3), (2, 2), use_bias=False)
x = common_layers.layer_norm(x)
x = tf.nn.relu(x)
xshape = common_layers.shape_list(x)
# apply a conv that will remove all frequencies and at the same time
# project the output into desired hidden_size
x = tf.pad(x, [[0, 0], [0, 2], [0, 0], [0, 0]])
x = tf.layers.conv2d(x,
p.hidden_size, (3, xshape[2]),
use_bias=False)
assert common_layers.shape_list(x)[2] == 1
x = common_layers.layer_norm(x)
x = tf.nn.relu(x)
return x
def _to_vocab_range(x):
"""Enforces that the vocab_ids in x are positive."""
return tf.SparseTensor(
indices=x.indices,
values=tf.mod(x.values, vocab_size),
dense_shape=x.dense_shape)
def detection_model(features, labels, mode, params):
num_classes = params['num_classes']
initial_weights_path = params.get('initial_weights_path', '')
log_dir = params['log_dir']
collect_priors_summary = params['collect_priors_summary']
data_format = params.get('data_format', 'NHWC')
depth_multiplier = params.get('depth_multiplier', 1.0)
priors_rule = params.get('priors_rule', 'caffe')
custom_priors = params.get('priors', [])
learning_rate = params.get('learning_rate', 0.01)
steps_per_epoch = params.get('steps_per_epoch', 1)
mobilenet_version = params.get('mobilenet_version', 'v2')
weight_regularization = params.get('weight_regularization', 4e-5)
optimizer_func = params.get(
'optimizer', lambda learning_rate: tf.train.AdagradOptimizer(
learning_rate=learning_rate))
# Override default FileWriter. Don't store the graph definition.
tf.summary.FileWriterCache._cache[log_dir] = tf.summary.FileWriter(
log_dir, graph=None)
if callable(learning_rate):
learning_rate = learning_rate()
is_training = True if mode == tf.estimator.ModeKeys.TRAIN else False
ssd = MobileNetSSD(
input_tensor=features,
num_classes=num_classes,
depth_multiplier=depth_multiplier,
is_training=is_training,
data_format=data_format,
priors_rule=priors_rule,
priors=custom_priors,
mobilenet_version=mobilenet_version,
weight_regularization=weight_regularization) # 1. Build model
if mode == tf.estimator.ModeKeys.PREDICT:
decoded_predictions = ssd.detection_output(
use_plain_caffe_format=False)
return tf.estimator.EstimatorSpec(mode,
predictions=decoded_predictions)
assert mode in (tf.estimator.ModeKeys.TRAIN, tf.estimator.ModeKeys.EVAL)
targets = ssd.create_targets(labels) # 2. Build GT from annotation
if collect_priors_summary:
with tf.name_scope('summary/'):
assigned_priors = create_tensors_and_streaming_ops_for_assigned_priors(
targets, ssd.priors_info, num_classes)
detailed_assigned_priors = get_detailed_assigned_priors_summary_tf(
assigned_priors, ssd.priors_info)
loss_func = MultiboxLoss(neg_pos_ratio=3.0) # 3. Build loss-object
eval_iteration = tf.get_variable('eval_iteration',
initializer=0,
dtype=tf.int32,
trainable=False)
if mode == tf.estimator.ModeKeys.EVAL:
eval_print_steps = steps_per_epoch // 50
eval_print_steps = 1 if eval_print_steps == 0 else eval_print_steps
every_eval_print_steps = tf.equal(
tf.mod(eval_iteration + 1, eval_print_steps), 0)
eval_iteration = tf.assign(eval_iteration, eval_iteration + 1)
targets = with_dependencies([eval_iteration], targets)
loss = loss_func.eval_summary(targets, ssd.predictions)
loss = tf.cond(
every_eval_print_steps, lambda: tf.Print(loss, [
tf.round(100 * eval_iteration / steps_per_epoch), loss
], '[%][loss]: '), lambda: loss)
eval_metric_ops = {}
for key, val in loss_func.eval_tensors.items():
eval_metric_ops['loss_function/' + key] = tf.metrics.mean(val)
if collect_priors_summary:
for key, metric_ops in assigned_priors.items(
): # We need only update ops
eval_metric_ops[key] = metric_ops
for key, assigned_priors_tensor in detailed_assigned_priors.items(
):
eval_metric_ops['prior_histogram/' +
key] = (assigned_priors_tensor, tf.no_op())
decoded_predictions = ssd.detection_output(
use_plain_caffe_format=False)
eval_metric_ops['predictions'] = tf.contrib.metrics.streaming_concat(
decoded_predictions)
return tf.estimator.EstimatorSpec(mode,
loss=loss,
eval_metric_ops=eval_metric_ops)
assert mode == tf.estimator.ModeKeys.TRAIN
if initial_weights_path:
ssd.load_weights(initial_weights_path)
bboxes = ssd._decode_boxes(ssd.predictions['locs'],
priors=ssd.priors[0, 0],
variance=ssd.priors[0, 1])
loss = loss_func.loss(targets, ssd.predictions,
bboxes) # 4. Compute loss with NMS
if collect_priors_summary:
with tf.name_scope('summary/'):
loss = with_dependencies(
[op for key, (tensor, op) in assigned_priors.items()], loss)
for name, assigned_priors_tensor in detailed_assigned_priors.items():
tf.summary.scalar(name, tf.reduce_sum(assigned_priors_tensor))
py_func_ops = []
priors_dir = os.path.join(log_dir, 'priors')
with tf.name_scope('write_histogram'):
every_epoch = tf.equal(
tf.mod(tf.train.get_global_step() + 1, steps_per_epoch), 0)
for name, (group, op) in assigned_priors.items():
def write_hist2d():
return tf.py_func(write_histogram_2d_tf, [
group,
pickle.dumps(ssd.priors_info), name,
tf.train.get_global_step(), priors_dir
], tf.bool)
write_hist2d_once_per_epoch = tf.cond(every_epoch,
write_hist2d, tf.no_op)
py_func_ops.append(write_hist2d_once_per_epoch)
loss = with_dependencies(py_func_ops, loss)
optimizer = optimizer_func(learning_rate)
tf.summary.scalar('learning_rate', learning_rate)
regularization_losses = tf.get_collection(
tf.GraphKeys.REGULARIZATION_LOSSES)
regularization_loss = tf.add_n(
regularization_losses, name='loss_function/regularization_losses_sum')
total_loss = tf.add(loss,
regularization_loss,
name='loss_function/total_loss')
tf.summary.scalar('loss_function/regularization_loss', regularization_loss)
with tf.variable_scope('train_loop'):
train_op = optimizer.minimize(total_loss,
global_step=tf.train.get_global_step())
return tf.estimator.EstimatorSpec(mode,
loss=total_loss,
train_op=train_op)
def get_unfactored_bilinear_classifier(self, layer, unlabeled_targets, token_weights, variable_scope=None, reuse=False):
""""""
recur_layer = layer
hidden_keep_prob = 1 if reuse else self.hidden_keep_prob
hidden_func = self.hidden_func
hidden_size = self.hidden_size
add_linear = self.add_linear
with tf.variable_scope(variable_scope or self.classname):
for i in six.moves.range(0, self.n_layers-1):
with tf.variable_scope('FC-%d' % i):
layer = classifiers.hidden(layer, 2*hidden_size,
hidden_func=hidden_func,
hidden_keep_prob=hidden_keep_prob)
with tf.variable_scope('FC-top'):
layers = classifiers.hiddens(layer, 2*[hidden_size],
hidden_func=hidden_func,
hidden_keep_prob=hidden_keep_prob)
layer1, layer2 = layers.pop(0), layers.pop(0)
with tf.variable_scope('Classifier'):
if self.diagonal:
logits = classifiers.diagonal_bilinear_classifier(
layer1, layer2, len(self),
hidden_keep_prob=hidden_keep_prob,
add_linear=add_linear)
else:
logits = classifiers.bilinear_classifier(
layer1, layer2, len(self),
hidden_keep_prob=hidden_keep_prob,
add_linear=add_linear)
bucket_size = tf.shape(layer)[-2]
#-------------------------------------------------------
# Process the targets
# c (*) (n x m) + (n x m)
#targets = len(self) * unlabeled_targets + self.placeholder
targets = bucket_size * self.placeholder + unlabeled_targets
#-------------------------------------------------------
# Process the logits
# (n x m x c x m) -> (n x m x cm)
reshaped_logits = tf.reshape(logits, tf.stack([-1, bucket_size, bucket_size * len(self)]))
#-------------------------------------------------------
# Compute probabilities/cross entropy
# (n x m x cm) -> (n x m x cm)
probabilities = tf.nn.softmax(reshaped_logits)
# (n x m x cm) -> (n x m x c x m)
probabilities = tf.reshape(probabilities, tf.stack([-1, bucket_size, len(self), bucket_size]))
# (n x m x c x m) -> (n x m x m x c)
probabilities = tf.transpose(probabilities, [0,1,3,2])
# (n x m), (n x m x cm), (n x m) -> ()
loss = tf.losses.sparse_softmax_cross_entropy(targets, reshaped_logits, weights=token_weights)
#-------------------------------------------------------
# Compute predictions/accuracy
# (n x m x cm) -> (n x m)
predictions = tf.argmax(reshaped_logits, axis=-1, output_type=tf.int32)
# (n x m), () -> (n x m)
unlabeled_predictions = tf.mod(predictions, bucket_size)
# (n x m) (*) (n x m) -> (n x m)
correct_tokens = nn.equal(predictions, targets) * token_weights
correct_unlabeled_tokens = nn.equal(unlabeled_predictions, unlabeled_targets) * token_weights
# (n x m) -> (n)
tokens_per_sequence = tf.reduce_sum(token_weights, axis=-1)
# (n x m) -> (n)
correct_tokens_per_sequence = tf.reduce_sum(correct_tokens, axis=-1)
correct_unlabeled_tokens_per_sequence = tf.reduce_sum(correct_unlabeled_tokens, axis=-1)
# (n), (n) -> (n)
correct_sequences = nn.equal(tokens_per_sequence, correct_tokens_per_sequence)
correct_unlabeled_sequences = nn.equal(tokens_per_sequence, correct_unlabeled_tokens_per_sequence)
#-----------------------------------------------------------
# Populate the output dictionary
outputs = {}
outputs['recur_layer'] = recur_layer
outputs['unlabeled_targets'] = unlabeled_targets
outputs['probabilities'] = probabilities
outputs['unlabeled_loss'] = tf.constant(0.)
outputs['loss'] = loss
outputs['unlabeled_predictions'] = unlabeled_predictions
outputs['label_predictions'] = predictions
outputs['n_correct_unlabeled_tokens'] = tf.reduce_sum(correct_unlabeled_tokens)
outputs['n_correct_unlabeled_sequences'] = tf.reduce_sum(correct_unlabeled_sequences)
outputs['n_correct_tokens'] = tf.reduce_sum(correct_tokens)
outputs['n_correct_sequences'] = tf.reduce_sum(correct_sequences)
return outputs
def pbc_tf(tup, shape):
"""Tensorflow implementation of `pbc` defined above."""
return list(tf.mod(tup, shape))
def InputBatch(self):
np.random.seed(1)
bs, sl = 10, 7
src_ids = tf.constant(
np.random.randint(low=0,
high=8192 - 1,
size=[bs, sl],
dtype=np.int32))
tgt_ids = tf.constant(
np.random.randint(low=0,
high=8192 - 1,
size=[bs, sl],
dtype=np.int32))
tgt_labels = tf.constant(
np.random.randint(low=0,
high=8192 - 1,
size=[bs, sl],
dtype=np.int32))
tgt_weights = tf.constant(np.ones(shape=[bs, sl], dtype=np.float32))
src_paddings = tf.zeros([bs, sl])
tgt_paddings = tf.zeros([bs, sl])
ret = py_utils.NestedMap()
ret.src = py_utils.NestedMap()
ret.tgt = py_utils.NestedMap()
if self.params.split:
src_ids = tf.split(src_ids, 2, 0)
src_paddings = tf.split(src_paddings, 2, 0)
tgt_ids = tf.split(tgt_ids, 2, 0)
tgt_labels = tf.split(tgt_labels, 2, 0)
tgt_paddings = tf.split(tgt_paddings, 2, 0)
tgt_weights = tf.split(tgt_weights, 2, 0)
ret.src.ids = tf.cond(
tf.equal(tf.mod(py_utils.GetOrCreateGlobalStep(), 2), 0),
lambda: src_ids[0], lambda: src_ids[1])
ret.src.paddings = tf.cond(
tf.equal(tf.mod(py_utils.GetOrCreateGlobalStep(), 2), 0),
lambda: src_paddings[0], lambda: src_paddings[1])
ret.tgt.ids = tf.cond(
tf.equal(tf.mod(py_utils.GetOrCreateGlobalStep(), 2), 0),
lambda: tgt_ids[0], lambda: tgt_ids[1])
ret.tgt.labels = tf.cond(
tf.equal(tf.mod(py_utils.GetOrCreateGlobalStep(), 2), 0),
lambda: tgt_labels[0], lambda: tgt_labels[1])
ret.tgt.paddings = tf.cond(
tf.equal(tf.mod(py_utils.GetOrCreateGlobalStep(), 2), 0),
lambda: tgt_paddings[0], lambda: tgt_paddings[1])
ret.tgt.weights = tf.cond(
tf.equal(tf.mod(py_utils.GetOrCreateGlobalStep(), 2), 0),
lambda: tgt_weights[0], lambda: tgt_weights[1])
else:
ret.src.ids = src_ids
ret.src.paddings = src_paddings
ret.tgt.ids = tgt_ids
ret.tgt.labels = tgt_labels
ret.tgt.paddings = tgt_paddings
ret.tgt.weights = tgt_weights
return ret
def call(self, inputs, training=None, mask=None):
input_shape = inputs.shape
min_pad_value = self.total_stride * int(
self.pad_image) if self.pad_image else 0
if self.channel_axis == 1:
pad_y = [
min_pad_value, min_pad_value + tf.mod(
self.total_stride - tf.mod(
input_shape[3], self.total_stride), self.total_stride)
]
pad_x = [
min_pad_value, min_pad_value + tf.mod(
self.total_stride - tf.mod(
input_shape[4], self.total_stride), self.total_stride)
]
paddings = [[0, 0], [0, 0], [0, 0], pad_y, pad_x]
crops = [[0, input_shape[0]], [0, input_shape[1]],
[0, self.last_depth],
[pad_y[0], pad_y[0] + input_shape[3]],
[pad_x[0], pad_x[0] + input_shape[4]]]
else:
pad_y = [
min_pad_value, min_pad_value + tf.mod(
self.total_stride - tf.mod(
input_shape[2], self.total_stride), self.total_stride)
]
pad_x = [
min_pad_value, min_pad_value + tf.mod(
self.total_stride - tf.mod(
input_shape[3], self.total_stride), self.total_stride)
]
paddings = [[0, 0], [0, 0], pad_y, pad_x, [0, 0]]
crops = [[0, input_shape[0]], [0, input_shape[1]],
[pad_y[0], input_shape[2] + pad_y[0]],
[pad_x[0], input_shape[3] + pad_x[0]],
[0, self.last_depth]]
inputs = tf.pad(inputs, paddings, "REFLECT")
input_shape = inputs.shape
skip_inputs = []
out_down = inputs
out_skip = tf.reshape(inputs, [
input_shape[0] * input_shape[1], input_shape[2], input_shape[3],
input_shape[4]
])
for down_layer in self.DownLayers:
skip_inputs.append(out_skip)
out_down, out_skip = down_layer(out_down,
training=training,
mask=mask)
up_input = out_skip
skip_inputs.reverse()
assert len(skip_inputs) == len(self.UpLayers)
for up_layer, skip_input in zip(self.UpLayers, skip_inputs):
up_input = up_layer((up_input, skip_input),
training=training,
mask=mask)
logits_output_shape = up_input.shape
logits_output = tf.reshape(up_input, [
input_shape[0], input_shape[1], logits_output_shape[1],
logits_output_shape[2], logits_output_shape[3]
])
logits_output = logits_output[crops[0][0]:crops[0][1],
crops[1][0]:crops[1][1],
crops[2][0]:crops[2][1],
crops[3][0]:crops[3][1],
crops[4][0]:crops[4][1]]
softmax_output = self.Softmax(logits_output)
return logits_output, softmax_output
tf.minimum 최소값
tf.cos 코사인
tf.sin 사인
tf.matmul 행렬의 곱
#====실습===============================
p1 = tf.placeholder("int32") #tf.placeholder: runtime시점에 값을 넣어 실행하겠다는 의미임 #값이 존재하지 않음 #초기화 시키지 않고 값이 나옴
p2 = tf.placeholder("int32")
y = tf.div(p1,p2) #더하기 메소드
sess = tf.Session()
y = tf.truediv(p1,p2)
sess.run(y, feed_dict={p1:9, p2:4})
y = tf.mod(p1,p2)
y = tf.abs(p1,p2)
y = tf.negative(p1,p2)
y = tf.sign(p1,p2)
y = tf.reciprocal(p1,p2)
y = tf.square(p1,p2)
y = tf.round(p1,p2)
y = tf.sqrt(p1,p2)
y = tf.pow(p1,p2)
y = tf.exp(p1,p2)
y = tf.log(p1,p2)
y = tf.maximum(p1,p2)
y = tf.minimum(p1,p2)
y = tf.cos(p1,p2)
y = tf.sin(p1,p2)
def get_control_flag(control, field):
return tf.equal(tf.mod(tf.floor_div(control, field), 2), 1)
import matplotlib.pyplot as plt import numpy as np import tensorflow as tf from tensorflow.python.framework import ops ops.reset_default_graph() # Open graph session sess = tf.Session() # div() vs truediv() vs floordiv() print(sess.run(tf.div(3, 4))) print(sess.run(tf.truediv(3, 4))) print(sess.run(tf.floordiv(3.0, 4.0))) # Mod function print(sess.run(tf.mod(22.0, 5.0))) # Cross Product print(sess.run(tf.cross([1., 0., 0.], [0., 1., 0.]))) # Trig functions print(sess.run(tf.sin(3.1416))) print(sess.run(tf.cos(3.1416))) # Tangent print(sess.run(tf.div(tf.sin(3.1416 / 4.), tf.cos(3.1416 / 4.)))) # Custom operation test_nums = range(15) #from tensorflow.python.ops import math_ops
def model_fn(features, labels, mode, params):
assert features is not None
assert labels is not None
batch_size = params['batch_size']
global_step = tf.train.get_or_create_global_step()
run_discriminator = tf.ceil(tf.div(tf.cast(tf.mod(global_step, (UPDATES_PER_GEN_UPDATE+1)), tf.float32), float(UPDATES_PER_GEN_UPDATE+1)))
with tf.variable_scope('runs'):
real_images = features
noise = tf.random_normal([batch_size, noise_dim])
fake_images, fake_intermed = generator(noise)
discriminator_real = discriminator(real_images)
discriminator_fake = discriminator(fake_images)
with tf.variable_scope('gradient-penalty'):
alpha = tf.random_uniform(shape=[batch_size, spec_dim[0], spec_dim[1], spec_dim[2]], minval=0., maxval=1.)
differences = fake_images-real_images
interpolates = real_images+(alpha*differences)
discriminator_interpolate = discriminator(interpolates)
def restore_batch_size(x):
return tf.tile(tf.reshape(x, [1, 1]), [batch_size, 1])
if mode == tf.estimator.ModeKeys.PREDICT:
test_images = {
'fake_images': fake_images,
'real_images': real_images,
'global_step': restore_batch_size(global_step),
}
return tf.contrib.tpu.TPUEstimatorSpec(mode, predictions=test_images)
with tf.variable_scope('costs'):
with tf.variable_scope('generator_cost'):
pre_gen_cost = -tf.reduce_mean(discriminator_fake)
gen_cost = pre_gen_cost*(1-run_discriminator)
with tf.variable_scope('discriminator_cost'):
original_cost = tf.reduce_mean(discriminator_fake)-tf.reduce_mean(discriminator_real)
with tf.variable_scope('gradient-penalty'):
gradients = tf.gradients(discriminator_interpolate, [interpolates])[0]
slopes = tf.sqrt(tf.reduce_sum(tf.square(gradients), reduction_indices=[1]))
gradient_penalty = tf.reduce_mean((slopes-1.)**2)
with tf.variable_scope('real-score-penalty'):
real_score_penalty = tf.reduce_mean(tf.square(discriminator_real))
pre_discriminator_cost = original_cost + gradient_penalty * gradient_penalty_weight + real_score_penalty * real_score_penalty_weight
discriminator_cost = pre_discriminator_cost*run_discriminator
if mode == tf.estimator.ModeKeys.EVAL:
return tf.contrib.tpu.TPUEstimatorSpec(mode, loss=0) # , eval_metric_ops=costs)
def restore_batch_size(x):
return tf.tile(tf.reshape(x, [1, 1]), [batch_size, 1])
if mode == tf.estimator.ModeKeys.TRAIN:
with tf.variable_scope('optimizers'):
gen_opt = tf.contrib.tpu.CrossShardOptimizer(tf.train.AdamOptimizer(ALPHA, BETA1, BETA2))
assert gen_opt is not None
discriminator_opt = tf.contrib.tpu.CrossShardOptimizer(tf.train.AdamOptimizer(ALPHA, BETA1, BETA2))
with tf.control_dependencies(tf.get_collection(tf.GraphKeys.UPDATE_OPS)):
discriminator_opt = discriminator_opt.minimize(discriminator_cost, var_list=tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope='runs/discriminator'))
assert discriminator_opt is not None
with tf.control_dependencies([discriminator_opt]):
gen_opt = gen_opt.minimize(gen_cost, var_list=tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope='runs/generator'))
assert discriminator_opt is not None
with tf.control_dependencies([gen_opt]):
opt = tf.assign_add(global_step, 1)
assert tf.tile(tf.reshape(global_step, [1, 1]), [batch_size, 1]) is not None
assert fake_images is not None
tensors_to_pass = [
restore_batch_size(global_step),
fake_images,
restore_batch_size(pre_gen_cost),
restore_batch_size(original_cost),
restore_batch_size(gradient_penalty),
restore_batch_size(real_score_penalty),
restore_batch_size(pre_discriminator_cost)
]
assert opt is not None
return tf.contrib.tpu.TPUEstimatorSpec(mode, train_op=opt, host_call=(host_call_fn, tensors_to_pass), loss=discriminator_cost+gen_cost)
return
def build_graph(self):
with tf.device('/cpu:0'):
"""Builds data processing graph using ``tf.data`` API."""
if self.params['mode'] != 'infer':
self._dataset = tf.data.Dataset.from_tensor_slices(self._files)
if self.params['shuffle']:
self._dataset = self._dataset.shuffle(self._size)
self._dataset = self._dataset.repeat()
self._dataset = self._dataset.prefetch(tf.contrib.data.AUTOTUNE)
self._dataset = self._dataset.map(
lambda line: tf.py_func(
self._parse_audio_transcript_element,
[line],
[self.params['dtype'], tf.int32, tf.int32, tf.int32, tf.float32],
stateful=False,
),
num_parallel_calls=8,
)
if self.params['max_duration'] > 0:
self._dataset = self._dataset.filter(
lambda x, x_len, y, y_len, duration:
tf.less_equal(duration, self.params['max_duration'])
)
if self.params['min_duration'] > 0:
self._dataset = self._dataset.filter(
lambda x, x_len, y, y_len, duration:
tf.greater_equal(duration, self.params['min_duration'])
)
self._dataset = self._dataset.map(
lambda x, x_len, y, y_len, duration:
[x, x_len, y, y_len],
num_parallel_calls=8,
)
self._dataset = self._dataset.padded_batch(
self.params['batch_size'],
padded_shapes=([None, self.params['num_audio_features']],
1, [None], 1),
padding_values=(
tf.cast(0, self.params['dtype']), 0, self.target_pad_value, 0),
)
else:
indices = self.split_data(
np.array(list(map(str, range(len(self.all_files)))))
)
self._dataset = tf.data.Dataset.from_tensor_slices(
np.hstack((indices[:, np.newaxis], self._files[:, np.newaxis]))
)
self._dataset = self._dataset.repeat()
self._dataset = self._dataset.prefetch(tf.contrib.data.AUTOTUNE)
self._dataset = self._dataset.map(
lambda line: tf.py_func(
self._parse_audio_element,
[line],
[self.params['dtype'], tf.int32, tf.int32, tf.float32],
stateful=False,
),
num_parallel_calls=8,
)
if self.params['max_duration'] > 0:
self._dataset = self._dataset.filter(
lambda x, x_len, idx, duration:
tf.less_equal(duration, self.params['max_duration'])
)
if self.params['min_duration'] > 0:
self._dataset = self._dataset.filter(
lambda x, x_len, y, y_len, duration:
tf.greater_equal(duration, self.params['min_duration'])
)
self._dataset = self._dataset.map(
lambda x, x_len, idx, duration:
[x, x_len, idx],
num_parallel_calls=16,
)
self._dataset = self._dataset.padded_batch(
self.params['batch_size'],
padded_shapes=([None, self.params['num_audio_features']], 1, 1)
)
self._iterator = self._dataset.prefetch(tf.contrib.data.AUTOTUNE)\
.make_initializable_iterator()
if self.params['mode'] != 'infer':
x, x_length, y, y_length = self._iterator.get_next()
# need to explicitly set batch size dimension
# (it is employed in the model)
y.set_shape([self.params['batch_size'], None])
y_length = tf.reshape(y_length, [self.params['batch_size']])
else:
x, x_length, x_id = self._iterator.get_next()
x_id = tf.reshape(x_id, [self.params['batch_size']])
x.set_shape([self.params['batch_size'], None,
self.params['num_audio_features']])
x_length = tf.reshape(x_length, [self.params['batch_size']])
pad_to = self.params.get("pad_to", 8)
if pad_to > 0 and self.params.get('backend') == 'librosa':
# we do padding with TF for librosa backend
num_pad = tf.mod(pad_to - tf.mod(tf.reduce_max(x_length), pad_to), pad_to)
x = tf.pad(x, [[0, 0], [0, num_pad], [0, 0]])
self._input_tensors = {}
self._input_tensors["source_tensors"] = [x, x_length]
if self.params['mode'] != 'infer':
self._input_tensors['target_tensors'] = [y, y_length]
else:
self._input_tensors['source_ids'] = [x_id]
def regDLF(y_true,
y_pred,
alpha=1,
beta=1,
gamma=0.01,
delta_v=0.5,
delta_d=1.5,
name='loss_discrim'):
def tf_norm(inputs, axis=1, epsilon=1e-7, name='safe_norm'):
squared_norm = tf.reduce_sum(tf.square(inputs),
axis=axis,
keep_dims=True)
safe_norm = tf.sqrt(squared_norm + epsilon)
return tf.identity(safe_norm, name=name)
###
lins = tf.linspace(0.0, DIMZ * DIMY * DIMX, DIMZ * DIMY * DIMX)
lins = tf.cast(lins, tf.int32)
# lins = lins / tf.reduce_max(lins) * 255
# lins = cvt2tanh(lins)
# lins = tf.reshape(lins, tf.shape(y_true), name='lins_3d')
# print lins
lins_z = tf.div(lins, (DIMY * DIMX))
lins_y = tf.div(tf.mod(lins, (DIMY * DIMX)), DIMY)
lins_x = tf.mod(tf.mod(lins, (DIMY * DIMX)), DIMY)
lins = tf.cast(lins, tf.float32)
lins_z = tf.cast(lins_z, tf.float32)
lins_y = tf.cast(lins_y, tf.float32)
lins_x = tf.cast(lins_x, tf.float32)
lins = lins / tf.reduce_max(lins) * 255
lins_z = lins_z / tf.reduce_max(lins_z) * 255
lins_y = lins_y / tf.reduce_max(lins_y) * 255
lins_x = lins_x / tf.reduce_max(lins_x) * 255
lins = cvt2tanh(lins)
lins_z = cvt2tanh(lins_z)
lins_y = cvt2tanh(lins_y)
lins_x = cvt2tanh(lins_x)
lins = tf.reshape(lins, tf.shape(y_true), name='lins')
lins_z = tf.reshape(lins_z, tf.shape(y_true), name='lins_z')
lins_y = tf.reshape(lins_y, tf.shape(y_true), name='lins_y')
lins_x = tf.reshape(lins_x, tf.shape(y_true), name='lins_x')
y_true = tf.reshape(y_true, [DIMZ * DIMY * DIMX])
y_pred = tf.concat([y_pred, lins, lins_z, lins_y, lins_x],
axis=-1)
nDim = tf.shape(y_pred)[-1]
X = tf.reshape(y_pred, [DIMZ * DIMY * DIMX, nDim])
uniqueLabels, uniqueInd = tf.unique(y_true)
numUnique = tf.size(
uniqueLabels) # Get the number of connected component
Sigma = tf.unsorted_segment_sum(X, uniqueInd, numUnique)
# ones_Sigma = tf.ones((tf.shape(X)[0], 1))
ones_Sigma = tf.ones_like(X)
ones_Sigma = tf.unsorted_segment_sum(ones_Sigma, uniqueInd,
numUnique)
mu = tf.divide(Sigma, ones_Sigma)
Lreg = tf.reduce_mean(tf.norm(mu, axis=1, ord=1))
T = tf.norm(tf.subtract(tf.gather(mu, uniqueInd), X),
axis=1,
ord=1)
T = tf.divide(T, Lreg)
T = tf.subtract(T, delta_v)
T = tf.clip_by_value(T, 0, T)
T = tf.square(T)
ones_Sigma = tf.ones_like(uniqueInd, dtype=tf.float32)
ones_Sigma = tf.unsorted_segment_sum(ones_Sigma, uniqueInd,
numUnique)
clusterSigma = tf.unsorted_segment_sum(T, uniqueInd, numUnique)
clusterSigma = tf.divide(clusterSigma, ones_Sigma)
# Lvar = tf.reduce_mean(clusterSigma, axis=0)
Lvar = tf.reduce_mean(clusterSigma)
mu_interleaved_rep = tf.tile(mu, [numUnique, 1])
mu_band_rep = tf.tile(mu, [1, numUnique])
mu_band_rep = tf.reshape(mu_band_rep,
(numUnique * numUnique, nDim))
mu_diff = tf.subtract(mu_band_rep, mu_interleaved_rep)
# Remove zero vector
# intermediate_tensor = reduce_sum(tf.abs(x), 1)
# zero_vector = tf.zeros(shape=(1,1), dtype=tf.float32)
# bool_mask = tf.not_equal(intermediate_tensor, zero_vector)
# omit_zeros = tf.boolean_mask(x, bool_mask)
intermediate_tensor = tf.reduce_sum(tf.abs(mu_diff), 1)
zero_vector = tf.zeros(shape=(1, 1), dtype=tf.float32)
bool_mask = tf.not_equal(intermediate_tensor, zero_vector)
omit_zeros = tf.boolean_mask(mu_diff, bool_mask)
mu_diff = tf.expand_dims(omit_zeros, axis=1)
print mu_diff
mu_diff = tf.norm(mu_diff, ord=1)
# squared_norm = tf.reduce_sum(tf.square(s), axis=axis,keep_dims=True)
# safe_norm = tf.sqrt(squared_norm + epsilon)
# squared_norm = tf.reduce_sum(tf.square(omit_zeros), axis=-1,keep_dims=True)
# safe_norm = tf.sqrt(squared_norm + 1e-6)
# mu_diff = safe_norm
mu_diff = tf.divide(mu_diff, Lreg)
mu_diff = tf.subtract(2 * delta_d, mu_diff)
mu_diff = tf.clip_by_value(mu_diff, 0, mu_diff)
mu_diff = tf.square(mu_diff)
numUniqueF = tf.cast(numUnique, tf.float32)
Ldist = tf.reduce_mean(mu_diff)
# L = alpha * Lvar + beta * Ldist + gamma * Lreg
# L = tf.reduce_mean(L, keep_dims=True)
L = tf.reduce_sum([alpha * Lvar, beta * Ldist, gamma * Lreg],
keep_dims=False)
print L
print Ldist
print Lvar
print Lreg
return tf.identity(L, name=name)
name='word_counter',
initializer=tf.constant_initializer(0),
trainable=False,
dtype=tf.int32)
add_words = tf.assign_add(word_counter, words)
# DATA PIPELINE (only want to go through data once)
# dataset = tf.contrib.data.Dataset.from_tensor_slices(filenames)
# dataset = tf.contrib.data.Dataset.list_files(FILE_PATTERN).enumerate()
time.sleep(2)
dataset = tf.contrib.data.Dataset.list_files(FILE_PATTERN)
# GOAL: Distribute workload across workers, giving each a different part of data.
# Assign file i work k if i % num_workers == 0.
dataset = (dataset.enumerate().filter(lambda idx, f: tf.equal(
tf.mod(idx, num_workers), FLAGS.task_index)).flat_map(
lambda idx, f: tf.contrib.data.TextLineDataset(f)))
# Use `Dataset.flat_map()` to transform each file as a separate nested dataset,
# and then concatenate their contents sequentially into a single "flat" dataset.
# Flatten text files.
# dataset = dataset.flat_map(
# lambda filename: (
# tf.contrib.data.TextLineDataset(filename)))
dataset = dataset.batch(16)
iterator = dataset.make_one_shot_iterator()
next_element = iterator.get_next()
def train(self, dataset, end_trigger):
with tf.Graph().as_default() as g:
generator_inputs = dataset.feature_tensors
real_data = dataset.label_tensors
counter = tf.Variable(0, dtype=tf.int32)
period = self._discriminator_steps + self._generator_steps
is_discriminator_phase = tf.less(tf.mod(counter, period), self._discriminator_steps)
with tf.variable_scope("generator"):
gen_data = self._call_fn_maybe_with_counter(self._generator_fn, counter,
generator_inputs)
with tf.variable_scope("discriminator"):
fake_d_outputs = self._call_fn_maybe_with_counter(self._discriminator_fn,
counter,
gen_data, generator_inputs)
with tf.variable_scope("discriminator", reuse=True):
real_d_outputs = self._call_fn_maybe_with_counter(self._discriminator_fn,
counter,
real_data, generator_inputs)
with tf.name_scope("generator_loss"):
generator_loss = self._call_fn_maybe_with_counter(self._generator_loss_fn,
counter,
fake_d_outputs)
with tf.name_scope("discriminator_loss"):
discriminator_loss = self._call_fn_maybe_with_counter(self._discriminator_loss_fn,
counter,
real_d_outputs,
fake_d_outputs)
generator_variables = tf.trainable_variables("generator")
generator_grads = tf.gradients(generator_loss, generator_variables)
discriminator_variables = tf.trainable_variables("discriminator")
discriminator_grads = tf.gradients(discriminator_loss, discriminator_variables)
variables = generator_variables + discriminator_variables
def true_fn():
return [tf.zeros_like(grad) for grad in generator_grads]
def false_fn():
return generator_grads
g_grads = tf.cond(is_discriminator_phase, true_fn=true_fn, false_fn=false_fn)
d_grads = tf.cond(is_discriminator_phase, lambda: discriminator_grads,
lambda: [tf.zeros_like(grad) for grad in discriminator_grads])
loss = tf.cond(is_discriminator_phase,
lambda: discriminator_loss,
lambda: generator_loss)
grads = g_grads + d_grads
with tf.control_dependencies(grads):
increase_counter = tf.assign_add(counter, 1)
g_param_size = sum([np.product(g.shape) for g in g_grads])
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
optimizer = TFOptimizer(loss, GanOptimMethod(self._discriminator_optim_method,
self._generator_optim_method,
g_param_size.value), sess=sess,
dataset=dataset, inputs=dataset._original_tensors,
grads=grads, variables=variables, graph=g,
updates=[increase_counter],
model_dir=self.checkpoint_path)
optimizer.optimize(end_trigger)
steps = sess.run(counter)
saver = tf.train.Saver()
saver.save(optimizer.sess, self.checkpoint_path, global_step=steps)
def __rmod__(self, other):
return tf.mod(other, self)
#tf.random_crop(value, size, seed=None, name=None) #tf.multinomial(logits, num_samples, seed=None, name=None) #tf.random_gamma(shape, alpha, beta=None, dtype=tf.float32, seed=None, name=None) # In[8]: # math operations a = tf.constant([3, 2]) b = tf.constant([2, 2]) tf.add(a, b) tf.add_n([a, b, b]) # a + b + b tf.multiply(a, b) # element wise #tf.matmul(a,b) # error tf.matmul(tf.reshape(a, shape=[1, 2]), tf.reshape(b, shape=[2, 1])) tf.div(a, b) tf.mod(a, b) # In[9]: # data types t_0 = 20 # treated as 0 d array tf.zeros_like(t_0) # ===> 0 tf.ones_like(t_0) # ===> 1 t_1 = ["apple", "banana", "orange"] # ==> treated as 1-D array tf.zeros_like(t_1) # ==> ['','',''] #tf.ones_like(t_1) # ==> error t_2 = [[True, False, False], [False, False, True], [False, True, False]] tf.zeros_like(t_2)
def __mod__(self, other):
return tf.mod(self, other)
def _boundary_circular(sample_coords, input_size):
return tf.mod(tf.mod(sample_coords, input_size) + input_size, input_size)
def build_orientation_map(pb):
pyramid_kernel_x = np.array(
[[0, 0, 1, 2, 3, 2, 1, 0, 0], [0, 1, 2, 3, 4, 3, 2, 1, 0],
[1, 2, 3, 4, 5, 4, 3, 2, 1], [0, 1, 2, 3, 4, 3, 2, 1, 0],
[0, 0, 1, 2, 3, 2, 1, 0, 0]], np.float)
pyramid_kernel_y = pyramid_kernel_x.transpose()
preprocessing_kernel_y = tf.constant(pyramid_kernel_y, tf.float32,
(9, 5, 1, 1), "pyramid_kernel_y")
preprocessing_kernel_x = tf.constant(pyramid_kernel_x, tf.float32,
(5, 9, 1, 1), "pyramid_kernel_x")
pb4d = tf.expand_dims(pb, -1, "pb4d")
pb_filtered_y = tf.nn.conv2d(pb4d,
preprocessing_kernel_y, (1, 1, 1, 1),
"SAME",
name="pb_filtered_y")
pb_filtered = tf.nn.conv2d(pb_filtered_y,
preprocessing_kernel_x, (1, 1, 1, 1),
"SAME",
name="pb_filtered")
gradient_kernel_y = tf.constant([-.5, 0, .5], tf.float32, (3, 1, 1, 1),
"gradient_kernel_y")
gradient_kernel_x = tf.constant([-.5, 0, .5], tf.float32, (1, 3, 1, 1),
"gradient_kernel_x")
padding_y = np.array([0, 0, 1, 1, 0, 0, 0, 0]).reshape(4, 2)
padding_x = np.array([0, 0, 0, 0, 1, 1, 0, 0]).reshape(4, 2)
pb_filtered_padded_y = tf.pad(pb_filtered, padding_y, "SYMMETRIC",
"pb_filtered_padded_y")
Oy = tf.nn.conv2d(pb_filtered_padded_y,
gradient_kernel_y, (1, 1, 1, 1),
"VALID",
name="Oy")
pb_filtered_padded_x = tf.pad(pb_filtered, padding_x, "SYMMETRIC",
"pb_filtered_padded_x")
Ox = tf.nn.conv2d(pb_filtered_padded_x,
gradient_kernel_x, (1, 1, 1, 1),
"VALID",
name="Ox")
Ox_padded_y = tf.pad(Ox, padding_y, "SYMMETRIC", "Ox_padded_y")
Oyx = tf.nn.conv2d(Ox_padded_y,
gradient_kernel_y, (1, 1, 1, 1),
"VALID",
name="Oyx")
Ox_padded_x = tf.pad(Ox, padding_x, "SYMMETRIC", "Ox_padded_x")
Oxx = tf.nn.conv2d(Ox_padded_x,
gradient_kernel_x, (1, 1, 1, 1),
"VALID",
name="Oxx")
Oy_padded_y = tf.pad(Oy, padding_y, "SYMMETRIC", "Oy_padded_y")
Oyy = tf.nn.conv2d(Oy_padded_y,
gradient_kernel_y, (1, 1, 1, 1),
"VALID",
name="Oyy")
Oy_padded_x = tf.pad(Oy, padding_x, "SYMMETRIC", "Oy_padded_x")
Oxy = tf.nn.conv2d(Oy_padded_x,
gradient_kernel_x, (1, 1, 1, 1),
"VALID",
name="Oxy")
Q = tf.atan(tf.divide(tf.multiply(Oyy, tf.sign(tf.negative(Oxy))),
tf.add(Oxx, tf.constant(1e-5))),
name="Q")
pi = tf.constant(np.pi, tf.float32, name="pi")
O = tf.mod(Q, pi, "O")
return O
def update_cross(x, label=label, shape=shape1):
i = tf.div(x[0], shape[0])
j = tf.mod(x[0], shape[0])
return tf.cond(tf.less(x[1], 1), lambda: tf.ones([
4,
], dtype=tf.int32), lambda: cross_layer(label, i, j))
def step(self, att_objects: List[Attention],
bs_state: SearchState) -> Tuple[SearchState, SearchStepOutput]:
# embed the previously decoded word
input_ = self._parent_decoder.embed_and_dropout(bs_state.last_word_ids)
# don't want to use this decoder with uninitialized parent
assert self._parent_decoder.step_scope.reuse
# run the parent decoder decoding step
# shapes:
# logits: beam x vocabulary
# state: beam x rnn_size
# attns: encoder x beam x context vector size
logits, state, attns = self._parent_decoder.step(
att_objects, input_, bs_state.last_state, bs_state.last_attns)
# mask the probabilities
# shape(logprobs) = beam x vocabulary
logprobs = (tf.expand_dims(1. - tf.to_float(bs_state.finished), 1) *
tf.nn.log_softmax(logits))
# update hypothesis scores
# shape(hyp_probs) = beam x vocabulary
hyp_probs = tf.expand_dims(bs_state.logprob_sum, 1) + logprobs
# update hypothesis lengths
hyp_lengths = bs_state.lengths + 1 - tf.to_int32(bs_state.finished)
# shape(scores) = beam x vocabulary
scores = hyp_probs / tf.expand_dims(self._length_penalty(hyp_lengths),
1)
# flatten so we can use top_k
scores_flat = tf.reshape(scores, [-1])
# shape(both) = beam
topk_scores, topk_indices = tf.nn.top_k(scores_flat, self._beam_size)
topk_scores.set_shape([self._beam_size])
topk_indices.set_shape([self._beam_size])
# flatten the hypothesis probabilities
hyp_probs_flat = tf.reshape(hyp_probs, [-1])
# select logprobs of the best hyps (disregard lenghts)
next_logprob_sum = tf.gather(hyp_probs_flat, topk_indices)
# pylint: disable=no-member
next_logprob_sum.set_shape([self._beam_size])
# pylint: enable=no-member
next_word_ids = tf.mod(topk_indices,
len(self._parent_decoder.vocabulary))
next_beam_ids = tf.div(topk_indices,
len(self._parent_decoder.vocabulary))
next_beam_prev_state = tf.gather(state, next_beam_ids)
next_beam_prev_attns = [tf.gather(a, next_beam_ids) for a in attns]
next_lengths = tf.gather(hyp_lengths, next_beam_ids)
# update finished flags
has_just_finished = tf.equal(next_word_ids, END_TOKEN_INDEX)
next_finished = tf.logical_or(
tf.gather(bs_state.finished, next_beam_ids), has_just_finished)
output = SearchStepOutput(scores=topk_scores,
parent_ids=next_beam_ids,
token_ids=next_word_ids)
search_state = SearchState(logprob_sum=next_logprob_sum,
lengths=next_lengths,
finished=next_finished,
last_word_ids=next_word_ids,
last_state=next_beam_prev_state,
last_attns=next_beam_prev_attns)
return search_state, output
def update_in(x, full_label=full_label, shape=shape):
i = tf.div(x[0], shape[0])
j = tf.mod(x[0], shape[0])
return tf.cond(tf.less(x[1], 1), lambda: tf.ones([
8,
], dtype=tf.int32), lambda: in_layer(full_label, i + 1, j + 1))
def loop_step(batch_index, ts, stop_decoder, states, alphas, cand_seqs,
cand_scores, completed_scores, completed_scores_scaled,
completed_seqs, completed_lens):
"""
Args:
batch_index: batch index
ts (int): time step
stop_decoder (bool): stop decoding
ys (?): [beam_size]
states (float): [beam_size, state_size]
alphas (float): [beam_size, alpha_size]
cand_scores: [beam_size], sequence score
cand_seqs: [beam_size, ts], ts increases over time
Returns:
logits shape: [beam_size, output_dim]
state: [beam_size, state_size]
alpha: [beam_size, alpha_size]
"""
# 1. get score from one step decoder
# logits = tf.one_hot(ts, depth=num_symbols, off_value=0.0, dtype=tf.float32)
if DEBUG: ts = tf.Print(ts, [ts], message='ts: ')
ys = cand_seqs[:, ts]
if DEBUG: ys = tf.Print(ys, [ys], message='Y(t-1): ')
logits, states, alphas = self.step(ys, states, alphas, batch_index)
if DEBUG: logits = tf.Print(logits, [logits], message='logits: ')
Z = tf.reduce_logsumexp(logits, 1, keep_dims=True)
if DEBUG: Z = tf.Print(Z, [Z], message='Z: ')
logprobs = tf.subtract(logits, Z) # [beam_size, num_symbols]
new_scores = tf.add(logprobs,
tf.expand_dims(cand_scores,
1)) # [beam_size, num_symbols]
if DEBUG:
new_scores = tf.Print(new_scores, [new_scores],
message='new_scores: ')
num_unstop_symbols = tf.shape(new_scores)[1] - 1
new_uncompleted_scores, new_completed_scores = tf.split(
new_scores, [num_unstop_symbols, 1], 1)
if DEBUG:
new_uncompleted_scores = tf.Print(
new_uncompleted_scores, [new_uncompleted_scores],
message='new_uncompleted_scores: ')
# 2. Update completed seqs --------------------------------------
# 2.1 update scores
new_completed_scores = tf.squeeze(new_completed_scores,
-1) # [beam_size]
all_completed_scores = tf.concat(
[completed_scores, new_completed_scores], 0) # [2*beam_size]
# 2.2 choose top K from scaled_scores
new_completed_scores_scaled = tf.div(new_completed_scores,
tf.to_float(ts + 1))
all_scores_scaled = tf.concat(
[completed_scores_scaled, new_completed_scores_scaled], 0)
completed_scores_scaled, indices = tf.nn.top_k(all_scores_scaled,
k=beam_size,
sorted=False)
if DEBUG:
indices = tf.Print(indices, [indices],
message='top K completed indices: ')
# 2.2 update len
new_completed_lens = tf.fill([beam_size], tf.add(ts,
1)) # [beam_size]
all_lens = tf.concat([completed_lens, new_completed_lens],
0) # [2*beam_size]
completed_lens = tf.gather(all_lens,
indices,
validate_indices=True,
axis=0) # [beam_size]
if DEBUG:
completed_lens = tf.Print(completed_lens, [completed_lens],
message='completed lens',
summarize=5)
# 2.3 update seqs
all_completed = tf.concat([completed_seqs, cand_seqs], 0)
completed_seqs = tf.gather(all_completed,
indices,
validate_indices=True,
axis=0) # [beam_size, ts]
if DEBUG:
completed_seqs = tf.Print(completed_seqs, [completed_seqs],
message='completed seqs: ',
summarize=MAX_STEPS + 2)
# 2.4 stop decoding loop
max_uncompleted = tf.reduce_max(new_uncompleted_scores)
completed_scores = tf.gather(all_completed_scores,
indices,
validate_indices=True,
axis=0)
min_completed = tf.reduce_min(completed_scores)
stop_decoder = tf.greater(min_completed, max_uncompleted)
# 2. Update completed seqs --------------------------------------
# 3. Update uncompleted sequences --------------------------------
# new_uncompleted_scores: [beam_size, num_symbols-1]
# top_k: [beam_size]. indices of top k scores
def f0():
return new_uncompleted_scores[0, :]
def f1():
return new_uncompleted_scores
un_scores = tf.cond(tf.equal(ts, 0), f0, f1)
new_flat = tf.squeeze(tf.reshape(
un_scores, [-1, 1])) # [beam_size*num_unstop_symbols]
# get top K symbols
cand_scores, flat_indices = tf.nn.top_k(new_flat,
k=beam_size,
sorted=False)
cand_parents = tf.div(flat_indices, num_unstop_symbols)
_ys = tf.mod(flat_indices,
num_unstop_symbols) # [beam_size], y(t) for next step
A = tf.gather(cand_seqs[:, 0:ts + 1],
cand_parents) #[beam_size, ts+1]
B = tf.expand_dims(_ys, -1) # [beam_size, 1]
C = tf.fill([beam_size, MAX_STEPS + 2 - ts - 2], stop_symbol)
cand_seqs = tf.concat([A, B, C], 1) # [beam_size, MAX_STEPS]
if DEBUG:
cand_seqs = tf.Print(cand_seqs, [cand_seqs],
message='cand seqs: ',
summarize=MAX_STEPS + 2)
cand_seqs = tf.reshape(cand_seqs, [beam_size, MAX_STEPS + 2])
cand_scores.set_shape([beam_size])
completed_seqs = tf.reshape(completed_seqs,
[beam_size, MAX_STEPS + 2])
s1_shape = [beam_size, self.attention_cell.state_size]
s2_shape = [beam_size, self.decoder_cell.state_size]
s3_shape = [beam_size, self.attn_context.context_size]
# prepare data for next step
# states = tf.gather(states, cand_parents, axis=0)
# states = self.select_states(states, cand_parents)
states = tuple(tf.gather(el, cand_parents) for el in states)
states[0].set_shape(s1_shape)
states[1].set_shape(s2_shape)
states[2].set_shape(s3_shape)
alphas = tf.gather(alphas, cand_parents, axis=1)
alphas_shape = [self.attn_context.num_encoder_states, beam_size]
alphas = tf.reshape(alphas, alphas_shape)
# alphas.set_shape(alphas_shape)
# 3. Update uncompleted sequences --------------------------------
ts = tf.add(ts, 1)
return batch_index, ts, stop_decoder, states, alphas, cand_seqs, \
cand_scores, completed_scores, completed_scores_scaled, \
completed_seqs, completed_lens
def model_creator(batch_size, name="default", dtype=np.float32):
"""Create MNIST autoencoder model. Dataset is part of model."""
model = Model(name)
def get_batch_size(data):
return 10000
init_dict = {}
global_vars = []
local_vars = []
# TODO: factor out to reuse between scripts
# TODO: change feed_dict logic to reuse value provided to VarStruct
# current situation makes reinitialization of global variable change
# it's value, counterinituitive
def init_var(val, name, is_global=False):
"""Helper to create variables with numpy or TF initial values."""
if isinstance(val, tf.Tensor):
var = u.get_variable(name=name, initializer=val, reuse=is_global)
else:
val = np.array(val)
assert u.is_numeric(val), "Non-numeric type."
var_struct = u.get_var(name=name, initializer=val, reuse=is_global)
holder = var_struct.val_
init_dict[holder] = val
var = var_struct.var
if is_global:
global_vars.append(var)
else:
local_vars.append(var)
return var
def nonlin(x):
return tf.sigmoid(x)
# TODO: rename into "nonlin_d"
def d_nonlin(y):
return y*(1-y)
patches = train_images[:,:args.batch_size];
test_patches = test_images[:,:args.batch_size];
if args.dataset == 'cifar':
input_dim = 3*32*32
elif args.dataset == 'mnist':
input_dim = 28*28
else:
assert False
if release_name == 'kfac_tiny':
fs = [args.batch_size, input_dim, 196, input_dim]
else:
fs = [args.batch_size, input_dim, 1024, 1024, 1024, 196, 1024, 1024, 1024,
input_dim]
def f(i): return fs[i+1] # W[i] has shape f[i] x f[i-1]
n = len(fs) - 2
# Full dataset from which new batches are sampled
X_full = init_var(train_images, "X_full", is_global=True)
X = init_var(patches, "X", is_global=False) # stores local batch per model
W = [None]*n
W.insert(0, X)
A = [None]*(n+2)
A[1] = W[0]
for i in range(1, n+1):
init_val = ng_init(f(i), f(i-1)).astype(dtype)
W[i] = init_var(init_val, "W_%d"%(i,), is_global=True)
A[i+1] = nonlin(kfac_lib.matmul(W[i], A[i]))
err = A[n+1] - A[1]
model.loss = u.L2(err) / (2 * get_batch_size(err))
# create test error eval
layer0 = init_var(test_patches, "X_test", is_global=True)
layer = layer0
for i in range(1, n+1):
layer = nonlin(W[i] @ layer)
verr = (layer - layer0)
model.vloss = u.L2(verr) / (2 * get_batch_size(verr))
# manually compute backprop to use for sanity checking
B = [None]*(n+1)
B2 = [None]*(n+1)
B[n] = err*d_nonlin(A[n+1])
_sampled_labels_live = tf.random_normal((f(n), f(-1)), dtype=dtype, seed=0)
if args.fixed_labels:
_sampled_labels_live = tf.ones(shape=(f(n), f(-1)), dtype=dtype)
_sampled_labels = init_var(_sampled_labels_live, "to_be_deleted",
is_global=False)
B2[n] = _sampled_labels*d_nonlin(A[n+1])
for i in range(n-1, -1, -1):
backprop = t(W[i+1]) @ B[i+1]
B[i] = backprop*d_nonlin(A[i+1])
backprop2 = t(W[i+1]) @ B2[i+1]
B2[i] = backprop2*d_nonlin(A[i+1])
# cov_A = [None]*(n+1) # covariance of activations[i]
# cov_B2 = [None]*(n+1) # covariance of synthetic backprops[i]
# vars_svd_A = [None]*(n+1)
# vars_svd_B2 = [None]*(n+1)
# dW = [None]*(n+1)
# pre_dW = [None]*(n+1) # preconditioned dW
# todo: decouple initial value from covariance update
# # maybe need start with identity and do running average
# for i in range(1,n+1):
# if regularized_svd:
# cov_A[i] = init_var(A[i]@t(A[i])/args.batch_size+args.Lambda*u.Identity(f(i-1)), "cov_A%d"%(i,))
# cov_B2[i] = init_var(B2[i]@t(B2[i])/args.batch_size+args.Lambda*u.Identity(f(i)), "cov_B2%d"%(i,))
# else:
# cov_A[i] = init_var(A[i]@t(A[i])/args.batch_size, "cov_A%d"%(i,))
# cov_B2[i] = init_var(B2[i]@t(B2[i])/args.batch_size, "cov_B2%d"%(i,))
# vars_svd_A[i] = u.SvdWrapper(cov_A[i],"svd_A_%d"%(i,), do_inverses=False)
# vars_svd_B2[i] = u.SvdWrapper(cov_B2[i],"svd_B2_%d"%(i,), do_inverses=False)
# whitened_A = u.cached_inverse(vars_svd_A[i], args.Lambda) @ A[i]
# whitened_B = u.cached_inverse(vars_svd_B2[i], args.Lambda) @ B[i]
# dW[i] = (B[i] @ t(A[i]))/args.batch_size
# pre_dW[i] = (whitened_B @ t(whitened_A))/args.batch_size
sampled_labels_live = A[n+1] + tf.random_normal((f(n), f(-1)),
dtype=dtype, seed=0)
if args.fixed_labels:
sampled_labels_live = A[n+1]+tf.ones(shape=(f(n), f(-1)), dtype=dtype)
sampled_labels = init_var(sampled_labels_live, "sampled_labels", is_global=False)
err2 = A[n+1] - sampled_labels
model.loss2 = u.L2(err2) / (2 * args.batch_size)
model.global_vars = global_vars
model.local_vars = local_vars
model.trainable_vars = W[1:]
# todo, we have 3 places where model step is tracked, reduce
model.step = init_var(u.as_int32(0), "step", is_global=False)
advance_step_op = model.step.assign_add(1)
assert get_batch_size(X_full) % args.batch_size == 0
batches_per_dataset = (get_batch_size(X_full) // args.batch_size)
batch_idx = tf.mod(model.step, batches_per_dataset)
start_idx = batch_idx * args.batch_size
advance_batch_op = X.assign(X_full[:,start_idx:start_idx + args.batch_size])
def advance_batch():
# print("Step for model(%s) is %s"%(model.name, u.eval(model.step)))
sess = u.get_default_session()
# TODO: get rid of _sampled_labels
sessrun([sampled_labels.initializer, _sampled_labels.initializer])
if args.advance_batch:
sessrun(advance_batch_op)
sessrun(advance_step_op)
model.advance_batch = advance_batch
# TODO: refactor this to take initial values out of Var struct
#global_init_op = tf.group(*[v.initializer for v in global_vars])
global_init_ops = [v.initializer for v in global_vars]
global_init_op = tf.group(*[v.initializer for v in global_vars])
global_init_query_ops = [tf.logical_not(tf.is_variable_initialized(v))
for v in global_vars]
def initialize_global_vars(verbose=False, reinitialize=False):
"""If reinitialize is false, will not reinitialize variables already
initialized."""
sess = u.get_default_session()
if not reinitialize:
uninited = sessrun(global_init_query_ops)
# use numpy boolean indexing to select list of initializers to run
to_initialize = list(np.asarray(global_init_ops)[uninited])
else:
to_initialize = global_init_ops
if verbose:
print("Initializing following:")
for v in to_initialize:
print(" " + v.name)
sessrun(to_initialize, feed_dict=init_dict)
model.initialize_global_vars = initialize_global_vars
# didn't quite work (can't initialize var in same run call as deps likely)
# enforce that batch is initialized before everything
# except fake labels opa
# for v in local_vars:
# if v != X and v != sampled_labels and v != _sampled_labels:
# print("Adding dep %s on %s"%(v.initializer.name, X.initializer.name))
# u.add_dep(v.initializer, on_op=X.initializer)
local_init_op = tf.group(*[v.initializer for v in local_vars],
name="%s_localinit"%(model.name))
print("Local vars:")
for v in local_vars:
print(v.name)
def initialize_local_vars():
sess = u.get_default_session()
sessrun(_sampled_labels.initializer, feed_dict=init_dict)
sessrun(X.initializer, feed_dict=init_dict)
sessrun(local_init_op, feed_dict=init_dict)
model.initialize_local_vars = initialize_local_vars
return model
def generate_detections_per_image_tpu(cls_outputs,
box_outputs,
anchor_boxes,
image_info,
pre_nms_num_detections=1000,
post_nms_num_detections=100,
nms_threshold=0.3,
bbox_reg_weights=(10., 10., 5., 5.)):
"""Generate the final detections per image given the model outputs.
Args:
cls_outputs: a tensor with shape [N, num_classes], which stacks class
logit outputs on all feature levels. The N is the number of total anchors
on all levels. The num_classes is the number of classes predicted by the
model. Note that the cls_outputs should be the output of softmax().
box_outputs: a tensor with shape [N, num_classes*4], which stacks box
regression outputs on all feature levels. The N is the number of total
anchors on all levels.
anchor_boxes: a tensor with shape [N, 4], which stacks anchors on all
feature levels. The N is the number of total anchors on all levels.
image_info: a tensor of shape [5] which encodes the input image's [height,
width, scale, original_height, original_width]
pre_nms_num_detections: an integer that specifies the number of candidates
before NMS.
post_nms_num_detections: an integer that specifies the number of candidates
after NMS.
nms_threshold: a float number to specify the IOU threshold of NMS.
bbox_reg_weights: a list of 4 float scalars, which are default weights on
(dx, dy, dw, dh) for normalizing bbox regression targets.
Returns:
detections: Tuple of tensors corresponding to number of valid boxes,
box coordinates, object categories for each boxes, and box scores
-- respectively.
"""
num_boxes, num_classes = cls_outputs.get_shape().as_list()
# Remove background class scores.
cls_outputs = cls_outputs[:, 1:num_classes]
top_k_scores, top_k_indices_with_classes = tf.nn.top_k(
tf.reshape(cls_outputs, [-1]), k=pre_nms_num_detections, sorted=False)
classes = tf.mod(top_k_indices_with_classes, num_classes - 1)
top_k_indices = tf.floordiv(top_k_indices_with_classes, num_classes - 1)
anchor_boxes = tf.gather(anchor_boxes, top_k_indices)
box_outputs = tf.reshape(box_outputs,
[num_boxes, num_classes, 4])[:, 1:num_classes, :]
class_indices = classes
box_outputs = tf.gather_nd(
box_outputs, tf.stack([top_k_indices, class_indices], axis=1))
# apply bounding box regression to anchors
boxes = box_utils.decode_boxes(box_outputs, anchor_boxes, bbox_reg_weights)
boxes = box_utils.clip_boxes(boxes, image_info[0], image_info[1])
list_of_all_boxes = []
list_of_all_scores = []
list_of_all_classes = []
# Skip background class.
for class_i in range(num_classes):
# Compute bitmask for the given classes.
class_i_bitmask = tf.cast(tf.equal(classes, class_i),
top_k_scores.dtype)
# This works because score is in [0, 1].
class_i_scores = top_k_scores * class_i_bitmask
# The TPU and CPU have different behaviors for
# tf.image.non_max_suppression_padded (b/116754376).
(class_i_post_nms_indices,
class_i_nms_num_valid) = tf.image.non_max_suppression_padded(
tf.to_float(boxes),
tf.to_float(class_i_scores),
post_nms_num_detections,
iou_threshold=nms_threshold,
score_threshold=0.05,
pad_to_max_output_size=True,
name='nms_detections_' + str(class_i))
class_i_post_nms_boxes = tf.gather(boxes, class_i_post_nms_indices)
class_i_post_nms_scores = tf.gather(class_i_scores,
class_i_post_nms_indices)
mask = tf.less(tf.range(post_nms_num_detections),
[class_i_nms_num_valid])
class_i_post_nms_scores = tf.where(
mask, class_i_post_nms_scores,
tf.zeros_like(class_i_post_nms_scores))
class_i_classes = tf.fill(tf.shape(class_i_post_nms_scores),
class_i + 1)
list_of_all_boxes.append(class_i_post_nms_boxes)
list_of_all_scores.append(class_i_post_nms_scores)
list_of_all_classes.append(class_i_classes)
post_nms_boxes = tf.concat(list_of_all_boxes, axis=0)
post_nms_scores = tf.concat(list_of_all_scores, axis=0)
post_nms_classes = tf.concat(list_of_all_classes, axis=0)
# sort all results.
post_nms_scores, sorted_indices = tf.nn.top_k(tf.to_float(post_nms_scores),
k=post_nms_num_detections,
sorted=True)
post_nms_boxes = tf.gather(post_nms_boxes, sorted_indices)
post_nms_classes = tf.gather(post_nms_classes, sorted_indices)
valid_mask = tf.where(tf.greater(post_nms_scores, 0),
tf.ones_like(post_nms_scores),
tf.zeros_like(post_nms_scores))
num_valid_boxes = tf.reduce_sum(valid_mask, axis=-1)
box_classes = tf.to_float(post_nms_classes)
return num_valid_boxes, post_nms_boxes, box_classes, post_nms_scores
def _boundary_circular(sample_coords, input_size):
sample_coords, input_size = _param_type_and_shape(sample_coords, input_size)
return tf.mod(tf.mod(sample_coords, input_size) + input_size, input_size)
def create_model(inputs, targets, classes_real, classes_fake):
def create_discriminator(discrim_inputs, discrim_targets):
n_layers = 3
layers = []
# 2x [batch, height, width, in_channels] => [batch, height, width, in_channels * 2]
input = tf.concat([discrim_inputs, discrim_targets], axis=3)
# layer_1: [batch, 256, 256, in_channels * 2] => [batch, 128, 128, ndf]
with tf.variable_scope("layer_1"):
convolved = conv(input, a.ndf, stride=2)
rectified = lrelu(convolved, 0.2)
layers.append(rectified)
# layer_2: [batch, 128, 128, ndf] => [batch, 64, 64, ndf * 2]
# layer_3: [batch, 64, 64, ndf * 2] => [batch, 32, 32, ndf * 4]
# layer_4: [batch, 32, 32, ndf * 4] => [batch, 31, 31, ndf * 8]
for i in range(n_layers):
with tf.variable_scope("layer_%d" % (len(layers) + 1)):
out_channels = a.ndf * min(2**(i + 1), 8)
stride = 1 if i == n_layers - 1 else 2 # last layer here has stride 1
convolved = conv(layers[-1], out_channels, stride=stride)
normalized = batchnorm(convolved)
rectified = lrelu(normalized, 0.2)
layers.append(rectified)
# layer_5: [batch, 31, 31, ndf * 8] => [batch, 30, 30, 1]
with tf.variable_scope("layer_%d" % (len(layers) + 1)):
convolved = conv(rectified, out_channels=1, stride=1)
#output = tf.sigmoid(convolved)
#layers.append(output)
layers.append(convolved)
# layer 6: [batch, 30, 30, 1] => [batch, 2k]
with tf.variable_scope("layer_%d" % (len(layers) + 1)):
conv_flat = tf.reshape(layers[-1], [-1, 30 * 30 * 1])
fully_connected = tf.layers.dense(conv_flat,
units=(2 * NUM_CLASSES))
layers.append(fully_connected)
return layers[-1]
with tf.variable_scope("generator") as scope:
out_channels = int(targets.get_shape()[-1])
outputs = create_generator(inputs, out_channels)
# create two copies of discriminator, one for real pairs and one for fake pairs
# they share the same underlying variables
with tf.name_scope("real_discriminator"):
with tf.variable_scope("discriminator"):
# 2x [batch, height, width, channels] => [batch, 2k]
real_outputs = create_discriminator(inputs, targets)
#real_outputs = tf.Print(real_outputs, [real_outputs, real_outputs.get_shape()], message='Real Output:',summarize=5)
with tf.name_scope("fake_discriminator"):
with tf.variable_scope("discriminator", reuse=True):
# 2x [batch, height, width, channels] => [batch, 2k]
fake_outputs = create_discriminator(inputs, outputs)
#fake_outputs = tf.Print(fake_outputs, [fake_outputs, fake_outputs.get_shape()], message='Fake Output:',summarize=5)
with tf.name_scope("discriminator_loss"):
shape = classes_real.get_shape().dims
real_penalty = tf.constant(a.penalties[0], shape=shape)
fake_penalty = tf.constant(a.penalties[0], shape=shape)
num_classes_const = tf.constant(NUM_CLASSES, shape=shape)
# A , F&C, has incorrect trailing / leading zeros and correct class
# B , C&F, has correct trailing / leading zeros and incorrect class
# C , F&F, has inccorect trailing / leading zeros and incorrect class
A_const = tf.constant(a.penalties[1], shape=shape)
B_const = tf.constant(a.penalties[2], shape=shape)
C_const = tf.constant(a.penalties[3], shape=shape)
# predictions
real_softmax = tf.nn.softmax(real_outputs, dim=-1)
fake_softmax = tf.nn.softmax(fake_outputs, dim=-1)
real_class_pred = tf.cast(tf.argmax(real_softmax, axis=1), tf.int32)
fake_class_pred = tf.cast(tf.argmax(fake_softmax, axis=1), tf.int32)
# determine if real / fake prediction was correct
real_binary = tf.greater(real_class_pred, num_classes_const)
fake_binary = tf.less(fake_class_pred, num_classes_const)
# determine if class prediction was correct
real_class_binary = tf.not_equal(
tf.mod(real_class_pred, num_classes_const), classes_real)
fake_class_binary = tf.not_equal(
tf.mod(fake_class_pred, num_classes_const), classes_real)
# get wrong wrong instances
real_ = tf.logical_and(real_binary, real_class_binary)
fake_ = tf.logical_and(fake_binary, fake_class_binary)
# penalty calc
real_penalty = tf.where(real_binary, A_const, real_penalty)
real_penalty = tf.where(real_class_binary, B_const, real_penalty)
real_penalty = tf.where(real_, C_const, real_penalty)
fake_penalty = tf.where(fake_binary, A_const, fake_penalty)
fake_penalty = tf.where(fake_class_binary, B_const, fake_penalty)
fake_penalty = tf.where(fake_, C_const, fake_penalty)
real_penalty = tf.cast(real_penalty, tf.float32)
fake_penalty = tf.cast(fake_penalty, tf.float32)
# calculate loss
predict_real = tf.reduce_sum(real_softmax[:, :NUM_CLASSES], axis=1)
predict_fake = tf.reduce_sum(fake_softmax[:, NUM_CLASSES:], axis=1)
discrim_unsupervised_loss = tf.reduce_sum(
-(tf.log(predict_real + EPS) + tf.log(predict_fake + EPS)))
real_cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(
labels=classes_real, logits=real_outputs)
fake_cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(
labels=classes_fake, logits=fake_outputs)
discrim_real_supervised_loss = tf.reduce_sum(
tf.multiply(real_cross_entropy, real_penalty))
discrim_fake_supervised_loss = tf.reduce_sum(
tf.multiply(fake_cross_entropy, fake_penalty))
discrim_loss = tf.add_n([
discrim_unsupervised_loss, discrim_real_supervised_loss,
discrim_fake_supervised_loss
])
with tf.name_scope("generator_loss"):
# abs(targets - outputs) => 0
gen_supervised_loss = tf.reduce_sum(
tf.nn.sparse_softmax_cross_entropy_with_logits(
labels=classes_real, logits=fake_outputs))
gen_loss_GAN = tf.reduce_sum(-tf.log(1 - predict_fake +
EPS)) + gen_supervised_loss
gen_loss_L1 = tf.reduce_mean(tf.abs(targets - outputs))
gen_loss = gen_loss_GAN * a.gan_weight + gen_loss_L1 * a.l1_weight
with tf.name_scope("discriminator_train"):
discrim_tvars = [
var for var in tf.trainable_variables()
if var.name.startswith("discriminator")
]
discrim_optim = tf.train.AdamOptimizer(a.lr, a.beta1)
discrim_grads_and_vars = discrim_optim.compute_gradients(
discrim_loss, var_list=discrim_tvars)
discrim_train = discrim_optim.apply_gradients(discrim_grads_and_vars)
with tf.name_scope("generator_train"):
with tf.control_dependencies([discrim_train]):
gen_tvars = [
var for var in tf.trainable_variables()
if var.name.startswith("generator")
]
gen_optim = tf.train.AdamOptimizer(a.lr, a.beta1)
gen_grads_and_vars = gen_optim.compute_gradients(
gen_loss, var_list=gen_tvars)
gen_train = gen_optim.apply_gradients(gen_grads_and_vars)
ema = tf.train.ExponentialMovingAverage(decay=0.99)
update_losses = ema.apply([discrim_loss, gen_loss_GAN, gen_loss_L1])
global_step = tf.contrib.framework.get_or_create_global_step()
incr_global_step = tf.assign(global_step, global_step + 1)
return Model(
predict_real=predict_real,
predict_fake=predict_fake,
discrim_loss=ema.average(discrim_loss),
discrim_grads_and_vars=discrim_grads_and_vars,
gen_loss_GAN=ema.average(gen_loss_GAN),
gen_loss_L1=ema.average(gen_loss_L1),
gen_grads_and_vars=gen_grads_and_vars,
outputs=outputs,
train=tf.group(update_losses, incr_global_step, gen_train),
)