def main():
tf.set_random_seed(10)
with tf.Session() as sess:
rnn_cell = tf.nn.rnn_cell.LSTMCell(10)
# defining initial state
initial_state = rnn_cell.zero_state(4, dtype=tf.float32)
inputs = tf.Variable(tf.random_uniform(shape = (4, 30, 100)), name='input')
inputs = tf.identity(inputs, "input_node")
# 'state' is a tensor of shape [batch_size, cell_state_size]
outputs, state = tf.nn.dynamic_rnn(rnn_cell, inputs, initial_state=initial_state, dtype=tf.float32)
y1 = tf.identity(outputs, 'outputs')
y2 = tf.identity(state, 'state')
t1 = tf.ones([4, 30, 10])
t2 = tf.ones([4, 10])
loss = tf.reduce_sum((y1 - t1) * (y1 - t1)) + tf.reduce_sum((y2 - t2) * (y2 - t2))
tf.identity(loss, name = "lstm_loss")
# tf.summary.FileWriter('/tmp/log', tf.get_default_graph())
net_outputs = map(lambda x: tf.get_default_graph().get_tensor_by_name(x), argv[2].split(','))
run_model(net_outputs, argv[1], None, argv[3] == 'True')
def predict(sdae, data_set, bias_node=False):
with sdae.session.graph.as_default():
labels_placeholder = tf.placeholder(tf.int32, shape=1,\
name='labels_placeholder')
examples_placeholder = tf.placeholder(tf.float32,\
shape=(1, sdae._net_shape[0]),\
name='input_pl')
logits = tf.identity(examples_placeholder)
for layer in sdae.get_layers:
if bias_node:
bias_n = tf.ones(shape=[1, 1], dtype=tf.float32)
logits = tf.concat(1, [bias_n, logits])
logits = layer.clean_activation(x_in=logits, use_fixed=False)
predictions = tf.argmax(logits, 1)
labels = tf.identity(labels_placeholder)
y_pred = []
y_true = []
for _ in xrange(data_set.num_examples):
feed_dict = fill_feed_dict(data_set,
examples_placeholder,
labels_placeholder, 1)
y_prediction, y_trues = sdae.session.run([predictions, labels], feed_dict=feed_dict)
y_pred += list(y_prediction)
y_true += list(y_trues)
# print(y_pred)
return y_pred, y_true
def testBijectorWithTrivialTransform(self):
flat_x_ = np.random.normal(0., 1., 8).astype(np.float32)
batched_x_ = np.random.normal(0., 1., (3, 8)).astype(np.float32)
for x_ in [flat_x_, batched_x_]:
nvp = tfb.RealNVP(
num_masked=4,
validate_args=True,
shift_and_log_scale_fn=lambda x, _: (x, x),
is_constant_jacobian=False)
x = tf.constant(x_)
forward_x = nvp.forward(x)
# Use identity to invalidate cache.
inverse_y = nvp.inverse(tf.identity(forward_x))
forward_inverse_y = nvp.forward(inverse_y)
fldj = nvp.forward_log_det_jacobian(x, event_ndims=1)
# Use identity to invalidate cache.
ildj = nvp.inverse_log_det_jacobian(tf.identity(forward_x), event_ndims=1)
forward_x_ = self.evaluate(forward_x)
inverse_y_ = self.evaluate(inverse_y)
forward_inverse_y_ = self.evaluate(forward_inverse_y)
ildj_ = self.evaluate(ildj)
fldj_ = self.evaluate(fldj)
self.assertEqual("real_nvp", nvp.name)
self.assertAllClose(forward_x_, forward_inverse_y_, rtol=1e-4, atol=0.)
self.assertAllClose(x_, inverse_y_, rtol=1e-4, atol=0.)
self.assertAllClose(ildj_, -fldj_, rtol=1e-6, atol=0.)
def testConstructionAndValue(self):
with self.test_session() as sess:
mu = [0.0, 0.1, 0.2]
sigma = tf.constant([1.1, 1.2, 1.3])
sigma2 = tf.constant([0.1, 0.2, 0.3])
with self.assertRaisesRegexp(ValueError, 'No value type currently set'):
prior = sg.DistributionTensor(distributions.Normal, mu=mu, sigma=sigma)
prior_0 = sg.DistributionTensor(
distributions.Normal, mu=mu, sigma=sigma,
dist_value_type=sg.SampleAndReshapeValue())
with sg.value_type(sg.SampleAndReshapeValue()):
prior = sg.DistributionTensor(distributions.Normal, mu=mu, sigma=sigma)
likelihood = sg.DistributionTensor(
distributions.Normal, mu=prior, sigma=sigma2)
coll = tf.get_collection(sg.STOCHASTIC_TENSOR_COLLECTION)
self.assertEqual(coll, [prior_0, prior, likelihood])
prior_0 = tf.identity(prior_0)
prior = tf.identity(prior) # Also works: tf.convert_to_tensor(prior)
likelihood = tf.identity(likelihood)
# Mostly a smoke test for now...
prior_0_val, prior_val, _ = sess.run(
[prior_0, prior, likelihood])
self.assertEqual(prior_0_val.shape, prior_val.shape)
# These are different random samples from the same distribution,
# so the values should differ.
self.assertGreater(np.abs(prior_0_val - prior_val).sum(), 1e-6)
def testBatchedBijectorWithMLPTransform(self):
x_ = np.random.normal(0., 1., (3, 8)).astype(np.float32)
nvp = tfb.RealNVP(
num_masked=4, validate_args=True, **self._real_nvp_kwargs)
x = tf.constant(x_)
forward_x = nvp.forward(x)
# Use identity to invalidate cache.
inverse_y = nvp.inverse(tf.identity(forward_x))
forward_inverse_y = nvp.forward(inverse_y)
fldj = nvp.forward_log_det_jacobian(x, event_ndims=1)
# Use identity to invalidate cache.
ildj = nvp.inverse_log_det_jacobian(tf.identity(forward_x), event_ndims=1)
self.evaluate(tf.global_variables_initializer())
[
forward_x_,
inverse_y_,
forward_inverse_y_,
ildj_,
fldj_,
] = self.evaluate([
forward_x,
inverse_y,
forward_inverse_y,
ildj,
fldj,
])
self.assertEqual("real_nvp", nvp.name)
self.assertAllClose(forward_x_, forward_inverse_y_, rtol=1e-4, atol=0.)
self.assertAllClose(x_, inverse_y_, rtol=1e-4, atol=0.)
self.assertAllClose(ildj_, -fldj_, rtol=1e-6, atol=0.)
def batch_norm(value, is_train = True, name = 'batch_norm', epsilon = 1e-5, momentum = 0.9):
#return value
with tf.variable_scope(name, reuse=tf.AUTO_REUSE):
ema = tf.train.ExponentialMovingAverage(decay = momentum)
shape = value.get_shape().as_list()[-1]
beta = bias('beta', [shape], bias_start = 0.0)
gamma = bias('gamma', [shape], bias_start = 1.0)
if is_train:
batch_mean, batch_variance = tf.nn.moments(value, [0, 1, 2], name = 'moments')
moving_mean = bias('moving_mean', [shape], 0.0, False)
moving_variance = bias('moving_variance', [shape], 1.0, False)
ema_apply_op = ema.apply([batch_mean, batch_variance])
assign_mean = moving_mean.assign(ema.average(batch_mean))
assign_variance = \
moving_variance.assign(ema.average(batch_variance))
with tf.control_dependencies([ema_apply_op]):
mean, variance = \
tf.identity(batch_mean), tf.identity(batch_variance)
with tf.control_dependencies([assign_mean, assign_variance]):
return tf.nn.batch_normalization(value, mean, variance, beta, gamma, 1e-5)
else:
mean = bias('moving_mean', [shape], 0.0, False)
variance = bias('moving_variance', [shape], 1.0, False)
return tf.nn.batch_normalization(value, mean, variance, beta, gamma, epsilon)
def _fn(*args):
p = tf.identity(proposal_log_prob_fn(*args), name="proposal_log_prob")
t = tf.identity(target_log_prob_fn(*args), name="target_log_prob")
dtype = p.dtype.base_dtype
beta = tf.cast(iter_ + 1, dtype) / tf.cast(num_steps, dtype)
return tf.identity(beta * t + (1. - beta) * p,
name="convex_combined_log_prob")
def model(inputs, is_training):
"""Constructs the ResNet model given the inputs."""
if data_format == 'channels_first':
# Convert the inputs from channels_last (NHWC) to channels_first (NCHW).
# This provides a large performance boost on GPU. See
# https://www.tensorflow.org/performance/performance_guide#data_formats
inputs = tf.transpose(inputs, [0, 3, 1, 2])
inputs = conv2d_fixed_padding(
inputs=inputs, filters=16, kernel_size=3, strides=1,
data_format=data_format)
inputs = tf.identity(inputs, 'initial_conv')
inputs = block_layer(
inputs=inputs, filters=16, block_fn=building_block, blocks=num_blocks,
strides=1, is_training=is_training, name='block_layer1',
data_format=data_format)
inputs = block_layer(
inputs=inputs, filters=32, block_fn=building_block, blocks=num_blocks,
strides=2, is_training=is_training, name='block_layer2',
data_format=data_format)
inputs = block_layer(
inputs=inputs, filters=64, block_fn=building_block, blocks=num_blocks,
strides=2, is_training=is_training, name='block_layer3',
data_format=data_format)
inputs = batch_norm_relu(inputs, is_training, data_format)
inputs = tf.layers.average_pooling2d(
inputs=inputs, pool_size=8, strides=1, padding='VALID',
data_format=data_format)
inputs = tf.identity(inputs, 'final_avg_pool')
inputs = tf.reshape(inputs, [-1, 64])
inputs = tf.layers.dense(inputs=inputs, units=num_classes)
inputs = tf.identity(inputs, 'final_dense')
return inputs
def _kl_entropy(self):
"""
Add to Graph:
1. KL divergence between old and new distributions
2. Entropy of present policy given states and actions
"""
log_det_cov_old = tf.reduce_sum(self.old_log_vars_ph)
log_det_cov_new = tf.reduce_sum(self.log_vars)
tr_old_new = tf.reduce_sum(tf.exp(self.old_log_vars_ph - self.log_vars))
#KL Divergence formultivariate normal ditributions
#https://en.wikipedia.org/wiki/Kullback%E2%80%93Leibler_divergence#Multivariate_normal_distributions
# log(sigma1/sigma0) = log(sigma1)-log(sigma0)
# tr: matrix trace
self.kl = 0.5 * tf.reduce_mean(log_det_cov_new - log_det_cov_old + tr_old_new +
# (mu1-mu0 )T*SIGMA^-1*(mu1-mu0):
tf.reduce_sum(tf.square(self.means - self.old_means_ph) /
tf.exp(self.log_vars), axis=1) -
self.act_dim)
# k = act_dim;
self.kl = tf.identity(self.kl, name="kl")
# simply the entropy formula of a multivariate normal distribution
# https://en.wikipedia.org/wiki/Multivariate_normal_distribution#Entropy
self.entropy = 0.5 * (self.act_dim * (np.log(2 * np.pi) + 1) +
tf.reduce_sum(self.log_vars))
self.entropy = tf.identity(self.entropy, name="entropy")
def train_phase(): mean, variance = tf.nn.moments(inputs, axis) update_moving_mean = moving_averages.assign_moving_average(moving_mean, mean, decay) update_moving_variance = moving_averages.assign_moving_average(moving_variance, variance, decay) with tf.control_dependencies([update_moving_mean, update_moving_variance]): return tf.identity(mean), tf.identity(variance)
def mean_var_with_update():
ema_apply_op = ema.apply([batch_mean, batch_var])
pop_mean_op = tf.assign(pop_mean, ema.average(batch_mean))
pop_var_op = tf.assign(pop_var, ema.average(batch_var))
with tf.control_dependencies([ema_apply_op, pop_mean_op, pop_var_op]):
return tf.identity(batch_mean), tf.identity(batch_var)
def testConstructionAndValue(self):
with self.test_session() as sess:
mu = [0.0, 0.1, 0.2]
sigma = tf.constant([1.1, 1.2, 1.3])
sigma2 = tf.constant([0.1, 0.2, 0.3])
prior_default = st.StochasticTensor(distributions.Normal(mu=mu, sigma=sigma))
self.assertTrue(isinstance(prior_default.value_type, st.SampleValue))
prior_0 = st.StochasticTensor(distributions.Normal(mu=mu, sigma=sigma), dist_value_type=st.SampleValue())
self.assertTrue(isinstance(prior_0.value_type, st.SampleValue))
with st.value_type(st.SampleValue()):
prior = st.StochasticTensor(distributions.Normal(mu=mu, sigma=sigma))
self.assertTrue(isinstance(prior.value_type, st.SampleValue))
likelihood = st.StochasticTensor(distributions.Normal(mu=prior, sigma=sigma2))
self.assertTrue(isinstance(likelihood.value_type, st.SampleValue))
coll = tf.get_collection(st.STOCHASTIC_TENSOR_COLLECTION)
self.assertEqual(coll, [prior_default, prior_0, prior, likelihood])
# Also works: tf.convert_to_tensor(prior)
prior_default = tf.identity(prior_default)
prior_0 = tf.identity(prior_0)
prior = tf.identity(prior)
likelihood = tf.identity(likelihood)
# Mostly a smoke test for now...
prior_0_val, prior_val, prior_default_val, _ = sess.run([prior_0, prior, prior_default, likelihood])
self.assertEqual(prior_0_val.shape, prior_val.shape)
self.assertEqual(prior_default_val.shape, prior_val.shape)
# These are different random samples from the same distribution,
# so the values should differ.
self.assertGreater(np.abs(prior_0_val - prior_val).sum(), 1e-6)
self.assertGreater(np.abs(prior_default_val - prior_val).sum(), 1e-6)
def _update_policy_step(self, observ, action, old_mean, old_logstd, advantage, length):
"""Compute the current policy loss and perform a gradient update step.
Args:
observ: Sequences of observations.
action: Sequences of actions.
old_mean: Sequences of action means of the behavioral policy.
old_logstd: Sequences of action log stddevs of the behavioral policy.
advantage: Sequences of advantages.
length: Batch of sequence lengths.
Returns:
Tuple of loss tensor and summary tensor.
"""
network = self._network(observ, length)
loss, summary = self._policy_loss(network.mean, network.logstd, old_mean, old_logstd, action,
advantage, length)
gradients, variables = (zip(*self._policy_optimizer.compute_gradients(loss)))
optimize = self._policy_optimizer.apply_gradients(zip(gradients, variables))
summary = tf.summary.merge([
summary,
tf.summary.scalar('gradient_norm', tf.global_norm(gradients)),
utility.gradient_summaries(zip(gradients, variables), dict(policy=r'.*'))
])
with tf.control_dependencies([optimize]):
return [tf.identity(loss), tf.identity(summary)]
def inference(inputs, is_training=True, scope=''):
batch_norm_params = {'decay': 0.99, 'epsilon': 0.001}
with scopes.arg_scope([ops.conv2d, ops.fc], weight_decay=0.0005,
is_training=is_training, batch_norm_params=batch_norm_params):
# get features from coarse layers
coarse_features = coarse_layers(inputs)
coarse_features_dim = coarse_features.get_shape()[1] # width
# calculate saliency scores and extract top k
coarse_output = top_layers(coarse_features)
coarse_h = entropy(tf.nn.softmax(coarse_output))
coarse_grads = tf.gradients(coarse_h, coarse_features, name='gradient_entropy')
top_k_values, top_k_idxs, M = identify_saliency(coarse_grads[0])
with tf.control_dependencies([top_k_idxs]):
top_k_idxs = tf.identity(top_k_idxs)
coarse_features = tf.identity(coarse_features)
# get features from fine layers
fine_features, src_idxs, k_patches = extract_features(inputs, top_k_idxs, coarse_features_dim)
# merge two feature maps
merged, flat_coarse, flat_fine = replace_features(coarse_features, fine_features, src_idxs)
raw_hint_loss = tf.reduce_sum(tf.square(flat_coarse - flat_fine), name='raw_hint_loss')
# scale hint loss per example in batch
# still does not match range of 5-25 shown in figure 2 in paper???
hint_loss = tf.div( raw_hint_loss, inputs.get_shape()[0].value*N_PATCHES, name='objective_hint')
tf.get_variable_scope().reuse_variables()
final_logits = top_layers(merged)
return final_logits, hint_loss
def _test_no_active_dims(Kern, sess):
S, N, M, D = 5, 4, 3, 2
X1 = tf.identity(np.random.randn(S, N, D))
X2 = tf.identity(np.random.randn(S, M, D))
kern = Kern(D) + gpflow.kernels.White(2)
compare_vs_map(X1, X2, kern, sess)
def test_rank_one_tensor_doesnt_raise_if_rank_just_right_static_rank(self):
with self.test_session():
tensor = tf.constant([1, 2], name="my_tensor")
desired_rank = 1
with tf.control_dependencies([tf.assert_rank_at_least(tensor,
desired_rank)]):
tf.identity(tensor).eval()
def test_rank_one_tensor_doesnt_raise_if_rank_just_right_dynamic_rank(self):
with self.test_session():
tensor = tf.placeholder(tf.float32, name="my_tensor")
desired_rank = 1
with tf.control_dependencies([tf.assert_rank_at_least(tensor,
desired_rank)]):
tf.identity(tensor).eval(feed_dict={tensor: [1, 2]})
def test_rank_one_tensor_raises_if_rank_too_small_dynamic_rank(self):
with self.test_session():
tensor = tf.placeholder(tf.float32, name="my_tensor")
desired_rank = 2
with tf.control_dependencies([tf.assert_rank(tensor, desired_rank)]):
with self.assertRaisesOpError("my_tensor.*rank"):
tf.identity(tensor).eval(feed_dict={tensor: [1, 2]})
def test_raises_if_rank_is_not_scalar_dynamic(self):
with self.test_session():
tensor = tf.constant([1, 2], dtype=tf.float32, name="my_tensor")
rank_tensor = tf.placeholder(tf.int32, name="rank_tensor")
with self.assertRaisesOpError("Rank must be a scalar"):
with tf.control_dependencies([tf.assert_rank(tensor, rank_tensor)]):
tf.identity(tensor).eval(feed_dict={rank_tensor: [1, 2]})
def parser(example):
features = {
'xywhc': tf.FixedLenFeature([150], tf.float32),
'img': tf.FixedLenFeature((), tf.string)}
feats = tf.parse_single_example(example, features)
coord = feats['xywhc']
coord = tf.reshape(coord, [30, 5])
img = tf.decode_raw(feats['img'], tf.float32)
img = tf.reshape(img, [416, 416, 3])
img = tf.image.resize_images(img, [cfg.train.image_resized, cfg.train.image_resized])
rnd = tf.less(tf.random_uniform(shape=[], minval=0, maxval=2), 1)
def flip_img_coord(_img, _coord):
zeros = tf.constant([[0, 0, 0, 0, 0]]*30, tf.float32)
img_flipped = tf.image.flip_left_right(_img)
idx_invalid = tf.reduce_all(tf.equal(coord, 0), axis=-1)
coord_temp = tf.concat([tf.minimum(tf.maximum(1 - _coord[:, :1], 0), 1),
_coord[:, 1:]], axis=-1)
coord_flipped = tf.where(idx_invalid, zeros, coord_temp)
return img_flipped, coord_flipped
img, coord = tf.cond(rnd, lambda: (tf.identity(img), tf.identity(coord)), lambda: flip_img_coord(img, coord))
img = tf.image.random_hue(img, max_delta=0.1)
img = tf.image.random_contrast(img, lower=0.8, upper=1.2)
img = tf.image.random_brightness(img, max_delta=0.1)
img = tf.image.random_saturation(img, lower=0.8, upper=1.2)
img = tf.minimum(img, 1.0)
img = tf.maximum(img, 0.0)
return img, coord
def test_rank_one_tensor_raises_if_rank_too_small_static_rank(self):
with self.test_session():
tensor = tf.constant([1, 2], name="my_tensor")
desired_rank = 2
with self.assertRaisesRegexp(ValueError, "my_tensor.*rank"):
with tf.control_dependencies([tf.assert_rank(tensor, desired_rank)]):
tf.identity(tensor).eval()
def __call__(self, x, train=True):
shape = x.get_shape().as_list()
if train:
with tf.variable_scope(self.name) as scope:
self.beta = tf.get_variable("beta", [shape[-1]],
initializer=tf.constant_initializer(0.))
self.gamma = tf.get_variable("gamma", [shape[-1]],
initializer=tf.random_normal_initializer(1., 0.02))
try:
batch_mean, batch_var = tf.nn.moments(x, [0, 1, 2], name='moments')
except:
batch_mean, batch_var = tf.nn.moments(x, [0, 1], name='moments')
ema_apply_op = self.ema.apply([batch_mean, batch_var])
self.ema_mean, self.ema_var = self.ema.average(batch_mean), self.ema.average(batch_var)
with tf.control_dependencies([ema_apply_op]):
mean, var = tf.identity(batch_mean), tf.identity(batch_var)
else:
mean, var = self.ema_mean, self.ema_var
normed = tf.nn.batch_norm_with_global_normalization(
x, mean, var, self.beta, self.gamma, self.epsilon, scale_after_normalization=True)
return normed
def mean_var_with_update():
if self.ema_apply_op is None:
self.ema_apply_op = self.ema.apply(
[self.batch_mean, self.batch_var])
with tf.control_dependencies([self.ema_apply_op]):
return tf.identity(self.batch_mean), \
tf.identity(self.batch_var)
def testBijector(self):
x_ = np.arange(3 * 4 * 2).astype(np.float32).reshape(3, 4, 2)
with self.test_session() as sess:
ma = tfb.MaskedAutoregressiveFlow(
validate_args=True, **self._autoregressive_flow_kwargs)
x = tf.constant(x_)
forward_x = ma.forward(x)
# Use identity to invalidate cache.
inverse_y = ma.inverse(tf.identity(forward_x))
fldj = ma.forward_log_det_jacobian(x, event_ndims=1)
# Use identity to invalidate cache.
ildj = ma.inverse_log_det_jacobian(tf.identity(forward_x), event_ndims=1)
tf.global_variables_initializer().run()
[
forward_x_,
inverse_y_,
ildj_,
fldj_,
] = sess.run([
forward_x,
inverse_y,
ildj,
fldj,
])
self.assertEqual("masked_autoregressive_flow", ma.name)
self.assertAllClose(forward_x_, forward_x_, rtol=1e-6, atol=0.)
self.assertAllClose(x_, inverse_y_, rtol=1e-5, atol=0.)
self.assertAllClose(ildj_, -fldj_, rtol=1e-6, atol=0.)
def __build_game_state_as_update_graph(self, training, global_step):
print('game_state_as_update')
with tf.variable_scope('game_state_as_update') as variable_scope:
seed = tf.placeholder(
tf.int64,
[self.seed_size],
'seed')
update_statistic = tf.placeholder(
tf.float32,
[None, self.update_statistic_size],
'update_statistic')
with tf.variable_scope('transformation') as \
transformation_variable_scope:
signal = self._game_state_as_update(
training, global_step,
seed, update_statistic)
output = tf.identity(signal, 'output')
output_gradient = tf.placeholder(
tf.float32,
[None, self.update_size],
'output_gradient')
update_statistic_gradient, = tf.gradients(
output, [update_statistic], output_gradient)
tf.identity(
update_statistic_gradient, 'update_statistic_gradient')
self.__model_gradients(
variable_scope, transformation_variable_scope, output,
output_gradient)
def test_raises_if_rank_is_not_integer_dynamic(self):
with self.test_session():
tensor = tf.constant([1, 2], dtype=tf.float32, name="my_tensor")
rank_tensor = tf.placeholder(tf.float32, name="rank_tensor")
with self.assertRaisesRegexp(TypeError, "must be of type <dtype: 'int32'>"):
with tf.control_dependencies([tf.assert_rank(tensor, rank_tensor)]):
tf.identity(tensor).eval(feed_dict={rank_tensor: 0.5})
def __init__(self, net, scope, classes, boxes_per_cell, training=False):
_, self.cell_height, self.cell_width, _ = tf.get_default_graph().get_tensor_by_name(scope + '/conv:0').get_shape().as_list()
cells = self.cell_height * self.cell_width
with tf.name_scope('regress'):
with tf.name_scope('inputs'):
end = cells * classes
self.prob = tf.reshape(net[:, :end], [-1, cells, 1, classes], name='prob')
inputs_remaining = tf.reshape(net[:, end:], [-1, cells, boxes_per_cell, 5], name='inputs_remaining')
self.iou = tf.identity(inputs_remaining[:, :, :, 0], name='iou')
self.offset_xy = tf.identity(inputs_remaining[:, :, :, 1:3], name='offset_xy')
wh01_sqrt_base = tf.identity(inputs_remaining[:, :, :, 3:], name='wh01_sqrt_base')
wh01 = tf.square(wh01_sqrt_base, name='wh01')
wh01_sqrt = tf.abs(wh01_sqrt_base, name='wh01_sqrt')
self.coords = tf.concat([self.offset_xy, wh01_sqrt], -1, name='coords')
self.wh = tf.identity(wh01 * [self.cell_width, self.cell_height], name='wh')
_wh = self.wh / 2
self.offset_xy_min = tf.identity(self.offset_xy - _wh, name='offset_xy_min')
self.offset_xy_max = tf.identity(self.offset_xy + _wh, name='offset_xy_max')
self.areas = tf.reduce_prod(self.wh, -1, name='areas')
if not training:
with tf.name_scope('detection'):
cell_xy = calc_cell_xy(self.cell_height, self.cell_width).reshape([1, cells, 1, 2])
self.xy = tf.identity(cell_xy + self.offset_xy, name='xy')
self.xy_min = tf.identity(cell_xy + self.offset_xy_min, name='xy_min')
self.xy_max = tf.identity(cell_xy + self.offset_xy_max, name='xy_max')
self.conf = tf.identity(tf.expand_dims(self.iou, -1) * self.prob, name='conf')
self.inputs = net
self.classes = classes
self.boxes_per_cell = boxes_per_cell
def setup(self):
"""Sets up all components of the computation graph."""
self.x, self.y = self.get_xy_placeholders()
with tf.variable_scope('core', reuse=None):
self.loss, self.gradient_ops = self.train(self.x, self.y)
with tf.variable_scope('core', reuse=True):
self.y_preds = self.eval(self.x, self.y)
# setup memory "reset" ops
(self.mem_keys, self.mem_vals,
self.mem_age, self.recent_idx) = self.memory.get()
self.mem_keys_reset = tf.placeholder(self.mem_keys.dtype,
tf.identity(self.mem_keys).shape)
self.mem_vals_reset = tf.placeholder(self.mem_vals.dtype,
tf.identity(self.mem_vals).shape)
self.mem_age_reset = tf.placeholder(self.mem_age.dtype,
tf.identity(self.mem_age).shape)
self.recent_idx_reset = tf.placeholder(self.recent_idx.dtype,
tf.identity(self.recent_idx).shape)
self.mem_reset_op = self.memory.set(self.mem_keys_reset,
self.mem_vals_reset,
self.mem_age_reset,
None)
def test_rank_zero_tensor_raises_if_rank_too_small_static_rank(self):
with self.test_session():
tensor = tf.constant(1, name="my_tensor")
desired_rank = 1
with self.assertRaisesRegexp(ValueError, "fail.*my_tensor.*must have rank 1"):
with tf.control_dependencies([tf.assert_rank(tensor, desired_rank, message="fail")]):
tf.identity(tensor).eval()
def _batch_norm(self, output,
lr_mult=1.0, scope='bn', restore=True):
with tf.variable_scope(scope):
shape = Network.shape(output)
# we don't squeeze only the last dimension, i.e. feature maps
squeeze_dims = range(len(shape)-1)
input_maps = shape[-1]
batch_mean, batch_var = tf.nn.moments(output, squeeze_dims, name='moments')
ema = tf.train.ExponentialMovingAverage(decay=self.decay)
ema_apply_op = ema.apply([batch_mean, batch_var])
# Needed for partial restoration from an existing model
self._set_restoring(ema.average(batch_mean),
Network._append(restore, 'moving_mean'))
self._set_restoring(ema.average(batch_var),
Network._append(restore, 'moving_variance'))
if self.is_train: # and lr_mult > 0):
with tf.control_dependencies([ema_apply_op]):
mean, var = tf.identity(batch_mean), tf.identity(batch_var)
else:
#mean, var = batch_mean, batch_var
mean, var = ema.average(batch_mean), ema.average(batch_var)
beta = self._constant_variable('beta', [input_maps], 0.0, 0.0,
lr_mult, Network._append(restore, 'beta'))
gamma = self._constant_variable('gamma', [input_maps], 1.0, 0.0,
lr_mult, Network._append(restore, 'gamma'))
output = tf.nn.batch_normalization(output, mean, var, beta, gamma, Network.BN_EPS)
return output
def __init__(self,
env_spec,
name='CategoricalGRUPolicy',
hidden_dim=32,
hidden_nonlinearity=tf.nn.tanh,
recurrent_nonlinearity=tf.nn.sigmoid,
recurrent_w_x_init=L.XavierUniformInitializer(),
recurrent_w_h_init=L.OrthogonalInitializer(),
output_nonlinearity=tf.nn.softmax,
output_w_init=L.XavierUniformInitializer(),
feature_network=None,
state_include_action=True,
gru_layer_cls=L.GRULayer):
"""
:param env_spec: A spec for the env.
:param hidden_dim: dimension of hidden layer
:param hidden_nonlinearity: nonlinearity used for each hidden layer
:return:
"""
assert isinstance(env_spec.action_space, Discrete)
self._prob_network_name = 'prob_network'
with tf.variable_scope(name, 'CategoricalGRUPolicy'):
Serializable.quick_init(self, locals())
super(CategoricalGRUPolicy, self).__init__(env_spec)
obs_dim = env_spec.observation_space.flat_dim
action_dim = env_spec.action_space.flat_dim
if state_include_action:
input_dim = obs_dim + action_dim
else:
input_dim = obs_dim
l_input = L.InputLayer(shape=(None, None, input_dim), name='input')
if feature_network is None:
feature_dim = input_dim
l_flat_feature = None
l_feature = l_input
else:
feature_dim = feature_network.output_layer.output_shape[-1]
l_flat_feature = feature_network.output_layer
l_feature = L.OpLayer(
l_flat_feature,
extras=[l_input],
name='reshape_feature',
op=lambda flat_feature, input: tf.reshape(
flat_feature,
tf.stack([
tf.shape(input)[0],
tf.shape(input)[1], feature_dim
])),
shape_op=lambda _, input_shape: (input_shape[
0], input_shape[1], feature_dim))
prob_network = GRUNetwork(
input_shape=(feature_dim, ),
input_layer=l_feature,
output_dim=env_spec.action_space.n,
hidden_dim=hidden_dim,
hidden_nonlinearity=hidden_nonlinearity,
recurrent_nonlinearity=recurrent_nonlinearity,
recurrent_w_x_init=recurrent_w_x_init,
recurrent_w_h_init=recurrent_w_h_init,
output_nonlinearity=output_nonlinearity,
output_w_init=output_w_init,
gru_layer_cls=gru_layer_cls,
name=self._prob_network_name)
self.prob_network = prob_network
self.feature_network = feature_network
self.l_input = l_input
self.state_include_action = state_include_action
flat_input_var = tf.placeholder(
dtype=tf.float32, shape=(None, input_dim), name='flat_input')
if feature_network is None:
feature_var = flat_input_var
else:
with tf.name_scope('feature_network', values=[flat_input_var]):
feature_var = L.get_output(
l_flat_feature,
{feature_network.input_layer: flat_input_var})
with tf.name_scope(self._prob_network_name, values=[feature_var]):
out_prob_step, out_prob_hidden = L.get_output(
[
prob_network.step_output_layer,
prob_network.step_hidden_layer
], {prob_network.step_input_layer: feature_var})
out_prob_step = tf.identity(out_prob_step, 'prob_step_output')
out_prob_hidden = tf.identity(out_prob_hidden,
'prob_step_hidden')
self.f_step_prob = tensor_utils.compile_function(
[flat_input_var, prob_network.step_prev_state_layer.input_var],
[out_prob_step, out_prob_hidden])
self.input_dim = input_dim
self.action_dim = action_dim
self.hidden_dim = hidden_dim
self.name = name
self.prev_actions = None
self.prev_hiddens = None
self.dist = RecurrentCategorical(env_spec.action_space.n)
out_layers = [prob_network.output_layer]
if feature_network is not None:
out_layers.append(feature_network.output_layer)
LayersPowered.__init__(self, out_layers)
def sync_batch_norm(inputs,
red_axises,
is_training=True,
scope=None,
bn_decay=0.999,
epsilon=0.001,
activation_fn=None,
updates_collections=tf.GraphKeys.UPDATE_OPS,
reuse=None,
variables_collections=None,
is_trainable=True,
num_dev=4):
'''
num_dev is how many gpus you use.
'''
# red_axises = [0, 1, 2]
num_outputs = inputs.get_shape().as_list()[-1]
if scope is None:
scope = 'BatchNorm'
with tf.variable_scope(scope, 'BatchNorm', reuse=reuse):
gamma = tf.get_variable(name='gamma',
shape=[num_outputs],
dtype=tf.float32,
initializer=tf.constant_initializer(1.0),
trainable=is_trainable,
collections=variables_collections)
beta = tf.get_variable(name='beta',
shape=[num_outputs],
dtype=tf.float32,
initializer=tf.constant_initializer(0.0),
trainable=is_trainable,
collections=variables_collections)
moving_mean = tf.get_variable(name='moving_mean',
shape=[num_outputs],
dtype=tf.float32,
initializer=tf.constant_initializer(0.0),
trainable=False,
collections=variables_collections)
moving_var = tf.get_variable(name='moving_variance',
shape=[num_outputs],
dtype=tf.float32,
initializer=tf.constant_initializer(1.0),
trainable=False,
collections=variables_collections)
#if is_training and is_trainable:
if is_training is not None and is_trainable is not None:
if num_dev == 1:
mean, var = tf.nn.moments(inputs, red_axises)
else:
shared_name = tf.get_variable_scope().name
batch_mean = tf.reduce_mean(inputs, axis=red_axises)
batch_mean_square = tf.reduce_mean(tf.square(inputs),
axis=red_axises)
batch_mean = gen_nccl_ops.nccl_all_reduce(
input=batch_mean,
reduction='sum',
num_devices=num_dev,
shared_name=shared_name + '_NCCL_mean') * (1.0 / num_dev)
batch_mean_square = gen_nccl_ops.nccl_all_reduce(
input=batch_mean_square,
reduction='sum',
num_devices=num_dev,
shared_name=shared_name + '_NCCL_mean_square') * (1.0 /
num_dev)
mean = batch_mean
var = batch_mean_square - tf.square(batch_mean)
outputs = tf.nn.batch_normalization(inputs, mean, var, beta, gamma,
epsilon)
if int(outputs.device[-1]) == 0:
update_moving_mean_op = tf.assign(
moving_mean,
moving_mean * bn_decay + mean * (1 - bn_decay))
update_moving_var_op = tf.assign(
moving_var, moving_var * bn_decay + var * (1 - bn_decay))
add_model_variable(moving_mean)
add_model_variable(moving_var)
if updates_collections is None:
with tf.control_dependencies(
[update_moving_mean_op, update_moving_var_op]):
outputs = tf.identity(outputs)
else:
# tf.add_to_collections(updates_collections, update_moving_mean_op)
# tf.add_to_collections(updates_collections, update_moving_var_op)
ops.add_to_collections(updates_collections,
update_moving_mean_op)
ops.add_to_collections(updates_collections,
update_moving_var_op)
outputs = tf.identity(outputs)
else:
outputs = tf.identity(outputs)
else:
outputs, _, _ = tf.nn.fused_batch_norm(inputs,
gamma,
beta,
mean=moving_mean,
variance=moving_var,
epsilon=epsilon,
is_training=False)
# if activation_fn is not None:
# outputs = activation_fn(outputs)
return outputs
def mean_var_with_update():
with tf.control_dependencies([ema_apply_op]):
return tf.identity(batch_mean), tf.identity(batch_var)
def model_fn(features, labels, mode, params):
"""Defines how to train, evaluate and predict from the transformer model."""
with tf.variable_scope("model"):
inputs, targets = features, labels
# Create model and get output logits.
model = transformer.Transformer(params,
mode == tf.estimator.ModeKeys.TRAIN)
logits = model(inputs, targets)
# When in prediction mode, the labels/targets is None. The model output
# is the prediction
if mode == tf.estimator.ModeKeys.PREDICT:
if params["use_tpu"]:
raise NotImplementedError(
"Prediction is not yet supported on TPUs.")
return tf.estimator.EstimatorSpec(
tf.estimator.ModeKeys.PREDICT,
predictions=logits,
export_outputs={
"translate": tf.estimator.export.PredictOutput(logits)
})
# Explicitly set the shape of the logits for XLA (TPU). This is needed
# because the logits are passed back to the host VM CPU for metric
# evaluation, and the shape of [?, ?, vocab_size] is too vague. However
# it is known from Transformer that the first two dimensions of logits
# are the dimensions of targets. Note that the ambiguous shape of logits is
# not a problem when computing xentropy, because padded_cross_entropy_loss
# resolves the shape on the TPU.
logits.set_shape(targets.shape.as_list() + logits.shape.as_list()[2:])
# Calculate model loss.
# xentropy contains the cross entropy loss of every nonpadding token in the
# targets.
xentropy, weights = metrics.padded_cross_entropy_loss(
logits, targets, params["label_smoothing"], params["vocab_size"])
loss = tf.reduce_sum(xentropy) / tf.reduce_sum(weights)
# Save loss as named tensor that will be logged with the logging hook.
tf.identity(loss, "cross_entropy")
if mode == tf.estimator.ModeKeys.EVAL:
if params["use_tpu"]:
# host call functions should only have tensors as arguments.
# This lambda pre-populates params so that metric_fn is
# TPUEstimator compliant.
metric_fn = lambda logits, labels: (metrics.get_eval_metrics(
logits, labels, params=params))
eval_metrics = (metric_fn, [logits, labels])
return tf.contrib.tpu.TPUEstimatorSpec(
mode=mode,
loss=loss,
predictions={"predictions": logits},
eval_metrics=eval_metrics)
return tf.estimator.EstimatorSpec(
mode=mode,
loss=loss,
predictions={"predictions": logits},
eval_metric_ops=metrics.get_eval_metrics(
logits, labels, params))
else:
train_op, metric_dict = get_train_op_and_metrics(loss, params)
# Epochs can be quite long. This gives some intermediate information
# in TensorBoard.
metric_dict["minibatch_loss"] = loss
if params["use_tpu"]:
return tf.contrib.tpu.TPUEstimatorSpec(
mode=mode,
loss=loss,
train_op=train_op,
host_call=tpu_util.construct_scalar_host_call(
metric_dict=metric_dict,
model_dir=params["model_dir"],
prefix="training/"))
record_scalars(metric_dict)
return tf.estimator.EstimatorSpec(mode=mode,
loss=loss,
train_op=train_op)
def data_loader(FLAGS):
with tf.device('/cpu:0'):
# Define the returned data batches
Data = collections.namedtuple('Data', 'paths_LR, paths_HR, inputs, targets, image_count, steps_per_epoch')
#Check the input directory
if (FLAGS.input_dir_LR == 'None') or (FLAGS.input_dir_HR == 'None'):
raise ValueError('Input directory is not provided')
if (not os.path.exists(FLAGS.input_dir_LR)) or (not os.path.exists(FLAGS.input_dir_HR)):
raise ValueError('Input directory not found')
image_list_LR = os.listdir(FLAGS.input_dir_LR)
image_list_LR = [_ for _ in image_list_LR if _.endswith('.png')]
if len(image_list_LR)==0:
raise Exception('No png files in the input directory')
image_list_LR_temp = sorted(image_list_LR)
image_list_LR = [os.path.join(FLAGS.input_dir_LR, _) for _ in image_list_LR_temp]
image_list_HR = [os.path.join(FLAGS.input_dir_HR, _) for _ in image_list_LR_temp]
image_list_LR_tensor = tf.convert_to_tensor(image_list_LR, dtype=tf.string)
image_list_HR_tensor = tf.convert_to_tensor(image_list_HR, dtype=tf.string)
with tf.variable_scope('load_image'):
# define the image list queue
# image_list_LR_queue = tf.train.string_input_producer(image_list_LR, shuffle=False, capacity=FLAGS.name_queue_capacity)
# image_list_HR_queue = tf.train.string_input_producer(image_list_HR, shuffle=False, capacity=FLAGS.name_queue_capacity)
#print('[Queue] image list queue use shuffle: %s'%(FLAGS.mode == 'Train'))
output = tf.train.slice_input_producer([image_list_LR_tensor, image_list_HR_tensor],
shuffle=False, capacity=FLAGS.name_queue_capacity)
# Reading and decode the images
reader = tf.WholeFileReader(name='image_reader')
image_LR = tf.read_file(output[0])
image_HR = tf.read_file(output[1])
input_image_LR = tf.image.decode_png(image_LR, channels=3)
input_image_HR = tf.image.decode_png(image_HR, channels=3)
input_image_LR = tf.image.convert_image_dtype(input_image_LR, dtype=tf.float32)
input_image_HR = tf.image.convert_image_dtype(input_image_HR, dtype=tf.float32)
assertion = tf.assert_equal(tf.shape(input_image_LR)[2], 3, message="image does not have 3 channels")
with tf.control_dependencies([assertion]):
input_image_LR = tf.identity(input_image_LR)
input_image_HR = tf.identity(input_image_HR)
# Normalize the low resolution image to [0, 1], high resolution to [-1, 1]
a_image = preprocessLR(input_image_LR)
b_image = preprocess(input_image_HR)
inputs, targets = [a_image, b_image]
# The data augmentation part
with tf.name_scope('data_preprocessing'):
with tf.name_scope('random_crop'):
# Check whether perform crop
if (FLAGS.random_crop is True) and FLAGS.mode == 'train':
print('[Config] Use random crop')
# Set the shape of the input image. the target will have 4X size
input_size = tf.shape(inputs)
target_size = tf.shape(targets)
offset_w = tf.cast(tf.floor(tf.random_uniform([], 0, tf.cast(input_size[1], tf.float32) - FLAGS.crop_size)),
dtype=tf.int32)
offset_h = tf.cast(tf.floor(tf.random_uniform([], 0, tf.cast(input_size[0], tf.float32) - FLAGS.crop_size)),
dtype=tf.int32)
if FLAGS.task == 'SRGAN' or FLAGS.task == 'SRResnet':
inputs = tf.image.crop_to_bounding_box(inputs, offset_h, offset_w, FLAGS.crop_size,
FLAGS.crop_size)
targets = tf.image.crop_to_bounding_box(targets, offset_h*4, offset_w*4, FLAGS.crop_size*4,
FLAGS.crop_size*4)
elif FLAGS.task == 'denoise':
inputs = tf.image.crop_to_bounding_box(inputs, offset_h, offset_w, FLAGS.crop_size,
FLAGS.crop_size)
targets = tf.image.crop_to_bounding_box(targets, offset_h, offset_w,
FLAGS.crop_size, FLAGS.crop_size)
# Do not perform crop
else:
inputs = tf.identity(inputs)
targets = tf.identity(targets)
with tf.variable_scope('random_flip'):
# Check for random flip:
if (FLAGS.flip is True) and (FLAGS.mode == 'train'):
print('[Config] Use random flip')
# Produce the decision of random flip
decision = tf.random_uniform([], 0, 1, dtype=tf.float32)
input_images = random_flip(inputs, decision)
target_images = random_flip(targets, decision)
else:
input_images = tf.identity(inputs)
target_images = tf.identity(targets)
if FLAGS.task == 'SRGAN' or FLAGS.task == 'SRResnet':
input_images.set_shape([FLAGS.crop_size, FLAGS.crop_size, 3])
target_images.set_shape([FLAGS.crop_size*4, FLAGS.crop_size*4, 3])
elif FLAGS.task == 'denoise':
input_images.set_shape([FLAGS.crop_size, FLAGS.crop_size, 3])
target_images.set_shape([FLAGS.crop_size, FLAGS.crop_size, 3])
if FLAGS.mode == 'train':
paths_LR_batch, paths_HR_batch, inputs_batch, targets_batch = tf.train.shuffle_batch([output[0], output[1], input_images, target_images],
batch_size=FLAGS.batch_size, capacity=FLAGS.image_queue_capacity+4*FLAGS.batch_size,
min_after_dequeue=FLAGS.image_queue_capacity, num_threads=FLAGS.queue_thread)
else:
paths_LR_batch, paths_HR_batch, inputs_batch, targets_batch = tf.train.batch([output[0], output[1], input_images, target_images],
batch_size=FLAGS.batch_size, num_threads=FLAGS.queue_thread, allow_smaller_final_batch=True)
steps_per_epoch = int(math.ceil(len(image_list_LR) / FLAGS.batch_size))
if FLAGS.task == 'SRGAN' or FLAGS.task == 'SRResnet':
inputs_batch.set_shape([FLAGS.batch_size, FLAGS.crop_size, FLAGS.crop_size, 3])
targets_batch.set_shape([FLAGS.batch_size, FLAGS.crop_size*4, FLAGS.crop_size*4, 3])
elif FLAGS.task == 'denoise':
inputs_batch.set_shape([FLAGS.batch_size, FLAGS.crop_size, FLAGS.crop_size, 3])
targets_batch.set_shape([FLAGS.batch_size, FLAGS.crop_size, FLAGS.crop_size, 3])
return Data(
paths_LR=paths_LR_batch,
paths_HR=paths_HR_batch,
inputs=inputs_batch,
targets=targets_batch,
image_count=len(image_list_LR),
steps_per_epoch=steps_per_epoch
)
def predict_mean_and_var(self, Fmu, Fvar):
return tf.identity(Fmu), Fvar + self.variance
def conditional_mean(self, F):
return tf.identity(F)
def build_rnn(cell, inputs): #cell是RNN的节点，inputs是字符串的长度
outputs, final_state = tf.nn.dynamic_rnn(cell, inputs, dtype=tf.float32) #一种方便的处理，可以避免填充
# 同样给final_state一个名字，后面要重新获取缓存
final_state = tf.identity(final_state, name="final_state")
return outputs, final_state
def main(unused_argv):
tf.logging.set_verbosity(tf.logging.INFO)
tf.logging.info('Prepare to export model to: %s', FLAGS.export_path)
with tf.Graph().as_default():
image, image_size, resized_image_size = _create_input_tensors()
model_options = common.ModelOptions(
outputs_to_num_classes={common.OUTPUT_TYPE: FLAGS.num_classes},
crop_size=FLAGS.crop_size,
atrous_rates=FLAGS.atrous_rates,
output_stride=FLAGS.output_stride)
if tuple(FLAGS.inference_scales) == (1.0, ):
tf.logging.info('Exported model performs single-scale inference.')
predictions = model.predict_labels(
image,
model_options=model_options,
image_pyramid=FLAGS.image_pyramid)
else:
tf.logging.info('Exported model performs multi-scale inference.')
if FLAGS.quantize_delay_step >= 0:
raise ValueError(
'Quantize mode is not supported with multi-scale test.')
predictions = model.predict_labels_multi_scale(
image,
model_options=model_options,
eval_scales=FLAGS.inference_scales,
add_flipped_images=FLAGS.add_flipped_images)
raw_predictions = tf.identity(
tf.cast(predictions[common.OUTPUT_TYPE], tf.float32),
_RAW_OUTPUT_NAME)
raw_probabilities = tf.identity(
predictions[common.OUTPUT_TYPE + model.PROB_SUFFIX],
_RAW_OUTPUT_PROB_NAME)
# Crop the valid regions from the predictions.
semantic_predictions = raw_predictions[:, :resized_image_size[0], :
resized_image_size[1]]
semantic_probabilities = raw_probabilities[:, :resized_image_size[0], :
resized_image_size[1]]
# Resize back the prediction to the original image size.
def _resize_label(label, label_size):
# Expand dimension of label to [1, height, width, 1] for resize operation.
label = tf.expand_dims(label, 3)
resized_label = tf.image.resize_images(
label,
label_size,
method=tf.image.ResizeMethod.NEAREST_NEIGHBOR,
align_corners=True)
return tf.cast(tf.squeeze(resized_label, 3), tf.int32)
semantic_predictions = _resize_label(semantic_predictions, image_size)
semantic_predictions = tf.identity(semantic_predictions,
name=_OUTPUT_NAME)
semantic_probabilities = tf.image.resize_bilinear(
semantic_probabilities,
image_size,
align_corners=True,
name=_OUTPUT_PROB_NAME)
if FLAGS.quantize_delay_step >= 0:
contrib_quantize.create_eval_graph()
saver = tf.train.Saver(tf.all_variables())
dirname = os.path.dirname(FLAGS.export_path)
tf.gfile.MakeDirs(dirname)
graph_def = tf.get_default_graph().as_graph_def(add_shapes=True)
freeze_graph.freeze_graph_with_def_protos(
graph_def,
saver.as_saver_def(),
tf.train.latest_checkpoint(
FLAGS.checkpoint_path
), # VALOHAI: Updated to fetch the latest checkpoint
_OUTPUT_NAME + ',' + _OUTPUT_PROB_NAME,
restore_op_name=None,
filename_tensor_name=None,
output_graph=FLAGS.export_path,
clear_devices=True,
initializer_nodes=None)
if FLAGS.save_inference_graph:
tf.train.write_graph(graph_def, dirname, 'inference_graph.pbtxt')
def auto_encoder(x, x_org, is_train, opt, opt_t=None):
# print x.get_shape() # batch L
if not opt_t: opt_t = opt
x_emb, W_norm = embedding(x, opt) # batch L emb
x_emb = tf.expand_dims(x_emb, 3) # batch L emb 1
res = {}
#res['W'] = W_norm
# cnn encoder
H_enc, res = conv_encoder(x_emb, is_train, opt, res)
# H_dec = layers.relu(Y4, 200, biases_initializer=biasInit)
H_dec = H_enc
# print x_rec.get_shape()
if opt.model == 'rnn_rnn':
loss, rec_sent_1, _ = seq2seq(x, x_org, opt)
_, rec_sent_2, _ = seq2seq(x, x_org, opt, feed_previous=True, is_reuse=True)
#res['logits'] = logits
res['rec_sents_feed_y'] = rec_sent_1
res['rec_sents'] = rec_sent_2
elif opt.model == 'cnn_rnn':
# lstm decoder
H_dec2 = tf.identity(H_dec)
if opt.rnn_share_emb:
loss, rec_sent_1, _ = lstm_decoder_embedding(H_dec2, x_org, W_norm, opt_t) #
_, rec_sent_2, _ = lstm_decoder_embedding(H_dec2, x_org, W_norm, opt_t, feed_previous=True, is_reuse=True)
else:
loss, rec_sent_1, _ = lstm_decoder(H_dec2, x_org, opt_t) #
_, rec_sent_2, _ = lstm_decoder(H_dec2, x_org, opt_t, feed_previous=True, is_reuse=True)
res['rec_sents_feed_y'] = rec_sent_1
res['rec_sents'] = rec_sent_2
# res['H1'],res['H2'],res['o1'],res['o2'] = H1, H2, o1, o2
else:
# deconv decoder
loss, res = deconv_decoder(H_dec, x_org, W_norm, is_train, opt_t, res)
# *tf.cast(tf.not_equal(x_temp,0), tf.float32)
tf.summary.scalar('loss', loss)
summaries = [
"learning_rate",
"loss",
# "gradients",
# "gradient_norm",
]
global_step = tf.Variable(0, trainable=False)
train_op = layers.optimize_loss(
loss,
global_step = global_step,
#aggregation_method=tf.AggregationMethod.EXPERIMENTAL_ACCUMULATE_N,
#framework.get_global_step(),
optimizer=opt.optimizer,
clip_gradients=(lambda grad: _clip_gradients_seperate_norm(grad, opt.clip_grad)) if opt.clip_grad else None,
learning_rate_decay_fn=lambda lr,g: tf.train.exponential_decay(learning_rate=lr, global_step = g, decay_rate=opt.decay_rate, decay_steps=3000),
learning_rate=opt.lr,
summaries = summaries
)
# optimizer = tf.train.AdamOptimizer(learning_rate=opt.lr) # Or another optimization algorithm.
# train_op = optimizer.minimize(
# loss,
# aggregation_method=tf.AggregationMethod.EXPERIMENTAL_ACCUMULATE_N)
return res, loss, train_op
def decoding_layer(self):
'''
构造Decoder层
参数：
- target_letter_to_int: target数据的映射表
- decoding_embedding_size: embed向量大小
- num_layers: 堆叠的RNN单元数量
- rnn_size: RNN单元的隐层结点数量
- target_sequence_length: target数据序列长度
- max_target_sequence_length: target数据序列最大长度
- encoder_state: encoder端编码的状态向量
- decoder_input: decoder端输入
'''
# 1. Embedding
decoder_embeddings = tf.Variable(tf.random_uniform([len(self.data.word_letter_to_int), self.args.decoding_embedding_size]))
decoder_embed_input = tf.nn.embedding_lookup(decoder_embeddings, self.decoder_input)
# 2. 构造Decoder中的RNN单元
def get_decoder_cell(rnn_size):
decoder_cell = tf.contrib.rnn.LSTMCell(rnn_size,
initializer=tf.random_uniform_initializer(-0.1, 0.1, seed=2))
single_cell=tf.contrib.rnn.DropoutWrapper(decoder_cell,output_keep_prob=self.drop_out)
return single_cell
cell = tf.contrib.rnn.MultiRNNCell([get_decoder_cell(self.args.rnn_size) for _ in range(self.args.num_layers)])
# 3. Output全连接层
output_layer = Dense(len(self.data.word_letter_to_int),
kernel_initializer = tf.truncated_normal_initializer(mean = 0.0, stddev=0.1))
# 4. Training decoder
with tf.variable_scope("decode"):
# 得到help对象
training_helper = tf.contrib.seq2seq.TrainingHelper(inputs=decoder_embed_input,
sequence_length=self.target_sequence_length,
time_major=False)
# 构造decoder
training_decoder = tf.contrib.seq2seq.BasicDecoder(cell,
training_helper,
self.encoder_state,
output_layer)
#tf.contrib.seq2seq.dynamic_decode执行decode,最终返回:(final_outputs, final_state, final_sequence_lengths)
self.training_decoder_output, _,_ = tf.contrib.seq2seq.dynamic_decode(training_decoder,
maximum_iterations=self.max_target_sequence_length)
#tf.identity是返回了一个一模一样新的tensor
self.training_logits = tf.identity(self.training_decoder_output.rnn_output, 'logits')
# 5. Predicting decoder
# Replicate encoder infos beam_width times
if (self.args.mode=='test'):
with tf.variable_scope("predict"):
decoder_initial_state = tf.contrib.seq2seq.tile_batch(self.encoder_state, multiplier=self.args.beam_size)
start_tokens = tf.tile(tf.constant([self.data.word_letter_to_int['<GO>']], dtype=tf.int32), [self.args.batch_size],
name='start_tokens')
# Define a beam-search decoder
decoder = tf.contrib.seq2seq.BeamSearchDecoder(
cell=cell,
embedding=decoder_embeddings,
start_tokens=start_tokens,
end_token=self.data.word_letter_to_int['<EOS>'],
initial_state=decoder_initial_state,
beam_width=self.args.beam_size,
output_layer=output_layer,
length_penalty_weight=0.0)
# Dynamic decoding
self.predict_decoder_outputs,_,_ = tf.contrib.seq2seq.dynamic_decode(decoder,
maximum_iterations=self.max_target_sequence_length)
self.predicts = tf.identity(tf.transpose(self.predict_decoder_outputs.predicted_ids, perm=[0, 2, 1]),'predictions')
def create_variables(self):
self.target_actor = self.actor.copy(scope="target_actor")
self.target_critic = self.critic.copy(scope="target_critic")
# FOR REGULAR ACTION SCORE COMPUTATION
with tf.name_scope("taking_action"):
# self.observation = tf.placeholder(tf.float32, (None, self.observation_size), name="observation")
# self.actor_val = tf.nn.sigmoid(self.actor(self.observation)) * 40 - 20;
self.actor_val = self.actor(self.observation_for_act)
self.actor_action = tf.identity(self.actor_val,
name="actor_action")
# tf.histogram_summary("actions", self.actor_action)
# FOR PREDICTING TARGET FUTURE REWARDS
with tf.name_scope("estimating_future_reward"):
# self.next_observation = tf.placeholder(tf.float32, (None, self.observation_size), name="next_observation")
# self.next_observation_mask = tf.placeholder(tf.float32, (None,), name="next_observation_mask")
# self.next_action = self.target_actor(self.next_observation) # ST
self.next_action = tf.stop_gradient(
self.target_actor(self.next_observation)) # ST
# print "next action: " + str(self.next_action)
# tf.histogram_summary("target_actions", self.next_action)
# self.next_value = self.target_critic([self.next_observation, self.next_action]) # ST
self.next_value = tf.stop_gradient(
tf.reshape(
self.target_critic(
[self.next_observation, self.next_action]),
[-1])) # ST
# self.rewards = tf.placeholder(tf.float32, (None,), name="rewards")
self.future_reward = self.rewards + self.discount_rate * self.next_observation_mask * self.next_value
with tf.name_scope("critic_update"):
##### ERROR FUNCTION #####
# self.given_action = tf.placeholder(tf.float32, (None, self.action_size), name="given_action")
self.value_given_action = tf.reshape(
self.critic([self.observation, self.given_action]), [-1])
# tf.scalar_summary("value_for_given_action", tf.reduce_mean(self.value_given_action))
temp_diff = self.value_given_action - self.future_reward
self.critic_error = tf.identity(tf.reduce_mean(
tf.square(temp_diff)),
name='critic_error')
##### OPTIMIZATION #####
critic_gradients = self.optimizer.compute_gradients(
self.critic_error, var_list=self.critic.variables())
# Add histograms for gradients.
for grad, var in critic_gradients:
# tf.histogram_summary('critic_update/' + var.name, var)
if grad is not None:
# tf.histogram_summary('critic_update/' + var.name + '/gradients', grad)
pass
self.critic_update = self.optimizer.apply_gradients(
critic_gradients, name='critic_train_op')
# tf.scalar_summary("critic_error", self.critic_error)
with tf.name_scope("actor_update"):
##### ERROR FUNCTION #####
# self.actor_score = self.critic([self.observation, self.actor_action])
self.actor_score = self.critic(
[self.observation,
self.actor(self.observation)])
##### OPTIMIZATION #####
# here we are maximizing actor score.
# only optimize actor variables here, while keeping critic constant
actor_gradients = self.optimizer.compute_gradients(
tf.reduce_mean(-self.actor_score),
var_list=self.actor.variables())
# Add histograms for gradients.
for grad, var in actor_gradients:
# tf.histogram_summary('actor_update/' + var.name, var)
if grad is not None:
# tf.histogram_summary('actor_update/' + var.name + '/gradients', grad)
pass
self.actor_update = self.optimizer.apply_gradients(
actor_gradients, name='actor_train_op')
# tf.scalar_summary("actor_score", tf.reduce_mean(self.actor_score))
# UPDATE TARGET NETWORK
with tf.name_scope("target_network_update"):
self.target_actor_update = ContinuousDeepQ.update_target_network(
self.actor, self.target_actor, self.target_actor_update_rate)
self.target_critic_update = ContinuousDeepQ.update_target_network(
self.critic, self.target_critic,
self.target_critic_update_rate)
self.update_all_targets = tf.group(self.target_actor_update,
self.target_critic_update,
name='target_networks_update')
# self.summarize = tf.merge_all_summaries()
self.no_op1 = tf.no_op()
except ValueError as err:
print(
'''Input file specified ({}) only holds the weights, and not the model defenition.
Save the model using mode.save(filename.h5) which will contain the network architecture
as well as its weights.
If the model is saved using model.save_weights(filename.h5), the model architecture is
expected to be saved separately in a json format and loaded prior to loading the weights.
Check the keras documentation for more details (https://keras.io/getting-started/faq/)'''
.format(weight_file_path))
raise err
num_output = args.num_outputs
pred = [None] * num_output
pred_node_names = [None] * num_output
for i in range(num_output):
pred_node_names[i] = args.output_node_prefix + str(i)
pred[i] = tf.identity(net_model.outputs[i], name=pred_node_names[i])
print('output nodes names are: ', pred_node_names)
# [optional] write graph definition in ascii
sess = K.get_session()
if args.graph_def:
f = args.output_graphdef_file
tf.train.write_graph(sess.graph.as_graph_def(),
output_fld,
f,
as_text=True)
print('saved the graph definition in ascii format at: ',
str(Path(output_fld) / f))
def main(_):
InputItemsLength = 24
ClipLength = 16
SZIE_W = InputItemsLength
CHANNEL_NUM = 1
CNN_0Dim_SIZE = None
numOutputVars = 2
#Cos Sin
# Import data
print("test data - end")
InSeqClips_Dataset_train = \
InputCSV.readCSV_trainingSeqDataSet_byLabeling_CosSinFloatNum (
strDir_Training = '2018.03.10 (Cosine, Sine)/2018.03.10_加了TargetDirection/' \
, ClipLength = ClipLength \
, InputItemsLength = InputItemsLength )
# test_dir="/tmp/code_3DCNN/Data_3DCNN/Test/",
print("end of read data")
## check point files path:
strRoot_Models = 'models/'
strNewVersionPath_FoldName = 'chkpt v0 from data v2018-0310/'
strChkptFileName = 'model_DirectionSensing.ckpt'
strChkptFolderPath = strRoot_Models + strNewVersionPath_FoldName + strChkptFileName
# Create the model
with tf.Graph().as_default():
#with graph.as_default():
clip = tf.placeholder(tf.float32,
[CNN_0Dim_SIZE, ClipLength, SZIE_W, CHANNEL_NUM],
name='placeHolder_x')
y_label = tf.placeholder(tf.float32, [CNN_0Dim_SIZE, numOutputVars],
name='placeHolder_y')
conv_result = Cnn2D.SeqCNN2D(clip, numOutputVars, 0.5)
y_conv_result = tf.identity(conv_result, name='y_conv_output_')
print(clip)
print(y_label)
print(y_conv_result)
with tf.name_scope('loss'):
loss = tf.reduce_mean(tf.squared_difference(
y_conv_result, y_label))
#tf.summary.scalar('entropy_loss', loss)
#with tf.name_scope('accuracy'):
# accuracy = tf.reduce_mean(tf.cast(tf.equal(y_conv_result, y_label), tf.float32))
learning_rate = 1e-4
optimizer = tf.train.AdamOptimizer(learning_rate).minimize(loss)
saver = tf.train.Saver()
#config.gpu_options.allow_growth = True
with tf.Session(config=tf.ConfigProto()) as sess:
##################################################################################################
saver.restore(sess, strChkptFolderPath)
### 使用 saver.restore，下面行就要註解起來，反之第一次執行，則要使用下面行，註解上面的saver.restore
#sess.run(tf.global_variables_initializer()) ; sess.run(tf.local_variables_initializer()) ;
##################################################################################################
step = 0
BATCH_SIZE = 50
End_of_EPOCH_NUM = 500000
SaveModelTiming_MultipleNumberOfEpoch = 50
for epoch in range(End_of_EPOCH_NUM):
loss_epoch = 0
for i in range(
len(InSeqClips_Dataset_train.clips) // BATCH_SIZE):
step += 1
batchClips, batchLabels = InSeqClips_Dataset_train.next_batch(
BATCH_SIZE)
_, loss_out = sess.run([optimizer, loss],
feed_dict={
clip: batchClips,
y_label: batchLabels
})
loss_epoch += loss_out
if i % 1 == 0:
#print('Epoch %d, Batch %d: Loss is %.9f; Accuracy is %.3f'%(epoch, i, loss_out, accuracy_out))
print('Epoch %d, Batch %d: Loss is %.5f; ' %
(epoch, i, loss_out))
############ for batch i loop ######################################################################
print('\n Epoch %d: Average loss is: %.9f; \n' %
(epoch, loss_epoch /
(len(InSeqClips_Dataset_train.clips) // BATCH_SIZE)))
# Save checkpoint, graph.pb
if epoch % SaveModelTiming_MultipleNumberOfEpoch == 0 and epoch != 0:
print("\n Save checkpoint ...\n")
#saver = tf.train.Saver()
saver.save(sess, strChkptFolderPath)
output_node_names = 'y_conv_output_'
#print(output_node_names) ;
graph = tf.get_default_graph()
input_graph_def = graph.as_graph_def()
output_graph_def = graph_util.convert_variables_to_constants(
sess, # The session is used to retrieve the weights
input_graph_def, # The graph_def is used to retrieve the nodes
output_node_names.split(
","
) # The output node names are used to select the usefull nodes
)
# save PB file ----------------------------------------------------
output_graph = "models/freeze/frozen_model_DirectionSensing.pb"
# Finally we serialize and dump the output graph to the filesystem
with tf.gfile.GFile(output_graph, "wb") as f:
f.write(output_graph_def.SerializeToString())
print("%d ops in the final graph." %
len(output_graph_def.node))
# end of Save checkpoint, graph.pb
## Shuffel_and_Reassign_Dataset
if epoch % 2 == 0 and epoch != 0:
InSeqClips_Dataset_train = InputCSV.DataSet.Shuffel_and_Reassign_Dataset(
)
########## end of for epoch loop ######################################################################################
print("start for save")
# Save checkpoint, graph.pb and tensorboard
#saver = tf.train.Saver()
saver.save(sess, strChkptFolderPath)
output_node_names = 'y_conv_output_'
print(output_node_names)
graph = tf.get_default_graph()
input_graph_def = graph.as_graph_def()
output_graph_def = graph_util.convert_variables_to_constants(
sess, # The session is used to retrieve the weights
input_graph_def, # The graph_def is used to retrieve the nodes
output_node_names.split(
","
) # The output node names are used to select the usefull nodes
)
output_graph = "models/freeze/frozen_model_DirectionSensing.pb"
# Finally we serialize and dump the output graph to the filesystem
with tf.gfile.GFile(output_graph, "wb") as f:
f.write(output_graph_def.SerializeToString())
print("%d ops in the final graph." %
len(output_graph_def.node))
tf.train.write_graph(sess.graph.as_graph_def(),
"models/freeze/",
"frozen_model_DirectionSensing.pb",
as_text=False)
print("end of the save")
####################################################################
print("---The end of our testing---")
def pick_pool_and_stack(inputs,
index_spec,
name='pps',
debug=False,
extra_chans=None):
"""Takes some previous layers and some indices, and creates a new data
layer by picking out the appropriate items then concatenating and/ or
pooling them together.
Yes, that description is extremely vague. In my defense, this layer is an
important part of my implementation strategy for action/proposition
modules, so it has to do a lot of unrelated things. Hopefully I'll be able
to write a better explanation later."""
assert len(index_spec) > 0, "need at least one incoming"
_check_index_spec(index_spec)
with tf.name_scope(name):
# out_stacks will be an array of len(self.index_spec) tensors, each N *
# L * D_i (where sum of D_is is the chans_out value returned by
# get_output_shape_for)
out_stacks = []
for chosen_input, pools in index_spec:
this_input = inputs[chosen_input]
if all(len(p) == 1 for p in pools):
# This is the fast path: we don't have to pool at all, because
# there's precisely one element in each pool. This always
# happens when constructing action modules, which never need
# any sort of pooling.
# FIXME: speed this up even further by doing combined gather &
# concat, instead of running concat separately (actually,
# before doing that, look at TF logs to see whether it's likely
# to be faster!).
np_indices = np.array([idx for idx, in pools], dtype=np.int32)
assert np_indices.ndim == 1, np_indices.shape
this_stack = tf.gather(this_input, np_indices, axis=1)
out_stacks.append(this_stack)
else:
# Slow path. Need to make new array & reduce over the first
# axis with segment_max.
trans_input = tf.transpose(this_input, (1, 0, 2))
flat_pools = sum(pools, [])
pool_segments = sum(
([r] * len(pools[r]) for r in range(len(pools))), [])
gathered = tf.gather(trans_input, flat_pools, axis=0)
if all(len(p) > 0 for p in pools):
# we can never end up with empty pools, so we just do
# segment_max, which seems to be a little faster
trans_pooled = tf.segment_max(gathered, pool_segments)
trans_pooled.set_shape((len(pools), trans_pooled.shape[1],
trans_pooled.shape[2]))
else:
# TODO: is there a faster way of doing this than
# unsorted_segment_max? I want something that will
# GRACEFULLY deal with missing segments; segment_max
# inserts 0 for missing interior segments (?!) and simply
# omits empty segments on the end (since it doesnt' know
# about them).
trans_pooled = tf.unsorted_segment_max(
gathered, pool_segments, len(pools))
# unsorted_segment_max will return FLOAT_MIN or something
# for empty segments; we get around that problem by
# clamping the value to the minimum possible activation
# output from the last layer
assert NONLINEARITY == 'elu', \
'minimum value of -1 is dependent on using elu'
trans_pooled = tf.maximum(trans_pooled, -1.0)
pooled = tf.transpose(trans_pooled, (1, 0, 2))
out_stacks.append(pooled)
# concat along input channels
if extra_chans is not None:
all_cat = out_stacks + list(extra_chans)
else:
all_cat = out_stacks
rv = tf.concat(all_cat, axis=2)
if debug:
this_in_shape = tf.shape(this_input)
in_bs = this_in_shape[0]
check_bs = tf.assert_equal(tf.shape(rv)[0],
in_bs,
message="returned batch size doesn't "
"match that of last input")
check_rank = tf.assert_rank(rv, 3, message='return rank is not 3')
with tf.control_dependencies([check_bs, check_rank]):
rv = tf.identity(rv)
return rv
def _run_op_test(self, op_fun, a, b, on_gpu):
"""Run a single test for a single op."""
# Preparations
op_name = op_fun.__name__
device_name = '/gpu:0' if on_gpu else '/cpu:0'
a = np.asarray(a, dtype=self.dtype.as_numpy_dtype())
b = np.asarray(b, dtype=self.dtype.as_numpy_dtype())
# Print
print2(
"--> %s: on_gpu=%s, a_shape=%s, b_shape=%s" %
(op_name, on_gpu, a.shape, b.shape), self.file)
# true_out = [email protected]()
true_out = np.sum(a.reshape(a.shape[0], 1, -1) * b, axis=2)
# Create graph
tf.reset_default_graph()
with tf.device(device_name):
# Create input
# We cannot use a constant here, since the operation will be
# pre-computed on a CPU for some cases (e.g. for int64 indices)
# To ensure that data is copied only once, we add an identity op
# which is served the input data and connected to all ops
a_pl = tf.placeholder(dtype=self.dtype, shape=(None, a.shape[1]))
a_op = tf.identity(a_pl)
if b.ndim == 2:
b_pl = tf.placeholder(dtype=self.dtype,
shape=(None, b.shape[1]))
else:
b_pl = tf.placeholder(dtype=self.dtype,
shape=(None, b.shape[1], b.shape[2]))
b_op = tf.identity(b_pl)
# Create ops
start_time = time.time()
ops = op_fun(a_op, b_op)
for _ in range(self.num_ops - 1):
# The tuple ensures that the next op waits for the output
# of the previous op, effectively stacking the ops
# but using the original input every time
ops = op_fun(tf.tuple([a_op, ops])[0], b_op)
setup_time = time.time() - start_time
# Get num of graph ops
graph_size = len(tf.get_default_graph().get_operations())
# Run multiple times
output_correct = True
with tf.Session(config=tf.ConfigProto(
allow_soft_placement=False,
log_device_placement=self.log_devs)) as sess:
run_times = []
for n in range(self.num_runs):
# Run
start_time = time.time()
out = sess.run(ops, feed_dict={a_pl: a, b_pl: b})
run_times.append(time.time() - start_time)
# Test value
try:
np.testing.assert_array_almost_equal(out,
true_out,
decimal=5)
except AssertionError:
output_correct = False
# Return stats
return OpTestResult(op_name, on_gpu, graph_size, setup_time, run_times,
output_correct)
def deploy(config,
model_fn,
args=None,
kwargs=None,
optimizer=None,
summarize_gradients=False):
"""Deploys a Slim-constructed model across multiple clones.
The deployment options are specified by the config object and support
deploying one or several clones on different GPUs and one or several replicas
of such clones.
The argument `model_fn` is called `config.num_clones` times to create the
model clones as `model_fn(*args, **kwargs)`.
The optional argument `optimizer` is an `Optimizer` object. If not `None`,
the deployed model is configured for training with that optimizer.
If `config` specifies deployment on multiple replicas then the default
tensorflow device is set appropriatly for each call to `model_fn` and for the
slim variable creation functions: model and global variables will be created
on the `ps` device, the clone operations will be on the `worker` device.
Args:
config: A `DeploymentConfig` object.
model_fn: A callable. Called as `model_fn(*args, **kwargs)`
args: Optional list of arguments to pass to `model_fn`.
kwargs: Optional list of keyword arguments to pass to `model_fn`.
optimizer: Optional `Optimizer` object. If passed the model is deployed
for training with that optimizer.
summarize_gradients: Whether or not add summaries to the gradients.
Returns:
A `DeployedModel` namedtuple.
"""
# Gather initial summaries.
summaries = set(
tf.compat.v1.get_collection(tf.compat.v1.GraphKeys.SUMMARIES))
# Create Clones.
clones = create_clones(config, model_fn, args, kwargs)
first_clone = clones[0]
# Gather update_ops from the first clone. These contain, for example,
# the updates for the batch_norm variables created by model_fn.
update_ops = tf.compat.v1.get_collection(tf.compat.v1.GraphKeys.UPDATE_OPS,
first_clone.scope)
train_op = None
total_loss = None
with tf.device(config.optimizer_device()):
if optimizer:
# Place the global step on the device storing the variables.
with tf.device(config.variables_device()):
global_step = tf.compat.v1.train.get_or_create_global_step()
# Compute the gradients for the clones.
total_loss, clones_gradients = optimize_clones(clones, optimizer)
if clones_gradients:
if summarize_gradients:
# Add summaries to the gradients.
summaries |= set(
_add_gradients_summaries(clones_gradients))
# Create gradient updates.
grad_updates = optimizer.apply_gradients(
clones_gradients, global_step=global_step)
update_ops.append(grad_updates)
update_op = tf.group(*update_ops)
with tf.control_dependencies([update_op]):
train_op = tf.identity(total_loss, name='train_op')
else:
clones_losses = []
regularization_losses = tf.get_collection(
tf.GraphKeys.REGULARIZATION_LOSSES)
for clone in clones:
with tf.name_scope(clone.scope):
clone_loss = _gather_clone_loss(clone, len(clones),
regularization_losses)
if clone_loss is not None:
clones_losses.append(clone_loss)
# Only use regularization_losses for the first clone
regularization_losses = None
if clones_losses:
total_loss = tf.add_n(clones_losses, name='total_loss')
# Add the summaries from the first clone. These contain the summaries
# created by model_fn and either optimize_clones() or _gather_clone_loss().
summaries |= set(
tf.compat.v1.get_collection(tf.compat.v1.GraphKeys.SUMMARIES,
first_clone.scope))
if total_loss is not None:
# Add total_loss to summary.
summaries.add(tf.compat.v1.summary.scalar('total_loss',
total_loss))
if summaries:
# Merge all summaries together.
summary_op = tf.compat.v1.summary.merge(list(summaries),
name='summary_op')
else:
summary_op = None
return DeployedModel(train_op, summary_op, total_loss, clones)
def group_return(*args): # Same as tf.group(), but returns the value of the last argument.
with tf.name_scope('GroupReturn'):
with tf.control_dependencies(args[:-1]):
return tf.identity(args[-1])
# Learning Rate
learning_rate = 0.01
with tf.device(device_name):
#building graph
train_graph = tf.Graph()
with train_graph.as_default():
embed_seq, input_data, targets, lr, target_sequence_length, max_target_sequence_length, source_sequence_length = get_inputs(
)
training_decoder_output, predicting_decoder_output = seq2seq_model(
embed_seq, input_data, targets, lr, target_sequence_length,
max_target_sequence_length, source_sequence_length,
len(source_letter_to_int), len(target_letter_to_int),
encoding_embedding_size, decoding_embedding_size, rnn_size,
num_layers)
training_logits = tf.identity(training_decoder_output.rnn_output,
'logits')
predicting_logits = tf.identity(
predicting_decoder_output.sample_id, name='predictions')
masks = tf.sequence_mask(target_sequence_length,
max_target_sequence_length,
dtype=tf.float32,
name='masks')
with tf.name_scope("optimization"):
# Loss function
cost = tf.contrib.seq2seq.sequence_loss(
training_logits, targets, masks)
# Optimizer
optimizer = tf.train.AdamOptimizer(lr)
import tensorflow as tf
import numpy as np
flag = tf.Variable(name="flag", initial_value=1, dtype=tf.int64)
a = tf.Variable(name="a", initial_value=3.0)
b = tf.Variable(name="b", initial_value=5.0)
d = tf.Variable(name="d", initial_value=21.0)
x = tf.where(flag > 0, tf.identity(b), tf.identity(d))
c = a * x
grad = tf.gradients(c, [a, b, d])
init = tf.global_variables_initializer()
sess = tf.Session()
sess.run(init)
result = sess.run([grad], feed_dict={flag: 0})
print("X")
def inception_v3(inputs,
dropout_keep_prob=0.8,
num_classes=1000,
is_training=True,
restore_logits=True,
scope=''):
"""Latest Inception from http://arxiv.org/abs/1512.00567.
"Rethinking the Inception Architecture for Computer Vision"
Christian Szegedy, Vincent Vanhoucke, Sergey Ioffe, Jonathon Shlens,
Zbigniew Wojna
Args:
inputs: a tensor of size [batch_size, height, width, channels].
dropout_keep_prob: dropout keep_prob.
num_classes: number of predicted classes.
is_training: whether is training or not.
restore_logits: whether or not the logits layers should be restored.
Useful for fine-tuning a model with different num_classes.
scope: Optional scope for op_scope.
Returns:
a list containing 'logits', 'aux_logits' Tensors.
"""
# end_points will collect relevant activations for external use, for example
# summaries or losses.
end_points = {}
with tf.op_scope([inputs], scope, 'inception_v3'):
with scopes.arg_scope(
[ops.conv2d, ops.fc, ops.batch_norm, ops.dropout],
is_training=is_training):
with scopes.arg_scope([ops.conv2d, ops.max_pool, ops.avg_pool],
stride=1,
padding='VALID'):
# 299 x 299 x 3
end_points['conv0'] = ops.conv2d(inputs,
32, [3, 3],
stride=2,
scope='conv0')
# 149 x 149 x 32
end_points['conv1'] = ops.conv2d(end_points['conv0'],
32, [3, 3],
scope='conv1')
# 147 x 147 x 32
end_points['conv2'] = ops.conv2d(end_points['conv1'],
64, [3, 3],
padding='SAME',
scope='conv2')
# 147 x 147 x 64
end_points['pool1'] = ops.max_pool(end_points['conv2'], [3, 3],
stride=2,
scope='pool1')
# 73 x 73 x 64
end_points['conv3'] = ops.conv2d(end_points['pool1'],
80, [1, 1],
scope='conv3')
# 73 x 73 x 80.
end_points['conv4'] = ops.conv2d(end_points['conv3'],
192, [3, 3],
scope='conv4')
# 71 x 71 x 192.
end_points['pool2'] = ops.max_pool(end_points['conv4'], [3, 3],
stride=2,
scope='pool2')
# 35 x 35 x 192.
net = end_points['pool2']
# Inception blocks
with scopes.arg_scope([ops.conv2d, ops.max_pool, ops.avg_pool],
stride=1,
padding='SAME'):
# mixed: 35 x 35 x 256.
with tf.variable_scope('mixed_35x35x256a'):
with tf.variable_scope('branch1x1'):
branch1x1 = ops.conv2d(net, 64, [1, 1])
with tf.variable_scope('branch5x5'):
branch5x5 = ops.conv2d(net, 48, [1, 1])
branch5x5 = ops.conv2d(branch5x5, 64, [5, 5])
with tf.variable_scope('branch3x3dbl'):
branch3x3dbl = ops.conv2d(net, 64, [1, 1])
branch3x3dbl = ops.conv2d(branch3x3dbl, 96, [3, 3])
branch3x3dbl = ops.conv2d(branch3x3dbl, 96, [3, 3])
with tf.variable_scope('branch_pool'):
branch_pool = ops.avg_pool(net, [3, 3])
branch_pool = ops.conv2d(branch_pool, 32, [1, 1])
net = tf.concat(
3, [branch1x1, branch5x5, branch3x3dbl, branch_pool])
end_points['mixed_35x35x256a'] = net
# mixed_1: 35 x 35 x 288.
with tf.variable_scope('mixed_35x35x288a'):
with tf.variable_scope('branch1x1'):
branch1x1 = ops.conv2d(net, 64, [1, 1])
with tf.variable_scope('branch5x5'):
branch5x5 = ops.conv2d(net, 48, [1, 1])
branch5x5 = ops.conv2d(branch5x5, 64, [5, 5])
with tf.variable_scope('branch3x3dbl'):
branch3x3dbl = ops.conv2d(net, 64, [1, 1])
branch3x3dbl = ops.conv2d(branch3x3dbl, 96, [3, 3])
branch3x3dbl = ops.conv2d(branch3x3dbl, 96, [3, 3])
with tf.variable_scope('branch_pool'):
branch_pool = ops.avg_pool(net, [3, 3])
branch_pool = ops.conv2d(branch_pool, 64, [1, 1])
net = tf.concat(
3, [branch1x1, branch5x5, branch3x3dbl, branch_pool])
end_points['mixed_35x35x288a'] = net
# mixed_2: 35 x 35 x 288.
with tf.variable_scope('mixed_35x35x288b'):
with tf.variable_scope('branch1x1'):
branch1x1 = ops.conv2d(net, 64, [1, 1])
with tf.variable_scope('branch5x5'):
branch5x5 = ops.conv2d(net, 48, [1, 1])
branch5x5 = ops.conv2d(branch5x5, 64, [5, 5])
with tf.variable_scope('branch3x3dbl'):
branch3x3dbl = ops.conv2d(net, 64, [1, 1])
branch3x3dbl = ops.conv2d(branch3x3dbl, 96, [3, 3])
branch3x3dbl = ops.conv2d(branch3x3dbl, 96, [3, 3])
with tf.variable_scope('branch_pool'):
branch_pool = ops.avg_pool(net, [3, 3])
branch_pool = ops.conv2d(branch_pool, 64, [1, 1])
net = tf.concat(
3, [branch1x1, branch5x5, branch3x3dbl, branch_pool])
end_points['mixed_35x35x288b'] = net
# mixed_3: 17 x 17 x 768.
with tf.variable_scope('mixed_17x17x768a'):
with tf.variable_scope('branch3x3'):
branch3x3 = ops.conv2d(net,
384, [3, 3],
stride=2,
padding='VALID')
with tf.variable_scope('branch3x3dbl'):
branch3x3dbl = ops.conv2d(net, 64, [1, 1])
branch3x3dbl = ops.conv2d(branch3x3dbl, 96, [3, 3])
branch3x3dbl = ops.conv2d(branch3x3dbl,
96, [3, 3],
stride=2,
padding='VALID')
with tf.variable_scope('branch_pool'):
branch_pool = ops.max_pool(net, [3, 3],
stride=2,
padding='VALID')
net = tf.concat(3, [branch3x3, branch3x3dbl, branch_pool])
end_points['mixed_17x17x768a'] = net
# mixed4: 17 x 17 x 768.
with tf.variable_scope('mixed_17x17x768b'):
with tf.variable_scope('branch1x1'):
branch1x1 = ops.conv2d(net, 192, [1, 1])
with tf.variable_scope('branch7x7'):
branch7x7 = ops.conv2d(net, 128, [1, 1])
branch7x7 = ops.conv2d(branch7x7, 128, [1, 7])
branch7x7 = ops.conv2d(branch7x7, 192, [7, 1])
with tf.variable_scope('branch7x7dbl'):
branch7x7dbl = ops.conv2d(net, 128, [1, 1])
branch7x7dbl = ops.conv2d(branch7x7dbl, 128, [7, 1])
branch7x7dbl = ops.conv2d(branch7x7dbl, 128, [1, 7])
branch7x7dbl = ops.conv2d(branch7x7dbl, 128, [7, 1])
branch7x7dbl = ops.conv2d(branch7x7dbl, 192, [1, 7])
with tf.variable_scope('branch_pool'):
branch_pool = ops.avg_pool(net, [3, 3])
branch_pool = ops.conv2d(branch_pool, 192, [1, 1])
net = tf.concat(
3, [branch1x1, branch7x7, branch7x7dbl, branch_pool])
end_points['mixed_17x17x768b'] = net
# mixed_5: 17 x 17 x 768.
with tf.variable_scope('mixed_17x17x768c'):
with tf.variable_scope('branch1x1'):
branch1x1 = ops.conv2d(net, 192, [1, 1])
with tf.variable_scope('branch7x7'):
branch7x7 = ops.conv2d(net, 160, [1, 1])
branch7x7 = ops.conv2d(branch7x7, 160, [1, 7])
branch7x7 = ops.conv2d(branch7x7, 192, [7, 1])
with tf.variable_scope('branch7x7dbl'):
branch7x7dbl = ops.conv2d(net, 160, [1, 1])
branch7x7dbl = ops.conv2d(branch7x7dbl, 160, [7, 1])
branch7x7dbl = ops.conv2d(branch7x7dbl, 160, [1, 7])
branch7x7dbl = ops.conv2d(branch7x7dbl, 160, [7, 1])
branch7x7dbl = ops.conv2d(branch7x7dbl, 192, [1, 7])
with tf.variable_scope('branch_pool'):
branch_pool = ops.avg_pool(net, [3, 3])
branch_pool = ops.conv2d(branch_pool, 192, [1, 1])
net = tf.concat(
3, [branch1x1, branch7x7, branch7x7dbl, branch_pool])
end_points['mixed_17x17x768c'] = net
# mixed_6: 17 x 17 x 768.
with tf.variable_scope('mixed_17x17x768d'):
with tf.variable_scope('branch1x1'):
branch1x1 = ops.conv2d(net, 192, [1, 1])
with tf.variable_scope('branch7x7'):
branch7x7 = ops.conv2d(net, 160, [1, 1])
branch7x7 = ops.conv2d(branch7x7, 160, [1, 7])
branch7x7 = ops.conv2d(branch7x7, 192, [7, 1])
with tf.variable_scope('branch7x7dbl'):
branch7x7dbl = ops.conv2d(net, 160, [1, 1])
branch7x7dbl = ops.conv2d(branch7x7dbl, 160, [7, 1])
branch7x7dbl = ops.conv2d(branch7x7dbl, 160, [1, 7])
branch7x7dbl = ops.conv2d(branch7x7dbl, 160, [7, 1])
branch7x7dbl = ops.conv2d(branch7x7dbl, 192, [1, 7])
with tf.variable_scope('branch_pool'):
branch_pool = ops.avg_pool(net, [3, 3])
branch_pool = ops.conv2d(branch_pool, 192, [1, 1])
net = tf.concat(
3, [branch1x1, branch7x7, branch7x7dbl, branch_pool])
end_points['mixed_17x17x768d'] = net
# mixed_7: 17 x 17 x 768.
with tf.variable_scope('mixed_17x17x768e'):
with tf.variable_scope('branch1x1'):
branch1x1 = ops.conv2d(net, 192, [1, 1])
with tf.variable_scope('branch7x7'):
branch7x7 = ops.conv2d(net, 192, [1, 1])
branch7x7 = ops.conv2d(branch7x7, 192, [1, 7])
branch7x7 = ops.conv2d(branch7x7, 192, [7, 1])
with tf.variable_scope('branch7x7dbl'):
branch7x7dbl = ops.conv2d(net, 192, [1, 1])
branch7x7dbl = ops.conv2d(branch7x7dbl, 192, [7, 1])
branch7x7dbl = ops.conv2d(branch7x7dbl, 192, [1, 7])
branch7x7dbl = ops.conv2d(branch7x7dbl, 192, [7, 1])
branch7x7dbl = ops.conv2d(branch7x7dbl, 192, [1, 7])
with tf.variable_scope('branch_pool'):
branch_pool = ops.avg_pool(net, [3, 3])
branch_pool = ops.conv2d(branch_pool, 192, [1, 1])
net = tf.concat(
3, [branch1x1, branch7x7, branch7x7dbl, branch_pool])
end_points['mixed_17x17x768e'] = net
# Auxiliary Head logits
aux_logits = tf.identity(end_points['mixed_17x17x768e'])
with tf.variable_scope('aux_logits'):
aux_logits = ops.avg_pool(aux_logits, [5, 5],
stride=3,
padding='VALID')
aux_logits = ops.conv2d(aux_logits,
128, [1, 1],
scope='proj')
# Shape of feature map before the final layer.
shape = aux_logits.get_shape()
aux_logits = ops.conv2d(aux_logits,
768,
shape[1:3],
stddev=0.01,
padding='VALID')
aux_logits = ops.flatten(aux_logits)
aux_logits = ops.fc(aux_logits,
num_classes,
activation=None,
stddev=0.001,
restore=restore_logits)
end_points['aux_logits'] = aux_logits
# mixed_8: 8 x 8 x 1280.
# Note that the scope below is not changed to not void previous
# checkpoints.
# (TODO) Fix the scope when appropriate.
with tf.variable_scope('mixed_17x17x1280a'):
with tf.variable_scope('branch3x3'):
branch3x3 = ops.conv2d(net, 192, [1, 1])
branch3x3 = ops.conv2d(branch3x3,
320, [3, 3],
stride=2,
padding='VALID')
with tf.variable_scope('branch7x7x3'):
branch7x7x3 = ops.conv2d(net, 192, [1, 1])
branch7x7x3 = ops.conv2d(branch7x7x3, 192, [1, 7])
branch7x7x3 = ops.conv2d(branch7x7x3, 192, [7, 1])
branch7x7x3 = ops.conv2d(branch7x7x3,
192, [3, 3],
stride=2,
padding='VALID')
with tf.variable_scope('branch_pool'):
branch_pool = ops.max_pool(net, [3, 3],
stride=2,
padding='VALID')
net = tf.concat(3, [branch3x3, branch7x7x3, branch_pool])
end_points['mixed_17x17x1280a'] = net
# mixed_9: 8 x 8 x 2048.
with tf.variable_scope('mixed_8x8x2048a'):
with tf.variable_scope('branch1x1'):
branch1x1 = ops.conv2d(net, 320, [1, 1])
with tf.variable_scope('branch3x3'):
branch3x3 = ops.conv2d(net, 384, [1, 1])
branch3x3 = tf.concat(3, [
ops.conv2d(branch3x3, 384, [1, 3]),
ops.conv2d(branch3x3, 384, [3, 1])
])
with tf.variable_scope('branch3x3dbl'):
branch3x3dbl = ops.conv2d(net, 448, [1, 1])
branch3x3dbl = ops.conv2d(branch3x3dbl, 384, [3, 3])
branch3x3dbl = tf.concat(3, [
ops.conv2d(branch3x3dbl, 384, [1, 3]),
ops.conv2d(branch3x3dbl, 384, [3, 1])
])
with tf.variable_scope('branch_pool'):
branch_pool = ops.avg_pool(net, [3, 3])
branch_pool = ops.conv2d(branch_pool, 192, [1, 1])
net = tf.concat(
3, [branch1x1, branch3x3, branch3x3dbl, branch_pool])
end_points['mixed_8x8x2048a'] = net
# mixed_10: 8 x 8 x 2048.
with tf.variable_scope('mixed_8x8x2048b'):
with tf.variable_scope('branch1x1'):
branch1x1 = ops.conv2d(net, 320, [1, 1])
with tf.variable_scope('branch3x3'):
branch3x3 = ops.conv2d(net, 384, [1, 1])
branch3x3 = tf.concat(3, [
ops.conv2d(branch3x3, 384, [1, 3]),
ops.conv2d(branch3x3, 384, [3, 1])
])
with tf.variable_scope('branch3x3dbl'):
branch3x3dbl = ops.conv2d(net, 448, [1, 1])
branch3x3dbl = ops.conv2d(branch3x3dbl, 384, [3, 3])
branch3x3dbl = tf.concat(3, [
ops.conv2d(branch3x3dbl, 384, [1, 3]),
ops.conv2d(branch3x3dbl, 384, [3, 1])
])
with tf.variable_scope('branch_pool'):
branch_pool = ops.avg_pool(net, [3, 3])
branch_pool = ops.conv2d(branch_pool, 192, [1, 1])
net = tf.concat(
3, [branch1x1, branch3x3, branch3x3dbl, branch_pool])
end_points['mixed_8x8x2048b'] = net
# Final pooling and prediction
with tf.variable_scope('logits'):
shape = net.get_shape()
net = ops.avg_pool(net,
shape[1:3],
padding='VALID',
scope='pool')
# 1 x 1 x 2048
net = ops.dropout(net, dropout_keep_prob, scope='dropout')
net = ops.flatten(net, scope='flatten')
# 2048
logits = ops.fc(net,
num_classes,
activation=None,
scope='logits',
restore=restore_logits)
# 1000
end_points['logits'] = logits
end_points['predictions'] = tf.nn.softmax(
logits, name='predictions')
return logits, end_points
def build_graph(self, *inputs):
is_training = get_current_tower_context().is_training
if config.MODE_MASK:
image, anchor_labels, anchor_boxes, gt_boxes, gt_labels, gt_masks = inputs
else:
image, anchor_labels, anchor_boxes, gt_boxes, gt_labels = inputs
image = self.preprocess(image) # 1CHW
featuremap = resnet_c4_backbone(image, config.RESNET_NUM_BLOCK[:3])
rpn_label_logits, rpn_box_logits = rpn_head('rpn', featuremap, 1024,
config.NUM_ANCHOR)
fm_anchors, anchor_labels, anchor_boxes = self.narrow_to_featuremap(
featuremap, get_all_anchors(), anchor_labels, anchor_boxes)
anchor_boxes_encoded = encode_bbox_target(anchor_boxes, fm_anchors)
image_shape2d = tf.shape(image)[2:] # h,w
pred_boxes_decoded = decode_bbox_target(
rpn_box_logits, fm_anchors) # fHxfWxNAx4, floatbox
proposal_boxes, proposal_scores = generate_rpn_proposals(
tf.reshape(pred_boxes_decoded, [-1, 4]),
tf.reshape(rpn_label_logits,
[-1]), image_shape2d, config.TRAIN_PRE_NMS_TOPK
if is_training else config.TEST_PRE_NMS_TOPK,
config.TRAIN_POST_NMS_TOPK
if is_training else config.TEST_POST_NMS_TOPK)
if is_training:
# sample proposal boxes in training
rcnn_boxes, rcnn_labels, fg_inds_wrt_gt = sample_fast_rcnn_targets(
proposal_boxes, gt_boxes, gt_labels)
else:
# The boxes to be used to crop RoIs.
# Use all proposal boxes in inference
rcnn_boxes = proposal_boxes
boxes_on_featuremap = rcnn_boxes * (1.0 / config.ANCHOR_STRIDE)
roi_resized = roi_align(featuremap, boxes_on_featuremap, 14)
# HACK to work around https://github.com/tensorflow/tensorflow/issues/14657
# which was fixed in TF 1.6
def ff_true():
feature_fastrcnn = resnet_conv5(
roi_resized, config.RESNET_NUM_BLOCK[-1]) # nxcx7x7
feature_gap = GlobalAvgPooling('gap',
feature_fastrcnn,
data_format='channels_first')
fastrcnn_label_logits, fastrcnn_box_logits = fastrcnn_outputs(
'fastrcnn', feature_gap, config.NUM_CLASS)
# Return C5 feature to be shared with mask branch
return feature_fastrcnn, fastrcnn_label_logits, fastrcnn_box_logits
def ff_false():
ncls = config.NUM_CLASS
return tf.zeros([0, 2048, 7,
7]), tf.zeros([0,
ncls]), tf.zeros([0, ncls - 1, 4])
if get_tf_version_number() >= 1.6:
feature_fastrcnn, fastrcnn_label_logits, fastrcnn_box_logits = ff_true(
)
else:
logger.warn("This example may drop support for TF < 1.6 soon.")
feature_fastrcnn, fastrcnn_label_logits, fastrcnn_box_logits = tf.cond(
tf.size(boxes_on_featuremap) > 0, ff_true, ff_false)
if is_training:
# rpn loss
rpn_label_loss, rpn_box_loss = rpn_losses(anchor_labels,
anchor_boxes_encoded,
rpn_label_logits,
rpn_box_logits)
# fastrcnn loss
matched_gt_boxes = tf.gather(gt_boxes, fg_inds_wrt_gt)
fg_inds_wrt_sample = tf.reshape(tf.where(rcnn_labels > 0),
[-1]) # fg inds w.r.t all samples
fg_sampled_boxes = tf.gather(rcnn_boxes, fg_inds_wrt_sample)
fg_fastrcnn_box_logits = tf.gather(fastrcnn_box_logits,
fg_inds_wrt_sample)
fastrcnn_label_loss, fastrcnn_box_loss = self.fastrcnn_training(
image, rcnn_labels, fg_sampled_boxes, matched_gt_boxes,
fastrcnn_label_logits, fg_fastrcnn_box_logits)
if config.MODE_MASK:
# maskrcnn loss
fg_labels = tf.gather(rcnn_labels, fg_inds_wrt_sample)
# In training, mask branch shares the same C5 feature.
fg_feature = tf.gather(feature_fastrcnn, fg_inds_wrt_sample)
mask_logits = maskrcnn_upXconv_head(
'maskrcnn', fg_feature, config.NUM_CLASS,
num_convs=0) # #fg x #cat x 14x14
matched_gt_masks = tf.gather(gt_masks,
fg_inds_wrt_gt) # nfg x H x W
target_masks_for_fg = crop_and_resize(
tf.expand_dims(matched_gt_masks, 1),
fg_sampled_boxes,
tf.range(tf.size(fg_inds_wrt_gt)),
14,
pad_border=False) # nfg x 1x14x14
target_masks_for_fg = tf.squeeze(target_masks_for_fg, 1,
'sampled_fg_mask_targets')
mrcnn_loss = maskrcnn_loss(mask_logits, fg_labels,
target_masks_for_fg)
else:
mrcnn_loss = 0.0
wd_cost = regularize_cost(
'(?:group1|group2|group3|rpn|fastrcnn|maskrcnn)/.*W',
l2_regularizer(1e-4),
name='wd_cost')
total_cost = tf.add_n([
rpn_label_loss, rpn_box_loss, fastrcnn_label_loss,
fastrcnn_box_loss, mrcnn_loss, wd_cost
], 'total_cost')
add_moving_summary(total_cost, wd_cost)
return total_cost
else:
final_boxes, final_labels = self.fastrcnn_inference(
image_shape2d, rcnn_boxes, fastrcnn_label_logits,
fastrcnn_box_logits)
if config.MODE_MASK:
# HACK to work around https://github.com/tensorflow/tensorflow/issues/14657
def f1():
roi_resized = roi_align(
featuremap, final_boxes * (1.0 / config.ANCHOR_STRIDE),
14)
feature_maskrcnn = resnet_conv5(
roi_resized, config.RESNET_NUM_BLOCK[-1])
mask_logits = maskrcnn_upXconv_head(
'maskrcnn', feature_maskrcnn, config.NUM_CLASS,
0) # #result x #cat x 14x14
indices = tf.stack([
tf.range(tf.size(final_labels)),
tf.to_int32(final_labels) - 1
],
axis=1)
final_mask_logits = tf.gather_nd(mask_logits,
indices) # #resultx14x14
return tf.sigmoid(final_mask_logits)
final_masks = tf.cond(
tf.size(final_labels) > 0, f1,
lambda: tf.zeros([0, 14, 14]))
tf.identity(final_masks, name='final_masks')
def load_examples():
if a.input_dir is None or not os.path.exists(a.input_dir):
raise Exception("input_dir does not exist")
input_paths = glob.glob(os.path.join(a.input_dir, "*.jpg"))
decode = tf.image.decode_jpeg
if len(input_paths) == 0:
input_paths = glob.glob(os.path.join(a.input_dir, "*.png"))
decode = tf.image.decode_png
if len(input_paths) == 0:
raise Exception("input_dir contains no image files")
def get_name(path):
name, _ = os.path.splitext(os.path.basename(path))
return name
# if the image names are numbers, sort by the value rather than asciibetically
# having sorted inputs means that the outputs are sorted in test mode
if all(get_name(path).isdigit() for path in input_paths):
input_paths = sorted(input_paths, key=lambda path: int(get_name(path)))
else:
input_paths = sorted(input_paths)
with tf.name_scope("load_images"):
path_queue = tf.train.string_input_producer(input_paths, shuffle=a.mode == "train")
reader = tf.WholeFileReader()
paths, contents = reader.read(path_queue)
raw_input = decode(contents)
raw_input = tf.image.convert_image_dtype(raw_input, dtype=tf.float32)
assertion = tf.assert_equal(tf.shape(raw_input)[2], 3, message="image does not have 3 channels")
with tf.control_dependencies([assertion]):
raw_input = tf.identity(raw_input)
raw_input.set_shape([None, None, 3])
if a.lab_colorization:
# load color and brightness from image, no B image exists here
lab = rgb_to_lab(raw_input)
L_chan, a_chan, b_chan = preprocess_lab(lab)
a_images = tf.expand_dims(L_chan, axis=2)
b_images = tf.stack([a_chan, b_chan], axis=2)
else:
# break apart image pair and move to range [-1, 1]
width = tf.shape(raw_input)[1] # [height, width, channels]
a_images = preprocess(raw_input[:,:width//2,:])
b_images = preprocess(raw_input[:,width//2:,:])
if a.which_direction == "AtoB":
inputs, targets = [a_images, b_images]
elif a.which_direction == "BtoA":
inputs, targets = [b_images, a_images]
else:
raise Exception("invalid direction")
# synchronize seed for image operations so that we do the same operations to both
# input and output images
seed = random.randint(0, 2**31 - 1)
def transform(image):
r = image
if a.flip:
r = tf.image.random_flip_left_right(r, seed=seed)
# area produces a nice downscaling, but does nearest neighbor for upscaling
# assume we're going to be doing downscaling here
r = tf.image.resize_images(r, [a.scale_size, a.scale_size], method=tf.image.ResizeMethod.AREA)
offset = tf.cast(tf.floor(tf.random_uniform([2], 0, a.scale_size - CROP_SIZE + 1, seed=seed)), dtype=tf.int32)
if a.scale_size > CROP_SIZE:
r = tf.image.crop_to_bounding_box(r, offset[0], offset[1], CROP_SIZE, CROP_SIZE)
elif a.scale_size < CROP_SIZE:
raise Exception("scale size cannot be less than crop size")
return r
with tf.name_scope("input_images"):
input_images = transform(inputs)
with tf.name_scope("target_images"):
target_images = transform(targets)
classes = []
for xy in range(0, len(input_paths)):
classes.append(input_paths[xy][len(input_paths[xy]) - 6:len(input_paths[xy]) - 5])
to_categorical = tf.keras.utils.to_categorical
classes_hot = to_categorical(classes, num_classes=2)
classes_queue = tf.train.input_producer(classes_hot, shuffle=a.mode == "train")
paths_batch, classes_batch, inputs_batch, targets_batch = tf.train.batch([paths, classes_queue.dequeue(), input_images, target_images], batch_size=a.batch_size)
steps_per_epoch = int(math.ceil(len(input_paths) / a.batch_size))
return Examples(
paths=paths_batch,
classes=classes_batch,
inputs=inputs_batch,
targets=targets_batch,
count=len(input_paths),
steps_per_epoch=steps_per_epoch,
)
def _inference(self, images, graph_part, cellist): # ,regularizer):
'''Method for recovering the network model provided by graph_part and cellist.
Args:
images: Images returned from Dataset() or inputs().
graph_part: The topology structure of th network given by adjacency table
cellist:
Returns:
Logits.'''
# print('Evaluater:starting to reconstruct the network')
nodelen = len(graph_part)
inputs = [0 for i in range(nodelen)
] # input list for every cell in network
inputs[0] = images
getinput = [
False for i in range(nodelen)
] # bool list for whether this cell has already got input or not
getinput[0] = True
for node in range(nodelen):
# print('Evaluater:right now we are processing node %d'%node,', ',cellist[node])
if cellist[node][0] == 'conv':
layer = self._makeconv(inputs[node], cellist[node], node)
layer = tf.nn.lrn(layer,
4,
bias=1.0,
alpha=0.001 / 9.0,
beta=0.75,
name='norm2')
elif cellist[node][0] == 'pooling':
layer = self._makepool(inputs[node], cellist[node])
elif cellist[node][0] == 'dense':
layer = self._makedense(inputs[node], cellist[node])
else:
print(
'WRONG!!!!! Notice that you got a layer type we cant process!',
cellist[node][0])
layer = []
# update inputs information of the cells below this cell
for j in graph_part[node]:
if getinput[
j]: # if this cell already got inputs from other cells precedes it
# padding
a = int(layer.shape[1])
b = int(inputs[j].shape[1])
pad = abs(a - b)
if layer.shape[1] > inputs[j].shape[1]:
inputs[j] = tf.pad(
inputs[j], [[0, 0], [0, pad], [0, pad], [0, 0]])
if layer.shape[1] < inputs[j].shape[1]:
layer = tf.pad(layer,
[[0, 0], [0, pad], [0, pad], [0, 0]])
inputs[j] = tf.concat([inputs[j], layer], 3)
else:
inputs[j] = layer
getinput[j] = True
# softmax
last_layer = tf.identity(layer, name="last_layer" + str(self.blocks))
# inputdim = layer.shape[3]
# kernel = tf.get_variable('weights', shape=[1, 1, inputdim, NUM_CLASSES],
# initializer=tf.truncated_normal_initializer(stddev=0.1))
# layer = tf.nn.conv2d(layer, kernel, [1, 1, 1, 1], padding='SAME')
# layer = tf.reshape(layer, [batch_size, -1])
# with tf.variable_scope('softmax_linear') as scope:
# weights = tf.get_variable('weights', shape=[layer.shape[-1], NUM_CLASSES],
# initializer=tf.truncated_normal_initializer(stddev=0.04)) # 1 / float(dim)))
# biases = tf.get_variable('biases', shape=[NUM_CLASSES], initializer=tf.constant_initializer(0.0))
# # softmax_linear = tf.nn.softmax(tf.matmul(layer, weights)+ biases, name=scope.name)
# softmax_linear = tf.add(tf.matmul(layer, weights), biases, name="last_layer")
# # tf.add_to_collection('losses', regularizer(weights))
return last_layer
def expanded_conv(input_tensor,
num_outputs,
expansion_size=expand_input_by_factor(6),
stride=1,
rate=1,
kernel_size=(3, 3),
residual=True,
normalizer_fn=None,
project_activation_fn=tf.identity,
split_projection=1,
split_expansion=1,
split_divisible_by=8,
expansion_transform=None,
depthwise_location='expansion',
depthwise_channel_multiplier=1,
endpoints=None,
use_explicit_padding=False,
padding='SAME',
scope=None):
"""Depthwise Convolution Block with expansion.
Builds a composite convolution that has the following structure
expansion (1x1) -> depthwise (kernel_size) -> projection (1x1)
Args:
input_tensor: input
num_outputs: number of outputs in the final layer.
expansion_size: the size of expansion, could be a constant or a callable.
If latter it will be provided 'num_inputs' as an input. For forward
compatibility it should accept arbitrary keyword arguments.
Default will expand the input by factor of 6.
stride: depthwise stride
rate: depthwise rate
kernel_size: depthwise kernel
residual: whether to include residual connection between input
and output.
normalizer_fn: batchnorm or otherwise
project_activation_fn: activation function for the project layer
split_projection: how many ways to split projection operator
(that is conv expansion->bottleneck)
split_expansion: how many ways to split expansion op
(that is conv bottleneck->expansion) ops will keep depth divisible
by this value.
split_divisible_by: make sure every split group is divisible by this number.
expansion_transform: Optional function that takes expansion
as a single input and returns output.
depthwise_location: where to put depthwise covnvolutions supported
values None, 'input', 'output', 'expansion'
depthwise_channel_multiplier: depthwise channel multiplier:
each input will replicated (with different filters)
that many times. So if input had c channels,
output will have c x depthwise_channel_multpilier.
endpoints: An optional dictionary into which intermediate endpoints are
placed. The keys "expansion_output", "depthwise_output",
"projection_output" and "expansion_transform" are always populated, even
if the corresponding functions are not invoked.
use_explicit_padding: Use 'VALID' padding for convolutions, but prepad
inputs so that the output dimensions are the same as if 'SAME' padding
were used.
padding: Padding type to use if `use_explicit_padding` is not set.
scope: optional scope.
Returns:
Tensor of depth num_outputs
Raises:
TypeError: on inval
"""
with tf.variable_scope(scope, default_name='expanded_conv') as s, \
tf.name_scope(s.original_name_scope):
prev_depth = input_tensor.get_shape().as_list()[3]
if depthwise_location not in [None, 'input', 'output', 'expansion']:
raise TypeError('%r is unknown value for depthwise_location' %
depthwise_location)
if use_explicit_padding:
if padding != 'SAME':
raise TypeError(
'`use_explicit_padding` should only be used with '
'"SAME" padding.')
padding = 'VALID'
depthwise_func = functools.partial(
slim.separable_conv2d,
num_outputs=None,
kernel_size=kernel_size,
depth_multiplier=depthwise_channel_multiplier,
stride=stride,
rate=rate,
normalizer_fn=normalizer_fn,
padding=padding,
scope='depthwise')
# b1 -> b2 * r -> b2
# i -> (o * r) (bottleneck) -> o
input_tensor = tf.identity(input_tensor, 'input')
net = input_tensor
if depthwise_location == 'input':
if use_explicit_padding:
net = _fixed_padding(net, kernel_size, rate)
net = depthwise_func(net, activation_fn=None)
if isinstance(expansion_size, collections.Callable):
inner_size = expansion_size(num_inputs=prev_depth)
else:
inner_size = expansion_size
if inner_size > net.shape[3]:
net = split_conv(net,
inner_size,
num_ways=split_expansion,
scope='expand',
divisible_by=split_divisible_by,
stride=1,
normalizer_fn=normalizer_fn)
net = tf.identity(net, 'expansion_output')
if endpoints is not None:
endpoints['expansion_output'] = net
if depthwise_location == 'expansion':
if use_explicit_padding:
net = _fixed_padding(net, kernel_size, rate)
net = depthwise_func(net)
net = tf.identity(net, name='depthwise_output')
if endpoints is not None:
endpoints['depthwise_output'] = net
if expansion_transform:
net = expansion_transform(expansion_tensor=net,
input_tensor=input_tensor)
# Note in contrast with expansion, we always have
# projection to produce the desired output size.
net = split_conv(net,
num_outputs,
num_ways=split_projection,
stride=1,
scope='project',
divisible_by=split_divisible_by,
normalizer_fn=normalizer_fn,
activation_fn=project_activation_fn)
if endpoints is not None:
endpoints['projection_output'] = net
if depthwise_location == 'output':
if use_explicit_padding:
net = _fixed_padding(net, kernel_size, rate)
net = depthwise_func(net, activation_fn=None)
if isinstance(residual, collections.Callable): # custom residual
net = residual(input_tensor=input_tensor, output_tensor=net)
elif (residual and
# stride check enforces that we don't add residuals when spatial
# dimensions are None
stride == 1 and
# Depth matches
net.get_shape().as_list()[3]
== input_tensor.get_shape().as_list()[3]):
net += input_tensor
return tf.identity(net, name='output')
def main():
epoch_count = 0
if a.seed is None:
a.seed = random.randint(0, 2**31 - 1)
tf.set_random_seed(a.seed)
np.random.seed(a.seed)
random.seed(a.seed)
if not os.path.exists(a.output_dir):
os.makedirs(a.output_dir)
if a.mode == "test" or a.mode == "export":
if a.checkpoint is None:
raise Exception("checkpoint required for test mode")
# load some options from the checkpoint
options = {"which_direction", "ngf", "ndf", "lab_colorization"}
with open(os.path.join(a.checkpoint, "options.json")) as f:
for key, val in json.loads(f.read()).items():
if key in options:
print("loaded", key, "=", val)
setattr(a, key, val)
# disable these features in test mode
a.scale_size = CROP_SIZE
a.flip = False
for k, v in a._get_kwargs():
print(k, "=", v)
with open(os.path.join(a.output_dir, "options.json"), "w") as f:
f.write(json.dumps(vars(a), sort_keys=True, indent=4))
if a.mode == "export":
# export the generator to a meta graph that can be imported later for standalone generation
if a.lab_colorization:
raise Exception("export not supported for lab_colorization")
input = tf.placeholder(tf.string, shape=[1])
input_data = tf.decode_base64(input[0])
input_image = tf.image.decode_png(input_data)
# remove alpha channel if present
input_image = tf.cond(tf.equal(tf.shape(input_image)[2], 4), lambda: input_image[:,:,:3], lambda: input_image)
# convert grayscale to RGB
input_image = tf.cond(tf.equal(tf.shape(input_image)[2], 1), lambda: tf.image.grayscale_to_rgb(input_image), lambda: input_image)
input_image = tf.image.convert_image_dtype(input_image, dtype=tf.float32)
input_image.set_shape([CROP_SIZE, CROP_SIZE, 3])
batch_input = tf.expand_dims(input_image, axis=0)
with tf.variable_scope("generator"):
batch_output = deprocess(create_generator(preprocess(batch_input), 3))
output_image = tf.image.convert_image_dtype(batch_output, dtype=tf.uint8)[0]
if a.output_filetype == "png":
output_data = tf.image.encode_png(output_image)
elif a.output_filetype == "jpeg":
output_data = tf.image.encode_jpeg(output_image, quality=80)
else:
raise Exception("invalid filetype")
output = tf.convert_to_tensor([tf.encode_base64(output_data)])
key = tf.placeholder(tf.string, shape=[1])
inputs = {
"key": key.name,
"input": input.name
}
tf.add_to_collection("inputs", json.dumps(inputs))
outputs = {
"key": tf.identity(key).name,
"output": output.name,
}
tf.add_to_collection("outputs", json.dumps(outputs))
init_op = tf.global_variables_initializer()
restore_saver = tf.train.Saver()
export_saver = tf.train.Saver()
with tf.Session() as sess:
sess.run(init_op)
print("loading model from checkpoint")
checkpoint = tf.train.latest_checkpoint(a.checkpoint)
restore_saver.restore(sess, checkpoint)
name = 'BotnetData'
restore_saver.restore(sess, '/home/shayan/CIFAR/tensorflow-adversarial-master/example/iter4_retrain/pix2pix_adv/Epoch_10/{}'.format(name))
print("exporting model")
export_saver.export_meta_graph(filename=os.path.join(a.output_dir, "export.meta"))
export_saver.save(sess, os.path.join(a.output_dir, "export"), write_meta_graph=False)
return
examples = load_examples()
print("examples count = %d" % examples.count)
# inputs and targets are [batch_size, height, width, channels]
# modelX = create_model(examples.inputs, examples.targets, examples.classes)
modelX = create_model(examples.inputs, examples.targets, examples.classes)
# undo colorization splitting on images that we use for display/output
if a.lab_colorization:
if a.which_direction == "AtoB":
# inputs is brightness, this will be handled fine as a grayscale image
# need to augment targets and outputs with brightness
targets = augment(examples.targets, examples.inputs)
outputs = augment(modelX.outputs, examples.inputs)
# inputs can be deprocessed normally and handled as if they are single channel
# grayscale images
inputs = deprocess(examples.inputs)
elif a.which_direction == "BtoA":
# inputs will be color channels only, get brightness from targets
inputs = augment(examples.inputs, examples.targets)
targets = deprocess(examples.targets)
outputs = deprocess(modelX.outputs)
else:
raise Exception("invalid direction")
else:
inputs = deprocess(examples.inputs)
targets = deprocess(examples.targets)
outputs = deprocess(modelX.outputs)
def convert(image):
if a.aspect_ratio != 1.0:
# upscale to correct aspect ratio
size = [CROP_SIZE, int(round(CROP_SIZE * a.aspect_ratio))]
image = tf.image.resize_images(image, size=size, method=tf.image.ResizeMethod.BICUBIC)
return tf.image.convert_image_dtype(image, dtype=tf.uint8, saturate=True)
# reverse any processing on images so they can be written to disk or displayed to user
with tf.name_scope("convert_inputs"):
converted_inputs = convert(inputs)
with tf.name_scope("convert_targets"):
converted_targets = convert(targets)
with tf.name_scope("convert_outputs"):
converted_outputs = convert(outputs)
with tf.name_scope("encode_images"):
display_fetches = {
"paths": examples.paths,
"inputs": tf.map_fn(tf.image.encode_png, converted_inputs, dtype=tf.string, name="input_pngs"),
"targets": tf.map_fn(tf.image.encode_png, converted_targets, dtype=tf.string, name="target_pngs"),
"outputs": tf.map_fn(tf.image.encode_png, converted_outputs, dtype=tf.string, name="output_pngs"),
}
# summaries
with tf.name_scope("inputs_summary"):
tf.summary.image("inputs", converted_inputs)
with tf.name_scope("targets_summary"):
tf.summary.image("targets", converted_targets)
with tf.name_scope("outputs_summary"):
tf.summary.image("outputs", converted_outputs)
with tf.name_scope("predict_real_summary"):
tf.summary.image("predict_real", tf.image.convert_image_dtype(modelX.predict_real, dtype=tf.uint8))
with tf.name_scope("predict_fake_summary"):
tf.summary.image("predict_fake", tf.image.convert_image_dtype(modelX.predict_fake, dtype=tf.uint8))
tf.summary.scalar("discriminator_loss", modelX.discrim_loss)
tf.summary.scalar("generator_loss_GAN", modelX.gen_loss_GAN)
tf.summary.scalar("generator_loss_L1", modelX.gen_loss_L1)
tf.summary.scalar("generator_loss", modelX.gen_loss_L1)
for var in tf.trainable_variables():
tf.summary.histogram(var.op.name + "/values", var)
for grad, var in modelX.discrim_grads_and_vars + modelX.gen_grads_and_vars:
tf.summary.histogram(var.op.name + "/gradients", grad)
with tf.name_scope("parameter_count"):
parameter_count = tf.reduce_sum([tf.reduce_prod(tf.shape(v)) for v in tf.trainable_variables()])
saver = tf.train.Saver(max_to_keep=1)
logdir = a.output_dir if (a.trace_freq > 0 or a.summary_freq > 0) else None
sv = tf.train.Supervisor(logdir=logdir, save_summaries_secs=0, saver=None)
with sv.managed_session() as sess:
print("parameter_count =", sess.run(parameter_count))
if a.checkpoint is not None:
print("loading model from checkpoint")
checkpoint = tf.train.latest_checkpoint(a.checkpoint)
saver.restore(sess, checkpoint)
max_steps = 2**32
if a.max_epochs is not None:
max_steps = examples.steps_per_epoch * a.max_epochs
if a.max_steps is not None:
max_steps = a.max_steps
if a.mode == "test":
# testing
# at most, process the test data once
start = time.time()
max_steps = min(examples.steps_per_epoch, max_steps)
for step in range(max_steps):
results = sess.run(display_fetches)
filesets = save_images(results)
for i, f in enumerate(filesets):
print("evaluated image", f["name"])
index_path = append_index(filesets)
print("wrote index at", index_path)
print("rate", (time.time() - start) / max_steps)
else:
# training
start = time.time()
for step in range(max_steps):
def should(freq):
return freq > 0 and ((step + 1) % freq == 0 or step == max_steps - 1)
options = None
run_metadata = None
if should(a.trace_freq):
options = tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE)
run_metadata = tf.RunMetadata()
fetches = {
"train": modelX.train,
"global_step": sv.global_step,
}
if should(a.progress_freq):
fetches["adv_real"] = modelX.adv_real
fetches["adv_fake"] = modelX.adv_fake
fetches["predict_real"] = modelX.predict_real
fetches["predict_fake"] = modelX.predict_fake
fetches["discrim_loss"] = modelX.discrim_loss
fetches["gen_loss_GAN"] = modelX.gen_loss_GAN
fetches["gen_loss_L1"] = modelX.gen_loss_L1
fetches["gen_loss"] = modelX.gen_loss
fetches["gan_total"] = modelX.gan_total
fetches["l1_total"] = modelX.l1_total
fetches["adver_total"] = modelX.adver_total
if should(a.summary_freq):
fetches["summary"] = sv.summary_op
if should(a.display_freq):
fetches["display"] = display_fetches
results = sess.run(fetches, options=options, run_metadata=run_metadata)
if should(a.summary_freq):
print("recording summary")
sv.summary_writer.add_summary(results["summary"], results["global_step"])
if should(a.display_freq):
print("saving display images")
filesets = save_images(results["display"], steIteration_5p=results["global_step"])
append_index(filesets, step=True)
if should(a.trace_freq):
print("recording trace")
sv.summary_writer.add_run_metadata(run_metadata, "step_%d" % results["global_step"])
if should(a.progress_freq):
# global_step will have the correct step count if we resume from a checkpoint
train_epoch = math.ceil(results["global_step"] / examples.steps_per_epoch)
train_step = (results["global_step"] - 1) % examples.steps_per_epoch + 1
rate = (step + 1) * a.batch_size / (time.time() - start)
remaining = (max_steps - step) * a.batch_size / rate
print("progress epoch %d step %d image/sec %0.1f remaining %dm" % (train_epoch, train_step, rate, remaining / 60))
print("adv_real", results["adv_real"])
print("adv_fake", results["adv_fake"])
# print("predict_real", results["predict_real"])
# print("predict_fake", results["predict_fake"])
print("discrim_loss", results["discrim_loss"])
print("gen_loss_GAN", results["gen_loss_GAN"])
print("gen_loss_L1", results["gen_loss_L1"])
print("gen_loss", results["gen_loss"])
print("gan_total", results["gan_total"])
print("l1_total", results["l1_total"])
print("adver_total", results["adver_total"])
if should(a.save_freq_epoch):
epoch_count = epoch_count + 1
print("saving model at every epoch")
saver.save(sess, os.path.join(a.output_dir, "model"), global_step=sv.global_step)
if not os.path.exists("/home/shayan/Proposal_Pix2Pix/Data/Training/FGSM/Iteration_5/Training_Out/Model_Epochs/" + "Epoch_" + str(epoch_count)):
os.mkdir("/home/shayan/Proposal_Pix2Pix/Data/Training/FGSM/Iteration_5/Training_Out/Model_Epochs/" + "Epoch_" + str(epoch_count))
from distutils.dir_util import copy_tree
copy_tree(a.output_dir, "/home/shayan/Proposal_Pix2Pix/Data/Training/FGSM/Iteration_5/Training_Out/Model_Epochs/" + "Epoch_" + str(epoch_count))
# if should(a.save_freq):
# print("saving model")
# saver.save(sess, os.path.join(a.output_dir, "model"), global_step=sv.global_step)
if sv.should_stop():
break
def main(args):
network = importlib.import_module(args.model_def, 'inference')
subdir = datetime.strftime(datetime.now(), '%Y%m%d-%H%M%S')
log_dir = os.path.join(os.path.expanduser(args.logs_base_dir), subdir)
if not os.path.isdir(log_dir): # Create the log directory if it doesn't exist
os.makedirs(log_dir)
model_dir = os.path.join(os.path.expanduser(args.models_base_dir), subdir)
if not os.path.isdir(model_dir): # Create the model directory if it doesn't exist
os.makedirs(model_dir)
# Store some git revision info in a text file in the log directory
src_path,_ = os.path.split(os.path.realpath(__file__))
facenet.store_revision_info(src_path, log_dir, ' '.join(sys.argv))
np.random.seed(seed=args.seed)
train_set = facenet.get_dataset(args.data_dir)
nrof_classes = len(train_set)
print('Model directory: %s' % model_dir)
print('Log directory: %s' % log_dir)
pretrained_model = None
if args.pretrained_model:
pretrained_model = os.path.expanduser(args.pretrained_model)
print('Pre-trained model: %s' % pretrained_model)
if args.lfw_dir:
print('LFW directory: %s' % args.lfw_dir)
# Read the file containing the pairs used for testing
pairs = lfw.read_pairs(os.path.expanduser(args.lfw_pairs))
# Get the paths for the corresponding images
lfw_paths, actual_issame = lfw.get_paths(os.path.expanduser(args.lfw_dir), pairs, args.lfw_file_ext)
with tf.Graph().as_default():
tf.set_random_seed(args.seed)
global_step = tf.Variable(0, trainable=False)
# Get a list of image paths and their labels
image_list, label_list = facenet.get_image_paths_and_labels(train_set)
# Read data and apply label preserving distortions
image_batch, label_batch = facenet.read_and_augument_data(image_list, label_list, args.image_size,
args.batch_size, args.max_nrof_epochs, args.random_crop, args.random_flip, args.random_rotate,
args.nrof_preprocess_threads)
print('Total number of classes: %d' % nrof_classes)
print('Total number of examples: %d' % len(image_list))
print('Building training graph')
# Placeholder for the learning rate
learning_rate_placeholder = tf.placeholder(tf.float32, name='learning_rate')
# Build the inference graph
prelogits, _ = network.inference(image_batch, args.keep_probability,
phase_train=True, weight_decay=args.weight_decay)
logits = slim.fully_connected(prelogits, len(train_set), activation_fn=None,
weights_initializer=tf.truncated_normal_initializer(stddev=0.1),
weights_regularizer=slim.l2_regularizer(args.weight_decay),
scope='Logits', reuse=False)
# Add DeCov regularization loss
if args.decov_loss_factor>0.0:
logits_decov_loss = facenet.decov_loss(logits) * args.decov_loss_factor
tf.add_to_collection(tf.GraphKeys.REGULARIZATION_LOSSES, logits_decov_loss)
# Add center loss
if args.center_loss_factor>0.0:
prelogits_center_loss, _ = facenet.center_loss(prelogits, label_batch, args.center_loss_alfa, nrof_classes)
tf.add_to_collection(tf.GraphKeys.REGULARIZATION_LOSSES, prelogits_center_loss * args.center_loss_factor)
learning_rate = tf.train.exponential_decay(learning_rate_placeholder, global_step,
args.learning_rate_decay_epochs*args.epoch_size, args.learning_rate_decay_factor, staircase=True)
tf.scalar_summary('learning_rate', learning_rate)
# Calculate the average cross entropy loss across the batch
cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(
logits, label_batch, name='cross_entropy_per_example')
cross_entropy_mean = tf.reduce_mean(cross_entropy, name='cross_entropy')
tf.add_to_collection('losses', cross_entropy_mean)
# Calculate the total losses
regularization_losses = tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES)
total_loss = tf.add_n([cross_entropy_mean] + regularization_losses, name='total_loss')
# Build a Graph that trains the model with one batch of examples and updates the model parameters
train_op = facenet.train(total_loss, global_step, args.optimizer,
learning_rate, args.moving_average_decay, tf.all_variables(), args.log_histograms)
# Evaluation
print('Building evaluation graph')
lfw_label_list = range(0,len(lfw_paths))
assert (len(lfw_paths) % args.lfw_batch_size == 0), "The number of images in the LFW test set need to be divisible by the lfw_batch_size"
eval_image_batch, eval_label_batch = facenet.read_and_augument_data(lfw_paths, lfw_label_list, args.image_size,
args.lfw_batch_size, None, False, False, False, args.nrof_preprocess_threads, shuffle=False)
# Node for input images
eval_image_batch.set_shape((None, args.image_size, args.image_size, 3))
eval_image_batch = tf.identity(eval_image_batch, name='input')
eval_prelogits, _ = network.inference(eval_image_batch, 1.0,
phase_train=False, weight_decay=0.0, reuse=True)
eval_embeddings = tf.nn.l2_normalize(eval_prelogits, 1, 1e-10, name='embeddings')
# Create a saver
saver = tf.train.Saver(tf.all_variables(), max_to_keep=3)
# Build the summary operation based on the TF collection of Summaries.
summary_op = tf.merge_all_summaries()
# Start running operations on the Graph.
gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=args.gpu_memory_fraction)
sess = tf.Session(config=tf.ConfigProto(gpu_options=gpu_options, log_device_placement=False))
sess.run(tf.initialize_all_variables())
sess.run(tf.initialize_local_variables())
summary_writer = tf.train.SummaryWriter(log_dir, sess.graph)
tf.train.start_queue_runners(sess=sess)
with sess.as_default():
if pretrained_model:
print('Restoring pretrained model: %s' % pretrained_model)
saver.restore(sess, pretrained_model)
# Training and validation loop
print('Running training')
epoch = 0
while epoch < args.max_nrof_epochs:
step = sess.run(global_step, feed_dict=None)
epoch = step // args.epoch_size
# Train for one epoch
train(args, sess, epoch, learning_rate_placeholder, global_step,
total_loss, train_op, summary_op, summary_writer, regularization_losses, args.learning_rate_schedule_file)
# Save variables and the metagraph if it doesn't exist already
save_variables_and_metagraph(sess, saver, summary_writer, model_dir, subdir, step)
# Evaluate on LFW
'''
if args.lfw_dir:
evaluate(sess, eval_embeddings, eval_label_batch, actual_issame, args.lfw_batch_size, args.seed,
args.lfw_nrof_folds, log_dir, step, summary_writer)
'''
return model_dir
def _testWhileLoopWritePackGradients(self, dynamic_size, dtype, legacy):
np_dtype = dtype.as_numpy_dtype
with self.test_session(use_gpu=self._use_gpu) as session:
v0 = tf.identity(np.arange(3*5, dtype=np_dtype).reshape(3, 5))
var = tf.Variable(np.arange(100, 105, dtype=np_dtype))
state0 = tf.identity(np.array([1] * 5, dtype=np_dtype))
ta = tensor_array_ops.TensorArray(
dtype=dtype, tensor_array_name="foo",
size=0 if dynamic_size else 3, dynamic_size=dynamic_size)
time_0 = tf.identity(0)
def body(time, ta_t, state):
sliced = tf.slice(v0, begin=tf.stack([time, 0]), size=[1, -1])
sliced = tf.squeeze(sliced)
out = sliced + var + state
state += sliced
ta_t = ta_t.write(time, out)
return (time+1, ta_t, state)
(unused_0, h_final, unused_2) = tf.while_loop(
cond=lambda time, unused_1, unused_2: time < 3,
body=body,
loop_vars=(time_0, ta, state0),
shape_invariants=(time_0.get_shape(),
tensor_shape.unknown_shape(),
tensor_shape.unknown_shape()),
parallel_iterations=3)
if legacy:
vout = h_final._legacy_pack()
else:
vout = h_final.pack()
grad_val = -np.arange(3*5, dtype=np_dtype).reshape(3, 5)
v0_grad = tf.gradients([vout], [v0], [grad_val])[0]
state0_grad = tf.gradients([vout], [state0], [grad_val])[0]
var_grad = tf.gradients([vout], [var], [grad_val])[0]
tf.global_variables_initializer().run()
state0_t, var_t, v0_t, vout_t, v0_grad_t, var_grad_t, state0_grad_t = (
session.run([state0, var, v0, vout, v0_grad, var_grad, state0_grad]))
just_v0_grad_t, = session.run([v0_grad])
# state = [ state0 | state0 + v0[0] | state0 + v0[0] + v0[1] ]
# vout = [ v0[0] + var + state[0] |
# v0[1] + var + state[1] |
# v0[2] + var + state[2] ]
# = [ v0[0] + var + state0 |
# v0[1] + var + state0 + v0[0] |
# v0[2] + var + state0 + v0[0] + v0[1] ]
#
# d(vout[0])/d(v0) = [1 | 0 | 0 ]
# d(vout[1])/d(v0) = [1 | 1 | 0 ]
# d(vout[2])/d(v0) = [1 | 1 | 1 ]
# d(vout)/d(var) = [1 | 1 | 1]
# d(vout)/d(state0) = [ 1 | 1 | 1 ]
state_per_time = np.array([
state0_t,
state0_t + v0_t[0, :],
state0_t + v0_t[0, :] + v0_t[1, :]])
# Compare forward prop
self.assertAllClose(v0_t + var_t + state_per_time, vout_t)
# Compare backward prop
expected_v0_grad_t = np.array([
grad_val[0, :] + grad_val[1, :] + grad_val[2, :],
grad_val[1, :] + grad_val[2, :],
grad_val[2, :]])
self.assertAllEqual(expected_v0_grad_t, v0_grad_t)
self.assertAllEqual(expected_v0_grad_t, just_v0_grad_t)
self.assertAllClose(grad_val.sum(axis=0), var_grad_t)
self.assertAllClose(grad_val.sum(axis=0), state0_grad_t)
def identity():
return tf.identity(inputs)
def tf_decoding(memory_tensor,
memory_sequence_length,
embedding_table,
decoding_args,
decoder_type,
kernel_initializer_range,
bias_initializer_range,
atol_threshold=1e-6):
with tf.variable_scope("transformer/decoder", reuse=tf.AUTO_REUSE):
# copy memory and memory_sequence_length by beam_width times
# if memory is [a, b, c], beam_width = 3, then the result is: [a a a b b b c c c ]
extended_memory = tf.contrib.seq2seq.tile_batch(
memory_tensor, multiplier=decoding_args.decoder_args.beam_width)
extended_memory_sequence_length = tf.contrib.seq2seq.tile_batch(
memory_sequence_length, multiplier=decoding_args.decoder_args.beam_width)
def _cond(word_ids, cum_log_probs, finished, step, outputs, my_cache, extra_vars, op_self_cache, op_mem_cache):
return tf.reduce_any(tf.logical_not(finished))
def _body(word_ids, cum_log_probs, finished, step, outputs, my_cache, extra_vars, op_self_cache, op_mem_cache):
# [batch_size * beam_width, hidden_dim]
inputs = tf.nn.embedding_lookup(embedding_table, word_ids)
# [batch_size * beam_width, 1, hidden_dim]
inputs = tf.expand_dims(inputs, 1)
tf_result = tf_decoder(decoder_args=decoding_args.decoder_args,
inputs=inputs,
memory=extended_memory,
memory_sequence_length=extended_memory_sequence_length,
step=step,
cache=my_cache,
kernel_initializer_range=kernel_initializer_range,
bias_initializer_range=bias_initializer_range)
decoder_vars = tf.global_variables()
decoder_vars_start_id = 0
while decoder_vars_start_id < len(decoder_vars):
if decoder_vars[decoder_vars_start_id].name.find("transformer/decoder") != -1:
break
decoder_vars_start_id += 1
decoder_vars = decoder_vars[decoder_vars_start_id:]
psuedo_input = []
if decoder_type == 2:
psuedo_input = tf_result
op_result, op_self_cache, op_mem_cache = op_decoder(inputs,
step,
extended_memory,
extended_memory_sequence_length,
op_self_cache,
op_mem_cache,
psuedo_input,
decoder_vars,
decoding_args.decoder_args,
decoding_args.encoder_hidden_dim)
result = None
if decoder_type == 0:
result = tf_result
elif decoder_type == 1:
result = op_result
elif decoder_type == 2:
result = tf_result
result_2 = op_result
diff = tf.math.reduce_max(tf.math.abs(result - result_2))
result = tf.Print(result, ["[INFO][PYTHON] step:", step, "max diff: ",
diff, tf.cond(diff < atol_threshold, lambda: "True", lambda: "False")])
else:
print("[TF][ERROR] decoder type is only 0 or 1 or 2.")
exit(-1)
# [batch_size * beam_width, hidden_dim]
result = tf.squeeze(result, axis=1)
logits = tf.layers.dense(result,
decoding_args.vocab_size,
use_bias=True,
bias_initializer=create_initializer(
bias_initializer_range, decoding_args.decoder_args.dtype),
kernel_initializer=create_initializer(
kernel_initializer_range, decoding_args.decoder_args.dtype),
activation=None)
end_ids = tf.fill([decoding_args.decoder_args.batch_size * decoding_args.decoder_args.beam_width],
decoding_args.end_id) # [batch_size * beam_width]
eos_max_prob = tf.one_hot(end_ids, decoding_args.vocab_size,
on_value=decoding_args.decoder_args.dtype.max,
off_value=decoding_args.decoder_args.dtype.min) # [batch_size * beam_width, vocab_size]
# [batch_size * beam_width, vocab_size]
logits = tf.where(finished, x=eos_max_prob, y=logits)
logits = tf.cast(logits, tf.float32)
# [batch_size * beam_width, vocab_size]
log_probs = tf.nn.log_softmax(logits)
output_id, next_cum_log_probs, finished, my_cache, \
extra_vars, op_self_cache = beam_search(decoding_args.decoder_args.beam_width,
decoding_args.vocab_size,
step,
log_probs,
cum_log_probs,
finished,
my_cache,
extra_vars,
op_self_cache)
outputs = outputs.write(step, output_id)
cum_log_probs = tf.where(
finished, x=cum_log_probs, y=next_cum_log_probs)
finished = tf.logical_or(finished, tf.equal(
output_id, decoding_args.end_id))
return output_id, cum_log_probs, finished, step + 1, outputs, my_cache, extra_vars, op_self_cache, op_mem_cache
# initialization
start_ids, step, outputs, tf_decoder_cache, finished, initial_log_probs, \
tf_sequence_lengths, extra_vars = initialize_decoding_variables(
decoding_args)
word_ids = tf.identity(start_ids, name="word_ids")
cum_log_probs = tf.identity(initial_log_probs, name="cum_log_probs")
# if use_op == False, these two caches are useless
op_self_cache, op_mem_cache = init_op_cache(decoding_args.decoder_args)
_, _, _, _, outputs, _, extra_vars, _, _ = tf.while_loop(
_cond,
_body,
loop_vars=(
word_ids,
cum_log_probs,
finished,
step,
outputs,
tf_decoder_cache,
extra_vars,
op_self_cache,
op_mem_cache
),
back_prop=False,
maximum_iterations=decoding_args.decoder_args.max_seq_len,
shape_invariants=(
start_ids.shape,
initial_log_probs.shape,
finished.shape,
step.shape,
tf.TensorShape(None),
tf.contrib.framework.nest.map_structure(
_get_shape_invariants, tf_decoder_cache),
tf.contrib.framework.nest.map_structure(
_get_shape_invariants, extra_vars),
tf.contrib.framework.nest.map_structure(
_get_shape_invariants, op_self_cache),
tf.contrib.framework.nest.map_structure(_get_shape_invariants, op_mem_cache))
)
tf_parent_ids = extra_vars[0].stack()
tf_sequence_lengths = extra_vars[1]
tf_output_ids = outputs.stack()
finalized_tf_output_ids, finalized_tf_sequence_lengths = finalize(decoding_args.decoder_args.beam_width,
tf_parent_ids,
tf_sequence_lengths,
tf_output_ids,
decoding_args.end_id,
decoding_args.decoder_args.max_seq_len)
finalized_tf_output_ids = tf.cast(
finalized_tf_output_ids, start_ids.dtype)
finalized_tf_sequence_lengths = tf.minimum(
finalized_tf_sequence_lengths + 1, tf.shape(finalized_tf_output_ids)[2])
return finalized_tf_output_ids, finalized_tf_sequence_lengths, tf_output_ids, tf_parent_ids, tf_sequence_lengths