def closure(inputs,
unused_conditioning=None,
weight_decay=2.5e-5,
is_training=True) -> tf.Tensor:
input_layer = tf.reshape(inputs, [-1, size])
outputs = tf.expand_dims(input_layer, 2)
outputs = conv1d(outputs,
filters=4,
kernel_size=2,
strides=1,
padding='same',
activation=leaky_relu)
outputs = conv1d(outputs,
filters=4,
kernel_size=2,
strides=1,
padding='same',
activation=leaky_relu)
outputs = conv1d(outputs,
filters=4,
kernel_size=2,
strides=1,
padding='same',
activation=leaky_relu)
outputs = conv1d(outputs,
filters=4,
kernel_size=2,
strides=1,
padding='same',
activation=leaky_relu)
outputs = max_pooling1d(outputs, pool_size=2, strides=1)
outputs = flatten(outputs)
outputs = fully_connected(outputs, 4, activation=leaky_relu)
outputs = fully_connected(outputs, 1, activation=leaky_relu)
return outputs
def attention_estimator(x, g):
with tf.variable_scope(None, 'attention_estimator'):
_, height, width, x_dim = x.get_shape().as_list()
g_dim = g.get_shape().as_list()[-1]
if not x_dim == g_dim:
x = dense(x, g_dim, use_bias=False)
c = tf.reduce_sum(x * tf.expand_dims(tf.expand_dims(g, 1), 1), axis=-1)
a = tf.nn.softmax(flatten(c))
a = tf.reshape(a, (-1, height, width))
g_out = x * tf.expand_dims(a, -1)
g_out = tf.reduce_sum(g_out, axis=[1, 2])
return g_out, a
def conv2d_rnn_encoder(inputs,
input_shape,
filters,
kernel_sizes,
strides,
activation,
latent_hidden_sizes,
latent_hidden_activation,
rnn_hidden_sizes,
rnn_cell_fn,
scope=None,
reuse=None):
with variable_scope.variable_scope(scope, default_name=scope, reuse=reuse):
inputs_shape = array_ops.shape(inputs)
batch_size = inputs_shape[0]
sequence_length = inputs_shape[1]
stacked_inputs = gen_array_ops.reshape(
inputs, [batch_size * sequence_length] + input_shape)
x, encoder, shapes = conv2d_encoder(
stacked_inputs, filters,
kernel_sizes, strides,
scope='encoder',
activation=activation,
reuse=reuse)
output_shape = array_ops.shape(x)
x = core.flatten(x)
x_size = x.get_shape()[-1]
x = gen_array_ops.reshape(
x, [batch_size, sequence_length, x_size])
for hidden in latent_hidden_sizes:
x = core.dense(
x, hidden,
activation=latent_hidden_activation)
outputs, states, initial_state_phs, zero_states = stacked_rnn_impl.stacked_rnn(
x, rnn_hidden_sizes, rnn_cell_fn,
scope='stacked_rnn',
reuse=reuse)
return ( # TODO(wenkesj): make this less crazy
outputs, states, initial_state_phs, zero_states,
encoder, shapes, output_shape, [x_size] + latent_hidden_sizes)
def test_file(name):
infile = np.loadtxt(name)
indata = infile[np.newaxis, :, :, np.newaxis]
conv1 = tf.nn.conv2d(
indata, conv1_weights_tf, strides=[1, 1, 1, 1
], padding='SAME') + conv1_biases
f**k = conv1.numpy()
fuck2 = np.einsum('bijc->bcij', f**k)
fuck3 = fuck2[0][0]
fuck3 = fuck2[0][1]
fuck3 = fuck2[0][2]
conv1 = tf.nn.relu(conv1)
conv1 = tf.nn.max_pool(conv1,
ksize=[1, 2, 2, 1],
strides=[1, 2, 2, 1],
padding='VALID')
conv2 = tf.nn.conv2d(
conv1, conv2_weights_tf, strides=[1, 1, 1, 1
], padding='VALID') + conv2_biases
conv2 = tf.nn.relu(conv2)
conv2 = tf.nn.max_pool(conv2,
ksize=[1, 2, 2, 1],
strides=[1, 2, 2, 1],
padding='VALID')
fc0 = flatten(conv2)
fc1 = tf.matmul(fc0, fc1_weights) + fc1_biases
fc1 = tf.nn.relu(fc1)
fc2 = tf.matmul(fc1, fc2_weights) + fc2_biases
fc2 = tf.nn.relu(fc2)
logits = tf.matmul(fc2, fc3_weights) + fc3_biases
return np.argmax(logits[0])
def testFunctionalFlatten(self):
x = array_ops.placeholder(shape=(None, 2, 3), dtype='float32')
y = core_layers.flatten(x, name='flatten')
self.assertEqual(y.get_shape().as_list(), [None, 6])
def testFunctionalFlatten(self): x = array_ops.placeholder(shape=(None, 2, 3), dtype='float32') y = core_layers.flatten(x, name='flatten') self.assertEqual(y.get_shape().as_list(), [None, 6])
def __init__(self,
in_shape,
lr=0.001,
embedding_dim=50,
num_projections=20):
'''
classes: List of class names or integers corrisponding to each class being classified
by the network. ie: ['left', 'straight', 'right'] or [0, 1, 2]
'''
# number of elements in the embedding vector
self.embedding_dim = embedding_dim
# number of test embeddings to project in tensorboard
self.num_projections = num_projections
# Define model
tf.reset_default_graph()
self.x = tf.placeholder(tf.float32,
shape=[
None,
] + in_shape,
name="x")
self.y = tf.placeholder(tf.float32,
shape=[
None,
] + in_shape,
name="y")
self._training = tf.placeholder(tf.bool)
self.training = tf.get_variable("training",
dtype=tf.bool,
initializer=True,
trainable=False)
self.set_training = self.training.assign(self._training)
self._embeddings = tf.placeholder(tf.float32,
shape=[None, self.embedding_dim])
self.embeddings = tf.get_variable(
"embeddings",
dtype=tf.float32,
shape=[self.num_projections, self.embedding_dim],
initializer=tf.zeros_initializer(),
trainable=False)
self.update_embeddings = self.embeddings.assign(self._embeddings)
paddings = tf.constant([[0, 0], [4, 4], [0, 0], [0, 0]])
relu = tf.nn.relu
sigmoid = tf.nn.sigmoid
with tf.name_scope("encoder"):
# Padding invector so reconstruction returns to the correct size.
x_padded = tf.pad(self.x, paddings, "SYMMETRIC")
# Encoder in num shape stride pad
enc1 = conv2d(x_padded,
24, (5, 5), (2, 2),
"same",
activation=relu,
kernel_initializer=xavier(),
name="enc1") # 64, 80,24
enc2 = conv2d(enc1,
32, (5, 5), (2, 2),
"same",
activation=relu,
kernel_initializer=xavier(),
name="enc2") # 32, 40,32
enc3 = conv2d(enc2,
64, (5, 5), (2, 2),
"same",
activation=relu,
kernel_initializer=xavier(),
name="enc3") # 16, 20,64
enc4 = conv2d(enc3,
64, (3, 3), (2, 2),
"same",
activation=relu,
kernel_initializer=xavier(),
name="enc4") # 8, 10,64
enc5 = conv2d(enc4,
64, (3, 3), (1, 1),
"same",
activation=relu,
kernel_initializer=xavier(),
name="enc5") # 8, 10,64
enc5f = flatten(enc5)
# in num
enc6 = dense(enc5f,
100,
activation=relu,
kernel_initializer=xavier(),
name="enc6")
enc6d = dropout(enc6,
rate=0.1,
training=self.training,
name="enc6d")
with tf.name_scope("embedding"):
# Autoencoder embedding
self.z = dense(enc6d,
self.embedding_dim,
activation=relu,
kernel_initializer=xavier(),
name="z")
tf.summary.histogram("z", self.z)
with tf.name_scope("decoder"):
# Decoder in num
dec1 = dense(self.z,
100,
activation=relu,
kernel_initializer=xavier(),
name="dec1")
dec2 = dense(dec1, (8 * 10 * 64),
activation=relu,
kernel_initializer=xavier(),
name="dec2")
dec2r = tf.reshape(dec2, (-1, 8, 10, 64))
# in num shape stride pad
dec3 = conv2d_transpose(dec2r,
64, (3, 3), (1, 1),
"same",
activation=relu,
name="dec3")
dec4 = conv2d_transpose(dec3,
64, (3, 3), (2, 2),
"same",
activation=relu,
name="dec4")
dec5 = conv2d_transpose(dec4,
32, (5, 5), (2, 2),
"same",
activation=relu,
name="dec5")
dec6 = conv2d_transpose(dec5,
24, (5, 5), (2, 2),
"same",
activation=relu,
name="dec6")
dec7 = conv2d_transpose(dec6,
3, (5, 5), (2, 2),
"same",
activation=None,
name="dec8")
self.dec = relu(dec7, name="reconstruction")
with tf.name_scope("loss"):
# # VAE Loss
# 1. Reconstruction loss: How far did we get from the actual image?
y_padded = tf.pad(self.y, paddings, "SYMMETRIC")
self.rec_loss = tf.reduce_sum(tf.square(y_padded - self.dec),
axis=[1, 2, 3])
self.loss = tf.reduce_mean(self.rec_loss, name='total_loss')
tf.summary.scalar("total_loss", self.loss)
update_tensor_dict(INPUTS, IMAGE_INPUT, self.x)
update_tensor_dict(OUTPUTS, EMBEDDING, self.z)
update_tensor_dict(OUTPUTS, RECONSTRUCTION, self.dec)
optimizer = tf.train.AdamOptimizer(learning_rate=lr)
self.train_step = optimizer.minimize(self.loss, name="train_step")
self.init_vars = tf.global_variables_initializer()
self.saver = tf.train.Saver()
def __init__(self,
in_shape,
lr=0.001,
embedding_dim=50,
num_projections=20):
'''
classes: List of class names or integers corrisponding to each class being classified
by the network. ie: ['left', 'straight', 'right'] or [0, 1, 2]
'''
# number of elements in the embedding vector
self.embedding_dim = embedding_dim
# number of test embeddings to project in tensorboard
self.num_projections = num_projections
# Define model
tf.reset_default_graph()
self.x = tf.placeholder(tf.float32,
shape=[
None,
] + in_shape,
name="x")
self.y = tf.placeholder(tf.float32,
shape=[
None,
] + in_shape,
name="y")
self._training = tf.placeholder(tf.bool)
self.training = tf.get_variable("training",
dtype=tf.bool,
initializer=True,
trainable=False)
self.set_training = self.training.assign(self._training)
self._beta = tf.placeholder(dtype=tf.float32)
self.beta = tf.get_variable("beta",
dtype=tf.float32,
initializer=0.,
trainable=False)
self.update_beta = self.beta.assign(self._beta)
self._embeddings = tf.placeholder(tf.float32,
shape=[None, self.embedding_dim])
self.embeddings = tf.get_variable(
"embeddings",
dtype=tf.float32,
shape=[self.num_projections, self.embedding_dim],
initializer=tf.zeros_initializer(),
trainable=False)
self.update_embeddings = self.embeddings.assign(self._embeddings)
paddings = tf.constant([[0, 0], [4, 4], [0, 0], [0, 0]])
relu = tf.nn.relu
sigmoid = tf.nn.sigmoid
with tf.name_scope("encoder"):
# Padding invector so reconstruction returns to the correct size.
#x_padded = tf.pad(self.x, paddings, "SYMMETRIC")
# Encoder in num shape stride pad
enc = conv2d(self.x,
32, (5, 5), (2, 2),
"valid",
activation=relu,
kernel_initializer=xavier()) # 59x79
print(f"enc1: {np.shape(enc)}")
enc = conv2d(enc,
62, (5, 5), (2, 2),
"valid",
activation=relu,
kernel_initializer=xavier()) # 28x38
print(f"enc2: {np.shape(enc)}")
enc = conv2d(enc,
128, (4, 4), (2, 2),
"valid",
activation=relu,
kernel_initializer=xavier()) # 13x18
print(f"enc3: {np.shape(enc)}")
enc = conv2d(enc,
256, (4, 4), (2, 2),
"valid",
activation=relu,
kernel_initializer=xavier()) # 5x8
print(f"enc4: {np.shape(enc)}")
enc = flatten(enc)
with tf.name_scope("sampling"):
# VAE sampling
'''
Note: exp(log(log_sigma / 2)) = sigma
'''
self.mu = dense(enc,
self.embedding_dim,
activation=None,
kernel_initializer=xavier(),
name="mu")
self.log_sigma = dense(enc,
self.embedding_dim,
activation=None,
kernel_initializer=xavier(),
name="log_sigma")
eps = tf.random_normal(shape=tf.shape(self.mu),
mean=0.0,
stddev=1.0,
dtype=tf.float32,
name="eps")
self.noisy_sigma = tf.exp(self.log_sigma) * eps
self.z = tf.add(self.mu, self.noisy_sigma, name="z")
#tf.summary.histogram("z", self.z)
#tf.summary.histogram("log_sigma", self.log_sigma)
#tf.summary.histogram("mu", self.mu)
#tf.summary.histogram("eps", eps)
with tf.name_scope("decoder"):
# Decoder in num
dec = dense(self.z, (256 * 5 * 7),
activation=relu,
kernel_initializer=xavier())
print(f"dec4: {np.shape(dec)}")
dec = tf.reshape(dec, (-1, 5, 7, 256))
print(f"dec3: {np.shape(dec)}")
# in num shape stride pad
dec = conv2d_transpose(dec,
128, (4, 4), (2, 2),
"valid",
activation=relu)
print(f"dec2: {np.shape(dec)}")
dec = conv2d_transpose(dec,
64, (4, 4), (2, 2),
"valid",
activation=relu)
print(f"dec1: {np.shape(dec)}")
dec = conv2d_transpose(dec,
32, (6, 6), (2, 2),
"valid",
activation=relu)
print(f"dec0: {np.shape(dec)}")
self.dec = conv2d_transpose(dec,
3, (10, 18), (2, 2),
"valid",
activation=sigmoid,
name="reconstruction")
print(f"out: {np.shape(self.dec)}")
# self.dec = relu(dec7, name="reconstruction")
with tf.name_scope("loss"):
# # VAE Loss
# 1. Reconstruction loss: How far did we get from the actual image?
#y_padded = tf.pad(self.y, paddings, "SYMMETRIC")
self.rec_loss = tf.reduce_sum(tf.square(self.y - self.dec),
axis=[1, 2, 3])
# Cross entropy loss from:
# https://stats.stackexchange.com/questions/332179/
# how-to-weight-kld-loss-vs-reconstruction-
# loss-in-variational-auto-encod
# Must have self.dec activation as sigmoid
# clipped = tf.clip_by_value(self.dec, 1e-8, 1-1e-8)
# self.rec_loss = -tf.reduce_sum(self.y * tf.log(clipped)
# + (1-self.y) * tf.log(1-clipped), axis=[1,2,3])
# 2. KL-Divergence: How far from the distribution 'z' is sampled
# from the desired zero mean unit variance?
self.kl_loss = 0.5 * tf.reduce_sum(
tf.square(self.mu) +
(tf.exp(2 * self.log_sigma)) - 2 * self.log_sigma - 1,
axis=1)
self.loss = tf.reduce_mean(self.rec_loss +
self.kl_loss * self.beta,
name='total_loss')
tf.summary.scalar("total_loss", self.loss)
tf.summary.scalar("kl_loss", tf.reduce_mean(self.kl_loss))
tf.summary.scalar("rec_loss", tf.reduce_mean(self.rec_loss))
tf.summary.scalar("beta", self.beta)
update_tensor_dict(INPUTS, IMAGE_INPUT, self.x)
update_tensor_dict(OUTPUTS, EMBEDDING, self.z)
update_tensor_dict(OUTPUTS, RECONSTRUCTION, self.dec)
optimizer = tf.train.AdamOptimizer(learning_rate=lr)
self.train_step = optimizer.minimize(self.loss, name="train_step")
self.init_vars = tf.global_variables_initializer()
self.saver = tf.train.Saver()
def __init__(self, in_shape, classes, lr=0.001):
'''
classes: List of class names or integers corrisponding to each class being classified
by the network. ie: ['left', 'straight', 'right'] or [0, 1, 2]
'''
# Define classes
self.num_bins = len(classes)
self.classes = np.array(classes, np.float32)
self.class_lookup = [c for c in classes]
# Define model
tf.reset_default_graph()
self.x = tf.placeholder(tf.float32,
shape=[
None,
] + in_shape,
name="input")
self.y_steering = tf.placeholder(tf.int32, shape=(None, ))
self.y_throttle = tf.placeholder(tf.float32, shape=(None, ))
self._training = tf.placeholder(tf.bool)
self.training = tf.get_variable("training",
dtype=tf.bool,
initializer=True,
trainable=False)
self.set_training = self.training.assign(self._training)
relu = tf.nn.relu
sigmoid = tf.nn.sigmoid
with tf.name_scope("donkey"):
# input num conv stride pad
conv = conv2d(self.x,
24, (5, 5), (2, 2),
"same",
activation=relu,
kernel_initializer=xavier(),
name="conv1")
conv = conv2d(conv,
32, (5, 5), (2, 2),
"same",
activation=relu,
kernel_initializer=xavier(),
name="conv2")
conv = conv2d(conv,
64, (5, 5), (2, 2),
"same",
activation=relu,
kernel_initializer=xavier(),
name="conv3")
conv = conv2d(conv,
64, (3, 3), (2, 2),
"same",
activation=relu,
kernel_initializer=xavier(),
name="conv4")
conv = conv2d(conv,
64, (3, 3), (1, 1),
"same",
activation=relu,
kernel_initializer=xavier(),
name="conv5")
conv = flatten(conv)
# in num
conv = dense(conv,
100,
activation=relu,
kernel_initializer=xavier(),
name="fc1")
conv = dropout(conv, rate=0.1, training=self.training)
conv = dense(conv,
50,
activation=relu,
kernel_initializer=xavier(),
name="fc2")
conv = dropout(conv, rate=0.1, training=self.training)
# Steering
self.logits = dense(conv,
self.num_bins,
activation=None,
kernel_initializer=xavier(),
name="logits")
self.steering_probs = tf.nn.softmax(self.logits,
name="steeringi_probs")
self.steering_prediction = tf.reduce_sum(
tf.multiply(self.steering_probs, self.classes),
axis=1,
name="steering_prediction")
# Throttle
self.throttle = dense(conv,
1,
sigmoid,
kernel_initializer=xavier(),
name="throttle")
# keep tensor names for easy freezing/loading later
self._TENSOR_DICT = {
common._IMAGE_INPUT: self.x.name,
common._STEERING_PREDICTION: self.steering_prediction.name,
common._STEERING_PROBS: self.steering_probs.name,
common._THROTTLE_PREDICTION: self.throttle.name
}
with tf.name_scope("loss"):
self.loss_steering = tf.nn.sparse_softmax_cross_entropy_with_logits(
labels=self.y_steering, logits=self.logits)
self.loss_steering = tf.reduce_mean(self.loss_steering)
self.loss_throttle = tf.reduce_mean(
(self.throttle - self.y_throttle)**2)
self.loss = 0.9 * self.loss_steering + 0.001 * self.loss_throttle
tf.summary.scalar("weighted_loss", self.loss)
tf.summary.scalar("steering_loss", self.loss_steering)
tf.summary.scalar("throttle_loss", self.loss_throttle)
optimizer = tf.train.AdamOptimizer(learning_rate=lr)
self.train_step = optimizer.minimize(self.loss)
self.init_vars = tf.global_variables_initializer()
self.saver = tf.train.Saver()
