def periodize(x, k):
input_shape = x.shape
print input_shape
output_shape = [np.shape(x)[0], input_shape[1], input_shape[2] // k, input_shape[3] // k]
reshape_shape = [np.shape(x)[0], input_shape[1],
input_shape[2] // output_shape[2], output_shape[2],
input_shape[3] // output_shape[3], output_shape[3]]
# Split x in two otherwise, tensor has too many dimensions
# and the tile op has no backward pass.
x0 = x[..., 0]
x1 = x[..., 1]
y0 = np.reshape(x0, np.stack(reshape_shape))
y1 = np.reshape(x1, np.stack(reshape_shape))
y0 = np.expand_dims(np.reduce_mean(np.reduce_mean(y0, axis=4), axis=2), axis=-1)
y1 = np.expand_dims(np.reduce_mean(np.reduce_mean(y1, axis=4), axis=2), axis=-1)
out = np.concatenate([y0, y1], axis=-1)
return out
def _make_plot(self,call_counter,feat,predicted,truth):
'''
input features as
B x P x P x F
with F = colours
truth as
B x P x P x T
with T = [mask, true_posx, true_posy, ID_0, ID_1, ID_2, true_width, true_height, n_objects]
all outputs in B x V x 1/F form except
n_active: B x 1
outdict['p_beta'] = tf.reshape(pred[:,:,:,0:1], reshaping)
outdict['p_pos'] = tf.reshape(pred[:,:,:,1:3], reshaping)
outdict['p_ID'] = tf.reshape(pred[:,:,:,3:6], reshaping)
outdict['p_dim'] = tf.reshape(pred[:,:,:,6:8], reshaping)
p_object = pred[:,0,0,8]
outdict['p_ccoords'] = tf.reshape(pred[:,:,:,9:], reshaping)
'''
feat = np.reshape(feat, [feat.shape[0],-1,feat.shape[3]])
predicted = np.reshape(predicted, [predicted.shape[0],-1,predicted.shape[3]])
truth = np.reshape(truth, [truth.shape[0],-1,truth.shape[3]])
p_beta = predicted[:,:,0]
lowest_t_b = 0.7
lowest_t_d = 0.01
acc = [(lowest_t_b, lowest_t_d, []),
(lowest_t_b, 0.05, []),
(lowest_t_b, 0.1, []),
(0.8, lowest_t_d, []),
(0.8, 0.05, []),
(0.8, 0.1, []),
(0.9, lowest_t_b, []),
(0.9, 0.05, []),
(0.9, 0.1, []),
] # make this a tuple of t_d t_b and the batch dim
for be in range(len(p_beta)):
bpred, btruth = select_points(predicted[be,:,9:], predicted[be], truth[be], predicted[be,:,0], lowest_t_b, lowest_t_d)
for i in range(len(acc))/3: #needs to be increasing
t_b = acc[i][0] ### needs to be changed.. won't work like that
for j in range(len(acc))/3:
t_d = acc[i][0]
tpred, ttruth = select_points(bpred[:,9:], bpred, btruth, bpred[:,0], t_d, t_b)
p_beta = tpred[:,0]
p_ccoords = tpred[:,9:]
p_pos = tpred[:,1:3]
p_ID = tpred[:,3:6]
t_mask = ttruth[:,0]
t_ccoords = ttruth[:,9:]
t_pos = ttruth[:,1:3]
t_ID = ttruth[:,3:6]
acc.append( np.reduce_mean(np.argmax(p_ID, axis=-1) * np.argmax(t_ID, axis=-1)) )
def __init__(self, name, input_size, output_size, size_layer, learning_rate):
with tf.variable_scope(name):
self.X = tf.placeholder(tf.float32, [None, 80, 80, 4])
self.Y = tf.placeholder(tf.float32, (None, output_size))
w_conv1 = tf.Variable(tf.truncated_normal([8, 8, 4, 32], stddev = 0.1))
conv1 = tf.nn.relu(conv_layer(self.X, w_conv1, stride = 4))
pooling1 = pooling(conv1)
w_conv2 = tf.Variable(tf.truncated_normal([4, 4, 32, 64], stddev = 0.1))
conv2 = tf.nn.relu(conv_layer(pooling1, w_conv2, stride = 2))
w_conv3 = tf.Variable(tf.truncated_normal([3, 3, 64, 64], stddev = 0.1))
conv3 = tf.nn.relu(conv_layer(conv2, w_conv3))
pulling_size = int(conv3.shape[1]) * int(conv3.shape[2]) * int(conv3.shape[3])
conv3 = tf.reshape(conv3, [-1, pulling_size])
w_fc1 = tf.Variable(tf.truncated_normal([pulling_size, 256], stddev = 0.1))
b_fc1 = tf.Variable(tf.truncated_normal([256], stddev = 0.01))
action_layer = tf.Variable(tf.random_normal([size_layer // 2, output_size]))
validation_layer = tf.Variable(tf.random_normal([size_layer // 2, 1]))
fc_1 = tf.nn.relu(tf.matmul(conv3, w_fc1))
layer_merge = tf.Variable(tf.random_normal([output_size, size_layer//2]))
layer_merge_out = tf.Variable(tf.random_normal([size_layer//2, 1]))
tensor_action, tensor_validation = tf.split(fc_1,2,1)
feed_action = tf.matmul(tensor_action, action_layer)
feed_validation = tf.matmul(tensor_validation, validation_layer)
feed_critic = feed_validation + tf.subtract(feed_action,tf.reduce_mean(feed_action,axis=1,keep_dims=True))
feed_critic = feed_critic + self.Y
feed_critic = tf.nn.relu(tf.matmul(feed_critic, layer_merge))
self.logits = tf.matmul(feed_critic, layer_merge_out)
self.cost = np.reduce_mean(tf.square(self.REWARD - self.logits))
self.optimizer = tf.train.AdamOptimizer(learning_rate).minimize(self.cost)
def get_error_metric(predicted_labels, true_labels, metric="MAE"):
errors = np.subtract(true_labels, predicted_labels)
if metric=="MAE":
metric = np.abs(errors)
elif metric=="MSE":
metric = np.square(errors)
return np.reduce_mean(metric)
def __init__(self, name, input_size, output_size, size_layer, learning_rate):
with tf.variable_scope(name):
self.X = tf.placeholder(tf.float32, [None, 80, 80, 4])
self.Y = tf.placeholder(tf.float32, (None, output_size))
w_conv1 = tf.Variable(tf.truncated_normal([8, 8, 4, 32], stddev = 0.1))
conv1 = tf.nn.relu(conv_layer(self.X, w_conv1, stride = 4))
pooling1 = pooling(conv1)
w_conv2 = tf.Variable(tf.truncated_normal([4, 4, 32, 64], stddev = 0.1))
conv2 = tf.nn.relu(conv_layer(pooling1, w_conv2, stride = 2))
w_conv3 = tf.Variable(tf.truncated_normal([3, 3, 64, 64], stddev = 0.1))
conv3 = tf.nn.relu(conv_layer(conv2, w_conv3))
pulling_size = int(conv3.shape[1]) * int(conv3.shape[2]) * int(conv3.shape[3])
conv3 = tf.reshape(tf.reshape(conv3, [-1, pulling_size]), [batch_size, 8, 512])
layer_merge = tf.Variable(tf.random_normal([output_size, size_layer//2]))
layer_merge_out = tf.Variable(tf.random_normal([size_layer//2, 1]))
cell = tf.nn.rnn_cell.LSTMCell(512, state_is_tuple = False)
self.hidden_layer = tf.placeholder(tf.float32, (None, 2 * 512))
self.rnn,self.last_state = tf.nn.dynamic_rnn(inputs=conv3,cell=cell,
dtype=tf.float32,
initial_state=self.hidden_layer)
w = tf.Variable(tf.random_normal([512, output_size]))
feed_critic = tf.nn.relu(tf.matmul(self.rnn[:,-1], w)) + self.Y
feed_critic = tf.nn.relu(tf.matmul(feed_critic, layer_merge))
self.logits = tf.matmul(feed_critic, layer_merge_out)
self.cost = np.reduce_mean(tf.square(self.REWARD - self.logits))
self.optimizer = tf.train.AdamOptimizer(learning_rate).minimize(self.cost)
def fNumpy(self, t, x, u, Du):
# u time derivative
Ut = -np.mean(np.where(x < 0, np.sin(x), x), axis=-1)
# u value
xSum = np.einsum("ij,j->i", x, np.arange(1., self.d + 1.))
cosU = np.cos(xSum)
UVal = -(self.T - t) * Ut + cosU
# U X derivarive (sum)
DUVal = (self.T - t) * np.mean(
np.where(x < 0, np.cos(x), np.ones(np.shape(x))),
axis=-1) - self.d * (self.d + 1.) / 2. * np.sin(xSum)
# sum of diag of Hessian
D2UVal = -cosU * self.d * (self.d + 1) * (2 * self.d + 1) / 6. - (
self.T - t) * np.reduce_mean(
np.where(x < 0, np.sin(x), np.zeros(np.shape(x))), axis=-1)
ret = -Ut - 0.5 * self.Sig * self.Sig * D2UVal - self.rescal * (
UVal * DUVal / self.d + UVal * UVal) + self.rescal * (
np.pow(u, 2.) + np.multiply(u, np.reduce_mean(Du, axis=-1)))
return ret.squeeze()
def __init__(self, name, input_size, output_size, size_layer, learning_rate):
with tf.variable_scope(name):
self.X = tf.placeholder(tf.float32, (None, input_size))
self.Y = tf.placeholder(tf.float32, (None, output_size))
self.REWARD = tf.placeholder(tf.float32, (None, 1))
layer_critic = tf.Variable(tf.random_normal([input_size, size_layer]))
output_critic = tf.Variable(tf.random_normal([size_layer, output_size]))
layer_merge = tf.Variable(tf.random_normal([output_size, size_layer//2]))
layer_merge_out = tf.Variable(tf.random_normal([size_layer//2, 1]))
feed_critic = tf.nn.relu(tf.matmul(self.X, layer_actor))
feed_critic = tf.nn.relu(tf.matmul(feed_critic, output_actor)) + self.Y
feed_critic = tf.nn.relu(tf.matmul(feed_critic, layer_merge))
self.logits = tf.matmul(feed_critic, layer_merge_out)
self.cost = np.reduce_mean(tf.square(self.REWARD - self.logits))
self.optimizer = tf.train.AdamOptimizer(learning_rate).minimize(self.cost)
def __init__(self, name, input_size, output_size, size_layer, learning_rate):
with tf.variable_scope(name):
self.X = tf.placeholder(tf.float32, (None, input_size))
self.Y = tf.placeholder(tf.float32, (None, output_size))
self.REWARD = tf.placeholder(tf.float32, (None, 1))
layer_merge = tf.Variable(tf.random_normal([output_size, size_layer//2]))
layer_merge_out = tf.Variable(tf.random_normal([size_layer//2, 1]))
cell = tf.nn.rnn_cell.LSTMCell(512, state_is_tuple = False)
self.hidden_layer = tf.placeholder(tf.float32, (None, 2 * 512))
self.rnn,self.last_state = tf.nn.dynamic_rnn(inputs=self.X,cell=cell,
dtype=tf.float32,
initial_state=self.hidden_layer)
w = tf.Variable(tf.random_normal([512, output_size]))
feed_critic = tf.nn.relu(tf.matmul(self.rnn[:,-1], w)) + self.Y
feed_critic = tf.nn.relu(tf.matmul(feed_critic, layer_merge))
self.logits = tf.matmul(feed_critic, layer_merge_out)
self.cost = np.reduce_mean(tf.square(self.REWARD - self.logits))
self.optimizer = tf.train.AdamOptimizer(learning_rate).minimize(self.cost)
def __init__(self, name, input_size, output_size, size_layer, learning_rate):
with tf.variable_scope(name):
self.X = tf.placeholder(tf.float32, (None, input_size))
self.Y = tf.placeholder(tf.float32, (None, output_size))
self.REWARD = tf.placeholder(tf.float32, (None, 1))
layer_critic = tf.Variable(tf.random_normal([input_size, size_layer]))
action_layer = tf.Variable(tf.random_normal([size_layer // 2, self.OUTPUT_SIZE]))
validation_layer = tf.Variable(tf.random_normal([size_layer // 2, 1]))
layer_merge = tf.Variable(tf.random_normal([output_size, size_layer//2]))
layer_merge_out = tf.Variable(tf.random_normal([size_layer//2, 1]))
feed_critic = tf.nn.relu(tf.matmul(self.X, layer_actor))
self.tensor_action, self.tensor_validation = tf.split(feed_critic,2,1)
self.feed_action = tf.matmul(self.tensor_action, action_layer)
self.feed_validation = tf.matmul(self.tensor_validation, validation_layer)
feed_critic = self.feed_validation + tf.subtract(self.feed_action,tf.reduce_mean(self.feed_action,axis=1,keep_dims=True))
feed_critic = tf.nn.relu(tf.matmul(feed_critic, output_actor)) + self.Y
feed_critic = tf.nn.relu(tf.matmul(feed_critic, layer_merge))
self.logits = tf.matmul(feed_critic, layer_merge_out)
self.cost = np.reduce_mean(tf.square(self.REWARD - self.logits))
self.optimizer = tf.train.AdamOptimizer(learning_rate).minimize(self.cost)
def _build_model(self, x, y):
"""Build the model function for federated learning.
Includes loss calculation and backprop.
"""
w0 = self.w0.read_value()
b0 = self.b0.read_value()
w1 = self.w1.read_value()
b1 = self.b1.read_value()
params = (w0, b0, w1, b1)
layer0 = np.matmul(x, w0) + b0
layer1 = np.scipy.special.expit(layer0)
layer2 = np.matmul(layer1, w1) + b1
predictions = layer2
loss = np.reduce_mean(
np.losses.sparse_softmax_cross_entropy(logits=predictions,
labels=y))
grads = np.gradients(ys=loss, xs=params)
return predictions, loss, grads
def error(self):
mistakes = tf.not_equal(tf.argmax(self.targets, 1), tf.argmax(self.predictions, 1))
mistakes = tf.reduce_mean(tf.cast(mistakes, tf.float32))
return mistakes
def cost(self):
cross_entropy = self.targets * tf.log(self.predictions)
cross_entropy = tf.reduce_mean(cross_entropy, reduction_indices=1)
return cross_entropy
def compute_mean_embedding(inputs):
not_pad = tf.math.count_nonzero(inputs, axis=-1)
n_words = tf.math.count_nonzero(not_pad, axis=-1, keepdims=True)
sqrt_n_words = tf.math.sqrt(tf.cast(n_words, tf.float32))
return tf.reduce_mean(inputs, axis=1) * sqrt_n_words
def PolicyEstimator(self, encoded_state):
with tf.variable_scope("policy_estimator"):
self.decay_epsilon = tf.train.polynomial_decay(
0.2,
tf.train.get_global_step(),
self.max_steps,
0.1,
power=1.0)
if self.action_type == 'continuous':
if self.a_size == 1 or self.use_multivariate_normal:
self.advantage = tf.placeholder(shape=[None],
dtype=tf.float32,
name="advantage")
self.old_action_probs = tf.placeholder(shape=[None, 1],
dtype=tf.float32)
elif self.a_size > 1:
self.advantage = tf.placeholder(shape=[None, 1],
dtype=tf.float32,
name="advantage")
self.old_action_probs = tf.placeholder(
shape=[None, self.a_size], dtype=tf.float32)
else:
self.advantage = tf.placeholder(shape=[None],
dtype=tf.float32,
name="advantage")
self.old_action_probs = tf.placeholder(
shape=[None, self.a_size], dtype=tf.float32)
if self.action_type == 'continuous':
mu = tf.layers.dense(encoded_state, self.a_size, None,
tf.contrib.layers.xavier_initializer())
sigma = tf.layers.dense(encoded_state, self.a_size, None,
tf.contrib.layers.xavier_initializer())
sigma = tf.nn.softplus(sigma) + 1e-10
if self.a_size == 1 or not self.use_multivariate_normal:
norm_dist = tf.contrib.distributions.Normal(mu, sigma)
else:
norm_dist = tf.contrib.distributions.MultivariateNormalDiag(
mu, sigma)
#action_tf_var can be backpropagated
self.action_tf_var = tf.squeeze(norm_dist.sample(1), axis=0)
self.action_tf_var = tf.clip_by_value(self.action_tf_var,
self.action_low,
self.action_high)
self.action_prob = norm_dist.prob(self.action_tf_var)
self.entropy = norm_dist.entropy()
self.mean_entropy = tf.reduce_mean(self.entropy)
if self.policy_type == 'policy_gradient':
self.policy_loss = -tf.log(
norm_dist.prob(self.action) + 1e-10) * self.advantage
self.policy_loss = np.reduce_mean(self.policy_loss)
elif self.policy_type == 'ppo':
#Clipped Surrogate Objective
ratio = self.action_prob / (self.old_action_probs + 1e-10)
a = ratio * self.advantage
b = tf.clip_by_value(ratio, 1.0 - self.decay_epsilon, 1.0 +
self.decay_epsilon) * self.advantage
self.policy_loss = -tf.reduce_mean(tf.minimum(a, b))
else:
self.action_probs = tf.layers.dense(
encoded_state,
self.a_size,
activation=tf.nn.softmax,
kernel_initializer=tf.contrib.layers.
variance_scaling_initializer(0.01))
self.picked_action_prob = tf.reduce_sum(
self.action_probs * tf.one_hot(self.action, self.a_size),
axis=1)
self.picked_old_action_prob = tf.reduce_sum(
self.old_action_probs *
tf.one_hot(self.action, self.a_size),
axis=1)
self.entropy = -tf.reduce_sum(
self.action_probs * tf.log(self.action_probs + 1e-10),
axis=1)
self.mean_entropy = tf.reduce_mean(self.entropy)
if self.policy_type == 'policy_gradient':
self.policy_loss = tf.reduce_mean(
-tf.log(self.picked_action_prob + 1e-10) *
self.advantage)
elif self.policy_type == 'ppo':
#Clipped Surrogate Objective
ratio = self.picked_action_prob / (
self.picked_old_action_prob + 1e-10)
a = ratio * self.advantage
b = tf.clip_by_value(ratio, 1.0 - self.decay_epsilon, 1.0 +
self.decay_epsilon) * self.advantage
self.policy_loss = -tf.reduce_mean(tf.minimum(a, b))
# Summaries for Tensorboard
self.policy_summaries = tf.summary.merge([
tf.summary.scalar("policy_loss", self.policy_loss),
tf.summary.scalar("entropy", self.mean_entropy)
# tf.summary.histografm("entropy", self.entropy)
])
