def minibatch_update(self,x,y,lr,regularization):
n_sample = x.shape[0]
info = x
hidden_cache = []
for i in xrange(self.n_hidden + 1):
if i == self.n_hidden:
probs = softmax(info.dot(self.W[i]) + self.b[i])
else:
info = sigmoid(info.dot(self.W[i]) + self.b[i])
hidden_cache.append(info)
loss = neg_log_likelihood(probs,y)
probs[np.arange(n_sample),y] -= 1.0
errors = probs
for i in range(self.n_hidden,-1,-1):
if i >= 1:
hidden_out = hidden_cache[i - 1]
grad_hidden_out = errors.dot(self.W[i].T)
self.W[i] -= (lr * (hidden_out.T).dot(errors) + regularization * self.W[i])
self.b[i] -= lr * np.sum(errors,axis = 0)
errors = hidden_out * (1 - hidden_out) * grad_hidden_out
else:
hidden_out = x
self.W[i] -= (lr * (hidden_out.T).dot(errors) + regularization * self.W[i])
self.b[i] -= lr * np.sum(errors,axis = 0)
return loss
def forwardPass(self, model, tree):
# Traverse the tree and populate word vectors and
# predictions, bottom up
if tree.is_leaf():
#word_index = model.word_lookup[tree.word]
word_index = self.getWordIndex(
model, tree.word)
tree.word_vector = model.L[:, word_index]
else:
left_child = tree.subtrees[0]
right_child = tree.subtrees[1]
self.forwardPass(model, left_child)
self.forwardPass(model, right_child)
tree.word_vector = self.composition(
model, left_child.word_vector, right_child.word_vector)
# hit elemenwise tanh
tree.word_vector = np.tanh(tree.word_vector)
# make softmax prediction
tree.prediction = utils.softmax(
model.Ws.dot(np.append(tree.word_vector, [1])))
# update (increment) loss
label_vector = self.getLabelVector(model, tree.label)
#self.loss += -1*sum(label_vector*np.log(tree.prediction))
self.loss += -1*label_vector.dot(np.log(tree.prediction))
def fprop(self, X):
X = np.array([np.array([float(x) for x in j]) for j in X])
X = X.transpose()
self._ha = np.dot(self._w1, X) + np.repeat(self._b1, len(X[0]), axis=1) # valeur des synapses entre x et hidden
self._hs = utils.relu(self._ha) # valeur hidden
self._oa = np.dot(self._w2, self._hs) + np.repeat(self._b2, len(X[0]), axis=1) # valeur entre hidden et sortie
self._os = utils.softmax(self._oa) # valeur de sortie
def forwardProp(self,node, correct=[], guess=[]):
cost = total = 0.0
# this is exactly the same setup as forwardProp in rnn.py
cost_l = total_l = cost_r = total_r = 0
node.fprop = True;
if node.isLeaf:
node.hActs1 = self.L[:, node.word];
else:
cost_l , total_l = self.forwardProp(node.left, correct, guess);
cost_r , total_r = self.forwardProp(node.right, correct, guess);
z1 = self.W1[:, :self.wvecDim].dot(node.left.hActs1) + \
self.W1[:, self.wvecDim:].dot(node.right.hActs1) + \
self.b1;
node.hActs1 = z1 * (z1> 0);
z2 = self.W2.dot(node.hActs1) + self.b2;
node.hActs2 = z2 * (z2 > 0)
node.probs = softmax(self.Ws.dot(node.hActs2) + self.bs);
cost = cost_l + cost_r -np.log(node.probs[node.label]);
total = total_l + total_r;
correct.append(node.label);
guess.append(np.argmax(node.probs))
return cost, total + 1
def _predictNode(self, node):
"""
Return the softmax sentiment prediction for the given word vector
WARNING: The node output(after activation fct) has to be already
computed (by the evaluateSample fct)
"""
z = np.dot(self.Ws, node.output) + self.bs
return utils.softmax(z)
def step_forward(self, x):
self.input_act[:] = x
pre_act = self.input_act.dot(self.Wih) + self.hidden_act.dot(
self.Whh) + self.hb
self.hidden_act[:] = self.hidden_non_linearity(pre_act)
print self.hidden_act
self.output_act[:] = softmax(np.dot(self.hidden_act, self.Who) +
self.ob)
return self.output_act.copy()
def process_intermediate_output(self, itr, X, y, net):
beg = itr * self.batch_size
end = beg + self.batch_size
weighted = utils.softmax(net.blobs["weighted_input"].data)
ip2 = utils.softmax(net.blobs["ip2"].data)
confidence = net.blobs["confidence"].data
y_predicted = weighted.argmax(axis=1)
self.uncertainty[beg:end, 0] = 1-confidence[xrange(y_predicted.shape[0]), y_predicted]
self.uncertainty[beg:end, 1] = utils.entropy(confidence)
self.uncertainty[beg:end, 2] = utils.entropy(ip2)
self.uncertainty[beg:end, 3] = utils.second_max(ip2)
self.uncertainty[beg:end, 4] = utils.entropy(weighted)
self.uncertainty[beg:end, 5] = utils.second_max(weighted)
self.correct[beg:end] = np.equal(y, y_predicted)
def get_dilation_model_camvid(input_shape, apply_softmax, input_tensor, classes):
if input_tensor is None:
model_in = Input(shape=input_shape)
else:
if not K.is_keras_tensor(input_tensor):
model_in = Input(tensor=input_tensor, shape=input_shape)
else:
model_in = input_tensor
h = Convolution2D(64, 3, 3, activation='relu', name='conv1_1')(model_in)
h = Convolution2D(64, 3, 3, activation='relu', name='conv1_2')(h)
h = MaxPooling2D(pool_size=(2, 2), strides=(2, 2), name='pool1')(h)
h = Convolution2D(128, 3, 3, activation='relu', name='conv2_1')(h)
h = Convolution2D(128, 3, 3, activation='relu', name='conv2_2')(h)
h = MaxPooling2D(pool_size=(2, 2), strides=(2, 2), name='pool2')(h)
h = Convolution2D(256, 3, 3, activation='relu', name='conv3_1')(h)
h = Convolution2D(256, 3, 3, activation='relu', name='conv3_2')(h)
h = Convolution2D(256, 3, 3, activation='relu', name='conv3_3')(h)
h = MaxPooling2D(pool_size=(2, 2), strides=(2, 2), name='pool3')(h)
h = Convolution2D(512, 3, 3, activation='relu', name='conv4_1')(h)
h = Convolution2D(512, 3, 3, activation='relu', name='conv4_2')(h)
h = Convolution2D(512, 3, 3, activation='relu', name='conv4_3')(h)
h = AtrousConvolution2D(512, 3, 3, atrous_rate=(2, 2), activation='relu', name='conv5_1')(h)
h = AtrousConvolution2D(512, 3, 3, atrous_rate=(2, 2), activation='relu', name='conv5_2')(h)
h = AtrousConvolution2D(512, 3, 3, atrous_rate=(2, 2), activation='relu', name='conv5_3')(h)
h = AtrousConvolution2D(4096, 7, 7, atrous_rate=(4, 4), activation='relu', name='fc6')(h)
h = Dropout(0.5, name='drop6')(h)
h = Convolution2D(4096, 1, 1, activation='relu', name='fc7')(h)
h = Dropout(0.5, name='drop7')(h)
h = Convolution2D(classes, 1, 1, name='final')(h)
h = ZeroPadding2D(padding=(1, 1))(h)
h = Convolution2D(classes, 3, 3, activation='relu', name='ctx_conv1_1')(h)
h = ZeroPadding2D(padding=(1, 1))(h)
h = Convolution2D(classes, 3, 3, activation='relu', name='ctx_conv1_2')(h)
h = ZeroPadding2D(padding=(2, 2))(h)
h = AtrousConvolution2D(classes, 3, 3, atrous_rate=(2, 2), activation='relu', name='ctx_conv2_1')(h)
h = ZeroPadding2D(padding=(4, 4))(h)
h = AtrousConvolution2D(classes, 3, 3, atrous_rate=(4, 4), activation='relu', name='ctx_conv3_1')(h)
h = ZeroPadding2D(padding=(8, 8))(h)
h = AtrousConvolution2D(classes, 3, 3, atrous_rate=(8, 8), activation='relu', name='ctx_conv4_1')(h)
h = ZeroPadding2D(padding=(16, 16))(h)
h = AtrousConvolution2D(classes, 3, 3, atrous_rate=(16, 16), activation='relu', name='ctx_conv5_1')(h)
h = ZeroPadding2D(padding=(1, 1))(h)
h = Convolution2D(classes, 3, 3, activation='relu', name='ctx_fc1')(h)
logits = Convolution2D(classes, 1, 1, name='ctx_final')(h)
if apply_softmax:
model_out = softmax(logits)
else:
model_out = logits
model = Model(input=model_in, output=model_out, name='dilation_camvid')
return model
def run(self):
p = softmax(self.Q, self.beta)
actions = (0, 1)
action = np.random.choice(actions, p=p)
reward = self.bandit.reward(action)
self.Q[action] += self.alpha * (reward - self.Q[action])
self.log["reward"].append(reward)
self.log["action"].append(action)
self.log["Q(0)"].append(self.Q[0])
self.log["Q(1)"].append(self.Q[1])
def construct_DNN(n_input, n_output, n_hid_layers=2, archi=128,
lr=1e-3, batchsize=40, dropout_rate=0.2, moment=0.95):
"""
Initialize and construct the deep neural netweok with dropout
update the DNN using momentum and minibatch
archi: number of neurons of each hidden layer
"""
# decide dropout or not, no dropout: stop_dropout > 1.05
x = T.fmatrix()
y_hat = T.fmatrix()
stop_dropout = T.scalar()
# initialize parameters
Ws, bs, cache_Ws, cache_bs = initialize_NNet(n_input, n_output,
archi, n_hid_layers)
# ############ construct the neural network ###############
Zs = []
As = []
# input layer
Zs.append(T.dot(x, Ws[0]) + bs[0].dimshuffle('x', 0))
As.append(maxout(Zs[0], stop_dropout, archi, dropout_rate) / stop_dropout)
# hidden layers
for i in range(n_hid_layers):
Zs.append(T.dot(As[i], Ws[i + 1]) + bs[i + 1].dimshuffle('x', 0))
act_out = maxout(Zs[i + 1], stop_dropout, archi, dropout_rate)
As.append(act_out / stop_dropout)
# output layer
z_out = T.dot(As[n_hid_layers], Ws[n_hid_layers + 1])
Zs.append(z_out + bs[n_hid_layers + 1].dimshuffle('x', 0))
y = softmax(Zs[-1] / stop_dropout)
# ############ construct the neural network ###############
forward = th.function([x, stop_dropout], y)
parameters = Ws + bs
moment_cache = cache_Ws + cache_bs
# objective is the binary crossentropy
Cost = ((-T.log((y * y_hat).sum(axis=1))).sum()) / batchsize
# calculate gradients
grads = T.grad(Cost, parameters, disconnected_inputs='ignore')
# update parameters using momentum
update_func = update(parameters, grads, moment_cache, lr, moment)
gradient_update = th.function(inputs=[x, y_hat, stop_dropout],
updates=update_func, outputs=Cost)
return gradient_update, forward
def run(self):
actions = self.bandit.actions
cues = self.bandit.cues
cue = self.bandit.get_cue()
p = softmax(self.Q[cue, :], self.beta)
action = np.random.choice(actions, p=p)
reward = self.bandit.reward(action)
self.Q[cue, action] += self.alpha * (reward - self.Q[cue, action])
self.log["reward"].append(reward)
self.log["action"].append(action)
self.log["cue"].append(cue)
for cue, action in product(cues, actions):
key = "Q({:d},{:d})".format(cue, action)
self.log[key].append(self.Q[cue, action])
def neg_log_likelihood(self, alphabeta):
df = self.df
alpha, beta = alphabeta
df = self.df[self.df['cue'].isin(self.cues)]
actions, rewards = df['action'].values, df['reward'].values
cues = df['cue'].values
prob_log = 0
Q = dict([[cue, np.zeros(self.n_actions)] for cue in self.cues])
k = 1
for action, reward, cue in zip(actions, rewards, cues):
if self.model == 'sample_average':
Q[cue][action] += alpha * (reward - Q[cue][action]) / k
k += 1
else:
Q[cue][action] += alpha * (reward - Q[cue][action])
prob_log += np.log(softmax(Q[cue], beta)[action])
return -prob_log
def neg_log_likelihood(self, alphabetas):
df = self.df
alphas = alphabetas[0::2]
betas = alphabetas[1::2]
df = self.df[self.df['cue'].isin(self.cues)]
actions, rewards = df['action'].values, df['reward'].values
cues = df['cue'].values
prob_log = 0
Q = dict([[cue, np.zeros(self.n_actions)] for cue in self.cues])
for action, reward, cue in zip(actions, rewards, cues):
alpha = alphas[self.cues.index(cue)]
beta = betas[self.cues.index(cue)]
Q[cue][action] += alpha * (reward - Q[cue][action])
prob_log += np.log(softmax(Q[cue], beta)[action])
return -prob_log
def __init__(self, rng, input, n_in, n_out,
W=None, b=None, name='tmp'):
"""
Typical hidden layer of a MLP: units are fully-connected and have
sigmoidal activation function. Weight matrix W is of shape (n_in,n_out)
and the bias vector b is of shape (n_out,).
NOTE : The nonlinearity used here is tanh
Hidden unit activation is given by: tanh(dot(input,W) + b)
:type rng: numpy.random.RandomState
:param rng: a random number generator used to initialize weights
:type input: theano.tensor.dmatrix
:param input: a symbolic tensor of shape (n_examples, n_in)
:type n_in: int
:param n_in: dimensionality of input
:type n_out: int
:param n_out: number of hidden units
"""
self.input = input
if W is None and b is None:
W_values = numpy.asarray(rng.uniform(
low=-numpy.sqrt(6. / (n_in + n_out)),
high=numpy.sqrt(6. / (n_in + n_out)),
size=(n_in, n_out)), dtype=theano.config.floatX)
self.W = theano.shared(value=W_values, name=name+'_W')
b_values = numpy.zeros((n_out,), dtype=theano.config.floatX)
self.b = theano.shared(value=b_values, name=name+'_b')
else:
self.W = W
self.b = b
self.output = softmax(TT.dot(input, self.W) + self.b)
# parameters of the model
self.params = [self.W, self.b]
def _forward_propagation(self, seq):
"""
Parameters
----------
seq : list
Variable length sequence of elements in the vocabulary.
Attributes
----------
self.h : np.array
Each row is for a hidden representation. The first row
is an all-0 initial state. The others correspond to
the inputs in seq.
self.y : np.array
The vector of predictions.
"""
self.h = np.zeros((len(seq)+1, self.hidden_dim))
for t in range(1, len(seq)+1):
word_rep = self.get_word_rep(seq[t-1])
self.h[t] = np.tanh(word_rep.dot(self.W_xh) + self.h[t-1].dot(self.W_hh))
self.y = softmax(self.h[-1].dot(self.W_hy) + self.b)
def forwardProp(self,node, correct = [], guess = []):
cost = total = 0.0
cost_l = total_l = cost_r = total_r = 0
node.fprop = True;
if node.isLeaf:
node.hActs1 = self.L[:, node.word];
else:
cost_l , total_l = self.forwardProp(node.left, correct, guess);
cost_r , total_r = self.forwardProp(node.right, correct, guess);
h = np.hstack([node.left.hActs1, node.right.hActs1])
tmp = np.tensordot(self.V, h, axes = 1)
z = np.dot(tmp,h);
z += np.dot(self.W, h) + self.b;
node.hActs1 = np.tanh(z) ;
node.probs = softmax(self.Ws.dot(node.hActs1) + self.bs);
cost = cost_l + cost_r -np.log(node.probs[node.label]);
total = total_l + total_r;
correct.append(node.label);
guess.append(np.argmax(node.probs))
return cost,total + 1
def train(self):
global_step = 0
training_rewards = []
for episode in range(1, self.num_episodes + 1):
state = self.env.reset()
episode_rewards = 0
episode_ave_max_q = 0
for time_step in range(self.episode_length):
action = self.actor.predict(
asset_inputs=np.array([state.asset_features]),
portfolio_inputs=np.array([state.portfolio_allocation]))[0]
#print("ACTION before:", softmax(action))
noise = self.actor_noise()
#print("NOISE:", noise)
action += noise
action = softmax(action) # take softmax here
#print("ACTION after:", action)
trans_state, reward, terminal, info = self.env.step(action)
episode_rewards += reward
# self.rpb.store_w_terminal(old_state=state,
# action=action,
# reward=reward,
# terminal=terminal,
# new_state=trans_state)
self.rpb.add(obs_t=state.features,
action=action,
reward=reward,
obs_tp1=trans_state.features,
done=terminal)
# if self.rpb.ready(self.batch_size):
if len(self.rpb._storage) >= self.batch_size:
# batch_states, batch_actions, batch_rewards, batch_terminal, batch_trans_state \
# = self.rpb.sample_batch(batch_size=self.batch_size)
experiences = self.rpb.sample(batch_size=self.batch_size,
beta=0.5)
batch_states, batch_actions, batch_rewards, batch_trans_state, batch_terminal, \
weights, rank_e_id = experiences
batch_asset_features, batch_portfolio = convert_features(
features=batch_states,
asset_features_shape=self.actor.asset_features_shape,
portfolio_features_shape=[self.actor.a_dim])
print("FEATS:", batch_asset_features)
batch_trans_asset_features, batch_trans_portfolio = \
convert_features(features=batch_trans_state,
asset_features_shape=self.actor.asset_features_shape,
portfolio_features_shape=[self.actor.a_dim])
weights = np.expand_dims(weights, axis=1)
target_actions = self.actor.predict_target(
asset_inputs=batch_trans_asset_features,
portfolio_inputs=batch_trans_portfolio
) # [batch_size, action_dim]
target_q = self.critic.predict_target(
asset_inputs=batch_trans_asset_features,
portfolio_inputs=
batch_trans_portfolio, # [batch_size, 1]
action=target_actions)
batch_y = []
for ind in range(self.batch_size):
if batch_terminal[ind]:
batch_y.append([batch_rewards[ind]])
else:
batch_y.append(batch_rewards[ind] +
self.gamma * target_q[ind])
batch_y = np.array(batch_y) # [batch_size, 1]
loss, out, _ = self.critic.train(
asset_inputs=batch_asset_features,
portfolio_inputs=batch_portfolio,
action=batch_actions,
predicted_q_value=batch_y,
weights=weights)
deltas = np.squeeze(np.abs(out - batch_y))
print(deltas)
deltas[deltas == 0] = 0.001
self.rpb.update_priorities(idxes=rank_e_id,
priorities=deltas)
policy_actions = self.actor.predict(
asset_inputs=batch_asset_features,
portfolio_inputs=batch_portfolio
) # [batch_size, num_assets]
policy_actions = softmax(policy_actions,
axis=-1) # take softmax here
action_grads = self.critic.action_gradients(
asset_inputs=batch_asset_features,
portfolio_inputs=batch_portfolio,
actions=policy_actions)[0]
self.actor.train(asset_inputs=batch_asset_features,
portfolio_inputs=batch_portfolio,
a_gradient=np.array(action_grads))
self.critic.update_target_network()
self.actor.update_target_network()
summary = self.sess.run(self.batch_summaries,
feed_dict={self.qfunc_loss: loss})
self.writer.add_summary(summary, global_step)
summary = self.sess.run(self.individual_summaries,
feed_dict={
self.actions: action,
self.prices: state.price,
self.individual_reward: reward,
self.individual_pnl: info['pnl'],
self.individual_tc: info['tc']
})
self.writer.add_summary(summary, global_step)
global_step += 1
state = trans_state
if terminal:
print("Episode number:", episode)
summary = self.sess.run(
self.episode_summaries,
feed_dict={self.episode_reward: episode_rewards})
self.writer.add_summary(summary, episode)
print("Reward:", episode_rewards)
break
elif time_step == (self.episode_length - 1):
print("Episode number:", episode)
summary = self.sess.run(
self.episode_summaries,
feed_dict={self.episode_reward: episode_rewards})
self.writer.add_summary(summary, episode)
print("Reward:", episode_rewards)
break
# summary = self.sess.run(self.summary_ops, feed_dict={self.episode_reward: episode_rewards})
# self.writer.add_summary(summary, global_step)
# epsiode_rewards.append(np.sum(rewards))
# if epsilon > 0.1:
# epsilon -= 2.0 / self.num_episodes
if (episode % 25) == 0:
self.infer(train=False, episode=episode)
def fprop(self, X):
X = np.array([[float(x)] for x in X])
self._ha = np.dot(self._w1, X) + self._b1 # valeur des synapses entre x et hidden
self._hs = utils.relu(self._ha) # valeur hidden
self._oa = np.dot(self._w2, self._hs) + self._b2 # valeur entre hidden et sortie
self._os = utils.softmax(self._oa) # valeur de sortie
def choose_action(self, context):
p = softmax(self.Q[context], self.beta)
actions = range(self.n)
action = np.random.choice(actions, p=p)
return action
def routing(input, b_IJ, num_outputs=6, num_dims=8):
''' The routing algorithm.
Args:
input: shape, num_caps_l meaning the number of capsule in the layer l.
num_outputs: the number of output capsules.
num_dims: the number of dimensions for output capsule.
Returns:
A Tensor of shape [batch_size, num_caps_l_plus_1, length(v_j)=16, 1]
representing the vector output `v_j` in the layer l+1
Notes:
u_i represents the vector output of capsule i in the layer l, and
v_j the vector output of capsule j in the layer l+1.
'''
# W: [1, num_caps_i, num_caps_j * len_v_j, len_u_j, 1]
input_shape = get_shape(input)
W = tf.get_variable('Weight',
shape=[1, input_shape[1], num_dims * num_outputs] +
input_shape[-2:],
dtype=tf.float32,
initializer=tf.random_normal_initializer(stddev=0.01))
biases = tf.get_variable('bias', shape=(1, 1, num_outputs, num_dims, 1))
# Eq.2, calc u_hat
# Since tf.matmul is a time-consuming op,
# A better solution is using element-wise multiply, reduce_sum and reshape
# ops instead. Matmul [a, b] x [b, c] is equal to a series ops as
# element-wise multiply [a*c, b] * [a*c, b], reduce_sum at axis=1 and
# reshape to [a, c]
input = tf.tile(input, [1, 1, num_dims * num_outputs, 1, 1])
# assert input.get_shape()
u_hat = reduce_sum(W * input, axis=3, keepdims=True)
u_hat = tf.reshape(u_hat,
shape=[-1, input_shape[1], num_outputs, num_dims, 1])
# assert u_hat.get_shape()
# In forward, u_hat_stopped = u_hat; in backward, no gradient passed back from u_hat_stopped to u_hat
u_hat_stopped = tf.stop_gradient(u_hat, name='stop_gradient')
n = 3 #3
# line 3,for r iterations do
for r_iter in range(n):
with tf.variable_scope('iter_' + str(r_iter)):
# line 4:
c_IJ = softmax(b_IJ, axis=2)
# At last iteration, use `u_hat` in order to receive gradients from the following graph
if r_iter == n - 1:
# line 5:
# weighting u_hat with c_IJ, element-wise in the last two dims
s_J = tf.multiply(c_IJ, u_hat)
# then sum in the second dim
s_J = reduce_sum(s_J, axis=1, keepdims=True) + biases
# assert s_J.get_shape()
# line 6:
# squash using Eq.1,
v_J = squash(s_J)
# assert v_J.get_shape()
elif r_iter < n - 1: # Inner iterations, do not apply backpropagation
s_J = tf.multiply(c_IJ, u_hat_stopped)
s_J = reduce_sum(s_J, axis=1, keepdims=True) + biases
v_J = squash(s_J)
# line 7:
# reshape & tile v_j from
# then matmul
# batch_size dim, resulting
v_J_tiled = tf.tile(v_J, [1, input_shape[1], 1, 1, 1])
u_produce_v = reduce_sum(u_hat_stopped * v_J_tiled,
axis=3,
keepdims=True)
# assert u_produce_v
# b_IJ += tf.reduce_sum(u_produce_v, axis=0, keep_dims=True)
b_IJ += u_produce_v
return (v_J)
remainder = outputs_chunk[l:] # new remainder
if avg_method == "avg-probs-ent": # entropy-weighted averaging
outputs_chunk = outputs_chunk[:l]
h = utils.entropy(outputs_chunk)
outputs_chunk *= np.exp(-h)[:, None]
outputs_chunk = outputs_chunk.reshape(l // num_test_tfs, num_test_tfs, outputs_chunk.shape[1]).sum(1)
z = np.exp(-h).reshape(l // num_test_tfs, num_test_tfs).sum(1)
outputs_chunk /= z[:, None]
else:
outputs_chunk = outputs_chunk[:l].reshape(l // num_test_tfs, num_test_tfs, outputs_chunk.shape[1]).mean(1)
outputs.append(outputs_chunk)
assert (remainder is None) or remainder.size == 0 # make sure we haven't left any predictions behind
outputs = np.vstack(outputs)
if avg_method == "avg-logits":
print "Passing averaged logits through the softmax"
outputs = utils.softmax(outputs)
elif avg_method == "avg-probs-geom":
print "Renormalizing geometrically averaged probabilities"
outputs = utils.softmax(outputs)
print "Saving"
np.save(target_path, outputs)
print " saved to %s" % target_path
def get_scores(model, loader, challenge=False, include_discarded=False):
model.eval()
predictions = []
labels = []
ids = []
with torch.set_grad_enabled(False):
for batch in tqdm(loader, 'Evaluating...', len(loader)):
x = batch['past_features' if args.task ==
'anticipation' else 'action_features']
if type(x) == list:
x = [xx.to(device) for xx in x]
else:
x = x.to(device)
y = batch['label'].numpy()
ids.append(batch['id'])
preds = model(x).cpu().numpy()[:, -args.S_ant:, :]
predictions.append(preds)
labels.append(y)
action_scores = np.concatenate(predictions)
labels = np.concatenate(labels)
ids = np.concatenate(ids)
actions = pd.read_csv(join(args.path_to_data, 'actions.csv'),
index_col='id')
vi = get_marginal_indexes(actions, 'verb')
ni = get_marginal_indexes(actions, 'noun')
action_probs = softmax(action_scores.reshape(-1, action_scores.shape[-1]))
verb_scores = marginalize(action_probs,
vi).reshape(action_scores.shape[0],
action_scores.shape[1], -1)
noun_scores = marginalize(action_probs,
ni).reshape(action_scores.shape[0],
action_scores.shape[1], -1)
if include_discarded:
dlab = np.array(loader.dataset.discarded_labels)
dislab = np.array(loader.dataset.discarded_ids)
ids = np.concatenate([ids, dislab])
num_disc = len(dlab)
labels = np.concatenate([labels, dlab])
verb_scores = np.concatenate(
(verb_scores, np.zeros((num_disc, *verb_scores.shape[1:]))))
noun_scores = np.concatenate(
(noun_scores, np.zeros((num_disc, *noun_scores.shape[1:]))))
action_scores = np.concatenate(
(action_scores, np.zeros((num_disc, *action_scores.shape[1:]))))
if labels.max() > 0 and not challenge:
return verb_scores, noun_scores, action_scores, labels[:,
0], labels[:,
1], labels[:,
2], ids
else:
return verb_scores, noun_scores, action_scores, ids
def build_arch(self):
with tf.variable_scope('Conv1_layer_init'):
conv_init1 = tf.contrib.layers.conv2d(self.X, num_outputs=16,
kernel_size=3, stride=2,
padding='VALID')
conv_init2 = tf.contrib.layers.conv2d(conv_init1, num_outputs=8,
kernel_size=4, stride=2,
padding='VALID')
conv_init3 = tf.contrib.layers.conv2d(conv_init2, num_outputs=1,
kernel_size=7, stride=2,
padding='VALID')
# conv_init3=tf.reshape(conv_init3,shape=(cfg.batch_size,28,28,1))
with tf.variable_scope('Conv1_layer'):
# Conv1, [batch_size, 20, 20, 256]
conv1 = tf.contrib.layers.conv2d(conv_init3, num_outputs=256,
kernel_size=9, stride=1,
padding='VALID')
assert conv1.get_shape() == [cfg.batch_size, 20, 20, 256]
# Primary Capsules layer, return [batch_size, 1152, 8, 1]
with tf.variable_scope('PrimaryCaps_layer'):
primaryCaps = CapsLayer(num_outputs=32, vec_len=8, with_routing=False, layer_type='CONV')
caps1 = primaryCaps(conv1, kernel_size=9, stride=2)
assert caps1.get_shape() == [cfg.batch_size, 1152, 8, 1]
# DigitCaps layer, return [batch_size, 10, 16, 1]
with tf.variable_scope('DigitCaps_layer'):
digitCaps = CapsLayer(num_outputs=12, vec_len=12, with_routing=True, layer_type='FC')
self.caps2 = digitCaps(caps1)
# Decoder structure in Fig. 2
# 1. Do masking, how:
with tf.variable_scope('Masking'):
# a). calc ||v_c||, then do softmax(||v_c||)
# [batch_size, 10, 16, 1] => [batch_size, 10, 1, 1]
self.v_length = tf.sqrt(reduce_sum(tf.square(self.caps2),
axis=2, keepdims=True) + epsilon)
self.softmax_v = softmax(self.v_length, axis=1)
assert self.softmax_v.get_shape() == [cfg.batch_size, 12, 1, 1]
# b). pick out the index of max softmax val of the 10 caps
# [batch_size, 10, 1, 1] => [batch_size] (index)
self.argmax_idx = tf.to_int32(tf.argmax(self.softmax_v, axis=1))
assert self.argmax_idx.get_shape() == [cfg.batch_size, 1, 1]
self.argmax_idx = tf.reshape(self.argmax_idx, shape=(cfg.batch_size, ))
# Method 1.
if not cfg.mask_with_y:
# c). indexing
# It's not easy to understand the indexing process with argmax_idx
# as we are 3-dim animal
masked_v = []
for batch_size in range(cfg.batch_size):
v = self.caps2[batch_size][self.argmax_idx[batch_size], :]
masked_v.append(tf.reshape(v, shape=(1, 1, 12, 1)))
self.masked_v = tf.concat(masked_v, axis=0)
assert self.masked_v.get_shape() == [cfg.batch_size, 1, 12, 1]
# Method 2. masking with true label, default mode
else:
# self.masked_v = tf.matmul(tf.squeeze(self.caps2), tf.reshape(self.Y, (-1, 10, 1)), transpose_a=True)
self.masked_v = tf.multiply(tf.squeeze(self.caps2), tf.reshape(self.Y, (-1, 12, 1)))
self.v_length = tf.sqrt(reduce_sum(tf.square(self.caps2), axis=2, keepdims=True) + epsilon)
def forward(self, x):
self.input_act[:] = x
self.hidden_act[:] = sigmoid(np.dot(self.input_act, self.Wih) + self.hb)
self.output_act[:] = softmax(np.dot(self.hidden_act, self.Who) +
self.ob)
def forward(self):
self.cache = softmax(self.x.data)
return Tensor(self.cache, diff=self.diff)
def test_softmax(arg, expected):
result = utils.softmax(arg).round(8)
expected = expected.round(8)
assert np.array_equal(result, expected)
def onlineUpdate(self,curr_data,data_object,complete=False,softmax_error=True,self_correct=True):
def getErrorMetric(epf,p=None,i=None,d=None,soft=None,last_epf=None,all_err=None):
'''
epf = error_per_frame = cumulative_error/current_frame (potentially a windowed epf to allow for quicker changes late in a task)
p = number of known tasks, also the value of the proportional gain
all_err = cumulative error for integral control
soft = error to softthreshold within
last_epf = previous value of epf for derivative control
Em = P*epf + D*(epf-last_epf) + I(all_err)
metric = max(1,Em), or if p,last_epf,and all_err are left as default None then epf is returned
'''
P = p #proportional gain
D = d #derivative gain
I = i #integral gain
Em = 0
control_terms = 0
#print 'epf = ', epf
#Proportional error addition
if p != None:
if soft != None:
Em += P*(epf-soft-1)
else:
Em += P*(epf-1)
#print 'prop Em = ', Em
else:
#print 'p = None'
control_terms += 1
#Derivative error addition
if last_epf != None:
Em += D*(epf-last_epf)
#print 'deriv Em = ', Em
else:
#print 'last epf = none'
control_terms += 1
#Integral error addition
if all_err != None:
Em += I*all_err
#print 'integral Em = ', Em
else:
#print 'all err = none'
control_terms += 1
#if no P,I,or D, return the error per frame alone
if control_terms == 3:
Em = epf
metric = max(1.0,Em)
return float(metric)
## use the new piece of data to get percent complete for each task model, also update the mixedRayleigh associated with each task
for t_id in self.task_check_order:
tid_task = self.known_tasks[t_id]
pct_complete,new_mixed = tid_task.getCurrentLabel(curr_data,data_object,self.curr_frame_count,mixed=self.mixed[t_id],kNN_number=self.kNN_number[t_id],complete_threshold=self.complete_threshold)
'''#determine how likely each task is to be the current task by computing dynamic time warping cost between the curr_labels and the expected path at this point - this was too slow
cost_threshold = 2
if self.curr_frame_count > self.min_frames_for_probability:
self.task_online_costs[t_id] = max(1.0,task_tools.getTaskMetric(tid_task.path,tid_task.times,tid_task.curr_labels,tid_task.curr_mixed_position,tid_task.frames_since_state_change,constraint=self.online_dtw_constraint)-cost_threshold)
if self.task_online_costs[t_id]>1.0:
self.task_online_costs[t_id] = 0.5*self.task_online_costs[t_id]**2
'''
#add current online error (or add new element if a new task has just been added)
expected_pct = 100*self.curr_frame_count/float(np.sum(self.known_tasks[t_id].times))
c = 0.0625
if len(self.cumulative_online_errors) < self.known_task_count:
self.cumulative_online_errors[t_id] = c*np.abs(expected_pct-pct_complete)**2
self.prev_error_per_frame[t_id] = 0
self.error_per_frame[t_id] = self.cumulative_online_errors[t_id]/float(self.curr_frame_count)
else:
new_error = c*np.abs(expected_pct-pct_complete)**2
self.cumulative_online_errors[t_id] += new_error
self.prev_error_per_frame[t_id] = deepcopy(self.error_per_frame[t_id])
base_epf = self.cumulative_online_errors[t_id]/float(self.curr_frame_count)
error_window = self.error_window
self.error_per_frame[t_id] = (base_epf*error_window+new_error)/(error_window+1) #essentially, (curr_epf*error_window + new_error)/(frame_window+1), this allows for more change later in a task
self.task_pct_complete[t_id] = pct_complete
self.mixed[t_id] = new_mixed
#print 'pct_complete: ', self.task_pct_complete[t_id]
## if enough frames have passed and a task exists, determine the probability that a particular task is being completed relative to other tasks
if self.curr_frame_count > self.min_frames_for_probability and self.known_task_count > 0:
#total_costs = np.sum(self.task_online_costs.values())
total_costs = self.known_task_count +1
pct_complete = 0
if total_costs > self.known_task_count:
self.error_metric = {}
temp = {}
#adjust errors using PID to extremify errors further
for t_id in self.task_check_order:
P = self.known_task_count
I = 0.1 #0.02
D = 50 #100
max_soft = 10
#time_at_4 = 30.0 #these two commented lines were used to determine the soft_thresh exponential constant (-0.0305)
#soft_thresh = max_soft*np.exp((np.log(4/max_soft)/time_at_4)*self.curr_frame_count)
soft_thresh = max_soft*np.exp(-0.0305*self.curr_frame_count)
last_err = self.prev_error_per_frame[t_id]
cumulative_error = self.cumulative_online_errors[t_id]
self.error_metric[t_id] = getErrorMetric(epf=self.error_per_frame[t_id],p=P,i=I,d=D,soft=soft_thresh,last_epf=last_err, all_err=cumulative_error)
#print self.error_metric[t_id]
temp[t_id] = 1/self.error_metric[t_id]
if softmax_error == False:
total_costs = np.sum(temp.values())
for t_id in self.task_check_order:
self.task_online_probability[t_id] = temp[t_id]/(1.*total_costs)
else:
metrics = [self.error_metric[x] for x in range(self.known_task_count)]
softmax_alpha = -1/40.0 #this alpha has bee chosen to that reasonable errors in the 50
soft_pcts = utils.softmax(np.array(metrics)-np.amin(metrics)/3.0,alpha=softmax_alpha,rescale_=False)
for t_id in self.task_check_order:
self.task_online_probability[t_id] = soft_pcts[t_id]
if self_correct and np.all(np.array(self.error_metric.values())>200):
#print 'in self-correct loop'
confusion_pct_addition = self.confusion_pct_addition #amount to add to pct_complete estimate if all errors are bad
pct_complete = self.curr_pct_complete_estimate + confusion_pct_addition
else:
#print 'no self-correction'
for t_id in self.task_check_order:
pct_complete += self.task_online_probability[t_id]*self.task_pct_complete[t_id]
else:
pct_complete = self.task_pct_complete[self.task_history[-1]]
else:
pct_complete = self.task_pct_complete[self.task_history[-1]]
self.curr_frame_count += 1 #increment the frame count
self.curr_pct_complete_estimate = pct_complete
return pct_complete
def answer(self, data):
return softmax(self._process_layers(self._weights, data, False))
def biuld_net(self):
# gragh = tf.Graph()
# with gragh.as_default():
###########
### set top conv
top_con = CNNs(self.x, 128, [9, 1], 2, "SAME", self.is_train)
self.primary_cap = layers_vector(
top_con,
32,
4, [9, 1],
2,
self.is_train,
shapes=[-1, self.next_length * 8, 16, 1])
# [-1,88*16,8,1]
#with tf.variable_scope("capsules_layers"):
fc_function = tf.reshape(self.primary_cap,
shape=(-1, self.primary_cap.shape[1].value, 1,
self.primary_cap.shape[-2].value, 1))
#with tf.variable_scope("routing"):
#[-1,88*16,1,8,1]
blu = tf.constant(np.zeros([
self.batch_size, self.primary_cap.shape[1].value, self.num_label,
1, 1
]),
dtype=tf.float32)
caps = routing(fc_function,
blu,
num_outputs=self.num_label,
num_dims=32)
#### [120,37,8,1]
top_conv_1 = CNNs(self.x, 128, [7, 1], 2, "SAME", self.is_train)
self.primary_cap_1 = layers_vector(
top_conv_1,
32,
4, [7, 1],
2,
self.is_train,
shapes=[-1, self.next_length * 16, 8, 1])
fc_function_1 = tf.reshape(
self.primary_cap_1,
shape=(-1, self.primary_cap_1.shape[1].value, 1,
self.primary_cap_1.shape[-2].value, 1))
blu_1 = tf.constant(np.zeros([
self.batch_size, self.primary_cap_1.shape[1].value, self.num_label,
1, 1
]),
dtype=tf.float32)
with tf.variable_scope("routint_1"):
caps_1 = routing(fc_function_1, blu_1, self.num_label, 16)
top_con_2 = CNNs(self.x, 128, [5, 1], 2, 'SAME', self.is_train)
self.primary_cap_2 = layers_vector(
top_con_2,
32,
4, [5, 1],
2,
self.is_train,
shapes=[-1, self.next_length * 32, 4, 1])
fc_function_2 = tf.reshape(
self.primary_cap_2,
shape=(-1, self.primary_cap_2.shape[1].value, 1,
self.primary_cap_2.shape[-2].value, 1))
blu_2 = tf.constant(np.zeros([
self.batch_size, self.primary_cap_2.shape[1].value, self.num_label,
1, 1
]),
dtype=tf.float32)
with tf.variable_scope("routing_2"):
caps_2 = routing(fc_function_2, blu_2, self.num_label, 8)
a = 3.0
b = 1.0
c = 1.0
# a = 3.0
# b = 1.0
caps = tf.concat([a * caps, b * caps_1, c * caps_2], axis=3)
# This is the best performance in our experiments.
self.caps = tf.squeeze(caps, axis=1)
v_length = tf.sqrt(
reduce_sum(tf.square(self.caps), axis=2, keepdims=True) +
eposilion)
softmax_v = softmax(v_length, axis=1)
#########[batch_size,num_label,1,1]
argmax_idx = tf.to_int32(tf.argmax(softmax_v, axis=1))
self.argmax_idx = tf.reshape(argmax_idx, shape=(self.batch_size, ))
###
self.masked_v = tf.multiply(
tf.squeeze(self.caps), tf.reshape(self.y, (-1, self.num_label, 1)))
self.v_length = tf.sqrt(
reduce_sum(tf.square(self.caps), axis=2, keepdims=True) +
eposilion)
########
# decoder
vector_j = tf.reshape(self.masked_v, shape=(self.batch_size, -1))
fc1 = tf.contrib.layers.fully_connected(vector_j, num_outputs=256)
fc1 = tf.contrib.layers.fully_connected(fc1, num_outputs=512)
self.decode = tf.contrib.layers.fully_connected(
fc1, num_outputs=self.length, activation_fn=tf.sigmoid)
def runSmc(args):
smcData, settings, do_metrics = args
print '\nInitializing SMC\n'
# precomputation
(particles, param, log_weights, cache, cache_tmp) = bdtsmc.init_smc(smcData, settings)
# Run smc
print '\nRunning SMC'
(particles, ess_itr, log_weights_itr, log_pd, particle_stats_itr_d, particles_itr_d, log_pd_islands) = \
bdtsmc.run_smc(particles, smcData, settings, param, log_weights, cache)
# Printing some diagnostics
print
print 'Estimate of log marginal probability i.e. log p(Y|X) = %s ' % log_pd
print 'Estimate of log marginal probability for different islands = %s' % log_pd_islands
print 'logsumexp(log_pd_islands) - np.max(log_pd_islands) = %s\n' % \
(logsumexp(log_pd_islands) - np.max(log_pd_islands))
if settings.debug == 1:
print 'log_weights_itr = \n%s' % log_weights_itr
# check if log_weights are computed correctly
for i_, p in enumerate(particles):
log_w = log_weights_itr[-1, i_] + np.log(settings.n_particles) - np.log(settings.n_islands)
logprior_p = p.compute_logprior()
loglik_p = p.compute_loglik()
logprob_p = p.compute_logprob()
if (np.abs(settings.ess_threshold) < 1e-15) and (settings.proposal == 'prior'):
# for the criterion above, only loglik should contribute to the weight update
try:
check_if_zero(log_w - loglik_p)
except AssertionError:
print 'Incorrect weight computation: log_w (smc) = %s, loglik_p = %s' % (log_w, loglik_p)
raise AssertionError
try:
check_if_zero(logprob_p - loglik_p - logprior_p)
except AssertionError:
print 'Incorrect weight computation'
print 'check if 0: %s, logprior_p = %s, loglik_p = %s' % (logprob_p - loglik_p - logprior_p, logprior_p, loglik_p)
raise AssertionError
# Evaluate
print 'Results on training data (log predictive prob is bogus)'
# log_predictive on training data is bogus ... you are computing something like \int_{\theta} p(data|\theta) p(\theta|data)
if settings.weight_islands == 1:
# each island's prediction is weighted by its marginal likelihood estimate which is equivalent to micro-averaging globally
weights_prediction = softmax(log_weights_itr[-1, :])
assert('islandv1' in settings.tag)
else:
# correction for macro-averaging predictions across islands
weights_prediction = np.ones(settings.n_particles) / settings.n_islands
n_particles_tmp = settings.n_particles / settings.n_islands
for i_ in range(settings.n_islands):
pid_min, pid_max = i_ * n_particles_tmp, (i_ + 1) * n_particles_tmp - 1
pid_range_tmp = range(pid_min, pid_max+1)
weights_prediction[pid_range_tmp] *= softmax(log_weights_itr[-1, pid_range_tmp])
(pred_prob_overall_train, metrics_train) = \
evaluate_predictions_smc(particles, smcData, smcData['x_train'], smcData['y_train'], settings, param, weights_prediction, do_metrics)
print '\nResults on test data'
(pred_prob_overall_test, metrics_test) = \
evaluate_predictions_smc(particles, smcData, smcData['x_test'], smcData['y_test'], settings, param, weights_prediction, do_metrics)
#return pred_prob_overall_test, particles, param, weights_prediction
return pred_prob_overall_test,
def get_dilation_model_cityscapes(input_shape, apply_softmax, input_tensor, classes):
if input_tensor is None:
model_in = Input(shape=input_shape)
else:
if not K.is_keras_tensor(input_tensor):
model_in = Input(tensor=input_tensor, shape=input_shape)
else:
model_in = input_tensor
h = Convolution2D(64, 3, 3, activation='relu', name='conv1_1')(model_in)
h = Convolution2D(64, 3, 3, activation='relu', name='conv1_2')(h)
h = MaxPooling2D(pool_size=(2, 2), strides=(2, 2), name='pool1')(h)
h = Convolution2D(128, 3, 3, activation='relu', name='conv2_1')(h)
h = Convolution2D(128, 3, 3, activation='relu', name='conv2_2')(h)
h = MaxPooling2D(pool_size=(2, 2), strides=(2, 2), name='pool2')(h)
h = Convolution2D(256, 3, 3, activation='relu', name='conv3_1')(h)
h = Convolution2D(256, 3, 3, activation='relu', name='conv3_2')(h)
h = Convolution2D(256, 3, 3, activation='relu', name='conv3_3')(h)
h = MaxPooling2D(pool_size=(2, 2), strides=(2, 2), name='pool3')(h)
h = Convolution2D(512, 3, 3, activation='relu', name='conv4_1')(h)
h = Convolution2D(512, 3, 3, activation='relu', name='conv4_2')(h)
h = Convolution2D(512, 3, 3, activation='relu', name='conv4_3')(h)
h = AtrousConvolution2D(512, 3, 3, atrous_rate=(2, 2), activation='relu', name='conv5_1')(h)
h = AtrousConvolution2D(512, 3, 3, atrous_rate=(2, 2), activation='relu', name='conv5_2')(h)
h = AtrousConvolution2D(512, 3, 3, atrous_rate=(2, 2), activation='relu', name='conv5_3')(h)
h = AtrousConvolution2D(4096, 7, 7, atrous_rate=(4, 4), activation='relu', name='fc6')(h)
h = Dropout(0.5, name='drop6')(h)
h = Convolution2D(4096, 1, 1, activation='relu', name='fc7')(h)
h = Dropout(0.5, name='drop7')(h)
h = Convolution2D(classes, 1, 1, name='final')(h)
h = ZeroPadding2D(padding=(1, 1))(h)
h = Convolution2D(classes, 3, 3, activation='relu', name='ctx_conv1_1')(h)
h = ZeroPadding2D(padding=(1, 1))(h)
h = Convolution2D(classes, 3, 3, activation='relu', name='ctx_conv1_2')(h)
h = ZeroPadding2D(padding=(2, 2))(h)
h = AtrousConvolution2D(classes, 3, 3, atrous_rate=(2, 2), activation='relu', name='ctx_conv2_1')(h)
h = ZeroPadding2D(padding=(4, 4))(h)
h = AtrousConvolution2D(classes, 3, 3, atrous_rate=(4, 4), activation='relu', name='ctx_conv3_1')(h)
h = ZeroPadding2D(padding=(8, 8))(h)
h = AtrousConvolution2D(classes, 3, 3, atrous_rate=(8, 8), activation='relu', name='ctx_conv4_1')(h)
h = ZeroPadding2D(padding=(16, 16))(h)
h = AtrousConvolution2D(classes, 3, 3, atrous_rate=(16, 16), activation='relu', name='ctx_conv5_1')(h)
h = ZeroPadding2D(padding=(32, 32))(h)
h = AtrousConvolution2D(classes, 3, 3, atrous_rate=(32, 32), activation='relu', name='ctx_conv6_1')(h)
h = ZeroPadding2D(padding=(64, 64))(h)
h = AtrousConvolution2D(classes, 3, 3, atrous_rate=(64, 64), activation='relu', name='ctx_conv7_1')(h)
h = ZeroPadding2D(padding=(1, 1))(h)
h = Convolution2D(classes, 3, 3, activation='relu', name='ctx_fc1')(h)
h = Convolution2D(classes, 1, 1, name='ctx_final')(h)
# the following two layers pretend to be a Deconvolution with grouping layer.
# never managed to implement it in Keras
# since it's just a gaussian upsampling trainable=False is recommended
h = UpSampling2D(size=(8, 8))(h)
logits = Convolution2D(classes, 16, 16, border_mode='same', bias=False, trainable=False, name='ctx_upsample')(h)
if apply_softmax:
model_out = softmax(logits)
else:
model_out = logits
model = Model(input=model_in, output=model_out, name='dilation_cityscapes')
return model
def routing(input, b_IJ):
''' The routing algorithm.
Args:
input: A Tensor with [batch_size, num_caps_l=1152, 1, length(u_i)=8, 1]
shape, num_caps_l meaning the number of capsule in the layer l.
Returns:
A Tensor of shape [batch_size, num_caps_l_plus_1, length(v_j)=16, 1]
representing the vector output `v_j` in the layer l+1
Notes:
u_i represents the vector output of capsule i in the layer l, and
v_j the vector output of capsule j in the layer l+1.
'''
# W: [1, num_caps_i, num_caps_j * len_v_j, len_u_j, 1]
W = tf.get_variable(
'Weight',
shape=(1, 1152, 144, 8, 1),
dtype=tf.float32,
initializer=tf.random_normal_initializer(stddev=cfg.stddev))
biases = tf.get_variable('bias', shape=(1, 1, 12, 12, 1))
# Eq.2, calc u_hat
# Since tf.matmul is a time-consuming op,
# A better solution is using element-wise multiply, reduce_sum and reshape
# ops instead. Matmul [a, b] x [b, c] is equal to a series ops as
# element-wise multiply [a*c, b] * [a*c, b], reduce_sum at axis=1 and
# reshape to [a, c]
input = tf.tile(input, [1, 1, 144, 1, 1])
assert input.get_shape() == [cfg.batch_size, 1152, 144, 8, 1]
u_hat = tf.reduce_sum(W * input, axis=3, keep_dims=True)
u_hat = tf.reshape(u_hat, shape=[-1, 1152, 12, 12, 1])
assert u_hat.get_shape() == [cfg.batch_size, 1152, 12, 12, 1]
# In forward, u_hat_stopped = u_hat; in backward, no gradient passed back from u_hat_stopped to u_hat
u_hat_stopped = tf.stop_gradient(u_hat, name='stop_gradient')
# line 3,for r iterations do
for r_iter in range(cfg.iter_routing):
with tf.variable_scope('iter_' + str(r_iter)):
# line 4:
# => [batch_size, 1152, 10, 1, 1]
c_IJ = softmax(b_IJ, axis=2)
# At last iteration, use `u_hat` in order to receive gradients from the following graph
if r_iter == cfg.iter_routing - 1:
# line 5:
# weighting u_hat with c_IJ, element-wise in the last two dims
# => [batch_size, 1152, 10, 16, 1]
s_J = tf.multiply(c_IJ, u_hat)
# then sum in the second dim, resulting in [batch_size, 1, 10, 16, 1]
s_J = reduce_sum(s_J, axis=1, keepdims=True) + biases
assert s_J.get_shape() == [cfg.batch_size, 1, 12, 12, 1]
# line 6:
# squash using Eq.1,
v_J = squash(s_J)
assert v_J.get_shape() == [cfg.batch_size, 1, 12, 12, 1]
elif r_iter < cfg.iter_routing - 1: # Inner iterations, do not apply backpropagation
s_J = tf.multiply(c_IJ, u_hat_stopped)
s_J = reduce_sum(s_J, axis=1, keepdims=True) + biases
v_J = squash(s_J)
# line 7:
# reshape & tile v_j from [batch_size ,1, 10, 16, 1] to [batch_size, 1152, 10, 16, 1]
# then matmul in the last tow dim: [16, 1].T x [16, 1] => [1, 1], reduce mean in the
# batch_size dim, resulting in [1, 1152, 10, 1, 1]
v_J_tiled = tf.tile(v_J, [1, 1152, 1, 1, 1])
u_produce_v = reduce_sum(u_hat_stopped * v_J_tiled,
axis=3,
keepdims=True)
assert u_produce_v.get_shape() == [
cfg.batch_size, 1152, 12, 1, 1
]
# b_IJ += tf.reduce_sum(u_produce_v, axis=0, keep_dims=True)
b_IJ += u_produce_v
return (v_J)
def forward(self, X):
Z = relu(X.dot(self.W1) + self.b1)
Ypred = softmax(Z.dot(self.W2) + self.b2)
return Ypred, Z
def forward(self, x, t):
self.t = t
self.y = softmax(x)
self.loss = cross_entropy_error(self.y, self.t)
return self.loss
def forward(self):
''' output produced isn't the loss as one would expect, '''
''' but it is the result of the softmax '''
batch_size = self.input_.shape[0]
for b in xrange(batch_size):
self.output[b, :] = softmax(self.input_[b])
def build_arch(self):
with tf.variable_scope('Conv1_layer'):
# Conv1, [batch_size, 20, 20, 256]
conv1 = tf.contrib.layers.conv2d(self.X, num_outputs=256,
kernel_size=9, stride=1,
padding='VALID')
assert conv1.get_shape() == [cfg.batch_size, 20, 20, 256]
# Primary Capsules layer, return [batch_size, 1152, 8, 1]
with tf.variable_scope('PrimaryCaps_layer'):
primaryCaps = CapsLayer(num_outputs=32, vec_len=8, with_routing=False, layer_type='CONV')
caps1 = primaryCaps(conv1, kernel_size=9, stride=2)
assert caps1.get_shape() == [cfg.batch_size, 1152, 8, 1]
# DigitCaps layer, return [batch_size, 10, 16, 1]
with tf.variable_scope('DigitCaps_layer'):
digitCaps = CapsLayer(num_outputs=10, vec_len=16, with_routing=True, layer_type='FC')
self.caps2 = digitCaps(caps1)
# Decoder structure in Fig. 2
# 1. Do masking, how:
with tf.variable_scope('Masking'):
# a). calc ||v_c||, then do softmax(||v_c||)
# [batch_size, 10, 16, 1] => [batch_size, 10, 1, 1]
self.v_length = tf.sqrt(reduce_sum(tf.square(self.caps2),
axis=2, keepdims=True) + epsilon)
self.softmax_v = softmax(self.v_length, axis=1)
assert self.softmax_v.get_shape() == [cfg.batch_size, 10, 1, 1]
# b). pick out the index of max softmax val of the 10 caps
# [batch_size, 10, 1, 1] => [batch_size] (index)
self.argmax_idx = tf.to_int32(tf.argmax(self.softmax_v, axis=1))
assert self.argmax_idx.get_shape() == [cfg.batch_size, 1, 1]
self.argmax_idx = tf.reshape(self.argmax_idx, shape=(cfg.batch_size, ))
# Method 1.
if not cfg.mask_with_y:
# c). indexing
# It's not easy to understand the indexing process with argmax_idx
# as we are 3-dim animal
masked_v = []
for batch_size in range(cfg.batch_size):
v = self.caps2[batch_size][self.argmax_idx[batch_size], :]
masked_v.append(tf.reshape(v, shape=(1, 1, 16, 1)))
self.masked_v = tf.concat(masked_v, axis=0)
assert self.masked_v.get_shape() == [cfg.batch_size, 1, 16, 1]
# Method 2. masking with true label, default mode
else:
# self.masked_v = tf.matmul(tf.squeeze(self.caps2), tf.reshape(self.Y, (-1, 10, 1)), transpose_a=True)
self.masked_v = tf.multiply(tf.squeeze(self.caps2), tf.reshape(self.Y, (-1, 10, 1)))
self.v_length = tf.sqrt(reduce_sum(tf.square(self.caps2), axis=2, keepdims=True) + epsilon)
# 2. Reconstructe the MNIST images with 3 FC layers
# [batch_size, 1, 16, 1] => [batch_size, 16] => [batch_size, 512]
with tf.variable_scope('Decoder'):
vector_j = tf.reshape(self.masked_v, shape=(cfg.batch_size, -1))
fc1 = tf.contrib.layers.fully_connected(vector_j, num_outputs=512)
assert fc1.get_shape() == [cfg.batch_size, 512]
fc2 = tf.contrib.layers.fully_connected(fc1, num_outputs=1024)
assert fc2.get_shape() == [cfg.batch_size, 1024]
self.decoded = tf.contrib.layers.fully_connected(fc2, num_outputs=784, activation_fn=tf.sigmoid)
def posteriorOverStates(self, Observation, CurrentTime, Policies,
PosteriorLastState, PastAction,
PriorPrecision, newB=None, PreUpd=False,
calc_Qs=False):
"""
Decision model for Active Inference. Takes as input the model
parameters in MDP, as well as an observation and the prior over the
current state.
The input PreUpd decides whether the output should be the final
value of precision (False) or the vector with all the updates for this
trial (True).
Parameters
----------
Observation: int
CurrentTime: int
Policies: np.array, shape=(nV, nT)
PosteriorLastState: np.array, shape=(nS)
PastAction: int
PriorPrecision: float
newB: np.array, shape=(nActions, nS, nS)
PreUpd: bool
Whether to return the Precision or the precision updates (of which
the last equals the Precision).
calc_Qs: bool
Whether to return the valuation of all the action sequences in
Policies.
Returns
-------
NOTE: Outputs are returned in the order listed here. All outputs with a
(#) next to their name will be "either/or".
x: np.array, shape=(nS)
Vector with the posteriors over states.
P: np.array, shape=(nActions)
Posteriors over actions
W(1): int
Precision. Note that if PreUpd is True, precisionUpdates is
returned instead of W.
precisionUpdates(1): np.array, shape=(self.N)
Precision updates for the current trial. The last one in the array
equals W above. Note that if PreUpd is True, precisionUpdates is
returned instead of W.
Q: np.array, shape=(nV)
Valuation of the different action sequences given in Policies. This
is only returned if calc_Qs is True.
"""
# print PriorPrecision, self.gamma
V = Policies
cNp = np.shape(V)[0]
w = np.array(range(cNp))
x = PosteriorLastState
W = PriorPrecision
a = PastAction
t = CurrentTime
T = self.T
u = np.zeros(cNp)
P = np.zeros(self.Nu)
# V can be given as the action sequences starting at trial t, instead
# of the full array (starting at trial 0). This is done to avoid doing
# extra calculations on ''repeated'' entries on V, when observations
# are being processed as independent (as oppossed to as part of a
# game). In this case, pad the left-hand side of the array with zeros,
# to make it consistent with everything else. These values will not be
# used for any calculations:
if V.shape[1] != T:
V = np.hstack(
[-np.ones((V.shape[0], T - V.shape[1]), dtype=int), V])
# A 'false' set of transition matrices can be fed to the Agent,
# depending on the newB input above. No input means that the ones from
# the actinf class are used:
if newB is None:
B = self.B
else:
if np.shape(newB) != np.shape(self.B):
raise ValueError('The provided transition matrices' +
' do not have the correct size')
B = newB
if t == 0:
v = self.lnA[Observation, :] + self.lnD
else:
v = self.lnA[Observation, :] + sp.log(sp.dot(B[a], x))
x = utils.softmax(v)
Q = np.zeros(cNp)
for k in range(cNp):
xt = x
for j in range(t, T):
# transition probability from current state
xt = sp.dot(B[V[k, j], :, :], xt)
# raise Exception('stooooop')
ot = sp.dot(self.A, xt)
# Predicted Divergence
Q[k] += self.H.dot(xt) + (self.lnC - np.log(ot)).dot(ot)
# self.oQ.append(Q)
# self.oV.append(V)
# Variational updates: calculate the distribution over actions, then
# the precision, and iterate N times.
precisionUpdates = []
b = self.alpha / W
for i in range(self.N):
# policy (u)
u[w] = utils.softmax(W * Q)
# precision (W)
b = self.lambd * b + (1 - self.lambd) * \
(self.beta - sp.dot(u[w], Q))
W = self.alpha / b
precisionUpdates.append(W)
# Calculate the posterior over policies and over actions.
for j in range(self.Nu):
P[j] = np.sum(u[w[utils.ismember(V[:, t], j)]])
list_return = [x, P]
if PreUpd is True:
list_return.append(precisionUpdates)
else:
list_return.append(W)
if calc_Qs is True:
list_return.append(Q)
return list_return
if test_preds.shape[1] > 5:
test_preds = test_preds[:, -5:].astype('float32')
np.set_printoptions(precision=3)
np.set_printoptions(suppress=True)
print "Orig test preds:\n\n"
print test_preds[:10], '\n'
if np.mean(test_preds) > 0:
# These are not log probs, so can do log.
test_preds = np.log(1e-5 + test_preds)
test_probs = softmax(test_preds, temp=pl_softmax_temp)
# Double ids so only every other.
images_test_pl = sorted(set(get_img_ids_from_dir(prefix_test)))
labels_test_pl = test_probs.reshape((-1, 2, 5))
print "\nImages for test:\n\n"
print images_test_pl[:5], '\n'
print "\nLabels for test:\n\n"
print labels_test_pl[:5], '\n'
# Add only test PL for now.
id_train_oversample, labels_train_oversample = oversample_set(id_train,
y_train,
sample_coefs)
def single(MDP):
"""
Original implementation of the Active Inference by the Friston group.
Translated to Python by Dario Cuevas.
"""
# Some stuff
alpha = 8
beta = 4
g = 1
lambd = 0
N = 4
T = np.shape(MDP['V'])[1]
# Read some numbers from the inputs
Ns = np.shape(MDP['B'])[1] # Number of hidden states
Nu = np.shape(MDP['B'])[0] # Number of actions
p0 = sp.exp(-16) # Smallest probability
A = MDP['A'] + p0
No = np.shape(MDP['A'])[0] # Number of outcomes
A = sp.dot(A,np.diag(1/np.sum(A,0)))
lnA = sp.log(A)
H = np.sum(A*lnA,0)
# transition probabilities
B = MDP['B'] + p0
for b in xrange(np.shape(B)[0]):
B[b] = B[b]/np.sum(B[b],axis=0)
# priors over last state (goals)
C = sp.tile(MDP['C'],(No,1)).T + p0
C = C/np.sum(C)
lnC = sp.log(C)
# priors over initial state
D = MDP['D']
D = D + p0
D = D/np.sum(D)
lnD = sp.log(D)
# policies and their expectations
V = MDP['V']
Np = np.shape(V)[0]
w = np.array(range(Np))
# initial states and outcomes
q = np.argmax(sp.dot(A,MDP['S']))
s = np.zeros((T))
s[0] = np.nonzero(MDP['S']==1)[0]
o = np.zeros((T))
o[0] = q
S = np.zeros((Ns,T))
S[s[0]][0] = 1
O = np.zeros((No,T))
O[q][0] = 1
U = np.zeros((Nu,T))
P = np.zeros((Nu,T))
x = np.zeros((Ns,T))
u = np.zeros((Np,T))
a = np.zeros((T))
W = np.zeros((T))
#solve
gamma = []
b = alpha/g
for t in xrange(T):
# Expectations of allowable policies (u) and current state (x)
if t>0:
# retain allowable policies (consistent with last action)
j = utils.ismember(V[:,t-1], a[t-1])
V = V[j,:]
w = w[j]
# current state (x)
v = lnA[o[t]] + sp.log(sp.dot(B[a[t-1]],x[:,t-1]))
x[:,t] = utils.softmax(v)
else:
# pdb.set_trace()
u[:,t] = np.ones(Np)/Np
v = lnA[int(o[t]),:] + lnD
x[:,t] = utils.softmax(v)
# value of policies (Q)
cNp = np.shape(V)[0]
Q = np.zeros(cNp)
for k in xrange(cNp):
# path integral of expected free energy (...)
xt = x[:,t]
for j in xrange(t,T):
# transition probability from current state
xt = sp.dot(B[V[k,j]],xt)
ot = sp.dot(A,xt)
# predicted divergence
Q[k] += sp.dot(H,xt) + sp.dot(lnC[:,j] - sp.log(ot),ot)
# Variational iterations
for i in xrange(N):
# policy (u)
u[w,t] = utils.softmax(sp.dot(W[t],Q))
# precision (W)
b = lambd*b + (1 - lambd)*(beta - sp.dot(u[w,t],Q))
W[t] = alpha/b
#simulated dopamine responses (precision as each iteration)
gamma.append(W[t])
for j in xrange(Nu):
for k in xrange(t,T):
P[j,k] = np.sum(u[w[utils.ismember(V[:,k],j)],t])
# next action
a[t] = np.nonzero(np.random.rand(1) < np.cumsum(P[:,t]))[0][0]
# save action
U[a[t],t] = 1
# sampling of next state (outcome)
if t<T-1:
#next sampled state
s[t+1] = np.nonzero(np.random.rand(1) <
np.cumsum(B[a[t],:,s[t]]))[0][0]
#next observed state
o[t+1] = np.nonzero(np.random.rand(1) <
np.cumsum(A[:,s[t+1]]))[0][0]
# save the outcome and state sampled
W[t+1] = W[t]
O[o[t+1]][t+1] = 1
S[s[t+1],t+1] = 1
oMDP = {}
oMDP['P'] = P
oMDP['Q'] = x
oMDP['O'] = O
oMDP['S'] = S
oMDP['U'] = U
oMDP['W'] = W
oMDP['s'] = s
oMDP['a'] = a
return oMDP
lbs = []
for _ in range(args.maxIter):
# Maximize ELBO
grads = elementwise_grad(elbo)(
(lambda_pi, lambda_phi, lambda_m, lambda_beta, lambda_nu, lambda_w))
# Variational parameter updates (gradient ascent)
lambda_pi -= ps['lambda_pi'] * grads[0]
lambda_phi -= ps['lambda_phi'] * grads[1]
lambda_m -= ps['lambda_m'] * grads[2]
lambda_beta -= ps['lambda_beta'] * grads[3]
lambda_nu -= ps['lambda_nu'] * grads[4]
lambda_w -= ps['lambda_w'] * grads[5]
lambda_phi = agnp.array([softmax(lambda_phi[i]) for i in range(N)])
lambda_beta = softplus(lambda_beta)
lambda_nu = softplus(lambda_nu)
lambda_pi = softplus(lambda_pi)
lambda_w = agnp.array(
[agnp.dot(lambda_w[k], lambda_w[k].T) for k in range(K)])
# ELBO computation
lb = elbo(
(lambda_pi, lambda_phi, lambda_m, lambda_beta, lambda_nu, lambda_w))
lbs.append(lb)
if VERBOSE:
print('\n******* ITERATION {} *******'.format(n_iters))
print('lambda_pi: {}'.format(lambda_pi))
print('lambda_beta: {}'.format(lambda_beta))
def feedforward(self, a):
for b, w in zip(self.biases[0:-1], self.weights[0:-1]):
a = utils.relu(np.dot(w, a) + b)
z = np.dot(self.weights[-1], a) + self.biases[-1]
a = utils.softmax(z)
return a
