def test_to_one_hot():
v = theano.tensor.ivector()
o = to_one_hot(v, 10)
f = theano.function([v], o)
out = f([1, 2, 3, 5, 6])
assert out.dtype == theano.config.floatX
assert np.allclose(
out,
[[0., 1., 0., 0., 0., 0., 0., 0., 0., 0.],
[0., 0., 1., 0., 0., 0., 0., 0., 0., 0.],
[0., 0., 0., 1., 0., 0., 0., 0., 0., 0.],
[0., 0., 0., 0., 0., 1., 0., 0., 0., 0.],
[0., 0., 0., 0., 0., 0., 1., 0., 0., 0.]])
v = theano.tensor.ivector()
o = to_one_hot(v, 10, dtype="int32")
f = theano.function([v], o)
out = f([1, 2, 3, 5, 6])
assert out.dtype == "int32"
assert np.allclose(
out,
[[0., 1., 0., 0., 0., 0., 0., 0., 0., 0.],
[0., 0., 1., 0., 0., 0., 0., 0., 0., 0.],
[0., 0., 0., 1., 0., 0., 0., 0., 0., 0.],
[0., 0., 0., 0., 0., 1., 0., 0., 0., 0.],
[0., 0., 0., 0., 0., 0., 1., 0., 0., 0.]])
def test_to_one_hot():
v = theano.tensor.ivector()
o = to_one_hot(v, 10)
f = theano.function([v], o)
out = f([1, 2, 3, 5, 6])
assert out.dtype == theano.config.floatX
assert numpy.allclose(
out,
[[0., 1., 0., 0., 0., 0., 0., 0., 0., 0.],
[0., 0., 1., 0., 0., 0., 0., 0., 0., 0.],
[0., 0., 0., 1., 0., 0., 0., 0., 0., 0.],
[0., 0., 0., 0., 0., 1., 0., 0., 0., 0.],
[0., 0., 0., 0., 0., 0., 1., 0., 0., 0.]])
v = theano.tensor.ivector()
o = to_one_hot(v, 10, dtype="int32")
f = theano.function([v], o)
out = f([1, 2, 3, 5, 6])
assert out.dtype == "int32"
assert numpy.allclose(
out,
[[0., 1., 0., 0., 0., 0., 0., 0., 0., 0.],
[0., 0., 1., 0., 0., 0., 0., 0., 0., 0.],
[0., 0., 0., 1., 0., 0., 0., 0., 0., 0.],
[0., 0., 0., 0., 0., 1., 0., 0., 0., 0.],
[0., 0., 0., 0., 0., 0., 1., 0., 0., 0.]])
def build_nnet(layer_sizes, normalize_layers=False):
X = T.vector(dtype='float32')
t = T.scalar(dtype='int32')
alpha = T.scalar(dtype='float32')
t_onehot = extra.to_one_hot(t.reshape((1, 1)), 10)
weights = []
# We always want to normalize the inputs to the first layer
Y, W = layer(normalize(X), 784, layer_sizes[0])
weights.append(W)
for l1, l2 in zip(layer_sizes[1:-1], layer_sizes[2:]):
if normalize_layers:
Y = normalize(Y)
Y, W = layer(Y, l1, l2)
weights.append(W)
if normalize_layers:
Y = normalize(Y)
Y, W = layer(Y, layer_sizes[-1], 10, activation=nnet.softmax)
weights.append(W)
mse = T.mean(T.sqr(Y - t_onehot))
updates = [(W, W - alpha * T.grad(cost=mse, wrt=W)) for W in weights]
prediction = T.argmax(Y)
confidence = T.max(Y)
eval_nnet = theano.function(inputs=[X], outputs=[prediction, confidence])
train_nnet = theano.function(inputs=[X, t, alpha], outputs=mse, updates=updates)
return eval_nnet, train_nnet
def test_class_idx_seq_to_1_of_k():
from theano.tensor.extra_ops import to_one_hot
v = theano.tensor.as_tensor_variable(numpy.array([1, 2, 3, 5, 6]))
out = to_one_hot(v, 10).eval()
assert numpy.allclose(out, [[0., 1., 0., 0., 0., 0., 0., 0., 0., 0.],
[0., 0., 1., 0., 0., 0., 0., 0., 0., 0.],
[0., 0., 0., 1., 0., 0., 0., 0., 0., 0.],
[0., 0., 0., 0., 0., 1., 0., 0., 0., 0.],
[0., 0., 0., 0., 0., 0., 1., 0., 0., 0.]])
out2 = class_idx_seq_to_1_of_k(v, 10).eval()
assert numpy.allclose(out, out2)
def encode(samples, max_int):
"""Convert inputs to one-hot matrix form.
The result is shape (S, R, M), where, as usual:
S - num samples, R - num registers, M - max int
"""
samples = np.asarray(samples)
# Encode each register separately.
# to_one_hot requires a 1-d vector.
encoded = []
for i in range(samples.shape[1]):
encoded.append(to_one_hot(samples[:, i], max_int).eval())
return np.asarray(encoded).swapaxes(0, 1)
def test_class_idx_seq_to_1_of_k():
from theano.tensor.extra_ops import to_one_hot
v = theano.tensor.as_tensor_variable(numpy.array([1, 2, 3, 5, 6]))
out = to_one_hot(v, 10).eval()
assert numpy.allclose(
out,
[[0., 1., 0., 0., 0., 0., 0., 0., 0., 0.],
[0., 0., 1., 0., 0., 0., 0., 0., 0., 0.],
[0., 0., 0., 1., 0., 0., 0., 0., 0., 0.],
[0., 0., 0., 0., 0., 1., 0., 0., 0., 0.],
[0., 0., 0., 0., 0., 0., 1., 0., 0., 0.]])
out2 = class_idx_seq_to_1_of_k(v, 10).eval()
assert numpy.allclose(out, out2)
def bce_of_unlabeled_d(p_vals,
num_classes,
negative_samples=False,
epsilon=1e-6):
p_vals = T.clip(p_vals, epsilon, 1 - epsilon)
p_vals = p_vals.reshape((-1, num_classes))
if not negative_samples:
p_max = p_vals.max(axis=1)
return bce(p_max, T.ones(p_max.shape)).mean()
else:
p_max_index = p_vals.argmax(axis=1)
p_max_index = to_one_hot(p_max_index, num_classes)
return bce(p_vals, p_max_index).mean()
def cross_entropy_loss(a, y, one_hot_num_classes=None):
r""" Compute the cross entropy loss over a softmax
.. math::
J = \frac{1}{T} \sum_t^T \sum_k - (y * \log a)_k
:param a: predictions
:param y: a one hot encoding of the target labels
:param one_hot_num_classes: if this option is specified, then ``y`` should be given
as a vector which will be converted to a one hot matrix
:return: cross entropy loss
"""
y = y if one_hot_num_classes is None else to_one_hot(y, one_hot_num_classes)
y = T.cast(y, config.floatX)
num_ex = T.cast(a.shape[0], config.floatX)
return - T.sum(y * T.log(a)) / num_ex
def forward(self, a, train=True):
""" Performs a lookup in the LookupTable for vectors corresponding to entries in ``a``
:param a: a 2-d array of indices that denote the indices to items in the lookup table
:return: a 2-d array A such that ``A[i, :]`` contain the concatenated vectors for words
requested by row ``i`` of ``a``
"""
if a.ndim < 2: # this is a scalar index
return self.E[a]
lookup = []
for col in range(self.window_size):
if self.advanced_indexing:
lookup += [self.E[a[:, col]]]
else:
one_hot = to_one_hot(a[:, col], nb_class=self.vocab_size)
lookup += [T.dot(one_hot, self.E)]
return T.concatenate(lookup, axis=1)
def log_likelihood_sym(self, actions_var, dist_info_vars, bernoulli = False):
"""
PS: x_var should be the samples from the distributions represented with dist_info_vars
"""
probs = dist_info_vars["prob"]
# Assume layout is N * A
if bernoulli:
actions_var = T.shape_padright(actions_var)
actions_var = theano.printing.Print('264 line actions_var:')(actions_var)
# probs = theano.printing.Print('265 line probs:')(probs)
res = T.sum(actions_var * T.log(probs + TINY) + (1 - actions_var) * T.log(1 - probs + TINY), axis=-1)
return res
# actions_var = theano.printing.Print('269 line actions_var:')(actions_var)
# probs = theano.printing.Print('270 line probs:')(probs)
oneHot = Ops.to_one_hot(actions_var,probs.shape[1])
# oneHot = theano.printing.Print('272 line oneHot:')(oneHot)
res = T.log(T.sum(probs*T.cast(oneHot,'float32'),axis=-1)+TINY)
return res
def apply(self, y, y_hat):
# find the unlabeled samples: a combination of oos and labeled samples that were moved by +50
unlabeled = (y >= y_hat.shape[1]).nonzero()
y = y[unlabeled]
y_hat = y_hat[unlabeled]
# return unlabeled samples that are in-set to their original value
y = T.switch(y <= 50, y, y-50)
# convert oos to 0
y = T.switch(y < y_hat.shape[1], y, 0)
# if maximal prob is below oos_thr then assume it is OOS
y_hat_argmax = T.switch(y_hat.max(axis=1) >= self.oos_thr, y_hat.argmax(axis=1), 0)
# locate mistakes
mistakes = T.neq(y, y_hat_argmax)
# compute the error rate for each label
yhot = to_one_hot(y, y_hat.shape[1], dtype=floatX)
yhot = yhot.T
mistakes = T.dot(yhot, mistakes) / (yhot.sum(axis=1) + np.float32(1e-6))
return (1. - self.poos)*mistakes[1:].mean() + self.poos * mistakes[0]
def apply(self, y, y_hat):
# find the unlabeled samples: a combination of oos and labeled samples that were moved by +50
unlabeled = (y >= y_hat.shape[1]).nonzero()
y = y[unlabeled]
y_hat = y_hat[unlabeled]
# return unlabeled samples that are in-set to their original value
y = T.switch(y <= 50, y, y - 50)
# convert oos to 0
y = T.switch(y < y_hat.shape[1], y, 0)
# if maximal prob is below oos_thr then assume it is OOS
y_hat_argmax = T.switch(
y_hat.max(axis=1) >= self.oos_thr, y_hat.argmax(axis=1), 0)
# locate mistakes
mistakes = T.neq(y, y_hat_argmax)
# compute the error rate for each label
yhot = to_one_hot(y, y_hat.shape[1], dtype=floatX)
yhot = yhot.T
mistakes = T.dot(yhot,
mistakes) / (yhot.sum(axis=1) + np.float32(1e-6))
return (1. - self.poos) * mistakes[1:].mean() + self.poos * mistakes[0]
q_values = DenseLayer(dense_2,
num_units = n_action,
nonlinearity = None,
W = Normal(0.1, 0.0),
b = Constant(0.0))
return q_values
X_next_state = T.fmatrix()
X_state = T.fmatrix()
X_action = T.bvector()
X_reward = T.fvector()
X_done = T.bvector()
X_action_hot = to_one_hot(X_action, n_action)
q_ = q_network(X_state); q = get_output(q_)
q_target_ = q_network(X_next_state); q_target = get_output(q_target_)
q_max = T.max(q_target, axis=1)
action = T.argmax(q, axis=1)
mu = theano.function(inputs = [X_state],
outputs = action,
allow_input_downcast = True)
loss = squared_error(X_reward + gamma * q_max * (1.0 - X_done), T.batched_dot(q, X_action_hot))
loss = loss.mean()
params = get_all_params(q_)
def module(max_int, mem):
"""Return the one-hot encoded constant."""
return to_one_hot(arr, max_int), mem
def __init__(self, n_vocab, n_mem, n_class, n_emb=50):
x1 = T.imatrix('parse1')
x2 = T.imatrix('parse2')
mask1 = T.imatrix('parse1_mask')
mask2 = T.imatrix('parse2_mask')
y = T.ivector('relation')
lookup = LookupTable(
n_vocab,
n_emb,
name='lookup',
weights_init=Uniform(0., width=0.01),
)
emb1 = lookup.apply(x1)
emb2 = lookup.apply(x2)
trans1 = Linear(n_emb,
n_mem * 4,
name='emb_to_h1',
weights_init=IsotropicGaussian(0.1),
biases_init=Constant(0.)).apply(emb1)
trans2 = Linear(n_emb,
n_mem * 4,
name='emb_to_h2',
weights_init=IsotropicGaussian(0.1),
biases_init=Constant(0.)).apply(emb2)
h1, c1 = LSTM(n_mem,
weights_init=IsotropicGaussian(0.1),
biases_init=Constant(0.),
name='lstm1').apply(trans1,
mask=T.cast(mask1,
theano.config.floatX))
h2, c2 = LSTM(n_mem,
weights_init=IsotropicGaussian(0.1),
biases_init=Constant(0.),
name='lstm2').apply(trans2,
mask=T.cast(mask2,
theano.config.floatX))
h_concat = T.concatenate([h1[-1], h2[-1]], axis=-1)
# debugging
self.x1, self.x2, self.mask1, self.mask2 = x1, x2, mask1, mask2
self.trans1, self.trans2 = trans1, trans2
self.emb1, self.emb2, self.h1, self.h2, self.h_concat = emb1, emb2, h1, h2, h_concat
score = Linear(
2 * n_mem,
n_class,
name='output',
weights_init=IsotropicGaussian(0.1),
biases_init=Constant(0.),
).apply(h_concat)
epsilon = 1e-7
self.y_prob = Softmax().apply(T.clip(score, epsilon, 1 - epsilon))
self.cost = CategoricalCrossEntropy().apply(to_one_hot(y, n_class),
self.y_prob)
self.cost.name = 'CrossEntropyCost'
eq = T.eq(self.y_prob.argmax(axis=-1), y)
self.acc = T.cast(eq, 'float32').mean()
self.acc.name = 'Accuracy'
self.lookup = lookup
def __call__(self, x):
import theano.tensor.extra_ops as extra_ops
y = extra_ops.to_one_hot(x.flatten(), self.n_classes)
if x.ndim == 1:
return y
return y.reshape((x.shape[0], x.shape[1], -1))
alpha_hinge=alpha_hinge,
delta=delta)
classifier_cost_eval = multiclass_hinge_loss(
predictions=predictions_eval,
targets=sym_y,
weight_decay=weight_decay_classifier,
alpha_decay=alpha_decay) # no hat loss for testing
cost_cla = classifier_cost_train
# generative objective
predictions_train_hard = predictions_train.argmax(axis=1)
predictions_eval_hard = predictions_eval.argmax(axis=1)
sym_l_in_y_train = to_one_hot(
T.concatenate([sym_y, predictions_train_hard[sym_batch_size_l:]], axis=0),
num_classes)
if distribution == 'bernoulli':
z_train, z_mu_train, z_log_var_train, x_mu_train = lasagne.layers.get_output(
[l_z, l_mu, l_log_var, l_dec_x_mu], {
l_in_x: sym_x,
l_in_y: sym_l_in_y_train
},
deterministic=False)
z_eval, z_mu_eval, z_log_var_eval, x_mu_eval = lasagne.layers.get_output(
[l_z, l_mu, l_log_var, l_dec_x_mu], {
l_in_x: sym_x,
l_in_y: to_one_hot(predictions_eval_hard, num_classes)
},
deterministic=True)
network = build_cnn(input_var)
prediction = lasagne.layers.get_output(network)
#y1_hot = Ex.to_one_hot(target_var,10)
#loss = T.mean(T.mul(y1_hot, T.nnet.relu(1 - prediction )) + 1.0/9*T.mul(1 - y1_hot, T.nnet.relu(1 + prediction )))
loss = lasagne.objectives.categorical_crossentropy(prediction, target_var)
loss = loss.mean()
# Get network params, with specifications of manually updated ones
params = lasagne.layers.get_all_params(network, trainable=True)
#updates = lasagne.updates.sgd(loss,params,learning_rate=0.01)
#updates = lasagne.updates.adam(loss,params)
updates = lasagne.updates.nesterov_momentum(loss, params, learning_rate=0.01)
test_prediction = lasagne.layers.get_output(network, deterministic=True)
y1_hot_t = Ex.to_one_hot(target_var, 10)
#test_loss = T.mean(T.mul(y1_hot_t, T.nnet.relu(1 - test_prediction)) + 1.0/9*T.mul(1 - y1_hot_t, T.nnet.relu(1 + test_prediction)))
test_loss = lasagne.objectives.categorical_crossentropy(
test_prediction, target_var)
test_loss = test_loss.mean()
test_acc = T.mean(T.eq(T.argmax(test_prediction, axis=1), target_var),
dtype=theano.config.floatX)
# Compile theano function computing the training validation loss and accuracy:
train_fn = theano.function([input_var, target_var], loss, updates=updates)
val_fn = theano.function([input_var, target_var], [test_loss, test_acc])
# The training loop
print("Starting training...")
num_epochs = 0
for epoch in range(num_epochs):
def __init__(self,
atari_env,
state_dimension,
action_dimension,
monitor_env=False,
learning_rate=0.001,
critic_update=10,
train_step=1,
gamma=0.95,
eps_max=1.0,
eps_min=0.1,
eps_decay=10000,
n_epochs=10000,
batch_size=32,
buffer_size=50000):
self.env = gym.make(atari_env)
if monitor_env:
None
self.state_dimension = state_dimension
self.action_dimension = action_dimension
self.learning_rate = learning_rate
self.critic_update = critic_update
self.train_step = train_step
self.gamma = gamma
self.eps_max = eps_max
self.eps_min = eps_min
self.eps_decay = eps_decay
self.n_epochs = n_epochs
self.batch_size = batch_size
self.buffer_size = buffer_size
self.experience_replay = []
def q_network(state):
input_state = InputLayer(input_var=state,
shape=(None, self.state_dimension[0],
self.state_dimension[1],
self.state_dimension[2]))
input_state = DimshuffleLayer(input_state, pattern=(0, 3, 1, 2))
conv = Conv2DLayer(input_state,
num_filters=32,
filter_size=(8, 8),
stride=(4, 4),
nonlinearity=rectify)
conv = Conv2DLayer(conv,
num_filters=64,
filter_size=(4, 4),
stride=(2, 2),
nonlinearity=rectify)
conv = Conv2DLayer(conv,
num_filters=64,
filter_size=(3, 3),
stride=(1, 1),
nonlinearity=rectify)
flatten = FlattenLayer(conv)
dense = DenseLayer(flatten, num_units=512, nonlinearity=rectify)
q_values = DenseLayer(dense,
num_units=self.action_dimension,
nonlinearity=linear)
return q_values
self.X_state = T.ftensor4()
self.X_action = T.bvector()
self.X_reward = T.fvector()
self.X_next_state = T.ftensor4()
self.X_done = T.bvector()
self.X_action_hot = to_one_hot(self.X_action, self.action_dimension)
self.q_ = q_network(self.X_state)
self.q = get_output(self.q_)
self.q_target_ = q_network(self.X_next_state)
self.q_target = get_output(self.q_target_)
self.q_max = T.max(self.q_target, axis=1)
self.action = T.argmax(self.q, axis=1)
self.mu = theano.function(inputs=[self.X_state],
outputs=self.action,
allow_input_downcast=True)
self.loss = squared_error(
self.X_reward + self.gamma * self.q_max * (1.0 - self.X_done),
T.batched_dot(self.q, self.X_action_hot))
self.loss = self.loss.mean()
self.params = get_all_params(self.q_)
self.grads = T.grad(self.loss, self.params)
self.normed_grads = total_norm_constraint(self.grads, 1.0)
self.updates = rmsprop(self.normed_grads,
self.params,
learning_rate=self.learning_rate)
self.update_network = theano.function(inputs=[
self.X_state, self.X_action, self.X_reward, self.X_next_state,
self.X_done
],
outputs=self.loss,
updates=self.updates,
allow_input_downcast=True)
input_state = InputLayer(input_var=state, shape=(None, n_input))
dense_1 = DenseLayer(input_state, num_units=n_input, nonlinearity=tanh)
dense_2 = DenseLayer(dense_1, num_units=n_input, nonlinearity=tanh)
probs = DenseLayer(dense_2, num_units=n_output, nonlinearity=softmax)
return probs
X_state = T.fmatrix()
X_action = T.bvector()
X_reward = T.fvector()
X_action_hot = to_one_hot(X_action, n_output)
prob_values = policy_network(X_state)
policy_ = get_output(prob_values)
policy = theano.function(inputs=[X_state],
outputs=policy_,
allow_input_downcast=True)
loss = categorical_crossentropy(policy_, X_action_hot) * X_reward
loss = loss.mean()
params = get_all_params(prob_values)
updates = adam(loss, params, learning_rate=learning_rate)
def __init__(self, n_vocab, n_mem, n_class, n_emb=50):
x1 = T.imatrix('parse1')
x2 = T.imatrix('parse2')
mask1 = T.imatrix('parse1_mask')
mask2 = T.imatrix('parse2_mask')
y = T.ivector('relation')
lookup = LookupTable(
n_vocab, n_emb,
name='lookup',
weights_init=Uniform(0., width=0.01),
)
emb1 = lookup.apply(x1)
emb2 = lookup.apply(x2)
trans1 = Linear(
n_emb, n_mem*4,
name='emb_to_h1',
weights_init=IsotropicGaussian(0.1),
biases_init=Constant(0.)
).apply(emb1)
trans2 = Linear(
n_emb, n_mem*4,
name='emb_to_h2',
weights_init=IsotropicGaussian(0.1),
biases_init=Constant(0.)
).apply(emb2)
h1, c1 = LSTM(
n_mem,
weights_init=IsotropicGaussian(0.1),
biases_init=Constant(0.),
name='lstm1'
).apply(trans1, mask=T.cast(mask1, theano.config.floatX))
h2, c2 = LSTM(
n_mem,
weights_init=IsotropicGaussian(0.1),
biases_init=Constant(0.),
name='lstm2'
).apply(trans2, mask=T.cast(mask2, theano.config.floatX))
h_concat = T.concatenate([h1[-1], h2[-1]], axis=-1)
# debugging
self.x1, self.x2, self.mask1, self.mask2 = x1, x2, mask1, mask2
self.trans1, self.trans2 = trans1, trans2
self.emb1, self.emb2, self.h1, self.h2, self.h_concat = emb1, emb2, h1, h2, h_concat
score = Linear(
2 * n_mem, n_class,
name='output',
weights_init=IsotropicGaussian(0.1),
biases_init=Constant(0.),
).apply(h_concat)
epsilon = 1e-7
self.y_prob = Softmax().apply(T.clip(score, epsilon, 1-epsilon))
self.cost = CategoricalCrossEntropy().apply(to_one_hot(y, n_class), self.y_prob)
self.cost.name = 'CrossEntropyCost'
eq = T.eq(self.y_prob.argmax(axis=-1), y)
self.acc = T.cast(eq, 'float32').mean()
self.acc.name = 'Accuracy'
self.lookup = lookup
