def forward(self, layers, truth, symm):
loss = torch.reduce_mean(torch.sort(layers[-1] - truth))
for l in layers:
loss += self.t1 * torch.reduce_mean(torch.sqrt(l - truth))
for s in symm:
loss += self.t2 * torch.reduce_mean(torch.sqrt(s))
return loss
def _layer_norm(self, inputs):
x = inputs
m = torch.reduce_mean(x, axis=-1, keepdims=True)
v = torch.reduce_mean(torch.square(x - m), axis=-1, keepdims=True)
y = (x - m) / torch.sqrt(v + self.cfg.eps)
y = y * self.norm_w + self.norm_b
return y
def q_train(make_obs_ph_n,
act_space_n,
q_index,
q_func,
optimizer,
grad_norm_clipping=None,
local_q_func=False,
scope="trainer",
reuse=None,
num_units=64):
with tf.variable_scope(scope, reuse=reuse):
# create distribtuions
act_pdtype_n = [make_pdtype(act_space) for act_space in act_space_n]
# set up placeholders
obs_ph_n = make_obs_ph_n
act_ph_n = [
act_pdtype_n[i].sample_placeholder([None], name="action" + str(i))
for i in range(len(act_space_n))
]
target_ph = tf.placeholder(tf.float32, [None], name="target")
q_input = tf.concat(obs_ph_n + act_ph_n, 1)
if local_q_func:
q_input = tf.concat([obs_ph_n[q_index], act_ph_n[q_index]], 1)
q = q_func(q_input, 1, scope="q_func", num_units=num_units)[:, 0]
q_func_vars = U.scope_vars(U.absolute_scope_name("q_func"))
q_loss = tf.reduce_mean(tf.square(q - target_ph))
# viscosity solution to Bellman differential equation in place of an initial condition
q_reg = tf.reduce_mean(tf.square(q))
loss = q_loss #+ 1e-3 * q_reg
optimize_expr = U.minimize_and_clip(optimizer, loss, q_func_vars,
grad_norm_clipping)
# Create callable functions
train = U.function(inputs=obs_ph_n + act_ph_n + [target_ph],
outputs=loss,
updates=[optimize_expr])
q_values = U.function(obs_ph_n + act_ph_n, q)
# target network
target_q = q_func(q_input,
1,
scope="target_q_func",
num_units=num_units)[:, 0]
target_q_func_vars = U.scope_vars(
U.absolute_scope_name("target_q_func"))
update_target_q = make_update_exp(q_func_vars, target_q_func_vars)
target_q_values = U.function(obs_ph_n + act_ph_n, target_q)
return train, update_target_q, {
'q_values': q_values,
'target_q_values': target_q_values
}
def predict():
"""Predict unseen images"""
"""Step 0: load data and trained model"""
mnist = input_data.read_data_sets("./data/", one_hot=True)
checkpoint_dir = sys.argv[1]
"""Step 1: build the rnn model"""
x = tf.placeholder("float", [None, n_steps, n_input])
y = tf.placeholder("float", [None, n_classes])
weights = tf.Variable(tf.random_normal([n_hidden, n_classes]), name='weights')
biases = tf.Variable(tf.random_normal([n_classes]), name='biases')
pred = rnn_model(x, weights, biases)
correct_pred = tf.equal(tf.argmax(pred, 1), tf.argmax(y, 1))
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))
"""Step 2: predict new images with the trained model"""
with tf.Session() as sess:
sess.run(tf.initialize_all_variables())
"""Step 2.0: load the trained model"""
checkpoint_file = tf.train.latest_checkpoint(checkpoint_dir + 'checkpoints')
print('Loaded the trained model: {}'.format(checkpoint_file))
saver = tf.train.Saver()
saver.restore(sess, checkpoint_file)
"""Step 2.1: predict new data"""
test_len = 500
test_data = mnist.test.images[:test_len].reshape((-1, n_steps, n_input))
test_label = mnist.test.labels[:test_len]
print("Testing Accuracy:", sess.run(accuracy, feed_dict={x: test_data, y: test_label}))
def tanimoto_loss(label, pred):
square = torch.square(pred)
sum_square = torch.sum(square)
product = torch.multiply(pred, label)
sum_product = torch.sum(product)
denomintor = torch.subtract(torch.add(sum_square, 1), sum_product)
loss = torch.divide(sum_product, denomintor)
loss = torch.reduce_mean(loss)
return 1.0 - loss
def E_log_p_Y(self, X, Y):
"""
Calculate the expectation of the data log likelihood under the variational distribution
with MC samples
"""
Fmean, Fvar = self._build_predict(X,
full_cov=False,
S=self.num_samples)
var_exp = self.likelihood.variational_expectations(Fmean, Fvar,
Y) # S, N, D
return th.reduce_mean(var_exp, 0) # N, D
def forward(self, inputs, **kw):
cfg = self.cfg
seq, typ, idx, val, fit, mlm = inputs
seq = y = self.trafo([[seq, typ], None], **kw)
fit_y = self.pool(torch.squeeze(y[:, 0:1, :], axis=1), **kw)
y = torch.gather(y, idx, axis=1)
y = self.norm(self.mlm_dense(y, **kw), **kw)
e = self.trafo.tok_embed.embeddings
y = torch.matmul(y, e, transpose_b=True)
y = torch.log_softmax(torch.bias_add(y, self.mlm_bias), axis=-1)
mlm_loss = -torch.reduce_sum(y * torch.one_hot(val, cfg.s_vocab),
axis=-1)
y = torch.matmul(fit_y, self.gain, transpose_b=True)
y = torch.log_softmax(torch.bias_add(y, self.bias), axis=-1)
fit_loss = -torch.reduce_sum(y * torch.one_hot(fit, 2), axis=-1)
loss = torch.reduce_sum(mlm * mlm_loss)
loss /= (torch.reduce_sum(mlm) + 1e-5) + torch.reduce_mean(fit_loss)
return seq, loss
def forward(self, inputs):
prev, x = inputs
y = x
if self.cmd:
cfg = self.cfg
for c in self.cmd:
if c == "a":
y = prev + x
elif c == "z":
y = prev + x * self.gamma
elif c == "n":
if cfg.norm_type == "layer":
y = _layer_norm(self, x)
elif cfg.norm_type == "batch":
y = self.batch(x)
elif cfg.norm_type == "l2":
m = torch.reduce_mean(x, axis=-1, keepdims=True)
n = torch.square(x - m)
n = torch.reduce_sum(n, axis=-1, keepdims=True)
y = (x - m) / torch.sqrt(n + cfg.eps)
y = y * self.gain + self.bias
elif cfg.norm_type == "group":
sh = torch.int_shape(x)
assert len(sh) == 4 and sh[-1] % cfg.n_groups == 0
gs = (cfg.n_groups, sh[-1] // cfg.n_groups)
x = torch.reshape(x, sh[:-1] + gs)
m, v = torch.moments(x, [1, 2, 4], keep_dims=True)
y = (x - m) / torch.sqrt(v + cfg.group_eps)
y = torch.reshape(y, sh) * self.gain + self.bias
elif cfg.norm_type == "noam":
y = torch.cast_to_floatx(torch.int_shape(x)[-1])
y = torch.l2_normalize(x, axis=-1) * torch.sqrt(y)
else:
assert cfg.norm_type == "none"
else:
assert c == "d"
y = self.drop(y)
x = y
return y
def _loss(i):
y = torch.log_softmax(pred[i], axis=-1)
y = torch.one_hot(span[:, i], self.slen) * y
return -torch.reduce_mean(torch.reduce_sum(y, axis=-1))
def p_train(make_obs_ph_n,
act_space_n,
p_index,
p_func,
q_func,
optimizer,
grad_norm_clipping=None,
local_q_func=False,
num_units=64,
scope="trainer",
reuse=None):
with tf.variable_scope(scope, reuse=reuse):
# create distribtuions
act_pdtype_n = [make_pdtype(act_space) for act_space in act_space_n]
# set up placeholders
obs_ph_n = make_obs_ph_n
act_ph_n = [
act_pdtype_n[i].sample_placeholder([None], name="action" + str(i))
for i in range(len(act_space_n))
]
p_input = obs_ph_n[p_index]
p = p_func(p_input,
int(act_pdtype_n[p_index].param_shape()[0]),
scope="p_func",
num_units=num_units)
p_func_vars = U.scope_vars(U.absolute_scope_name("p_func"))
# wrap parameters in distribution
act_pd = act_pdtype_n[p_index].pdfromflat(p)
act_sample = act_pd.sample()
p_reg = tf.reduce_mean(tf.square(act_pd.flatparam()))
act_input_n = act_ph_n + []
act_input_n[p_index] = act_pd.sample()
q_input = tf.concat(obs_ph_n + act_input_n, 1)
if local_q_func:
q_input = tf.concat([obs_ph_n[p_index], act_input_n[p_index]], 1)
q = q_func(q_input, 1, scope="q_func", reuse=True,
num_units=num_units)[:, 0]
pg_loss = -tf.reduce_mean(q)
loss = pg_loss + p_reg * 1e-3
optimize_expr = U.minimize_and_clip(optimizer, loss, p_func_vars,
grad_norm_clipping)
# Create callable functions
train = U.function(inputs=obs_ph_n + act_ph_n,
outputs=loss,
updates=[optimize_expr])
act = U.function(inputs=[obs_ph_n[p_index]], outputs=act_sample)
p_values = U.function([obs_ph_n[p_index]], p)
# target network
target_p = p_func(p_input,
int(act_pdtype_n[p_index].param_shape()[0]),
scope="target_p_func",
num_units=num_units)
target_p_func_vars = U.scope_vars(
U.absolute_scope_name("target_p_func"))
update_target_p = make_update_exp(p_func_vars, target_p_func_vars)
target_act_sample = act_pdtype_n[p_index].pdfromflat(target_p).sample()
target_act = U.function(inputs=[obs_ph_n[p_index]],
outputs=target_act_sample)
return act, train, update_target_p, {
'p_values': p_values,
'target_act': target_act
}
