def learn(self, batch_size):
exps = self.memory.sample(batch_size)
obss, actions, rewards, obs_nexts, dones = self._process(exps)
# Update critic
dy.renew_cg()
target_actions = self.actor_target(obs_nexts, batched=True)
target_values = self.critic_target(dy.concatenate([dy.inputTensor(obs_nexts, batched=True), target_actions]),
batched=True)
target_values = rewards + 0.99 * target_values.npvalue() * (1 - dones)
dy.renew_cg()
values = self.critic(np.concatenate([obss, actions]), batched=True)
loss = dy.mean_batches((values - dy.inputTensor(target_values, batched=True)) ** 2)
loss_value_critic = loss.npvalue()
loss.backward()
self.trainer_critic.update()
# update actor
dy.renew_cg()
actions = self.actor(obss, batched=True)
obs_and_actions = dy.concatenate([dy.inputTensor(obss, batched=True), actions])
loss = -dy.mean_batches(self.critic(obs_and_actions, batched=True))
loss_value_actor = loss.npvalue()
loss.backward()
self.trainer_actor.update()
self.noise_stddev = (
self.noise_stddev - self.noise_stddev_decrease) if self.noise_stddev > self.noise_stddev_lower else self.noise_stddev_lower
self.actor_target.update(self.actor, soft=True)
self.critic_target.update(self.critic, soft=True)
return loss_value_actor + loss_value_critic
def _loss(outputs, labels):
losses = [
dy.pickneglogsoftmax_batch(out, label)
for out, label in zip(outputs, labels)
]
loss = dy.mean_batches(dy.average(losses))
return loss
def compose(
self, embeds: Union[dy.Expression,
List[dy.Expression]]) -> dy.Expression:
if type(embeds) != list:
return dy.mean_batches(embeds)
else:
return dy.average(embeds)
def on_calc_additional_loss(self, reward):
if not self.learn_segmentation:
return None
ret = LossBuilder()
if self.length_prior_alpha > 0:
reward += self.segment_length_prior * self.length_prior_alpha
reward = dy.cdiv(reward - dy.mean_batches(reward),
dy.std_batches(reward))
# Baseline Loss
if self.use_baseline:
baseline_loss = []
for i, baseline in enumerate(self.bs):
baseline_loss.append(dy.squared_distance(reward, baseline))
ret.add_loss("Baseline", dy.esum(baseline_loss))
# Reinforce Loss
lmbd = self.lmbd.get_value(self.warmup_counter)
if lmbd > 0.0:
reinforce_loss = []
# Calculating the loss of the baseline and reinforce
for i in range(len(self.segment_decisions)):
ll = dy.pick_batch(self.segment_logsoftmaxes[i],
self.segment_decisions[i])
if self.use_baseline:
r_i = reward - self.bs[i]
else:
r_i = reward
reinforce_loss.append(dy.logistic(r_i) * ll)
ret.add_loss("Reinforce", -dy.esum(reinforce_loss) * lmbd)
# Total Loss
return ret
def _step(self, loader, update, log, reporting_fns, verbose=None):
steps = len(loader)
pg = create_progress_bar(steps)
cm = ConfusionMatrix(self.labels)
epoch_loss = 0
epoch_div = 0
for batch_dict in pg(loader):
dy.renew_cg()
inputs = self.model.make_input(batch_dict)
ys = inputs.pop('y')
preds = self.model.forward(inputs)
losses = self.model.loss(preds, ys)
loss = dy.mean_batches(losses)
batchsz = self._get_batchsz(batch_dict)
lossv = loss.npvalue().item() * batchsz
epoch_loss += lossv
epoch_div += batchsz
_add_to_cm(cm, ys, preds.npvalue())
update(loss)
log(self.optimizer.global_step, lossv, batchsz, reporting_fns)
metrics = cm.get_all_metrics()
metrics['avg_loss'] = epoch_loss / float(epoch_div)
verbose_output(verbose, cm)
return metrics
def on_calc_additional_loss(self, translator_loss):
if not self.learn_segmentation or self.segment_decisions is None:
return None
reward = -translator_loss["mle"]
if not self.log_reward:
reward = dy.exp(reward)
reward = dy.nobackprop(reward)
# Make sure that reward is not scalar, but rather based on the each batch item
assert reward.dim()[1] == len(self.src_sent)
# Mask
enc_mask = self.enc_mask.get_active_one_mask().transpose() if self.enc_mask is not None else None
# Compose the lose
ret = LossBuilder()
## Length prior
alpha = self.length_prior_alpha.value() if self.length_prior_alpha is not None else 0
if alpha > 0:
reward += self.segment_length_prior * alpha
# reward z-score normalization
if self.z_normalization:
reward = dy.cdiv(reward-dy.mean_batches(reward), dy.std_batches(reward) + EPS)
## Baseline Loss
if self.use_baseline:
baseline_loss = []
for i, baseline in enumerate(self.bs):
loss = dy.squared_distance(reward, baseline)
if enc_mask is not None:
loss = dy.cmult(dy.inputTensor(enc_mask[i], batched=True), loss)
baseline_loss.append(loss)
ret.add_loss("Baseline", dy.esum(baseline_loss))
if self.print_sample:
print(dy.exp(self.segment_logsoftmaxes[i]).npvalue().transpose()[0])
## Reinforce Loss
lmbd = self.lmbd.value()
if lmbd > 0.0:
reinforce_loss = []
# Calculating the loss of the baseline and reinforce
for i in range(len(self.segment_decisions)):
ll = dy.pick_batch(self.segment_logsoftmaxes[i], self.segment_decisions[i])
if self.use_baseline:
r_i = reward - dy.nobackprop(self.bs[i])
else:
r_i = reward
if enc_mask is not None:
ll = dy.cmult(dy.inputTensor(enc_mask[i], batched=True), ll)
reinforce_loss.append(r_i * -ll)
loss = dy.esum(reinforce_loss) * lmbd
ret.add_loss("Reinforce", loss)
if self.confidence_penalty:
ls_loss = self.confidence_penalty(self.segment_logsoftmaxes, enc_mask)
ret.add_loss("Confidence Penalty", ls_loss)
# Total Loss
return ret
def learn(self, batch_size):
exps = self.memory.sample(batch_size)
obss, actions, rewards, obs_nexts, dones = self._process(exps)
# Update critic
dy.renew_cg()
target_actions = self.actor_target(obs_nexts, batched=True)
target_values = self.critic_target(dy.concatenate(
[dy.inputTensor(obs_nexts, batched=True), target_actions]),
batched=True)
target_values = rewards + 0.99 * target_values.npvalue() * (1 - dones)
dy.renew_cg()
values = self.critic(np.concatenate([obss, actions]), batched=True)
loss = dy.mean_batches(
(values - dy.inputTensor(target_values, batched=True))**2)
loss_value_critic = loss.npvalue()
loss.backward()
self.trainer_critic.update()
# update actor
dy.renew_cg()
actions = self.actor(obss, batched=True)
obs_and_actions = dy.concatenate(
[dy.inputTensor(obss, batched=True), actions])
loss = -dy.mean_batches(self.critic(obs_and_actions, batched=True))
loss_value_actor = loss.npvalue()
loss.backward()
self.trainer_actor.update()
self.noise_stddev = (
self.noise_stddev - self.noise_stddev_decrease
) if self.noise_stddev > self.noise_stddev_lower else self.noise_stddev_lower
self.actor_target.update(self.actor, soft=True)
self.critic_target.update(self.critic, soft=True)
return loss_value_actor + loss_value_critic
def test():
lbls = []
imgs = []
for lbl, img in test_data:
lbls.append(lbl)
imgs.append(img)
dy.renew_cg()
losses = network.create_network_return_loss(imgs, lbls, dropout=False)
loss = dy.mean_batches(losses)
predicts = network.create_network_return_best(imgs, dropout=False)
correct = np.sum(lbls == predicts[0])
print('\nTest set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\n'.format(
loss.value(), correct, len(test_data),
100. * correct / len(test_data)))
def mean(x, dim=None, include_batch_dim=False):
if isinstance(x, list):
return dy.average(x)
head_shape, batch_size = x.dim()
if dim is None:
# warning: dynet only implement 2 or lower dims for mean_elems
x = dy.mean_elems(x)
if include_batch_dim and batch_size > 1:
return dy.mean_batches(x)
else:
return x
else:
if dim == -1:
dim = len(head_shape) - 1
return dy.mean_dim(x, d=[dim], b=include_batch_dim)
def _step(self,
loader,
update,
log,
reporting_fns,
verbose=None,
output=None,
txts=None):
steps = len(loader)
pg = create_progress_bar(steps)
cm = ConfusionMatrix(self.labels)
epoch_loss = 0
epoch_div = 0
handle = None
line_number = 0
if output is not None and txts is not None:
handle = open(output, "w")
for batch_dict in pg(loader):
dy.renew_cg()
inputs = self.model.make_input(batch_dict)
ys = inputs.pop('y')
preds = self.model.forward(inputs)
losses = self.model.loss(preds, ys)
loss = dy.mean_batches(losses)
batchsz = self._get_batchsz(batch_dict)
lossv = loss.npvalue().item() * batchsz
if handle is not None:
for p, y in zip(preds, ys):
handle.write('{}\t{}\t{}\n'.format(
" ".join(txts[line_number]), self.model.labels[p],
self.model.labels[y]))
line_number += 1
epoch_loss += lossv
epoch_div += batchsz
_add_to_cm(cm, ys, preds.npvalue())
update(loss)
log(self.optimizer.global_step, lossv, batchsz, reporting_fns)
metrics = cm.get_all_metrics()
metrics['avg_loss'] = epoch_loss / float(epoch_div)
verbose_output(verbose, cm)
if handle is not None:
handle.close()
return metrics
def decode_loss(self, encodings, trg): y, masksy = self.prepare_batch(trg, self.__trg_eos) slen, batch_size = y.shape self.__dec.init_params(encodings, batch_size, self.__train_flag) context = dy.zeros((self.__enc.output_dim, ), batch_size=batch_size) errs = [] for cur_word, next_word, mask in zip(y, y[1:], masksy[1:]): hidden, embs, _ = self.__dec.next(cur_word, context, self.__train_flag) context, _ = self.attend(encodings, hidden) score = self.__dec.score(hidden, context, embs, self.__train_flag) masksy_embs = dy.inputTensor(mask, batched=True) loss = self.cross_entropy_loss(score, next_word, cur_word) loss = dy.cmult(loss, masksy_embs) errs.append(loss) error = dy.mean_batches(dy.esum(errs)) return error
def decode_loss(self, encodings, trg):
"""Compute the negative conditional log likelihood of the target sentence
given the encoding of the source sentence
:param encodings: Source sentence encodings obtained with self.encode
:param trg: List of target sentences
:returns: Expression of the loss averaged on the minibatch
"""
y, masksy = self.prepare_batch(trg, self.trg_eos)
slen, bsize = y.shape
# Init decoder
self.dec.init(encodings,
y,
self.usr.user_vector,
test=self.test,
update=self.update)
# Initialize context
context = dy.zeroes((self.enc.dim, ), batch_size=bsize)
# Process user token if necessary
if self.user_token:
_, _, _ = self.dec.next(self.usr.user_vector,
context,
test=self.test)
# Start decoding
errs = []
for cw, nw, mask in zip(y, y[1:], masksy[1:]):
# Run LSTM
h, e, _ = self.dec.next(cw, context, test=self.test)
# Compute next context
context, _ = self.attend(encodings, h)
# Score
s = self.dec.s(h, context, e, test=self.test)
masksy_e = dy.inputTensor(mask, batched=True)
# Loss
loss = self.cross_entropy_loss(s, nw, cw)
loss = dy.cmult(loss, masksy_e)
errs.append(loss)
# Add all losses together
err = dy.mean_batches(dy.esum(errs))
return err
def calc_loss(self, rewards):
loss = FactoredLossExpr()
## Z-Normalization
if self.z_normalization:
reward_batches = dy.concatenate_to_batch(rewards)
mean_batches = dy.mean_batches(reward_batches)
std_batches = dy.std_batches(reward_batches)
rewards = [
dy.cdiv(reward - mean_batches, std_batches)
for reward in rewards
]
## Calculate baseline
if self.baseline is not None:
pred_reward, baseline_loss = self.calc_baseline_loss(rewards)
loss.add_loss("rl_baseline", baseline_loss)
## Calculate Confidence Penalty
if self.confidence_penalty:
loss.add_loss("rl_confpen",
self.confidence_penalty.calc_loss(self.policy_lls))
## Calculate Reinforce Loss
reinf_loss = []
# Loop through all action in one sequence
for i, (policy,
action_sample) in enumerate(zip(self.policy_lls,
self.actions)):
# Discount the reward if we use baseline
if self.baseline is not None:
rewards = [reward - pred_reward[i] for reward in rewards]
# Main Reinforce calculation
sample_loss = []
for action, reward in zip(action_sample, rewards):
ll = dy.pick_batch(policy, action)
if self.valid_pos is not None:
ll = dy.pick_batch_elems(ll, self.valid_pos[i])
reward = dy.pick_batch_elems(reward, self.valid_pos[i])
sample_loss.append(dy.sum_batches(ll * reward))
# Take the average of the losses accross multiple samples
reinf_loss.append(dy.esum(sample_loss) / len(sample_loss))
loss.add_loss("rl_reinf", self.weight * -dy.esum(reinf_loss))
## the composed losses
return loss
def train(epoch):
random.shuffle(train_data)
i = 0
epoch_start = time.time()
while i < len(train_data):
dy.renew_cg()
lbls = []
imgs = []
for lbl, img in train_data[i:i + args.batch_size]:
lbls.append(lbl)
imgs.append(img)
losses = network.create_network_return_loss(imgs, lbls, dropout=True)
loss = dy.mean_batches(losses)
if (int(i / args.batch_size)) % args.log_interval == 0:
print('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format(
epoch, i, len(train_data), 100. * i / len(train_data),
loss.value()))
loss.backward()
trainer.update()
i += args.batch_size
epoch_end = time.time()
print("{} s per epoch".format(epoch_end - epoch_start))
def train(epoch):
random.shuffle(train_data)
i = 0
epoch_start = time.time()
while i < len(train_data):
dy.renew_cg()
lbls = []
imgs = []
for lbl, img in train_data[i:i+args.batch_size]:
lbls.append(lbl)
imgs.append(img)
losses = network.create_network_return_loss(imgs, lbls, dropout=True)
loss = dy.mean_batches(losses)
if (int(i/args.batch_size)) % args.log_interval == 0:
print('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format(
epoch, i, len(train_data),
100. * i/len(train_data), loss.value()))
loss.backward()
trainer.update()
i += args.batch_size
epoch_end = time.time()
print("{} s per epoch".format(epoch_end-epoch_start))
def _loss(outputs, labels):
losses = [dy.pickneglogsoftmax_batch(out, label) for out, label in zip(outputs, labels)]
loss = dy.mean_batches(dy.average(losses))
return loss
def _train(self, ts, **kwargs):
self.model.train = True
reporting_fns = kwargs.get('reporting_fns', [])
epoch_loss = 0
epoch_norm = 0
auto_norm = 0
metrics = {}
steps = len(ts)
last = steps
losses = []
i = 1
pg = create_progress_bar(steps)
dy.renew_cg()
for batch_dict in pg(ts):
inputs = self.model.make_input(batch_dict)
y = inputs.pop('y')
pred = self.model.compute_unaries(inputs)
bsz = self._get_batchsz(y)
if self.autobatchsz is None:
losses = self.model.loss(pred, y)
loss = dy.mean_batches(losses)
lossv = loss.npvalue().item()
report_loss = lossv * bsz
epoch_loss += report_loss
epoch_norm += bsz
self.nstep_agg += report_loss
self.nstep_div += bsz
loss.backward()
self.optimizer.update()
dy.renew_cg()
# TODO: Abstract this somewhat, or else once we have a batched tagger have 2 trainers
if (self.optimizer.global_step + 1) % self.nsteps == 0:
metrics = self.calc_metrics(self.nstep_agg, self.nstep_div)
self.report(self.optimizer.global_step + 1, metrics,
self.nstep_start, 'Train', 'STEP',
reporting_fns, self.nsteps)
self.reset_nstep()
else:
loss = self.model.loss(pred, y)
losses.append(loss)
self.nstep_div += bsz
epoch_norm += bsz
auto_norm += bsz
if i % self.autobatchsz == 0 or i == last:
loss = dy.average(losses)
lossv = loss.npvalue().item()
loss.backward()
self.optimizer.update()
report_loss = lossv * auto_norm
epoch_loss += report_loss
self.nstep_agg += report_loss
losses = []
dy.renew_cg()
if (self.optimizer.global_step + 1) % self.nsteps == 0:
metrics = self.calc_metrics(self.nstep_agg,
self.nstep_div)
self.report(self.optimizer.global_step + 1, metrics,
self.nstep_start, 'Train', 'STEP',
reporting_fnsa, self.nsteps)
self.reset_nstep()
auto_norm = 0
i += 1
metrics = self.calc_metrics(epoch_loss, epoch_norm)
return metrics
def predict_loss(self, encodings, usr):
avg_enc = dy.mean_dim(encodings, 1)
h = dy.rectify(dy.affine_transform([self.bh, self.Wh, avg_enc]))
s = dy.affine_transform([self.bu, self.Wu, h])
return dy.mean_batches(dy.squared_distance(s, self.usr_vec))
def predict_loss(self, encodings, usr):
avg_enc = dy.mean_dim(encodings, 1)
h = dy.rectify(dy.affine_transform([self.bh, self.Wh, avg_enc]))
s = dy.affine_transform([self.bu, self.Wu, h])
return dy.mean_batches(dy.pickneglogsoftmax(s, usr))
def train(args, network, train_batches, dev_batches, log=None):
"""Estimate model parameters on `train_batches`
with early stopping on`dev_batches`"""
# Logger
log = log or util.Logger(verbose=args.verbose, flush=True)
# Optimizer
trainer = dy.AdamTrainer(network.pc, alpha=args.lr)
# Start training
log("Starting training")
best_accuracy = 0
deadline = 0
running_nll = n_processed = 0
report_every = ceil(len(train_batches) / 10)
# Start training
for epoch in range(1, args.n_epochs + 1):
# Time the epoch
start_time = time.time()
for batch, y in train_batches:
# Renew the computation graph
dy.renew_cg()
# Initialize layers
network.init(test=False, update=True)
# Compute logits
logits = network(batch)
# Loss function
nll = dy.mean_batches(dy.pickneglogsoftmax_batch(logits, y))
# Backward pass
nll.backward()
# Update the parameters
trainer.update()
# Keep track of the nll
running_nll += nll.value() * batch.batch_size
n_processed += batch.batch_size
# Print the current loss from time to time
if train_batches.just_passed_multiple(report_every):
avg_nll = running_nll / n_processed
log(f"Epoch {epoch}@{train_batches.percentage_done():.0f}%: "
f"NLL={avg_nll:.3f}")
running_nll = n_processed = 0
# End of epoch logging
avg_nll = running_nll / n_processed
log(f"Epoch {epoch}@100%: NLL={avg_nll:.3f}")
log(f"Took {time.time()-start_time:.1f}s")
log("=" * 20)
# Validate
accuracy = evaluate(args, network, dev_batches)
# Print final result
log(f"Dev accuracy: {accuracy*100:.2f}%")
# Early stopping
if accuracy > best_accuracy:
best_accuracy = accuracy
dynn.io.save(network.pc, args.model_file)
deadline = 0
else:
if deadline < args.patience:
dynn.io.populate(network.pc, args.model_file)
trainer.learning_rate *= args.lr_decay
deadline += 1
else:
log("Early stopping with best accuracy "
f"{best_accuracy*100:.2f}%")
break
# Load best model
dynn.io.populate(network.pc, args.model_file)
return best_accuracy
def _train(self, ts, **kwargs):
self.model.train = True
reporting_fns = kwargs.get('reporting_fns', [])
epoch_loss = 0
epoch_norm = 0
auto_norm = 0
metrics = {}
steps = len(ts)
last = steps
losses = []
i = 1
pg = create_progress_bar(steps)
dy.renew_cg()
for batch_dict in pg(ts):
inputs = self.model.make_input(batch_dict)
y = inputs.pop('y')
pred = self.model.compute_unaries(inputs)
bsz = self._get_batchsz(y)
if self.autobatchsz is None:
losses = self.model.loss(pred, y)
loss = dy.mean_batches(losses)
lossv = loss.npvalue().item()
report_loss = lossv * bsz
epoch_loss += report_loss
epoch_norm += bsz
self.nstep_agg += report_loss
self.nstep_div += bsz
loss.backward()
self.optimizer.update()
dy.renew_cg()
# TODO: Abstract this somewhat, or else once we have a batched tagger have 2 trainers
if (self.optimizer.global_step + 1) % self.nsteps == 0:
metrics = self.calc_metrics(self.nstep_agg, self.nstep_div)
self.report(
self.optimizer.global_step + 1, metrics, self.nstep_start,
'Train', 'STEP', reporting_fns, self.nsteps
)
self.reset_nstep()
else:
loss = self.model.loss(pred, y)
losses.append(loss)
self.nstep_div += bsz
epoch_norm += bsz
auto_norm += bsz
if i % self.autobatchsz == 0 or i == last:
loss = dy.average(losses)
lossv = loss.npvalue().item()
loss.backward()
self.optimizer.update()
report_loss = lossv * auto_norm
epoch_loss += report_loss
self.nstep_agg += report_loss
losses = []
dy.renew_cg()
if (self.optimizer.global_step + 1) % self.nsteps == 0:
metrics = self.calc_metrics(self.nstep_agg, self.nstep_div)
self.report(
self.optimizer.global_step + 1, metrics, self.nstep_start,
'Train', 'STEP', reporting_fnsa, self.nsteps
)
self.reset_nstep()
auto_norm = 0
i += 1
metrics = self.calc_metrics(epoch_loss, epoch_norm)
return metrics
