def manage_best_checkpoints(self, step, score):
score_filename = os.path.join(self.checkpoint_dir, 'scores.txt')
# try loading previous scores
try:
with open(score_filename) as f:
# list of pairs (score, step)
scores = [(float(line.split()[0]), int(line.split()[1]))
for line in f]
except IOError:
scores = []
if any(step_ >= step for _, step_ in scores):
utils.warn('inconsistent scores.txt file')
best_scores = sorted(scores,
reverse=not self.reversed_scores)[:self.keep_best]
def full_path(filename):
return os.path.join(self.checkpoint_dir, filename)
lower = lambda x, y: y < x if self.reversed_scores else lambda x, y: x < y
if any(lower(score_, score)
for score_, _ in best_scores) or not best_scores:
# if this checkpoint is in the top, save it under a special name
prefix = 'translate-{}'.format(step)
dest_prefix = 'best-{}'.format(step)
for filename in os.listdir(self.checkpoint_dir):
if filename.startswith(prefix):
dest_filename = filename.replace(prefix, dest_prefix)
shutil.copy(full_path(filename), full_path(dest_filename))
# FIXME (wrong `best`)
# also copy to `best` if this checkpoint is the absolute best
if all(lower(score_, score) for score_, _ in best_scores):
dest_filename = filename.replace(prefix, 'best')
shutil.copy(full_path(filename),
full_path(dest_filename))
best_scores = sorted(best_scores + [(score, step)],
reverse=not self.reversed_scores)
for _, step_ in best_scores[self.keep_best:]:
# remove checkpoints that are not in the top anymore
prefix = 'best-{}'.format(step_)
for filename in os.listdir(self.checkpoint_dir):
if filename.startswith(prefix):
os.remove(full_path(filename))
# save bleu scores
scores.append((score, step))
with open(score_filename, 'w') as f:
for score_, step_ in scores:
f.write('{:.2f} {}\n'.format(score_, step_))
def manage_best_checkpoints(self, step, score):
score_filename = os.path.join(self.checkpoint_dir, 'scores.txt')
# try loading previous scores
try:
with open(score_filename) as f:
# list of pairs (score, step)
scores = [(float(line.split()[0]), int(line.split()[1]))
for line in f]
except IOError:
scores = []
if any(step_ >= step for _, step_ in scores):
utils.warn('inconsistent scores.txt file')
best_scores = sorted(scores, reverse=True)[:self.keep_best]
if any(score_ < score for score_, _ in best_scores) or not best_scores:
shutil.copy(
os.path.join(self.checkpoint_dir, 'translate-{}'.format(step)),
os.path.join(self.checkpoint_dir, 'best-{}'.format(step)))
if all(score_ < score for score_, _ in best_scores):
path = os.path.abspath(
os.path.join(self.checkpoint_dir, 'best'))
try: # remove old links
os.remove(path)
except OSError:
pass
# make symbolic links to best model
os.symlink('{}-{}'.format(path, step), path)
best_scores = sorted(best_scores + [(score, step)], reverse=True)
for _, step_ in best_scores[self.keep_best:]:
# remove checkpoints that are not in the top anymore
try:
os.remove(
os.path.join(self.checkpoint_dir,
'best-{}'.format(step_)))
except OSError:
pass
# save bleu scores
scores.append((score, step))
with open(score_filename, 'w') as f:
for score_, step_ in scores:
f.write('{} {}\n'.format(score_, step_))