def __init__(self, denominator, max_width=None, eta_every=1):
self._eta = ETA(denominator=denominator)
self.max_width = max_width
self.eta_every = eta_every
self.force_done = False
self._eta_string = ''
self._eta_count = 1
def test_linear_slope_1():
eta = ETA(100)
eta._timing_data = deque([(10, 10), (20, 20), (30, 30), (40, 40)])
getattr(eta, '_calculate')()
assert 100 == eta.eta_epoch
assert 1.0 == eta.rate
assert 1.0 == eta.rate_unstable
def test_linear_transform_undefined():
eta = ETA()
eta._timing_data = deque([(1.2, 22), (2.4, 58), (3.1, 102), (4.4, 118)])
getattr(eta, '_calculate')()
assert eta.eta_epoch is None
assert 30 < eta.rate < 35
assert 12 < eta.rate_unstable < 13
def test_linear_transform():
"""Wolfram Alpha:
x is the timestamp. y is the numerator. 120 is the denominator.
linear fit {1.2, 22},{2.4, 58},{3.1, 102},{4.4, 118}
The closer we get to 100%, the more vertical shift/transform is applied to the line.
As we near the end we want the line to get closer to the last point on the graph.
This avoids having 99% with an ETA in the past.
"""
eta = ETA(120)
eta._timing_data = deque([(1.2, 22), (2.4, 58), (3.1, 102), (4.4, 118)])
getattr(eta, '_calculate')()
assert 4.4 < eta.eta_epoch < 4.6
assert 30 < eta.rate < 35
assert 12 < eta.rate_unstable < 13
def gather_results(self, workers, log_queue, test_split=False):
'''
check for logs while waiting for workers
'''
results = {}
eval_type = 'subgoal' if self.args.subgoals else 'task'
lock = filelock.FileLock(self.results_path + '.lock')
eta = ETA(self.num_trials, scope=32)
while True:
if log_queue.qsize() > 0:
# there is a new log entry available, process it
log_entry, trial_uid, model_path = log_queue.get()
# load old results (if available)
with lock:
if os.path.exists(self.results_path):
with open(self.results_path, 'r') as results_file:
results = json.load(results_file)
eval_epoch = os.path.basename(model_path)
# update the old results with the new log entry
if eval_epoch not in results:
results[eval_epoch] = {}
if eval_type not in results[eval_epoch]:
results[eval_epoch][eval_type] = {}
if trial_uid in results[eval_epoch][eval_type] and not test_split:
success_prev = results[eval_epoch][eval_type][trial_uid]['success']
success_curr = log_entry['success']
if success_prev != success_curr:
print(colored(
'WARNING: trial {} result has changed from {} to {}'.format(
trial_uid,
'success' if success_prev else 'fail',
'success' if success_curr else 'fail'), 'yellow'))
results[eval_epoch][eval_type][trial_uid] = log_entry
# print updated results
self.num_trials_done += 1
eta.numerator = self.num_trials_done
if not test_split:
successes = [
log['success'] for log in results[eval_epoch][eval_type].values()]
print(colored(
'{:4d}/{} trials are done (current SR = {:.1f}), ETA = {}, elapsed = {}'.format(
self.num_trials_done,
self.num_trials,
100 * sum(successes) / len(successes),
time.strftime('%H:%M:%S', time.gmtime(eta.eta_seconds)),
time.strftime('%H:%M:%S', time.gmtime(eta.elapsed))),
'green'))
# make a backup copy of results file before writing
eval_util.save_with_backup(results, self.results_path, lock)
# update info.json file
model_util.update_log(
self.args.dout, stage='eval',
update='increase', progress=1)
# check whether all workers have exited (exitcode == None means they are still running)
all_terminated = all([worker.exitcode is not None for worker in workers])
if all_terminated and log_queue.qsize() == 0:
if self.num_trials_left > 0:
print(colored('WARNING: only {}/{} trials were evaluated'.format(
self.num_trials_done, self.num_trials), 'red'))
# our mission is over
break
time.sleep(1)
print(colored('Evaluation is complete', 'green'))
