def restore(cls, directory, new_stop=None, context={}):
logger.info("Starting run restore...")
directory = Path(directory)
backup_tape = directory / "bak-tape.son"
(directory / "tape.son").rename(backup_tape)
old_tape = Tape.restore(backup_tape)
runner_config = old_tape.metadata["run"]
if new_stop is not None:
runner_config["stop"] = to_config(new_stop)
run = Run.from_config(runner_config, context=context)
new_tape = Tape.new(metadata=run.get_config(),
filename=directory / "tape.son")
evals = ResultDB()
state = State.from_tape(
search=run.search, tape=old_tape, evals=evals, new_tape=new_tape
) # now we have successfully replayed the optimisation so far
run._prepare(directory, evals, state, msg="Recovered")
return run
def __call__(self, f, variables):
start = time.monotonic()
qmmlpack = import_qmmlpack("run local grid search")
logger.info("Starting local grid search with {} variables.".format(
len(variables)))
result = qmmlpack.local_grid_search(
f=f,
variables=variables,
evalmonitor=log_during_eval,
resolution=self.resolution,
maxevals=self.maxevals,
rng=self.rng,
)
end = time.monotonic()
result["duration"] = end - start
result["best"] = result["best_valpow"]
logger.info(
f"Ended local grid search with loss {result['best_f']} at {result['best']}."
)
return result
def shutdown(self):
self.pool.stop() # no point in waiting for things
try:
self.pool.join(timeout=1.0)
logger.info("Successfully and peacefully shut down pool.")
except TimeoutError:
logger.info(
"Failed to peacefully shut down pool... but no worries.")
def write_results(self):
for i, config in enumerate(self.state.evals.top_suggestions()):
save_yaml(self.work_directory / f"suggestion-{i}", config)
logger.info(f"Saved top 5 suggestions into {self.work_directory}.")
refined = self.state.evals.top_refined_suggestions()
if any([r != {} for r in refined]):
for i, config in enumerate(
self.state.evals.top_refined_suggestions()):
save_yaml(self.work_directory / f"refined_suggestion-{i}",
config)
logger.info(
f"Saved top 5 refined suggestions into {self.work_directory}.")
def _prepare(self, work_directory, evals, state, msg="Prepared"):
self.work_directory = work_directory
self.pool = EvaluationPool(
evals=evals,
max_workers=self.context["max_workers"],
evaluator_config=self.evaluator_config,
evaluator_context=self.context,
trial_timeout=self.trial_timeout,
caught_exceptions=self.caught_exceptions,
)
self.state = state
logger.addHandler(FileHandler(f"{self.work_directory}/log.log"))
logger.setLevel(INFO)
logger.info(
f"{msg} runner {self.name} in folder {self.work_directory}.")
self.ready = True
def run(self, duration=float("inf")):
"""Run for duration minutes."""
assert self.ready, "prepare() must be called before starting run."
duration = (duration * 60.0) - self.context["shutdown_duration"]
start = time.monotonic()
end = start + duration
futures = {}
# when recovering, first re-submit the running trials
if len(self.state.live_trials) > 0:
logger.info(
f"Re-submitting {len(self.state.live_trials)} trials to the pool."
)
for tid, suggestion in self.state.live_trials.items():
f = self.pool.schedule(suggestion)
futures[f] = tid
while time.monotonic() < end and not self.stop.done(self.state):
status = self.write_status("Running.", len(futures),
time.monotonic() - start, duration)
logger.info(status)
done, running = wait(
futures,
timeout=self.context["wait_per_loop"],
return_when=FIRST_COMPLETED,
)
if not self.stop.done(self.state):
for f in done:
tid = futures[f]
result = self.pool.finish(f)
self.state.submit(tid, result)
del futures[f]
n_new_trials = max(
0, self.context["max_workers"] + 1 - len(running))
for i in range(n_new_trials):
tid, suggestion = self.state.suggest()
f = self.pool.schedule(suggestion)
futures[f] = tid
runtime = time.monotonic() - start
logger.info(
f"Finished run {self.name} in {runtime:.2f}s. Starting shutdown..."
)
self.write_status(f"{self.name}: Done, saving results.", 0, runtime,
duration)
self.write_results()
self.write_status(f"{self.name}: Done, initiating shutdown.", 0,
runtime, duration)
self.pool.shutdown()
self.write_status(f"{self.name}: Done. Have a good day!", 0, runtime,
duration)
def checkout(cls, directory):
"""Get read-only run from directory.
This is the canonical way of obtaining the results of a
finished (or, inadvisably, an ongoing) run. It returns a
Run instance that cannot be run(), but is otherwise in the
same state as the run that is being checked out.
"""
directory = Path(directory)
logger.info(
"Starting run checkout... (this will not yield a runnable instance)."
)
original_tape = Tape.restore(directory / "tape.son")
run = Run.from_config(original_tape.metadata["run"])
state = State.from_tape(search=run.search, tape=original_tape)
run.state = state
run.readonly = True
return run