def _checkpoint(self, i):
print("Fetching results of step {} at: ".format(i))
print(datetime.datetime.now())
for i, host in enumerate(self.hosts):
if host == ':':
command = "mv {local_scratch}/experiments/* ./experiments"
self.execute_command(command, robust=True)
command = "rm -rf {local_scratch}/experiments"
self.execute_command(command, robust=True)
command = "cp -ru {local_scratch}/{archive_root} ."
self.execute_command(command, robust=True)
else:
command = (
"rsync -az {rsync_verbosity} --timeout=300 -e \"ssh {ssh_options}\" "
"{host}:{local_scratch}/experiments/ ./experiments".format(
host=host, **self.__dict__))
self.execute_command(command,
frmt=False,
robust=True,
output="loud")
command = "rm -rf {local_scratch}/experiments"
self.ssh_execute(command, host, robust=True, output="loud")
command = (
"rsync -az {rsync_verbosity} --timeout=300 -e \"ssh {ssh_options}\" "
"{host}:{local_scratch}/{archive_root} .".format(
host=host, **self.__dict__))
self.execute_command(command,
frmt=False,
robust=True,
output="loud")
self.execute_command("zip -rq results {archive_root}", robust=True)
try:
from dps.hyper import HyperSearch
search = HyperSearch('.')
with redirect_stream('stdout', 'results.txt', tee=False):
search.print_summary(print_config=False, verbose=False)
print(search.job.summary(verbose=False))
except Exception:
job_path = 'results.zip' if os.path.exists(
'results.zip') else 'orig.zip'
assert os.path.exists(job_path)
job = ReadOnlyJob(job_path)
print(job.summary(verbose=False))
def _summarize_search_cmd(path, no_config, verbose, criteria, maximize):
search = HyperSearch(path)
search.print_summary(print_config=not no_config, verbose=verbose, criteria=criteria, maximize=maximize)