def load_xps(save_as): """Load `xps` (as a simple list) from dir.""" save_as = Path(save_as).expanduser() files = [d / "xp" for d in uplink.list_job_dirs(save_as)] def load_any(filepath): """Load any/all `xp's` from `filepath`.""" with open(filepath, "rb") as F: # If experiment crashed, then xp will be empty try: data = dill.load(F) except EOFError: return [] # Always return list try: return data["xps"] except KeyError: return [data["xp"]] print("Loading %d files from %s" % (len(files), save_as)) xps = [] # NB: progbar wont clean up properly w/ list compr. for f in tqdm(files, desc="Loading"): xps.extend(load_any(f)) if len(xps) < len(files): print(f"{len(files)-len(xps)} files could not be loaded.") return xps
def load_xps(save_as): """Load `xps` (as a `list`) from given dir.""" save_as = Path(save_as).expanduser() files = [d / "xp" for d in uplink.list_job_dirs(save_as)] if not files: raise FileNotFoundError(f"No results found at {save_as}.") def load_any(filepath): """Load any/all `xp's` from `filepath`.""" with open(filepath, "rb") as F: # If experiment crashed, then xp will be empty try: data = dill.load(F) except EOFError: return [] # Always return list try: return data["xps"] except KeyError: return [data["xp"]] print("Loading %d files from %s" % (len(files), save_as)) xps = [] # NB: progbar wont clean up properly w/ list compr. for f in tqdm(files, desc="Loading"): xps.extend(load_any(f)) if len(xps) == 0: raise RuntimeError("No completed experiments found.") elif len(xps) < len(files): print( len(files) - len(xps), "files could not be loaded,", "presumably because their respective jobs crashed.") return xps
def overwrite_xps(xps, save_as, nDir=100): """Save xps in save_as, but safely (by first saving to tmp).""" save_xps(xps, save_as / "tmp", nDir) # Delete for d in tqdm(uplink.list_job_dirs(save_as), desc="Deleting old"): shutil.rmtree(d) # Mv up from tmp/ -- goes quick, coz there are not many. for d in os.listdir(save_as / "tmp"): shutil.move(save_as / "tmp" / d, save_as / d) shutil.rmtree(save_as / "tmp")