def load_xps(save_as):
"""Load `xps` (as a simple list) from dir."""
save_as = Path(save_as).expanduser()
files = [d / "xp" for d in uplink.list_job_dirs(save_as)]
def load_any(filepath):
"""Load any/all `xp's` from `filepath`."""
with open(filepath, "rb") as F:
# If experiment crashed, then xp will be empty
try:
data = dill.load(F)
except EOFError:
return []
# Always return list
try:
return data["xps"]
except KeyError:
return [data["xp"]]
print("Loading %d files from %s" % (len(files), save_as))
xps = [] # NB: progbar wont clean up properly w/ list compr.
for f in tqdm(files, desc="Loading"):
xps.extend(load_any(f))
if len(xps) < len(files):
print(f"{len(files)-len(xps)} files could not be loaded.")
return xps
def load_xps(save_as):
"""Load `xps` (as a `list`) from given dir."""
save_as = Path(save_as).expanduser()
files = [d / "xp" for d in uplink.list_job_dirs(save_as)]
if not files:
raise FileNotFoundError(f"No results found at {save_as}.")
def load_any(filepath):
"""Load any/all `xp's` from `filepath`."""
with open(filepath, "rb") as F:
# If experiment crashed, then xp will be empty
try:
data = dill.load(F)
except EOFError:
return []
# Always return list
try:
return data["xps"]
except KeyError:
return [data["xp"]]
print("Loading %d files from %s" % (len(files), save_as))
xps = [] # NB: progbar wont clean up properly w/ list compr.
for f in tqdm(files, desc="Loading"):
xps.extend(load_any(f))
if len(xps) == 0:
raise RuntimeError("No completed experiments found.")
elif len(xps) < len(files):
print(
len(files) - len(xps), "files could not be loaded,",
"presumably because their respective jobs crashed.")
return xps
def overwrite_xps(xps, save_as, nDir=100):
"""Save xps in save_as, but safely (by first saving to tmp)."""
save_xps(xps, save_as / "tmp", nDir)
# Delete
for d in tqdm(uplink.list_job_dirs(save_as), desc="Deleting old"):
shutil.rmtree(d)
# Mv up from tmp/ -- goes quick, coz there are not many.
for d in os.listdir(save_as / "tmp"):
shutil.move(save_as / "tmp" / d, save_as / d)
shutil.rmtree(save_as / "tmp")