def git_info(ctx):
sys.path.insert(0, os.path.join(APP_ROOT, "nbviewer"))
try:
from utils import git_info, GIT_INFO_JSON
info = git_info(APP_ROOT, force_git=True)
except Exception as e:
print("Failed to get git info", e)
return
print("Writing git info to %s" % GIT_INFO_JSON)
with open(GIT_INFO_JSON, "w") as f:
json.dump(info, f)
sys.path.pop(0)
if any(l < 0 for l in session.losses.values()) or all(
l == 0 for l in session.losses.values()):
raise ValueError(f'Invalid losses: {session.losses}')
if len(experiment.sessions) > 0 and ('state_dict' not in experiment.model or
'state_dict' not in experiment.optimizer):
raise ValueError(
f'Model and optimizer state dicts are required to restore training')
# Experiment computed fields
experiment.epoch = sum((s.epochs for s in experiment.sessions), 0)
# Session computed fields
session.status = 'NEW'
session.datetime_started = None
session.datetime_completed = None
git = git_info()
if git is not None:
session.git = git
if 'cuda' in session.device:
session.cuda = cuda_info()
# Resolving paths
rand_id = ''.join(chr(random.randint(ord('A'), ord('Z'))) for _ in range(6))
session.data.path = Path(session.data.path.replace(
'{name}', experiment.name)).expanduser().resolve().as_posix()
session.log.folder = session.log.folder \
.replace('{name}', experiment.name) \
.replace('{tags}', '_'.join(experiment.tags)) \
.replace('{rand}', rand_id)
if len(session.checkpoint.when) > 0:
if len(session.log.when) > 0: