def save_training_meta(args):
# Comment out, since rank is not saved to args. Safeguard save_training_meta already in training scripts.
# if args.rank > 0:
# return
# args is an EasyDict object, treat it the same as a normal dict
os.makedirs(join(args.output_dir, 'log'), exist_ok=True)
os.makedirs(join(args.output_dir, 'ckpt'), exist_ok=True)
# training args
save_args_path = join(args.output_dir, 'log', 'hps.json')
save_json(vars(args), save_args_path, save_pretty=True)
# model args
model_config = load_json(args.model_config)
save_model_config_path = join(args.output_dir, 'log', 'model_config.json')
save_json(model_config, save_model_config_path, save_pretty=True)
# git info
try:
LOGGER.info("Waiting on git info....")
c = subprocess.run(["git", "rev-parse", "--abbrev-ref", "HEAD"],
timeout=10,
stdout=subprocess.PIPE)
git_branch_name = c.stdout.decode().strip()
LOGGER.info("Git branch: %s", git_branch_name)
c = subprocess.run(["git", "rev-parse", "HEAD"],
timeout=10,
stdout=subprocess.PIPE)
git_sha = c.stdout.decode().strip()
LOGGER.info("Git SHA: %s", git_sha)
git_dir = abspath(dirname(__file__))
git_status = subprocess.check_output(['git', 'status', '--short'],
cwd=git_dir,
universal_newlines=True).strip()
with open(join(args.output_dir, 'log', 'git_info.json'),
'w') as writer:
json.dump(
{
'branch': git_branch_name,
'is_dirty': bool(git_status),
'status': git_status,
'sha': git_sha
},
writer,
indent=4)
except (subprocess.TimeoutExpired, subprocess.CalledProcessError) as e:
LOGGER.exception(e)
LOGGER.warn("Git info not found. Saving code into zip instead...")
# save a copy of the codebase.
# !!!Do not store heavy file in your codebase when using it.
code_dir = dirname(dirname(realpath(__file__)))
code_zip_filename = os.path.join(args.output_dir, "code.zip")
LOGGER.info(f"Saving code from {code_dir} to {code_zip_filename}...")
make_zipfile(code_dir,
code_zip_filename,
enclosing_dir="code",
exclude_dirs_substring="results",
exclude_dirs=["results", "debug_results", "__pycache__"],
exclude_extensions=[".pyc", ".ipynb", ".swap"])
LOGGER.info("Saving code done.")
def obtain_system_token():
LOGGER.info('Trying to read token from file: %s', CONFIG.TOKEN_FILE)
token_file = open(CONFIG.TOKEN_FILE, 'r')
file_content = token_file.read().splitlines()
if len(file_content) >= 2 and file_content[0] == TOKEN_START and file_content[2] == TOKEN_END:
LOGGER.info('Reading token from file succeeded')
return file_content[1]
else:
LOGGER.warn('Reading token from file failed')
return aquire_token()
def save_training_meta(args):
if args.rank > 0:
return
if not exists(args.output_dir):
os.makedirs(join(args.output_dir, "log"))
os.makedirs(join(args.output_dir, "ckpt"))
with open(join(args.output_dir, "log", "hps.json"), "w") as writer:
json.dump(vars(args), writer, indent=4)
model_config = json.load(open(args.model_config))
with open(join(args.output_dir, "log", "model.json"), "w") as writer:
json.dump(model_config, writer, indent=4)
# git info
try:
LOGGER.info("Waiting on git info....")
c = subprocess.run(
["git", "rev-parse", "--abbrev-ref", "HEAD"],
timeout=10,
stdout=subprocess.PIPE,
)
git_branch_name = c.stdout.decode().strip()
LOGGER.info("Git branch: %s", git_branch_name)
c = subprocess.run(
["git", "rev-parse", "HEAD"], timeout=10, stdout=subprocess.PIPE
)
git_sha = c.stdout.decode().strip()
LOGGER.info("Git SHA: %s", git_sha)
git_dir = abspath(dirname(__file__))
git_status = subprocess.check_output(
["git", "status", "--short"], cwd=git_dir, universal_newlines=True
).strip()
with open(join(args.output_dir, "log", "git_info.json"), "w") as writer:
json.dump(
{
"branch": git_branch_name,
"is_dirty": bool(git_status),
"status": git_status,
"sha": git_sha,
},
writer,
indent=4,
)
except subprocess.TimeoutExpired as e:
LOGGER.exception(e)
LOGGER.warn("Git info not found. Moving right along...")
def save_training_meta(args):
if args.rank > 0:
return
if not exists(args.output_dir):
os.makedirs(join(args.output_dir, 'log'))
os.makedirs(join(args.output_dir, 'ckpt'))
with open(join(args.output_dir, 'log', 'hps.json'), 'w') as writer:
json.dump(vars(args), writer, indent=4)
model_config = json.load(open(args.model_config))
with open(join(args.output_dir, 'log', 'model.json'), 'w') as writer:
json.dump(model_config, writer, indent=4)
return # no need to store git info
# git info
try:
LOGGER.info("Waiting on git info....")
c = subprocess.run(["git", "rev-parse", "--abbrev-ref", "HEAD"],
timeout=10, stdout=subprocess.PIPE)
git_branch_name = c.stdout.decode().strip()
LOGGER.info("Git branch: %s", git_branch_name)
c = subprocess.run(["git", "rev-parse", "HEAD"],
timeout=10, stdout=subprocess.PIPE)
git_sha = c.stdout.decode().strip()
LOGGER.info("Git SHA: %s", git_sha)
git_dir = abspath(dirname(__file__))
git_status = subprocess.check_output(
['git', 'status', '--short'],
cwd=git_dir, universal_newlines=True).strip()
with open(join(args.output_dir, 'log', 'git_info.json'),
'w') as writer:
json.dump({'branch': git_branch_name,
'is_dirty': bool(git_status),
'status': git_status,
'sha': git_sha},
writer, indent=4)
except subprocess.TimeoutExpired as e:
LOGGER.exception(e)
LOGGER.warn("Git info not found. Moving right along...")
