def save_training_meta(args): # Comment out, since rank is not saved to args. Safeguard save_training_meta already in training scripts. # if args.rank > 0: # return # args is an EasyDict object, treat it the same as a normal dict os.makedirs(join(args.output_dir, 'log'), exist_ok=True) os.makedirs(join(args.output_dir, 'ckpt'), exist_ok=True) # training args save_args_path = join(args.output_dir, 'log', 'hps.json') save_json(vars(args), save_args_path, save_pretty=True) # model args model_config = load_json(args.model_config) save_model_config_path = join(args.output_dir, 'log', 'model_config.json') save_json(model_config, save_model_config_path, save_pretty=True) # git info try: LOGGER.info("Waiting on git info....") c = subprocess.run(["git", "rev-parse", "--abbrev-ref", "HEAD"], timeout=10, stdout=subprocess.PIPE) git_branch_name = c.stdout.decode().strip() LOGGER.info("Git branch: %s", git_branch_name) c = subprocess.run(["git", "rev-parse", "HEAD"], timeout=10, stdout=subprocess.PIPE) git_sha = c.stdout.decode().strip() LOGGER.info("Git SHA: %s", git_sha) git_dir = abspath(dirname(__file__)) git_status = subprocess.check_output(['git', 'status', '--short'], cwd=git_dir, universal_newlines=True).strip() with open(join(args.output_dir, 'log', 'git_info.json'), 'w') as writer: json.dump( { 'branch': git_branch_name, 'is_dirty': bool(git_status), 'status': git_status, 'sha': git_sha }, writer, indent=4) except (subprocess.TimeoutExpired, subprocess.CalledProcessError) as e: LOGGER.exception(e) LOGGER.warn("Git info not found. Saving code into zip instead...") # save a copy of the codebase. # !!!Do not store heavy file in your codebase when using it. code_dir = dirname(dirname(realpath(__file__))) code_zip_filename = os.path.join(args.output_dir, "code.zip") LOGGER.info(f"Saving code from {code_dir} to {code_zip_filename}...") make_zipfile(code_dir, code_zip_filename, enclosing_dir="code", exclude_dirs_substring="results", exclude_dirs=["results", "debug_results", "__pycache__"], exclude_extensions=[".pyc", ".ipynb", ".swap"]) LOGGER.info("Saving code done.")
def obtain_system_token(): LOGGER.info('Trying to read token from file: %s', CONFIG.TOKEN_FILE) token_file = open(CONFIG.TOKEN_FILE, 'r') file_content = token_file.read().splitlines() if len(file_content) >= 2 and file_content[0] == TOKEN_START and file_content[2] == TOKEN_END: LOGGER.info('Reading token from file succeeded') return file_content[1] else: LOGGER.warn('Reading token from file failed') return aquire_token()
def save_training_meta(args): if args.rank > 0: return if not exists(args.output_dir): os.makedirs(join(args.output_dir, "log")) os.makedirs(join(args.output_dir, "ckpt")) with open(join(args.output_dir, "log", "hps.json"), "w") as writer: json.dump(vars(args), writer, indent=4) model_config = json.load(open(args.model_config)) with open(join(args.output_dir, "log", "model.json"), "w") as writer: json.dump(model_config, writer, indent=4) # git info try: LOGGER.info("Waiting on git info....") c = subprocess.run( ["git", "rev-parse", "--abbrev-ref", "HEAD"], timeout=10, stdout=subprocess.PIPE, ) git_branch_name = c.stdout.decode().strip() LOGGER.info("Git branch: %s", git_branch_name) c = subprocess.run( ["git", "rev-parse", "HEAD"], timeout=10, stdout=subprocess.PIPE ) git_sha = c.stdout.decode().strip() LOGGER.info("Git SHA: %s", git_sha) git_dir = abspath(dirname(__file__)) git_status = subprocess.check_output( ["git", "status", "--short"], cwd=git_dir, universal_newlines=True ).strip() with open(join(args.output_dir, "log", "git_info.json"), "w") as writer: json.dump( { "branch": git_branch_name, "is_dirty": bool(git_status), "status": git_status, "sha": git_sha, }, writer, indent=4, ) except subprocess.TimeoutExpired as e: LOGGER.exception(e) LOGGER.warn("Git info not found. Moving right along...")
def save_training_meta(args): if args.rank > 0: return if not exists(args.output_dir): os.makedirs(join(args.output_dir, 'log')) os.makedirs(join(args.output_dir, 'ckpt')) with open(join(args.output_dir, 'log', 'hps.json'), 'w') as writer: json.dump(vars(args), writer, indent=4) model_config = json.load(open(args.model_config)) with open(join(args.output_dir, 'log', 'model.json'), 'w') as writer: json.dump(model_config, writer, indent=4) return # no need to store git info # git info try: LOGGER.info("Waiting on git info....") c = subprocess.run(["git", "rev-parse", "--abbrev-ref", "HEAD"], timeout=10, stdout=subprocess.PIPE) git_branch_name = c.stdout.decode().strip() LOGGER.info("Git branch: %s", git_branch_name) c = subprocess.run(["git", "rev-parse", "HEAD"], timeout=10, stdout=subprocess.PIPE) git_sha = c.stdout.decode().strip() LOGGER.info("Git SHA: %s", git_sha) git_dir = abspath(dirname(__file__)) git_status = subprocess.check_output( ['git', 'status', '--short'], cwd=git_dir, universal_newlines=True).strip() with open(join(args.output_dir, 'log', 'git_info.json'), 'w') as writer: json.dump({'branch': git_branch_name, 'is_dirty': bool(git_status), 'status': git_status, 'sha': git_sha}, writer, indent=4) except subprocess.TimeoutExpired as e: LOGGER.exception(e) LOGGER.warn("Git info not found. Moving right along...")