def setup_logger( exp_prefix="default", variant=None, text_log_file="debug.log", variant_log_file="variant.json", tabular_log_file="progress.csv", snapshot_mode="last", snapshot_gap=1, log_tabular_only=False, log_dir=None, git_infos=None, script_name=None, **create_log_dir_kwargs ): """ Set up logger to have some reasonable default settings. Will save log output to based_log_dir/exp_prefix/exp_name. exp_name will be auto-generated to be unique. If log_dir is specified, then that directory is used as the output dir. :param exp_prefix: The sub-directory for this specific experiment. :param variant: :param text_log_file: :param variant_log_file: :param tabular_log_file: :param snapshot_mode: :param log_tabular_only: :param snapshot_gap: :param log_dir: :param git_infos: :param script_name: If set, save the script name to this. :return: """ if git_infos is None: git_infos = get_git_infos(conf.CODE_DIRS_TO_MOUNT) first_time = log_dir is None if first_time: log_dir = create_log_dir(exp_prefix, **create_log_dir_kwargs) if variant is not None: logger.log("Variant:") logger.log(json.dumps(dict_to_safe_json(variant), indent=2)) variant_log_path = osp.join(log_dir, variant_log_file) logger.log_variant(variant_log_path, variant) tabular_log_path = osp.join(log_dir, tabular_log_file) text_log_path = osp.join(log_dir, text_log_file) logger.add_text_output(text_log_path) if first_time: logger.add_tabular_output(tabular_log_path) else: logger._add_output(tabular_log_path, logger._tabular_outputs, logger._tabular_fds, mode='a') for tabular_fd in logger._tabular_fds: logger._tabular_header_written.add(tabular_fd) logger.set_snapshot_dir(log_dir) logger.set_snapshot_mode(snapshot_mode) logger.set_snapshot_gap(snapshot_gap) logger.set_log_tabular_only(log_tabular_only) exp_name = log_dir.split("/")[-1] logger.push_prefix("[%s] " % exp_name) if git_infos is not None: for ( directory, code_diff, code_diff_staged, commit_hash, branch_name ) in git_infos: if directory[-1] == '/': directory = directory[:-1] diff_file_name = directory[1:].replace("/", "-") + ".patch" diff_staged_file_name = ( directory[1:].replace("/", "-") + "_staged.patch" ) if code_diff is not None and len(code_diff) > 0: with open(osp.join(log_dir, diff_file_name), "w") as f: f.write(code_diff + '\n') if code_diff_staged is not None and len(code_diff_staged) > 0: with open(osp.join(log_dir, diff_staged_file_name), "w") as f: f.write(code_diff_staged + '\n') with open(osp.join(log_dir, "git_infos.txt"), "a") as f: f.write("directory: {}\n".format(directory)) f.write("git hash: {}\n".format(commit_hash)) f.write("git branch name: {}\n\n".format(branch_name)) if script_name is not None: with open(osp.join(log_dir, "script_name.txt"), "w") as f: f.write(script_name) return log_dir
def setup_logger( exp_prefix="default", exp_id=0, seed=0, variant=None, base_log_dir=None, text_log_file="debug.log", variant_log_file="variant.json", tabular_log_file="progress.csv", snapshot_mode="last", snapshot_gap=1, log_tabular_only=False, log_dir=None, git_info=None, script_name=None, ): """ Set up logger to have some reasonable default settings. Will save log output to based_log_dir/exp_prefix/exp_name. exp_name will be auto-generated to be unique. If log_dir is specified, then that directory is used as the output dir. :param exp_prefix: The sub-directory for this specific experiment. :param exp_id: The number of the specific experiment run within this experiment. :param variant: :param base_log_dir: The directory where all log should be saved. :param text_log_file: :param variant_log_file: :param tabular_log_file: :param snapshot_mode: :param log_tabular_only: :param snapshot_gap: :param log_dir: :param git_info: :param script_name: If set, save the script name to this. :return: """ first_time = log_dir is None if first_time: log_dir = create_log_dir(exp_prefix, exp_id=exp_id, seed=seed, base_log_dir=base_log_dir) if variant is not None: logger.log("Variant:") logger.log(json.dumps(dict_to_safe_json(variant), indent=2)) variant_log_path = osp.join(log_dir, variant_log_file) logger.log_variant(variant_log_path, variant) tabular_log_path = osp.join(log_dir, tabular_log_file) text_log_path = osp.join(log_dir, text_log_file) logger.add_text_output(text_log_path) if first_time: logger.add_tabular_output(tabular_log_path) else: logger._add_output(tabular_log_path, logger._tabular_outputs, logger._tabular_fds, mode='a') for tabular_fd in logger._tabular_fds: logger._tabular_header_written.add(tabular_fd) logger.set_snapshot_dir(log_dir) logger.set_snapshot_mode(snapshot_mode) logger.set_snapshot_gap(snapshot_gap) logger.set_log_tabular_only(log_tabular_only) exp_name = log_dir.split("/")[-1] logger.push_prefix("[%s] " % exp_name) if git_info is not None: code_diff, commit_hash, branch_name = git_info if code_diff is not None: with open(osp.join(log_dir, "code.diff"), "w") as f: f.write(code_diff) with open(osp.join(log_dir, "git_info.txt"), "w") as f: f.write("git hash: {}".format(commit_hash)) f.write('\n') f.write("git branch name: {}".format(branch_name)) if script_name is not None: with open(osp.join(log_dir, "script_name.txt"), "w") as f: f.write(script_name) return log_dir