def _get_info_from_the_folder(folder_path):
print("process the folder: {}".format(folder_path))
arguments_path = os.path.join(folder_path, "arguments.pickle")
# collect runtime json info for one rank.
sub_folder_paths = sorted([
sub_folder_path for sub_folder_path in list_files(folder_path)
if ".tar" not in sub_folder_path and "pickle" not in sub_folder_path
])
# return the information.
return (
folder_path,
{
"arguments":
_get_arguments(load_pickle(arguments_path)),
# single worker records.
"single_records":
_parse_runtime_infos(os.path.join(sub_folder_paths[0])),
# multiple workers records.
"multi_records": [
_parse_runtime_infos(os.path.join(sub_folder_path))
for sub_folder_path in sub_folder_paths
],
},
)
def _get_info_from_the_folder(folder_path):
print("process the folder: {}".format(folder_path))
arguments_path = os.path.join(folder_path, "arguments.pickle")
# collect runtime json info for one rank.
sub_folder_paths = sorted(
[
sub_folder_path
for sub_folder_path in list_files(folder_path)
if ".tar" not in sub_folder_path and "pickle" not in sub_folder_path
]
)
# return the information.
return (
folder_path,
{
"arguments": _get_arguments(load_pickle(arguments_path)),
# single worker records.
"single_records": _parse_runtime_info(
os.path.join(sub_folder_paths[0], "log.json")
),
# records w.r.t. averaged model
"averaged_records": _parse_averaged_info(
[
os.path.join(sub_folder_path, "log.json")
for sub_folder_path in sub_folder_paths
]
),
},
)
def load_raw_info_from_experiments(root_path):
"""load experiments.
root_path: a directory with a list of different trials.
"""
exp_folder_paths = [
folder_path for folder_path in list_files(root_path)
if "pickle" not in folder_path
]
info = []
for folder_path in exp_folder_paths:
try:
element_of_info = _get_info_from_the_folder(folder_path)
info.append(element_of_info)
except Exception as e:
print("error: {}".format(e))
return info
def _get_info_from_the_folder(folder_path):
print("process the folder: {}".format(folder_path))
arguments_path = os.path.join(folder_path, "arguments.json")
# collect runtime json info for one rank.
sub_folder_paths = sorted([
sub_folder_path for sub_folder_path in list_files(folder_path)
if ".tar" not in sub_folder_path and "pickle" not in sub_folder_path
])
# return the information.
return (
folder_path,
{
"arguments": read_json(arguments_path),
"single_records": _parse_runtime_infos(sub_folder_paths[0]),
},
)