def main(xargs):
R.reset_default_uri(xargs.save_dir)
experiments = R.list_experiments()
key_map = {
"IC": "IC",
"ICIR": "ICIR",
"Rank IC": "Rank_IC",
"Rank ICIR": "Rank_ICIR",
"excess_return_with_cost.annualized_return": "Annualized_Return",
"excess_return_with_cost.information_ratio": "Information_Ratio",
"excess_return_with_cost.max_drawdown": "Max_Drawdown",
}
all_keys = list(key_map.values())
print("There are {:} experiments.".format(len(experiments)))
head_strs, value_strs, names = [], [], []
for idx, (key, experiment) in enumerate(experiments.items()):
if experiment.id == "0":
continue
recorders = experiment.list_recorders()
recorders, not_finished = filter_finished(recorders)
print(
"====>>>> {:02d}/{:02d}-th experiment {:9s} has {:02d}/{:02d} finished recorders."
.format(idx, len(experiments), experiment.name, len(recorders),
len(recorders) + not_finished))
result = QResult()
for recorder_id, recorder in recorders.items():
result.update(recorder.list_metrics(), key_map)
head_str, value_str = result.info(all_keys, show=False)
head_strs.append(head_str)
value_strs.append(value_str)
names.append(experiment.name)
compare_results(head_strs, value_strs, names, space=10)
def query_info(save_dir, verbose, name_filter, key_map):
if isinstance(save_dir, list):
results = []
for x in save_dir:
x = query_info(x, verbose, name_filter, key_map)
results.extend(x)
return results
# Here, the save_dir must be a string
R.set_uri(str(save_dir))
experiments = R.list_experiments()
if verbose:
print("There are {:} experiments.".format(len(experiments)))
qresults = []
for idx, (key, experiment) in enumerate(experiments.items()):
if experiment.id == "0":
continue
if (name_filter is not None
and re.fullmatch(name_filter, experiment.name) is None):
continue
recorders = experiment.list_recorders()
recorders, not_finished = filter_finished(recorders)
if verbose:
print(
"====>>>> {:02d}/{:02d}-th experiment {:9s} has {:02d}/{:02d} finished recorders."
.format(
idx + 1,
len(experiments),
experiment.name,
len(recorders),
len(recorders) + not_finished,
))
result = QResult(experiment.name)
for recorder_id, recorder in recorders.items():
file_names = [
"results-train.pkl", "results-valid.pkl", "results-test.pkl"
]
date2IC = OrderedDict()
for file_name in file_names:
xtemp = recorder.load_object(file_name)["all-IC"]
timestamps, values = xtemp.index.tolist(), xtemp.tolist()
for timestamp, value in zip(timestamps, values):
add_to_dict(date2IC, timestamp, value)
result.update(recorder.list_metrics(), key_map)
result.append_path(
os.path.join(recorder.uri, recorder.experiment_id,
recorder.id))
result.append_date2ICs(date2IC)
if not len(result):
print("There are no valid recorders for {:}".format(experiment))
continue
else:
if verbose:
print("There are {:} valid recorders for {:}".format(
len(recorders), experiment.name))
qresults.append(result)
return qresults
def query_info(save_dir, verbose, name_filter):
R.set_uri(save_dir)
experiments = R.list_experiments()
key_map = {
# "RMSE": "RMSE",
"IC": "IC",
"ICIR": "ICIR",
"Rank IC": "Rank_IC",
"Rank ICIR": "Rank_ICIR",
# "excess_return_with_cost.annualized_return": "Annualized_Return",
# "excess_return_with_cost.information_ratio": "Information_Ratio",
# "excess_return_with_cost.max_drawdown": "Max_Drawdown",
}
all_keys = list(key_map.values())
if verbose:
print("There are {:} experiments.".format(len(experiments)))
head_strs, value_strs, names = [], [], []
for idx, (key, experiment) in enumerate(experiments.items()):
if experiment.id == "0":
continue
if name_filter is not None and re.match(name_filter,
experiment.name) is None:
continue
recorders = experiment.list_recorders()
recorders, not_finished = filter_finished(recorders)
if verbose:
print(
"====>>>> {:02d}/{:02d}-th experiment {:9s} has {:02d}/{:02d} finished recorders."
.format(
idx + 1,
len(experiments),
experiment.name,
len(recorders),
len(recorders) + not_finished,
))
result = QResult()
for recorder_id, recorder in recorders.items():
result.update(recorder.list_metrics(), key_map)
if not len(result):
print("There are no valid recorders for {:}".format(experiment))
continue
else:
print("There are {:} valid recorders for {:}".format(
len(recorders), experiment.name))
head_str, value_str = result.info(all_keys, verbose=verbose)
head_strs.append(head_str)
value_strs.append(value_str)
names.append(experiment.name)
info_str_dict = compare_results(head_strs,
value_strs,
names,
space=10,
verbose=verbose)
info_value_dict = dict(heads=head_strs, values=value_strs, names=names)
return info_str_dict, info_value_dict
def query_info(save_dir, verbose, name_filter, key_map):
R.set_uri(save_dir)
experiments = R.list_experiments()
if verbose:
print("There are {:} experiments.".format(len(experiments)))
qresults = []
for idx, (key, experiment) in enumerate(experiments.items()):
if experiment.id == "0":
continue
if (name_filter is not None
and re.fullmatch(name_filter, experiment.name) is None):
continue
recorders = experiment.list_recorders()
recorders, not_finished = filter_finished(recorders)
if verbose:
print(
"====>>>> {:02d}/{:02d}-th experiment {:9s} has {:02d}/{:02d} finished recorders."
.format(
idx + 1,
len(experiments),
experiment.name,
len(recorders),
len(recorders) + not_finished,
))
result = QResult(experiment.name)
for recorder_id, recorder in recorders.items():
result.update(recorder.list_metrics(), key_map)
result.append_path(
os.path.join(recorder.uri, recorder.experiment_id,
recorder.id))
if not len(result):
print("There are no valid recorders for {:}".format(experiment))
continue
else:
print("There are {:} valid recorders for {:}".format(
len(recorders), experiment.name))
qresults.append(result)
return qresults
