def main(xargs): R.reset_default_uri(xargs.save_dir) experiments = R.list_experiments() key_map = { "IC": "IC", "ICIR": "ICIR", "Rank IC": "Rank_IC", "Rank ICIR": "Rank_ICIR", "excess_return_with_cost.annualized_return": "Annualized_Return", "excess_return_with_cost.information_ratio": "Information_Ratio", "excess_return_with_cost.max_drawdown": "Max_Drawdown", } all_keys = list(key_map.values()) print("There are {:} experiments.".format(len(experiments))) head_strs, value_strs, names = [], [], [] for idx, (key, experiment) in enumerate(experiments.items()): if experiment.id == "0": continue recorders = experiment.list_recorders() recorders, not_finished = filter_finished(recorders) print( "====>>>> {:02d}/{:02d}-th experiment {:9s} has {:02d}/{:02d} finished recorders." .format(idx, len(experiments), experiment.name, len(recorders), len(recorders) + not_finished)) result = QResult() for recorder_id, recorder in recorders.items(): result.update(recorder.list_metrics(), key_map) head_str, value_str = result.info(all_keys, show=False) head_strs.append(head_str) value_strs.append(value_str) names.append(experiment.name) compare_results(head_strs, value_strs, names, space=10)
def query_info(save_dir, verbose, name_filter, key_map): if isinstance(save_dir, list): results = [] for x in save_dir: x = query_info(x, verbose, name_filter, key_map) results.extend(x) return results # Here, the save_dir must be a string R.set_uri(str(save_dir)) experiments = R.list_experiments() if verbose: print("There are {:} experiments.".format(len(experiments))) qresults = [] for idx, (key, experiment) in enumerate(experiments.items()): if experiment.id == "0": continue if (name_filter is not None and re.fullmatch(name_filter, experiment.name) is None): continue recorders = experiment.list_recorders() recorders, not_finished = filter_finished(recorders) if verbose: print( "====>>>> {:02d}/{:02d}-th experiment {:9s} has {:02d}/{:02d} finished recorders." .format( idx + 1, len(experiments), experiment.name, len(recorders), len(recorders) + not_finished, )) result = QResult(experiment.name) for recorder_id, recorder in recorders.items(): file_names = [ "results-train.pkl", "results-valid.pkl", "results-test.pkl" ] date2IC = OrderedDict() for file_name in file_names: xtemp = recorder.load_object(file_name)["all-IC"] timestamps, values = xtemp.index.tolist(), xtemp.tolist() for timestamp, value in zip(timestamps, values): add_to_dict(date2IC, timestamp, value) result.update(recorder.list_metrics(), key_map) result.append_path( os.path.join(recorder.uri, recorder.experiment_id, recorder.id)) result.append_date2ICs(date2IC) if not len(result): print("There are no valid recorders for {:}".format(experiment)) continue else: if verbose: print("There are {:} valid recorders for {:}".format( len(recorders), experiment.name)) qresults.append(result) return qresults
def query_info(save_dir, verbose, name_filter): R.set_uri(save_dir) experiments = R.list_experiments() key_map = { # "RMSE": "RMSE", "IC": "IC", "ICIR": "ICIR", "Rank IC": "Rank_IC", "Rank ICIR": "Rank_ICIR", # "excess_return_with_cost.annualized_return": "Annualized_Return", # "excess_return_with_cost.information_ratio": "Information_Ratio", # "excess_return_with_cost.max_drawdown": "Max_Drawdown", } all_keys = list(key_map.values()) if verbose: print("There are {:} experiments.".format(len(experiments))) head_strs, value_strs, names = [], [], [] for idx, (key, experiment) in enumerate(experiments.items()): if experiment.id == "0": continue if name_filter is not None and re.match(name_filter, experiment.name) is None: continue recorders = experiment.list_recorders() recorders, not_finished = filter_finished(recorders) if verbose: print( "====>>>> {:02d}/{:02d}-th experiment {:9s} has {:02d}/{:02d} finished recorders." .format( idx + 1, len(experiments), experiment.name, len(recorders), len(recorders) + not_finished, )) result = QResult() for recorder_id, recorder in recorders.items(): result.update(recorder.list_metrics(), key_map) if not len(result): print("There are no valid recorders for {:}".format(experiment)) continue else: print("There are {:} valid recorders for {:}".format( len(recorders), experiment.name)) head_str, value_str = result.info(all_keys, verbose=verbose) head_strs.append(head_str) value_strs.append(value_str) names.append(experiment.name) info_str_dict = compare_results(head_strs, value_strs, names, space=10, verbose=verbose) info_value_dict = dict(heads=head_strs, values=value_strs, names=names) return info_str_dict, info_value_dict
def query_info(save_dir, verbose, name_filter, key_map): R.set_uri(save_dir) experiments = R.list_experiments() if verbose: print("There are {:} experiments.".format(len(experiments))) qresults = [] for idx, (key, experiment) in enumerate(experiments.items()): if experiment.id == "0": continue if (name_filter is not None and re.fullmatch(name_filter, experiment.name) is None): continue recorders = experiment.list_recorders() recorders, not_finished = filter_finished(recorders) if verbose: print( "====>>>> {:02d}/{:02d}-th experiment {:9s} has {:02d}/{:02d} finished recorders." .format( idx + 1, len(experiments), experiment.name, len(recorders), len(recorders) + not_finished, )) result = QResult(experiment.name) for recorder_id, recorder in recorders.items(): result.update(recorder.list_metrics(), key_map) result.append_path( os.path.join(recorder.uri, recorder.experiment_id, recorder.id)) if not len(result): print("There are no valid recorders for {:}".format(experiment)) continue else: print("There are {:} valid recorders for {:}".format( len(recorders), experiment.name)) qresults.append(result) return qresults