def __init__( self, experiment, process_list=[], rec_key_func=None, rec_filter_func=None, artifacts_path={"pred": "pred.pkl"}, artifacts_key=None, ): """ Init RecorderCollector. Args: experiment (Experiment or str): an instance of an Experiment or the name of an Experiment process_list (list or Callable): the list of processors or the instance of a processor to process dict. rec_key_func (Callable): a function to get the key of a recorder. If None, use recorder id. rec_filter_func (Callable, optional): filter the recorder by return True or False. Defaults to None. artifacts_path (dict, optional): The artifacts name and its path in Recorder. Defaults to {"pred": "pred.pkl", "IC": "sig_analysis/ic.pkl"}. artifacts_key (str or List, optional): the artifacts key you want to get. If None, get all artifacts. """ super().__init__(process_list=process_list) if isinstance(experiment, str): experiment = R.get_exp(experiment_name=experiment) self.experiment = experiment self.artifacts_path = artifacts_path if rec_key_func is None: rec_key_func = lambda rec: rec.info["id"] if artifacts_key is None: artifacts_key = list(self.artifacts_path.keys()) self.rec_key_func = rec_key_func self.artifacts_key = artifacts_key self.rec_filter_func = rec_filter_func
def get_all_results(folders) -> dict: results = dict() for fn in folders: exp = R.get_exp(experiment_name=fn, create=False) recorders = exp.list_recorders() result = dict() result["annualized_return_with_cost"] = list() result["information_ratio_with_cost"] = list() result["max_drawdown_with_cost"] = list() result["ic"] = list() result["icir"] = list() result["rank_ic"] = list() result["rank_icir"] = list() for recorder_id in recorders: if recorders[recorder_id].status == "FINISHED": recorder = R.get_recorder(recorder_id=recorder_id, experiment_name=fn) metrics = recorder.list_metrics() result["annualized_return_with_cost"].append(metrics["excess_return_with_cost.annualized_return"]) result["information_ratio_with_cost"].append(metrics["excess_return_with_cost.information_ratio"]) result["max_drawdown_with_cost"].append(metrics["excess_return_with_cost.max_drawdown"]) result["ic"].append(metrics["IC"]) result["icir"].append(metrics["ICIR"]) result["rank_ic"].append(metrics["Rank IC"]) result["rank_icir"].append(metrics["Rank ICIR"]) results[fn] = result return results
def reset(self): print("========== reset ==========") if isinstance(self.trainer, TrainerRM): TaskManager(task_pool=self.task_pool).remove() exp = R.get_exp(experiment_name=self.experiment_name) for rid in exp.list_recorders(): exp.delete_recorder(rid)
def reset(self): for task in self.tasks + self.add_tasks: name_id = task["model"]["class"] exp = R.get_exp(experiment_name=name_id) for rid in exp.list_recorders(): exp.delete_recorder(rid) if os.path.exists(self._ROLLING_MANAGER_PATH): os.remove(self._ROLLING_MANAGER_PATH)
def check_diff_freq(self): self._init_qlib() exp = R.get_exp(experiment_name="backtest") rec = next(iter(exp.list_recorders().values()) ) # assuming this will get the latest recorder for check_key in "account", "total_turnover", "total_cost": check_key = "total_cost" acc_dict = {} for freq in ["30minute", "5minute", "1day"]: acc_dict[freq] = rec.load_object( f"portfolio_analysis/report_normal_{freq}.pkl")[check_key] acc_df = pd.DataFrame(acc_dict) acc_resam = acc_df.resample("1d").last().dropna() assert (acc_resam["30minute"] == acc_resam["1day"]).all()
def meta_inference(self): """ Leverage meta-model for inference: - Given - baseline tasks - input for meta model(internal data) - meta model (its learnt knowledge on proxy forecasting model is expected to transfer to normal forecasting model) """ # 1) get meta model exp = R.get_exp(experiment_name=self.meta_exp_name) rec = exp.list_recorders(rtype=exp.RT_L)[0] meta_model: MetaModelDS = rec.load_object("model") # 2) # we are transfer to knowledge of meta model to final forecasting tasks. # Create MetaTaskDataset for the final forecasting tasks # Aligning the setting of it to the MetaTaskDataset when training Meta model is necessary # 2.1) get previous config param = rec.list_params() trunc_days = int(param["trunc_days"]) step = int(param["step"]) hist_step_n = int(param["hist_step_n"]) fill_method = param.get("fill_method", "max") rb = RollingBenchmark(model_type=self.forecast_model) task_l = rb.create_rolling_tasks() # 2.2) create meta dataset for final dataset kwargs = dict( task_tpl=task_l, step=step, segments=0.0, # all the tasks are for testing trunc_days=trunc_days, hist_step_n=hist_step_n, fill_method=fill_method, task_mode=MetaTask.PROC_MODE_TRANSFER, ) with self._internal_data_path.open("rb") as f: internal_data = pickle.load(f) mds = MetaDatasetDS(exp_name=internal_data, **kwargs) # 3) meta model make inference and get new qlib task new_tasks = meta_model.inference(mds) with self._task_path.open("wb") as f: pickle.dump(new_tasks, f)
def __init__( self, experiment, process_list=[], rec_key_func=None, rec_filter_func=None, artifacts_path={"pred": "pred.pkl"}, artifacts_key=None, list_kwargs={}, status: Iterable = {Recorder.STATUS_FI}, ): """ Init RecorderCollector. Args: experiment: (Experiment or str): an instance of an Experiment or the name of an Experiment (Callable): an callable function, which returns a list of experiments process_list (list or Callable): the list of processors or the instance of a processor to process dict. rec_key_func (Callable): a function to get the key of a recorder. If None, use recorder id. rec_filter_func (Callable, optional): filter the recorder by return True or False. Defaults to None. artifacts_path (dict, optional): The artifacts name and its path in Recorder. Defaults to {"pred": "pred.pkl", "IC": "sig_analysis/ic.pkl"}. artifacts_key (str or List, optional): the artifacts key you want to get. If None, get all artifacts. list_kwargs (str): arguments for list_recorders function. status (Iterable): only collect recorders with specific status. None indicating collecting all the recorders """ super().__init__(process_list=process_list) if isinstance(experiment, str): experiment = R.get_exp(experiment_name=experiment) assert isinstance(experiment, (Experiment, Callable)) self.experiment = experiment self.artifacts_path = artifacts_path if rec_key_func is None: def rec_key_func(rec): return rec.info["id"] if artifacts_key is None: artifacts_key = list(self.artifacts_path.keys()) self.rec_key_func = rec_key_func self.artifacts_key = artifacts_key self.rec_filter_func = rec_filter_func self.list_kwargs = list_kwargs self.status = status
def list_recorders(experiment, rec_filter_func=None): """ List all recorders which can pass the filter in an experiment. Args: experiment (str or Experiment): the name of an Experiment or an instance rec_filter_func (Callable, optional): return True to retain the given recorder. Defaults to None. Returns: dict: a dict {rid: recorder} after filtering. """ if isinstance(experiment, str): experiment = R.get_exp(experiment_name=experiment) recs = experiment.list_recorders() recs_flt = {} for rid, rec in recs.items(): if rec_filter_func is None or rec_filter_func(rec): recs_flt[rid] = rec return recs_flt
def get_all_results(folders) -> dict: results = dict() for fn in folders: try: exp = R.get_exp(experiment_name=fn, create=False) except ValueError: # No experiment results continue recorders = exp.list_recorders() result = dict() result["annualized_return_with_cost"] = list() result["information_ratio_with_cost"] = list() result["max_drawdown_with_cost"] = list() result["ic"] = list() result["icir"] = list() result["rank_ic"] = list() result["rank_icir"] = list() for recorder_id in recorders: if recorders[recorder_id].status == "FINISHED": recorder = R.get_recorder(recorder_id=recorder_id, experiment_name=fn) metrics = recorder.list_metrics() if "1day.excess_return_with_cost.annualized_return" not in metrics: print(f"{recorder_id} is skipped due to incomplete result") continue result["annualized_return_with_cost"].append( metrics["1day.excess_return_with_cost.annualized_return"]) result["information_ratio_with_cost"].append( metrics["1day.excess_return_with_cost.information_ratio"]) result["max_drawdown_with_cost"].append( metrics["1day.excess_return_with_cost.max_drawdown"]) result["ic"].append(metrics["IC"]) result["icir"].append(metrics["ICIR"]) result["rank_ic"].append(metrics["Rank IC"]) result["rank_icir"].append(metrics["Rank ICIR"]) results[fn] = result return results
def reset(self): TaskManager(self.task_pool).remove() exp = R.get_exp(experiment_name=self.exp_name) for rid in exp.list_recorders(): exp.delete_recorder(rid)