def test_basic(self, dataloader_dummy, network_dummy): metrics_table = generate_metrics_table({'bm_1': network_dummy}, dataloader_dummy, {'rets': MeanReturns()}) assert isinstance(metrics_table, pd.DataFrame) assert len(metrics_table) == len(dataloader_dummy.dataset) assert {'metric', 'value', 'benchmark', 'timestamp'} == set(metrics_table.columns.to_list())
def test_errors(self, dataloader_dummy, network_dummy): with pytest.raises(TypeError): generate_metrics_table({'bm_1': 'WRONG'}, dataloader_dummy, {'metric': MeanReturns()}) with pytest.raises(TypeError): generate_metrics_table({'bm_1': network_dummy}, 'FAKE', {'metric': MeanReturns()}) with pytest.raises(TypeError): generate_metrics_table({'bm_1': network_dummy}, dataloader_dummy, {'metric': 'FAKE'})
# %% # During training, the only mandatory metric/loss was the loss criterion that we tried to minimize. # Naturally, one might be interested in many other metrics to evaluate the performance. See below # an example. metrics = { 'MaxDD': MaximumDrawdown(), 'Sharpe': SharpeRatio(), 'MeanReturn': MeanReturns() } # %% # Let us now use the above created objects. We first generate a table with all metrics over all # samples and for all benchmarks. This is done via :code:`generate_metrics_table`. metrics_table = generate_metrics_table(benchmarks, dataloader_test, metrics) # %% # And then we plot it with :code:`plot_metrics`. plot_metrics(metrics_table) # %% # Each plot represents a different metric. The x-axis represents the timestamps in our # test set. The different colors are capturing different models. How is the value of a metric # computed? We assume that the investor predicts the portfolio at time x and buys it. He then # holds it for :code:`horizon` timesteps. The actual metric is then computed over this time horizon. # %% # Finally, we are also interested in how the allocation/prediction looks like at each time step. # We can use the :code:`generate_weights_table` function to create a :code:`pd.DataFrame`. weight_table = generate_weights_table(network, dataloader_test)