def performance( self, data, era_as_str=True, region_as_str=True, columns=['logloss', 'auc', 'acc', 'ystd', 'sharpe', 'consis'], sort_by='logloss'): df, info = metrics_per_name(data, self, columns=columns, era_as_str=era_as_str, region_as_str=region_as_str) if sort_by in columns: if sort_by == 'logloss': df = df.sort_values(by='logloss', ascending=True) elif sort_by == 'auc': df = df.sort_values(by='auc', ascending=False) elif sort_by == 'acc': df = df.sort_values(by='acc', ascending=False) elif sort_by == 'ystd': df = df.sort_values(by='ystd', ascending=False) elif sort_by == 'sharpe': df = df.sort_values(by='sharpe', ascending=False) elif sort_by == 'consis': by = ['consis'] ascending = [False] if 'logloss' in df: by.append('logloss') ascending.append('True') df = df.sort_values(by=by, ascending=ascending) else: raise ValueError("`sort_by` name not recognized") return df
def test_metrics_per_name(): "make sure metrics_per_name runs" d = testing.micro_data() p = testing.micro_prediction() metrics_per_name(d, p, 1) metrics_per_name(d, p, 2, join='yhat') metrics_per_name(d, p, 3, columns=['sharpe']) assert_raises(ValueError, metrics_per_name, d, p, 4, 'data', ['wtf'])
def metric_per_tournament(self, data, metric='corr'): """DataFrame containing given metric versus tournament""" dfs = [] for t_int, t_name in nx.tournament_iter(active_only=False): df, info = metrics_per_name(data, self, t_int, columns=[metric], split_pairs=False) df.columns = [t_name] dfs.append(df) df = pd.concat(dfs, axis=1) df.insert(df.shape[1], 'mean', df.mean(axis=1)) df = df.sort_values('mean') return df