def performance(
self,
data,
era_as_str=True,
region_as_str=True,
columns=['logloss', 'auc', 'acc', 'ystd', 'sharpe', 'consis'],
sort_by='logloss'):
df, info = metrics_per_name(data,
self,
columns=columns,
era_as_str=era_as_str,
region_as_str=region_as_str)
if sort_by in columns:
if sort_by == 'logloss':
df = df.sort_values(by='logloss', ascending=True)
elif sort_by == 'auc':
df = df.sort_values(by='auc', ascending=False)
elif sort_by == 'acc':
df = df.sort_values(by='acc', ascending=False)
elif sort_by == 'ystd':
df = df.sort_values(by='ystd', ascending=False)
elif sort_by == 'sharpe':
df = df.sort_values(by='sharpe', ascending=False)
elif sort_by == 'consis':
by = ['consis']
ascending = [False]
if 'logloss' in df:
by.append('logloss')
ascending.append('True')
df = df.sort_values(by=by, ascending=ascending)
else:
raise ValueError("`sort_by` name not recognized")
return df
def test_metrics_per_name():
"make sure metrics_per_name runs"
d = testing.micro_data()
p = testing.micro_prediction()
metrics_per_name(d, p, 1)
metrics_per_name(d, p, 2, join='yhat')
metrics_per_name(d, p, 3, columns=['sharpe'])
assert_raises(ValueError, metrics_per_name, d, p, 4, 'data', ['wtf'])
def metric_per_tournament(self, data, metric='corr'):
"""DataFrame containing given metric versus tournament"""
dfs = []
for t_int, t_name in nx.tournament_iter(active_only=False):
df, info = metrics_per_name(data,
self,
t_int,
columns=[metric],
split_pairs=False)
df.columns = [t_name]
dfs.append(df)
df = pd.concat(dfs, axis=1)
df.insert(df.shape[1], 'mean', df.mean(axis=1))
df = df.sort_values('mean')
return df