def main():
'''Evaluate a trained model against logistic regression.'''
parser = argparse.ArgumentParser()
parser.add_argument("path", help="The path to the file.", type=Path)
parser.add_argument('--rolling',
help='Set a rolling window.',
type=int,
default=0)
args = parser.parse_args()
dataframe = pd.read_csv(args.path)
axes = defaultdict(lambda: plt.figure().add_subplot(111))
columns = dataframe.columns
pyplot_attr = {
'title': 'Performance on {} data set',
'xlabel': 'Epoch',
}
dataframe = dataframe.fillna(method='bfill')
if args.rolling != 0:
dataframe = dataframe.rolling(args.rolling).mean()
for name in columns:
utils_plot.plot_sequence(axes[name], dataframe[name], label=name)
for column in columns:
pyplot_attr['ylabel'] = column.capitalize()
utils_plot.set_attributes(axes[column], pyplot_attr)
plt.show()
def run_lr(path,
trials=10,
batch_size=None,
total_epochs=40,
data_set='mnist'):
'''Run both agent evaluationg and logistic classification training.'''
path = Path(path)
infos = list(
chain.from_iterable([
task_lr(i,
batch_size=batch_size,
total_epochs=total_epochs,
data_set=data_set) for i in trange(trials)
]))
dataframe_lc = pd.DataFrame.from_dict(infos)
columns = [
'accuracy' if col == 'acc' else col for col in dataframe_lc.columns
]
dataframe_lc.columns = columns
axes = defaultdict(lambda: plt.figure().add_subplot(111))
pyplot_attr = {
'title': 'Performance on {} data set'.format(data_set.upper()),
'xlabel': 'Epoch',
}
# columns = set(dataframe_rl.select_dtypes('number').columns) - {'epoch'}
for column in columns:
pyplot_attr['ylabel'] = column.capitalize()
utils_plot.set_attributes(axes[column], pyplot_attr)
dataframe_lc.columns = columns
plot_results(axes, dataframe_lc, 'epoch', 'Logistic Classification')
for axis in axes.values():
utils_plot.add_legend(axis)
plt.show()
def run_multi(path, trials=10, batch_size=None, total_epochs=40,
data_set='mnist'):
'''Run both agent evaluation and logistic classification training.'''
path = Path(path)
infos = list(chain.from_iterable([task(path, i, batch_size=batch_size,
total_epochs=total_epochs,
data_set=data_set)
for i in trange(trials)]))
dataframe_rl = pd.DataFrame(infos)
dataframe_rl.to_csv(str(path / 'dataframe_rl.csv'))
infos = list(chain.from_iterable([task_lr(i, batch_size=batch_size,
total_epochs=total_epochs,
data_set=data_set)
for i in trange(trials)]))
dataframe_lc = pd.DataFrame.from_dict(infos)
dataframe_lc.to_csv(str(path / 'dataframe_lc.csv'))
columns = ['accuracy' if col == 'acc' else col
for col in dataframe_lc.columns]
dataframe_lc.columns = columns
axes = defaultdict(lambda: plt.figure().add_subplot(111))
pyplot_attr = {
'title': 'Performance on {} data set'.format(data_set.upper()),
'xlabel': 'Epoch',
}
columns = set(dataframe_rl.select_dtypes('number').columns) - {'epoch'}
for column in columns:
pyplot_attr['ylabel'] = column.capitalize()
utils_plot.set_attributes(axes[column], pyplot_attr)
plot_results(axes, dataframe_rl, 'epoch', 'RL Gradient Descent')
plot_results(axes, dataframe_lc, 'epoch', 'Gradient Descent')
for name, axis in axes.items():
utils_plot.add_legend(axis)
axis.figure.savefig(str(path / '{}.png'.format(name)))
def plot_hyperparamsearch_alg2(dataframe):
'''Plot experiments contained in dataframe.'''
dataframe['learning_rate'] = np.log10(dataframe['learning_rate'])
groups = dataframe.groupby(['learning_rate', 'gamma', 'alg', 'index'])
mean_df = groups.mean()
std_df = groups.std()
(lrates, gammas, algs) = utils_pandas.create_grid(mean_df, 3)
target = np.reshape([
g['objective'].mean()
for _, g in utils_pandas.iterate_levels(mean_df, 3)
], lrates.shape)
metric_min = np.nanmin(target)
metric_max = np.nanmax(target)
for i, alg in enumerate(algs[0, 0, :]):
fig = plt.figure()
ax = fig.add_subplot(111, projection='3d')
ax.plot_surface(lrates[..., i],
gammas[..., i],
target[..., i],
cmap=None,
linewidth=0,
antialiased=False)
attr = {
'xlabel': 'Learning Rate',
'ylabel': 'Gamma',
'zlabel': 'Mean Loss',
'zmin': metric_min,
'zmax': metric_max,
'title': 'Model: {}'.format(alg)
}
utils_plot.set_attributes(ax, attr)
pos = np.nanargmin(target[..., i])
loss = target[..., i].ravel()[pos]
lrate = lrates[..., i].ravel()[pos]
gamma = gammas[..., i].ravel()[pos]
print(alg, 10**lrate, gamma, loss)
for col in ['objective', 'accuracy']:
ylabel = 'loss' if col == 'objective' else col
metric_mean = mean_df.loc[(lrate, gamma, alg), col]
metric_std = std_df.loc[(lrate, gamma, alg), col]
fig = plt.figure()
ax = fig.add_subplot(111)
attr = {
'xlabel': 'steps',
'ylabel': ylabel,
'title': 'Model: {}'.format(alg)
}
utils_plot.set_attributes(ax, attr)
utils_plot.plot_sequence(ax, metric_mean)
utils_plot.fill_between(ax, metric_mean, metric_std, alpha=0.1)
plt.show()