def filter_and_print_tables(dataset):
from plotting import plot_group_overlap
from utils import make_issue_specific_figure_folder
def make_dataframe(l):
ldf = concat(l, axis=1)
ldf.columns = columns
tdf = ldf.transpose()
tdf.columns = utils.get_latex_par_names([tdf], as_row = True).split(',')
tdf = tdf.transpose()
return tdf
fnums = [1,2,3,4,5,6,7,8,9]
#fit, par, gen, ids = IO.load_pickled_generation_dataframe(dataset)
fit, par = utils.load_d10d11()
masks, filters = apply_filters(fit, fnums = fnums)
jaccard = calculate_jaccard(masks)
columns = map(lambda x: 'F%s'%x, fnums)
parmeans = make_dataframe(map(lambda f: par[f()].mean(), filters))
fitmeans = make_dataframe(map(lambda f: fit[f()].mean(), filters))
counts = make_dataframe(map(lambda f: par[f()].count(), filters))
full_table = concat([parmeans, fitmeans])
counts = counts.iloc[0]
counts.name = 'Count'
full_table = full_table.append(counts)
tex = full_table.to_latex(float_format=lambda x: str(round(x,1)))
full_tex = utils.prettify_table(tex, 'table:manual_filtering_%s'%dataset, 'XXX').replace('nan','N/A')
print full_tex
folder = make_issue_specific_figure_folder('manual_filtering', dataset)
plot_group_overlap(folder + 'group_overlap', jaccard)
return jaccard
def latency_vs_fitness_with_lines_for_agent_ratio(dataset):
from plotting import multiline_xy_plot
from utils import make_issue_specific_figure_folder
def get_ssmmlat_mask(l, u):
return (p.ssmm_latency_mu > l) & (p.ssmm_latency_mu < u)
def get_sclat_mask(l, u):
return (p.sc_latency_mu > l) & (p.sc_latency_mu < u)
def zip_to_tuples(r): return zip(r[:-1], r[1::])
def calc_and_plot(ratio_direction):
for fitness in f.columns:
ssmm_ys = list()
sc_ys = list()
legend_labels = list()
for ratio_lower, ratio_upper in zip_to_tuples(ratio_range):
ratio_mask = (ratio_lower < p.ratio) & (p.ratio < ratio_upper)
ssmm_lat_range = concat(map(lambda l: f[get_ssmmlat_mask(l,l+20) & ratio_mask].mean(), ssmmlatencyrange), axis=1).transpose()
ssmm_ys.append(ssmm_lat_range[fitness])
sc_lat_range = concat(map(lambda l: f[get_sclat_mask(l,l+20) & ratio_mask].mean(), sclatencyrange), axis=1).transpose()
sc_ys.append(sc_lat_range[fitness])
legend_labels.append(r'$\displaystyle %s < %s < %s$'%(round(ratio_lower,1), fl(ratio_direction, mathmode = False), round(ratio_upper,1)))
filename = '%s_%s_%s_mmlatency.png'%(folder, ratio_direction, fitness)
multiline_xy_plot(ssmm_lat_range.index, ssmm_ys, xlabel = 'ssmm_latency_mu', ylabel = fitness, legend_labels = legend_labels, filename = filename)
filename = '%s_%s_%s_sclatency.png'%(folder, ratio_direction, fitness)
multiline_xy_plot(sc_lat_range.index, sc_ys, xlabel = 'sc_latency_mu', ylabel = fitness, legend_labels = legend_labels, filename = filename)
ssmmlatencyrange = range(100)
sclatencyrange = range(100)
folder = make_issue_specific_figure_folder('latency_vs_fitness_with_lines_for_agent_ratio', dataset)
if dataset == 'd10d11':
f, p = utils.load_d10d11()
else:
f,p,g, i=IO.load_pickled_generation_dataframe(dataset_name=dataset)
if dataset == 'd10':
p['sc_nAgents'] = 150
elif dataset == 'd11':
p['ssmm_nAgents'] = 52
#nssmm_mask = p.ssmm_nAgents > 50
#f = f[nssmm_mask]
#p = p[nssmm_mask]
p['ratio'] = p['sc_nAgents'].astype(float) / p['ssmm_nAgents']
ratio_range = np.linspace(0,3,6)
calc_and_plot('ratioagent')
p['ratio'] = p['ssmm_nAgents'].astype(float) / p['sc_nAgents']
ratio_range = [0,0.01, 0.2,0.35,0.6,1]
calc_and_plot('ratioagentinv')
def issue_101_plot_pars_vs_fitness(dataset, overshoot_threshold, preloaded_data = None):
from plotting import get_pretty_xy_plot, make_pretty_scatter_plot
from numpy import where
def get_plots_to_make(fitness_types):
plots_to_make = list()
for fitness_type in fitness_types:
for stat in stats:
plots_to_make.append((fitness_type, stat))
return plots_to_make
def mkplot(all_data, groupby, plots_to_make):
g = all_data.groupby(groupby)
#x = g.groups.keys()
s = all_data.sort(groupby)
sorted_x, index_order = zip(*sorted(zip(g.groups.keys(), range(len(g.groups.keys())))))
for attr, stat in plots_to_make:
print groupby, attr, stat
y = getattr(g[attr],stat)()
filename = '%s%s__vs__%s(%s)'%(folder, groupby, attr, stat)
ax, fig = get_pretty_xy_plot(sorted_x, y, groupby, '%s (%s)'%(attr, stat), filename, g[attr].std()/2, save_figure = False)
filename = '%s%s__vs__%s(%s)_scatter'%(folder, groupby, attr, stat)
make_pretty_scatter_plot(s[groupby], s[attr], groupby, '%s (%s)'%(attr, stat), filename, ax=ax, fig=fig)
def run_analysis(groups, data, plots_to_make):
for groupby in groups:
mkplot(data, groupby, plots_to_make)
folder = make_issue_specific_figure_folder('101_pars_vs_fits', dataset)
stats = ['mean']
if dataset == 'd10d11':
f, p = utils.load_d10d11()
else:
f,p,g, i=IO.load_pickled_generation_dataframe(dataset_name=dataset)
if 'dataset' == 'd10':
p['sc_nAgents'] = 150
elif 'dataset' == 'd11':
p['ssmm_nAgents'] = 52
if preloaded_data is None:
fit, par, gen, ids = IO.load_pickled_generation_dataframe(dataset)
else:
try:
fit = preloaded_data['fit']
par = preloaded_data['par']
except KeyError, e:
print "Provide dict with keys 'fit' and 'par' containing dataframes for fit and par data"
print e
sys.exit(1)
def agent_ratio_to_latency_ratio(n_ar_bins = 10, n_lr_bins = 10, dataset = 'd10d11'):
from plotting import plot_image_matrix
from utils import make_issue_specific_figure_folder
folder = make_issue_specific_figure_folder('latency_vs_agent_ratios', dataset)
fit, par = utils.load_d10d11()
#fit, par, gen, ids = IO.load_pickled_generation_dataframe('d10')
agent_ratio = par.sc_nAgents.astype(float) / par.ssmm_nAgents
latency_ratio = par['ssmm_latency_mu'].astype(float) / par['sc_latency_mu']
inf_mask = agent_ratio != np.inf
agent_ratio_mask = (agent_ratio < 5)
latency_ratio_mask = (latency_ratio > 0) & (latency_ratio < 5)
#nsc_mask = par.sc_nAgents > 0
#nssmm_mask = (20 < par.ssmm_nAgents) & (par.ssmm_nAgents < 50)
#total_mask = inf_mask & agent_ratio_mask & latency_ratio_mask & nsc_mask & nssmm_mask
total_mask = inf_mask & agent_ratio_mask & latency_ratio_mask
par = par[total_mask]
fit = fit[total_mask]
agent_ratio = agent_ratio[total_mask]
latency_ratio = latency_ratio[total_mask]
ar_hn, ar_histbins = np.histogram(agent_ratio, bins = n_ar_bins)
lr_hn, lr_histbins = np.histogram(latency_ratio, bins = n_lr_bins)
print ar_histbins
overshoot_mean = np.zeros((n_ar_bins, n_lr_bins))
roundstable_mean = np.zeros((n_ar_bins, n_lr_bins))
stdev_mean = np.zeros((n_ar_bins, n_lr_bins))
timeto_mean = np.zeros((n_ar_bins, n_lr_bins))
test = np.zeros((n_ar_bins, n_lr_bins))
for i, (ar_lower, ar_upper) in enumerate(zip(ar_histbins[:-1], ar_histbins[1::])):
for j, (lr_lower, lr_upper) in enumerate(zip(lr_histbins[:-1], lr_histbins[1::])):
index_mask = (ar_lower < agent_ratio) & (agent_ratio < ar_upper) & (lr_lower < latency_ratio) & (latency_ratio < lr_upper)
means = fit[index_mask].mean()
overshoot_mean[i][j] = means['overshoot']
roundstable_mean[i][j] = means['round_stable']
stdev_mean[i][j] = means['stdev']
timeto_mean[i][j] = means['time_to_reach_new_fundamental']
test[i][j] = j
x_ticklabels = map(lambda x: round(x,1), lr_histbins)
y_ticklabels = map(lambda x: round(x,1), ar_histbins)
plot_image_matrix(folder + 'overshoot.png', overshoot_mean, x_ticklabels, y_ticklabels)
plot_image_matrix(folder + 'stdev.png', stdev_mean, x_ticklabels, y_ticklabels)
plot_image_matrix(folder + 'time_to_reach_new_fundamental.png', timeto_mean, x_ticklabels, y_ticklabels)
plot_image_matrix(folder + 'round_stable.png', roundstable_mean, x_ticklabels, y_ticklabels)
x_ticklabels = range(10)
y_ticklabels = range(10)
plot_image_matrix(folder + 'test.png', test, x_ticklabels, y_ticklabels)
return overshoot_mean, stdev_mean, roundstable_mean, timeto_mean
def faster_mm_makes_worse_markets(dataset):
from plotting import multiline_xy_plot
from utils import make_issue_specific_figure_folder
def get_mmlat_mask(l, u):
return (p.ssmm_latency_mu > l) & (p.ssmm_latency_mu < u)
def get_ssmmlatencyrange_mean(agent_mask, ssmmlatencyrange = range(1,100), nsc_lower = 0):
return concat(map(lambda l: f[get_mmlat_mask(l,l+20) & agent_mask].mean(), ssmmlatencyrange), axis=1).transpose()
def get_sclat_mask(l, u):
return (p.sc_latency_mu > l) & (p.sc_latency_mu < u)
def get_sclatencyrange_mean(agent_mask, sclatencyrange = range(1,100), nsc_lower = 0):
return concat(map(lambda l: f[get_sclat_mask(l,l+20) & agent_mask].mean(), sclatencyrange), axis=1).transpose()
def get_nchartist_mask(lower, upper):
return (p.sc_nAgents >= lower) & (p.sc_nAgents < upper)
def get_nmm_mask(lower, upper):
return (p.ssmm_nAgents >= lower) & (p.ssmm_nAgents < upper)
def zip_to_tuples(r): return zip(r[:-1], r[1::])
ssmmlatencyrange = range(80)
sclatencyrange = range(100)
if dataset == 'd10d11':
f, p = utils.load_d10d11()
else:
f,p,g, i=IO.load_pickled_generation_dataframe(dataset_name=dataset)
folder = make_issue_specific_figure_folder('faster_mm_makes_worse_markets', dataset)
try:
for fitness in f.columns:
filename = folder + fitness + '_SC_mmlatency.png'
xlabel = 'Market maker latency'
ylabel = fitness
legend_labels = list()
ys = list()
for nsc_lower, nsc_upper in zip_to_tuples(np.linspace(0,500,6)):
nchartist_mask = get_nchartist_mask(nsc_lower, nsc_upper)
means = get_ssmmlatencyrange_mean(nchartist_mask, ssmmlatencyrange, nsc_lower = nsc_lower)
ys.append(means[fitness])
legend_labels.append('%s <= # SC < %s'%(nsc_lower, nsc_upper))
multiline_xy_plot(means.index, ys, xlabel, ylabel, legend_labels, filename, y_errorbars=None, save_figure = True)
filename = folder + fitness + '_SC_sclatency.png'
xlabel = 'Chartist latency'
legend_labels = list()
ys = list()
for nsc_lower, nsc_upper in zip_to_tuples(np.linspace(0,500,6)):
nchartist_mask = get_nchartist_mask(nsc_lower, nsc_upper)
means = get_sclatencyrange_mean(nchartist_mask, sclatencyrange, nsc_lower = nsc_lower)
ys.append(means[fitness])
legend_labels.append('%s <= # SC < %s'%(nsc_lower, nsc_upper))
multiline_xy_plot(means.index, ys, xlabel, ylabel, legend_labels, filename, y_errorbars=None, save_figure = True)
except AttributeError:
pass
try:
for fitness in f.columns:
filename = folder + fitness + '_MM_mmlatency.png'
xlabel = 'Market maker latency'
ylabel = fitness
legend_labels = list()
ys = list()
for nmm_lower, nmm_upper in zip_to_tuples(range(0,150,25)):
n_mm_mask = get_nmm_mask(nmm_lower, nmm_upper)
means = get_ssmmlatencyrange_mean(n_mm_mask, ssmmlatencyrange, nsc_lower = nsc_lower)
ys.append(means[fitness])
legend_labels.append('%s <= # MM < %s'%(nmm_lower, nmm_upper))
multiline_xy_plot(means.index, ys, xlabel, ylabel, legend_labels, filename, y_errorbars=None, save_figure = True)
filename = folder + fitness + '_MM_sclatency.png'
xlabel = 'Chartist latency'
ylabel = fitness
legend_labels = list()
ys = list()
for nmm_lower, nmm_upper in zip_to_tuples(range(0,150,25)):
n_mm_mask = get_nmm_mask(nmm_lower, nmm_upper)
means = get_sclatencyrange_mean(n_mm_mask, sclatencyrange, nsc_lower = nsc_lower)
ys.append(means[fitness])
legend_labels.append('%s <= # MM < %s'%(nmm_lower, nmm_upper))
multiline_xy_plot(means.index, ys, xlabel, ylabel, legend_labels, filename, y_errorbars=None, save_figure = True)
except AttributeError:
pass