def filter_and_print_tables(dataset):
	from plotting import plot_group_overlap
	from utils import make_issue_specific_figure_folder

	def make_dataframe(l):
		ldf = concat(l, axis=1)
		ldf.columns = columns
		tdf = ldf.transpose()
		tdf.columns = utils.get_latex_par_names([tdf], as_row = True).split(',')
		tdf = tdf.transpose()
		return tdf
	
	fnums = [1,2,3,4,5,6,7,8,9]
	#fit, par, gen, ids = IO.load_pickled_generation_dataframe(dataset)
	fit, par = utils.load_d10d11()
	masks, filters = apply_filters(fit, fnums = fnums)
	jaccard = calculate_jaccard(masks)
	
	columns = map(lambda x: 'F%s'%x, fnums)
	parmeans = make_dataframe(map(lambda f: par[f()].mean(), filters))
	fitmeans = make_dataframe(map(lambda f: fit[f()].mean(), filters))
	counts = make_dataframe(map(lambda f: par[f()].count(), filters))
	
	full_table = concat([parmeans, fitmeans])
	
	counts = counts.iloc[0]
	counts.name = 'Count'
	full_table = full_table.append(counts)
	tex = full_table.to_latex(float_format=lambda x: str(round(x,1)))
	full_tex = utils.prettify_table(tex, 'table:manual_filtering_%s'%dataset, 'XXX').replace('nan','N/A')
	print full_tex
	
	folder = make_issue_specific_figure_folder('manual_filtering', dataset)
	plot_group_overlap(folder + 'group_overlap', jaccard)
	return jaccard
def latency_vs_fitness_with_lines_for_agent_ratio(dataset):
	from plotting import multiline_xy_plot
	from utils import make_issue_specific_figure_folder
	def get_ssmmlat_mask(l, u): 
		return (p.ssmm_latency_mu > l) & (p.ssmm_latency_mu < u)

	def get_sclat_mask(l, u): 
		return (p.sc_latency_mu > l) & (p.sc_latency_mu < u)

	def zip_to_tuples(r): return zip(r[:-1], r[1::])


	def calc_and_plot(ratio_direction):	
		for fitness in f.columns:
			ssmm_ys = list()
			sc_ys = list()
			legend_labels = list()
			for ratio_lower, ratio_upper in zip_to_tuples(ratio_range):
				ratio_mask = (ratio_lower < p.ratio) & (p.ratio < ratio_upper)
				ssmm_lat_range = concat(map(lambda l: f[get_ssmmlat_mask(l,l+20) & ratio_mask].mean(), ssmmlatencyrange), axis=1).transpose()
				ssmm_ys.append(ssmm_lat_range[fitness])
				sc_lat_range = concat(map(lambda l: f[get_sclat_mask(l,l+20) & ratio_mask].mean(), sclatencyrange), axis=1).transpose()
				sc_ys.append(sc_lat_range[fitness])
				legend_labels.append(r'$\displaystyle %s < %s < %s$'%(round(ratio_lower,1), fl(ratio_direction, mathmode = False), round(ratio_upper,1)))
			filename = '%s_%s_%s_mmlatency.png'%(folder, ratio_direction, fitness)
			
			multiline_xy_plot(ssmm_lat_range.index, ssmm_ys, xlabel = 'ssmm_latency_mu', ylabel = fitness, legend_labels = legend_labels, filename = filename)
			filename = '%s_%s_%s_sclatency.png'%(folder, ratio_direction, fitness)
			
			multiline_xy_plot(sc_lat_range.index, sc_ys, xlabel = 'sc_latency_mu', ylabel = fitness, legend_labels = legend_labels, filename = filename)


	ssmmlatencyrange = range(100)
	sclatencyrange = range(100)
	
	folder = make_issue_specific_figure_folder('latency_vs_fitness_with_lines_for_agent_ratio', dataset)
	
	if dataset == 'd10d11':
		f, p = utils.load_d10d11()
	else:
		f,p,g, i=IO.load_pickled_generation_dataframe(dataset_name=dataset)
		if dataset == 'd10':
			p['sc_nAgents'] = 150
		elif dataset == 'd11':
			p['ssmm_nAgents'] = 52
	
	#nssmm_mask = p.ssmm_nAgents > 50
	#f = f[nssmm_mask]
	#p = p[nssmm_mask]

	p['ratio'] = p['sc_nAgents'].astype(float) / p['ssmm_nAgents']
	ratio_range = np.linspace(0,3,6)
	calc_and_plot('ratioagent')

	p['ratio'] = p['ssmm_nAgents'].astype(float) / p['sc_nAgents']
	ratio_range = [0,0.01, 0.2,0.35,0.6,1]
	calc_and_plot('ratioagentinv')
def issue_101_plot_pars_vs_fitness(dataset, overshoot_threshold, preloaded_data = None):
	from plotting import get_pretty_xy_plot, make_pretty_scatter_plot
	from numpy import where


	def get_plots_to_make(fitness_types):
		plots_to_make = list()
		for fitness_type in fitness_types:
			for stat in stats:
				plots_to_make.append((fitness_type, stat))
		return plots_to_make


	def mkplot(all_data, groupby, plots_to_make):
		g = all_data.groupby(groupby)
		#x = g.groups.keys()
		s = all_data.sort(groupby)
		sorted_x, index_order = zip(*sorted(zip(g.groups.keys(), range(len(g.groups.keys())))))
		for attr, stat in plots_to_make:
			print groupby, attr, stat
			y = getattr(g[attr],stat)()
			filename = '%s%s__vs__%s(%s)'%(folder, groupby, attr, stat)
			ax, fig = get_pretty_xy_plot(sorted_x, y, groupby, '%s (%s)'%(attr, stat), filename, g[attr].std()/2, save_figure = False)
			filename = '%s%s__vs__%s(%s)_scatter'%(folder, groupby, attr, stat)
			make_pretty_scatter_plot(s[groupby], s[attr], groupby, '%s (%s)'%(attr, stat), filename, ax=ax, fig=fig)
	
	def run_analysis(groups, data, plots_to_make):
		for groupby in groups:
			mkplot(data, groupby, plots_to_make)

	
	folder = make_issue_specific_figure_folder('101_pars_vs_fits', dataset)
	stats = ['mean']

	if dataset == 'd10d11':
		f, p = utils.load_d10d11()
	else:
		f,p,g, i=IO.load_pickled_generation_dataframe(dataset_name=dataset)
		if 'dataset' == 'd10':
			p['sc_nAgents'] = 150
		elif 'dataset' == 'd11':
			p['ssmm_nAgents'] = 52

	

	if preloaded_data is None: 
		fit, par, gen, ids = IO.load_pickled_generation_dataframe(dataset)
	else:
		try:
			fit = preloaded_data['fit']
			par = preloaded_data['par']
		except KeyError, e:
			print "Provide dict with keys 'fit' and 'par' containing dataframes for fit and par data"
			print e
			sys.exit(1)
def agent_ratio_to_latency_ratio(n_ar_bins = 10, n_lr_bins = 10, dataset = 'd10d11'):
	from plotting import plot_image_matrix
	from utils import make_issue_specific_figure_folder
	folder = make_issue_specific_figure_folder('latency_vs_agent_ratios', dataset)
	
	fit, par = utils.load_d10d11()

	#fit, par, gen, ids = IO.load_pickled_generation_dataframe('d10')
	agent_ratio = par.sc_nAgents.astype(float) / par.ssmm_nAgents
	latency_ratio = par['ssmm_latency_mu'].astype(float) / par['sc_latency_mu']

	inf_mask = agent_ratio != np.inf

	agent_ratio_mask = (agent_ratio < 5)
	latency_ratio_mask = (latency_ratio > 0) & (latency_ratio < 5)
	
	#nsc_mask = par.sc_nAgents > 0
	#nssmm_mask = (20 < par.ssmm_nAgents) & (par.ssmm_nAgents < 50)

	#total_mask = inf_mask & agent_ratio_mask & latency_ratio_mask & nsc_mask & nssmm_mask
	total_mask = inf_mask & agent_ratio_mask & latency_ratio_mask
	par = par[total_mask]
	fit = fit[total_mask]
	agent_ratio = agent_ratio[total_mask]
	latency_ratio = latency_ratio[total_mask]

	ar_hn, ar_histbins = np.histogram(agent_ratio, bins = n_ar_bins)
	lr_hn, lr_histbins = np.histogram(latency_ratio, bins = n_lr_bins)
	print ar_histbins
	overshoot_mean = np.zeros((n_ar_bins, n_lr_bins))
	roundstable_mean = np.zeros((n_ar_bins, n_lr_bins))
	stdev_mean = np.zeros((n_ar_bins, n_lr_bins))
	timeto_mean = np.zeros((n_ar_bins, n_lr_bins))
	test = np.zeros((n_ar_bins, n_lr_bins))
	for i, (ar_lower, ar_upper) in enumerate(zip(ar_histbins[:-1], ar_histbins[1::])):
		for j, (lr_lower, lr_upper) in enumerate(zip(lr_histbins[:-1], lr_histbins[1::])):
			index_mask = (ar_lower < agent_ratio) & (agent_ratio < ar_upper) & (lr_lower < latency_ratio) & (latency_ratio < lr_upper)
			means = fit[index_mask].mean()
			overshoot_mean[i][j] = means['overshoot']
			roundstable_mean[i][j] = means['round_stable']
			stdev_mean[i][j] = means['stdev']
			timeto_mean[i][j] = means['time_to_reach_new_fundamental']
			test[i][j] = j
	x_ticklabels = map(lambda x: round(x,1), lr_histbins)
	y_ticklabels = map(lambda x: round(x,1), ar_histbins)
	plot_image_matrix(folder + 'overshoot.png', overshoot_mean, x_ticklabels, y_ticklabels)
	plot_image_matrix(folder + 'stdev.png', stdev_mean, x_ticklabels, y_ticklabels)
	plot_image_matrix(folder + 'time_to_reach_new_fundamental.png', timeto_mean, x_ticklabels, y_ticklabels)
	plot_image_matrix(folder + 'round_stable.png', roundstable_mean, x_ticklabels, y_ticklabels)
	x_ticklabels = range(10)
	y_ticklabels = range(10)
	plot_image_matrix(folder + 'test.png', test, x_ticklabels, y_ticklabels)
	return overshoot_mean, stdev_mean, roundstable_mean, timeto_mean
def faster_mm_makes_worse_markets(dataset):
	from plotting import multiline_xy_plot
	from utils import make_issue_specific_figure_folder
	def get_mmlat_mask(l, u): 
		return (p.ssmm_latency_mu > l) & (p.ssmm_latency_mu < u)

	def get_ssmmlatencyrange_mean(agent_mask, ssmmlatencyrange = range(1,100), nsc_lower = 0):
		return concat(map(lambda l: f[get_mmlat_mask(l,l+20) & agent_mask].mean(), ssmmlatencyrange), axis=1).transpose()

	def get_sclat_mask(l, u): 
		return (p.sc_latency_mu > l) & (p.sc_latency_mu < u)

	def get_sclatencyrange_mean(agent_mask, sclatencyrange = range(1,100), nsc_lower = 0):
		return concat(map(lambda l: f[get_sclat_mask(l,l+20) & agent_mask].mean(), sclatencyrange), axis=1).transpose()

	def get_nchartist_mask(lower, upper):
		return (p.sc_nAgents >= lower) & (p.sc_nAgents < upper)

	def get_nmm_mask(lower, upper):
		return (p.ssmm_nAgents >= lower) & (p.ssmm_nAgents < upper)
	
	def zip_to_tuples(r): return zip(r[:-1], r[1::])

	ssmmlatencyrange = range(80)
	sclatencyrange = range(100)

	

	if dataset == 'd10d11':
		f, p = utils.load_d10d11()
	else:
		f,p,g, i=IO.load_pickled_generation_dataframe(dataset_name=dataset)

	folder = make_issue_specific_figure_folder('faster_mm_makes_worse_markets', dataset)
	try:
		for fitness in f.columns:
			filename = folder + fitness + '_SC_mmlatency.png'
			xlabel = 'Market maker latency'
			ylabel = fitness
			legend_labels = list()
			ys = list()
			for nsc_lower, nsc_upper in zip_to_tuples(np.linspace(0,500,6)):
				nchartist_mask = get_nchartist_mask(nsc_lower, nsc_upper)	
				means = get_ssmmlatencyrange_mean(nchartist_mask, ssmmlatencyrange, nsc_lower = nsc_lower)
				ys.append(means[fitness])
				legend_labels.append('%s <= # SC < %s'%(nsc_lower, nsc_upper))
			multiline_xy_plot(means.index, ys, xlabel, ylabel, legend_labels, filename, y_errorbars=None, save_figure = True)

			filename = folder + fitness + '_SC_sclatency.png'
			xlabel = 'Chartist latency'
			legend_labels = list()
			ys = list()
			for nsc_lower, nsc_upper in zip_to_tuples(np.linspace(0,500,6)):
				nchartist_mask = get_nchartist_mask(nsc_lower, nsc_upper)	
				means = get_sclatencyrange_mean(nchartist_mask, sclatencyrange, nsc_lower = nsc_lower)
				ys.append(means[fitness])
				legend_labels.append('%s <= # SC < %s'%(nsc_lower, nsc_upper))
			multiline_xy_plot(means.index, ys, xlabel, ylabel, legend_labels, filename, y_errorbars=None, save_figure = True)
	except AttributeError:
		pass

	try:	
		for fitness in f.columns:
			filename = folder + fitness + '_MM_mmlatency.png'
			xlabel = 'Market maker latency'
			ylabel = fitness
			legend_labels = list()
			ys = list()
			for nmm_lower, nmm_upper in zip_to_tuples(range(0,150,25)):
				n_mm_mask = get_nmm_mask(nmm_lower, nmm_upper)	
				means = get_ssmmlatencyrange_mean(n_mm_mask, ssmmlatencyrange, nsc_lower = nsc_lower)
				ys.append(means[fitness])
				legend_labels.append('%s <= # MM < %s'%(nmm_lower, nmm_upper))
			multiline_xy_plot(means.index, ys, xlabel, ylabel, legend_labels, filename, y_errorbars=None, save_figure = True)

			filename = folder + fitness + '_MM_sclatency.png'
			xlabel = 'Chartist latency'
			ylabel = fitness
			legend_labels = list()
			ys = list()
			for nmm_lower, nmm_upper in zip_to_tuples(range(0,150,25)):
				n_mm_mask = get_nmm_mask(nmm_lower, nmm_upper)	
				means = get_sclatencyrange_mean(n_mm_mask, sclatencyrange, nsc_lower = nsc_lower)
				ys.append(means[fitness])
				legend_labels.append('%s <= # MM < %s'%(nmm_lower, nmm_upper))
			multiline_xy_plot(means.index, ys, xlabel, ylabel, legend_labels, filename, y_errorbars=None, save_figure = True)		
	except AttributeError:
		pass