/
trials.py
114 lines (84 loc) · 4.01 KB
/
trials.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
#!/usr/bin/env python
import sys
from queries import queries
import math
from timeit import Timer
import matplotlib.pyplot as plt
grams_setup="ss = __import__('scrabble-suggester');name='suggester';Q='{}';K={};printing=False"
grams_fn = 'ss.main([name,Q,K],printing)'
def time(qsize,k,number=10,sample_size=100): # Sample size = 500
"""Returns a list of times of trials of the k top results of a query of size qsize"""
return [_time(q,k,number) for q,k in queries(qsize=qsize,K=k,sample_size=sample_size)] # times each query
def _time(q,k,number):
"""Returns the average time that one query with Q=q and K=k takes"""
timer = Timer(grams_fn,grams_setup.format(q,k)) # make this less hard-coded
return min(timer.repeat(3,number=number))/number
def avg(ls):
return sum(ls)/float(len(ls))
def stdev(ls):
return math.sqrt(variance(ls))
def variance(ls):
return avg(map(lambda x: (x - avg(ls))**2, ls))
def stats(values):
"""Returns the sample mean, standard deviation, maxvalue and
sample size"""
return avg(values),stdev(values),max(values),len(values)
class TrialRunner(object):
def __init__(self,filename_k='plotallK.png',filename_q='plotallQ.png'):
self._k_vals = [10,50,100,500,1000]
self._q_sizes = [1,2,5,10,20]
self._colors = ["blue","red","green","yellow","purple"]
self._plot_title = 'Average Query Times by {}'
self._y_label = "Time (s)"
self._log = {'K':{},'Q':{}} #Ex: self._log['K'][10]=> all times for K=10
self._filename_k = filename_k
self._filename_q= filename_q
def plotK(self,kvals):
self._plot(x_vals=kvals,y_func=self._kys,labelstr="Q:{}",plot_vals=self._q_sizes,
title=self._plot_title.format('K-values'),xlabel='K',ylabel=self._y_label,filename=self._filename_k)
def plotQ(self,qsizes):
self._plot(x_vals=qsizes,y_func=self._qys,labelstr="K:{}",plot_vals=self._k_vals,
title=self._plot_title.format('Q-Length'),xlabel='Q-Length',ylabel=self._y_label,filename=self._filename_q)
def _kys(self,ks,qsize):
"""Returns a list of timings for k values in ks, where qsize is kept constant"""
return [self.get_time(qsize,k) for k in ks]
def _qys(self,qsizes,k):
"""Returns a list of timings for q values in qsizes, where k is kept constant"""
return [self.get_time(qsize,k) for qsize in qsizes]
def _log_times(self,qsize,k,times):
"""For each set of trials with a given qsize & k value, logs the times
generated by the trials in both the Q log and the K log"""
Qentry,Kentry = self._log['Q'].setdefault(qsize,[]),self._log['K'].setdefault(k,[])
Qentry.extend(times)
Kentry.extend(times)
def get_time(self,qsize,k):
"""Times queries of size qsize with k value = k and returns their average"""
ts = time(qsize,k)
self._log_times(qsize,k,ts)
return avg(ts)
def _plot(self,x_vals,y_func,labelstr,plot_vals,title,xlabel,ylabel,filename):
x_data = x_vals
y_colors = (color for color in self._colors)
fig = plt.figure()
ax = fig.add_subplot(1,1,1)
for plot_val in plot_vals:
color = y_colors.next()
y_vals = y_func(x_data,plot_val)
ax.scatter(x_data,y_vals,color=color,label=labelstr.format(plot_val))
ax.set_title(title)
ax.set_xlabel(xlabel)
ax.set_ylabel(ylabel)
plt.legend(loc=2)
fig.savefig(filename)
def print_stats(self,axis):
axis_log = self._log[axis]
print '\n{:^10}|{:^20}|{:^20}|{:^20}|{:^20}|'.format(axis,'Mean','StdDev','Max','Sample Size')
print '{:-^10}|{:-^20}|{:-^20}|{:-^20}{:^20}|'.format('','','','','')
for key,values in sorted(axis_log.items()):
print '{:^10}|{:^20}|{:^20}|{:^20}{:^20}|'.format(key,*stats(values))
if __name__ == '__main__':
t = TrialRunner()
t.plotQ(qsizes=range(1,30))
t.plotK(kvals=[1,2,5,10,20,50,100,200,500,1000])
t.print_stats('Q')
t.print_stats('K')