def update(self):
self.progress_.show()
n_text = DB.fetchone("""select count(*) from text""", (0,))[0]
self.progress_.inc(2)
n_res = DB.fetchone("""select count(*) from result""", (0,))[0]
self.progress_.inc(2)
n_words = DB.fetchall(
"""select count(*),sum(count) from statistic
group by type order by type"""
)
self.progress_.inc(2)
if len(n_words) != 3:
n_words = [(0, 0), (0, 0), (0, 0)]
n_first = DB.fetchone("""select w from result order by w asc limit 1""", (time.time(),))[0]
self.progress_.hide()
self.stats_.setText(
locale.format_string(
"""Texts: %d
Results: %d
Analysis data: %d (%d keys, %d trigrams, %d words)
%d characters and %d words typed total\n"""
+ ("First result was %.2f days ago.\n" % ((time.time() - n_first) / 86400.0)),
tuple(
[n_text, n_res, sum(map(lambda x: x[0], n_words))]
+ map(lambda x: x[0], n_words)
+ [n_words[0][1], n_words[2][1]]
),
True,
)
)
def nextText(self):
type = Settings.get('select_method')
if type != 1:
# Not in order
v = DB.execute("select id,source,text from text where disabled is null order by random() limit %d" % Settings.get('num_rand')).fetchall()
if len(v) == 0:
v = None
elif type == 2:
v = min(v, key=self.diff_eval)
elif type == 3:
v = max(v, key=self.diff_eval)
else:
v = v[0] # random, just pick the first
else:
# Fetch in order
lastid = (0,)
g = DB.fetchone("""select r.text_id
from result as r left join source as s on (r.source = s.rowid)
where (s.discount is null) or (s.discount = 1) order by r.w desc limit 1""", None)
if g is not None:
lastid = DB.fetchone("select rowid from text where id = ?", lastid, g)
v = DB.fetchone("select id,source,text from text where rowid > ? and disabled is null order by rowid asc limit 1", None, lastid)
if v is None:
v = self.defaultText
self.emit(SIGNAL("setText"), v)
def populateData(self, idxs):
if len(idxs) == 0:
return map(
list,
DB.fetchall("""
select s.rowid,s.name,t.count,r.count,r.wpm,ifelse(nullif(t.dis,t.count),'No','Yes')
from source as s
left join (select source,count(*) as count,count(disabled) as dis from text group by source) as t
on (s.rowid = t.source)
left join (select source,count(*) as count,avg(wpm) as wpm from result group by source) as r
on (t.source = r.source)
where s.disabled is null
order by s.name"""))
if len(idxs) > 1:
return []
r = self.rows[idxs[0]]
return map(
list,
DB.fetchall(
"""select t.rowid,substr(t.text,0,40)||"...",length(t.text),r.count,r.m,ifelse(t.disabled,'Yes','No')
from (select rowid,* from text where source = ?) as t
left join (select text_id,count(*) as count,agg_median(wpm) as m from result group by text_id) as r
on (t.id = r.text_id)
order by t.rowid""", (r[0], )))
def cleanup(self):
s_in_day = 24*60*60
now = time.time()
pending = []
for idx, grp, lim in [
(1, 30, Settings.get("group_month")),
(2, 7, Settings.get("group_week")),
(3, 1, Settings.get("group_day")),
]:
minimum = now - s_in_day * lim
binsize = s_in_day * grp
pending.extend(DB.fetchall(f"""
select avg(w), data, type, agg_mean(time, count), sum(count), sum(mistakes),
agg_median(viscosity)
from statistic where w <= {minimum}
group by data, type, cast(w/{binsize} as int)"""))
self.progressbar.set_fraction(idx/5)
DB.executemany("""insert into statistic (w, data, type, time, count, mistakes, viscosity)
values (?,?,?,?,?,?,?)""", pending)
self.progressbar.set_fraction(4/5)
# FIXME vacuum not supported
# DB.execute("vacuum")
self.progressbar.set_fraction(5/5)
DB.commit()
self.progressbar.set_fraction(0)
def next_text(self):
kind = Settings.get("select_method")
if kind != 1:
# Not in order
targets = DB.execute(
f"""select id,source,text from text where disabled is null
order by random() limit {Settings.get("num_rand")}"""
).fetchall()
if not targets:
target = None
elif kind == 2:
target = min(targets, key=self.diff_eval)
elif kind == 3:
target = max(targets, key=self.diff_eval)
else:
target = targets[0] # random, just pick the first
else:
# Fetch in order
prev = (0, )
result = DB.fetchone(
"""select r.text_id
from result as r left join source as s on (r.source = s.rowid)
where (s.discount is null) or (s.discount = 1) order by r.w desc limit 1""",
None)
if result is not None:
prev = DB.fetchone("select rowid from text where id = ?", prev,
result)
target = DB.fetchone(
"""select id,source,text from text
where rowid > ? and disabled is null order by rowid asc limit 1""",
None, prev)
if target is None:
target = self.default_text
self.emit("set-text", *target)
def populateData(self, idxs):
if len(idxs) == 0:
return map(
list,
DB.fetchall(
"""
select s.rowid,s.name,t.count,r.count,r.wpm,ifelse(nullif(t.dis,t.count),'No','Yes')
from source as s
left join (select source,count(*) as count,count(disabled) as dis from text group by source) as t
on (s.rowid = t.source)
left join (select source,count(*) as count,avg(wpm) as wpm from result group by source) as r
on (t.source = r.source)
where s.disabled is null
order by s.name"""
),
)
if len(idxs) > 1:
return []
r = self.rows[idxs[0]]
return map(
list,
DB.fetchall(
"""select t.rowid,substr(t.text,0,40)||"...",length(t.text),r.count,r.m,ifelse(t.disabled,'Yes','No')
from (select rowid,* from text where source = ?) as t
left join (select text_id,count(*) as count,agg_median(wpm) as m from result group by text_id) as r
on (t.id = r.text_id)
order by t.rowid""",
(r[0],),
),
)
def update(self):
self.progress_.show()
n_text = DB.fetchone('''select count(*) from text''', (0, ))[0]
self.progress_.inc(2)
n_res = DB.fetchone('''select count(*) from result''', (0, ))[0]
self.progress_.inc(2)
n_words = DB.fetchall('''select count(*), sum(count) from statistic
group by type order by type''')
self.progress_.inc(2)
if len(n_words) != 3:
n_words = [(0, 0), (0, 0), (0, 0)]
n_first = DB.fetchone(
'''select w from result order by w asc limit 1''',
(time.time(), ))[0]
self.progress_.hide()
self.stats_.setText(
locale.format_string(
'''Texts: %d
Results: %d
Analysis data: %d (%d keys, %d trigrams, %d words)
%d characters and %d words typed total\n''' +
("First result was %.2f days ago.\n" %
((time.time() - n_first) / 86400.0)),
tuple([n_text, n_res,
sum(map(lambda x: x[0], n_words))] +
list(map(lambda x: x[0], n_words)) +
[n_words[0][1], n_words[2][1]]), True))
def removeSelected(self):
cats, texts = self.getSelected()
DB.executemany("delete from text where rowid = ?",
map(lambda x:(x, ), texts))
self.removeUnused()
self.update()
DB.commit()
def removeSelected(self):
cats, texts = self.getSelected()
DB.executemany("delete from text where rowid = ?",
map(lambda x: (x, ), texts))
self.removeUnused()
self.update()
DB.commit()
def nextText(self):
type = Settings.get('select_method')
if type != 1:
# Not in order
v = DB.execute("select id,source,text from text where disabled is null order by random() limit %d" % Settings.get('num_rand')).fetchall()
if len(v) == 0:
v = None
elif type == 2:
v = min(v, key=self.diff_eval)
elif type == 3:
v = max(v, key=self.diff_eval)
else:
v = v[0] # random, just pick the first
else:
# Fetch in order
lastid = (0,)
g = DB.fetchone("""select r.text_id
from result as r left join source as s on (r.source = s.rowid)
where (s.discount is null) or (s.discount = 1) order by r.w desc limit 1""", None)
if g is not None:
lastid = DB.fetchone("select rowid from text where id = ?", lastid, g)
v = DB.fetchone("select id,source,text from text where rowid > ? and disabled is null order by rowid asc limit 1", None, lastid)
if v is None:
v = self.defaultText
self.emit(SIGNAL("setText"), v)
def insertStats(self, now, vals):
DB.executemany_(
'''insert into statistic
(time, viscosity, w, count, mistakes, type, data, source) values (?, ?, ?, ?, ?, ?, ?, ?)''',
vals)
DB.executemany_(
'insert into mistake (w, target, mistake, count) values (?, ?, ?, ?)',
[(now, k[0], k[1], v)
for k, v in self.typer.getMistakes().items()])
def updateData(self, *args):
if self.editflag:
return
where = []
if self.cb_source.currentIndex() <= 0:
pass
elif self.cb_source.currentIndex() == 1: # last text
where.append(
'r.text_id = (select text_id from result order by w desc limit 1)'
)
elif self.cb_source.currentIndex() == 2: # all texts
where.append('s.discount is null')
elif self.cb_source.currentIndex() == 3: # all lessons texts
where.append('s.discount is not null')
else:
s = self.cb_source.itemData(self.cb_source.currentIndex())
where.append('r.source = %d' % s.toInt()[0])
if len(where) > 0:
where = 'where ' + ' and '.join(where)
else:
where = ""
g = Settings.get('perf_group_by')
if g == 0: # no grouping
sql = '''select text_id,w,s.name,wpm,100.0*accuracy,viscosity
from result as r left join source as s on (r.source = s.rowid)
%s %s
order by w desc limit %d'''
elif g:
sql = '''select agg_first(text_id),avg(r.w) as w,count(r.rowid) || ' result(s)',agg_median(r.wpm),
100.0*agg_median(r.accuracy),agg_median(r.viscosity)
from result as r left join source as s on (r.source = s.rowid)
%s %s
order by w desc limit %d'''
group = ''
if g == 1: # by Settings.get('def_group_by')
DB.resetCounter()
gn = Settings.get('def_group_by')
if gn <= 1:
gn = 1
group = "group by cast(counter()/%d as int)" % gn
elif g == 2: # by sitting
mis = Settings.get('minutes_in_sitting') * 60.0
DB.resetTimeGroup()
group = "group by time_group(%f, r.w)" % mis
elif g == 3: # by day
group = "group by cast((r.w+4*3600)/86400 as int)"
n = Settings.get("perf_items")
sql = sql % (where, group, n)
self.model.setData(map(list, DB.fetchall(sql)))
self.updateGraph()
def updateData(self, *args):
if self.editflag:
return
where = []
if self.cb_source.currentIndex() <= 0:
pass
elif self.cb_source.currentIndex() == 1: # last text
where.append(
"r.text_id = (select text_id from result order by w desc limit 1)"
)
elif self.cb_source.currentIndex() == 2: # all texts
where.append("s.discount is null")
elif self.cb_source.currentIndex() == 3: # all lessons texts
where.append("s.discount is not null")
else:
s = self.cb_source.itemData(self.cb_source.currentIndex())
where.append("r.source = %d" % s.toInt()[0])
if len(where) > 0:
where = "where " + " and ".join(where)
else:
where = ""
g = Settings.get("perf_group_by")
if g == 0: # no grouping
sql = """select text_id,w,s.name,wpm,100.0*accuracy,viscosity
from result as r left join source as s on (r.source = s.rowid)
%s %s
order by w desc limit %d"""
elif g:
sql = """select agg_first(text_id),avg(r.w) as w,count(r.rowid) || ' result(s)',agg_median(r.wpm),
100.0*agg_median(r.accuracy),agg_median(r.viscosity)
from result as r left join source as s on (r.source = s.rowid)
%s %s
order by w desc limit %d"""
group = ""
if g == 1: # by Settings.get('def_group_by')
DB.resetCounter()
gn = Settings.get("def_group_by")
if gn <= 1:
gn = 1
group = "group by cast(counter()/%d as int)" % gn
elif g == 2: # by sitting
mis = Settings.get("minutes_in_sitting") * 60.0
DB.resetTimeGroup()
group = "group by time_group(%f, r.w)" % mis
elif g == 3: # by day
group = "group by cast((r.w+4*3600)/86400 as int)"
n = Settings.get("perf_items")
sql = sql % (where, group, n)
self.model.setData(map(list, DB.fetchall(sql)))
self.updateGraph()
def cleanup(self):
day = 24 * 60 * 60
now = time.time()
q = []
self.progress_.show()
for grp, lim in [(30.0, Settings.get('group_month')),
(7.0, Settings.get('group_week')),
(1.0, Settings.get('group_day'))]:
w = now - day * lim
g = grp * day
q.extend(
DB.fetchall('''
select avg(w), data, type, agg_mean(time, count), sum(count), sum(mistakes), agg_median(viscosity)
from statistic where w <= %f
group by data, type, cast(w/%f as int)''' % (w, g)))
self.progress_.inc()
DB.execute('''delete from statistic where w <= ?''', (w, ))
self.progress_.inc()
DB.executemany(
'''insert into statistic (w, data, type, time, count, mistakes, viscosity)
VALUES (?, ?, ?, ?, ?, ?, ?)''', q)
self.progress_.inc()
DB.commit()
DB.execute('vacuum')
self.progress_.inc()
self.progress_.hide()
def cleanup(self):
day = 24*60*60
now = time.time()
q = []
self.progress_.show()
for grp, lim in [(30.0, Settings.get('group_month')),
(7.0, Settings.get('group_week')),
(1.0, Settings.get('group_day'))]:
w = now - day*lim
g = grp * day
q.extend(DB.fetchall('''
select avg(w), data, type, agg_mean(time, count), sum(count), sum(mistakes), agg_median(viscosity)
from statistic where w <= %f
group by data, type, cast(w/%f as int)''' % (w, g)))
self.progress_.inc()
DB.execute('''delete from statistic where w <= ?''', (w, ))
self.progress_.inc()
DB.executemany('''insert into statistic (w, data, type, time, count, mistakes, viscosity)
VALUES (?, ?, ?, ?, ?, ?, ?)''', q)
self.progress_.inc()
DB.execute('vacuum')
self.progress_.inc()
DB.commit()
self.progress_.hide()
def toggle_selected(self):
model, paths = self.tree.get_selection().get_selected_rows()
for path in paths:
if model.iter_depth(model.get_iter(path)) == 0:
continue
row = Gtk.TreeModelRow(model, path)
DB.execute("update text set disabled = 1 where rowid=?",
(row[0], ))
self.update()
DB.commit()
def update_data(self):
if self.editflag:
return
where = []
where_query = ""
selected = self.cb_source.get_active_id()
if selected == "last text":
where.append(
"r.text_id = (select text_id from result order by w desc limit 1)"
)
elif selected == "all texts":
where.append("s.discount is null")
elif selected == "all lessons":
where.append("s.discount is not null")
elif selected and selected.isdigit():
rowid = int(selected)
where.append(f"r.source = {rowid}")
if where:
where_query = "where " + " and ".join(where)
sql_template = """select agg_first(text_id),avg(r.w) as w,count(r.rowid)
|| ' result(s)',agg_median(r.wpm),
100.0*agg_median(r.accuracy),agg_median(r.viscosity)
from result as r left join source as s on (r.source = s.rowid)
%s %s
order by w desc limit %d"""
groupby = Settings.get("perf_group_by")
group = ""
print(groupby)
if groupby == 1: # by def_group_by
DB.reset_counter()
group = "group by cast(counter()/%d as int)" % max(
Settings.get("def_group_by"), 1)
elif groupby == 2: # by sitting
mis = Settings.get("minutes_in_sitting") * 60.0
DB.reset_time_group()
group = "group by time_group(%f, r.w)" % mis
elif groupby == 3: # by day
group = "group by cast((r.w+4*3600)/86400 as int)"
elif not groupby: # no grouping
sql_template = """select text_id,w,s.name,wpm,100.0*accuracy,viscosity
from result as r left join source as s on (r.source = s.rowid)
%s %s
order by w desc limit %d"""
items = Settings.get("perf_items")
sql = sql_template % (where_query, group, items)
self.model.set_stats([list(r) for r in DB.fetchall(sql)])
self.update_graph()
def setImpList(self, files):
self.sender().hide()
self.progress.show()
for x in map(unicode, files):
self.progress.setValue(0)
fname = path.basename(x)
lm = LessonMiner(x)
self.connect(lm, SIGNAL("progress(int)"), self.progress.setValue)
self.addTexts(fname, lm, update=False)
self.progress.hide()
self.update()
DB.commit()
def setImpList(self, files):
self.sender().hide()
self.progress.show()
for x in map(unicode, files):
self.progress.setValue(0)
fname = path.basename(x)
lm = LessonMiner(x)
self.connect(lm, SIGNAL("progress(int)"), self.progress.setValue)
self.addTexts(fname, lm, update=False)
self.progress.hide()
self.update()
DB.commit()
def addTexts(self, source, texts, lesson=None, update=True):
id = DB.getSource(source, lesson)
r = []
for x in texts:
h = hashlib.sha1()
h.update(x.encode('utf-8'))
txt_id = h.hexdigest()
dis = 1 if lesson == 2 else None
try:
DB.execute("insert into text (id,text,source,disabled) values (?,?,?,?)",
(txt_id, x, id, dis))
r.append(txt_id)
except Exception, e:
pass # silently skip ...
def lastText(self):
# Fetch in order
lastResultGuid = DB.fetchone("""select r.text_id
from result as r left join source as s on (r.source = s.rowid)
where (s.discount is null) or (s.discount = 1) order by r.w desc limit 1""", None)
if lastResultGuid is not None:
v = DB.fetchone("select id, source, text from text where id = ?", None, lastResultGuid)
else:
v = self.defaultText
if v is None:
v = self.defaultText
self.emit(SIGNAL("setText"), v)
def addTexts(self, source, texts, lesson=None, update=True):
id = DB.getSource(source, lesson)
r = []
for x in texts:
h = hashlib.sha1()
h.update(x.encode('utf-8'))
txt_id = h.hexdigest()
dis = 1 if lesson == 2 else None
try:
DB.execute(
"insert into text (id,text,source,disabled) values (?,?,?,?)",
(txt_id, x, id, dis))
r.append(txt_id)
except Exception, e:
pass # silently skip ...
def newReview(self, review):
q = self.addTexts("<Reviews>", [review], lesson=2, update=False)
if q:
v = DB.fetchone("select id,source,text from text where id = ?", self.defaultText, q)
self.emit(SIGNAL("setText"), v)
else:
self.nextText()
def addFromTyped(self):
words = [
x[0] for x in DB.fetchall(
'select distinct data from statistic where type = 2 order by random()'
)
]
self.filterWords(words)
def newReview(self, review):
q = self.addTexts("<Reviews>", [review], lesson=2, update=False)
if q:
v = DB.fetchone("select id,source,text from text where id = ?", self.defaultText, q)
self.emit(SIGNAL("setText"), v)
else:
self.nextText()
def AddUPCItem(self, strUPC):
if(self.Closed==True):
return False
for item in self.Items:
if(item.RID==strUPC and item.Type==CartItem.ItemType.UPCItem):
item.Qty+=1
self.CalculateCart()
return True
ci=CartItem()
UPCItemRS=posDB.GetUPCItemDetails(strUPC)
if(UPCItemRS==[]):
#display screen for new item
print("to do - new item add")
return False
UPCItemRS=UPCItemRS[0]
ci.RID=strUPC
ci.Type=CartItem.ItemType.UPCItem
ci.Name=UPCItemRS[0]
ci.Desc=UPCItemRS[1]
ci.Price=UPCItemRS[2]
ci.TaxRate=UPCItemRS[3]
ci.Qty=1
self.Items.append(ci)
self.CalculateCart()
self.SaveCartToFile()
return True
def LoadFoodItems(event, arg):
print("to do: LoadFoodItems - Add style to buttons", arg)
#multiple pages
#style buttons
#different size buttons
#print(formRegister.frameItems.children)
#remove all existing buttons
for child in formRegister.frameItems.winfo_children():
child.destroy()
#create buttons
intRow = 0
intCol = -1
curFoodItemGroups = posDB.GetFoodItemsByGroupRID(arg)
for rowTab in curFoodItemGroups:
if (intCol == 6):
if (intRow == 10):
#stop loading
break
intCol = 0
intRow += 1
else:
intCol += 1
btn = tk.Button(formRegister.frameItems)
btn.place(x=intCol * 105, y=intRow * 50 + 10, height=42, width=97)
btn.configure(text=rowTab[2])
btn.bind('<Button-1>',
lambda event, arg=rowTab[0]: AddFoodItemToCart(event, arg))
def setTarget(self, text, guid):
self.editflag = True
self.target = text
self.when = [0] * (len(self.target) + 1)
# time for each character typed
self.times = [0] * len(self.target)
# whether each character was a mistake
self.mistake = [False] * len(self.target)
# mistake characters ( must be what was actually typed )
self.mistakes = {} #collections.defaultdict(lambda: [])
self.where = 0
self.clear()
self.setPalette(self.palettes['inactive'])
self.setText(self.getWaitText())
self.selectAll()
self.editflag = False
self.is_lesson = DB.fetchone(
"select discount from source where rowid=?", (None, ), (guid, ))[0]
if self.is_lesson:
self.mins = (Settings.get("min_lesson_wpm"),
Settings.get("min_lesson_acc"))
else:
self.mins = (Settings.get("min_wpm"), Settings.get("min_acc"))
def getStats(self):
if self.when[0] == -1:
t = self.times[1:]
t.sort(reverse=True)
v = DB.fetchone('select time from statistic where type = 0 and data = ? order by rowid desc limit 1', (t[len(t)//5], ), (self.target[0], ))
self.times[0] = v[0]
self.when[0] = self.when[1] - self.times[0]
return self.when[self.where]-self.when[0], self.where, self.times, self.mistake, self.getMistakes()
def new_review(self, review):
added = self.add_texts("<Reviews>", [review], lesson=2, update=False)
if added:
tgt = DB.fetchone("select id,source,text from text where id = ?",
self.default_text, added)
self.emit("set-text", *tgt)
else:
self.next_text()
def addFromTyped(self):
words = [
x[0]
for x in DB.fetchall(
"select distinct data from statistic where type = 2 order by random()"
)
]
self.filterWords(words)
def double_clicked(self, treeview, where, _column):
row = Gtk.TreeModelRow(treeview.get_model(), where)
target = DB.fetchone("select id,source,text from text where id = ?",
None, (row[0], ))
if target is None:
return
self.emit("set-text", *target)
self.emit("go-to-text")
def getStats(self):
if self.when[0] == -1:
t = self.times[1:]
t.sort(reverse=True)
v = DB.fetchone('select time from statistic where type = 0 and data = ? order by rowid desc limit 1', (t[len(t)//5], ), (self.target[0], ))
self.times[0] = v[0]
self.when[0] = self.when[1] - self.times[0]
return self.when[self.where]-self.when[0], self.where, self.times, self.mistake, self.getMistakes()
def updateResultLabel(self):
spc = self.typer.getSpeed()
accuracy = self.typer.getAccuracy()
v2 = DB.fetchone("""select agg_median(wpm), agg_median(acc) from
(select wpm, 100.0*accuracy as acc from result order by w desc limit %d)""" % Settings.get('def_group_by'), (0.0, 100.0))
if Settings.get('show_since_fail_counter'):
self.result.setText("Last: %.1fwpm (%.1f%%), last 10 average: %.1fwpm (%.1f%%) \n\nPerfect Count: (Current :%1d) (Max : %1d) (Last : %1d)" % ((spc, 100.0*accuracy) + v2 + ( self.typer.count, self.typer.max_count, self.typer.last_count)))
else:
self.result.setText("Last: %.1fwpm (%.1f%%), last 10 average: %.1fwpm (%.1f%%)" % ((spc, 100.0*accuracy) + v2 ))
def getStats(self):
if self.when[0] == -1:
# my refactoring mean this may never get hit, I'm not sure what when and times are for, so i'm not sure if I'm breaking some edge case here??
t = self.times[1:]
t.sort(reverse=True)
v = DB.fetchone('select time from statistic where type = 0 and data = ? order by rowid desc limit 1', (t[len(t)//5], ), (self.target[0], ))
self.times[0] = v[0]
self.when[0] = self.when[1] - self.times[0]
return self.getElapsed(), self.where, self.times, self.mistake, self.getMistakes()
def doubleClicked(self, idx):
r = self.model.rows[idx.row()]
v = DB.fetchone('select id,source,text from text where id = ?', None, (r[0], ))
if v == None:
return # silently ignore
self.emit(SIGNAL("setText"), v)
self.emit(SIGNAL("gotoText"))
def removeUnused(self):
DB.execute('''
delete from source where rowid in (
select s.rowid from source as s
left join result as r on (s.rowid=r.source)
left join text as t on (t.source=s.rowid)
group by s.rowid
having count(r.rowid) = 0 and count(t.rowid) = 0
)''')
DB.execute('''
update source set disabled = 1 where rowid in (
select s.rowid from source as s
left join result as r on (s.rowid=r.source)
left join text as t on (t.source=s.rowid)
group by s.rowid
having count(r.rowid) > 0 and count(t.rowid) = 0
)''')
self.emit(SIGNAL("refreshSources"))
def updateResultLabel(self):
spc = self.typer.getSpeed()
accuracy = self.typer.getAccuracy()
v2 = DB.fetchone("""select agg_median(wpm), agg_median(acc) from
(select wpm, 100.0*accuracy as acc from result order by w desc limit %d)""" % Settings.get('def_group_by'), (0.0, 100.0))
if Settings.get('show_since_fail_counter'):
self.result.setText("Last: %.1fwpm (%.1f%%), last 10 average: %.1fwpm (%.1f%%) \n\nPerfect Count: (Current :%1d) (Max : %1d) (Last : %1d)" % ((spc, 100.0*accuracy) + v2 + ( self.typer.count, self.typer.max_count, self.typer.last_count)))
else:
self.result.setText("Last: %.1fwpm (%.1f%%), last 10 average: %.1fwpm (%.1f%%)" % ((spc, 100.0*accuracy) + v2 ))
def removeUnused(self):
DB.execute('''
delete from source where rowid in (
select s.rowid from source as s
left join result as r on (s.rowid=r.source)
left join text as t on (t.source=s.rowid)
group by s.rowid
having count(r.rowid) = 0 and count(t.rowid) = 0
)''')
DB.execute('''
update source set disabled = 1 where rowid in (
select s.rowid from source as s
left join result as r on (s.rowid=r.source)
left join text as t on (t.source=s.rowid)
group by s.rowid
having count(r.rowid) > 0 and count(t.rowid) = 0
)''')
self.emit(SIGNAL("refreshSources"))
def getStats(self):
if self.when[0] == -1:
# my refactoring mean this may never get hit, I'm not sure what when and times are for, so i'm not sure if I'm breaking some edge case here??
t = self.times[1:]
t.sort(reverse=True)
v = DB.fetchone('select time from statistic where type = 0 and data = ? order by rowid desc limit 1', (t[len(t)//5], ), (self.target[0], ))
self.times[0] = v[0]
self.when[0] = self.when[1] - self.times[0]
return self.getElapsed(), self.where, self.times, self.mistake, self.getMistakes()
def get_stats(self):
if self.when[0] == -1:
times = sorted(self.times[1:], reverse=True)
self.times[0] = DB.fetchone(
"select time from statistic where type=0 and data=? order by rowid desc limit 1",
(times[len(times) // 5], ), (self.target[0], ))[0]
self.when[0] = self.when[1] - self.times[0]
return (self.when[self.where] - self.when[0], self.where, self.times,
self.mistake, self.get_mistakes())
def add_files(self):
filepicker = Gtk.FileChooserDialog()
filepicker.add_button(Gtk.STOCK_CANCEL, Gtk.ResponseType.CANCEL)
filepicker.add_button(Gtk.STOCK_OPEN, Gtk.ResponseType.ACCEPT)
result = filepicker.run()
fname = filepicker.get_filename()
filepicker.destroy()
if result == Gtk.ResponseType.CANCEL or fname is None:
return
lminer = LessonMiner(fname)
lminer.connect("progress",
lambda p: self.progress.set_fraction(p / 100))
self.add_texts(fname, lminer, update=False)
self.progress.set_fraction(0)
self.update()
DB.commit()
def doubleClicked(self, idx):
p = idx.parent()
if not p.isValid():
return
q = self.model.data(idx, Qt.UserRole)
v = DB.fetchall('select id,source,text from text where rowid = ?', (q[0], ))
self.cur = v[0] if len(v) > 0 else self.defaultText
self.emit(SIGNAL("setText"), self.cur)
self.emit(SIGNAL("gotoText"))
def doubleClicked(self, idx):
p = idx.parent()
if not p.isValid():
return
q = self.model.data(idx, Qt.UserRole)
v = DB.fetchall("select id,source,text from text where rowid = ?", (q[0],))
self.cur = v[0] if len(v) > 0 else self.defaultText
self.emit(SIGNAL("setText"), self.cur)
self.emit(SIGNAL("gotoText"))
def refreshSources(self):
self.editflag = True
self.cb_source.clear()
self.cb_source.addItem("<ALL>")
self.cb_source.addItem("<LAST TEXT>")
self.cb_source.addItem("<ALL TEXTS>")
self.cb_source.addItem("<ALL LESSONS>")
for id, v in DB.fetchall('select rowid,abbreviate(name,30) from source order by name'):
self.cb_source.addItem(v, QVariant(id))
self.editflag = False
def addTexts(self, source, texts, lesson=None, update=True):
id = DB.getSource(source, lesson)
r = []
for x in texts:
x = re.sub(Settings.get('sentence_strip'), ' ', x)
h = hashlib.sha1()
h.update(x.encode('utf-8'))
txt_id = h.hexdigest()
dis = 1 if lesson == 2 else None
try:
DB.execute("insert into text (id, text, source, disabled) values (?, ?, ?, ?)", (txt_id, x, id, dis))
except Exception:
pass # silently skip ...
r.append(txt_id)
if update:
self.update()
if lesson:
DB.commit()
return r
def disableSelected(self):
cats, texts = self.getSelected()
DB.setRegex(Settings.get('text_regex'))
DB.executemany("""update text set disabled = ifelse(disabled,NULL,1)
where rowid = ? and regex_match(text) = 1""",
map(lambda x:(x, ), texts))
DB.executemany("""update text set disabled = ifelse(disabled,NULL,1)
where source = ? and regex_match(text) = 1""",
map(lambda x:(x, ), cats))
self.update()
def getStats(self):
#TODO: redo when, times to avoid guessing time taken to hit zeroth char from old stat
if self.when[0] == -1:
t = self.times[1:]
t.sort(reverse=True)
v = DB.fetchone('select time from statistic where type = 0 and data = ? order by rowid desc limit 1', (t[len(t)//5], ), (self.target[0], ))
self.when[0] = self.when[1] - v[0]
if not self.when[-1]:
self.when[-1] = timer()
self.when = list(linearly_interpolate(self.when))
for i in range(len(self.times)):
#prevent division by zero when 0 time
time = self.when[i+1] - self.when[i]
self.times[i] = MINIMUM_CHAR_TYPING_TIME if time == 0 else time
return self.when[-1]-self.when[0], len(self.target), self.times, self.mistake, self.getMistakes()
def __init__(self, *args):
super(StringStats, self).__init__(*args)
self.model = WordModel()
tw = AmphTree(self.model)
tw.setIndentation(0)
tw.setUniformRowHeights(True)
tw.setRootIsDecorated(False)
self.stats = tw
ob = SettingsCombo('ana_which', [
('wpm asc', 'slowest'),
('wpm desc', 'fastest'),
('viscosity desc', 'least fluid'),
('viscosity asc', 'most fluid'),
('accuracy asc', 'least accurate'),
('misses desc', 'most mistyped'),
('total desc', 'most common'),
('damage desc', 'most damaging'),
])
wc = SettingsCombo('ana_what', ['keys', 'trigrams', 'words'])
lim = SettingsEdit('ana_many')
s = DB.fetchall("select rowid, name from source")
source = SettingsCombo('ana_source', [
(-1, 'all')
] + s)
self.w_count = SettingsEdit('ana_count')
self.connect(Settings, SIGNAL("change_ana_which"), self.update)
self.connect(Settings, SIGNAL("change_ana_what"), self.update)
self.connect(Settings, SIGNAL("change_ana_many"), self.update)
self.connect(Settings, SIGNAL("change_ana_count"), self.update)
self.connect(Settings, SIGNAL("history"), self.update)
self.setLayout(AmphBoxLayout([
["Display statistics about the", ob, wc, "from ", source, None, AmphButton("Update List", self.update)],
["Limit list to", lim, "items of count ", self.w_count," and older than", SettingsEdit("history"), "hours",
None, AmphButton("Send List to Lesson Generator",
lambda: self.emit(SIGNAL("lessonStrings"), [x[0] for x in self.model.words]))],
(self.stats, 1)
]))
def setSelect(self, v):
if v == 0 or v == 1:
self.diff_eval = lambda x: 1
self.nextText()
return
hist = time.time() - 86400.0 * Settings.get("history")
tri = dict(
DB.execute(
"""
select data,agg_median(time) as wpm from statistic
where w >= ? and type = 1
group by data""",
(hist,),
).fetchall()
) # [(t, (m, c)) for t, m, c in
g = tri.values()
if len(g) == 0:
return lambda x: 1
g.sort(reverse=True)
expect = g[len(g) // 4]
def _func(v):
text = v[2]
v = 0
s = 0.0
for i in xrange(0, len(text) - 2):
t = text[i : i + 3]
if t in tri:
s += tri[t]
else:
# print "|", t,
s += expect
v += 1
avg = s / (len(text) - 2)
# print text
# print " v=%d,s=%f" % (v, 12.0/avg), "ex:", expect
return 12.0 / avg
self.diff_eval = _func
self.nextText()
def update(self, *arg):
ord = Settings.get('ana_which')
cat = Settings.get('ana_what')
limit = Settings.get('ana_many')
count = Settings.get('ana_count')
hist = time.time() - Settings.get('history') * 86400.0
sql = """select data,12.0/time as wpm,
100.0-100.0*misses/cast(total as real) as accuracy,
viscosity,total,misses,
total*time*time*(1.0+misses/total) as damage
from
(select data,agg_median(time) as time,agg_median(viscosity) as viscosity,
sum(count) as total,sum(mistakes) as misses
from statistic where w >= ? and type = ? group by data)
where total >= ?
order by %s limit %d""" % (ord, limit)
self.model.setData(DB.fetchall(sql, (hist, cat, count)))
def setTarget(self, text, guid):
self.editflag = True
self.target = text
self.when = [0] * (len(self.target)+1)
# time for each character typed
self.times = [0] * len(self.target)
# whether each character was a mistake
self.mistake = [False] * len(self.target)
# mistake characters ( must be what was actually typed )
self.mistakes = {} #collections.defaultdict(lambda: [])
self.where = 0
self.clear()
self.setPalette(self.palettes['inactive'])
self.setText(self.getWaitText())
self.selectAll()
self.editflag = False
self.is_lesson = DB.fetchone("select discount from source where rowid=?", (None, ), (guid, ))[0]
if self.is_lesson:
self.mins = (Settings.get("min_lesson_wpm"), Settings.get("min_lesson_acc"))
else:
self.mins = (Settings.get("min_wpm"), Settings.get("min_acc"))
def isLesson(self):
is_lesson = DB.fetchone("select discount from source where rowid=?", (None, ), (self.text[1], ))[0]
return is_lesson
def insertStats(self, now, vals):
DB.executemany_('''insert into statistic
(time, viscosity, w, count, mistakes, type, data, source) values (?, ?, ?, ?, ?, ?, ?, ?)''', vals)
DB.executemany_('insert into mistake (w, target, mistake, count) values (?, ?, ?, ?)',
[(now, k[0], k[1], v) for k, v in self.typer.getMistakes().iteritems()])
def done(self):
now = time.time()
elapsed, chars, times, mis, mistakes = self.typer.getStats()
assert chars == len(self.text[2])
accuracy = 1.0 - len(filter(None, mis)) / chars
spc = elapsed / chars
viscosity = sum(map(lambda x: ((x-spc)/spc)**2, times)) / chars
DB.execute('insert into result (w,text_id,source,wpm,accuracy,viscosity) values (?,?,?,?,?,?)',
(now, self.text[0], self.text[1], 12.0/spc, accuracy, viscosity))
v2 = DB.fetchone("""select agg_median(wpm),agg_median(acc) from
(select wpm,100.0*accuracy as acc from result order by w desc limit %d)""" % Settings.get('def_group_by'), (0.0, 100.0))
self.result.setText("Last: %.1fwpm (%.1f%%), last 10 average: %.1fwpm (%.1f%%)"
% ((12.0/spc, 100.0*accuracy) + v2))
self.emit(SIGNAL("statsChanged"))
stats = collections.defaultdict(Statistic)
visc = collections.defaultdict(Statistic)
text = self.text[2]
for c, t, m in zip(text, times, mis):
stats[c].append(t, m)
visc[c].append(((t-spc)/spc)**2)
def gen_tup(s, e):
perch = sum(times[s:e])/(e-s)
visc = sum(map(lambda x: ((x-perch)/perch)**2, times[s:e]))/(e-s)
return (text[s:e], perch, len(filter(None, mis[s:e])), visc)
for tri, t, m, v in [gen_tup(i, i+3) for i in xrange(0, chars-2)]:
stats[tri].append(t, m > 0)
visc[tri].append(v)
regex = re.compile(r"(\w|'(?![A-Z]))+(-\w(\w|')*)*")
for w, t, m, v in [gen_tup(*x.span()) for x in regex.finditer(text) if x.end()-x.start() > 3]:
stats[w].append(t, m > 0)
visc[w].append(v)
def type(k):
if len(k) == 1:
return 0
elif len(k) == 3:
return 1
return 2
vals = []
for k, s in stats.iteritems():
v = visc[k].median()
vals.append( (s.median(), v*100.0, now, len(s), s.flawed(), type(k), k) )
is_lesson = DB.fetchone("select discount from source where rowid=?", (None,), (self.text[1], ))[0]
if Settings.get('use_lesson_stats') or not is_lesson:
DB.executemany_('''insert into statistic
(time,viscosity,w,count,mistakes,type,data) values (?,?,?,?,?,?,?)''', vals)
DB.executemany_('insert into mistake (w,target,mistake,count) values (?,?,?,?)',
[(now, k[0], k[1], v) for k, v in mistakes.iteritems()])
if is_lesson:
mins = (Settings.get("min_lesson_wpm"), Settings.get("min_lesson_acc"))
else:
mins = (Settings.get("min_wpm"), Settings.get("min_acc"))
if 12.0/spc < mins[0] or accuracy < mins[1]/100.0:
self.setText(self.text)
elif not is_lesson and Settings.get('auto_review'):
ws = filter(lambda x: x[5] == 2, vals)
if len(ws) == 0:
self.emit(SIGNAL("wantText"))
return
ws.sort(key=lambda x: (x[4],x[0]), reverse=True)
i = 0
while ws[i][4] != 0:
i += 1
i += (len(ws) - i) // 4
self.emit(SIGNAL("wantReview"), map(lambda x:x[6], ws[0:i]))
else:
self.emit(SIGNAL("wantText"))
def enableAll(self):
DB.execute("update text set disabled = null where disabled is not null")
self.update()
def removeDisabled(self):
DB.execute("delete from text where disabled is not null")
self.removeUnused()
self.update()
DB.commit()
