def cleanup(self):
day = 24 * 60 * 60
now = time.time()
q = []
self.progress_.show()
for grp, lim in [(30.0, Settings.get('group_month')),
(7.0, Settings.get('group_week')),
(1.0, Settings.get('group_day'))]:
w = now - day * lim
g = grp * day
q.extend(
DB.fetchall('''
select avg(w), data, type, agg_mean(time, count), sum(count), sum(mistakes), agg_median(viscosity)
from statistic where w <= %f
group by data, type, cast(w/%f as int)''' % (w, g)))
self.progress_.inc()
DB.execute('''delete from statistic where w <= ?''', (w, ))
self.progress_.inc()
DB.executemany(
'''insert into statistic (w, data, type, time, count, mistakes, viscosity)
VALUES (?, ?, ?, ?, ?, ?, ?)''', q)
self.progress_.inc()
DB.commit()
DB.execute('vacuum')
self.progress_.inc()
self.progress_.hide()
def cleanup(self):
day = 24*60*60
now = time.time()
q = []
self.progress_.show()
for grp, lim in [(30.0, Settings.get('group_month')),
(7.0, Settings.get('group_week')),
(1.0, Settings.get('group_day'))]:
w = now - day*lim
g = grp * day
q.extend(DB.fetchall('''
select avg(w), data, type, agg_mean(time, count), sum(count), sum(mistakes), agg_median(viscosity)
from statistic where w <= %f
group by data, type, cast(w/%f as int)''' % (w, g)))
self.progress_.inc()
DB.execute('''delete from statistic where w <= ?''', (w, ))
self.progress_.inc()
DB.executemany('''insert into statistic (w, data, type, time, count, mistakes, viscosity)
VALUES (?, ?, ?, ?, ?, ?, ?)''', q)
self.progress_.inc()
DB.execute('vacuum')
self.progress_.inc()
DB.commit()
self.progress_.hide()
def toggle_selected(self):
model, paths = self.tree.get_selection().get_selected_rows()
for path in paths:
if model.iter_depth(model.get_iter(path)) == 0:
continue
row = Gtk.TreeModelRow(model, path)
DB.execute("update text set disabled = 1 where rowid=?",
(row[0], ))
self.update()
DB.commit()
def addTexts(self, source, texts, lesson=None, update=True):
id = DB.getSource(source, lesson)
r = []
for x in texts:
h = hashlib.sha1()
h.update(x.encode('utf-8'))
txt_id = h.hexdigest()
dis = 1 if lesson == 2 else None
try:
DB.execute("insert into text (id,text,source,disabled) values (?,?,?,?)",
(txt_id, x, id, dis))
r.append(txt_id)
except Exception, e:
pass # silently skip ...
def addTexts(self, source, texts, lesson=None, update=True):
id = DB.getSource(source, lesson)
r = []
for x in texts:
h = hashlib.sha1()
h.update(x.encode('utf-8'))
txt_id = h.hexdigest()
dis = 1 if lesson == 2 else None
try:
DB.execute(
"insert into text (id,text,source,disabled) values (?,?,?,?)",
(txt_id, x, id, dis))
r.append(txt_id)
except Exception, e:
pass # silently skip ...
def next_text(self):
kind = Settings.get("select_method")
if kind != 1:
# Not in order
targets = DB.execute(
f"""select id,source,text from text where disabled is null
order by random() limit {Settings.get("num_rand")}"""
).fetchall()
if not targets:
target = None
elif kind == 2:
target = min(targets, key=self.diff_eval)
elif kind == 3:
target = max(targets, key=self.diff_eval)
else:
target = targets[0] # random, just pick the first
else:
# Fetch in order
prev = (0, )
result = DB.fetchone(
"""select r.text_id
from result as r left join source as s on (r.source = s.rowid)
where (s.discount is null) or (s.discount = 1) order by r.w desc limit 1""",
None)
if result is not None:
prev = DB.fetchone("select rowid from text where id = ?", prev,
result)
target = DB.fetchone(
"""select id,source,text from text
where rowid > ? and disabled is null order by rowid asc limit 1""",
None, prev)
if target is None:
target = self.default_text
self.emit("set-text", *target)
def nextText(self):
type = Settings.get('select_method')
if type != 1:
# Not in order
v = DB.execute("select id,source,text from text where disabled is null order by random() limit %d" % Settings.get('num_rand')).fetchall()
if len(v) == 0:
v = None
elif type == 2:
v = min(v, key=self.diff_eval)
elif type == 3:
v = max(v, key=self.diff_eval)
else:
v = v[0] # random, just pick the first
else:
# Fetch in order
lastid = (0,)
g = DB.fetchone("""select r.text_id
from result as r left join source as s on (r.source = s.rowid)
where (s.discount is null) or (s.discount = 1) order by r.w desc limit 1""", None)
if g is not None:
lastid = DB.fetchone("select rowid from text where id = ?", lastid, g)
v = DB.fetchone("select id,source,text from text where rowid > ? and disabled is null order by rowid asc limit 1", None, lastid)
if v is None:
v = self.defaultText
self.emit(SIGNAL("setText"), v)
def nextText(self):
type = Settings.get('select_method')
if type != 1:
# Not in order
v = DB.execute("select id,source,text from text where disabled is null order by random() limit %d" % Settings.get('num_rand')).fetchall()
if len(v) == 0:
v = None
elif type == 2:
v = min(v, key=self.diff_eval)
elif type == 3:
v = max(v, key=self.diff_eval)
else:
v = v[0] # random, just pick the first
else:
# Fetch in order
lastid = (0,)
g = DB.fetchone("""select r.text_id
from result as r left join source as s on (r.source = s.rowid)
where (s.discount is null) or (s.discount = 1) order by r.w desc limit 1""", None)
if g is not None:
lastid = DB.fetchone("select rowid from text where id = ?", lastid, g)
v = DB.fetchone("select id,source,text from text where rowid > ? and disabled is null order by rowid asc limit 1", None, lastid)
if v is None:
v = self.defaultText
self.emit(SIGNAL("setText"), v)
def removeUnused(self):
DB.execute('''
delete from source where rowid in (
select s.rowid from source as s
left join result as r on (s.rowid=r.source)
left join text as t on (t.source=s.rowid)
group by s.rowid
having count(r.rowid) = 0 and count(t.rowid) = 0
)''')
DB.execute('''
update source set disabled = 1 where rowid in (
select s.rowid from source as s
left join result as r on (s.rowid=r.source)
left join text as t on (t.source=s.rowid)
group by s.rowid
having count(r.rowid) > 0 and count(t.rowid) = 0
)''')
self.emit(SIGNAL("refreshSources"))
def removeUnused(self):
DB.execute('''
delete from source where rowid in (
select s.rowid from source as s
left join result as r on (s.rowid=r.source)
left join text as t on (t.source=s.rowid)
group by s.rowid
having count(r.rowid) = 0 and count(t.rowid) = 0
)''')
DB.execute('''
update source set disabled = 1 where rowid in (
select s.rowid from source as s
left join result as r on (s.rowid=r.source)
left join text as t on (t.source=s.rowid)
group by s.rowid
having count(r.rowid) > 0 and count(t.rowid) = 0
)''')
self.emit(SIGNAL("refreshSources"))
def addTexts(self, source, texts, lesson=None, update=True):
id = DB.getSource(source, lesson)
r = []
for x in texts:
x = re.sub(Settings.get('sentence_strip'), ' ', x)
h = hashlib.sha1()
h.update(x.encode('utf-8'))
txt_id = h.hexdigest()
dis = 1 if lesson == 2 else None
try:
DB.execute("insert into text (id, text, source, disabled) values (?, ?, ?, ?)", (txt_id, x, id, dis))
except Exception:
pass # silently skip ...
r.append(txt_id)
if update:
self.update()
if lesson:
DB.commit()
return r
def add_texts(self, source, texts, lesson=None, update=True):
idx = DB.get_source(source, lesson)
out = []
for text in texts:
hasher = hashlib.sha1()
hasher.update(text.encode("utf-8"))
text_hash = hasher.hexdigest()
dis = 1 if lesson == 2 else None
try:
DB.execute(
"insert into text (id,text,source,disabled) values (?,?,?,?)",
(text_hash, text, idx, dis))
out.append(text_hash)
except Exception:
# TODO properly handle exception
pass # silently skip ...
if update:
self.update()
if lesson:
DB.commit()
return out
def addTexts(self, source, texts, lesson=None, update=True):
source = source +' Length:' + str(Settings.get('min_chars'))
id = DB.getSource(source, lesson)
r = []
for x in texts:
x = self.cleanText(x)
h = hashlib.sha1()
h.update(x.encode('utf-8'))
txt_id = h.hexdigest()
dis = 1 if lesson == 2 else None
try:
DB.execute("insert into text (id, text, source, disabled) values (?, ?, ?, ?)", (txt_id, x, id, dis))
except Exception:
pass # silently skip ...
r.append(txt_id)
if update:
self.update()
if lesson:
DB.commit()
return r
def setSelect(self, v):
if v == 0 or v == 1:
self.diff_eval = lambda x: 1
self.nextText()
return
hist = time.time() - 86400.0 * Settings.get("history")
tri = dict(
DB.execute(
"""
select data,agg_median(time) as wpm from statistic
where w >= ? and type = 1
group by data""",
(hist,),
).fetchall()
) # [(t, (m, c)) for t, m, c in
g = tri.values()
if len(g) == 0:
return lambda x: 1
g.sort(reverse=True)
expect = g[len(g) // 4]
def _func(v):
text = v[2]
v = 0
s = 0.0
for i in xrange(0, len(text) - 2):
t = text[i : i + 3]
if t in tri:
s += tri[t]
else:
# print "|", t,
s += expect
v += 1
avg = s / (len(text) - 2)
# print text
# print " v=%d,s=%f" % (v, 12.0/avg), "ex:", expect
return 12.0 / avg
self.diff_eval = _func
self.nextText()
def setSelect(self, v):
if v == 0 or v == 1:
self.diff_eval = lambda x: 1
self.nextText()
return
hist = time.time() - 86400.0 * Settings.get("history")
tri = dict(
DB.execute(
"""
select data,agg_median(time) as wpm from statistic
where w >= ? and type = 1
group by data""",
(hist, ),
).fetchall()) # [(t, (m, c)) for t, m, c in
g = tri.values()
if len(g) == 0:
return lambda x: 1
g.sort(reverse=True)
expect = g[len(g) // 4]
def _func(v):
text = v[2]
v = 0
s = 0.0
for i in xrange(0, len(text) - 2):
t = text[i:i + 3]
if t in tri:
s += tri[t]
else:
# print "|", t,
s += expect
v += 1
avg = s / (len(text) - 2)
# print text
# print " v=%d,s=%f" % (v, 12.0/avg), "ex:", expect
return 12.0 / avg
self.diff_eval = _func
self.nextText()
def set_select(self):
method = Settings.get("select_method")
if method in (0, 1):
self.diff_eval = lambda x: 1
self.next_text()
return
hist = time.time() - 86400 * Settings.get("history")
tri = dict(
DB.execute(
"""
select data,agg_median(time) as wpm from statistic
where w >= ? and type = 1
group by data""", (hist, )).fetchall())
vals = list(tri.values())
if not vals:
self.diff_eval = lambda x: 1
self.next_text()
return
vals.sort(reverse=True)
expect = vals[len(vals) // 4]
def func(target):
text = target[2]
# FIXME what does v do here?
# v = 0
total = 0.0
for i in range(0, len(text) - 2):
trigram = text[i:i + 3]
if trigram in tri:
total += tri[trigram]
else:
total += expect
# v += 1
avg = total / (len(text) - 2)
return 12.0 / avg
self.diff_eval = func
self.next_text()
def delete_disabled(self):
DB.execute('delete from text where disabled is not null')
DB.execute("""
delete from source where rowid in (
select s.rowid from source as s
left join result as r on (s.rowid=r.source)
left join text as t on (t.source=s.rowid)
group by s.rowid
having count(r.rowid) = 0 and count(t.rowid) = 0
)""")
DB.execute("""
update source set disabled = 1 where rowid in (
select s.rowid from source as s
left join result as r on (s.rowid=r.source)
left join text as t on (t.source=s.rowid)
group by s.rowid
having count(r.rowid) > 0 and count(t.rowid) = 0
)""")
self.emit("refresh-sources")
self.update()
DB.commit()
def insertResults(self, now):
return DB.execute(
'insert into result (w, text_id, source, wpm, accuracy, viscosity) values (?, ?, ?, ?, ?, ?)',
(now, self.text[0], self.text[1], 12.0 / self.typer.getRawSpeed(),
self.typer.getAccuracy(), self.typer.getViscosity()))
def done(self):
now = time.time()
elapsed, chars, times, mis, mistakes = self.typer.getStats()
assert chars == len(self.text[2])
accuracy = 1.0 - len(filter(None, mis)) / chars
spc = elapsed / chars
viscosity = sum(map(lambda x: ((x-spc)/spc)**2, times)) / chars
DB.execute('insert into result (w,text_id,source,wpm,accuracy,viscosity) values (?,?,?,?,?,?)',
(now, self.text[0], self.text[1], 12.0/spc, accuracy, viscosity))
v2 = DB.fetchone("""select agg_median(wpm),agg_median(acc) from
(select wpm,100.0*accuracy as acc from result order by w desc limit %d)""" % Settings.get('def_group_by'), (0.0, 100.0))
self.result.setText("Last: %.1fwpm (%.1f%%), last 10 average: %.1fwpm (%.1f%%)"
% ((12.0/spc, 100.0*accuracy) + v2))
self.emit(SIGNAL("statsChanged"))
stats = collections.defaultdict(Statistic)
visc = collections.defaultdict(Statistic)
text = self.text[2]
for c, t, m in zip(text, times, mis):
stats[c].append(t, m)
visc[c].append(((t-spc)/spc)**2)
def gen_tup(s, e):
perch = sum(times[s:e])/(e-s)
visc = sum(map(lambda x: ((x-perch)/perch)**2, times[s:e]))/(e-s)
return (text[s:e], perch, len(filter(None, mis[s:e])), visc)
for tri, t, m, v in [gen_tup(i, i+3) for i in xrange(0, chars-2)]:
stats[tri].append(t, m > 0)
visc[tri].append(v)
regex = re.compile(r"(\w|'(?![A-Z]))+(-\w(\w|')*)*")
for w, t, m, v in [gen_tup(*x.span()) for x in regex.finditer(text) if x.end()-x.start() > 3]:
stats[w].append(t, m > 0)
visc[w].append(v)
def type(k):
if len(k) == 1:
return 0
elif len(k) == 3:
return 1
return 2
vals = []
for k, s in stats.iteritems():
v = visc[k].median()
vals.append( (s.median(), v*100.0, now, len(s), s.flawed(), type(k), k) )
is_lesson = DB.fetchone("select discount from source where rowid=?", (None,), (self.text[1], ))[0]
if Settings.get('use_lesson_stats') or not is_lesson:
DB.executemany_('''insert into statistic
(time,viscosity,w,count,mistakes,type,data) values (?,?,?,?,?,?,?)''', vals)
DB.executemany_('insert into mistake (w,target,mistake,count) values (?,?,?,?)',
[(now, k[0], k[1], v) for k, v in mistakes.iteritems()])
if is_lesson:
mins = (Settings.get("min_lesson_wpm"), Settings.get("min_lesson_acc"))
else:
mins = (Settings.get("min_wpm"), Settings.get("min_acc"))
if 12.0/spc < mins[0] or accuracy < mins[1]/100.0:
self.setText(self.text)
elif not is_lesson and Settings.get('auto_review'):
ws = filter(lambda x: x[5] == 2, vals)
if len(ws) == 0:
self.emit(SIGNAL("wantText"))
return
ws.sort(key=lambda x: (x[4],x[0]), reverse=True)
i = 0
while ws[i][4] != 0:
i += 1
i += (len(ws) - i) // 4
self.emit(SIGNAL("wantReview"), map(lambda x:x[6], ws[0:i]))
else:
self.emit(SIGNAL("wantText"))
def enableAll(self):
DB.execute("update text set disabled = null where disabled is not null")
self.update()
def removeDisabled(self):
DB.execute("delete from text where disabled is not null")
self.removeUnused()
self.update()
DB.commit()
def done(self):
now = time.time()
elapsed, chars, times, mis, mistakes = self.typer.getStats()
assert chars == len(self.text[2])
accuracy = 1.0 - len(filter(None, mis)) / chars
spc = elapsed / chars
viscosity = sum(map(lambda x: ((x - spc) / spc)**2, times)) / chars
DB.execute(
'insert into result (w,text_id,source,wpm,accuracy,viscosity) values (?,?,?,?,?,?)',
(now, self.text[0], self.text[1], 12.0 / spc, accuracy, viscosity))
v2 = DB.fetchone(
"""select agg_median(wpm),agg_median(acc) from
(select wpm,100.0*accuracy as acc from result order by w desc limit %d)"""
% Settings.get('def_group_by'), (0.0, 100.0))
self.result.setText(
"Last: %.1fwpm (%.1f%%), last 10 average: %.1fwpm (%.1f%%)" %
((12.0 / spc, 100.0 * accuracy) + v2))
self.emit(SIGNAL("statsChanged"))
stats = collections.defaultdict(Statistic)
visc = collections.defaultdict(Statistic)
text = self.text[2]
for c, t, m in zip(text, times, mis):
stats[c].append(t, m)
visc[c].append(((t - spc) / spc)**2)
def gen_tup(s, e):
perch = sum(times[s:e]) / (e - s)
visc = sum(map(lambda x:
((x - perch) / perch)**2, times[s:e])) / (e - s)
return (text[s:e], perch, len(filter(None, mis[s:e])), visc)
for tri, t, m, v in [gen_tup(i, i + 3) for i in xrange(0, chars - 2)]:
stats[tri].append(t, m > 0)
visc[tri].append(v)
regex = re.compile(r"(\w|'(?![A-Z]))+(-\w(\w|')*)*")
for w, t, m, v in [
gen_tup(*x.span()) for x in regex.finditer(text)
if x.end() - x.start() > 3
]:
stats[w].append(t, m > 0)
visc[w].append(v)
def type(k):
if len(k) == 1:
return 0
elif len(k) == 3:
return 1
return 2
vals = []
for k, s in stats.iteritems():
v = visc[k].median()
vals.append(
(s.median(), v * 100.0, now, len(s), s.flawed(), type(k), k))
is_lesson = DB.fetchone("select discount from source where rowid=?",
(None, ), (self.text[1], ))[0]
if Settings.get('use_lesson_stats') or not is_lesson:
DB.executemany_(
'''insert into statistic
(time,viscosity,w,count,mistakes,type,data) values (?,?,?,?,?,?,?)''',
vals)
DB.executemany_(
'insert into mistake (w,target,mistake,count) values (?,?,?,?)',
[(now, k[0], k[1], v) for k, v in mistakes.iteritems()])
if is_lesson:
mins = (Settings.get("min_lesson_wpm"),
Settings.get("min_lesson_acc"))
else:
mins = (Settings.get("min_wpm"), Settings.get("min_acc"))
if 12.0 / spc < mins[0] or accuracy < mins[1] / 100.0:
self.setText(self.text)
elif not is_lesson and Settings.get('auto_review'):
ws = filter(lambda x: x[5] == 2, vals)
if len(ws) == 0:
self.emit(SIGNAL("wantText"))
return
ws.sort(key=lambda x: (x[4], x[0]), reverse=True)
i = 0
while ws[i][4] != 0:
i += 1
i += (len(ws) - i) // 4
self.emit(SIGNAL("wantReview"), map(lambda x: x[6], ws[0:i]))
else:
self.emit(SIGNAL("wantText"))
def enableAll(self):
DB.execute(
'update text set disabled = null where disabled is not null')
self.update()
def removeDisabled(self):
DB.execute('delete from text where disabled is not null')
self.removeUnused()
self.update()
DB.commit()
def done(self):
print("DONE")
# TODO split into smaller bits
now = timer()
elapsed, chars, times, mis, mistakes = self.typer.get_stats()
text = self.text[2]
assert chars == len(text)
accuracy = 1.0 - sum(1 for f in mis if f) / chars
spc = elapsed / chars
viscosity = sum((t / spc - 1)**2 for t in times) / chars
DB.execute(
"""insert into result (w, text_id, source, wpm, accuracy, viscosity)
values (?,?,?,?,?,?)""",
(now, self.text[0], self.text[1], 12.0 / spc, accuracy, viscosity))
wpm_median, acc_median = DB.fetchone(
f"""select agg_median(wpm),agg_median(acc) from
(select wpm,100.0*accuracy as acc from result order by w desc limit
{Settings.get("def_group_by")})""", (0.0, 100.0))
self.result.set_text(
"Last: {:.1f}wpm ({:.1f}%), last 10 average: {:.1f}wpm ({:.1f}%)".
format(12.0 / spc, 100.0 * accuracy, wpm_median, acc_median))
self.emit("stats-changed")
stats = collections.defaultdict(Statistic)
viscs = collections.defaultdict(Statistic)
for char, time, mistake in zip(text, times, mis):
stats[char].append(time, mistake)
viscs[char].append((time / spc - 1)**2)
def gen_tup(start, end):
span = end - start
char_avg = sum(times[start:end]) / span
visc = sum((t / char_avg - 1)**2 for t in times[start:end]) / span
return (text[start:end], char_avg,
sum(1 for f in mis[start:end] if f), visc)
for trigraph, time, mist, visc in [
gen_tup(i, i + 3) for i in range(0, chars - 2)
]:
stats[trigraph].append(time, mist > 0)
viscs[trigraph].append(visc)
regex = re.compile(r"(\w|'(?![A-Z]))+(-\w(\w|')*)*")
for word, time, mist, visc in [
gen_tup(*m.span()) for m in regex.finditer(text)
if m.end() - m.start() > 3
]:
stats[word].append(time, mist > 0)
viscs[word].append(visc)
def kind(key):
if len(key) == 1:
return 0
if len(key) == 3:
return 1
return 2
vals = []
for key, stat in stats.items():
visc = viscs[key].median()
vals.append((stat.median(), visc * 100.0, now, len(stat),
stat.flawed(), kind(key), key))
is_lesson = DB.fetchone("select discount from source where rowid=?",
(None, ), (self.text[1], ))[0]
if Settings.get("use_lesson_stats") or not is_lesson:
DB.executemany(
"""insert into statistic (time,viscosity,w,count,mistakes,type,data)
values (?,?,?,?,?,?,?)""", vals)
DB.executemany(
"insert into mistake (w,target,mistake,count) values (?,?,?,?)",
[(now, k[0], k[1], v) for k, v in mistakes.items()])
if is_lesson:
mins = (Settings.get("min_lesson_wpm"),
Settings.get("min_lesson_acc"))
else:
mins = (Settings.get("min_wpm"), Settings.get("min_acc"))
if 12.0 / spc < mins[0] or accuracy < mins[1] / 100.0:
self.set_target(self.text)
elif not is_lesson and Settings.get('auto_review'):
words = [x for x in vals if x[5] == 2]
if not words:
self.emit("want-text")
return
words.sort(key=lambda x: (x[4], x[0]), reverse=True)
i = 0
while words[i][4] != 0:
i += 1
i += (len(words) - i) // 4
# TODO support want-review
# self.emit("want-review", [x[6] for x in words[0:i]])
else:
self.emit("want-text")
def removeDisabled(self):
DB.execute('delete from text where disabled = 1')
self.removeUnused()
self.update()
DB.commit()
def insertResults(self, now):
return DB.execute('insert into result (w, text_id, source, wpm, accuracy, viscosity) values (?, ?, ?, ?, ?, ?)',
(now, self.text[0], self.text[1], 12.0/self.typer.getRawSpeed(), self.typer.getAccuracy(), self.typer.getViscosity()))
