def cleanup(self): day = 24 * 60 * 60 now = time.time() q = [] self.progress_.show() for grp, lim in [(30.0, Settings.get('group_month')), (7.0, Settings.get('group_week')), (1.0, Settings.get('group_day'))]: w = now - day * lim g = grp * day q.extend( DB.fetchall(''' select avg(w), data, type, agg_mean(time, count), sum(count), sum(mistakes), agg_median(viscosity) from statistic where w <= %f group by data, type, cast(w/%f as int)''' % (w, g))) self.progress_.inc() DB.execute('''delete from statistic where w <= ?''', (w, )) self.progress_.inc() DB.executemany( '''insert into statistic (w, data, type, time, count, mistakes, viscosity) VALUES (?, ?, ?, ?, ?, ?, ?)''', q) self.progress_.inc() DB.commit() DB.execute('vacuum') self.progress_.inc() self.progress_.hide()
def cleanup(self): day = 24*60*60 now = time.time() q = [] self.progress_.show() for grp, lim in [(30.0, Settings.get('group_month')), (7.0, Settings.get('group_week')), (1.0, Settings.get('group_day'))]: w = now - day*lim g = grp * day q.extend(DB.fetchall(''' select avg(w), data, type, agg_mean(time, count), sum(count), sum(mistakes), agg_median(viscosity) from statistic where w <= %f group by data, type, cast(w/%f as int)''' % (w, g))) self.progress_.inc() DB.execute('''delete from statistic where w <= ?''', (w, )) self.progress_.inc() DB.executemany('''insert into statistic (w, data, type, time, count, mistakes, viscosity) VALUES (?, ?, ?, ?, ?, ?, ?)''', q) self.progress_.inc() DB.execute('vacuum') self.progress_.inc() DB.commit() self.progress_.hide()
def toggle_selected(self): model, paths = self.tree.get_selection().get_selected_rows() for path in paths: if model.iter_depth(model.get_iter(path)) == 0: continue row = Gtk.TreeModelRow(model, path) DB.execute("update text set disabled = 1 where rowid=?", (row[0], )) self.update() DB.commit()
def addTexts(self, source, texts, lesson=None, update=True): id = DB.getSource(source, lesson) r = [] for x in texts: h = hashlib.sha1() h.update(x.encode('utf-8')) txt_id = h.hexdigest() dis = 1 if lesson == 2 else None try: DB.execute("insert into text (id,text,source,disabled) values (?,?,?,?)", (txt_id, x, id, dis)) r.append(txt_id) except Exception, e: pass # silently skip ...
def addTexts(self, source, texts, lesson=None, update=True): id = DB.getSource(source, lesson) r = [] for x in texts: h = hashlib.sha1() h.update(x.encode('utf-8')) txt_id = h.hexdigest() dis = 1 if lesson == 2 else None try: DB.execute( "insert into text (id,text,source,disabled) values (?,?,?,?)", (txt_id, x, id, dis)) r.append(txt_id) except Exception, e: pass # silently skip ...
def next_text(self): kind = Settings.get("select_method") if kind != 1: # Not in order targets = DB.execute( f"""select id,source,text from text where disabled is null order by random() limit {Settings.get("num_rand")}""" ).fetchall() if not targets: target = None elif kind == 2: target = min(targets, key=self.diff_eval) elif kind == 3: target = max(targets, key=self.diff_eval) else: target = targets[0] # random, just pick the first else: # Fetch in order prev = (0, ) result = DB.fetchone( """select r.text_id from result as r left join source as s on (r.source = s.rowid) where (s.discount is null) or (s.discount = 1) order by r.w desc limit 1""", None) if result is not None: prev = DB.fetchone("select rowid from text where id = ?", prev, result) target = DB.fetchone( """select id,source,text from text where rowid > ? and disabled is null order by rowid asc limit 1""", None, prev) if target is None: target = self.default_text self.emit("set-text", *target)
def nextText(self): type = Settings.get('select_method') if type != 1: # Not in order v = DB.execute("select id,source,text from text where disabled is null order by random() limit %d" % Settings.get('num_rand')).fetchall() if len(v) == 0: v = None elif type == 2: v = min(v, key=self.diff_eval) elif type == 3: v = max(v, key=self.diff_eval) else: v = v[0] # random, just pick the first else: # Fetch in order lastid = (0,) g = DB.fetchone("""select r.text_id from result as r left join source as s on (r.source = s.rowid) where (s.discount is null) or (s.discount = 1) order by r.w desc limit 1""", None) if g is not None: lastid = DB.fetchone("select rowid from text where id = ?", lastid, g) v = DB.fetchone("select id,source,text from text where rowid > ? and disabled is null order by rowid asc limit 1", None, lastid) if v is None: v = self.defaultText self.emit(SIGNAL("setText"), v)
def removeUnused(self): DB.execute(''' delete from source where rowid in ( select s.rowid from source as s left join result as r on (s.rowid=r.source) left join text as t on (t.source=s.rowid) group by s.rowid having count(r.rowid) = 0 and count(t.rowid) = 0 )''') DB.execute(''' update source set disabled = 1 where rowid in ( select s.rowid from source as s left join result as r on (s.rowid=r.source) left join text as t on (t.source=s.rowid) group by s.rowid having count(r.rowid) > 0 and count(t.rowid) = 0 )''') self.emit(SIGNAL("refreshSources"))
def addTexts(self, source, texts, lesson=None, update=True): id = DB.getSource(source, lesson) r = [] for x in texts: x = re.sub(Settings.get('sentence_strip'), ' ', x) h = hashlib.sha1() h.update(x.encode('utf-8')) txt_id = h.hexdigest() dis = 1 if lesson == 2 else None try: DB.execute("insert into text (id, text, source, disabled) values (?, ?, ?, ?)", (txt_id, x, id, dis)) except Exception: pass # silently skip ... r.append(txt_id) if update: self.update() if lesson: DB.commit() return r
def add_texts(self, source, texts, lesson=None, update=True): idx = DB.get_source(source, lesson) out = [] for text in texts: hasher = hashlib.sha1() hasher.update(text.encode("utf-8")) text_hash = hasher.hexdigest() dis = 1 if lesson == 2 else None try: DB.execute( "insert into text (id,text,source,disabled) values (?,?,?,?)", (text_hash, text, idx, dis)) out.append(text_hash) except Exception: # TODO properly handle exception pass # silently skip ... if update: self.update() if lesson: DB.commit() return out
def addTexts(self, source, texts, lesson=None, update=True): source = source +' Length:' + str(Settings.get('min_chars')) id = DB.getSource(source, lesson) r = [] for x in texts: x = self.cleanText(x) h = hashlib.sha1() h.update(x.encode('utf-8')) txt_id = h.hexdigest() dis = 1 if lesson == 2 else None try: DB.execute("insert into text (id, text, source, disabled) values (?, ?, ?, ?)", (txt_id, x, id, dis)) except Exception: pass # silently skip ... r.append(txt_id) if update: self.update() if lesson: DB.commit() return r
def setSelect(self, v): if v == 0 or v == 1: self.diff_eval = lambda x: 1 self.nextText() return hist = time.time() - 86400.0 * Settings.get("history") tri = dict( DB.execute( """ select data,agg_median(time) as wpm from statistic where w >= ? and type = 1 group by data""", (hist,), ).fetchall() ) # [(t, (m, c)) for t, m, c in g = tri.values() if len(g) == 0: return lambda x: 1 g.sort(reverse=True) expect = g[len(g) // 4] def _func(v): text = v[2] v = 0 s = 0.0 for i in xrange(0, len(text) - 2): t = text[i : i + 3] if t in tri: s += tri[t] else: # print "|", t, s += expect v += 1 avg = s / (len(text) - 2) # print text # print " v=%d,s=%f" % (v, 12.0/avg), "ex:", expect return 12.0 / avg self.diff_eval = _func self.nextText()
def setSelect(self, v): if v == 0 or v == 1: self.diff_eval = lambda x: 1 self.nextText() return hist = time.time() - 86400.0 * Settings.get("history") tri = dict( DB.execute( """ select data,agg_median(time) as wpm from statistic where w >= ? and type = 1 group by data""", (hist, ), ).fetchall()) # [(t, (m, c)) for t, m, c in g = tri.values() if len(g) == 0: return lambda x: 1 g.sort(reverse=True) expect = g[len(g) // 4] def _func(v): text = v[2] v = 0 s = 0.0 for i in xrange(0, len(text) - 2): t = text[i:i + 3] if t in tri: s += tri[t] else: # print "|", t, s += expect v += 1 avg = s / (len(text) - 2) # print text # print " v=%d,s=%f" % (v, 12.0/avg), "ex:", expect return 12.0 / avg self.diff_eval = _func self.nextText()
def set_select(self): method = Settings.get("select_method") if method in (0, 1): self.diff_eval = lambda x: 1 self.next_text() return hist = time.time() - 86400 * Settings.get("history") tri = dict( DB.execute( """ select data,agg_median(time) as wpm from statistic where w >= ? and type = 1 group by data""", (hist, )).fetchall()) vals = list(tri.values()) if not vals: self.diff_eval = lambda x: 1 self.next_text() return vals.sort(reverse=True) expect = vals[len(vals) // 4] def func(target): text = target[2] # FIXME what does v do here? # v = 0 total = 0.0 for i in range(0, len(text) - 2): trigram = text[i:i + 3] if trigram in tri: total += tri[trigram] else: total += expect # v += 1 avg = total / (len(text) - 2) return 12.0 / avg self.diff_eval = func self.next_text()
def delete_disabled(self): DB.execute('delete from text where disabled is not null') DB.execute(""" delete from source where rowid in ( select s.rowid from source as s left join result as r on (s.rowid=r.source) left join text as t on (t.source=s.rowid) group by s.rowid having count(r.rowid) = 0 and count(t.rowid) = 0 )""") DB.execute(""" update source set disabled = 1 where rowid in ( select s.rowid from source as s left join result as r on (s.rowid=r.source) left join text as t on (t.source=s.rowid) group by s.rowid having count(r.rowid) > 0 and count(t.rowid) = 0 )""") self.emit("refresh-sources") self.update() DB.commit()
def insertResults(self, now): return DB.execute( 'insert into result (w, text_id, source, wpm, accuracy, viscosity) values (?, ?, ?, ?, ?, ?)', (now, self.text[0], self.text[1], 12.0 / self.typer.getRawSpeed(), self.typer.getAccuracy(), self.typer.getViscosity()))
def done(self): now = time.time() elapsed, chars, times, mis, mistakes = self.typer.getStats() assert chars == len(self.text[2]) accuracy = 1.0 - len(filter(None, mis)) / chars spc = elapsed / chars viscosity = sum(map(lambda x: ((x-spc)/spc)**2, times)) / chars DB.execute('insert into result (w,text_id,source,wpm,accuracy,viscosity) values (?,?,?,?,?,?)', (now, self.text[0], self.text[1], 12.0/spc, accuracy, viscosity)) v2 = DB.fetchone("""select agg_median(wpm),agg_median(acc) from (select wpm,100.0*accuracy as acc from result order by w desc limit %d)""" % Settings.get('def_group_by'), (0.0, 100.0)) self.result.setText("Last: %.1fwpm (%.1f%%), last 10 average: %.1fwpm (%.1f%%)" % ((12.0/spc, 100.0*accuracy) + v2)) self.emit(SIGNAL("statsChanged")) stats = collections.defaultdict(Statistic) visc = collections.defaultdict(Statistic) text = self.text[2] for c, t, m in zip(text, times, mis): stats[c].append(t, m) visc[c].append(((t-spc)/spc)**2) def gen_tup(s, e): perch = sum(times[s:e])/(e-s) visc = sum(map(lambda x: ((x-perch)/perch)**2, times[s:e]))/(e-s) return (text[s:e], perch, len(filter(None, mis[s:e])), visc) for tri, t, m, v in [gen_tup(i, i+3) for i in xrange(0, chars-2)]: stats[tri].append(t, m > 0) visc[tri].append(v) regex = re.compile(r"(\w|'(?![A-Z]))+(-\w(\w|')*)*") for w, t, m, v in [gen_tup(*x.span()) for x in regex.finditer(text) if x.end()-x.start() > 3]: stats[w].append(t, m > 0) visc[w].append(v) def type(k): if len(k) == 1: return 0 elif len(k) == 3: return 1 return 2 vals = [] for k, s in stats.iteritems(): v = visc[k].median() vals.append( (s.median(), v*100.0, now, len(s), s.flawed(), type(k), k) ) is_lesson = DB.fetchone("select discount from source where rowid=?", (None,), (self.text[1], ))[0] if Settings.get('use_lesson_stats') or not is_lesson: DB.executemany_('''insert into statistic (time,viscosity,w,count,mistakes,type,data) values (?,?,?,?,?,?,?)''', vals) DB.executemany_('insert into mistake (w,target,mistake,count) values (?,?,?,?)', [(now, k[0], k[1], v) for k, v in mistakes.iteritems()]) if is_lesson: mins = (Settings.get("min_lesson_wpm"), Settings.get("min_lesson_acc")) else: mins = (Settings.get("min_wpm"), Settings.get("min_acc")) if 12.0/spc < mins[0] or accuracy < mins[1]/100.0: self.setText(self.text) elif not is_lesson and Settings.get('auto_review'): ws = filter(lambda x: x[5] == 2, vals) if len(ws) == 0: self.emit(SIGNAL("wantText")) return ws.sort(key=lambda x: (x[4],x[0]), reverse=True) i = 0 while ws[i][4] != 0: i += 1 i += (len(ws) - i) // 4 self.emit(SIGNAL("wantReview"), map(lambda x:x[6], ws[0:i])) else: self.emit(SIGNAL("wantText"))
def enableAll(self): DB.execute("update text set disabled = null where disabled is not null") self.update()
def removeDisabled(self): DB.execute("delete from text where disabled is not null") self.removeUnused() self.update() DB.commit()
def done(self): now = time.time() elapsed, chars, times, mis, mistakes = self.typer.getStats() assert chars == len(self.text[2]) accuracy = 1.0 - len(filter(None, mis)) / chars spc = elapsed / chars viscosity = sum(map(lambda x: ((x - spc) / spc)**2, times)) / chars DB.execute( 'insert into result (w,text_id,source,wpm,accuracy,viscosity) values (?,?,?,?,?,?)', (now, self.text[0], self.text[1], 12.0 / spc, accuracy, viscosity)) v2 = DB.fetchone( """select agg_median(wpm),agg_median(acc) from (select wpm,100.0*accuracy as acc from result order by w desc limit %d)""" % Settings.get('def_group_by'), (0.0, 100.0)) self.result.setText( "Last: %.1fwpm (%.1f%%), last 10 average: %.1fwpm (%.1f%%)" % ((12.0 / spc, 100.0 * accuracy) + v2)) self.emit(SIGNAL("statsChanged")) stats = collections.defaultdict(Statistic) visc = collections.defaultdict(Statistic) text = self.text[2] for c, t, m in zip(text, times, mis): stats[c].append(t, m) visc[c].append(((t - spc) / spc)**2) def gen_tup(s, e): perch = sum(times[s:e]) / (e - s) visc = sum(map(lambda x: ((x - perch) / perch)**2, times[s:e])) / (e - s) return (text[s:e], perch, len(filter(None, mis[s:e])), visc) for tri, t, m, v in [gen_tup(i, i + 3) for i in xrange(0, chars - 2)]: stats[tri].append(t, m > 0) visc[tri].append(v) regex = re.compile(r"(\w|'(?![A-Z]))+(-\w(\w|')*)*") for w, t, m, v in [ gen_tup(*x.span()) for x in regex.finditer(text) if x.end() - x.start() > 3 ]: stats[w].append(t, m > 0) visc[w].append(v) def type(k): if len(k) == 1: return 0 elif len(k) == 3: return 1 return 2 vals = [] for k, s in stats.iteritems(): v = visc[k].median() vals.append( (s.median(), v * 100.0, now, len(s), s.flawed(), type(k), k)) is_lesson = DB.fetchone("select discount from source where rowid=?", (None, ), (self.text[1], ))[0] if Settings.get('use_lesson_stats') or not is_lesson: DB.executemany_( '''insert into statistic (time,viscosity,w,count,mistakes,type,data) values (?,?,?,?,?,?,?)''', vals) DB.executemany_( 'insert into mistake (w,target,mistake,count) values (?,?,?,?)', [(now, k[0], k[1], v) for k, v in mistakes.iteritems()]) if is_lesson: mins = (Settings.get("min_lesson_wpm"), Settings.get("min_lesson_acc")) else: mins = (Settings.get("min_wpm"), Settings.get("min_acc")) if 12.0 / spc < mins[0] or accuracy < mins[1] / 100.0: self.setText(self.text) elif not is_lesson and Settings.get('auto_review'): ws = filter(lambda x: x[5] == 2, vals) if len(ws) == 0: self.emit(SIGNAL("wantText")) return ws.sort(key=lambda x: (x[4], x[0]), reverse=True) i = 0 while ws[i][4] != 0: i += 1 i += (len(ws) - i) // 4 self.emit(SIGNAL("wantReview"), map(lambda x: x[6], ws[0:i])) else: self.emit(SIGNAL("wantText"))
def enableAll(self): DB.execute( 'update text set disabled = null where disabled is not null') self.update()
def removeDisabled(self): DB.execute('delete from text where disabled is not null') self.removeUnused() self.update() DB.commit()
def done(self): print("DONE") # TODO split into smaller bits now = timer() elapsed, chars, times, mis, mistakes = self.typer.get_stats() text = self.text[2] assert chars == len(text) accuracy = 1.0 - sum(1 for f in mis if f) / chars spc = elapsed / chars viscosity = sum((t / spc - 1)**2 for t in times) / chars DB.execute( """insert into result (w, text_id, source, wpm, accuracy, viscosity) values (?,?,?,?,?,?)""", (now, self.text[0], self.text[1], 12.0 / spc, accuracy, viscosity)) wpm_median, acc_median = DB.fetchone( f"""select agg_median(wpm),agg_median(acc) from (select wpm,100.0*accuracy as acc from result order by w desc limit {Settings.get("def_group_by")})""", (0.0, 100.0)) self.result.set_text( "Last: {:.1f}wpm ({:.1f}%), last 10 average: {:.1f}wpm ({:.1f}%)". format(12.0 / spc, 100.0 * accuracy, wpm_median, acc_median)) self.emit("stats-changed") stats = collections.defaultdict(Statistic) viscs = collections.defaultdict(Statistic) for char, time, mistake in zip(text, times, mis): stats[char].append(time, mistake) viscs[char].append((time / spc - 1)**2) def gen_tup(start, end): span = end - start char_avg = sum(times[start:end]) / span visc = sum((t / char_avg - 1)**2 for t in times[start:end]) / span return (text[start:end], char_avg, sum(1 for f in mis[start:end] if f), visc) for trigraph, time, mist, visc in [ gen_tup(i, i + 3) for i in range(0, chars - 2) ]: stats[trigraph].append(time, mist > 0) viscs[trigraph].append(visc) regex = re.compile(r"(\w|'(?![A-Z]))+(-\w(\w|')*)*") for word, time, mist, visc in [ gen_tup(*m.span()) for m in regex.finditer(text) if m.end() - m.start() > 3 ]: stats[word].append(time, mist > 0) viscs[word].append(visc) def kind(key): if len(key) == 1: return 0 if len(key) == 3: return 1 return 2 vals = [] for key, stat in stats.items(): visc = viscs[key].median() vals.append((stat.median(), visc * 100.0, now, len(stat), stat.flawed(), kind(key), key)) is_lesson = DB.fetchone("select discount from source where rowid=?", (None, ), (self.text[1], ))[0] if Settings.get("use_lesson_stats") or not is_lesson: DB.executemany( """insert into statistic (time,viscosity,w,count,mistakes,type,data) values (?,?,?,?,?,?,?)""", vals) DB.executemany( "insert into mistake (w,target,mistake,count) values (?,?,?,?)", [(now, k[0], k[1], v) for k, v in mistakes.items()]) if is_lesson: mins = (Settings.get("min_lesson_wpm"), Settings.get("min_lesson_acc")) else: mins = (Settings.get("min_wpm"), Settings.get("min_acc")) if 12.0 / spc < mins[0] or accuracy < mins[1] / 100.0: self.set_target(self.text) elif not is_lesson and Settings.get('auto_review'): words = [x for x in vals if x[5] == 2] if not words: self.emit("want-text") return words.sort(key=lambda x: (x[4], x[0]), reverse=True) i = 0 while words[i][4] != 0: i += 1 i += (len(words) - i) // 4 # TODO support want-review # self.emit("want-review", [x[6] for x in words[0:i]]) else: self.emit("want-text")
def removeDisabled(self): DB.execute('delete from text where disabled = 1') self.removeUnused() self.update() DB.commit()
def insertResults(self, now): return DB.execute('insert into result (w, text_id, source, wpm, accuracy, viscosity) values (?, ?, ?, ?, ?, ?)', (now, self.text[0], self.text[1], 12.0/self.typer.getRawSpeed(), self.typer.getAccuracy(), self.typer.getViscosity()))