def createTagsTab(self): self.frame3 = QGroupBox("Tags") self.tabWidget.addTab(self.frame3, "Tags") vbox = QVBoxLayout() self.frame3.setLayout(vbox) vbox.setContentsMargins(0, 20, 0, 0) label = QLabel( "This plugin will add and delete following tags from your matched notes. Hover your mouse over text entries to see tooltip info." ) label.setWordWrap(True) vbox.addWidget(label) vbox.addSpacing(50) grid = QGridLayout() vbox.addLayout(grid) tagList = [ ("Vocab note:", 'Tag_Vocab', 'Note that is optimal to learn (one unknown word.)'), ("Compehension note:", 'Tag_Comprehension', 'Note that only has mature words (optimal for sentence learning).' ), ("Fresh vocab note:", 'Tag_Fresh', 'Note that does not contain unknown words, but one or\nmore unmature (card with recently learned morphmes).' ), ("Not ready:", 'Tag_NotReady', 'Note that has two or more unknown words.'), ("Already known:", 'Tag_AlreadyKnown', 'You can add this tag to a note.\nAfter a recalc of the database, all in this sentence words are marked as known.\nPress \'K\' while reviewing to tag current card.' ), ("Priority:", 'Tag_Priority', 'Morpheme is in priority.db.'), ("Too Short:", 'Tag_TooShort', 'Sentence is too short.'), ("Too Long:", 'Tag_TooLong', 'Sentence is too long.'), ] self.tagEntryList = [] numberOfColumns = 2 for i, (name, key, tooltipInfo) in enumerate(tagList): entry = QLineEdit(jcfg(key)) entry.setToolTip(tooltipInfo) self.tagEntryList.append((key, entry)) grid.addWidget(QLabel(name), i // numberOfColumns, (i % numberOfColumns) * 2 + 0) grid.addWidget(entry, i // numberOfColumns, (i % numberOfColumns) * 2 + 1) vbox.addSpacing(50) self.checkboxSetNotRequiredTags = QCheckBox( "Add tags even if not required") self.checkboxSetNotRequiredTags.setCheckState( Qt.Checked if jcfg('Option_SetNotRequiredTags') else Qt.Unchecked) vbox.addWidget(self.checkboxSetNotRequiredTags) vbox.addStretch()
def createExtraFieldsTab(self): self.frame2 = QWidget() self.tabWidget.addTab(self.frame2, "Extra Fields") vbox = QVBoxLayout(); self.frame2.setLayout(vbox); vbox.setContentsMargins(0, 20, 0, 0) label = QLabel("This plugin will attempt to change the data in following fields. Every field that has a (*) is REQUIRED IN EVERY NOTE for MorphMan to work correctly. The other fields are optional. Hover your mouse over text entries to see tooltip info.") label.setWordWrap(True) vbox.addWidget(label) vbox.addSpacing(50) grid = QGridLayout(); vbox.addLayout(grid) numberOfColumns = 2 fieldsList = [ ("Focus morph (*):", "Field_FocusMorph", "Stores the unknown morpheme for sentences with one unmature word.\nGets cleared as soon as all works are mature."), ("MorphMan Index:", "Field_MorphManIndex", "Difficulty of card. This will be set to `due` time of card."), ("Unmatures", "Field_Unmatures", "Comma-separated list of unmature words."), ("Unmatures count:", "Field_UnmatureMorphCount", "Number of unmature words on this note."), ("Unknowns:", "Field_Unknowns", "Comma-separated list of unknown morphemes."), ("Unknown count:", "Field_UnknownMorphCount", "Number of unknown morphemes on this note."), ("Unknown frequency:", "Field_UnknownFreq", "Average of how many times the unknowns appear in your collection.") ] self.fieldEntryList = [] for i, (name, key, tooltipInfo) in enumerate(fieldsList): entry = QLineEdit(jcfg(key)) entry.setToolTip(tooltipInfo) self.fieldEntryList.append((key, entry)) grid.addWidget(QLabel(name), i // numberOfColumns, (i % numberOfColumns) * 2 + 0) grid.addWidget(entry, i // numberOfColumns, (i % numberOfColumns) * 2 + 1) vbox.addStretch()
def createNoteFilterTab(self): self.frame1 = QWidget() self.tabWidget.addTab(self.frame1, "Note Filter") vbox = QVBoxLayout(); self.frame1.setLayout(vbox); vbox.setContentsMargins(0, 20, 0, 0) self.tableModel = QStandardItemModel(0, 5) self.tableView = QTableView() self.tableView.setModel(self.tableModel) self.tableView.horizontalHeader().setResizeMode(QHeaderView.Stretch) self.tableView.setSelectionBehavior(QAbstractItemView.SelectRows) self.tableView.setSelectionMode(QAbstractItemView.SingleSelection) self.tableModel.setHeaderData(0, Qt.Horizontal, "Note type") self.tableModel.setHeaderData(1, Qt.Horizontal, "Tags") self.tableModel.setHeaderData(2, Qt.Horizontal, "Fields") self.tableModel.setHeaderData(3, Qt.Horizontal, "Morphemizer") self.tableModel.setHeaderData(4, Qt.Horizontal, "Modify?") rowData = jcfg('Filter') self.tableModel.setRowCount(len(rowData)) self.rowGui = [] for i, row in enumerate(rowData): self.setTableRow(i, row) label = QLabel("Any card that has the given `Note type` and all of the given `Tags` will have its `Fields` analyzed with the specified `Morphemizer`. A morphemizer specifies how words are extraced from a sentence. `Fields` and `Tags` are both comma-separated lists. If `Tags` is empty, there are no tag restrictions. If `Modify` is deactivated, the note will only be analyzed.\n\nIf a note is matched multple times, only the first filter in this list will be used.") label.setWordWrap(True) vbox.addWidget(label) vbox.addSpacing(20) vbox.addWidget(self.tableView) hbox = QHBoxLayout(); vbox.addLayout(hbox) self.clone = mkBtn("Clone", self.onClone, self, hbox) self.delete = mkBtn("Delete", self.onDelete, self, hbox) self.up = mkBtn("Up", self.onUp, self, hbox) self.down = mkBtn("Down", self.onDown, self, hbox)
def createGeneralTab(self): self.frame4 = QGroupBox("General") self.tabWidget.addTab(self.frame4, "General") vbox = QVBoxLayout(); self.frame4.setLayout(vbox); vbox.setContentsMargins(0, 20, 0, 0) label = QLabel("MorphMan will reorder the cards so that the easiest cards are at the front. To avoid getting new cards that are too easy, MorphMan will skip certain new cards. You can customize the skip behavior here:") label.setWordWrap(True) vbox.addWidget(label) vbox.addSpacing(20) grid = QVBoxLayout(); vbox.addLayout(grid); grid.setContentsMargins(20, 0, 0, 0) optionList = [ ("Skip comprehension cards", 'Option_SkipComprehensionCards', 'Note that only has mature words (optimal for sentence learning but not for acquiring new vocabulary).'), ("Skip cards with fresh vocabulary", 'Option_SkipFreshVocabCards', 'Note that does not contain unknown words, but one or\nmore unmature (card with recently learned morphmes). Enable to\nskip to first card that has unknown vocabulary.'), ("Skip card if focus morph was already seen today", 'Option_SkipFocusMorphSeenToday', 'This improves the \'new cards\'-queue without having to recalculate the databases.'), ] self.boolOptionList = [] for i, (name, key, tooltipInfo) in enumerate(optionList): checkBox = QCheckBox(name) checkBox.setCheckState(Qt.Checked if jcfg(key) else Qt.Unchecked) checkBox.setToolTip(tooltipInfo) self.boolOptionList.append((key, checkBox)) grid.addWidget(checkBox) grid.addSpacing(15) vbox.addStretch()
def my_getNewCard(self, _old): '''Continually call _getNewCard until we get one with a focusMorph we haven't seen before. Also skip bad vocab cards. :type self: anki.sched.Scheduler :type _old: Callable ''' while True: C = partial(cfg, None, self.col.decks.active()[0]) if not C('next new card feature'): return _old(self) if not C('new card merged fill'): c = _old(self) ''' :type c: anki.cards.Card ''' else: # pop from opposite direction and skip sibling spacing if not self._fillNew(): return (id, due) = self._newQueue.pop(0) c = self.col.getCard(id) self.newCount -= 1 if not c: return # no more cards n = c.note() try: fm = focus(n) # fm is either the focusMorph or empty except KeyError: return c # card has no focusMorph field -> assume it's good # determine if good vocab word based on whether k+1 # defaults to whether has focus morph if no k+N field or disabled try: goodVocab = n[jcfg('Field_UnknownMorphCount')] == '1' except KeyError: goodVocab = fm # even if it is not a good vocabulary card, we have no choice when there are no other cards available if (not goodVocab and not n.hasTag(jcfg('Tag_NotReady'))) or n.hasTag( jcfg('Tag_AlreadyKnown')) or fm in seenMorphs: self.buryCards([c.id]) self.newCount += 1 # the card was quaried from the "new queue" so we have to increase the "new counter" back to its original value continue break return c
def createTagsTab(self): self.frame3 = QGroupBox("Tags") self.tabWidget.addTab(self.frame3, "Tags") vbox = QVBoxLayout(); self.frame3.setLayout(vbox); vbox.setContentsMargins(0, 20, 0, 0) label = QLabel("This plugin will add and delete following tags from your matched notes. Hover your mouse over text entries to see tooltip info.") label.setWordWrap(True) vbox.addWidget(label) vbox.addSpacing(50) grid = QGridLayout(); vbox.addLayout(grid) tagList = [ ("Vocab note:", 'Tag_Vocab', 'Note that is optimal to learn (one unknown word.)'), ("Compehension note:", 'Tag_Comprehension', 'Note that only has mature words (optimal for sentence learning).'), ("Fresh vocab note:", 'Tag_Fresh', 'Note that does not contain unknown words, but one or\nmore unmature (card with recently learned morphmes).'), ("Not ready:", 'Tag_NotReady', 'Note that has two or more unknown words.'), ("Already known:", 'Tag_AlreadyKnown', 'You can add this tag to a note.\nAfter a recalc of the database, all in this sentence words are marked as known.\nPress \'K\' while reviewing to tag current card.'), ("Priority:", 'Tag_Priority', 'Morpheme is in priority.db.'), ("Too Short:", 'Tag_TooShort', 'Sentence is too short.'), ("Too Long:", 'Tag_TooLong', 'Sentence is too long.'), ] self.tagEntryList = [] numberOfColumns = 2 for i, (name, key, tooltipInfo) in enumerate(tagList): entry = QLineEdit(jcfg(key)) entry.setToolTip(tooltipInfo) self.tagEntryList.append((key, entry)) grid.addWidget(QLabel(name), i // numberOfColumns, (i % numberOfColumns) * 2 + 0) grid.addWidget(entry, i // numberOfColumns, (i % numberOfColumns) * 2 + 1) vbox.addSpacing(50) self.checkboxSetNotRequiredTags = QCheckBox("Add tags even if not required") self.checkboxSetNotRequiredTags.setCheckState(Qt.Checked if jcfg('Option_SetNotRequiredTags') else Qt.Unchecked) vbox.addWidget(self.checkboxSetNotRequiredTags) vbox.addStretch()
def createExtraFieldsTab(self): self.frame2 = QWidget() self.tabWidget.addTab(self.frame2, "Extra Fields") vbox = QVBoxLayout() self.frame2.setLayout(vbox) vbox.setContentsMargins(0, 20, 0, 0) label = QLabel( "This plugin will attempt to change the data in following fields. Every field that has a (*) is REQUIRED IN EVERY NOTE for MorphMan to work correctly. The other fields are optional. Hover your mouse over text entries to see tooltip info." ) label.setWordWrap(True) vbox.addWidget(label) vbox.addSpacing(50) grid = QGridLayout() vbox.addLayout(grid) numberOfColumns = 2 fieldsList = [ ("Focus morph (*):", "Field_FocusMorph", "Stores the unknown morpheme for sentences with one unmature word.\nGets cleared as soon as all works are mature." ), ("MorphMan Index:", "Field_MorphManIndex", "Difficulty of card. This will be set to `due` time of card."), ("Unmatures", "Field_Unmatures", "Comma-separated list of unmature words."), ("Unmatures count:", "Field_UnmatureMorphCount", "Number of unmature words on this note."), ("Unknowns:", "Field_Unknowns", "Comma-separated list of unknown morphemes."), ("Unknown count:", "Field_UnknownMorphCount", "Number of unknown morphemes on this note."), ("Unknown frequency:", "Field_UnknownFreq", "Average of how many times the unknowns appear in your collection." ) ] self.fieldEntryList = [] for i, (name, key, tooltipInfo) in enumerate(fieldsList): entry = QLineEdit(jcfg(key)) entry.setToolTip(tooltipInfo) self.fieldEntryList.append((key, entry)) grid.addWidget(QLabel(name), i / numberOfColumns, (i % numberOfColumns) * 2 + 0) grid.addWidget(entry, i / numberOfColumns, (i % numberOfColumns) * 2 + 1) vbox.addStretch()
def setKnownAndSkip( self ): #2 '''Set card as alreadyKnown and skip along with all other cards with same focusMorph. Useful if you see a focusMorph you already know from external knowledge :type self: aqt.reviewer.Reviewer ''' self.mw.checkpoint( _("Set already known focus morph") ) n = self.card.note() n.addTag(jcfg('Tag_AlreadyKnown')) n.flush() markFocusSeen( self, n ) # "new counter" might have been decreased (but "new card" was not answered # so it shouldn't) -> this function recomputes "new counter" self.mw.col.reset() # skip card self.nextCard()
def createNoteFilterTab(self): self.frame1 = QWidget() self.tabWidget.addTab(self.frame1, "Note Filter") vbox = QVBoxLayout() self.frame1.setLayout(vbox) vbox.setContentsMargins(0, 20, 0, 0) self.tableModel = QStandardItemModel(0, 5) self.tableView = QTableView() self.tableView.setModel(self.tableModel) self.tableView.horizontalHeader().setResizeMode(QHeaderView.Stretch) self.tableView.setSelectionBehavior(QAbstractItemView.SelectRows) self.tableView.setSelectionMode(QAbstractItemView.SingleSelection) self.tableModel.setHeaderData(0, Qt.Horizontal, "Note type") self.tableModel.setHeaderData(1, Qt.Horizontal, "Tags") self.tableModel.setHeaderData(2, Qt.Horizontal, "Fields") self.tableModel.setHeaderData(3, Qt.Horizontal, "Morphemizer") self.tableModel.setHeaderData(4, Qt.Horizontal, "Modify?") rowData = jcfg('Filter') self.tableModel.setRowCount(len(rowData)) self.rowGui = [] for i, row in enumerate(rowData): self.setTableRow(i, row) label = QLabel( "Any card that has the given `Note type` and all of the given `Tags` will have its `Fields` analyzed with the specified `Morphemizer`. A morphemizer specifies how words are extraced from a sentence. `Fields` and `Tags` are both comma-separated lists. If `Tags` is empty, there are no tag restrictions. If `Modify` is deactivated, the note will only be analyzed.\n\nIf a note is matched multple times, the first line will take precedence." ) label.setWordWrap(True) vbox.addWidget(label) vbox.addSpacing(20) vbox.addWidget(self.tableView) hbox = QHBoxLayout() vbox.addLayout(hbox) self.clone = mkBtn("Clone", self.onClone, self, hbox) self.delete = mkBtn("Delete", self.onDelete, self, hbox) self.up = mkBtn("Up", self.onUp, self, hbox) self.down = mkBtn("Down", self.onDown, self, hbox)
def createGeneralTab(self): self.frame4 = QGroupBox("General") self.tabWidget.addTab(self.frame4, "General") vbox = QVBoxLayout() self.frame4.setLayout(vbox) vbox.setContentsMargins(0, 20, 0, 0) label = QLabel( "MorphMan will reorder the cards so that the easiest cards are at the front. To avoid getting new cards that are too easy, MorphMan will skip certain new cards. You can customize the skip behavior here:" ) label.setWordWrap(True) vbox.addWidget(label) vbox.addSpacing(20) grid = QVBoxLayout() vbox.addLayout(grid) grid.setContentsMargins(20, 0, 0, 0) optionList = [ ("Skip comprehension cards", 'Option_SkipComprehensionCards', 'Note that only has mature words (optimal for sentence learning but not for acquiring new vocabulary).' ), ("Skip cards with fresh vocabulary", 'Option_SkipFreshVocabCards', 'Note that does not contain unknown words, but one or\nmore unmature (card with recently learned morphmes). Enable to\nskip to first card that has unknown vocabulary.' ), ("Skip card if focus morph was already seen today", 'Option_SkipFocusMorphSeenToday', 'This improves the \'new cards\'-queue without having to recalculate the databases.' ), ] self.boolOptionList = [] for i, (name, key, tooltipInfo) in enumerate(optionList): checkBox = QCheckBox(name) checkBox.setCheckState(Qt.Checked if jcfg(key) else Qt.Unchecked) checkBox.setToolTip(tooltipInfo) self.boolOptionList.append((key, checkBox)) grid.addWidget(checkBox) grid.addSpacing(15) vbox.addStretch()
def mkAllDb(allDb=None): import config reload(config) t_0, db, TAG = time.time(), mw.col.db, mw.col.tags N_notes = db.scalar('select count() from notes') N_enabled_notes = 0 # for providing an error message if there is no note that is used for processing mw.progress.start(label='Prep work for all.db creation', max=N_notes, immediate=True) if not allDb: allDb = MorphDb() fidDb = allDb.fidDb() locDb = allDb.locDb(recalc=False) # fidDb() already forces locDb recalc mw.progress.update(label='Generating all.db data') for i, (nid, mid, flds, guid, tags) in enumerate( db.execute('select id, mid, flds, guid, tags from notes')): if i % 500 == 0: mw.progress.update(value=i) C = partial(cfg, mid, None) note = mw.col.getNote(nid) notecfg = getFilter(note) if notecfg is None: continue morphemizer = getMorphemizerByName(notecfg['Morphemizer']) N_enabled_notes += 1 mats = [(0.5 if ivl == 0 and ctype == 1 else ivl) for ivl, ctype in db.execute( 'select ivl, type from cards where nid = :nid', nid=nid)] if C('ignore maturity'): mats = [0 for mat in mats] ts, alreadyKnownTag = TAG.split(tags), jcfg('Tag_AlreadyKnown') if alreadyKnownTag in ts: mats += [C('threshold_mature') + 1] for fieldName in notecfg['Fields']: try: # if doesn't have field, continue #fieldValue = normalizeFieldValue( getField( fieldName, flds, mid ) ) fieldValue = extractFieldData(fieldName, flds, mid) except KeyError: continue except TypeError: mname = mw.col.models.get(mid)['name'] errorMsg( u'Failed to get field "{field}" from a note of model "{model}". Please fix your config.py file to match your collection appropriately and ignore the following error.' .format(model=mname, field=fieldName)) raise loc = fidDb.get((nid, guid, fieldName), None) if not loc: loc = AnkiDeck(nid, fieldName, fieldValue, guid, mats) ms = getMorphemes(morphemizer, fieldValue, ts) if ms: #TODO: this needed? should we change below too then? #printf( ' .loc for %d[%s]' % ( nid, fieldName ) ) locDb[loc] = ms else: # mats changed -> new loc (new mats), move morphs if loc.fieldValue == fieldValue and loc.maturities != mats: #printf( ' .mats for %d[%s]' % ( nid, fieldName ) ) newLoc = AnkiDeck(nid, fieldName, fieldValue, guid, mats) locDb[newLoc] = locDb.pop(loc) # field changed -> new loc, new morphs elif loc.fieldValue != fieldValue: #printf( ' .morphs for %d[%s]' % ( nid, fieldName ) ) newLoc = AnkiDeck(nid, fieldName, fieldValue, guid, mats) ms = getMorphemes(morphemizer, fieldValue, ts) locDb.pop(loc) locDb[newLoc] = ms if N_enabled_notes == 0: mw.progress.finish() errorMsg( u'There is no card that can be analyzed or be moved. Add cards or (re-)check your configuration under "Tools -> MorhpMan Preferences" or in "Anki/addons/morph/config.py" for mistakes.' ) return None printf('Processed all %d notes in %f sec' % (N_notes, time.time() - t_0)) mw.progress.update(value=i, label='Creating all.db object') allDb.clear() allDb.addFromLocDb(locDb) if cfg1('saveDbs'): mw.progress.update(value=i, label='Saving all.db to disk') allDb.save(cfg1('path_all')) printf('Processed all %d notes + saved all.db in %f sec' % (N_notes, time.time() - t_0)) mw.progress.finish() return allDb
def updateNotes(allDb): t_0, now, db, TAG = time.time(), intTime(), mw.col.db, mw.col.tags ds, nid2mmi = [], {} N_notes = db.scalar('select count() from notes') mw.progress.start(label='Updating data', max=N_notes, immediate=True) fidDb = allDb.fidDb() locDb = allDb.locDb(recalc=False) # fidDb() already forces locDb recalc # read tag names compTag, vocabTag, freshTag, notReadyTag, alreadyKnownTag, priorityTag, tooShortTag, tooLongTag = tagNames = jcfg( 'Tag_Comprehension'), jcfg('Tag_Vocab'), jcfg('Tag_Fresh'), jcfg( 'Tag_NotReady'), jcfg('Tag_AlreadyKnown'), jcfg( 'Tag_Priority'), jcfg('Tag_TooShort'), jcfg('Tag_TooLong') TAG.register(tagNames) badLengthTag = jcfg2().get('Tag_BadLength') # handle secondary databases mw.progress.update(label='Creating seen/known/mature from all.db') seenDb = filterDbByMat(allDb, cfg1('threshold_seen')) knownDb = filterDbByMat(allDb, cfg1('threshold_known')) matureDb = filterDbByMat(allDb, cfg1('threshold_mature')) mw.progress.update(label='Loading priority.db') priorityDb = MorphDb(cfg1('path_priority'), ignoreErrors=True).db if cfg1('saveDbs'): mw.progress.update(label='Saving seen/known/mature dbs') seenDb.save(cfg1('path_seen')) knownDb.save(cfg1('path_known')) matureDb.save(cfg1('path_mature')) mw.progress.update(label='Updating notes') for i, (nid, mid, flds, guid, tags) in enumerate( db.execute('select id, mid, flds, guid, tags from notes')): if i % 500 == 0: mw.progress.update(value=i) C = partial(cfg, mid, None) note = mw.col.getNote(nid) notecfg = getFilter(note) if notecfg is None or not notecfg['Modify']: continue # Get all morphemes for note morphemes = set() for fieldName in notecfg['Fields']: try: loc = fidDb[(nid, guid, fieldName)] morphemes.update(locDb[loc]) except KeyError: continue # Determine un-seen/known/mature and i+N unseens, unknowns, unmatures, newKnowns = set(), set(), set(), set() for morpheme in morphemes: if morpheme not in seenDb.db: unseens.add(morpheme) if morpheme not in knownDb.db: unknowns.add(morpheme) if morpheme not in matureDb.db: unmatures.add(morpheme) if morpheme not in matureDb.db and morpheme in knownDb.db: newKnowns.add(morpheme) # Determine MMI - Morph Man Index N, N_s, N_k, N_m = len(morphemes), len(unseens), len(unknowns), len( unmatures) # Bail early for lite update if N_k > 2 and C('only update k+2 and below'): continue # average frequency of unknowns (ie. how common the word is within your collection) F_k = 0 for focusMorph in unknowns: # focusMorph used outside loop F_k += allDb.frequency(focusMorph) F_k_avg = F_k // N_k if N_k > 0 else F_k usefulness = F_k_avg # add bonus for morphs in priority.db isPriority = False for focusMorph in unknowns: if focusMorph in priorityDb: isPriority = True usefulness += C('priority.db weight') # add bonus for studying recent learned knowns (reinforce) for morpheme in newKnowns: locs = allDb.db[morpheme] if locs: ivl = min(1, max(loc.maturity for loc in locs)) usefulness += C( 'reinforce new vocab weight' ) // ivl #TODO: maybe average this so it doesnt favor long sentences if any(morpheme.pos == u'動詞' for morpheme in unknowns): #FIXME: this isn't working??? usefulness += C('verb bonus') usefulness = 999 - min(999, usefulness) # difference from optimal length range (too little context vs long sentence) lenDiffRaw = min(N - C('min good sentence length'), max(0, N - C('max good sentence length'))) lenDiff = min(9, abs(lenDiffRaw)) # calculate mmi mmi = 10000 * N_k + 1000 * lenDiff + usefulness if C('set due based on mmi'): nid2mmi[nid] = mmi # Fill in various fields/tags on the note based on cfg ts, fs = TAG.split(tags), splitFields(flds) # clear any 'special' tags, the appropriate will be set in the next few lines ts = [ t for t in ts if t not in [notReadyTag, compTag, vocabTag, freshTag] ] # determine card type if N_m == 0: # sentence comprehension card, m+0 ts = ts + [compTag] setField(mid, fs, jcfg('Field_FocusMorph'), u'') elif N_k == 1: # new vocab card, k+1 ts = ts + [vocabTag] setField(mid, fs, jcfg('Field_FocusMorph'), u'%s' % focusMorph.base) elif N_k > 1: # M+1+ and K+2+ ts = ts + [notReadyTag] setField(mid, fs, jcfg('Field_FocusMorph'), u'') elif N_m == 1: # we have k+0, and m+1, so this card does not introduce a new vocabulary -> card for newly learned morpheme ts = ts + [freshTag] setField(mid, fs, jcfg('Field_FocusMorph'), u'%s' % list(unmatures)[0].base) else: # only case left: we have k+0, but m+2 or higher, so this card does not introduce a new vocabulary -> card for newly learned morpheme ts = ts + [freshTag] setField(mid, fs, jcfg('Field_FocusMorph'), u'') # set type agnostic fields setField(mid, fs, jcfg('Field_UnknownMorphCount'), u'%d' % N_k) setField(mid, fs, jcfg('Field_UnmatureMorphCount'), u'%d' % N_m) setField(mid, fs, jcfg('Field_MorphManIndex'), u'%d' % mmi) setField(mid, fs, jcfg('Field_Unknowns'), u', '.join(u.base for u in unknowns)) setField(mid, fs, jcfg('Field_Unmatures'), u', '.join(u.base for u in unmatures)) setField(mid, fs, jcfg('Field_UnknownFreq'), u'%d' % F_k_avg) # remove deprecated tag if badLengthTag is not None and badLengthTag in ts: ts.remove(badLengthTag) # other tags if priorityTag in ts: ts.remove(priorityTag) if isPriority: ts.append(priorityTag) if tooShortTag in ts: ts.remove(tooShortTag) if lenDiffRaw < 0: ts.append(tooShortTag) if tooLongTag in ts: ts.remove(tooLongTag) if lenDiffRaw > 0: ts.append(tooLongTag) # remove unnecessary tags if not jcfg('Option_SetNotRequiredTags'): unnecessary = [priorityTag, tooShortTag, tooLongTag] ts = [tag for tag in ts if tag not in unnecessary] # update sql db tags_ = TAG.join(TAG.canonify(ts)) flds_ = joinFields(fs) if flds != flds_ or tags != tags_: # only update notes that have changed csum = fieldChecksum(fs[0]) sfld = stripHTML(fs[getSortFieldIndex(mid)]) ds.append({ 'now': now, 'tags': tags_, 'flds': flds_, 'sfld': sfld, 'csum': csum, 'usn': mw.col.usn(), 'nid': nid }) mw.progress.update(value=i, label='Updating anki database...') mw.col.db.executemany( 'update notes set tags=:tags, flds=:flds, sfld=:sfld, csum=:csum, mod=:now, usn=:usn where id=:nid', ds) # Now reorder new cards based on MMI mw.progress.update(value=i, label='Updating new card ordering...') ds = [] # "type = 0": new cards # "type = 1": learning cards [is supposed to be learning: in my case no learning card had this type] # "type = 2": review cards for (cid, nid, due) in db.execute('select id, nid, due from cards where type = 0'): if nid in nid2mmi: # owise it was disabled due_ = nid2mmi[nid] if due != due_: # only update cards that have changed ds.append({ 'now': now, 'due': due_, 'usn': mw.col.usn(), 'cid': cid }) mw.col.db.executemany( 'update cards set due=:due, mod=:now, usn=:usn where id=:cid', ds) mw.reset() printf('Updated notes in %f sec' % (time.time() - t_0)) mw.progress.finish() return knownDb
def my_getNewCard( self, _old ): '''Continually call _getNewCard until we get one with a focusMorph we haven't seen before. Also skip bad vocab cards. :type self: anki.sched.Scheduler :type _old: Callable ''' while True: C = partial( cfg, None, self.col.decks.active()[0] ) if not C('next new card feature'): return _old( self ) if not C('new card merged fill'): c = _old( self ) ''' :type c: anki.cards.Card ''' else: # pop from opposite direction and skip sibling spacing if not self._fillNew(): return ( id, due ) = self._newQueue.pop( 0 ) c = self.col.getCard( id ) self.newCount -= 1 if not c: return # no more cards n = c.note() # find the right morphemizer for this note, so we can apply model-dependent options (modify off == disable skip feature) from morphemes import getMorphemes from util import getFilter notefilter = getFilter(n) if notefilter is None: return c # this note is not configured in any filter -> proceed like normal without MorphMan-plugin if not notefilter['Modify']: return c # the deck should not be modified -> the user probably doesn't want the 'skip mature' feature # get the focus morph try: focusMorph = focus( n ) # field contains either the focusMorph or is empty except KeyError: tooltip( _( 'Encountered card without the \'focus morph\' field configured in the preferences. Please check your MorphMan settings and note models.') ) return c # card has no focusMorph field -> undefined behavior -> just proceed like normal # evaluate all conditions, on which this card might be skipped/buried isVocabCard = n.hasTag(jcfg('Tag_Vocab')) isNotReady = n.hasTag(jcfg('Tag_NotReady')) isComprehensionCard = n.hasTag(jcfg('Tag_Comprehension')) isFreshVocab = n.hasTag(jcfg('Tag_Fresh')) isAlreadyKnown = n.hasTag( jcfg('Tag_AlreadyKnown') ) skipComprehension = jcfg('Option_SkipComprehensionCards') skipFresh = jcfg('Option_SkipFreshVocabCards') skipFocusMorphSeenToday = jcfg('Option_SkipFocusMorphSeenToday') skipCondition1 = (isComprehensionCard and skipComprehension) skipCondition2 = (isFreshVocab and skipFresh) skipCondition3 = isAlreadyKnown # the user requested that the vocabulary does not have to be shown skipCondition4 = (focusMorph in seenMorphs and skipFocusMorphSeenToday) # we already learned that/saw that today #skipCondition5 = not (isVocabCard or isNotReady) # even if it is not a good vocabulary card, we have no choice when there are no other cards available # skip/bury card if any skip condition is true if skipCondition1 or skipCondition2 or skipCondition3 or skipCondition4: self.buryCards( [ c.id ] ) self.newCount += 1 # the card was quaried from the "new queue" so we have to increase the "new counter" back to its original value continue break return c
def focusName( n ): return jcfg('Field_FocusMorph') # TODO remove argument n def focus( n ): return n[ focusName(n) ]
def mkAllDb( allDb=None ): import config; reload(config) t_0, db, TAG = time.time(), mw.col.db, mw.col.tags N_notes = db.scalar( 'select count() from notes' ) N_enabled_notes = 0 # for providing an error message if there is no note that is used for processing mw.progress.start( label='Prep work for all.db creation', max=N_notes, immediate=True ) if not allDb: allDb = MorphDb() fidDb = allDb.fidDb() locDb = allDb.locDb( recalc=False ) # fidDb() already forces locDb recalc mw.progress.update( label='Generating all.db data' ) for i,( nid, mid, flds, guid, tags ) in enumerate( db.execute( 'select id, mid, flds, guid, tags from notes' ) ): if i % 500 == 0: mw.progress.update( value=i ) C = partial( cfg, mid, None ) note = mw.col.getNote(nid) notecfg = getFilter(note) if notecfg is None: continue morphemizer = getMorphemizerByName(notecfg['Morphemizer']) N_enabled_notes += 1 mats = [ ( 0.5 if ivl == 0 and ctype == 1 else ivl ) for ivl, ctype in db.execute( 'select ivl, type from cards where nid = :nid', nid=nid ) ] if C('ignore maturity'): mats = [ 0 for mat in mats ] ts, alreadyKnownTag = TAG.split( tags ), jcfg('Tag_AlreadyKnown') if alreadyKnownTag in ts: mats += [ C('threshold_mature')+1 ] for fieldName in notecfg['Fields']: try: # if doesn't have field, continue #fieldValue = normalizeFieldValue( getField( fieldName, flds, mid ) ) fieldValue = extractFieldData( fieldName, flds, mid ) except KeyError: continue except TypeError: mname = mw.col.models.get( mid )[ 'name' ] errorMsg( u'Failed to get field "{field}" from a note of model "{model}". Please fix your config.py file to match your collection appropriately and ignore the following error.'.format( model=mname, field=fieldName ) ) raise loc = fidDb.get( ( nid, guid, fieldName ), None ) if not loc: loc = AnkiDeck( nid, fieldName, fieldValue, guid, mats ) ms = getMorphemes(morphemizer, fieldValue, ts) if ms: #TODO: this needed? should we change below too then? #printf( ' .loc for %d[%s]' % ( nid, fieldName ) ) locDb[ loc ] = ms else: # mats changed -> new loc (new mats), move morphs if loc.fieldValue == fieldValue and loc.maturities != mats: #printf( ' .mats for %d[%s]' % ( nid, fieldName ) ) newLoc = AnkiDeck( nid, fieldName, fieldValue, guid, mats ) locDb[ newLoc ] = locDb.pop( loc ) # field changed -> new loc, new morphs elif loc.fieldValue != fieldValue: #printf( ' .morphs for %d[%s]' % ( nid, fieldName ) ) newLoc = AnkiDeck( nid, fieldName, fieldValue, guid, mats ) ms = getMorphemes(morphemizer, fieldValue, ts) locDb.pop( loc ) locDb[ newLoc ] = ms if N_enabled_notes == 0: mw.progress.finish() errorMsg(u'There is no card that can be analyzed or be moved. Add cards or (re-)check your configuration under "Tools -> MorhpMan Preferences" or in "Anki/addons/morph/config.py" for mistakes.') return None printf( 'Processed all %d notes in %f sec' % ( N_notes, time.time() - t_0 ) ) mw.progress.update( value=i, label='Creating all.db object' ) allDb.clear() allDb.addFromLocDb( locDb ) if cfg1('saveDbs'): mw.progress.update( value=i, label='Saving all.db to disk' ) allDb.save( cfg1('path_all') ) printf( 'Processed all %d notes + saved all.db in %f sec' % ( N_notes, time.time() - t_0 ) ) mw.progress.finish() return allDb
def updateNotes( allDb ): t_0, now, db, TAG = time.time(), intTime(), mw.col.db, mw.col.tags ds, nid2mmi = [], {} N_notes = db.scalar( 'select count() from notes' ) mw.progress.start( label='Updating data', max=N_notes, immediate=True ) fidDb = allDb.fidDb() locDb = allDb.locDb( recalc=False ) # fidDb() already forces locDb recalc # read tag names compTag, vocabTag, freshTag, notReadyTag, alreadyKnownTag, priorityTag, tooShortTag, tooLongTag = tagNames = jcfg('Tag_Comprehension'), jcfg('Tag_Vocab'), jcfg('Tag_Fresh'), jcfg('Tag_NotReady'), jcfg('Tag_AlreadyKnown'), jcfg('Tag_Priority'), jcfg('Tag_TooShort'), jcfg('Tag_TooLong') TAG.register( tagNames ) badLengthTag = jcfg2().get('Tag_BadLength') # handle secondary databases mw.progress.update( label='Creating seen/known/mature from all.db' ) seenDb = filterDbByMat( allDb, cfg1('threshold_seen') ) knownDb = filterDbByMat( allDb, cfg1('threshold_known') ) matureDb = filterDbByMat( allDb, cfg1('threshold_mature') ) mw.progress.update( label='Loading priority.db' ) priorityDb = MorphDb( cfg1('path_priority'), ignoreErrors=True ).db if cfg1('saveDbs'): mw.progress.update( label='Saving seen/known/mature dbs' ) seenDb.save( cfg1('path_seen') ) knownDb.save( cfg1('path_known') ) matureDb.save( cfg1('path_mature') ) mw.progress.update( label='Updating notes' ) for i,( nid, mid, flds, guid, tags ) in enumerate( db.execute( 'select id, mid, flds, guid, tags from notes' ) ): if i % 500 == 0: mw.progress.update( value=i ) C = partial( cfg, mid, None ) note = mw.col.getNote(nid) notecfg = getFilter(note) if notecfg is None or not notecfg['Modify']: continue # Get all morphemes for note morphemes = set() for fieldName in notecfg['Fields']: try: loc = fidDb[ ( nid, guid, fieldName ) ] morphemes.update( locDb[ loc ] ) except KeyError: continue # Determine un-seen/known/mature and i+N unseens, unknowns, unmatures, newKnowns = set(), set(), set(), set() for morpheme in morphemes: if morpheme not in seenDb.db: unseens.add( morpheme ) if morpheme not in knownDb.db: unknowns.add( morpheme ) if morpheme not in matureDb.db: unmatures.add( morpheme ) if morpheme not in matureDb.db and morpheme in knownDb.db: newKnowns.add( morpheme ) # Determine MMI - Morph Man Index N, N_s, N_k, N_m = len( morphemes ), len( unseens ), len( unknowns ), len( unmatures ) # Bail early for lite update if N_k > 2 and C('only update k+2 and below'): continue # average frequency of unknowns (ie. how common the word is within your collection) F_k = 0 for focusMorph in unknowns: # focusMorph used outside loop F_k += allDb.frequency(focusMorph) F_k_avg = F_k // N_k if N_k > 0 else F_k usefulness = F_k_avg # add bonus for morphs in priority.db isPriority = False for focusMorph in unknowns: if focusMorph in priorityDb: isPriority = True usefulness += C('priority.db weight') # add bonus for studying recent learned knowns (reinforce) for morpheme in newKnowns: locs = allDb.db[ morpheme ] if locs: ivl = min( 1, max( loc.maturity for loc in locs ) ) usefulness += C('reinforce new vocab weight') // ivl #TODO: maybe average this so it doesnt favor long sentences if any( morpheme.pos == u'動詞' for morpheme in unknowns ): #FIXME: this isn't working??? usefulness += C('verb bonus') usefulness = 999 - min( 999, usefulness ) # difference from optimal length range (too little context vs long sentence) lenDiffRaw = min(N - C('min good sentence length'), max(0, N - C('max good sentence length'))) lenDiff = min(9, abs(lenDiffRaw)) # calculate mmi mmi = 10000*N_k + 1000*lenDiff + usefulness if C('set due based on mmi'): nid2mmi[ nid ] = mmi # Fill in various fields/tags on the note based on cfg ts, fs = TAG.split( tags ), splitFields( flds ) # clear any 'special' tags, the appropriate will be set in the next few lines ts = [ t for t in ts if t not in [ notReadyTag, compTag, vocabTag, freshTag ] ] # determine card type if N_m == 0: # sentence comprehension card, m+0 ts = ts + [ compTag ] setField( mid, fs, jcfg('Field_FocusMorph'), u'' ) elif N_k == 1: # new vocab card, k+1 ts = ts + [ vocabTag ] setField( mid, fs, jcfg('Field_FocusMorph'), u'%s' % focusMorph.base ) elif N_k > 1: # M+1+ and K+2+ ts = ts + [ notReadyTag ] setField( mid, fs, jcfg('Field_FocusMorph'), u'') elif N_m == 1: # we have k+0, and m+1, so this card does not introduce a new vocabulary -> card for newly learned morpheme ts = ts + [ freshTag ] setField( mid, fs, jcfg('Field_FocusMorph'), u'%s' % list(unmatures)[0].base) else: # only case left: we have k+0, but m+2 or higher, so this card does not introduce a new vocabulary -> card for newly learned morpheme ts = ts + [ freshTag ] setField( mid, fs, jcfg('Field_FocusMorph'), u'') # set type agnostic fields setField( mid, fs, jcfg('Field_UnknownMorphCount'), u'%d' % N_k ) setField( mid, fs, jcfg('Field_UnmatureMorphCount'), u'%d' % N_m ) setField( mid, fs, jcfg('Field_MorphManIndex'), u'%d' % mmi ) setField( mid, fs, jcfg('Field_Unknowns'), u', '.join( u.base for u in unknowns ) ) setField( mid, fs, jcfg('Field_Unmatures'), u', '.join( u.base for u in unmatures ) ) setField( mid, fs, jcfg('Field_UnknownFreq'), u'%d' % F_k_avg ) # remove deprecated tag if badLengthTag is not None and badLengthTag in ts: ts.remove( badLengthTag ) # other tags if priorityTag in ts: ts.remove( priorityTag ) if isPriority: ts.append( priorityTag ) if tooShortTag in ts: ts.remove( tooShortTag ) if lenDiffRaw < 0: ts.append( tooShortTag ) if tooLongTag in ts: ts.remove( tooLongTag ) if lenDiffRaw > 0: ts.append( tooLongTag ) # remove unnecessary tags if not jcfg('Option_SetNotRequiredTags'): unnecessary = [priorityTag, tooShortTag, tooLongTag] ts = [tag for tag in ts if tag not in unnecessary] # update sql db tags_ = TAG.join( TAG.canonify( ts ) ) flds_ = joinFields( fs ) if flds != flds_ or tags != tags_: # only update notes that have changed csum = fieldChecksum( fs[0] ) sfld = stripHTML( fs[ getSortFieldIndex( mid ) ] ) ds.append( { 'now':now, 'tags':tags_, 'flds':flds_, 'sfld':sfld, 'csum':csum, 'usn':mw.col.usn(), 'nid':nid } ) mw.progress.update( value=i, label='Updating anki database...' ) mw.col.db.executemany( 'update notes set tags=:tags, flds=:flds, sfld=:sfld, csum=:csum, mod=:now, usn=:usn where id=:nid', ds ) # Now reorder new cards based on MMI mw.progress.update( value=i, label='Updating new card ordering...' ) ds = [] # "type = 0": new cards # "type = 1": learning cards [is supposed to be learning: in my case no learning card had this type] # "type = 2": review cards for ( cid, nid, due ) in db.execute( 'select id, nid, due from cards where type = 0' ): if nid in nid2mmi: # owise it was disabled due_ = nid2mmi[ nid ] if due != due_: # only update cards that have changed ds.append( { 'now':now, 'due':due_, 'usn':mw.col.usn(), 'cid':cid } ) mw.col.db.executemany( 'update cards set due=:due, mod=:now, usn=:usn where id=:cid', ds ) mw.reset() printf( 'Updated notes in %f sec' % ( time.time() - t_0 ) ) mw.progress.finish() return knownDb