def createTagsTab(self):
self.frame3 = QGroupBox("Tags")
self.tabWidget.addTab(self.frame3, "Tags")
vbox = QVBoxLayout()
self.frame3.setLayout(vbox)
vbox.setContentsMargins(0, 20, 0, 0)
label = QLabel(
"This plugin will add and delete following tags from your matched notes. Hover your mouse over text entries to see tooltip info."
)
label.setWordWrap(True)
vbox.addWidget(label)
vbox.addSpacing(50)
grid = QGridLayout()
vbox.addLayout(grid)
tagList = [
("Vocab note:", 'Tag_Vocab',
'Note that is optimal to learn (one unknown word.)'),
("Compehension note:", 'Tag_Comprehension',
'Note that only has mature words (optimal for sentence learning).'
),
("Fresh vocab note:", 'Tag_Fresh',
'Note that does not contain unknown words, but one or\nmore unmature (card with recently learned morphmes).'
),
("Not ready:", 'Tag_NotReady',
'Note that has two or more unknown words.'),
("Already known:", 'Tag_AlreadyKnown',
'You can add this tag to a note.\nAfter a recalc of the database, all in this sentence words are marked as known.\nPress \'K\' while reviewing to tag current card.'
),
("Priority:", 'Tag_Priority', 'Morpheme is in priority.db.'),
("Too Short:", 'Tag_TooShort', 'Sentence is too short.'),
("Too Long:", 'Tag_TooLong', 'Sentence is too long.'),
]
self.tagEntryList = []
numberOfColumns = 2
for i, (name, key, tooltipInfo) in enumerate(tagList):
entry = QLineEdit(jcfg(key))
entry.setToolTip(tooltipInfo)
self.tagEntryList.append((key, entry))
grid.addWidget(QLabel(name), i // numberOfColumns,
(i % numberOfColumns) * 2 + 0)
grid.addWidget(entry, i // numberOfColumns,
(i % numberOfColumns) * 2 + 1)
vbox.addSpacing(50)
self.checkboxSetNotRequiredTags = QCheckBox(
"Add tags even if not required")
self.checkboxSetNotRequiredTags.setCheckState(
Qt.Checked if jcfg('Option_SetNotRequiredTags') else Qt.Unchecked)
vbox.addWidget(self.checkboxSetNotRequiredTags)
vbox.addStretch()
def createExtraFieldsTab(self):
self.frame2 = QWidget()
self.tabWidget.addTab(self.frame2, "Extra Fields")
vbox = QVBoxLayout(); self.frame2.setLayout(vbox); vbox.setContentsMargins(0, 20, 0, 0)
label = QLabel("This plugin will attempt to change the data in following fields. Every field that has a (*) is REQUIRED IN EVERY NOTE for MorphMan to work correctly. The other fields are optional. Hover your mouse over text entries to see tooltip info.")
label.setWordWrap(True)
vbox.addWidget(label)
vbox.addSpacing(50)
grid = QGridLayout(); vbox.addLayout(grid)
numberOfColumns = 2
fieldsList = [
("Focus morph (*):", "Field_FocusMorph", "Stores the unknown morpheme for sentences with one unmature word.\nGets cleared as soon as all works are mature."),
("MorphMan Index:", "Field_MorphManIndex", "Difficulty of card. This will be set to `due` time of card."),
("Unmatures", "Field_Unmatures", "Comma-separated list of unmature words."),
("Unmatures count:", "Field_UnmatureMorphCount", "Number of unmature words on this note."),
("Unknowns:", "Field_Unknowns", "Comma-separated list of unknown morphemes."),
("Unknown count:", "Field_UnknownMorphCount", "Number of unknown morphemes on this note."),
("Unknown frequency:", "Field_UnknownFreq", "Average of how many times the unknowns appear in your collection.")
]
self.fieldEntryList = []
for i, (name, key, tooltipInfo) in enumerate(fieldsList):
entry = QLineEdit(jcfg(key))
entry.setToolTip(tooltipInfo)
self.fieldEntryList.append((key, entry))
grid.addWidget(QLabel(name), i // numberOfColumns, (i % numberOfColumns) * 2 + 0)
grid.addWidget(entry, i // numberOfColumns, (i % numberOfColumns) * 2 + 1)
vbox.addStretch()
def createNoteFilterTab(self):
self.frame1 = QWidget()
self.tabWidget.addTab(self.frame1, "Note Filter")
vbox = QVBoxLayout(); self.frame1.setLayout(vbox); vbox.setContentsMargins(0, 20, 0, 0)
self.tableModel = QStandardItemModel(0, 5)
self.tableView = QTableView()
self.tableView.setModel(self.tableModel)
self.tableView.horizontalHeader().setResizeMode(QHeaderView.Stretch)
self.tableView.setSelectionBehavior(QAbstractItemView.SelectRows)
self.tableView.setSelectionMode(QAbstractItemView.SingleSelection)
self.tableModel.setHeaderData(0, Qt.Horizontal, "Note type")
self.tableModel.setHeaderData(1, Qt.Horizontal, "Tags")
self.tableModel.setHeaderData(2, Qt.Horizontal, "Fields")
self.tableModel.setHeaderData(3, Qt.Horizontal, "Morphemizer")
self.tableModel.setHeaderData(4, Qt.Horizontal, "Modify?")
rowData = jcfg('Filter')
self.tableModel.setRowCount(len(rowData))
self.rowGui = []
for i, row in enumerate(rowData):
self.setTableRow(i, row)
label = QLabel("Any card that has the given `Note type` and all of the given `Tags` will have its `Fields` analyzed with the specified `Morphemizer`. A morphemizer specifies how words are extraced from a sentence. `Fields` and `Tags` are both comma-separated lists. If `Tags` is empty, there are no tag restrictions. If `Modify` is deactivated, the note will only be analyzed.\n\nIf a note is matched multple times, only the first filter in this list will be used.")
label.setWordWrap(True)
vbox.addWidget(label)
vbox.addSpacing(20)
vbox.addWidget(self.tableView)
hbox = QHBoxLayout(); vbox.addLayout(hbox)
self.clone = mkBtn("Clone", self.onClone, self, hbox)
self.delete = mkBtn("Delete", self.onDelete, self, hbox)
self.up = mkBtn("Up", self.onUp, self, hbox)
self.down = mkBtn("Down", self.onDown, self, hbox)
def createGeneralTab(self):
self.frame4 = QGroupBox("General")
self.tabWidget.addTab(self.frame4, "General")
vbox = QVBoxLayout(); self.frame4.setLayout(vbox); vbox.setContentsMargins(0, 20, 0, 0)
label = QLabel("MorphMan will reorder the cards so that the easiest cards are at the front. To avoid getting new cards that are too easy, MorphMan will skip certain new cards. You can customize the skip behavior here:")
label.setWordWrap(True)
vbox.addWidget(label)
vbox.addSpacing(20)
grid = QVBoxLayout(); vbox.addLayout(grid); grid.setContentsMargins(20, 0, 0, 0)
optionList = [
("Skip comprehension cards", 'Option_SkipComprehensionCards', 'Note that only has mature words (optimal for sentence learning but not for acquiring new vocabulary).'),
("Skip cards with fresh vocabulary", 'Option_SkipFreshVocabCards', 'Note that does not contain unknown words, but one or\nmore unmature (card with recently learned morphmes). Enable to\nskip to first card that has unknown vocabulary.'),
("Skip card if focus morph was already seen today", 'Option_SkipFocusMorphSeenToday', 'This improves the \'new cards\'-queue without having to recalculate the databases.'),
]
self.boolOptionList = []
for i, (name, key, tooltipInfo) in enumerate(optionList):
checkBox = QCheckBox(name)
checkBox.setCheckState(Qt.Checked if jcfg(key) else Qt.Unchecked)
checkBox.setToolTip(tooltipInfo)
self.boolOptionList.append((key, checkBox))
grid.addWidget(checkBox)
grid.addSpacing(15)
vbox.addStretch()
def my_getNewCard(self, _old):
'''Continually call _getNewCard until we get one with a focusMorph we haven't
seen before. Also skip bad vocab cards.
:type self: anki.sched.Scheduler
:type _old: Callable
'''
while True:
C = partial(cfg, None, self.col.decks.active()[0])
if not C('next new card feature'):
return _old(self)
if not C('new card merged fill'):
c = _old(self)
''' :type c: anki.cards.Card '''
else: # pop from opposite direction and skip sibling spacing
if not self._fillNew(): return
(id, due) = self._newQueue.pop(0)
c = self.col.getCard(id)
self.newCount -= 1
if not c: return # no more cards
n = c.note()
try:
fm = focus(n) # fm is either the focusMorph or empty
except KeyError:
return c # card has no focusMorph field -> assume it's good
# determine if good vocab word based on whether k+1
# defaults to whether has focus morph if no k+N field or disabled
try:
goodVocab = n[jcfg('Field_UnknownMorphCount')] == '1'
except KeyError:
goodVocab = fm
# even if it is not a good vocabulary card, we have no choice when there are no other cards available
if (not goodVocab and not n.hasTag(jcfg('Tag_NotReady'))) or n.hasTag(
jcfg('Tag_AlreadyKnown')) or fm in seenMorphs:
self.buryCards([c.id])
self.newCount += 1 # the card was quaried from the "new queue" so we have to increase the "new counter" back to its original value
continue
break
return c
def createTagsTab(self):
self.frame3 = QGroupBox("Tags")
self.tabWidget.addTab(self.frame3, "Tags")
vbox = QVBoxLayout(); self.frame3.setLayout(vbox); vbox.setContentsMargins(0, 20, 0, 0)
label = QLabel("This plugin will add and delete following tags from your matched notes. Hover your mouse over text entries to see tooltip info.")
label.setWordWrap(True)
vbox.addWidget(label)
vbox.addSpacing(50)
grid = QGridLayout(); vbox.addLayout(grid)
tagList = [
("Vocab note:", 'Tag_Vocab', 'Note that is optimal to learn (one unknown word.)'),
("Compehension note:", 'Tag_Comprehension', 'Note that only has mature words (optimal for sentence learning).'),
("Fresh vocab note:", 'Tag_Fresh', 'Note that does not contain unknown words, but one or\nmore unmature (card with recently learned morphmes).'),
("Not ready:", 'Tag_NotReady', 'Note that has two or more unknown words.'),
("Already known:", 'Tag_AlreadyKnown', 'You can add this tag to a note.\nAfter a recalc of the database, all in this sentence words are marked as known.\nPress \'K\' while reviewing to tag current card.'),
("Priority:", 'Tag_Priority', 'Morpheme is in priority.db.'),
("Too Short:", 'Tag_TooShort', 'Sentence is too short.'),
("Too Long:", 'Tag_TooLong', 'Sentence is too long.'),
]
self.tagEntryList = []
numberOfColumns = 2
for i, (name, key, tooltipInfo) in enumerate(tagList):
entry = QLineEdit(jcfg(key))
entry.setToolTip(tooltipInfo)
self.tagEntryList.append((key, entry))
grid.addWidget(QLabel(name), i // numberOfColumns, (i % numberOfColumns) * 2 + 0)
grid.addWidget(entry, i // numberOfColumns, (i % numberOfColumns) * 2 + 1)
vbox.addSpacing(50)
self.checkboxSetNotRequiredTags = QCheckBox("Add tags even if not required")
self.checkboxSetNotRequiredTags.setCheckState(Qt.Checked if jcfg('Option_SetNotRequiredTags') else Qt.Unchecked)
vbox.addWidget(self.checkboxSetNotRequiredTags)
vbox.addStretch()
def createExtraFieldsTab(self):
self.frame2 = QWidget()
self.tabWidget.addTab(self.frame2, "Extra Fields")
vbox = QVBoxLayout()
self.frame2.setLayout(vbox)
vbox.setContentsMargins(0, 20, 0, 0)
label = QLabel(
"This plugin will attempt to change the data in following fields. Every field that has a (*) is REQUIRED IN EVERY NOTE for MorphMan to work correctly. The other fields are optional. Hover your mouse over text entries to see tooltip info."
)
label.setWordWrap(True)
vbox.addWidget(label)
vbox.addSpacing(50)
grid = QGridLayout()
vbox.addLayout(grid)
numberOfColumns = 2
fieldsList = [
("Focus morph (*):", "Field_FocusMorph",
"Stores the unknown morpheme for sentences with one unmature word.\nGets cleared as soon as all works are mature."
),
("MorphMan Index:", "Field_MorphManIndex",
"Difficulty of card. This will be set to `due` time of card."),
("Unmatures", "Field_Unmatures",
"Comma-separated list of unmature words."),
("Unmatures count:", "Field_UnmatureMorphCount",
"Number of unmature words on this note."),
("Unknowns:", "Field_Unknowns",
"Comma-separated list of unknown morphemes."),
("Unknown count:", "Field_UnknownMorphCount",
"Number of unknown morphemes on this note."),
("Unknown frequency:", "Field_UnknownFreq",
"Average of how many times the unknowns appear in your collection."
)
]
self.fieldEntryList = []
for i, (name, key, tooltipInfo) in enumerate(fieldsList):
entry = QLineEdit(jcfg(key))
entry.setToolTip(tooltipInfo)
self.fieldEntryList.append((key, entry))
grid.addWidget(QLabel(name), i / numberOfColumns,
(i % numberOfColumns) * 2 + 0)
grid.addWidget(entry, i / numberOfColumns,
(i % numberOfColumns) * 2 + 1)
vbox.addStretch()
def setKnownAndSkip( self ): #2
'''Set card as alreadyKnown and skip along with all other cards with same focusMorph.
Useful if you see a focusMorph you already know from external knowledge
:type self: aqt.reviewer.Reviewer '''
self.mw.checkpoint( _("Set already known focus morph") )
n = self.card.note()
n.addTag(jcfg('Tag_AlreadyKnown'))
n.flush()
markFocusSeen( self, n )
# "new counter" might have been decreased (but "new card" was not answered
# so it shouldn't) -> this function recomputes "new counter"
self.mw.col.reset()
# skip card
self.nextCard()
def createNoteFilterTab(self):
self.frame1 = QWidget()
self.tabWidget.addTab(self.frame1, "Note Filter")
vbox = QVBoxLayout()
self.frame1.setLayout(vbox)
vbox.setContentsMargins(0, 20, 0, 0)
self.tableModel = QStandardItemModel(0, 5)
self.tableView = QTableView()
self.tableView.setModel(self.tableModel)
self.tableView.horizontalHeader().setResizeMode(QHeaderView.Stretch)
self.tableView.setSelectionBehavior(QAbstractItemView.SelectRows)
self.tableView.setSelectionMode(QAbstractItemView.SingleSelection)
self.tableModel.setHeaderData(0, Qt.Horizontal, "Note type")
self.tableModel.setHeaderData(1, Qt.Horizontal, "Tags")
self.tableModel.setHeaderData(2, Qt.Horizontal, "Fields")
self.tableModel.setHeaderData(3, Qt.Horizontal, "Morphemizer")
self.tableModel.setHeaderData(4, Qt.Horizontal, "Modify?")
rowData = jcfg('Filter')
self.tableModel.setRowCount(len(rowData))
self.rowGui = []
for i, row in enumerate(rowData):
self.setTableRow(i, row)
label = QLabel(
"Any card that has the given `Note type` and all of the given `Tags` will have its `Fields` analyzed with the specified `Morphemizer`. A morphemizer specifies how words are extraced from a sentence. `Fields` and `Tags` are both comma-separated lists. If `Tags` is empty, there are no tag restrictions. If `Modify` is deactivated, the note will only be analyzed.\n\nIf a note is matched multple times, the first line will take precedence."
)
label.setWordWrap(True)
vbox.addWidget(label)
vbox.addSpacing(20)
vbox.addWidget(self.tableView)
hbox = QHBoxLayout()
vbox.addLayout(hbox)
self.clone = mkBtn("Clone", self.onClone, self, hbox)
self.delete = mkBtn("Delete", self.onDelete, self, hbox)
self.up = mkBtn("Up", self.onUp, self, hbox)
self.down = mkBtn("Down", self.onDown, self, hbox)
def createGeneralTab(self):
self.frame4 = QGroupBox("General")
self.tabWidget.addTab(self.frame4, "General")
vbox = QVBoxLayout()
self.frame4.setLayout(vbox)
vbox.setContentsMargins(0, 20, 0, 0)
label = QLabel(
"MorphMan will reorder the cards so that the easiest cards are at the front. To avoid getting new cards that are too easy, MorphMan will skip certain new cards. You can customize the skip behavior here:"
)
label.setWordWrap(True)
vbox.addWidget(label)
vbox.addSpacing(20)
grid = QVBoxLayout()
vbox.addLayout(grid)
grid.setContentsMargins(20, 0, 0, 0)
optionList = [
("Skip comprehension cards", 'Option_SkipComprehensionCards',
'Note that only has mature words (optimal for sentence learning but not for acquiring new vocabulary).'
),
("Skip cards with fresh vocabulary", 'Option_SkipFreshVocabCards',
'Note that does not contain unknown words, but one or\nmore unmature (card with recently learned morphmes). Enable to\nskip to first card that has unknown vocabulary.'
),
("Skip card if focus morph was already seen today",
'Option_SkipFocusMorphSeenToday',
'This improves the \'new cards\'-queue without having to recalculate the databases.'
),
]
self.boolOptionList = []
for i, (name, key, tooltipInfo) in enumerate(optionList):
checkBox = QCheckBox(name)
checkBox.setCheckState(Qt.Checked if jcfg(key) else Qt.Unchecked)
checkBox.setToolTip(tooltipInfo)
self.boolOptionList.append((key, checkBox))
grid.addWidget(checkBox)
grid.addSpacing(15)
vbox.addStretch()
def mkAllDb(allDb=None):
import config
reload(config)
t_0, db, TAG = time.time(), mw.col.db, mw.col.tags
N_notes = db.scalar('select count() from notes')
N_enabled_notes = 0 # for providing an error message if there is no note that is used for processing
mw.progress.start(label='Prep work for all.db creation',
max=N_notes,
immediate=True)
if not allDb: allDb = MorphDb()
fidDb = allDb.fidDb()
locDb = allDb.locDb(recalc=False) # fidDb() already forces locDb recalc
mw.progress.update(label='Generating all.db data')
for i, (nid, mid, flds, guid, tags) in enumerate(
db.execute('select id, mid, flds, guid, tags from notes')):
if i % 500 == 0: mw.progress.update(value=i)
C = partial(cfg, mid, None)
note = mw.col.getNote(nid)
notecfg = getFilter(note)
if notecfg is None: continue
morphemizer = getMorphemizerByName(notecfg['Morphemizer'])
N_enabled_notes += 1
mats = [(0.5 if ivl == 0 and ctype == 1 else ivl)
for ivl, ctype in db.execute(
'select ivl, type from cards where nid = :nid', nid=nid)]
if C('ignore maturity'):
mats = [0 for mat in mats]
ts, alreadyKnownTag = TAG.split(tags), jcfg('Tag_AlreadyKnown')
if alreadyKnownTag in ts:
mats += [C('threshold_mature') + 1]
for fieldName in notecfg['Fields']:
try: # if doesn't have field, continue
#fieldValue = normalizeFieldValue( getField( fieldName, flds, mid ) )
fieldValue = extractFieldData(fieldName, flds, mid)
except KeyError:
continue
except TypeError:
mname = mw.col.models.get(mid)['name']
errorMsg(
u'Failed to get field "{field}" from a note of model "{model}". Please fix your config.py file to match your collection appropriately and ignore the following error.'
.format(model=mname, field=fieldName))
raise
loc = fidDb.get((nid, guid, fieldName), None)
if not loc:
loc = AnkiDeck(nid, fieldName, fieldValue, guid, mats)
ms = getMorphemes(morphemizer, fieldValue, ts)
if ms: #TODO: this needed? should we change below too then?
#printf( ' .loc for %d[%s]' % ( nid, fieldName ) )
locDb[loc] = ms
else:
# mats changed -> new loc (new mats), move morphs
if loc.fieldValue == fieldValue and loc.maturities != mats:
#printf( ' .mats for %d[%s]' % ( nid, fieldName ) )
newLoc = AnkiDeck(nid, fieldName, fieldValue, guid, mats)
locDb[newLoc] = locDb.pop(loc)
# field changed -> new loc, new morphs
elif loc.fieldValue != fieldValue:
#printf( ' .morphs for %d[%s]' % ( nid, fieldName ) )
newLoc = AnkiDeck(nid, fieldName, fieldValue, guid, mats)
ms = getMorphemes(morphemizer, fieldValue, ts)
locDb.pop(loc)
locDb[newLoc] = ms
if N_enabled_notes == 0:
mw.progress.finish()
errorMsg(
u'There is no card that can be analyzed or be moved. Add cards or (re-)check your configuration under "Tools -> MorhpMan Preferences" or in "Anki/addons/morph/config.py" for mistakes.'
)
return None
printf('Processed all %d notes in %f sec' % (N_notes, time.time() - t_0))
mw.progress.update(value=i, label='Creating all.db object')
allDb.clear()
allDb.addFromLocDb(locDb)
if cfg1('saveDbs'):
mw.progress.update(value=i, label='Saving all.db to disk')
allDb.save(cfg1('path_all'))
printf('Processed all %d notes + saved all.db in %f sec' %
(N_notes, time.time() - t_0))
mw.progress.finish()
return allDb
def updateNotes(allDb):
t_0, now, db, TAG = time.time(), intTime(), mw.col.db, mw.col.tags
ds, nid2mmi = [], {}
N_notes = db.scalar('select count() from notes')
mw.progress.start(label='Updating data', max=N_notes, immediate=True)
fidDb = allDb.fidDb()
locDb = allDb.locDb(recalc=False) # fidDb() already forces locDb recalc
# read tag names
compTag, vocabTag, freshTag, notReadyTag, alreadyKnownTag, priorityTag, tooShortTag, tooLongTag = tagNames = jcfg(
'Tag_Comprehension'), jcfg('Tag_Vocab'), jcfg('Tag_Fresh'), jcfg(
'Tag_NotReady'), jcfg('Tag_AlreadyKnown'), jcfg(
'Tag_Priority'), jcfg('Tag_TooShort'), jcfg('Tag_TooLong')
TAG.register(tagNames)
badLengthTag = jcfg2().get('Tag_BadLength')
# handle secondary databases
mw.progress.update(label='Creating seen/known/mature from all.db')
seenDb = filterDbByMat(allDb, cfg1('threshold_seen'))
knownDb = filterDbByMat(allDb, cfg1('threshold_known'))
matureDb = filterDbByMat(allDb, cfg1('threshold_mature'))
mw.progress.update(label='Loading priority.db')
priorityDb = MorphDb(cfg1('path_priority'), ignoreErrors=True).db
if cfg1('saveDbs'):
mw.progress.update(label='Saving seen/known/mature dbs')
seenDb.save(cfg1('path_seen'))
knownDb.save(cfg1('path_known'))
matureDb.save(cfg1('path_mature'))
mw.progress.update(label='Updating notes')
for i, (nid, mid, flds, guid, tags) in enumerate(
db.execute('select id, mid, flds, guid, tags from notes')):
if i % 500 == 0: mw.progress.update(value=i)
C = partial(cfg, mid, None)
note = mw.col.getNote(nid)
notecfg = getFilter(note)
if notecfg is None or not notecfg['Modify']: continue
# Get all morphemes for note
morphemes = set()
for fieldName in notecfg['Fields']:
try:
loc = fidDb[(nid, guid, fieldName)]
morphemes.update(locDb[loc])
except KeyError:
continue
# Determine un-seen/known/mature and i+N
unseens, unknowns, unmatures, newKnowns = set(), set(), set(), set()
for morpheme in morphemes:
if morpheme not in seenDb.db: unseens.add(morpheme)
if morpheme not in knownDb.db: unknowns.add(morpheme)
if morpheme not in matureDb.db: unmatures.add(morpheme)
if morpheme not in matureDb.db and morpheme in knownDb.db:
newKnowns.add(morpheme)
# Determine MMI - Morph Man Index
N, N_s, N_k, N_m = len(morphemes), len(unseens), len(unknowns), len(
unmatures)
# Bail early for lite update
if N_k > 2 and C('only update k+2 and below'): continue
# average frequency of unknowns (ie. how common the word is within your collection)
F_k = 0
for focusMorph in unknowns: # focusMorph used outside loop
F_k += allDb.frequency(focusMorph)
F_k_avg = F_k // N_k if N_k > 0 else F_k
usefulness = F_k_avg
# add bonus for morphs in priority.db
isPriority = False
for focusMorph in unknowns:
if focusMorph in priorityDb:
isPriority = True
usefulness += C('priority.db weight')
# add bonus for studying recent learned knowns (reinforce)
for morpheme in newKnowns:
locs = allDb.db[morpheme]
if locs:
ivl = min(1, max(loc.maturity for loc in locs))
usefulness += C(
'reinforce new vocab weight'
) // ivl #TODO: maybe average this so it doesnt favor long sentences
if any(morpheme.pos == u'動詞'
for morpheme in unknowns): #FIXME: this isn't working???
usefulness += C('verb bonus')
usefulness = 999 - min(999, usefulness)
# difference from optimal length range (too little context vs long sentence)
lenDiffRaw = min(N - C('min good sentence length'),
max(0, N - C('max good sentence length')))
lenDiff = min(9, abs(lenDiffRaw))
# calculate mmi
mmi = 10000 * N_k + 1000 * lenDiff + usefulness
if C('set due based on mmi'):
nid2mmi[nid] = mmi
# Fill in various fields/tags on the note based on cfg
ts, fs = TAG.split(tags), splitFields(flds)
# clear any 'special' tags, the appropriate will be set in the next few lines
ts = [
t for t in ts
if t not in [notReadyTag, compTag, vocabTag, freshTag]
]
# determine card type
if N_m == 0: # sentence comprehension card, m+0
ts = ts + [compTag]
setField(mid, fs, jcfg('Field_FocusMorph'), u'')
elif N_k == 1: # new vocab card, k+1
ts = ts + [vocabTag]
setField(mid, fs, jcfg('Field_FocusMorph'),
u'%s' % focusMorph.base)
elif N_k > 1: # M+1+ and K+2+
ts = ts + [notReadyTag]
setField(mid, fs, jcfg('Field_FocusMorph'), u'')
elif N_m == 1: # we have k+0, and m+1, so this card does not introduce a new vocabulary -> card for newly learned morpheme
ts = ts + [freshTag]
setField(mid, fs, jcfg('Field_FocusMorph'),
u'%s' % list(unmatures)[0].base)
else: # only case left: we have k+0, but m+2 or higher, so this card does not introduce a new vocabulary -> card for newly learned morpheme
ts = ts + [freshTag]
setField(mid, fs, jcfg('Field_FocusMorph'), u'')
# set type agnostic fields
setField(mid, fs, jcfg('Field_UnknownMorphCount'), u'%d' % N_k)
setField(mid, fs, jcfg('Field_UnmatureMorphCount'), u'%d' % N_m)
setField(mid, fs, jcfg('Field_MorphManIndex'), u'%d' % mmi)
setField(mid, fs, jcfg('Field_Unknowns'),
u', '.join(u.base for u in unknowns))
setField(mid, fs, jcfg('Field_Unmatures'),
u', '.join(u.base for u in unmatures))
setField(mid, fs, jcfg('Field_UnknownFreq'), u'%d' % F_k_avg)
# remove deprecated tag
if badLengthTag is not None and badLengthTag in ts:
ts.remove(badLengthTag)
# other tags
if priorityTag in ts: ts.remove(priorityTag)
if isPriority: ts.append(priorityTag)
if tooShortTag in ts: ts.remove(tooShortTag)
if lenDiffRaw < 0: ts.append(tooShortTag)
if tooLongTag in ts: ts.remove(tooLongTag)
if lenDiffRaw > 0: ts.append(tooLongTag)
# remove unnecessary tags
if not jcfg('Option_SetNotRequiredTags'):
unnecessary = [priorityTag, tooShortTag, tooLongTag]
ts = [tag for tag in ts if tag not in unnecessary]
# update sql db
tags_ = TAG.join(TAG.canonify(ts))
flds_ = joinFields(fs)
if flds != flds_ or tags != tags_: # only update notes that have changed
csum = fieldChecksum(fs[0])
sfld = stripHTML(fs[getSortFieldIndex(mid)])
ds.append({
'now': now,
'tags': tags_,
'flds': flds_,
'sfld': sfld,
'csum': csum,
'usn': mw.col.usn(),
'nid': nid
})
mw.progress.update(value=i, label='Updating anki database...')
mw.col.db.executemany(
'update notes set tags=:tags, flds=:flds, sfld=:sfld, csum=:csum, mod=:now, usn=:usn where id=:nid',
ds)
# Now reorder new cards based on MMI
mw.progress.update(value=i, label='Updating new card ordering...')
ds = []
# "type = 0": new cards
# "type = 1": learning cards [is supposed to be learning: in my case no learning card had this type]
# "type = 2": review cards
for (cid, nid,
due) in db.execute('select id, nid, due from cards where type = 0'):
if nid in nid2mmi: # owise it was disabled
due_ = nid2mmi[nid]
if due != due_: # only update cards that have changed
ds.append({
'now': now,
'due': due_,
'usn': mw.col.usn(),
'cid': cid
})
mw.col.db.executemany(
'update cards set due=:due, mod=:now, usn=:usn where id=:cid', ds)
mw.reset()
printf('Updated notes in %f sec' % (time.time() - t_0))
mw.progress.finish()
return knownDb
def my_getNewCard( self, _old ):
'''Continually call _getNewCard until we get one with a focusMorph we haven't
seen before. Also skip bad vocab cards.
:type self: anki.sched.Scheduler
:type _old: Callable
'''
while True:
C = partial( cfg, None, self.col.decks.active()[0] )
if not C('next new card feature'):
return _old( self )
if not C('new card merged fill'):
c = _old( self )
''' :type c: anki.cards.Card '''
else: # pop from opposite direction and skip sibling spacing
if not self._fillNew(): return
( id, due ) = self._newQueue.pop( 0 )
c = self.col.getCard( id )
self.newCount -= 1
if not c: return # no more cards
n = c.note()
# find the right morphemizer for this note, so we can apply model-dependent options (modify off == disable skip feature)
from morphemes import getMorphemes
from util import getFilter
notefilter = getFilter(n)
if notefilter is None: return c # this note is not configured in any filter -> proceed like normal without MorphMan-plugin
if not notefilter['Modify']: return c # the deck should not be modified -> the user probably doesn't want the 'skip mature' feature
# get the focus morph
try: focusMorph = focus( n ) # field contains either the focusMorph or is empty
except KeyError:
tooltip( _( 'Encountered card without the \'focus morph\' field configured in the preferences. Please check your MorphMan settings and note models.') )
return c # card has no focusMorph field -> undefined behavior -> just proceed like normal
# evaluate all conditions, on which this card might be skipped/buried
isVocabCard = n.hasTag(jcfg('Tag_Vocab'))
isNotReady = n.hasTag(jcfg('Tag_NotReady'))
isComprehensionCard = n.hasTag(jcfg('Tag_Comprehension'))
isFreshVocab = n.hasTag(jcfg('Tag_Fresh'))
isAlreadyKnown = n.hasTag( jcfg('Tag_AlreadyKnown') )
skipComprehension = jcfg('Option_SkipComprehensionCards')
skipFresh = jcfg('Option_SkipFreshVocabCards')
skipFocusMorphSeenToday = jcfg('Option_SkipFocusMorphSeenToday')
skipCondition1 = (isComprehensionCard and skipComprehension)
skipCondition2 = (isFreshVocab and skipFresh)
skipCondition3 = isAlreadyKnown # the user requested that the vocabulary does not have to be shown
skipCondition4 = (focusMorph in seenMorphs and skipFocusMorphSeenToday) # we already learned that/saw that today
#skipCondition5 = not (isVocabCard or isNotReady) # even if it is not a good vocabulary card, we have no choice when there are no other cards available
# skip/bury card if any skip condition is true
if skipCondition1 or skipCondition2 or skipCondition3 or skipCondition4:
self.buryCards( [ c.id ] )
self.newCount += 1 # the card was quaried from the "new queue" so we have to increase the "new counter" back to its original value
continue
break
return c
def focusName( n ): return jcfg('Field_FocusMorph') # TODO remove argument n
def focus( n ): return n[ focusName(n) ]
def mkAllDb( allDb=None ):
import config; reload(config)
t_0, db, TAG = time.time(), mw.col.db, mw.col.tags
N_notes = db.scalar( 'select count() from notes' )
N_enabled_notes = 0 # for providing an error message if there is no note that is used for processing
mw.progress.start( label='Prep work for all.db creation', max=N_notes, immediate=True )
if not allDb: allDb = MorphDb()
fidDb = allDb.fidDb()
locDb = allDb.locDb( recalc=False ) # fidDb() already forces locDb recalc
mw.progress.update( label='Generating all.db data' )
for i,( nid, mid, flds, guid, tags ) in enumerate( db.execute( 'select id, mid, flds, guid, tags from notes' ) ):
if i % 500 == 0: mw.progress.update( value=i )
C = partial( cfg, mid, None )
note = mw.col.getNote(nid)
notecfg = getFilter(note)
if notecfg is None: continue
morphemizer = getMorphemizerByName(notecfg['Morphemizer'])
N_enabled_notes += 1
mats = [ ( 0.5 if ivl == 0 and ctype == 1 else ivl ) for ivl, ctype in db.execute( 'select ivl, type from cards where nid = :nid', nid=nid ) ]
if C('ignore maturity'):
mats = [ 0 for mat in mats ]
ts, alreadyKnownTag = TAG.split( tags ), jcfg('Tag_AlreadyKnown')
if alreadyKnownTag in ts:
mats += [ C('threshold_mature')+1 ]
for fieldName in notecfg['Fields']:
try: # if doesn't have field, continue
#fieldValue = normalizeFieldValue( getField( fieldName, flds, mid ) )
fieldValue = extractFieldData( fieldName, flds, mid )
except KeyError: continue
except TypeError:
mname = mw.col.models.get( mid )[ 'name' ]
errorMsg( u'Failed to get field "{field}" from a note of model "{model}". Please fix your config.py file to match your collection appropriately and ignore the following error.'.format( model=mname, field=fieldName ) )
raise
loc = fidDb.get( ( nid, guid, fieldName ), None )
if not loc:
loc = AnkiDeck( nid, fieldName, fieldValue, guid, mats )
ms = getMorphemes(morphemizer, fieldValue, ts)
if ms: #TODO: this needed? should we change below too then?
#printf( ' .loc for %d[%s]' % ( nid, fieldName ) )
locDb[ loc ] = ms
else:
# mats changed -> new loc (new mats), move morphs
if loc.fieldValue == fieldValue and loc.maturities != mats:
#printf( ' .mats for %d[%s]' % ( nid, fieldName ) )
newLoc = AnkiDeck( nid, fieldName, fieldValue, guid, mats )
locDb[ newLoc ] = locDb.pop( loc )
# field changed -> new loc, new morphs
elif loc.fieldValue != fieldValue:
#printf( ' .morphs for %d[%s]' % ( nid, fieldName ) )
newLoc = AnkiDeck( nid, fieldName, fieldValue, guid, mats )
ms = getMorphemes(morphemizer, fieldValue, ts)
locDb.pop( loc )
locDb[ newLoc ] = ms
if N_enabled_notes == 0:
mw.progress.finish()
errorMsg(u'There is no card that can be analyzed or be moved. Add cards or (re-)check your configuration under "Tools -> MorhpMan Preferences" or in "Anki/addons/morph/config.py" for mistakes.')
return None
printf( 'Processed all %d notes in %f sec' % ( N_notes, time.time() - t_0 ) )
mw.progress.update( value=i, label='Creating all.db object' )
allDb.clear()
allDb.addFromLocDb( locDb )
if cfg1('saveDbs'):
mw.progress.update( value=i, label='Saving all.db to disk' )
allDb.save( cfg1('path_all') )
printf( 'Processed all %d notes + saved all.db in %f sec' % ( N_notes, time.time() - t_0 ) )
mw.progress.finish()
return allDb
def updateNotes( allDb ):
t_0, now, db, TAG = time.time(), intTime(), mw.col.db, mw.col.tags
ds, nid2mmi = [], {}
N_notes = db.scalar( 'select count() from notes' )
mw.progress.start( label='Updating data', max=N_notes, immediate=True )
fidDb = allDb.fidDb()
locDb = allDb.locDb( recalc=False ) # fidDb() already forces locDb recalc
# read tag names
compTag, vocabTag, freshTag, notReadyTag, alreadyKnownTag, priorityTag, tooShortTag, tooLongTag = tagNames = jcfg('Tag_Comprehension'), jcfg('Tag_Vocab'), jcfg('Tag_Fresh'), jcfg('Tag_NotReady'), jcfg('Tag_AlreadyKnown'), jcfg('Tag_Priority'), jcfg('Tag_TooShort'), jcfg('Tag_TooLong')
TAG.register( tagNames )
badLengthTag = jcfg2().get('Tag_BadLength')
# handle secondary databases
mw.progress.update( label='Creating seen/known/mature from all.db' )
seenDb = filterDbByMat( allDb, cfg1('threshold_seen') )
knownDb = filterDbByMat( allDb, cfg1('threshold_known') )
matureDb = filterDbByMat( allDb, cfg1('threshold_mature') )
mw.progress.update( label='Loading priority.db' )
priorityDb = MorphDb( cfg1('path_priority'), ignoreErrors=True ).db
if cfg1('saveDbs'):
mw.progress.update( label='Saving seen/known/mature dbs' )
seenDb.save( cfg1('path_seen') )
knownDb.save( cfg1('path_known') )
matureDb.save( cfg1('path_mature') )
mw.progress.update( label='Updating notes' )
for i,( nid, mid, flds, guid, tags ) in enumerate( db.execute( 'select id, mid, flds, guid, tags from notes' ) ):
if i % 500 == 0: mw.progress.update( value=i )
C = partial( cfg, mid, None )
note = mw.col.getNote(nid)
notecfg = getFilter(note)
if notecfg is None or not notecfg['Modify']: continue
# Get all morphemes for note
morphemes = set()
for fieldName in notecfg['Fields']:
try:
loc = fidDb[ ( nid, guid, fieldName ) ]
morphemes.update( locDb[ loc ] )
except KeyError: continue
# Determine un-seen/known/mature and i+N
unseens, unknowns, unmatures, newKnowns = set(), set(), set(), set()
for morpheme in morphemes:
if morpheme not in seenDb.db: unseens.add( morpheme )
if morpheme not in knownDb.db: unknowns.add( morpheme )
if morpheme not in matureDb.db: unmatures.add( morpheme )
if morpheme not in matureDb.db and morpheme in knownDb.db:
newKnowns.add( morpheme )
# Determine MMI - Morph Man Index
N, N_s, N_k, N_m = len( morphemes ), len( unseens ), len( unknowns ), len( unmatures )
# Bail early for lite update
if N_k > 2 and C('only update k+2 and below'): continue
# average frequency of unknowns (ie. how common the word is within your collection)
F_k = 0
for focusMorph in unknowns: # focusMorph used outside loop
F_k += allDb.frequency(focusMorph)
F_k_avg = F_k // N_k if N_k > 0 else F_k
usefulness = F_k_avg
# add bonus for morphs in priority.db
isPriority = False
for focusMorph in unknowns:
if focusMorph in priorityDb:
isPriority = True
usefulness += C('priority.db weight')
# add bonus for studying recent learned knowns (reinforce)
for morpheme in newKnowns:
locs = allDb.db[ morpheme ]
if locs:
ivl = min( 1, max( loc.maturity for loc in locs ) )
usefulness += C('reinforce new vocab weight') // ivl #TODO: maybe average this so it doesnt favor long sentences
if any( morpheme.pos == u'動詞' for morpheme in unknowns ): #FIXME: this isn't working???
usefulness += C('verb bonus')
usefulness = 999 - min( 999, usefulness )
# difference from optimal length range (too little context vs long sentence)
lenDiffRaw = min(N - C('min good sentence length'),
max(0, N - C('max good sentence length')))
lenDiff = min(9, abs(lenDiffRaw))
# calculate mmi
mmi = 10000*N_k + 1000*lenDiff + usefulness
if C('set due based on mmi'):
nid2mmi[ nid ] = mmi
# Fill in various fields/tags on the note based on cfg
ts, fs = TAG.split( tags ), splitFields( flds )
# clear any 'special' tags, the appropriate will be set in the next few lines
ts = [ t for t in ts if t not in [ notReadyTag, compTag, vocabTag, freshTag ] ]
# determine card type
if N_m == 0: # sentence comprehension card, m+0
ts = ts + [ compTag ]
setField( mid, fs, jcfg('Field_FocusMorph'), u'' )
elif N_k == 1: # new vocab card, k+1
ts = ts + [ vocabTag ]
setField( mid, fs, jcfg('Field_FocusMorph'), u'%s' % focusMorph.base )
elif N_k > 1: # M+1+ and K+2+
ts = ts + [ notReadyTag ]
setField( mid, fs, jcfg('Field_FocusMorph'), u'')
elif N_m == 1: # we have k+0, and m+1, so this card does not introduce a new vocabulary -> card for newly learned morpheme
ts = ts + [ freshTag ]
setField( mid, fs, jcfg('Field_FocusMorph'), u'%s' % list(unmatures)[0].base)
else: # only case left: we have k+0, but m+2 or higher, so this card does not introduce a new vocabulary -> card for newly learned morpheme
ts = ts + [ freshTag ]
setField( mid, fs, jcfg('Field_FocusMorph'), u'')
# set type agnostic fields
setField( mid, fs, jcfg('Field_UnknownMorphCount'), u'%d' % N_k )
setField( mid, fs, jcfg('Field_UnmatureMorphCount'), u'%d' % N_m )
setField( mid, fs, jcfg('Field_MorphManIndex'), u'%d' % mmi )
setField( mid, fs, jcfg('Field_Unknowns'), u', '.join( u.base for u in unknowns ) )
setField( mid, fs, jcfg('Field_Unmatures'), u', '.join( u.base for u in unmatures ) )
setField( mid, fs, jcfg('Field_UnknownFreq'), u'%d' % F_k_avg )
# remove deprecated tag
if badLengthTag is not None and badLengthTag in ts:
ts.remove( badLengthTag )
# other tags
if priorityTag in ts: ts.remove( priorityTag )
if isPriority: ts.append( priorityTag )
if tooShortTag in ts: ts.remove( tooShortTag )
if lenDiffRaw < 0: ts.append( tooShortTag )
if tooLongTag in ts: ts.remove( tooLongTag )
if lenDiffRaw > 0: ts.append( tooLongTag )
# remove unnecessary tags
if not jcfg('Option_SetNotRequiredTags'):
unnecessary = [priorityTag, tooShortTag, tooLongTag]
ts = [tag for tag in ts if tag not in unnecessary]
# update sql db
tags_ = TAG.join( TAG.canonify( ts ) )
flds_ = joinFields( fs )
if flds != flds_ or tags != tags_: # only update notes that have changed
csum = fieldChecksum( fs[0] )
sfld = stripHTML( fs[ getSortFieldIndex( mid ) ] )
ds.append( { 'now':now, 'tags':tags_, 'flds':flds_, 'sfld':sfld, 'csum':csum, 'usn':mw.col.usn(), 'nid':nid } )
mw.progress.update( value=i, label='Updating anki database...' )
mw.col.db.executemany( 'update notes set tags=:tags, flds=:flds, sfld=:sfld, csum=:csum, mod=:now, usn=:usn where id=:nid', ds )
# Now reorder new cards based on MMI
mw.progress.update( value=i, label='Updating new card ordering...' )
ds = []
# "type = 0": new cards
# "type = 1": learning cards [is supposed to be learning: in my case no learning card had this type]
# "type = 2": review cards
for ( cid, nid, due ) in db.execute( 'select id, nid, due from cards where type = 0' ):
if nid in nid2mmi: # owise it was disabled
due_ = nid2mmi[ nid ]
if due != due_: # only update cards that have changed
ds.append( { 'now':now, 'due':due_, 'usn':mw.col.usn(), 'cid':cid } )
mw.col.db.executemany( 'update cards set due=:due, mod=:now, usn=:usn where id=:cid', ds )
mw.reset()
printf( 'Updated notes in %f sec' % ( time.time() - t_0 ) )
mw.progress.finish()
return knownDb