def _reqForTemplate(self, m, flds, t):
a = []
b = []
cloze = "cloze" in t['qfmt']
reqstrs = []
if cloze:
# need a cloze-specific filler
cloze = ""
nums = re.findall("\{\{cloze:(\d+):", t['qfmt'])
for n in nums:
n = int(n)
cloze += "{{c%d::foo}}" % n
# record that we require a specific string for generation
reqstrs.append("{{c%d::" % n)
return 'all', [], reqstrs
for f in flds:
a.append(cloze if cloze else "1")
b.append("")
data = [1, 1, m['id'], 1, t['ord'], "", joinFields(a)]
full = self.col._renderQA(data)['q']
data = [1, 1, m['id'], 1, t['ord'], "", joinFields(b)]
empty = self.col._renderQA(data)['q']
# if full and empty are the same, the template is invalid and there is
# no way to satisfy it
if full == empty:
return "none", [], []
type = 'all'
req = []
for i in range(len(flds)):
tmp = a[:]
tmp[i] = ""
data[6] = joinFields(tmp)
# if the result is same as empty, field is required
if self.col._renderQA(data)['q'] == empty:
req.append(i)
if req:
return type, req, reqstrs
# if there are no required fields, switch to any mode
type = 'any'
req = []
for i in range(len(flds)):
tmp = b[:]
tmp[i] = "1"
data[6] = joinFields(tmp)
# if not the same as empty, this field can make the card non-blank
if self.col._renderQA(data)['q'] != empty:
req.append(i)
return type, req, reqstrs
def _reqForTemplate(self, m, flds, t):
a = []
b = []
cloze = "cloze" in t['qfmt']
reqstrs = []
if cloze:
# need a cloze-specific filler
cloze = ""
nums = re.findall("\{\{cloze:(\d+):", t['qfmt'])
for n in nums:
n = int(n)
cloze += "{{c%d::foo}}" % n
# record that we require a specific string for generation
reqstrs.append("{{c%d::" % n)
return 'all', [], reqstrs
for f in flds:
a.append(cloze if cloze else "1")
b.append("")
data = [1, 1, m['id'], 1, t['ord'], "", joinFields(a)]
full = self.col._renderQA(data)['q']
data = [1, 1, m['id'], 1, t['ord'], "", joinFields(b)]
empty = self.col._renderQA(data)['q']
# if full and empty are the same, the template is invalid and there is
# no way to satisfy it
if full == empty:
return "none", [], []
type = 'all'
req = []
for i in range(len(flds)):
tmp = a[:]
tmp[i] = ""
data[6] = joinFields(tmp)
# if the result is same as empty, field is required
if self.col._renderQA(data)['q'] == empty:
req.append(i)
if req:
return type, req, reqstrs
# if there are no required fields, switch to any mode
type = 'any'
req = []
for i in range(len(flds)):
tmp = b[:]
tmp[i] = "1"
data[6] = joinFields(tmp)
# if not the same as empty, this field can make the card non-blank
if self.col._renderQA(data)['q'] != empty:
req.append(i)
return type, req, reqstrs
def _transformFields(self, fn):
self.deck.modSchema()
r = []
for (id, flds) in self.deck.db.execute(
"select id, flds from facts where mid = ?", self.id):
r.append((joinFields(fn(splitFields(flds))), id))
self.deck.db.executemany("update facts set flds = ? where id = ?", r)
def _changeNotes(self, nids, newModel, map):
"""Change the note whose ids are nid to the model newModel, reorder
fields according to map. Write the change in the database
Note that if a field is mapped to nothing, it is lost
keyword arguments:
nids -- the list of id of notes to change
newmodel -- the model of destination of the note
map -- the dictionnary sending to each fields'ord of the old model a field'ord of the new model
"""
d = [
] #The list of dictionnaries, containing the information relating to the new cards
nfields = len(newModel['flds'])
for (nid, flds) in self.col.db.execute(
"select id, flds from notes where id in " + ids2str(nids)):
newflds = {}
flds = splitFields(flds)
for old, new in list(map.items()):
newflds[new] = flds[old]
flds = []
for c in range(nfields):
flds.append(newflds.get(c, ""))
flds = joinFields(flds)
d.append(
dict(nid=nid,
flds=flds,
mid=newModel['id'],
m=intTime(),
u=self.col.usn()))
self.col.db.executemany(
"update notes set flds=:flds,mid=:mid,mod=:m,usn=:u where id = :nid",
d)
self.col.updateFieldCache(nids)
def _mungeMedia(self, mid: NotetypeId, fieldsStr: str) -> str:
fields = splitFields(fieldsStr)
def repl(match):
fname = match.group("fname")
srcData = self._srcMediaData(fname)
dstData = self._dstMediaData(fname)
if not srcData:
# file was not in source, ignore
return match.group(0)
# if model-local file exists from a previous import, use that
name, ext = os.path.splitext(fname)
lname = f"{name}_{mid}{ext}"
if self.dst.media.have(lname):
return match.group(0).replace(fname, lname)
# if missing or the same, pass unmodified
elif not dstData or srcData == dstData:
# need to copy?
if not dstData:
self._writeDstMedia(fname, srcData)
return match.group(0)
# exists but does not match, so we need to dedupe
self._writeDstMedia(lname, srcData)
return match.group(0).replace(fname, lname)
for idx, field in enumerate(fields):
fields[idx] = self.dst.media.transformNames(field, repl)
return joinFields(fields)
def _mungeMedia(self, mid, fields):
fields = splitFields(fields)
def repl(match):
fname = match.group("fname")
srcData = self._srcMediaData(fname)
dstData = self._dstMediaData(fname)
if not srcData:
# file was not in source, ignore
return match.group(0)
# if model-local file exists from a previous import, use that
name, ext = os.path.splitext(fname)
lname = "%s_%s%s" % (name, mid, ext)
if self.dst.media.have(lname):
return match.group(0).replace(fname, lname)
# if missing or the same, pass unmodified
elif not dstData or srcData == dstData:
# need to copy?
if not dstData:
self._writeDstMedia(fname, srcData)
return match.group(0)
# exists but does not match, so we need to dedupe
self._writeDstMedia(lname, srcData)
return match.group(0).replace(fname, lname)
for i in range(len(fields)):
fields[i] = self.dst.media.transformNames(fields[i], repl)
return joinFields(fields)
def _mungeMedia(self, mid, fields):
fields = splitFields(fields)
def repl(match):
fname = match.group("fname")
srcData = self._srcMediaData(fname)
dstData = self._dstMediaData(fname)
if not srcData:
# file was not in source, ignore
return match.group(0)
# if model-local file exists from a previous import, use that
name, ext = os.path.splitext(fname)
lname = "%s_%s%s" % (name, mid, ext)
if self.dst.media.have(lname):
return match.group(0).replace(fname, lname)
# if missing or the same, pass unmodified
elif not dstData or srcData == dstData:
# need to copy?
if not dstData:
self._writeDstMedia(fname, srcData)
return match.group(0)
# exists but does not match, so we need to dedupe
self._writeDstMedia(lname, srcData)
return match.group(0).replace(fname, lname)
for i in range(len(fields)):
fields[i] = self.dst.media.transformNames(fields[i], repl)
return joinFields(fields)
def _changeNotes(
self, nids: List[int], newModel: NoteType, map: Dict[int, Union[None, int]]
) -> None:
d = []
nfields = len(newModel["flds"])
for (nid, flds) in self.col.db.execute(
"select id, flds from notes where id in " + ids2str(nids)
):
newflds = {}
flds = splitFields(flds)
for old, new in list(map.items()):
newflds[new] = flds[old]
flds = []
for c in range(nfields):
flds.append(newflds.get(c, ""))
flds = joinFields(flds)
d.append(
dict(
nid=nid,
flds=flds,
mid=newModel["id"],
m=intTime(),
u=self.col.usn(),
)
)
self.col.db.executemany(
"update notes set flds=:flds,mid=:mid,mod=:m,usn=:u where id = :nid", d
)
self.col.updateFieldCache(nids)
def _transformFields(self, fn):
self.deck.modSchema()
r = []
for (id, flds) in self.deck.db.execute(
"select id, flds from facts where mid = ?", self.id):
r.append((joinFields(fn(splitFields(flds))), id))
self.deck.db.executemany("update facts set flds = ? where id = ?", r)
def _changeNotes(
self,
nids: List[anki.notes.NoteId],
newModel: NotetypeDict,
map: Dict[int, Union[None, int]],
) -> None:
d = []
nfields = len(newModel["flds"])
for (nid, flds) in self.col.db.execute(
f"select id, flds from notes where id in {ids2str(nids)}"):
newflds = {}
flds = splitFields(flds)
for old, new in list(map.items()):
newflds[new] = flds[old]
flds = []
for c in range(nfields):
flds.append(newflds.get(c, ""))
flds = joinFields(flds)
d.append((
flds,
newModel["id"],
intTime(),
self.col.usn(),
nid,
))
self.col.db.executemany(
"update notes set flds=?,mid=?,mod=?,usn=? where id = ?", d)
def findReplace(
col: _Collection,
nids: List[int],
src: str,
dst: str,
regex: bool = False,
field: Optional[str] = None,
fold: bool = True,
) -> int:
"Find and replace fields in a note."
mmap: Dict[str, Any] = {}
if field:
for m in col.models.all():
for f in m["flds"]:
if f["name"].lower() == field.lower():
mmap[str(m["id"])] = f["ord"]
if not mmap:
return 0
# find and gather replacements
if not regex:
src = re.escape(src)
dst = dst.replace("\\", "\\\\")
if fold:
src = "(?i)" + src
compiled_re = re.compile(src)
def repl(s: str):
return compiled_re.sub(dst, s)
d = []
snids = ids2str(nids)
nids = []
for nid, mid, flds in col.db.execute(
"select id, mid, flds from notes where id in " + snids):
origFlds = flds
# does it match?
sflds = splitFields(flds)
if field:
try:
ord = mmap[str(mid)]
sflds[ord] = repl(sflds[ord])
except KeyError:
# note doesn't have that field
continue
else:
for c in range(len(sflds)):
sflds[c] = repl(sflds[c])
flds = joinFields(sflds)
if flds != origFlds:
nids.append(nid)
d.append(dict(nid=nid, flds=flds, u=col.usn(), m=intTime()))
if not d:
return 0
# replace
col.db.executemany(
"update notes set flds=:flds,mod=:m,usn=:u where id=:nid", d)
col.updateFieldCache(nids)
col.genCards(nids)
return len(d)
def _transformFields(self, m, fn):
# model hasn't been added yet?
if not m["id"]:
return
r = []
for (id, flds) in self.col.db.execute("select id, flds from notes where mid = ?", m["id"]):
r.append((joinFields(fn(splitFields(flds))), intTime(), self.col.usn(), id))
self.col.db.executemany("update notes set flds=?,mod=?,usn=? where id = ?", r)
def _reqForTemplate(self, m, flds, t):
"""A rule which is supposed to determine whether a card should be
generated or not according to its fields.
See ../documentation/templates_generation_rules.md
"""
a = []
b = []
for f in flds:
a.append("ankiflag")
b.append("")
data = [1, 1, m['id'], 1, t['ord'], "", joinFields(a), 0]
# The html of the card at position ord where each field's content is "ankiflag"
full = self.col._renderQA(data)['q']
data = [1, 1, m['id'], 1, t['ord'], "", joinFields(b), 0]
# The html of the card at position ord where each field's content is the empty string ""
empty = self.col._renderQA(data)['q']
# if full and empty are the same, the template is invalid and there is
# no way to satisfy it
if full == empty:
return "none", [], []
type = 'all'
req = []
for i in range(len(flds)):
tmp = a[:]
tmp[i] = ""
data[6] = joinFields(tmp)
# if no field content appeared, field is required
if "ankiflag" not in self.col._renderQA(data)['q']:
req.append(i)
if req:
return type, req
# if there are no required fields, switch to any mode
type = 'any'
req = []
for i in range(len(flds)):
tmp = b[:]
tmp[i] = "1"
data[6] = joinFields(tmp)
# if not the same as empty, this field can make the card non-blank
if self.col._renderQA(data)['q'] != empty:
req.append(i)
return type, req
def findTemplates(self, note):
"Return (active), non-empty templates."
model = note.model()
avail = self.models.availOrds(model, joinFields(note.fields))
ok = []
for t in model['tmpls']:
if t['ord'] in avail:
ok.append(t)
return ok
def findReplace(col,
nids,
src,
dst,
regex=False,
field=None,
fold=True) -> int:
"Find and replace fields in a note."
mmap = {}
if field:
for m in col.models.all():
for f in m['flds']:
if f['name'].lower() == field.lower():
mmap[str(m['id'])] = f['ord']
if not mmap:
return 0
# find and gather replacements
if not regex:
src = re.escape(src)
dst = dst.replace("\\", "\\\\")
if fold:
src = "(?i)" + src
regex = re.compile(src)
def repl(str):
return re.sub(regex, dst, str)
d = []
snids = ids2str(nids)
nids = []
for nid, mid, flds in col.db.execute(
"select id, mid, flds from notes where id in " + snids):
origFlds = flds
# does it match?
sflds = splitFields(flds)
if field:
try:
ord = mmap[str(mid)]
sflds[ord] = repl(sflds[ord])
except KeyError:
# note doesn't have that field
continue
else:
for c in range(len(sflds)):
sflds[c] = repl(sflds[c])
flds = joinFields(sflds)
if flds != origFlds:
nids.append(nid)
d.append(dict(nid=nid, flds=flds, u=col.usn(), m=intTime()))
if not d:
return 0
# replace
col.db.executemany(
"update notes set flds=:flds,mod=:m,usn=:u where id=:nid", d)
col.updateFieldCache(nids)
col.genCards(nids)
return len(d)
def findTemplates(self, note):
"Return (active), non-empty templates."
ok = []
model = note.model()
avail = self.models.availOrds(model, joinFields(note.fields))
ok = []
for t in model['tmpls']:
if t['ord'] in avail:
ok.append(t)
return ok
def _transformFields(self, m: NoteType, fn: Callable) -> None:
# model hasn't been added yet?
if not m["id"]:
return
r = []
for (id, flds) in self.col.db.execute(
"select id, flds from notes where mid = ?", m["id"]
):
r.append((joinFields(fn(splitFields(flds))), intTime(), self.col.usn(), id))
self.col.db.executemany("update notes set flds=?,mod=?,usn=? where id = ?", r)
def addTab(self, t):
c = self.connect
w = QWidget()
l = QHBoxLayout()
l.setMargin(0)
l.setSpacing(3)
left = QWidget()
# template area
tform = aqt.forms.template.Ui_Form()
tform.setupUi(left)
tform.label1.setText(u" →")
tform.label2.setText(u" →")
tform.labelc1.setText(u" ↗")
tform.labelc2.setText(u" ↘")
if self.style().objectName() == "gtk+":
# gtk+ requires margins in inner layout
tform.tlayout1.setContentsMargins(0, 11, 0, 0)
tform.tlayout2.setContentsMargins(0, 11, 0, 0)
tform.tlayout3.setContentsMargins(0, 11, 0, 0)
if len(self.cards) > 1:
tform.groupBox_3.setTitle(_(
"Styling (shared between cards)"))
c(tform.front, SIGNAL("textChanged()"), self.saveCard)
c(tform.css, SIGNAL("textChanged()"), self.saveCard)
c(tform.back, SIGNAL("textChanged()"), self.saveCard)
l.addWidget(left, 5)
# preview area
right = QWidget()
pform = aqt.forms.preview.Ui_Form()
pform.setupUi(right)
if self.style().objectName() == "gtk+":
# gtk+ requires margins in inner layout
pform.frontPrevBox.setContentsMargins(0, 11, 0, 0)
pform.backPrevBox.setContentsMargins(0, 11, 0, 0)
# for cloze notes, show that it's one of n cards
if self.model['type'] == MODEL_CLOZE:
cnt = len(self.mm.availOrds(
self.model, joinFields(self.note.fields)))
for g in pform.groupBox, pform.groupBox_2:
g.setTitle(g.title() + _(" (1 of %d)") % max(cnt, 1))
pform.frontWeb = AnkiWebView()
pform.frontPrevBox.addWidget(pform.frontWeb)
pform.backWeb = AnkiWebView()
pform.backPrevBox.addWidget(pform.backWeb)
def linkClicked(url):
openLink(url)
for wig in pform.frontWeb, pform.backWeb:
wig.page().setLinkDelegationPolicy(
QWebPage.DelegateExternalLinks)
c(wig, SIGNAL("linkClicked(QUrl)"), linkClicked)
l.addWidget(right, 5)
w.setLayout(l)
self.forms.append({'tform': tform, 'pform': pform})
self.tabs.addTab(w, t['name'])
def addTab(self, t):
c = self.connect
w = QWidget()
l = QHBoxLayout()
l.setMargin(0)
l.setSpacing(3)
left = QWidget()
# template area
tform = aqt.forms.template.Ui_Form()
tform.setupUi(left)
tform.label1.setText(u" →")
tform.label2.setText(u" →")
tform.labelc1.setText(u" ↗")
tform.labelc2.setText(u" ↘")
if self.style().objectName() == "gtk+":
# gtk+ requires margins in inner layout
tform.tlayout1.setContentsMargins(0, 11, 0, 0)
tform.tlayout2.setContentsMargins(0, 11, 0, 0)
tform.tlayout3.setContentsMargins(0, 11, 0, 0)
if len(self.cards) > 1:
tform.groupBox_3.setTitle(_("Styling (shared between cards)"))
c(tform.front, SIGNAL("textChanged()"), self.saveCard)
c(tform.css, SIGNAL("textChanged()"), self.saveCard)
c(tform.back, SIGNAL("textChanged()"), self.saveCard)
l.addWidget(left, 5)
# preview area
right = QWidget()
pform = aqt.forms.preview.Ui_Form()
pform.setupUi(right)
if self.style().objectName() == "gtk+":
# gtk+ requires margins in inner layout
pform.frontPrevBox.setContentsMargins(0, 11, 0, 0)
pform.backPrevBox.setContentsMargins(0, 11, 0, 0)
# for cloze notes, show that it's one of n cards
if self.model['type'] == MODEL_CLOZE:
cnt = len(
self.mm.availOrds(self.model, joinFields(self.note.fields)))
for g in pform.groupBox, pform.groupBox_2:
g.setTitle(g.title() + _(" (1 of %d)") % max(cnt, 1))
pform.frontWeb = AnkiWebView()
pform.frontPrevBox.addWidget(pform.frontWeb)
pform.backWeb = AnkiWebView()
pform.backPrevBox.addWidget(pform.backWeb)
def linkClicked(url):
openLink(url)
for wig in pform.frontWeb, pform.backWeb:
wig.page().setLinkDelegationPolicy(QWebPage.DelegateExternalLinks)
c(wig, SIGNAL("linkClicked(QUrl)"), linkClicked)
l.addWidget(right, 5)
w.setLayout(l)
self.forms.append({'tform': tform, 'pform': pform})
self.tabs.addTab(w, t['name'])
def _transformFields(self, m, fn):
"""TODO"""
# model hasn't been added yet?
if not m['id']:
return
r = []
for (id, flds) in self.col.db.execute(
"select id, flds from notes where mid = ?", m['id']):
r.append((joinFields(fn(splitFields(flds))),
intTime(), self.col.usn(), id))
self.col.db.executemany(
"update notes set flds=?,mod=?,usn=? where id = ?", r)
def test_availOrds():
d = getEmptyCol()
m = d.models.current()
mm = d.models
t = m['tmpls'][0]
f = d.newNote()
f['Front'] = "1"
# simple templates
assert mm.availOrds(m, joinFields(f.fields)) == [0]
t['qfmt'] = "{{Back}}"
mm.save(m, templates=True)
assert not mm.availOrds(m, joinFields(f.fields))
# AND
t['qfmt'] = "{{#Front}}{{#Back}}{{Front}}{{/Back}}{{/Front}}"
mm.save(m, templates=True)
assert not mm.availOrds(m, joinFields(f.fields))
t['qfmt'] = "{{#Front}}\n{{#Back}}\n{{Front}}\n{{/Back}}\n{{/Front}}"
mm.save(m, templates=True)
assert not mm.availOrds(m, joinFields(f.fields))
# OR
t['qfmt'] = "{{Front}}\n{{Back}}"
mm.save(m, templates=True)
assert mm.availOrds(m, joinFields(f.fields)) == [0]
t['Front'] = ""
t['Back'] = "1"
assert mm.availOrds(m, joinFields(f.fields)) == [0]
def test_availOrds():
d = getEmptyDeck()
m = d.models.current(); mm = d.models
t = m['tmpls'][0]
f = d.newNote()
f['Front'] = "1"
# simple templates
assert mm.availOrds(m, joinFields(f.fields)) == [0]
t['qfmt'] = "{{Back}}"
mm.save(m, templates=True)
assert not mm.availOrds(m, joinFields(f.fields))
# AND
t['qfmt'] = "{{#Front}}{{#Back}}{{Front}}{{/Back}}{{/Front}}"
mm.save(m, templates=True)
assert not mm.availOrds(m, joinFields(f.fields))
t['qfmt'] = "{{#Front}}\n{{#Back}}\n{{Front}}\n{{/Back}}\n{{/Front}}"
mm.save(m, templates=True)
assert not mm.availOrds(m, joinFields(f.fields))
# OR
t['qfmt'] = "{{Front}}\n{{Back}}"
mm.save(m, templates=True)
assert mm.availOrds(m, joinFields(f.fields)) == [0]
t['Front'] = ""
t['Back'] = "1"
assert mm.availOrds(m, joinFields(f.fields)) == [0]
def processFields(self, note, fields=None):
if not fields:
fields = [""]*len(self.model['flds'])
for c, f in enumerate(self.mapping):
if not f:
continue
elif f == "_tags":
note.tags.extend(self.col.tags.split(note.fields[c]))
else:
sidx = self._fmap[f][0]
fields[sidx] = note.fields[c]
note.fieldsStr = joinFields(fields)
return self.col.models.availOrds(self.model, note.fieldsStr)
def processFields(self, note, fields=None):
if not fields:
fields = [""] * len(self.model['flds'])
for c, f in enumerate(self.mapping):
if not f:
continue
elif f == "_tags":
note.tags.extend(self.col.tags.split(note.fields[c]))
else:
sidx = self._fmap[f][0]
fields[sidx] = note.fields[c]
note.fieldsStr = joinFields(fields)
return self.col.models.availOrds(self.model, note.fieldsStr)
def _reqForTemplate(
self, m: Dict[str, Any], flds: List[str],
t: Dict[str, Any]) -> Tuple[Union[str, List[int]], ...]:
a = []
b = []
for f in flds:
a.append("ankiflag")
b.append("")
data = [1, 1, m['id'], 1, t['ord'], "", joinFields(a), 0]
full = self.col._renderQA(data)['q']
data = [1, 1, m['id'], 1, t['ord'], "", joinFields(b), 0]
empty = self.col._renderQA(data)['q']
# if full and empty are the same, the template is invalid and there is
# no way to satisfy it
if full == empty:
return "none", [], []
type = 'all'
req = []
for i in range(len(flds)):
tmp = a[:]
tmp[i] = ""
data[6] = joinFields(tmp)
# if no field content appeared, field is required
if "ankiflag" not in self.col._renderQA(data)['q']:
req.append(i)
if req:
return type, req
# if there are no required fields, switch to any mode
type = 'any'
req = []
for i in range(len(flds)):
tmp = b[:]
tmp[i] = "1"
data[6] = joinFields(tmp)
# if not the same as empty, this field can make the card non-blank
if self.col._renderQA(data)['q'] != empty:
req.append(i)
return type, req
def processFields(self,
note: ForeignNote,
fields: Optional[List[str]] = None) -> None:
if not fields:
fields = [""] * len(self.model["flds"])
for c, f in enumerate(self.mapping):
if not f:
continue
elif f == "_tags":
note.tags.extend(self.col.tags.split(note.fields[c]))
else:
sidx = self._fmap[f][0]
fields[sidx] = note.fields[c]
note.fieldsStr = joinFields(fields)
def _transformFields(self, m, fn):
"""For each note of the model m, apply m to the set of field's value,
and save the note modified.
fn -- a function taking and returning a list of field."""
# model hasn't been added yet?
if not m['id']:
return
r = []
for (id, flds) in self.col.db.execute(
"select id, flds from notes where mid = ?", m['id']):
r.append((joinFields(fn(splitFields(flds))), intTime(),
self.col.usn(), id))
self.col.db.executemany(
"update notes set flds=?,mod=?,usn=? where id = ?", r)
def _reqForTemplate(self, m, flds, t):
a = []
b = []
for f in flds:
a.append("ankiflag")
b.append("")
data = [1, 1, m['id'], 1, t['ord'], "", joinFields(a)]
full = self.col._renderQA(data)['q']
data = [1, 1, m['id'], 1, t['ord'], "", joinFields(b)]
empty = self.col._renderQA(data)['q']
# if full and empty are the same, the template is invalid and there is
# no way to satisfy it
if full == empty:
return "none", [], []
type = 'all'
req = []
for i in range(len(flds)):
tmp = a[:]
tmp[i] = ""
data[6] = joinFields(tmp)
# if no field content appeared, field is required
if "ankiflag" not in self.col._renderQA(data)['q']:
req.append(i)
if req:
return type, req
# if there are no required fields, switch to any mode
type = 'any'
req = []
for i in range(len(flds)):
tmp = b[:]
tmp[i] = "1"
data[6] = joinFields(tmp)
# if not the same as empty, this field can make the card non-blank
if self.col._renderQA(data)['q'] != empty:
req.append(i)
return type, req
def _changeNotes(self, nids, newModel, map):
d = []
nfields = len(newModel["flds"])
for (nid, flds) in self.col.db.execute("select id, flds from notes where id in " + ids2str(nids)):
newflds = {}
flds = splitFields(flds)
for old, new in map.items():
newflds[new] = flds[old]
flds = []
for c in range(nfields):
flds.append(newflds.get(c, ""))
flds = joinFields(flds)
d.append(dict(nid=nid, flds=flds, mid=newModel["id"], m=intTime(), u=self.col.usn()))
self.col.db.executemany("update notes set flds=:flds,mid=:mid,mod=:m,usn=:u where id = :nid", d)
self.col.updateFieldCache(nids)
def findReplace(col, nids, src, dst, regex=False, field=None, fold=True):
"Find and replace fields in a note."
mmap = {}
if field:
for m in col.models.all():
for f in m['flds']:
if f['name'] == field:
mmap[str(m['id'])] = f['ord']
if not mmap:
return 0
# find and gather replacements
if not regex:
src = re.escape(src)
if fold:
src = "(?i)"+src
regex = re.compile(src)
def repl(str):
return re.sub(regex, dst, str)
d = []
snids = ids2str(nids)
nids = []
for nid, mid, flds in col.db.execute(
"select id, mid, flds from notes where id in "+snids):
origFlds = flds
# does it match?
sflds = splitFields(flds)
if field:
try:
ord = mmap[str(mid)]
sflds[ord] = repl(sflds[ord])
except KeyError:
# note doesn't have that field
continue
else:
for c in range(len(sflds)):
sflds[c] = repl(sflds[c])
flds = joinFields(sflds)
if flds != origFlds:
nids.append(nid)
d.append(dict(nid=nid,flds=flds,u=col.usn(),m=intTime()))
if not d:
return 0
# replace
col.db.executemany(
"update notes set flds=:flds,mod=:m,usn=:u where id=:nid", d)
col.updateFieldCache(nids)
col.genCards(nids)
return len(d)
def addTab(self, t):
w = QWidget()
l = QHBoxLayout()
l.setContentsMargins(0,0,0,0)
l.setSpacing(3)
left = QWidget()
# template area
tform = aqt.forms.template.Ui_Form()
tform.setupUi(left)
tform.label1.setText(" →")
tform.label2.setText(" →")
tform.labelc1.setText(" ↗")
tform.labelc2.setText(" ↘")
if self.style().objectName() == "gtk+":
# gtk+ requires margins in inner layout
tform.tlayout1.setContentsMargins(0, 11, 0, 0)
tform.tlayout2.setContentsMargins(0, 11, 0, 0)
tform.tlayout3.setContentsMargins(0, 11, 0, 0)
if len(self.cards) > 1:
tform.groupBox_3.setTitle(_(
"Styling (shared between cards)"))
tform.front.textChanged.connect(self.saveCard)
tform.css.textChanged.connect(self.saveCard)
tform.back.textChanged.connect(self.saveCard)
l.addWidget(left, 5)
# preview area
right = QWidget()
pform = aqt.forms.preview.Ui_Form()
pform.setupUi(right)
if self.style().objectName() == "gtk+":
# gtk+ requires margins in inner layout
pform.frontPrevBox.setContentsMargins(0, 11, 0, 0)
pform.backPrevBox.setContentsMargins(0, 11, 0, 0)
# for cloze notes, show that it's one of n cards
if self.model['type'] == MODEL_CLOZE:
cnt = len(self.mm.availOrds(
self.model, joinFields(self.note.fields)))
for g in pform.groupBox, pform.groupBox_2:
g.setTitle(g.title() + _(" (1 of %d)") % max(cnt, 1))
pform.frontWeb = AnkiWebView()
pform.frontPrevBox.addWidget(pform.frontWeb)
pform.backWeb = AnkiWebView()
pform.backPrevBox.addWidget(pform.backWeb)
l.addWidget(right, 5)
w.setLayout(l)
self.forms.append({'tform': tform, 'pform': pform})
self.tabs.addTab(w, t['name'])
def addTab(self, t):
w = QWidget()
l = QHBoxLayout()
l.setContentsMargins(0,0,0,0)
l.setSpacing(3)
left = QWidget()
# template area
tform = aqt.forms.template.Ui_Form()
tform.setupUi(left)
tform.label1.setText(" →")
tform.label2.setText(" →")
tform.labelc1.setText(" ↗")
tform.labelc2.setText(" ↘")
if self.style().objectName() == "gtk+":
# gtk+ requires margins in inner layout
tform.tlayout1.setContentsMargins(0, 11, 0, 0)
tform.tlayout2.setContentsMargins(0, 11, 0, 0)
tform.tlayout3.setContentsMargins(0, 11, 0, 0)
if len(self.cards) > 1:
tform.groupBox_3.setTitle(_(
"Styling (shared between cards)"))
tform.front.textChanged.connect(self.saveCard)
tform.css.textChanged.connect(self.saveCard)
tform.back.textChanged.connect(self.saveCard)
l.addWidget(left, 5)
# preview area
right = QWidget()
pform = aqt.forms.preview.Ui_Form()
pform.setupUi(right)
if self.style().objectName() == "gtk+":
# gtk+ requires margins in inner layout
pform.frontPrevBox.setContentsMargins(0, 11, 0, 0)
pform.backPrevBox.setContentsMargins(0, 11, 0, 0)
# for cloze notes, show that it's one of n cards
if self.model['type'] == MODEL_CLOZE:
cnt = len(self.mm.availOrds(
self.model, joinFields(self.note.fields)))
for g in pform.groupBox, pform.groupBox_2:
g.setTitle(g.title() + _(" (1 of %d)") % max(cnt, 1))
pform.frontWeb = AnkiWebView()
pform.frontPrevBox.addWidget(pform.frontWeb)
pform.backWeb = AnkiWebView()
pform.backPrevBox.addWidget(pform.backWeb)
l.addWidget(right, 5)
w.setLayout(l)
self.forms.append({'tform': tform, 'pform': pform})
self.tabs.addTab(w, t['name'])
def _changeFacts(self, fids, newModel, map):
d = []
nfields = len(newModel.fields)
for (fid, flds) in self.deck.db.execute(
"select id, flds from facts where id in "+ids2str(fids)):
newflds = {}
flds = splitFields(flds)
for old, new in map.items():
newflds[new] = flds[old]
flds = []
for c in range(nfields):
flds.append(newflds.get(c, ""))
flds = joinFields(flds)
d.append(dict(fid=fid, flds=flds, mid=newModel.id))
self.deck.db.executemany(
"update facts set flds=:flds, mid=:mid where id = :fid", d)
self.deck.updateFieldCache(fids)
def getNoteData(self, sortId, question, answerDicts, ref, siblings,
connections):
"""returns a list of all content needed to create the a new note and
the media contained in that note in a list"""
noteList = []
media = []
# Set field Reference
noteList.append('<ul>%s</ul>' % ref)
# Set field Question
qtContent, qtMedia = getNodeContent(tagList=self.tagList, tag=question)
noteList.append(qtContent)
media.append(qtMedia)
# Set Answer fields
aId = 0
for answerDict in answerDicts:
if answerDict['isAnswer']:
aId += 1
# noinspection PyTypeChecker
anContent, anMedia = getNodeContent(tagList=self.tagList,
tag=answerDict['nodeTag'])
noteList.append(anContent)
media.append(anMedia)
answerDict['aId'] = str(aId)
# noinspection PyShadowingNames
for i in range(aId, X_MAX_ANSWERS):
noteList.append('')
# set field ID
noteList.append(sortId)
# set field Meta
meta = self.getXMindMeta(question=question, answerDicts=answerDicts,
siblings=siblings, connections=connections)
noteList.append(meta)
nId = timestampID(self.col.db, "notes")
noteData = [nId, guid64(), self.model['id'], intTime(), self.col.usn(),
self.currentSheetImport['tag'], joinFields(noteList), "",
"", 0, ""]
return noteData, media
def maybeSync(self, sheetId, noteList):
if self.repair:
existingNotes = list(self.col.db.execute(
"select id, flds from notes where tags like '%" +
self.currentSheetImport['tag'].replace(" ", "") + "%'"))
else:
existingNotes = getNotesFromSheet(sheetId=sheetId, col=self.col)
if existingNotes:
notesToAdd = []
notesToUpdate = []
oldQIdList = list(map(lambda n: json.loads(
splitFields(n[1])[list(X_FLDS.keys()).index('mt')])[
'questionId'], existingNotes))
for newNote in noteList:
newFields = splitFields(newNote[6])
newMeta = json.loads(newFields[list(X_FLDS.keys()).index('mt')])
newQId = newMeta['questionId']
try:
if self.repair:
print('')
newQtxAw = joinFields(newFields[1:22])
oldTpl = tuple(
filter(lambda n: newQtxAw in n[1], existingNotes))[
0]
noteId = existingNotes.index(oldTpl)
else:
noteId = oldQIdList.index(newQId)
# if the fields are different, add it to notes to be updated
if not existingNotes[noteId][1] == newNote[6]:
notesToUpdate.append(
[existingNotes[noteId], newNote])
del existingNotes[noteId]
del oldQIdList[noteId]
except (ValueError, IndexError):
notesToAdd.append(newNote)
self.addNew(notesToAdd)
self.log[0][1] += len(notesToAdd)
self.addUpdates(notesToUpdate)
self.log[1][1] += len(notesToUpdate)
self.removeOld(existingNotes)
self.log[2][1] += len(existingNotes)
self.col.save()
else:
notesToAdd = noteList
self.addNew(notesToAdd)
self.log[0][1] += len(notesToAdd)
def _changeFacts(self, fids, newModel, map):
d = []
nfields = len(newModel.fields)
for (fid, flds) in self.deck.db.execute(
"select id, flds from facts where id in " + ids2str(fids)):
newflds = {}
flds = splitFields(flds)
for old, new in map.items():
newflds[new] = flds[old]
flds = []
for c in range(nfields):
flds.append(newflds.get(c, ""))
flds = joinFields(flds)
d.append(dict(fid=fid, flds=flds, mid=newModel.id))
self.deck.db.executemany(
"update facts set flds=:flds, mid=:mid where id = :fid", d)
self.deck.updateFieldCache(fids)
def dump_collection(colpath, includetags, includedecknames):
sql_attach = 'ATTACH DATABASE "%s" AS db2' % colpath
mw.col.db.execute(sql_attach)
sql_names_of_tables = """SELECT name FROM sqlite_master WHERE type='table'"""
tables = mw.col.db.all(sql_names_of_tables)
tags_anon = {}
for t in tables:
tabname = t[0]
sql_copy_over = """INSERT OR REPLACE INTO db2.{0} SELECT * FROM {0};""".format(tabname)
try:
mw.col.db.execute(sql_copy_over)
# e.g. the add-on "symbols as you type" inserts another table into the database, which was
# recommended in https://apps.ankiweb.net/docs/addons20.html
except:
continue
else:
if tabname == "notes":
for nid, flds, tags in mw.col.db.all("""SELECT id, flds, tags from db2.notes"""):
flist = splitFields(flds)
for i, v in enumerate(flist):
flist[i] = "..." # len(v)
joined = joinFields(flist)
if includetags:
tags = ""
else:
t = tags.strip().split()
alist = []
for i in t:
if i not in tags_anon:
tags_anon[i] = randstr(12)
alist.append(tags_anon[i])
tags = ', tags="' + mw.col.tags.join(mw.col.tags.canonify(alist)) + '"'
attrs = 'flds = "{}", sfld = ""{}'.format(joined, tags)
mw.col.db.execute("UPDATE db2.notes SET {} WHERE id = {}".format(attrs, nid))
if not includedecknames:
decks = mw.col.db.first("select decks from db2.col")[0]
deckdict = json.loads(decks)
old_to_new = decknames_to_anon_dict()
for props in deckdict.values():
props['name'] = old_to_new[props['name']]
mw.col.db.execute("update db2.col set decks=?", json.dumps(deckdict))
if not includetags:
mw.col.db.execute('UPDATE db2.col SET tags = "{}"') # clear tag cache
mw.col.db.commit()
mw.col.db.execute('DETACH DATABASE db2')
def processFields(self,
note: ForeignNote,
fields: Optional[List[str]] = None) -> None:
if not fields:
fields = [""] * len(self.model["flds"])
for c, f in enumerate(self.mapping):
if not f:
continue
elif f == "_tags":
note.tags.extend(self.col.tags.split(note.fields[c]))
else:
sidx = self._fmap[f][0]
fields[sidx] = note.fields[c]
note.fieldsStr = joinFields(fields)
# temporary fix for the following issue until we can update the code:
# https://forums.ankiweb.net/t/python-checksum-rust-checksum/8195/16
if self.col.get_config_bool(Config.Bool.NORMALIZE_NOTE_TEXT):
note.fieldsStr = unicodedata.normalize("NFC", note.fieldsStr)
def _changeNotes(self, nids, newModel, map):
d = []
nfields = len(newModel['flds'])
for (nid, flds) in self.col.db.execute(
"select id, flds from notes where id in "+ids2str(nids)):
newflds = {}
flds = splitFields(flds)
for old, new in map.items():
newflds[new] = flds[old]
flds = []
for c in range(nfields):
flds.append(newflds.get(c, ""))
flds = joinFields(flds)
d.append(dict(nid=nid, flds=flds, mid=newModel['id'],
m=intTime(),u=self.col.usn()))
self.col.db.executemany(
"update notes set flds=:flds,mid=:mid,mod=:m,usn=:u where id = :nid", d)
self.col.updateFieldCache(nids)
def processFields(self,
note: ForeignNote,
fields: Optional[List[str]] = None) -> Any:
if not fields:
fields = [""] * len(self.model['flds'])
for c, f in enumerate(self.mapping):
if not f:
continue
elif f == "_tags":
note.tags.extend(self.col.tags.split(note.fields[c]))
else:
sidx = self._fmap[f][0]
fields[sidx] = note.fields[c]
note.fieldsStr = joinFields(fields)
ords = self.col.models.availOrds(self.model, note.fieldsStr)
if not ords:
self._emptyNotes = True
return ords
def isDuplicate(self, card):
#query the db directly since Anki's search will not match ':' and '(', eg
fields = joinFields([card.front, card.back])
q = 'select id from notes where flds = ?'
noteIds = mw.col.db.list(q, fields)
if len(noteIds) == 0:
# no notes match, we are done
return False
else:
# notes match, but are they in the current deck?
deckId = mw.col.conf['curDeck']
q = 'select id from cards where nid = ? and did = ?'
for noteId in noteIds:
cardIds = mw.col.db.list(q, noteId, deckId)
if len(cardIds) > 0:
return True
return False # matches were found, but not in the current deck
def findReplace(deck, fids, src, dst, regex=False, field=None, fold=True):
"Find and replace fields in a fact."
mmap = {}
if field:
for m in deck.models().values():
for f in m.fields:
if f['name'] == field:
mmap[m.id] = f['ord']
if not mmap:
return 0
# find and gather replacements
if not regex:
src = re.escape(src)
if fold:
src = "(?i)" + src
regex = re.compile(src)
def repl(str):
return re.sub(regex, dst, str)
d = []
for fid, mid, flds in deck.db.execute(
"select id, mid, flds from facts where id in " + ids2str(fids)):
origFlds = flds
# does it match?
sflds = splitFields(flds)
if field:
ord = mmap[mid]
sflds[ord] = repl(sflds[ord])
else:
for c in range(len(sflds)):
sflds[c] = repl(sflds[c])
flds = joinFields(sflds)
if flds != origFlds:
d.append(dict(fid=fid, flds=flds))
if not d:
return 0
# replace
deck.db.executemany("update facts set flds = :flds where id=:fid", d)
deck.updateFieldCache(fids)
return len(d)
def findReplace(deck, fids, src, dst, regex=False, field=None, fold=True):
"Find and replace fields in a fact."
mmap = {}
if field:
for m in deck.models().values():
for f in m.fields:
if f['name'] == field:
mmap[m.id] = f['ord']
if not mmap:
return 0
# find and gather replacements
if not regex:
src = re.escape(src)
if fold:
src = "(?i)"+src
regex = re.compile(src)
def repl(str):
return re.sub(regex, dst, str)
d = []
for fid, mid, flds in deck.db.execute(
"select id, mid, flds from facts where id in "+ids2str(fids)):
origFlds = flds
# does it match?
sflds = splitFields(flds)
if field:
ord = mmap[mid]
sflds[ord] = repl(sflds[ord])
else:
for c in range(len(sflds)):
sflds[c] = repl(sflds[c])
flds = joinFields(sflds)
if flds != origFlds:
d.append(dict(fid=fid, flds=flds))
if not d:
return 0
# replace
deck.db.executemany("update facts set flds = :flds where id=:fid", d)
deck.updateFieldCache(fids)
return len(d)
def mainBackend(modelName,tag,destField,expr):
# Main backend code called by the GUI to do the replace all/field fill operation
# Assign collection to col
col=mw.col
# Get the model id corresponding to modelName
model=findModel(col,modelName)
# Get the index belonging to destination field
destFieldIdx=findFieldIdx(model,destField)
# Run a database query and return list of noteid,fields of notes with specified model ID and tag
noteid,tags,flds=fetchData(col,model,tag)
# Render exp for each note
newflds=[]
for idx in range(len(flds)):
flist=splitFields(flds[idx])
fdic=makeFieldDic(col,model,flist)
fdic['Tags'] = tags [idx]
fdic['Type'] = model['name']
flist[destFieldIdx]=template.render(expr,fdic)
newflds.append([joinFields(flist),noteid[idx]])
# Write the new data to the database
col.db.executemany("update notes set flds=? where id=?",newflds)
# Return the number of notes which were updated
return len(noteid)
def mainBackend(modelName, tag, destField, expr):
# Main backend code called by the GUI to do the replace all/field fill operation
# Assign collection to col
col = mw.col
# Get the model id corresponding to modelName
model = findModel(col, modelName)
# Get the index belonging to destination field
destFieldIdx = findFieldIdx(model, destField)
# Run a database query and return list of noteid,fields of notes with specified model ID and tag
noteid, tags, flds = fetchData(col, model, tag)
# Render exp for each note
newflds = []
for idx in range(len(flds)):
flist = splitFields(flds[idx])
fdic = makeFieldDic(col, model, flist)
fdic['Tags'] = tags[idx]
fdic['Type'] = model['name']
flist[destFieldIdx] = template.render(expr, fdic)
newflds.append([joinFields(flist), noteid[idx]])
# Write the new data to the database
col.db.executemany("update notes set flds=? where id=?", newflds)
# Return the number of notes which were updated
return len(noteid)
def updateCompleteDeck(self, data):
self.startEditing()
did = self.decks().id(data['deck'])
self.decks().flush()
model_manager = self.collection().models
for _, card in data['cards'].items():
self.database().execute(
'replace into cards (id, nid, did, ord, type, queue, due, ivl, factor, reps, lapses, left, '
'mod, usn, odue, odid, flags, data) '
'values (' + '?,' * (12 + 6 - 1) + '?)',
card['id'], card['nid'], did, card['ord'], card['type'], card['queue'], card['due'],
card['ivl'], card['factor'], card['reps'], card['lapses'], card['left'],
intTime(), -1, 0, 0, 0, 0
)
note = data['notes'][str(card['nid'])]
tags = self.collection().tags.join(self.collection().tags.canonify(note['tags']))
self.database().execute(
'replace into notes(id, mid, tags, flds,'
'guid, mod, usn, flags, data, sfld, csum) values (' + '?,' * (4 + 7 - 1) + '?)',
note['id'], note['mid'], tags, joinFields(note['fields']),
guid64(), intTime(), -1, 0, 0, '', fieldChecksum(note['fields'][0])
)
model = data['models'][str(note['mid'])]
if not model_manager.get(model['id']):
model_o = model_manager.new(model['name'])
for field_name in model['fields']:
field = model_manager.newField(field_name)
model_manager.addField(model_o, field)
for template_name in model['templateNames']:
template = model_manager.newTemplate(template_name)
model_manager.addTemplate(model_o, template)
model_o['id'] = model['id']
model_manager.update(model_o)
model_manager.flush()
self.stopEditing()
def joinedFields(self):
return joinFields(self.fields)
def updateNotes( allDb ):
t_0, now, db, TAG = time.time(), intTime(), mw.col.db, mw.col.tags
ds, nid2mmi = [], {}
N_notes = db.scalar( 'select count() from notes' )
mw.progress.start( label='Updating data', max=N_notes, immediate=True )
fidDb = allDb.fidDb()
locDb = allDb.locDb( recalc=False ) # fidDb() already forces locDb recalc
# handle secondary databases
mw.progress.update( label='Creating seen/known/mature from all.db' )
seenDb = filterDbByMat( allDb, cfg1('threshold_seen') )
knownDb = filterDbByMat( allDb, cfg1('threshold_known') )
matureDb = filterDbByMat( allDb, cfg1('threshold_mature') )
mw.progress.update( label='Loading priority.db' )
priorityDb = MorphDb( cfg1('path_priority'), ignoreErrors=True ).db
if cfg1('saveDbs'):
mw.progress.update( label='Saving seen/known/mature dbs' )
seenDb.save( cfg1('path_seen') )
knownDb.save( cfg1('path_known') )
matureDb.save( cfg1('path_mature') )
mw.progress.update( label='Calculating frequency information' )
pops = [ len( locs ) for locs in allDb.db.values() ]
pops = [ n for n in pops if n > 1 ]
mw.progress.update( label='Updating notes' )
for i,( nid, mid, flds, guid, tags ) in enumerate( db.execute( 'select id, mid, flds, guid, tags from notes' ) ):
if i % 500 == 0: mw.progress.update( value=i )
C = partial( cfg, mid, None )
if not C('enabled'): continue
# Get all morphemes for note
ms = set()
for fieldName in C('morph_fields'):
try:
loc = fidDb[ ( nid, guid, fieldName ) ]
ms.update( locDb[ loc ] )
except KeyError: continue
ms = [ m for m in ms if m.pos not in C('morph_blacklist') ]
# Determine un-seen/known/mature and i+N
unseens, unknowns, unmatures, newKnowns = set(), set(), set(), set()
for m in ms:
if m not in seenDb.db: unseens.add( m )
if m not in knownDb.db: unknowns.add( m )
if m not in matureDb.db: unmatures.add( m )
if m not in matureDb.db and m in knownDb.db:
newKnowns.add( m )
# Determine MMI - Morph Man Index
N, N_s, N_k, N_m = len( ms ), len( unseens ), len( unknowns ), len( unmatures )
# Bail early for lite update
if N_k > 2 and C('only update k+2 and below'): continue
# average frequency of unknowns (ie. how common the word is within your collection)
F_k = 0
for focusMorph in unknowns: # focusMorph used outside loop
F_k += len( allDb.db[ focusMorph ] )
F_k_avg = F_k / N_k if N_k > 0 else F_k
usefulness = F_k_avg
# add bonus for morphs in priority.db
isPriority = False
for focusMorph in unknowns:
if focusMorph in priorityDb:
isPriority = True
usefulness += C('priority.db weight')
# add bonus for studying recent learned knowns (reinforce)
for m in newKnowns:
locs = allDb.db[ m ]
if locs:
ivl = min( 1, max( loc.maturity for loc in locs ) )
usefulness += C('reinforce new vocab weight') / ivl #TODO: maybe average this so it doesnt favor long sentences
if any( m.pos == u'動詞' for m in unknowns ): #FIXME: this isn't working???
usefulness += C('verb bonus')
usefulness = 999 - min( 999, usefulness )
# difference from optimal length (too little context vs long sentence)
lenDiff = max( 0, min( 9, abs( C('optimal sentence length') - N ) -2 ) )
# calculate mmi
mmi = 10000*N_k + 1000*lenDiff + usefulness
if C('set due based on mmi'):
nid2mmi[ nid ] = mmi
# Fill in various fields/tags on the note based on cfg
ts, fs = TAG.split( tags ), splitFields( flds )
# determine card type
compTag, vocabTag, notReadyTag, alreadyKnownTag, priorityTag = tagNames = C('tag_comprehension'), C('tag_vocab'), C('tag_notReady'), C('tag_alreadyKnown'), C('tag_priority')
if N_m == 0: # sentence comprehension card, m+0
ts = [ compTag ] + [ t for t in ts if t not in [ vocabTag, notReadyTag ] ]
setField( mid, fs, C('focusMorph'), u'' )
elif N_k == 1: # new vocab card, k+1
ts = [ vocabTag ] + [ t for t in ts if t not in [ compTag, notReadyTag ] ]
setField( mid, fs, C('focusMorph'), u'%s' % focusMorph.base )
elif N_k > 1: # M+1+ and K+2+
ts = [ notReadyTag ] + [ t for t in ts if t not in [ compTag, vocabTag ] ]
# set type agnostic fields
setField( mid, fs, C('k+N'), u'%d' % N_k )
setField( mid, fs, C('m+N'), u'%d' % N_m )
setField( mid, fs, C('morphManIndex'), u'%d' % mmi )
setField( mid, fs, C('unknowns'), u', '.join( u.base for u in unknowns ) )
setField( mid, fs, C('unmatures'), u', '.join( u.base for u in unmatures ) )
setField( mid, fs, C('unknownFreq'), u'%d' % F_k_avg )
# other tags
if priorityTag in ts: ts.remove( priorityTag )
if isPriority: ts.append( priorityTag )
# update sql db
tags_ = TAG.join( TAG.canonify( ts ) )
flds_ = joinFields( fs )
if flds != flds_ or tags != tags_: # only update notes that have changed
csum = fieldChecksum( fs[0] )
sfld = stripHTML( fs[ getSortFieldIndex( mid ) ] )
ds.append( { 'now':now, 'tags':tags_, 'flds':flds_, 'sfld':sfld, 'csum':csum, 'usn':mw.col.usn(), 'nid':nid } )
mw.progress.update( value=i, label='Updating anki database...' )
mw.col.db.executemany( 'update notes set tags=:tags, flds=:flds, sfld=:sfld, csum=:csum, mod=:now, usn=:usn where id=:nid', ds )
TAG.register( tagNames )
# Now reorder new cards based on MMI
mw.progress.update( value=i, label='Updating new card ordering...' )
ds = []
for ( cid, nid, due ) in db.execute( 'select id, nid, due from cards where type = 0' ):
if nid in nid2mmi: # owise it was disabled
due_ = nid2mmi[ nid ]
if due != due_: # only update cards that have changed
ds.append( { 'now':now, 'due':due_, 'usn':mw.col.usn(), 'cid':cid } )
mw.col.db.executemany( 'update cards set due=:due, mod=:now, usn=:usn where id=:cid', ds )
mw.reset()
printf( 'Updated notes in %f sec' % ( time.time() - t_0 ) )
mw.progress.finish()
def updateMainArea(self):
if self._isCloze():
cnt = len(self.mm.availOrds(
self.model, joinFields(self.note.fields)))
for g in self.pform.groupBox, self.pform.groupBox_2:
g.setTitle(g.title() + _(" (1 of %d)") % max(cnt, 1))
def findTemplates(self, note):
"Return (active), non-empty templates."
model = note.model()
avail = self.models.availOrds(model, joinFields(note.fields))
return self._tmplsFromOrds(model, avail)
def updateNotes( allDb ):
t_0, now, db, TAG = time.time(), intTime(), mw.col.db, mw.col.tags
ds, nid2mmi = [], {}
N_notes = db.scalar( 'select count() from notes' )
mw.progress.start( label='Updating data', max=N_notes, immediate=True )
fidDb = allDb.fidDb()
locDb = allDb.locDb( recalc=False ) # fidDb() already forces locDb recalc
# read tag names
compTag, vocabTag, freshTag, notReadyTag, alreadyKnownTag, priorityTag, tooShortTag, tooLongTag = tagNames = jcfg('Tag_Comprehension'), jcfg('Tag_Vocab'), jcfg('Tag_Fresh'), jcfg('Tag_NotReady'), jcfg('Tag_AlreadyKnown'), jcfg('Tag_Priority'), jcfg('Tag_TooShort'), jcfg('Tag_TooLong')
TAG.register( tagNames )
badLengthTag = jcfg2().get('Tag_BadLength')
# handle secondary databases
mw.progress.update( label='Creating seen/known/mature from all.db' )
seenDb = filterDbByMat( allDb, cfg1('threshold_seen') )
knownDb = filterDbByMat( allDb, cfg1('threshold_known') )
matureDb = filterDbByMat( allDb, cfg1('threshold_mature') )
mw.progress.update( label='Loading priority.db' )
priorityDb = MorphDb( cfg1('path_priority'), ignoreErrors=True ).db
if cfg1('saveDbs'):
mw.progress.update( label='Saving seen/known/mature dbs' )
seenDb.save( cfg1('path_seen') )
knownDb.save( cfg1('path_known') )
matureDb.save( cfg1('path_mature') )
mw.progress.update( label='Updating notes' )
for i,( nid, mid, flds, guid, tags ) in enumerate( db.execute( 'select id, mid, flds, guid, tags from notes' ) ):
if i % 500 == 0: mw.progress.update( value=i )
C = partial( cfg, mid, None )
note = mw.col.getNote(nid)
notecfg = getFilter(note)
if notecfg is None or not notecfg['Modify']: continue
# Get all morphemes for note
morphemes = set()
for fieldName in notecfg['Fields']:
try:
loc = fidDb[ ( nid, guid, fieldName ) ]
morphemes.update( locDb[ loc ] )
except KeyError: continue
# Determine un-seen/known/mature and i+N
unseens, unknowns, unmatures, newKnowns = set(), set(), set(), set()
for morpheme in morphemes:
if morpheme not in seenDb.db: unseens.add( morpheme )
if morpheme not in knownDb.db: unknowns.add( morpheme )
if morpheme not in matureDb.db: unmatures.add( morpheme )
if morpheme not in matureDb.db and morpheme in knownDb.db:
newKnowns.add( morpheme )
# Determine MMI - Morph Man Index
N, N_s, N_k, N_m = len( morphemes ), len( unseens ), len( unknowns ), len( unmatures )
# Bail early for lite update
if N_k > 2 and C('only update k+2 and below'): continue
# average frequency of unknowns (ie. how common the word is within your collection)
F_k = 0
for focusMorph in unknowns: # focusMorph used outside loop
F_k += allDb.frequency(focusMorph)
F_k_avg = F_k // N_k if N_k > 0 else F_k
usefulness = F_k_avg
# add bonus for morphs in priority.db
isPriority = False
for focusMorph in unknowns:
if focusMorph in priorityDb:
isPriority = True
usefulness += C('priority.db weight')
# add bonus for studying recent learned knowns (reinforce)
for morpheme in newKnowns:
locs = allDb.db[ morpheme ]
if locs:
ivl = min( 1, max( loc.maturity for loc in locs ) )
usefulness += C('reinforce new vocab weight') // ivl #TODO: maybe average this so it doesnt favor long sentences
if any( morpheme.pos == u'動詞' for morpheme in unknowns ): #FIXME: this isn't working???
usefulness += C('verb bonus')
usefulness = 999 - min( 999, usefulness )
# difference from optimal length range (too little context vs long sentence)
lenDiffRaw = min(N - C('min good sentence length'),
max(0, N - C('max good sentence length')))
lenDiff = min(9, abs(lenDiffRaw))
# calculate mmi
mmi = 10000*N_k + 1000*lenDiff + usefulness
if C('set due based on mmi'):
nid2mmi[ nid ] = mmi
# Fill in various fields/tags on the note based on cfg
ts, fs = TAG.split( tags ), splitFields( flds )
# clear any 'special' tags, the appropriate will be set in the next few lines
ts = [ t for t in ts if t not in [ notReadyTag, compTag, vocabTag, freshTag ] ]
# determine card type
if N_m == 0: # sentence comprehension card, m+0
ts = ts + [ compTag ]
setField( mid, fs, jcfg('Field_FocusMorph'), u'' )
elif N_k == 1: # new vocab card, k+1
ts = ts + [ vocabTag ]
setField( mid, fs, jcfg('Field_FocusMorph'), u'%s' % focusMorph.base )
elif N_k > 1: # M+1+ and K+2+
ts = ts + [ notReadyTag ]
setField( mid, fs, jcfg('Field_FocusMorph'), u'')
elif N_m == 1: # we have k+0, and m+1, so this card does not introduce a new vocabulary -> card for newly learned morpheme
ts = ts + [ freshTag ]
setField( mid, fs, jcfg('Field_FocusMorph'), u'%s' % list(unmatures)[0].base)
else: # only case left: we have k+0, but m+2 or higher, so this card does not introduce a new vocabulary -> card for newly learned morpheme
ts = ts + [ freshTag ]
setField( mid, fs, jcfg('Field_FocusMorph'), u'')
# set type agnostic fields
setField( mid, fs, jcfg('Field_UnknownMorphCount'), u'%d' % N_k )
setField( mid, fs, jcfg('Field_UnmatureMorphCount'), u'%d' % N_m )
setField( mid, fs, jcfg('Field_MorphManIndex'), u'%d' % mmi )
setField( mid, fs, jcfg('Field_Unknowns'), u', '.join( u.base for u in unknowns ) )
setField( mid, fs, jcfg('Field_Unmatures'), u', '.join( u.base for u in unmatures ) )
setField( mid, fs, jcfg('Field_UnknownFreq'), u'%d' % F_k_avg )
# remove deprecated tag
if badLengthTag is not None and badLengthTag in ts:
ts.remove( badLengthTag )
# other tags
if priorityTag in ts: ts.remove( priorityTag )
if isPriority: ts.append( priorityTag )
if tooShortTag in ts: ts.remove( tooShortTag )
if lenDiffRaw < 0: ts.append( tooShortTag )
if tooLongTag in ts: ts.remove( tooLongTag )
if lenDiffRaw > 0: ts.append( tooLongTag )
# remove unnecessary tags
if not jcfg('Option_SetNotRequiredTags'):
unnecessary = [priorityTag, tooShortTag, tooLongTag]
ts = [tag for tag in ts if tag not in unnecessary]
# update sql db
tags_ = TAG.join( TAG.canonify( ts ) )
flds_ = joinFields( fs )
if flds != flds_ or tags != tags_: # only update notes that have changed
csum = fieldChecksum( fs[0] )
sfld = stripHTML( fs[ getSortFieldIndex( mid ) ] )
ds.append( { 'now':now, 'tags':tags_, 'flds':flds_, 'sfld':sfld, 'csum':csum, 'usn':mw.col.usn(), 'nid':nid } )
mw.progress.update( value=i, label='Updating anki database...' )
mw.col.db.executemany( 'update notes set tags=:tags, flds=:flds, sfld=:sfld, csum=:csum, mod=:now, usn=:usn where id=:nid', ds )
# Now reorder new cards based on MMI
mw.progress.update( value=i, label='Updating new card ordering...' )
ds = []
# "type = 0": new cards
# "type = 1": learning cards [is supposed to be learning: in my case no learning card had this type]
# "type = 2": review cards
for ( cid, nid, due ) in db.execute( 'select id, nid, due from cards where type = 0' ):
if nid in nid2mmi: # owise it was disabled
due_ = nid2mmi[ nid ]
if due != due_: # only update cards that have changed
ds.append( { 'now':now, 'due':due_, 'usn':mw.col.usn(), 'cid':cid } )
mw.col.db.executemany( 'update cards set due=:due, mod=:now, usn=:usn where id=:cid', ds )
mw.reset()
printf( 'Updated notes in %f sec' % ( time.time() - t_0 ) )
mw.progress.finish()
return knownDb
def isEmpty(self):
ords = self.col.models.availOrds(
self.model(), joinFields(self.note().fields))
if self.ord not in ords:
return True
def run(self):
tree = Tree(self.file[:-4])
queue = tree.parse()
deck_name = tree.name
col = self.col
# Setup a deck
deck = col.decks.id(deck_name)
col.decks.select(deck)
# Make new models
new_models = []
old_models = []
old_model_ids = []
for note in queue:
m = col.models.byName(note["ffsModel"]["name"])
if not m:
unique = True
for n in new_models:
if note["ffsModel"]["name"] == n["ffsModel"]["name"]:
unique = False
break
if unique:
new_models.append(note)
else:
unique = True
for n in old_models:
if note["ffsModel"]["name"] == n["ffsModel"]["name"]:
unique = False
break
if unique:
old_models.append(note)
note["ffsModel"]["id"] = m["id"]
self.handle_models(new_models, True)
# Check for updates
nids = []
nids_decks = {}
update = []
update_nids = []
add = []
delete = []
tags_to_register = ["ffsi:added", "ffsi:changed", "ffsi:owned"]
addedc = 0
changedc = 0
deletedc = 0
for note in col.db.execute("select * from notes"):
note = list(note)
match = None
deletable = False
m = col.models.get(note[2])
tags = col.tags.split(note[5])
field_map = col.models.fieldMap(m)
fields = splitFields(note[6])
if "Filename" not in field_map:
if col.tags.inList("ffsi:owned", tags):
raise ValueError("ffs owned note tampered with.")
continue
filename = fields[field_map["Filename"][0]]
deck_check = filename.split("/")[0]
if col.tags.inList("ffsi:owned", tags) and deck_name == deck_check:
deletable = True
for n in queue:
om = col.models.byName(n["ffsModel"]["name"])
if n["Filename"] == filename:
flds = []
changed = False
if "fields" in n["ffsModel"]:
flds.append(n["Filename"])
for field in n["ffsModel"]["fields"].split():
if field not in n:
raise ValueError("Note missing model field," + " in file: {0}".format(n["Filename"]))
flds.append(n[field])
else:
for field in col.models.fieldNames(om):
flds.append(n[field])
if int(note[2]) != int(om["id"]):
changed = True
old_model_ids.append(note[2])
note[2] = int(om["id"])
if note[6] != joinFields(flds):
changed = True
note[6] = joinFields(flds)
new_tags = col.tags.addToStr("ffsi:owned", "")
note[5] = col.tags.remFromStr("ffsi:added", note[5])
note[5] = col.tags.remFromStr("ffsi:changed", note[5])
if "tags" in n:
for t in n["tags"].split():
new_tags = col.tags.addToStr(t, new_tags)
tags_to_register.extend(col.tags.split(new_tags))
else:
new_tags = note[5]
if note[5] != new_tags:
changed = True
if changed:
new_tags = col.tags.addToStr("ffsi:changed", new_tags)
changedc = changedc + 1
note[5] = new_tags
update.append(note)
update_nids.append(note[0])
match = n
if match:
queue.remove(match)
elif deletable:
delete.append(note[0])
deletedc = deletedc + 1
self.handle_models(old_models)
for note in queue:
add.append(note)
col.remNotes(delete)
col.db.executemany("insert or replace into notes values (?,?,?,?,?,?,?,?,?,?,?)", update)
col.genCards(update_nids)
col.updateFieldCache(update_nids)
col.tags.registerNotes(update_nids)
for textfile in add:
m = col.models.byName(textfile["ffsModel"]["name"])
col.models.setCurrent(m)
note = col.newNote()
new_tags = ["ffsi:owned", "ffsi:added"]
if "tags" in textfile:
custom_tags = textfile["tags"].split()
new_tags.extend(custom_tags)
tags_to_register.extend(custom_tags)
[note.addTag(tag) for tag in list(set(new_tags))]
fields = col.models.fieldNames(m)
for field in fields:
note[field] = textfile[field]
col.addNote(note)
nids.append(note.id)
nids_decks[note.id] = textfile["ffsDeckname"]
addedc = addedc + 1
col.genCards(nids)
added_cards = []
for card in col.db.execute("select * from cards"):
card = list(card)
if card[1] in nids:
added_cards.append(card)
totalc = addedc + changedc + deletedc
self.log.append(
ngettext(
"{0} card changed, " + "({1}, {2}, {3}) note (added, changed, or deleted).",
"{0} cards changed. " + "({1}, {2}, {3}) notes (added, changed, or deleted).",
totalc,
).format(len(added_cards), addedc, changedc, deletedc)
)
# TODO improve this loop
for card in added_cards:
deck_name = nids_decks[card[1]]
deck = col.decks.id(deck_name)
col.decks.setDeck([card[0]], deck)
# Set the default model to something standard
basic = col.models.byName("Basic")
if basic:
col.models.setCurrent(basic)
# Register all of our tags
col.tags.register(list(set(tags_to_register)))
# Cleanup empty decks
children_decks = col.decks.children(deck)
for child in children_decks:
cids = col.decks.cids(child[1], True)
if len(cids) == 0:
col.decks.rem(child[1])
# Cleanup models
for note in old_models:
m = col.models.byName(note["ffsModel"]["name"])
if m:
nids = col.models.nids(m)
if len(nids) == 0:
col.models.rem(m)
for i in old_model_ids:
m = col.models.get(i)
if m:
nids = col.models.nids(m)
if len(nids) == 0:
col.models.rem(m)
col.conf["nextPos"] = col.db.scalar("select max(due)+1 from cards where type = 0") or 0
col.save()
col.db.execute("vacuum")
col.db.execute("analyze")
col.fixIntegrity()
