def availOrds(self, m, flds):
"Given a joined field string, return available template ordinals."
if m['type'] == MODEL_CLOZE:
return self._availClozeOrds(m, flds)
fields = {}
for c, f in enumerate(splitFields(flds)):
fields[c] = f.strip()
avail = []
for ord, type, req in m['req']:
# unsatisfiable template
if type == "none":
continue
# AND requirement?
elif type == "all":
ok = True
for idx in req:
if not fields[idx]:
# missing and was required
ok = False
break
if not ok:
continue
# OR requirement?
elif type == "any":
ok = False
for idx in req:
if fields[idx]:
ok = True
break
if not ok:
continue
avail.append(ord)
return avail
def _findField(self, field, val):
field = field.lower()
val = val.replace("*", "%")
# find models that have that field
mods = {}
for m in self.col.models.all():
for f in m['flds']:
if unicodedata.normalize("NFC", f['name'].lower()) == field:
mods[str(m['id'])] = (m, f['ord'])
if not mods:
# nothing has that field
return
# gather nids
regex = re.escape(val).replace("_", ".").replace(re.escape("%"), ".*")
nids = []
for (id, mid, flds) in self.col.db.execute(
"""
select id, mid, flds from notes
where mid in %s and flds like ? escape '\\'""" % (ids2str(list(mods.keys()))),
"%" + val + "%"):
flds = splitFields(flds)
ord = mods[str(mid)][1]
strg = flds[ord]
try:
if re.search("(?si)^" + regex + "$", strg):
nids.append(id)
except sre_constants.error:
return
if not nids:
return "0"
return "n.id in %s" % ids2str(nids)
def _mungeMedia(self, mid, fields):
fields = splitFields(fields)
def repl(match):
fname = match.group("fname")
srcData = self._srcMediaData(fname)
dstData = self._dstMediaData(fname)
if not srcData:
# file was not in source, ignore
return match.group(0)
# if model-local file exists from a previous import, use that
name, ext = os.path.splitext(fname)
lname = "%s_%s%s" % (name, mid, ext)
if self.dst.media.have(lname):
return match.group(0).replace(fname, lname)
# if missing or the same, pass unmodified
elif not dstData or srcData == dstData:
# need to copy?
if not dstData:
self._writeDstMedia(fname, srcData)
return match.group(0)
# exists but does not match, so we need to dedupe
self._writeDstMedia(lname, srcData)
return match.group(0).replace(fname, lname)
for i in range(len(fields)):
fields[i] = self.dst.media.transformNames(fields[i], repl)
return joinFields(fields)
def load(self):
(self.guid, self.mid, self.mod, self.usn, self.tags, self.fields,
self.flags, self.data) = self.col.db.first(
"""
select guid, mid, mod, usn, tags, flds, flags, data
from notes where id = ?""", self.id)
self.fields = splitFields(self.fields)
self.tags = self.col.tags.split(self.tags)
self._model = self.col.models.get(self.mid)
self._fmap = self.col.models.fieldMap(self._model)
self.scm = self.col.scm
def _transformFields(self, m, fn):
# model hasn't been added yet?
if not m['id']:
return
r = []
for (id, flds) in self.col.db.execute(
"select id, flds from notes where mid = ?", m['id']):
r.append((joinFields(fn(splitFields(flds))),
intTime(), self.col.usn(), id))
self.col.db.executemany(
"update notes set flds=?,mod=?,usn=? where id = ?", r)
def dupeOrEmpty(self):
"1 if first is empty; 2 if first is a duplicate, False otherwise."
val = self.fields[0]
if not val.strip():
return 1
csum = fieldChecksum(val)
# find any matching csums and compare
for flds in self.col.db.list(
"select flds from notes where csum = ? and id != ? and mid = ?",
csum, self.id or 0, self.mid):
if stripHTMLMedia(splitFields(flds)[0]) == stripHTMLMedia(
self.fields[0]):
return 2
return False
def findReplace(col, nids, src, dst, regex=False, field=None, fold=True):
"Find and replace fields in a note."
mmap = {}
if field:
for m in col.models.all():
for f in m['flds']:
if f['name'].lower() == field.lower():
mmap[str(m['id'])] = f['ord']
if not mmap:
return 0
# find and gather replacements
if not regex:
src = re.escape(src)
if fold:
src = "(?i)" + src
regex = re.compile(src)
def repl(str):
return re.sub(regex, dst, str)
d = []
snids = ids2str(nids)
nids = []
for nid, mid, flds in col.db.execute(
"select id, mid, flds from notes where id in " + snids):
origFlds = flds
# does it match?
sflds = splitFields(flds)
if field:
try:
ord = mmap[str(mid)]
sflds[ord] = repl(sflds[ord])
except KeyError:
# note doesn't have that field
continue
else:
for c in range(len(sflds)):
sflds[c] = repl(sflds[c])
flds = joinFields(sflds)
if flds != origFlds:
nids.append(nid)
d.append(dict(nid=nid, flds=flds, u=col.usn(), m=intTime()))
if not d:
return 0
# replace
col.db.executemany(
"update notes set flds=:flds,mod=:m,usn=:u where id=:nid", d)
col.updateFieldCache(nids)
col.genCards(nids)
return len(d)
def updateFieldCache(self, nids):
"Update field checksums and sort cache, after find&replace, etc."
snids = ids2str(nids)
r = []
for (nid, mid, flds) in self._fieldData(snids):
fields = splitFields(flds)
model = self.models.get(mid)
if not model:
# note points to invalid model
continue
r.append((stripHTMLMedia(fields[self.models.sortIdx(model)]),
fieldChecksum(fields[0]), nid))
# apply, relying on calling code to bump usn+mod
self.db.executemany("update notes set sfld=?, csum=? where id=?", r)
def _findDupes(self, args):
# caller must call stripHTMLMedia on passed val
(val, args) = args
try:
mid, val = val.split(",", 1)
except OSError:
return
csum = fieldChecksum(val)
nids = []
for nid, flds in self.col.db.execute(
"select id, flds from notes where mid=? and csum=?", mid,
csum):
if stripHTMLMedia(splitFields(flds)[0]) == val:
nids.append(nid)
return "n.id in %s" % ids2str(nids)
def _renderQA(self, data, qfmt=None, afmt=None):
"Returns hash of id, question, answer."
# data is [cid, nid, mid, did, ord, tags, flds, cardFlags]
# unpack fields and create dict
flist = splitFields(data[6])
fields = {}
model = self.models.get(data[2])
for (name, (idx, conf)) in list(self.models.fieldMap(model).items()):
fields[name] = flist[idx]
fields['Tags'] = data[5].strip()
fields['Type'] = model['name']
fields['Deck'] = self.decks.name(data[3])
fields['Subdeck'] = fields['Deck'].split('::')[-1]
fields['CardFlag'] = self._flagNameFromCardFlags(data[7])
if model['type'] == MODEL_STD:
template = model['tmpls'][data[4]]
else:
template = model['tmpls'][0]
fields['Card'] = template['name']
fields['c%d' % (data[4] + 1)] = "1"
# render q & a
d = dict(id=data[0])
qfmt = qfmt or template['qfmt']
afmt = afmt or template['afmt']
for (type, format) in (("q", qfmt), ("a", afmt)):
if type == "q":
format = re.sub("{{(?!type:)(.*?)cloze:",
r"{{\1cq-%d:" % (data[4] + 1), format)
format = format.replace("<%cloze:",
"<%%cq:%d:" % (data[4] + 1))
else:
format = re.sub("{{(.*?)cloze:", r"{{\1ca-%d:" % (data[4] + 1),
format)
format = format.replace("<%cloze:",
"<%%ca:%d:" % (data[4] + 1))
fields['FrontSide'] = stripSounds(d['q'])
fields = runFilter("mungeFields", fields, model, data, self)
html = anki_lib.template.render(format, fields)
d[type] = runFilter("mungeQA", html, type, fields, model, data,
self)
# empty cloze?
if type == 'q' and model['type'] == MODEL_CLOZE:
if not self.models._availClozeOrds(model, data[6], False):
d['q'] += ("<p>" + _(
"Please edit this note and add some cloze deletions. (%s)"
) % ("<a href=%s#cloze>%s</a>" % (HELP_SITE, _("help"))))
return d
def _availClozeOrds(self, m, flds, allowEmpty=True):
sflds = splitFields(flds)
map = self.fieldMap(m)
ords = set()
matches = re.findall("{{[^}]*?cloze:(?:[^}]?:)*(.+?)}}", m['tmpls'][0]['qfmt'])
matches += re.findall("<%cloze:(.+?)%>", m['tmpls'][0]['qfmt'])
for fname in matches:
if fname not in map:
continue
ord = map[fname][0]
ords.update([int(m)-1 for m in re.findall(
r"(?s){{c(\d+)::.+?}}", sflds[ord])])
if -1 in ords:
ords.remove(-1)
if not ords and allowEmpty:
# empty clozes use first ord
return [0]
return list(ords)
def _changeNotes(self, nids, newModel, map):
d = []
nfields = len(newModel['flds'])
for (nid, flds) in self.col.db.execute(
"select id, flds from notes where id in "+ids2str(nids)):
newflds = {}
flds = splitFields(flds)
for old, new in list(map.items()):
newflds[new] = flds[old]
flds = []
for c in range(nfields):
flds.append(newflds.get(c, ""))
flds = joinFields(flds)
d.append(dict(nid=nid, flds=flds, mid=newModel['id'],
m=intTime(),u=self.col.usn()))
self.col.db.executemany(
"update notes set flds=:flds,mid=:mid,mod=:m,usn=:u where id = :nid", d)
self.col.updateFieldCache(nids)
def doExport(self, file):
cardIds = self.cardIds()
data = []
for id, flds, tags in self.col.db.execute("""
select guid, flds, tags from notes
where id in
(select nid from cards
where cards.id in %s)""" % ids2str(cardIds)):
row = []
# note id
if self.includeID:
row.append(str(id))
# fields
row.extend([self.processText(f) for f in splitFields(flds)])
# tags
if self.includeTags:
row.append(tags.strip())
data.append("\t".join(row))
self.count = len(data)
out = "\n".join(data)
file.write(out.encode("utf-8"))
def findDupes(col, fieldName, search=""):
# limit search to notes with applicable field name
if search:
search = "(" + search + ") "
search += "'%s:*'" % fieldName
# go through notes
vals = {}
dupes = []
fields = {}
def ordForMid(mid):
if mid not in fields:
model = col.models.get(mid)
for c, f in enumerate(model['flds']):
if f['name'].lower() == fieldName.lower():
fields[mid] = c
break
return fields[mid]
for nid, mid, flds in col.db.all(
"select id, mid, flds from notes where id in " +
ids2str(col.findNotes(search))):
flds = splitFields(flds)
ord = ordForMid(mid)
if ord is None:
continue
val = flds[ord]
val = stripHTMLMedia(val)
# empty does not count as duplicate
if not val:
continue
if val not in vals:
vals[val] = []
vals[val].append(nid)
if len(vals[val]) == 2:
dupes.append((val, vals[val]))
return dupes
def importNotes(self, notes):
"Convert each card into a note, apply attributes and add to col."
assert self.mappingOk()
# note whether tags are mapped
self._tagsMapped = False
for f in self.mapping:
if f == "_tags":
self._tagsMapped = True
# gather checks for duplicate comparison
csums = {}
for csum, id in self.col.db.execute(
"select csum, id from notes where mid = ?", self.model['id']):
if csum in csums:
csums[csum].append(id)
else:
csums[csum] = [id]
firsts = {}
fld0idx = self.mapping.index(self.model['flds'][0]['name'])
self._fmap = self.col.models.fieldMap(self.model)
self._nextID = timestampID(self.col.db, "notes")
# loop through the notes
updates = []
updateLog = []
updateLogTxt = _("First field matched: %s")
dupeLogTxt = _("Added duplicate with first field: %s")
new = []
self._ids = []
self._cards = []
self._emptyNotes = False
dupeCount = 0
dupes = []
for n in notes:
for c in range(len(n.fields)):
if not self.allowHTML:
n.fields[c] = html.escape(n.fields[c], quote=False)
n.fields[c] = n.fields[c].strip()
if not self.allowHTML:
n.fields[c] = n.fields[c].replace("\n", "<br>")
n.fields[c] = unicodedata.normalize("NFC", n.fields[c])
n.tags = [unicodedata.normalize("NFC", t) for t in n.tags]
fld0 = n.fields[fld0idx]
csum = fieldChecksum(fld0)
# first field must exist
if not fld0:
self.log.append(
_("Empty first field: %s") % " ".join(n.fields))
continue
# earlier in import?
if fld0 in firsts and self.importMode != 2:
# duplicates in source file; log and ignore
self.log.append(_("Appeared twice in file: %s") % fld0)
continue
firsts[fld0] = True
# already exists?
found = False
if csum in csums:
# csum is not a guarantee; have to check
for id in csums[csum]:
flds = self.col.db.scalar(
"select flds from notes where id = ?", id)
sflds = splitFields(flds)
if fld0 == sflds[0]:
# duplicate
found = True
if self.importMode == 0:
data = self.updateData(n, id, sflds)
if data:
updates.append(data)
updateLog.append(updateLogTxt % fld0)
dupeCount += 1
found = True
elif self.importMode == 1:
dupeCount += 1
elif self.importMode == 2:
# allow duplicates in this case
if fld0 not in dupes:
# only show message once, no matter how many
# duplicates are in the collection already
updateLog.append(dupeLogTxt % fld0)
dupes.append(fld0)
found = False
# newly add
if not found:
data = self.newData(n)
if data:
new.append(data)
# note that we've seen this note once already
firsts[fld0] = True
self.addNew(new)
self.addUpdates(updates)
# make sure to update sflds, etc
self.col.updateFieldCache(self._ids)
# generate cards
if self.col.genCards(self._ids):
self.log.insert(
0, _("Empty cards found. Please run Tools>Empty Cards."))
# apply scheduling updates
self.updateCards()
# we randomize or order here, to ensure that siblings
# have the same due#
did = self.col.decks.selected()
conf = self.col.decks.confForDid(did)
# in order due?
if conf['new']['order'] == NEW_CARDS_RANDOM:
self.col.sched.randomizeCards(did)
part1 = ngettext("%d note added", "%d notes added",
len(new)) % len(new)
part2 = ngettext("%d note updated", "%d notes updated",
self.updateCount) % self.updateCount
if self.importMode == 0:
unchanged = dupeCount - self.updateCount
elif self.importMode == 1:
unchanged = dupeCount
else:
unchanged = 0
part3 = ngettext("%d note unchanged", "%d notes unchanged",
unchanged) % unchanged
self.log.append("%s, %s, %s." % (part1, part2, part3))
self.log.extend(updateLog)
if self._emptyNotes:
self.log.append(
_("""\
One or more notes were not imported, because they didn't generate any cards. \
This can happen when you have empty fields or when you have not mapped the \
content in the text file to the correct fields."""))
self.total = len(self._ids)
