def availOrds(self, m, flds): "Given a joined field string, return available template ordinals." if m['type'] == MODEL_CLOZE: return self._availClozeOrds(m, flds) fields = {} for c, f in enumerate(splitFields(flds)): fields[c] = f.strip() avail = [] for ord, type, req in m['req']: # unsatisfiable template if type == "none": continue # AND requirement? elif type == "all": ok = True for idx in req: if not fields[idx]: # missing and was required ok = False break if not ok: continue # OR requirement? elif type == "any": ok = False for idx in req: if fields[idx]: ok = True break if not ok: continue avail.append(ord) return avail
def _findField(self, field, val): field = field.lower() val = val.replace("*", "%") # find models that have that field mods = {} for m in self.col.models.all(): for f in m['flds']: if unicodedata.normalize("NFC", f['name'].lower()) == field: mods[str(m['id'])] = (m, f['ord']) if not mods: # nothing has that field return # gather nids regex = re.escape(val).replace("_", ".").replace(re.escape("%"), ".*") nids = [] for (id, mid, flds) in self.col.db.execute( """ select id, mid, flds from notes where mid in %s and flds like ? escape '\\'""" % (ids2str(list(mods.keys()))), "%" + val + "%"): flds = splitFields(flds) ord = mods[str(mid)][1] strg = flds[ord] try: if re.search("(?si)^" + regex + "$", strg): nids.append(id) except sre_constants.error: return if not nids: return "0" return "n.id in %s" % ids2str(nids)
def _mungeMedia(self, mid, fields): fields = splitFields(fields) def repl(match): fname = match.group("fname") srcData = self._srcMediaData(fname) dstData = self._dstMediaData(fname) if not srcData: # file was not in source, ignore return match.group(0) # if model-local file exists from a previous import, use that name, ext = os.path.splitext(fname) lname = "%s_%s%s" % (name, mid, ext) if self.dst.media.have(lname): return match.group(0).replace(fname, lname) # if missing or the same, pass unmodified elif not dstData or srcData == dstData: # need to copy? if not dstData: self._writeDstMedia(fname, srcData) return match.group(0) # exists but does not match, so we need to dedupe self._writeDstMedia(lname, srcData) return match.group(0).replace(fname, lname) for i in range(len(fields)): fields[i] = self.dst.media.transformNames(fields[i], repl) return joinFields(fields)
def load(self): (self.guid, self.mid, self.mod, self.usn, self.tags, self.fields, self.flags, self.data) = self.col.db.first( """ select guid, mid, mod, usn, tags, flds, flags, data from notes where id = ?""", self.id) self.fields = splitFields(self.fields) self.tags = self.col.tags.split(self.tags) self._model = self.col.models.get(self.mid) self._fmap = self.col.models.fieldMap(self._model) self.scm = self.col.scm
def _transformFields(self, m, fn): # model hasn't been added yet? if not m['id']: return r = [] for (id, flds) in self.col.db.execute( "select id, flds from notes where mid = ?", m['id']): r.append((joinFields(fn(splitFields(flds))), intTime(), self.col.usn(), id)) self.col.db.executemany( "update notes set flds=?,mod=?,usn=? where id = ?", r)
def dupeOrEmpty(self): "1 if first is empty; 2 if first is a duplicate, False otherwise." val = self.fields[0] if not val.strip(): return 1 csum = fieldChecksum(val) # find any matching csums and compare for flds in self.col.db.list( "select flds from notes where csum = ? and id != ? and mid = ?", csum, self.id or 0, self.mid): if stripHTMLMedia(splitFields(flds)[0]) == stripHTMLMedia( self.fields[0]): return 2 return False
def findReplace(col, nids, src, dst, regex=False, field=None, fold=True): "Find and replace fields in a note." mmap = {} if field: for m in col.models.all(): for f in m['flds']: if f['name'].lower() == field.lower(): mmap[str(m['id'])] = f['ord'] if not mmap: return 0 # find and gather replacements if not regex: src = re.escape(src) if fold: src = "(?i)" + src regex = re.compile(src) def repl(str): return re.sub(regex, dst, str) d = [] snids = ids2str(nids) nids = [] for nid, mid, flds in col.db.execute( "select id, mid, flds from notes where id in " + snids): origFlds = flds # does it match? sflds = splitFields(flds) if field: try: ord = mmap[str(mid)] sflds[ord] = repl(sflds[ord]) except KeyError: # note doesn't have that field continue else: for c in range(len(sflds)): sflds[c] = repl(sflds[c]) flds = joinFields(sflds) if flds != origFlds: nids.append(nid) d.append(dict(nid=nid, flds=flds, u=col.usn(), m=intTime())) if not d: return 0 # replace col.db.executemany( "update notes set flds=:flds,mod=:m,usn=:u where id=:nid", d) col.updateFieldCache(nids) col.genCards(nids) return len(d)
def updateFieldCache(self, nids): "Update field checksums and sort cache, after find&replace, etc." snids = ids2str(nids) r = [] for (nid, mid, flds) in self._fieldData(snids): fields = splitFields(flds) model = self.models.get(mid) if not model: # note points to invalid model continue r.append((stripHTMLMedia(fields[self.models.sortIdx(model)]), fieldChecksum(fields[0]), nid)) # apply, relying on calling code to bump usn+mod self.db.executemany("update notes set sfld=?, csum=? where id=?", r)
def _findDupes(self, args): # caller must call stripHTMLMedia on passed val (val, args) = args try: mid, val = val.split(",", 1) except OSError: return csum = fieldChecksum(val) nids = [] for nid, flds in self.col.db.execute( "select id, flds from notes where mid=? and csum=?", mid, csum): if stripHTMLMedia(splitFields(flds)[0]) == val: nids.append(nid) return "n.id in %s" % ids2str(nids)
def _renderQA(self, data, qfmt=None, afmt=None): "Returns hash of id, question, answer." # data is [cid, nid, mid, did, ord, tags, flds, cardFlags] # unpack fields and create dict flist = splitFields(data[6]) fields = {} model = self.models.get(data[2]) for (name, (idx, conf)) in list(self.models.fieldMap(model).items()): fields[name] = flist[idx] fields['Tags'] = data[5].strip() fields['Type'] = model['name'] fields['Deck'] = self.decks.name(data[3]) fields['Subdeck'] = fields['Deck'].split('::')[-1] fields['CardFlag'] = self._flagNameFromCardFlags(data[7]) if model['type'] == MODEL_STD: template = model['tmpls'][data[4]] else: template = model['tmpls'][0] fields['Card'] = template['name'] fields['c%d' % (data[4] + 1)] = "1" # render q & a d = dict(id=data[0]) qfmt = qfmt or template['qfmt'] afmt = afmt or template['afmt'] for (type, format) in (("q", qfmt), ("a", afmt)): if type == "q": format = re.sub("{{(?!type:)(.*?)cloze:", r"{{\1cq-%d:" % (data[4] + 1), format) format = format.replace("<%cloze:", "<%%cq:%d:" % (data[4] + 1)) else: format = re.sub("{{(.*?)cloze:", r"{{\1ca-%d:" % (data[4] + 1), format) format = format.replace("<%cloze:", "<%%ca:%d:" % (data[4] + 1)) fields['FrontSide'] = stripSounds(d['q']) fields = runFilter("mungeFields", fields, model, data, self) html = anki_lib.template.render(format, fields) d[type] = runFilter("mungeQA", html, type, fields, model, data, self) # empty cloze? if type == 'q' and model['type'] == MODEL_CLOZE: if not self.models._availClozeOrds(model, data[6], False): d['q'] += ("<p>" + _( "Please edit this note and add some cloze deletions. (%s)" ) % ("<a href=%s#cloze>%s</a>" % (HELP_SITE, _("help")))) return d
def _availClozeOrds(self, m, flds, allowEmpty=True): sflds = splitFields(flds) map = self.fieldMap(m) ords = set() matches = re.findall("{{[^}]*?cloze:(?:[^}]?:)*(.+?)}}", m['tmpls'][0]['qfmt']) matches += re.findall("<%cloze:(.+?)%>", m['tmpls'][0]['qfmt']) for fname in matches: if fname not in map: continue ord = map[fname][0] ords.update([int(m)-1 for m in re.findall( r"(?s){{c(\d+)::.+?}}", sflds[ord])]) if -1 in ords: ords.remove(-1) if not ords and allowEmpty: # empty clozes use first ord return [0] return list(ords)
def _changeNotes(self, nids, newModel, map): d = [] nfields = len(newModel['flds']) for (nid, flds) in self.col.db.execute( "select id, flds from notes where id in "+ids2str(nids)): newflds = {} flds = splitFields(flds) for old, new in list(map.items()): newflds[new] = flds[old] flds = [] for c in range(nfields): flds.append(newflds.get(c, "")) flds = joinFields(flds) d.append(dict(nid=nid, flds=flds, mid=newModel['id'], m=intTime(),u=self.col.usn())) self.col.db.executemany( "update notes set flds=:flds,mid=:mid,mod=:m,usn=:u where id = :nid", d) self.col.updateFieldCache(nids)
def doExport(self, file): cardIds = self.cardIds() data = [] for id, flds, tags in self.col.db.execute(""" select guid, flds, tags from notes where id in (select nid from cards where cards.id in %s)""" % ids2str(cardIds)): row = [] # note id if self.includeID: row.append(str(id)) # fields row.extend([self.processText(f) for f in splitFields(flds)]) # tags if self.includeTags: row.append(tags.strip()) data.append("\t".join(row)) self.count = len(data) out = "\n".join(data) file.write(out.encode("utf-8"))
def findDupes(col, fieldName, search=""): # limit search to notes with applicable field name if search: search = "(" + search + ") " search += "'%s:*'" % fieldName # go through notes vals = {} dupes = [] fields = {} def ordForMid(mid): if mid not in fields: model = col.models.get(mid) for c, f in enumerate(model['flds']): if f['name'].lower() == fieldName.lower(): fields[mid] = c break return fields[mid] for nid, mid, flds in col.db.all( "select id, mid, flds from notes where id in " + ids2str(col.findNotes(search))): flds = splitFields(flds) ord = ordForMid(mid) if ord is None: continue val = flds[ord] val = stripHTMLMedia(val) # empty does not count as duplicate if not val: continue if val not in vals: vals[val] = [] vals[val].append(nid) if len(vals[val]) == 2: dupes.append((val, vals[val])) return dupes
def importNotes(self, notes): "Convert each card into a note, apply attributes and add to col." assert self.mappingOk() # note whether tags are mapped self._tagsMapped = False for f in self.mapping: if f == "_tags": self._tagsMapped = True # gather checks for duplicate comparison csums = {} for csum, id in self.col.db.execute( "select csum, id from notes where mid = ?", self.model['id']): if csum in csums: csums[csum].append(id) else: csums[csum] = [id] firsts = {} fld0idx = self.mapping.index(self.model['flds'][0]['name']) self._fmap = self.col.models.fieldMap(self.model) self._nextID = timestampID(self.col.db, "notes") # loop through the notes updates = [] updateLog = [] updateLogTxt = _("First field matched: %s") dupeLogTxt = _("Added duplicate with first field: %s") new = [] self._ids = [] self._cards = [] self._emptyNotes = False dupeCount = 0 dupes = [] for n in notes: for c in range(len(n.fields)): if not self.allowHTML: n.fields[c] = html.escape(n.fields[c], quote=False) n.fields[c] = n.fields[c].strip() if not self.allowHTML: n.fields[c] = n.fields[c].replace("\n", "<br>") n.fields[c] = unicodedata.normalize("NFC", n.fields[c]) n.tags = [unicodedata.normalize("NFC", t) for t in n.tags] fld0 = n.fields[fld0idx] csum = fieldChecksum(fld0) # first field must exist if not fld0: self.log.append( _("Empty first field: %s") % " ".join(n.fields)) continue # earlier in import? if fld0 in firsts and self.importMode != 2: # duplicates in source file; log and ignore self.log.append(_("Appeared twice in file: %s") % fld0) continue firsts[fld0] = True # already exists? found = False if csum in csums: # csum is not a guarantee; have to check for id in csums[csum]: flds = self.col.db.scalar( "select flds from notes where id = ?", id) sflds = splitFields(flds) if fld0 == sflds[0]: # duplicate found = True if self.importMode == 0: data = self.updateData(n, id, sflds) if data: updates.append(data) updateLog.append(updateLogTxt % fld0) dupeCount += 1 found = True elif self.importMode == 1: dupeCount += 1 elif self.importMode == 2: # allow duplicates in this case if fld0 not in dupes: # only show message once, no matter how many # duplicates are in the collection already updateLog.append(dupeLogTxt % fld0) dupes.append(fld0) found = False # newly add if not found: data = self.newData(n) if data: new.append(data) # note that we've seen this note once already firsts[fld0] = True self.addNew(new) self.addUpdates(updates) # make sure to update sflds, etc self.col.updateFieldCache(self._ids) # generate cards if self.col.genCards(self._ids): self.log.insert( 0, _("Empty cards found. Please run Tools>Empty Cards.")) # apply scheduling updates self.updateCards() # we randomize or order here, to ensure that siblings # have the same due# did = self.col.decks.selected() conf = self.col.decks.confForDid(did) # in order due? if conf['new']['order'] == NEW_CARDS_RANDOM: self.col.sched.randomizeCards(did) part1 = ngettext("%d note added", "%d notes added", len(new)) % len(new) part2 = ngettext("%d note updated", "%d notes updated", self.updateCount) % self.updateCount if self.importMode == 0: unchanged = dupeCount - self.updateCount elif self.importMode == 1: unchanged = dupeCount else: unchanged = 0 part3 = ngettext("%d note unchanged", "%d notes unchanged", unchanged) % unchanged self.log.append("%s, %s, %s." % (part1, part2, part3)) self.log.extend(updateLog) if self._emptyNotes: self.log.append( _("""\ One or more notes were not imported, because they didn't generate any cards. \ This can happen when you have empty fields or when you have not mapped the \ content in the text file to the correct fields.""")) self.total = len(self._ids)