def _findField(self, field, val): field = field.lower() val = val.replace("*", "%") # find models that have that field mods = {} for m in self.col.models.all(): for f in m['flds']: if f['name'].lower() == field: mods[str(m['id'])] = (m, f['ord']) if not mods: # nothing has that field return # gather nids regex = val.replace("_", ".").replace("%", ".*") nids = [] for (id,mid,flds) in self.col.db.execute(""" select id, mid, flds from notes where mid in %s and flds like ? escape '\\'""" % ( ids2str(mods.keys())), "%"+val+"%"): flds = splitFields(flds) ord = mods[str(mid)][1] strg = flds[ord] try: if re.search("(?i)^"+regex+"$", strg): nids.append(id) except sre_constants.error: return if not nids: return return "n.id in %s" % ids2str(nids)
def get_notes_field(col, field, query=""): """Returns a list of (note-id, field-contents) tuples for all notes whose type contains field, restricted by an optional query.""" models = field_map(col, field) if not models: return [] # The following was mostly copied from anki.find.findNotes finder = anki.find.Finder(col) tokens = finder._tokenize(query) preds, args = finder._where(tokens) if preds is None: return [] if preds: preds = "(" + preds + ")" else: preds = "1" sql = """select distinct n.id, n.mid, n.flds from cards c join notes n on c.nid=n.id where n.mid in %s and %s""" % ( ids2str(models.keys()), preds) res = col.db.all(sql, *args) return [ (id, splitFields(flds)[models[mid]]) for id, mid, flds in res ]
def fieldUnique(self, name): (ord, conf) = self._fmap[name] if not conf['uniq']: return True val = self[name] if not val: return True csum = fieldChecksum(val) if self.id: lim = "and fid != :fid" else: lim = "" fids = self.deck.db.list( "select fid from fsums where csum = ? and fid != ? and mid = ?", csum, self.id or 0, self.mid) if not fids: return True # grab facts with the same checksums, and see if they're actually # duplicates for flds in self.deck.db.list("select flds from facts where id in "+ ids2str(fids)): fields = splitFields(flds) if fields[ord] == val: return False return True
def availOrds(self, m, flds): "Given a joined field string, return available template ordinals." if m['type'] == MODEL_CLOZE: return self._availClozeOrds(m, flds) fields = {} for c, f in enumerate(splitFields(flds)): fields[c] = f.strip() avail = [] for ord, type, req in m['req']: # unsatisfiable template if type == "none": continue # AND requirement? elif type == "all": ok = True for idx in req: if not fields[idx]: # missing and was required ok = False break if not ok: continue # OR requirement? elif type == "any": ok = False for idx in req: if fields[idx]: ok = True break if not ok: continue avail.append(ord) return avail
def _transformFields(self, fn): self.deck.modSchema() r = [] for (id, flds) in self.deck.db.execute( "select id, flds from facts where mid = ?", self.id): r.append((joinFields(fn(splitFields(flds))), id)) self.deck.db.executemany("update facts set flds = ? where id = ?", r)
def findDupes(col, fieldName, search=""): # limit search to notes with applicable field name if search: search = "("+search+") " search += "'%s:*'" % fieldName # go through notes vals = {} dupes = [] fields = {} def ordForMid(mid): if mid not in fields: model = col.models.get(mid) for c, f in enumerate(model['flds']): if f['name'].lower() == fieldName.lower(): fields[mid] = c break return fields[mid] for nid, mid, flds in col.db.all( "select id, mid, flds from notes where id in "+ids2str( col.findNotes(search))): flds = splitFields(flds) ord = ordForMid(mid) if ord is None: continue val = flds[ord] val = stripHTMLMedia(val) # empty does not count as duplicate if not val: continue if val not in vals: vals[val] = [] vals[val].append(nid) if len(vals[val]) == 2: dupes.append((val, vals[val])) return dupes
def _findField(self, token, isNeg): field = value = '' parts = token.split(':', 1); field = parts[0].lower() value = "%" + parts[1].replace("*", "%") + "%" # find models that have that field mods = {} for m in self.col.models.all(): for f in m['flds']: if f['name'].lower() == field: mods[str(m['id'])] = (m, f['ord']) if not mods: # nothing has that field self.lims['valid'] = False return # gather nids regex = value.replace("_", ".").replace("%", ".*") nids = [] for (id,mid,flds) in self.col.db.execute(""" select id, mid, flds from notes where mid in %s and flds like ? escape '\\'""" % ( ids2str(mods.keys())), "%" if self.full else value): flds = splitFields(flds) ord = mods[str(mid)][1] strg = flds[ord] if self.full: strg = stripHTML(strg) if re.search(regex, strg): nids.append(id) extra = "not" if isNeg else "" self.lims['preds'].append(""" n.mid in %s and n.id %s in %s""" % ( ids2str(mods.keys()), extra, ids2str(nids)))
def findDupes(col, fieldName, search=""): # limit search to notes with applicable field name if search: search = "("+search+") " search += "'%s:*'" % fieldName # go through notes vals = {} dupes = [] fields = {} def ordForMid(mid): if mid not in fields: model = col.models.get(mid) fields[mid] = col.models.fieldMap(model)[fieldName][0] return fields[mid] for nid, mid, flds in col.db.all( "select id, mid, flds from notes where id in "+ids2str( col.findNotes(search))): flds = splitFields(flds) val = flds[ordForMid(mid)] # empty does not count as duplicate if not val: continue if val not in vals: vals[val] = [] vals[val].append(nid) if len(vals[val]) == 2: dupes.append((val, vals[val])) return dupes
def _renderQA(self, data): "Returns hash of id, question, answer." # data is [cid, nid, mid, did, ord, tags, flds] # unpack fields and create dict flist = splitFields(data[6]) fields = {} model = self.models.get(data[2]) for (name, (idx, conf)) in self.models.fieldMap(model).items(): fields[name] = flist[idx] fields['Tags'] = data[5] fields['Type'] = model['name'] fields['Deck'] = self.decks.name(data[3]) template = model['tmpls'][data[4]] fields['Card'] = template['name'] # render q & a d = dict(id=data[0]) for (type, format) in (("q", template['qfmt']), ("a", template['afmt'])): if type == "q": format = format.replace("cloze:", "cq:") else: format = format.replace("cloze:", "ca:") fields = runFilter("mungeFields", fields, model, data, self) html = anki.template.render(format, fields) d[type] = runFilter( "mungeQA", html, type, fields, model, data, self) return d
def _findField(self, token, isNeg): field = value = '' parts = token.split(':', 1); field = parts[0].lower() value = "%" + parts[1].replace("*", "%") + "%" # find models that have that field mods = {} for m in self.deck.models().values(): for f in m.fields: if f['name'].lower() == field: mods[m.id] = (m, f['ord']) if not mods: # nothing has that field self.lims['valid'] = False return # gather fids regex = value.replace("%", ".*") fids = [] for (id,mid,flds) in self.deck.db.execute(""" select id, mid, flds from facts where mid in %s and flds like ? escape '\\'""" % ( ids2str(mods.keys())), "%" if self.full else value): flds = splitFields(flds) ord = mods[mid][1] str = flds[ord] if self.full: str = stripHTML(str) if re.search(regex, str): fids.append(id) extra = "not" if isNeg else "" self.lims['fact'].append("id %s in %s" % (extra, ids2str(fids)))
def _mungeMedia(self, mid, fields): fields = splitFields(fields) def repl(match): fname = match.group("fname") srcData = self._srcMediaData(fname) dstData = self._dstMediaData(fname) if not srcData: # file was not in source, ignore return match.group(0) # if model-local file exists from a previous import, use that name, ext = os.path.splitext(fname) lname = "%s_%s%s" % (name, mid, ext) if self.dst.media.have(lname): return match.group(0).replace(fname, lname) # if missing or the same, pass unmodified elif not dstData or srcData == dstData: # need to copy? if not dstData: self._writeDstMedia(fname, srcData) return match.group(0) # exists but does not match, so we need to dedupe self._writeDstMedia(lname, srcData) return match.group(0).replace(fname, lname) for i in range(len(fields)): fields[i] = self.dst.media.transformNames(fields[i], repl) return joinFields(fields)
def _renderQA(self, model, gname, data): "Returns hash of id, question, answer." # data is [cid, fid, mid, gid, ord, tags, flds] # unpack fields and create dict flist = splitFields(data[6]) fields = {} for (name, (idx, conf)) in model.fieldMap().items(): fields[name] = flist[idx] if fields[name]: fields[name] = '<span class="fm%s-%s">%s</span>' % ( hexifyID(data[2]), hexifyID(idx), fields[name]) else: fields[name] = "" fields['Tags'] = data[5] fields['Model'] = model.name fields['Group'] = gname template = model.templates[data[4]] fields['Template'] = template['name'] # render q & a d = dict(id=data[0]) for (type, format) in (("q", template['qfmt']), ("a", template['afmt'])): if type == "q": format = format.replace("cloze:", "cq:") else: if model.conf['clozectx']: name = "cactx:" else: name = "ca:" format = format.replace("cloze:", name) fields = runFilter("mungeFields", fields, model, gname, data, self) html = anki.template.render(format, fields) d[type] = runFilter( "mungeQA", html, type, fields, model, gname, data, self) return d
def extractFieldData( fname, flds, mid ): """ :type str fname: The field name (like u'Expression') :type str flds: A string containing all field data for the model (created by anki.utils.joinFields()) :type int mid: the modelId depicing the model for the "flds" data """ idx = getFieldIndex( fname, mid ) return stripHTML( splitFields( flds )[ idx ] )
def _transformFields(self, m, fn): # model hasn't been added yet? if not m["id"]: return r = [] for (id, flds) in self.col.db.execute("select id, flds from notes where mid = ?", m["id"]): r.append((joinFields(fn(splitFields(flds))), intTime(), self.col.usn(), id)) self.col.db.executemany("update notes set flds=?,mod=?,usn=? where id = ?", r)
def get_evernote_guid_from_anki_fields(fields): if isinstance(fields, dict): if not FIELDS.EVERNOTE_GUID in fields: return None return fields[FIELDS.EVERNOTE_GUID].replace(FIELDS.EVERNOTE_GUID_PREFIX, '') if is_str(fields): fields = splitFields(fields) return fields[FIELDS.ORD.EVERNOTE_GUID].replace(FIELDS.EVERNOTE_GUID_PREFIX, '')
def isDupe(self, data): "Takes field, model and returns True if the field is a dupe and False otherwise." # find any matching csums and compare csum = fieldChecksum(data["field"]) mid = mw.col.models.byName(data["model"])["id"] for flds in mw.col.db.list( "select flds from notes where csum = ? and id != ? and mid = ?", csum, 0, mid): if splitFields(flds)[0] == data["field"]: return True return False
def updateFieldCache(self, nids): "Update field checksums and sort cache, after find&replace, etc." snids = ids2str(nids) r = [] for (nid, mid, flds) in self._fieldData(snids): fields = splitFields(flds) model = self.models.get(mid) r.append((stripHTML(fields[self.models.sortIdx(model)]), fieldChecksum(fields[0]), nid)) # apply, relying on calling code to bump usn+mod self.db.executemany("update notes set sfld=?, csum=? where id=?", r)
def onRemNotes(self, col, nids): path = os.path.join(self.pm.profileFolder(), "deleted.txt") existed = os.path.exists(path) with open(path, "ab") as f: if not existed: f.write(b"nid\tmid\tfields\n") for id, mid, flds in col.db.execute( "select id, mid, flds from notes where id in %s" % ids2str(nids)): fields = splitFields(flds) f.write(("\t".join([str(id), str(mid)] + fields)).encode("utf8")) f.write(b"\n")
def load(self): (self.guid, self.mid, self.mod, self.usn, self.tags, self.fields, self.flags, self.data) = self.col.db.first( """ select guid, mid, mod, usn, tags, flds, flags, data from notes where id = ?""", self.id, ) self.fields = splitFields(self.fields) self.tags = self.col.tags.split(self.tags) self._model = self.col.models.get(self.mid) self._fmap = self.col.models.fieldMap(self._model) self.scm = self.col.scm
def dupeOrEmpty(self): "1 if first is empty; 2 if first is a duplicate, False otherwise." val = self.fields[0] if not val.strip(): return 1 csum = fieldChecksum(val) # find any matching csums and compare for flds in self.col.db.list( "select flds from notes where csum = ? and id != ? and mid = ?", csum, self.id or 0, self.mid): if splitFields(flds)[0] == self.fields[0]: return 2 return False
def load(self): """Given a note knowing its collection and its id, choosing this card from the database.""" (self.guid, self.mid, self.mod, self.usn, self.tags, self.fields, self.flags, self.data) = self.col.db.first( """ select guid, mid, mod, usn, tags, flds, flags, data from notes where id = ?""", self.id) self.fields = splitFields(self.fields) self.tags = self.col.tags.split(self.tags) self._model = self.col.models.get(self.mid) self._fmap = self.col.models.fieldMap(self._model) self.scm = self.col.scm
def load(self): (self.mid, self.gid, self.crt, self.mod, self.tags, self.fields, self.data) = self.deck.db.first(""" select mid, gid, crt, mod, tags, flds, data from facts where id = ?""", self.id) self.fields = splitFields(self.fields) self.tags = parseTags(self.tags) self._model = self.deck.getModel(self.mid) self._fmap = self._model.fieldMap()
def getNotes2Sync(self): """gets all notes with fields that were changed after their last import""" xMid = xModelId(self.mw.col) xNotes = list( self.mw.col.db.execute( "select id, mod, flds from notes where mid = %s" % xMid)) for xNote in xNotes: fields = splitFields(xNote[2]) meta = json.loads(fields[23]) # If the last time the note was edited was at least 10 Seconds after it was imported if xNote[1] > (meta['lastSync'] + 10): self.notes2Sync.append( dict(meta=meta, fields=fields, nid=xNote[0]))
def _availClozeOrds(self, m, flds): sflds = splitFields(flds) map = self.fieldMap(m) ords = set() for fname in re.findall("{{cloze:(.+?)}}", m['tmpls'][0]['qfmt']): if fname not in map: continue ord = map[fname][0] ords.update([int(m)-1 for m in re.findall( "{{c(\d+)::[^}]*?}}", sflds[ord])]) if -1 in ords: ords.remove(-1) return list(ords)
def _renderQA(self, data: QAData, qfmt: None = None, afmt: None = None) -> Dict: "Returns hash of id, question, answer." # data is [cid, nid, mid, did, ord, tags, flds, cardFlags] # unpack fields and create dict flist = splitFields(data[6]) fields = {} model = self.models.get(data[2]) assert model for (name, (idx, conf)) in list(self.models.fieldMap(model).items()): fields[name] = flist[idx] fields["Tags"] = data[5].strip() fields["Type"] = model["name"] fields["Deck"] = self.decks.name(data[3]) fields["Subdeck"] = fields["Deck"].split("::")[-1] fields["CardFlag"] = self._flagNameFromCardFlags(data[7]) if model["type"] == MODEL_STD: template = model["tmpls"][data[4]] else: template = model["tmpls"][0] fields["Card"] = template["name"] fields["c%d" % (data[4] + 1)] = "1" # render q & a d: Dict[str, Any] = dict(id=data[0]) qfmt = qfmt or template["qfmt"] afmt = afmt or template["afmt"] for (type, format) in (("q", qfmt), ("a", afmt)): if type == "q": format = re.sub("{{(?!type:)(.*?)cloze:", r"{{\1cq-%d:" % (data[4] + 1), format) format = format.replace("<%cloze:", "<%%cq:%d:" % (data[4] + 1)) else: format = re.sub("{{(.*?)cloze:", r"{{\1ca-%d:" % (data[4] + 1), format) format = format.replace("<%cloze:", "<%%ca:%d:" % (data[4] + 1)) fields["FrontSide"] = stripSounds(d["q"]) fields = runFilter("mungeFields", fields, model, data, self) html = anki.template.render(format, fields) d[type] = runFilter("mungeQA", html, type, fields, model, data, self) # empty cloze? if type == "q" and model["type"] == MODEL_CLOZE: if not self.models._availClozeOrds(model, data[6], False): d["q"] += "<p>" + _( "Please edit this note and add some cloze deletions. (%s)" ) % ("<a href=%s#cloze>%s</a>" % (HELP_SITE, _("help"))) return d
def findReplace(col, nids, src, dst, regex=False, field=None, fold=True) -> int: "Find and replace fields in a note." mmap = {} if field: for m in col.models.all(): for f in m["flds"]: if f["name"].lower() == field.lower(): mmap[str(m["id"])] = f["ord"] if not mmap: return 0 # find and gather replacements if not regex: src = re.escape(src) dst = dst.replace("\\", "\\\\") if fold: src = "(?i)" + src regex = re.compile(src) def repl(str): return re.sub(regex, dst, str) d = [] snids = ids2str(nids) nids = [] for nid, mid, flds in col.db.execute( "select id, mid, flds from notes where id in " + snids ): origFlds = flds # does it match? sflds = splitFields(flds) if field: try: ord = mmap[str(mid)] sflds[ord] = repl(sflds[ord]) except KeyError: # note doesn't have that field continue else: for c in range(len(sflds)): sflds[c] = repl(sflds[c]) flds = joinFields(sflds) if flds != origFlds: nids.append(nid) d.append(dict(nid=nid, flds=flds, u=col.usn(), m=intTime())) if not d: return 0 # replace col.db.executemany("update notes set flds=:flds,mod=:m,usn=:u where id=:nid", d) col.updateFieldCache(nids) col.genCards(nids) return len(d)
def _findDupes(self, val): # caller must call stripHTMLMedia on passed val try: mid, val = val.split(",", 1) except OSError: return csum = fieldChecksum(val) nids = [] for nid, flds in self.col.db.execute( "select id, flds from notes where mid=? and csum=?", mid, csum): if stripHTMLMedia(splitFields(flds)[0]) == val: nids.append(nid) return "n.id in %s" % ids2str(nids)
def __init__(self, collection, config, added_contents): self.collection = collection self.model = self.collection.models.byName(config['model_name']) self._find_field_indexes(config) self.added_normalized_contents = set( map(normalized_content, added_contents)) note_fields = self.collection.db.list( 'select flds from notes where mid = ?', self.model['id']) self.present_normalized_contents = { normalized_content(splitFields(fields)[self.content_field_index]) for fields in note_fields }
def updateFieldCache(self, nids): "Update field checksums and sort cache, after find&replace, etc." snids = ids2str(nids) r = [] for (nid, mid, flds) in self._fieldData(snids): fields = splitFields(flds) model = self.models.get(mid) if not model: # note points to invalid model continue r.append((stripHTMLMedia(fields[self.models.sortIdx(model)]), fieldChecksum(fields[0]), nid)) # apply, relying on calling code to bump usn+mod self.db.executemany("update notes set sfld=?, csum=? where id=?", r)
def _availClozeOrds(self, m, flds, allowEmpty=True): sflds = splitFields(flds) map = self.fieldMap(m) ords = set() for fname in re.findall("{{cloze:(.+?)}}", m["tmpls"][0]["qfmt"]): if fname not in map: continue ord = map[fname][0] ords.update([int(m) - 1 for m in re.findall("{{c(\d+)::.+?}}", sflds[ord])]) if -1 in ords: ords.remove(-1) if not ords and allowEmpty: # empty clozes use first ord return [0] return list(ords)
def _transformFields(self, m, fn): """For each note of the model m, apply m to the set of field's value, and save the note modified. fn -- a function taking and returning a list of field.""" # model hasn't been added yet? if not m['id']: return r = [] for (id, flds) in self.col.db.execute( "select id, flds from notes where mid = ?", m['id']): r.append((joinFields(fn(splitFields(flds))), intTime(), self.col.usn(), id)) self.col.db.executemany( "update notes set flds=?,mod=?,usn=? where id = ?", r)
def _changeNotes(self, nids, newModel, map): d = [] nfields = len(newModel["flds"]) for (nid, flds) in self.col.db.execute("select id, flds from notes where id in " + ids2str(nids)): newflds = {} flds = splitFields(flds) for old, new in map.items(): newflds[new] = flds[old] flds = [] for c in range(nfields): flds.append(newflds.get(c, "")) flds = joinFields(flds) d.append(dict(nid=nid, flds=flds, mid=newModel["id"], m=intTime(), u=self.col.usn())) self.col.db.executemany("update notes set flds=:flds,mid=:mid,mod=:m,usn=:u where id = :nid", d) self.col.updateFieldCache(nids)
def makeTOC(chapId, focusNid=-1): global currentChap global relatedChapters # On recupere le modele du sommaire (les titres des differentes parties) (chapitre, noteType, partsRaw) = mw.col.db.execute( "SELECT chapitre, noteType, toc FROM `CHAP.chapters` WHERE id=%d" % (chapId)).fetchone() notes = {} for n in noteType.split('\n'): infos = n.split('::') notes[int(infos[0])] = int(infos[2]) parts = partsRaw.split("\n") html = "<ul>" i = 1 for p in parts: html += "<li><h2>%s</h2><ul>" % (p) # Enfin, on recupere toutes les notes qui sont dans cette partie for nid, mid, flds in mw.col.db.execute(""" SELECT N.id, N.mid, N.flds FROM notes AS N JOIN `CHAP.toc`AS T ON T.noteId = N.id WHERE chapId=%d AND part=%d ORDER BY position""" % (chapId, i)): fields = splitFields(flds) span = "<span>" if (nid == focusNid): span = "<span id='currentCard'>" html += """<li>%s<a onclick="py.link('%s');">%s</a></span></li>""" % ( span, nid, fields[notes[int(mid)]]) html += "</ul></li>" i += 1 html += "</ul>" (header, JS_load) = makeHeader(chapId) currentChap = chapId utils.sideWidgets["toc"].update(""" border-width:2px; border-style:solid; margin:10px;""", """ %s <h1><u><i>Sommaire</i> : %s</u></h1><br>%s""" % (header, chapitre, html), JS=JS_load) # On recupere egalements tous les chapitres en realtions relatedChapters = [] for id, cha, nt in mw.col.db.execute( """ SELECT C.id, C.chapitre, C.noteType FROM `CHAP.chapters` AS C""" ): if str(nt) == str(noteType): relatedChapters.append((cha, id))
def rewriteRef(key): all = match.group(0) fname = match.group("fname") if all in state['mflds']: # we've converted this field before new = state['mflds'][all] else: # get field name and any prefix/suffix m2 = re.match( "([^{]*)\{\{\{?(?:text:)?([^}]+)\}\}\}?(.*)", fname) # not a field reference? if not m2: return pre, ofld, suf = m2.groups() # get index of field name try: idx = col.models.fieldMap(m)[ofld][0] except: # invalid field or tag reference; don't rewrite return # find a free field name while 1: state['fields'] += 1 fld = "Media %d" % state['fields'] if fld not in col.models.fieldMap(m).keys(): break # add the new field f = col.models.newField(fld) f['qsize'] = 20 f['qcol'] = '#000' col.models.addField(m, f) # loop through notes and write reference into new field data = [] for id, flds in self.col.db.execute( "select id, flds from notes where id in "+ ids2str(col.models.nids(m))): sflds = splitFields(flds) ref = all.replace(fname, pre+sflds[idx]+suf) data.append((flds+ref, id)) # update notes col.db.executemany("update notes set flds=? where id=?", data) # note field for future state['mflds'][fname] = fld new = fld # rewrite reference in template t[key] = t[key].replace(all, "{{{%s}}}" % new)
def findReplace(col, nids, src, dst, regex=False, field=None, fold=True): "Find and replace fields in a note." mmap = {} if field: for m in col.models.all(): for f in m['flds']: if f['name'] == field: mmap[str(m['id'])] = f['ord'] if not mmap: return 0 # find and gather replacements if not regex: src = re.escape(src) if fold: src = "(?i)"+src regex = re.compile(src) def repl(str): return re.sub(regex, dst, str) d = [] snids = ids2str(nids) nids = [] for nid, mid, flds in col.db.execute( "select id, mid, flds from notes where id in "+snids): origFlds = flds # does it match? sflds = splitFields(flds) if field: try: ord = mmap[str(mid)] sflds[ord] = repl(sflds[ord]) except KeyError: # note doesn't have that field continue else: for c in range(len(sflds)): sflds[c] = repl(sflds[c]) flds = joinFields(sflds) if flds != origFlds: nids.append(nid) d.append(dict(nid=nid,flds=flds,u=col.usn(),m=intTime())) if not d: return 0 # replace col.db.executemany( "update notes set flds=:flds,mod=:m,usn=:u where id=:nid", d) col.updateFieldCache(nids) col.genCards(nids) return len(d)
def load(self): (self.guid, self.mid, self.mod, self.usn, self.tags, self.fields, self.flags, self.data) = self.col.db.first(""" select guid, mid, mod, usn, tags, flds, flags, data from notes where id = ?""", self.id) self.fields = splitFields(self.fields) self.tags = self.col.tags.split(self.tags) self._model = self.col.models.get(self.mid) self._fmap = self.col.models.fieldMap(self._model) self.scm = self.col.scm
def rewriteRef(key): all = match.group(0) fname = match.group("fname") if all in state['mflds']: # we've converted this field before new = state['mflds'][all] else: # get field name and any prefix/suffix m2 = re.match("([^{]*)\{\{\{?(?:text:)?([^}]+)\}\}\}?(.*)", fname) # not a field reference? if not m2: return pre, ofld, suf = m2.groups() # get index of field name try: idx = self.col.models.fieldMap(m)[ofld][0] except: # invalid field or tag reference; don't rewrite return # find a free field name while 1: state['fields'] += 1 fld = "Media %d" % state['fields'] if fld not in list(self.col.models.fieldMap(m).keys()): break # add the new field f = self.col.models.newField(fld) f['qsize'] = 20 f['qcol'] = '#000' self.col.models.addField(m, f) # loop through notes and write reference into new field data = [] for id, flds in self.col.db.execute( "select id, flds from notes where id in " + ids2str(self.col.models.nids(m))): sflds = splitFields(flds) ref = all.replace(fname, pre + sflds[idx] + suf) data.append((flds + ref, id)) # update notes self.col.db.executemany("update notes set flds=? where id=?", data) # note field for future state['mflds'][fname] = fld new = fld # rewrite reference in template t[key] = t[key].replace(all, "{{{%s}}}" % new)
def myRenderQA(self, data, qfmt=None, afmt=None): "Returns hash of id, question, answer." # data is [cid, nid, mid, did, ord, tags, flds, flags] # unpack fields and create dict flist = splitFields(data[6]) fields = {} model = self.models.get(data[2]) for (name, (idx, conf)) in self.models.fieldMap(model).items(): fields[name] = flist[idx] if len(data) > 7: shakyWarningFieldName = _getShakyWarningFieldName(data[7]) if shakyWarningFieldName: fields[shakyWarningFieldName] = shakyWarningFieldName fields['Tags'] = data[5].strip() fields['Type'] = model['name'] fields['Deck'] = self.decks.name(data[3]) if model['type'] == MODEL_STD: template = model['tmpls'][data[4]] else: template = model['tmpls'][0] fields['Card'] = template['name'] fields['c%d' % (data[4]+1)] = "1" # render q & a d = dict(id=data[0]) qfmt = qfmt or template['qfmt'] afmt = afmt or template['afmt'] for (type, format) in (("q", qfmt), ("a", afmt)): if type == "q": format = re.sub("{{(?!type:)(.*?)cloze:", r"{{\1cq-%d:" % (data[4]+1), format) format = format.replace("<%cloze:", "<%%cq:%d:" % ( data[4]+1)) else: format = re.sub("{{(.*?)cloze:", r"{{\1ca-%d:" % (data[4]+1), format) format = format.replace("<%cloze:", "<%%ca:%d:" % ( data[4]+1)) fields['FrontSide'] = stripSounds(d['q']) fields = runFilter("mungeFields", fields, model, data, self) html = anki.template.render(format, fields) d[type] = runFilter( "mungeQA", html, type, fields, model, data, self) # empty cloze? if type == 'q' and model['type'] == MODEL_CLOZE: if not self.models._availClozeOrds(model, data[6], False): d['q'] += ("<p>" + _( "Please edit this note and add some cloze deletions. (%s)") % ( "<a href=%s#cloze>%s</a>" % (HELP_SITE, _("help")))) return d
def _changeFacts(self, fids, newModel, map): d = [] nfields = len(newModel.fields) for (fid, flds) in self.deck.db.execute( "select id, flds from facts where id in " + ids2str(fids)): newflds = {} flds = splitFields(flds) for old, new in map.items(): newflds[new] = flds[old] flds = [] for c in range(nfields): flds.append(newflds.get(c, "")) flds = joinFields(flds) d.append(dict(fid=fid, flds=flds, mid=newModel.id)) self.deck.db.executemany( "update facts set flds=:flds, mid=:mid where id = :fid", d) self.deck.updateFieldCache(fids)
def extract_field(model_id, field_name) -> Iterable[Tuple[int, str]]: # type works better in future anki model = self.models.get(model_id) assert model # type is optional, but None should never come back note_ids = self.findNotes(" ".join(search_filters + [f'note:{model["name"]}'])) field_ord: int = next(field["ord"] for field in model["flds"] if field["name"] == field_name) assert self.db for note_id, fields in self.db.all( "select id, flds from notes where id in " + ids2str(note_ids)): value = splitFields(fields)[field_ord] yield note_id, stripHTMLMedia(value)
def doExport(self, file): writer = csv.writer(file) cardIds = self.cardIds() self.count = 0 for _id, flds, tags in self.col.db.execute(self.db_query % ids2str(cardIds)): row = [] # note id # if self.includeID: # row.append(str(_id)) # fields row.extend([self.escapeText(f) for f in splitFields(flds)]) # tags if self.includeTags: row.append(tags.strip()) self.count += 1 writer.writerow([x for x in row])
def _renderQA(self, data, qfmt=None, afmt=None): "Returns hash of id, question, answer." # data is [cid, nid, mid, did, ord, tags, flds] # unpack fields and create dict flist = splitFields(data[6]) fields = {} model = self.models.get(data[2]) for (name, (idx, conf)) in list(self.models.fieldMap(model).items()): fields[name] = flist[idx] fields['Tags'] = data[5].strip() fields['Type'] = model['name'] fields['Deck'] = self.decks.name(data[3]) fields['Subdeck'] = fields['Deck'].split('::')[-1] if model['type'] == MODEL_STD: template = model['tmpls'][data[4]] else: template = model['tmpls'][0] fields['Card'] = template['name'] fields['c%d' % (data[4] + 1)] = "1" # render q & a d = dict(id=data[0]) qfmt = qfmt or template['qfmt'] afmt = afmt or template['afmt'] for (type, format) in (("q", qfmt), ("a", afmt)): if type == "q": format = re.sub("{{(?!type:)(.*?)cloze:", r"{{\1cq-%d:" % (data[4] + 1), format) format = format.replace("<%cloze:", "<%%cq:%d:" % (data[4] + 1)) else: format = re.sub("{{(.*?)cloze:", r"{{\1ca-%d:" % (data[4] + 1), format) format = format.replace("<%cloze:", "<%%ca:%d:" % (data[4] + 1)) fields['FrontSide'] = stripSounds(d['q']) fields = runFilter("mungeFields", fields, model, data, self) html = anki.template.render(format, fields) d[type] = runFilter("mungeQA", html, type, fields, model, data, self) # empty cloze? if type == 'q' and model['type'] == MODEL_CLOZE: if not self.models._availClozeOrds(model, data[6], False): d['q'] += ("<p>" + _( "Please edit this note and add some cloze deletions. (%s)" ) % ("<a href=%s#cloze>%s</a>" % (HELP_SITE, _("help")))) return d
def _changeFacts(self, fids, newModel, map): d = [] nfields = len(newModel.fields) for (fid, flds) in self.deck.db.execute( "select id, flds from facts where id in "+ids2str(fids)): newflds = {} flds = splitFields(flds) for old, new in map.items(): newflds[new] = flds[old] flds = [] for c in range(nfields): flds.append(newflds.get(c, "")) flds = joinFields(flds) d.append(dict(fid=fid, flds=flds, mid=newModel.id)) self.deck.db.executemany( "update facts set flds=:flds, mid=:mid where id = :fid", d) self.deck.updateFieldCache(fids)
def _availClozeOrds(self, m, flds, allowEmpty=True): sflds = splitFields(flds) map = self.fieldMap(m) ords = set() for fname in re.findall("{{cloze:(.+?)}}", m['tmpls'][0]['qfmt']): if fname not in map: continue ord = map[fname][0] ords.update([ int(m) - 1 for m in re.findall("{{c(\d+)::.+?}}", sflds[ord]) ]) if -1 in ords: ords.remove(-1) if not ords and allowEmpty: # empty clozes use first ord return [0] return list(ords)
def dump_collection(colpath, includetags, includedecknames): sql_attach = 'ATTACH DATABASE "%s" AS db2' % colpath mw.col.db.execute(sql_attach) sql_names_of_tables = """SELECT name FROM sqlite_master WHERE type='table'""" tables = mw.col.db.all(sql_names_of_tables) tags_anon = {} for t in tables: tabname = t[0] sql_copy_over = """INSERT OR REPLACE INTO db2.{0} SELECT * FROM {0};""".format(tabname) try: mw.col.db.execute(sql_copy_over) # e.g. the add-on "symbols as you type" inserts another table into the database, which was # recommended in https://apps.ankiweb.net/docs/addons20.html except: continue else: if tabname == "notes": for nid, flds, tags in mw.col.db.all("""SELECT id, flds, tags from db2.notes"""): flist = splitFields(flds) for i, v in enumerate(flist): flist[i] = "..." # len(v) joined = joinFields(flist) if includetags: tags = "" else: t = tags.strip().split() alist = [] for i in t: if i not in tags_anon: tags_anon[i] = randstr(12) alist.append(tags_anon[i]) tags = ', tags="' + mw.col.tags.join(mw.col.tags.canonify(alist)) + '"' attrs = 'flds = "{}", sfld = ""{}'.format(joined, tags) mw.col.db.execute("UPDATE db2.notes SET {} WHERE id = {}".format(attrs, nid)) if not includedecknames: decks = mw.col.db.first("select decks from db2.col")[0] deckdict = json.loads(decks) old_to_new = decknames_to_anon_dict() for props in deckdict.values(): props['name'] = old_to_new[props['name']] mw.col.db.execute("update db2.col set decks=?", json.dumps(deckdict)) if not includetags: mw.col.db.execute('UPDATE db2.col SET tags = "{}"') # clear tag cache mw.col.db.commit() mw.col.db.execute('DETACH DATABASE db2')
def onRemNotes(self, col, nids): """Append (id, model id and fields) to the end of deleted.txt This is done for each id of nids. This method is added to the hook remNotes; and executed on note deletion. """ path = os.path.join(self.pm.profileFolder(), "deleted.txt") existed = os.path.exists(path) with open(path, "ab") as f: if not existed: f.write(b"nid\tmid\tfields\n") for id, mid, flds in col.db.execute( "select id, mid, flds from notes where id in %s" % ids2str(nids)): fields = splitFields(flds) f.write( ("\t".join([str(id), str(mid)] + fields)).encode("utf8")) f.write(b"\n")
def _availClozeOrds(self, m: NoteType, flds: str, allowEmpty: bool = True) -> List: sflds = splitFields(flds) map = self.fieldMap(m) ords = set() matches = re.findall("{{[^}]*?cloze:(?:[^}]?:)*(.+?)}}", m['tmpls'][0]['qfmt']) matches += re.findall("<%cloze:(.+?)%>", m['tmpls'][0]['qfmt']) for fname in matches: if fname not in map: continue ord = map[fname][0] ords.update([int(m)-1 for m in re.findall( r"(?s){{c(\d+)::.+?}}", sflds[ord])]) if -1 in ords: ords.remove(-1) if not ords and allowEmpty: # empty clozes use first ord return [0] return list(ords)
def _changeNotes(self, nids, newModel, map): d = [] nfields = len(newModel['flds']) for (nid, flds) in self.col.db.execute( "select id, flds from notes where id in "+ids2str(nids)): newflds = {} flds = splitFields(flds) for old, new in list(map.items()): newflds[new] = flds[old] flds = [] for c in range(nfields): flds.append(newflds.get(c, "")) flds = joinFields(flds) d.append(dict(nid=nid, flds=flds, mid=newModel['id'], m=intTime(),u=self.col.usn())) self.col.db.executemany( "update notes set flds=:flds,mid=:mid,mod=:m,usn=:u where id = :nid", d) self.col.updateFieldCache(nids)
def _availClozeOrds(self, m, flds, allowEmpty=True): sflds = splitFields(flds) map = self.fieldMap(m) ords = set() matches = re.findall("{{[^}]*?cloze:(?:[^}]?:)*(.+?)}}", m['tmpls'][0]['qfmt']) matches += re.findall("<%cloze:(.+?)%>", m['tmpls'][0]['qfmt']) for fname in matches: if fname not in map: continue ord = map[fname][0] ords.update([int(m)-1 for m in re.findall( "{{c(\d+)::.+?}}", sflds[ord])]) if -1 in ords: ords.remove(-1) if not ords and allowEmpty: # empty clozes use first ord return [0] return list(ords)
def lilypond_check(self, local=None): files = [] for nid, mid, fields in self.col.db.execute( "select id, mid, flds from notes"): model = self.col.models.get(mid) note = self.col.getNote(nid) data = [None, note.id] flist = splitFields(fields) fields = {} for (name, (idx, conf)) in list(self.col.models.fieldMap(model).items()): fields[name] = flist[idx] (fields, note_files) = mungeFieldsWithFileList(fields, model, data, self.col) files = files + note_files anki_results = anki_check(self, local) files_to_delete = [x for x in anki_results[1] if x not in files] return (anki_results[0], files_to_delete, anki_results[2])
def updateFieldCache(self, fids, csum=True): "Update field checksums and sort cache, after find&replace, etc." sfids = ids2str(fids) mods = self.models() r = [] r2 = [] for (fid, mid, flds) in self._fieldData(sfids): fields = splitFields(flds) model = mods[mid] if csum: for f in model.fields: if f['uniq'] and fields[f['ord']]: r.append((fid, mid, fieldChecksum(fields[f['ord']]))) r2.append((stripHTML(fields[model.sortIdx()]), fid)) if csum: self.db.execute("delete from fsums where fid in "+sfids) self.db.executemany("insert into fsums values (?,?,?)", r) self.db.executemany("update facts set sfld = ? where id = ?", r2)
def availOrds(self, m, flds, changedOrNewReq=None): #oldModel = None, newTemplatesData = None """Given a joined field string, return template ordinals which should be seen. See ../documentation/templates_generation_rules.md for the detail """ if m['type'] == MODEL_CLOZE: return self._availClozeOrds(m, flds) fields = {} for c, f in enumerate(splitFields(flds)): fields[c] = f.strip() avail = [] #List of ord cards which would be generated for tup in m['req']: # print(f"""tup is {tup}. # m['req'] is {m['req']} # m is {m}""") ord, type, req = tup if changedOrNewReq is not None and ord not in changedOrNewReq: continue # unsatisfiable template if type == "none": continue # AND requirement? elif type == "all": ok = True for idx in req: if not fields[idx]: # missing and was required ok = False break if not ok: continue # OR requirement? elif type == "any": ok = False for idx in req: if fields[idx]: ok = True break if not ok: continue avail.append(ord) return avail
def _changeNotes( self, nids: List[int], newModel: NoteType, map: Dict[int, Union[None, int]] ) -> None: d = [] nfields = len(newModel["flds"]) for (nid, flds) in self.col.db.execute( "select id, flds from notes where id in " + ids2str(nids) ): newflds = {} flds = splitFields(flds) for old, new in list(map.items()): newflds[new] = flds[old] flds = [] for c in range(nfields): flds.append(newflds.get(c, "")) flds = joinFields(flds) d.append((flds, newModel["id"], intTime(), self.col.usn(), nid,)) self.col.db.executemany( "update notes set flds=?,mid=?,mod=?,usn=? where id = ?", d )
def findReplace(deck, fids, src, dst, regex=False, field=None, fold=True): "Find and replace fields in a fact." mmap = {} if field: for m in deck.models().values(): for f in m.fields: if f['name'] == field: mmap[m.id] = f['ord'] if not mmap: return 0 # find and gather replacements if not regex: src = re.escape(src) if fold: src = "(?i)" + src regex = re.compile(src) def repl(str): return re.sub(regex, dst, str) d = [] for fid, mid, flds in deck.db.execute( "select id, mid, flds from facts where id in " + ids2str(fids)): origFlds = flds # does it match? sflds = splitFields(flds) if field: ord = mmap[mid] sflds[ord] = repl(sflds[ord]) else: for c in range(len(sflds)): sflds[c] = repl(sflds[c]) flds = joinFields(sflds) if flds != origFlds: d.append(dict(fid=fid, flds=flds)) if not d: return 0 # replace deck.db.executemany("update facts set flds = :flds where id=:fid", d) deck.updateFieldCache(fids) return len(d)
def doExport(self, file: BufferedWriter) -> None: cardIds = self.cardIds() data = [] for id, flds, tags in self.col.db.execute(""" select guid, flds, tags from notes where id in (select nid from cards where cards.id in %s)""" % ids2str(cardIds)): row = [] # note id if self.includeID: row.append(str(id)) # fields row.extend([self.processText(f) for f in splitFields(flds)]) # tags if self.includeTags: row.append(tags.strip()) data.append("\t".join(row)) self.count = len(data) out = "\n".join(data) file.write(out.encode("utf-8"))
def updateChapters(): global chapters global notes for id, mid, flds in mw.col.db.execute("SELECT id,mid,flds FROM notes"): if mid in midFilter: fields = splitFields(flds) chapter = fields[chapterField[mid]] if not (chapter in chapters): chapters[chapter] = [] chapters[chapter].append(id) notes[id] = chapter # Si cette note n'est pas dans la table "todo", on l'ajoute count = 0 for cardId in mw.col.db.execute( "SELECT cardId FROM todo WHERE `cardId`=%d" % id): count += 1 if count == 0: mw.col.db.execute( "INSERT INTO `todo`(`cardId`,`chapitre`,`logicOrder`,`done`) VALUES (%d,'%s',0,0)" % (id, chapter)) tooltip("TODO : database loaded !", period=1000)
def availOrds(self, m, flds): "Given a joined field string, return available template ordinals." fields = {} for c, f in enumerate(splitFields(flds)): fields[c] = f.strip() avail = [] for ord, type, req, reqstrs in m['req']: # unsatisfiable template if type == "none": continue # AND requirement? elif type == "all": ok = True for idx in req: if not fields[idx]: # missing and was required ok = False break if not ok: continue # OR requirement? elif type == "any": ok = False for idx in req: if fields[idx]: ok = True break if not ok: continue # extra cloze requirement? ok = True for s in reqstrs: if s not in flds: # required cloze string was missing ok = False break if not ok: continue avail.append(ord) return avail