def _findField(self, field, val): field = field.lower() val = val.replace("*", "%") # find models that have that field mods = {} for m in self.col.models.all(): for f in m['flds']: if unicodedata.normalize("NFC", f['name'].lower()) == field: mods[str(m['id'])] = (m, f['ord']) if not mods: # nothing has that field return # gather nids regex = re.escape(val).replace("_", ".").replace(re.escape("%"), ".*") nids = [] for (id, mid, flds) in self.col.db.execute( """ select id, mid, flds from notes where mid in %s and flds like ? escape '\\'""" % (ids2str(list(mods.keys()))), "%" + val + "%"): flds = splitFields(flds) ord = mods[str(mid)][1] strg = flds[ord] try: if re.search("(?si)^" + regex + "$", strg): nids.append(id) except sre_constants.error: return if not nids: return "0" return "n.id in %s" % ids2str(nids)
def renderQA(self, ids=None, type="card"): # gather metadata if type == "card": where = "and c.id in " + ids2str(ids) elif type == "note": where = "and f.id in " + ids2str(ids) elif type == "model": where = "and m.id in " + ids2str(ids) elif type == "all": where = "" else: raise Exception() return [self._renderQA(row) for row in self._qaData(where)]
def _burySiblings(self, card): toBury = [] nconf = self._newConf(card) buryNew = nconf.get("bury", True) rconf = self._revConf(card) buryRev = rconf.get("bury", True) # loop through and remove from queues for cid, queue in self.col.db.execute( """ select id, queue from cards where nid=? and id!=? and (queue=0 or (queue=2 and due<=?))""", card.nid, card.id, self.today): if queue == 2: if buryRev: toBury.append(cid) # if bury disabled, we still discard to give same-day spacing try: self._revQueue.remove(cid) except ValueError: pass else: # if bury disabled, we still discard to give same-day spacing if buryNew: toBury.append(cid) try: self._newQueue.remove(cid) except ValueError: pass # then bury if toBury: self.col.db.execute( "update cards set queue=-2,mod=?,usn=? where id in " + ids2str(toBury), intTime(), self.col.usn()) self.col.log(toBury)
def unsuspendCards(self, ids): "Unsuspend cards." self.col.log(ids) self.col.db.execute( "update cards set queue=type,mod=?,usn=? " "where queue = -1 and id in " + ids2str(ids), intTime(), self.col.usn())
def _changeCards(self, nids, oldModel, newModel, map): d = [] deleted = [] for (cid, ord) in self.col.db.execute( "select id, ord from cards where nid in "+ids2str(nids)): # if the src model is a cloze, we ignore the map, as the gui # doesn't currently support mapping them if oldModel['type'] == MODEL_CLOZE: new = ord if newModel['type'] != MODEL_CLOZE: # if we're mapping to a regular note, we need to check if # the destination ord is valid if len(newModel['tmpls']) <= ord: new = None else: # mapping from a regular note, so the map should be valid new = map[ord] if new is not None: d.append(dict( cid=cid,new=new,u=self.col.usn(),m=intTime())) else: deleted.append(cid) self.col.db.executemany( "update cards set ord=:new,usn=:u,mod=:m where id=:cid", d) self.col.remCards(deleted)
def remTemplate(self, m, template): "False if removing template would leave orphan notes." assert len(m['tmpls']) > 1 # find cards using this template ord = m['tmpls'].index(template) cids = self.col.db.list(""" select c.id from cards c, notes f where c.nid=f.id and mid = ? and ord = ?""", m['id'], ord) # all notes with this template must have at least two cards, or we # could end up creating orphaned notes if self.col.db.scalar(""" select nid, count() from cards where nid in (select nid from cards where id in %s) group by nid having count() < 2 limit 1""" % ids2str(cids)): return False # ok to proceed; remove cards self.col.modSchema(check=True) self.col.remCards(cids) # shift ordinals self.col.db.execute(""" update cards set ord = ord - 1, usn = ?, mod = ? where nid in (select id from notes where mid = ?) and ord > ?""", self.col.usn(), intTime(), m['id'], ord) m['tmpls'].remove(template) self._updateTemplOrds(m) self.save(m) return True
def _findDeck(self, args): # if searching for all decks, skip (val, args) = args if val == "*": return "skip" # deck types elif val == "filtered": return "c.odid" def dids(did): if not did: return None return [did] + [a[1] for a in self.col.decks.children(did)] # current deck? ids = None if val.lower() == "current": ids = dids(self.col.decks.current()['id']) elif "*" not in val: # single deck ids = dids(self.col.decks.id(val, create=False)) else: # wildcard ids = set() val = re.escape(val).replace(r"\*", ".*") for d in self.col.decks.all(): if re.match("(?i)" + val, unicodedata.normalize("NFC", d['name'])): ids.update(dids(d['id'])) if not ids: return sids = ids2str(ids) return "c.did in %s or c.odid in %s" % (sids, sids)
def remCards(self, ids, notes=True): "Bulk delete cards by ID." if not ids: return sids = ids2str(ids) nids = self.db.list("select nid from cards where id in " + sids) # remove cards self._logRem(ids, REM_CARD) self.db.execute("delete from cards where id in " + sids) # then notes if not notes: return nids = self.db.list(""" select id from notes where id in %s and id not in (select nid from cards)""" % ids2str(nids)) self._remNotes(nids)
def buryCards(self, cids): self.col.log(cids) self.remFromDyn(cids) self.removeLrn(cids) self.col.db.execute( """ update cards set queue=-2,mod=?,usn=? where id in """ + ids2str(cids), intTime(), self.col.usn())
def suspendCards(self, ids): "Suspend cards." self.col.log(ids) self.remFromDyn(ids) self.removeLrn(ids) self.col.db.execute( "update cards set queue=-1,mod=?,usn=? where id in " + ids2str(ids), intTime(), self.col.usn())
def unburyCardsForDeck(self): sids = ids2str(self.col.decks.active()) self.col.log( self.col.db.list( "select id from cards where queue = -2 and did in %s" % sids)) self.col.db.execute( "update cards set mod=?,usn=?,queue=type where queue = -2 and did in %s" % sids, intTime(), self.col.usn())
def emptyCardReport(self, cids): rep = "" for ords, cnt, flds in self.db.all(""" select group_concat(ord+1), count(), flds from cards c, notes n where c.nid = n.id and c.id in %s group by nid""" % ids2str(cids)): rep += _("Empty card numbers: %(c)s\nFields: %(f)s\n\n") % dict( c=ords, f=flds.replace("\x1f", " / ")) return rep
def cids(self, did, children=False): if not children: return self.col.db.list("select id from cards where did=?", did) dids = [did] for name, id in self.children(did): dids.append(id) return self.col.db.list("select id from cards where did in " + ids2str(dids))
def _findModel(self, args): (val, args) = args ids = [] val = val.lower() for m in self.col.models.all(): if unicodedata.normalize("NFC", m['name'].lower()) == val: ids.append(m['id']) return "n.mid in %s" % ids2str(ids)
def _remNotes(self, ids): "Bulk delete notes by ID. Don't call this directly." if not ids: return strids = ids2str(ids) # we need to log these independently of cards, as one side may have # more card templates runHook("remNotes", self, ids) self._logRem(ids, REM_NOTE) self.db.execute("delete from notes where id in %s" % strids)
def removeLrn(self, ids=None): "Remove cards from the learning queues." if ids: extra = " and id in " + ids2str(ids) else: # benchmarks indicate it's about 10x faster to search all decks # with the index than scan the table extra = " and did in " + ids2str(self.col.decks.allIds()) # review cards in relearning self.col.db.execute(""" update cards set due = odue, queue = 2, mod = %d, usn = %d, odue = 0 where queue in (1,3) and type = 2 %s """ % (intTime(), self.col.usn(), extra)) # new cards in learning self.forgetCards( self.col.db.list("select id from cards where queue in (1,3) %s" % extra))
def forgetCards(self, ids): "Put cards at the end of the new queue." self.remFromDyn(ids) self.col.db.execute( "update cards set type=0,queue=0,ivl=0,due=0,odue=0,factor=?" " where id in " + ids2str(ids), STARTING_FACTOR) pmax = self.col.db.scalar( "select max(due) from cards where type=0") or 0 # takes care of mod + usn self.sortCards(ids, start=pmax + 1) self.col.log(ids)
def byDeck(self, did, children=False): basequery = "select n.tags from cards c, notes n WHERE c.nid = n.id" if not children: query = basequery + " AND c.did=?" res = self.col.db.list(query, did) return list(set(self.split(" ".join(res)))) dids = [did] for name, id in self.col.decks.children(did): dids.append(id) query = basequery + " AND c.did IN " + ids2str(dids) res = self.col.db.list(query) return list(set(self.split(" ".join(res))))
def newerRows(self, data, table, modIdx): ids = (r[0] for r in data) lmods = {} for id, mod in self.col.db.execute( "select id, mod from %s where id in %s and %s" % ( table, ids2str(ids), self.usnLim())): lmods[id] = mod update = [] for r in data: if r[0] not in lmods or lmods[r[0]] < r[modIdx]: update.append(r) self.col.log(table, data) return update
def doExport(self, file): ids = sorted(self.cardIds()) strids = ids2str(ids) def esc(s): # strip off the repeated question in answer if exists s = re.sub("(?si)^.*<hr id=answer>\n*", "", s) return self.processText(s) out = "" for cid in ids: c = self.col.getCard(cid) out += esc(c.q()) out += "\t" + esc(c.a()) + "\n" file.write(out.encode("utf-8"))
def fieldNamesForNotes(col, nids): downcasedNames = set() origNames = [] mids = col.db.list("select distinct mid from notes where id in %s" % ids2str(nids)) for mid in mids: model = col.models.get(mid) for field in col.models.fieldNames(model): if field.lower() not in downcasedNames: downcasedNames.add(field.lower()) origNames.append(field) return sorted(origNames, key=lambda x: x.lower())
def registerNotes(self, nids=None): "Add any missing tags from notes to the tags list." # when called without an argument, the old list is cleared first. if nids: lim = " where id in " + ids2str(nids) else: lim = "" self.tags = {} self.changed = True self.register( set( self.split(" ".join( self.col.db.list("select distinct tags from notes" + lim)))))
def updateFieldCache(self, nids): "Update field checksums and sort cache, after find&replace, etc." snids = ids2str(nids) r = [] for (nid, mid, flds) in self._fieldData(snids): fields = splitFields(flds) model = self.models.get(mid) if not model: # note points to invalid model continue r.append((stripHTMLMedia(fields[self.models.sortIdx(model)]), fieldChecksum(fields[0]), nid)) # apply, relying on calling code to bump usn+mod self.db.executemany("update notes set sfld=?, csum=? where id=?", r)
def findReplace(col, nids, src, dst, regex=False, field=None, fold=True): "Find and replace fields in a note." mmap = {} if field: for m in col.models.all(): for f in m['flds']: if f['name'].lower() == field.lower(): mmap[str(m['id'])] = f['ord'] if not mmap: return 0 # find and gather replacements if not regex: src = re.escape(src) if fold: src = "(?i)" + src regex = re.compile(src) def repl(str): return re.sub(regex, dst, str) d = [] snids = ids2str(nids) nids = [] for nid, mid, flds in col.db.execute( "select id, mid, flds from notes where id in " + snids): origFlds = flds # does it match? sflds = splitFields(flds) if field: try: ord = mmap[str(mid)] sflds[ord] = repl(sflds[ord]) except KeyError: # note doesn't have that field continue else: for c in range(len(sflds)): sflds[c] = repl(sflds[c]) flds = joinFields(sflds) if flds != origFlds: nids.append(nid) d.append(dict(nid=nid, flds=flds, u=col.usn(), m=intTime())) if not d: return 0 # replace col.db.executemany( "update notes set flds=:flds,mod=:m,usn=:u where id=:nid", d) col.updateFieldCache(nids) col.genCards(nids) return len(d)
def basicCheck(self): "Basic integrity check for syncing. True if ok." # cards without notes if self.db.scalar(""" select 1 from cards where nid not in (select id from notes) limit 1"""): return # notes without cards or models if self.db.scalar(""" select 1 from notes where id not in (select distinct nid from cards) or mid not in %s limit 1""" % ids2str(self.models.ids())): return # invalid ords for m in self.models.all(): # ignore clozes if m['type'] != MODEL_STD: continue if self.db.scalar( """ select 1 from cards where ord not in %s and nid in ( select id from notes where mid = ?) limit 1""" % ids2str([t['ord'] for t in m['tmpls']]), m['id']): return return True
def resetCards(self, ids): "Completely reset cards for export." sids = ids2str(ids) # we want to avoid resetting due number of existing new cards on export nonNew = self.col.db.list( "select id from cards where id in %s and (queue != 0 or type != 0)" % sids) # reset all cards self.col.db.execute( "update cards set reps=0,lapses=0,odid=0,odue=0,queue=0" " where id in %s" % sids) # and forget any non-new cards, changing their due numbers self.forgetCards(nonNew) self.col.log(ids)
def _findDupes(self, args): # caller must call stripHTMLMedia on passed val (val, args) = args try: mid, val = val.split(",", 1) except OSError: return csum = fieldChecksum(val) nids = [] for nid, flds in self.col.db.execute( "select id, flds from notes where mid=? and csum=?", mid, csum): if stripHTMLMedia(splitFields(flds)[0]) == val: nids.append(nid) return "n.id in %s" % ids2str(nids)
def _deckAge(self, by): lim = self._revlogLimit() if lim: lim = " where " + lim if by == 'review': t = self.col.db.scalar("select id from revlog %s order by id limit 1" % lim) elif by == 'add': lim = "where did in %s" % ids2str(self.col.decks.active()) t = self.col.db.scalar("select id from cards %s order by id limit 1" % lim) if not t: period = 1 else: period = max( 1, int(1+((self.col.sched.dayCutoff - (t/1000)) / 86400))) return period
def _changeNotes(self, nids, newModel, map): d = [] nfields = len(newModel['flds']) for (nid, flds) in self.col.db.execute( "select id, flds from notes where id in "+ids2str(nids)): newflds = {} flds = splitFields(flds) for old, new in list(map.items()): newflds[new] = flds[old] flds = [] for c in range(nfields): flds.append(newflds.get(c, "")) flds = joinFields(flds) d.append(dict(nid=nid, flds=flds, mid=newModel['id'], m=intTime(),u=self.col.usn())) self.col.db.executemany( "update notes set flds=:flds,mid=:mid,mod=:m,usn=:u where id = :nid", d) self.col.updateFieldCache(nids)
def doExport(self, file): cardIds = self.cardIds() data = [] for id, flds, tags in self.col.db.execute(""" select guid, flds, tags from notes where id in (select nid from cards where cards.id in %s)""" % ids2str(cardIds)): row = [] # note id if self.includeID: row.append(str(id)) # fields row.extend([self.processText(f) for f in splitFields(flds)]) # tags if self.includeTags: row.append(tags.strip()) data.append("\t".join(row)) self.count = len(data) out = "\n".join(data) file.write(out.encode("utf-8"))