def JxTagDuplicates(Query): # Query = """select fields.value, facts.id, facts.created, facts.tags from fields,facts,fieldModels,models where # facts.id = fields.factId and fields.fieldModelId = fieldModels.id and facts.modelId = models.id and # fieldModels.name = "Expression" and models.tags like "%Japanese%" group by facts.id order by fields.value """ Rows = mw.deck.s.all(Query) Seen={} Duplicates = {} for (Field,Id,Time,Tags) in Rows: Entry = Field.strip(u' ') if Entry in Seen: (lastId,lastTime,lastTags) = Seen[Entry] if lastTime > Time: # first duplicate younger Duplicates[lastId] = Entry Seen[Entry] = (Id,Time,canonifyTags(addTags(lastTags,Tags))) else: # second duplicate younger Duplicates[Id] = Entry Seen[Entry] = (lastId,lastTime,canonifyTags(addTags(lastTags,Tags))) else: Seen[Entry] = (Id,Time,Tags) MasterDuplicates = [] Html = u"""<style> li {font-size: x-large;}</style><h2>Duplicated Entries</h2><ul>""" for Entry in Duplicates.values(): (Id,Time,Tags) = Seen[Entry] MasterDuplicates.append(Id) Html += u"""<li>%s</li>""" % Entry mw.deck.s.statement("update facts set tags = :tags, modified = :t where id =:id",id=Id, t=time.time(),tags=canonifyTags(Tags)) Html += u"""</ul>""" mw.deck.addTags(MasterDuplicates,u"JxMasterDuplicate") mw.deck.addTags(Duplicates.keys(),u"JxDuplicate") mw.deck.deleteTags(MasterDuplicates,u"JxDuplicate") mw.help.showText(Html)
def doMark(self): if self.currentCard.hasTag("Marked"): self.currentCard.fact.tags = canonifyTags(deleteTags( "Marked", self.currentCard.fact.tags)) else: self.currentCard.fact.tags = canonifyTags(addTags( "Marked", self.currentCard.fact.tags)) self.currentCard.fact.setModified(textChanged=True) self.deck.updateFactTags([self.currentCard.fact.id]) for card in self.currentCard.fact.cards: self.deck.updatePriority(card) self.deck.setModified()
def update_fact_with_node(self, somefact, mydeck=None): # update fact fields for k, v in self.node_into_fields().iteritems(): try: # db access layer complains about this if isinstance(v, str): v = unicode(v) somefact[k] = v except KeyError: logging.critical("anki is complaining that %s does not exist" % k) raise # update tags somefact.tags = canonifyTags(self.get_tags()) somefact.setModified(textChanged=True) # update the deck. needed if updating cards if mydeck is not None: # should be OK if we're using this function to add a new fact mydeck.updateFactTags([somefact.id]) for card in somefact.cards: mydeck.updatePriority(card) self.update_num_fact_cards(somefact, mydeck)
def moveTags(): for fact in mw.deck.s.query(Fact).all(): old = fact.tags fact.tags = canonifyTags(fact.tags + "," + ",".join([c.tags for c in fact.cards])) fact.setModified() mw.deck.setModified() mw.reset()
def moveTags(): for fact in mw.deck.s.query(Fact).all(): old = fact.tags fact.tags = canonifyTags(fact.tags + "," + ",".join( [c.tags for c in fact.cards])) fact.setModified() mw.deck.setModified() mw.reset()
def addCards(self, cards): "Add facts in bulk from foreign cards." # map tags field to attr try: idx = self.mapping.index(0) for c in cards: c.tags += " " + c.fields[idx] except ValueError: pass # add facts self.deck.updateProgress() factIds = [genID() for n in range(len(cards))] self.deck.s.execute(factsTable.insert(), [{'modelId': self.model.id, 'tags': canonifyTags(self.tagsToAdd + " " + cards[n].tags), 'id': factIds[n]} for n in range(len(cards))]) self.deck.factCount += len(factIds) self.deck.s.execute(""" delete from factsDeleted where factId in (%s)""" % ",".join([str(s) for s in factIds])) # add all the fields self.deck.updateProgress() for fm in self.model.fieldModels: try: index = self.mapping.index(fm) except ValueError: index = None data = [{'factId': factIds[m], 'fieldModelId': fm.id, 'ordinal': fm.ordinal, 'id': genID(), 'value': (index is not None and cards[m].fields[index] or u"")} for m in range(len(cards))] self.deck.s.execute(fieldsTable.insert(), data) # and cards self.deck.updateProgress() now = time.time() active = 0 for cm in self.model.cardModels: self._now = now if cm.active: active += 1 data = [self.addMeta({ 'id': genID(), 'factId': factIds[m], 'cardModelId': cm.id, 'ordinal': cm.ordinal, 'question': u"", 'answer': u"", 'type': 2},cards[m]) for m in range(len(cards))] self.deck.s.execute(cardsTable.insert(), data) self.deck.updateProgress() self.deck.updateCardsFromFactIds(factIds) self.deck.cardCount += len(cards) * active self.total = len(factIds)
def markAndDelete(): undo = _("MarkDelete") mw.deck.setUndoStart(undo) mw.currentCard.fact.tags = canonifyTags(mw.currentCard.fact.tags + "," + "MarkDelete") mw.currentCard.fact.setModified() mw.deck.updateFactTags([mw.currentCard.fact.id]) mw.deck.deleteCard(mw.currentCard.id) mw.reset() mw.deck.setUndoEnd(undo)
def onTagChange(self): if not self.fact: return old = self.fact.tags self.fact.tags = canonifyTags(unicode(self.tags.text())) if old != self.fact.tags: self.deck.s.flush() self.deck.updateFactTags([self.fact.id]) self.deck.updatePriorities([c.id for c in self.fact.cards]) self.fact.setModified(textChanged=True) self.deck.flushMod() if self.onChange: self.onChange("tag")
def markDuplicateFacts(self, deck): ankiDeck = AnkiHelper.getDeck(deck.path) cards = self.factsService.getAllCardsOrderByScore(deck = deck) ankiFactsId = list() ankiFacts = AnkiHelper.getFacts(ankiDeck) ankiFactsDict = dict() for ankiFact in ankiFacts: ankiFactsDict[ankiFact.id] = ankiFact uniqueMorphemes = dict() #i = 0 for card in cards: fact = self.factsService.getFactById(card.factId) morphemes = self.factsService.getMorphemes(fact) factHasNewMorphemes = False for morpheme in morphemes: if morpheme.id not in uniqueMorphemes: uniqueMorphemes[morpheme.id] = morpheme.id factHasNewMorphemes = True try: ankiFact = ankiFactsDict[fact.ankiFactId] except Exception:continue ankiFact.tags = canonifyTags(deleteTags(u'LxDuplicate', ankiFact.tags)) if factHasNewMorphemes == False: log(str(fact) + " is Duplicate") ankiFact.tags = canonifyTags(addTags(u'LxDuplicate', ankiFact.tags)) ankiFactsId.append(int(ankiFact.id)) log(ankiFactsId) ankiDeck.updateFactTags(ankiFactsId) ankiDeck.save() ankiDeck.close()
def cardIsUnique(self, card): fieldsAsTags = [] for n in range(len(self.mapping)): if self.mapping[n] and self.mapping[n].unique: if card.fields[n] in self.uniqueCache[self.mapping[n].id]: if not self.tagDuplicates: self.log.append("Fact has duplicate '%s': %s" % (self.mapping[n].name, ", ".join(card.fields))) return False fieldsAsTags.append(self.mapping[n].name.replace(" ", "-")) else: self.uniqueCache[self.mapping[n].id][card.fields[n]] = 1 if fieldsAsTags: card.tags += u" Duplicate:" + ("+".join(fieldsAsTags)) card.tags = canonifyTags(card.tags) return True
def accept(self): self.hide() self.deck.startProgress() n = 0 suspended = [] for item in self.items: idx = self.dialog.list.indexFromItem(item) if self.dialog.list.selectionModel().isSelected(idx): suspended.append(self.tags[n]) n += 1 self.deck.suspended = canonifyTags(joinTags(suspended)) self.deck.setModified() self.deck.updateAllPriorities(partial=True, dirty=False) self.deck.finishProgress() QDialog.accept(self)
def cardIsUnique(self, card): fieldsAsTags = [] for n in range(len(self.mapping)): if self.mapping[n] and self.mapping[n].unique: if card.fields[n] in self.uniqueCache[self.mapping[n].id]: if not self.tagDuplicates: self.log.append( "Fact has duplicate '%s': %s" % (self.mapping[n].name, ", ".join(card.fields))) return False fieldsAsTags.append(self.mapping[n].name.replace(" ", "-")) else: self.uniqueCache[self.mapping[n].id][card.fields[n]] = 1 if fieldsAsTags: card.tags += u" Duplicate:" + ("+".join(fieldsAsTags)) card.tags = canonifyTags(card.tags) return True
def doUpdate(self): # grab the data from the external file cards = self.foreignCards() # grab data from db fields = self.deck.db.all(""" select factId, value from fields where fieldModelId = :id and value != ''""", id=self.updateKey[1]) # hash it vhash = {} fids = [] for (fid, val) in fields: fids.append(fid) vhash[val] = fid # prepare tags tagsIdx = None try: tagsIdx = self.mapping.index(0) for c in cards: c.tags = canonifyTags(self.tagsToAdd + " " + c.fields[tagsIdx]) except ValueError: pass # look for matches upcards = [] newcards = [] for c in cards: v = c.fields[self.updateKey[0]] if v in vhash: # ignore empty keys if v: # fid, card upcards.append((vhash[v], c)) else: newcards.append(c) # update fields for fm in self.model.fieldModels: if fm.id == self.updateKey[1]: # don't update key continue try: index = self.mapping.index(fm) except ValueError: # not mapped continue data = [{'fid': fid, 'fmid': fm.id, 'v': c.fields[index], 'chk': self.maybeChecksum(c.fields[index], fm.unique)} for (fid, c) in upcards] self.deck.db.execute(""" update fields set value = :v, chksum = :chk where factId = :fid and fieldModelId = :fmid""", data) # update tags if tagsIdx is not None: data = [{'fid': fid, 't': c.fields[tagsIdx]} for (fid, c) in upcards] self.deck.db.execute( "update facts set tags = :t where id = :fid", data) # rebuild caches cids = self.deck.db.column0( "select id from cards where factId in %s" % ids2str(fids)) self.deck.updateCardTags(cids) self.deck.updateCardsFromFactIds(fids) self.total = len(cards) self.deck.setModified()
def doUpdate(self): # grab the data from the external file cards = self.foreignCards() # grab data from db fields = self.deck.db.all(""" select factId, value from fields where fieldModelId = :id and value != ''""", id=self.updateKey[1]) # hash it vhash = {} fids = [] for (fid, val) in fields: fids.append(fid) vhash[val] = fid # prepare tags tagsIdx = None try: tagsIdx = self.mapping.index(0) for c in cards: c.tags = canonifyTags(self.tagsToAdd + " " + c.fields[tagsIdx]) except ValueError: pass # look for matches upcards = [] newcards = [] for c in cards: v = c.fields[self.updateKey[0]] if v in vhash: # ignore empty keys if v: # fid, card upcards.append((vhash[v], c)) else: newcards.append(c) # update fields for fm in self.model.fieldModels: if fm.id == self.updateKey[1]: # don't update key continue try: index = self.mapping.index(fm) except ValueError: # not mapped continue data = [{ 'fid': fid, 'fmid': fm.id, 'v': c.fields[index], 'chk': self.maybeChecksum(c.fields[index], fm.unique) } for (fid, c) in upcards] self.deck.db.execute( """ update fields set value = :v, chksum = :chk where factId = :fid and fieldModelId = :fmid""", data) # update tags if tagsIdx is not None: data = [{ 'fid': fid, 't': c.fields[tagsIdx] } for (fid, c) in upcards] self.deck.db.execute("update facts set tags = :t where id = :fid", data) # rebuild caches cids = self.deck.db.column0("select id from cards where factId in %s" % ids2str(fids)) self.deck.updateCardTags(cids) self.deck.updateCardsFromFactIds(fids) self.total = len(cards) self.deck.setModified()
def addCards(self, cards): "Add facts in bulk from foreign cards." # map tags field to attr try: idx = self.mapping.index(0) for c in cards: c.tags += " " + c.fields[idx] except ValueError: pass # add facts factIds = [genID() for n in range(len(cards))] factCreated = {} def fudgeCreated(d, tmp=[]): if not tmp: tmp.append(time.time()) else: tmp[0] += 0.0001 d['created'] = tmp[0] factCreated[d['id']] = d['created'] return d self.deck.db.execute(factsTable.insert(), [ fudgeCreated( { 'modelId': self.model.id, 'tags': canonifyTags(self.tagsToAdd + " " + cards[n].tags), 'id': factIds[n] }) for n in range(len(cards)) ]) self.deck.db.execute(""" delete from factsDeleted where factId in (%s)""" % ",".join([str(s) for s in factIds])) # add all the fields for fm in self.model.fieldModels: try: index = self.mapping.index(fm) except ValueError: index = None data = [{ 'factId': factIds[m], 'fieldModelId': fm.id, 'ordinal': fm.ordinal, 'id': genID(), 'value': (index is not None and cards[m].fields[index] or u""), 'chksum': self.maybeChecksum( index is not None and cards[m].fields[index] or u"", fm.unique) } for m in range(len(cards))] self.deck.db.execute(fieldsTable.insert(), data) # and cards active = 0 for cm in self.model.cardModels: if cm.active: active += 1 data = [ self.addMeta( { 'id': genID(), 'factId': factIds[m], 'factCreated': factCreated[factIds[m]], 'cardModelId': cm.id, 'ordinal': cm.ordinal, 'question': u"", 'answer': u"" }, cards[m]) for m in range(len(cards)) ] self.deck.db.execute(cardsTable.insert(), data) self.deck.updateCardsFromFactIds(factIds) self.total = len(factIds)
def doDeckUpdate(self): # IO () fidDb = self.allDb().fidDb() fid2cardsDb = self.fid2cardsDb() locDb = self.allDb().locDb() rankDb = R.mkRankDb(self.knownDb()) knownDbDb = self.knownDb().db matureDbDb = self.matureDb().db fs = self.getFacts() # cache lookups fieldNames = self.cfg['morph fields'] ipnKnownField = self.cfg['i+N known field'] ipnMatureField = self.cfg['i+N mature field'] unknownsField = self.cfg['unknowns field'] unmaturesField = self.cfg['unmatures field'] vrField = self.cfg['vocab rank field'] mmiField = self.cfg['morph man index field'] ip1knownField = self.cfg['copy i+1 known to'] ip0matField = self.cfg['copy i+0 mature to'] i, lfs = 0, len(fs) start = time.time() fstart = time.time() mmiDict = {} # Map Fact -> MorphManIndex for f in fs: # first get all morphems for this fact ms = set() for fieldName in fieldNames: try: loc = fidDb[(f.id, fieldName)] ms.update(locDb[loc]) except KeyError: continue # now determine unknowns and iPlusN - don't count multiple instances # of a morpheme as an increase in difficulty, so use a set unknowns = set() for m in ms: if m not in knownDbDb: unknowns.add(m) N_k = len(unknowns) unmatures = set() for m in ms: if m not in matureDbDb: unmatures.add(m) N_m = len(unmatures) # determine vocab rank and morph man overall difficulty index vr = R.rankMorphemes(rankDb, ms) mmi = N_k * 10000 + len(ms) * 1000 + vr mmiDict[f] = mmi try: f[ipnKnownField] = u'%d' % N_k except KeyError: pass try: f[ipnMatureField] = u'%d' % N_m except KeyError: pass try: f[mmiField] = u'%d' % mmi except KeyError: pass try: f[vrField] = u'%d' % vr except KeyError: pass try: f[unknownsField] = u','.join(u.base for u in unknowns) except KeyError: pass try: f[unmaturesField] = u','.join(u.base for u in unmatures) except KeyError: pass if len(ms) == 0: f.tags = canonifyTags(addTags(u'noMorphemes', f.tags)) # Help automate vocab card -> sentence card promotion try: f[ip0matField] = u'' except KeyError: pass try: f[ip1knownField] = u'' except KeyError: pass f.tags = canonifyTags( deleteTags(u'ip0mature ip1known notReady', f.tags)) if N_m == 0: # is i+0 mature, make it a sentence card f.tags = canonifyTags( addTags(u'ip0mature ip0matureEver', f.tags)) try: f[ip0matField] = u' '.join(f[name] for name in fieldNames) except KeyError: pass elif N_k == 1: # is i+1 known, make it a vocab card f.tags = canonifyTags(addTags(u'ip1known ip1knownEver', f.tags)) try: f[ip1knownField] = u'%s' % unknowns.pop().base except KeyError: pass else: # is neither, make it a neither card f.tags = canonifyTags(addTags(u'notReady', f.tags)) #f.setModified( textChanged=True, deck=self.deck ) # now display progress i += 1 if i % 100 == 0: fend = time.time() log(' %d / %d = %d%% in %f sec' % (i, lfs, 100. * i / lfs, fend - fstart)) fstart = time.time() # rebuild tags self.deck.updateFactTags((f.id for f in fs)) # sort new cards by Morph Man Index debug('Set fact fields, now changing card creation times for ordering') newtime = 0.0 try: for f in sorted(fs, key=lambda x: mmiDict[x]): for c in fid2cardsDb[f.id]: c.created = newtime newtime += CARD_CREATION_TIME_INC except KeyError: log('! no morph man index field for sorting') # save deck and timestamps self.deck.save() end = time.time() log('Proccessed all facts in %f sec' % (end - start)) self.cfg['last deck update'] = time.time() self.cfg['last deck update took'] = end - start
def addCards(self, cards): "Add facts in bulk from foreign cards." # map tags field to attr try: idx = self.mapping.index(0) for c in cards: c.tags += " " + c.fields[idx] except ValueError: pass # add facts factIds = [genID() for n in range(len(cards))] factCreated = {} def fudgeCreated(d, tmp=[]): if not tmp: tmp.append(time.time()) else: tmp[0] += 0.0001 d['created'] = tmp[0] factCreated[d['id']] = d['created'] return d self.deck.db.execute(factsTable.insert(), [fudgeCreated({'modelId': self.model.id, 'tags': canonifyTags(self.tagsToAdd + " " + cards[n].tags), 'id': factIds[n]}) for n in range(len(cards))]) self.deck.db.execute(""" delete from factsDeleted where factId in (%s)""" % ",".join([str(s) for s in factIds])) # add all the fields for fm in self.model.fieldModels: try: index = self.mapping.index(fm) except ValueError: index = None data = [{'factId': factIds[m], 'fieldModelId': fm.id, 'ordinal': fm.ordinal, 'id': genID(), 'value': (index is not None and cards[m].fields[index] or u""), 'chksum': self.maybeChecksum( index is not None and cards[m].fields[index] or u"", fm.unique) } for m in range(len(cards))] self.deck.db.execute(fieldsTable.insert(), data) # and cards active = 0 for cm in self.model.cardModels: if cm.active: active += 1 data = [self.addMeta({ 'id': genID(), 'factId': factIds[m], 'factCreated': factCreated[factIds[m]], 'cardModelId': cm.id, 'ordinal': cm.ordinal, 'question': u"", 'answer': u"" },cards[m]) for m in range(len(cards))] self.deck.db.execute(cardsTable.insert(), data) self.deck.updateCardsFromFactIds(factIds) self.total = len(factIds)
def doDeckUpdate( self ): # IO () fidDb = self.allDb().fidDb() fid2cardsDb = self.fid2cardsDb() locDb = self.allDb().locDb() rankDb = R.mkRankDb( self.knownDb() ) knownDbDb = self.knownDb().db matureDbDb = self.matureDb().db fs = self.getFacts() # cache lookups fieldNames = self.cfg['morph fields'] ipnKnownField = self.cfg['i+N known field'] ipnMatureField = self.cfg['i+N mature field'] unknownsField = self.cfg['unknowns field'] unmaturesField = self.cfg['unmatures field'] vrField = self.cfg['vocab rank field'] mmiField = self.cfg['morph man index field'] ip1knownField = self.cfg['copy i+1 known to'] ip0matField = self.cfg['copy i+0 mature to'] i, lfs = 0, len(fs) start = time.time() fstart = time.time() mmiDict = {} # Map Fact -> MorphManIndex hasModification = False mod_fs = set() for f in fs: # first get all morphems for this fact ms = set() for fieldName in fieldNames: try: loc = fidDb[ (f.id, fieldName) ] ms.update( locDb[ loc ] ) except KeyError: log ("Error while loading morphems : " + fieldName) continue # now determine unknowns and iPlusN - don't count multiple instances # of a morpheme as an increase in difficulty, so use a set unknowns = set() #log (M.ms2str(ms)) for m in ms: if m not in knownDbDb: unknowns.add( m ) N_k = len( unknowns ) unmatures = set() for m in ms: if m not in matureDbDb: unmatures.add( m ) N_m = len( unmatures ) # determine vocab rank and morph man overall difficulty index vr = R.rankMorphemes( rankDb, ms ) mmi = N_k*1000 + N_m*100 + vr*25 try: if f[ mmiField ] == "" or int(mmi) != int(f[ mmiField ].decode()): #log(f[ mmiField ]) #log("Change ! != " + str(int(mmi))) f.setModified(textChanged=True, deck=self.deck) hasModification = True mod_fs.add(int(f.id)) except KeyError: pass mmiDict[ f ] = mmi try: f[ ipnKnownField ] = u'%d' % N_k except KeyError: pass try: f[ ipnMatureField ] = u'%d' % N_m except KeyError: pass try: f[ mmiField ] = u'%d' % mmi except KeyError: pass try: f[ vrField ] = u'%d' % vr except KeyError: pass #log(','.join( u.base for u in unknowns )) try: f[ unknownsField ] = u','.join( u.base for u in unknowns ) except KeyError: pass #log(','.join( u.base for u in unmatures )) try: f[ unmaturesField ] = u','.join( u.base for u in unmatures ) except KeyError: pass if len(ms) == 0: f.tags = canonifyTags( addTags( u'noMorphemes', f.tags ) ) # Help automate vocab card -> sentence card promotion try: f[ ip0matField ] = u'' except KeyError: pass try: f[ ip1knownField ] = u'' except KeyError: pass #before_tags = f.tags f.tags = canonifyTags( deleteTags( u'ipXmature,ip0mature,ip1known,notReady', f.tags ) ) # Verification du nombre unknows et matures. Si changement alors, mettre la carte en modifier if N_k > 1: #log("Before Tags :" + before_tags) #log("After First Tags: " + f.tags) f.tags = canonifyTags( addTags( u'notReady', f.tags ) ) #log("After Remove Tags: " + f.tags) else: if N_m == 0: f.tags = canonifyTags( addTags( u'ip0mature ip0matureEver', f.tags ) ) try: f[ ip0matField ] = u' '.join( f[ name ] for name in fieldNames ) except KeyError: pass elif N_k == 1: f.tags = canonifyTags( addTags( u'ip1known ip1knownEver', f.tags ) ) try: f[ ip1knownField ] = u'%s' % unknowns.pop().base except KeyError: pass else: f.tags = canonifyTags( addTags( u'ipXmature ipXmatureEver', f.tags ) ) try: f[ ip0matField ] = u' '.join( f[ name ] for name in fieldNames ) except KeyError: pass #if N_m == 0: # is i+0 mature, make it a sentence card # f.tags = canonifyTags( addTags( u'ip0mature ip0matureEver', f.tags ) ) # try: f[ ip0matField ] = u' '.join( f[ name ] for name in fieldNames ) # except KeyError: pass #elif N_k == 1: # is i+1 known, make it a vocab card # f.tags = canonifyTags( addTags( u'ip1known ip1knownEver', f.tags ) ) # try: f[ ip1knownField ] = u'%s' % unknowns.pop().base # except KeyError: pass #else: # is neither, make it a neither card # f.tags = canonifyTags( addTags( u'notReady', f.tags ) ) #f.setModified( textChanged=True, deck=self.deck ) # now display progress i += 1 if i % 100 == 0: fend = time.time() log(' %d / %d = %d%% in %f sec' % ( i, lfs, 100.*i/lfs, fend-fstart ) ) fstart = time.time() # rebuild tags self.deck.updateFactTags( ( f.id for f in fs ) ) # sort new cards by Morph Man Index debug( 'Set fact fields, now changing card creation times for ordering' ) newtime = 472777200.0 # 25 Decembre 1984 :) #newCardIds = set() try: for f in sorted( fs, key=lambda x: mmiDict[ x ] ): for c in fid2cardsDb[ f.id ]: if self.deck.cardIsNew(c): #newCardIds.add(c.id) #c.created = newtime c.due = newtime c.combinedDue = newtime newtime += CARD_CREATION_TIME_INC except KeyError: log( '! no morph man index field for sorting' ) # save deck #self.deck.orderNewCards() if hasModification: self.deck.setModified() self.deck.updateCardQACacheFromIds(mod_fs, type="facts") self.deck.save() # reorder new cards and save # self.deck.orderNewCards() # self.deck.save() end = time.time() log( 'Proccessed all facts in %f sec' % ( end-start ) ) self.cfg['last deck update'] = time.time() self.cfg['last deck update took'] = end-start
def stringTags(self): return canonifyTags(self.tags)
def doDeckUpdate( self ): # IO () fidDb = self.allDb().fidDb() fid2cardsDb = self.fid2cardsDb() locDb = self.allDb().locDb() rankDb = R.mkRankDb( self.knownDb() ) knownDbDb = self.knownDb().db matureDbDb = self.matureDb().db fs = self.getFacts() # cache lookups fieldNames = self.cfg['morph fields'] ipnKnownField = self.cfg['i+N known field'] ipnMatureField = self.cfg['i+N mature field'] unknownsField = self.cfg['unknowns field'] unmaturesField = self.cfg['unmatures field'] vrField = self.cfg['vocab rank field'] mmiField = self.cfg['morph man index field'] ip1knownField = self.cfg['copy i+1 known to'] ip0matField = self.cfg['copy i+0 mature to'] i, lfs = 0, len(fs) start = time.time() fstart = time.time() mmiDict = {} # Map Fact -> MorphManIndex for f in fs: # first get all morphems for this fact ms = set() for fieldName in fieldNames: try: loc = fidDb[ (f.id, fieldName) ] ms.update( locDb[ loc ] ) except KeyError: continue # now determine unknowns and iPlusN - don't count multiple instances # of a morpheme as an increase in difficulty, so use a set unknowns = set() for m in ms: if m not in knownDbDb: unknowns.add( m ) N_k = len( unknowns ) unmatures = set() for m in ms: if m not in matureDbDb: unmatures.add( m ) N_m = len( unmatures ) # determine vocab rank and morph man overall difficulty index vr = R.rankMorphemes( rankDb, ms ) mmi = N_k*10000 + len(ms)*1000 + vr mmiDict[ f ] = mmi try: f[ ipnKnownField ] = u'%d' % N_k except KeyError: pass try: f[ ipnMatureField ] = u'%d' % N_m except KeyError: pass try: f[ mmiField ] = u'%d' % mmi except KeyError: pass try: f[ vrField ] = u'%d' % vr except KeyError: pass try: f[ unknownsField ] = u','.join( u.base for u in unknowns ) except KeyError: pass try: f[ unmaturesField ] = u','.join( u.base for u in unmatures ) except KeyError: pass if len(ms) == 0: f.tags = canonifyTags( addTags( u'noMorphemes', f.tags ) ) # Help automate vocab card -> sentence card promotion try: f[ ip0matField ] = u'' except KeyError: pass try: f[ ip1knownField ] = u'' except KeyError: pass f.tags = canonifyTags( deleteTags( u'ip0mature ip1known notReady', f.tags ) ) if N_m == 0: # is i+0 mature, make it a sentence card f.tags = canonifyTags( addTags( u'ip0mature ip0matureEver', f.tags ) ) try: f[ ip0matField ] = u' '.join( f[ name ] for name in fieldNames ) except KeyError: pass elif N_k == 1: # is i+1 known, make it a vocab card f.tags = canonifyTags( addTags( u'ip1known ip1knownEver', f.tags ) ) try: f[ ip1knownField ] = u'%s' % unknowns.pop().base except KeyError: pass else: # is neither, make it a neither card f.tags = canonifyTags( addTags( u'notReady', f.tags ) ) #f.setModified( textChanged=True, deck=self.deck ) # now display progress i += 1 if i % 100 == 0: fend = time.time() log(' %d / %d = %d%% in %f sec' % ( i, lfs, 100.*i/lfs, fend-fstart ) ) fstart = time.time() # rebuild tags self.deck.updateFactTags( ( f.id for f in fs ) ) # sort new cards by Morph Man Index debug( 'Set fact fields, now changing card creation times for ordering' ) newtime = 0.0 try: for f in sorted( fs, key=lambda x: mmiDict[ x ] ): for c in fid2cardsDb[ f.id ]: c.created = newtime newtime += CARD_CREATION_TIME_INC except KeyError: log( '! no morph man index field for sorting' ) # save deck and timestamps self.deck.save() end = time.time() log( 'Proccessed all facts in %f sec' % ( end-start ) ) self.cfg['last deck update'] = time.time() self.cfg['last deck update took'] = end-start
def importIknowItem(item, sentenceModel, vocabModel, importSettings, progress): idQuery = mw.deck.s.query(Field).filter_by(value=item.uniqIdStr()) field = idQuery.first() if field: return False#is duplicate of an existing iknow ID, so return immediately try: expQuery = mw.deck.s.query(Field).filter_by(value=item.expression) field = expQuery.first() if field: return False #duplicate expression, the definition may be different but we don't want dup cards except: pass #might fail due to null expression which we catch other ways if item.type == "item": model = vocabModel elif item.type == "sentence": model = sentenceModel fact = mw.deck.newFact(model) if importSettings.tagsOnImport and len(importSettings.tagsOnImport.strip()) > 0: fact.tags = canonifyTags(importSettings.tagsOnImport) fact['iKnowID'] = item.uniqIdStr() fact['Expression'] = item.expression fact['Meaning'] = item.meaning fact['iKnowType'] = item.type fact['Reading'] = item.reading if item.image_uri: if DOWNLOAD_IMAGES: #downloading images from smart.fm may be prohibited by smart.fm in certain cases. please only enable DOWNLOAD_IMAGES if you are sure that the list you are downloading has public domain (non-copyrighted or free for use) images. (filePath, headers) = urllib.urlretrieve(item.image_uri) path = mw.deck.addMedia(filePath) fact['Image_URI'] = u'<img src="%s" alt="[No Image]">' % item.image_uri else: fact['Image_URI'] = u'<img src="%s" alt="[No Image]">' % item.image_uri else: fact['Image_URI'] = u"" if importSettings.downloadAudio and item.audio_uri: tries = 0 gotAudioForItem = False gotItemWithNoData = False filePath = None headers = None path = None # make several attempts to download audio from smart.fm while not gotAudioForItem and tries < 3: tries += 1 try: (filePath, headers) = urllib.urlretrieve(item.audio_uri) if os.path.exists(filePath) and os.path.getsize(filePath) > 0: gotAudioForItem = True elif not os.path.exists(filePath): progress.logMsg("WARN INFO: downloading URL %s with urlretrieve but the returned path %s does not exist." % (item.audio_uri.encode('utf-8'), filePath.encode('utf-8'))) else: progress.logMsg("WARN INFO: downloading URL %s with urlretrieve but the returned path %s has no data." % (item.audio_uri.encode('utf-8'), filePath.encode('utf-8'))) gotItemWithNoData = True except IOError: progress.logMsg("ERROR INFO: downloading URL %s but got IOError: %s" % (item.audio_uri.encode('utf-8'), traceback.format_exc().encode('utf-8'))) if not gotAudioForItem and not gotItemWithNoData: #sometimes smart.fm reports that an item has an audio file, gives its URL, and even serves a response from that URL - except the URL has 0 bytes of data. so it downloads as a blank file. in this infrequent case, don't raise an error but just skip the audio. raise AudioDownloadError, "Failed to get audio for an item after 3 tries, cancelling import. Error with URI %s" % item.audio_uri if gotItemWithNoData: progress.logMsg("WARN INFO: skipping audio for %s, file had 0 bytes on smart.fm." % item.uniqIdStr()) if gotAudioForItem: # now try to add the media file to the deck try: path = mw.deck.addMedia(filePath) progress.logMsg("Successfully added media from URI %s which was downloaded to %s to the media dir at %s" % (item.audio_uri.encode('utf-8'), filePath.encode('utf-8'), mw.deck.mediaDir().encode('utf-8'))) except: raise AddMediaException, ("Failed to add media from URI %s \nwhich was downloaded to %s \nto the media dir at %s" % (item.audio_uri.encode('utf-8'), filePath.encode('utf-8'), mw.deck.mediaDir().encode('utf-8'))) + traceback.format_exc().encode('utf-8') fact['Audio'] = u"[sound:%s]" % path elif item.audio_uri: fact['Audio'] = u"[sound:%s]" % item.audio_uri else: fact['Audio'] = u"" newfact = mw.deck.addFact(fact) cardIds = [card.id for card in newfact.cards] if importSettings.rescheduleMin and importSettings.rescheduleMax: mw.deck.rescheduleCards(cardIds, importSettings.rescheduleMin, importSettings.rescheduleMax) mw.deck.save() return True
def markFacts(self, language): facts = self.factsService.getAllFactsByLanguage(language) ankiFactsId = list() #i = 0 for fact in facts: try: ankiFact = self.ankiDeckFacts[fact.deck.path][fact.ankiFactId] except Exception: continue deck = self.decks[fact.deck.path] fields = deck.fields morphemes = self.factsService.getMorphemes(fact) morphemesScore, matureMorphemes, knownMorphemes, learnMorphemes, unknownMorphemes = self.getMorphemesScore(morphemes) if fact.statusChanged == True: # FIXME: sure ? if fields[Deck.LEARNX_SCORE_KEY][1]: try: ankiFact[fields[Deck.LEARNX_SCORE_KEY][0]] = u'%d' % int(fact.score) except KeyError: pass if fields[Deck.VOCAB_SCORE_KEY][1]: try: ankiFact[fields[Deck.VOCAB_SCORE_KEY][0]] = u'%d' % int(morphemesScore) except KeyError: pass if fields[Deck.UNKNOWNS_KEY][1]: try: ankiFact[fields[Deck.UNKNOWNS_KEY][0]] = u','.join(u for u in unknownMorphemes) except KeyError: pass if fields[Deck.LEARNTS_KEY][1]: try: ankiFact[fields[Deck.LEARNTS_KEY][0]] = u','.join(u for u in learnMorphemes) except KeyError: pass if fields[Deck.KNOWNS_KEY][1]: try: ankiFact[fields[Deck.KNOWNS_KEY][0]] = u','.join(u for u in knownMorphemes) except KeyError: pass if fields[Deck.MATURES_KEY][1]: try: ankiFact[fields[Deck.MATURES_KEY][0]] = u','.join(u for u in matureMorphemes) except KeyError: pass if len(unknownMorphemes) == 1: if fields[Deck.COPY_UNKNOWN_1_TO_KEY][1]: try: ankiFact[fields[Deck.COPY_UNKNOWN_1_TO_KEY][0]] = u','.join(u for u in unknownMorphemes) except KeyError: pass elif len(unknownMorphemes) == 0: if fields[Deck.COPY_MATURE_TO_KEY][1]: try: ankiFact[fields[Deck.COPY_MATURE_TO_KEY][0]] = u'%s' % ankiFact[deck.expressionField] except KeyError: pass ankiFact.tags = canonifyTags(deleteTags(fact.getAllStatusTag(), ankiFact.tags)) ankiFact.tags = canonifyTags(addTags(fact.getStatusTag(), ankiFact.tags)) if deck.definitionField: try: ankiFact.tags = canonifyTags(deleteTags(u'LxDefKnown,LxDefMatch', ankiFact.tags)) definition = self.factsService.getDefinition(fact) if definition and definition.definitionHash and int(definition.definitionHash) != 0: defMorphemes = self.morphemesService.getMorphemesDefinition(definition) dictMorphemesScore, defMatureMorphemes, defKnownMorphemes, defLearnMorphemes, defUnknownMorphemes = self.getMorphemesScore(defMorphemes) if len(defUnknownMorphemes) == 0: ankiFact.tags = canonifyTags(addTags(u'LxDefKnown', ankiFact.tags)) defKeyMorphemes = self.morphemesService.getMorphemesDefinitionKey(definition) defKeyMorphemesBase = "".join(m.morphLemme.base for m in defKeyMorphemes) if len(unknownMorphemes) == 1 and unknownMorphemes[0] in defKeyMorphemesBase: ankiFact.tags = canonifyTags(addTags(u'LxDefMatch', ankiFact.tags)) if fields[Deck.DEFINITION_SCORE_KEY][1]: try: ankiFact[fields[Deck.DEFINITION_SCORE_KEY][0]] = u'%d' % int(dictMorphemesScore) except KeyError: pass except KeyError: pass ankiFactsId.append(ankiFact.id) self.saveDecks(ankiFactsId, language)