def JxTagDuplicates(Query): # Query = """select fields.value, facts.id, facts.created, facts.tags from fields,facts,fieldModels,models where # facts.id = fields.factId and fields.fieldModelId = fieldModels.id and facts.modelId = models.id and # fieldModels.name = "Expression" and models.tags like "%Japanese%" group by facts.id order by fields.value """ Rows = mw.deck.s.all(Query) Seen={} Duplicates = {} for (Field,Id,Time,Tags) in Rows: Entry = Field.strip(u' ') if Entry in Seen: (lastId,lastTime,lastTags) = Seen[Entry] if lastTime > Time: # first duplicate younger Duplicates[lastId] = Entry Seen[Entry] = (Id,Time,canonifyTags(addTags(lastTags,Tags))) else: # second duplicate younger Duplicates[Id] = Entry Seen[Entry] = (lastId,lastTime,canonifyTags(addTags(lastTags,Tags))) else: Seen[Entry] = (Id,Time,Tags) MasterDuplicates = [] Html = u"""<style> li {font-size: x-large;}</style><h2>Duplicated Entries</h2><ul>""" for Entry in Duplicates.values(): (Id,Time,Tags) = Seen[Entry] MasterDuplicates.append(Id) Html += u"""<li>%s</li>""" % Entry mw.deck.s.statement("update facts set tags = :tags, modified = :t where id =:id",id=Id, t=time.time(),tags=canonifyTags(Tags)) Html += u"""</ul>""" mw.deck.addTags(MasterDuplicates,u"JxMasterDuplicate") mw.deck.addTags(Duplicates.keys(),u"JxDuplicate") mw.deck.deleteTags(MasterDuplicates,u"JxDuplicate") mw.help.showText(Html)
def updateCard(self): basename = os.path.basename(self.fileName) if not basename: raise IOError('No name to add to card found in \'%\'' % self.fileName) self.card.fact[AudioField] += u"[sound:%s]" % basename self.card.fact.tags = addTags(DOWNLOAD_SUCCESS_TAG, self.card.fact.tags) self.card.fact.setModified(textChanged=True, deck=self.card.deck) self.card.deck.save()
def retrieveFile(self): self.fileName, retrieveHeader = urllib.urlretrieve(self.buildQueryUrl(), self.fileName) self.retrievedHash = hashlib.sha256(file(self.fileName, 'r').read()) if self.retrievedHash.hexdigest() in BlacklistHashes: self.card.fact.tags = addTags(DOWNLOAD_FAILURE_TAG, self.card.fact.tags) self.card.fact.setModified(textChanged=True,deck=self.card.deck) self.card.deck.save() raise ValueError('Retrieved file is in Blacklist. (No pronunciation found.)')
def tagCurrentCard(tag): mw.deck.setUndoStart(_("Suspend")) mw.currentCard.fact.tags = addTags(tag, mw.currentCard.fact.tags) mw.currentCard.fact.setModified() for card in mw.currentCard.fact.cards: mw.deck.updatePriority(card) mw.deck.setModified() mw.lastScheduledTime = None mw.reset() mw.deck.setUndoEnd(_("Suspend"))
def retrieveFile(self): self.fileName, retrieveHeader = urllib.urlretrieve( self.buildQueryUrl(), self.fileName) self.retrievedHash = hashlib.sha256(file(self.fileName, 'r').read()) if self.retrievedHash.hexdigest() in BlacklistHashes: self.card.fact.tags = addTags(DOWNLOAD_FAILURE_TAG, self.card.fact.tags) self.card.fact.setModified(textChanged=True, deck=self.card.deck) self.card.deck.save() raise ValueError( 'Retrieved file is in Blacklist. (No pronunciation found.)')
def doMark(self): if self.currentCard.hasTag("Marked"): self.currentCard.fact.tags = canonifyTags(deleteTags( "Marked", self.currentCard.fact.tags)) else: self.currentCard.fact.tags = canonifyTags(addTags( "Marked", self.currentCard.fact.tags)) self.currentCard.fact.setModified(textChanged=True) self.deck.updateFactTags([self.currentCard.fact.id]) for card in self.currentCard.fact.cards: self.deck.updatePriority(card) self.deck.setModified()
def switch(string): myAction = mw.mainWin.__getattribute__('action'+ string) choice = mw.config.get('SBT_'+ string,0) if choice == 0: block(string) #mw.deck.lowPriority = addTags(myDict[string],mw.deck.lowPriority) myAction.setIcon(myPictures['NoNew'+string]) elif choice == 1: #mw.deck.lowPriority = deleteTags(myDict[string],mw.deck.lowPriority) mw.deck.suspended = addTags(myDict[string],mw.deck.suspended) myAction.setIcon(myPictures['No'+string]) else: unblock(string) mw.deck.suspended = deleteTags(myDict[string],mw.deck.suspended) myAction.setIcon(myPictures[string]) mw.config['SBT_'+ string] = (choice + 1) % 3 mw.deck.updateAllPriorities()
def newLoadDeck(deckPath, sync=True, interactive=True, uprecent=True,media=None): code = oldLoadDeck(deckPath, sync, interactive, uprecent,media) if code and mw.deck: for string in myDict.iterkeys(): myAction = mw.mainWin.__getattribute__('action'+ string) choice = mw.config.get('SBT_'+ string,0) if choice == 0: myAction.setStatusTip(string) pass elif choice == 1: block(string) #mw.deck.lowPriority = addTags(myDict[string],mw.deck.lowPriority) myAction.setStatusTip('No new '+string) else: mw.deck.suspended = addTags(myDict[string],mw.deck.suspended) myAction.setStatusTip('No '+string) mw.deck.updateAllPriorities() return code
def markDuplicateFacts(self, deck): ankiDeck = AnkiHelper.getDeck(deck.path) cards = self.factsService.getAllCardsOrderByScore(deck = deck) ankiFactsId = list() ankiFacts = AnkiHelper.getFacts(ankiDeck) ankiFactsDict = dict() for ankiFact in ankiFacts: ankiFactsDict[ankiFact.id] = ankiFact uniqueMorphemes = dict() #i = 0 for card in cards: fact = self.factsService.getFactById(card.factId) morphemes = self.factsService.getMorphemes(fact) factHasNewMorphemes = False for morpheme in morphemes: if morpheme.id not in uniqueMorphemes: uniqueMorphemes[morpheme.id] = morpheme.id factHasNewMorphemes = True try: ankiFact = ankiFactsDict[fact.ankiFactId] except Exception:continue ankiFact.tags = canonifyTags(deleteTags(u'LxDuplicate', ankiFact.tags)) if factHasNewMorphemes == False: log(str(fact) + " is Duplicate") ankiFact.tags = canonifyTags(addTags(u'LxDuplicate', ankiFact.tags)) ankiFactsId.append(int(ankiFact.id)) log(ankiFactsId) ankiDeck.updateFactTags(ankiFactsId) ankiDeck.save() ankiDeck.close()
def doDeckUpdate(self): # IO () fidDb = self.allDb().fidDb() fid2cardsDb = self.fid2cardsDb() locDb = self.allDb().locDb() rankDb = R.mkRankDb(self.knownDb()) knownDbDb = self.knownDb().db matureDbDb = self.matureDb().db fs = self.getFacts() # cache lookups fieldNames = self.cfg['morph fields'] ipnKnownField = self.cfg['i+N known field'] ipnMatureField = self.cfg['i+N mature field'] unknownsField = self.cfg['unknowns field'] unmaturesField = self.cfg['unmatures field'] vrField = self.cfg['vocab rank field'] mmiField = self.cfg['morph man index field'] ip1knownField = self.cfg['copy i+1 known to'] ip0matField = self.cfg['copy i+0 mature to'] i, lfs = 0, len(fs) start = time.time() fstart = time.time() mmiDict = {} # Map Fact -> MorphManIndex for f in fs: # first get all morphems for this fact ms = set() for fieldName in fieldNames: try: loc = fidDb[(f.id, fieldName)] ms.update(locDb[loc]) except KeyError: continue # now determine unknowns and iPlusN - don't count multiple instances # of a morpheme as an increase in difficulty, so use a set unknowns = set() for m in ms: if m not in knownDbDb: unknowns.add(m) N_k = len(unknowns) unmatures = set() for m in ms: if m not in matureDbDb: unmatures.add(m) N_m = len(unmatures) # determine vocab rank and morph man overall difficulty index vr = R.rankMorphemes(rankDb, ms) mmi = N_k * 10000 + len(ms) * 1000 + vr mmiDict[f] = mmi try: f[ipnKnownField] = u'%d' % N_k except KeyError: pass try: f[ipnMatureField] = u'%d' % N_m except KeyError: pass try: f[mmiField] = u'%d' % mmi except KeyError: pass try: f[vrField] = u'%d' % vr except KeyError: pass try: f[unknownsField] = u','.join(u.base for u in unknowns) except KeyError: pass try: f[unmaturesField] = u','.join(u.base for u in unmatures) except KeyError: pass if len(ms) == 0: f.tags = canonifyTags(addTags(u'noMorphemes', f.tags)) # Help automate vocab card -> sentence card promotion try: f[ip0matField] = u'' except KeyError: pass try: f[ip1knownField] = u'' except KeyError: pass f.tags = canonifyTags( deleteTags(u'ip0mature ip1known notReady', f.tags)) if N_m == 0: # is i+0 mature, make it a sentence card f.tags = canonifyTags( addTags(u'ip0mature ip0matureEver', f.tags)) try: f[ip0matField] = u' '.join(f[name] for name in fieldNames) except KeyError: pass elif N_k == 1: # is i+1 known, make it a vocab card f.tags = canonifyTags(addTags(u'ip1known ip1knownEver', f.tags)) try: f[ip1knownField] = u'%s' % unknowns.pop().base except KeyError: pass else: # is neither, make it a neither card f.tags = canonifyTags(addTags(u'notReady', f.tags)) #f.setModified( textChanged=True, deck=self.deck ) # now display progress i += 1 if i % 100 == 0: fend = time.time() log(' %d / %d = %d%% in %f sec' % (i, lfs, 100. * i / lfs, fend - fstart)) fstart = time.time() # rebuild tags self.deck.updateFactTags((f.id for f in fs)) # sort new cards by Morph Man Index debug('Set fact fields, now changing card creation times for ordering') newtime = 0.0 try: for f in sorted(fs, key=lambda x: mmiDict[x]): for c in fid2cardsDb[f.id]: c.created = newtime newtime += CARD_CREATION_TIME_INC except KeyError: log('! no morph man index field for sorting') # save deck and timestamps self.deck.save() end = time.time() log('Proccessed all facts in %f sec' % (end - start)) self.cfg['last deck update'] = time.time() self.cfg['last deck update took'] = end - start
def doDeckUpdate( self ): # IO () fidDb = self.allDb().fidDb() fid2cardsDb = self.fid2cardsDb() locDb = self.allDb().locDb() rankDb = R.mkRankDb( self.knownDb() ) knownDbDb = self.knownDb().db matureDbDb = self.matureDb().db fs = self.getFacts() # cache lookups fieldNames = self.cfg['morph fields'] ipnKnownField = self.cfg['i+N known field'] ipnMatureField = self.cfg['i+N mature field'] unknownsField = self.cfg['unknowns field'] unmaturesField = self.cfg['unmatures field'] vrField = self.cfg['vocab rank field'] mmiField = self.cfg['morph man index field'] ip1knownField = self.cfg['copy i+1 known to'] ip0matField = self.cfg['copy i+0 mature to'] i, lfs = 0, len(fs) start = time.time() fstart = time.time() mmiDict = {} # Map Fact -> MorphManIndex hasModification = False mod_fs = set() for f in fs: # first get all morphems for this fact ms = set() for fieldName in fieldNames: try: loc = fidDb[ (f.id, fieldName) ] ms.update( locDb[ loc ] ) except KeyError: log ("Error while loading morphems : " + fieldName) continue # now determine unknowns and iPlusN - don't count multiple instances # of a morpheme as an increase in difficulty, so use a set unknowns = set() #log (M.ms2str(ms)) for m in ms: if m not in knownDbDb: unknowns.add( m ) N_k = len( unknowns ) unmatures = set() for m in ms: if m not in matureDbDb: unmatures.add( m ) N_m = len( unmatures ) # determine vocab rank and morph man overall difficulty index vr = R.rankMorphemes( rankDb, ms ) mmi = N_k*1000 + N_m*100 + vr*25 try: if f[ mmiField ] == "" or int(mmi) != int(f[ mmiField ].decode()): #log(f[ mmiField ]) #log("Change ! != " + str(int(mmi))) f.setModified(textChanged=True, deck=self.deck) hasModification = True mod_fs.add(int(f.id)) except KeyError: pass mmiDict[ f ] = mmi try: f[ ipnKnownField ] = u'%d' % N_k except KeyError: pass try: f[ ipnMatureField ] = u'%d' % N_m except KeyError: pass try: f[ mmiField ] = u'%d' % mmi except KeyError: pass try: f[ vrField ] = u'%d' % vr except KeyError: pass #log(','.join( u.base for u in unknowns )) try: f[ unknownsField ] = u','.join( u.base for u in unknowns ) except KeyError: pass #log(','.join( u.base for u in unmatures )) try: f[ unmaturesField ] = u','.join( u.base for u in unmatures ) except KeyError: pass if len(ms) == 0: f.tags = canonifyTags( addTags( u'noMorphemes', f.tags ) ) # Help automate vocab card -> sentence card promotion try: f[ ip0matField ] = u'' except KeyError: pass try: f[ ip1knownField ] = u'' except KeyError: pass #before_tags = f.tags f.tags = canonifyTags( deleteTags( u'ipXmature,ip0mature,ip1known,notReady', f.tags ) ) # Verification du nombre unknows et matures. Si changement alors, mettre la carte en modifier if N_k > 1: #log("Before Tags :" + before_tags) #log("After First Tags: " + f.tags) f.tags = canonifyTags( addTags( u'notReady', f.tags ) ) #log("After Remove Tags: " + f.tags) else: if N_m == 0: f.tags = canonifyTags( addTags( u'ip0mature ip0matureEver', f.tags ) ) try: f[ ip0matField ] = u' '.join( f[ name ] for name in fieldNames ) except KeyError: pass elif N_k == 1: f.tags = canonifyTags( addTags( u'ip1known ip1knownEver', f.tags ) ) try: f[ ip1knownField ] = u'%s' % unknowns.pop().base except KeyError: pass else: f.tags = canonifyTags( addTags( u'ipXmature ipXmatureEver', f.tags ) ) try: f[ ip0matField ] = u' '.join( f[ name ] for name in fieldNames ) except KeyError: pass #if N_m == 0: # is i+0 mature, make it a sentence card # f.tags = canonifyTags( addTags( u'ip0mature ip0matureEver', f.tags ) ) # try: f[ ip0matField ] = u' '.join( f[ name ] for name in fieldNames ) # except KeyError: pass #elif N_k == 1: # is i+1 known, make it a vocab card # f.tags = canonifyTags( addTags( u'ip1known ip1knownEver', f.tags ) ) # try: f[ ip1knownField ] = u'%s' % unknowns.pop().base # except KeyError: pass #else: # is neither, make it a neither card # f.tags = canonifyTags( addTags( u'notReady', f.tags ) ) #f.setModified( textChanged=True, deck=self.deck ) # now display progress i += 1 if i % 100 == 0: fend = time.time() log(' %d / %d = %d%% in %f sec' % ( i, lfs, 100.*i/lfs, fend-fstart ) ) fstart = time.time() # rebuild tags self.deck.updateFactTags( ( f.id for f in fs ) ) # sort new cards by Morph Man Index debug( 'Set fact fields, now changing card creation times for ordering' ) newtime = 472777200.0 # 25 Decembre 1984 :) #newCardIds = set() try: for f in sorted( fs, key=lambda x: mmiDict[ x ] ): for c in fid2cardsDb[ f.id ]: if self.deck.cardIsNew(c): #newCardIds.add(c.id) #c.created = newtime c.due = newtime c.combinedDue = newtime newtime += CARD_CREATION_TIME_INC except KeyError: log( '! no morph man index field for sorting' ) # save deck #self.deck.orderNewCards() if hasModification: self.deck.setModified() self.deck.updateCardQACacheFromIds(mod_fs, type="facts") self.deck.save() # reorder new cards and save # self.deck.orderNewCards() # self.deck.save() end = time.time() log( 'Proccessed all facts in %f sec' % ( end-start ) ) self.cfg['last deck update'] = time.time() self.cfg['last deck update took'] = end-start
def doDeckUpdate( self ): # IO () fidDb = self.allDb().fidDb() fid2cardsDb = self.fid2cardsDb() locDb = self.allDb().locDb() rankDb = R.mkRankDb( self.knownDb() ) knownDbDb = self.knownDb().db matureDbDb = self.matureDb().db fs = self.getFacts() # cache lookups fieldNames = self.cfg['morph fields'] ipnKnownField = self.cfg['i+N known field'] ipnMatureField = self.cfg['i+N mature field'] unknownsField = self.cfg['unknowns field'] unmaturesField = self.cfg['unmatures field'] vrField = self.cfg['vocab rank field'] mmiField = self.cfg['morph man index field'] ip1knownField = self.cfg['copy i+1 known to'] ip0matField = self.cfg['copy i+0 mature to'] i, lfs = 0, len(fs) start = time.time() fstart = time.time() mmiDict = {} # Map Fact -> MorphManIndex for f in fs: # first get all morphems for this fact ms = set() for fieldName in fieldNames: try: loc = fidDb[ (f.id, fieldName) ] ms.update( locDb[ loc ] ) except KeyError: continue # now determine unknowns and iPlusN - don't count multiple instances # of a morpheme as an increase in difficulty, so use a set unknowns = set() for m in ms: if m not in knownDbDb: unknowns.add( m ) N_k = len( unknowns ) unmatures = set() for m in ms: if m not in matureDbDb: unmatures.add( m ) N_m = len( unmatures ) # determine vocab rank and morph man overall difficulty index vr = R.rankMorphemes( rankDb, ms ) mmi = N_k*10000 + len(ms)*1000 + vr mmiDict[ f ] = mmi try: f[ ipnKnownField ] = u'%d' % N_k except KeyError: pass try: f[ ipnMatureField ] = u'%d' % N_m except KeyError: pass try: f[ mmiField ] = u'%d' % mmi except KeyError: pass try: f[ vrField ] = u'%d' % vr except KeyError: pass try: f[ unknownsField ] = u','.join( u.base for u in unknowns ) except KeyError: pass try: f[ unmaturesField ] = u','.join( u.base for u in unmatures ) except KeyError: pass if len(ms) == 0: f.tags = canonifyTags( addTags( u'noMorphemes', f.tags ) ) # Help automate vocab card -> sentence card promotion try: f[ ip0matField ] = u'' except KeyError: pass try: f[ ip1knownField ] = u'' except KeyError: pass f.tags = canonifyTags( deleteTags( u'ip0mature ip1known notReady', f.tags ) ) if N_m == 0: # is i+0 mature, make it a sentence card f.tags = canonifyTags( addTags( u'ip0mature ip0matureEver', f.tags ) ) try: f[ ip0matField ] = u' '.join( f[ name ] for name in fieldNames ) except KeyError: pass elif N_k == 1: # is i+1 known, make it a vocab card f.tags = canonifyTags( addTags( u'ip1known ip1knownEver', f.tags ) ) try: f[ ip1knownField ] = u'%s' % unknowns.pop().base except KeyError: pass else: # is neither, make it a neither card f.tags = canonifyTags( addTags( u'notReady', f.tags ) ) #f.setModified( textChanged=True, deck=self.deck ) # now display progress i += 1 if i % 100 == 0: fend = time.time() log(' %d / %d = %d%% in %f sec' % ( i, lfs, 100.*i/lfs, fend-fstart ) ) fstart = time.time() # rebuild tags self.deck.updateFactTags( ( f.id for f in fs ) ) # sort new cards by Morph Man Index debug( 'Set fact fields, now changing card creation times for ordering' ) newtime = 0.0 try: for f in sorted( fs, key=lambda x: mmiDict[ x ] ): for c in fid2cardsDb[ f.id ]: c.created = newtime newtime += CARD_CREATION_TIME_INC except KeyError: log( '! no morph man index field for sorting' ) # save deck and timestamps self.deck.save() end = time.time() log( 'Proccessed all facts in %f sec' % ( end-start ) ) self.cfg['last deck update'] = time.time() self.cfg['last deck update took'] = end-start
def markFacts(self, language): facts = self.factsService.getAllFactsByLanguage(language) ankiFactsId = list() #i = 0 for fact in facts: try: ankiFact = self.ankiDeckFacts[fact.deck.path][fact.ankiFactId] except Exception: continue deck = self.decks[fact.deck.path] fields = deck.fields morphemes = self.factsService.getMorphemes(fact) morphemesScore, matureMorphemes, knownMorphemes, learnMorphemes, unknownMorphemes = self.getMorphemesScore(morphemes) if fact.statusChanged == True: # FIXME: sure ? if fields[Deck.LEARNX_SCORE_KEY][1]: try: ankiFact[fields[Deck.LEARNX_SCORE_KEY][0]] = u'%d' % int(fact.score) except KeyError: pass if fields[Deck.VOCAB_SCORE_KEY][1]: try: ankiFact[fields[Deck.VOCAB_SCORE_KEY][0]] = u'%d' % int(morphemesScore) except KeyError: pass if fields[Deck.UNKNOWNS_KEY][1]: try: ankiFact[fields[Deck.UNKNOWNS_KEY][0]] = u','.join(u for u in unknownMorphemes) except KeyError: pass if fields[Deck.LEARNTS_KEY][1]: try: ankiFact[fields[Deck.LEARNTS_KEY][0]] = u','.join(u for u in learnMorphemes) except KeyError: pass if fields[Deck.KNOWNS_KEY][1]: try: ankiFact[fields[Deck.KNOWNS_KEY][0]] = u','.join(u for u in knownMorphemes) except KeyError: pass if fields[Deck.MATURES_KEY][1]: try: ankiFact[fields[Deck.MATURES_KEY][0]] = u','.join(u for u in matureMorphemes) except KeyError: pass if len(unknownMorphemes) == 1: if fields[Deck.COPY_UNKNOWN_1_TO_KEY][1]: try: ankiFact[fields[Deck.COPY_UNKNOWN_1_TO_KEY][0]] = u','.join(u for u in unknownMorphemes) except KeyError: pass elif len(unknownMorphemes) == 0: if fields[Deck.COPY_MATURE_TO_KEY][1]: try: ankiFact[fields[Deck.COPY_MATURE_TO_KEY][0]] = u'%s' % ankiFact[deck.expressionField] except KeyError: pass ankiFact.tags = canonifyTags(deleteTags(fact.getAllStatusTag(), ankiFact.tags)) ankiFact.tags = canonifyTags(addTags(fact.getStatusTag(), ankiFact.tags)) if deck.definitionField: try: ankiFact.tags = canonifyTags(deleteTags(u'LxDefKnown,LxDefMatch', ankiFact.tags)) definition = self.factsService.getDefinition(fact) if definition and definition.definitionHash and int(definition.definitionHash) != 0: defMorphemes = self.morphemesService.getMorphemesDefinition(definition) dictMorphemesScore, defMatureMorphemes, defKnownMorphemes, defLearnMorphemes, defUnknownMorphemes = self.getMorphemesScore(defMorphemes) if len(defUnknownMorphemes) == 0: ankiFact.tags = canonifyTags(addTags(u'LxDefKnown', ankiFact.tags)) defKeyMorphemes = self.morphemesService.getMorphemesDefinitionKey(definition) defKeyMorphemesBase = "".join(m.morphLemme.base for m in defKeyMorphemes) if len(unknownMorphemes) == 1 and unknownMorphemes[0] in defKeyMorphemesBase: ankiFact.tags = canonifyTags(addTags(u'LxDefMatch', ankiFact.tags)) if fields[Deck.DEFINITION_SCORE_KEY][1]: try: ankiFact[fields[Deck.DEFINITION_SCORE_KEY][0]] = u'%d' % int(dictMorphemesScore) except KeyError: pass except KeyError: pass ankiFactsId.append(ankiFact.id) self.saveDecks(ankiFactsId, language)