def JxTagDuplicates(Query):
# Query = """select fields.value, facts.id, facts.created, facts.tags from fields,facts,fieldModels,models where
# facts.id = fields.factId and fields.fieldModelId = fieldModels.id and facts.modelId = models.id and
# fieldModels.name = "Expression" and models.tags like "%Japanese%" group by facts.id order by fields.value """
Rows = mw.deck.s.all(Query)
Seen={}
Duplicates = {}
for (Field,Id,Time,Tags) in Rows:
Entry = Field.strip(u' ')
if Entry in Seen:
(lastId,lastTime,lastTags) = Seen[Entry]
if lastTime > Time:
# first duplicate younger
Duplicates[lastId] = Entry
Seen[Entry] = (Id,Time,canonifyTags(addTags(lastTags,Tags)))
else:
# second duplicate younger
Duplicates[Id] = Entry
Seen[Entry] = (lastId,lastTime,canonifyTags(addTags(lastTags,Tags)))
else:
Seen[Entry] = (Id,Time,Tags)
MasterDuplicates = []
Html = u"""<style> li {font-size: x-large;}</style><h2>Duplicated Entries</h2><ul>"""
for Entry in Duplicates.values():
(Id,Time,Tags) = Seen[Entry]
MasterDuplicates.append(Id)
Html += u"""<li>%s</li>""" % Entry
mw.deck.s.statement("update facts set tags = :tags, modified = :t where id =:id",id=Id, t=time.time(),tags=canonifyTags(Tags))
Html += u"""</ul>"""
mw.deck.addTags(MasterDuplicates,u"JxMasterDuplicate")
mw.deck.addTags(Duplicates.keys(),u"JxDuplicate")
mw.deck.deleteTags(MasterDuplicates,u"JxDuplicate")
mw.help.showText(Html)
def doMark(self):
if self.currentCard.hasTag("Marked"):
self.currentCard.fact.tags = canonifyTags(deleteTags(
"Marked", self.currentCard.fact.tags))
else:
self.currentCard.fact.tags = canonifyTags(addTags(
"Marked", self.currentCard.fact.tags))
self.currentCard.fact.setModified(textChanged=True)
self.deck.updateFactTags([self.currentCard.fact.id])
for card in self.currentCard.fact.cards:
self.deck.updatePriority(card)
self.deck.setModified()
def update_fact_with_node(self, somefact, mydeck=None):
# update fact fields
for k, v in self.node_into_fields().iteritems():
try:
# db access layer complains about this
if isinstance(v, str):
v = unicode(v)
somefact[k] = v
except KeyError:
logging.critical("anki is complaining that %s does not exist" %
k)
raise
# update tags
somefact.tags = canonifyTags(self.get_tags())
somefact.setModified(textChanged=True)
# update the deck. needed if updating cards
if mydeck is not None: # should be OK if we're using this function to add a new fact
mydeck.updateFactTags([somefact.id])
for card in somefact.cards:
mydeck.updatePriority(card)
self.update_num_fact_cards(somefact, mydeck)
def update_fact_with_node(self, somefact, mydeck=None):
# update fact fields
for k, v in self.node_into_fields().iteritems():
try:
# db access layer complains about this
if isinstance(v, str):
v = unicode(v)
somefact[k] = v
except KeyError:
logging.critical("anki is complaining that %s does not exist" % k)
raise
# update tags
somefact.tags = canonifyTags(self.get_tags())
somefact.setModified(textChanged=True)
# update the deck. needed if updating cards
if mydeck is not None: # should be OK if we're using this function to add a new fact
mydeck.updateFactTags([somefact.id])
for card in somefact.cards:
mydeck.updatePriority(card)
self.update_num_fact_cards(somefact, mydeck)
def moveTags():
for fact in mw.deck.s.query(Fact).all():
old = fact.tags
fact.tags = canonifyTags(fact.tags + "," +
",".join([c.tags for c in fact.cards]))
fact.setModified()
mw.deck.setModified()
mw.reset()
def moveTags():
for fact in mw.deck.s.query(Fact).all():
old = fact.tags
fact.tags = canonifyTags(fact.tags + "," + ",".join(
[c.tags for c in fact.cards]))
fact.setModified()
mw.deck.setModified()
mw.reset()
def addCards(self, cards):
"Add facts in bulk from foreign cards."
# map tags field to attr
try:
idx = self.mapping.index(0)
for c in cards:
c.tags += " " + c.fields[idx]
except ValueError:
pass
# add facts
self.deck.updateProgress()
factIds = [genID() for n in range(len(cards))]
self.deck.s.execute(factsTable.insert(),
[{'modelId': self.model.id,
'tags': canonifyTags(self.tagsToAdd + " " + cards[n].tags),
'id': factIds[n]} for n in range(len(cards))])
self.deck.factCount += len(factIds)
self.deck.s.execute("""
delete from factsDeleted
where factId in (%s)""" % ",".join([str(s) for s in factIds]))
# add all the fields
self.deck.updateProgress()
for fm in self.model.fieldModels:
try:
index = self.mapping.index(fm)
except ValueError:
index = None
data = [{'factId': factIds[m],
'fieldModelId': fm.id,
'ordinal': fm.ordinal,
'id': genID(),
'value': (index is not None and
cards[m].fields[index] or u"")}
for m in range(len(cards))]
self.deck.s.execute(fieldsTable.insert(),
data)
# and cards
self.deck.updateProgress()
now = time.time()
active = 0
for cm in self.model.cardModels:
self._now = now
if cm.active:
active += 1
data = [self.addMeta({
'id': genID(),
'factId': factIds[m],
'cardModelId': cm.id,
'ordinal': cm.ordinal,
'question': u"",
'answer': u"",
'type': 2},cards[m]) for m in range(len(cards))]
self.deck.s.execute(cardsTable.insert(),
data)
self.deck.updateProgress()
self.deck.updateCardsFromFactIds(factIds)
self.deck.cardCount += len(cards) * active
self.total = len(factIds)
def markAndDelete():
undo = _("MarkDelete")
mw.deck.setUndoStart(undo)
mw.currentCard.fact.tags = canonifyTags(mw.currentCard.fact.tags +
"," + "MarkDelete")
mw.currentCard.fact.setModified()
mw.deck.updateFactTags([mw.currentCard.fact.id])
mw.deck.deleteCard(mw.currentCard.id)
mw.reset()
mw.deck.setUndoEnd(undo)
def markAndDelete():
undo = _("MarkDelete")
mw.deck.setUndoStart(undo)
mw.currentCard.fact.tags = canonifyTags(mw.currentCard.fact.tags + "," +
"MarkDelete")
mw.currentCard.fact.setModified()
mw.deck.updateFactTags([mw.currentCard.fact.id])
mw.deck.deleteCard(mw.currentCard.id)
mw.reset()
mw.deck.setUndoEnd(undo)
def onTagChange(self):
if not self.fact:
return
old = self.fact.tags
self.fact.tags = canonifyTags(unicode(self.tags.text()))
if old != self.fact.tags:
self.deck.s.flush()
self.deck.updateFactTags([self.fact.id])
self.deck.updatePriorities([c.id for c in self.fact.cards])
self.fact.setModified(textChanged=True)
self.deck.flushMod()
if self.onChange:
self.onChange("tag")
def markDuplicateFacts(self, deck):
ankiDeck = AnkiHelper.getDeck(deck.path)
cards = self.factsService.getAllCardsOrderByScore(deck = deck)
ankiFactsId = list()
ankiFacts = AnkiHelper.getFacts(ankiDeck)
ankiFactsDict = dict()
for ankiFact in ankiFacts:
ankiFactsDict[ankiFact.id] = ankiFact
uniqueMorphemes = dict()
#i = 0
for card in cards:
fact = self.factsService.getFactById(card.factId)
morphemes = self.factsService.getMorphemes(fact)
factHasNewMorphemes = False
for morpheme in morphemes:
if morpheme.id not in uniqueMorphemes:
uniqueMorphemes[morpheme.id] = morpheme.id
factHasNewMorphemes = True
try:
ankiFact = ankiFactsDict[fact.ankiFactId]
except Exception:continue
ankiFact.tags = canonifyTags(deleteTags(u'LxDuplicate', ankiFact.tags))
if factHasNewMorphemes == False:
log(str(fact) + " is Duplicate")
ankiFact.tags = canonifyTags(addTags(u'LxDuplicate', ankiFact.tags))
ankiFactsId.append(int(ankiFact.id))
log(ankiFactsId)
ankiDeck.updateFactTags(ankiFactsId)
ankiDeck.save()
ankiDeck.close()
def cardIsUnique(self, card):
fieldsAsTags = []
for n in range(len(self.mapping)):
if self.mapping[n] and self.mapping[n].unique:
if card.fields[n] in self.uniqueCache[self.mapping[n].id]:
if not self.tagDuplicates:
self.log.append("Fact has duplicate '%s': %s" % (self.mapping[n].name, ", ".join(card.fields)))
return False
fieldsAsTags.append(self.mapping[n].name.replace(" ", "-"))
else:
self.uniqueCache[self.mapping[n].id][card.fields[n]] = 1
if fieldsAsTags:
card.tags += u" Duplicate:" + ("+".join(fieldsAsTags))
card.tags = canonifyTags(card.tags)
return True
def accept(self):
self.hide()
self.deck.startProgress()
n = 0
suspended = []
for item in self.items:
idx = self.dialog.list.indexFromItem(item)
if self.dialog.list.selectionModel().isSelected(idx):
suspended.append(self.tags[n])
n += 1
self.deck.suspended = canonifyTags(joinTags(suspended))
self.deck.setModified()
self.deck.updateAllPriorities(partial=True, dirty=False)
self.deck.finishProgress()
QDialog.accept(self)
def cardIsUnique(self, card):
fieldsAsTags = []
for n in range(len(self.mapping)):
if self.mapping[n] and self.mapping[n].unique:
if card.fields[n] in self.uniqueCache[self.mapping[n].id]:
if not self.tagDuplicates:
self.log.append(
"Fact has duplicate '%s': %s" %
(self.mapping[n].name, ", ".join(card.fields)))
return False
fieldsAsTags.append(self.mapping[n].name.replace(" ", "-"))
else:
self.uniqueCache[self.mapping[n].id][card.fields[n]] = 1
if fieldsAsTags:
card.tags += u" Duplicate:" + ("+".join(fieldsAsTags))
card.tags = canonifyTags(card.tags)
return True
def doUpdate(self):
# grab the data from the external file
cards = self.foreignCards()
# grab data from db
fields = self.deck.db.all("""
select factId, value from fields where fieldModelId = :id
and value != ''""",
id=self.updateKey[1])
# hash it
vhash = {}
fids = []
for (fid, val) in fields:
fids.append(fid)
vhash[val] = fid
# prepare tags
tagsIdx = None
try:
tagsIdx = self.mapping.index(0)
for c in cards:
c.tags = canonifyTags(self.tagsToAdd + " " + c.fields[tagsIdx])
except ValueError:
pass
# look for matches
upcards = []
newcards = []
for c in cards:
v = c.fields[self.updateKey[0]]
if v in vhash:
# ignore empty keys
if v:
# fid, card
upcards.append((vhash[v], c))
else:
newcards.append(c)
# update fields
for fm in self.model.fieldModels:
if fm.id == self.updateKey[1]:
# don't update key
continue
try:
index = self.mapping.index(fm)
except ValueError:
# not mapped
continue
data = [{'fid': fid,
'fmid': fm.id,
'v': c.fields[index],
'chk': self.maybeChecksum(c.fields[index], fm.unique)}
for (fid, c) in upcards]
self.deck.db.execute("""
update fields set value = :v, chksum = :chk where factId = :fid
and fieldModelId = :fmid""", data)
# update tags
if tagsIdx is not None:
data = [{'fid': fid,
't': c.fields[tagsIdx]}
for (fid, c) in upcards]
self.deck.db.execute(
"update facts set tags = :t where id = :fid",
data)
# rebuild caches
cids = self.deck.db.column0(
"select id from cards where factId in %s" %
ids2str(fids))
self.deck.updateCardTags(cids)
self.deck.updateCardsFromFactIds(fids)
self.total = len(cards)
self.deck.setModified()
def doUpdate(self):
# grab the data from the external file
cards = self.foreignCards()
# grab data from db
fields = self.deck.db.all("""
select factId, value from fields where fieldModelId = :id
and value != ''""",
id=self.updateKey[1])
# hash it
vhash = {}
fids = []
for (fid, val) in fields:
fids.append(fid)
vhash[val] = fid
# prepare tags
tagsIdx = None
try:
tagsIdx = self.mapping.index(0)
for c in cards:
c.tags = canonifyTags(self.tagsToAdd + " " + c.fields[tagsIdx])
except ValueError:
pass
# look for matches
upcards = []
newcards = []
for c in cards:
v = c.fields[self.updateKey[0]]
if v in vhash:
# ignore empty keys
if v:
# fid, card
upcards.append((vhash[v], c))
else:
newcards.append(c)
# update fields
for fm in self.model.fieldModels:
if fm.id == self.updateKey[1]:
# don't update key
continue
try:
index = self.mapping.index(fm)
except ValueError:
# not mapped
continue
data = [{
'fid': fid,
'fmid': fm.id,
'v': c.fields[index],
'chk': self.maybeChecksum(c.fields[index], fm.unique)
} for (fid, c) in upcards]
self.deck.db.execute(
"""
update fields set value = :v, chksum = :chk where factId = :fid
and fieldModelId = :fmid""", data)
# update tags
if tagsIdx is not None:
data = [{
'fid': fid,
't': c.fields[tagsIdx]
} for (fid, c) in upcards]
self.deck.db.execute("update facts set tags = :t where id = :fid",
data)
# rebuild caches
cids = self.deck.db.column0("select id from cards where factId in %s" %
ids2str(fids))
self.deck.updateCardTags(cids)
self.deck.updateCardsFromFactIds(fids)
self.total = len(cards)
self.deck.setModified()
def addCards(self, cards):
"Add facts in bulk from foreign cards."
# map tags field to attr
try:
idx = self.mapping.index(0)
for c in cards:
c.tags += " " + c.fields[idx]
except ValueError:
pass
# add facts
factIds = [genID() for n in range(len(cards))]
factCreated = {}
def fudgeCreated(d, tmp=[]):
if not tmp:
tmp.append(time.time())
else:
tmp[0] += 0.0001
d['created'] = tmp[0]
factCreated[d['id']] = d['created']
return d
self.deck.db.execute(factsTable.insert(), [
fudgeCreated(
{
'modelId': self.model.id,
'tags': canonifyTags(self.tagsToAdd + " " + cards[n].tags),
'id': factIds[n]
}) for n in range(len(cards))
])
self.deck.db.execute("""
delete from factsDeleted
where factId in (%s)""" % ",".join([str(s) for s in factIds]))
# add all the fields
for fm in self.model.fieldModels:
try:
index = self.mapping.index(fm)
except ValueError:
index = None
data = [{
'factId':
factIds[m],
'fieldModelId':
fm.id,
'ordinal':
fm.ordinal,
'id':
genID(),
'value': (index is not None and cards[m].fields[index] or u""),
'chksum':
self.maybeChecksum(
index is not None and cards[m].fields[index] or u"",
fm.unique)
} for m in range(len(cards))]
self.deck.db.execute(fieldsTable.insert(), data)
# and cards
active = 0
for cm in self.model.cardModels:
if cm.active:
active += 1
data = [
self.addMeta(
{
'id': genID(),
'factId': factIds[m],
'factCreated': factCreated[factIds[m]],
'cardModelId': cm.id,
'ordinal': cm.ordinal,
'question': u"",
'answer': u""
}, cards[m]) for m in range(len(cards))
]
self.deck.db.execute(cardsTable.insert(), data)
self.deck.updateCardsFromFactIds(factIds)
self.total = len(factIds)
def doDeckUpdate(self): # IO ()
fidDb = self.allDb().fidDb()
fid2cardsDb = self.fid2cardsDb()
locDb = self.allDb().locDb()
rankDb = R.mkRankDb(self.knownDb())
knownDbDb = self.knownDb().db
matureDbDb = self.matureDb().db
fs = self.getFacts()
# cache lookups
fieldNames = self.cfg['morph fields']
ipnKnownField = self.cfg['i+N known field']
ipnMatureField = self.cfg['i+N mature field']
unknownsField = self.cfg['unknowns field']
unmaturesField = self.cfg['unmatures field']
vrField = self.cfg['vocab rank field']
mmiField = self.cfg['morph man index field']
ip1knownField = self.cfg['copy i+1 known to']
ip0matField = self.cfg['copy i+0 mature to']
i, lfs = 0, len(fs)
start = time.time()
fstart = time.time()
mmiDict = {} # Map Fact -> MorphManIndex
for f in fs:
# first get all morphems for this fact
ms = set()
for fieldName in fieldNames:
try:
loc = fidDb[(f.id, fieldName)]
ms.update(locDb[loc])
except KeyError:
continue
# now determine unknowns and iPlusN - don't count multiple instances
# of a morpheme as an increase in difficulty, so use a set
unknowns = set()
for m in ms:
if m not in knownDbDb:
unknowns.add(m)
N_k = len(unknowns)
unmatures = set()
for m in ms:
if m not in matureDbDb:
unmatures.add(m)
N_m = len(unmatures)
# determine vocab rank and morph man overall difficulty index
vr = R.rankMorphemes(rankDb, ms)
mmi = N_k * 10000 + len(ms) * 1000 + vr
mmiDict[f] = mmi
try:
f[ipnKnownField] = u'%d' % N_k
except KeyError:
pass
try:
f[ipnMatureField] = u'%d' % N_m
except KeyError:
pass
try:
f[mmiField] = u'%d' % mmi
except KeyError:
pass
try:
f[vrField] = u'%d' % vr
except KeyError:
pass
try:
f[unknownsField] = u','.join(u.base for u in unknowns)
except KeyError:
pass
try:
f[unmaturesField] = u','.join(u.base for u in unmatures)
except KeyError:
pass
if len(ms) == 0:
f.tags = canonifyTags(addTags(u'noMorphemes', f.tags))
# Help automate vocab card -> sentence card promotion
try:
f[ip0matField] = u''
except KeyError:
pass
try:
f[ip1knownField] = u''
except KeyError:
pass
f.tags = canonifyTags(
deleteTags(u'ip0mature ip1known notReady', f.tags))
if N_m == 0: # is i+0 mature, make it a sentence card
f.tags = canonifyTags(
addTags(u'ip0mature ip0matureEver', f.tags))
try:
f[ip0matField] = u' '.join(f[name] for name in fieldNames)
except KeyError:
pass
elif N_k == 1: # is i+1 known, make it a vocab card
f.tags = canonifyTags(addTags(u'ip1known ip1knownEver',
f.tags))
try:
f[ip1knownField] = u'%s' % unknowns.pop().base
except KeyError:
pass
else: # is neither, make it a neither card
f.tags = canonifyTags(addTags(u'notReady', f.tags))
#f.setModified( textChanged=True, deck=self.deck )
# now display progress
i += 1
if i % 100 == 0:
fend = time.time()
log(' %d / %d = %d%% in %f sec' %
(i, lfs, 100. * i / lfs, fend - fstart))
fstart = time.time()
# rebuild tags
self.deck.updateFactTags((f.id for f in fs))
# sort new cards by Morph Man Index
debug('Set fact fields, now changing card creation times for ordering')
newtime = 0.0
try:
for f in sorted(fs, key=lambda x: mmiDict[x]):
for c in fid2cardsDb[f.id]:
c.created = newtime
newtime += CARD_CREATION_TIME_INC
except KeyError:
log('! no morph man index field for sorting')
# save deck and timestamps
self.deck.save()
end = time.time()
log('Proccessed all facts in %f sec' % (end - start))
self.cfg['last deck update'] = time.time()
self.cfg['last deck update took'] = end - start
def addCards(self, cards):
"Add facts in bulk from foreign cards."
# map tags field to attr
try:
idx = self.mapping.index(0)
for c in cards:
c.tags += " " + c.fields[idx]
except ValueError:
pass
# add facts
factIds = [genID() for n in range(len(cards))]
factCreated = {}
def fudgeCreated(d, tmp=[]):
if not tmp:
tmp.append(time.time())
else:
tmp[0] += 0.0001
d['created'] = tmp[0]
factCreated[d['id']] = d['created']
return d
self.deck.db.execute(factsTable.insert(),
[fudgeCreated({'modelId': self.model.id,
'tags': canonifyTags(self.tagsToAdd + " " + cards[n].tags),
'id': factIds[n]}) for n in range(len(cards))])
self.deck.db.execute("""
delete from factsDeleted
where factId in (%s)""" % ",".join([str(s) for s in factIds]))
# add all the fields
for fm in self.model.fieldModels:
try:
index = self.mapping.index(fm)
except ValueError:
index = None
data = [{'factId': factIds[m],
'fieldModelId': fm.id,
'ordinal': fm.ordinal,
'id': genID(),
'value': (index is not None and
cards[m].fields[index] or u""),
'chksum': self.maybeChecksum(
index is not None and
cards[m].fields[index] or u"", fm.unique)
}
for m in range(len(cards))]
self.deck.db.execute(fieldsTable.insert(),
data)
# and cards
active = 0
for cm in self.model.cardModels:
if cm.active:
active += 1
data = [self.addMeta({
'id': genID(),
'factId': factIds[m],
'factCreated': factCreated[factIds[m]],
'cardModelId': cm.id,
'ordinal': cm.ordinal,
'question': u"",
'answer': u""
},cards[m]) for m in range(len(cards))]
self.deck.db.execute(cardsTable.insert(),
data)
self.deck.updateCardsFromFactIds(factIds)
self.total = len(factIds)
def doDeckUpdate( self ): # IO ()
fidDb = self.allDb().fidDb()
fid2cardsDb = self.fid2cardsDb()
locDb = self.allDb().locDb()
rankDb = R.mkRankDb( self.knownDb() )
knownDbDb = self.knownDb().db
matureDbDb = self.matureDb().db
fs = self.getFacts()
# cache lookups
fieldNames = self.cfg['morph fields']
ipnKnownField = self.cfg['i+N known field']
ipnMatureField = self.cfg['i+N mature field']
unknownsField = self.cfg['unknowns field']
unmaturesField = self.cfg['unmatures field']
vrField = self.cfg['vocab rank field']
mmiField = self.cfg['morph man index field']
ip1knownField = self.cfg['copy i+1 known to']
ip0matField = self.cfg['copy i+0 mature to']
i, lfs = 0, len(fs)
start = time.time()
fstart = time.time()
mmiDict = {} # Map Fact -> MorphManIndex
hasModification = False
mod_fs = set()
for f in fs:
# first get all morphems for this fact
ms = set()
for fieldName in fieldNames:
try:
loc = fidDb[ (f.id, fieldName) ]
ms.update( locDb[ loc ] )
except KeyError:
log ("Error while loading morphems : " + fieldName)
continue
# now determine unknowns and iPlusN - don't count multiple instances
# of a morpheme as an increase in difficulty, so use a set
unknowns = set()
#log (M.ms2str(ms))
for m in ms:
if m not in knownDbDb:
unknowns.add( m )
N_k = len( unknowns )
unmatures = set()
for m in ms:
if m not in matureDbDb:
unmatures.add( m )
N_m = len( unmatures )
# determine vocab rank and morph man overall difficulty index
vr = R.rankMorphemes( rankDb, ms )
mmi = N_k*1000 + N_m*100 + vr*25
try:
if f[ mmiField ] == "" or int(mmi) != int(f[ mmiField ].decode()):
#log(f[ mmiField ])
#log("Change ! != " + str(int(mmi)))
f.setModified(textChanged=True, deck=self.deck)
hasModification = True
mod_fs.add(int(f.id))
except KeyError: pass
mmiDict[ f ] = mmi
try: f[ ipnKnownField ] = u'%d' % N_k
except KeyError: pass
try: f[ ipnMatureField ] = u'%d' % N_m
except KeyError: pass
try: f[ mmiField ] = u'%d' % mmi
except KeyError: pass
try: f[ vrField ] = u'%d' % vr
except KeyError: pass
#log(','.join( u.base for u in unknowns ))
try: f[ unknownsField ] = u','.join( u.base for u in unknowns )
except KeyError: pass
#log(','.join( u.base for u in unmatures ))
try: f[ unmaturesField ] = u','.join( u.base for u in unmatures )
except KeyError: pass
if len(ms) == 0:
f.tags = canonifyTags( addTags( u'noMorphemes', f.tags ) )
# Help automate vocab card -> sentence card promotion
try: f[ ip0matField ] = u''
except KeyError: pass
try: f[ ip1knownField ] = u''
except KeyError: pass
#before_tags = f.tags
f.tags = canonifyTags( deleteTags( u'ipXmature,ip0mature,ip1known,notReady', f.tags ) )
# Verification du nombre unknows et matures. Si changement alors, mettre la carte en modifier
if N_k > 1:
#log("Before Tags :" + before_tags)
#log("After First Tags: " + f.tags)
f.tags = canonifyTags( addTags( u'notReady', f.tags ) )
#log("After Remove Tags: " + f.tags)
else:
if N_m == 0:
f.tags = canonifyTags( addTags( u'ip0mature ip0matureEver', f.tags ) )
try: f[ ip0matField ] = u' '.join( f[ name ] for name in fieldNames )
except KeyError: pass
elif N_k == 1:
f.tags = canonifyTags( addTags( u'ip1known ip1knownEver', f.tags ) )
try: f[ ip1knownField ] = u'%s' % unknowns.pop().base
except KeyError: pass
else:
f.tags = canonifyTags( addTags( u'ipXmature ipXmatureEver', f.tags ) )
try: f[ ip0matField ] = u' '.join( f[ name ] for name in fieldNames )
except KeyError: pass
#if N_m == 0: # is i+0 mature, make it a sentence card
# f.tags = canonifyTags( addTags( u'ip0mature ip0matureEver', f.tags ) )
# try: f[ ip0matField ] = u' '.join( f[ name ] for name in fieldNames )
# except KeyError: pass
#elif N_k == 1: # is i+1 known, make it a vocab card
# f.tags = canonifyTags( addTags( u'ip1known ip1knownEver', f.tags ) )
# try: f[ ip1knownField ] = u'%s' % unknowns.pop().base
# except KeyError: pass
#else: # is neither, make it a neither card
# f.tags = canonifyTags( addTags( u'notReady', f.tags ) )
#f.setModified( textChanged=True, deck=self.deck )
# now display progress
i += 1
if i % 100 == 0:
fend = time.time()
log(' %d / %d = %d%% in %f sec' % ( i, lfs, 100.*i/lfs, fend-fstart ) )
fstart = time.time()
# rebuild tags
self.deck.updateFactTags( ( f.id for f in fs ) )
# sort new cards by Morph Man Index
debug( 'Set fact fields, now changing card creation times for ordering' )
newtime = 472777200.0 # 25 Decembre 1984 :)
#newCardIds = set()
try:
for f in sorted( fs, key=lambda x: mmiDict[ x ] ):
for c in fid2cardsDb[ f.id ]:
if self.deck.cardIsNew(c):
#newCardIds.add(c.id)
#c.created = newtime
c.due = newtime
c.combinedDue = newtime
newtime += CARD_CREATION_TIME_INC
except KeyError:
log( '! no morph man index field for sorting' )
# save deck
#self.deck.orderNewCards()
if hasModification:
self.deck.setModified()
self.deck.updateCardQACacheFromIds(mod_fs, type="facts")
self.deck.save()
# reorder new cards and save
# self.deck.orderNewCards()
# self.deck.save()
end = time.time()
log( 'Proccessed all facts in %f sec' % ( end-start ) )
self.cfg['last deck update'] = time.time()
self.cfg['last deck update took'] = end-start
def stringTags(self):
return canonifyTags(self.tags)
def doDeckUpdate( self ): # IO ()
fidDb = self.allDb().fidDb()
fid2cardsDb = self.fid2cardsDb()
locDb = self.allDb().locDb()
rankDb = R.mkRankDb( self.knownDb() )
knownDbDb = self.knownDb().db
matureDbDb = self.matureDb().db
fs = self.getFacts()
# cache lookups
fieldNames = self.cfg['morph fields']
ipnKnownField = self.cfg['i+N known field']
ipnMatureField = self.cfg['i+N mature field']
unknownsField = self.cfg['unknowns field']
unmaturesField = self.cfg['unmatures field']
vrField = self.cfg['vocab rank field']
mmiField = self.cfg['morph man index field']
ip1knownField = self.cfg['copy i+1 known to']
ip0matField = self.cfg['copy i+0 mature to']
i, lfs = 0, len(fs)
start = time.time()
fstart = time.time()
mmiDict = {} # Map Fact -> MorphManIndex
for f in fs:
# first get all morphems for this fact
ms = set()
for fieldName in fieldNames:
try:
loc = fidDb[ (f.id, fieldName) ]
ms.update( locDb[ loc ] )
except KeyError:
continue
# now determine unknowns and iPlusN - don't count multiple instances
# of a morpheme as an increase in difficulty, so use a set
unknowns = set()
for m in ms:
if m not in knownDbDb:
unknowns.add( m )
N_k = len( unknowns )
unmatures = set()
for m in ms:
if m not in matureDbDb:
unmatures.add( m )
N_m = len( unmatures )
# determine vocab rank and morph man overall difficulty index
vr = R.rankMorphemes( rankDb, ms )
mmi = N_k*10000 + len(ms)*1000 + vr
mmiDict[ f ] = mmi
try: f[ ipnKnownField ] = u'%d' % N_k
except KeyError: pass
try: f[ ipnMatureField ] = u'%d' % N_m
except KeyError: pass
try: f[ mmiField ] = u'%d' % mmi
except KeyError: pass
try: f[ vrField ] = u'%d' % vr
except KeyError: pass
try: f[ unknownsField ] = u','.join( u.base for u in unknowns )
except KeyError: pass
try: f[ unmaturesField ] = u','.join( u.base for u in unmatures )
except KeyError: pass
if len(ms) == 0:
f.tags = canonifyTags( addTags( u'noMorphemes', f.tags ) )
# Help automate vocab card -> sentence card promotion
try: f[ ip0matField ] = u''
except KeyError: pass
try: f[ ip1knownField ] = u''
except KeyError: pass
f.tags = canonifyTags( deleteTags( u'ip0mature ip1known notReady', f.tags ) )
if N_m == 0: # is i+0 mature, make it a sentence card
f.tags = canonifyTags( addTags( u'ip0mature ip0matureEver', f.tags ) )
try: f[ ip0matField ] = u' '.join( f[ name ] for name in fieldNames )
except KeyError: pass
elif N_k == 1: # is i+1 known, make it a vocab card
f.tags = canonifyTags( addTags( u'ip1known ip1knownEver', f.tags ) )
try: f[ ip1knownField ] = u'%s' % unknowns.pop().base
except KeyError: pass
else: # is neither, make it a neither card
f.tags = canonifyTags( addTags( u'notReady', f.tags ) )
#f.setModified( textChanged=True, deck=self.deck )
# now display progress
i += 1
if i % 100 == 0:
fend = time.time()
log(' %d / %d = %d%% in %f sec' % ( i, lfs, 100.*i/lfs, fend-fstart ) )
fstart = time.time()
# rebuild tags
self.deck.updateFactTags( ( f.id for f in fs ) )
# sort new cards by Morph Man Index
debug( 'Set fact fields, now changing card creation times for ordering' )
newtime = 0.0
try:
for f in sorted( fs, key=lambda x: mmiDict[ x ] ):
for c in fid2cardsDb[ f.id ]:
c.created = newtime
newtime += CARD_CREATION_TIME_INC
except KeyError:
log( '! no morph man index field for sorting' )
# save deck and timestamps
self.deck.save()
end = time.time()
log( 'Proccessed all facts in %f sec' % ( end-start ) )
self.cfg['last deck update'] = time.time()
self.cfg['last deck update took'] = end-start
def stringTags(self):
return canonifyTags(self.tags)
def importIknowItem(item, sentenceModel, vocabModel, importSettings, progress):
idQuery = mw.deck.s.query(Field).filter_by(value=item.uniqIdStr())
field = idQuery.first()
if field:
return False#is duplicate of an existing iknow ID, so return immediately
try:
expQuery = mw.deck.s.query(Field).filter_by(value=item.expression)
field = expQuery.first()
if field:
return False #duplicate expression, the definition may be different but we don't want dup cards
except:
pass #might fail due to null expression which we catch other ways
if item.type == "item":
model = vocabModel
elif item.type == "sentence":
model = sentenceModel
fact = mw.deck.newFact(model)
if importSettings.tagsOnImport and len(importSettings.tagsOnImport.strip()) > 0:
fact.tags = canonifyTags(importSettings.tagsOnImport)
fact['iKnowID'] = item.uniqIdStr()
fact['Expression'] = item.expression
fact['Meaning'] = item.meaning
fact['iKnowType'] = item.type
fact['Reading'] = item.reading
if item.image_uri:
if DOWNLOAD_IMAGES:
#downloading images from smart.fm may be prohibited by smart.fm in certain cases. please only enable DOWNLOAD_IMAGES if you are sure that the list you are downloading has public domain (non-copyrighted or free for use) images.
(filePath, headers) = urllib.urlretrieve(item.image_uri)
path = mw.deck.addMedia(filePath)
fact['Image_URI'] = u'<img src="%s" alt="[No Image]">' % item.image_uri
else:
fact['Image_URI'] = u'<img src="%s" alt="[No Image]">' % item.image_uri
else:
fact['Image_URI'] = u""
if importSettings.downloadAudio and item.audio_uri:
tries = 0
gotAudioForItem = False
gotItemWithNoData = False
filePath = None
headers = None
path = None
# make several attempts to download audio from smart.fm
while not gotAudioForItem and tries < 3:
tries += 1
try:
(filePath, headers) = urllib.urlretrieve(item.audio_uri)
if os.path.exists(filePath) and os.path.getsize(filePath) > 0:
gotAudioForItem = True
elif not os.path.exists(filePath):
progress.logMsg("WARN INFO: downloading URL %s with urlretrieve but the returned path %s does not exist." % (item.audio_uri.encode('utf-8'), filePath.encode('utf-8')))
else:
progress.logMsg("WARN INFO: downloading URL %s with urlretrieve but the returned path %s has no data." % (item.audio_uri.encode('utf-8'), filePath.encode('utf-8')))
gotItemWithNoData = True
except IOError:
progress.logMsg("ERROR INFO: downloading URL %s but got IOError: %s" % (item.audio_uri.encode('utf-8'), traceback.format_exc().encode('utf-8')))
if not gotAudioForItem and not gotItemWithNoData:
#sometimes smart.fm reports that an item has an audio file, gives its URL, and even serves a response from that URL - except the URL has 0 bytes of data. so it downloads as a blank file. in this infrequent case, don't raise an error but just skip the audio.
raise AudioDownloadError, "Failed to get audio for an item after 3 tries, cancelling import. Error with URI %s" % item.audio_uri
if gotItemWithNoData:
progress.logMsg("WARN INFO: skipping audio for %s, file had 0 bytes on smart.fm." % item.uniqIdStr())
if gotAudioForItem:
# now try to add the media file to the deck
try:
path = mw.deck.addMedia(filePath)
progress.logMsg("Successfully added media from URI %s which was downloaded to %s to the media dir at %s" % (item.audio_uri.encode('utf-8'), filePath.encode('utf-8'), mw.deck.mediaDir().encode('utf-8')))
except:
raise AddMediaException, ("Failed to add media from URI %s \nwhich was downloaded to %s \nto the media dir at %s" % (item.audio_uri.encode('utf-8'), filePath.encode('utf-8'), mw.deck.mediaDir().encode('utf-8'))) + traceback.format_exc().encode('utf-8')
fact['Audio'] = u"[sound:%s]" % path
elif item.audio_uri:
fact['Audio'] = u"[sound:%s]" % item.audio_uri
else:
fact['Audio'] = u""
newfact = mw.deck.addFact(fact)
cardIds = [card.id for card in newfact.cards]
if importSettings.rescheduleMin and importSettings.rescheduleMax:
mw.deck.rescheduleCards(cardIds, importSettings.rescheduleMin, importSettings.rescheduleMax)
mw.deck.save()
return True
def markFacts(self, language):
facts = self.factsService.getAllFactsByLanguage(language)
ankiFactsId = list()
#i = 0
for fact in facts:
try:
ankiFact = self.ankiDeckFacts[fact.deck.path][fact.ankiFactId]
except Exception: continue
deck = self.decks[fact.deck.path]
fields = deck.fields
morphemes = self.factsService.getMorphemes(fact)
morphemesScore, matureMorphemes, knownMorphemes, learnMorphemes, unknownMorphemes = self.getMorphemesScore(morphemes)
if fact.statusChanged == True: # FIXME: sure ?
if fields[Deck.LEARNX_SCORE_KEY][1]:
try: ankiFact[fields[Deck.LEARNX_SCORE_KEY][0]] = u'%d' % int(fact.score)
except KeyError: pass
if fields[Deck.VOCAB_SCORE_KEY][1]:
try: ankiFact[fields[Deck.VOCAB_SCORE_KEY][0]] = u'%d' % int(morphemesScore)
except KeyError: pass
if fields[Deck.UNKNOWNS_KEY][1]:
try: ankiFact[fields[Deck.UNKNOWNS_KEY][0]] = u','.join(u for u in unknownMorphemes)
except KeyError: pass
if fields[Deck.LEARNTS_KEY][1]:
try: ankiFact[fields[Deck.LEARNTS_KEY][0]] = u','.join(u for u in learnMorphemes)
except KeyError: pass
if fields[Deck.KNOWNS_KEY][1]:
try: ankiFact[fields[Deck.KNOWNS_KEY][0]] = u','.join(u for u in knownMorphemes)
except KeyError: pass
if fields[Deck.MATURES_KEY][1]:
try: ankiFact[fields[Deck.MATURES_KEY][0]] = u','.join(u for u in matureMorphemes)
except KeyError: pass
if len(unknownMorphemes) == 1:
if fields[Deck.COPY_UNKNOWN_1_TO_KEY][1]:
try: ankiFact[fields[Deck.COPY_UNKNOWN_1_TO_KEY][0]] = u','.join(u for u in unknownMorphemes)
except KeyError: pass
elif len(unknownMorphemes) == 0:
if fields[Deck.COPY_MATURE_TO_KEY][1]:
try: ankiFact[fields[Deck.COPY_MATURE_TO_KEY][0]] = u'%s' % ankiFact[deck.expressionField]
except KeyError: pass
ankiFact.tags = canonifyTags(deleteTags(fact.getAllStatusTag(), ankiFact.tags))
ankiFact.tags = canonifyTags(addTags(fact.getStatusTag(), ankiFact.tags))
if deck.definitionField:
try:
ankiFact.tags = canonifyTags(deleteTags(u'LxDefKnown,LxDefMatch', ankiFact.tags))
definition = self.factsService.getDefinition(fact)
if definition and definition.definitionHash and int(definition.definitionHash) != 0:
defMorphemes = self.morphemesService.getMorphemesDefinition(definition)
dictMorphemesScore, defMatureMorphemes, defKnownMorphemes, defLearnMorphemes, defUnknownMorphemes = self.getMorphemesScore(defMorphemes)
if len(defUnknownMorphemes) == 0:
ankiFact.tags = canonifyTags(addTags(u'LxDefKnown', ankiFact.tags))
defKeyMorphemes = self.morphemesService.getMorphemesDefinitionKey(definition)
defKeyMorphemesBase = "".join(m.morphLemme.base for m in defKeyMorphemes)
if len(unknownMorphemes) == 1 and unknownMorphemes[0] in defKeyMorphemesBase:
ankiFact.tags = canonifyTags(addTags(u'LxDefMatch', ankiFact.tags))
if fields[Deck.DEFINITION_SCORE_KEY][1]:
try: ankiFact[fields[Deck.DEFINITION_SCORE_KEY][0]] = u'%d' % int(dictMorphemesScore)
except KeyError: pass
except KeyError: pass
ankiFactsId.append(ankiFact.id)
self.saveDecks(ankiFactsId, language)
