def doDeckUpdate(self): # IO ()
fidDb = self.allDb().fidDb()
fid2cardsDb = self.fid2cardsDb()
locDb = self.allDb().locDb()
rankDb = R.mkRankDb(self.knownDb())
knownDbDb = self.knownDb().db
matureDbDb = self.matureDb().db
fs = self.getFacts()
# cache lookups
fieldNames = self.cfg['morph fields']
ipnKnownField = self.cfg['i+N known field']
ipnMatureField = self.cfg['i+N mature field']
unknownsField = self.cfg['unknowns field']
unmaturesField = self.cfg['unmatures field']
vrField = self.cfg['vocab rank field']
mmiField = self.cfg['morph man index field']
ip1knownField = self.cfg['copy i+1 known to']
ip0matField = self.cfg['copy i+0 mature to']
i, lfs = 0, len(fs)
start = time.time()
fstart = time.time()
mmiDict = {} # Map Fact -> MorphManIndex
for f in fs:
# first get all morphems for this fact
ms = set()
for fieldName in fieldNames:
try:
loc = fidDb[(f.id, fieldName)]
ms.update(locDb[loc])
except KeyError:
continue
# now determine unknowns and iPlusN - don't count multiple instances
# of a morpheme as an increase in difficulty, so use a set
unknowns = set()
for m in ms:
if m not in knownDbDb:
unknowns.add(m)
N_k = len(unknowns)
unmatures = set()
for m in ms:
if m not in matureDbDb:
unmatures.add(m)
N_m = len(unmatures)
# determine vocab rank and morph man overall difficulty index
vr = R.rankMorphemes(rankDb, ms)
mmi = N_k * 10000 + len(ms) * 1000 + vr
mmiDict[f] = mmi
try:
f[ipnKnownField] = u'%d' % N_k
except KeyError:
pass
try:
f[ipnMatureField] = u'%d' % N_m
except KeyError:
pass
try:
f[mmiField] = u'%d' % mmi
except KeyError:
pass
try:
f[vrField] = u'%d' % vr
except KeyError:
pass
try:
f[unknownsField] = u','.join(u.base for u in unknowns)
except KeyError:
pass
try:
f[unmaturesField] = u','.join(u.base for u in unmatures)
except KeyError:
pass
if len(ms) == 0:
f.tags = canonifyTags(addTags(u'noMorphemes', f.tags))
# Help automate vocab card -> sentence card promotion
try:
f[ip0matField] = u''
except KeyError:
pass
try:
f[ip1knownField] = u''
except KeyError:
pass
f.tags = canonifyTags(
deleteTags(u'ip0mature ip1known notReady', f.tags))
if N_m == 0: # is i+0 mature, make it a sentence card
f.tags = canonifyTags(
addTags(u'ip0mature ip0matureEver', f.tags))
try:
f[ip0matField] = u' '.join(f[name] for name in fieldNames)
except KeyError:
pass
elif N_k == 1: # is i+1 known, make it a vocab card
f.tags = canonifyTags(addTags(u'ip1known ip1knownEver',
f.tags))
try:
f[ip1knownField] = u'%s' % unknowns.pop().base
except KeyError:
pass
else: # is neither, make it a neither card
f.tags = canonifyTags(addTags(u'notReady', f.tags))
#f.setModified( textChanged=True, deck=self.deck )
# now display progress
i += 1
if i % 100 == 0:
fend = time.time()
log(' %d / %d = %d%% in %f sec' %
(i, lfs, 100. * i / lfs, fend - fstart))
fstart = time.time()
# rebuild tags
self.deck.updateFactTags((f.id for f in fs))
# sort new cards by Morph Man Index
debug('Set fact fields, now changing card creation times for ordering')
newtime = 0.0
try:
for f in sorted(fs, key=lambda x: mmiDict[x]):
for c in fid2cardsDb[f.id]:
c.created = newtime
newtime += CARD_CREATION_TIME_INC
except KeyError:
log('! no morph man index field for sorting')
# save deck and timestamps
self.deck.save()
end = time.time()
log('Proccessed all facts in %f sec' % (end - start))
self.cfg['last deck update'] = time.time()
self.cfg['last deck update took'] = end - start
def doDeckUpdate( self ): # IO ()
fidDb = self.allDb().fidDb()
fid2cardsDb = self.fid2cardsDb()
locDb = self.allDb().locDb()
rankDb = R.mkRankDb( self.knownDb() )
knownDbDb = self.knownDb().db
matureDbDb = self.matureDb().db
fs = self.getFacts()
# cache lookups
fieldNames = self.cfg['morph fields']
ipnKnownField = self.cfg['i+N known field']
ipnMatureField = self.cfg['i+N mature field']
unknownsField = self.cfg['unknowns field']
unmaturesField = self.cfg['unmatures field']
vrField = self.cfg['vocab rank field']
mmiField = self.cfg['morph man index field']
ip1knownField = self.cfg['copy i+1 known to']
ip0matField = self.cfg['copy i+0 mature to']
i, lfs = 0, len(fs)
start = time.time()
fstart = time.time()
mmiDict = {} # Map Fact -> MorphManIndex
hasModification = False
mod_fs = set()
for f in fs:
# first get all morphems for this fact
ms = set()
for fieldName in fieldNames:
try:
loc = fidDb[ (f.id, fieldName) ]
ms.update( locDb[ loc ] )
except KeyError:
log ("Error while loading morphems : " + fieldName)
continue
# now determine unknowns and iPlusN - don't count multiple instances
# of a morpheme as an increase in difficulty, so use a set
unknowns = set()
#log (M.ms2str(ms))
for m in ms:
if m not in knownDbDb:
unknowns.add( m )
N_k = len( unknowns )
unmatures = set()
for m in ms:
if m not in matureDbDb:
unmatures.add( m )
N_m = len( unmatures )
# determine vocab rank and morph man overall difficulty index
vr = R.rankMorphemes( rankDb, ms )
mmi = N_k*1000 + N_m*100 + vr*25
try:
if f[ mmiField ] == "" or int(mmi) != int(f[ mmiField ].decode()):
#log(f[ mmiField ])
#log("Change ! != " + str(int(mmi)))
f.setModified(textChanged=True, deck=self.deck)
hasModification = True
mod_fs.add(int(f.id))
except KeyError: pass
mmiDict[ f ] = mmi
try: f[ ipnKnownField ] = u'%d' % N_k
except KeyError: pass
try: f[ ipnMatureField ] = u'%d' % N_m
except KeyError: pass
try: f[ mmiField ] = u'%d' % mmi
except KeyError: pass
try: f[ vrField ] = u'%d' % vr
except KeyError: pass
#log(','.join( u.base for u in unknowns ))
try: f[ unknownsField ] = u','.join( u.base for u in unknowns )
except KeyError: pass
#log(','.join( u.base for u in unmatures ))
try: f[ unmaturesField ] = u','.join( u.base for u in unmatures )
except KeyError: pass
if len(ms) == 0:
f.tags = canonifyTags( addTags( u'noMorphemes', f.tags ) )
# Help automate vocab card -> sentence card promotion
try: f[ ip0matField ] = u''
except KeyError: pass
try: f[ ip1knownField ] = u''
except KeyError: pass
#before_tags = f.tags
f.tags = canonifyTags( deleteTags( u'ipXmature,ip0mature,ip1known,notReady', f.tags ) )
# Verification du nombre unknows et matures. Si changement alors, mettre la carte en modifier
if N_k > 1:
#log("Before Tags :" + before_tags)
#log("After First Tags: " + f.tags)
f.tags = canonifyTags( addTags( u'notReady', f.tags ) )
#log("After Remove Tags: " + f.tags)
else:
if N_m == 0:
f.tags = canonifyTags( addTags( u'ip0mature ip0matureEver', f.tags ) )
try: f[ ip0matField ] = u' '.join( f[ name ] for name in fieldNames )
except KeyError: pass
elif N_k == 1:
f.tags = canonifyTags( addTags( u'ip1known ip1knownEver', f.tags ) )
try: f[ ip1knownField ] = u'%s' % unknowns.pop().base
except KeyError: pass
else:
f.tags = canonifyTags( addTags( u'ipXmature ipXmatureEver', f.tags ) )
try: f[ ip0matField ] = u' '.join( f[ name ] for name in fieldNames )
except KeyError: pass
#if N_m == 0: # is i+0 mature, make it a sentence card
# f.tags = canonifyTags( addTags( u'ip0mature ip0matureEver', f.tags ) )
# try: f[ ip0matField ] = u' '.join( f[ name ] for name in fieldNames )
# except KeyError: pass
#elif N_k == 1: # is i+1 known, make it a vocab card
# f.tags = canonifyTags( addTags( u'ip1known ip1knownEver', f.tags ) )
# try: f[ ip1knownField ] = u'%s' % unknowns.pop().base
# except KeyError: pass
#else: # is neither, make it a neither card
# f.tags = canonifyTags( addTags( u'notReady', f.tags ) )
#f.setModified( textChanged=True, deck=self.deck )
# now display progress
i += 1
if i % 100 == 0:
fend = time.time()
log(' %d / %d = %d%% in %f sec' % ( i, lfs, 100.*i/lfs, fend-fstart ) )
fstart = time.time()
# rebuild tags
self.deck.updateFactTags( ( f.id for f in fs ) )
# sort new cards by Morph Man Index
debug( 'Set fact fields, now changing card creation times for ordering' )
newtime = 472777200.0 # 25 Decembre 1984 :)
#newCardIds = set()
try:
for f in sorted( fs, key=lambda x: mmiDict[ x ] ):
for c in fid2cardsDb[ f.id ]:
if self.deck.cardIsNew(c):
#newCardIds.add(c.id)
#c.created = newtime
c.due = newtime
c.combinedDue = newtime
newtime += CARD_CREATION_TIME_INC
except KeyError:
log( '! no morph man index field for sorting' )
# save deck
#self.deck.orderNewCards()
if hasModification:
self.deck.setModified()
self.deck.updateCardQACacheFromIds(mod_fs, type="facts")
self.deck.save()
# reorder new cards and save
# self.deck.orderNewCards()
# self.deck.save()
end = time.time()
log( 'Proccessed all facts in %f sec' % ( end-start ) )
self.cfg['last deck update'] = time.time()
self.cfg['last deck update took'] = end-start
def pre( ed ):
if not util.requireKnownDb(): return 'BAIL'
kdb = M.loadDb( util.knownDbPath )
rdb = R.mkRankDb( kdb )
return { 'rdb':rdb, 'mp':M.mecab(None) }
def doDeckUpdate( self ): # IO ()
fidDb = self.allDb().fidDb()
fid2cardsDb = self.fid2cardsDb()
locDb = self.allDb().locDb()
rankDb = R.mkRankDb( self.knownDb() )
knownDbDb = self.knownDb().db
matureDbDb = self.matureDb().db
fs = self.getFacts()
# cache lookups
fieldNames = self.cfg['morph fields']
ipnKnownField = self.cfg['i+N known field']
ipnMatureField = self.cfg['i+N mature field']
unknownsField = self.cfg['unknowns field']
unmaturesField = self.cfg['unmatures field']
vrField = self.cfg['vocab rank field']
mmiField = self.cfg['morph man index field']
ip1knownField = self.cfg['copy i+1 known to']
ip0matField = self.cfg['copy i+0 mature to']
i, lfs = 0, len(fs)
start = time.time()
fstart = time.time()
mmiDict = {} # Map Fact -> MorphManIndex
for f in fs:
# first get all morphems for this fact
ms = set()
for fieldName in fieldNames:
try:
loc = fidDb[ (f.id, fieldName) ]
ms.update( locDb[ loc ] )
except KeyError:
continue
# now determine unknowns and iPlusN - don't count multiple instances
# of a morpheme as an increase in difficulty, so use a set
unknowns = set()
for m in ms:
if m not in knownDbDb:
unknowns.add( m )
N_k = len( unknowns )
unmatures = set()
for m in ms:
if m not in matureDbDb:
unmatures.add( m )
N_m = len( unmatures )
# determine vocab rank and morph man overall difficulty index
vr = R.rankMorphemes( rankDb, ms )
mmi = N_k*10000 + len(ms)*1000 + vr
mmiDict[ f ] = mmi
try: f[ ipnKnownField ] = u'%d' % N_k
except KeyError: pass
try: f[ ipnMatureField ] = u'%d' % N_m
except KeyError: pass
try: f[ mmiField ] = u'%d' % mmi
except KeyError: pass
try: f[ vrField ] = u'%d' % vr
except KeyError: pass
try: f[ unknownsField ] = u','.join( u.base for u in unknowns )
except KeyError: pass
try: f[ unmaturesField ] = u','.join( u.base for u in unmatures )
except KeyError: pass
if len(ms) == 0:
f.tags = canonifyTags( addTags( u'noMorphemes', f.tags ) )
# Help automate vocab card -> sentence card promotion
try: f[ ip0matField ] = u''
except KeyError: pass
try: f[ ip1knownField ] = u''
except KeyError: pass
f.tags = canonifyTags( deleteTags( u'ip0mature ip1known notReady', f.tags ) )
if N_m == 0: # is i+0 mature, make it a sentence card
f.tags = canonifyTags( addTags( u'ip0mature ip0matureEver', f.tags ) )
try: f[ ip0matField ] = u' '.join( f[ name ] for name in fieldNames )
except KeyError: pass
elif N_k == 1: # is i+1 known, make it a vocab card
f.tags = canonifyTags( addTags( u'ip1known ip1knownEver', f.tags ) )
try: f[ ip1knownField ] = u'%s' % unknowns.pop().base
except KeyError: pass
else: # is neither, make it a neither card
f.tags = canonifyTags( addTags( u'notReady', f.tags ) )
#f.setModified( textChanged=True, deck=self.deck )
# now display progress
i += 1
if i % 100 == 0:
fend = time.time()
log(' %d / %d = %d%% in %f sec' % ( i, lfs, 100.*i/lfs, fend-fstart ) )
fstart = time.time()
# rebuild tags
self.deck.updateFactTags( ( f.id for f in fs ) )
# sort new cards by Morph Man Index
debug( 'Set fact fields, now changing card creation times for ordering' )
newtime = 0.0
try:
for f in sorted( fs, key=lambda x: mmiDict[ x ] ):
for c in fid2cardsDb[ f.id ]:
c.created = newtime
newtime += CARD_CREATION_TIME_INC
except KeyError:
log( '! no morph man index field for sorting' )
# save deck and timestamps
self.deck.save()
end = time.time()
log( 'Proccessed all facts in %f sec' % ( end-start ) )
self.cfg['last deck update'] = time.time()
self.cfg['last deck update took'] = end-start