Example #1
0
    def doDeckUpdate(self):  # IO ()
        fidDb = self.allDb().fidDb()
        fid2cardsDb = self.fid2cardsDb()
        locDb = self.allDb().locDb()
        rankDb = R.mkRankDb(self.knownDb())
        knownDbDb = self.knownDb().db
        matureDbDb = self.matureDb().db
        fs = self.getFacts()

        # cache lookups
        fieldNames = self.cfg['morph fields']
        ipnKnownField = self.cfg['i+N known field']
        ipnMatureField = self.cfg['i+N mature field']
        unknownsField = self.cfg['unknowns field']
        unmaturesField = self.cfg['unmatures field']
        vrField = self.cfg['vocab rank field']
        mmiField = self.cfg['morph man index field']
        ip1knownField = self.cfg['copy i+1 known to']
        ip0matField = self.cfg['copy i+0 mature to']

        i, lfs = 0, len(fs)
        start = time.time()
        fstart = time.time()
        mmiDict = {}  # Map Fact -> MorphManIndex
        for f in fs:
            # first get all morphems for this fact
            ms = set()
            for fieldName in fieldNames:
                try:
                    loc = fidDb[(f.id, fieldName)]
                    ms.update(locDb[loc])
                except KeyError:
                    continue
            # now determine unknowns and iPlusN - don't count multiple instances
            # of a morpheme as an increase in difficulty, so use a set
            unknowns = set()
            for m in ms:
                if m not in knownDbDb:
                    unknowns.add(m)
            N_k = len(unknowns)

            unmatures = set()
            for m in ms:
                if m not in matureDbDb:
                    unmatures.add(m)
            N_m = len(unmatures)
            # determine vocab rank and morph man overall difficulty index
            vr = R.rankMorphemes(rankDb, ms)
            mmi = N_k * 10000 + len(ms) * 1000 + vr
            mmiDict[f] = mmi

            try:
                f[ipnKnownField] = u'%d' % N_k
            except KeyError:
                pass
            try:
                f[ipnMatureField] = u'%d' % N_m
            except KeyError:
                pass
            try:
                f[mmiField] = u'%d' % mmi
            except KeyError:
                pass
            try:
                f[vrField] = u'%d' % vr
            except KeyError:
                pass
            try:
                f[unknownsField] = u','.join(u.base for u in unknowns)
            except KeyError:
                pass
            try:
                f[unmaturesField] = u','.join(u.base for u in unmatures)
            except KeyError:
                pass

            if len(ms) == 0:
                f.tags = canonifyTags(addTags(u'noMorphemes', f.tags))

            # Help automate vocab card -> sentence card promotion
            try:
                f[ip0matField] = u''
            except KeyError:
                pass
            try:
                f[ip1knownField] = u''
            except KeyError:
                pass
            f.tags = canonifyTags(
                deleteTags(u'ip0mature ip1known notReady', f.tags))

            if N_m == 0:  # is i+0 mature, make it a sentence card
                f.tags = canonifyTags(
                    addTags(u'ip0mature ip0matureEver', f.tags))
                try:
                    f[ip0matField] = u' '.join(f[name] for name in fieldNames)
                except KeyError:
                    pass
            elif N_k == 1:  # is i+1 known, make it a vocab card
                f.tags = canonifyTags(addTags(u'ip1known ip1knownEver',
                                              f.tags))
                try:
                    f[ip1knownField] = u'%s' % unknowns.pop().base
                except KeyError:
                    pass
            else:  # is neither, make it a neither card
                f.tags = canonifyTags(addTags(u'notReady', f.tags))

            #f.setModified( textChanged=True, deck=self.deck )

            # now display progress
            i += 1
            if i % 100 == 0:
                fend = time.time()
                log('    %d / %d = %d%% in %f sec' %
                    (i, lfs, 100. * i / lfs, fend - fstart))
                fstart = time.time()

        # rebuild tags
        self.deck.updateFactTags((f.id for f in fs))

        # sort new cards by Morph Man Index
        debug('Set fact fields, now changing card creation times for ordering')
        newtime = 0.0
        try:
            for f in sorted(fs, key=lambda x: mmiDict[x]):
                for c in fid2cardsDb[f.id]:
                    c.created = newtime
                    newtime += CARD_CREATION_TIME_INC
        except KeyError:
            log('! no morph man index field for sorting')

        # save deck and timestamps
        self.deck.save()
        end = time.time()
        log('Proccessed all facts in %f sec' % (end - start))
        self.cfg['last deck update'] = time.time()
        self.cfg['last deck update took'] = end - start
Example #2
0
    def doDeckUpdate( self ): # IO ()
        fidDb = self.allDb().fidDb()
        fid2cardsDb = self.fid2cardsDb()
        locDb = self.allDb().locDb()
        rankDb = R.mkRankDb( self.knownDb() )
        knownDbDb = self.knownDb().db
        matureDbDb = self.matureDb().db
        fs = self.getFacts()

        # cache lookups
        fieldNames = self.cfg['morph fields']
        ipnKnownField = self.cfg['i+N known field']
        ipnMatureField = self.cfg['i+N mature field']
        unknownsField = self.cfg['unknowns field']
        unmaturesField = self.cfg['unmatures field']
        vrField = self.cfg['vocab rank field']
        mmiField = self.cfg['morph man index field']
        ip1knownField = self.cfg['copy i+1 known to']
        ip0matField = self.cfg['copy i+0 mature to']

        i, lfs = 0, len(fs)
        start = time.time()
        fstart = time.time()
        mmiDict = {} # Map Fact -> MorphManIndex
        hasModification = False
        mod_fs = set()
        for f in fs:
            # first get all morphems for this fact
            ms = set()
            for fieldName in fieldNames:
                try:
                    loc = fidDb[ (f.id, fieldName) ]
                    ms.update( locDb[ loc ] )
                except KeyError:
                    log ("Error while loading morphems : " + fieldName)
                    continue
            # now determine unknowns and iPlusN - don't count multiple instances
            # of a morpheme as an increase in difficulty, so use a set
            unknowns = set()
            #log (M.ms2str(ms))
            for m in ms:
                if m not in knownDbDb:
                    unknowns.add( m )
            N_k = len( unknowns )

            unmatures = set()
            for m in ms:
                if m not in matureDbDb:
                    unmatures.add( m )
            N_m = len( unmatures )
            # determine vocab rank and morph man overall difficulty index
            vr = R.rankMorphemes( rankDb, ms )
            mmi = N_k*1000 + N_m*100 + vr*25
            
            try:
                if f[ mmiField ] == "" or int(mmi) != int(f[ mmiField ].decode()):
                    #log(f[ mmiField ])
                    #log("Change ! != " + str(int(mmi)))
                    f.setModified(textChanged=True, deck=self.deck)
                    hasModification = True
                    mod_fs.add(int(f.id))
            except KeyError: pass
            mmiDict[ f ] = mmi
            
            try: f[ ipnKnownField ] = u'%d' % N_k
            except KeyError: pass
            try: f[ ipnMatureField ] = u'%d' % N_m
            except KeyError: pass
            try: f[ mmiField ] = u'%d' % mmi
            except KeyError: pass
            try: f[ vrField ] = u'%d' % vr
            except KeyError: pass
            #log(','.join( u.base for u in unknowns ))
            try: f[ unknownsField ] = u','.join( u.base for u in unknowns )
            except KeyError: pass
            #log(','.join( u.base for u in unmatures ))
            try: f[ unmaturesField ] = u','.join( u.base for u in unmatures )
            except KeyError: pass

            if len(ms) == 0:
                f.tags = canonifyTags( addTags( u'noMorphemes', f.tags ) )

            # Help automate vocab card -> sentence card promotion
            try: f[ ip0matField ] = u''
            except KeyError: pass
            try: f[ ip1knownField ] = u''
            except KeyError: pass
            #before_tags = f.tags
            f.tags = canonifyTags( deleteTags( u'ipXmature,ip0mature,ip1known,notReady', f.tags ) )

            # Verification du nombre unknows et matures. Si changement alors, mettre la carte en modifier
            if N_k > 1:
                #log("Before Tags :" + before_tags)
                #log("After First Tags: " + f.tags)
                f.tags = canonifyTags( addTags( u'notReady', f.tags ) )
                #log("After Remove Tags: " + f.tags)
            else:
                if N_m == 0:
                    f.tags = canonifyTags( addTags( u'ip0mature ip0matureEver', f.tags ) )
                    try: f[ ip0matField ] = u' '.join( f[ name ] for name in fieldNames )
                    except KeyError: pass
                elif N_k == 1:
                    f.tags = canonifyTags( addTags( u'ip1known ip1knownEver', f.tags ) )
                    try: f[ ip1knownField ] = u'%s' % unknowns.pop().base
                    except KeyError: pass
                else:
                    f.tags = canonifyTags( addTags( u'ipXmature ipXmatureEver', f.tags ) )
                    try: f[ ip0matField ] = u' '.join( f[ name ] for name in fieldNames )
                    except KeyError: pass
            #if N_m == 0: # is i+0 mature, make it a sentence card
            #    f.tags = canonifyTags( addTags( u'ip0mature ip0matureEver', f.tags ) )
            #    try: f[ ip0matField ] = u' '.join( f[ name ] for name in fieldNames )
            #    except KeyError: pass
            #elif N_k == 1: # is i+1 known, make it a vocab card
            #    f.tags = canonifyTags( addTags( u'ip1known ip1knownEver', f.tags ) )
            #    try: f[ ip1knownField ] = u'%s' % unknowns.pop().base
            #    except KeyError: pass
            #else: # is neither, make it a neither card
            #    f.tags = canonifyTags( addTags( u'notReady', f.tags ) )

            #f.setModified( textChanged=True, deck=self.deck )

            # now display progress
            i += 1
            if i % 100 == 0:
                fend = time.time()
                log('    %d / %d = %d%% in %f sec' % ( i, lfs, 100.*i/lfs, fend-fstart ) )
                fstart = time.time()

        # rebuild tags
        self.deck.updateFactTags( ( f.id for f in fs ) )

        # sort new cards by Morph Man Index
        debug( 'Set fact fields, now changing card creation times for ordering' )
        newtime = 472777200.0 # 25 Decembre 1984 :)
        #newCardIds = set()
        try:
            for f in sorted( fs, key=lambda x: mmiDict[ x ] ):
                for c in fid2cardsDb[ f.id ]:
                    if self.deck.cardIsNew(c):
                        #newCardIds.add(c.id)
                        #c.created = newtime
                        c.due = newtime
                        c.combinedDue = newtime
                    newtime += CARD_CREATION_TIME_INC
        except KeyError:
            log( '! no morph man index field for sorting' )

        # save deck
        #self.deck.orderNewCards()
        if hasModification:
            self.deck.setModified()
            self.deck.updateCardQACacheFromIds(mod_fs, type="facts")
        self.deck.save()
        # reorder new cards and save
        # self.deck.orderNewCards()
        # self.deck.save()
        end = time.time()
        log( 'Proccessed all facts in %f sec' % ( end-start ) )
        self.cfg['last deck update'] = time.time()
        self.cfg['last deck update took'] = end-start
Example #3
0
def pre( ed ):
    if not util.requireKnownDb(): return 'BAIL'
    kdb = M.loadDb( util.knownDbPath )
    rdb = R.mkRankDb( kdb )
    return { 'rdb':rdb, 'mp':M.mecab(None) }
Example #4
0
    def doDeckUpdate( self ): # IO ()
        fidDb = self.allDb().fidDb()
        fid2cardsDb = self.fid2cardsDb()
        locDb = self.allDb().locDb()
        rankDb = R.mkRankDb( self.knownDb() )
        knownDbDb = self.knownDb().db
        matureDbDb = self.matureDb().db
        fs = self.getFacts()

        # cache lookups
        fieldNames = self.cfg['morph fields']
        ipnKnownField = self.cfg['i+N known field']
        ipnMatureField = self.cfg['i+N mature field']
        unknownsField = self.cfg['unknowns field']
        unmaturesField = self.cfg['unmatures field']
        vrField = self.cfg['vocab rank field']
        mmiField = self.cfg['morph man index field']
        ip1knownField = self.cfg['copy i+1 known to']
        ip0matField = self.cfg['copy i+0 mature to']

        i, lfs = 0, len(fs)
        start = time.time()
        fstart = time.time()
        mmiDict = {} # Map Fact -> MorphManIndex
        for f in fs:
            # first get all morphems for this fact
            ms = set()
            for fieldName in fieldNames:
                try:
                    loc = fidDb[ (f.id, fieldName) ]
                    ms.update( locDb[ loc ] )
                except KeyError:
                    continue
            # now determine unknowns and iPlusN - don't count multiple instances
            # of a morpheme as an increase in difficulty, so use a set
            unknowns = set()
            for m in ms:
                if m not in knownDbDb:
                    unknowns.add( m )
            N_k = len( unknowns )

            unmatures = set()
            for m in ms:
                if m not in matureDbDb:
                    unmatures.add( m )
            N_m = len( unmatures )
            # determine vocab rank and morph man overall difficulty index
            vr = R.rankMorphemes( rankDb, ms )
            mmi = N_k*10000 + len(ms)*1000 + vr
	    mmiDict[ f ] = mmi

            try: f[ ipnKnownField ] = u'%d' % N_k
            except KeyError: pass
            try: f[ ipnMatureField ] = u'%d' % N_m
            except KeyError: pass
            try: f[ mmiField ] = u'%d' % mmi
            except KeyError: pass
            try: f[ vrField ] = u'%d' % vr
            except KeyError: pass
            try: f[ unknownsField ] = u','.join( u.base for u in unknowns )
            except KeyError: pass
            try: f[ unmaturesField ] = u','.join( u.base for u in unmatures )
            except KeyError: pass

            if len(ms) == 0:
                f.tags = canonifyTags( addTags( u'noMorphemes', f.tags ) )

            # Help automate vocab card -> sentence card promotion
            try: f[ ip0matField ] = u''
            except KeyError: pass
            try: f[ ip1knownField ] = u''
            except KeyError: pass
            f.tags = canonifyTags( deleteTags( u'ip0mature ip1known notReady', f.tags ) )

            if N_m == 0: # is i+0 mature, make it a sentence card
                f.tags = canonifyTags( addTags( u'ip0mature ip0matureEver', f.tags ) )
                try: f[ ip0matField ] = u' '.join( f[ name ] for name in fieldNames )
                except KeyError: pass
            elif N_k == 1: # is i+1 known, make it a vocab card
                f.tags = canonifyTags( addTags( u'ip1known ip1knownEver', f.tags ) )
                try: f[ ip1knownField ] = u'%s' % unknowns.pop().base
                except KeyError: pass
            else: # is neither, make it a neither card
                f.tags = canonifyTags( addTags( u'notReady', f.tags ) )

            #f.setModified( textChanged=True, deck=self.deck )

            # now display progress
            i += 1
            if i % 100 == 0:
                fend = time.time()
                log('    %d / %d = %d%% in %f sec' % ( i, lfs, 100.*i/lfs, fend-fstart ) )
                fstart = time.time()

        # rebuild tags
        self.deck.updateFactTags( ( f.id for f in fs ) )

        # sort new cards by Morph Man Index
        debug( 'Set fact fields, now changing card creation times for ordering' )
        newtime = 0.0
	try:
            for f in sorted( fs, key=lambda x: mmiDict[ x ] ):
                for c in fid2cardsDb[ f.id ]:
                    c.created = newtime
                    newtime += CARD_CREATION_TIME_INC
        except KeyError:
            log( '! no morph man index field for sorting' )

        # save deck and timestamps
        self.deck.save()
        end = time.time()
        log( 'Proccessed all facts in %f sec' % ( end-start ) )
        self.cfg['last deck update'] = time.time()
        self.cfg['last deck update took'] = end-start