コード例 #1
0
 def __init__(self, serviceLocator):
     self.serviceLocator = serviceLocator
     self.fact_dao = FactDao()
     self.card_dao = CardDao()
     self.morpheme_dao = MorphemeDao()
     self.deck_dao = DeckDao()
     self.definitionDao = DefinitionDao()
コード例 #2
0
class FactsService:
    def __init__(self, serviceLocator):
        self.serviceLocator = serviceLocator
        self.fact_dao = FactDao()
        self.card_dao = CardDao()
        self.morpheme_dao = MorphemeDao()
        self.deck_dao = DeckDao()
        self.definitionDao = DefinitionDao()
        
    def setupServices(self):
        self.decksService = self.serviceLocator.getDecksService()
        
    def getFact(self, deck, ankiFactId):
        fact = self.fact_dao.findByAnkiFactId(deck, ankiFactId)
        if fact == None:
            fact = Fact(deck.id, ankiFactId, None, None, False, Fact.STATUS_NONE, False, 0)
            fact.deck = deck
            fact = self.fact_dao.insert(fact)

        return fact
    
    def getDefinition(self, fact):
        return self.definitionDao.getDefinition(fact.id)

    def getFactById(self, ankiFactId):
        fact = self.fact_dao.findById(ankiFactId)
        fact.deck = self.deck_dao.findById(fact.deckId)
        
        return fact
    
    def getAllFacts(self, deck, ankiFacts):
        
        facts = []
        factsToInsert = []
        i = 0
        for ankiFact in ankiFacts:
            fact = self.fact_dao.findByAnkiFactId(deck, ankiFact.id)
            if fact == None:
                fact = Fact(deck.id, ankiFact.id, None, None, False, Fact.STATUS_NONE, False, 0)
                fact.deck = deck
                factsToInsert.append(fact)
            fact.ankiFactIndex = i
            fact.ankiLastModified = ankiFact.modified
            facts.append(fact)
            i += 1

        if len(factsToInsert) > 0:
            self.fact_dao.insertAll(factsToInsert)
        
        return facts
    
    def getAllCardsOrderByScore(self, deck = None, language = None):
        if language:
            decksId = self.decksService.listDecksIdByLanguage(language)
        else:
            decksId = [deck.id]
        log(decksId)
        return self.card_dao.getCardsOrderByScore(decksId)
    
    def calcCardStatus(self, deck, ankiCard):
        if ankiCard.interval < deck.learnTreshold: 
            return Card.STATUS_NONE
        if ankiCard.interval < deck.knownTreshold:
            return Card.STATUS_LEARNT
        if ankiCard.interval < deck.matureTreshold:
            return Card.STATUS_KNOWN
        return Card.STATUS_MATURE
        
    def getAllCardsChanged(self, deck, ankiCards):
        
        changedCards = []
        cardsToInsert = []
        cardsToUpdate = []
        for ankiCard in ankiCards:
            card = self.card_dao.findById(deck, ankiCard.id)
            status = self.calcCardStatus(deck, ankiCard)
            if card == None:
                fact = self.getFact(deck, ankiCard.fact.id)
                card = Card(deck.id, fact.id, ankiCard.id, status, True)
                card.deck = deck
                card.fact = fact
                cardsToInsert.append(card)
                changedCards.append(card)
            else:
                if status != card.status:
                    card.statusChanged = True
                    changedCards.append(card)
                card.status = status               
                cardsToUpdate.append(card)  
            card.ankiLastModified = ankiCard.modified
            
        if len(cardsToInsert) > 0:
            self.card_dao.insertAll(cardsToInsert)
        
        if len(cardsToUpdate) > 0:
            self.card_dao.updateAll(cardsToUpdate)
        
        return changedCards
    
    def getAllFactsChanged(self, language):
        return self.getAllFactsByLanguage(language, 1)
    
    def getAllFactsByLanguage(self, language, statusChanged = None):
        
        decksId = self.decksService.listDecksIdByLanguage(language)
        facts = self.fact_dao.selectAll(decksId, statusChanged) 
        decks = dict()
        for fact in facts:
            deck = None
            try:
                deck = decks[fact.deckId]
            except KeyError as e:
                deck = self.deck_dao.findById(fact.deckId)
                decks[fact.deckId] = deck
            fact.deck = deck
        return facts
    
    def computeFactsMaturity(self, language):
        
        decksId = self.decksService.listDecksIdByLanguage(language)
        facts = self.fact_dao.selectAll(decksId, None) #FIXME: Buggggg with other decks !! replace with all facts for now ?
        for fact in facts:
            morphemes = self.morpheme_dao.getMorphemesFromFact(fact)
        
            matureMorphemes = []
            knownMorphemes = []
            learnMorphemes = []
            unknownMorphemes = []
            morphemesScore = 0
            for morpheme in morphemes:
                if morpheme.status == Morpheme.STATUS_MATURE:
                    matureMorphemes.append(morpheme)
                if morpheme.status == Morpheme.STATUS_KNOWN:
                    knownMorphemes.append(morpheme)
                if morpheme.status == Morpheme.STATUS_LEARNT:
                    learnMorphemes.append(morpheme)
                if morpheme.status == Morpheme.STATUS_NONE:
                    unknownMorphemes.append(morpheme)
                morphemesScore += morpheme.score
            total = len(morphemes)
            mature = len(matureMorphemes)
            known = len(knownMorphemes)
            learnt = len(learnMorphemes)
            unknown = len(unknownMorphemes)
            
            status = fact.status
            if unknown == 0:
                if learnt == 0 and known == 0:
                    status = Fact.STATUS_REVIEW_EASY
                elif learnt > 0:
                    status = Fact.STATUS_REVIEW_HARD
                else :
                    status = Fact.STATUS_REVIEW_MEDIUM
            elif unknown == 1:
                if learnt == 0 and known == 0:
                    status = Fact.STATUS_LEARN_EASY
                elif learnt > 0:
                    status = Fact.STATUS_LEARN_HARD
                else :
                    status = Fact.STATUS_LEARN_MEDIUM
            else:
                status = Fact.STATUS_TOO_DIFFICULT

            if status != fact.status:
                fact.status = status
                fact.statusChanged = True
            
            score = (mature * 1 + known * 2 + learnt * 6 + unknown * 30) * 300 + morphemesScore
            if score != fact.score:
                fact.score = score
                fact.statusChanged = True
            
        self.fact_dao.updateAll(facts) 

        #self.morpheme_dao.clearMorphemesStatus()

        return facts

    def changeMorphemes(self, fact, morphemes):
        self.fact_dao.insertFactMorphemes(fact, morphemes)
        
        fact.morphemes = morphemes
        return fact
    
    def getMorphemes(self, fact):
        return self.morpheme_dao.getMorphemesFromFact(fact, True)

    def getAllNewCards(self, language):
        decksId = self.decksService.listDecksIdByLanguage(language)
        return self.card_dao.getAllNewCards(decksId)

    def clearAllFactsStatus(self, language):
        decksId = self.decksService.listDecksIdByLanguage(language)
        self.fact_dao.clearAllFactsStatus(decksId)
コード例 #3
0
class MorphemesService:
    
    def __init__(self, serviceLocator):
        self.serviceLocator = serviceLocator
        
        self.lemmeDao = MorphemeLemmeDao()
        self.morphemeDao = MorphemeDao()
        self.factDao = FactDao()
        self.cardDao = CardDao()
        self.deckDao = DeckDao()
        self.definitionDao = DefinitionDao()
    
    def setupServices(self):
        self.decksService = self.serviceLocator.getDecksService()
        self.factsService = self.serviceLocator.getFactsService()

    def extractMorphemes(self, expression, deck, language):
        
        morphemes = language.posTagger.posMorphemes(expression, deck, language)
        return morphemes
    
    #def analyze(self, expression, language):
    #    morphemes = self.extractMorphemes(expression, None, None)
    #    return morphemes
    
    def getList(self, dict):
        dictList = list()
        for key, value in dict.iteritems():
            dictList.append(value)
        return dictList
    
    # Store temporarly (not in database) definition morphemes and score in facts
    # Dont work with lemmatizater
    def getMorphemesFromDB(self, expression, deck, language):
        
        # Unique Morphemes
        log("Extract Morphemes")
        morphLemmes = self.extractMorphemes(expression, deck, language)
        factMorphemes = list()
        for morphemeLemme in morphLemmes:
            morphLemmeDB = self.lemmeDao.find(morphemeLemme)
            if morphLemmeDB == None:
                morpheme = Morpheme(0, 0, -1, None, 0, -1)
                morpheme.morphLemme = morphemeLemme
            else:
                morpheme = self.morphemeDao.findByLemmeId(morphLemmeDB.id)
                if morpheme == None:
                    continue
                morpheme.morphLemme = morphLemmeDB
            if morpheme not in factMorphemes:
                factMorphemes.append(morpheme)
        return factMorphemes
    
    def analyseDefinition(self, deck, language, facts, ankiFacts):
        
        definitions = self.definitionDao.getDefinitions(facts)
        
        modifiedDefinition = list()
        keyModifiedDefinition = list()
        defModifiedDefinition = list()
        
        for definition in definitions:
            fact = definition.fact
            if fact.ankiLastModified == fact.lastUpdated:
                continue
            
            isModified = False
            definitionExpression = ankiFacts[fact.ankiFactIndex][deck.definitionField]
            if definition.definitionHash == None or int(definition.definitionHash) != hash(definitionExpression):
                definition.definitionMorphemes = self.getMorphemesFromDB(definitionExpression, deck, language)
                definition.definitionHash = hash(definitionExpression)
                defModifiedDefinition.append(definition)
                isModified = True
            
            definitionKeyExpression = ankiFacts[fact.ankiFactIndex][deck.definitionKeyField]
            if definition.definitionKeyHash == None or int(definition.definitionKeyHash) != hash(definitionKeyExpression):
                definition.definitionMorphemesKey = self.getMorphemesFromDB(definitionKeyExpression, deck, language)
                definition.definitionKeyHash = hash(definitionKeyExpression)
                keyModifiedDefinition.append(definition)
                isModified = True
            
            if isModified == True:
                modifiedDefinition.append(definition)
        
        if len(defModifiedDefinition) > 0:
            self.definitionDao.updateAllDefinitionsMorphemes(defModifiedDefinition)
        if len(keyModifiedDefinition) > 0:
            self.definitionDao.updateAllDefinitionsKeysMorphemes(keyModifiedDefinition)
        if len(modifiedDefinition) > 0:
            self.definitionDao.updateAll(modifiedDefinition)
    
    def getMorphemesDefinition(self, definition):
        morphemesId = self.definitionDao.getAllDefinitionMorphemes(definition)
        
        morphemes = list()
        for morphemeId in morphemesId:
            morpheme = self.morphemeDao.findById(morphemeId)
            if morpheme:
                morpheme.morphLemme = self.lemmeDao.findById(morpheme.morphLemmeId)
                if morpheme.morphLemme:
                    morphemes.append(morpheme)
            
        return morphemes
    
    def getMorphemesDefinitionKey(self, definition):
        morphemesId = self.definitionDao.getAllDefinitionKeyMorphemes(definition)
        
        morphemes = list()
        for morphemeId in morphemesId:
            morpheme = self.morphemeDao.findById(morphemeId)
            if morpheme:
                morpheme.morphLemme = self.lemmeDao.findById(morpheme.morphLemmeId)
                if morpheme.morphLemme:
                    morphemes.append(morpheme)
            
        return morphemes
    
    def analyzeMorphemes(self, facts, deck, language):
        
        # Unique Morphemes
        #log("Extract Morphemes")
        allUniqueMorphLemmes = dict()
        for fact in facts:
            morphLemmes = self.extractMorphemes(fact.expression, deck, language)
            factMorphLemmes = list()
            for morphLemme in morphLemmes:
                if morphLemme in allUniqueMorphLemmes:
                    morphLemme = allUniqueMorphLemmes[morphLemme]
                else:
                    allUniqueMorphLemmes[morphLemme] = morphLemme
                factMorphLemmes.append(morphLemme)
            fact.morphLemmes = factMorphLemmes
        
        log("Lemmatize Morphemes : " + str(len(allUniqueMorphLemmes)))
        if language.lemmatizer:
            language.lemmatizer.lemmatizeMorphemes(self.getList(allUniqueMorphLemmes))
        
        self.filterMorphLemmes(self.getList(allUniqueMorphLemmes))
        
        log("Compute Facts <-> Morphemes")
        allMorphemes = dict()
        for fact in facts:
            factUniqueMorphemes = dict()
            for morphLemme in fact.morphLemmes:
                morpheme = Morpheme(Morpheme.STATUS_NONE, False, morphLemme.id, morphLemme, 0)
                if morpheme in allMorphemes:    
                    morpheme = allMorphemes[morpheme]
                else:
                    allMorphemes[morpheme] = morpheme
                if morpheme not in factUniqueMorphemes:
                    factUniqueMorphemes[morpheme] = morpheme
            fact.morphemes = self.getList(factUniqueMorphemes)
            fact.expressionHash = hash(fact.expression)
            fact.lastUpdated = fact.ankiLastModified
            fact.expression = None
        
        allMorphemesList = self.getList(allMorphemes)
        log("All Unique Morphemes (Lemmatized): " + str(len(allMorphemesList)))
        
        log("persistAll")
        self.lemmeDao.persistAll(allMorphemesList)
        self.morphemeDao.persistAll(allMorphemesList)
        
        log("insertAllFactMorphemes")
        self.factDao.insertAllFactMorphemes(facts)
        
        log("updateAll")
        self.factDao.updateAll(facts)
        
        for fact in facts:
            fact.morphLemmes = None
            fact.morphemes = None
    
    def computeMorphemesMaturity(self, cards):
        
        log("computeMorphemesMaturity")
        
        morphemesInCards = dict()
        for card in cards:
            morphemes = self.morphemeDao.getMorphemesFromCard(card) # Very Fast 45s pour 20.000
            #log("Morphemes: " + str(len(morphemes)))
            for morpheme in morphemes:
                if morpheme.id not in morphemesInCards:
                    morphemesInCards[morpheme.id] = morpheme
            card.lastUpdated = card.ankiLastModified
        
        morphemesInCardsList = self.getList(morphemesInCards)
        log("get New Status for morphemes: " + str(len(morphemesInCardsList)))
        
        modifiedMorphemes = list()
        for morpheme in morphemesInCardsList:
            #log("morpheme: " + str(morpheme))
            status = self.morphemeDao.getNewStatus(morpheme) # Slow :/ XXX: Optimize
            #log("status: " + str(status))
            if morpheme.status != status:
                log("status modified")
                morpheme.status = status
                morpheme.statusChanged = True
                modifiedMorphemes.append(morpheme)

        if len(modifiedMorphemes) > 0:
            self.morphemeDao.updateAll(modifiedMorphemes)
        self.cardDao.updateAll(cards)
    
    #def analyze(self, fact):
        
    #    morphemes = self.extractMorphemes(fact.expression)
        
    #    attachedMorphemes = []
    #    for morpheme in morphemes:
    #        morpheme.morphLemme = self.lemmeDao.persist(morpheme.morphLemme)
    #        morpheme = self.morphemeDao.persist(morpheme)
    #        attachedMorphemes.append(morpheme)
        
    #    self.factsService.changeMorphemes(fact, attachedMorphemes)
        
    #    return morphemes
        
    def getAllPOS(self, language):
        try:
            return language.posOptions["activatedPos"]
        except Exception: pass
  
    def getAllSubPOS(self, language):
        try:
            return language.posOptions["activatedSubPos"]
        except Exception: pass
    
    def getMorphemes(self, searchText = None, decksId = None):
        
        if searchText == None or searchText == "":
            return self.lemmeDao.getMorphemes(decksId)
        else:
            searchExpressions = searchText.split(" ")
            status_changed = None
            status = None
            pos = None
            subPos = None
            expressions = []
            
            for searchExpression in searchExpressions:
                uni = unicode(searchExpression)
                if searchExpression == "is:changed":
                    status_changed = True
                elif searchExpression == "-is:changed":
                    status_changed = False
                elif searchExpression == "status:None":
                    status = Morpheme.STATUS_NONE
                elif searchExpression == "status:Learnt":
                    status = Morpheme.STATUS_LEARNT
                elif searchExpression == "status:Known":
                    status = Morpheme.STATUS_KNOWN
                elif searchExpression == "status:Mature":
                    status = Morpheme.STATUS_MATURE
                elif uni.startswith("pos:"):
                    pos = uni.split(":")[1]                    
                elif uni.startswith("sub_pos:"):
                    subPos = uni.split(":")[1]
                else:
                    expressions.append(uni)
            return self.lemmeDao.getMorphemes(decksId, expressions, status, status_changed, pos, subPos)