def __init__(self, serviceLocator):
self.serviceLocator = serviceLocator
self.fact_dao = FactDao()
self.card_dao = CardDao()
self.morpheme_dao = MorphemeDao()
self.deck_dao = DeckDao()
self.definitionDao = DefinitionDao()
class FactsService:
def __init__(self, serviceLocator):
self.serviceLocator = serviceLocator
self.fact_dao = FactDao()
self.card_dao = CardDao()
self.morpheme_dao = MorphemeDao()
self.deck_dao = DeckDao()
self.definitionDao = DefinitionDao()
def setupServices(self):
self.decksService = self.serviceLocator.getDecksService()
def getFact(self, deck, ankiFactId):
fact = self.fact_dao.findByAnkiFactId(deck, ankiFactId)
if fact == None:
fact = Fact(deck.id, ankiFactId, None, None, False, Fact.STATUS_NONE, False, 0)
fact.deck = deck
fact = self.fact_dao.insert(fact)
return fact
def getDefinition(self, fact):
return self.definitionDao.getDefinition(fact.id)
def getFactById(self, ankiFactId):
fact = self.fact_dao.findById(ankiFactId)
fact.deck = self.deck_dao.findById(fact.deckId)
return fact
def getAllFacts(self, deck, ankiFacts):
facts = []
factsToInsert = []
i = 0
for ankiFact in ankiFacts:
fact = self.fact_dao.findByAnkiFactId(deck, ankiFact.id)
if fact == None:
fact = Fact(deck.id, ankiFact.id, None, None, False, Fact.STATUS_NONE, False, 0)
fact.deck = deck
factsToInsert.append(fact)
fact.ankiFactIndex = i
fact.ankiLastModified = ankiFact.modified
facts.append(fact)
i += 1
if len(factsToInsert) > 0:
self.fact_dao.insertAll(factsToInsert)
return facts
def getAllCardsOrderByScore(self, deck = None, language = None):
if language:
decksId = self.decksService.listDecksIdByLanguage(language)
else:
decksId = [deck.id]
log(decksId)
return self.card_dao.getCardsOrderByScore(decksId)
def calcCardStatus(self, deck, ankiCard):
if ankiCard.interval < deck.learnTreshold:
return Card.STATUS_NONE
if ankiCard.interval < deck.knownTreshold:
return Card.STATUS_LEARNT
if ankiCard.interval < deck.matureTreshold:
return Card.STATUS_KNOWN
return Card.STATUS_MATURE
def getAllCardsChanged(self, deck, ankiCards):
changedCards = []
cardsToInsert = []
cardsToUpdate = []
for ankiCard in ankiCards:
card = self.card_dao.findById(deck, ankiCard.id)
status = self.calcCardStatus(deck, ankiCard)
if card == None:
fact = self.getFact(deck, ankiCard.fact.id)
card = Card(deck.id, fact.id, ankiCard.id, status, True)
card.deck = deck
card.fact = fact
cardsToInsert.append(card)
changedCards.append(card)
else:
if status != card.status:
card.statusChanged = True
changedCards.append(card)
card.status = status
cardsToUpdate.append(card)
card.ankiLastModified = ankiCard.modified
if len(cardsToInsert) > 0:
self.card_dao.insertAll(cardsToInsert)
if len(cardsToUpdate) > 0:
self.card_dao.updateAll(cardsToUpdate)
return changedCards
def getAllFactsChanged(self, language):
return self.getAllFactsByLanguage(language, 1)
def getAllFactsByLanguage(self, language, statusChanged = None):
decksId = self.decksService.listDecksIdByLanguage(language)
facts = self.fact_dao.selectAll(decksId, statusChanged)
decks = dict()
for fact in facts:
deck = None
try:
deck = decks[fact.deckId]
except KeyError as e:
deck = self.deck_dao.findById(fact.deckId)
decks[fact.deckId] = deck
fact.deck = deck
return facts
def computeFactsMaturity(self, language):
decksId = self.decksService.listDecksIdByLanguage(language)
facts = self.fact_dao.selectAll(decksId, None) #FIXME: Buggggg with other decks !! replace with all facts for now ?
for fact in facts:
morphemes = self.morpheme_dao.getMorphemesFromFact(fact)
matureMorphemes = []
knownMorphemes = []
learnMorphemes = []
unknownMorphemes = []
morphemesScore = 0
for morpheme in morphemes:
if morpheme.status == Morpheme.STATUS_MATURE:
matureMorphemes.append(morpheme)
if morpheme.status == Morpheme.STATUS_KNOWN:
knownMorphemes.append(morpheme)
if morpheme.status == Morpheme.STATUS_LEARNT:
learnMorphemes.append(morpheme)
if morpheme.status == Morpheme.STATUS_NONE:
unknownMorphemes.append(morpheme)
morphemesScore += morpheme.score
total = len(morphemes)
mature = len(matureMorphemes)
known = len(knownMorphemes)
learnt = len(learnMorphemes)
unknown = len(unknownMorphemes)
status = fact.status
if unknown == 0:
if learnt == 0 and known == 0:
status = Fact.STATUS_REVIEW_EASY
elif learnt > 0:
status = Fact.STATUS_REVIEW_HARD
else :
status = Fact.STATUS_REVIEW_MEDIUM
elif unknown == 1:
if learnt == 0 and known == 0:
status = Fact.STATUS_LEARN_EASY
elif learnt > 0:
status = Fact.STATUS_LEARN_HARD
else :
status = Fact.STATUS_LEARN_MEDIUM
else:
status = Fact.STATUS_TOO_DIFFICULT
if status != fact.status:
fact.status = status
fact.statusChanged = True
score = (mature * 1 + known * 2 + learnt * 6 + unknown * 30) * 300 + morphemesScore
if score != fact.score:
fact.score = score
fact.statusChanged = True
self.fact_dao.updateAll(facts)
#self.morpheme_dao.clearMorphemesStatus()
return facts
def changeMorphemes(self, fact, morphemes):
self.fact_dao.insertFactMorphemes(fact, morphemes)
fact.morphemes = morphemes
return fact
def getMorphemes(self, fact):
return self.morpheme_dao.getMorphemesFromFact(fact, True)
def getAllNewCards(self, language):
decksId = self.decksService.listDecksIdByLanguage(language)
return self.card_dao.getAllNewCards(decksId)
def clearAllFactsStatus(self, language):
decksId = self.decksService.listDecksIdByLanguage(language)
self.fact_dao.clearAllFactsStatus(decksId)
class MorphemesService:
def __init__(self, serviceLocator):
self.serviceLocator = serviceLocator
self.lemmeDao = MorphemeLemmeDao()
self.morphemeDao = MorphemeDao()
self.factDao = FactDao()
self.cardDao = CardDao()
self.deckDao = DeckDao()
self.definitionDao = DefinitionDao()
def setupServices(self):
self.decksService = self.serviceLocator.getDecksService()
self.factsService = self.serviceLocator.getFactsService()
def extractMorphemes(self, expression, deck, language):
morphemes = language.posTagger.posMorphemes(expression, deck, language)
return morphemes
#def analyze(self, expression, language):
# morphemes = self.extractMorphemes(expression, None, None)
# return morphemes
def getList(self, dict):
dictList = list()
for key, value in dict.iteritems():
dictList.append(value)
return dictList
# Store temporarly (not in database) definition morphemes and score in facts
# Dont work with lemmatizater
def getMorphemesFromDB(self, expression, deck, language):
# Unique Morphemes
log("Extract Morphemes")
morphLemmes = self.extractMorphemes(expression, deck, language)
factMorphemes = list()
for morphemeLemme in morphLemmes:
morphLemmeDB = self.lemmeDao.find(morphemeLemme)
if morphLemmeDB == None:
morpheme = Morpheme(0, 0, -1, None, 0, -1)
morpheme.morphLemme = morphemeLemme
else:
morpheme = self.morphemeDao.findByLemmeId(morphLemmeDB.id)
if morpheme == None:
continue
morpheme.morphLemme = morphLemmeDB
if morpheme not in factMorphemes:
factMorphemes.append(morpheme)
return factMorphemes
def analyseDefinition(self, deck, language, facts, ankiFacts):
definitions = self.definitionDao.getDefinitions(facts)
modifiedDefinition = list()
keyModifiedDefinition = list()
defModifiedDefinition = list()
for definition in definitions:
fact = definition.fact
if fact.ankiLastModified == fact.lastUpdated:
continue
isModified = False
definitionExpression = ankiFacts[fact.ankiFactIndex][deck.definitionField]
if definition.definitionHash == None or int(definition.definitionHash) != hash(definitionExpression):
definition.definitionMorphemes = self.getMorphemesFromDB(definitionExpression, deck, language)
definition.definitionHash = hash(definitionExpression)
defModifiedDefinition.append(definition)
isModified = True
definitionKeyExpression = ankiFacts[fact.ankiFactIndex][deck.definitionKeyField]
if definition.definitionKeyHash == None or int(definition.definitionKeyHash) != hash(definitionKeyExpression):
definition.definitionMorphemesKey = self.getMorphemesFromDB(definitionKeyExpression, deck, language)
definition.definitionKeyHash = hash(definitionKeyExpression)
keyModifiedDefinition.append(definition)
isModified = True
if isModified == True:
modifiedDefinition.append(definition)
if len(defModifiedDefinition) > 0:
self.definitionDao.updateAllDefinitionsMorphemes(defModifiedDefinition)
if len(keyModifiedDefinition) > 0:
self.definitionDao.updateAllDefinitionsKeysMorphemes(keyModifiedDefinition)
if len(modifiedDefinition) > 0:
self.definitionDao.updateAll(modifiedDefinition)
def getMorphemesDefinition(self, definition):
morphemesId = self.definitionDao.getAllDefinitionMorphemes(definition)
morphemes = list()
for morphemeId in morphemesId:
morpheme = self.morphemeDao.findById(morphemeId)
if morpheme:
morpheme.morphLemme = self.lemmeDao.findById(morpheme.morphLemmeId)
if morpheme.morphLemme:
morphemes.append(morpheme)
return morphemes
def getMorphemesDefinitionKey(self, definition):
morphemesId = self.definitionDao.getAllDefinitionKeyMorphemes(definition)
morphemes = list()
for morphemeId in morphemesId:
morpheme = self.morphemeDao.findById(morphemeId)
if morpheme:
morpheme.morphLemme = self.lemmeDao.findById(morpheme.morphLemmeId)
if morpheme.morphLemme:
morphemes.append(morpheme)
return morphemes
def analyzeMorphemes(self, facts, deck, language):
# Unique Morphemes
#log("Extract Morphemes")
allUniqueMorphLemmes = dict()
for fact in facts:
morphLemmes = self.extractMorphemes(fact.expression, deck, language)
factMorphLemmes = list()
for morphLemme in morphLemmes:
if morphLemme in allUniqueMorphLemmes:
morphLemme = allUniqueMorphLemmes[morphLemme]
else:
allUniqueMorphLemmes[morphLemme] = morphLemme
factMorphLemmes.append(morphLemme)
fact.morphLemmes = factMorphLemmes
log("Lemmatize Morphemes : " + str(len(allUniqueMorphLemmes)))
if language.lemmatizer:
language.lemmatizer.lemmatizeMorphemes(self.getList(allUniqueMorphLemmes))
self.filterMorphLemmes(self.getList(allUniqueMorphLemmes))
log("Compute Facts <-> Morphemes")
allMorphemes = dict()
for fact in facts:
factUniqueMorphemes = dict()
for morphLemme in fact.morphLemmes:
morpheme = Morpheme(Morpheme.STATUS_NONE, False, morphLemme.id, morphLemme, 0)
if morpheme in allMorphemes:
morpheme = allMorphemes[morpheme]
else:
allMorphemes[morpheme] = morpheme
if morpheme not in factUniqueMorphemes:
factUniqueMorphemes[morpheme] = morpheme
fact.morphemes = self.getList(factUniqueMorphemes)
fact.expressionHash = hash(fact.expression)
fact.lastUpdated = fact.ankiLastModified
fact.expression = None
allMorphemesList = self.getList(allMorphemes)
log("All Unique Morphemes (Lemmatized): " + str(len(allMorphemesList)))
log("persistAll")
self.lemmeDao.persistAll(allMorphemesList)
self.morphemeDao.persistAll(allMorphemesList)
log("insertAllFactMorphemes")
self.factDao.insertAllFactMorphemes(facts)
log("updateAll")
self.factDao.updateAll(facts)
for fact in facts:
fact.morphLemmes = None
fact.morphemes = None
def computeMorphemesMaturity(self, cards):
log("computeMorphemesMaturity")
morphemesInCards = dict()
for card in cards:
morphemes = self.morphemeDao.getMorphemesFromCard(card) # Very Fast 45s pour 20.000
#log("Morphemes: " + str(len(morphemes)))
for morpheme in morphemes:
if morpheme.id not in morphemesInCards:
morphemesInCards[morpheme.id] = morpheme
card.lastUpdated = card.ankiLastModified
morphemesInCardsList = self.getList(morphemesInCards)
log("get New Status for morphemes: " + str(len(morphemesInCardsList)))
modifiedMorphemes = list()
for morpheme in morphemesInCardsList:
#log("morpheme: " + str(morpheme))
status = self.morphemeDao.getNewStatus(morpheme) # Slow :/ XXX: Optimize
#log("status: " + str(status))
if morpheme.status != status:
log("status modified")
morpheme.status = status
morpheme.statusChanged = True
modifiedMorphemes.append(morpheme)
if len(modifiedMorphemes) > 0:
self.morphemeDao.updateAll(modifiedMorphemes)
self.cardDao.updateAll(cards)
#def analyze(self, fact):
# morphemes = self.extractMorphemes(fact.expression)
# attachedMorphemes = []
# for morpheme in morphemes:
# morpheme.morphLemme = self.lemmeDao.persist(morpheme.morphLemme)
# morpheme = self.morphemeDao.persist(morpheme)
# attachedMorphemes.append(morpheme)
# self.factsService.changeMorphemes(fact, attachedMorphemes)
# return morphemes
def getAllPOS(self, language):
try:
return language.posOptions["activatedPos"]
except Exception: pass
def getAllSubPOS(self, language):
try:
return language.posOptions["activatedSubPos"]
except Exception: pass
def getMorphemes(self, searchText = None, decksId = None):
if searchText == None or searchText == "":
return self.lemmeDao.getMorphemes(decksId)
else:
searchExpressions = searchText.split(" ")
status_changed = None
status = None
pos = None
subPos = None
expressions = []
for searchExpression in searchExpressions:
uni = unicode(searchExpression)
if searchExpression == "is:changed":
status_changed = True
elif searchExpression == "-is:changed":
status_changed = False
elif searchExpression == "status:None":
status = Morpheme.STATUS_NONE
elif searchExpression == "status:Learnt":
status = Morpheme.STATUS_LEARNT
elif searchExpression == "status:Known":
status = Morpheme.STATUS_KNOWN
elif searchExpression == "status:Mature":
status = Morpheme.STATUS_MATURE
elif uni.startswith("pos:"):
pos = uni.split(":")[1]
elif uni.startswith("sub_pos:"):
subPos = uni.split(":")[1]
else:
expressions.append(uni)
return self.lemmeDao.getMorphemes(decksId, expressions, status, status_changed, pos, subPos)