Exemplo n.º 1
0
    def testGetOrCreateKeyword(self):
        # interactive
        tui.addInputAnswers("y")
        dbutils.getOrCreateKeyword("k1")
        self._assertOneObject(Keyword.selectBy(name="k1"))

        # !interactive
        dbutils.getOrCreateKeyword("k2", interactive=False)
        self._assertOneObject(Keyword.selectBy(name="k2"))
Exemplo n.º 2
0
    def get_keyword_quotes(self, keywords, n=25):

        # function to convert incompatible nltk POS tags
        def get_pos(treebank_tag):
            if treebank_tag.startswith('J'):
                return 'a'
            elif treebank_tag.startswith('V'):
                return 'v'
            elif treebank_tag.startswith('N'):
                return 'n'
            elif treebank_tag.startswith('R'):
                return 'r'
            else:
                return ''

        keywords = nltk.tokenize.word_tokenize(keywords)
        keywords = nltk.pos_tag(keywords)
        keywords = [
            self.lemmatize(w[0], get_pos(w[1]))
            if get_pos(w[1]) != '' else w[0] for w in keywords
        ]
        keywords = [k for k in keywords if k in self.model.wv.vocab]

        # if no words in the vocab, just return garbage...
        if len(keywords) == 0:
            keywords = ['']
        # duplicate short keyword strings to get in enough training
        base_keywords = keywords
        while len(keywords) < 20:
            keywords = keywords + base_keywords

        # create new docvec using model
        v = self.model.infer_vector(keywords, steps=300)
        sims = self.index[v]
        sims = sorted(enumerate(sims), key=lambda item: -item[1])
        sims = sims[:n]
        ids = [i[0] + 1 for i in sims]

        # cache keyword's similar quotes in the db. This means when this
        # keyword is needed in the future (e.g., to update PCA coords), vector
        # and similarity isn't recomputed
        quotebytes = np.array(ids).tostring()
        keyword = Keyword(quotes=quotebytes)
        session = Session()
        session.add(keyword)
        session.commit()
        keyword_id = keyword.id

        # retrieve actual quotes
        quotes = session.query(Quote).filter(Quote.id.in_(ids)).all()
        quotes_dict = {q.id: q for q in quotes}
        quotes = [quotes_dict[i] for i in ids]
        source_ids = set(
            [q.source_id for q in quotes if q.source_id is not None])
        sources = session.query(Source).filter(Source.id.in_(source_ids)).all()
        person_ids = set([q.person_id for q in quotes])
        people = session.query(Person).filter(Person.id.in_(person_ids)).all()
        Session.remove()
        return quotes, sources, people, keyword_id
Exemplo n.º 3
0
def warnIfKeywordDoesNotExist(keywordFilters):
    """Warn user is keyword does not exist
    @return: True if at least one keyword does not exist, else False"""
    doesNotExist = False
    for keyword in [k.name for k in keywordFilters]:
            if Keyword.select(LIKE(Keyword.q.name, keyword)).count() == 0:
                tui.error("Keyword %s is unknown." % keyword)
                doesNotExist = True
    return doesNotExist
Exemplo n.º 4
0
 def do_k_remove(self, line):
     """Remove a keyword"""
     keywords=Keyword.select(LIKE(Keyword.q.name, line))
     if keywords.count()==0:
         print "Sorry, no keyword name matching %s exists. Use k_list to see all defined keywords and k_add to create keywords" % line 
         return
     for keyword in keywords:
         if keyword.tasks:
             print "The keyword %s is used by the following tasks: %s" % (keyword.name,
                                                                          ", ".join(str(task.id) for task in keyword.tasks))
             answer=raw_input("Do you really want to remove this keyword ? (y/n)")
             if answer!="y":
                 print "Skipping deletion of keyword %s" % keyword.name
                 continue
         keyword.destroySelf()
         print "Keyword %s has been removed" % keyword.name
Exemplo n.º 5
0
def getOrCreateKeyword(keywordName, interactive=True):
    """Get a keyword by its name. Create it if needed
    @param keywordName: keyword name as a string
    @param interactive: Ask user before creating keyword (this is the default)
    @type interactive: Bool
    @return: Keyword instance or None if user cancel creation"""
    result = Keyword.selectBy(name=keywordName)
    result = list(result)
    if len(result):
        return result[0]

    if interactive and not tui.confirm("Keyword '%s' does not exist, create it" % keywordName):
        return None
    keyword = Keyword(name=keywordName)
    print "Added keyword '%s'" % keywordName
    return keyword
Exemplo n.º 6
0
 def do_k_list(self, line):
     """List all keywords."""
     for keyword in Keyword.select():
         tasks=", ".join(str(task.id) for task in keyword.tasks)
         print "%s (tasks: %s)" % (keyword.name, tasks)
Exemplo n.º 7
0
    def do_t_list(self, line):
        doneRangeList= ["today", "thisweek", "thismonth"]

        def keywordDictIsSubsetOf(taskKeywordDict, wantedKeywordDict):
            # Returns true if taskKeywordDict is a subset of wantedKeywordDict
            # taskKeywordDict is considered a subset of wantedKeywordDict if:
            # 1. All wantedKeywordDict keys are in taskKeywordDict
            # 2. All wantedKeywordDict valued keywords have the same value
            #    in taskKeywordDict
            for wantedKeyword, wantedValue in wantedKeywordDict.items():
                if not wantedKeyword in taskKeywordDict:
                    return False
                if wantedValue and taskKeywordDict[wantedKeyword] != wantedValue:
                    return False
            return True

        def taskHasWantedKeywordDict(task, wantedKeywordDict):
            """
            @param task: task object
            @param wantedKeywordDict: dict of name/value of wanted keyword
            # a task is considered a subset of wantedKeywordDict if:
            # 1. All wantedKeywordDict keys are in task or project keywords
            # 2. All wantedKeywordDict valued keywords have the same value
            #    in task or project keyword"""
            for wantedKeyword, wantedValue in wantedKeywordDict.items():
                taskFilters=[Task.q.id==task.id,
                         TaskKeyword.q.taskID==task.id,
                         TaskKeyword.q.keywordID==Keyword.q.id,
                         LIKE(Keyword.q.name, wantedKeyword)]

                projectFilters=[Project.q.id==task.projectID,
                                ProjectKeyword.q.projectID==Project.q.id,
                                ProjectKeyword.q.keyword==Keyword.q.id,
                                LIKE(Keyword.q.name, wantedKeyword)]

                if wantedValue:
                    taskFilters.append(TaskKeyword.q.value==wantedValue)
                    projectFilters.append(ProjectKeyword.q.value==wantedValue)

                if Task.select(AND(*taskFilters)).count()==0 and Task.select(AND(*projectFilters)).count()==0:
                    return False
            # All critera were met, return ok
            return True

        def createFilterFromRange(_range):
            # Parse the _range string and return an SQLObject filter
            minDate = date.today()
            if _range == "today":
                pass
            elif _range == "thisweek":
                minDate -= timedelta(minDate.weekday())
            elif _range == "thismonth":
                minDate = minDate.replace(day = 1)
            else:
                raise YokadiException("Invalid range value '%s'" % _range)

            return Task.q.doneDate>=minDate

        def selectRendererClass():
            if options.format != "auto":
                return gRendererClassDict[options.format]

            defaultRendererClass = TextListRenderer
            if not options.output:
                return defaultRendererClass

            ext = os.path.splitext(options.output)[1]
            if not ext:
                return defaultRendererClass

            return gRendererClassDict.get(ext[1:], defaultRendererClass)

        #BUG: completion based on parameter position is broken when parameter is given
        parser = self.parser_t_list()
        options, args = parser.parse_args(line)
        if len(args) > 0:
            projectName, keywordDict = parseutils.extractKeywords(u" ".join(args))
        else:
            projectName = ""
            keywordDict = {}

        if not projectName:
            # Take all project if none provided
            projectName="%"

        projectList = Project.select(LIKE(Project.q.name, projectName))

        if projectList.count()==0:
            tui.error("Found no project matching '%s'" % projectName)
            return

        # Check keywords exist
        for keyword in keywordDict.keys():
            if Keyword.select(LIKE(Keyword.q.name, keyword)).count()==0:
                tui.error("Keyword %s is unknown." % keyword)

        # Filtering and sorting according to parameters
        filters=[]
        order=-Task.q.urgency, Task.q.creationDate
        limit=None
        if options.done:
            filters.append(Task.q.status=='done')
            if options.done != "all":
                filters.append(createFilterFromRange(options.done))
        elif not options.all:
            filters.append(Task.q.status!='done')
        if options.topUrgent:
            order=-Task.q.urgency
            limit=5
        if options.topDue:
            filters.append(Task.q.dueDate!=None)
            order=Task.q.dueDate
            limit=5
        if options.search:
            for word in options.search:
                filters.append(OR(LIKE(Task.q.title, "%"+word+"%"),
                                  LIKE(Task.q.description, "%"+word+"%")))

        # Define output
        if options.output:
            out = open(options.output, "w")
        else:
            out = tui.stdout

        # Instantiate renderer
        rendererClass = selectRendererClass()
        renderer = rendererClass(out)

        # Fill the renderer
        if options.keyword:
            if options.keyword.startswith("@"):
                options.keyword = options.keyword[1:]
            for keyword in Keyword.select(LIKE(Keyword.q.name, options.keyword)):
                if unicode(keyword.name).startswith("_") and not options.keyword.startswith("_"):
                    #BUG: cannot filter on db side because sqlobject does not understand ESCAPE needed whith _
                    continue
                taskList = Task.select(AND(TaskKeyword.q.taskID == Task.q.id,
                                           TaskKeyword.q.keywordID == keyword.id,
                                           *filters),
                                        orderBy=order, limit=limit)
                taskList = list(taskList)
                if keywordDict:
                    # FIXME: factorize (see project oriented rendering below)
                    taskList = [x for x in taskList if taskHasWantedKeywordDict(x, keywordDict)]
                if projectList:
                    taskList = [x for x in taskList if x.project in projectList]
                if len(taskList) == 0:
                    continue

                renderer.addTaskList(unicode(keyword), taskList)
            # Call renderer
            renderer.end()
        else:
            hiddenProjectNames = []
            for project in projectList:
                if not project.active:
                    hiddenProjectNames.append(project.name)
                    continue
                taskList = Task.select(AND(Task.q.projectID == project.id, *filters),
                                       orderBy=order, limit=limit)

                if keywordDict:
                    taskList = [x for x in taskList if taskHasWantedKeywordDict(x, keywordDict)]
                else:
                    taskList = list(taskList)
    
                if len(taskList) == 0:
                    continue

                renderer.addTaskList(unicode(project), taskList)
            renderer.end()
    
            if len(hiddenProjectNames) > 0:
                tui.info("hidden projects: %s" % ", ".join(hiddenProjectNames))
Exemplo n.º 8
0
    def do_t_list(self, line):

        def selectRendererClass():
            if options.format != "auto":
                return gRendererClassDict[options.format]

            defaultRendererClass = TextListRenderer
            if not options.output:
                return defaultRendererClass

            ext = os.path.splitext(options.output)[1]
            if not ext:
                return defaultRendererClass

            return gRendererClassDict.get(ext[1:], defaultRendererClass)

        #BUG: completion based on parameter position is broken when parameter is given
        parser = self.parser_t_list()
        options, args = parser.parse_args(line)
        if len(args) > 0:
            projectName, keywordFilters = parseutils.extractKeywords(u" ".join(args))
        else:
            projectName = ""
            keywordFilters = []

        if self.kFilters:
            # Add keyword filter
            keywordFilters.extend(self.kFilters)

        if not projectName:
            if self.pFilter:
                # If a project filter is defined, use it as none was provided
                projectName = self.pFilter
            else:
                # Take all project if none provided
                projectName = "%"

        if projectName.startswith("!"):
            projectName = projectName[1:]
            projectList = Project.select(NOT(LIKE(Project.q.name, projectName)))
        else:
            projectList = Project.select(LIKE(Project.q.name, projectName))

        if projectList.count() == 0:
            tui.error("Found no project matching '%s'" % projectName)
            return

        # Check keywords exist
        parseutils.warnIfKeywordDoesNotExist(keywordFilters)

        # Filtering and sorting according to parameters
        filters = []

        # Filter on keywords
        for keywordFilter in keywordFilters:
            filters.append(keywordFilter.filter())

        order = -Task.q.urgency, Task.q.creationDate
        limit = None
        if options.done:
            filters.append(Task.q.status == 'done')
            if options.done != "all":
                filters.append(parseutils.createFilterFromRange(options.done))
        elif not options.all:
            filters.append(Task.q.status != 'done')
        if options.topUrgent:
            order = -Task.q.urgency
            limit = 5
        if options.topDue:
            filters.append(Task.q.dueDate != None)
            order = Task.q.dueDate
            limit = 5
        if options.overdue:
            filters.append(Task.q.dueDate < datetime.now())
            order = Task.q.dueDate
        if options.search:
            for word in options.search:
                filters.append(OR(LIKE(Task.q.title, "%" + word + "%"),
                                  LIKE(Task.q.description, "%" + word + "%")))

        # Define output
        if options.output:
            out = open(options.output, "w")
        else:
            out = tui.stdout

        # Instantiate renderer
        rendererClass = selectRendererClass()
        renderer = rendererClass(out)

        # Fill the renderer
        if options.keyword:
            if options.keyword.startswith("@"):
                options.keyword = options.keyword[1:]
            for keyword in Keyword.select(LIKE(Keyword.q.name, options.keyword)):
                if unicode(keyword.name).startswith("_") and not options.keyword.startswith("_"):
                    #BUG: cannot filter on db side because sqlobject does not understand ESCAPE needed whith _
                    continue
                taskList = Task.select(AND(TaskKeyword.q.keywordID == keyword.id,
                                           *filters),
                                       orderBy=order, limit=limit, distinct=True,
                                       join=LEFTJOINOn(Task, TaskKeyword, Task.q.id == TaskKeyword.q.taskID))
                taskList = list(taskList)
                if projectList:
                    taskList = [x for x in taskList if x.project in projectList]
                if len(taskList) == 0:
                    continue

                renderer.addTaskList(unicode(keyword), taskList)
            # Call renderer
            renderer.end()
        else:
            hiddenProjectNames = []
            for project in projectList:
                if not project.active:
                    hiddenProjectNames.append(project.name)
                    continue
                taskList = Task.select(AND(Task.q.projectID == project.id, *filters),
                                       orderBy=order, limit=limit, distinct=True,
                                       join=LEFTJOINOn(Task, TaskKeyword, Task.q.id == TaskKeyword.q.taskID))
                taskList = list(taskList)

                if len(taskList) == 0:
                    continue

                renderer.addTaskList(unicode(project), taskList)
            renderer.end()

            if len(hiddenProjectNames) > 0:
                tui.info("hidden projects: %s" % ", ".join(hiddenProjectNames))