def testGetOrCreateKeyword(self):
# interactive
tui.addInputAnswers("y")
dbutils.getOrCreateKeyword("k1")
self._assertOneObject(Keyword.selectBy(name="k1"))
# !interactive
dbutils.getOrCreateKeyword("k2", interactive=False)
self._assertOneObject(Keyword.selectBy(name="k2"))
def get_keyword_quotes(self, keywords, n=25):
# function to convert incompatible nltk POS tags
def get_pos(treebank_tag):
if treebank_tag.startswith('J'):
return 'a'
elif treebank_tag.startswith('V'):
return 'v'
elif treebank_tag.startswith('N'):
return 'n'
elif treebank_tag.startswith('R'):
return 'r'
else:
return ''
keywords = nltk.tokenize.word_tokenize(keywords)
keywords = nltk.pos_tag(keywords)
keywords = [
self.lemmatize(w[0], get_pos(w[1]))
if get_pos(w[1]) != '' else w[0] for w in keywords
]
keywords = [k for k in keywords if k in self.model.wv.vocab]
# if no words in the vocab, just return garbage...
if len(keywords) == 0:
keywords = ['']
# duplicate short keyword strings to get in enough training
base_keywords = keywords
while len(keywords) < 20:
keywords = keywords + base_keywords
# create new docvec using model
v = self.model.infer_vector(keywords, steps=300)
sims = self.index[v]
sims = sorted(enumerate(sims), key=lambda item: -item[1])
sims = sims[:n]
ids = [i[0] + 1 for i in sims]
# cache keyword's similar quotes in the db. This means when this
# keyword is needed in the future (e.g., to update PCA coords), vector
# and similarity isn't recomputed
quotebytes = np.array(ids).tostring()
keyword = Keyword(quotes=quotebytes)
session = Session()
session.add(keyword)
session.commit()
keyword_id = keyword.id
# retrieve actual quotes
quotes = session.query(Quote).filter(Quote.id.in_(ids)).all()
quotes_dict = {q.id: q for q in quotes}
quotes = [quotes_dict[i] for i in ids]
source_ids = set(
[q.source_id for q in quotes if q.source_id is not None])
sources = session.query(Source).filter(Source.id.in_(source_ids)).all()
person_ids = set([q.person_id for q in quotes])
people = session.query(Person).filter(Person.id.in_(person_ids)).all()
Session.remove()
return quotes, sources, people, keyword_id
def warnIfKeywordDoesNotExist(keywordFilters):
"""Warn user is keyword does not exist
@return: True if at least one keyword does not exist, else False"""
doesNotExist = False
for keyword in [k.name for k in keywordFilters]:
if Keyword.select(LIKE(Keyword.q.name, keyword)).count() == 0:
tui.error("Keyword %s is unknown." % keyword)
doesNotExist = True
return doesNotExist
def do_k_remove(self, line):
"""Remove a keyword"""
keywords=Keyword.select(LIKE(Keyword.q.name, line))
if keywords.count()==0:
print "Sorry, no keyword name matching %s exists. Use k_list to see all defined keywords and k_add to create keywords" % line
return
for keyword in keywords:
if keyword.tasks:
print "The keyword %s is used by the following tasks: %s" % (keyword.name,
", ".join(str(task.id) for task in keyword.tasks))
answer=raw_input("Do you really want to remove this keyword ? (y/n)")
if answer!="y":
print "Skipping deletion of keyword %s" % keyword.name
continue
keyword.destroySelf()
print "Keyword %s has been removed" % keyword.name
def getOrCreateKeyword(keywordName, interactive=True):
"""Get a keyword by its name. Create it if needed
@param keywordName: keyword name as a string
@param interactive: Ask user before creating keyword (this is the default)
@type interactive: Bool
@return: Keyword instance or None if user cancel creation"""
result = Keyword.selectBy(name=keywordName)
result = list(result)
if len(result):
return result[0]
if interactive and not tui.confirm("Keyword '%s' does not exist, create it" % keywordName):
return None
keyword = Keyword(name=keywordName)
print "Added keyword '%s'" % keywordName
return keyword
def do_k_list(self, line):
"""List all keywords."""
for keyword in Keyword.select():
tasks=", ".join(str(task.id) for task in keyword.tasks)
print "%s (tasks: %s)" % (keyword.name, tasks)
def do_t_list(self, line):
doneRangeList= ["today", "thisweek", "thismonth"]
def keywordDictIsSubsetOf(taskKeywordDict, wantedKeywordDict):
# Returns true if taskKeywordDict is a subset of wantedKeywordDict
# taskKeywordDict is considered a subset of wantedKeywordDict if:
# 1. All wantedKeywordDict keys are in taskKeywordDict
# 2. All wantedKeywordDict valued keywords have the same value
# in taskKeywordDict
for wantedKeyword, wantedValue in wantedKeywordDict.items():
if not wantedKeyword in taskKeywordDict:
return False
if wantedValue and taskKeywordDict[wantedKeyword] != wantedValue:
return False
return True
def taskHasWantedKeywordDict(task, wantedKeywordDict):
"""
@param task: task object
@param wantedKeywordDict: dict of name/value of wanted keyword
# a task is considered a subset of wantedKeywordDict if:
# 1. All wantedKeywordDict keys are in task or project keywords
# 2. All wantedKeywordDict valued keywords have the same value
# in task or project keyword"""
for wantedKeyword, wantedValue in wantedKeywordDict.items():
taskFilters=[Task.q.id==task.id,
TaskKeyword.q.taskID==task.id,
TaskKeyword.q.keywordID==Keyword.q.id,
LIKE(Keyword.q.name, wantedKeyword)]
projectFilters=[Project.q.id==task.projectID,
ProjectKeyword.q.projectID==Project.q.id,
ProjectKeyword.q.keyword==Keyword.q.id,
LIKE(Keyword.q.name, wantedKeyword)]
if wantedValue:
taskFilters.append(TaskKeyword.q.value==wantedValue)
projectFilters.append(ProjectKeyword.q.value==wantedValue)
if Task.select(AND(*taskFilters)).count()==0 and Task.select(AND(*projectFilters)).count()==0:
return False
# All critera were met, return ok
return True
def createFilterFromRange(_range):
# Parse the _range string and return an SQLObject filter
minDate = date.today()
if _range == "today":
pass
elif _range == "thisweek":
minDate -= timedelta(minDate.weekday())
elif _range == "thismonth":
minDate = minDate.replace(day = 1)
else:
raise YokadiException("Invalid range value '%s'" % _range)
return Task.q.doneDate>=minDate
def selectRendererClass():
if options.format != "auto":
return gRendererClassDict[options.format]
defaultRendererClass = TextListRenderer
if not options.output:
return defaultRendererClass
ext = os.path.splitext(options.output)[1]
if not ext:
return defaultRendererClass
return gRendererClassDict.get(ext[1:], defaultRendererClass)
#BUG: completion based on parameter position is broken when parameter is given
parser = self.parser_t_list()
options, args = parser.parse_args(line)
if len(args) > 0:
projectName, keywordDict = parseutils.extractKeywords(u" ".join(args))
else:
projectName = ""
keywordDict = {}
if not projectName:
# Take all project if none provided
projectName="%"
projectList = Project.select(LIKE(Project.q.name, projectName))
if projectList.count()==0:
tui.error("Found no project matching '%s'" % projectName)
return
# Check keywords exist
for keyword in keywordDict.keys():
if Keyword.select(LIKE(Keyword.q.name, keyword)).count()==0:
tui.error("Keyword %s is unknown." % keyword)
# Filtering and sorting according to parameters
filters=[]
order=-Task.q.urgency, Task.q.creationDate
limit=None
if options.done:
filters.append(Task.q.status=='done')
if options.done != "all":
filters.append(createFilterFromRange(options.done))
elif not options.all:
filters.append(Task.q.status!='done')
if options.topUrgent:
order=-Task.q.urgency
limit=5
if options.topDue:
filters.append(Task.q.dueDate!=None)
order=Task.q.dueDate
limit=5
if options.search:
for word in options.search:
filters.append(OR(LIKE(Task.q.title, "%"+word+"%"),
LIKE(Task.q.description, "%"+word+"%")))
# Define output
if options.output:
out = open(options.output, "w")
else:
out = tui.stdout
# Instantiate renderer
rendererClass = selectRendererClass()
renderer = rendererClass(out)
# Fill the renderer
if options.keyword:
if options.keyword.startswith("@"):
options.keyword = options.keyword[1:]
for keyword in Keyword.select(LIKE(Keyword.q.name, options.keyword)):
if unicode(keyword.name).startswith("_") and not options.keyword.startswith("_"):
#BUG: cannot filter on db side because sqlobject does not understand ESCAPE needed whith _
continue
taskList = Task.select(AND(TaskKeyword.q.taskID == Task.q.id,
TaskKeyword.q.keywordID == keyword.id,
*filters),
orderBy=order, limit=limit)
taskList = list(taskList)
if keywordDict:
# FIXME: factorize (see project oriented rendering below)
taskList = [x for x in taskList if taskHasWantedKeywordDict(x, keywordDict)]
if projectList:
taskList = [x for x in taskList if x.project in projectList]
if len(taskList) == 0:
continue
renderer.addTaskList(unicode(keyword), taskList)
# Call renderer
renderer.end()
else:
hiddenProjectNames = []
for project in projectList:
if not project.active:
hiddenProjectNames.append(project.name)
continue
taskList = Task.select(AND(Task.q.projectID == project.id, *filters),
orderBy=order, limit=limit)
if keywordDict:
taskList = [x for x in taskList if taskHasWantedKeywordDict(x, keywordDict)]
else:
taskList = list(taskList)
if len(taskList) == 0:
continue
renderer.addTaskList(unicode(project), taskList)
renderer.end()
if len(hiddenProjectNames) > 0:
tui.info("hidden projects: %s" % ", ".join(hiddenProjectNames))
def do_t_list(self, line):
def selectRendererClass():
if options.format != "auto":
return gRendererClassDict[options.format]
defaultRendererClass = TextListRenderer
if not options.output:
return defaultRendererClass
ext = os.path.splitext(options.output)[1]
if not ext:
return defaultRendererClass
return gRendererClassDict.get(ext[1:], defaultRendererClass)
#BUG: completion based on parameter position is broken when parameter is given
parser = self.parser_t_list()
options, args = parser.parse_args(line)
if len(args) > 0:
projectName, keywordFilters = parseutils.extractKeywords(u" ".join(args))
else:
projectName = ""
keywordFilters = []
if self.kFilters:
# Add keyword filter
keywordFilters.extend(self.kFilters)
if not projectName:
if self.pFilter:
# If a project filter is defined, use it as none was provided
projectName = self.pFilter
else:
# Take all project if none provided
projectName = "%"
if projectName.startswith("!"):
projectName = projectName[1:]
projectList = Project.select(NOT(LIKE(Project.q.name, projectName)))
else:
projectList = Project.select(LIKE(Project.q.name, projectName))
if projectList.count() == 0:
tui.error("Found no project matching '%s'" % projectName)
return
# Check keywords exist
parseutils.warnIfKeywordDoesNotExist(keywordFilters)
# Filtering and sorting according to parameters
filters = []
# Filter on keywords
for keywordFilter in keywordFilters:
filters.append(keywordFilter.filter())
order = -Task.q.urgency, Task.q.creationDate
limit = None
if options.done:
filters.append(Task.q.status == 'done')
if options.done != "all":
filters.append(parseutils.createFilterFromRange(options.done))
elif not options.all:
filters.append(Task.q.status != 'done')
if options.topUrgent:
order = -Task.q.urgency
limit = 5
if options.topDue:
filters.append(Task.q.dueDate != None)
order = Task.q.dueDate
limit = 5
if options.overdue:
filters.append(Task.q.dueDate < datetime.now())
order = Task.q.dueDate
if options.search:
for word in options.search:
filters.append(OR(LIKE(Task.q.title, "%" + word + "%"),
LIKE(Task.q.description, "%" + word + "%")))
# Define output
if options.output:
out = open(options.output, "w")
else:
out = tui.stdout
# Instantiate renderer
rendererClass = selectRendererClass()
renderer = rendererClass(out)
# Fill the renderer
if options.keyword:
if options.keyword.startswith("@"):
options.keyword = options.keyword[1:]
for keyword in Keyword.select(LIKE(Keyword.q.name, options.keyword)):
if unicode(keyword.name).startswith("_") and not options.keyword.startswith("_"):
#BUG: cannot filter on db side because sqlobject does not understand ESCAPE needed whith _
continue
taskList = Task.select(AND(TaskKeyword.q.keywordID == keyword.id,
*filters),
orderBy=order, limit=limit, distinct=True,
join=LEFTJOINOn(Task, TaskKeyword, Task.q.id == TaskKeyword.q.taskID))
taskList = list(taskList)
if projectList:
taskList = [x for x in taskList if x.project in projectList]
if len(taskList) == 0:
continue
renderer.addTaskList(unicode(keyword), taskList)
# Call renderer
renderer.end()
else:
hiddenProjectNames = []
for project in projectList:
if not project.active:
hiddenProjectNames.append(project.name)
continue
taskList = Task.select(AND(Task.q.projectID == project.id, *filters),
orderBy=order, limit=limit, distinct=True,
join=LEFTJOINOn(Task, TaskKeyword, Task.q.id == TaskKeyword.q.taskID))
taskList = list(taskList)
if len(taskList) == 0:
continue
renderer.addTaskList(unicode(project), taskList)
renderer.end()
if len(hiddenProjectNames) > 0:
tui.info("hidden projects: %s" % ", ".join(hiddenProjectNames))