def _addProjectFilter(self):
if self.projectName is None:
return
self._filters.append(
IN(
Task.q.project,
Select(Project.q.id, LIKE(Project.q.name, "%" + self.projectName + "%"))
))
def getBugBranchesForBugTasks(self, tasks):
"""See `IBugBranchSet`."""
bug_ids = [task.bugID for task in tasks]
if not bug_ids:
return []
bugbranches = BugBranch.select(IN(BugBranch.q.bugID, bug_ids),
orderBy=['branch'])
return bugbranches.prejoin(
['branch', 'branch.owner', 'branch.product'])
def getSpecificationBranchesForBranches(self, branches, user):
"""See `ISpecificationBranchSet`."""
branch_ids = [branch.id for branch in branches]
if not branch_ids:
return []
# When specification gain the ability to be private, this
# method will need to be updated to enforce the privacy checks.
return SpecificationBranch.select(
IN(SpecificationBranch.q.branchID, branch_ids))
def hasAccess(self, member=False, branch="master", tag="", action="R"):
from Phoenix.Models import Privilege
from re import match
from sqlobject import AND, OR, IN
if not member:
privileges = Privilege.selectBy(repository=self, public=True)
else:
privileges = Privilege.select(
AND(
Privilege.q.repository == self.id,
OR(Privilege.q.member == member.id,
IN(Privilege.q.role, member.roles),
Privilege.q.public == 1)))
print privileges
if privileges.count() > 0:
for p in privileges:
if branch and match(p.branch, branch) and action in p.crud:
return True
if tag and match(p.tag, tag) and action in p.crud:
return True
return False
def retrieve_sites(results):
'''\
Retrieves the site objects associated with the results.
'''
# get a list of all the site keys
site_keys = set()
for r in results:
site_keys.add(r['siteKey'])
# retrieve the site objects from the database
site_objects = Site.select(IN(Site.q.key, list(site_keys)))
# convert results into a dict with site key as the key
sites = {}
for s in site_objects:
sites[s.key] = s
# place site objects into the dict
for r in results:
site_key = r['siteKey']
r['site'] = sites[site_key]
def finalise_question(self, q):
"""
Massages and serialises the question object so it can be inserted into
the search index in the form that we want.
Things this does -
* fetch comments for question and answers and attach them to the objects
* creates the 'text' field for the search index that contains all the
text of the question (title, question, answers and all comments).
* serialises answers to JSON
* creates dict that maps to the search index document schema
* remove unwanted attributes from the q object and serialise question to
JSON
* add question JSON to document
* commit document to search index.
"""
# find and attach any comments for this question and its answers
# get the set of post ids
post_ids = set()
post_ids.add(q['id'])
for a in q['answers']:
post_ids.add(a['id'])
# get the comments
comment_objs = Comment.select(AND(Comment.q.siteId == self.site.id,
IN(Comment.q.postId, list(post_ids))))
# sort the comments out into a dict keyed on the post id
comments = { }
for c in comment_objs:
# convert comment object to a JSON-serialisable object
comment_json = { }
for f in Comment.json_fields:
comment_json[f] = getattr(c, f)
# we already know that this comment comes from the current site, so
# we only need to filter on post ID
if not comments.has_key(c.postId):
comments[c.postId] = [ ]
comments[c.postId].append(comment_json)
# add comments to the question
if comments.has_key(q['id']):
q['comments'].extend(comments[q['id']])
if len(q['comments']) != q['commentCount']:
print('Post ID [%s] expected to have %d comments, but has %d instead. Ignoring inconsistency.' % (q['id'], q['commentCount'], len(q['comments'])))
# add comments to the answers
for a in q['answers']:
if comments.has_key(a['id']):
a['comments'].extend(comments[a['id']])
if len(a['comments']) != a['commentCount']:
print('Post ID [%s] expected to have %d comments, but has %d instead. Ignoring inconsistency.' % (a['id'], a['commentCount'], len(a['comments'])))
doc = { }
# create the text field contents
search_text = [ ]
# question bits
search_text.append(q['title'])
search_text.append(q['body'])
for c in q['comments']:
search_text.append(c['text'])
# answer bits
for a in q['answers']:
search_text.append(a['body'])
for c in a['comments']:
search_text.append(c['text'])
search_text = ' '.join(search_text)
doc['text'] = search_text
# serialise answers to JSON
doc['answers-json'] = [ json.dumps(a, default=self.json_default_handler) for a in q['answers'] ]
# map other fields to search index doc
# this is the ID for Solr to uniquely identify this question across all
# sites
doc['documentId'] = self.site.key + '-' + str(q['id'])
doc['id'] = str(q['id'])
doc['siteKey'] = self.site.key
doc['creationDate'] = q['creationDate']
# the XML field name is score, but score is a reserved lucene keyword
doc['votes'] = q['score']
doc['viewCount'] = q['viewCount']
doc['title'] = q['title']
doc['answerId'] = [ str(a['id']) for a in q['answers'] ]
doc['ownerUserId'] = q['ownerUserId']
if 'lastEditorUserId' in q:
doc['lastEditorUserId'] = q['lastEditorUserId']
doc['lastActivityDate'] = q['lastActivityDate']
if 'communityOwnedDate' in q:
doc['communityOwnedDate'] = q['communityOwnedDate']
if 'closedDate' in q:
doc['closedDate'] = q['closedDate']
if 'tags' in q:
# parse tags into a list
doc['tags'] = PostContentHandler.TAGS_RE.findall(q['tags'])
# serialise question to JSON (the q object has cruft we don't want)
question_obj = { }
question_obj['id'] = q['id']
if 'acceptedAnswerId' in q:
# check that the accepted answer is in the question's answers section.
# sometimes they're not, e.g. if the question was merged - the
# acceptedAnswerId would point to an answer in another question
# instead. We don't deal with merged questions yet, so this option
# means questions won't appear to have answers when they don't.
if q['acceptedAnswerId'] in post_ids:
question_obj['acceptedAnswerId'] = q['acceptedAnswerId']
else:
print 'Question [ID# %i] had an unknown answer. Possibly been merged or migrated. Ignoring inconsistency.' % (q['id'], )
question_obj['creationDate'] = q['creationDate']
question_obj['score'] = q['score']
question_obj['viewCount'] = q['viewCount']
question_obj['body'] = q['body']
question_obj['ownerUserId'] = q['ownerUserId']
if 'lastEditorUserId' in q:
question_obj['lastEditorUserId'] = q['lastEditorUserId']
if 'LastEditDate' in q:
question_obj['lastEditDate'] = q['lastEditDate']
question_obj['lastActivityDate'] = q['lastActivityDate']
if 'communityOwnedDate' in q:
question_obj['communityOwnedDate'] = q['communityOwnedDate']
if 'closedDate' in q:
question_obj['closedDate'] = q['closedDate']
question_obj['title'] = q['title']
if 'tags' in q:
question_obj['tags'] = PostContentHandler.TAGS_RE.findall(q['tags'])
question_obj['favoriteCount'] = q['favoriteCount']
question_obj['comments'] = q['comments']
doc['question-json'] = json.dumps(question_obj, default=self.json_default_handler)
return doc
def retrieve_users(results, question_only=False, ignore_comments=False):
'''\
Retrieves the user objects associated with the question objects.
'''
# get a list of all the user IDs
user_ids_by_site = {}
for r in results:
site_key = r['siteKey']
if site_key not in user_ids_by_site.keys():
user_ids_by_site[site_key] = set()
# the search result object itself
for k in r.keys():
if k.lower().endswith('userid'):
user_ids_by_site[site_key].add(r[k])
# the question object
question = r['question']
for k in question.keys():
if k.lower().endswith('userid'):
user_ids_by_site[site_key].add(question[k])
comments = question.get('comments')
if not ignore_comments and comments:
for c in comments:
for ck in c.keys():
if ck.lower().endswith('userid'):
user_ids_by_site[site_key].add(c[ck])
# the answers
answers = r.get('answers')
if not question_only and answers:
for a in answers:
for k in a.keys():
if k.lower().endswith('userid'):
user_ids_by_site[site_key].add(a[k])
comments = a.get('comments')
if not ignore_comments and comments:
for c in comments:
for ck in c.keys():
if ck.lower().endswith('userid'):
user_ids_by_site[site_key].add(c[ck])
# retrieve the user objects from the database by site
users_by_site = {}
for site_key in user_ids_by_site.keys():
site = Site.select(Site.q.key == site_key).getOne()
user_objects = User.select(
AND(User.q.site == site,
IN(User.q.sourceId, list(user_ids_by_site[site_key]))))
# convert results into a dict with user id as the key
users = {}
for u in user_objects:
users[u.sourceId] = u
users_by_site[site_key] = users
# place user objects into the dict
for r in results:
site_key = r['siteKey']
# the search result object itself
for k in r.keys():
if k.lower().endswith('userid'):
# use the same field name, minus the 'Id' on the end.
r[k[:-2]] = users_by_site[site_key].get(r[k])
# the question object
question = r['question']
for k in question.keys():
if k.lower().endswith('userid'):
# use the same field name, minus the 'Id' on the end.
question[k[:-2]] = users_by_site[site_key].get(question[k])
comments = question.get('comments')
if not ignore_comments and comments:
for c in comments:
for ck in c.keys():
if ck.lower().endswith('userid'):
# use the same field name, minus the 'Id' on the end.
c[ck[:-2]] = users_by_site[site_key].get(c[ck])
# the answers
answers = r.get('answers')
if not question_only and answers:
for a in answers:
for k in a.keys():
if k.lower().endswith('userid'):
# use the same field name, minus the 'Id' on the end.
a[k[:-2]] = users_by_site[site_key].get(a[k])
comments = a.get('comments')
if not ignore_comments and comments:
for c in comments:
for ck in c.keys():
if ck.lower().endswith('userid'):
# use the same field name, minus the 'Id' on the end.
c[ck[:-2]] = users_by_site[site_key].get(c[ck])
def get_api_keys(mss, extension_id, apis=None):
"""
For a given extension, get all the API keys out of the database
`apis` is the result of something like get_extension_api_query() -
if you don't provide it then mqlread will be run to fill it in for
the given extension_id
"""
# get a list of all keys that this extension needs, grouped by API
# (because, in fact, an extension might use APIs that share
# overlapping keys)
if apis is None:
q = get_extension_api_query(extension_id, optional=False)
apis = mss.mqlread(q)
if not apis:
return None
# ok, now authenticate
mss.authenticate()
context = get_context(mss)
# to fetch them from the database, we want a flat list of all unique ids
all_keys = set()
for api in apis:
for api_key in api["api_keys"]:
all_keys.add(api_key)
if api["access_token"]:
all_keys.add(api["access_token"]["id"])
if api["consumer_token"]:
all_keys.add(api["consumer_token"]["id"])
conn = get_sql_connection(mss)
# now query the provider database for all of these specific keys
foreign_key_list = mwOAuthProviderToken.select(AND(
mwOAuthProviderToken.q.context == context,
IN(mwOAuthProviderToken.q.apiKeyId, all_keys)),
connection=conn)
# generate a map of id->key data so we can access it below
foreign_keys = {}
for foreign_key in foreign_key_list:
info = {"id": foreign_key.apiKeyId, "key": foreign_key.key}
if foreign_key.secret:
info["secret"] = foreign_key.secret
foreign_keys[foreign_key.apiKeyId] = info
# now generate a datastructure similar to the mqlread
# something like
# [{ "id": "/netflix/queue_info",
# "consumer_token": {
# "id": "/netflix/consumer_token",
# "key": "ccc",
# "secret": "secretccc",
# },
# "access_token": {
# "id": "/netflix/access_token",
# "key": "aaa",
# "secret": "secretaaa",
# },
# },
# { "id": "/netflix/movie_info",
# "consumer_token": {
# "id": "/netflix/consumer_token",
# "key": "ccc",
# "secret": "secretccc",
# },
# "api_keys": [{
# "id": "/netflix/affiliate_code",
# "key": "fff"
# }]
# }]
api_manifest = []
for api in apis:
api_info = {"id": api["id"]}
api_manifest.append(api_info)
for special_key in ("consumer_token", "access_token"):
if api.get(special_key):
# map "consumer_token" to "/netflix/consumer_token"
special_key_id = api[special_key]["id"]
# even if we dont' have the key, include dummy entry
# meaning that the API requires the key
api_info[special_key] = {"id": special_key_id}
if special_key_id in foreign_keys:
# key and secret MUST be there
foreign_key = foreign_keys[special_key_id]
api_info[special_key]["key"] = foreign_key["key"]
api_info[special_key]["secret"] = foreign_key["secret"]
for api_key in api["api_keys"]:
api_key_id = api_key["id"]
# put a dummy entry in, meaning the API requires/expects
# the key
api_key_info = {
"id": api_key_id,
}
api_info.setdefault("api_keys", []).append(api_key_info)
if api_key_id in foreign_keys:
foreign_key = foreign_keys[api_key_id]
if foreign_key.get("key"):
api_key_info["key"] = foreign_key["key"]
if foreign_key.get("secret"):
api_key_info["secret"] = foreign_key["secret"]
return api_manifest