def get_user_repos(http_client, access_token):
""" Get a user's repos. """
data = []
# get the results in groups of 100
first_page = yield torngithub.github_request(
http_client,
'/user/repos?page=1&per_page=100',
access_token=access_token)
data.extend(first_page.body)
# get the number of pages (repos // 100 + 1)
max_pages = get_last_page_num(first_page.headers.get('Link', ''))
repos_list = yield [
torngithub.github_request(http_client,
'/user/repos?per_page=100&page=' + str(i),
access_token=access_token)
for i in range(2, max_pages + 1)
]
for repo in repos_list:
data.extend(res.body)
raise tornado.gen.Return(data)
def __create_manifest_file__(self,
name,
access_token,
github_username,
callback=None):
github_collection_name = "brainspell-collection-{}".format(name)
initial_manifest_contents = []
manifest_contents_encoded = b64encode(
json_encode(initial_manifest_contents).encode("utf-8")).decode(
'utf-8')
create_manifest_body = {
"message": "creating manifest.json file",
"content": manifest_contents_encoded
}
create_manifest_ress = yield [
torngithub.github_request(
self.get_auth_http_client(),
'/repos/{owner}/{repo}/contents/{path}'.format(
owner=github_username,
repo=github_collection_name,
path="manifest.json"),
callback=callback,
access_token=access_token,
method="PUT",
body=create_manifest_body)
]
return create_manifest_ress
def __create_repo_on_github__(self,
name,
description,
access_token,
callback=None):
github_collection_name = "brainspell-collection-{}".format(name)
create_repo_body = {
"name": github_collection_name,
"description": description,
"homepage": "https://brainspell-neo.herokuapp.com",
"private": False,
"has_issues": True,
"has_projects": True,
"has_wiki": True
}
# not blocking because torngithub is asynchronous
create_repo_ress = yield [
torngithub.github_request(self.get_auth_http_client(),
'/user/repos',
callback,
access_token=access_token,
method="POST",
body=create_repo_body)
]
return create_repo_ress
def get_my_stars(http_client, access_token):
data = []
first_page = yield torngithub.github_request(
http_client, '/user/starred?page=1&per_page=100',
access_token=access_token)
log.info(first_page.headers.get('Link', ''))
data.extend(first_page.body)
max_pages = get_last_page_num(first_page.headers.get('Link', ''))
ress = yield [torngithub.github_request(
http_client, '/user/starred?per_page=100&page=' + str(i),
access_token=access_token) for i in range(2, max_pages + 1)]
for res in ress:
data.extend(res.body)
raise tornado.gen.Return(data)
def get_my_stars(http_client, access_token):
data = []
first_page = yield torngithub.github_request(
http_client,
'/user/starred?page=1&per_page=100',
access_token=access_token)
log.info(first_page.headers.get('Link', ''))
data.extend(first_page.body)
max_pages = get_last_page_num(first_page.headers.get('Link', ''))
ress = yield [
torngithub.github_request(http_client,
'/user/starred?per_page=100&page=' + str(i),
access_token=access_token)
for i in range(2, max_pages + 1)
]
for res in ress:
data.extend(res.body)
raise tornado.gen.Return(data)
def process(self, response, args):
collection = args["name"]
pmid = args["pmid"]
if remove_article_from_brainspell_database_collection(
collection, pmid, args["key"]):
github_username = get_github_username_from_api_key(args["key"])
github_collection_name = "brainspell-collection-{}".format(
collection)
try:
content_data = yield torngithub.github_request(
self.get_auth_http_client(),
'/repos/{owner}/{repo}/contents/{path}'.format(
owner=github_username,
repo=github_collection_name,
path="manifest.json"),
access_token=args["github_access_token"],
method="GET")
content = content_data["body"]["content"]
# get the SHA so we can update this file
manifest_sha = content_data["body"]["sha"]
# parse the manifest.json file for the collection
collection_contents = json_decode(
b64decode(content.encode('utf-8')).decode('utf-8'))
collection_contents = [
c for c in collection_contents if str(c["pmid"]) != pmid
]
manifest_contents_encoded = b64encode(
json_encode(collection_contents).encode("utf-8")).decode(
'utf-8')
update_manifest_body = {
"message": "creating manifest.json file",
"content": manifest_contents_encoded,
"sha": manifest_sha
}
update_manifest_ress = yield [
torngithub.github_request(
self.get_auth_http_client(),
'/repos/{owner}/{repo}/contents/{path}'.format(
owner=github_username,
repo=github_collection_name,
path="manifest.json"),
access_token=args["github_access_token"],
method="PUT",
body=update_manifest_body)
]
except Exception as e:
print(e)
response["success"] = 0
response[
"description"] = "There was some failure in communicating with GitHub."
finally:
if response["success"] != 0:
# only remove from Brainspell collection if the GitHub
# operation succeeded
remove_article_from_brainspell_database_collection(
collection, pmid, args["key"], False)
else:
response["success"] = 0
response[
"description"] = "That article doesn't exist in that collection."
self.finish_async(response)
def callback_function(github_response=None):
# check if the article is already in this collection
# doesn't matter if we're bulk adding
if args["bulk_add"] == 1 or add_article_to_brainspell_database_collection(
name, args["pmid"], args["key"]):
pmid = args["pmid"]
# idempotent operation to create the Brainspell collection
# if it doesn't already exist (guaranteed that it exists on
# GitHub)
add_collection_to_brainspell_database(name, "None",
args["key"], False)
# make a single PMID into a list so we can treat it the same
# way
if args["bulk_add"] == 0:
pmid = "[" + pmid + "]"
pmid_list = json_decode(pmid)
# add all of the PMIDs to GitHub, then insert to Brainspell
github_username = get_github_username_from_api_key(args["key"])
github_collection_name = "brainspell-collection-{}".format(
name)
content_data = yield torngithub.github_request(
self.get_auth_http_client(),
'/repos/{owner}/{repo}/contents/{path}'.format(
owner=github_username,
repo=github_collection_name,
path="manifest.json"),
access_token=args["github_access_token"],
method="GET")
content = content_data["body"]["content"]
# get the SHA so we can update this file
manifest_sha = content_data["body"]["sha"]
# parse the manifest.json file for the collection
collection_contents = json_decode(
b64decode(content.encode('utf-8')).decode('utf-8'))
article_set = set([c["pmid"] for c in collection_contents])
for p in pmid_list:
if p not in article_set:
# create the entry to add to the GitHub repo
article = list(get_article_object(p))[0]
article_entry = {
"pmid": p,
"title": article.title,
"reference": article.reference,
"doi": article.doi,
"notes": "Here are my notes on this article."
}
collection_contents.append(article_entry)
article_set.add(p)
manifest_contents_encoded = b64encode(
json_encode(collection_contents).encode("utf-8")).decode(
'utf-8')
update_manifest_body = {
"message": "creating manifest.json file",
"content": manifest_contents_encoded,
"sha": manifest_sha
}
update_manifest_ress = yield [
torngithub.github_request(
self.get_auth_http_client(),
'/repos/{owner}/{repo}/contents/{path}'.format(
owner=github_username,
repo=github_collection_name,
path="manifest.json"),
access_token=args["github_access_token"],
method="PUT",
body=update_manifest_body)
]
# actually add the article(s) if the request succeeds
bulk_add_articles_to_brainspell_database_collection(
name, pmid_list, args["key"], False)
else:
response["success"] = 0
response[
"description"] = "That article already exists in that collection."
self.finish_async(response)
def process(self, response, args):
collections_list = []
# get all repos for an authenticated user using GitHub
# need to use GitHub, because not storing "contributors_url" in
# Brainspell's database
data = yield get_user_repos(self.get_auth_http_client(),
args["github_access_token"])
repos = [
d for d in data if d["name"].startswith("brainspell-collection-")
]
if args["force_github_refresh"] == 1:
remove_all_brainspell_database_collections(args["key"])
# get all repos using the Brainspell database
brainspell_cache = get_brainspell_collections_from_api_key(args["key"])
# for each repo
for repo_contents in repos:
# guaranteed that the name starts with "brainspell-collection"
# (from above)
collection_name = repo_contents["name"][
len("brainspell-collection-"):]
repo = {
# take the "brainspell-collection" off of the name
"name": collection_name,
"description": repo_contents["description"]
}
# get the contributor info for this collection
contributor_info = yield torngithub.github_request(
self.get_auth_http_client(),
repo_contents["contributors_url"].replace(
"https://api.github.com", ""),
access_token=args["github_access_token"],
method="GET")
repo["contributors"] = []
if contributor_info["body"]:
repo["contributors"] = contributor_info["body"]
# get the PMIDs in the collection
description = repo_contents["description"]
valid_collection = True
# if there's a new GitHub collection added to the user's profile,
# then add it
if collection_name not in brainspell_cache:
try:
# fetch the PMIDs from GitHub
github_username = get_github_username_from_api_key(
args["key"])
content_data = yield torngithub.github_request(
self.get_auth_http_client(),
'/repos/{owner}/{repo}/contents/{path}'.format(
owner=github_username,
repo=repo_contents["name"],
path="manifest.json"),
access_token=args["github_access_token"],
method="GET")
content = content_data["body"]["content"]
# parse the manifest.json file for the collection
collection_contents = json_decode(
b64decode(content.encode('utf-8')).decode('utf-8'))
pmids = [c["pmid"] for c in collection_contents]
add_collection_to_brainspell_database(
collection_name, description, args["key"], False)
brainspell_cache[collection_name] = {
"description": description,
"pmids": pmids
}
if len(pmids) != 0:
# if PMIDs were fetched from GitHub
bulk_add_articles_to_brainspell_database_collection(
collection_name, pmids, args["key"], False)
except BaseException:
# some error occurred with GitHub
# there was potentially no manifest file
valid_collection = False
if valid_collection:
# accounts for malformed collections, and excludes them
pmids = brainspell_cache[collection_name]["pmids"]
# determine if the pmid (if given) is in this collection
repo["in_collection"] = any(
[str(pmid) == args["pmid"] for pmid in pmids])
# convert PeeWee article object to dict
def parse_article_object(article_object):
return {
"title": article_object.title,
"reference": article_object.reference,
"pmid": article_object.pmid
}
# get article information from each pmid from the database
repo["contents"] = [
parse_article_object(next(get_article_object(p)))
for p in pmids
]
collections_list.append(repo)
response["collections"] = collections_list
self.finish_async(response)
