def get_user_repos(http_client, access_token): """ Get a user's repos. """ data = [] # get the results in groups of 100 first_page = yield torngithub.github_request( http_client, '/user/repos?page=1&per_page=100', access_token=access_token) data.extend(first_page.body) # get the number of pages (repos // 100 + 1) max_pages = get_last_page_num(first_page.headers.get('Link', '')) repos_list = yield [ torngithub.github_request(http_client, '/user/repos?per_page=100&page=' + str(i), access_token=access_token) for i in range(2, max_pages + 1) ] for repo in repos_list: data.extend(res.body) raise tornado.gen.Return(data)
def __create_manifest_file__(self, name, access_token, github_username, callback=None): github_collection_name = "brainspell-collection-{}".format(name) initial_manifest_contents = [] manifest_contents_encoded = b64encode( json_encode(initial_manifest_contents).encode("utf-8")).decode( 'utf-8') create_manifest_body = { "message": "creating manifest.json file", "content": manifest_contents_encoded } create_manifest_ress = yield [ torngithub.github_request( self.get_auth_http_client(), '/repos/{owner}/{repo}/contents/{path}'.format( owner=github_username, repo=github_collection_name, path="manifest.json"), callback=callback, access_token=access_token, method="PUT", body=create_manifest_body) ] return create_manifest_ress
def __create_repo_on_github__(self, name, description, access_token, callback=None): github_collection_name = "brainspell-collection-{}".format(name) create_repo_body = { "name": github_collection_name, "description": description, "homepage": "https://brainspell-neo.herokuapp.com", "private": False, "has_issues": True, "has_projects": True, "has_wiki": True } # not blocking because torngithub is asynchronous create_repo_ress = yield [ torngithub.github_request(self.get_auth_http_client(), '/user/repos', callback, access_token=access_token, method="POST", body=create_repo_body) ] return create_repo_ress
def get_my_stars(http_client, access_token): data = [] first_page = yield torngithub.github_request( http_client, '/user/starred?page=1&per_page=100', access_token=access_token) log.info(first_page.headers.get('Link', '')) data.extend(first_page.body) max_pages = get_last_page_num(first_page.headers.get('Link', '')) ress = yield [torngithub.github_request( http_client, '/user/starred?per_page=100&page=' + str(i), access_token=access_token) for i in range(2, max_pages + 1)] for res in ress: data.extend(res.body) raise tornado.gen.Return(data)
def get_my_stars(http_client, access_token): data = [] first_page = yield torngithub.github_request( http_client, '/user/starred?page=1&per_page=100', access_token=access_token) log.info(first_page.headers.get('Link', '')) data.extend(first_page.body) max_pages = get_last_page_num(first_page.headers.get('Link', '')) ress = yield [ torngithub.github_request(http_client, '/user/starred?per_page=100&page=' + str(i), access_token=access_token) for i in range(2, max_pages + 1) ] for res in ress: data.extend(res.body) raise tornado.gen.Return(data)
def process(self, response, args): collection = args["name"] pmid = args["pmid"] if remove_article_from_brainspell_database_collection( collection, pmid, args["key"]): github_username = get_github_username_from_api_key(args["key"]) github_collection_name = "brainspell-collection-{}".format( collection) try: content_data = yield torngithub.github_request( self.get_auth_http_client(), '/repos/{owner}/{repo}/contents/{path}'.format( owner=github_username, repo=github_collection_name, path="manifest.json"), access_token=args["github_access_token"], method="GET") content = content_data["body"]["content"] # get the SHA so we can update this file manifest_sha = content_data["body"]["sha"] # parse the manifest.json file for the collection collection_contents = json_decode( b64decode(content.encode('utf-8')).decode('utf-8')) collection_contents = [ c for c in collection_contents if str(c["pmid"]) != pmid ] manifest_contents_encoded = b64encode( json_encode(collection_contents).encode("utf-8")).decode( 'utf-8') update_manifest_body = { "message": "creating manifest.json file", "content": manifest_contents_encoded, "sha": manifest_sha } update_manifest_ress = yield [ torngithub.github_request( self.get_auth_http_client(), '/repos/{owner}/{repo}/contents/{path}'.format( owner=github_username, repo=github_collection_name, path="manifest.json"), access_token=args["github_access_token"], method="PUT", body=update_manifest_body) ] except Exception as e: print(e) response["success"] = 0 response[ "description"] = "There was some failure in communicating with GitHub." finally: if response["success"] != 0: # only remove from Brainspell collection if the GitHub # operation succeeded remove_article_from_brainspell_database_collection( collection, pmid, args["key"], False) else: response["success"] = 0 response[ "description"] = "That article doesn't exist in that collection." self.finish_async(response)
def callback_function(github_response=None): # check if the article is already in this collection # doesn't matter if we're bulk adding if args["bulk_add"] == 1 or add_article_to_brainspell_database_collection( name, args["pmid"], args["key"]): pmid = args["pmid"] # idempotent operation to create the Brainspell collection # if it doesn't already exist (guaranteed that it exists on # GitHub) add_collection_to_brainspell_database(name, "None", args["key"], False) # make a single PMID into a list so we can treat it the same # way if args["bulk_add"] == 0: pmid = "[" + pmid + "]" pmid_list = json_decode(pmid) # add all of the PMIDs to GitHub, then insert to Brainspell github_username = get_github_username_from_api_key(args["key"]) github_collection_name = "brainspell-collection-{}".format( name) content_data = yield torngithub.github_request( self.get_auth_http_client(), '/repos/{owner}/{repo}/contents/{path}'.format( owner=github_username, repo=github_collection_name, path="manifest.json"), access_token=args["github_access_token"], method="GET") content = content_data["body"]["content"] # get the SHA so we can update this file manifest_sha = content_data["body"]["sha"] # parse the manifest.json file for the collection collection_contents = json_decode( b64decode(content.encode('utf-8')).decode('utf-8')) article_set = set([c["pmid"] for c in collection_contents]) for p in pmid_list: if p not in article_set: # create the entry to add to the GitHub repo article = list(get_article_object(p))[0] article_entry = { "pmid": p, "title": article.title, "reference": article.reference, "doi": article.doi, "notes": "Here are my notes on this article." } collection_contents.append(article_entry) article_set.add(p) manifest_contents_encoded = b64encode( json_encode(collection_contents).encode("utf-8")).decode( 'utf-8') update_manifest_body = { "message": "creating manifest.json file", "content": manifest_contents_encoded, "sha": manifest_sha } update_manifest_ress = yield [ torngithub.github_request( self.get_auth_http_client(), '/repos/{owner}/{repo}/contents/{path}'.format( owner=github_username, repo=github_collection_name, path="manifest.json"), access_token=args["github_access_token"], method="PUT", body=update_manifest_body) ] # actually add the article(s) if the request succeeds bulk_add_articles_to_brainspell_database_collection( name, pmid_list, args["key"], False) else: response["success"] = 0 response[ "description"] = "That article already exists in that collection." self.finish_async(response)
def process(self, response, args): collections_list = [] # get all repos for an authenticated user using GitHub # need to use GitHub, because not storing "contributors_url" in # Brainspell's database data = yield get_user_repos(self.get_auth_http_client(), args["github_access_token"]) repos = [ d for d in data if d["name"].startswith("brainspell-collection-") ] if args["force_github_refresh"] == 1: remove_all_brainspell_database_collections(args["key"]) # get all repos using the Brainspell database brainspell_cache = get_brainspell_collections_from_api_key(args["key"]) # for each repo for repo_contents in repos: # guaranteed that the name starts with "brainspell-collection" # (from above) collection_name = repo_contents["name"][ len("brainspell-collection-"):] repo = { # take the "brainspell-collection" off of the name "name": collection_name, "description": repo_contents["description"] } # get the contributor info for this collection contributor_info = yield torngithub.github_request( self.get_auth_http_client(), repo_contents["contributors_url"].replace( "https://api.github.com", ""), access_token=args["github_access_token"], method="GET") repo["contributors"] = [] if contributor_info["body"]: repo["contributors"] = contributor_info["body"] # get the PMIDs in the collection description = repo_contents["description"] valid_collection = True # if there's a new GitHub collection added to the user's profile, # then add it if collection_name not in brainspell_cache: try: # fetch the PMIDs from GitHub github_username = get_github_username_from_api_key( args["key"]) content_data = yield torngithub.github_request( self.get_auth_http_client(), '/repos/{owner}/{repo}/contents/{path}'.format( owner=github_username, repo=repo_contents["name"], path="manifest.json"), access_token=args["github_access_token"], method="GET") content = content_data["body"]["content"] # parse the manifest.json file for the collection collection_contents = json_decode( b64decode(content.encode('utf-8')).decode('utf-8')) pmids = [c["pmid"] for c in collection_contents] add_collection_to_brainspell_database( collection_name, description, args["key"], False) brainspell_cache[collection_name] = { "description": description, "pmids": pmids } if len(pmids) != 0: # if PMIDs were fetched from GitHub bulk_add_articles_to_brainspell_database_collection( collection_name, pmids, args["key"], False) except BaseException: # some error occurred with GitHub # there was potentially no manifest file valid_collection = False if valid_collection: # accounts for malformed collections, and excludes them pmids = brainspell_cache[collection_name]["pmids"] # determine if the pmid (if given) is in this collection repo["in_collection"] = any( [str(pmid) == args["pmid"] for pmid in pmids]) # convert PeeWee article object to dict def parse_article_object(article_object): return { "title": article_object.title, "reference": article_object.reference, "pmid": article_object.pmid } # get article information from each pmid from the database repo["contents"] = [ parse_article_object(next(get_article_object(p))) for p in pmids ] collections_list.append(repo) response["collections"] = collections_list self.finish_async(response)