def get_csv(file_name): token = '' # GitHub TOKEN repo_path = 'CSSEGISandData/COVID-19' daily_reports_dir_path = 'https://github.com/CSSEGISandData/COVID-19/raw/master/csse_covid_19_data/csse_covid_19_daily_reports/' time_series_dir_path = 'https://github.com/CSSEGISandData/COVID-19/raw/master/csse_covid_19_data/csse_covid_19_time_series/' dr_repo_file_list = 'csse_covid_19_data/csse_covid_19_daily_reports' git = Github(token) repo = git.get_repo(repo_path) daily_reports_file_list = repo.get_contents(dr_repo_file_list) if file_name == 'daily_reports': daily_reports_file_path = daily_reports_dir_path + str( daily_reports_file_list[-2]).split('/')[-1].split(".")[0] + '.csv' req = requests.get(daily_reports_file_path) url_content = req.content csv_file = open('daily_report.csv', 'wb') csv_file.write(url_content) csv_file.close() else: confirmed_global_file_path = time_series_dir_path + 'time_series_covid19_confirmed_global.csv' deaths_global_file_path = time_series_dir_path + 'time_series_covid19_deaths_global.csv' recovered_global_file_path = time_series_dir_path + 'time_series_covid19_recovered_global.csv' return confirmed_global_file_path, deaths_global_file_path, recovered_global_file_path
def get_kaggle_credential_from_secret_repository( repository: str, location: str, gh_token: str) -> KaggleCredential: gh = Github(gh_token) cred_repo = gh.get_repo(repository) kaggle_cred_res = cred_repo.get_contents(location) kaggle_cred = json.loads(kaggle_cred_res.decoded_content) if 'key' not in kaggle_cred or 'username' not in kaggle_cred: raise RuntimeError('Cannot find username or key in kaggle.json file.') kaggle_username = kaggle_cred['username'] kaggle_key = kaggle_cred['key'] ret_cred = KaggleCredential(kaggle_username, kaggle_key) return ret_cred
import pandas as pd import os desired_width = 320 pd.set_option('display.width', desired_width) pd.set_option('display.max_columns', 10) # making path for loading currPath = os.getcwd() inPath = os.path.join(currPath, "output") # import from github -- need token from github.MainClass import Github g = Github('PUT_YOUR_TOKEN_HERE') repo = g.get_repo("Transcranial-Solutions/ICONProject") # making proper logo web address & name using logo file_list = repo.get_contents("vote_analysis/output/logos") split_file_list = lambda x: str(x).split('"')[1] file_list = list(map(split_file_list, file_list)) split_logo_list = lambda x: str(x).split('/')[-1].split('.')[0] logo_list = list(map(split_logo_list, file_list)) github_dir_path = 'https://raw.githubusercontent.com/Transcranial-Solutions/ICONProject/master/' web_list = [github_dir_path + s for s in file_list] # dataframe - name by logo and web address logo_df = pd.DataFrame({'name_by_logo': logo_list, 'logo': web_list}) logo_df = logo_df.iloc[logo_df.logo.str.lower().argsort()].reset_index(drop=True)
class GithubService: """Contains convenience methods and properties for Github-related functionality. An adapter to the functionality of the PyGithub library. """ def __init__(self, access_token: str): """Constructor. :param str access_token: the access token to use for connecting """ self.client = Github(login_or_token=access_token) @lru_cache(maxsize=None) def get_repo(self, repo_name: str) -> Repository: """Return the repository object with the given name. :param str repo_name: the full name of the repository :return: the repository or None if not found :rtype: github.Repository.Repository """ return self.client.get_repo(repo_name) @lru_cache(maxsize=None) def get_pr(self, repo_name: str, pr_num: int): """Return the pull request object with the given number. :param str repo_name: the name of the repository the PR is in :param int pr_num: the identifier of the pull request :return: the pull request object :rtype: PullRequest """ repo = self.get_repo(repo_name) if repo: return repo.get_pull(pr_num) def create_pr_comment(self, repo_name: str, pr_num: int, body: str) -> dict: """Create a comment on the pull request with the given info. :param str repo_name: the name of the repository the PR is in :param int pr_num: the identifier of the pull request :param str body: the body of the comment to add :return: a dictionary with information about the created comment :rtype: dict """ pr = self.get_pr(repo_name, pr_num) issue = pr.as_issue() comment = issue.create_comment(body) return {'id': comment.id, 'html_url': comment.html_url} def get_pr_comments(self, repo_name: str, pr_num: int) -> List[dict]: """Return a list of comments on the PR with the given number. :param str repo_name: the name of the repository the PR is in :param int pr_num: the identifier of the pull request :return: a list of all comments, formatted as: [ {'id': <comment_id>, 'body': <body>, 'updated_at': <updated_at>}, ... ] :rtype: list """ pr = self.get_pr(repo_name, pr_num) issue = pr.as_issue() comments = issue.get_comments() return [ {'id': comment.id, 'body': comment.body, 'updated_at': comment.updated_at} for comment in comments ] def delete_pr_comment(self, repo_name: str, pr_num: int, comment_id: int) -> bool: """Delete the PR comment with the given id :param str repo_name: the name of the repository the PR is in :param int pr_num: the identifier of the pull request :param int comment_id: the ID of the comment to delete :return: True if found and deleted successfully, False otherwise :rtype: bool """ pr = self.get_pr(repo_name, pr_num) issue = pr.as_issue() comment = issue.get_comment(comment_id) if comment is None: return False comment.delete() return True
def read_from_github(): # Input: no # Outputs: the main datasets ready to use # Function: reads the files from GitHub # Online Github path from github.MainClass import Github token = 'f1d4c930800cb3e782c1b61b86101c97deefc4c7' # connected to GitHub using my credentials g = Github(token) # load the repo repo = g.get_repo("CSSEGISandData/COVID-19") file_list = repo.get_contents("csse_covid_19_data/csse_covid_19_time_series") github_dir_path = 'https://github.com/CSSEGISandData/COVID-19/raw/master/csse_covid_19_data/csse_covid_19_time_series/' file_path_confirmed_US = github_dir_path + str(file_list[-5]).split('/')[-1].split(".")[0]+ '.csv' confirmed_US = pd.read_csv(file_path_confirmed_US, error_bad_lines=False) confirmed_US = confirmed_US.drop(['UID', 'iso2', 'iso3', 'code3', 'FIPS', 'Admin2', 'Combined_Key'], axis=1) confirmed_US=confirmed_US.rename(columns = {'Province_State':'Province/State', 'Country_Region':'Country/Region', 'Long_':'Long'}) confirmed_info = confirmed_US.groupby(["Province/State"])['Lat','Long','Country/Region'].agg({ 'Lat' :lambda x: x.mean(), 'Long' :lambda x: x.mean(), 'Country/Region':lambda x: x.unique() }) cases = confirmed_US.drop(columns=['Country/Region', 'Lat', 'Long']).set_index('Province/State') cases = cases.groupby(["Province/State"]).sum() confirmed_US = confirmed_info.merge(cases, left_index=True, right_index=True).reset_index() confirmed_US = confirmed_US[confirmed_US['Long'].isna() == False] confirmed_US = confirmed_US[confirmed_US['Lat'].isna() == False] confirmed_US = confirmed_US[confirmed_US['Long'] != 0] confirmed_US = confirmed_US[confirmed_US['Lat'] != 0] file_path_confirmed = github_dir_path + str(file_list[-4]).split('/')[-1].split(".")[0]+ '.csv' confirmed = pd.read_csv(file_path_confirmed, error_bad_lines=False) confirmed = pd.concat([confirmed_US,confirmed]) file_path_died_US = github_dir_path + str(file_list[-3]).split('/')[-1].split(".")[0]+ '.csv' died_US = pd.read_csv(file_path_died_US, error_bad_lines=False) died_US = died_US.drop(['UID', 'iso2', 'iso3', 'code3', 'FIPS', 'Admin2', 'Combined_Key', 'Population'], axis=1) died_US=died_US.rename(columns = {'Province_State':'Province/State', 'Country_Region':'Country/Region', 'Long_':'Long'}) died_info = died_US.groupby(["Province/State"])['Lat','Long','Country/Region'].agg({ 'Lat' :lambda x: x.mean(), 'Long' :lambda x: x.mean(), 'Country/Region':lambda x: x.unique() }) deaths = died_US.drop(columns=['Country/Region', 'Lat', 'Long']).set_index('Province/State') deaths = deaths.groupby(["Province/State"]).sum() died_US = died_info.merge(deaths, left_index=True, right_index=True).reset_index() died_US = died_US[died_US['Long'].isna() == False] died_US = died_US[died_US['Lat'].isna() == False] died_US = died_US[died_US['Long'] != 0] died_US = died_US[died_US['Lat'] != 0] # file_path_died = github_dir_path + str(file_list[-2]).split('/')[-1].split(".")[0]+ '.csv' died = pd.read_csv(file_path_died, error_bad_lines=False) died = pd.concat([died_US,died]) file_path_recovered = github_dir_path + str(file_list[-1]).split('/')[-1].split(".")[0]+ '.csv' recovered = pd.read_csv(file_path_recovered, error_bad_lines=False) # preprocessing to add total column confirmed = preprocessing(confirmed) died = preprocessing(died) recovered = preprocessing(recovered) # demographics population_statistics = pd.read_csv("population_statistics.csv") return confirmed, died, recovered, population_statistics
import csv import requests import os import sys from github.MainClass import Github import pandas as pd # BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) # sys.path.insert(0, BASE_DIR) from simple_prognozer.config import TOKEN REPO_PATH = 'CSSEGISandData/COVID-19' GIT = Github(TOKEN) REPO = GIT.get_repo(REPO_PATH) DR_REPO_FILE_LIST = 'csse_covid_19_data/csse_covid_19_daily_reports' DAILY_REPORTS_DIR_PATH = ('https://github.com/CSSEGISandData/COVID-19/raw/' 'master/csse_covid_19_data' '/csse_covid_19_daily_reports/' ) def get_csv(file_name): print('Getting daily report csv...') daily_reports_file_list = REPO.get_contents(DR_REPO_FILE_LIST) if file_name == 'daily_reports': daily_reports_file_path = DAILY_REPORTS_DIR_PATH +\ str(daily_reports_file_list[-2]).split('/')[-1].split(".")[
minify_inited = False # Init flask app = Flask(__name__, template_folder="game") if compress_inited: Compress(app) if minify_inited: minify(app=app, js=False, caching_limit=0) import before_after_req # noqa: F401,E402 # Init github if os.getenv("GITHUB_VERSION_PAT") is not None: github_instance = Github(os.getenv("GITHUB_VERSION_PAT")) else: github_instance = Github() rep = github_instance.get_repo("KTibow/scorecard") def sleep(timefor): """ Sleep, but catch interrupts and whatever. Args: timefor: The amount of time to sleep in seconds. """ for _index in range(round(timefor * 16.0)): time.sleep(1 / 16) commit_number = 0