Beispiel #1
0
def get_csv(file_name):
    token = ''  # GitHub TOKEN

    repo_path = 'CSSEGISandData/COVID-19'
    daily_reports_dir_path = 'https://github.com/CSSEGISandData/COVID-19/raw/master/csse_covid_19_data/csse_covid_19_daily_reports/'
    time_series_dir_path = 'https://github.com/CSSEGISandData/COVID-19/raw/master/csse_covid_19_data/csse_covid_19_time_series/'
    dr_repo_file_list = 'csse_covid_19_data/csse_covid_19_daily_reports'

    git = Github(token)
    repo = git.get_repo(repo_path)
    daily_reports_file_list = repo.get_contents(dr_repo_file_list)

    if file_name == 'daily_reports':
        daily_reports_file_path = daily_reports_dir_path + str(
            daily_reports_file_list[-2]).split('/')[-1].split(".")[0] + '.csv'
        req = requests.get(daily_reports_file_path)
        url_content = req.content
        csv_file = open('daily_report.csv', 'wb')
        csv_file.write(url_content)
        csv_file.close()
    else:
        confirmed_global_file_path = time_series_dir_path + 'time_series_covid19_confirmed_global.csv'
        deaths_global_file_path = time_series_dir_path + 'time_series_covid19_deaths_global.csv'
        recovered_global_file_path = time_series_dir_path + 'time_series_covid19_recovered_global.csv'
        return confirmed_global_file_path, deaths_global_file_path, recovered_global_file_path
Beispiel #2
0
    def find_my_contributions(cls, account):
        """Find and save the user contributions. A contribution is a merged
        Pull Request opened by the user.
        """
        connection = Github(login_or_token=account.github_token)

        # Set query to find contributions
        qualifiers = {'is': 'merged', 'type': 'pr',
                      'author': account.user.username}

        # Search GH API for projects.
        issues = connection.search_issues(
            query='', sort='updated', order='desc', **qualifiers)

        i = 0
        issues_page = issues.get_page(i)
        while issues_page:
            for issue in issues_page:

                attributes = {'account': account, 'title': issue.title,
                              'url': issue.html_url,
                              'repo': issue.repository.name,
                              'repo_url': issue.repository.html_url,
                              'merged': issue.updated_at}

                if not Contribution.objects.filter(**attributes):
                    contribution = Contribution(**attributes)
                    contribution.save()
            i += 1
            issues_page = issues.get_page(i)
    def run(self):
        if self.GITHUB_TOKEN is None:
            logging.critical('No github OAuth token defined in the GITHUB_TOKEN env variable')
            sys.exit(1)
        if self.SSH_PKEY is None:
            logging.critical('SSH_KEY not configured, please set it to you private SSH key file')
            sys.exit(1)

        github = Github(self.GITHUB_TOKEN)
        ssh = SSHClient()
        ssh.load_system_host_keys()
        ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
        self.SSH_PKEY = os.path.expanduser(self.SSH_PKEY)

        orga = github.get_organization('nuxeo')  # type: Organization
        repo = orga.get_repo('nuxeo.com')  # type: Repository
        opened_pulls = [('/var/www/nuxeo.com/pr-%d.' % pull.number) + self.PREVIEW_DOMAIN for pull in repo.get_pulls()]

        try:
            proxy = ProxyCommand(('ssh -i %s -W 10.10.0.63:22 ' % self.SSH_PKEY) + self.BASTION_IP)
            ssh.connect('10.10.0.63', username='******', sock=proxy, key_filename=self.SSH_PKEY)
            _, stdout, _ = ssh.exec_command('ls -d /var/www/nuxeo.com/pr-*')
            [ssh.exec_command('rm -rf ' + line.strip()) for line in stdout.readlines() if line.strip() not in opened_pulls]
            ssh.close()
        except SSHException, e:
            logging.critical('Could work on remote: %s', e)
            sys.exit(1)
    def run(self):
        if self.GITHUB_TOKEN is None:
            logging.critical('No github OAuth token defined in the GITHUB_TOKEN env variable')
            sys.exit(1)

        github = Github(self.GITHUB_TOKEN)
        qualifiers = {
            'query': '',
            'in': 'file',
            'org': 'nuxeo',
            'filename': 'README.md'
        }
        logging.info('Qualifiers: %s', qualifiers)
        search = github.search_code(**qualifiers)  # type: PaginatedList
        logging.info('Number of results: %d', search.totalCount)
        for result in search:  # type: ContentFile
            try:
                logging.info('Updating nuxeo/%s', result.repository.name)
                new_content = result.decoded_content
                commit = result.repository.update_file(
                    '/'+result.path,
                    'NXBT-1198: Remove Company name',
                    new_content,
                    result.sha)['commit']
                logging.info('Updated nuxeo/%s: %s', result.repository.name, commit.html_url)
            except Exception, e:
                logging.warn('Failed failed nuxeo/%s: %s', result.repository.name, e)
Beispiel #5
0
def get_kaggle_credential_from_secret_repository(
        repository: str, location: str, gh_token: str) -> KaggleCredential:
    gh = Github(gh_token)
    cred_repo = gh.get_repo(repository)
    kaggle_cred_res = cred_repo.get_contents(location)
    kaggle_cred = json.loads(kaggle_cred_res.decoded_content)
    if 'key' not in kaggle_cred or 'username' not in kaggle_cred:
        raise RuntimeError('Cannot find username or key in kaggle.json file.')
    kaggle_username = kaggle_cred['username']
    kaggle_key = kaggle_cred['key']
    ret_cred = KaggleCredential(kaggle_username, kaggle_key)
    return ret_cred
Beispiel #6
0
    def try_auth(self) -> Optional[Github]:
        """Create Github client object and attempt hitting API with token"""

        token = self.config.get("token")

        if token:
            try:
                gh_client = Github(token)
                _ = gh_client.get_rate_limit()
                return gh_client

            except BadCredentialsException:
                return None
Beispiel #7
0
def main() -> None:
    try:
        github_token: str = os.environ[ENV_GITHUB_TOKEN]
    except KeyError:
        print(f'Environment variable {ENV_GITHUB_TOKEN} must be defined.')
        sys.exit(1)
    gh = Github(login_or_token=github_token)
    GoPRMaker(gh).run()
Beispiel #8
0
def get_github(repo):
    login_or_token, password = None, None
    if repo.is_private:
        message = "This environment references a private repository: {}".format(
            repo.url)
        login_or_token, password = get_github_credentials(message, force=True)
        if not login_or_token and not password:
            raise EnvironmentException("Github credentials required")
    return Github(login_or_token=login_or_token, password=password)
Beispiel #9
0
def get_github(repo):
    token = None
    if repo.is_private:
        message = "This environment references a private repository: {}".format(
            repo.url)
        token = get_github_token(message, required=True)
        if not token:
            raise EnvironmentException("Github credentials required")
    return Github(login_or_token=token)
Beispiel #10
0
def github(monkeypatch, github_user, github_organization):
    target = Github(login_or_token='_test_user_',
                    password='******',
                    user_agent='MedusaTests')
    monkeypatch.setattr(target, 'get_user',
                        lambda *args, **kwargs: github_user)
    monkeypatch.setattr(target, 'get_organization',
                        lambda login: github_organization)
    return target
Beispiel #11
0
def repo_details(request, login, repo):
    """
    :param request: A url request.
    :param login: An argument from a url
    :param repo: An argument from a ulr
    :return: If the login and the repository exists, returns its statistics
             and adds the login and the name of repository to 'visited'
             cookies.
             If there is not such login and/or repository, returns a 404 error
             message.
             If there is an GitHub API rate limit, returns an 403 error
             message.
    """
    try:
        repository = Github().get_user(login).get_repo(repo)

        session = request.session
        owner = repository.owner.login
        name = repository.name
        if 'visited' in session and session['visited'] is not None:
            if '%s/%s' % (owner, name) not in session['visited']:
                session['visited'] += '---%s/%s' % (owner, name)
        else:
            session['visited'] = '%s/%s' % (owner, name)
            now = datetime.now().time().strftime('%H:%M:%S').split(':')
            # The day's number of seconds minus an amount of ones
            # since the start of the current day.
            seconds_to_midnight = 86400 - (int(now[0]) * 3600 +
                                           int(now[1]) * 60 + int(now[2]))
            # Setting up how much time the cookies file will exist.
            session.set_expiry(seconds_to_midnight)

        week = week_statistics(repository)
        month = month_statistics(repository)

        return render(request, 'statistics_app/repo_details.html',
                      {'repo': repository,
                       'week_commits': week[0],
                       'week_contributors': week[1],
                       'week_days_names': week[2],
                       'month_commits': month[0],
                       'month_contributors': month[1],
                       'month_intervals': month[2],
                       'rate_limit': False, 'not_found': False})

    except (UnknownObjectException, UnicodeEncodeError):
        return render(request, 'statistics_app/repo_details.html',
                      {'not_found': True, 'rate_limit': False})

    except GithubException:
        return render(request, 'statistics_app/repo_details.html',
                      {'rate_limit': True, 'not_found': False})
    def __init__(self):
        # static basedir for github action container
        self.config['base_dir'] = '/github/workspace/'

        # config map
        config = {
            'arg_tag': 'INPUT_TAG',
            'arg_tag_pattern': 'INPUT_TAG_PATTERN',
            'arg_release_name': 'INPUT_RELEASE_NAME',
            'arg_release_desc': 'INPUT_RELEASE_DESCRIPTION',
            'arg_prerelease': 'INPUT_PRERELEASE',
            'arg_assets': 'INPUT_ASSETS',
            'arg_auto_increment': 'INPUT_AUTO_INCREMENT',
            'arg_repo_name': 'INPUT_REPO_NAME',
            'arg_get_last_tag': 'INPUT_GET_LAST_TAG',
            'base_dir': 'INPUT_BASE_DIR'
        }

        for c_key, c_value in config.items():
            value = os.environ.get(c_value)

            if not self.is_empty(value):
                self.config[c_key] = value

        # validate required config
        if (not self.validate_config("tag", self.get_config('arg_tag'))
                and not self.validate_config("assets",
                                             self.get_config('arg_assets'))
                and not self.validate_config(
                    "repo name", self.get_config('arg_repo_name'))):
            self.workflow("error", "Missing required fields")
            exit(1)

        # make sure tag pattern was supplied
        if self.validate_config("tag pattern",
                                self.get_config('arg_tag_pattern')):
            try:
                re_pattern = self.get_config('arg_tag_pattern')
                re.compile(f'{re_pattern}')
            except re.error:
                self.workflow("error", "invalid tag pattern regex")
                exit(1)

        # validate the assets
        self.validate_assets(self.get_config('arg_assets'))

        # connect to github
        self.gh = Github(os.environ.get('INPUT_TOKEN'))

        # run app
        self.run()
Beispiel #13
0
    def find_my_projects(cls, account, type):
        """Find projects for a user, based on hers/his needs: to practice or
        to learn. Save the data in the database.
        """
        connection = Github(login_or_token=account.github_token)

        # Find user Skills
        repos = connection.get_user().get_repos()
        cls.find_account_skills(account, repos)

        # Search for projects based on what skills he wants to practice
        wanted_skills = cls.get_account_wanted_skills(account, type)
        # If the user doesn't have any skills and she/he wants to practice,
        # return none, let him start a project on his own / learn.
        if len(wanted_skills) == 0 and type == ProjectTypes.PRACTICE:
            return

        # We need the names for the GH query.
        wanted_languages = [skill.name.lower() for skill in wanted_skills]

        # Set query to find an active project
        stars = '%d..%d' % (MIN_STARS, MAX_STARS)
        forks = '%d..%d' % (MIN_FORKS, MAX_FORKS)
        pushed = '>%s' % get_last_month()
        qualifiers = {'stars': stars, 'forks': forks, 'pushed': pushed}

        # Search GH API for projects per each language.
        repos = []
        for wanted_language in wanted_languages:
            query = 'language:%s' % wanted_language

            repos_per_lang = connection.search_repositories(
                query=query, sort='stars', order='desc', **qualifiers)

            repos.extend(repos_per_lang.get_page(0)[:MAX_PROJECTS])

        # Save the projects which are a match for the user.
        cls.find_account_projects(account, repos, type)
    def run(self):
        if self.GITHUB_TOKEN is None:
            logging.critical('No github OAuth token defined in the GITHUB_TOKEN env variable')
            sys.exit(1)

        github = Github(self.GITHUB_TOKEN)
        captain_hooks = CaptainHooksClient(self.GITHUB_TOKEN)

        logging.info('Fetching organization %s', self.ORGANIZATION)
        orga = github.get_organization(self.ORGANIZATION)  # type: Organization
        logging.info('Fetching organization repositories')
        for repo in orga.get_repos():  # type: Repository
            for hook in repo.get_hooks():  # type: Hook
                logging.info('%s/%s:backup: ' + json.dumps(hook.raw_data), orga.login, repo.name)
            logging.info('%s/%s:updating', orga.login, repo.name)
            try:
                captain_hooks.setup_webhooks(orga.login, repo.name, {
                    'absent': [
                        {'url': 'http://qapreprod.in.nuxeo.com/jenkins/github-webhook/'},
                        {'url': 'https://qa.nuxeo.org/githooks/send-email'},
                        # {'url': 'https://app.review.ninja/github/webhook'},
                    ],
                    'present': [
                        {
                            'name': 'web',
                            'config': {
                                'content_type': 'json',
                                'url': 'https://hooks.nuxeo.org/hook/'
                            },
                            'events': ['push'],
                            'active': True
                        },
                    ]
                })
                logging.info('%s/%s:done', orga.login, repo.name)
            except CaptainHooksClientException, e:
                logging.warn('%s/%s:failed: %s', orga.login, repo.name, e)
Beispiel #15
0
def main() -> None:
    parser = ArgumentParser()
    parser.add_argument('--update-version-only',
                        type=bool,
                        default=False,
                        help='Exit after updating the GO_VERSION file')
    args: Namespace = parser.parse_args()

    try:
        github_token: str = os.environ[ENV_GITHUB_TOKEN]
    except KeyError:
        print(f'Environment variable {ENV_GITHUB_TOKEN} must be defined.')
        sys.exit(1)
    gh = Github(login_or_token=github_token)
    GoPRMaker(gh).run(args.update_version_only)
Beispiel #16
0
def main() -> None:
    """
	The entrypoint for running the PR-maker.
	"""
    parser = ArgumentParser()
    parser.add_argument('--update-version-only',
                        action="store_true",
                        help='Exit after updating the GO_VERSION file')
    args: Namespace = parser.parse_args()

    try:
        github_token = os.environ[ENV_GITHUB_TOKEN]
    except KeyError:
        print(f'Environment variable {ENV_GITHUB_TOKEN} must be defined.')
        sys.exit(1)
    gh_api = Github(login_or_token=github_token)
    GoPRMaker(gh_api).run(args.update_version_only)
Beispiel #17
0
    def create_pr(self, latest_go_version: str, commit_message: str,
                  owner: str, source_branch_name: str,
                  target_branch: str) -> None:
        """
		Creates the pull request to update the Go version.
		"""
        prs = self.gh_api.search_issues(
            f'repo:{self.repo.full_name} is:pr is:open head:{source_branch_name}'
        )
        for list_item in prs:
            pull_request = self.repo.get_pull(list_item.number)
            if pull_request.head.ref != source_branch_name:
                continue
            print(
                f'Pull request for branch {source_branch_name} already exists:\n{pull_request.html_url}'
            )
            return

        milestone_url = self.get_go_milestone(latest_go_version)
        pr_body = _get_pr_body(latest_go_version, milestone_url)

        try:
            pr_github_token = getenv(ENV_PR_GITHUB_TOKEN)
            self.gh_api = Github(login_or_token=pr_github_token)
            self.repo = self.get_repo(getenv(ENV_GITHUB_REPOSITORY))
        except KeyError:
            print(
                f'Token in {ENV_PR_GITHUB_TOKEN} is invalid, creating the PR using the '
                f'{ENV_GITHUB_TOKEN} token')
            pass

        pull_request = self.repo.create_pull(
            title=commit_message,
            body=pr_body,
            head=f'{owner}:{source_branch_name}',
            base=target_branch,
            maintainer_can_modify=True,
        )
        try:
            go_version_label = self.repo.get_label('go version')
            pull_request.add_to_labels(go_version_label)
        except UnknownObjectException:
            print('Unable to find a label named "go version"', file=sys.stderr)
        print(f'Created pull request {pull_request.html_url}')
# getting the right logo link
# need github api token for this

import pandas as pd
import os
desired_width = 320
pd.set_option('display.width', desired_width)
pd.set_option('display.max_columns', 10)

# making path for loading
currPath = os.getcwd()
inPath = os.path.join(currPath, "output")

# import from github -- need token
from github.MainClass import Github
g = Github('PUT_YOUR_TOKEN_HERE')

repo = g.get_repo("Transcranial-Solutions/ICONProject")

# making proper logo web address & name using logo
file_list = repo.get_contents("vote_analysis/output/logos")
split_file_list = lambda x: str(x).split('"')[1]
file_list = list(map(split_file_list, file_list))

split_logo_list = lambda x: str(x).split('/')[-1].split('.')[0]
logo_list = list(map(split_logo_list, file_list))

github_dir_path = 'https://raw.githubusercontent.com/Transcranial-Solutions/ICONProject/master/'
web_list = [github_dir_path + s for s in file_list]

# dataframe - name by logo and web address
Beispiel #19
0
from dataclasses import dataclass, field
from time import sleep
from typing import Iterator, List, Optional, Tuple, Iterable, Dict

from github import Repository, PullRequest, Issue, PullRequestReview
from github.AuthenticatedUser import AuthenticatedUser
from github.MainClass import Github
from plumbum import cli, local, FG, ProcessExecutionError
from plumbum.cli import switch
import plumbum.colors as colors

GITHUB_TOKEN = os.environ["GITHUB_TOKEN"]
if not GITHUB_TOKEN:
    raise Exception(
        "GitHub token not specified in environment. Please set GITHUB_TOKEN")
github: Github = Github(GITHUB_TOKEN)

git = local["git"]
bash = local["bash"]

verbose = colors.dim
branch_color = colors.green
sha_color = colors.yellow


class RemoteRepo:
    def __init__(self, repo: Repository):
        self._repo = repo

    def get_user_prs(self, user: AuthenticatedUser) -> List['PrInfo']:
        issues: Iterable[Issue] = github.search_issues(
Beispiel #20
0
 def __login(self):
     if hasattr(self.__config, "git_token") and self.__config.git_token:
         self.__github = Github(self.__config.git_token)
     else:
         self.__github = Github(self.__config.git_username, self.__config.git_password)
Beispiel #21
0
try:
    from flask_minify import minify
except ModuleNotFoundError:
    minify_inited = False

# Init flask
app = Flask(__name__, template_folder="game")
if compress_inited:
    Compress(app)
if minify_inited:
    minify(app=app, js=False, caching_limit=0)
import before_after_req  # noqa: F401,E402

# Init github
if os.getenv("GITHUB_VERSION_PAT") is not None:
    github_instance = Github(os.getenv("GITHUB_VERSION_PAT"))
else:
    github_instance = Github()
rep = github_instance.get_repo("KTibow/scorecard")


def sleep(timefor):
    """
    Sleep, but catch interrupts and whatever.

    Args:
        timefor: The amount of time to sleep in seconds.
    """
    for _index in range(round(timefor * 16.0)):
        time.sleep(1 / 16)
Beispiel #22
0
 def github(self):
     return Github(login_or_token=self.github_api_key)
Beispiel #23
0
from github.GithubException import GithubException
from github.MainClass import Github
from github.NamedUser import NamedUser as User
from github.Organization import Organization as Org
from github.Team import Team

from autobot import ORG_NAME
from autobot.concepts import Group

# GH_API_KEY = os.environ["GITHUB_API_KEY"]
_GH_API_KEY = "ca9c8efd282ec1bc79dd10e9a84d2883f0104a51"

_api: Github = Github(_GH_API_KEY, api_preview=True)
_org: Org = _api.get_organization(ORG_NAME)
_team: Team = None
_coord_team: Team = None

_gen_err = "There's something wrong surrounding the GitHub API."


# region Actual calls to the GitHub API
def get_github_user(user: str) -> User:
    return _api.get_user(user)


def create_semester_team(grp: Group) -> int:
    # TODO: Enable nested teams (waiting on PyGitHub to implement this)
    global _team
    _team = _org.create_team(grp.sem.short, privacy="public")
    return _team.id
Beispiel #24
0
def init(args: Namespace) -> tuple:
    """ Init some standard vars """
    g: Github = Github(args.organisation_token)
    org: Organization = g.get_organization(args.organisation_slug)
    team: Team = org.get_team_by_slug(args.team_slug)
    return g, org, team
def read_from_github():
    # Input: no 
    # Outputs: the main datasets ready to use
    # Function: reads the files from GitHub
    # Online Github path
    from github.MainClass import Github
    token = 'f1d4c930800cb3e782c1b61b86101c97deefc4c7'
  	# connected to GitHub using my credentials
    g = Github(token)
    # load the repo
    repo = g.get_repo("CSSEGISandData/COVID-19")
    file_list = repo.get_contents("csse_covid_19_data/csse_covid_19_time_series")
    github_dir_path = 'https://github.com/CSSEGISandData/COVID-19/raw/master/csse_covid_19_data/csse_covid_19_time_series/'
    
    file_path_confirmed_US = github_dir_path  + str(file_list[-5]).split('/')[-1].split(".")[0]+ '.csv'
    confirmed_US = pd.read_csv(file_path_confirmed_US, error_bad_lines=False)
    confirmed_US = confirmed_US.drop(['UID', 'iso2', 'iso3', 'code3', 'FIPS', 'Admin2', 'Combined_Key'], axis=1)
    confirmed_US=confirmed_US.rename(columns = {'Province_State':'Province/State', 
                                          'Country_Region':'Country/Region',
                                          'Long_':'Long'})
    confirmed_info = confirmed_US.groupby(["Province/State"])['Lat','Long','Country/Region'].agg({
            'Lat'      :lambda x: x.mean(),
            'Long'     :lambda x: x.mean(),
            'Country/Region':lambda x: x.unique()
            })
    cases = confirmed_US.drop(columns=['Country/Region', 'Lat', 'Long']).set_index('Province/State')
    cases = cases.groupby(["Province/State"]).sum()
    confirmed_US = confirmed_info.merge(cases, left_index=True, right_index=True).reset_index()    
    confirmed_US = confirmed_US[confirmed_US['Long'].isna() == False]
    confirmed_US = confirmed_US[confirmed_US['Lat'].isna() == False]
    confirmed_US = confirmed_US[confirmed_US['Long'] != 0]
    confirmed_US = confirmed_US[confirmed_US['Lat'] != 0]
    
    file_path_confirmed = github_dir_path  + str(file_list[-4]).split('/')[-1].split(".")[0]+ '.csv'
    confirmed = pd.read_csv(file_path_confirmed, error_bad_lines=False)
    
    confirmed = pd.concat([confirmed_US,confirmed])
     
    file_path_died_US = github_dir_path  + str(file_list[-3]).split('/')[-1].split(".")[0]+ '.csv'
    died_US = pd.read_csv(file_path_died_US, error_bad_lines=False)
    
    died_US = died_US.drop(['UID', 'iso2', 'iso3', 'code3', 'FIPS', 'Admin2', 'Combined_Key', 'Population'], axis=1)
    died_US=died_US.rename(columns = {'Province_State':'Province/State', 
                                          'Country_Region':'Country/Region',
                                          'Long_':'Long'})
    died_info = died_US.groupby(["Province/State"])['Lat','Long','Country/Region'].agg({
            'Lat'      :lambda x: x.mean(),
            'Long'     :lambda x: x.mean(),
            'Country/Region':lambda x: x.unique()
            })
    deaths = died_US.drop(columns=['Country/Region', 'Lat', 'Long']).set_index('Province/State')
    deaths = deaths.groupby(["Province/State"]).sum()
    died_US = died_info.merge(deaths, left_index=True, right_index=True).reset_index()    
    died_US = died_US[died_US['Long'].isna() == False]
    died_US = died_US[died_US['Lat'].isna() == False]
    died_US = died_US[died_US['Long'] != 0]
    died_US = died_US[died_US['Lat'] != 0]
    
#    
    file_path_died = github_dir_path  + str(file_list[-2]).split('/')[-1].split(".")[0]+ '.csv'
    died = pd.read_csv(file_path_died, error_bad_lines=False)
    
    
    died = pd.concat([died_US,died])

    file_path_recovered = github_dir_path  + str(file_list[-1]).split('/')[-1].split(".")[0]+ '.csv'
    recovered = pd.read_csv(file_path_recovered, error_bad_lines=False)

    # preprocessing to add total column
    confirmed = preprocessing(confirmed)
    died = preprocessing(died)
    recovered = preprocessing(recovered)
    
    # demographics
    population_statistics = pd.read_csv("population_statistics.csv")

    return confirmed, died, recovered, population_statistics
Beispiel #26
0
import sys
import os
from github.MainClass import Github

foldername = str(sys.argv[1])
path = os.environ.get('mp')  # add projects dirctory to the env vars
token = os.environ.get('gt')  # add github token to the env vars
_dir = path + '/' + foldername

g = Github(token)
user = g.get_user()
login = user.login
repo = user.create_repo(foldername)

commands = [
    f'echo "# {repo.name}" >> README.md', 'git init',
    f'git remote add origin https://github.com/{login}/{foldername}.git',
    'git add .', 'git commit -m "Initial commit"', 'git push -u origin master'
]

os.mkdir(_dir)
os.chdir(_dir)

for c in commands:
    os.system(c)

print(f'{foldername} created locally')
os.system('code .')
Beispiel #27
0
class GithubService:
    """Contains convenience methods and properties for Github-related
    functionality.

    An adapter to the functionality of the PyGithub library.
    """

    def __init__(self, access_token: str):
        """Constructor.

        :param str access_token: the access token to use for connecting
        """
        self.client = Github(login_or_token=access_token)

    @lru_cache(maxsize=None)
    def get_repo(self, repo_name: str) -> Repository:
        """Return the repository object with the given name.

        :param str repo_name: the full name of the repository
        :return: the repository or None if not found
        :rtype: github.Repository.Repository
        """
        return self.client.get_repo(repo_name)

    @lru_cache(maxsize=None)
    def get_pr(self, repo_name: str, pr_num: int):
        """Return the pull request object with the given number.

        :param str repo_name: the name of the repository the PR is in
        :param int pr_num: the identifier of the pull request
        :return: the pull request object
        :rtype: PullRequest
        """
        repo = self.get_repo(repo_name)
        if repo:
            return repo.get_pull(pr_num)

    def create_pr_comment(self, repo_name: str, pr_num: int, body: str) -> dict:
        """Create a comment on the pull request with the given info.

        :param str repo_name: the name of the repository the PR is in
        :param int pr_num: the identifier of the pull request
        :param str body: the body of the comment to add
        :return: a dictionary with information about the created comment
        :rtype: dict
        """
        pr = self.get_pr(repo_name, pr_num)
        issue = pr.as_issue()
        comment = issue.create_comment(body)
        return {'id': comment.id, 'html_url': comment.html_url}

    def get_pr_comments(self, repo_name: str, pr_num: int) -> List[dict]:
        """Return a list of comments on the PR with the given number.

        :param str repo_name: the name of the repository the PR is in
        :param int pr_num: the identifier of the pull request
        :return: a list of all comments, formatted as:
            [
              {'id': <comment_id>, 'body': <body>, 'updated_at': <updated_at>},
              ...
            ]
        :rtype: list
        """
        pr = self.get_pr(repo_name, pr_num)
        issue = pr.as_issue()
        comments = issue.get_comments()
        return [
            {'id': comment.id, 'body': comment.body, 'updated_at': comment.updated_at}
            for comment in comments
        ]

    def delete_pr_comment(self, repo_name: str, pr_num: int, comment_id: int) -> bool:
        """Delete the PR comment with the given id

        :param str repo_name: the name of the repository the PR is in
        :param int pr_num: the identifier of the pull request
        :param int comment_id: the ID of the comment to delete
        :return: True if found and deleted successfully, False otherwise
        :rtype: bool
        """
        pr = self.get_pr(repo_name, pr_num)
        issue = pr.as_issue()
        comment = issue.get_comment(comment_id)
        if comment is None:
            return False

        comment.delete()
        return True
Beispiel #28
0
class GitHub:

    def __init__(self, config: Config) -> None:
        self.__config: Config = config

        self.__authenticate_git_user()
        self.__initialize_repository_object()

    def __initialize_repository_object(self):
        self.__cache = GitHubCache()
        try:
            org = self.__github.get_organization(self.__config.git_repo_org)
            if self.__config.debug:
                log_debug("Organization found.")
        except UnknownObjectException:
            if self.__config.debug:
                log_debug("Organization not found. Try interpreting " + self.__config.git_repo_org + " as user...")
            org = self.__github.get_user(self.__config.git_repo_org)
            if self.__config.debug:
                log_debug("User found.")

        self.__repo: Repository = org.get_repo(self.__config.git_repo_name)

    # This script is responsible for the authentication of git user
    def __authenticate_git_user(self):
        while True:
            if not hasattr(self.__config, "two_factor_authentication"):
                self.__config.two_factor_authentication = prompt_yesno_question(
                    "Are you using two-factor authentication on GitHub?")
            if self.__config.two_factor_authentication:
                self.__config.git_token = getpass.getpass("> Please enter your token: ")
                while not self.__config.git_token:
                    self.__config.git_token = getpass.getpass("> Please enter your token: ")
            else:
                if not hasattr(self.__config, "git_username"):
                    self.__config.git_username = prompt_enter_value("your git user name")
                else:
                    log_info("The stored Github username is {}".format(self.__config.git_username))
                self.__config.git_password = getpass.getpass("> Please enter your password: "******"> Please enter your password: "******"Authenticated.")
                break
            except BadCredentialsException:
                log_info("Authentication error, please try again.")
                continue

    def __login(self):
        if hasattr(self.__config, "git_token") and self.__config.git_token:
            self.__github = Github(self.__config.git_token)
        else:
            self.__github = Github(self.__config.git_username, self.__config.git_password)

    def find_issue(self, issue_number: int) -> Issue:
        '''Search for the Release issue to be used, if not found, exit'''
        # caching!
        if issue_number in self.__cache.issues:
            log_info("Issue with number " + str(issue_number) + " found.")
            return self.__cache.issues[issue_number]
        else:
            log_debug("Issue not found in cache, retrieving from GitHub...")

        try:
            self.__cache.issues.update({issue_number: self.__repo.get_issue(issue_number)})
            log_info("Issue with number " + str(issue_number) + " found.")
            return self.__cache.issues[issue_number]
        except UnknownObjectException:
            return None

    def create_issue(self, title: str, milestone: Milestone, body: str) -> int:
        '''Function creates an issue in git hub with title,milestone,body,labels passed'''
        if self.__config.dry_run:
            log_info_dry('Skipping creation of issue with title ' + str(title))
            return 0
        if self.__config.debug and not prompt_yesno_question(
            '[DEBUG] Would now create GitHub issue with title="' + title + '", milestone=' + str(
                milestone) + '. Continue?'):
            sys.exit()

        log_info('Create GitHub issue with title "' + title + '"...')

        try:
            issue: Issue = self.__repo.create_issue(title=title, body=body, milestone=milestone,
                                                    labels=[self.__config.issue_label_name, "CI/CD"],
                                                    assignee=self.__github.get_user().login)
            self.__config.github_issue_no = issue.number
            self.__cache.issues.update({issue.number: issue})
            return self.__config.github_issue_no
        except GithubException as e:
            print(str(e))
            return 0

    def __request_milestone_list(self) -> PaginatedList:
        # caching!
        try:
            return self.__cache.milestones
        except AttributeError:
            log_debug("Milestones not found in cache, retrieving from GitHub...")

        try:
            milestones: PaginatedList = self.__repo.get_milestones(state="all")
            self.__cache.milestones = milestones
            return milestones
        except GithubException as e:
            log_error('Could not retrieve milestones')
            print(str(e))
            sys.exit()

    def find_release_milestone(self) -> Milestone:
        milestones: PaginatedList = self.__request_milestone_list()

        for milestone in milestones:
            milestone_title_in_git = milestone.title
            if self.__config.expected_milestone_name in milestone_title_in_git:
                return milestone
        return None

    def find_milestone(self, module: str, version: str) -> Milestone:
        milestones: PaginatedList = self.__request_milestone_list()

        search_title = self.__config.expected_raw_milestone_names.get(module) + version
        log_debug("Trying to search milestone: " + search_title)
        for milestone in milestones:
            if milestone.title == search_title:
                return milestone
        return None

    def create_next_release_milestone(self) -> Milestone:
        new_mile_title = self.__config.expected_milestone_name.replace(self.__config.release_version,
                                                                       self.__config.next_version)
        if self.__config.dry_run:
            log_info_dry("Would now create a new milestone with title '" + new_mile_title + "'.")
            return None

        log_info("Creating milestone '" + new_mile_title + "' for next release...")
        try:
            milestone: Milestone = self.__repo.create_milestone(title=new_mile_title, state="open")
            log_info("New milestone created!")
            return milestone
        except GithubException as e:
            log_info("Could not create milestone!")
            print(str(e))
            return None

    def create_release(self, closed_milestone: Milestone, core_version_in_eclipse_pom: str) -> GitRelease:
        if self.__config.dry_run:
            log_info_dry("Would create a new GitHub release")
            return None

        url_milestone = self.__config.github_closed_milestone_url(closed_milestone.number)
        release_title = self.__config.cobigenwiki_title_name + " v" + self.__config.release_version
        release_text = "[ChangeLog](" + url_milestone + ")"
        if "eclipse" in self.__config.branch_to_be_released and core_version_in_eclipse_pom:
            cobigen_core_milestone: Milestone = self.find_milestone("dev_core", core_version_in_eclipse_pom)
            if cobigen_core_milestone.state == "closed":
                core_url_milestone = self.__config.github_closed_milestone_url(cobigen_core_milestone.number)
                release_text = release_text + "\n also includes \n" + "[ChangeLog CobiGen Core](" + core_url_milestone + ")"
            else:
                log_info("Core version " + core_version_in_eclipse_pom +
                         " is not yet released. This should be released before releasing cobigen-eclipse")
                sys.exit()

        try:
            release: GitRelease = self.__repo.create_git_release(self.__config.tag_name, release_title,
                                                                 release_text, draft=False, prerelease=False,
                                                                 target_commitish="master")

            content_type = "application/java-archive"
            if self.__config.branch_to_be_released == self.__config.branch_eclipseplugin:
                content_type = "application/zip"

            for root, dirs, files in os.walk(
                os.path.join(self.__config.build_folder_abs, self.__config.build_artifacts_root_search_path)):
                dirs[:] = [d for d in dirs if
                           d not in [".settings", "src", "repository", "repository-upload", "classes", "apidocs"]]
                for fname in files:
                    fpath = os.path.join(root, fname)
                    # To prevent uploading of unnecessary zip/jar files.
                    if (fname.endswith("jar") or fname.endswith("zip")) and self.__config.release_version in fname and 'nexus-staging' in fpath:
                        log_info("Uploading file " + fname + " from " + fpath + " ...")
                        try:
                            asset: GitReleaseAsset = release.upload_asset(path=fpath, label=fname,
                                                                          content_type=content_type)
                            log_info("Uploaded " + str(asset.size) + "kb!")
                        except GithubException as e:
                            log_error("Upload failed!")
                            if self.__config.debug:
                                print(str(e))

            # workaround as of https://github.com/PyGithub/PyGithub/issues/779
            self.__login()
            self.__initialize_repository_object()

            return release
        except GithubException as e:
            log_error("Could not create release.")
            print(str(e))
            sys.exit()
Beispiel #29
0
    def __init__(self, access_token: str):
        """Constructor.

        :param str access_token: the access token to use for connecting
        """
        self.client = Github(login_or_token=access_token)
Beispiel #30
0
def run():
    # Init some variables
    args = docopt(__doc__)
    ctx = Context()
    ctx.debugging = args["--debug"]

    # Check for dotenv file & load variables
    deliverit.dotenv.load(ctx)

    # read config file
    config_filepath = args["--config-file"] or (".deliverit.yaml" if Path(
        ".deliverit.yaml").is_file() else ".deliverit.yml")

    config = deliverit.config.load(config_filepath,
                                   cli_args=args,
                                   has_git_remote=has_git_remote())

    # Read manifest file to get some info
    # TODO: support multiple manifest files
    (
        ctx.old_version,
        ctx.package_name,
        ctx.repository_url,
    ) = deliverit.manifest_file.load(config.manifest_file)

    if ctx.old_version is None and config.tag_name is None:
        raise ConfigurationError("Please set either manifest_file or tag_name")
    ctx.old_version = ctx.old_version or get_current_version_from_git_tag(
        tag_template=config.tag_name, fallback_version=Version(0, 1, 0))
    ctx.package_name = ctx.package_name or config.package_name
    if ctx.package_name is None:
        raise ConfigurationError(
            "Could not detect the package name. Set it explicitly with package_name"
        )
    ctx.repository_url = ctx.repository_url or config.repository_url
    if ctx.repository_url is None:
        raise ConfigurationError(
            "Could not detect the github repository's URL. Set it explicitly with repository_url"
        )

    # Check if repository is hosted on github
    if not is_hosted_on_github(ctx.repository_url):
        raise NotImplementedError("Your repository is not hosted on github")

    # Get the repository name
    ctx.repository_owner, ctx.repository_name = split_repository_name(
        ctx.repository_url)
    ctx.repository_full_name = f"{ctx.repository_owner}/{ctx.repository_name}"

    # Compute new version
    ctx.version_bump = ("major" if args["major"] else "minor" if args["minor"]
                        else "patch" if args["patch"] else None)
    if ctx.version_bump is None:
        raise ValueError("No version bump specified.")
    ctx.new_version = ctx.old_version.bump(ctx.version_bump)

    # Compute some configurable values
    version_tag = config.tag_name.format(new=ctx.new_version)

    # Log info
    print(f"""\
Releasing a new {em(ctx.version_bump)} version!

     Upgrading package {em(ctx.package_name)} by {em(ctx.repository_owner)}
             hosted at {em(ctx.repository_url)}
          published on {em(config.registry)}
          from version {em(ctx.old_version)}
            to version {em(ctx.new_version)}
""")

    # Make the step function
    step = make_step_function(args, config)

    # Modify the changelog
    step(
        "update_changelog",
        "Update the changelog",
        lambda: deliverit.changelog.update(
            ctx,
            ctx.apply(config.changelog),
            config.tag_name.replace("{new}", "{t}"),
        ),
    )

    # Codemods
    for declaration in config.version_declarations:
        new_content = ctx.apply(declaration.replace)
        step(
            "update_code_version",
            f"Replace {declaration.search} with {new_content} in {ctx.apply(declaration.in_)}",
            lambda: deliverit.version_declaration.update(ctx, declaration),
        )

    # Bump version
    step(
        "bump_manifest_version",
        "Bump the poetry version",
        command=ctx.apply(config.steps.bump_manifest_version),
    )

    # Add all changes
    step(
        "git_add",
        "Add changes",
        command=(
            "git",
            "add",
            *[ctx.apply(f.in_) for f in config.version_declarations],
            config.changelog,
            config.manifest_file,
        ),
    )

    # Commit
    step(
        "git_commit",
        "Commit the version bump",
        command=("git", "commit", "-m", ctx.apply(config.commit_message)),
    )

    # Add tag to commit
    try:
        latest_commit_hash = get_latest_commit_hash()
    except CalledProcessError:
        print(red("Could not get the latest commit's hash"))
        exit(1)
    step(
        "git_tag",
        f"Add tag {version_tag} to commit {latest_commit_hash[:7]}",
        command=(
            "git",
            "tag",
            "-a",
            version_tag,
            latest_commit_hash,
            "-m",
            ctx.apply(config.commit_message),
        ),
    )

    # Push
    step("git_push", "Push changes", command=("git", "push"))

    # Push tags
    step(
        "git_push_tag",
        f"Push the tag {version_tag}",
        command=("git", "push", "origin", version_tag),
    )

    # Build
    step(
        "build_for_registry",
        "Build for registry",
        command=config.steps.build_for_registry,
    )

    # Publish
    step(
        "publish_to_registry",
        f"Publish to {config.registry}",
        command=config.steps.publish_to_registry,
    )

    # Start a Github API session
    gh = Github(getenv("GITHUB_TOKEN"))

    # Get the release notes
    release_notes = get_release_notes_for_version(
        ctx.new_version,
        Path(config.changelog).read_text("utf-8"))

    release = step(
        "create_github_release",
        "Create a GitHub release",
        lambda: create_github_release(
            ctx,
            gh,
            ctx.apply(config.tag_name),
            ctx.apply(config.release_title),
            message=release_notes,
        ),
    )

    step(
        "add_assets_to_github_release",
        "Upload assets to the Github release",
        lambda: upload_assets_to_release(
            ctx,
            release,
            assets=config.release_assets,
        ),
    )

    step(
        "close_milestone",
        "Close the milestone",
        lambda: close_milestone(ctx, gh, ctx.apply(config.milestone_title)),
    )
import csv
import requests
import os
import sys

from github.MainClass import Github
import pandas as pd

# BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
# sys.path.insert(0, BASE_DIR)
from simple_prognozer.config import TOKEN


REPO_PATH = 'CSSEGISandData/COVID-19'
GIT = Github(TOKEN)
REPO = GIT.get_repo(REPO_PATH)
DR_REPO_FILE_LIST = 'csse_covid_19_data/csse_covid_19_daily_reports'
DAILY_REPORTS_DIR_PATH = ('https://github.com/CSSEGISandData/COVID-19/raw/'
                          'master/csse_covid_19_data'
                          '/csse_covid_19_daily_reports/'
                          )


def get_csv(file_name):
    print('Getting daily report csv...')

    daily_reports_file_list = REPO.get_contents(DR_REPO_FILE_LIST)

    if file_name == 'daily_reports':
        daily_reports_file_path = DAILY_REPORTS_DIR_PATH +\