예제 #1
0
class OpenElections:

    def __init__(self):
        self.github = GitHub()
        self.openelex = self.github.organization('openelections')

    def repos_report(self):
        repos = []
        for repo in self.repos():
            payload = {
                'state': '',
                'repo_type': '',
                'repo_name': repo.name,
                'open_issues': repo.open_issues_count,
                'url': repo.url,
            }
            match = re.search(r'openelections-(sources|data|results)-(\w{2})$', repo.name)
            if match:
                repo_type, state = match.groups()
                payload.update({'repo_type': repo_type, 'state': state.upper()})
            repos.append(payload)
        return repos

    def issues_report(self):
        issues = []
        for repo in self.repos_with_issues():
            for issue in repo.issues:
                payload = {
                    'state': '',
                    'repo_type': '',
                    'repo_name': repo.name,
                    'number': issue.number,
                    'title': issue.title,
                    'assignee': ','.join(issue.assignees),
                    'url': issue.url,
                    'created_date': self._simple_date(issue.created_at),
                    'created_by': issue.author,
                    'labels': ','.join(issue.labels),
                }
                match = re.search(r'openelections-(sources|data|results)-(\w{2})$', repo.name)
                if match:
                    repo_type, state = match.groups()
                    payload.update({'repo_type': repo_type, 'state': state.upper()})
                issues.append(payload)
        return issues

    def repos(self):
        query = ReposQuery()
        repos = query.run()
        return repos

    def repos_with_issues(self):
        query = ReposWithIssuesQuery()
        repos = query.run()
        return repos

    def _simple_date(self, date_str):
        return datetime.datetime\
                .strptime(date_str,"%Y-%m-%dT%H:%M:%SZ")\
                .strftime("%Y-%m-%d")
예제 #2
0
def resolve_team_members(
    github_team_name: str,
    github_api: GitHub,
) -> typing.Union[typing.Generator[str, None, None], list]:
    not_none(github_team_name)
    org_name, team_name = github_team_name.split('/') # always of form 'org/name'
    organisation = github_api.organization(org_name)
    # unfortunately, we have to look-up the team (no api to retrieve it by name)
    team_or_none = _first(filter(lambda team: team.slug == team_name, organisation.teams()))
    if not team_or_none:
        logger.warning('failed to lookup team {t}'.format(t=team_name))
        return []
    for member in map(github_api.user, team_or_none.members()):
        if member.email:
            yield member.email
        else:
            logger.warning(f'no email found for GitHub user {member}')
def main():
    user = input('GitHub username: '******'GitHub password: '******'''Given a function and some arguments, tries to execute the function
        or waits until the ratelimit is reset.'''

        success = False
        while not success:
            try:
                result = function(*args, **kargs)
                success = True
            except GitHubError as e:
                if e.code == 403:
                    # rate limit exceeded
                    reset = gh.rate_limit()['rate']['reset']
                    wait_time = int(reset - time.time()) + 1

                    print("Not enough API calls. Waiting for",
                          int(wait_time / 60), "minutes and", wait_time % 60,
                          "seconds")

                    time.sleep(wait_time)
                else:
                    raise e

        return result

    # get awesome developers
    devmine = call_or_wait(
        lambda: gh.organization("DevMine").iter_public_members())
    # the lambda is used to make the call lazy, as there are multiple API
    # calls involved in the statement

    devs = {dev.login for dev in devmine}

    # get random developers
    for repo in call_or_wait(gh.iter_all_repos,
                             number=settings.REPOS_COUNT,
                             since=settings.REPOS_SINCE_ID):
        # skip organizations and forked repository to find users that actually
        # added new content
        if repo.private or repo.fork or (repo.owner.type == 'Organization'):
            continue
        devs.add(repo.owner.login)

    print(len(devs), "developers fetched")

    fpu = codecs.open(settings.USERS_DATASET, 'a', 'utf-8')
    fpr = codecs.open(settings.REPOS_DATASET, 'a', 'utf-8')

    # dump developers and their repositories
    users_processed = 0
    for dev in devs:
        u = call_or_wait(gh.user, dev)
        json.dump(u.to_json(), fpu)
        fpu.write("\n")
        users_processed += 1
        if users_processed % 100 == 0:
            print("Repos feched for", users_processed, "developers")
        for repo in call_or_wait(gh.iter_user_repos, u.login):
            json.dump(repo.to_json(), fpr)
            fpr.write("\n")

    fpu.close()
    fpr.close()
예제 #4
0
 def __init__(self, gh: GitHub, organization: str) -> None:
     self._gh = gh
     self._org = gh.organization(organization)
예제 #5
0
def main():
    user = input('GitHub username: '******'GitHub password: '******'''Given a function and some arguments, tries to execute the function
        or waits until the ratelimit is reset.'''

        success = False
        while not success:
            try:
                result = function(*args, **kargs)
                success = True
            except GitHubError as e:
                if e.code == 403:
                    # rate limit exceeded
                    reset = gh.rate_limit()['rate']['reset']
                    wait_time = int(reset - time.time()) + 1

                    print("Not enough API calls. Waiting for",
                          int(wait_time / 60), "minutes and",
                          wait_time % 60, "seconds")

                    time.sleep(wait_time)
                else:
                    raise e

        return result

    # get awesome developers
    devmine = call_or_wait(lambda:
                           gh.organization("DevMine").iter_public_members())
    # the lambda is used to make the call lazy, as there are multiple API
    # calls involved in the statement

    devs = {dev.login for dev in devmine}

    # get random developers
    for repo in call_or_wait(gh.iter_all_repos,
                             number=settings.REPOS_COUNT,
                             since=settings.REPOS_SINCE_ID):
        # skip organizations and forked repository to find users that actually
        # added new content
        if repo.private or repo.fork or (repo.owner.type == 'Organization'):
            continue
        devs.add(repo.owner.login)

    print(len(devs), "developers fetched")

    fpu = codecs.open(settings.USERS_DATASET, 'a', 'utf-8')
    fpr = codecs.open(settings.REPOS_DATASET, 'a', 'utf-8')

    # dump developers and their repositories
    users_processed = 0
    for dev in devs:
        u = call_or_wait(gh.user, dev)
        json.dump(u.to_json(), fpu)
        fpu.write("\n")
        users_processed += 1
        if users_processed % 100 == 0:
            print("Repos feched for", users_processed, "developers")
        for repo in call_or_wait(gh.iter_user_repos, u.login):
            json.dump(repo.to_json(), fpr)
            fpr.write("\n")

    fpu.close()
    fpr.close()
예제 #6
0
    'invenio-search': '1cd5740aae6022ffb0a781c3d63dd3b26b83dc61',
    'invenio-sequencegenerator': '1b55943e1f5e8dbf992b9d44412b2a783c415068',
    'invenio-webhooks': '43e2f1e7670781d57f7d1cf9c03bdf91f9afe23b',
}

link_search_name = 'https://github.com/inveniosoftware/{0}/search?q=author-name:{1}&type=Commits'
link_search_login = '******'

with open('./people.yaml') as f:
    people = yaml.load(f)

cern_people = set(p for k, g in people.items() if 'cern' in k for p in g)
external = defaultdict(lambda: defaultdict(list))

g = GitHub(token='<token>')
org = g.organization('inveniosoftware')

for repo in org.repositories():
    if repo.fork or 'flask' in repo.name.lower():
        # Remove forks and flask packages
        # flask-menu
        # flask-breadcrumbs
        # flask-sso
        # flask-iiif
        # flask-sitemap
        # flask-cli
        # flask-celeryext
        # flask-security-fork
        # flask-webpackext
        continue
    if repo.name in base_commits:
예제 #7
0
class GithubAuditor():
    """
    Implements Github3py client and manages connections and credentials
    """
    def __init__(self):
        self.load_app_credentials()
        # self.github = GitHub()
        self.github = GitHub(self.user, token=self.token)
        # TODO Can we use client credentials somehow with something like the below?
        # print(f"Connecting to API as app: {self.app_id}")
        # self.github.login_as_app_installation(
        #     self.private_key_bytes,
        #     self.app_id,
        #     self.install_id
        # )

    def load_app_credentials(self):
        """
        Load credentials defined in environment variables
        """
        if "AUTOSNYK_APP" in os.environ:
            self.app_id = int(os.environ["AUTOSNYK_APP"])

        if "AUTOSNYK_INSTALL" in os.environ:
            self.install_id = int(os.environ["AUTOSNYK_INSTALL"])

        if "AUTOSNYK_USER" in os.environ:
            self.user = os.environ["AUTOSNYK_USER"]

        if "AUTOSNYK_TOKEN" in os.environ:
            self.token = os.environ["AUTOSNYK_TOKEN"]

        if "AUTOSNYK_KEY" in os.environ:
            key_path = os.environ["AUTOSNYK_KEY"]
            with open(key_path, "rb") as key_file:
                self.private_key = serialization.load_pem_private_key(
                    key_file.read(), password=None, backend=default_backend())
                self.private_key_bytes = self.private_key.private_bytes(
                    encoding=serialization.Encoding.PEM,
                    format=serialization.PrivateFormat.PKCS8,
                    encryption_algorithm=serialization.NoEncryption())

    def usage(self):
        say = "python app.py get_repos [org_name]"
        print(self.github.octocat(say))

    def get_repos(self, org_name):
        print(f"get_repos for {org_name}")
        by_status = {"open": [], "private": []}
        for repo in self.github.repositories_by(org_name):
            # print(repo.as_json())
            # print(repo.name)
            if repo.private:
                by_status["private"].append(repo)
            else:
                by_status["open"].append(repo)

        for status, repos in by_status.items():
            count = str(len(repos))
            print(f"{status}: {count}")

    def get_org(self, org_name):
        org = self.github.organization(org_name)
        return org

    def get_org_teams(self, org_name):
        org = self.get_org(org_name)
        teams = org.teams()
        # for team in teams:
        #     #print(team.as_json())
        #     print(team.slug)
        return teams

    def get_team_repos(self, org_name, team_slug):
        teams = self.get_org_teams(org_name)
        repos = []
        for team in teams:
            if team.slug == team_slug:
                repos = team.repositories()
        return repos

    def list_team_repos(self, org_name, team_slug):
        repos = self.get_team_repos(org_name, team_slug)
        for repo in repos:
            print(repo.name)

    def clone_team_repos(self, org_name, team_slug):
        initial_directory = os.getcwd()
        repos = self.get_team_repos(org_name, team_slug)
        checkout_into = f"repos/{org_name}/{team_slug}"
        os.makedirs(checkout_into, exist_ok=True)
        cloned = []
        for repo in repos:
            is_checked_out = os.path.exists(f"{checkout_into}/{repo.name}")
            if not repo.private:
                # if it's already there make sure it's current
                if is_checked_out:
                    os.chdir(checkout_into)
                    print(os.getcwd())
                    # os.system("git reset --hard origin/master")
                    # os.system("git pull")
                    os.chdir(initial_directory)
                else:
                    self.clone_repo(repo, checkout_into)
                cloned.append(repo)
        return cloned

    def clone_repo(self, repo, checkout_into):
        clone_url = repo.clone_url
        print(clone_url)
        return_to = os.getcwd()
        os.chdir(checkout_into)
        os.system(f"git clone {clone_url}")
        os.chdir(return_to)

    def empty_team_repos(self, org_name, team_slug):
        checkout_into = f"repos/{org_name}/{team_slug}"
        #os.removedirs(checkout_into)
        shutil.rmtree(checkout_into)

    def say(self, text):
        self.github.octocat(say=text)