def show_repos(request): """Get repositories for organizations.""" context = set_up_context(request) if request.method == 'POST': form_orgs = OrgNamesForm(request.POST) if form_orgs.is_valid(): session = requests.Session() repos_choices = [] for org_name in form_orgs.cleaned_data['organizations'].split(): org_data = github.get_org_data(org_name, session) org, _ = misc.get_or_create_record(Organization, org_data) repos_data = [ repo for repo in github.get_org_repos(org_name, session) ] for repo_data in repos_data: misc.get_or_create_record( org, repo_data, { 'is_tracked': False, 'is_visible': False }, ) repos_choices.append( (repo_data['id'], repo_data['name']), ) session.close() form_repos = RepoNamesForm(choices=repos_choices) context['form_repos'] = form_repos else: form_orgs = OrgNamesForm() context['form_orgs'] = form_orgs return TemplateResponse(request, 'admin/configuration.html', context)
def get_or_create_contributor(login): """Return a contributor object.""" try: return Contributor.objects.get(login=login) except Contributor.DoesNotExist: user_data = github.get_user_data(login, session) contributor, _ = misc.get_or_create_record(Contributor, user_data) return contributor
def handle_user_post_save(sender, **kwargs): """Tie a user with a contributor profile.""" site_user = kwargs['instance'] if site_user.has_usable_password(): # The user has registered with the default backend return if kwargs['created']: contributor, _ = misc.get_or_create_record( apps.get_model('contributors.Contributor'), misc.get_contributor_data(site_user.username), ) contributor.user = site_user contributor.save()
def handle(self, *args, **options): # noqa: WPS """Collect data from GitHub.""" logger.info("Data collection started") if options['repo']: data_of_orgs_and_repos = github.get_data_of_orgs_and_repos( repo_full_names=options['repo'], ) elif options['org']: data_of_orgs_and_repos = github.get_data_of_orgs_and_repos( org_names=options['org'], ) else: raise management.base.CommandError( "Provide a list of organizations or repositories", ) session = requests.Session() for org_data in data_of_orgs_and_repos.values(): org, _ = misc.get_or_create_record( Organization, org_data['details'], ) logger.info(org) ignored_repos = [ repo.name for repo in Repository.objects.filter(is_tracked=False, ) ] repos_to_process = [ repo for repo in org_data['repos'] if repo['name'] not in ignored_repos ] number_of_repos = len(repos_to_process) for i, repo_data in enumerate(repos_to_process, start=1): # noqa: WPS111,E501 repo, _ = misc.get_or_create_record(org, repo_data) logger.info(f"{repo} ({i}/{number_of_repos})") # noqa: G004 if repo_data['size'] == 0: logger.info("Empty repository") continue logger.info("Processing pull requests") prs = github.get_repo_prs(org, repo, session) total_prs_per_user = github.get_total_prs_per_user(prs) logger.info("Processing issues") issues_and_prs = github.get_repo_issues(org, repo, session) issues = [ issue for issue in issues_and_prs if 'pull_request' not in issue ] total_issues_per_user = github.get_total_issues_per_user( issues, ) logger.info("Processing comments") comments = github.get_repo_comments(org, repo, session) review_comments = github.get_repo_review_comments( org, repo, session, ) total_comments_per_user = misc.merge_dicts( github.get_total_comments_per_user(comments), github.get_total_comments_per_user(review_comments), ) ignored_contributors = [ contrib.login for contrib in Contributor.objects.filter( is_tracked=False, ) ] try: contributors = [ contrib for contrib in github.get_repo_contributors( org, repo, session, ) if contrib['login'] not in ignored_contributors ] except github.Accepted: logger.info("Data is not ready. Will retry") self.repos_to_rehandle.append(repo.full_name) continue logger.info("Processing commits") total_commits_per_user = { contributor['login']: contributor['total'] for contributor in contributors } logger.info("Processing commits stats") total_additions_per_user = github.get_total_additions_per_user( contributors, ) total_deletions_per_user = github.get_total_deletions_per_user( contributors, ) contributions_to_totals_mapping = { 'commits': total_commits_per_user, 'pull_requests': total_prs_per_user, 'issues': total_issues_per_user, 'comments': total_comments_per_user, 'additions': total_additions_per_user, 'deletions': total_deletions_per_user, } contributors_logins = [ contributor['login'] for contributor in contributors ] logger.info("Finishing insertions") for login in contributors_logins: contributor = get_or_create_contributor(login) Contribution.objects.update_or_create( repository=repo, contributor=contributor, defaults={ contrib_type: totals.get(login, 0) for contrib_type, totals in contributions_to_totals_mapping.items() }, ) session.close() if self.repos_to_rehandle: logger.info("Rehandling some repositories") management.call_command('fetchdata', repo=self.repos_to_rehandle) else: logger.info( self.style.SUCCESS( "Data fetched from GitHub and saved to the database", ))
def get_or_create_contributor(login): """Return a contributor object.""" user_data = github.get_user_data(login) contributor, _ = misc.get_or_create_record(Contributor, user_data) return contributor
def handle(self, *args, **options): # noqa: WPS110,WPS213 """Collect data from GitHub.""" logger.info("Data collection started") if options['repo']: data_of_orgs_and_repos = github.get_data_of_orgs_and_repos( repo_full_names=options['repo'], ) elif options['org']: data_of_orgs_and_repos = github.get_data_of_orgs_and_repos( org_names=options['org'], ) else: raise management.base.CommandError( "Provide a list of organizations or repositories", ) for org_data in data_of_orgs_and_repos.values(): org, _ = misc.get_or_create_record( Organization, org_data['details'], ) logger.info(org) repos_to_process = [ repo for repo in org_data['repos'] if repo['name'] not in IGNORED_REPOSITORIES ] number_of_repos = len(repos_to_process) for i, repo_data in enumerate(repos_to_process, start=1): # noqa: WPS111,E501 repo, _ = misc.get_or_create_record(org, repo_data) logger.info(f"{repo} ({i}/{number_of_repos})") # noqa: G004 if repo_data['size'] == 0: logger.info("Empty repository") continue logger.info("Processing issues and pull requests") create_contributions( repo, github.get_repo_issues(org, repo, session), user_field='user', id_field='id', ) logger.info("Processing commits") create_contributions( repo, github.get_repo_commits_except_merges( org, repo, session=session, ), user_field='author', id_field='sha', type_='cit', ) logger.info("Processing comments") create_contributions( repo, github.get_all_types_of_comments(org, repo, session), user_field='user', id_field='id', type_='cnt', ) session.close() logger.info(self.style.SUCCESS( "Data fetched from GitHub and saved to the database", ))
def create_contributions( # noqa: C901,WPS231 repo, contrib_data, user_field=None, id_field=None, type_=None, ): """Create a contribution record.""" for contrib in contrib_data: contrib_author = contrib[user_field] if contrib_author is None or contrib_author['type'] == 'Bot': continue contrib_author_login = contrib_author['login'] if contrib_author_login in IGNORED_CONTRIBUTORS: continue if not type_: pr_or_iss = 'pr' if 'pull_request' in contrib else 'iss' if type_ == 'cit': datetime = contrib['commit']['author']['date'] else: datetime = contrib['created_at'] contribution, created = Contribution.objects.get_or_create( id=contrib[id_field], defaults={ 'repository': repo, 'contributor': misc.get_or_create_record( Contributor, misc.get_contributor_data( contrib_author_login, session, ), )[0], 'type': type_ or pr_or_iss, 'html_url': contrib['html_url'], 'created_at': dateparse.parse_datetime(datetime), }, ) if created and type_ == 'cit': commit_data = github.get_commit_data( repo.organization, repo, contribution.id, session, ) commit_stats = commit_data['stats'] CommitStats.objects.create( commit=contribution, additions=commit_stats['additions'], deletions=commit_stats['deletions'], ) with suppress(NameError): if pr_or_iss: IssueInfo.objects.update_or_create( issue=contribution, defaults={ 'title': contrib['title'], 'is_open': contrib['state'] == 'open', }, )