def show_repos(request):
"""Get repositories for organizations."""
context = set_up_context(request)
if request.method == 'POST':
form_orgs = OrgNamesForm(request.POST)
if form_orgs.is_valid():
session = requests.Session()
repos_choices = []
for org_name in form_orgs.cleaned_data['organizations'].split():
org_data = github.get_org_data(org_name, session)
org, _ = misc.get_or_create_record(Organization, org_data)
repos_data = [
repo for repo in github.get_org_repos(org_name, session)
]
for repo_data in repos_data:
misc.get_or_create_record(
org,
repo_data,
{
'is_tracked': False,
'is_visible': False
},
)
repos_choices.append(
(repo_data['id'], repo_data['name']), )
session.close()
form_repos = RepoNamesForm(choices=repos_choices)
context['form_repos'] = form_repos
else:
form_orgs = OrgNamesForm()
context['form_orgs'] = form_orgs
return TemplateResponse(request, 'admin/configuration.html', context)
def get_or_create_contributor(login):
"""Return a contributor object."""
try:
return Contributor.objects.get(login=login)
except Contributor.DoesNotExist:
user_data = github.get_user_data(login, session)
contributor, _ = misc.get_or_create_record(Contributor, user_data)
return contributor
def handle_user_post_save(sender, **kwargs):
"""Tie a user with a contributor profile."""
site_user = kwargs['instance']
if site_user.has_usable_password():
# The user has registered with the default backend
return
if kwargs['created']:
contributor, _ = misc.get_or_create_record(
apps.get_model('contributors.Contributor'),
misc.get_contributor_data(site_user.username),
)
contributor.user = site_user
contributor.save()
def handle(self, *args, **options): # noqa: WPS
"""Collect data from GitHub."""
logger.info("Data collection started")
if options['repo']:
data_of_orgs_and_repos = github.get_data_of_orgs_and_repos(
repo_full_names=options['repo'], )
elif options['org']:
data_of_orgs_and_repos = github.get_data_of_orgs_and_repos(
org_names=options['org'], )
else:
raise management.base.CommandError(
"Provide a list of organizations or repositories", )
session = requests.Session()
for org_data in data_of_orgs_and_repos.values():
org, _ = misc.get_or_create_record(
Organization,
org_data['details'],
)
logger.info(org)
ignored_repos = [
repo.name
for repo in Repository.objects.filter(is_tracked=False, )
]
repos_to_process = [
repo for repo in org_data['repos']
if repo['name'] not in ignored_repos
]
number_of_repos = len(repos_to_process)
for i, repo_data in enumerate(repos_to_process,
start=1): # noqa: WPS111,E501
repo, _ = misc.get_or_create_record(org, repo_data)
logger.info(f"{repo} ({i}/{number_of_repos})") # noqa: G004
if repo_data['size'] == 0:
logger.info("Empty repository")
continue
logger.info("Processing pull requests")
prs = github.get_repo_prs(org, repo, session)
total_prs_per_user = github.get_total_prs_per_user(prs)
logger.info("Processing issues")
issues_and_prs = github.get_repo_issues(org, repo, session)
issues = [
issue for issue in issues_and_prs
if 'pull_request' not in issue
]
total_issues_per_user = github.get_total_issues_per_user(
issues, )
logger.info("Processing comments")
comments = github.get_repo_comments(org, repo, session)
review_comments = github.get_repo_review_comments(
org,
repo,
session,
)
total_comments_per_user = misc.merge_dicts(
github.get_total_comments_per_user(comments),
github.get_total_comments_per_user(review_comments),
)
ignored_contributors = [
contrib.login for contrib in Contributor.objects.filter(
is_tracked=False, )
]
try:
contributors = [
contrib for contrib in github.get_repo_contributors(
org,
repo,
session,
) if contrib['login'] not in ignored_contributors
]
except github.Accepted:
logger.info("Data is not ready. Will retry")
self.repos_to_rehandle.append(repo.full_name)
continue
logger.info("Processing commits")
total_commits_per_user = {
contributor['login']: contributor['total']
for contributor in contributors
}
logger.info("Processing commits stats")
total_additions_per_user = github.get_total_additions_per_user(
contributors, )
total_deletions_per_user = github.get_total_deletions_per_user(
contributors, )
contributions_to_totals_mapping = {
'commits': total_commits_per_user,
'pull_requests': total_prs_per_user,
'issues': total_issues_per_user,
'comments': total_comments_per_user,
'additions': total_additions_per_user,
'deletions': total_deletions_per_user,
}
contributors_logins = [
contributor['login'] for contributor in contributors
]
logger.info("Finishing insertions")
for login in contributors_logins:
contributor = get_or_create_contributor(login)
Contribution.objects.update_or_create(
repository=repo,
contributor=contributor,
defaults={
contrib_type: totals.get(login, 0)
for contrib_type, totals in
contributions_to_totals_mapping.items()
},
)
session.close()
if self.repos_to_rehandle:
logger.info("Rehandling some repositories")
management.call_command('fetchdata', repo=self.repos_to_rehandle)
else:
logger.info(
self.style.SUCCESS(
"Data fetched from GitHub and saved to the database", ))
def get_or_create_contributor(login):
"""Return a contributor object."""
user_data = github.get_user_data(login)
contributor, _ = misc.get_or_create_record(Contributor, user_data)
return contributor
def handle(self, *args, **options): # noqa: WPS110,WPS213
"""Collect data from GitHub."""
logger.info("Data collection started")
if options['repo']:
data_of_orgs_and_repos = github.get_data_of_orgs_and_repos(
repo_full_names=options['repo'],
)
elif options['org']:
data_of_orgs_and_repos = github.get_data_of_orgs_and_repos(
org_names=options['org'],
)
else:
raise management.base.CommandError(
"Provide a list of organizations or repositories",
)
for org_data in data_of_orgs_and_repos.values():
org, _ = misc.get_or_create_record(
Organization, org_data['details'],
)
logger.info(org)
repos_to_process = [
repo for repo in org_data['repos']
if repo['name'] not in IGNORED_REPOSITORIES
]
number_of_repos = len(repos_to_process)
for i, repo_data in enumerate(repos_to_process, start=1): # noqa: WPS111,E501
repo, _ = misc.get_or_create_record(org, repo_data)
logger.info(f"{repo} ({i}/{number_of_repos})") # noqa: G004
if repo_data['size'] == 0:
logger.info("Empty repository")
continue
logger.info("Processing issues and pull requests")
create_contributions(
repo,
github.get_repo_issues(org, repo, session),
user_field='user',
id_field='id',
)
logger.info("Processing commits")
create_contributions(
repo,
github.get_repo_commits_except_merges(
org, repo, session=session,
),
user_field='author',
id_field='sha',
type_='cit',
)
logger.info("Processing comments")
create_contributions(
repo,
github.get_all_types_of_comments(org, repo, session),
user_field='user',
id_field='id',
type_='cnt',
)
session.close()
logger.info(self.style.SUCCESS(
"Data fetched from GitHub and saved to the database",
))
def create_contributions( # noqa: C901,WPS231
repo,
contrib_data,
user_field=None,
id_field=None,
type_=None,
):
"""Create a contribution record."""
for contrib in contrib_data:
contrib_author = contrib[user_field]
if contrib_author is None or contrib_author['type'] == 'Bot':
continue
contrib_author_login = contrib_author['login']
if contrib_author_login in IGNORED_CONTRIBUTORS:
continue
if not type_:
pr_or_iss = 'pr' if 'pull_request' in contrib else 'iss'
if type_ == 'cit':
datetime = contrib['commit']['author']['date']
else:
datetime = contrib['created_at']
contribution, created = Contribution.objects.get_or_create(
id=contrib[id_field],
defaults={
'repository':
repo,
'contributor':
misc.get_or_create_record(
Contributor,
misc.get_contributor_data(
contrib_author_login,
session,
),
)[0],
'type':
type_ or pr_or_iss,
'html_url':
contrib['html_url'],
'created_at':
dateparse.parse_datetime(datetime),
},
)
if created and type_ == 'cit':
commit_data = github.get_commit_data(
repo.organization,
repo,
contribution.id,
session,
)
commit_stats = commit_data['stats']
CommitStats.objects.create(
commit=contribution,
additions=commit_stats['additions'],
deletions=commit_stats['deletions'],
)
with suppress(NameError):
if pr_or_iss:
IssueInfo.objects.update_or_create(
issue=contribution,
defaults={
'title': contrib['title'],
'is_open': contrib['state'] == 'open',
},
)
