def _get_last_years_commits(self, repo_url):
try:
activity = get_response(urljoin(repo_url + '/', "stats/commit_activity"), self._headers)
except TaskError as e:
self.log.debug(e)
return []
return [x['total'] for x in activity]
def _rate_limit_exceeded(cls, headers):
"""Return True if Github API rate limit has been exceeded."""
# avoid cyclic import
from f8a_worker.utils import get_response
response = get_response(urljoin(cls.GITHUB_API, "rate_limit"), headers=headers)
remaining_attempts = response.get('rate', {}).get('remaining', 0)
return remaining_attempts == 0
def execute(self, arguments):
"""Task code.
:param arguments: dictionary with task arguments
:return: {}, results
"""
self._strict_assert(arguments.get('ecosystem'))
self._strict_assert(arguments.get('name'))
rdb_session = StoragePool.get_connected_storage(
'BayesianPostgres').session
name = arguments['name']
ecosystem = arguments['ecosystem']
if ecosystem == 'go':
name = quote(name, safe='')
project_url = self.configuration.libraries_io_project_url(
Ecosystem.by_name(rdb_session, ecosystem), name)
project = get_response(project_url)
versions = project['versions']
details = {
'dependent_repositories': {
'count': project['dependent_repos_count']
},
'dependents': {
'count': project['dependents_count']
},
'releases': {
'count': len(versions),
'recent': self.recent_releases(versions)
}
}
return {'status': 'success', 'summary': [], 'details': details}
def _get_last_years_commits(self, repo_url):
"""Get weekly commit activity for last year."""
try:
activity = get_response(urljoin(repo_url + '/', "stats/commit_activity"), self._headers)
except NotABugTaskError as e:
logger.debug(e)
return []
return [x['total'] for x in activity]
def _get_last_years_commits(self, repo_url):
"""Get weekly commit activity for last year."""
try:
activity = get_response(
urljoin(repo_url + '/', "stats/commit_activity"))
except NotABugTaskError as e:
self.log.debug(e)
return []
return [x.get('total', 0) for x in activity]
def execute(self, arguments):
"""Task code.
:param arguments: dictionary with task arguments
:return: {}, results
"""
result_data = {'status': 'unknown', 'summary': [], 'details': {}}
# For testing purposes, a repo may be specified at task creation time
if self._repo_name is None:
# Otherwise, get the repo name from earlier Mercator scan results
self._repo_name = self._get_repo_name(arguments['url'])
if self._repo_name is None:
# Not a GitHub hosted project
return result_data
repo_url = urljoin(self.configuration.GITHUB_API + "repos/",
self._repo_name)
try:
repo = get_response(repo_url)
if not repo:
raise NotABugFatalTaskError(
'Page not found on {}'.format(repo_url))
except NotABugTaskError as e:
self.log.error(e)
raise NotABugFatalTaskError from e
result_data['status'] = 'success'
issues = {}
# Get Repo Statistics
notoriety = self._get_repo_stats(repo)
if notoriety:
issues.update(notoriety)
issues['topics'] = repo.get('topics', [])
issues['license'] = repo.get('license') or {}
# Get Commit Statistics
last_year_commits = self._get_last_years_commits(repo['url'])
commits = {
'last_year_commits': {
'sum': sum(last_year_commits),
'weekly': last_year_commits
}
}
t_stamp = datetime.datetime.utcnow()
refreshed_on = {'updated_on': t_stamp.strftime("%Y-%m-%d %H:%M:%S")}
issues.update(refreshed_on)
issues.update(commits)
# Get PR/Issue details for previous Month and Year
gh_pr_issue_details = get_gh_pr_issue_counts(repo['full_name'])
issues.update(gh_pr_issue_details)
result_data['details'] = issues
return result_data
def _get_repo_stats(self, repo):
try:
contributors = get_response(repo['contributors_url'], self._headers)
except TaskError as e:
self.log.debug(e)
contributors = {}
d = {'contributors_count': len(list(contributors)) if contributors is not None else 'N/A'}
for prop in REPO_PROPS:
d[prop] = repo.get(prop, -1)
return d
def execute(self, arguments):
"""Task code.
:param arguments: dictionary with task arguments
:return: {}, results
"""
result_data = {'status': 'unknown', 'summary': [], 'details': {}}
# For testing purposes, a repo may be specified at task creation time
if self._repo_name is None:
# Otherwise, get the repo name from earlier Mercator scan results
self._repo_name = self._get_repo_name(arguments['url'])
if self._repo_name is None:
# Not a GitHub hosted project
return result_data
try:
_, header = self.configuration.select_random_github_token()
self._headers.update(header)
except F8AConfigurationException as e:
self.log.error(e)
raise FatalTaskError from e
repo_url = urljoin(self.configuration.GITHUB_API + "repos/",
self._repo_name)
try:
repo = get_response(repo_url, self._headers)
except TaskError as e:
self.log.error(e)
raise FatalTaskError from e
result_data['status'] = 'success'
issues = {}
# Get Repo Statistics
notoriety = self._get_repo_stats(repo)
if notoriety:
issues.update(notoriety)
issues['topics'] = repo.get('topics', [])
issues['license'] = repo.get('license') or {}
# Get Commit Statistics
last_year_commits = self._get_last_years_commits(repo['url'])
commits = {
'last_year_commits': {
'sum': sum(last_year_commits),
'weekly': last_year_commits
}
}
issues.update(commits)
result_data['details'] = issues
return result_data
def _get_repo_stats(self, repo):
"""Collect various repository properties."""
try:
if repo.get('contributors_url', ''):
contributors = get_response(repo.get('contributors_url', ''),
self._headers)
else:
contributors = {}
except NotABugTaskError as e:
self.log.debug(e)
contributors = {}
d = {
'contributors_count':
len(list(contributors)) if contributors is not None else 'N/A'
}
for prop in REPO_PROPS:
d[prop] = repo.get(prop, -1)
return d
def execute(self, arguments):
"""Task entrypoint."""
self._strict_assert(arguments.get('ecosystem'))
self._strict_assert(arguments.get('name'))
name = arguments['name']
ecosystem = arguments['ecosystem']
if ecosystem == 'go':
name = quote(name, safe='')
project_url = self.configuration.libraries_io_project_url(ecosystem, name)
project = get_response(project_url)
versions = project['versions']
details = {'dependent_repositories': {'count': project['dependent_repos_count']},
'dependents': {'count': project['dependents_count']},
'releases': {'count': len(versions),
'recent': self.recent_releases(versions)
}
}
return {'status': 'success',
'summary': [],
'details': details}
def execute(self, arguments):
"""Task code.
:param arguments: dictionary with task arguments
:return: {}, results
"""
result_data = {'status': 'unknown', 'summary': [], 'details': {}}
if arguments['ecosystem'] == 'golang':
go_obj = GolangUtils(arguments.get('name'))
url = go_obj.get_gh_link()
if url:
arguments['url'] = url
else:
return result_data
# For testing purposes, a repo may be specified at task creation time
if self._repo_name is None:
# Otherwise, get the repo name from URL
self._repo_name = self._get_repo_name(arguments['url'])
if self._repo_name is None:
# Not a GitHub hosted project
return result_data
repo_url = urljoin(self.configuration.GITHUB_API + "repos/",
self._repo_name)
repo = {}
try:
repo = get_response(repo_url)
if not repo:
raise NotABugFatalTaskError(
'Page not found on {}'.format(repo_url))
except NotABugTaskError as e:
logger.error(e)
result_data['status'] = 'success'
issues = {}
# Get Repo Statistics
notoriety = self._get_repo_stats(repo)
if notoriety:
issues.update(notoriety)
issues['topics'] = repo.get('topics', [])
issues['license'] = repo.get('license') or {}
# Get Commit Statistics
last_year_commits = self._get_last_years_commits(repo.get('url', ''))
commits = {
'last_year_commits': {
'sum': sum(last_year_commits),
'weekly': last_year_commits
}
}
t_stamp = datetime.datetime.utcnow()
refreshed_on = {'updated_on': t_stamp.strftime("%Y-%m-%d %H:%M:%S")}
issues.update(refreshed_on)
issues.update(commits)
# Get PR/Issue details for previous Month and Year
gh_pr_issue_details = get_gh_pr_issue_counts(self._repo_name)
issues.update(gh_pr_issue_details)
result_data['details'] = issues
# Store github details for being used in Data-Importer
store_data_to_s3(arguments,
StoragePool.get_connected_storage('S3GitHub'),
result_data)
return result_data
