def _get_last_years_commits(self, repo_url): try: activity = get_response(urljoin(repo_url + '/', "stats/commit_activity"), self._headers) except TaskError as e: self.log.debug(e) return [] return [x['total'] for x in activity]
def _rate_limit_exceeded(cls, headers): """Return True if Github API rate limit has been exceeded.""" # avoid cyclic import from f8a_worker.utils import get_response response = get_response(urljoin(cls.GITHUB_API, "rate_limit"), headers=headers) remaining_attempts = response.get('rate', {}).get('remaining', 0) return remaining_attempts == 0
def execute(self, arguments): """Task code. :param arguments: dictionary with task arguments :return: {}, results """ self._strict_assert(arguments.get('ecosystem')) self._strict_assert(arguments.get('name')) rdb_session = StoragePool.get_connected_storage( 'BayesianPostgres').session name = arguments['name'] ecosystem = arguments['ecosystem'] if ecosystem == 'go': name = quote(name, safe='') project_url = self.configuration.libraries_io_project_url( Ecosystem.by_name(rdb_session, ecosystem), name) project = get_response(project_url) versions = project['versions'] details = { 'dependent_repositories': { 'count': project['dependent_repos_count'] }, 'dependents': { 'count': project['dependents_count'] }, 'releases': { 'count': len(versions), 'recent': self.recent_releases(versions) } } return {'status': 'success', 'summary': [], 'details': details}
def _get_last_years_commits(self, repo_url): """Get weekly commit activity for last year.""" try: activity = get_response(urljoin(repo_url + '/', "stats/commit_activity"), self._headers) except NotABugTaskError as e: logger.debug(e) return [] return [x['total'] for x in activity]
def _get_last_years_commits(self, repo_url): """Get weekly commit activity for last year.""" try: activity = get_response( urljoin(repo_url + '/', "stats/commit_activity")) except NotABugTaskError as e: self.log.debug(e) return [] return [x.get('total', 0) for x in activity]
def execute(self, arguments): """Task code. :param arguments: dictionary with task arguments :return: {}, results """ result_data = {'status': 'unknown', 'summary': [], 'details': {}} # For testing purposes, a repo may be specified at task creation time if self._repo_name is None: # Otherwise, get the repo name from earlier Mercator scan results self._repo_name = self._get_repo_name(arguments['url']) if self._repo_name is None: # Not a GitHub hosted project return result_data repo_url = urljoin(self.configuration.GITHUB_API + "repos/", self._repo_name) try: repo = get_response(repo_url) if not repo: raise NotABugFatalTaskError( 'Page not found on {}'.format(repo_url)) except NotABugTaskError as e: self.log.error(e) raise NotABugFatalTaskError from e result_data['status'] = 'success' issues = {} # Get Repo Statistics notoriety = self._get_repo_stats(repo) if notoriety: issues.update(notoriety) issues['topics'] = repo.get('topics', []) issues['license'] = repo.get('license') or {} # Get Commit Statistics last_year_commits = self._get_last_years_commits(repo['url']) commits = { 'last_year_commits': { 'sum': sum(last_year_commits), 'weekly': last_year_commits } } t_stamp = datetime.datetime.utcnow() refreshed_on = {'updated_on': t_stamp.strftime("%Y-%m-%d %H:%M:%S")} issues.update(refreshed_on) issues.update(commits) # Get PR/Issue details for previous Month and Year gh_pr_issue_details = get_gh_pr_issue_counts(repo['full_name']) issues.update(gh_pr_issue_details) result_data['details'] = issues return result_data
def _get_repo_stats(self, repo): try: contributors = get_response(repo['contributors_url'], self._headers) except TaskError as e: self.log.debug(e) contributors = {} d = {'contributors_count': len(list(contributors)) if contributors is not None else 'N/A'} for prop in REPO_PROPS: d[prop] = repo.get(prop, -1) return d
def execute(self, arguments): """Task code. :param arguments: dictionary with task arguments :return: {}, results """ result_data = {'status': 'unknown', 'summary': [], 'details': {}} # For testing purposes, a repo may be specified at task creation time if self._repo_name is None: # Otherwise, get the repo name from earlier Mercator scan results self._repo_name = self._get_repo_name(arguments['url']) if self._repo_name is None: # Not a GitHub hosted project return result_data try: _, header = self.configuration.select_random_github_token() self._headers.update(header) except F8AConfigurationException as e: self.log.error(e) raise FatalTaskError from e repo_url = urljoin(self.configuration.GITHUB_API + "repos/", self._repo_name) try: repo = get_response(repo_url, self._headers) except TaskError as e: self.log.error(e) raise FatalTaskError from e result_data['status'] = 'success' issues = {} # Get Repo Statistics notoriety = self._get_repo_stats(repo) if notoriety: issues.update(notoriety) issues['topics'] = repo.get('topics', []) issues['license'] = repo.get('license') or {} # Get Commit Statistics last_year_commits = self._get_last_years_commits(repo['url']) commits = { 'last_year_commits': { 'sum': sum(last_year_commits), 'weekly': last_year_commits } } issues.update(commits) result_data['details'] = issues return result_data
def _get_repo_stats(self, repo): """Collect various repository properties.""" try: if repo.get('contributors_url', ''): contributors = get_response(repo.get('contributors_url', ''), self._headers) else: contributors = {} except NotABugTaskError as e: self.log.debug(e) contributors = {} d = { 'contributors_count': len(list(contributors)) if contributors is not None else 'N/A' } for prop in REPO_PROPS: d[prop] = repo.get(prop, -1) return d
def execute(self, arguments): """Task entrypoint.""" self._strict_assert(arguments.get('ecosystem')) self._strict_assert(arguments.get('name')) name = arguments['name'] ecosystem = arguments['ecosystem'] if ecosystem == 'go': name = quote(name, safe='') project_url = self.configuration.libraries_io_project_url(ecosystem, name) project = get_response(project_url) versions = project['versions'] details = {'dependent_repositories': {'count': project['dependent_repos_count']}, 'dependents': {'count': project['dependents_count']}, 'releases': {'count': len(versions), 'recent': self.recent_releases(versions) } } return {'status': 'success', 'summary': [], 'details': details}
def execute(self, arguments): """Task code. :param arguments: dictionary with task arguments :return: {}, results """ result_data = {'status': 'unknown', 'summary': [], 'details': {}} if arguments['ecosystem'] == 'golang': go_obj = GolangUtils(arguments.get('name')) url = go_obj.get_gh_link() if url: arguments['url'] = url else: return result_data # For testing purposes, a repo may be specified at task creation time if self._repo_name is None: # Otherwise, get the repo name from URL self._repo_name = self._get_repo_name(arguments['url']) if self._repo_name is None: # Not a GitHub hosted project return result_data repo_url = urljoin(self.configuration.GITHUB_API + "repos/", self._repo_name) repo = {} try: repo = get_response(repo_url) if not repo: raise NotABugFatalTaskError( 'Page not found on {}'.format(repo_url)) except NotABugTaskError as e: logger.error(e) result_data['status'] = 'success' issues = {} # Get Repo Statistics notoriety = self._get_repo_stats(repo) if notoriety: issues.update(notoriety) issues['topics'] = repo.get('topics', []) issues['license'] = repo.get('license') or {} # Get Commit Statistics last_year_commits = self._get_last_years_commits(repo.get('url', '')) commits = { 'last_year_commits': { 'sum': sum(last_year_commits), 'weekly': last_year_commits } } t_stamp = datetime.datetime.utcnow() refreshed_on = {'updated_on': t_stamp.strftime("%Y-%m-%d %H:%M:%S")} issues.update(refreshed_on) issues.update(commits) # Get PR/Issue details for previous Month and Year gh_pr_issue_details = get_gh_pr_issue_counts(self._repo_name) issues.update(gh_pr_issue_details) result_data['details'] = issues # Store github details for being used in Data-Importer store_data_to_s3(arguments, StoragePool.get_connected_storage('S3GitHub'), result_data) return result_data