def get_config_params(self): """Get configuration parameters to be rendered into the conf file.""" # TODO this should be handled better in the theme conf_py_path = os.path.join(os.path.sep, self.version.get_conf_py_path(), '') remote_version = self.version.commit_name github_user, github_repo = version_utils.get_github_username_repo( url=self.project.repo) github_version_is_editable = (self.version.type == 'branch') display_github = github_user is not None bitbucket_user, bitbucket_repo = version_utils.get_bitbucket_username_repo( url=self.project.repo) bitbucket_version_is_editable = (self.version.type == 'branch') display_bitbucket = bitbucket_user is not None # Avoid hitting database and API if using Docker build environment if getattr(settings, 'DONT_HIT_API', False): versions = self.project.active_versions() downloads = self.version.get_downloads(pretty=True) else: versions = self.project.api_versions() downloads = api.version(self.version.pk).get()['downloads'] data = { 'current_version': self.version.verbose_name, 'project': self.project, 'settings': settings, 'static_path': SPHINX_STATIC_DIR, 'template_path': SPHINX_TEMPLATE_DIR, 'conf_py_path': conf_py_path, 'api_host': getattr(settings, 'PUBLIC_API_URL', 'https://readthedocs.org'), 'commit': self.project.vcs_repo(self.version.slug).commit, 'versions': versions, 'downloads': downloads, # GitHub 'github_user': github_user, 'github_repo': github_repo, 'github_version': remote_version, 'github_version_is_editable': github_version_is_editable, 'display_github': display_github, # BitBucket 'bitbucket_user': bitbucket_user, 'bitbucket_repo': bitbucket_repo, 'bitbucket_version': remote_version, 'bitbucket_version_is_editable': bitbucket_version_is_editable, 'display_bitbucket': display_bitbucket, } finalize_sphinx_context_data.send( sender=self.__class__, build_env=self.build_env, data=data, ) return data
def setup_webhook(self, project): """Set up GitHub project webhook for project :param project: project to set up webhook for :type project: Project :returns: boolean based on webhook set up success :rtype: bool """ session = self.get_session() owner, repo = build_utils.get_github_username_repo(url=project.repo) data = json.dumps({ 'name': 'readthedocs', 'active': True, 'config': {'url': 'https://{domain}/github'.format(domain=settings.PRODUCTION_DOMAIN)} }) resp = None try: resp = session.post( ('https://api.github.com/repos/{owner}/{repo}/hooks' .format(owner=owner, repo=repo)), data=data, headers={'content-type': 'application/json'} ) if resp.status_code == 201: log.info('GitHub webhook creation successful for project: %s', project) return True except RequestException: log.error('GitHub webhook creation failed for project: %s', project, exc_info=True) pass else: log.error('GitHub webhook creation failed for project: %s', project) return False
def setup_webhook(self, project): """ Set up GitHub project webhook for project. :param project: project to set up webhook for :type project: Project :returns: boolean based on webhook set up success, and requests Response object :rtype: (Bool, Response) """ session = self.get_session() owner, repo = build_utils.get_github_username_repo(url=project.repo) integration, _ = Integration.objects.get_or_create( project=project, integration_type=Integration.GITHUB_WEBHOOK, ) data = self.get_webhook_data(project, integration) resp = None try: resp = session.post( ('https://api.github.com/repos/{owner}/{repo}/hooks' .format(owner=owner, repo=repo)), data=data, headers={'content-type': 'application/json'} ) # GitHub will return 200 if already synced if resp.status_code in [200, 201]: recv_data = resp.json() integration.provider_data = recv_data integration.save() log.info('GitHub webhook creation successful for project: %s', project) return (True, resp) # Catch exceptions with request or deserializing JSON except (RequestException, ValueError): log.exception( 'GitHub webhook creation failed for project: %s', project, ) else: log.error( 'GitHub webhook creation failed for project: %s', project, ) # Response data should always be JSON, still try to log if not # though try: debug_data = resp.json() except ValueError: debug_data = resp.content log.debug( 'GitHub webhook creation failure response: %s', debug_data, ) return (False, resp)
def add_github_webhook(session, project): owner, repo = build_utils.get_github_username_repo(url=project.repo) data = json.dumps({ 'name': 'readthedocs', 'active': True, 'config': {'url': 'https://{domain}/github'.format(domain=settings.PRODUCTION_DOMAIN)} }) resp = session.post( 'https://api.github.com/repos/{owner}/{repo}/hooks'.format(owner=owner, repo=repo), data=data, headers={'content-type': 'application/json'} ) log.info("Creating GitHub webhook response code: {code}".format(code=resp.status_code)) return resp
def get_github_url( self, docroot, filename, source_suffix='.rst', action='view', ): """ Return a GitHub URL for a given filename. :param docroot: Location of documentation in repository :param filename: Name of file :param source_suffix: File suffix of documentation format :param action: `view` (default) or `edit` """ repo_url = self.project.repo if 'github' not in repo_url: return '' if not docroot: return '' # Normalize /docroot/ docroot = '/' + docroot.strip('/') + '/' if action == 'view': action_string = 'blob' elif action == 'edit': action_string = 'edit' user, repo = get_github_username_repo(repo_url) if not user and not repo: return '' repo = repo.rstrip('/') if not filename: # If there isn't a filename, we don't need a suffix source_suffix = '' return GITHUB_URL.format( user=user, repo=repo, version=self.commit_name, docroot=docroot, path=filename, source_suffix=source_suffix, action=action_string, )
def setup_webhook(self, project): """Set up GitHub project webhook for project :param project: project to set up webhook for :type project: Project :returns: boolean based on webhook set up success, and requests Response object :rtype: (Bool, Response) """ session = self.get_session() owner, repo = build_utils.get_github_username_repo(url=project.repo) data = json.dumps({ 'name': 'web', 'active': True, 'config': { 'url': 'https://{domain}{path}'.format( domain=settings.PRODUCTION_DOMAIN, path=reverse( 'api_webhook_github', kwargs={'project_slug': project.slug} ) ), 'content_type': 'json', }, 'events': ['push', 'pull_request'], }) resp = None try: resp = session.post( ('https://api.github.com/repos/{owner}/{repo}/hooks' .format(owner=owner, repo=repo)), data=data, headers={'content-type': 'application/json'} ) # GitHub will return 200 if already synced if resp.status_code in [200, 201]: log.info('GitHub webhook creation successful for project: %s', project) return (True, resp) except RequestException: log.error('GitHub webhook creation failed for project: %s', project, exc_info=True) pass else: log.error('GitHub webhook creation failed for project: %s', project) log.debug('GitHub webhook creation failure response: %s', dict(resp)) return (False, resp)
def add_github_webhook(session, project): owner, repo = build_utils.get_github_username_repo(url=project.repo) data = json.dumps( { "name": "readthedocs", "active": True, "config": {"url": "https://{domain}/github".format(domain=settings.PRODUCTION_DOMAIN)}, } ) resp = session.post( "https://api.github.com/repos/{owner}/{repo}/hooks".format(owner=owner, repo=repo), data=data, headers={"content-type": "application/json"}, ) log.info("Creating GitHub webhook response code: {code}".format(code=resp.status_code)) return resp
def setup_webhook(self, project): """ Set up GitHub project webhook for project. :param project: project to set up webhook for :type project: Project :returns: boolean based on webhook set up success, and requests Response object :rtype: (Bool, Response) """ session = self.get_session() owner, repo = build_utils.get_github_username_repo(url=project.repo) integration, _ = Integration.objects.get_or_create( project=project, integration_type=Integration.GITHUB_WEBHOOK, ) data = self.get_webhook_data(project, integration) resp = None try: resp = session.post( ('https://api.github.com/repos/{owner}/{repo}/hooks'.format( owner=owner, repo=repo)), data=data, headers={'content-type': 'application/json'}) # GitHub will return 200 if already synced if resp.status_code in [200, 201]: recv_data = resp.json() integration.provider_data = recv_data integration.save() log.info('GitHub webhook creation successful for project: %s', project) return (True, resp) # Catch exceptions with request or deserializing JSON except (RequestException, ValueError): log.error('GitHub webhook creation failed for project: %s', project, exc_info=True) else: log.error('GitHub webhook creation failed for project: %s', project) # Response data should always be JSON, still try to log if not though try: debug_data = resp.json() except ValueError: debug_data = resp.content log.debug('GitHub webhook creation failure response: %s', debug_data) return (False, resp)
def setup_webhook(self, project): """Set up GitHub project webhook for project :param project: project to set up webhook for :type project: Project :returns: boolean based on webhook set up success, and requests Response object :rtype: (Bool, Response) """ session = self.get_session() owner, repo = build_utils.get_github_username_repo(url=project.repo) data = json.dumps({ 'name': 'web', 'active': True, 'config': { 'url': 'https://{domain}{path}'.format( domain=settings.PRODUCTION_DOMAIN, path=reverse('api_webhook_github', kwargs={'project_slug': project.slug})), 'content_type': 'json', }, 'events': ['push', 'pull_request'], }) resp = None try: resp = session.post( ('https://api.github.com/repos/{owner}/{repo}/hooks'.format( owner=owner, repo=repo)), data=data, headers={'content-type': 'application/json'}) # GitHub will return 200 if already synced if resp.status_code in [200, 201]: log.info('GitHub webhook creation successful for project: %s', project) return (True, resp) except RequestException: log.error('GitHub webhook creation failed for project: %s', project, exc_info=True) pass else: log.error('GitHub webhook creation failed for project: %s', project) log.debug('GitHub webhook creation failure response: %s', resp.content) return (False, resp)
def handle_project_import(sender, **kwargs): """ Add post-commit hook on project import. """ project = sender request = kwargs.get('request') for provider in ['github', 'bitbucket']: if provider in project.repo: session = oauth_utils.get_oauth_session(user=request.user, provider=provider) if not session: break if provider == 'github': try: owner, repo = build_utils.get_github_username_repo(url=project.repo) data = json.dumps({ 'name': 'readthedocs', 'active': True, 'config': {'url': 'https://{domain}/github'.format(domain=settings.PRODUCTION_DOMAIN)} }) resp = session.post( 'https://api.github.com/repos/{owner}/{repo}/hooks'.format(owner=owner, repo=repo), data=data, headers={'content-type': 'application/json'} ) log.info("Creating GitHub webhook response code: {code}".format(code=resp.status_code)) if resp.status_code == 201: messages.success(request, _('GitHub webhook activated')) except: log.exception('GitHub Hook creation failed', exc_info=True) elif provider == 'bitbucket': try: owner, repo = build_utils.get_bitbucket_username_repo(url=project.repo) data = { 'type': 'POST', 'url': 'https://{domain}/bitbucket'.format(domain=settings.PRODUCTION_DOMAIN), } resp = session.post( 'https://api.bitbucket.org/1.0/repositories/{owner}/{repo}/services'.format(owner=owner, repo=repo), data=data, ) log.info("Creating BitBucket webhook response code: {code}".format(code=resp.status_code)) if resp.status_code == 200: messages.success(request, _('BitBucket webhook activated')) except: log.exception('BitBucket Hook creation failed', exc_info=True)
def handle(self, *args, **options): # pylint: disable=too-many-locals token = os.environ.get('GITHUB_AUTH_TOKEN') if not token: print('Invalid GitHub token, exiting') return for project in Project.objects.filter( programming_language__in=['none', '', 'words'] ).filter( repo__contains='github' ): repo_url = project.repo user, repo = get_github_username_repo(repo_url) if not user: print('No GitHub repo for %s' % repo_url) continue cache_key = '%s-%s' % (user, repo) top_lang = cache.get(cache_key, None) if not top_lang: url = '{github_api_url}/repos/{user}/{repo}/languages'.format( github_api_url=GitHubOAuth2Adapter.api_url, user=user, repo=repo, ) # We need this to get around GitHub's rate limiting headers = {'Authorization': 'token {token}'.format(token=token)} resp = requests.get(url, headers=headers) languages = resp.json() if not languages: continue sorted_langs = sorted(list(languages.items()), key=lambda x: x[1], reverse=True) print('Sorted langs: %s ' % sorted_langs) top_lang = sorted_langs[0][0] else: print('Cached top_lang: %s' % top_lang) if top_lang in PL_DICT: slug = PL_DICT[top_lang] print('Setting %s to %s' % (repo_url, slug)) Project.objects.filter(pk=project.pk).update(programming_language=slug) else: print('Language unknown: %s' % top_lang) cache.set(cache_key, top_lang, 60 * 600)
def add_github_webhook(session, project): owner, repo = build_utils.get_github_username_repo(url=project.repo) data = json.dumps({ 'name': 'readthedocs', 'active': True, 'config': { 'url': 'https://{domain}/github'.format(domain=settings.PRODUCTION_DOMAIN) } }) resp = session.post( 'https://api.github.com/repos/{owner}/{repo}/hooks'.format(owner=owner, repo=repo), data=data, headers={'content-type': 'application/json'}) log.info("Creating GitHub webhook response code: {code}".format( code=resp.status_code)) return resp
def vcs_url(self): """ Generate VCS (github, gitlab, bitbucket) URL for this version. Example: https://github.com/rtfd/readthedocs.org/tree/3.4.2/. External Version Example: https://github.com/rtfd/readthedocs.org/pull/99/. """ if self.type == EXTERNAL: if 'github' in self.project.repo: user, repo = get_github_username_repo(self.project.repo) return GITHUB_PULL_REQUEST_URL.format( user=user, repo=repo, number=self.verbose_name, ) if 'gitlab' in self.project.repo: user, repo = get_gitlab_username_repo(self.project.repo) return GITLAB_MERGE_REQUEST_URL.format( user=user, repo=repo, number=self.verbose_name, ) # TODO: Add VCS URL for BitBucket. return '' url = '' if self.slug == STABLE: slug_url = self.ref elif self.slug == LATEST: slug_url = self.project.default_branch or self.project.vcs_repo( ).fallback_branch else: slug_url = self.slug if ('github' in self.project.repo) or ('gitlab' in self.project.repo): url = f'/tree/{slug_url}/' if 'bitbucket' in self.project.repo: slug_url = self.identifier url = f'/src/{slug_url}' # TODO: improve this replacing return self.project.repo.replace('git://', 'https://').replace( '.git', '') + url
def setup_webhook(self, project): """Set up GitHub project webhook for project :param project: project to set up webhook for :type project: Project :returns: boolean based on webhook set up success :rtype: bool """ session = self.get_session() owner, repo = build_utils.get_github_username_repo(url=project.repo) data = json.dumps({ 'name': 'readthedocs', 'active': True, 'config': { 'url': 'https://{domain}/github'.format( domain=settings.PRODUCTION_DOMAIN) } }) resp = None try: resp = session.post( ('https://api.github.com/repos/{owner}/{repo}/hooks'.format( owner=owner, repo=repo)), data=data, headers={'content-type': 'application/json'}) if resp.status_code == 201: log.info('GitHub webhook creation successful for project: %s', project) return True except RequestException: log.error('GitHub webhook creation failed for project: %s', project, exc_info=True) pass else: log.error('GitHub webhook creation failed for project: %s', project) return False
def append_conf(self, **kwargs): """Modify the given ``conf.py`` file from a whitelisted user's project. """ # Pull config data try: conf_py_path = self.version.get_conf_py_path() except ProjectImportError: master_doc = self.create_index(extension='rst') self._write_config(master_doc=master_doc) project = self.project # Open file for appending. outfile_path = project.conf_file(self.version.slug) try: outfile = codecs.open(outfile_path, encoding='utf-8', mode='a') except IOError: trace = sys.exc_info()[2] raise ProjectImportError('Conf file not found'), None, trace try: outfile.write("\n") # TODO this should be handled better in the theme conf_py_path = os.path.join(os.path.sep, self.version.get_conf_py_path(), '') remote_version = self.version.commit_name github_user, github_repo = version_utils.get_github_username_repo( url=self.project.repo) github_version_is_editable = (self.version.type == 'branch') display_github = github_user is not None bitbucket_user, bitbucket_repo = version_utils.get_bitbucket_username_repo( url=self.project.repo) bitbucket_version_is_editable = (self.version.type == 'branch') display_bitbucket = bitbucket_user is not None rtd_ctx = { 'current_version': self.version.verbose_name, 'project': project, 'settings': settings, 'static_path': SPHINX_STATIC_DIR, 'template_path': SPHINX_TEMPLATE_DIR, 'conf_py_path': conf_py_path, 'api_host': getattr(settings, 'PUBLIC_API_URL', 'https://readthedocs.org'), # GitHub 'github_user': github_user, 'github_repo': github_repo, 'github_version': remote_version, 'github_version_is_editable': github_version_is_editable, 'display_github': display_github, # BitBucket 'bitbucket_user': bitbucket_user, 'bitbucket_repo': bitbucket_repo, 'bitbucket_version': remote_version, 'bitbucket_version_is_editable': bitbucket_version_is_editable, 'display_bitbucket': display_bitbucket, 'commit': self.project.vcs_repo(self.version.slug).commit, } # Avoid hitting database and API if using Docker build environment if getattr(settings, 'DONT_HIT_API', False): rtd_ctx['versions'] = project.active_versions() rtd_ctx['downloads'] = self.version.get_downloads(pretty=True) else: rtd_ctx['versions'] = project.api_versions() rtd_ctx['downloads'] = (api.version(self.version.pk) .get()['downloads']) rtd_string = template_loader.get_template('doc_builder/conf.py.tmpl').render(rtd_ctx) outfile.write(rtd_string) finally: outfile.close() # Print the contents of conf.py in order to make the rendered # configfile visible in the build logs self.run( 'cat', os.path.basename(outfile_path), cwd=os.path.dirname(outfile_path), )
def send_build_status(self, build, commit, state): """ Create GitHub commit status for project. :param build: Build to set up commit status for :type build: Build :param state: build state failure, pending, or success. :type state: str :param commit: commit sha of the pull request :type commit: str :returns: boolean based on commit status creation was successful or not. :rtype: Bool """ session = self.get_session() project = build.project owner, repo = build_utils.get_github_username_repo(url=project.repo) # select the correct state and description. github_build_state = SELECT_BUILD_STATUS[state]['github'] description = SELECT_BUILD_STATUS[state]['description'] target_url = build.get_full_url() if state == BUILD_STATUS_SUCCESS: target_url = build.version.get_absolute_url() context = f'{settings.RTD_BUILD_STATUS_API_NAME}:{project.slug}' data = { 'state': github_build_state, 'target_url': target_url, 'description': description, 'context': context, } resp = None try: statuses_url = f'https://api.github.com/repos/{owner}/{repo}/statuses/{commit}' resp = session.post( statuses_url, data=json.dumps(data), headers={'content-type': 'application/json'}, ) if resp.status_code == 201: log.info( "GitHub commit status created for project: %s, commit status: %s", project, github_build_state, ) return True if resp.status_code in [401, 403, 404]: log.info( 'GitHub project does not exist or user does not have ' 'permissions: project=%s, user=%s, status=%s, url=%s', project, self.user, resp.status_code, statuses_url, ) return False log.warning( 'Unknown GitHub status API response: project=%s, user=%s, status_code=%s', project, self.user, resp.status_code) return False # Catch exceptions with request or deserializing JSON except (RequestException, ValueError): log.exception( 'GitHub commit status creation failed for project: %s', project, ) # Response data should always be JSON, still try to log if not # though if resp is not None: try: debug_data = resp.json() except ValueError: debug_data = resp.content else: debug_data = resp log.debug( 'GitHub commit status creation failure response: %s', debug_data, ) return False
def setup_webhook(self, project, integration=None): """ Set up GitHub project webhook for project. :param project: project to set up webhook for :type project: Project :param integration: Integration for the project :type integration: Integration :returns: boolean based on webhook set up success, and requests Response object :rtype: (Bool, Response) """ session = self.get_session() owner, repo = build_utils.get_github_username_repo(url=project.repo) if not integration: integration, _ = Integration.objects.get_or_create( project=project, integration_type=Integration.GITHUB_WEBHOOK, ) if not integration.secret: integration.recreate_secret() data = self.get_webhook_data(project, integration) resp = None try: resp = session.post( ('https://api.github.com/repos/{owner}/{repo}/hooks'.format( owner=owner, repo=repo)), data=data, headers={'content-type': 'application/json'}, ) # GitHub will return 200 if already synced if resp.status_code in [200, 201]: recv_data = resp.json() integration.provider_data = recv_data integration.save() log.info( 'GitHub webhook creation successful for project: %s', project, ) return (True, resp) if resp.status_code in [401, 403, 404]: log.info( 'GitHub project does not exist or user does not have ' 'permissions: project=%s', project, ) # All other status codes will flow to the `else` clause below # Catch exceptions with request or deserializing JSON except (RequestException, ValueError): log.exception( 'GitHub webhook creation failed for project: %s', project, ) else: log.error( 'GitHub webhook creation failed for project: %s', project, ) # Response data should always be JSON, still try to log if not # though try: debug_data = resp.json() except ValueError: debug_data = resp.content log.debug( 'GitHub webhook creation failure response: %s', debug_data, ) # Always remove the secret and return False if we don't return True above integration.remove_secret() return (False, resp)
def append_conf(self, **kwargs): """Modify the given ``conf.py`` file from a whitelisted user's project. """ # Pull config data try: conf_py_path = self.version.get_conf_py_path() except ProjectImportError: self._write_config() self.create_index(extension='rst') project = self.version.project # Open file for appending. try: outfile = codecs.open(project.conf_file(self.version.slug), encoding='utf-8', mode='a') except IOError: trace = sys.exc_info()[2] raise ProjectImportError('Conf file not found'), None, trace outfile.write("\n") conf_py_path = self.version.get_conf_py_path() remote_version = self.version.get_vcs_slug() github_user, github_repo = version_utils.get_github_username_repo( url=self.version.project.repo) github_version_is_editable = (self.version.type == 'branch') display_github = github_user is not None bitbucket_user, bitbucket_repo = version_utils.get_bitbucket_username_repo( url=self.version.project.repo) bitbucket_version_is_editable = (self.version.type == 'branch') display_bitbucket = bitbucket_user is not None rtd_ctx = Context({ 'current_version': self.version.verbose_name, 'project': project, 'settings': settings, 'static_path': STATIC_DIR, 'template_path': TEMPLATE_DIR, 'conf_py_path': conf_py_path, 'api_host': getattr(settings, 'SLUMBER_API_HOST', 'https://readthedocs.org'), # GitHub 'github_user': github_user, 'github_repo': github_repo, 'github_version': remote_version, 'github_version_is_editable': github_version_is_editable, 'display_github': display_github, # BitBucket 'bitbucket_user': bitbucket_user, 'bitbucket_repo': bitbucket_repo, 'bitbucket_version': remote_version, 'bitbucket_version_is_editable': bitbucket_version_is_editable, 'display_bitbucket': display_bitbucket, 'commit': self.version.project.vcs_repo(self.version.slug).commit, }) # Avoid hitting database and API if using Docker build environment if getattr(settings, 'DONT_HIT_API', False): rtd_ctx['versions'] = project.active_versions() rtd_ctx['downloads'] = self.version.get_downloads(pretty=True) else: rtd_ctx['versions'] = project.api_versions() rtd_ctx['downloads'] = (api.version(self.version.pk) .get()['downloads']) rtd_string = template_loader.get_template('doc_builder/conf.py.tmpl').render(rtd_ctx) outfile.write(rtd_string)
def get_config_params(self): """Get configuration parameters to be rendered into the conf file.""" # TODO this should be handled better in the theme conf_py_path = os.path.join( os.path.sep, os.path.dirname( os.path.relpath( self.config_file, self.project.checkout_path(self.version.slug), ), ), '', ) remote_version = self.version.commit_name github_user, github_repo = version_utils.get_github_username_repo( url=self.project.repo, ) github_version_is_editable = (self.version.type == 'branch') display_github = github_user is not None bitbucket_user, bitbucket_repo = version_utils.get_bitbucket_username_repo( # noqa url=self.project.repo, ) bitbucket_version_is_editable = (self.version.type == 'branch') display_bitbucket = bitbucket_user is not None gitlab_user, gitlab_repo = version_utils.get_gitlab_username_repo( url=self.project.repo, ) gitlab_version_is_editable = (self.version.type == 'branch') display_gitlab = gitlab_user is not None versions = [] downloads = [] subproject_urls = [] # Avoid hitting database and API if using Docker build environment if settings.DONT_HIT_API: if self.project.has_feature(Feature.ALL_VERSIONS_IN_HTML_CONTEXT): versions = self.project.active_versions() else: versions = self.project.active_versions().filter( privacy_level=PUBLIC, ) downloads = self.version.get_downloads(pretty=True) subproject_urls = self.project.get_subproject_urls() else: try: versions = self.project.api_versions() if not self.project.has_feature( Feature.ALL_VERSIONS_IN_HTML_CONTEXT): versions = [ v for v in versions if v.privacy_level == PUBLIC ] downloads = api.version(self.version.pk).get()['downloads'] subproject_urls = self.project.get_subproject_urls() except ConnectionError: log.exception( 'Timeout while fetching versions/downloads/subproject_urls for Sphinx context. ' 'project: %s version: %s', self.project.slug, self.version.slug, ) data = { 'html_theme': 'sphinx_rtd_theme', 'html_theme_import': 'sphinx_rtd_theme', 'current_version': self.version.verbose_name, 'project': self.project, 'version': self.version, 'settings': settings, 'conf_py_path': conf_py_path, 'api_host': settings.PUBLIC_API_URL, 'commit': self.project.vcs_repo(self.version.slug).commit, 'versions': versions, 'downloads': downloads, 'subproject_urls': subproject_urls, # GitHub 'github_user': github_user, 'github_repo': github_repo, 'github_version': remote_version, 'github_version_is_editable': github_version_is_editable, 'display_github': display_github, # BitBucket 'bitbucket_user': bitbucket_user, 'bitbucket_repo': bitbucket_repo, 'bitbucket_version': remote_version, 'bitbucket_version_is_editable': bitbucket_version_is_editable, 'display_bitbucket': display_bitbucket, # GitLab 'gitlab_user': gitlab_user, 'gitlab_repo': gitlab_repo, 'gitlab_version': remote_version, 'gitlab_version_is_editable': gitlab_version_is_editable, 'display_gitlab': display_gitlab, # Features 'dont_overwrite_sphinx_context': self.project.has_feature(Feature.DONT_OVERWRITE_SPHINX_CONTEXT, ), } finalize_sphinx_context_data.send( sender=self.__class__, build_env=self.build_env, data=data, ) return data
def send_build_status(self, build, commit, state, link_to_build=False): """ Create GitHub commit status for project. :param build: Build to set up commit status for :type build: Build :param state: build state failure, pending, or success. :type state: str :param commit: commit sha of the pull request :type commit: str :param link_to_build: If true, link to the build page regardless the state. :returns: boolean based on commit status creation was successful or not. :rtype: Bool """ session = self.get_session() project = build.project owner, repo = build_utils.get_github_username_repo(url=project.repo) # select the correct state and description. github_build_state = SELECT_BUILD_STATUS[state]['github'] description = SELECT_BUILD_STATUS[state]['description'] target_url = build.get_full_url() statuses_url = f'https://api.github.com/repos/{owner}/{repo}/statuses/{commit}' if not link_to_build and state == BUILD_STATUS_SUCCESS: target_url = build.version.get_absolute_url() context = f'{settings.RTD_BUILD_STATUS_API_NAME}:{project.slug}' data = { 'state': github_build_state, 'target_url': target_url, 'description': description, 'context': context, } log.bind( project_slug=project.slug, commit_status=github_build_state, user_username=self.user.username, statuses_url=statuses_url, ) resp = None try: resp = session.post( statuses_url, data=json.dumps(data), headers={'content-type': 'application/json'}, ) log.bind(http_status_code=resp.status_code) if resp.status_code == 201: log.debug("GitHub commit status created for project.") return True if resp.status_code in [401, 403, 404]: log.info( 'GitHub project does not exist or user does not have permissions.' ) return False try: debug_data = resp.json() except ValueError: debug_data = resp.content log.warning( 'GitHub commit status creation failed. Unknown GitHub response.', debug_data=debug_data, ) # Catch exceptions with request or deserializing JSON except (RequestException, ValueError): log.exception('GitHub commit status creation failed for project.') return False
def append_conf(self, **kwargs): """Modify the given ``conf.py`` file from a whitelisted user's project. """ # Pull config data try: conf_py_path = self.version.get_conf_py_path() except ProjectImportError: master_doc = self.create_index(extension="rst") self._write_config(master_doc=master_doc) project = self.project # Open file for appending. outfile_path = project.conf_file(self.version.slug) try: outfile = codecs.open(outfile_path, encoding="utf-8", mode="a") except IOError: trace = sys.exc_info()[2] raise ProjectImportError("Conf file not found"), None, trace try: outfile.write("\n") # TODO this should be handled better in the theme conf_py_path = os.path.join(os.path.sep, self.version.get_conf_py_path(), "") remote_version = self.version.commit_name github_user, github_repo = version_utils.get_github_username_repo(url=self.project.repo) github_version_is_editable = self.version.type == "branch" display_github = github_user is not None bitbucket_user, bitbucket_repo = version_utils.get_bitbucket_username_repo(url=self.project.repo) bitbucket_version_is_editable = self.version.type == "branch" display_bitbucket = bitbucket_user is not None rtd_ctx = { "current_version": self.version.verbose_name, "project": project, "settings": settings, "static_path": SPHINX_STATIC_DIR, "template_path": SPHINX_TEMPLATE_DIR, "conf_py_path": conf_py_path, "api_host": getattr(settings, "SLUMBER_API_HOST", "https://readthedocs.org"), # GitHub "github_user": github_user, "github_repo": github_repo, "github_version": remote_version, "github_version_is_editable": github_version_is_editable, "display_github": display_github, # BitBucket "bitbucket_user": bitbucket_user, "bitbucket_repo": bitbucket_repo, "bitbucket_version": remote_version, "bitbucket_version_is_editable": bitbucket_version_is_editable, "display_bitbucket": display_bitbucket, "commit": self.project.vcs_repo(self.version.slug).commit, } # Avoid hitting database and API if using Docker build environment if getattr(settings, "DONT_HIT_API", False): rtd_ctx["versions"] = project.active_versions() rtd_ctx["downloads"] = self.version.get_downloads(pretty=True) else: rtd_ctx["versions"] = project.api_versions() rtd_ctx["downloads"] = api.version(self.version.pk).get()["downloads"] rtd_string = template_loader.get_template("doc_builder/conf.py.tmpl").render(rtd_ctx) outfile.write(rtd_string) finally: outfile.close() # Print the contents of conf.py in order to make the rendered # configfile visible in the build logs self.run("cat", os.path.basename(outfile_path), cwd=os.path.dirname(outfile_path))
def get_config_params(self): """Get configuration parameters to be rendered into the conf file.""" # TODO this should be handled better in the theme conf_py_path = os.path.join( os.path.sep, os.path.dirname( os.path.relpath( self.config_file, self.project.checkout_path(self.version.slug), ), ), '', ) remote_version = self.version.commit_name github_user, github_repo = version_utils.get_github_username_repo( url=self.project.repo, ) github_version_is_editable = (self.version.type == 'branch') display_github = github_user is not None bitbucket_user, bitbucket_repo = version_utils.get_bitbucket_username_repo( # noqa url=self.project.repo, ) bitbucket_version_is_editable = (self.version.type == 'branch') display_bitbucket = bitbucket_user is not None gitlab_user, gitlab_repo = version_utils.get_gitlab_username_repo( url=self.project.repo, ) gitlab_version_is_editable = (self.version.type == 'branch') display_gitlab = gitlab_user is not None # Avoid hitting database and API if using Docker build environment if getattr(settings, 'DONT_HIT_API', False): versions = self.project.active_versions() downloads = self.version.get_downloads(pretty=True) else: versions = self.project.api_versions() downloads = api.version(self.version.pk).get()['downloads'] data = { 'html_theme': 'sphinx_rtd_theme', 'html_theme_import': 'sphinx_rtd_theme', 'current_version': self.version.verbose_name, 'project': self.project, 'version': self.version, 'settings': settings, 'conf_py_path': conf_py_path, 'api_host': getattr( settings, 'PUBLIC_API_URL', 'https://readthedocs.org', ), 'commit': self.project.vcs_repo(self.version.slug).commit, 'versions': versions, 'downloads': downloads, # GitHub 'github_user': github_user, 'github_repo': github_repo, 'github_version': remote_version, 'github_version_is_editable': github_version_is_editable, 'display_github': display_github, # BitBucket 'bitbucket_user': bitbucket_user, 'bitbucket_repo': bitbucket_repo, 'bitbucket_version': remote_version, 'bitbucket_version_is_editable': bitbucket_version_is_editable, 'display_bitbucket': display_bitbucket, # GitLab 'gitlab_user': gitlab_user, 'gitlab_repo': gitlab_repo, 'gitlab_version': remote_version, 'gitlab_version_is_editable': gitlab_version_is_editable, 'display_gitlab': display_gitlab, # Features 'dont_overwrite_sphinx_context': self.project.has_feature(Feature.DONT_OVERWRITE_SPHINX_CONTEXT, ), } finalize_sphinx_context_data.send( sender=self.__class__, build_env=self.build_env, data=data, ) return data
def get_config_params(self): """Get configuration parameters to be rendered into the conf file.""" # TODO this should be handled better in the theme conf_py_path = os.path.join( os.path.sep, os.path.dirname( os.path.relpath( self.config_file, self.project.checkout_path(self.version.slug), ), ), '', ) remote_version = self.version.commit_name github_user, github_repo = version_utils.get_github_username_repo( url=self.project.repo, ) github_version_is_editable = (self.version.type == 'branch') display_github = github_user is not None bitbucket_user, bitbucket_repo = version_utils.get_bitbucket_username_repo( # noqa url=self.project.repo, ) bitbucket_version_is_editable = (self.version.type == 'branch') display_bitbucket = bitbucket_user is not None gitlab_user, gitlab_repo = version_utils.get_gitlab_username_repo( url=self.project.repo, ) gitlab_version_is_editable = (self.version.type == 'branch') display_gitlab = gitlab_user is not None # Avoid hitting database and API if using Docker build environment if settings.DONT_HIT_API: versions = self.project.active_versions() downloads = self.version.get_downloads(pretty=True) else: versions = self.project.api_versions() downloads = api.version(self.version.pk).get()['downloads'] data = { 'html_theme': 'sphinx_rtd_theme', 'html_theme_import': 'sphinx_rtd_theme', 'current_version': self.version.verbose_name, 'project': self.project, 'version': self.version, 'settings': settings, 'conf_py_path': conf_py_path, 'api_host': settings.PUBLIC_API_URL, 'commit': self.project.vcs_repo(self.version.slug).commit, 'versions': versions, 'downloads': downloads, # GitHub 'github_user': github_user, 'github_repo': github_repo, 'github_version': remote_version, 'github_version_is_editable': github_version_is_editable, 'display_github': display_github, # BitBucket 'bitbucket_user': bitbucket_user, 'bitbucket_repo': bitbucket_repo, 'bitbucket_version': remote_version, 'bitbucket_version_is_editable': bitbucket_version_is_editable, 'display_bitbucket': display_bitbucket, # GitLab 'gitlab_user': gitlab_user, 'gitlab_repo': gitlab_repo, 'gitlab_version': remote_version, 'gitlab_version_is_editable': gitlab_version_is_editable, 'display_gitlab': display_gitlab, # Features 'dont_overwrite_sphinx_context': self.project.has_feature( Feature.DONT_OVERWRITE_SPHINX_CONTEXT, ), 'use_pdf_latexmk': self.project.has_feature( Feature.USE_PDF_LATEXMK, ), } finalize_sphinx_context_data.send( sender=self.__class__, build_env=self.build_env, data=data, ) return data
def append_conf(self, **kwargs): """Modify the given ``conf.py`` file from a whitelisted user's project. """ # Pull config data try: conf_py_path = self.version.get_conf_py_path() except ProjectImportError: master_doc = self.create_index(extension='rst') self._write_config(master_doc=master_doc) project = self.project # Open file for appending. outfile_path = project.conf_file(self.version.slug) try: outfile = codecs.open(outfile_path, encoding='utf-8', mode='a') except IOError: trace = sys.exc_info()[2] raise ProjectImportError('Conf file not found'), None, trace try: outfile.write("\n") # TODO this should be handled better in the theme conf_py_path = os.path.join(os.path.sep, self.version.get_conf_py_path(), '') remote_version = self.version.commit_name github_user, github_repo = version_utils.get_github_username_repo( url=self.project.repo) github_version_is_editable = (self.version.type == 'branch') display_github = github_user is not None bitbucket_user, bitbucket_repo = version_utils.get_bitbucket_username_repo( url=self.project.repo) bitbucket_version_is_editable = (self.version.type == 'branch') display_bitbucket = bitbucket_user is not None rtd_ctx = { 'current_version': self.version.verbose_name, 'project': project, 'settings': settings, 'static_path': SPHINX_STATIC_DIR, 'template_path': SPHINX_TEMPLATE_DIR, 'conf_py_path': conf_py_path, 'api_host': getattr(settings, 'PUBLIC_API_URL', 'https://readthedocs.org'), # GitHub 'github_user': github_user, 'github_repo': github_repo, 'github_version': remote_version, 'github_version_is_editable': github_version_is_editable, 'display_github': display_github, # BitBucket 'bitbucket_user': bitbucket_user, 'bitbucket_repo': bitbucket_repo, 'bitbucket_version': remote_version, 'bitbucket_version_is_editable': bitbucket_version_is_editable, 'display_bitbucket': display_bitbucket, 'commit': self.project.vcs_repo(self.version.slug).commit, } # Avoid hitting database and API if using Docker build environment if getattr(settings, 'DONT_HIT_API', False): rtd_ctx['versions'] = project.active_versions() rtd_ctx['downloads'] = self.version.get_downloads(pretty=True) else: rtd_ctx['versions'] = project.api_versions() rtd_ctx['downloads'] = (api.version( self.version.pk).get()['downloads']) rtd_string = template_loader.get_template( 'doc_builder/conf.py.tmpl').render(rtd_ctx) outfile.write(rtd_string) finally: outfile.close() # Print the contents of conf.py in order to make the rendered # configfile visible in the build logs self.run( 'cat', os.path.basename(outfile_path), cwd=os.path.dirname(outfile_path), )
def get_provider_data(self, project, integration): """ Gets provider data from GitHub Webhooks API. :param project: project :type project: Project :param integration: Integration for the project :type integration: Integration :returns: Dictionary containing provider data from the API or None :rtype: dict """ if integration.provider_data: return integration.provider_data session = self.get_session() owner, repo = build_utils.get_github_username_repo(url=project.repo) rtd_webhook_url = 'https://{domain}{path}'.format( domain=settings.PRODUCTION_DOMAIN, path=reverse( 'api_webhook', kwargs={ 'project_slug': project.slug, 'integration_pk': integration.pk, }, ) ) try: resp = session.get( ( 'https://api.github.com/repos/{owner}/{repo}/hooks' .format(owner=owner, repo=repo) ), ) if resp.status_code == 200: recv_data = resp.json() for webhook_data in recv_data: if webhook_data["config"]["url"] == rtd_webhook_url: integration.provider_data = webhook_data integration.save() log.info( 'GitHub integration updated with provider data for project: %s', project, ) break else: log.info( 'GitHub project does not exist or user does not have ' 'permissions: project=%s', project, ) except Exception: log.exception( 'GitHub webhook Listing failed for project: %s', project, ) return integration.provider_data
def setup_webhook(self, project, integration=None): """ Set up GitHub project webhook for project. :param project: project to set up webhook for :type project: Project :param integration: Integration for the project :type integration: Integration :returns: boolean based on webhook set up success, and requests Response object :rtype: (Bool, Response) """ session = self.get_session() owner, repo = build_utils.get_github_username_repo(url=project.repo) if not integration: integration, _ = Integration.objects.get_or_create( project=project, integration_type=Integration.GITHUB_WEBHOOK, ) if not integration.secret: integration.recreate_secret() data = self.get_webhook_data(project, integration) url = f'https://api.github.com/repos/{owner}/{repo}/hooks' log.bind( url=url, project_slug=project.slug, integration_id=integration.pk, ) resp = None try: resp = session.post( url, data=data, headers={'content-type': 'application/json'}, ) log.bind(http_status_code=resp.status_code) # GitHub will return 200 if already synced if resp.status_code in [200, 201]: recv_data = resp.json() integration.provider_data = recv_data integration.save() log.debug('GitHub webhook creation successful for project.') return (True, resp) if resp.status_code in [401, 403, 404]: log.warning( 'GitHub project does not exist or user does not have permissions.' ) else: # Unknown response from GitHub try: debug_data = resp.json() except ValueError: debug_data = resp.content log.warning( 'GitHub webhook creation failed for project. Unknown response from GitHub.', debug_data=debug_data, ) # Catch exceptions with request or deserializing JSON except (RequestException, ValueError): log.exception('GitHub webhook creation failed for project.') # Always remove the secret and return False if we don't return True above integration.remove_secret() return (False, resp)
def get_commit_url(self): """Return the commit URL.""" repo_url = self.project.repo if self.is_external: if 'github' in repo_url: user, repo = get_github_username_repo(repo_url) if not user and not repo: return '' repo = repo.rstrip('/') return GITHUB_PULL_REQUEST_COMMIT_URL.format( user=user, repo=repo, number=self.version.verbose_name, commit=self.commit ) if 'gitlab' in repo_url: user, repo = get_gitlab_username_repo(repo_url) if not user and not repo: return '' repo = repo.rstrip('/') return GITLAB_MERGE_REQUEST_COMMIT_URL.format( user=user, repo=repo, number=self.version.verbose_name, commit=self.commit ) # TODO: Add External Version Commit URL for BitBucket. else: if 'github' in repo_url: user, repo = get_github_username_repo(repo_url) if not user and not repo: return '' repo = repo.rstrip('/') return GITHUB_COMMIT_URL.format( user=user, repo=repo, commit=self.commit ) if 'gitlab' in repo_url: user, repo = get_gitlab_username_repo(repo_url) if not user and not repo: return '' repo = repo.rstrip('/') return GITLAB_COMMIT_URL.format( user=user, repo=repo, commit=self.commit ) if 'bitbucket' in repo_url: user, repo = get_bitbucket_username_repo(repo_url) if not user and not repo: return '' repo = repo.rstrip('/') return BITBUCKET_COMMIT_URL.format( user=user, repo=repo, commit=self.commit ) return None