def symlink_cnames(version): """ OLD Link from HOME/user_builds/cnames/<cname> -> HOME/user_builds/<project>/rtd-builds/ NEW Link from HOME/user_builds/cnametoproject/<cname> -> HOME/user_builds/<project>/ """ try: redis_conn = redis.Redis(**settings.REDIS) cnames = redis_conn.smembers('rtd_slug:v1:%s' % version.project.slug) except redis.ConnectionError: log.error(LOG_TEMPLATE.format(project=version.project.slug, version=version.slug, msg='Failed to symlink cnames, Redis error.'), exc_info=True) return for cname in cnames: log.debug(LOG_TEMPLATE.format(project=version.project.slug, version=version.slug, msg="Symlinking CNAME: %s" % cname)) docs_dir = version.project.rtd_build_path(version.slug) # Chop off the version from the end. docs_dir = '/'.join(docs_dir.split('/')[:-1]) # Old symlink location -- Keep this here til we change nginx over symlink = version.project.cnames_symlink_path(cname) run_on_app_servers('mkdir -p %s' % '/'.join(symlink.split('/')[:-1])) run_on_app_servers('ln -nsf %s %s' % (docs_dir, symlink)) # New symlink location new_docs_dir = version.project.doc_path new_cname_symlink = os.path.join(getattr(settings, 'SITE_ROOT'), 'cnametoproject', cname) run_on_app_servers('mkdir -p %s' % '/'.join(new_cname_symlink.split('/')[:-1])) run_on_app_servers('ln -nsf %s %s' % (new_docs_dir, new_cname_symlink))
def fileify(version_pk, commit): """ Create ImportedFile objects for all of a version's files. This is a prereq for indexing the docs for search. It also causes celery-haystack to kick off an index of the file. """ version = Version.objects.get(pk=version_pk) project = version.project path = project.rtd_build_path(version.slug) if path: log.info(LOG_TEMPLATE.format( project=project.slug, version=version.slug, msg='Creating ImportedFiles')) for root, dirnames, filenames in os.walk(path): for filename in filenames: if fnmatch.fnmatch(filename, '*.html'): dirpath = os.path.join(root.replace(path, '').lstrip('/'), filename.lstrip('/')) obj, created = ImportedFile.objects.get_or_create( project=project, version=version, path=dirpath, name=filename, commit=commit, ) if not created: obj.save() # Delete ImportedFiles from previous versions ImportedFile.objects.filter(project=project, version=version).exclude(commit=commit).delete() else: log.info(LOG_TEMPLATE.format(project=project.slug, version=version.slug, msg='No ImportedFile files'))
def finish_build(version, build, results): """ Build Finished, do house keeping bits """ (ret, out, err) = results['html'] if 'no targets are out of date.' in out: log.info(LOG_TEMPLATE.format(project=version.project.slug, version=version.slug, msg="Build Unchanged")) else: if ret == 0: log.info(LOG_TEMPLATE.format(project=version.project.slug, version=version.slug, msg="Successful Build")) #update_search(version) #fileify.delay(version.pk) symlinks.symlink_cnames(version) symlinks.symlink_translations(version) symlinks.symlink_subprojects(version) if version.project.single_version: symlinks.symlink_single_version(version) else: symlinks.remove_symlink_single_version(version) # This requires database access, must disable it for now. #send_notifications(version, build) else: log.warning(LOG_TEMPLATE.format(project=version.project.slug, version=version.slug, msg="Failed HTML Build"))
def setup_vcs(version, build, api): """ Update the checkout of the repo to make sure it's the latest. This also syncs versions in the DB. """ log.info( LOG_TEMPLATE.format(project=version.project.slug, version=version.slug, msg='Updating docs from VCS')) try: update_output = update_imported_docs(version.pk, api) commit = version.project.vcs_repo(version.slug).commit if commit: build['commit'] = commit except ProjectImportError, err: log.error(LOG_TEMPLATE.format( project=version.project.slug, version=version.slug, msg='Failed to import project; skipping build'), exc_info=True) build['state'] = 'finished' build['setup_error'] = ('Failed to import project; skipping build.\n' '\nError\n-----\n\n%s' % err.message) api.build(build['id']).put(build) return False
def update_imported_docs(version_pk, api=None): """ Check out or update the given project's repository. """ if api is None: api = tastyapi.api version_data = api.version(version_pk).get() version = make_api_version(version_data) project = version.project ret_dict = {} # Make Dirs if not os.path.exists(project.doc_path): os.makedirs(project.doc_path) with project.repo_nonblockinglock(version=version, max_lock_age=getattr(settings, "REPO_LOCK_SECONDS", 30)): if not project.vcs_repo(): raise ProjectImportError(("Repo type '{0}' unknown".format(project.repo_type))) # Get the actual code on disk if version: log.info( LOG_TEMPLATE.format( project=project.slug, version=version.slug, msg="Checking out version {slug}: {identifier}".format( slug=version.slug, identifier=version.identifier ), ) ) version_slug = version.slug version_repo = project.vcs_repo(version_slug) ret_dict["checkout"] = version_repo.checkout(version.identifier) else: # Does this ever get called? log.info(LOG_TEMPLATE.format(project=project.slug, version=version.slug, msg="Updating to latest revision")) version_slug = "latest" version_repo = project.vcs_repo(version_slug) ret_dict["checkout"] = version_repo.update() # Update tags/version version_post_data = {"repo": version_repo.repo_url} if version_repo.supports_tags: version_post_data["tags"] = [ {"identifier": v.identifier, "verbose_name": v.verbose_name} for v in version_repo.tags ] if version_repo.supports_branches: version_post_data["branches"] = [ {"identifier": v.identifier, "verbose_name": v.verbose_name} for v in version_repo.branches ] try: apiv2.project(project.pk).sync_versions.post(version_post_data) except Exception, e: print "Sync Verisons Exception: %s" % e.message
def update_docs(pk, version_pk=None, build_pk=None, record=True, docker=False, pdf=True, man=True, epub=True, dash=True, search=True, force=False, intersphinx=True, localmedia=True, api=None, **kwargs): """ The main entry point for updating documentation. It handles all of the logic around whether a project is imported or we created it. Then it will build the html docs and other requested parts. `pk` Primary key of the project to update `record` Whether or not to keep a record of the update in the database. Useful for preventing changes visible to the end-user when running commands from the shell, for example. """ # Dependency injection to allow for testing if api is None: api = tastyapi.api project_data = api.project(pk).get() project = make_api_project(project_data) log.info(LOG_TEMPLATE.format(project=project.slug, version='', msg='Building')) version = ensure_version(api, project, version_pk) build = create_build(build_pk) results = {} # Build Servery stuff try: record_build(api=api, build=build, record=record, results=results, state='cloning') vcs_results = setup_vcs(version, build, api) if vcs_results: results.update(vcs_results) if docker or settings.DOCKER_ENABLE: record_build(api=api, build=build, record=record, results=results, state='building') docker = DockerEnvironment(version) build_results = docker.build() results.update(build_results) else: record_build(api=api, build=build, record=record, results=results, state='installing') setup_results = setup_environment(version) results.update(setup_results) record_build(api=api, build=build, record=record, results=results, state='building') build_results = build_docs(version, force, pdf, man, epub, dash, search, localmedia) results.update(build_results) except vcs_support_utils.LockTimeout, e: results['checkout'] = (423, "", "Version locked, retrying in 5 minutes.") log.info(LOG_TEMPLATE.format(project=version.project.slug, version=version.slug, msg="Unable to lock, will retry")) # http://celery.readthedocs.org/en/3.0/userguide/tasks.html#retrying # Should completely retry the task for us until max_retries is exceeded update_docs.retry(exc=e, throw=False)
def fileify(version_pk): """ Create ImportedFile objects for all of a version's files. This is a prereq for indexing the docs for search. It also causes celery-haystack to kick off an index of the file. """ version_data = api.version(version_pk).get() version = make_api_version(version_data) project = version.project path = project.rtd_build_path(version.slug) log.info(LOG_TEMPLATE.format(project=project.slug, version=version.slug, msg='Indexing files')) if path: for root, dirnames, filenames in os.walk(path): for filename in filenames: if fnmatch.fnmatch(filename, '*.html'): dirpath = os.path.join(root.replace(path, '').lstrip('/'), filename.lstrip('/')) if getattr(settings, 'DONT_HIT_DB', True): api.file.post(dict( project="/api/v1/project/%s/" % project.pk, version="/api/v1/version/%s/" % version.pk, path=dirpath, name=filename)) else: obj, created = ImportedFile.objects.get_or_create( project=project, version=version, path=dirpath, name=filename) if not created: obj.save()
def symlink_subprojects(version): """ Link from HOME/user_builds/project/subprojects/<project> -> HOME/user_builds/<project>/rtd-builds/ """ # Subprojects subprojects = apiv2.project( version.project.pk).subprojects.get()['subprojects'] for subproject_data in subprojects: slugs = [subproject_data['slug']] if '_' in slugs[0]: slugs.append(slugs[0].replace('_', '-')) for subproject_slug in slugs: log.debug( LOG_TEMPLATE.format(project=version.project.slug, version=version.slug, msg="Symlinking subproject: %s" % subproject_slug)) # The directory for this specific subproject symlink = version.project.subprojects_symlink_path(subproject_slug) run_on_app_servers('mkdir -p %s' % '/'.join(symlink.split('/')[:-1])) # Where the actual docs live docs_dir = os.path.join(settings.DOCROOT, subproject_slug, 'rtd-builds') run_on_app_servers('ln -nsf %s %s' % (docs_dir, symlink))
def symlink_translations(version): """ Link from HOME/user_builds/project/translations/<lang> -> HOME/user_builds/<project>/rtd-builds/ """ if getattr(settings, 'DONT_HIT_DB', True): translations = [(trans['slug'], trans['language']) for trans in apiv2.project(version.project.pk).translations.get()['translations']] else: translations = [(trans.slug, trans.language) for trans in version.project.translations.all()] for slug, language in translations: log.debug(LOG_TEMPLATE.format(project=version.project.slug, version=version.slug, msg="Symlinking translation: %s->%s" % (language, slug))) # The directory for this specific translation symlink = version.project.translations_symlink_path(language) run_on_app_servers('mkdir -p %s' % '/'.join(symlink.split('/')[:-1])) # Where the actual docs live docs_dir = os.path.join(settings.DOCROOT, slug, 'rtd-builds') run_on_app_servers('ln -nsf %s %s' % (docs_dir, symlink)) # Hack in the en version for backwards compat symlink = version.project.translations_symlink_path('en') run_on_app_servers('mkdir -p %s' % '/'.join(symlink.split('/')[:-1])) docs_dir = os.path.join(version.project.doc_path, 'rtd-builds') run_on_app_servers('ln -nsf %s %s' % (docs_dir, symlink)) # Add the main language project to nginx too if version.project.language is not 'en': symlink = version.project.translations_symlink_path(version.project.language) run_on_app_servers('mkdir -p %s' % '/'.join(symlink.split('/')[:-1])) docs_dir = os.path.join(settings.DOCROOT, version.project.slug.replace('_', '-'), 'rtd-builds') run_on_app_servers('ln -nsf %s %s' % (docs_dir, symlink))
def record_pdf(api, record, results, state, version): if not record or 'sphinx' not in version.project.documentation_type: return None try: if 'pdf' in results: pdf_exit = results['pdf'][0] pdf_success = pdf_exit == 0 pdf_output = results['pdf'][1] pdf_error = results['pdf'][2] else: pdf_exit = 999 pdf_success = False pdf_output = pdf_error = "PDF Failed" api.build.post(dict( state=state, project='/api/v1/project/%s/' % version.project.pk, version='/api/v1/version/%s/' % version.pk, success=pdf_success, type='pdf', output=pdf_output, error=pdf_error, exit_code=pdf_exit, )) except Exception: log.error(LOG_TEMPLATE.format(project=version.project.slug, version=version.slug, msg="Unable to post a new build"), exc_info=True)
def docker_build(version_pk, pdf=True, man=True, epub=True, dash=True, search=True, force=False, intersphinx=True, localmedia=True): """ The code that executes inside of docker """ version_data = api.version(version_pk).get() version = make_api_version(version_data) environment_results = setup_environment(version) results = build_docs(version, force, pdf, man, epub, dash, search, localmedia) results.update(environment_results) try: number = uuid.uuid4() path = os.path.join(version.project.doc_path, 'build.json') fh = open(path, 'w') json.dump(results, fh) fh.close() except IOError as e: log.debug( LOG_TEMPLATE.format( project=version.project.slug, version='', msg='Cannot write to build.json: {0}'.format(e))) return None return number
def symlink_translations(version): """ Link from HOME/user_builds/project/translations/<lang> -> HOME/user_builds/<project>/rtd-builds/ """ translations = apiv2.project(version.project.pk).translations.get()['translations'] for translation_data in translations: translation_slug = translation_data['slug'].replace('_', '-') translation_language = translation_data['language'] log.debug(LOG_TEMPLATE.format(project=version.project.slug, version=version.slug, msg="Symlinking translation: %s->%s" % (translation_language, translation_slug))) # The directory for this specific translation symlink = version.project.translations_symlink_path(translation_language) run_on_app_servers('mkdir -p %s' % '/'.join(symlink.split('/')[:-1])) # Where the actual docs live docs_dir = os.path.join(settings.DOCROOT, translation_slug, 'rtd-builds') run_on_app_servers('ln -nsf %s %s' % (docs_dir, symlink)) # Hack in the en version for backwards compat symlink = version.project.translations_symlink_path('en') run_on_app_servers('mkdir -p %s' % '/'.join(symlink.split('/')[:-1])) docs_dir = os.path.join(version.project.doc_path, 'rtd-builds') run_on_app_servers('ln -nsf %s %s' % (docs_dir, symlink)) # Add the main language project to nginx too if version.project.language is not 'en': symlink = version.project.translations_symlink_path(version.project.language) run_on_app_servers('mkdir -p %s' % '/'.join(symlink.split('/')[:-1])) docs_dir = os.path.join(settings.DOCROOT, version.project.slug.replace('_', '-'), 'rtd-builds') run_on_app_servers('ln -nsf %s %s' % (docs_dir, symlink))
def record_pdf(api, record, results, state, version): if not record or 'sphinx' not in version.project.documentation_type: return None try: if 'pdf' in results: pdf_exit = results['pdf'][0] pdf_success = pdf_exit == 0 pdf_output = results['pdf'][1] pdf_error = results['pdf'][2] else: pdf_exit = 999 pdf_success = False pdf_output = pdf_error = "PDF Failed" pdf_output = pdf_output.decode('utf-8', 'ignore') pdf_error = pdf_error.decode('utf-8', 'ignore') api.build.post( dict( state=state, project='/api/v1/project/%s/' % version.project.pk, version='/api/v1/version/%s/' % version.pk, success=pdf_success, type='pdf', output=pdf_output, error=pdf_error, exit_code=pdf_exit, )) except Exception: log.error(LOG_TEMPLATE.format(project=version.project.slug, version=version.slug, msg="Unable to post a new build"), exc_info=True)
def email_notification(version, build, email): log.debug( LOG_TEMPLATE.format(project=version.project.slug, version=version.slug, msg='sending email to: %s' % email)) context = { 'version': version, 'project': version.project, 'build': build, 'build_url': 'https://{0}{1}'.format( getattr(settings, 'PRODUCTION_DOMAIN', 'readthedocs.org'), build.get_absolute_url()), 'unsub_url': 'https://{0}{1}'.format( getattr(settings, 'PRODUCTION_DOMAIN', 'readthedocs.org'), reverse('projects_notifications', args=[version.project.slug])), } if build.commit: title = _('Failed: {project.name} ({commit})').format( commit=build.commit[:8], **context) else: title = _('Failed: {project.name} ({version.verbose_name})').format( **context) send_email(email, title, template='projects/email/build_failed.txt', template_html='projects/email/build_failed.html', context=context)
def symlink_subprojects(version): """ Link from HOME/user_builds/project/subprojects/<project> -> HOME/user_builds/<project>/rtd-builds/ """ # Subprojects if getattr(settings, 'DONT_HIT_DB', True): subproject_slugs = [data['slug'] for data in apiv2.project(version.project.pk).subprojects.get()['subprojects']] else: rels = version.project.subprojects.all() subproject_slugs = [rel.child.slug for rel in rels] for slug in subproject_slugs: slugs = [slug] if '_' in slugs[0]: slugs.append(slugs[0].replace('_', '-')) for subproject_slug in slugs: log.debug(LOG_TEMPLATE.format(project=version.project.slug, version=version.slug, msg="Symlinking subproject: %s" % subproject_slug)) # The directory for this specific subproject symlink = version.project.subprojects_symlink_path(subproject_slug) run_on_app_servers('mkdir -p %s' % '/'.join(symlink.split('/')[:-1])) # Where the actual docs live docs_dir = os.path.join(settings.DOCROOT, subproject_slug, 'rtd-builds') run_on_app_servers('ln -nsf %s %s' % (docs_dir, symlink))
def fileify(version_pk): """ Create ImportedFile objects for all of a version's files. This is a prereq for indexing the docs for search. It also causes celery-haystack to kick off an index of the file. """ version_data = api.version(version_pk).get() version = make_api_version(version_data) project = version.project path = project.rtd_build_path(version.slug) log.info(LOG_TEMPLATE.format( project=project.slug, version=version.slug, msg='Indexing files')) if path: for root, dirnames, filenames in os.walk(path): for filename in filenames: if fnmatch.fnmatch(filename, '*.html'): dirpath = os.path.join(root.replace(path, '').lstrip('/'), filename.lstrip('/')) if getattr(settings, 'DONT_HIT_DB', True): api.file.post(dict( project="/api/v1/project/%s/" % project.pk, version="/api/v1/version/%s/" % version.pk, path=dirpath, name=filename)) else: obj, created = ImportedFile.objects.get_or_create( project=project, version=version, path=dirpath, name=filename) if not created: obj.save()
def email_notification(version, build, email): log.debug( LOG_TEMPLATE.format(project=version.project.slug, version=version.slug, msg="sending email to: %s" % email) ) context = { "version": version, "project": version.project, "build": build, "build_url": "https://{0}{1}".format( getattr(settings, "PRODUCTION_DOMAIN", "readthedocs.org"), build.get_absolute_url() ), "unsub_url": "https://{0}{1}".format( getattr(settings, "PRODUCTION_DOMAIN", "readthedocs.org"), reverse("projects_notifications", args=[version.project.slug]), ), } if build.commit: title = _("Failed: {project.name} ({commit})").format(commit=build.commit[:8], **context) else: title = _("Failed: {project.name} ({version.verbose_name})").format(**context) send_email( email, title, template="projects/email/build_failed.txt", template_html="projects/email/build_failed.html", context=context, )
def docker_build(version_pk, pdf=True, man=True, epub=True, dash=True, search=True, force=False, intersphinx=True, localmedia=True): """ The code that executes inside of docker """ version_data = api.version(version_pk).get() version = make_api_version(version_data) environment_results = setup_environment(version) results = build_docs(version=version, force=force, pdf=pdf, man=man, epub=epub, dash=dash, search=search, localmedia=localmedia) results.update(environment_results) try: number = uuid.uuid4() path = os.path.join(version.project.doc_path, 'build.json') fh = open(path, 'w') json.dump(results, fh) fh.close() except IOError as e: log.debug(LOG_TEMPLATE.format( project=version.project.slug, version='', msg='Cannot write to build.json: {0}'.format(e) )) return None return number
def update_static_metadata(project_pk): """Update static metadata JSON file Metadata settings include the following project settings: version The default version for the project, default: `latest` language The default language for the project, default: `en` languages List of languages built by linked translation projects. """ project_base = apiv2.project(project_pk) project_data = project_base.get() project = make_api_project(project_data) translations = project_base.translations.get()["translations"] languages = set([translation["language"] for translation in translations if "language" in translation]) # Convert to JSON safe types metadata = {"version": project.default_version, "language": project.language, "languages": list(languages)} try: path = project.static_metadata_path() fh = open(path, "w") json.dump(metadata, fh) fh.close() copy_file_to_app_servers(path, path) except IOError as e: log.debug( LOG_TEMPLATE.format(project=project.slug, version="", msg="Cannot write to metadata.json: {0}".format(e)) )
def email_notification(version, build, email): log.debug(LOG_TEMPLATE.format(project=version.project.slug, version=version.slug, msg='sending email to: %s' % email)) context = {'version': version, 'project': version.project, 'build': build, 'build_url': 'https://{0}{1}'.format( getattr(settings, 'PRODUCTION_DOMAIN', 'readthedocs.org'), build.get_absolute_url()), 'unsub_url': 'https://{0}{1}'.format( getattr(settings, 'PRODUCTION_DOMAIN', 'readthedocs.org'), reverse('projects_notifications', args=[version.project.slug])), } if build.commit: title = _('Failed: {project.name} ({commit})').format(commit=build.commit[:8], **context) else: title = _('Failed: {project.name} ({version.verbose_name})').format(**context) send_email( email, title, template='projects/email/build_failed.txt', template_html='projects/email/build_failed.html', context=context )
def update_static_metadata(project_pk): """Update static metadata JSON file Metadata settings include the following project settings: version The default version for the project, default: `latest` language The default language for the project, default: `en` languages List of languages built by linked translation projects. """ project_base = apiv2.project(project_pk) project_data = project_base.get() project = make_api_project(project_data) log.info( LOG_TEMPLATE.format( project=project.slug, version='', msg='Updating static metadata', )) translations = project_base.translations.get()['translations'] languages = set([ translation['language'] for translation in translations if 'language' in translation ]) # Convert to JSON safe types metadata = { 'version': project.default_version, 'language': project.language, 'languages': list(languages), 'single_version': project.single_version, } try: path = project.static_metadata_path() fh = open(path, 'w') json.dump(metadata, fh) fh.close() copy_file_to_app_servers(path, path) except IOError as e: log.debug( LOG_TEMPLATE.format( project=project.slug, version='', msg='Cannot write to metadata.json: {0}'.format(e)))
def update_static_metadata(project_pk): """Update static metadata JSON file Metadata settings include the following project settings: version The default version for the project, default: `latest` language The default language for the project, default: `en` languages List of languages built by linked translation projects. """ project_base = apiv2.project(project_pk) project_data = project_base.get() project = make_api_project(project_data) log.info(LOG_TEMPLATE.format( project=project.slug, version='', msg='Updating static metadata', )) translations = project_base.translations.get()['translations'] languages = set([ translation['language'] for translation in translations if 'language' in translation ]) # Convert to JSON safe types metadata = { 'version': project.default_version, 'language': project.language, 'languages': list(languages), 'single_version': project.single_version, } try: path = project.static_metadata_path() fh = open(path, 'w') json.dump(metadata, fh) fh.close() copy_file_to_app_servers(path, path) except IOError as e: log.debug(LOG_TEMPLATE.format( project=project.slug, version='', msg='Cannot write to metadata.json: {0}'.format(e) ))
def fileify(version_pk, commit): """ Create ImportedFile objects for all of a version's files. This is a prereq for indexing the docs for search. It also causes celery-haystack to kick off an index of the file. """ version = Version.objects.get(pk=version_pk) project = version.project if not commit: log.info( LOG_TEMPLATE.format( project=project.slug, version=version.slug, msg= 'Imported File not being built because no commit information')) path = project.rtd_build_path(version.slug) if path: log.info( LOG_TEMPLATE.format(project=project.slug, version=version.slug, msg='Creating ImportedFiles')) for root, dirnames, filenames in os.walk(path): for filename in filenames: if fnmatch.fnmatch(filename, '*.html'): dirpath = os.path.join( root.replace(path, '').lstrip('/'), filename.lstrip('/')) obj, created = ImportedFile.objects.get_or_create( project=project, version=version, path=dirpath, name=filename, commit=commit, ) if not created: obj.save() # Delete ImportedFiles from previous versions ImportedFile.objects.filter( project=project, version=version).exclude(commit=commit).delete() else: log.info( LOG_TEMPLATE.format(project=project.slug, version=version.slug, msg='No ImportedFile files'))
def remove_symlink_single_version(version): """Remove single_version symlink""" log.debug( LOG_TEMPLATE.format(project=version.project.slug, version=version.project.default_version, msg="Removing symlink for single_version")) symlink = version.project.single_version_symlink_path() run_on_app_servers('rm -f %s' % symlink)
def remove_symlink_single_version(version): """Remove single_version symlink""" log.debug(LOG_TEMPLATE.format( project=version.project.slug, version=version.project.get_default_version(), msg="Removing symlink for single_version") ) symlink = version.project.single_version_symlink_path() run_on_app_servers('rm -f %s' % symlink)
def setup_vcs(version, build, api): """ Update the checkout of the repo to make sure it's the latest. This also syncs versions in the DB. """ log.info(LOG_TEMPLATE.format(project=version.project.slug, version=version.slug, msg='Updating docs from VCS')) try: update_output = update_imported_docs(version.pk, api) except ProjectImportError, err: log.error(LOG_TEMPLATE.format(project=version.project.slug, version=version.slug, msg='Failed to import project; skipping build'), exc_info=True) build['state'] = 'finished' build['setup_error'] = ( 'Failed to import project; skipping build.\n' '\nError\n-----\n\n%s' % err.message ) api.build(build['id']).put(build) return False
def setup_vcs(version, build, api): """ Update the checkout of the repo to make sure it's the latest. This also syncs versions in the DB. """ log.info(LOG_TEMPLATE.format(project=version.project.slug, version=version.slug, msg='Updating docs from VCS')) try: update_output = update_imported_docs(version.pk, api) commit = version.project.vcs_repo(version.slug).commit if commit: build['commit'] = commit except ProjectImportError: log.error(LOG_TEMPLATE.format(project=version.project.slug, version=version.slug, msg='Failed to import project; skipping build'), exc_info=True) raise return update_output
def update_static_metadata(project_pk, path=None): """Update static metadata JSON file Metadata settings include the following project settings: version The default version for the project, default: `latest` language The default language for the project, default: `en` languages List of languages built by linked translation projects. """ project = Project.objects.get(pk=project_pk) if not path: path = project.static_metadata_path() log.info( LOG_TEMPLATE.format( project=project.slug, version='', msg='Updating static metadata', )) translations = [trans.language for trans in project.translations.all()] languages = set(translations) # Convert to JSON safe types metadata = { 'version': project.default_version, 'language': project.language, 'languages': list(languages), 'single_version': project.single_version, } try: fh = open(path, 'w+') json.dump(metadata, fh) fh.close() Syncer.copy(path, path, host=socket.gethostname(), file=True) except (AttributeError, IOError) as e: log.debug( LOG_TEMPLATE.format( project=project.slug, version='', msg='Cannot write to metadata.json: {0}'.format(e)))
def update_static_metadata(project_pk, path=None): """Update static metadata JSON file Metadata settings include the following project settings: version The default version for the project, default: `latest` language The default language for the project, default: `en` languages List of languages built by linked translation projects. """ project = Project.objects.get(pk=project_pk) if not path: path = project.static_metadata_path() log.info(LOG_TEMPLATE.format( project=project.slug, version='', msg='Updating static metadata', )) translations = [trans.language for trans in project.translations.all()] languages = set(translations) # Convert to JSON safe types metadata = { 'version': project.default_version, 'language': project.language, 'languages': list(languages), 'single_version': project.single_version, } try: fh = open(path, 'w+') json.dump(metadata, fh) fh.close() Syncer.copy(path, path, host=socket.gethostname(), file=True) except (AttributeError, IOError) as e: log.debug(LOG_TEMPLATE.format( project=project.slug, version='', msg='Cannot write to metadata.json: {0}'.format(e) ))
def webhook_notification(version, build, hook_url): data = json.dumps( { "name": project.name, "slug": project.slug, "build": {"id": build.id, "success": build.success, "date": build.date.strftime("%Y-%m-%d %H:%M:%S")}, } ) log.debug(LOG_TEMPLATE.format(project=project.slug, version="", msg="sending notification to: %s" % hook_url)) requests.post(hook_url, data=data)
def setup_vcs(version, build, api): """ Update the checkout of the repo to make sure it's the latest. This also syncs versions in the DB. """ log.info(LOG_TEMPLATE.format(project=version.project.slug, version=version.slug, msg="Updating docs from VCS")) try: update_output = update_imported_docs(version.pk, api) except ProjectImportError, err: log.error( LOG_TEMPLATE.format( project=version.project.slug, version=version.slug, msg="Failed to import project; skipping build" ), exc_info=True, ) build["state"] = "finished" build["setup_error"] = "Failed to import project; skipping build.\n" "\nError\n-----\n\n%s" % err.message api.build(build["id"]).put(build) return False
def ensure_version(api, project, version_pk): """ Ensure we're using a sane version. This also creates the "latest" version if it doesn't exist. """ if version_pk: version_data = api.version(version_pk).get() else: branch = project.default_branch or project.vcs_repo().fallback_branch try: # Use latest version version_data = (api.version( project.slug).get(slug='latest')['objects'][0]) except (slumber.exceptions.HttpClientError, IndexError): # Create the latest version since it doesn't exist version_data = dict( project='/api/v1/project/%s/' % project.pk, slug='latest', type='branch', active=True, verbose_name='latest', identifier=branch, ) try: version_data = api.version.post(version_data) except Exception as e: log.info( LOG_TEMPLATE.format( project=project.slug, version='', msg='Exception in creating version: %s' % e)) raise e version = make_api_version(version_data) if not version_pk: # Lots of course correction. to_save = False if not version.verbose_name: version_data['verbose_name'] = 'latest' to_save = True if not version.active: version_data['active'] = True to_save = True if version.identifier != branch: version_data['identifier'] = branch to_save = True if to_save: version_data['project'] = ("/api/v1/version/%s/" % version_data['project'].pk) api.version(version.pk).put(version_data) return version
def webhook_notification(version, build, hook_url): data = json.dumps({ 'name': project.name, 'slug': project.slug, 'build': { 'id': build.id, 'success': build.success, 'date': build.date.strftime('%Y-%m-%d %H:%M:%S'), } }) log.debug(LOG_TEMPLATE.format(project=project.slug, version='', msg='sending notification to: %s' % hook_url)) requests.post(hook_url, data=data)
def ensure_version(api, project, version_pk): """ Ensure we're using a sane version. This also creates the "latest" version if it doesn't exist. """ if version_pk: version_data = api.version(version_pk).get() else: branch = project.default_branch or project.vcs_repo().fallback_branch try: # Use latest version version_data = (api.version(project.slug) .get(slug='latest')['objects'][0]) except (slumber.exceptions.HttpClientError, IndexError): # Create the latest version since it doesn't exist version_data = dict( project='/api/v1/project/%s/' % project.pk, slug='latest', type='branch', active=True, verbose_name='latest', identifier=branch, ) try: version_data = api.version.post(version_data) except Exception as e: log.info(LOG_TEMPLATE.format( project=project.slug, version='', msg='Exception in creating version: %s' % e)) raise e version = make_api_version(version_data) if not version_pk: # Lots of course correction. to_save = False if not version.verbose_name: version_data['verbose_name'] = 'latest' to_save = True if not version.active: version_data['active'] = True to_save = True if version.identifier != branch: version_data['identifier'] = branch to_save = True if to_save: version_data['project'] = ("/api/v1/project/%s/" % version_data['project'].pk) api.version(version.pk).put(version_data) return version
def ensure_version(api, project, version_pk): """ Ensure we're using a sane version. This also creates the "latest" version if it doesn't exist. """ if version_pk: version_data = api.version(version_pk).get() else: branch = project.default_branch or project.vcs_repo().fallback_branch try: # Use latest version version_data = api.version(project.slug).get(slug="latest")["objects"][0] except (slumber.exceptions.HttpClientError, IndexError): # Create the latest version since it doesn't exist version_data = dict( project="/api/v1/project/%s/" % project.pk, slug="latest", type="branch", active=True, verbose_name="latest", identifier=branch, ) try: version_data = api.version.post(version_data) except Exception as e: log.info( LOG_TEMPLATE.format(project=project.slug, version="", msg="Exception in creating version: %s" % e) ) raise e version = make_api_version(version_data) if not version_pk: # Lots of course correction. to_save = False if not version.verbose_name: version_data["verbose_name"] = "latest" to_save = True if not version.active: version_data["active"] = True to_save = True if version.identifier != branch: version_data["identifier"] = branch to_save = True if to_save: version_data["project"] = "/api/v1/version/%s/" % version_data["project"].pk api.version(version.pk).put(version_data) return version
def symlink_single_version(version): """ Link from HOME/user_builds/<project>/single_version -> HOME/user_builds/<project>/rtd-builds/<default_version>/ """ default_version = version.project.get_default_version() log.debug(LOG_TEMPLATE.format(project=version.project.slug, version=default_version, msg="Symlinking single_version")) # The single_version directory symlink = version.project.single_version_symlink_path() run_on_app_servers('mkdir -p %s' % '/'.join(symlink.split('/')[:-1])) # Where the actual docs live docs_dir = os.path.join(settings.DOCROOT, version.project.slug, 'rtd-builds', default_version) run_on_app_servers('ln -nsf %s %s' % (docs_dir, symlink))
def record_pdf(api, record, results, state, version): if not record: return None try: api.build.post(dict( state=state, project='/api/v1/project/%s/' % version.project.pk, version='/api/v1/version/%s/' % version.pk, success=results['pdf'][0] == 0, type='pdf', output=results['pdf'][1], error=results['pdf'][2], exit_code=results['pdf'][0], )) except UnicodeDecodeError, e: log.error(LOG_TEMPLATE.format(project=version.project.slug, version=version.slug, msg="Unable to post a new build"), exc_info=True)
def symlink_subprojects(version): """ Link from HOME/user_builds/project/subprojects/<project> -> HOME/user_builds/<project>/rtd-builds/ """ # Subprojects subprojects = apiv2.project(version.project.pk).subprojects.get()['subprojects'] for subproject_data in subprojects: subproject_slug = subproject_data['slug'] log.debug(LOG_TEMPLATE.format(project=version.project.slug, version=version.slug, msg="Symlinking subproject: %s" % subproject_slug)) # The directory for this specific subproject symlink = version.project.subprojects_symlink_path(subproject_slug) run_on_app_servers('mkdir -p %s' % '/'.join(symlink.split('/')[:-1])) # Where the actual docs live docs_dir = os.path.join(settings.DOCROOT, subproject_slug, 'rtd-builds') run_on_app_servers('ln -nsf %s %s' % (docs_dir, symlink))
def record_pdf(api, record, results, state, version): if not record or "sphinx" not in version.project.documentation_type: return None if not version.project.enable_pdf_build: return None try: if "pdf" in results: pdf_exit = results["pdf"][0] pdf_success = pdf_exit == 0 pdf_output = results["pdf"][1] pdf_error = results["pdf"][2] else: pdf_exit = 999 pdf_success = False pdf_output = pdf_error = "PDF Failed" pdf_output = pdf_output.decode("utf-8", "ignore") pdf_error = pdf_error.decode("utf-8", "ignore") if "Output written on" in pdf_output: pdf_success = True api.build.post( dict( state=state, project="/api/v1/project/%s/" % version.project.pk, version="/api/v1/version/%s/" % version.pk, success=pdf_success, type="pdf", output=pdf_output, error=pdf_error, exit_code=pdf_exit, builder=socket.gethostname(), ) ) except Exception: log.error( LOG_TEMPLATE.format(project=version.project.slug, version=version.slug, msg="Unable to post a new build"), exc_info=True, )
def symlink_translations(version): """ Link from HOME/user_builds/project/translations/<lang> -> HOME/user_builds/<project>/rtd-builds/ """ translations = {} if getattr(settings, 'DONT_HIT_DB', True): for trans in (api .project(version.project.pk) .translations.get()['translations']): translations[trans['language']] = trans['slug'] else: for trans in version.project.translations.all(): translations[trans.language] = trans.slug # Default language, and pointer for 'en' version_slug = version.project.slug.replace('_', '-') translations[version.project.language] = version_slug if not translations.has_key('en'): translations['en'] = version_slug run_on_app_servers( 'mkdir -p {0}' .format(os.path.join(version.project.doc_path, 'translations'))) for (language, slug) in translations.items(): log.debug(LOG_TEMPLATE.format( project=version.project.slug, version=version.slug, msg="Symlinking translation: %s->%s" % (language, slug) )) # The directory for this specific translation symlink = version.project.translations_symlink_path(language) translation_path = os.path.join(settings.DOCROOT, slug, 'rtd-builds') run_on_app_servers('ln -nsf {0} {1}'.format(translation_path, symlink))
def symlink_translations(version): """ Link from HOME/user_builds/project/translations/<lang> -> HOME/user_builds/<project>/rtd-builds/ """ translations = {} if getattr(settings, 'DONT_HIT_DB', True): for trans in (apiv2 .project(version.project.pk) .translations.get()['translations']): translations[trans['language']] = trans['slug'] else: for trans in version.project.translations.all(): translations[trans.language] = trans.slug # Default language, and pointer for 'en' version_slug = version.project.slug.replace('_', '-') translations[version.project.language] = version_slug if not translations.has_key('en'): translations['en'] = version_slug run_on_app_servers( 'mkdir -p {0}' .format(os.path.join(version.project.doc_path, 'translations'))) for (language, slug) in translations.items(): log.debug(LOG_TEMPLATE.format( project=version.project.slug, version=version.slug, msg="Symlinking translation: %s->%s" % (language, slug) )) # The directory for this specific translation symlink = version.project.translations_symlink_path(language) translation_path = os.path.join(settings.DOCROOT, slug, 'rtd-builds') run_on_app_servers('ln -nsf {0} {1}'.format(translation_path, symlink))
def record_pdf(record, results, state, version): if not record or 'sphinx' not in version.project.documentation_type: return None if not version.project.enable_pdf_build: return None try: if 'pdf' in results: pdf_exit = results['pdf'][0] pdf_success = pdf_exit == 0 pdf_output = results['pdf'][1] pdf_error = results['pdf'][2] else: pdf_exit = 999 pdf_success = False pdf_output = pdf_error = "PDF Failed" pdf_output = pdf_output.decode('utf-8', 'ignore') pdf_error = pdf_error.decode('utf-8', 'ignore') if 'Output written on' in pdf_output: pdf_success = True api_v1.build.post(dict( state=state, project='/api/v1/project/%s/' % version.project.pk, version='/api/v1/version/%s/' % version.pk, success=pdf_success, type='pdf', output=pdf_output, error=pdf_error, exit_code=pdf_exit, builder=socket.gethostname(), )) except Exception: log.error(LOG_TEMPLATE.format(project=version.project.slug, version=version.slug, msg="Unable to post a new build"), exc_info=True)
def build(self, **kwargs): project = self.version.project checkout_path = project.checkout_path(self.version.slug) site_path = os.path.join(checkout_path, 'site') os.chdir(checkout_path) # Pull mkdocs config data user_config = yaml.safe_load(open('mkdocs.yml', 'r')) docs_dir = user_config.get('docs_dir', 'docs') # Set mkdocs config values MEDIA_URL = getattr(settings, 'MEDIA_URL', 'https://media.readthedocs.org') if 'extra_javascript' in user_config: user_config['extra_javascript'].append( '%sjavascript/jquery/jquery-2.0.3.min.js' % MEDIA_URL) user_config['extra_javascript'].append('readthedocs-data.js') user_config['extra_javascript'].append( 'readthedocs-dynamic-include.js') user_config['extra_javascript'].append( '%sjavascript/readthedocs-doc-embed.js' % MEDIA_URL) else: user_config['extra_javascript'] = [ '%sjavascript/jquery/jquery-2.0.3.min.js' % MEDIA_URL, 'readthedocs-data.js', 'readthedocs-dynamic-include.js', '%sjavascript/readthedocs-doc-embed.js' % MEDIA_URL, ] if 'extra_css' in user_config: user_config['extra_css'].append( 'https://media.readthedocs.org/css/badge_only.css') user_config['extra_css'].append( 'https://media.readthedocs.org/css/readthedocs-doc-embed.css') else: user_config['extra_css'] = [ 'https://media.readthedocs.org/css/badge_only.css', 'https://media.readthedocs.org/css/readthedocs-doc-embed.css', ] if 'pages' not in user_config: user_config['pages'] = [] for root, dirnames, filenames in os.walk(docs_dir): for filename in filenames: if fnmatch.fnmatch(filename, '*.md'): full_path = os.path.join(root.replace( docs_dir, ''), filename.lstrip('/')).lstrip('/') user_config['pages'].append([full_path]) yaml.dump(user_config, open('mkdocs.yml', 'w')) # RTD javascript writing READTHEDOCS_DATA = { 'project': project.slug, 'version': self.version.slug, 'language': project.language, 'page': None, 'theme': "readthedocs", 'docroot': docs_dir, 'source_suffix': ".md", 'api_host': getattr(settings, 'SLUMBER_API_HOST', 'https://readthedocs.org'), 'commit': self.version.project.vcs_repo(self.version.slug).commit, } data_json = json.dumps(READTHEDOCS_DATA, indent=4) data_ctx = Context({ 'data_json': data_json, 'current_version': READTHEDOCS_DATA['version'], 'slug': READTHEDOCS_DATA['project'], 'html_theme': READTHEDOCS_DATA['theme'], 'pagename': None, }) data_string = template_loader.get_template( 'doc_builder/data.js.tmpl').render(data_ctx) data_file = open(os.path.join(docs_dir, 'readthedocs-data.js'), 'w+') data_file.write(data_string) data_file.close() include_ctx = Context({ 'global_analytics_code': getattr(settings, 'GLOBAL_ANALYTICS_CODE', 'UA-17997319-1'), 'user_analytics_code': project.analytics_code, }) include_string = template_loader.get_template( 'doc_builder/include.js.tmpl').render(include_ctx) include_file = open( os.path.join(docs_dir, 'readthedocs-dynamic-include.js'), 'w+') include_file.write(include_string) include_file.close() # Actual build if project.use_virtualenv: build_command = "%s build --site-dir=site --theme=mkdocs" % ( project.venv_bin(version=self.version.slug, bin='mkdocs')) else: build_command = "mkdocs build --site-dir=site --theme=mkdocs" results = run(build_command, shell=True) try: # Index Search page_list = [] log.info( LOG_TEMPLATE.format(project=self.version.project.slug, version=self.version.slug, msg='Indexing files')) for root, dirnames, filenames in os.walk(site_path): for filename in filenames: if fnmatch.fnmatch(filename, '*.html'): full_path = os.path.join(root, filename.lstrip('/')) relative_path = os.path.join( root.replace(site_path, '').lstrip('/'), filename.lstrip('/')) relative_path = re.sub('.html$', '', relative_path) html = parse_content_from_file( documentation_type='mkdocs', file_path=full_path) headers = parse_headers_from_file( documentation_type='mkdocs', file_path=full_path) sections = parse_sections_from_file( documentation_type='mkdocs', file_path=full_path) page_list.append({ 'content': html, 'path': relative_path, 'title': sections[0]['title'], 'headers': headers, 'sections': sections }) data = { 'page_list': page_list, 'version_pk': self.version.pk, 'project_pk': self.version.project.pk } log_msg = ' '.join([page['path'] for page in page_list]) log.info("(Search Index) Sending Data: %s [%s]" % (self.version.project.slug, log_msg)) apiv2.index_search.post({'data': data}) except: log.error('Search indexing failed') return results
def update_docs(pk, version_pk=None, build_pk=None, record=True, docker=False, pdf=True, man=True, epub=True, dash=True, search=True, force=False, intersphinx=True, localmedia=True, api=None, basic=False, **kwargs): """ The main entry point for updating documentation. It handles all of the logic around whether a project is imported or we created it. Then it will build the html docs and other requested parts. `pk` Primary key of the project to update `record` Whether or not to keep a record of the update in the database. Useful for preventing changes visible to the end-user when running commands from the shell, for example. """ # Dependency injection to allow for testing if api is None: api = tastyapi.api apiv2 = tastyapi.apiv2 else: apiv2 = api start_time = datetime.datetime.utcnow() try: project_data = api.project(pk).get() except HttpClientError: log.exception( LOG_TEMPLATE.format( project=pk, version='', msg='Failed to get project data on build. Erroring.')) project = make_api_project(project_data) # Don't build skipped projects if project.skip: log.info( LOG_TEMPLATE.format(project=project.slug, version='', msg='Skipping')) return else: log.info( LOG_TEMPLATE.format(project=project.slug, version='', msg='Building')) version = ensure_version(api, project, version_pk) build = create_build(build_pk) results = {} # Build Servery stuff try: record_build(api=api, build=build, record=record, results=results, state='cloning') vcs_results = setup_vcs(version, build, api) if vcs_results: results.update(vcs_results) if project.documentation_type == 'auto': update_documentation_type(version, apiv2) if docker or settings.DOCKER_ENABLE: record_build(api=api, build=build, record=record, results=results, state='building') docker = DockerEnvironment(version) build_results = docker.build() results.update(build_results) else: record_build(api=api, build=build, record=record, results=results, state='installing') setup_results = setup_environment(version) results.update(setup_results) record_build(api=api, build=build, record=record, results=results, state='building') build_results = build_docs(version, force, pdf, man, epub, dash, search, localmedia) results.update(build_results) except vcs_support_utils.LockTimeout, e: results['checkout'] = (423, "", "Version locked, retrying in 5 minutes.") log.info( LOG_TEMPLATE.format(project=version.project.slug, version=version.slug, msg="Unable to lock, will retry")) # http://celery.readthedocs.org/en/3.0/userguide/tasks.html#retrying # Should completely retry the task for us until max_retries is exceeded update_docs.retry(exc=e, throw=False)
def build_docs(version, force, pdf, man, epub, dash, search, localmedia): """ This handles the actual building of the documentation """ project = version.project results = {} before_build.send(sender=version) with project.repo_nonblockinglock(version=version, max_lock_age=getattr( settings, 'REPO_LOCK_SECONDS', 30)): html_builder = builder_loading.get(project.documentation_type)(version) if force: html_builder.force() html_builder.append_conf() results['html'] = html_builder.build() if results['html'][0] == 0: html_builder.move() # Gracefully attempt to move files via task on web workers. try: move_files.delay( version_pk=version.pk, html=True, hostname=socket.gethostname(), ) except socket.error: pass fake_results = (999, "Project Skipped, Didn't build", "Project Skipped, Didn't build") if 'mkdocs' in project.documentation_type: if search: try: search_builder = builder_loading.get('mkdocs_json')( version) results['search'] = search_builder.build() if results['search'][0] == 0: search_builder.move() except: log.error(LOG_TEMPLATE.format(project=project.slug, version=version.slug, msg="JSON Build Error"), exc_info=True) if 'sphinx' in project.documentation_type: # Search builder. Creates JSON from docs and sends it to the # server. if search: try: search_builder = builder_loading.get('sphinx_search')( version) results['search'] = search_builder.build() if results['search'][0] == 0: # Copy json for safe keeping search_builder.move() except: log.error(LOG_TEMPLATE.format(project=project.slug, version=version.slug, msg="JSON Build Error"), exc_info=True) # Local media builder for singlepage HTML download archive if localmedia: try: localmedia_builder = builder_loading.get( 'sphinx_singlehtmllocalmedia')(version) results['localmedia'] = localmedia_builder.build() if results['localmedia'][0] == 0: localmedia_builder.move() except: log.error(LOG_TEMPLATE.format( project=project.slug, version=version.slug, msg="Local Media HTML Build Error"), exc_info=True) # Optional build steps if version.project.slug not in HTML_ONLY and not project.skip: if pdf: pdf_builder = builder_loading.get('sphinx_pdf')(version) results['pdf'] = pdf_builder.build() # Always move pdf results even when there's an error. # if pdf_results[0] == 0: pdf_builder.move() else: results['pdf'] = fake_results if epub: epub_builder = builder_loading.get('sphinx_epub')(version) results['epub'] = epub_builder.build() if results['epub'][0] == 0: epub_builder.move() else: results['epub'] = fake_results after_build.send(sender=version) return results
def setup_environment(version): """ Build the virtualenv and install the project into it. Always build projects with a virtualenv. """ ret_dict = {} project = version.project build_dir = os.path.join(project.venv_path(version=version.slug), 'build') if os.path.exists(build_dir): log.info( LOG_TEMPLATE.format(project=project.slug, version=version.slug, msg='Removing existing build dir')) shutil.rmtree(build_dir) if project.use_system_packages: site_packages = '--system-site-packages' else: site_packages = '--no-site-packages' # Here the command has been modified to support different # interpreters. ret_dict['venv'] = run('{cmd} {site_packages} {path}'.format( cmd='virtualenv-2.7 -p {interpreter}'.format( interpreter=project.python_interpreter), site_packages=site_packages, path=project.venv_path(version=version.slug))) # Other code expects sphinx-build to be installed inside the # virtualenv. Using the -I option makes sure it gets installed # even if it is already installed system-wide (and # --system-site-packages is used) if project.use_system_packages: ignore_option = '-I' else: ignore_option = '' wheeldir = os.path.join(settings.SITE_ROOT, 'deploy', 'wheels') ret_dict['doc_builder'] = run(( '{cmd} install --use-wheel --find-links={wheeldir} -U {ignore_option} ' 'sphinx==1.2.2 virtualenv==1.10.1 setuptools==1.1 docutils==0.11 mkdocs==0.11.1 mock==1.0.1 pillow==2.6.1' ' readthedocs-sphinx-ext==0.4.4 sphinx-rtd-theme==0.1.6 ').format( cmd=project.venv_bin(version=version.slug, bin='pip'), ignore_option=ignore_option, wheeldir=wheeldir, )) # Handle requirements requirements_file_path = project.requirements_file checkout_path = project.checkout_path(version.slug) if not requirements_file_path: docs_dir = builder_loading.get( project.documentation_type)(version).docs_dir() for path in [docs_dir, '']: for req_file in ['pip_requirements.txt', 'requirements.txt']: test_path = os.path.join(checkout_path, path, req_file) print('Testing %s' % test_path) if os.path.exists(test_path): requirements_file_path = test_path break if requirements_file_path: os.chdir(checkout_path) ret_dict['requirements'] = run( '{cmd} install --exists-action=w -r {requirements}'.format( cmd=project.venv_bin(version=version.slug, bin='pip'), requirements=requirements_file_path)) # Handle setup.py os.chdir(project.checkout_path(version.slug)) if os.path.isfile("setup.py"): if getattr(settings, 'USE_PIP_INSTALL', False): ret_dict['install'] = run( '{cmd} install --ignore-installed .'.format( cmd=project.venv_bin(version=version.slug, bin='pip'))) else: ret_dict['install'] = run('{cmd} setup.py install --force'.format( cmd=project.venv_bin(version=version.slug, bin='python'))) else: ret_dict['install'] = (999, "", "No setup.py, skipping install") return ret_dict
def update_imported_docs(version_pk, api=None): """ Check out or update the given project's repository. """ if api is None: api = tastyapi.api version_data = api.version(version_pk).get() version = make_api_version(version_data) project = version.project ret_dict = {} # Make Dirs if not os.path.exists(project.doc_path): os.makedirs(project.doc_path) if not project.vcs_repo(): raise ProjectImportError( ("Repo type '{0}' unknown".format(project.repo_type))) with project.repo_nonblockinglock(version=version, max_lock_age=getattr( settings, 'REPO_LOCK_SECONDS', 30)): before_vcs.send(sender=version) # Get the actual code on disk if version: log.info( LOG_TEMPLATE.format( project=project.slug, version=version.slug, msg='Checking out version {slug}: {identifier}'.format( slug=version.slug, identifier=version.identifier))) version_slug = version.slug version_repo = project.vcs_repo(version_slug) ret_dict['checkout'] = version_repo.checkout(version.identifier, ) else: # Does this ever get called? log.info( LOG_TEMPLATE.format(project=project.slug, version=version.slug, msg='Updating to latest revision')) version_slug = 'latest' version_repo = project.vcs_repo(version_slug) ret_dict['checkout'] = version_repo.update() after_vcs.send(sender=version) # Update tags/version version_post_data = {'repo': version_repo.repo_url} if version_repo.supports_tags: version_post_data['tags'] = [{ 'identifier': v.identifier, 'verbose_name': v.verbose_name, } for v in version_repo.tags] if version_repo.supports_branches: version_post_data['branches'] = [{ 'identifier': v.identifier, 'verbose_name': v.verbose_name, } for v in version_repo.branches] try: apiv2.project(project.pk).sync_versions.post(version_post_data) except Exception, e: print "Sync Versions Exception: %s" % e.message
LOG_TEMPLATE.format(project=version.project.slug, version=version.slug, msg="Unable to lock, will retry")) # http://celery.readthedocs.org/en/3.0/userguide/tasks.html#retrying # Should completely retry the task for us until max_retries is exceeded update_docs.retry(exc=e, throw=False) except ProjectImportError, e: results['checkout'] = ( 404, "", 'Failed to import project; skipping build.\n\nError\n-----\n\n%s' % e.message) # Close out build in finally with error. pass except Exception, e: log.error(LOG_TEMPLATE.format(project=version.project.slug, version=version.slug, msg="Top-level Build Failure"), exc_info=True) results['checkout'] = (404, "", 'Top-level Build Failure: %s' % e.message) finally: record_build(api=api, build=build, record=record, results=results, state='finished', start_time=start_time) record_pdf(api=api, record=record, results=results, state='finished',
def build_docs(version, force, pdf, man, epub, dash, search, localmedia): """ This handles the actual building of the documentation """ project = version.project results = {} if 'sphinx' in project.documentation_type: try: project.conf_file(version.slug) except ProjectImportError: results['html'] = (999, 'Conf file not found.', '') return results with project.repo_nonblockinglock(version=version, max_lock_age=getattr( settings, 'REPO_LOCK_SECONDS', 30)): html_builder = builder_loading.get(project.documentation_type)(version) if force: html_builder.force() #html_builder.clean() if 'sphinx' in project.documentation_type: html_builder.append_conf() results['html'] = html_builder.build() if results['html'][0] == 0: html_builder.move() fake_results = (999, "Project Skipped, Didn't build", "Project Skipped, Didn't build") if 'sphinx' in project.documentation_type: # Search builder. Creates JSON from docs and sends it to the server. if search: try: search_builder = builder_loading.get('sphinx_search')( version) results['search'] = search_builder.build() if results['search'][0] == 0: # Copy json for safe keeping search_builder.move() except: log.error(LOG_TEMPLATE.format(project=project.slug, version=version.slug, msg="JSON Build Error"), exc_info=True) # Local media builder for singlepage HTML download archive if localmedia: try: localmedia_builder = builder_loading.get( 'sphinx_singlehtmllocalmedia')(version) results['localmedia'] = localmedia_builder.build() if results['localmedia'][0] == 0: localmedia_builder.move() except: log.error(LOG_TEMPLATE.format( project=project.slug, version=version.slug, msg="Local Media HTML Build Error"), exc_info=True) # Optional build steps if version.project.slug not in HTML_ONLY and not project.skip: if pdf: pdf_builder = builder_loading.get('sphinx_pdf')(version) results['pdf'] = pdf_builder.build() # Always move pdf results even when there's an error. #if pdf_results[0] == 0: pdf_builder.move() else: results['pdf'] = fake_results if epub: epub_builder = builder_loading.get('sphinx_epub')(version) results['epub'] = epub_builder.build() if results['epub'][0] == 0: epub_builder.move() else: results['epub'] = fake_results return results
def setup_environment(version): """ Build the virtualenv and install the project into it. """ ret_dict = {} project = version.project if project.use_virtualenv: build_dir = os.path.join(project.venv_path(version=version.slug), 'build') if os.path.exists(build_dir): log.info( LOG_TEMPLATE.format(project=project.slug, version=version.slug, msg='Removing existing build dir')) shutil.rmtree(build_dir) if project.use_system_packages: site_packages = '--system-site-packages' else: site_packages = '--no-site-packages' # Here the command has been modified to support different # interpreters. ret_dict['venv'] = run('{cmd} {site_packages} {path}'.format( cmd='virtualenv-2.7 -p {interpreter}'.format( interpreter=project.python_interpreter), site_packages=site_packages, path=project.venv_path(version=version.slug))) # Other code expects sphinx-build to be installed inside the # virtualenv. Using the -I option makes sure it gets installed # even if it is already installed system-wide (and # --system-site-packages is used) if project.use_system_packages: ignore_option = '-I' else: ignore_option = '' sphinx = 'sphinx==1.2.2' if project.python_interpreter != 'python3': ret_dict['sphinx'] = run(( '{cmd} install -U {ignore_option} {sphinx} ' 'virtualenv==1.10.1 setuptools==1.1 ' 'docutils==0.11 git+git://github.com/ericholscher/readthedocs-sphinx-ext#egg=readthedocs_ext' ).format(cmd=project.venv_bin(version=version.slug, bin='pip'), sphinx=sphinx, ignore_option=ignore_option)) else: # python 3 specific hax ret_dict['sphinx'] = run(( '{cmd} install -U {ignore_option} {sphinx} ' 'virtualenv==1.9.1 docutils==0.11 git+git://github.com/ericholscher/readthedocs-sphinx-ext#egg=readthedocs_ext' ).format(cmd=project.venv_bin(version=version.slug, bin='pip'), sphinx=sphinx, ignore_option=ignore_option)) if project.requirements_file: os.chdir(project.checkout_path(version.slug)) ret_dict['requirements'] = run( '{cmd} install --exists-action=w -r {requirements}'.format( cmd=project.venv_bin(version=version.slug, bin='pip'), requirements=project.requirements_file)) os.chdir(project.checkout_path(version.slug)) if os.path.isfile("setup.py"): if getattr(settings, 'USE_PIP_INSTALL', False): ret_dict['install'] = run( '{cmd} install --ignore-installed .'.format( cmd=project.venv_bin(version=version.slug, bin='pip'))) else: ret_dict['install'] = run( '{cmd} setup.py install --force'.format( cmd=project.venv_bin(version=version.slug, bin='python'))) else: ret_dict['install'] = (999, "", "No setup.py, skipping install") return ret_dict
def update_docs(pk, version_pk=None, record=True, docker=False, pdf=True, man=True, epub=True, dash=True, search=True, force=False, intersphinx=True, localmedia=True, api=None, **kwargs): """ The main entry point for updating documentation. It handles all of the logic around whether a project is imported or we created it. Then it will build the html docs and other requested parts. `pk` Primary key of the project to update `record` Whether or not to keep a record of the update in the database. Useful for preventing changes visible to the end-user when running commands from the shell, for example. """ # Dependency injection to allow for testing if api is None: api = tastyapi.api project_data = api.project(pk).get() project = make_api_project(project_data) log.info( LOG_TEMPLATE.format(project=project.slug, version='', msg='Building')) version = ensure_version(api, project, version_pk) build = create_build(version, api, record) results = {} try: record_build(api=api, build=build, record=record, results=results, state='cloning') vcs_results = setup_vcs(version, build, api) results.update(vcs_results) if docker: record_build(api=api, build=build, record=record, results=results, state='building') build_results = run_docker(version) results.update(build_results) else: record_build(api=api, build=build, record=record, results=results, state='installing') setup_results = setup_environment(version) results.update(setup_results) record_build(api=api, build=build, record=record, results=results, state='building') build_results = build_docs(version, force, pdf, man, epub, dash, search, localmedia) results.update(build_results) move_files(version, results) record_pdf(api=api, record=record, results=results, state='finished', version=version) finish_build(version=version, build=build, results=results) if results['html'][0] == 0: # Mark version active on the site version_data = api.version(version.pk).get() version_data['active'] = True version_data['built'] = True # Need to delete this because a bug in tastypie breaks on the users # list. del version_data['project'] try: api.version(version.pk).put(version_data) except Exception, e: log.error(LOG_TEMPLATE.format( project=version.project.slug, version=version.slug, msg="Unable to put a new version"), exc_info=True) except vcs_support_utils.LockTimeout, e: results['checkout'] = (999, "", "Version locked, retrying in 5 minutes.") log.info( LOG_TEMPLATE.format(project=version.project.slug, version=version.slug, msg="Unable to lock, will retry")) # http://celery.readthedocs.org/en/3.0/userguide/tasks.html#retrying # Should completely retry the task for us until max_retries is exceeded update_docs.retry(exc=e, throw=False)
# Mark version active on the site version_data = api.version(version.pk).get() version_data['active'] = True version_data['built'] = True # Need to delete this because a bug in tastypie breaks on the users # list. del version_data['project'] try: api.version(version.pk).put(version_data) except Exception, e: log.error(LOG_TEMPLATE.format(project=version.project.slug, version=version.slug, msg="Unable to put a new version"), exc_info=True) except Exception, e: log.error(LOG_TEMPLATE.format(project=version.project.slug, version=version.slug, msg="Top-level Build Failure"), exc_info=True) finally: record_build(api=api, build=build, record=record, results=results, state='finished') log.info(LOG_TEMPLATE.format(project=version.project.slug, version='', msg='Build finished')) def move_files(version, results): if results['html'][0] == 0: from_path = version.project.artifact_path(version=version.slug, type=version.project.documentation_type) target = version.project.rtd_build_path(version.slug) core_utils.copy(from_path, target) if 'sphinx' in version.project.documentation_type: if 'localmedia' in results and results['localmedia'][0] == 0: from_path = version.project.artifact_path(version=version.slug, type='sphinx_localmedia') to_path = os.path.join(settings.MEDIA_ROOT, 'htmlzip', version.project.slug, version.slug) core_utils.copy(from_path, to_path) if 'search' in results and results['search'][0] == 0: from_path = version.project.artifact_path(version=version.slug, type='sphinx_search') to_path = os.path.join(settings.MEDIA_ROOT, 'json', version.project.slug, version.slug)
def build_docs(version, force, pdf, man, epub, dash, search, localmedia): """ This handles the actual building of the documentation """ project = version.project results = {} if 'sphinx' in project.documentation_type: try: project.conf_file(version.slug) except ProjectImportError: results['html'] = (999, 'Conf file not found.', '') return results with project.repo_nonblockinglock(version=version, max_lock_age=getattr(settings, 'REPO_LOCK_SECONDS', 30)): html_builder = builder_loading.get(project.documentation_type)(version) if force: html_builder.force() # html_builder.clean() if 'sphinx' in project.documentation_type: html_builder.append_conf() results['html'] = html_builder.build() if results['html'][0] == 0: html_builder.move() fake_results = (999, "Project Skipped, Didn't build", "Project Skipped, Didn't build") if 'sphinx' in project.documentation_type: # Search builder. Creates JSON from docs and sends it to the # server. if search: try: search_builder = builder_loading.get( 'sphinx_search')(version) results['search'] = search_builder.build() if results['search'][0] == 0: # Copy json for safe keeping search_builder.move() except: log.error(LOG_TEMPLATE.format( project=project.slug, version=version.slug, msg="JSON Build Error"), exc_info=True) # Local media builder for singlepage HTML download archive if localmedia: try: localmedia_builder = builder_loading.get( 'sphinx_singlehtmllocalmedia')(version) results['localmedia'] = localmedia_builder.build() if results['localmedia'][0] == 0: localmedia_builder.move() except: log.error(LOG_TEMPLATE.format( project=project.slug, version=version.slug, msg="Local Media HTML Build Error"), exc_info=True) # Optional build steps if version.project.slug not in HTML_ONLY and not project.skip: if pdf: pdf_builder = builder_loading.get('sphinx_pdf')(version) results['pdf'] = pdf_builder.build() # Always move pdf results even when there's an error. # if pdf_results[0] == 0: pdf_builder.move() else: results['pdf'] = fake_results if epub: epub_builder = builder_loading.get('sphinx_epub')(version) results['epub'] = epub_builder.build() if results['epub'][0] == 0: epub_builder.move() else: results['epub'] = fake_results return results