class Project(models.Model): """Project model.""" # Auto fields pub_date = models.DateTimeField(_('Publication date'), auto_now_add=True) modified_date = models.DateTimeField(_('Modified date'), auto_now=True) # Generally from conf.py users = models.ManyToManyField(User, verbose_name=_('User'), related_name='projects') # A DNS label can contain up to 63 characters. name = models.CharField(_('Name'), max_length=63) slug = models.SlugField(_('Slug'), max_length=63, unique=True) description = models.TextField(_('Description'), blank=True, help_text=_('The reStructuredText ' 'description of the project')) repo = models.CharField(_('Repository URL'), max_length=255, validators=[validate_repository_url], help_text=_('Hosted documentation repository URL')) repo_type = models.CharField(_('Repository type'), max_length=10, choices=constants.REPO_CHOICES, default='git') project_url = models.URLField(_('Project homepage'), blank=True, help_text=_('The project\'s homepage')) canonical_url = models.URLField( _('Canonical URL'), blank=True, help_text=_('URL that documentation is expected to serve from')) single_version = models.BooleanField( _('Single version'), default=False, help_text=_( 'A single version site has no translations and only your ' '"latest" version, served at the root of the domain. Use ' 'this with caution, only turn it on if you will <b>never</b> ' 'have multiple versions of your docs.')) default_version = models.CharField( _('Default version'), max_length=255, default=LATEST, help_text=_('The version of your project that / redirects to')) # In default_branch, None means the backend should choose the # appropriate branch. Eg 'master' for git default_branch = models.CharField( _('Default branch'), max_length=255, default=None, null=True, blank=True, help_text=_('What branch "latest" points to. Leave empty ' 'to use the default value for your VCS (eg. ' '<code>trunk</code> or <code>master</code>).')) requirements_file = models.CharField( _('Requirements file'), max_length=255, default=None, null=True, blank=True, help_text=_( 'A <a ' 'href="https://pip.pypa.io/en/latest/user_guide.html#requirements-files">' 'pip requirements file</a> needed to build your documentation. ' 'Path from the root of your project.')) documentation_type = models.CharField( _('Documentation type'), max_length=20, choices=constants.DOCUMENTATION_CHOICES, default='sphinx', help_text=_( 'Type of documentation you are building. <a href="' 'http://www.sphinx-doc.org/en/stable/builders.html#sphinx.builders.html.' 'DirectoryHTMLBuilder">More info</a>.')) # Project features cdn_enabled = models.BooleanField(_('CDN Enabled'), default=False) analytics_code = models.CharField( _('Analytics code'), max_length=50, null=True, blank=True, help_text=_('Google Analytics Tracking ID ' '(ex. <code>UA-22345342-1</code>). ' 'This may slow down your page loads.')) container_image = models.CharField(_('Alternative container image'), max_length=64, null=True, blank=True) container_mem_limit = models.CharField( _('Container memory limit'), max_length=10, null=True, blank=True, help_text=_('Memory limit in Docker format ' '-- example: <code>512m</code> or <code>1g</code>')) container_time_limit = models.IntegerField( _('Container time limit in seconds'), null=True, blank=True, ) build_queue = models.CharField(_('Alternate build queue id'), max_length=32, null=True, blank=True) allow_promos = models.BooleanField( _('Allow paid advertising'), default=True, help_text=_('If unchecked, users will still see community ads.')) ad_free = models.BooleanField( _('Ad-free'), default=False, help_text='If checked, do not show advertising for this project', ) show_version_warning = models.BooleanField( _('Show version warning'), default=False, help_text=_('Show warning banner in non-stable nor latest versions.')) # Sphinx specific build options. enable_epub_build = models.BooleanField( _('Enable EPUB build'), default=True, help_text=_( 'Create a EPUB version of your documentation with each build.')) enable_pdf_build = models.BooleanField( _('Enable PDF build'), default=True, help_text=_( 'Create a PDF version of your documentation with each build.')) # Other model data. path = models.CharField(_('Path'), max_length=255, editable=False, help_text=_('The directory where ' '<code>conf.py</code> lives')) conf_py_file = models.CharField( _('Python configuration file'), max_length=255, default='', blank=True, help_text=_('Path from project root to <code>conf.py</code> file ' '(ex. <code>docs/conf.py</code>). ' 'Leave blank if you want us to find it for you.')) featured = models.BooleanField(_('Featured'), default=False) skip = models.BooleanField(_('Skip'), default=False) install_project = models.BooleanField( _('Install Project'), help_text=_( 'Install your project inside a virtualenv using <code>setup.py ' 'install</code>'), default=False) # This model attribute holds the python interpreter used to create the # virtual environment python_interpreter = models.CharField( _('Python Interpreter'), max_length=20, choices=constants.PYTHON_CHOICES, default='python', help_text=_('The Python interpreter used to create the virtual ' 'environment.')) use_system_packages = models.BooleanField( _('Use system packages'), help_text=_('Give the virtual environment access to the global ' 'site-packages dir.'), default=False) privacy_level = models.CharField( _('Privacy Level'), max_length=20, choices=constants.PRIVACY_CHOICES, default=getattr(settings, 'DEFAULT_PRIVACY_LEVEL', 'public'), help_text=_('Level of privacy that you want on the repository. ' 'Protected means public but not in listings.')) version_privacy_level = models.CharField( _('Version Privacy Level'), max_length=20, choices=constants.PRIVACY_CHOICES, default=getattr(settings, 'DEFAULT_PRIVACY_LEVEL', 'public'), help_text=_('Default level of privacy you want on built ' 'versions of documentation.')) # Subprojects related_projects = models.ManyToManyField( 'self', verbose_name=_('Related projects'), blank=True, symmetrical=False, through=ProjectRelationship) # Language bits language = models.CharField(_('Language'), max_length=20, default='en', help_text=_( 'The language the project ' 'documentation is rendered in. ' "Note: this affects your project's URL."), choices=constants.LANGUAGES) programming_language = models.CharField( _('Programming Language'), max_length=20, default='words', help_text=_( 'The primary programming language the project is written in.'), choices=constants.PROGRAMMING_LANGUAGES, blank=True) # A subproject pointed at its main language, so it can be tracked main_language_project = models.ForeignKey('self', related_name='translations', on_delete=models.SET_NULL, blank=True, null=True) has_valid_webhook = models.BooleanField( default=False, help_text=_('This project has been built with a webhook')) has_valid_clone = models.BooleanField( default=False, help_text=_('This project has been successfully cloned')) tags = TaggableManager(blank=True) objects = ProjectQuerySet.as_manager() all_objects = models.Manager() class Meta(object): ordering = ('slug', ) permissions = ( # Translators: Permission around whether a user can view the # project ('view_project', _('View Project')), ) def __str__(self): return self.name def save(self, *args, **kwargs): # pylint: disable=arguments-differ from readthedocs.projects import tasks first_save = self.pk is None if not self.slug: # Subdomains can't have underscores in them. self.slug = slugify(self.name) if not self.slug: raise Exception(_('Model must have slug')) if self.documentation_type == 'auto': # This used to determine the type and automatically set the # documentation type to Sphinx for rST and Mkdocs for markdown. # It now just forces Sphinx, due to markdown support. self.documentation_type = 'sphinx' super(Project, self).save(*args, **kwargs) for owner in self.users.all(): assign('view_project', owner, self) try: latest = self.versions.filter(slug=LATEST).first() default_branch = self.get_default_branch() if latest and latest.identifier != default_branch: latest.identifier = default_branch latest.save() except Exception: log.exception('Failed to update latest identifier') try: if not first_save: log.info( 'Re-symlinking project and subprojects: project=%s', self.slug, ) broadcast( type='app', task=tasks.symlink_project, args=[self.pk], ) log.info( 'Re-symlinking superprojects: project=%s', self.slug, ) for relationship in self.superprojects.all(): broadcast( type='app', task=tasks.symlink_project, args=[relationship.parent.pk], ) except Exception: log.exception('failed to symlink project') try: if not first_save: broadcast( type='app', task=tasks.update_static_metadata, args=[self.pk], ) except Exception: log.exception('failed to update static metadata') try: branch = self.default_branch or self.vcs_repo().fallback_branch if not self.versions.filter(slug=LATEST).exists(): self.versions.create_latest(identifier=branch) except Exception: log.exception('Error creating default branches') def get_absolute_url(self): return reverse('projects_detail', args=[self.slug]) def get_docs_url(self, version_slug=None, lang_slug=None, private=None): """ Return a URL for the docs. Always use http for now, to avoid content warnings. """ return resolve(project=self, version_slug=version_slug, language=lang_slug, private=private) def get_builds_url(self): return reverse('builds_project_list', kwargs={ 'project_slug': self.slug, }) def get_canonical_url(self): if getattr(settings, 'DONT_HIT_DB', True): return api.project(self.pk).canonical_url().get()['url'] return self.get_docs_url() def get_subproject_urls(self): """ List subproject URLs. This is used in search result linking """ if getattr(settings, 'DONT_HIT_DB', True): return [(proj['slug'], proj['canonical_url']) for proj in ( api.project(self.pk).subprojects().get()['subprojects'])] return [(proj.child.slug, proj.child.get_docs_url()) for proj in self.subprojects.all()] def get_production_media_path(self, type_, version_slug, include_file=True): """ Used to see if these files exist so we can offer them for download. :param type_: Media content type, ie - 'pdf', 'zip' :param version_slug: Project version slug for lookup :param include_file: Include file name in return :type include_file: bool :returns: Full path to media file or path """ if getattr(settings, 'DEFAULT_PRIVACY_LEVEL', 'public') == 'public' or settings.DEBUG: path = os.path.join(settings.MEDIA_ROOT, type_, self.slug, version_slug) else: path = os.path.join(settings.PRODUCTION_MEDIA_ARTIFACTS, type_, self.slug, version_slug) if include_file: path = os.path.join( path, '%s.%s' % (self.slug, type_.replace('htmlzip', 'zip'))) return path def get_production_media_url(self, type_, version_slug, full_path=True): """Get the URL for downloading a specific media file.""" try: path = reverse('project_download_media', kwargs={ 'project_slug': self.slug, 'type_': type_, 'version_slug': version_slug, }) except NoReverseMatch: return '' if full_path: path = '//%s%s' % (settings.PRODUCTION_DOMAIN, path) return path def subdomain(self): """Get project subdomain from resolver.""" return resolve_domain(self) def get_downloads(self): downloads = {} downloads['htmlzip'] = self.get_production_media_url( 'htmlzip', self.get_default_version()) downloads['epub'] = self.get_production_media_url( 'epub', self.get_default_version()) downloads['pdf'] = self.get_production_media_url( 'pdf', self.get_default_version()) return downloads @property def clean_repo(self): if self.repo.startswith('http://github.com'): return self.repo.replace('http://github.com', 'https://github.com') return self.repo # Doc PATH: # MEDIA_ROOT/slug/checkouts/version/<repo> @property def doc_path(self): return os.path.join(settings.DOCROOT, self.slug.replace('_', '-')) def checkout_path(self, version=LATEST): return os.path.join(self.doc_path, 'checkouts', version) @property def pip_cache_path(self): """Path to pip cache.""" if getattr(settings, 'GLOBAL_PIP_CACHE', False) and settings.DEBUG: return settings.GLOBAL_PIP_CACHE return os.path.join(self.doc_path, '.cache', 'pip') # # Paths for symlinks in project doc_path. # def translations_symlink_path(self, language=None): """Path in the doc_path that we symlink translations.""" if not language: language = self.language return os.path.join(self.doc_path, 'translations', language) # # End symlink paths # def full_doc_path(self, version=LATEST): """The path to the documentation root in the project.""" doc_base = self.checkout_path(version) for possible_path in ['docs', 'doc', 'Doc']: if os.path.exists(os.path.join(doc_base, '%s' % possible_path)): return os.path.join(doc_base, '%s' % possible_path) # No docs directory, docs are at top-level. return doc_base def artifact_path(self, type_, version=LATEST): """The path to the build html docs in the project.""" return os.path.join(self.doc_path, 'artifacts', version, type_) def full_build_path(self, version=LATEST): """The path to the build html docs in the project.""" return os.path.join(self.conf_dir(version), '_build', 'html') def full_latex_path(self, version=LATEST): """The path to the build LaTeX docs in the project.""" return os.path.join(self.conf_dir(version), '_build', 'latex') def full_epub_path(self, version=LATEST): """The path to the build epub docs in the project.""" return os.path.join(self.conf_dir(version), '_build', 'epub') # There is currently no support for building man/dash formats, but we keep # the support there for existing projects. They might have already existing # legacy builds. def full_man_path(self, version=LATEST): """The path to the build man docs in the project.""" return os.path.join(self.conf_dir(version), '_build', 'man') def full_dash_path(self, version=LATEST): """The path to the build dash docs in the project.""" return os.path.join(self.conf_dir(version), '_build', 'dash') def full_json_path(self, version=LATEST): """The path to the build json docs in the project.""" if 'sphinx' in self.documentation_type: return os.path.join(self.conf_dir(version), '_build', 'json') if 'mkdocs' in self.documentation_type: return os.path.join(self.checkout_path(version), '_build', 'json') def full_singlehtml_path(self, version=LATEST): """The path to the build singlehtml docs in the project.""" return os.path.join(self.conf_dir(version), '_build', 'singlehtml') def rtd_build_path(self, version=LATEST): """The destination path where the built docs are copied.""" return os.path.join(self.doc_path, 'rtd-builds', version) def static_metadata_path(self): """The path to the static metadata JSON settings file.""" return os.path.join(self.doc_path, 'metadata.json') def conf_file(self, version=LATEST): """Find a ``conf.py`` file in the project checkout.""" if self.conf_py_file: conf_path = os.path.join( self.checkout_path(version), self.conf_py_file, ) if os.path.exists(conf_path): log.info('Inserting conf.py file path from model') return conf_path log.warning("Conf file specified on model doesn't exist") files = self.find('conf.py', version) if not files: files = self.full_find('conf.py', version) if len(files) == 1: return files[0] for filename in files: # When multiples conf.py files, we look up the first one that # contains the `doc` word in its path and return this one if filename.find('doc', 70) != -1: return filename # If the project has more than one conf.py file but none of them have # the `doc` word in the path, we raise an error informing this to the user if len(files) > 1: raise ProjectConfigurationError( ProjectConfigurationError.MULTIPLE_CONF_FILES) raise ProjectConfigurationError(ProjectConfigurationError.NOT_FOUND) def conf_dir(self, version=LATEST): conf_file = self.conf_file(version) if conf_file: return os.path.dirname(conf_file) @property def is_imported(self): return bool(self.repo) @property def has_good_build(self): return self.builds.filter(success=True).exists() @property def has_versions(self): return self.versions.exists() @property def has_aliases(self): return self.aliases.exists() def has_pdf(self, version_slug=LATEST): if not self.enable_pdf_build: return False return os.path.exists( self.get_production_media_path(type_='pdf', version_slug=version_slug)) def has_epub(self, version_slug=LATEST): if not self.enable_epub_build: return False return os.path.exists( self.get_production_media_path(type_='epub', version_slug=version_slug)) def has_htmlzip(self, version_slug=LATEST): return os.path.exists( self.get_production_media_path(type_='htmlzip', version_slug=version_slug)) @property def sponsored(self): return False def vcs_repo(self, version=LATEST, environment=None): """ Return a Backend object for this project able to handle VCS commands. :param environment: environment to run the commands :type environment: doc_builder.environments.BuildEnvironment :param version: version slug for the backend (``LATEST`` by default) :type version: str """ # TODO: this seems to be the only method that receives a # ``version.slug`` instead of a ``Version`` instance (I prefer an # instance here) backend = backend_cls.get(self.repo_type) if not backend: repo = None else: repo = backend(self, version, environment) return repo def repo_nonblockinglock(self, version, max_lock_age=None): """ Return a ``NonBlockingLock`` to acquire the lock via context manager. :param version: project's version that want to get the lock for. :param max_lock_age: time (in seconds) to consider the lock's age is old and grab it anyway. It default to the ``container_time_limit`` of the project or the default ``DOCKER_LIMITS['time']`` or ``REPO_LOCK_SECONDS`` or 30 """ if max_lock_age is None: max_lock_age = (self.container_time_limit or getattr( settings, 'DOCKER_LIMITS', {}).get('time') or getattr(settings, 'REPO_LOCK_SECONDS', 30)) return NonBlockingLock( project=self, version=version, max_lock_age=max_lock_age, ) def repo_lock(self, version, timeout=5, polling_interval=5): return Lock(self, version, timeout, polling_interval) def find(self, filename, version): """ Find files inside the project's ``doc`` path. :param filename: Filename to search for in project checkout :param version: Version instance to set version checkout path """ matches = [] for root, __, filenames in os.walk(self.full_doc_path(version)): for match in fnmatch.filter(filenames, filename): matches.append(os.path.join(root, match)) return matches def full_find(self, filename, version): """ Find files inside a project's checkout path. :param filename: Filename to search for in project checkout :param version: Version instance to set version checkout path """ matches = [] for root, __, filenames in os.walk(self.checkout_path(version)): for match in fnmatch.filter(filenames, filename): matches.append(os.path.join(root, match)) return matches def get_latest_build(self, finished=True): """ Get latest build for project. :param finished: Return only builds that are in a finished state """ kwargs = {'type': 'html'} if finished: kwargs['state'] = 'finished' return self.builds.filter(**kwargs).first() def api_versions(self): from readthedocs.builds.models import APIVersion ret = [] for version_data in api.project( self.pk).active_versions.get()['versions']: version = APIVersion(**version_data) ret.append(version) return sort_version_aware(ret) def active_versions(self): from readthedocs.builds.models import Version versions = Version.objects.public(project=self, only_active=True) return (versions.filter(built=True, active=True) | versions.filter(active=True, uploaded=True)) def ordered_active_versions(self, user=None): from readthedocs.builds.models import Version kwargs = { 'project': self, 'only_active': True, } if user: kwargs['user'] = user versions = Version.objects.public(**kwargs) return sort_version_aware(versions) def all_active_versions(self): """ Get queryset with all active versions. .. note:: This is a temporary workaround for activate_versions filtering out things that were active, but failed to build :returns: :py:class:`Version` queryset """ return self.versions.filter(active=True) def get_stable_version(self): return self.versions.filter(slug=STABLE).first() def update_stable_version(self): """ Returns the version that was promoted to be the new stable version. Return ``None`` if no update was mode or if there is no version on the project that can be considered stable. """ versions = self.versions.all() new_stable = determine_stable_version(versions) if new_stable: current_stable = self.get_stable_version() if current_stable: identifier_updated = (new_stable.identifier != current_stable.identifier) if identifier_updated and current_stable.active and current_stable.machine: log.info( 'Update stable version: {project}:{version}'.format( project=self.slug, version=new_stable.identifier)) current_stable.identifier = new_stable.identifier current_stable.save() return new_stable else: log.info( 'Creating new stable version: {project}:{version}'.format( project=self.slug, version=new_stable.identifier)) current_stable = self.versions.create_stable( type=new_stable.type, identifier=new_stable.identifier) return new_stable def versions_from_branch_name(self, branch): return (self.versions.filter(identifier=branch) | self.versions.filter(identifier='remotes/origin/%s' % branch) | self.versions.filter(identifier='origin/%s' % branch) | self.versions.filter(verbose_name=branch)) def get_default_version(self): """ Get the default version (slug). Returns self.default_version if the version with that slug actually exists (is built and published). Otherwise returns 'latest'. """ # latest is a special case where we don't have to check if it exists if self.default_version == LATEST: return self.default_version # check if the default_version exists version_qs = self.versions.filter(slug=self.default_version, active=True) if version_qs.exists(): return self.default_version return LATEST def get_default_branch(self): """Get the version representing 'latest'.""" if self.default_branch: return self.default_branch return self.vcs_repo().fallback_branch def add_subproject(self, child, alias=None): subproject, __ = ProjectRelationship.objects.get_or_create( parent=self, child=child, alias=alias, ) return subproject def remove_subproject(self, child): ProjectRelationship.objects.filter(parent=self, child=child).delete() @property def features(self): return Feature.objects.for_project(self) def has_feature(self, feature_id): """ Does project have existing feature flag. If the feature has a historical True value before the feature was added, we consider the project to have the flag. This is used for deprecating a feature or changing behavior for new projects """ return self.features.filter(feature_id=feature_id).exists() def get_feature_value(self, feature, positive, negative): """ Look up project feature, return corresponding value. If a project has a feature, return ``positive``, otherwise return ``negative`` """ return positive if self.has_feature(feature) else negative @property def show_advertising(self): """ Whether this project is ad-free :returns: ``True`` if advertising should be shown and ``False`` otherwise :rtype: bool """ if self.ad_free or self.gold_owners.exists(): return False return True @property def environment_variables(self): """ Environment variables to build this particular project. :returns: dictionary with all the variables {name: value} :rtype: dict """ return { variable.name: variable.value for variable in self.environmentvariable_set.all() }
class Project(models.Model): """Project model.""" # Auto fields pub_date = models.DateTimeField(_('Publication date'), auto_now_add=True) modified_date = models.DateTimeField(_('Modified date'), auto_now=True) # Generally from conf.py users = models.ManyToManyField(User, verbose_name=_('User'), related_name='projects') name = models.CharField(_('Name'), max_length=255) slug = models.SlugField(_('Slug'), max_length=255, unique=True) description = models.TextField(_('Description'), blank=True, help_text=_('The reStructuredText ' 'description of the project')) repo = models.CharField(_('Repository URL'), max_length=255, help_text=_('Hosted documentation repository URL')) repo_type = models.CharField(_('Repository type'), max_length=10, choices=constants.REPO_CHOICES, default='git') project_url = models.URLField(_('Project homepage'), blank=True, help_text=_('The project\'s homepage')) canonical_url = models.URLField(_('Canonical URL'), blank=True, help_text=_('URL that documentation is expected to serve from')) version = models.CharField(_('Version'), max_length=100, blank=True, help_text=_('Project version these docs apply ' 'to, i.e. 1.0a')) copyright = models.CharField(_('Copyright'), max_length=255, blank=True, help_text=_('Project copyright information')) theme = models.CharField( _('Theme'), max_length=20, choices=constants.DEFAULT_THEME_CHOICES, default=constants.THEME_DEFAULT, help_text=(u'<a href="http://sphinx.pocoo.org/theming.html#builtin-' 'themes" target="_blank">%s</a>') % _('Examples')) suffix = models.CharField(_('Suffix'), max_length=10, editable=False, default='.rst') single_version = models.BooleanField( _('Single version'), default=False, help_text=_('A single version site has no translations and only your ' '"latest" version, served at the root of the domain. Use ' 'this with caution, only turn it on if you will <b>never</b> ' 'have multiple versions of your docs.')) default_version = models.CharField( _('Default version'), max_length=255, default=LATEST, help_text=_('The version of your project that / redirects to')) # In default_branch, None means the backend should choose the # appropriate branch. Eg 'master' for git default_branch = models.CharField( _('Default branch'), max_length=255, default=None, null=True, blank=True, help_text=_('What branch "latest" points to. Leave empty ' 'to use the default value for your VCS (eg. ' '<code>trunk</code> or <code>master</code>).')) requirements_file = models.CharField( _('Requirements file'), max_length=255, default=None, null=True, blank=True, help_text=_( 'A <a ' 'href="https://pip.pypa.io/en/latest/user_guide.html#requirements-files">' 'pip requirements file</a> needed to build your documentation. ' 'Path from the root of your project.')) documentation_type = models.CharField( _('Documentation type'), max_length=20, choices=constants.DOCUMENTATION_CHOICES, default='sphinx', help_text=_('Type of documentation you are building. <a href="http://' 'sphinx-doc.org/builders.html#sphinx.builders.html.' 'DirectoryHTMLBuilder">More info</a>.')) # Project features allow_comments = models.BooleanField(_('Allow Comments'), default=False) comment_moderation = models.BooleanField(_('Comment Moderation'), default=False) cdn_enabled = models.BooleanField(_('CDN Enabled'), default=False) analytics_code = models.CharField( _('Analytics code'), max_length=50, null=True, blank=True, help_text=_("Google Analytics Tracking ID " "(ex. <code>UA-22345342-1</code>). " "This may slow down your page loads.")) container_image = models.CharField( _('Alternative container image'), max_length=64, null=True, blank=True) container_mem_limit = models.CharField( _('Container memory limit'), max_length=10, null=True, blank=True, help_text=_("Memory limit in Docker format " "-- example: <code>512m</code> or <code>1g</code>")) container_time_limit = models.CharField( _('Container time limit'), max_length=10, null=True, blank=True) build_queue = models.CharField( _('Alternate build queue id'), max_length=32, null=True, blank=True) allow_promos = models.BooleanField( _('Allow paid advertising'), default=True, help_text=_( "If unchecked, users will still see community ads.")) # Sphinx specific build options. enable_epub_build = models.BooleanField( _('Enable EPUB build'), default=True, help_text=_( 'Create a EPUB version of your documentation with each build.')) enable_pdf_build = models.BooleanField( _('Enable PDF build'), default=True, help_text=_( 'Create a PDF version of your documentation with each build.')) # Other model data. path = models.CharField(_('Path'), max_length=255, editable=False, help_text=_("The directory where " "<code>conf.py</code> lives")) conf_py_file = models.CharField( _('Python configuration file'), max_length=255, default='', blank=True, help_text=_('Path from project root to <code>conf.py</code> file ' '(ex. <code>docs/conf.py</code>). ' 'Leave blank if you want us to find it for you.')) featured = models.BooleanField(_('Featured'), default=False) skip = models.BooleanField(_('Skip'), default=False) mirror = models.BooleanField(_('Mirror'), default=False) install_project = models.BooleanField( _('Install Project'), help_text=_("Install your project inside a virtualenv using <code>setup.py " "install</code>"), default=False ) # This model attribute holds the python interpreter used to create the # virtual environment python_interpreter = models.CharField( _('Python Interpreter'), max_length=20, choices=constants.PYTHON_CHOICES, default='python', help_text=_("(Beta) The Python interpreter used to create the virtual " "environment.")) use_system_packages = models.BooleanField( _('Use system packages'), help_text=_("Give the virtual environment access to the global " "site-packages dir."), default=False ) django_packages_url = models.CharField(_('Django Packages URL'), max_length=255, blank=True) privacy_level = models.CharField( _('Privacy Level'), max_length=20, choices=constants.PRIVACY_CHOICES, default=getattr(settings, 'DEFAULT_PRIVACY_LEVEL', 'public'), help_text=_("(Beta) Level of privacy that you want on the repository. " "Protected means public but not in listings.")) version_privacy_level = models.CharField( _('Version Privacy Level'), max_length=20, choices=constants.PRIVACY_CHOICES, default=getattr( settings, 'DEFAULT_PRIVACY_LEVEL', 'public'), help_text=_("(Beta) Default level of privacy you want on built " "versions of documentation.")) # Subprojects related_projects = models.ManyToManyField( 'self', verbose_name=_('Related projects'), blank=True, symmetrical=False, through=ProjectRelationship) # Language bits language = models.CharField(_('Language'), max_length=20, default='en', help_text=_("The language the project " "documentation is rendered in. " "Note: this affects your project's URL."), choices=constants.LANGUAGES) programming_language = models.CharField( _('Programming Language'), max_length=20, default='words', help_text=_("The primary programming language the project is written in."), choices=constants.PROGRAMMING_LANGUAGES, blank=True) # A subproject pointed at its main language, so it can be tracked main_language_project = models.ForeignKey('self', related_name='translations', on_delete=models.SET_NULL, blank=True, null=True) # Version State num_major = models.IntegerField( _('Number of Major versions'), default=2, null=True, blank=True, help_text=_("2 means supporting 3.X.X and 2.X.X, but not 1.X.X") ) num_minor = models.IntegerField( _('Number of Minor versions'), default=2, null=True, blank=True, help_text=_("2 means supporting 2.2.X and 2.1.X, but not 2.0.X") ) num_point = models.IntegerField( _('Number of Point versions'), default=2, null=True, blank=True, help_text=_("2 means supporting 2.2.2 and 2.2.1, but not 2.2.0") ) has_valid_webhook = models.BooleanField( default=False, help_text=_('This project has been built with a webhook') ) has_valid_clone = models.BooleanField( default=False, help_text=_('This project has been successfully cloned') ) tags = TaggableManager(blank=True) objects = ProjectQuerySet.as_manager() all_objects = models.Manager() class Meta(object): ordering = ('slug',) permissions = ( # Translators: Permission around whether a user can view the # project ('view_project', _('View Project')), ) def __str__(self): return self.name def sync_supported_versions(self): supported = self.supported_versions() if supported: self.versions.filter( verbose_name__in=supported).update(supported=True) self.versions.exclude( verbose_name__in=supported).update(supported=False) self.versions.filter(verbose_name=LATEST_VERBOSE_NAME).update(supported=True) def save(self, *args, **kwargs): # pylint: disable=arguments-differ from readthedocs.projects import tasks first_save = self.pk is None if not self.slug: # Subdomains can't have underscores in them. self.slug = slugify(self.name) if self.slug == '': raise Exception(_("Model must have slug")) super(Project, self).save(*args, **kwargs) for owner in self.users.all(): assign('view_project', owner, self) try: if self.default_branch: latest = self.versions.get(slug=LATEST) if latest.identifier != self.default_branch: latest.identifier = self.default_branch latest.save() except Exception: log.exception('Failed to update latest identifier') # Add exceptions here for safety try: self.sync_supported_versions() except Exception: log.exception('failed to sync supported versions') try: if not first_save: broadcast(type='app', task=tasks.symlink_project, args=[self.pk]) except Exception: log.exception('failed to symlink project') try: if not first_save: broadcast(type='app', task=tasks.update_static_metadata, args=[self.pk]) except Exception: log.exception('failed to update static metadata') try: branch = self.default_branch or self.vcs_repo().fallback_branch if not self.versions.filter(slug=LATEST).exists(): self.versions.create_latest(identifier=branch) except Exception: log.exception('Error creating default branches') def get_absolute_url(self): return reverse('projects_detail', args=[self.slug]) def get_docs_url(self, version_slug=None, lang_slug=None, private=None): """ Return a URL for the docs. Always use http for now, to avoid content warnings. """ return resolve(project=self, version_slug=version_slug, language=lang_slug, private=private) def get_builds_url(self): return reverse('builds_project_list', kwargs={ 'project_slug': self.slug, }) def get_canonical_url(self): if getattr(settings, 'DONT_HIT_DB', True): return api.project(self.pk).canonical_url().get()['url'] return self.get_docs_url() def get_subproject_urls(self): """ List subproject URLs. This is used in search result linking """ if getattr(settings, 'DONT_HIT_DB', True): return [(proj['slug'], proj['canonical_url']) for proj in ( api.project(self.pk) .subprojects() .get()['subprojects'])] return [(proj.child.slug, proj.child.get_docs_url()) for proj in self.subprojects.all()] def get_production_media_path(self, type_, version_slug, include_file=True): """ Used to see if these files exist so we can offer them for download. :param type_: Media content type, ie - 'pdf', 'zip' :param version_slug: Project version slug for lookup :param include_file: Include file name in return :type include_file: bool :returns: Full path to media file or path """ if getattr(settings, 'DEFAULT_PRIVACY_LEVEL', 'public') == 'public' or settings.DEBUG: path = os.path.join( settings.MEDIA_ROOT, type_, self.slug, version_slug) else: path = os.path.join( settings.PRODUCTION_MEDIA_ARTIFACTS, type_, self.slug, version_slug) if include_file: path = os.path.join( path, '%s.%s' % (self.slug, type_.replace('htmlzip', 'zip'))) return path def get_production_media_url(self, type_, version_slug, full_path=True): """Get the URL for downloading a specific media file.""" try: path = reverse('project_download_media', kwargs={ 'project_slug': self.slug, 'type_': type_, 'version_slug': version_slug, }) except NoReverseMatch: return '' if full_path: path = '//%s%s' % (settings.PRODUCTION_DOMAIN, path) return path def subdomain(self): """Get project subdomain from resolver.""" return resolve_domain(self) def get_downloads(self): downloads = {} downloads['htmlzip'] = self.get_production_media_url( 'htmlzip', self.get_default_version()) downloads['epub'] = self.get_production_media_url( 'epub', self.get_default_version()) downloads['pdf'] = self.get_production_media_url( 'pdf', self.get_default_version()) return downloads @property def clean_repo(self): if self.repo.startswith('http://github.com'): return self.repo.replace('http://github.com', 'https://github.com') return self.repo # Doc PATH: # MEDIA_ROOT/slug/checkouts/version/<repo> @property def doc_path(self): return os.path.join(settings.DOCROOT, self.slug.replace('_', '-')) def checkout_path(self, version=LATEST): return os.path.join(self.doc_path, 'checkouts', version) @property def pip_cache_path(self): """Path to pip cache.""" if getattr(settings, 'GLOBAL_PIP_CACHE', False): return settings.GLOBAL_PIP_CACHE return os.path.join(self.doc_path, '.cache', 'pip') # # Paths for symlinks in project doc_path. # def translations_symlink_path(self, language=None): """Path in the doc_path that we symlink translations.""" if not language: language = self.language return os.path.join(self.doc_path, 'translations', language) # # End symlink paths # def full_doc_path(self, version=LATEST): """The path to the documentation root in the project.""" doc_base = self.checkout_path(version) for possible_path in ['docs', 'doc', 'Doc']: if os.path.exists(os.path.join(doc_base, '%s' % possible_path)): return os.path.join(doc_base, '%s' % possible_path) # No docs directory, docs are at top-level. return doc_base def artifact_path(self, type_, version=LATEST): """The path to the build html docs in the project.""" return os.path.join(self.doc_path, "artifacts", version, type_) def full_build_path(self, version=LATEST): """The path to the build html docs in the project.""" return os.path.join(self.conf_dir(version), "_build", "html") def full_latex_path(self, version=LATEST): """The path to the build LaTeX docs in the project.""" return os.path.join(self.conf_dir(version), "_build", "latex") def full_epub_path(self, version=LATEST): """The path to the build epub docs in the project.""" return os.path.join(self.conf_dir(version), "_build", "epub") # There is currently no support for building man/dash formats, but we keep # the support there for existing projects. They might have already existing # legacy builds. def full_man_path(self, version=LATEST): """The path to the build man docs in the project.""" return os.path.join(self.conf_dir(version), "_build", "man") def full_dash_path(self, version=LATEST): """The path to the build dash docs in the project.""" return os.path.join(self.conf_dir(version), "_build", "dash") def full_json_path(self, version=LATEST): """The path to the build json docs in the project.""" if 'sphinx' in self.documentation_type: return os.path.join(self.conf_dir(version), "_build", "json") elif 'mkdocs' in self.documentation_type: return os.path.join(self.checkout_path(version), "_build", "json") def full_singlehtml_path(self, version=LATEST): """The path to the build singlehtml docs in the project.""" return os.path.join(self.conf_dir(version), "_build", "singlehtml") def rtd_build_path(self, version=LATEST): """The destination path where the built docs are copied.""" return os.path.join(self.doc_path, 'rtd-builds', version) def static_metadata_path(self): """The path to the static metadata JSON settings file.""" return os.path.join(self.doc_path, 'metadata.json') def conf_file(self, version=LATEST): """Find a ``conf.py`` file in the project checkout.""" if self.conf_py_file: conf_path = os.path.join(self.checkout_path(version), self.conf_py_file) if os.path.exists(conf_path): log.info('Inserting conf.py file path from model') return conf_path else: log.warning("Conf file specified on model doesn't exist") files = self.find('conf.py', version) if not files: files = self.full_find('conf.py', version) if len(files) == 1: return files[0] for filename in files: if filename.find('doc', 70) != -1: return filename raise ProjectConfigurationError( ProjectConfigurationError.NOT_FOUND ) def conf_dir(self, version=LATEST): conf_file = self.conf_file(version) if conf_file: return os.path.dirname(conf_file) @property def is_type_sphinx(self): """Is project type Sphinx.""" return 'sphinx' in self.documentation_type @property def is_type_mkdocs(self): """Is project type Mkdocs.""" return 'mkdocs' in self.documentation_type @property def is_imported(self): return bool(self.repo) @property def has_good_build(self): return self.builds.filter(success=True).exists() @property def has_versions(self): return self.versions.exists() @property def has_aliases(self): return self.aliases.exists() def has_pdf(self, version_slug=LATEST): if not self.enable_pdf_build: return False return os.path.exists(self.get_production_media_path( type_='pdf', version_slug=version_slug)) def has_epub(self, version_slug=LATEST): if not self.enable_epub_build: return False return os.path.exists(self.get_production_media_path( type_='epub', version_slug=version_slug)) def has_htmlzip(self, version_slug=LATEST): return os.path.exists(self.get_production_media_path( type_='htmlzip', version_slug=version_slug)) @property def sponsored(self): return False def vcs_repo(self, version=LATEST): backend = backend_cls.get(self.repo_type) if not backend: repo = None else: proj = VCSProject( self.name, self.default_branch, self.checkout_path(version), self.clean_repo) repo = backend(proj, version) return repo def repo_nonblockinglock(self, version, max_lock_age=5): return NonBlockingLock(project=self, version=version, max_lock_age=max_lock_age) def repo_lock(self, version, timeout=5, polling_interval=5): return Lock(self, version, timeout, polling_interval) def find(self, filename, version): """ Find files inside the project's ``doc`` path. :param filename: Filename to search for in project checkout :param version: Version instance to set version checkout path """ matches = [] for root, __, filenames in os.walk(self.full_doc_path(version)): for match in fnmatch.filter(filenames, filename): matches.append(os.path.join(root, match)) return matches def full_find(self, filename, version): """ Find files inside a project's checkout path. :param filename: Filename to search for in project checkout :param version: Version instance to set version checkout path """ matches = [] for root, __, filenames in os.walk(self.checkout_path(version)): for match in fnmatch.filter(filenames, filename): matches.append(os.path.join(root, match)) return matches def get_latest_build(self, finished=True): """ Get latest build for project. finished -- Return only builds that are in a finished state """ kwargs = {'type': 'html'} if finished: kwargs['state'] = 'finished' return self.builds.filter(**kwargs).first() def api_versions(self): from readthedocs.builds.models import APIVersion ret = [] for version_data in api.project(self.pk).active_versions.get()['versions']: version = APIVersion(**version_data) ret.append(version) return sort_version_aware(ret) def active_versions(self): from readthedocs.builds.models import Version versions = Version.objects.public(project=self, only_active=True) return (versions.filter(built=True, active=True) | versions.filter(active=True, uploaded=True)) def ordered_active_versions(self, user=None): from readthedocs.builds.models import Version kwargs = { 'project': self, 'only_active': True, } if user: kwargs['user'] = user versions = Version.objects.public(**kwargs) return sort_version_aware(versions) def all_active_versions(self): """ Get queryset with all active versions. .. note:: This is a temporary workaround for activate_versions filtering out things that were active, but failed to build :returns: :py:class:`Version` queryset """ return self.versions.filter(active=True) def supported_versions(self): """ Get the list of supported versions. :returns: List of version strings. """ if not self.num_major or not self.num_minor or not self.num_point: return [] version_identifiers = self.versions.values_list('verbose_name', flat=True) return version_windows( version_identifiers, major=self.num_major, minor=self.num_minor, point=self.num_point, ) def get_stable_version(self): return self.versions.filter(slug=STABLE).first() def update_stable_version(self): """ Returns the version that was promoted to be the new stable version. Return ``None`` if no update was mode or if there is no version on the project that can be considered stable. """ versions = self.versions.all() new_stable = determine_stable_version(versions) if new_stable: current_stable = self.get_stable_version() if current_stable: identifier_updated = ( new_stable.identifier != current_stable.identifier) if identifier_updated and current_stable.active and current_stable.machine: log.info( "Update stable version: {project}:{version}".format( project=self.slug, version=new_stable.identifier)) current_stable.identifier = new_stable.identifier current_stable.save() return new_stable else: log.info( "Creating new stable version: {project}:{version}".format( project=self.slug, version=new_stable.identifier)) current_stable = self.versions.create_stable( type=new_stable.type, identifier=new_stable.identifier) return new_stable def versions_from_branch_name(self, branch): return ( self.versions.filter(identifier=branch) | self.versions.filter(identifier='remotes/origin/%s' % branch) | self.versions.filter(identifier='origin/%s' % branch) ) def get_default_version(self): """ Get the default version (slug). Returns self.default_version if the version with that slug actually exists (is built and published). Otherwise returns 'latest'. """ # latest is a special case where we don't have to check if it exists if self.default_version == LATEST: return self.default_version # check if the default_version exists version_qs = self.versions.filter( slug=self.default_version, active=True ) if version_qs.exists(): return self.default_version return LATEST def get_default_branch(self): """Get the version representing 'latest'.""" if self.default_branch: return self.default_branch return self.vcs_repo().fallback_branch def add_subproject(self, child, alias=None): subproject, __ = ProjectRelationship.objects.get_or_create( parent=self, child=child, alias=alias, ) return subproject def remove_subproject(self, child): ProjectRelationship.objects.filter(parent=self, child=child).delete() return def moderation_queue(self): # non-optimal SQL warning. from readthedocs.comments.models import DocumentComment queue = [] comments = DocumentComment.objects.filter(node__project=self) for comment in comments: if not comment.has_been_approved_since_most_recent_node_change(): queue.append(comment) return queue def add_node(self, content_hash, page, version, commit): """ Add comment node. :param content_hash: Hash of node content :param page: Doc page for node :param version: Slug for project version to apply node to :type version: str :param commit: Commit that node was updated in :type commit: str """ from readthedocs.comments.models import NodeSnapshot, DocumentNode project_obj = Project.objects.get(slug=self.slug) version_obj = project_obj.versions.get(slug=version) try: NodeSnapshot.objects.get(hash=content_hash, node__project=project_obj, node__version=version_obj, node__page=page, commit=commit) return False # ie, no new node was created. except NodeSnapshot.DoesNotExist: DocumentNode.objects.create( hash=content_hash, page=page, project=project_obj, version=version_obj, commit=commit ) return True # ie, it's True that a new node was created. def add_comment(self, version_slug, page, content_hash, commit, user, text): """ Add comment to node. :param version_slug: Version slug to use for node lookup :param page: Page to attach comment to :param content_hash: Hash of content to apply comment to :param commit: Commit that updated comment :param user: :py:class:`User` instance that created comment :param text: Comment text """ from readthedocs.comments.models import DocumentNode try: node = self.nodes.from_hash(version_slug, page, content_hash) except DocumentNode.DoesNotExist: version = self.versions.get(slug=version_slug) node = self.nodes.create(version=version, page=page, hash=content_hash, commit=commit) return node.comments.create(user=user, text=text) @property def features(self): return Feature.objects.for_project(self) def has_feature(self, feature_id): """ Does project have existing feature flag. If the feature has a historical True value before the feature was added, we consider the project to have the flag. This is used for deprecating a feature or changing behavior for new projects """ return self.features.filter(feature_id=feature_id).exists() def get_feature_value(self, feature, positive, negative): """ Look up project feature, return corresponding value. If a project has a feature, return ``positive``, otherwise return ``negative`` """ return positive if self.has_feature(feature) else negative