def copy(cls, path, target, host, is_file=False, **kwargs): # pylint: disable=arguments-differ """ A better copy command that works from the webs. Respects the ``MULTIPLE_APP_SERVERS`` setting when copying. """ if not is_file: path += '/' log.info('Remote Pull %s to %s', path, target) if not is_file and not os.path.exists(target): safe_makedirs(target) if is_file: # Create containing directory if it doesn't exist directory = os.path.dirname(target) safe_makedirs(directory) # Add a slash when copying directories sync_cmd = "rsync -e 'ssh -T' -av --delete {user}@{host}:{path} {target}".format( host=host, path=path, user=settings.SYNC_USER, target=target, ) ret = os.system(sync_cmd) if ret != 0: log.debug( 'Copy error to app servers. Command: [%s] Return: [%s]', sync_cmd, ret, )
def copy(cls, path, target, host, is_file=False, **kwargs): # pylint: disable=arguments-differ """ A better copy command that works from the webs. Respects the ``MULTIPLE_APP_SERVERS`` setting when copying. """ sync_user = getattr(settings, 'SYNC_USER', getpass.getuser()) if not is_file: path += '/' log.info('Remote Pull %s to %s', path, target) if not is_file and not os.path.exists(target): safe_makedirs(target) if is_file: # Create containing directory if it doesn't exist directory = os.path.dirname(target) safe_makedirs(directory) # Add a slash when copying directories sync_cmd = "rsync -e 'ssh -T' -av --delete {user}@{host}:{path} {target}".format( host=host, path=path, user=sync_user, target=target, ) ret = os.system(sync_cmd) if ret != 0: log.debug( 'Copy error to app servers. Command: [%s] Return: [%s]', sync_cmd, ret, )
def symlink_versions(self): """ Symlink project's versions. Link from $WEB_ROOT/<project>/<language>/<version>/ -> HOME/user_builds/<project>/rtd-builds/<version> """ versions = set() version_dir = os.path.join(self.WEB_ROOT, self.project.slug, self.project.language) # Include active public versions, # as well as public versions that are built but not active, for archived versions version_queryset = self.get_version_queryset() if version_queryset.count(): if not os.path.exists(version_dir): safe_makedirs(version_dir) for version in version_queryset: self._log(u"Symlinking Version: %s" % version) symlink = os.path.join(version_dir, version.slug) docs_dir = os.path.join(settings.DOCROOT, self.project.slug, 'rtd-builds', version.slug) run(['ln', '-nsf', docs_dir, symlink]) versions.add(version.slug) # Remove old symlinks if os.path.exists(version_dir): for old_ver in os.listdir(version_dir): if old_ver not in versions: os.unlink(os.path.join(version_dir, old_ver))
def symlink_translations(self): """Symlink project translations Link from $WEB_ROOT/<project>/<language>/ -> $WEB_ROOT/<translation>/<language>/ """ translations = {} for trans in self.get_translations(): translations[trans.language] = trans.slug # Make sure the language directory is a directory language_dir = os.path.join(self.project_root, self.project.language) if os.path.islink(language_dir): os.unlink(language_dir) if not os.path.lexists(language_dir): safe_makedirs(language_dir) for (language, slug) in list(translations.items()): self._log(u"Symlinking translation: {0}->{1}".format( language, slug)) symlink = os.path.join(self.project_root, language) docs_dir = os.path.join(self.WEB_ROOT, slug, language) run(['ln', '-nsf', docs_dir, symlink]) # Remove old symlinks for lang in os.listdir(self.project_root): if (lang not in translations and lang not in ['projects', self.project.language]): to_delete = os.path.join(self.project_root, lang) if os.path.islink(to_delete): os.unlink(to_delete) else: shutil.rmtree(to_delete)
def symlink_versions(self): """Symlink project's versions Link from $WEB_ROOT/<project>/<language>/<version>/ -> HOME/user_builds/<project>/rtd-builds/<version> """ versions = set() version_dir = os.path.join(self.WEB_ROOT, self.project.slug, self.project.language) # Include active public versions, # as well as public versions that are built but not active, for archived versions version_queryset = self.get_version_queryset() if version_queryset.count(): if not os.path.exists(version_dir): safe_makedirs(version_dir) for version in version_queryset: self._log(u"Symlinking Version: %s" % version) symlink = os.path.join(version_dir, version.slug) docs_dir = os.path.join(settings.DOCROOT, self.project.slug, 'rtd-builds', version.slug) run(['ln', '-nsf', docs_dir, symlink]) versions.add(version.slug) # Remove old symlinks if os.path.exists(version_dir): for old_ver in os.listdir(version_dir): if old_ver not in versions: os.unlink(os.path.join(version_dir, old_ver))
def symlink_translations(self): """ Symlink project translations. Link from $WEB_ROOT/<project>/<language>/ -> $WEB_ROOT/<translation>/<language>/ """ translations = {} for trans in self.get_translations(): translations[trans.language] = trans.slug # Make sure the language directory is a directory language_dir = os.path.join(self.project_root, self.project.language) if os.path.islink(language_dir): os.unlink(language_dir) if not os.path.lexists(language_dir): safe_makedirs(language_dir) for (language, slug) in list(translations.items()): self._log(u"Symlinking translation: {0}->{1}".format(language, slug)) symlink = os.path.join(self.project_root, language) docs_dir = os.path.join(self.WEB_ROOT, slug, language) run(['ln', '-nsf', docs_dir, symlink]) # Remove old symlinks for lang in os.listdir(self.project_root): if (lang not in translations and lang not in ['projects', self.project.language]): to_delete = os.path.join(self.project_root, lang) if os.path.islink(to_delete): os.unlink(to_delete) else: shutil.rmtree(to_delete)
def copy(cls, path, target, host, is_file=False, **__): """ A better copy command that works from the webs. Respects the ``MULTIPLE_APP_SERVERS`` setting when copying. """ sync_user = getattr(settings, 'SYNC_USER', getpass.getuser()) if not is_file: path += "/" log.info("Remote Pull %s to %s", path, target) if not is_file and not os.path.exists(target): safe_makedirs(target) # Add a slash when copying directories sync_cmd = "rsync -e 'ssh -T' -av --delete {user}@{host}:{path} {target}".format( host=host, path=path, user=sync_user, target=target, ) ret = os.system(sync_cmd) if ret != 0: log.debug( "Copy error to app servers. Command: [%s] Return: [%s]", sync_cmd, ret, )
def symlink_versions(self): """ Symlink project's versions. Link from $WEB_ROOT/<project>/<language>/<version>/ -> HOME/user_builds/<project>/rtd-builds/<version> """ versions = set() version_dir = os.path.join(self.WEB_ROOT, self.project.slug, self.project.language) # Include active public versions, # as well as public versions that are built but not active, for archived versions version_queryset = self.get_version_queryset() if version_queryset.count(): if not os.path.exists(version_dir): safe_makedirs(version_dir) for version in version_queryset: log_msg = 'Symlinking Version: {}'.format(version) log.info( constants.LOG_TEMPLATE.format(project=self.project.slug, version='', msg=log_msg)) symlink = os.path.join(version_dir, version.slug) docs_dir = os.path.join(settings.DOCROOT, self.project.slug, 'rtd-builds', version.slug) self.environment.run('ln', '-nsf', docs_dir, symlink) versions.add(version.slug) # Remove old symlinks if os.path.exists(version_dir): for old_ver in os.listdir(version_dir): if old_ver not in versions: safe_unlink(os.path.join(version_dir, old_ver))
def symlink_subprojects(self): """Symlink project subprojects Link from $WEB_ROOT/projects/<project> -> $WEB_ROOT/<project> """ subprojects = set() rels = self.get_subprojects() if rels.count(): # Don't creat the `projects/` directory unless subprojects exist. if not os.path.exists(self.subproject_root): safe_makedirs(self.subproject_root) for rel in rels: # A mapping of slugs for the subproject URL to the actual built # documentation from_to = OrderedDict({rel.child.slug: rel.child.slug}) subprojects.add(rel.child.slug) if rel.alias: from_to[rel.alias] = rel.child.slug subprojects.add(rel.alias) for from_slug, to_slug in list(from_to.items()): self._log(u"Symlinking subproject: {0} -> {1}".format( from_slug, to_slug)) symlink = os.path.join(self.subproject_root, from_slug) docs_dir = os.path.join(self.WEB_ROOT, to_slug) symlink_dir = os.sep.join(symlink.split(os.path.sep)[:-1]) if not os.path.lexists(symlink_dir): safe_makedirs(symlink_dir) run('ln -nsf %s %s' % (docs_dir, symlink)) # Remove old symlinks if os.path.exists(self.subproject_root): for subproj in os.listdir(self.subproject_root): if subproj not in subprojects: os.unlink(os.path.join(self.subproject_root, subproj))
def copy(cls, path, target, host, is_file=False, **__): """ A better copy command that works from the webs. Respects the ``MULTIPLE_APP_SERVERS`` setting when copying. """ sync_user = getattr(settings, 'SYNC_USER', getpass.getuser()) if not is_file: path += "/" log.info("Remote Pull %s to %s", path, target) if not is_file and not os.path.exists(target): safe_makedirs(target) # Add a slash when copying directories sync_cmd = "rsync -e 'ssh -T' -av --delete {user}@{host}:{path} {target}".format( host=host, path=path, user=sync_user, target=target, ) ret = os.system(sync_cmd) if ret != 0: log.error( "Copy error to app servers. Command: [%s] Return: [%s]", sync_cmd, ret, )
def symlink_subprojects(self): """ Symlink project subprojects. Link from $WEB_ROOT/projects/<project> -> $WEB_ROOT/<project> """ subprojects = set() rels = self.get_subprojects() if rels.count(): # Don't create the `projects/` directory unless subprojects exist. if not os.path.exists(self.subproject_root): safe_makedirs(self.subproject_root) for rel in rels: # A mapping of slugs for the subproject URL to the actual built # documentation from_to = OrderedDict({rel.child.slug: rel.child.slug}) subprojects.add(rel.child.slug) if rel.alias: from_to[rel.alias] = rel.child.slug subprojects.add(rel.alias) for from_slug, to_slug in list(from_to.items()): log_msg = 'Symlinking subproject: {} -> {}'.format( from_slug, to_slug, ) log.debug( constants.LOG_TEMPLATE, { 'project': self.project.slug, 'version': '', 'msg': log_msg, } ) symlink = os.path.join(self.subproject_root, from_slug) docs_dir = os.path.join( self.WEB_ROOT, to_slug, ) symlink_dir = os.sep.join(symlink.split(os.path.sep)[:-1]) if not os.path.lexists(symlink_dir): safe_makedirs(symlink_dir) # TODO this should use os.symlink, not a call to shell. For now, # this passes command as a list to be explicit about escaping # characters like spaces. result = self.environment.run('ln', '-nsf', docs_dir, symlink) if result.exit_code > 0: log.error( 'Could not symlink path: status=%d error=%s', result.exit_code, result.error, ) # Remove old symlinks if os.path.exists(self.subproject_root): for subproj in os.listdir(self.subproject_root): if subproj not in subprojects: safe_unlink(os.path.join(self.subproject_root, subproj))
def symlink_subprojects(self): """ Symlink project subprojects. Link from $WEB_ROOT/projects/<project> -> $WEB_ROOT/<project> """ subprojects = set() rels = self.get_subprojects() if rels.count(): # Don't create the `projects/` directory unless subprojects exist. if not os.path.exists(self.subproject_root): safe_makedirs(self.subproject_root) for rel in rels: # A mapping of slugs for the subproject URL to the actual built # documentation from_to = OrderedDict({rel.child.slug: rel.child.slug}) subprojects.add(rel.child.slug) if rel.alias: from_to[rel.alias] = rel.child.slug subprojects.add(rel.alias) for from_slug, to_slug in list(from_to.items()): log_msg = 'Symlinking subproject: {} -> {}'.format( from_slug, to_slug, ) log.debug( constants.LOG_TEMPLATE.format( project=self.project.slug, version='', msg=log_msg, ), ) symlink = os.path.join(self.subproject_root, from_slug) docs_dir = os.path.join( self.WEB_ROOT, to_slug, ) symlink_dir = os.sep.join(symlink.split(os.path.sep)[:-1]) if not os.path.lexists(symlink_dir): safe_makedirs(symlink_dir) # TODO this should use os.symlink, not a call to shell. For now, # this passes command as a list to be explicit about escaping # characters like spaces. result = self.environment.run('ln', '-nsf', docs_dir, symlink) if result.exit_code > 0: log.error( 'Could not symlink path: status=%d error=%s', result.exit_code, result.error, ) # Remove old symlinks if os.path.exists(self.subproject_root): for subproj in os.listdir(self.subproject_root): if subproj not in subprojects: safe_unlink(os.path.join(self.subproject_root, subproj))
def sanity_check(self): """ Make sure the project_root is the proper structure before continuing. This will leave it in the proper state for the single_project setting. """ if os.path.islink( self.project_root) and not self.project.single_version: log.info( constants.LOG_TEMPLATE, { 'project': self.project.slug, 'version': '', 'msg': 'Removing single version symlink', }) safe_unlink(self.project_root) safe_makedirs(self.project_root) elif (self.project.single_version and not os.path.islink(self.project_root) and os.path.exists(self.project_root)): shutil.rmtree(self.project_root) elif not os.path.lexists(self.project_root): safe_makedirs(self.project_root) # CNAME root directories if not os.path.lexists(self.CNAME_ROOT): safe_makedirs(self.CNAME_ROOT) if not os.path.lexists(self.PROJECT_CNAME_ROOT): safe_makedirs(self.PROJECT_CNAME_ROOT)
def sanity_check(self): """ Make sure the project_root is the proper structure before continuing. This will leave it in the proper state for the single_project setting. """ if os.path.islink(self.project_root) and not self.project.single_version: log.info( constants.LOG_TEMPLATE, { 'project': self.project.slug, 'version': '', 'msg': 'Removing single version symlink', } ) safe_unlink(self.project_root) safe_makedirs(self.project_root) elif (self.project.single_version and not os.path.islink(self.project_root) and os.path.exists(self.project_root)): shutil.rmtree(self.project_root) elif not os.path.lexists(self.project_root): safe_makedirs(self.project_root) # CNAME root directories if not os.path.lexists(self.CNAME_ROOT): safe_makedirs(self.CNAME_ROOT) if not os.path.lexists(self.PROJECT_CNAME_ROOT): safe_makedirs(self.PROJECT_CNAME_ROOT)
def symlink_versions(self): """ Symlink project's versions. Link from $WEB_ROOT/<project>/<language>/<version>/ -> HOME/user_builds/<project>/rtd-builds/<version> """ versions = set() version_dir = os.path.join( self.WEB_ROOT, self.project.slug, self.project.language, ) # Include active public versions, # as well as public versions that are built but not active, for archived versions version_queryset = self.get_version_queryset() if version_queryset.count(): if not os.path.exists(version_dir): safe_makedirs(version_dir) for version in version_queryset: log_msg = 'Symlinking Version: {}'.format(version) log.debug( constants.LOG_TEMPLATE, { 'project': self.project.slug, 'version': '', 'msg': log_msg, } ) symlink = os.path.join(version_dir, version.slug) docs_dir = os.path.join( settings.DOCROOT, self.project.slug, 'rtd-builds', version.slug, ) self.environment.run('ln', '-nsf', docs_dir, symlink) versions.add(version.slug) # Remove old symlinks if os.path.exists(version_dir): for old_ver in os.listdir(version_dir): if old_ver not in versions: safe_unlink(os.path.join(version_dir, old_ver))
def symlink_translations(self): """ Symlink project translations. Link from $WEB_ROOT/<project>/<language>/ -> $WEB_ROOT/<translation>/<language>/ """ translations = {} for trans in self.get_translations(): translations[trans.language] = trans.slug # Make sure the language directory is a directory language_dir = os.path.join(self.project_root, self.project.language) if os.path.islink(language_dir): safe_unlink(language_dir) if not os.path.lexists(language_dir): safe_makedirs(language_dir) for (language, slug) in list(translations.items()): log_msg = 'Symlinking translation: {}->{}'.format(language, slug) log.debug( constants.LOG_TEMPLATE, { 'project': self.project.slug, 'version': '', 'msg': log_msg, } ) symlink = os.path.join(self.project_root, language) docs_dir = os.path.join(self.WEB_ROOT, slug, language) self.environment.run('ln', '-nsf', docs_dir, symlink) # Remove old symlinks for lang in os.listdir(self.project_root): if (lang not in translations and lang not in ['projects', self.project.language]): to_delete = os.path.join(self.project_root, lang) if os.path.islink(to_delete): safe_unlink(to_delete) else: shutil.rmtree(to_delete)
def symlink_subprojects(self): """ Symlink project subprojects. Link from $WEB_ROOT/projects/<project> -> $WEB_ROOT/<project> """ subprojects = set() rels = self.get_subprojects() if rels.count(): # Don't creat the `projects/` directory unless subprojects exist. if not os.path.exists(self.subproject_root): safe_makedirs(self.subproject_root) for rel in rels: # A mapping of slugs for the subproject URL to the actual built # documentation from_to = OrderedDict({rel.child.slug: rel.child.slug}) subprojects.add(rel.child.slug) if rel.alias: from_to[rel.alias] = rel.child.slug subprojects.add(rel.alias) for from_slug, to_slug in list(from_to.items()): self._log(u"Symlinking subproject: {0} -> {1}".format(from_slug, to_slug)) symlink = os.path.join(self.subproject_root, from_slug) docs_dir = os.path.join( self.WEB_ROOT, to_slug ) symlink_dir = os.sep.join(symlink.split(os.path.sep)[:-1]) if not os.path.lexists(symlink_dir): safe_makedirs(symlink_dir) # TODO this should use os.symlink, not a call to shell. For now, # this passes command as a list to be explicit about escaping # characters like spaces. status, _, stderr = run(['ln', '-nsf', docs_dir, symlink]) if status > 0: log.error('Could not symlink path: status=%d error=%s', status, stderr) # Remove old symlinks if os.path.exists(self.subproject_root): for subproj in os.listdir(self.subproject_root): if subproj not in subprojects: os.unlink(os.path.join(self.subproject_root, subproj))
def copy(cls, path, target, is_file=False, **kwargs): """A copy command that works with files or directories.""" log.info('Local Copy %s to %s', path, target) if is_file: if path == target: # Don't copy the same file over itself return if os.path.exists(target): os.remove(target) # Create containing directory if it doesn't exist directory = os.path.dirname(target) safe_makedirs(directory) shutil.copy2(path, target) else: if os.path.exists(target): shutil.rmtree(target) shutil.copytree(path, target)
def symlink_translations(self): """ Symlink project translations. Link from $WEB_ROOT/<project>/<language>/ -> $WEB_ROOT/<translation>/<language>/ """ translations = {} for trans in self.get_translations(): translations[trans.language] = trans.slug # Make sure the language directory is a directory language_dir = os.path.join(self.project_root, self.project.language) if os.path.islink(language_dir): safe_unlink(language_dir) if not os.path.lexists(language_dir): safe_makedirs(language_dir) for (language, slug) in list(translations.items()): log_msg = 'Symlinking translation: {}->{}'.format(language, slug) log.debug( constants.LOG_TEMPLATE.format( project=self.project.slug, version='', msg=log_msg, ), ) symlink = os.path.join(self.project_root, language) docs_dir = os.path.join(self.WEB_ROOT, slug, language) self.environment.run('ln', '-nsf', docs_dir, symlink) # Remove old symlinks for lang in os.listdir(self.project_root): if (lang not in translations and lang not in ['projects', self.project.language]): to_delete = os.path.join(self.project_root, lang) if os.path.islink(to_delete): safe_unlink(to_delete) else: shutil.rmtree(to_delete)
def symlink_subprojects(self): """ Symlink project subprojects. Link from $WEB_ROOT/projects/<project> -> $WEB_ROOT/<project> """ subprojects = set() rels = self.get_subprojects() if rels.count(): # Don't creat the `projects/` directory unless subprojects exist. if not os.path.exists(self.subproject_root): safe_makedirs(self.subproject_root) for rel in rels: # A mapping of slugs for the subproject URL to the actual built # documentation from_to = OrderedDict({rel.child.slug: rel.child.slug}) subprojects.add(rel.child.slug) if rel.alias: from_to[rel.alias] = rel.child.slug subprojects.add(rel.alias) for from_slug, to_slug in list(from_to.items()): self._log(u"Symlinking subproject: {0} -> {1}".format( from_slug, to_slug)) symlink = os.path.join(self.subproject_root, from_slug) docs_dir = os.path.join(self.WEB_ROOT, to_slug) symlink_dir = os.sep.join(symlink.split(os.path.sep)[:-1]) if not os.path.lexists(symlink_dir): safe_makedirs(symlink_dir) # TODO this should use os.symlink, not a call to shell. For now, # this passes command as a list to be explicit about escaping # characters like spaces. status, _, stderr = run(['ln', '-nsf', docs_dir, symlink]) if status > 0: log.error('Could not symlink path: status=%d error=%s', status, stderr) # Remove old symlinks if os.path.exists(self.subproject_root): for subproj in os.listdir(self.subproject_root): if subproj not in subprojects: os.unlink(os.path.join(self.subproject_root, subproj))
def symlink_subprojects(self): """Symlink project subprojects Link from $WEB_ROOT/projects/<project> -> $WEB_ROOT/<project> """ subprojects = set() rels = self.get_subprojects() if rels.count(): # Don't creat the `projects/` directory unless subprojects exist. if not os.path.exists(self.subproject_root): safe_makedirs(self.subproject_root) for rel in rels: # A mapping of slugs for the subproject URL to the actual built # documentation from_to = OrderedDict({rel.child.slug: rel.child.slug}) subprojects.add(rel.child.slug) if rel.alias: from_to[rel.alias] = rel.child.slug subprojects.add(rel.alias) for from_slug, to_slug in list(from_to.items()): self._log(u"Symlinking subproject: {0} -> {1}".format(from_slug, to_slug)) symlink = os.path.join(self.subproject_root, from_slug) docs_dir = os.path.join( self.WEB_ROOT, to_slug ) symlink_dir = os.sep.join(symlink.split(os.path.sep)[:-1]) if not os.path.lexists(symlink_dir): safe_makedirs(symlink_dir) run('ln -nsf %s %s' % (docs_dir, symlink)) # Remove old symlinks if os.path.exists(self.subproject_root): for subproj in os.listdir(self.subproject_root): if subproj not in subprojects: os.unlink(os.path.join(self.subproject_root, subproj))
def sanity_check(self): """ Make sure the project_root is the proper structure before continuing. This will leave it in the proper state for the single_project setting. """ if os.path.islink(self.project_root) and not self.project.single_version: self._log("Removing single version symlink") os.unlink(self.project_root) safe_makedirs(self.project_root) elif (self.project.single_version and not os.path.islink(self.project_root) and os.path.exists(self.project_root)): shutil.rmtree(self.project_root) elif not os.path.lexists(self.project_root): safe_makedirs(self.project_root) # CNAME root directories if not os.path.lexists(self.CNAME_ROOT): safe_makedirs(self.CNAME_ROOT) if not os.path.lexists(self.PROJECT_CNAME_ROOT): safe_makedirs(self.PROJECT_CNAME_ROOT)
def sanity_check(self): """ Make sure the project_root is the proper structure before continuing. This will leave it in the proper state for the single_project setting. """ if os.path.islink( self.project_root) and not self.project.single_version: self._log("Removing single version symlink") os.unlink(self.project_root) safe_makedirs(self.project_root) elif (self.project.single_version and not os.path.islink(self.project_root) and os.path.exists(self.project_root)): shutil.rmtree(self.project_root) elif not os.path.lexists(self.project_root): safe_makedirs(self.project_root) # CNAME root directories if not os.path.lexists(self.CNAME_ROOT): safe_makedirs(self.CNAME_ROOT) if not os.path.lexists(self.PROJECT_CNAME_ROOT): safe_makedirs(self.PROJECT_CNAME_ROOT)