def test_normal_execution(self):
"""Normal build in passing state"""
self.mocks.configure_mock("process", {"communicate.return_value": ("This is okay", "")})
type(self.mocks.process).returncode = PropertyMock(return_value=0)
build_env = LocalEnvironment(version=self.version, project=self.project, build={})
with build_env:
build_env.run("echo", "test")
self.assertTrue(self.mocks.process.communicate.called)
self.assertTrue(build_env.done)
self.assertTrue(build_env.successful)
self.assertEqual(len(build_env.commands), 1)
self.assertEqual(build_env.commands[0].output, u"This is okay")
def test_failing_execution(self):
"""Build in failing state"""
self.mocks.configure_mock("process", {"communicate.return_value": ("This is not okay", "")})
type(self.mocks.process).returncode = PropertyMock(return_value=1)
build_env = LocalEnvironment(version=self.version, project=self.project, build={})
with build_env:
build_env.run("echo", "test")
self.fail("This should be unreachable")
self.assertTrue(self.mocks.process.communicate.called)
self.assertTrue(build_env.done)
self.assertTrue(build_env.failed)
self.assertEqual(len(build_env.commands), 1)
self.assertEqual(build_env.commands[0].output, u"This is not okay")
def test_failing_execution(self):
"""Build in failing state."""
self.mocks.configure_mock(
'process', {'communicate.return_value': (b'This is not okay', '')})
type(self.mocks.process).returncode = PropertyMock(return_value=1)
build_env = LocalEnvironment(
version=self.version,
project=self.project,
build={'id': DUMMY_BUILD_ID},
)
with build_env:
build_env.run('echo', 'test')
self.fail('This should be unreachable')
self.assertTrue(self.mocks.process.communicate.called)
self.assertTrue(build_env.done)
self.assertTrue(build_env.failed)
self.assertEqual(len(build_env.commands), 1)
self.assertEqual(build_env.commands[0].output, u'This is not okay')
# api() is not called anymore, we use api_v2 instead
self.assertFalse(self.mocks.api()(DUMMY_BUILD_ID).put.called)
self.mocks.mocks['api_v2.build']().put.assert_called_with({
'id':
DUMMY_BUILD_ID,
'version':
self.version.pk,
'success':
False,
'project':
self.project.pk,
'setup_error':
u'',
'length':
mock.ANY,
'error':
'',
'setup':
u'',
'output':
u'',
'state':
u'finished',
'builder':
mock.ANY,
'exit_code':
1,
})
def test_normal_execution(self):
'''Normal build in passing state'''
self.mocks.configure_mock('process', {
'communicate.return_value': ('This is okay', '')})
type(self.mocks.process).returncode = PropertyMock(return_value=0)
build_env = LocalEnvironment(version=self.version, project=self.project,
build={})
with build_env:
build_env.run('echo', 'test')
self.assertTrue(self.mocks.process.communicate.called)
self.assertTrue(build_env.done)
self.assertTrue(build_env.successful)
self.assertEqual(len(build_env.commands), 1)
self.assertEqual(build_env.commands[0].output, u'This is okay')
def test_failing_execution(self):
'''Build in failing state'''
self.mocks.configure_mock('process', {
'communicate.return_value': ('This is not okay', '')})
type(self.mocks.process).returncode = PropertyMock(return_value=1)
build_env = LocalEnvironment(version=self.version, project=self.project,
build={})
with build_env:
build_env.run('echo', 'test')
self.fail('This should be unreachable')
self.assertTrue(self.mocks.process.communicate.called)
self.assertTrue(build_env.done)
self.assertTrue(build_env.failed)
self.assertEqual(len(build_env.commands), 1)
self.assertEqual(build_env.commands[0].output, u'This is not okay')
def test_failing_execution(self):
"""Build in failing state."""
self.mocks.configure_mock('process', {
'communicate.return_value': (b'This is not okay', '')
})
type(self.mocks.process).returncode = PropertyMock(return_value=1)
build_env = LocalEnvironment(
version=self.version,
project=self.project,
build={'id': DUMMY_BUILD_ID},
)
with build_env:
build_env.run('echo', 'test')
self.fail('This should be unreachable')
self.assertTrue(self.mocks.process.communicate.called)
self.assertTrue(build_env.done)
self.assertTrue(build_env.failed)
self.assertEqual(len(build_env.commands), 1)
self.assertEqual(build_env.commands[0].output, u'This is not okay')
# api() is not called anymore, we use api_v2 instead
self.assertFalse(self.mocks.api()(DUMMY_BUILD_ID).put.called)
self.mocks.mocks['api_v2.build']().put.assert_called_with({
'id': DUMMY_BUILD_ID,
'version': self.version.pk,
'success': False,
'project': self.project.pk,
'setup_error': u'',
'length': mock.ANY,
'error': '',
'setup': u'',
'output': u'',
'state': u'finished',
'builder': mock.ANY,
'exit_code': 1,
})
class Symlink:
"""Base class for symlinking of projects."""
def __init__(self, project):
self.project = project
self.project_root = os.path.join(
self.WEB_ROOT,
project.slug,
)
self.subproject_root = os.path.join(
self.project_root,
'projects',
)
self.environment = LocalEnvironment(project)
self.sanity_check()
def sanity_check(self):
"""
Make sure the project_root is the proper structure before continuing.
This will leave it in the proper state for the single_project setting.
"""
if os.path.islink(
self.project_root) and not self.project.single_version:
log.info(
constants.LOG_TEMPLATE, {
'project': self.project.slug,
'version': '',
'msg': 'Removing single version symlink',
})
safe_unlink(self.project_root)
safe_makedirs(self.project_root)
elif (self.project.single_version
and not os.path.islink(self.project_root)
and os.path.exists(self.project_root)):
shutil.rmtree(self.project_root)
elif not os.path.lexists(self.project_root):
safe_makedirs(self.project_root)
# CNAME root directories
if not os.path.lexists(self.CNAME_ROOT):
safe_makedirs(self.CNAME_ROOT)
if not os.path.lexists(self.PROJECT_CNAME_ROOT):
safe_makedirs(self.PROJECT_CNAME_ROOT)
def run(self):
"""
Create proper symlinks in the right order.
Since we have a small nest of directories and symlinks, the ordering of
these calls matter, so we provide this helper to make life easier.
"""
# Outside of the web root
self.symlink_cnames()
# Build structure inside symlink zone
if self.project.single_version:
self.symlink_single_version()
self.symlink_subprojects()
else:
self.symlink_translations()
self.symlink_subprojects()
self.symlink_versions()
def symlink_cnames(self, domain=None):
"""
Symlink project CNAME domains.
Link from HOME/$CNAME_ROOT/<cname> ->
HOME/$WEB_ROOT/<project>
Also give cname -> project link
Link from HOME/public_cname_project/<cname> ->
HOME/<project>/
"""
if domain:
domains = [domain]
else:
domains = Domain.objects.filter(project=self.project).values_list(
'domain', flat=True)
for dom in domains:
log_msg = 'Symlinking CNAME: {} -> {}'.format(
dom,
self.project.slug,
)
log.debug(constants.LOG_TEMPLATE, {
'project': self.project.slug,
'version': '',
'msg': log_msg,
})
# CNAME to doc root
symlink = os.path.join(self.CNAME_ROOT, dom)
self.environment.run('ln', '-nsf', self.project_root, symlink)
# Project symlink
project_cname_symlink = os.path.join(
self.PROJECT_CNAME_ROOT,
dom,
)
self.environment.run(
'ln',
'-nsf',
self.project.doc_path,
project_cname_symlink,
)
def remove_symlink_cname(self, domain):
"""
Remove CNAME symlink.
:param domain: domain for which symlink is to be removed
:type domain: str
"""
log_msg = 'Removing symlink for CNAME {}'.format(domain)
log.debug(constants.LOG_TEMPLATE, {
'project': self.project.slug,
'version': '',
'msg': log_msg,
})
symlink = os.path.join(self.CNAME_ROOT, domain)
safe_unlink(symlink)
def symlink_subprojects(self):
"""
Symlink project subprojects.
Link from $WEB_ROOT/projects/<project> -> $WEB_ROOT/<project>
"""
subprojects = set()
rels = self.get_subprojects()
if rels.count():
# Don't create the `projects/` directory unless subprojects exist.
if not os.path.exists(self.subproject_root):
safe_makedirs(self.subproject_root)
for rel in rels:
# A mapping of slugs for the subproject URL to the actual built
# documentation
from_to = OrderedDict({rel.child.slug: rel.child.slug})
subprojects.add(rel.child.slug)
if rel.alias:
from_to[rel.alias] = rel.child.slug
subprojects.add(rel.alias)
for from_slug, to_slug in list(from_to.items()):
log_msg = 'Symlinking subproject: {} -> {}'.format(
from_slug,
to_slug,
)
log.debug(constants.LOG_TEMPLATE, {
'project': self.project.slug,
'version': '',
'msg': log_msg,
})
symlink = os.path.join(self.subproject_root, from_slug)
docs_dir = os.path.join(
self.WEB_ROOT,
to_slug,
)
symlink_dir = os.sep.join(symlink.split(os.path.sep)[:-1])
if not os.path.lexists(symlink_dir):
safe_makedirs(symlink_dir)
# TODO this should use os.symlink, not a call to shell. For now,
# this passes command as a list to be explicit about escaping
# characters like spaces.
result = self.environment.run('ln', '-nsf', docs_dir, symlink)
if result.exit_code > 0:
log.error(
'Could not symlink path: status=%d error=%s',
result.exit_code,
result.error,
)
# Remove old symlinks
if os.path.exists(self.subproject_root):
for subproj in os.listdir(self.subproject_root):
if subproj not in subprojects:
safe_unlink(os.path.join(self.subproject_root, subproj))
def symlink_translations(self):
"""
Symlink project translations.
Link from $WEB_ROOT/<project>/<language>/ ->
$WEB_ROOT/<translation>/<language>/
"""
translations = {}
for trans in self.get_translations():
translations[trans.language] = trans.slug
# Make sure the language directory is a directory
language_dir = os.path.join(self.project_root, self.project.language)
if os.path.islink(language_dir):
safe_unlink(language_dir)
if not os.path.lexists(language_dir):
safe_makedirs(language_dir)
for (language, slug) in list(translations.items()):
log_msg = 'Symlinking translation: {}->{}'.format(language, slug)
log.debug(constants.LOG_TEMPLATE, {
'project': self.project.slug,
'version': '',
'msg': log_msg,
})
symlink = os.path.join(self.project_root, language)
docs_dir = os.path.join(self.WEB_ROOT, slug, language)
self.environment.run('ln', '-nsf', docs_dir, symlink)
# Remove old symlinks
for lang in os.listdir(self.project_root):
if (lang not in translations
and lang not in ['projects', self.project.language]):
to_delete = os.path.join(self.project_root, lang)
if os.path.islink(to_delete):
safe_unlink(to_delete)
else:
shutil.rmtree(to_delete)
def symlink_single_version(self):
"""
Symlink project single version.
Link from:
$WEB_ROOT/<project> -> HOME/user_builds/<project>/rtd-builds/latest/
"""
version = self.get_default_version()
# Clean up symlinks
symlink = self.project_root
if os.path.islink(symlink):
safe_unlink(symlink)
elif os.path.exists(symlink):
shutil.rmtree(symlink)
# Create symlink
if version is not None:
docs_dir = os.path.join(
settings.DOCROOT,
self.project.slug,
'rtd-builds',
version.slug,
)
self.environment.run('ln', '-nsf', docs_dir, symlink)
def symlink_versions(self):
"""
Symlink project's versions.
Link from $WEB_ROOT/<project>/<language>/<version>/ ->
HOME/user_builds/<project>/rtd-builds/<version>
"""
versions = set()
version_dir = os.path.join(
self.WEB_ROOT,
self.project.slug,
self.project.language,
)
# Include active public versions,
# as well as public versions that are built but not active, for archived versions
version_queryset = self.get_version_queryset()
if version_queryset.count():
if not os.path.exists(version_dir):
safe_makedirs(version_dir)
for version in version_queryset:
log_msg = 'Symlinking Version: {}'.format(version)
log.debug(constants.LOG_TEMPLATE, {
'project': self.project.slug,
'version': '',
'msg': log_msg,
})
symlink = os.path.join(version_dir, version.slug)
docs_dir = os.path.join(
settings.DOCROOT,
self.project.slug,
'rtd-builds',
version.slug,
)
self.environment.run('ln', '-nsf', docs_dir, symlink)
versions.add(version.slug)
# Remove old symlinks
if os.path.exists(version_dir):
for old_ver in os.listdir(version_dir):
if old_ver not in versions:
safe_unlink(os.path.join(version_dir, old_ver))
def get_default_version(self):
"""Look up project default version, return None if not found."""
default_version = self.project.get_default_version()
try:
return self.get_version_queryset().get(slug=default_version)
except Version.DoesNotExist:
return None
class Symlink:
"""Base class for symlinking of projects."""
def __init__(self, project):
self.project = project
self.project_root = os.path.join(
self.WEB_ROOT,
project.slug,
)
self.subproject_root = os.path.join(
self.project_root,
'projects',
)
self.environment = LocalEnvironment(project)
self.sanity_check()
def sanity_check(self):
"""
Make sure the project_root is the proper structure before continuing.
This will leave it in the proper state for the single_project setting.
"""
if os.path.islink(self.project_root) and not self.project.single_version:
log.info(
constants.LOG_TEMPLATE,
{
'project': self.project.slug,
'version': '',
'msg': 'Removing single version symlink',
}
)
safe_unlink(self.project_root)
safe_makedirs(self.project_root)
elif (self.project.single_version and
not os.path.islink(self.project_root) and
os.path.exists(self.project_root)):
shutil.rmtree(self.project_root)
elif not os.path.lexists(self.project_root):
safe_makedirs(self.project_root)
# CNAME root directories
if not os.path.lexists(self.CNAME_ROOT):
safe_makedirs(self.CNAME_ROOT)
if not os.path.lexists(self.PROJECT_CNAME_ROOT):
safe_makedirs(self.PROJECT_CNAME_ROOT)
def run(self):
"""
Create proper symlinks in the right order.
Since we have a small nest of directories and symlinks, the ordering of
these calls matter, so we provide this helper to make life easier.
"""
# Outside of the web root
self.symlink_cnames()
# Build structure inside symlink zone
if self.project.single_version:
self.symlink_single_version()
self.symlink_subprojects()
else:
self.symlink_translations()
self.symlink_subprojects()
self.symlink_versions()
def symlink_cnames(self, domain=None):
"""
Symlink project CNAME domains.
Link from HOME/$CNAME_ROOT/<cname> ->
HOME/$WEB_ROOT/<project>
Also give cname -> project link
Link from HOME/public_cname_project/<cname> ->
HOME/<project>/
"""
if domain:
domains = [domain]
else:
domains = Domain.objects.filter(project=self.project).values_list('domain', flat=True)
for dom in domains:
log_msg = 'Symlinking CNAME: {} -> {}'.format(
dom,
self.project.slug,
)
log.debug(
constants.LOG_TEMPLATE,
{
'project': self.project.slug,
'version': '',
'msg': log_msg,
}
)
# CNAME to doc root
symlink = os.path.join(self.CNAME_ROOT, dom)
self.environment.run('ln', '-nsf', self.project_root, symlink)
# Project symlink
project_cname_symlink = os.path.join(
self.PROJECT_CNAME_ROOT,
dom,
)
self.environment.run(
'ln',
'-nsf',
self.project.doc_path,
project_cname_symlink,
)
def remove_symlink_cname(self, domain):
"""
Remove CNAME symlink.
:param domain: domain for which symlink is to be removed
:type domain: str
"""
log_msg = 'Removing symlink for CNAME {}'.format(domain)
log.debug(
constants.LOG_TEMPLATE,
{
'project': self.project.slug,
'version': '',
'msg': log_msg,
}
)
symlink = os.path.join(self.CNAME_ROOT, domain)
safe_unlink(symlink)
def symlink_subprojects(self):
"""
Symlink project subprojects.
Link from $WEB_ROOT/projects/<project> -> $WEB_ROOT/<project>
"""
subprojects = set()
rels = self.get_subprojects()
if rels.count():
# Don't create the `projects/` directory unless subprojects exist.
if not os.path.exists(self.subproject_root):
safe_makedirs(self.subproject_root)
for rel in rels:
# A mapping of slugs for the subproject URL to the actual built
# documentation
from_to = OrderedDict({rel.child.slug: rel.child.slug})
subprojects.add(rel.child.slug)
if rel.alias:
from_to[rel.alias] = rel.child.slug
subprojects.add(rel.alias)
for from_slug, to_slug in list(from_to.items()):
log_msg = 'Symlinking subproject: {} -> {}'.format(
from_slug,
to_slug,
)
log.debug(
constants.LOG_TEMPLATE,
{
'project': self.project.slug,
'version': '',
'msg': log_msg,
}
)
symlink = os.path.join(self.subproject_root, from_slug)
docs_dir = os.path.join(
self.WEB_ROOT,
to_slug,
)
symlink_dir = os.sep.join(symlink.split(os.path.sep)[:-1])
if not os.path.lexists(symlink_dir):
safe_makedirs(symlink_dir)
# TODO this should use os.symlink, not a call to shell. For now,
# this passes command as a list to be explicit about escaping
# characters like spaces.
result = self.environment.run('ln', '-nsf', docs_dir, symlink)
if result.exit_code > 0:
log.error(
'Could not symlink path: status=%d error=%s',
result.exit_code,
result.error,
)
# Remove old symlinks
if os.path.exists(self.subproject_root):
for subproj in os.listdir(self.subproject_root):
if subproj not in subprojects:
safe_unlink(os.path.join(self.subproject_root, subproj))
def symlink_translations(self):
"""
Symlink project translations.
Link from $WEB_ROOT/<project>/<language>/ ->
$WEB_ROOT/<translation>/<language>/
"""
translations = {}
for trans in self.get_translations():
translations[trans.language] = trans.slug
# Make sure the language directory is a directory
language_dir = os.path.join(self.project_root, self.project.language)
if os.path.islink(language_dir):
safe_unlink(language_dir)
if not os.path.lexists(language_dir):
safe_makedirs(language_dir)
for (language, slug) in list(translations.items()):
log_msg = 'Symlinking translation: {}->{}'.format(language, slug)
log.debug(
constants.LOG_TEMPLATE,
{
'project': self.project.slug,
'version': '',
'msg': log_msg,
}
)
symlink = os.path.join(self.project_root, language)
docs_dir = os.path.join(self.WEB_ROOT, slug, language)
self.environment.run('ln', '-nsf', docs_dir, symlink)
# Remove old symlinks
for lang in os.listdir(self.project_root):
if (lang not in translations and
lang not in ['projects', self.project.language]):
to_delete = os.path.join(self.project_root, lang)
if os.path.islink(to_delete):
safe_unlink(to_delete)
else:
shutil.rmtree(to_delete)
def symlink_single_version(self):
"""
Symlink project single version.
Link from:
$WEB_ROOT/<project> -> HOME/user_builds/<project>/rtd-builds/latest/
"""
version = self.get_default_version()
# Clean up symlinks
symlink = self.project_root
if os.path.islink(symlink):
safe_unlink(symlink)
elif os.path.exists(symlink):
shutil.rmtree(symlink)
# Create symlink
if version is not None:
docs_dir = os.path.join(
settings.DOCROOT,
self.project.slug,
'rtd-builds',
version.slug,
)
self.environment.run('ln', '-nsf', docs_dir, symlink)
def symlink_versions(self):
"""
Symlink project's versions.
Link from $WEB_ROOT/<project>/<language>/<version>/ ->
HOME/user_builds/<project>/rtd-builds/<version>
"""
versions = set()
version_dir = os.path.join(
self.WEB_ROOT,
self.project.slug,
self.project.language,
)
# Include active public versions,
# as well as public versions that are built but not active, for archived versions
version_queryset = self.get_version_queryset()
if version_queryset.count():
if not os.path.exists(version_dir):
safe_makedirs(version_dir)
for version in version_queryset:
log_msg = 'Symlinking Version: {}'.format(version)
log.debug(
constants.LOG_TEMPLATE,
{
'project': self.project.slug,
'version': '',
'msg': log_msg,
}
)
symlink = os.path.join(version_dir, version.slug)
docs_dir = os.path.join(
settings.DOCROOT,
self.project.slug,
'rtd-builds',
version.slug,
)
self.environment.run('ln', '-nsf', docs_dir, symlink)
versions.add(version.slug)
# Remove old symlinks
if os.path.exists(version_dir):
for old_ver in os.listdir(version_dir):
if old_ver not in versions:
safe_unlink(os.path.join(version_dir, old_ver))
def get_default_version(self):
"""Look up project default version, return None if not found."""
default_version = self.project.get_default_version()
try:
return self.get_version_queryset().get(slug=default_version)
except Version.DoesNotExist:
return None