def upload_rpms(scratch_directory, target_bucket, version, build_server):
"""
Upload RPMS from build server to yum repository.
:param FilePath scratch_directory: Temporary directory to download
repository to.
:param bytes target_bucket: S3 bucket to upload repository to.
:param bytes version: Version to download RPMs for.
:param bytes build_server: Server to download new RPMs from.
"""
if not (is_release(version)
or is_weekly_release(version)
or is_pre_release(version)):
raise NotARelease()
if get_doc_version(version) != version:
raise DocumentationRelease()
is_dev = not is_release(version)
if is_dev:
target_distro_suffix = "-testing"
else:
target_distro_suffix = ""
operating_systems = [
{'distro': 'fedora', 'version': '20', 'arch': 'x86_64'},
{'distro': 'centos', 'version': '7', 'arch': 'x86_64'},
]
for operating_system in operating_systems:
yield update_repo(
rpm_directory=scratch_directory.child(
b'{}-{}-{}'.format(
operating_system['distro'],
operating_system['version'],
operating_system['arch'])),
target_bucket=target_bucket,
target_key=os.path.join(
operating_system['distro'] + target_distro_suffix,
operating_system['version'],
operating_system['arch']),
source_repo=os.path.join(
build_server, b'results/omnibus',
version,
b'{}-{}'.format(
operating_system['distro'],
operating_system['version'])),
packages=FLOCKER_PACKAGES,
flocker_version=version,
distro_name=operating_system['distro'],
distro_version=operating_system['version'],
)
def calculate_base_branch(version, path):
"""
The branch a release branch is created from depends on the release
type and sometimes which pre-releases have preceeded this.
:param bytes version: The version of Flocker to get a base branch for.
:param bytes path: See :func:`git.Repo.init`.
:returns: The base branch from which the new release branch was created.
"""
if not (is_release(version) or is_weekly_release(version)
or is_pre_release(version)):
raise NotARelease()
repo = Repo(path=path, search_parent_directories=True)
existing_tags = [tag for tag in repo.tags if tag.name == version]
if existing_tags:
raise TagExists()
# We always base releases off master now.
base_branch_name = 'master'
# We create a new branch from a branch, not a tag, because a maintenance
# or documentation change may have been applied to the branch and not the
# tag.
# The branch must be available locally for the next step.
repo.git.checkout(base_branch_name)
return (branch for branch in repo.branches
if branch.name == base_branch_name).next()
def calculate_base_branch(version, path):
"""
The branch a release branch is created from depends on the release
type and sometimes which pre-releases have preceeded this.
:param bytes version: The version of Flocker to get a base branch for.
:param bytes path: See :func:`git.Repo.init`.
:returns: The base branch from which the new release branch was created.
"""
if not (is_release(version)
or is_weekly_release(version)
or is_pre_release(version)):
raise NotARelease()
repo = Repo(path=path, search_parent_directories=True)
existing_tags = [tag for tag in repo.tags if tag.name == version]
if existing_tags:
raise TagExists()
# We always base releases off master now.
base_branch_name = 'master'
# We create a new branch from a branch, not a tag, because a maintenance
# or documentation change may have been applied to the branch and not the
# tag.
# The branch must be available locally for the next step.
repo.git.checkout(base_branch_name)
return (
branch for branch in repo.branches if
branch.name == base_branch_name).next()
def get_expected_redirects(flocker_version):
"""
Get the expected redirects for a given version of Flocker, if that version
has been published successfully. Documentation versions (e.g. 0.3.0.post2)
are published to their release version counterparts (e.g. 0.3.0).
:param bytes flocker_version: The version of Flocker for which to get
expected redirects.
:return: Dictionary mapping paths to the path to which they are expected to
redirect.
"""
published_version = get_doc_version(flocker_version)
if is_release(published_version):
expected_redirects = {
'/':
'/en/' + published_version + '/',
'/en/':
'/en/' + published_version + '/',
'/en/latest':
'/en/' + published_version + '/',
'/en/latest/faq/index.html':
'/en/' + published_version + '/faq/index.html',
}
else:
expected_redirects = {
'/en/devel':
'/en/' + published_version + '/',
'/en/devel/faq/index.html':
'/en/' + published_version + '/faq/index.html',
}
return expected_redirects
def get_expected_redirects(flocker_version):
"""
Get the expected redirects for a given version of Flocker, if that version
has been published successfully. Documentation versions (e.g. 0.3.0.post2)
are published to their release version counterparts (e.g. 0.3.0).
:param bytes flocker_version: The version of Flocker for which to get
expected redirects.
:return: Dictionary mapping paths to the path to which they are expected to
redirect.
"""
published_version = get_doc_version(flocker_version)
if is_release(published_version):
expected_redirects = {
'/': '/en/' + published_version + '/',
'/en/': '/en/' + published_version + '/',
'/en/latest': '/en/' + published_version + '/',
'/en/latest/faq/index.html':
'/en/' + published_version + '/faq/index.html',
}
else:
expected_redirects = {
'/en/devel': '/en/' + published_version + '/',
'/en/devel/faq/index.html':
'/en/' + published_version + '/faq/index.html',
}
return expected_redirects
def parseArgs(self):
version = self['flocker-version']
if not (is_release(version) or is_weekly_release(version)
or is_pre_release(version)):
raise NotARelease()
if get_doc_version(version) != version:
raise DocumentationRelease()
def parseArgs(self):
version = self['flocker-version']
if not (is_release(version)
or is_weekly_release(version)
or is_pre_release(version)):
raise NotARelease()
if get_doc_version(version) != version:
raise DocumentationRelease()
def get_repo_options(flocker_version):
"""
Get a list of options for enabling necessary yum repositories.
:param bytes flocker_version: The version of Flocker to get options for.
:return: List of bytes for enabling (or not) a testing repository.
"""
is_dev = not is_release(flocker_version)
if is_dev:
return ['--enablerepo=clusterhq-testing']
else:
return []
def get_repo_options(flocker_version):
"""
Get a list of options for enabling necessary yum repositories.
:param bytes flocker_version: The version of Flocker to get options for.
:return: List of bytes for enabling (or not) a testing repository.
"""
is_dev = not is_release(flocker_version)
if is_dev:
return ['--enablerepo=clusterhq-testing']
else:
return []
def upload_rpms(scratch_directory, target_bucket, version, build_server):
"""
The ClusterHQ yum repository contains packages for Flocker, as well as the
dependencies which aren't available in Fedora 20 or CentOS 7. It is
currently hosted on Amazon S3. When doing a release, we want to add the
new Flocker packages, while preserving the existing packages in the
repository. To do this, we download the current repository, add the new
package, update the metadata, and then upload the repository.
:param FilePath scratch_directory: Temporary directory to download
repository to.
:param bytes target_bucket: S3 bucket to upload repository to.
:param bytes version: Version to download RPMs for.
:param bytes build_server: Server to download new RPMs from.
"""
is_dev = not is_release(version)
if is_dev:
target_distro_suffix = "-testing"
else:
target_distro_suffix = ""
operating_systems = [
{'distro': 'fedora', 'version': '20', 'arch': 'x86_64'},
{'distro': 'centos', 'version': '7', 'arch': 'x86_64'},
{'distro': 'ubuntu', 'version': '14.04', 'arch': 'amd64'},
]
for operating_system in operating_systems:
yield update_repo(
package_directory=scratch_directory.child(
b'{}-{}-{}'.format(
operating_system['distro'],
operating_system['version'],
operating_system['arch'])),
target_bucket=target_bucket,
target_key=os.path.join(
operating_system['distro'] + target_distro_suffix,
operating_system['version'],
operating_system['arch']),
source_repo=os.path.join(
build_server, b'results/omnibus',
version,
b'{}-{}'.format(
operating_system['distro'],
operating_system['version'])),
packages=FLOCKER_PACKAGES,
flocker_version=version,
distro_name=operating_system['distro'],
distro_version=operating_system['version'],
)
extlinks = {
'issue': ('https://clusterhq.atlassian.net/browse/FLOC-%s', 'issue '),
}
# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
from flocker import __version__
from flocker.common.version import get_doc_version, is_release
# The short X.Y version.
version = get_doc_version(__version__)
html_context = {
# This is used to show the development version warning.
'is_release': is_release(__version__),
}
# The full version, including alpha/beta/rc tags.
release = version
# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
# We override with our own variant to improve search results slightly.
from sphinx.search.en import SearchEnglish
from sphinx.search import languages as sphinx_languages
class FlockerLanguage(SearchEnglish):
"""
Workaround for https://bitbucket.org/birkenfeld/sphinx/issue/1529/
def publish_docs(flocker_version, doc_version, environment):
"""
Publish the Flocker documentation. The documentation for each version of
Flocker is uploaded to a development bucket on S3 by the build server and
this copies the documentation for a particular ``flocker_version`` and
publishes it as ``doc_version``. Attempting to publish documentation to a
staging environment as a documentation version publishes it as the version
being updated.
:param bytes flocker_version: The version of Flocker to publish the
documentation for.
:param bytes doc_version: The version to publish the documentation as.
:param Environments environment: The environment to publish the
documentation to.
:raises NotARelease: Raised if trying to publish to a version that isn't a
release.
:raises NotTagged: Raised if publishing to production and the version being
published version isn't tagged.
"""
if not (is_release(doc_version)
or is_weekly_release(doc_version)
or is_pre_release(doc_version)):
raise NotARelease()
if environment == Environments.PRODUCTION:
if get_doc_version(flocker_version) != doc_version:
raise NotTagged()
configuration = DOCUMENTATION_CONFIGURATIONS[environment]
dev_prefix = '%s/' % (flocker_version,)
version_prefix = 'en/%s/' % (get_doc_version(doc_version),)
is_dev = not is_release(doc_version)
if is_dev:
stable_prefix = "en/devel/"
else:
stable_prefix = "en/latest/"
# Get the list of keys in the new documentation.
new_version_keys = yield Effect(
ListS3Keys(bucket=configuration.dev_bucket,
prefix=dev_prefix))
# Get the list of keys already existing for the given version.
# This should only be non-empty for documentation releases.
existing_version_keys = yield Effect(
ListS3Keys(bucket=configuration.documentation_bucket,
prefix=version_prefix))
# Copy the new documentation to the documentation bucket.
yield Effect(
CopyS3Keys(source_bucket=configuration.dev_bucket,
source_prefix=dev_prefix,
destination_bucket=configuration.documentation_bucket,
destination_prefix=version_prefix,
keys=new_version_keys))
# Delete any keys that aren't in the new documentation.
yield Effect(
DeleteS3Keys(bucket=configuration.documentation_bucket,
prefix=version_prefix,
keys=existing_version_keys - new_version_keys))
# Update the key used for error pages if we're publishing to staging or if
# we're publishing a marketing release to production.
if ((environment is Environments.STAGING) or
(environment is Environments.PRODUCTION and not is_dev)):
yield Effect(
UpdateS3ErrorPage(bucket=configuration.documentation_bucket,
target_prefix=version_prefix))
# Update the redirect for the stable URL (en/latest/ or en/devel/)
# to point to the new version. Returns the old target.
old_prefix = yield Effect(
UpdateS3RoutingRule(bucket=configuration.documentation_bucket,
prefix=stable_prefix,
target_prefix=version_prefix))
# If we have changed versions, get all the keys from the old version
if old_prefix:
previous_version_keys = yield Effect(
ListS3Keys(bucket=configuration.documentation_bucket,
prefix=old_prefix))
else:
previous_version_keys = set()
# The changed keys are the new keys, the keys that were deleted from this
# version, and the keys for the previous version.
changed_keys = (new_version_keys |
existing_version_keys |
previous_version_keys)
# S3 serves /index.html when given /, so any changed /index.html means
# that / changed as well.
# Note that we check for '/index.html' but remove 'index.html'
changed_keys |= {key_name[:-len('index.html')]
for key_name in changed_keys
if key_name.endswith('/index.html')}
# Always update the root.
changed_keys |= {''}
# The full paths are all the changed keys under the stable prefix, and
# the new version prefix. This set is slightly bigger than necessary.
changed_paths = {prefix + key_name
for key_name in changed_keys
for prefix in [stable_prefix, version_prefix]}
# Invalidate all the changed paths in cloudfront.
yield Effect(
CreateCloudFrontInvalidation(cname=configuration.cloudfront_cname,
paths=changed_paths))
extlinks = {
'issue': ('https://clusterhq.atlassian.net/browse/FLOC-%s', 'issue '),
}
# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
from flocker import __version__
from flocker.common.version import get_doc_version, is_release
# The short X.Y version.
version = get_doc_version(__version__)
html_context = {
# This is used to show the development version warning.
'is_release': is_release(__version__),
}
# The full version, including alpha/beta/rc tags.
release = version
# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
# We override with our own variant to improve search results slightly.
from sphinx.search.en import SearchEnglish
from sphinx.search import languages as sphinx_languages
class FlockerLanguage(SearchEnglish):
"""
Workaround for https://bitbucket.org/birkenfeld/sphinx/issue/1529/
def test_redirects_main(args, base_path, top_level):
"""
Tests redirects to Flocker documentation.
:param list args: The arguments passed to the script.
:param FilePath base_path: The executable being run.
:param FilePath top_level: The top-level of the flocker repository.
"""
options = TestRedirectsOptions()
try:
options.parseOptions(args)
except UsageError as e:
sys.stderr.write("%s: %s\n" % (base_path.basename(), e))
raise SystemExit(1)
doc_version = options['doc-version']
document_configuration = DOCUMENTATION_CONFIGURATIONS[options.environment]
base_url = 'https://' + document_configuration.cloudfront_cname
is_dev = not is_release(doc_version)
if is_dev:
expected_redirects = {
'/en/devel': '/en/' + doc_version + '/',
'/en/devel/faq/index.html':
'/en/' + doc_version + '/faq/index.html',
}
else:
expected_redirects = {
'/': '/en/' + doc_version + '/',
'/en/': '/en/' + doc_version + '/',
'/en/latest': '/en/' + doc_version + '/',
'/en/latest/faq/index.html':
'/en/' + doc_version + '/faq/index.html',
}
failed_redirects = []
for path in expected_redirects:
original_url = base_url + path
expected_url = base_url + expected_redirects[path]
final_url = requests.get(original_url).url
if expected_url != final_url:
failed_redirects.append(original_url)
message = (
"'{original_url}' expected to redirect to '{expected_url}', "
"instead redirects to '{final_url}'.\n").format(
original_url=original_url,
expected_url=expected_url,
final_url=final_url,
)
sys.stderr.write(message)
if len(failed_redirects):
raise SystemExit(1)
else:
print 'All tested redirects work correctly.'
def publish_docs(flocker_version, doc_version, environment):
"""
Publish the Flocker documentation. The documentation for each version of
Flocker is uploaded to a development bucket on S3 by the build server and
this copies the documentation for a particular ``flocker_version`` and
publishes it as ``doc_version``. Attempting to publish documentation to a
staging environment as a documentation version publishes it as the version
being updated.
:param bytes flocker_version: The version of Flocker to publish the
documentation for.
:param bytes doc_version: The version to publish the documentation as.
:param Environments environment: The environment to publish the
documentation to.
:raises NotARelease: Raised if trying to publish to a version that isn't a
release.
:raises NotTagged: Raised if publishing to production and the version being
published version isn't tagged.
"""
if not (is_release(doc_version) or is_weekly_release(doc_version)
or is_pre_release(doc_version)):
raise NotARelease()
if environment == Environments.PRODUCTION:
if get_doc_version(flocker_version) != doc_version:
raise NotTagged()
configuration = DOCUMENTATION_CONFIGURATIONS[environment]
dev_prefix = '%s/' % (flocker_version, )
version_prefix = 'en/%s/' % (get_doc_version(doc_version), )
is_dev = not is_release(doc_version)
if is_dev:
stable_prefix = "en/devel/"
else:
stable_prefix = "en/latest/"
# Get the list of keys in the new documentation.
new_version_keys = yield Effect(
ListS3Keys(bucket=configuration.dev_bucket, prefix=dev_prefix))
# Get the list of keys already existing for the given version.
# This should only be non-empty for documentation releases.
existing_version_keys = yield Effect(
ListS3Keys(bucket=configuration.documentation_bucket,
prefix=version_prefix))
# Copy the new documentation to the documentation bucket.
yield Effect(
CopyS3Keys(source_bucket=configuration.dev_bucket,
source_prefix=dev_prefix,
destination_bucket=configuration.documentation_bucket,
destination_prefix=version_prefix,
keys=new_version_keys))
# Delete any keys that aren't in the new documentation.
yield Effect(
DeleteS3Keys(bucket=configuration.documentation_bucket,
prefix=version_prefix,
keys=existing_version_keys - new_version_keys))
# Update the key used for error pages if we're publishing to staging or if
# we're publishing a marketing release to production.
if ((environment is Environments.STAGING)
or (environment is Environments.PRODUCTION and not is_dev)):
yield Effect(
UpdateS3ErrorPage(bucket=configuration.documentation_bucket,
target_prefix=version_prefix))
# Update the redirect for the stable URL (en/latest/ or en/devel/)
# to point to the new version. Returns the old target.
old_prefix = yield Effect(
UpdateS3RoutingRule(bucket=configuration.documentation_bucket,
prefix=stable_prefix,
target_prefix=version_prefix))
# If we have changed versions, get all the keys from the old version
if old_prefix:
previous_version_keys = yield Effect(
ListS3Keys(bucket=configuration.documentation_bucket,
prefix=old_prefix))
else:
previous_version_keys = set()
# The changed keys are the new keys, the keys that were deleted from this
# version, and the keys for the previous version.
changed_keys = (new_version_keys | existing_version_keys
| previous_version_keys)
# S3 serves /index.html when given /, so any changed /index.html means
# that / changed as well.
# Note that we check for '/index.html' but remove 'index.html'
changed_keys |= {
key_name[:-len('index.html')]
for key_name in changed_keys if key_name.endswith('/index.html')
}
# Always update the root.
changed_keys |= {''}
# The full paths are all the changed keys under the stable prefix, and
# the new version prefix. This set is slightly bigger than necessary.
changed_paths = {
prefix + key_name
for key_name in changed_keys
for prefix in [stable_prefix, version_prefix]
}
# Invalidate all the changed paths in cloudfront.
yield Effect(
CreateCloudFrontInvalidation(cname=configuration.cloudfront_cname,
paths=changed_paths))
def calculate_base_branch(version, path):
"""
The branch a release branch is created from depends on the release
type and sometimes which pre-releases have preceeded this.
:param bytes version: The version of Flocker to get a base branch for.
:param bytes path: See :func:`git.Repo.init`.
:returns: The base branch from which the new release branch was created.
"""
if not (is_release(version)
or is_weekly_release(version)
or is_pre_release(version)):
raise NotARelease()
repo = Repo(path=path, search_parent_directories=True)
existing_tags = [tag for tag in repo.tags if tag.name == version]
if existing_tags:
raise TagExists()
release_branch_prefix = 'release/flocker-'
if is_weekly_release(version):
base_branch_name = 'master'
elif is_pre_release(version) and get_pre_release(version) == 1:
base_branch_name = 'master'
elif get_doc_version(version) != version:
base_branch_name = release_branch_prefix + get_doc_version(version)
else:
if is_pre_release(version):
target_version = target_release(version)
else:
target_version = version
pre_releases = []
for tag in repo.tags:
try:
if (is_pre_release(tag.name) and
target_version == target_release(tag.name)):
pre_releases.append(tag.name)
except UnparseableVersion:
# The Flocker repository contains versions which are not
# currently considered valid versions.
pass
if not pre_releases:
raise NoPreRelease()
latest_pre_release = sorted(
pre_releases,
key=lambda pre_release: get_pre_release(pre_release))[-1]
if (is_pre_release(version) and get_pre_release(version) >
get_pre_release(latest_pre_release) + 1):
raise MissingPreRelease()
base_branch_name = release_branch_prefix + latest_pre_release
# We create a new branch from a branch, not a tag, because a maintenance
# or documentation change may have been applied to the branch and not the
# tag.
# The branch must be available locally for the next step.
repo.git.checkout(base_branch_name)
return (
branch for branch in repo.branches if
branch.name == base_branch_name).next()
def calculate_base_branch(version, path):
"""
The branch a release branch is created from depends on the release
type and sometimes which pre-releases have preceeded this.
:param bytes version: The version of Flocker to get a base branch for.
:param bytes path: See :func:`git.Repo.init`.
:returns: The base branch from which the new release branch was created.
"""
if not (is_release(version) or is_weekly_release(version)
or is_pre_release(version)):
raise NotARelease()
repo = Repo(path=path, search_parent_directories=True)
existing_tags = [tag for tag in repo.tags if tag.name == version]
if existing_tags:
raise TagExists()
release_branch_prefix = 'release/flocker-'
if is_weekly_release(version):
base_branch_name = 'master'
elif is_pre_release(version) and get_pre_release(version) == 1:
base_branch_name = 'master'
elif get_doc_version(version) != version:
base_branch_name = release_branch_prefix + get_doc_version(version)
else:
if is_pre_release(version):
target_version = target_release(version)
else:
target_version = version
pre_releases = []
for tag in repo.tags:
try:
if (is_pre_release(tag.name)
and target_version == target_release(tag.name)):
pre_releases.append(tag.name)
except UnparseableVersion:
# The Flocker repository contains versions which are not
# currently considered valid versions.
pass
if not pre_releases:
raise NoPreRelease()
latest_pre_release = sorted(
pre_releases,
key=lambda pre_release: get_pre_release(pre_release))[-1]
if (is_pre_release(version) and get_pre_release(version) >
get_pre_release(latest_pre_release) + 1):
raise MissingPreRelease()
base_branch_name = release_branch_prefix + latest_pre_release
# We create a new branch from a branch, not a tag, because a maintenance
# or documentation change may have been applied to the branch and not the
# tag.
# The branch must be available locally for the next step.
repo.git.checkout(base_branch_name)
return (branch for branch in repo.branches
if branch.name == base_branch_name).next()
