def install_cli_commands_yum(distribution, package_source):
"""
Install Flocker CLI on CentOS.
The ClusterHQ repo is added for downloading latest releases. If
``package_source`` contains a branch, then a BuildBot repo will also
be added to the package search path, to use in-development packages.
Note, the ClusterHQ repo is always enabled, to provide dependencies.
:param bytes distribution: The distribution the node is running.
:param PackageSource package_source: The source from which to install the
package.
:return: a sequence of commands to run on the distribution
"""
if package_source.branch:
# A development branch has been selected - add its Buildbot repo
use_development_branch = True
result_path = posixpath.join(
'/results/omnibus/', package_source.branch, distribution)
base_url = urljoin(package_source.build_server, result_path)
else:
use_development_branch = False
commands = [
sudo(command="yum install -y " + get_repository_url(
distribution=distribution,
flocker_version=get_installable_version(version))),
]
if use_development_branch:
repo = dedent(b"""\
[clusterhq-build]
name=clusterhq-build
baseurl=%s
gpgcheck=0
enabled=0
""") % (base_url,)
commands.append(put(content=repo,
path='/tmp/clusterhq-build.repo'))
commands.append(sudo_from_args([
'cp', '/tmp/clusterhq-build.repo',
'/etc/yum.repos.d/clusterhq-build.repo']))
repo_options = ['--enablerepo=clusterhq-build']
else:
repo_options = get_repo_options(
flocker_version=get_installable_version(version))
if package_source.os_version:
package = 'clusterhq-flocker-cli-%s' % (package_source.os_version,)
else:
package = 'clusterhq-flocker-cli'
# Install Flocker CLI and all dependencies
commands.append(sudo_from_args(
["yum", "install"] + repo_options + ["-y", package]))
return sequence(commands)
def install_cli_commands_yum(distribution, package_source):
"""
Install Flocker CLI on CentOS.
The ClusterHQ repo is added for downloading latest releases. If
``package_source`` contains a branch, then a BuildBot repo will also
be added to the package search path, to use in-development packages.
Note, the ClusterHQ repo is always enabled, to provide dependencies.
:param bytes distribution: The distribution the node is running.
:param PackageSource package_source: The source from which to install the
package.
:return: a sequence of commands to run on the distribution
"""
if package_source.branch:
# A development branch has been selected - add its Buildbot repo
use_development_branch = True
result_path = posixpath.join(
'/results/omnibus/', package_source.branch, distribution)
base_url = urljoin(package_source.build_server, result_path)
else:
use_development_branch = False
commands = [
sudo(command="yum install -y " + get_repository_url(
distribution=distribution,
flocker_version=get_installable_version(version))),
]
if use_development_branch:
repo = dedent(b"""\
[clusterhq-build]
name=clusterhq-build
baseurl=%s
gpgcheck=0
enabled=0
""") % (base_url,)
commands.append(put(content=repo,
path='/tmp/clusterhq-build.repo'))
commands.append(sudo_from_args([
'cp', '/tmp/clusterhq-build.repo',
'/etc/yum.repos.d/clusterhq-build.repo']))
repo_options = ['--enablerepo=clusterhq-build']
else:
repo_options = get_repo_options(
flocker_version=get_installable_version(version))
if package_source.os_version:
package = 'clusterhq-flocker-cli-%s' % (package_source.os_version,)
else:
package = 'clusterhq-flocker-cli'
# Install Flocker CLI and all dependencies
commands.append(sudo_from_args(
["yum", "install"] + repo_options + ["-y", package]))
return sequence(commands)
def install_commands_yum(package_name, distribution, package_source,
base_url):
"""
Install Flocker package on CentOS.
The ClusterHQ repo is added for downloading latest releases. If
``package_source`` contains a branch, then a BuildBot repo will also
be added to the package search path, to use in-development packages.
Note, the ClusterHQ repo is always enabled, to provide dependencies.
:param str package_name: The name of the package to install.
:param bytes distribution: The distribution the node is running.
:param PackageSource package_source: The source from which to install the
package.
:param base_url: URL of repository, or ``None`` if we're not using
development branch.
:return: a sequence of commands to run on the distribution
"""
commands = [
# If package has previously been installed, 'yum install' fails,
# so check if it is installed first.
run(
command="yum list installed clusterhq-release || yum install -y {0}".format( # noqa
get_repository_url(
distribution=distribution,
flocker_version=get_installable_version(version)))),
]
if base_url is not None:
repo = dedent(b"""\
[clusterhq-build]
name=clusterhq-build
baseurl=%s
gpgcheck=0
enabled=0
""") % (base_url,)
commands.append(put(content=repo,
path='/tmp/clusterhq-build.repo'))
commands.append(run_from_args([
'cp', '/tmp/clusterhq-build.repo',
'/etc/yum.repos.d/clusterhq-build.repo']))
repo_options = ['--enablerepo=clusterhq-build']
else:
repo_options = get_repo_options(
flocker_version=get_installable_version(version))
if package_source.os_version:
package_name += '-%s' % (package_source.os_version,)
# Install package and all dependencies:
commands.append(run_from_args(
["yum", "install"] + repo_options + ["-y", package_name]))
return sequence(commands)
def test_version_prompt(self):
"""
The ``version-prompt`` directive replaces the placemarker
``|latest-installable|`` in a source file with the current
installable version in the output file.
"""
source_directory = self.make_temporary_directory()
source_file = source_directory.child('contents.rst')
source_file.setContent(dedent('''
.. version-prompt:: bash $
$ PRE-|latest-installable|-POST
'''))
destination_directory = self.make_temporary_directory()
run_process([
'sphinx-build', '-b', 'html',
'-C', # don't look for config file, use -D flags instead
'-D', 'extensions=flocker.docs.version_extensions',
# directory containing source/config files
source_directory.path,
# directory containing build files
destination_directory.path,
source_file.path]) # source file to process
expected = 'PRE-{}-POST'.format(get_installable_version(version))
content = destination_directory.child('contents.html').getContent()
self.assertIn(expected, content)
def make_changed_file(path, env):
"""
Given the path to a template file, write a new file with:
* The same filename, except without '.template' at the end.
* A placeholder in the new file changed to the latest installable
version of Flocker.
This new file will be deleted on build completion.
:param unicode path: The path to a template file.
:param sphinx.environment.BuildEnvironment env: The Sphinx build
environment
"""
def remove_file(path):
try:
os.remove(path)
except OSError:
pass
latest = get_installable_version(version)
new_path = remove_extension(path)
with open(path, 'r') as templated_file:
with open(new_path, 'w') as new_file:
new_file.write(templated_file.read().replace(PLACEHOLDER, latest))
env.app.connect('build-finished',
lambda self, *args: remove_file(new_path))
def make_changed_file(path, env):
"""
Given the path to a template file, write a new file with:
* The same filename, except without '.template' at the end.
* A placeholder in the new file changed to the latest installable
version of Flocker.
This new file will be deleted on build completion.
:param unicode path: The path to a template file.
:param sphinx.environment.BuildEnvironment env: The Sphinx build
environment
"""
def remove_file(path):
try:
os.remove(path)
except OSError:
pass
latest = get_installable_version(version)
new_path = remove_extension(path)
with open(path, 'r') as templated_file:
with open(new_path, 'w') as new_file:
new_file.write(templated_file.read().replace(PLACEHOLDER, latest))
env.app.connect('build-finished',
lambda self, *args: remove_file(new_path))
def task_cli_pip_install(
venv_name='flocker-client', package_source=PackageSource()):
"""
Install the Flocker client into a virtualenv using pip.
:param bytes venv_name: Name for the virtualenv.
:param package_source: Package source description
:return: an Effect to install the client.
"""
vers = package_source.version
if vers is None:
vers = version
url = (
'https://{bucket}.s3.amazonaws.com/{key}/'
'Flocker-{version}-py2-none-any.whl'.format(
bucket=ARCHIVE_BUCKET, key='python',
version=get_installable_version(vers))
)
return sequence([
run_from_args(
['virtualenv', '--python=/usr/bin/python2.7', venv_name]),
run_from_args(['source', '{}/bin/activate'.format(venv_name)]),
run_from_args(['pip', 'install', '--upgrade', 'pip']),
run_from_args(
['pip', 'install', url]),
])
def test_version_prompt(self):
"""
The ``version-prompt`` directive replaces the placemarker
``|latest-installable|`` in a source file with the current
installable version in the output file.
"""
temp_dir = FilePath(self.mktemp())
temp_dir.makedirs()
source_file = temp_dir.child('contents.rst')
source_file.setContent(
dedent('''
.. version-prompt:: bash $
$ PRE-|latest-installable|-POST
'''))
run_process([
'sphinx-build',
'-b',
'html',
'-C', # don't look for config file, use -D flags instead
'-D',
'extensions=flocker.docs.version_extensions',
temp_dir.path, # directory containing source/config files
temp_dir.path, # directory containing build files
source_file.path
]) # source file to process
expected = 'PRE-{}-POST'.format(get_installable_version(version))
content = temp_dir.child('contents.html').getContent()
self.assertIn(expected, content)
def task_cli_pip_install(
venv_name='flocker-client', package_source=PackageSource()):
"""
Install the Flocker client into a virtualenv using pip.
:param bytes venv_name: Name for the virtualenv.
:param package_source: Package source description
:return: an Effect to install the client.
"""
vers = package_source.version
if vers is None:
vers = version
url = (
'https://{bucket}.s3.amazonaws.com/{key}/'
'Flocker-{version}-py2-none-any.whl'.format(
bucket=ARCHIVE_BUCKET, key='python',
version=get_installable_version(vers))
)
return sequence([
run_from_args(
['virtualenv', '--python=/usr/bin/python2.7', venv_name]),
run_from_args(['source', '{}/bin/activate'.format(venv_name)]),
run_from_args(['pip', 'install', '--upgrade', 'pip']),
run_from_args(
['pip', 'install', url]),
])
def run(self):
task = getattr(tasks, 'task_%s' % (self.arguments[0], ))
prompt = self.options.get('prompt', '$')
if len(self.arguments) > 1:
# Some tasks can include the latest installable version as (part
# of) an argument. This replaces a placeholder with that version.
arguments = self.arguments[1].split()
latest = get_installable_version(version)
task_arguments = [
item.replace(PLACEHOLDER, latest).encode("utf-8")
for item in arguments
]
else:
task_arguments = []
commands = task(*task_arguments)
lines = ['.. prompt:: bash %s,> auto' % (prompt, ), '']
try:
command_lines = run_for_docs(commands)
except NoPerformerFoundError as e:
raise self.error("task: %s not supported" %
(type(e.args[0]).__name__, ))
except SequenceFailed as e:
print e.error
for command_line in command_lines:
# handler can return either a string or a list. If it returns a
# list, treat the elements after the first as continuation lines.
if isinstance(command_line, list):
lines.append(' %s %s' % (
prompt,
command_line[0],
))
lines.extend(
[' > %s' % (line, ) for line in command_line[1:]])
else:
lines.append(' %s %s' % (
prompt,
command_line,
))
# The following three lines record (some?) of the dependencies of the
# directive, so automatic regeneration happens. Specifically, it
# records this file, and the file where the task is declared.
task_file = getsourcefile(task)
tasks_file = getsourcefile(tasks)
self.state.document.settings.record_dependencies.add(task_file)
self.state.document.settings.record_dependencies.add(tasks_file)
self.state.document.settings.record_dependencies.add(__file__)
node = nodes.Element()
text = StringList(lines)
self.state.nested_parse(text, self.content_offset, node)
return node.children
def _get_wheel_version(package_source):
"""
Get the latest available wheel version for the specified package source.
If package source version is not set, the latest installable release
will be installed. Note, branch is never used for wheel
installations, since the wheel file is not created for branches.
:param PackageSource package_source: The source from which to install the
package.
:return: a string containing the previous installable version of
either the package version, or, if that is not specified, of the
current version.
"""
return get_installable_version(package_source.version or version)
def _get_wheel_version(package_source):
"""
Get the latest available wheel version for the specified package source.
If package source version is not set, the latest installable release
will be installed. Note, branch is never used for wheel
installations, since the wheel file is not created for branches.
:param PackageSource package_source: The source from which to install the
package.
:return: a string containing the previous installable version of
either the package version, or, if that is not specified, of the
current version.
"""
return get_installable_version(package_source.version or version)
def run(self):
task = getattr(tasks, 'task_%s' % (self.arguments[0],))
prompt = self.options.get('prompt', '$')
if len(self.arguments) > 1:
# Some tasks can include the latest installable version as (part
# of) an argument. This replaces a placeholder with that version.
arguments = self.arguments[1].split()
latest = get_installable_version(version)
task_arguments = [item.replace(PLACEHOLDER, latest).encode("utf-8")
for item in arguments]
else:
task_arguments = []
commands = task(*task_arguments)
lines = ['.. prompt:: bash %s,> auto' % (prompt,), '']
try:
command_lines = run_for_docs(commands)
except NoPerformerFoundError as e:
raise self.error("task: %s not supported"
% (type(e.args[0]).__name__,))
except SequenceFailed as e:
print e.error
for command_line in command_lines:
# handler can return either a string or a list. If it returns a
# list, treat the elements after the first as continuation lines.
if isinstance(command_line, list):
lines.append(' %s %s' % (prompt, command_line[0],))
lines.extend([' > %s' % (line,)
for line in command_line[1:]])
else:
lines.append(' %s %s' % (prompt, command_line,))
# The following three lines record (some?) of the dependencies of the
# directive, so automatic regeneration happens. Specifically, it
# records this file, and the file where the task is declared.
task_file = getsourcefile(task)
tasks_file = getsourcefile(tasks)
self.state.document.settings.record_dependencies.add(task_file)
self.state.document.settings.record_dependencies.add(tasks_file)
self.state.document.settings.record_dependencies.add(__file__)
node = nodes.Element()
text = StringList(lines)
self.state.nested_parse(text, self.content_offset, node)
return node.children
def cli_pkg_test(package_source=PackageSource()):
"""
Check that the Flocker CLI is working and has the expected version.
:param PackageSource package_source: The source from which to install the
package.
:return: An ``Effect`` to pass to a ``Dispatcher`` that supports
``Sequence``, ``Run``, ``Sudo``, ``Comment``, and ``Put``.
"""
expected = package_source.version
if not expected:
if package_source.branch:
# If branch is set but version isn't, we don't know the
# latest version. In this case, just check that the version
# can be displayed.
return run('flocker-deploy --version')
else:
# If neither branch nor version is set, the latest
# installable release will be installed.
expected = get_installable_version(version)
return run('test `flocker-deploy --version` = {}'.format(quote(expected)))
def cli_pkg_test(package_source=PackageSource()):
"""
Check that the Flocker CLI is working and has the expected version.
:param PackageSource package_source: The source from which to install the
package.
:return: An ``Effect`` to pass to a ``Dispatcher`` that supports
``Sequence``, ``Run``, ``Sudo``, ``Comment``, and ``Put``.
"""
expected = package_source.version
if not expected:
if package_source.branch:
# If branch is set but version isn't, we don't know the
# latest version. In this case, just check that the version
# can be displayed.
return run('flocker-deploy --version')
else:
# If neither branch nor version is set, the latest
# installable release will be installed.
expected = get_installable_version(version)
return run('test `flocker-deploy --version` = {}'.format(quote(expected)))
def install_cli_commands_ubuntu(distribution, package_source):
"""
Install flocker CLI on Ubuntu.
The ClusterHQ repo is added for downloading latest releases. If
``package_source`` contains a branch, then a BuildBot repo will also
be added to the package search path, to use in-development packages.
Note, the ClusterHQ repo is always enabled, to provide dependencies.
:param bytes distribution: The distribution the node is running.
:param PackageSource package_source: The source from which to install the
package.
:return: a sequence of commands to run on the distribution
"""
if package_source.branch:
# A development branch has been selected - add its Buildbot repo
use_development_branch = True
result_path = posixpath.join(
'/results/omnibus/', package_source.branch, distribution)
base_url = urljoin(package_source.build_server, result_path)
else:
use_development_branch = False
commands = [
# Minimal images often have cleared apt caches and are missing
# packages that are common in a typical release. These commands
# ensure that we start from a good base system with the required
# capabilities, particularly that the add-apt-repository command
# is available, and HTTPS URLs are supported.
sudo_from_args(["apt-get", "update"]),
sudo_from_args([
"apt-get", "-y", "install", "apt-transport-https",
"software-properties-common"]),
# Add ClusterHQ repo for installation of Flocker packages.
sudo(command='add-apt-repository -y "deb {} /"'.format(
get_repository_url(
distribution=distribution,
flocker_version=get_installable_version(version))))
]
if use_development_branch:
# Add BuildBot repo for running tests
commands.append(sudo_from_args([
"add-apt-repository", "-y", "deb {} /".format(base_url)]))
# During a release, the ClusterHQ repo may contain packages with
# a higher version number than the Buildbot repo for a branch.
# Use a pin file to ensure that any Buildbot repo has higher
# priority than the ClusterHQ repo.
buildbot_host = urlparse(package_source.build_server).hostname
commands.append(put(dedent('''\
Package: *
Pin: origin {}
Pin-Priority: 900
'''.format(buildbot_host)), '/tmp/apt-pref'))
commands.append(sudo_from_args([
'mv', '/tmp/apt-pref', '/etc/apt/preferences.d/buildbot-900']))
# Update to read package info from new repos
commands.append(sudo_from_args(["apt-get", "update"]))
if package_source.os_version:
package = 'clusterhq-flocker-cli=%s' % (package_source.os_version,)
else:
package = 'clusterhq-flocker-cli'
# Install Flocker CLI and all dependencies
commands.append(sudo_from_args([
'apt-get', '-y', '--force-yes', 'install', package]))
return sequence(commands)
def install_commands_yum(package_name, distribution, package_source, base_url):
"""
Install Flocker package on CentOS.
The ClusterHQ repo is added for downloading latest releases. If
``package_source`` contains a branch, then a BuildBot repo will also
be added to the package search path, to use in-development packages.
Note, the ClusterHQ repo is always enabled, to provide dependencies.
:param str package_name: The name of the package to install.
:param bytes distribution: The distribution the node is running.
:param PackageSource package_source: The source from which to install the
package.
:param base_url: URL of repository, or ``None`` if we're not using
development branch.
:return: a sequence of commands to run on the distribution
"""
flocker_version = package_source.version
if not flocker_version:
# support empty values other than None, as '' sometimes used to
# indicate latest version, due to previous behaviour
flocker_version = get_installable_version(version)
repo_package_name = 'clusterhq-release'
commands = [
# If package has previously been installed, 'yum install' fails,
# so check if it is installed first.
run(command="yum list installed {} || yum install -y {}".format(
quote(repo_package_name),
get_repository_url(distribution=distribution,
flocker_version=flocker_version))),
]
if base_url is not None:
repo = dedent(b"""\
[clusterhq-build]
name=clusterhq-build
baseurl=%s
gpgcheck=0
enabled=0
# There is a distinct clusterhq-build repository for each branch.
# The metadata across these different repositories varies. Version
# numbers are not comparable. A version which exists in one likely
# does not exist in another. In order to support switching between
# branches (and therefore between clusterhq-build repositories),
# tell yum to always update metadata for this repository.
metadata_expire=0
""") % (base_url, )
commands.append(put(content=repo, path='/tmp/clusterhq-build.repo'))
commands.append(
run_from_args([
'cp', '/tmp/clusterhq-build.repo',
'/etc/yum.repos.d/clusterhq-build.repo'
]))
repo_options = ['--enablerepo=clusterhq-build']
else:
repo_options = get_repo_options(
flocker_version=get_installable_version(version))
os_version = package_source.os_version()
if os_version:
package_name += '-%s' % (os_version, )
# Install package and all dependencies:
commands.append(
run_from_args(["yum", "install"] + repo_options +
["-y", package_name]))
return sequence(commands)
def run(self):
latest = get_installable_version(version)
self.content = [
item.replace(PLACEHOLDER, latest) for item in self.content
]
return sphinx_prompt.PromptDirective.run(self)
def install_commands_ubuntu(package_name, distribution, package_source,
base_url):
"""
Install Flocker package on Ubuntu.
The ClusterHQ repo is added for downloading latest releases. If
``package_source`` contains a branch, then a BuildBot repo will also
be added to the package search path, to use in-development packages.
Note, the ClusterHQ repo is always enabled, to provide dependencies.
:param bytes distribution: The distribution the node is running.
:param PackageSource package_source: The source from which to install the
package.
:param base_url: URL of repository, or ``None`` if we're not using
development branch.
:return: a sequence of commands to run on the distribution
"""
commands = [
# Minimal images often have cleared apt caches and are missing
# packages that are common in a typical release. These commands
# ensure that we start from a good base system with the required
# capabilities, particularly that the add-apt-repository command
# is available, and HTTPS URLs are supported.
run_from_args(["apt-get", "update"]),
run_from_args([
"apt-get", "-y", "install", "apt-transport-https",
"software-properties-common"]),
# Add ClusterHQ repo for installation of Flocker packages.
run(command='add-apt-repository -y "deb {} /"'.format(
get_repository_url(
distribution=distribution,
flocker_version=get_installable_version(version))))
]
if base_url is not None:
# Add BuildBot repo for running tests
commands.append(run_from_args([
"add-apt-repository", "-y", "deb {} /".format(base_url)]))
# During a release, the ClusterHQ repo may contain packages with
# a higher version number than the Buildbot repo for a branch.
# Use a pin file to ensure that any Buildbot repo has higher
# priority than the ClusterHQ repo.
buildbot_host = urlparse(package_source.build_server).hostname
commands.append(put(dedent('''\
Package: *
Pin: origin {}
Pin-Priority: 900
'''.format(buildbot_host)), '/tmp/apt-pref'))
commands.append(run_from_args([
'mv', '/tmp/apt-pref', '/etc/apt/preferences.d/buildbot-900']))
# Update to read package info from new repos
commands.append(run_from_args(["apt-get", "update"]))
if package_source.os_version:
package_name += '=%s' % (package_source.os_version,)
# Install package and all dependencies
commands.append(run_from_args([
'apt-get', '-y', '--force-yes', 'install', package_name]))
return sequence(commands)
def install_commands_ubuntu(package_name, distribution, package_source,
base_url):
"""
Install Flocker package on Ubuntu.
The ClusterHQ repo is added for downloading latest releases. If
``package_source`` contains a branch, then a BuildBot repo will also
be added to the package search path, to use in-development packages.
Note, the ClusterHQ repo is always enabled, to provide dependencies.
:param bytes distribution: The distribution the node is running.
:param PackageSource package_source: The source from which to install the
package.
:param base_url: URL of repository, or ``None`` if we're not using
development branch.
:return: a sequence of commands to run on the distribution
"""
flocker_version = package_source.version
if not flocker_version:
# support empty values other than None, as '' sometimes used to
# indicate latest version, due to previous behaviour
flocker_version = get_installable_version(version)
commands = [
# Minimal images often have cleared apt caches and are missing
# packages that are common in a typical release. These commands
# ensure that we start from a good base system with the required
# capabilities, particularly that the add-apt-repository command
# is available, and HTTPS URLs are supported.
run_from_args(["apt-get", "update"]),
run_from_args([
"apt-get", "-y", "install", "apt-transport-https",
"software-properties-common"
]),
# Add ClusterHQ repo for installation of Flocker packages.
run(command='add-apt-repository -y "deb {} /"'.format(
get_repository_url(distribution=distribution,
flocker_version=flocker_version)))
]
if base_url is not None:
# Add BuildBot repo for running tests
commands.append(
run_from_args(
["add-apt-repository", "-y", "deb {} /".format(base_url)]))
# During a release, the ClusterHQ repo may contain packages with
# a higher version number than the Buildbot repo for a branch.
# Use a pin file to ensure that any Buildbot repo has higher
# priority than the ClusterHQ repo. We only add the Buildbot
# repo when a branch is specified, so it wil not interfere with
# attempts to install a release (when no branch is specified).
buildbot_host = urlparse(package_source.build_server).hostname
commands.append(
put(
dedent('''\
Package: *
Pin: origin {}
Pin-Priority: 700
'''.format(buildbot_host)), '/tmp/apt-pref'))
commands.append(
run_from_args(
['mv', '/tmp/apt-pref',
'/etc/apt/preferences.d/buildbot-700']))
# Update to read package info from new repos
commands.append(run_from_args(["apt-get", "update"]))
os_version = package_source.os_version()
if os_version:
# Set the version of the top-level package
package_name += '=%s' % (os_version, )
# If a specific version is required, ensure that the version for
# all ClusterHQ packages is consistent. This prevents conflicts
# between the top-level package, which may depend on a lower
# version of a dependency, and apt, which wants to install the
# most recent version. Note that this trumps the Buildbot
# pinning above.
commands.append(
put(
dedent('''\
Package: clusterhq-*
Pin: version {}
Pin-Priority: 900
'''.format(os_version)), '/tmp/apt-pref'))
commands.append(
run_from_args([
'mv', '/tmp/apt-pref', '/etc/apt/preferences.d/clusterhq-900'
]))
# Install package and all dependencies
commands.append(
run_from_args(
['apt-get', '-y', '--force-yes', 'install', package_name]))
return sequence(commands)
def test_upgrade(self, cluster):
"""
Given a dataset created and used with the previously installable
version of flocker, uninstalling the previous version of flocker and
installing HEAD does not destroy the data on the dataset.
"""
node = cluster.nodes[0]
SAMPLE_STR = '123456' * 100
upgrade_from_version = get_installable_version(HEAD_FLOCKER_VERSION)
# Get the initial flocker version and setup a cleanup call to restore
# flocker to that version when the test is done.
d = cluster.client.version()
original_package_source = [None]
def setup_restore_original_flocker(version):
version_bytes = version.get('flocker', u'').encode('ascii')
original_package_source[0] = (self._get_package_source(
default_version=version_bytes or None))
self.addCleanup(lambda: cluster.install_flocker_version(
original_package_source[0]))
return version
d.addCallback(setup_restore_original_flocker)
# Double check that the nodes are clean before we destroy the persisted
# state.
d.addCallback(lambda _: cluster.clean_nodes())
# Downgrade flocker to the most recent released version.
d.addCallback(lambda _: cluster.install_flocker_version(
PackageSource(version=upgrade_from_version),
destroy_persisted_state=True))
# Create a dataset with the code from the most recent release.
d.addCallback(lambda _: create_dataset(self, cluster, node=node))
first_dataset = [None]
# Write some data to a file in the dataset.
def write_to_file(dataset):
first_dataset[0] = dataset
return node.run_as_root([
'bash', '-c',
'echo "%s" > %s' %
(SAMPLE_STR, os.path.join(dataset.path.path, 'test.txt'))
])
d.addCallback(write_to_file)
# Upgrade flocker to the code under test.
d.addCallback(lambda _: cluster.install_flocker_version(
original_package_source[0]))
# Create a new dataset to convince ourselves that the new code is
# running.
d.addCallback(lambda _: create_dataset(self, cluster, node=node))
# Wait for the first dataset to be mounted again.
d.addCallback(lambda _: cluster.wait_for_dataset(first_dataset[0]))
# Verify that the file still has its contents.
def cat_and_verify_file(dataset):
output = []
file_catting = node.run_as_root([
'bash', '-c',
'cat %s' % (os.path.join(dataset.path.path, 'test.txt'))
],
handle_stdout=output.append)
def verify_file(_):
file_contents = ''.join(output)
self.assertEqual(file_contents, SAMPLE_STR)
file_catting.addCallback(verify_file)
return file_catting
d.addCallback(cat_and_verify_file)
return d
def run(self):
latest = get_installable_version(version)
self.content = [item.replace(PLACEHOLDER, latest) for
item in self.content]
return CodeBlock.run(self)
def install_commands_yum(package_name, distribution, package_source, base_url):
"""
Install Flocker package on CentOS.
The ClusterHQ repo is added for downloading latest releases. If
``package_source`` contains a branch, then a BuildBot repo will also
be added to the package search path, to use in-development packages.
Note, the ClusterHQ repo is always enabled, to provide dependencies.
:param str package_name: The name of the package to install.
:param bytes distribution: The distribution the node is running.
:param PackageSource package_source: The source from which to install the
package.
:param base_url: URL of repository, or ``None`` if we're not using
development branch.
:return: a sequence of commands to run on the distribution
"""
flocker_version = package_source.version
if not flocker_version:
# support empty values other than None, as '' sometimes used to
# indicate latest version, due to previous behaviour
flocker_version = get_installable_version(version)
repo_package_name = 'clusterhq-release'
commands = [
# If package has previously been installed, 'yum install' fails,
# so check if it is installed first.
run(
command="yum list installed {} || yum install -y {}".format(
quote(repo_package_name),
get_repository_url(
distribution=distribution,
flocker_version=flocker_version))),
]
if base_url is not None:
repo = dedent(b"""\
[clusterhq-build]
name=clusterhq-build
baseurl=%s
gpgcheck=0
enabled=0
# There is a distinct clusterhq-build repository for each branch.
# The metadata across these different repositories varies. Version
# numbers are not comparable. A version which exists in one likely
# does not exist in another. In order to support switching between
# branches (and therefore between clusterhq-build repositories),
# tell yum to always update metadata for this repository.
metadata_expire=0
""") % (base_url,)
commands.append(put(content=repo,
path='/tmp/clusterhq-build.repo'))
commands.append(run_from_args([
'cp', '/tmp/clusterhq-build.repo',
'/etc/yum.repos.d/clusterhq-build.repo']))
repo_options = ['--enablerepo=clusterhq-build']
else:
repo_options = get_repo_options(
flocker_version=get_installable_version(version))
os_version = package_source.os_version()
if os_version:
package_name += '-%s' % (os_version,)
# Install package and all dependencies:
commands.append(run_from_args(
["yum", "install"] + repo_options + ["-y", package_name]))
return sequence(commands)
def task_install_flocker(
distribution=None,
package_source=PackageSource(),
):
"""
Install flocker cluster on a distribution.
The ClusterHQ repo is added for downloading latest releases. If
``package_source`` contains a branch, then a BuildBot repo will also
be added to the package search path, to use in-development packages.
Note, the ClusterHQ repo is always enabled, to provide dependencies.
:param bytes distribution: The distribution the node is running.
:param PackageSource package_source: The source from which to install the
package.
:raises: ``UnsupportedDistribution`` if the distribution is unsupported.
"""
if package_source.branch:
# A development branch has been selected - add its Buildbot repo
use_development_branch = True
result_path = posixpath.join(
'/results/omnibus/', package_source.branch, distribution)
base_url = urljoin(package_source.build_server, result_path)
else:
use_development_branch = False
if distribution in ('ubuntu-14.04', 'ubuntu-15.04'):
commands = [
# Ensure add-apt-repository command and HTTPS URLs are supported
# FLOC-1880 will ensure these are necessary and sufficient
run_from_args([
"apt-get", "-y", "install", "apt-transport-https",
"software-properties-common"]),
# Add Docker repo for recent Docker versions
run_from_args([
"add-apt-repository", "-y", "ppa:james-page/docker"]),
# Add ClusterHQ repo for installation of Flocker packages.
run(command='add-apt-repository -y "deb {} /"'.format(
get_repository_url(
distribution=distribution,
flocker_version=get_installable_version(version)))),
]
if use_development_branch:
# Add BuildBot repo for testing
commands.append(run_from_args([
"add-apt-repository", "-y", "deb {} /".format(base_url)]))
# During a release, the ClusterHQ repo may contain packages with
# a higher version number than the Buildbot repo for a branch.
# Use a pin file to ensure that any Buildbot repo has higher
# priority than the ClusterHQ repo.
buildbot_host = urlparse(package_source.build_server).hostname
commands.append(put(
dedent('''\
Package: *
Pin: origin {}
Pin-Priority: 900
'''.format(buildbot_host)),
'/etc/apt/preferences.d/buildbot-900'))
commands += [
# Update to read package info from new repos
run_from_args([
"apt-get", "update"]),
]
if package_source.os_version:
package = 'clusterhq-flocker-node=%s' % (
package_source.os_version,)
else:
package = 'clusterhq-flocker-node'
# Install Flocker node and all dependencies
commands.append(run_from_args([
'apt-get', '-y', '--force-yes', 'install', package]))
return sequence(commands)
elif distribution in ('centos-7',):
commands = [
run(command="yum clean all"),
run(command="yum install -y " + get_repository_url(
distribution=distribution,
flocker_version=get_installable_version(version)))
]
if use_development_branch:
repo = dedent(b"""\
[clusterhq-build]
name=clusterhq-build
baseurl=%s
gpgcheck=0
enabled=0
""") % (base_url,)
commands.append(put(content=repo,
path='/etc/yum.repos.d/clusterhq-build.repo'))
repo_options = ['--enablerepo=clusterhq-build']
else:
repo_options = get_repo_options(
flocker_version=get_installable_version(version))
if package_source.os_version:
package = 'clusterhq-flocker-node-%s' % (
package_source.os_version,)
else:
package = 'clusterhq-flocker-node'
commands.append(run_from_args(
["yum", "install"] + repo_options + ["-y", package]))
return sequence(commands)
else:
raise UnsupportedDistribution()
def install_commands_ubuntu(package_name, distribution, package_source,
base_url):
"""
Install Flocker package on Ubuntu.
The ClusterHQ repo is added for downloading latest releases. If
``package_source`` contains a branch, then a BuildBot repo will also
be added to the package search path, to use in-development packages.
Note, the ClusterHQ repo is always enabled, to provide dependencies.
:param bytes distribution: The distribution the node is running.
:param PackageSource package_source: The source from which to install the
package.
:param base_url: URL of repository, or ``None`` if we're not using
development branch.
:return: a sequence of commands to run on the distribution
"""
flocker_version = package_source.version
if not flocker_version:
# support empty values other than None, as '' sometimes used to
# indicate latest version, due to previous behaviour
flocker_version = get_installable_version(version)
commands = [
# Minimal images often have cleared apt caches and are missing
# packages that are common in a typical release. These commands
# ensure that we start from a good base system with the required
# capabilities, particularly that the add-apt-repository command
# is available, and HTTPS URLs are supported.
run_from_args(["apt-get", "update"]),
run_from_args([
"apt-get", "-y", "install", "apt-transport-https",
"software-properties-common"]),
# Add ClusterHQ repo for installation of Flocker packages.
run(command='add-apt-repository -y "deb {} /"'.format(
get_repository_url(
distribution=distribution,
flocker_version=flocker_version)))
]
if base_url is not None:
# Add BuildBot repo for running tests
commands.append(run_from_args([
"add-apt-repository", "-y", "deb {} /".format(base_url)]))
# During a release, the ClusterHQ repo may contain packages with
# a higher version number than the Buildbot repo for a branch.
# Use a pin file to ensure that any Buildbot repo has higher
# priority than the ClusterHQ repo. We only add the Buildbot
# repo when a branch is specified, so it wil not interfere with
# attempts to install a release (when no branch is specified).
buildbot_host = urlparse(package_source.build_server).hostname
commands.append(put(dedent('''\
Package: *
Pin: origin {}
Pin-Priority: 700
'''.format(buildbot_host)), '/tmp/apt-pref'))
commands.append(run_from_args([
'mv', '/tmp/apt-pref', '/etc/apt/preferences.d/buildbot-700']))
# Update to read package info from new repos
commands.append(run_from_args(["apt-get", "update"]))
os_version = package_source.os_version()
if os_version:
# Set the version of the top-level package
package_name += '=%s' % (os_version,)
# If a specific version is required, ensure that the version for
# all ClusterHQ packages is consistent. This prevents conflicts
# between the top-level package, which may depend on a lower
# version of a dependency, and apt, which wants to install the
# most recent version. Note that this trumps the Buildbot
# pinning above.
commands.append(put(dedent('''\
Package: clusterhq-*
Pin: version {}
Pin-Priority: 900
'''.format(os_version)), '/tmp/apt-pref'))
commands.append(run_from_args([
'mv', '/tmp/apt-pref', '/etc/apt/preferences.d/clusterhq-900']))
# Install package and all dependencies
commands.append(run_from_args([
'apt-get', '-y', '--force-yes', 'install', package_name]))
return sequence(commands)
def run(self):
latest = get_installable_version(version)
self.content = [item.replace(PLACEHOLDER, latest) for
item in self.content]
return sphinx_prompt.PromptDirective.run(self)
def run(self):
latest = get_installable_version(version)
self.content = [
item.replace(PLACEHOLDER, latest) for item in self.content
]
return CodeBlock.run(self)
def install_cli_commands_ubuntu(distribution, package_source):
"""
Install flocker CLI on Ubuntu.
The ClusterHQ repo is added for downloading latest releases. If
``package_source`` contains a branch, then a BuildBot repo will also
be added to the package search path, to use in-development packages.
Note, the ClusterHQ repo is always enabled, to provide dependencies.
:param bytes distribution: The distribution the node is running.
:param PackageSource package_source: The source from which to install the
package.
:return: a sequence of commands to run on the distribution
"""
if package_source.branch:
# A development branch has been selected - add its Buildbot repo
use_development_branch = True
result_path = posixpath.join(
'/results/omnibus/', package_source.branch, distribution)
base_url = urljoin(package_source.build_server, result_path)
else:
use_development_branch = False
commands = [
# Minimal images often have cleared apt caches and are missing
# packages that are common in a typical release. These commands
# ensure that we start from a good base system with the required
# capabilities, particularly that the add-apt-repository command
# and HTTPS URLs are supported.
# FLOC-1880 will ensure these are necessary and sufficient.
sudo_from_args(["apt-get", "update"]),
sudo_from_args([
"apt-get", "-y", "install", "apt-transport-https",
"software-properties-common"]),
# Add ClusterHQ repo for installation of Flocker packages.
sudo(command='add-apt-repository -y "deb {} /"'.format(
get_repository_url(
distribution=distribution,
flocker_version=get_installable_version(version))))
]
if use_development_branch:
# Add BuildBot repo for running tests
commands.append(sudo_from_args([
"add-apt-repository", "-y", "deb {} /".format(base_url)]))
# During a release, the ClusterHQ repo may contain packages with
# a higher version number than the Buildbot repo for a branch.
# Use a pin file to ensure that any Buildbot repo has higher
# priority than the ClusterHQ repo.
buildbot_host = urlparse(package_source.build_server).hostname
commands.append(put(dedent('''\
Package: *
Pin: origin {}
Pin-Priority: 900
'''.format(buildbot_host)), '/tmp/apt-pref'))
commands.append(sudo_from_args([
'mv', '/tmp/apt-pref', '/etc/apt/preferences.d/buildbot-900']))
# Update to read package info from new repos
commands.append(sudo_from_args(["apt-get", "update"]))
if package_source.os_version:
package = 'clusterhq-flocker-cli=%s' % (package_source.os_version,)
else:
package = 'clusterhq-flocker-cli'
# Install Flocker CLI and all dependencies
commands.append(sudo_from_args([
'apt-get', '-y', '--force-yes', 'install', package]))
return sequence(commands)
def test_upgrade(self, cluster):
"""
Given a dataset created and used with the previously installable
version of flocker, uninstalling the previous version of flocker and
installing HEAD does not destroy the data on the dataset.
"""
node = cluster.nodes[0]
SAMPLE_STR = '123456' * 100
upgrade_from_version = get_installable_version(HEAD_FLOCKER_VERSION)
# Get the initial flocker version and setup a cleanup call to restore
# flocker to that version when the test is done.
d = cluster.client.version()
original_package_source = [None]
def setup_restore_original_flocker(version):
version_bytes = version.get('flocker', u'').encode('ascii')
original_package_source[0] = (
self._get_package_source(
default_version=version_bytes or None)
)
self.addCleanup(
lambda: cluster.install_flocker_version(
original_package_source[0]))
return version
d.addCallback(setup_restore_original_flocker)
# Double check that the nodes are clean before we destroy the persisted
# state.
d.addCallback(lambda _: cluster.clean_nodes())
# Downgrade flocker to the most recent released version.
d.addCallback(
lambda _: cluster.install_flocker_version(
PackageSource(version=upgrade_from_version),
destroy_persisted_state=True
)
)
# Create a dataset with the code from the most recent release.
d.addCallback(lambda _: create_dataset(self, cluster, node=node))
first_dataset = [None]
# Write some data to a file in the dataset.
def write_to_file(dataset):
first_dataset[0] = dataset
return node.run_as_root(
['bash', '-c', 'echo "%s" > %s' % (
SAMPLE_STR, os.path.join(dataset.path.path, 'test.txt'))])
d.addCallback(write_to_file)
# Upgrade flocker to the code under test.
d.addCallback(lambda _: cluster.install_flocker_version(
original_package_source[0]))
# Create a new dataset to convince ourselves that the new code is
# running.
d.addCallback(lambda _: create_dataset(self, cluster, node=node))
# Wait for the first dataset to be mounted again.
d.addCallback(lambda _: cluster.wait_for_dataset(first_dataset[0]))
# Verify that the file still has its contents.
def cat_and_verify_file(dataset):
output = []
file_catting = node.run_as_root(
['bash', '-c', 'cat %s' % (
os.path.join(dataset.path.path, 'test.txt'))],
handle_stdout=output.append)
def verify_file(_):
file_contents = ''.join(output)
self.assertEqual(file_contents, SAMPLE_STR)
file_catting.addCallback(verify_file)
return file_catting
d.addCallback(cat_and_verify_file)
return d
def task_install_flocker(
distribution=None,
package_source=PackageSource(),
):
"""
Install flocker cluster on a distribution.
The ClusterHQ repo is added for downloading latest releases. If
``package_source`` contains a branch, then a BuildBot repo will also
be added to the package search path, to use in-development packages.
Note, the ClusterHQ repo is always enabled, to provide dependencies.
:param bytes distribution: The distribution the node is running.
:param PackageSource package_source: The source from which to install the
package.
:raises: ``UnsupportedDistribution`` if the distribution is unsupported.
"""
if package_source.branch:
# A development branch has been selected - add its Buildbot repo
use_development_branch = True
result_path = posixpath.join(
'/results/omnibus/', package_source.branch, distribution)
base_url = urljoin(package_source.build_server, result_path)
else:
use_development_branch = False
if distribution in ('ubuntu-14.04', 'ubuntu-15.04'):
commands = [
# Ensure add-apt-repository command and HTTPS URLs are supported
# FLOC-1880 will ensure these are necessary and sufficient
run_from_args([
"apt-get", "-y", "install", "apt-transport-https",
"software-properties-common"]),
# Add Docker repo for recent Docker versions
run_from_args([
"add-apt-repository", "-y", "ppa:james-page/docker"]),
# Add ClusterHQ repo for installation of Flocker packages.
run(command='add-apt-repository -y "deb {} /"'.format(
get_repository_url(
distribution=distribution,
flocker_version=get_installable_version(version)))),
]
if use_development_branch:
# Add BuildBot repo for testing
commands.append(run_from_args([
"add-apt-repository", "-y", "deb {} /".format(base_url)]))
# During a release, the ClusterHQ repo may contain packages with
# a higher version number than the Buildbot repo for a branch.
# Use a pin file to ensure that any Buildbot repo has higher
# priority than the ClusterHQ repo.
buildbot_host = urlparse(package_source.build_server).hostname
commands.append(put(
dedent('''\
Package: *
Pin: origin {}
Pin-Priority: 900
'''.format(buildbot_host)),
'/etc/apt/preferences.d/buildbot-900'))
commands += [
# Update to read package info from new repos
run_from_args([
"apt-get", "update"]),
]
if package_source.os_version:
package = 'clusterhq-flocker-node=%s' % (
package_source.os_version,)
else:
package = 'clusterhq-flocker-node'
# Install Flocker node and all dependencies
commands.append(run_from_args([
'apt-get', '-y', '--force-yes', 'install', package]))
return sequence(commands)
elif distribution in ('centos-7',):
commands = [
run(command="yum clean all"),
run(command="yum install -y " + get_repository_url(
distribution=distribution,
flocker_version=get_installable_version(version)))
]
if use_development_branch:
repo = dedent(b"""\
[clusterhq-build]
name=clusterhq-build
baseurl=%s
gpgcheck=0
enabled=0
""") % (base_url,)
commands.append(put(content=repo,
path='/etc/yum.repos.d/clusterhq-build.repo'))
repo_options = ['--enablerepo=clusterhq-build']
else:
repo_options = get_repo_options(
flocker_version=get_installable_version(version))
if package_source.os_version:
package = 'clusterhq-flocker-node-%s' % (
package_source.os_version,)
else:
package = 'clusterhq-flocker-node'
commands.append(run_from_args(
["yum", "install"] + repo_options + ["-y", package]))
return sequence(commands)
else:
raise UnsupportedDistribution()
