def install_cli_commands_yum(distribution, package_source): """ Install Flocker CLI on CentOS. The ClusterHQ repo is added for downloading latest releases. If ``package_source`` contains a branch, then a BuildBot repo will also be added to the package search path, to use in-development packages. Note, the ClusterHQ repo is always enabled, to provide dependencies. :param bytes distribution: The distribution the node is running. :param PackageSource package_source: The source from which to install the package. :return: a sequence of commands to run on the distribution """ if package_source.branch: # A development branch has been selected - add its Buildbot repo use_development_branch = True result_path = posixpath.join( '/results/omnibus/', package_source.branch, distribution) base_url = urljoin(package_source.build_server, result_path) else: use_development_branch = False commands = [ sudo(command="yum install -y " + get_repository_url( distribution=distribution, flocker_version=get_installable_version(version))), ] if use_development_branch: repo = dedent(b"""\ [clusterhq-build] name=clusterhq-build baseurl=%s gpgcheck=0 enabled=0 """) % (base_url,) commands.append(put(content=repo, path='/tmp/clusterhq-build.repo')) commands.append(sudo_from_args([ 'cp', '/tmp/clusterhq-build.repo', '/etc/yum.repos.d/clusterhq-build.repo'])) repo_options = ['--enablerepo=clusterhq-build'] else: repo_options = get_repo_options( flocker_version=get_installable_version(version)) if package_source.os_version: package = 'clusterhq-flocker-cli-%s' % (package_source.os_version,) else: package = 'clusterhq-flocker-cli' # Install Flocker CLI and all dependencies commands.append(sudo_from_args( ["yum", "install"] + repo_options + ["-y", package])) return sequence(commands)
def install_cli_commands_yum(distribution, package_source): """ Install Flocker CLI on CentOS. The ClusterHQ repo is added for downloading latest releases. If ``package_source`` contains a branch, then a BuildBot repo will also be added to the package search path, to use in-development packages. Note, the ClusterHQ repo is always enabled, to provide dependencies. :param bytes distribution: The distribution the node is running. :param PackageSource package_source: The source from which to install the package. :return: a sequence of commands to run on the distribution """ if package_source.branch: # A development branch has been selected - add its Buildbot repo use_development_branch = True result_path = posixpath.join( '/results/omnibus/', package_source.branch, distribution) base_url = urljoin(package_source.build_server, result_path) else: use_development_branch = False commands = [ sudo(command="yum install -y " + get_repository_url( distribution=distribution, flocker_version=get_installable_version(version))), ] if use_development_branch: repo = dedent(b"""\ [clusterhq-build] name=clusterhq-build baseurl=%s gpgcheck=0 enabled=0 """) % (base_url,) commands.append(put(content=repo, path='/tmp/clusterhq-build.repo')) commands.append(sudo_from_args([ 'cp', '/tmp/clusterhq-build.repo', '/etc/yum.repos.d/clusterhq-build.repo'])) repo_options = ['--enablerepo=clusterhq-build'] else: repo_options = get_repo_options( flocker_version=get_installable_version(version)) if package_source.os_version: package = 'clusterhq-flocker-cli-%s' % (package_source.os_version,) else: package = 'clusterhq-flocker-cli' # Install Flocker CLI and all dependencies commands.append(sudo_from_args( ["yum", "install"] + repo_options + ["-y", package])) return sequence(commands)
def install_commands_yum(package_name, distribution, package_source, base_url): """ Install Flocker package on CentOS. The ClusterHQ repo is added for downloading latest releases. If ``package_source`` contains a branch, then a BuildBot repo will also be added to the package search path, to use in-development packages. Note, the ClusterHQ repo is always enabled, to provide dependencies. :param str package_name: The name of the package to install. :param bytes distribution: The distribution the node is running. :param PackageSource package_source: The source from which to install the package. :param base_url: URL of repository, or ``None`` if we're not using development branch. :return: a sequence of commands to run on the distribution """ commands = [ # If package has previously been installed, 'yum install' fails, # so check if it is installed first. run( command="yum list installed clusterhq-release || yum install -y {0}".format( # noqa get_repository_url( distribution=distribution, flocker_version=get_installable_version(version)))), ] if base_url is not None: repo = dedent(b"""\ [clusterhq-build] name=clusterhq-build baseurl=%s gpgcheck=0 enabled=0 """) % (base_url,) commands.append(put(content=repo, path='/tmp/clusterhq-build.repo')) commands.append(run_from_args([ 'cp', '/tmp/clusterhq-build.repo', '/etc/yum.repos.d/clusterhq-build.repo'])) repo_options = ['--enablerepo=clusterhq-build'] else: repo_options = get_repo_options( flocker_version=get_installable_version(version)) if package_source.os_version: package_name += '-%s' % (package_source.os_version,) # Install package and all dependencies: commands.append(run_from_args( ["yum", "install"] + repo_options + ["-y", package_name])) return sequence(commands)
def test_version_prompt(self): """ The ``version-prompt`` directive replaces the placemarker ``|latest-installable|`` in a source file with the current installable version in the output file. """ source_directory = self.make_temporary_directory() source_file = source_directory.child('contents.rst') source_file.setContent(dedent(''' .. version-prompt:: bash $ $ PRE-|latest-installable|-POST ''')) destination_directory = self.make_temporary_directory() run_process([ 'sphinx-build', '-b', 'html', '-C', # don't look for config file, use -D flags instead '-D', 'extensions=flocker.docs.version_extensions', # directory containing source/config files source_directory.path, # directory containing build files destination_directory.path, source_file.path]) # source file to process expected = 'PRE-{}-POST'.format(get_installable_version(version)) content = destination_directory.child('contents.html').getContent() self.assertIn(expected, content)
def make_changed_file(path, env): """ Given the path to a template file, write a new file with: * The same filename, except without '.template' at the end. * A placeholder in the new file changed to the latest installable version of Flocker. This new file will be deleted on build completion. :param unicode path: The path to a template file. :param sphinx.environment.BuildEnvironment env: The Sphinx build environment """ def remove_file(path): try: os.remove(path) except OSError: pass latest = get_installable_version(version) new_path = remove_extension(path) with open(path, 'r') as templated_file: with open(new_path, 'w') as new_file: new_file.write(templated_file.read().replace(PLACEHOLDER, latest)) env.app.connect('build-finished', lambda self, *args: remove_file(new_path))
def make_changed_file(path, env): """ Given the path to a template file, write a new file with: * The same filename, except without '.template' at the end. * A placeholder in the new file changed to the latest installable version of Flocker. This new file will be deleted on build completion. :param unicode path: The path to a template file. :param sphinx.environment.BuildEnvironment env: The Sphinx build environment """ def remove_file(path): try: os.remove(path) except OSError: pass latest = get_installable_version(version) new_path = remove_extension(path) with open(path, 'r') as templated_file: with open(new_path, 'w') as new_file: new_file.write(templated_file.read().replace(PLACEHOLDER, latest)) env.app.connect('build-finished', lambda self, *args: remove_file(new_path))
def task_cli_pip_install( venv_name='flocker-client', package_source=PackageSource()): """ Install the Flocker client into a virtualenv using pip. :param bytes venv_name: Name for the virtualenv. :param package_source: Package source description :return: an Effect to install the client. """ vers = package_source.version if vers is None: vers = version url = ( 'https://{bucket}.s3.amazonaws.com/{key}/' 'Flocker-{version}-py2-none-any.whl'.format( bucket=ARCHIVE_BUCKET, key='python', version=get_installable_version(vers)) ) return sequence([ run_from_args( ['virtualenv', '--python=/usr/bin/python2.7', venv_name]), run_from_args(['source', '{}/bin/activate'.format(venv_name)]), run_from_args(['pip', 'install', '--upgrade', 'pip']), run_from_args( ['pip', 'install', url]), ])
def test_version_prompt(self): """ The ``version-prompt`` directive replaces the placemarker ``|latest-installable|`` in a source file with the current installable version in the output file. """ temp_dir = FilePath(self.mktemp()) temp_dir.makedirs() source_file = temp_dir.child('contents.rst') source_file.setContent( dedent(''' .. version-prompt:: bash $ $ PRE-|latest-installable|-POST ''')) run_process([ 'sphinx-build', '-b', 'html', '-C', # don't look for config file, use -D flags instead '-D', 'extensions=flocker.docs.version_extensions', temp_dir.path, # directory containing source/config files temp_dir.path, # directory containing build files source_file.path ]) # source file to process expected = 'PRE-{}-POST'.format(get_installable_version(version)) content = temp_dir.child('contents.html').getContent() self.assertIn(expected, content)
def task_cli_pip_install( venv_name='flocker-client', package_source=PackageSource()): """ Install the Flocker client into a virtualenv using pip. :param bytes venv_name: Name for the virtualenv. :param package_source: Package source description :return: an Effect to install the client. """ vers = package_source.version if vers is None: vers = version url = ( 'https://{bucket}.s3.amazonaws.com/{key}/' 'Flocker-{version}-py2-none-any.whl'.format( bucket=ARCHIVE_BUCKET, key='python', version=get_installable_version(vers)) ) return sequence([ run_from_args( ['virtualenv', '--python=/usr/bin/python2.7', venv_name]), run_from_args(['source', '{}/bin/activate'.format(venv_name)]), run_from_args(['pip', 'install', '--upgrade', 'pip']), run_from_args( ['pip', 'install', url]), ])
def run(self): task = getattr(tasks, 'task_%s' % (self.arguments[0], )) prompt = self.options.get('prompt', '$') if len(self.arguments) > 1: # Some tasks can include the latest installable version as (part # of) an argument. This replaces a placeholder with that version. arguments = self.arguments[1].split() latest = get_installable_version(version) task_arguments = [ item.replace(PLACEHOLDER, latest).encode("utf-8") for item in arguments ] else: task_arguments = [] commands = task(*task_arguments) lines = ['.. prompt:: bash %s,> auto' % (prompt, ), ''] try: command_lines = run_for_docs(commands) except NoPerformerFoundError as e: raise self.error("task: %s not supported" % (type(e.args[0]).__name__, )) except SequenceFailed as e: print e.error for command_line in command_lines: # handler can return either a string or a list. If it returns a # list, treat the elements after the first as continuation lines. if isinstance(command_line, list): lines.append(' %s %s' % ( prompt, command_line[0], )) lines.extend( [' > %s' % (line, ) for line in command_line[1:]]) else: lines.append(' %s %s' % ( prompt, command_line, )) # The following three lines record (some?) of the dependencies of the # directive, so automatic regeneration happens. Specifically, it # records this file, and the file where the task is declared. task_file = getsourcefile(task) tasks_file = getsourcefile(tasks) self.state.document.settings.record_dependencies.add(task_file) self.state.document.settings.record_dependencies.add(tasks_file) self.state.document.settings.record_dependencies.add(__file__) node = nodes.Element() text = StringList(lines) self.state.nested_parse(text, self.content_offset, node) return node.children
def _get_wheel_version(package_source): """ Get the latest available wheel version for the specified package source. If package source version is not set, the latest installable release will be installed. Note, branch is never used for wheel installations, since the wheel file is not created for branches. :param PackageSource package_source: The source from which to install the package. :return: a string containing the previous installable version of either the package version, or, if that is not specified, of the current version. """ return get_installable_version(package_source.version or version)
def _get_wheel_version(package_source): """ Get the latest available wheel version for the specified package source. If package source version is not set, the latest installable release will be installed. Note, branch is never used for wheel installations, since the wheel file is not created for branches. :param PackageSource package_source: The source from which to install the package. :return: a string containing the previous installable version of either the package version, or, if that is not specified, of the current version. """ return get_installable_version(package_source.version or version)
def run(self): task = getattr(tasks, 'task_%s' % (self.arguments[0],)) prompt = self.options.get('prompt', '$') if len(self.arguments) > 1: # Some tasks can include the latest installable version as (part # of) an argument. This replaces a placeholder with that version. arguments = self.arguments[1].split() latest = get_installable_version(version) task_arguments = [item.replace(PLACEHOLDER, latest).encode("utf-8") for item in arguments] else: task_arguments = [] commands = task(*task_arguments) lines = ['.. prompt:: bash %s,> auto' % (prompt,), ''] try: command_lines = run_for_docs(commands) except NoPerformerFoundError as e: raise self.error("task: %s not supported" % (type(e.args[0]).__name__,)) except SequenceFailed as e: print e.error for command_line in command_lines: # handler can return either a string or a list. If it returns a # list, treat the elements after the first as continuation lines. if isinstance(command_line, list): lines.append(' %s %s' % (prompt, command_line[0],)) lines.extend([' > %s' % (line,) for line in command_line[1:]]) else: lines.append(' %s %s' % (prompt, command_line,)) # The following three lines record (some?) of the dependencies of the # directive, so automatic regeneration happens. Specifically, it # records this file, and the file where the task is declared. task_file = getsourcefile(task) tasks_file = getsourcefile(tasks) self.state.document.settings.record_dependencies.add(task_file) self.state.document.settings.record_dependencies.add(tasks_file) self.state.document.settings.record_dependencies.add(__file__) node = nodes.Element() text = StringList(lines) self.state.nested_parse(text, self.content_offset, node) return node.children
def cli_pkg_test(package_source=PackageSource()): """ Check that the Flocker CLI is working and has the expected version. :param PackageSource package_source: The source from which to install the package. :return: An ``Effect`` to pass to a ``Dispatcher`` that supports ``Sequence``, ``Run``, ``Sudo``, ``Comment``, and ``Put``. """ expected = package_source.version if not expected: if package_source.branch: # If branch is set but version isn't, we don't know the # latest version. In this case, just check that the version # can be displayed. return run('flocker-deploy --version') else: # If neither branch nor version is set, the latest # installable release will be installed. expected = get_installable_version(version) return run('test `flocker-deploy --version` = {}'.format(quote(expected)))
def cli_pkg_test(package_source=PackageSource()): """ Check that the Flocker CLI is working and has the expected version. :param PackageSource package_source: The source from which to install the package. :return: An ``Effect`` to pass to a ``Dispatcher`` that supports ``Sequence``, ``Run``, ``Sudo``, ``Comment``, and ``Put``. """ expected = package_source.version if not expected: if package_source.branch: # If branch is set but version isn't, we don't know the # latest version. In this case, just check that the version # can be displayed. return run('flocker-deploy --version') else: # If neither branch nor version is set, the latest # installable release will be installed. expected = get_installable_version(version) return run('test `flocker-deploy --version` = {}'.format(quote(expected)))
def install_cli_commands_ubuntu(distribution, package_source): """ Install flocker CLI on Ubuntu. The ClusterHQ repo is added for downloading latest releases. If ``package_source`` contains a branch, then a BuildBot repo will also be added to the package search path, to use in-development packages. Note, the ClusterHQ repo is always enabled, to provide dependencies. :param bytes distribution: The distribution the node is running. :param PackageSource package_source: The source from which to install the package. :return: a sequence of commands to run on the distribution """ if package_source.branch: # A development branch has been selected - add its Buildbot repo use_development_branch = True result_path = posixpath.join( '/results/omnibus/', package_source.branch, distribution) base_url = urljoin(package_source.build_server, result_path) else: use_development_branch = False commands = [ # Minimal images often have cleared apt caches and are missing # packages that are common in a typical release. These commands # ensure that we start from a good base system with the required # capabilities, particularly that the add-apt-repository command # is available, and HTTPS URLs are supported. sudo_from_args(["apt-get", "update"]), sudo_from_args([ "apt-get", "-y", "install", "apt-transport-https", "software-properties-common"]), # Add ClusterHQ repo for installation of Flocker packages. sudo(command='add-apt-repository -y "deb {} /"'.format( get_repository_url( distribution=distribution, flocker_version=get_installable_version(version)))) ] if use_development_branch: # Add BuildBot repo for running tests commands.append(sudo_from_args([ "add-apt-repository", "-y", "deb {} /".format(base_url)])) # During a release, the ClusterHQ repo may contain packages with # a higher version number than the Buildbot repo for a branch. # Use a pin file to ensure that any Buildbot repo has higher # priority than the ClusterHQ repo. buildbot_host = urlparse(package_source.build_server).hostname commands.append(put(dedent('''\ Package: * Pin: origin {} Pin-Priority: 900 '''.format(buildbot_host)), '/tmp/apt-pref')) commands.append(sudo_from_args([ 'mv', '/tmp/apt-pref', '/etc/apt/preferences.d/buildbot-900'])) # Update to read package info from new repos commands.append(sudo_from_args(["apt-get", "update"])) if package_source.os_version: package = 'clusterhq-flocker-cli=%s' % (package_source.os_version,) else: package = 'clusterhq-flocker-cli' # Install Flocker CLI and all dependencies commands.append(sudo_from_args([ 'apt-get', '-y', '--force-yes', 'install', package])) return sequence(commands)
def install_commands_yum(package_name, distribution, package_source, base_url): """ Install Flocker package on CentOS. The ClusterHQ repo is added for downloading latest releases. If ``package_source`` contains a branch, then a BuildBot repo will also be added to the package search path, to use in-development packages. Note, the ClusterHQ repo is always enabled, to provide dependencies. :param str package_name: The name of the package to install. :param bytes distribution: The distribution the node is running. :param PackageSource package_source: The source from which to install the package. :param base_url: URL of repository, or ``None`` if we're not using development branch. :return: a sequence of commands to run on the distribution """ flocker_version = package_source.version if not flocker_version: # support empty values other than None, as '' sometimes used to # indicate latest version, due to previous behaviour flocker_version = get_installable_version(version) repo_package_name = 'clusterhq-release' commands = [ # If package has previously been installed, 'yum install' fails, # so check if it is installed first. run(command="yum list installed {} || yum install -y {}".format( quote(repo_package_name), get_repository_url(distribution=distribution, flocker_version=flocker_version))), ] if base_url is not None: repo = dedent(b"""\ [clusterhq-build] name=clusterhq-build baseurl=%s gpgcheck=0 enabled=0 # There is a distinct clusterhq-build repository for each branch. # The metadata across these different repositories varies. Version # numbers are not comparable. A version which exists in one likely # does not exist in another. In order to support switching between # branches (and therefore between clusterhq-build repositories), # tell yum to always update metadata for this repository. metadata_expire=0 """) % (base_url, ) commands.append(put(content=repo, path='/tmp/clusterhq-build.repo')) commands.append( run_from_args([ 'cp', '/tmp/clusterhq-build.repo', '/etc/yum.repos.d/clusterhq-build.repo' ])) repo_options = ['--enablerepo=clusterhq-build'] else: repo_options = get_repo_options( flocker_version=get_installable_version(version)) os_version = package_source.os_version() if os_version: package_name += '-%s' % (os_version, ) # Install package and all dependencies: commands.append( run_from_args(["yum", "install"] + repo_options + ["-y", package_name])) return sequence(commands)
def run(self): latest = get_installable_version(version) self.content = [ item.replace(PLACEHOLDER, latest) for item in self.content ] return sphinx_prompt.PromptDirective.run(self)
def install_commands_ubuntu(package_name, distribution, package_source, base_url): """ Install Flocker package on Ubuntu. The ClusterHQ repo is added for downloading latest releases. If ``package_source`` contains a branch, then a BuildBot repo will also be added to the package search path, to use in-development packages. Note, the ClusterHQ repo is always enabled, to provide dependencies. :param bytes distribution: The distribution the node is running. :param PackageSource package_source: The source from which to install the package. :param base_url: URL of repository, or ``None`` if we're not using development branch. :return: a sequence of commands to run on the distribution """ commands = [ # Minimal images often have cleared apt caches and are missing # packages that are common in a typical release. These commands # ensure that we start from a good base system with the required # capabilities, particularly that the add-apt-repository command # is available, and HTTPS URLs are supported. run_from_args(["apt-get", "update"]), run_from_args([ "apt-get", "-y", "install", "apt-transport-https", "software-properties-common"]), # Add ClusterHQ repo for installation of Flocker packages. run(command='add-apt-repository -y "deb {} /"'.format( get_repository_url( distribution=distribution, flocker_version=get_installable_version(version)))) ] if base_url is not None: # Add BuildBot repo for running tests commands.append(run_from_args([ "add-apt-repository", "-y", "deb {} /".format(base_url)])) # During a release, the ClusterHQ repo may contain packages with # a higher version number than the Buildbot repo for a branch. # Use a pin file to ensure that any Buildbot repo has higher # priority than the ClusterHQ repo. buildbot_host = urlparse(package_source.build_server).hostname commands.append(put(dedent('''\ Package: * Pin: origin {} Pin-Priority: 900 '''.format(buildbot_host)), '/tmp/apt-pref')) commands.append(run_from_args([ 'mv', '/tmp/apt-pref', '/etc/apt/preferences.d/buildbot-900'])) # Update to read package info from new repos commands.append(run_from_args(["apt-get", "update"])) if package_source.os_version: package_name += '=%s' % (package_source.os_version,) # Install package and all dependencies commands.append(run_from_args([ 'apt-get', '-y', '--force-yes', 'install', package_name])) return sequence(commands)
def install_commands_ubuntu(package_name, distribution, package_source, base_url): """ Install Flocker package on Ubuntu. The ClusterHQ repo is added for downloading latest releases. If ``package_source`` contains a branch, then a BuildBot repo will also be added to the package search path, to use in-development packages. Note, the ClusterHQ repo is always enabled, to provide dependencies. :param bytes distribution: The distribution the node is running. :param PackageSource package_source: The source from which to install the package. :param base_url: URL of repository, or ``None`` if we're not using development branch. :return: a sequence of commands to run on the distribution """ flocker_version = package_source.version if not flocker_version: # support empty values other than None, as '' sometimes used to # indicate latest version, due to previous behaviour flocker_version = get_installable_version(version) commands = [ # Minimal images often have cleared apt caches and are missing # packages that are common in a typical release. These commands # ensure that we start from a good base system with the required # capabilities, particularly that the add-apt-repository command # is available, and HTTPS URLs are supported. run_from_args(["apt-get", "update"]), run_from_args([ "apt-get", "-y", "install", "apt-transport-https", "software-properties-common" ]), # Add ClusterHQ repo for installation of Flocker packages. run(command='add-apt-repository -y "deb {} /"'.format( get_repository_url(distribution=distribution, flocker_version=flocker_version))) ] if base_url is not None: # Add BuildBot repo for running tests commands.append( run_from_args( ["add-apt-repository", "-y", "deb {} /".format(base_url)])) # During a release, the ClusterHQ repo may contain packages with # a higher version number than the Buildbot repo for a branch. # Use a pin file to ensure that any Buildbot repo has higher # priority than the ClusterHQ repo. We only add the Buildbot # repo when a branch is specified, so it wil not interfere with # attempts to install a release (when no branch is specified). buildbot_host = urlparse(package_source.build_server).hostname commands.append( put( dedent('''\ Package: * Pin: origin {} Pin-Priority: 700 '''.format(buildbot_host)), '/tmp/apt-pref')) commands.append( run_from_args( ['mv', '/tmp/apt-pref', '/etc/apt/preferences.d/buildbot-700'])) # Update to read package info from new repos commands.append(run_from_args(["apt-get", "update"])) os_version = package_source.os_version() if os_version: # Set the version of the top-level package package_name += '=%s' % (os_version, ) # If a specific version is required, ensure that the version for # all ClusterHQ packages is consistent. This prevents conflicts # between the top-level package, which may depend on a lower # version of a dependency, and apt, which wants to install the # most recent version. Note that this trumps the Buildbot # pinning above. commands.append( put( dedent('''\ Package: clusterhq-* Pin: version {} Pin-Priority: 900 '''.format(os_version)), '/tmp/apt-pref')) commands.append( run_from_args([ 'mv', '/tmp/apt-pref', '/etc/apt/preferences.d/clusterhq-900' ])) # Install package and all dependencies commands.append( run_from_args( ['apt-get', '-y', '--force-yes', 'install', package_name])) return sequence(commands)
def test_upgrade(self, cluster): """ Given a dataset created and used with the previously installable version of flocker, uninstalling the previous version of flocker and installing HEAD does not destroy the data on the dataset. """ node = cluster.nodes[0] SAMPLE_STR = '123456' * 100 upgrade_from_version = get_installable_version(HEAD_FLOCKER_VERSION) # Get the initial flocker version and setup a cleanup call to restore # flocker to that version when the test is done. d = cluster.client.version() original_package_source = [None] def setup_restore_original_flocker(version): version_bytes = version.get('flocker', u'').encode('ascii') original_package_source[0] = (self._get_package_source( default_version=version_bytes or None)) self.addCleanup(lambda: cluster.install_flocker_version( original_package_source[0])) return version d.addCallback(setup_restore_original_flocker) # Double check that the nodes are clean before we destroy the persisted # state. d.addCallback(lambda _: cluster.clean_nodes()) # Downgrade flocker to the most recent released version. d.addCallback(lambda _: cluster.install_flocker_version( PackageSource(version=upgrade_from_version), destroy_persisted_state=True)) # Create a dataset with the code from the most recent release. d.addCallback(lambda _: create_dataset(self, cluster, node=node)) first_dataset = [None] # Write some data to a file in the dataset. def write_to_file(dataset): first_dataset[0] = dataset return node.run_as_root([ 'bash', '-c', 'echo "%s" > %s' % (SAMPLE_STR, os.path.join(dataset.path.path, 'test.txt')) ]) d.addCallback(write_to_file) # Upgrade flocker to the code under test. d.addCallback(lambda _: cluster.install_flocker_version( original_package_source[0])) # Create a new dataset to convince ourselves that the new code is # running. d.addCallback(lambda _: create_dataset(self, cluster, node=node)) # Wait for the first dataset to be mounted again. d.addCallback(lambda _: cluster.wait_for_dataset(first_dataset[0])) # Verify that the file still has its contents. def cat_and_verify_file(dataset): output = [] file_catting = node.run_as_root([ 'bash', '-c', 'cat %s' % (os.path.join(dataset.path.path, 'test.txt')) ], handle_stdout=output.append) def verify_file(_): file_contents = ''.join(output) self.assertEqual(file_contents, SAMPLE_STR) file_catting.addCallback(verify_file) return file_catting d.addCallback(cat_and_verify_file) return d
def run(self): latest = get_installable_version(version) self.content = [item.replace(PLACEHOLDER, latest) for item in self.content] return CodeBlock.run(self)
def install_commands_yum(package_name, distribution, package_source, base_url): """ Install Flocker package on CentOS. The ClusterHQ repo is added for downloading latest releases. If ``package_source`` contains a branch, then a BuildBot repo will also be added to the package search path, to use in-development packages. Note, the ClusterHQ repo is always enabled, to provide dependencies. :param str package_name: The name of the package to install. :param bytes distribution: The distribution the node is running. :param PackageSource package_source: The source from which to install the package. :param base_url: URL of repository, or ``None`` if we're not using development branch. :return: a sequence of commands to run on the distribution """ flocker_version = package_source.version if not flocker_version: # support empty values other than None, as '' sometimes used to # indicate latest version, due to previous behaviour flocker_version = get_installable_version(version) repo_package_name = 'clusterhq-release' commands = [ # If package has previously been installed, 'yum install' fails, # so check if it is installed first. run( command="yum list installed {} || yum install -y {}".format( quote(repo_package_name), get_repository_url( distribution=distribution, flocker_version=flocker_version))), ] if base_url is not None: repo = dedent(b"""\ [clusterhq-build] name=clusterhq-build baseurl=%s gpgcheck=0 enabled=0 # There is a distinct clusterhq-build repository for each branch. # The metadata across these different repositories varies. Version # numbers are not comparable. A version which exists in one likely # does not exist in another. In order to support switching between # branches (and therefore between clusterhq-build repositories), # tell yum to always update metadata for this repository. metadata_expire=0 """) % (base_url,) commands.append(put(content=repo, path='/tmp/clusterhq-build.repo')) commands.append(run_from_args([ 'cp', '/tmp/clusterhq-build.repo', '/etc/yum.repos.d/clusterhq-build.repo'])) repo_options = ['--enablerepo=clusterhq-build'] else: repo_options = get_repo_options( flocker_version=get_installable_version(version)) os_version = package_source.os_version() if os_version: package_name += '-%s' % (os_version,) # Install package and all dependencies: commands.append(run_from_args( ["yum", "install"] + repo_options + ["-y", package_name])) return sequence(commands)
def task_install_flocker( distribution=None, package_source=PackageSource(), ): """ Install flocker cluster on a distribution. The ClusterHQ repo is added for downloading latest releases. If ``package_source`` contains a branch, then a BuildBot repo will also be added to the package search path, to use in-development packages. Note, the ClusterHQ repo is always enabled, to provide dependencies. :param bytes distribution: The distribution the node is running. :param PackageSource package_source: The source from which to install the package. :raises: ``UnsupportedDistribution`` if the distribution is unsupported. """ if package_source.branch: # A development branch has been selected - add its Buildbot repo use_development_branch = True result_path = posixpath.join( '/results/omnibus/', package_source.branch, distribution) base_url = urljoin(package_source.build_server, result_path) else: use_development_branch = False if distribution in ('ubuntu-14.04', 'ubuntu-15.04'): commands = [ # Ensure add-apt-repository command and HTTPS URLs are supported # FLOC-1880 will ensure these are necessary and sufficient run_from_args([ "apt-get", "-y", "install", "apt-transport-https", "software-properties-common"]), # Add Docker repo for recent Docker versions run_from_args([ "add-apt-repository", "-y", "ppa:james-page/docker"]), # Add ClusterHQ repo for installation of Flocker packages. run(command='add-apt-repository -y "deb {} /"'.format( get_repository_url( distribution=distribution, flocker_version=get_installable_version(version)))), ] if use_development_branch: # Add BuildBot repo for testing commands.append(run_from_args([ "add-apt-repository", "-y", "deb {} /".format(base_url)])) # During a release, the ClusterHQ repo may contain packages with # a higher version number than the Buildbot repo for a branch. # Use a pin file to ensure that any Buildbot repo has higher # priority than the ClusterHQ repo. buildbot_host = urlparse(package_source.build_server).hostname commands.append(put( dedent('''\ Package: * Pin: origin {} Pin-Priority: 900 '''.format(buildbot_host)), '/etc/apt/preferences.d/buildbot-900')) commands += [ # Update to read package info from new repos run_from_args([ "apt-get", "update"]), ] if package_source.os_version: package = 'clusterhq-flocker-node=%s' % ( package_source.os_version,) else: package = 'clusterhq-flocker-node' # Install Flocker node and all dependencies commands.append(run_from_args([ 'apt-get', '-y', '--force-yes', 'install', package])) return sequence(commands) elif distribution in ('centos-7',): commands = [ run(command="yum clean all"), run(command="yum install -y " + get_repository_url( distribution=distribution, flocker_version=get_installable_version(version))) ] if use_development_branch: repo = dedent(b"""\ [clusterhq-build] name=clusterhq-build baseurl=%s gpgcheck=0 enabled=0 """) % (base_url,) commands.append(put(content=repo, path='/etc/yum.repos.d/clusterhq-build.repo')) repo_options = ['--enablerepo=clusterhq-build'] else: repo_options = get_repo_options( flocker_version=get_installable_version(version)) if package_source.os_version: package = 'clusterhq-flocker-node-%s' % ( package_source.os_version,) else: package = 'clusterhq-flocker-node' commands.append(run_from_args( ["yum", "install"] + repo_options + ["-y", package])) return sequence(commands) else: raise UnsupportedDistribution()
def install_commands_ubuntu(package_name, distribution, package_source, base_url): """ Install Flocker package on Ubuntu. The ClusterHQ repo is added for downloading latest releases. If ``package_source`` contains a branch, then a BuildBot repo will also be added to the package search path, to use in-development packages. Note, the ClusterHQ repo is always enabled, to provide dependencies. :param bytes distribution: The distribution the node is running. :param PackageSource package_source: The source from which to install the package. :param base_url: URL of repository, or ``None`` if we're not using development branch. :return: a sequence of commands to run on the distribution """ flocker_version = package_source.version if not flocker_version: # support empty values other than None, as '' sometimes used to # indicate latest version, due to previous behaviour flocker_version = get_installable_version(version) commands = [ # Minimal images often have cleared apt caches and are missing # packages that are common in a typical release. These commands # ensure that we start from a good base system with the required # capabilities, particularly that the add-apt-repository command # is available, and HTTPS URLs are supported. run_from_args(["apt-get", "update"]), run_from_args([ "apt-get", "-y", "install", "apt-transport-https", "software-properties-common"]), # Add ClusterHQ repo for installation of Flocker packages. run(command='add-apt-repository -y "deb {} /"'.format( get_repository_url( distribution=distribution, flocker_version=flocker_version))) ] if base_url is not None: # Add BuildBot repo for running tests commands.append(run_from_args([ "add-apt-repository", "-y", "deb {} /".format(base_url)])) # During a release, the ClusterHQ repo may contain packages with # a higher version number than the Buildbot repo for a branch. # Use a pin file to ensure that any Buildbot repo has higher # priority than the ClusterHQ repo. We only add the Buildbot # repo when a branch is specified, so it wil not interfere with # attempts to install a release (when no branch is specified). buildbot_host = urlparse(package_source.build_server).hostname commands.append(put(dedent('''\ Package: * Pin: origin {} Pin-Priority: 700 '''.format(buildbot_host)), '/tmp/apt-pref')) commands.append(run_from_args([ 'mv', '/tmp/apt-pref', '/etc/apt/preferences.d/buildbot-700'])) # Update to read package info from new repos commands.append(run_from_args(["apt-get", "update"])) os_version = package_source.os_version() if os_version: # Set the version of the top-level package package_name += '=%s' % (os_version,) # If a specific version is required, ensure that the version for # all ClusterHQ packages is consistent. This prevents conflicts # between the top-level package, which may depend on a lower # version of a dependency, and apt, which wants to install the # most recent version. Note that this trumps the Buildbot # pinning above. commands.append(put(dedent('''\ Package: clusterhq-* Pin: version {} Pin-Priority: 900 '''.format(os_version)), '/tmp/apt-pref')) commands.append(run_from_args([ 'mv', '/tmp/apt-pref', '/etc/apt/preferences.d/clusterhq-900'])) # Install package and all dependencies commands.append(run_from_args([ 'apt-get', '-y', '--force-yes', 'install', package_name])) return sequence(commands)
def run(self): latest = get_installable_version(version) self.content = [item.replace(PLACEHOLDER, latest) for item in self.content] return sphinx_prompt.PromptDirective.run(self)
def run(self): latest = get_installable_version(version) self.content = [ item.replace(PLACEHOLDER, latest) for item in self.content ] return CodeBlock.run(self)
def install_cli_commands_ubuntu(distribution, package_source): """ Install flocker CLI on Ubuntu. The ClusterHQ repo is added for downloading latest releases. If ``package_source`` contains a branch, then a BuildBot repo will also be added to the package search path, to use in-development packages. Note, the ClusterHQ repo is always enabled, to provide dependencies. :param bytes distribution: The distribution the node is running. :param PackageSource package_source: The source from which to install the package. :return: a sequence of commands to run on the distribution """ if package_source.branch: # A development branch has been selected - add its Buildbot repo use_development_branch = True result_path = posixpath.join( '/results/omnibus/', package_source.branch, distribution) base_url = urljoin(package_source.build_server, result_path) else: use_development_branch = False commands = [ # Minimal images often have cleared apt caches and are missing # packages that are common in a typical release. These commands # ensure that we start from a good base system with the required # capabilities, particularly that the add-apt-repository command # and HTTPS URLs are supported. # FLOC-1880 will ensure these are necessary and sufficient. sudo_from_args(["apt-get", "update"]), sudo_from_args([ "apt-get", "-y", "install", "apt-transport-https", "software-properties-common"]), # Add ClusterHQ repo for installation of Flocker packages. sudo(command='add-apt-repository -y "deb {} /"'.format( get_repository_url( distribution=distribution, flocker_version=get_installable_version(version)))) ] if use_development_branch: # Add BuildBot repo for running tests commands.append(sudo_from_args([ "add-apt-repository", "-y", "deb {} /".format(base_url)])) # During a release, the ClusterHQ repo may contain packages with # a higher version number than the Buildbot repo for a branch. # Use a pin file to ensure that any Buildbot repo has higher # priority than the ClusterHQ repo. buildbot_host = urlparse(package_source.build_server).hostname commands.append(put(dedent('''\ Package: * Pin: origin {} Pin-Priority: 900 '''.format(buildbot_host)), '/tmp/apt-pref')) commands.append(sudo_from_args([ 'mv', '/tmp/apt-pref', '/etc/apt/preferences.d/buildbot-900'])) # Update to read package info from new repos commands.append(sudo_from_args(["apt-get", "update"])) if package_source.os_version: package = 'clusterhq-flocker-cli=%s' % (package_source.os_version,) else: package = 'clusterhq-flocker-cli' # Install Flocker CLI and all dependencies commands.append(sudo_from_args([ 'apt-get', '-y', '--force-yes', 'install', package])) return sequence(commands)
def test_upgrade(self, cluster): """ Given a dataset created and used with the previously installable version of flocker, uninstalling the previous version of flocker and installing HEAD does not destroy the data on the dataset. """ node = cluster.nodes[0] SAMPLE_STR = '123456' * 100 upgrade_from_version = get_installable_version(HEAD_FLOCKER_VERSION) # Get the initial flocker version and setup a cleanup call to restore # flocker to that version when the test is done. d = cluster.client.version() original_package_source = [None] def setup_restore_original_flocker(version): version_bytes = version.get('flocker', u'').encode('ascii') original_package_source[0] = ( self._get_package_source( default_version=version_bytes or None) ) self.addCleanup( lambda: cluster.install_flocker_version( original_package_source[0])) return version d.addCallback(setup_restore_original_flocker) # Double check that the nodes are clean before we destroy the persisted # state. d.addCallback(lambda _: cluster.clean_nodes()) # Downgrade flocker to the most recent released version. d.addCallback( lambda _: cluster.install_flocker_version( PackageSource(version=upgrade_from_version), destroy_persisted_state=True ) ) # Create a dataset with the code from the most recent release. d.addCallback(lambda _: create_dataset(self, cluster, node=node)) first_dataset = [None] # Write some data to a file in the dataset. def write_to_file(dataset): first_dataset[0] = dataset return node.run_as_root( ['bash', '-c', 'echo "%s" > %s' % ( SAMPLE_STR, os.path.join(dataset.path.path, 'test.txt'))]) d.addCallback(write_to_file) # Upgrade flocker to the code under test. d.addCallback(lambda _: cluster.install_flocker_version( original_package_source[0])) # Create a new dataset to convince ourselves that the new code is # running. d.addCallback(lambda _: create_dataset(self, cluster, node=node)) # Wait for the first dataset to be mounted again. d.addCallback(lambda _: cluster.wait_for_dataset(first_dataset[0])) # Verify that the file still has its contents. def cat_and_verify_file(dataset): output = [] file_catting = node.run_as_root( ['bash', '-c', 'cat %s' % ( os.path.join(dataset.path.path, 'test.txt'))], handle_stdout=output.append) def verify_file(_): file_contents = ''.join(output) self.assertEqual(file_contents, SAMPLE_STR) file_catting.addCallback(verify_file) return file_catting d.addCallback(cat_and_verify_file) return d
def task_install_flocker( distribution=None, package_source=PackageSource(), ): """ Install flocker cluster on a distribution. The ClusterHQ repo is added for downloading latest releases. If ``package_source`` contains a branch, then a BuildBot repo will also be added to the package search path, to use in-development packages. Note, the ClusterHQ repo is always enabled, to provide dependencies. :param bytes distribution: The distribution the node is running. :param PackageSource package_source: The source from which to install the package. :raises: ``UnsupportedDistribution`` if the distribution is unsupported. """ if package_source.branch: # A development branch has been selected - add its Buildbot repo use_development_branch = True result_path = posixpath.join( '/results/omnibus/', package_source.branch, distribution) base_url = urljoin(package_source.build_server, result_path) else: use_development_branch = False if distribution in ('ubuntu-14.04', 'ubuntu-15.04'): commands = [ # Ensure add-apt-repository command and HTTPS URLs are supported # FLOC-1880 will ensure these are necessary and sufficient run_from_args([ "apt-get", "-y", "install", "apt-transport-https", "software-properties-common"]), # Add Docker repo for recent Docker versions run_from_args([ "add-apt-repository", "-y", "ppa:james-page/docker"]), # Add ClusterHQ repo for installation of Flocker packages. run(command='add-apt-repository -y "deb {} /"'.format( get_repository_url( distribution=distribution, flocker_version=get_installable_version(version)))), ] if use_development_branch: # Add BuildBot repo for testing commands.append(run_from_args([ "add-apt-repository", "-y", "deb {} /".format(base_url)])) # During a release, the ClusterHQ repo may contain packages with # a higher version number than the Buildbot repo for a branch. # Use a pin file to ensure that any Buildbot repo has higher # priority than the ClusterHQ repo. buildbot_host = urlparse(package_source.build_server).hostname commands.append(put( dedent('''\ Package: * Pin: origin {} Pin-Priority: 900 '''.format(buildbot_host)), '/etc/apt/preferences.d/buildbot-900')) commands += [ # Update to read package info from new repos run_from_args([ "apt-get", "update"]), ] if package_source.os_version: package = 'clusterhq-flocker-node=%s' % ( package_source.os_version,) else: package = 'clusterhq-flocker-node' # Install Flocker node and all dependencies commands.append(run_from_args([ 'apt-get', '-y', '--force-yes', 'install', package])) return sequence(commands) elif distribution in ('centos-7',): commands = [ run(command="yum clean all"), run(command="yum install -y " + get_repository_url( distribution=distribution, flocker_version=get_installable_version(version))) ] if use_development_branch: repo = dedent(b"""\ [clusterhq-build] name=clusterhq-build baseurl=%s gpgcheck=0 enabled=0 """) % (base_url,) commands.append(put(content=repo, path='/etc/yum.repos.d/clusterhq-build.repo')) repo_options = ['--enablerepo=clusterhq-build'] else: repo_options = get_repo_options( flocker_version=get_installable_version(version)) if package_source.os_version: package = 'clusterhq-flocker-node-%s' % ( package_source.os_version,) else: package = 'clusterhq-flocker-node' commands.append(run_from_args( ["yum", "install"] + repo_options + ["-y", package])) return sequence(commands) else: raise UnsupportedDistribution()