def dataset_backend(self):
"""
Get the storage driver the acceptance testing nodes will use.
:return: A constant from ``DatasetBackend`` matching the name of the
backend chosen by the command-line options.
"""
configuration = self.dataset_backend_configuration()
# Avoid requiring repetition of the backend name when it is the same as
# the name of the configuration section. But allow it so that there
# can be "great-openstack-provider" and "better-openstack-provider"
# sections side-by-side that both use "openstack" backend but configure
# it slightly differently.
dataset_backend_name = configuration.get("backend", self["dataset-backend"])
try:
return DatasetBackend.lookupByName(dataset_backend_name)
except ValueError:
raise UsageError("Unknown dataset backend: {}".format(dataset_backend_name))
def dataset_backend(self):
"""
Get the storage driver the acceptance testing nodes will use.
:return: A constant from ``DatasetBackend`` matching the name of the
backend chosen by the command-line options.
"""
configuration = self.dataset_backend_configuration()
# Avoid requiring repetition of the backend name when it is the same as
# the name of the configuration section. But allow it so that there
# can be "great-openstack-provider" and "better-openstack-provider"
# sections side-by-side that both use "openstack" backend but configure
# it slightly differently.
dataset_backend_name = configuration.get("backend",
self["dataset-backend"])
try:
return DatasetBackend.lookupByName(dataset_backend_name)
except ValueError:
raise UsageError(
"Unknown dataset backend: {}".format(dataset_backend_name))
class CommonOptions(Options):
"""
Options common to ``run-acceptance-tests`` and ``setup-cluster``.
"""
optParameters = [
['distribution', None, None,
'The target distribution. '
'One of {}.'.format(', '.join(DISTRIBUTIONS))],
['provider', None, 'vagrant',
'The compute-resource provider to test against. '
'One of {}.'],
['dataset-backend', None, 'zfs',
'The dataset backend to test against. '
'One of {}'.format(', '.join(backend.name for backend
in DatasetBackend.iterconstants()))],
['config-file', None, None,
'Configuration for compute-resource providers and dataset backends.'],
['branch', None, None, 'Branch to grab packages from'],
['flocker-version', None, None, 'Version of flocker to install'],
['build-server', None, 'http://build.clusterhq.com/',
'Base URL of build server for package downloads'],
['number-of-nodes', None,
int(os.environ.get("FLOCKER_ACCEPTANCE_NUM_NODES", 2)),
'Number of nodes to start; default is 2 unless you set the deprecated'
' environment variable which was previous way to do this.', int],
]
def __init__(self, top_level):
"""
:param FilePath top_level: The top-level of the flocker repository.
"""
Options.__init__(self)
self.docs['provider'] = self.docs['provider'].format(
self._get_provider_names()
)
self.top_level = top_level
self['variants'] = []
def _get_provider_names(self):
"""
Find the names of all supported "providers" (eg Vagrant, Rackspace).
:return: A ``list`` of ``str`` giving all such names.
"""
return prefixedMethodNames(self.__class__, "_runner_")
def opt_variant(self, arg):
"""
Specify a variant of the provisioning to run.
Supported variants: distro-testing, docker-head, zfs-testing.
"""
self['variants'].append(Variants.lookupByValue(arg))
def dataset_backend_configuration(self):
"""
Get the configuration corresponding to storage driver chosen by the
command line options.
"""
drivers = self['config'].get('storage-drivers', {})
configuration = drivers.get(self['dataset-backend'], {})
return configuration
def dataset_backend(self):
"""
Get the storage driver the acceptance testing nodes will use.
:return: A constant from ``DatasetBackend`` matching the name of the
backend chosen by the command-line options.
"""
configuration = self.dataset_backend_configuration()
# Avoid requiring repetition of the backend name when it is the same as
# the name of the configuration section. But allow it so that there
# can be "great-openstack-provider" and "better-openstack-provider"
# sections side-by-side that both use "openstack" backend but configure
# it slightly differently.
dataset_backend_name = configuration.get(
"backend", self["dataset-backend"]
)
try:
return DatasetBackend.lookupByName(dataset_backend_name)
except ValueError:
raise UsageError(
"Unknown dataset backend: {}".format(
dataset_backend_name
)
)
def postOptions(self):
if self['distribution'] is None:
raise UsageError("Distribution required.")
if self['config-file'] is not None:
config_file = FilePath(self['config-file'])
self['config'] = yaml.safe_load(config_file.getContent())
else:
self['config'] = {}
if self.get('cert-directory') is None:
self['cert-directory'] = FilePath(mkdtemp())
provider = self['provider'].lower()
provider_config = self['config'].get(provider, {})
package_source = PackageSource(
version=self['flocker-version'],
branch=self['branch'],
build_server=self['build-server'],
)
try:
get_runner = getattr(self, "_runner_" + provider.upper())
except AttributeError:
raise UsageError(
"Provider {!r} not supported. Available providers: {}".format(
provider, ', '.join(
name.lower() for name in self._get_provider_names()
)
)
)
else:
self.runner = get_runner(
package_source=package_source,
dataset_backend=self.dataset_backend(),
provider_config=provider_config,
)
def _make_cluster_identity(self, dataset_backend):
"""
Build a cluster identity based on the parameters.
"""
cluster_id = make_cluster_id(
TestTypes.ACCEPTANCE,
_provider_for_cluster_id(dataset_backend),
)
return ClusterIdentity(
purpose=u'acceptance-testing',
prefix=u'acceptance-test',
name=b'acceptance-cluster',
id=cluster_id,
)
def _provider_config_missing(self, provider):
"""
:param str provider: The name of the missing provider.
:raise: ``UsageError`` indicating which provider configuration was
missing.
"""
raise UsageError(
"Configuration file must include a "
"{!r} config stanza.".format(provider)
)
def _runner_VAGRANT(self, package_source,
dataset_backend, provider_config):
"""
:param PackageSource package_source: The source of omnibus packages.
:param DatasetBackend dataset_backend: A ``DatasetBackend`` constant.
:param provider_config: The ``vagrant`` section of the acceptance
testing configuration file. Since the Vagrant runner accepts no
configuration, this is ignored.
:returns: ``VagrantRunner``
"""
return VagrantRunner(
config=self['config'],
top_level=self.top_level,
distribution=self['distribution'],
package_source=package_source,
variants=self['variants'],
dataset_backend=dataset_backend,
dataset_backend_configuration=self.dataset_backend_configuration()
)
def _runner_MANAGED(self, package_source, dataset_backend,
provider_config):
"""
:param PackageSource package_source: The source of omnibus packages.
:param DatasetBackend dataset_backend: A ``DatasetBackend`` constant.
:param provider_config: The ``managed`` section of the acceptance
testing configuration file. The section of the configuration
file should look something like:
managed:
addresses:
- "172.16.255.240"
- "172.16.255.241"
distribution: "centos-7"
:returns: ``ManagedRunner``.
"""
if provider_config is None:
self._provider_config_missing("managed")
if not provider_config.get("upgrade"):
package_source = None
return ManagedRunner(
node_addresses=provider_config['addresses'],
package_source=package_source,
# TODO LATER Might be nice if this were part of
# provider_config. See FLOC-2078.
distribution=self['distribution'],
dataset_backend=dataset_backend,
dataset_backend_configuration=self.dataset_backend_configuration(),
identity=self._make_cluster_identity(dataset_backend),
cert_path=self['cert-directory'],
logging_config=self['config'].get('logging'),
)
def _libcloud_runner(self, package_source, dataset_backend,
provider, provider_config):
"""
Run some nodes using ``libcloud``.
By default, two nodes are run. This can be overridden by using
the ``--number-of-nodes`` command line option.
:param PackageSource package_source: The source of omnibus packages.
:param DatasetBackend dataset_backend: A ``DatasetBackend`` constant.
:param provider: The name of the cloud provider of nodes for the tests.
:param provider_config: The ``managed`` section of the acceptance
:returns: ``LibcloudRunner``.
"""
if provider_config is None:
self._provider_config_missing(provider)
provisioner = CLOUD_PROVIDERS[provider](**provider_config)
return LibcloudRunner(
config=self['config'],
top_level=self.top_level,
distribution=self['distribution'],
package_source=package_source,
provisioner=provisioner,
dataset_backend=dataset_backend,
dataset_backend_configuration=self.dataset_backend_configuration(),
variants=self['variants'],
num_nodes=self['number-of-nodes'],
identity=self._make_cluster_identity(dataset_backend),
cert_path=self['cert-directory'],
)
def _runner_RACKSPACE(self, package_source, dataset_backend,
provider_config):
"""
:param PackageSource package_source: The source of omnibus packages.
:param DatasetBackend dataset_backend: A ``DatasetBackend`` constant.
:param dict provider_config: The ``rackspace`` section of the
acceptance testing configuration file. See the linked
documentation for the form of that section.
:see: :ref:`acceptance-testing-rackspace-config`
"""
return self._libcloud_runner(
package_source, dataset_backend, "rackspace", provider_config
)
def _runner_AWS(self, package_source, dataset_backend,
provider_config):
"""
:param PackageSource package_source: The source of omnibus packages.
:param DatasetBackend dataset_backend: A ``DatasetBackend`` constant.
:param dict provider_config: The ``aws`` section of the acceptance
testing configuration file. See the linked documentation for the
form of that section.
:see: :ref:`acceptance-testing-aws-config`
"""
return self._libcloud_runner(
package_source, dataset_backend, "aws", provider_config
)
class RunOptions(Options):
description = "Run the acceptance tests."
optParameters = [
[
'distribution', None, None, 'The target distribution. '
'One of {}.'.format(', '.join(DISTRIBUTIONS))
],
[
'provider', None, 'vagrant',
'The compute-resource provider to test against. '
'One of {}.'
],
[
'dataset-backend', None, 'zfs',
'The dataset backend to test against. '
'One of {}'.format(', '.join(
backend.name for backend in DatasetBackend.iterconstants()))
],
[
'config-file', None, None,
'Configuration for compute-resource providers and dataset backends.'
],
['branch', None, None, 'Branch to grab packages from'],
[
'flocker-version', None, flocker.__version__,
'Version of flocker to install'
],
[
'build-server', None, 'http://build.clusterhq.com/',
'Base URL of build server for package downloads'
],
]
optFlags = [
["keep", "k", "Keep VMs around, if the tests fail."],
[
"no-pull", None,
"Do not pull any Docker images when provisioning nodes."
],
]
synopsis = ('Usage: run-acceptance-tests --distribution <distribution> '
'[--provider <provider>] [<test-cases>]')
def __init__(self, top_level):
"""
:param FilePath top_level: The top-level of the flocker repository.
"""
Options.__init__(self)
self.docs['provider'] = self.docs['provider'].format(
self._get_provider_names())
self.top_level = top_level
self['variants'] = []
def _get_provider_names(self):
"""
Find the names of all supported "providers" (eg Vagrant, Rackspace).
:return: A ``list`` of ``str`` giving all such names.
"""
return prefixedMethodNames(self.__class__, "_runner_")
def opt_variant(self, arg):
"""
Specify a variant of the provisioning to run.
Supported variants: distro-testing, docker-head, zfs-testing.
"""
self['variants'].append(Variants.lookupByValue(arg))
def parseArgs(self, *trial_args):
self['trial-args'] = trial_args
def dataset_backend_configuration(self):
"""
Get the configuration corresponding to storage driver chosen by the
command line options.
"""
drivers = self['config'].get('storage-drivers', {})
configuration = drivers.get(self['dataset-backend'], {})
return configuration
def dataset_backend(self):
"""
Get the storage driver the acceptance testing nodes will use.
:return: A constant from ``DatasetBackend`` matching the name of the
backend chosen by the command-line options.
"""
configuration = self.dataset_backend_configuration()
# Avoid requiring repetition of the backend name when it is the same as
# the name of the configuration section. But allow it so that there
# can be "great-openstack-provider" and "better-openstack-provider"
# sections side-by-side that both use "openstack" backend but configure
# it slightly differently.
dataset_backend_name = configuration.get("backend",
self["dataset-backend"])
try:
return DatasetBackend.lookupByName(dataset_backend_name)
except ValueError:
raise UsageError(
"Unknown dataset backend: {}".format(dataset_backend_name))
def postOptions(self):
if self['distribution'] is None:
raise UsageError("Distribution required.")
if self['config-file'] is not None:
config_file = FilePath(self['config-file'])
self['config'] = yaml.safe_load(config_file.getContent())
else:
self['config'] = {}
provider = self['provider'].lower()
provider_config = self['config'].get(provider, {})
if self['flocker-version']:
rpm_version = make_rpm_version(self['flocker-version'])
os_version = "%s-%s" % (rpm_version.version, rpm_version.release)
if os_version.endswith('.dirty'):
os_version = os_version[:-len('.dirty')]
else:
os_version = None
package_source = PackageSource(
version=self['flocker-version'],
os_version=os_version,
branch=self['branch'],
build_server=self['build-server'],
)
try:
get_runner = getattr(self, "_runner_" + provider.upper())
except AttributeError:
raise UsageError(
"Provider {!r} not supported. Available providers: {}".format(
provider,
', '.join(name.lower()
for name in self._get_provider_names())))
else:
self.runner = get_runner(
package_source=package_source,
dataset_backend=self.dataset_backend(),
provider_config=provider_config,
)
def _provider_config_missing(self, provider):
"""
:param str provider: The name of the missing provider.
:raise: ``UsageError`` indicating which provider configuration was
missing.
"""
raise UsageError("Configuration file must include a "
"{!r} config stanza.".format(provider))
def _runner_VAGRANT(self, package_source, dataset_backend,
provider_config):
"""
:param PackageSource package_source: The source of omnibus packages.
:param DatasetBackend dataset_backend: A ``DatasetBackend`` constant.
:param provider_config: The ``vagrant`` section of the acceptance
testing configuration file. Since the Vagrant runner accepts no
configuration, this is ignored.
:returns: ``VagrantRunner``
"""
return VagrantRunner(
config=self['config'],
top_level=self.top_level,
distribution=self['distribution'],
package_source=package_source,
variants=self['variants'],
dataset_backend=dataset_backend,
dataset_backend_configuration=self.dataset_backend_configuration())
def _runner_MANAGED(self, package_source, dataset_backend,
provider_config):
"""
:param PackageSource package_source: The source of omnibus packages.
:param DatasetBackend dataset_backend: A ``DatasetBackend`` constant.
:param provider_config: The ``managed`` section of the acceptance
testing configuration file. The section of the configuration
file should look something like:
managed:
addresses:
- "172.16.255.240"
- "172.16.255.241"
distribution: "centos-7"
:returns: ``ManagedRunner``.
"""
if provider_config is None:
self._provider_config_missing("managed")
if not provider_config.get("upgrade"):
package_source = None
return ManagedRunner(
node_addresses=provider_config['addresses'],
package_source=package_source,
# TODO LATER Might be nice if this were part of
# provider_config. See FLOC-2078.
distribution=self['distribution'],
dataset_backend=dataset_backend,
dataset_backend_configuration=self.dataset_backend_configuration(),
)
def _libcloud_runner(self, package_source, dataset_backend, provider,
provider_config):
"""
Run some nodes using ``libcloud``.
:param PackageSource package_source: The source of omnibus packages.
:param DatasetBackend dataset_backend: A ``DatasetBackend`` constant.
:param provider: The name of the cloud provider of nodes for the tests.
:param provider_config: The ``managed`` section of the acceptance
:returns: ``LibcloudRunner``.
"""
if provider_config is None:
self._provider_config_missing(provider)
provisioner = CLOUD_PROVIDERS[provider](**provider_config)
return LibcloudRunner(
config=self['config'],
top_level=self.top_level,
distribution=self['distribution'],
package_source=package_source,
provisioner=provisioner,
dataset_backend=dataset_backend,
dataset_backend_configuration=self.dataset_backend_configuration(),
variants=self['variants'],
)
def _runner_RACKSPACE(self, package_source, dataset_backend,
provider_config):
"""
:param PackageSource package_source: The source of omnibus packages.
:param DatasetBackend dataset_backend: A ``DatasetBackend`` constant.
:param provider_config: The ``rackspace`` section of the acceptance
testing configuration file. The section of the configuration
file should look something like:
rackspace:
region: <rackspace region, e.g. "iad">
username: <rackspace username>
key: <access key>
keyname: <ssh-key-name>
:see: :ref:`acceptance-testing-rackspace-config`
"""
return self._libcloud_runner(package_source, dataset_backend,
"rackspace", provider_config)
def _runner_AWS(self, package_source, dataset_backend, provider_config):
"""
:param PackageSource package_source: The source of omnibus packages.
:param DatasetBackend dataset_backend: A ``DatasetBackend`` constant.
:param provider_config: The ``aws`` section of the acceptance testing
configuration file. The section of the configuration file should
look something like:
aws:
region: <aws region, e.g. "us-west-2">
zone: <aws zone, e.g. "us-west-2a">
access_key: <aws access key>
secret_access_token: <aws secret access token>
keyname: <ssh-key-name>
security_groups: ["<permissive security group>"]
:see: :ref:`acceptance-testing-aws-config`
"""
return self._libcloud_runner(package_source, dataset_backend, "aws",
provider_config)
def postOptions(self):
if self['distribution'] is None:
raise UsageError("Distribution required.")
try:
self.dataset_backend = DatasetBackend.lookupByName(
self['dataset-backend'])
except ValueError:
raise UsageError("Unknown dataset backend: %s"
% (self['dataset-backend']))
if self['config-file'] is not None:
config_file = FilePath(self['config-file'])
self['config'] = yaml.safe_load(config_file.getContent())
else:
self['config'] = {}
if self['flocker-version']:
rpm_version = make_rpm_version(self['flocker-version'])
os_version = "%s-%s" % (rpm_version.version, rpm_version.release)
if os_version.endswith('.dirty'):
os_version = os_version[:-len('.dirty')]
else:
os_version = None
package_source = PackageSource(
version=self['flocker-version'],
os_version=os_version,
branch=self['branch'],
build_server=self['build-server'],
)
if self['provider'] not in PROVIDERS:
raise UsageError(
"Provider %r not supported. Available providers: %s"
% (self['provider'], ', '.join(PROVIDERS)))
if self['provider'] in CLOUD_PROVIDERS:
# Configuration must include credentials etc for cloud providers.
try:
provider_config = self['config'][self['provider']]
except KeyError:
raise UsageError(
"Configuration file must include a "
"{!r} config stanza.".format(self['provider'])
)
provisioner = CLOUD_PROVIDERS[self['provider']](**provider_config)
self.runner = LibcloudRunner(
config=self['config'],
top_level=self.top_level,
distribution=self['distribution'],
package_source=package_source,
provisioner=provisioner,
dataset_backend=self.dataset_backend,
variants=self['variants'],
)
else:
self.runner = VagrantRunner(
config=self['config'],
top_level=self.top_level,
distribution=self['distribution'],
package_source=package_source,
variants=self['variants'],
)
