def dataset_backend(self): """ Get the storage driver the acceptance testing nodes will use. :return: A constant from ``DatasetBackend`` matching the name of the backend chosen by the command-line options. """ configuration = self.dataset_backend_configuration() # Avoid requiring repetition of the backend name when it is the same as # the name of the configuration section. But allow it so that there # can be "great-openstack-provider" and "better-openstack-provider" # sections side-by-side that both use "openstack" backend but configure # it slightly differently. dataset_backend_name = configuration.get("backend", self["dataset-backend"]) try: return DatasetBackend.lookupByName(dataset_backend_name) except ValueError: raise UsageError("Unknown dataset backend: {}".format(dataset_backend_name))
def dataset_backend(self): """ Get the storage driver the acceptance testing nodes will use. :return: A constant from ``DatasetBackend`` matching the name of the backend chosen by the command-line options. """ configuration = self.dataset_backend_configuration() # Avoid requiring repetition of the backend name when it is the same as # the name of the configuration section. But allow it so that there # can be "great-openstack-provider" and "better-openstack-provider" # sections side-by-side that both use "openstack" backend but configure # it slightly differently. dataset_backend_name = configuration.get("backend", self["dataset-backend"]) try: return DatasetBackend.lookupByName(dataset_backend_name) except ValueError: raise UsageError( "Unknown dataset backend: {}".format(dataset_backend_name))
class CommonOptions(Options): """ Options common to ``run-acceptance-tests`` and ``setup-cluster``. """ optParameters = [ ['distribution', None, None, 'The target distribution. ' 'One of {}.'.format(', '.join(DISTRIBUTIONS))], ['provider', None, 'vagrant', 'The compute-resource provider to test against. ' 'One of {}.'], ['dataset-backend', None, 'zfs', 'The dataset backend to test against. ' 'One of {}'.format(', '.join(backend.name for backend in DatasetBackend.iterconstants()))], ['config-file', None, None, 'Configuration for compute-resource providers and dataset backends.'], ['branch', None, None, 'Branch to grab packages from'], ['flocker-version', None, None, 'Version of flocker to install'], ['build-server', None, 'http://build.clusterhq.com/', 'Base URL of build server for package downloads'], ['number-of-nodes', None, int(os.environ.get("FLOCKER_ACCEPTANCE_NUM_NODES", 2)), 'Number of nodes to start; default is 2 unless you set the deprecated' ' environment variable which was previous way to do this.', int], ] def __init__(self, top_level): """ :param FilePath top_level: The top-level of the flocker repository. """ Options.__init__(self) self.docs['provider'] = self.docs['provider'].format( self._get_provider_names() ) self.top_level = top_level self['variants'] = [] def _get_provider_names(self): """ Find the names of all supported "providers" (eg Vagrant, Rackspace). :return: A ``list`` of ``str`` giving all such names. """ return prefixedMethodNames(self.__class__, "_runner_") def opt_variant(self, arg): """ Specify a variant of the provisioning to run. Supported variants: distro-testing, docker-head, zfs-testing. """ self['variants'].append(Variants.lookupByValue(arg)) def dataset_backend_configuration(self): """ Get the configuration corresponding to storage driver chosen by the command line options. """ drivers = self['config'].get('storage-drivers', {}) configuration = drivers.get(self['dataset-backend'], {}) return configuration def dataset_backend(self): """ Get the storage driver the acceptance testing nodes will use. :return: A constant from ``DatasetBackend`` matching the name of the backend chosen by the command-line options. """ configuration = self.dataset_backend_configuration() # Avoid requiring repetition of the backend name when it is the same as # the name of the configuration section. But allow it so that there # can be "great-openstack-provider" and "better-openstack-provider" # sections side-by-side that both use "openstack" backend but configure # it slightly differently. dataset_backend_name = configuration.get( "backend", self["dataset-backend"] ) try: return DatasetBackend.lookupByName(dataset_backend_name) except ValueError: raise UsageError( "Unknown dataset backend: {}".format( dataset_backend_name ) ) def postOptions(self): if self['distribution'] is None: raise UsageError("Distribution required.") if self['config-file'] is not None: config_file = FilePath(self['config-file']) self['config'] = yaml.safe_load(config_file.getContent()) else: self['config'] = {} if self.get('cert-directory') is None: self['cert-directory'] = FilePath(mkdtemp()) provider = self['provider'].lower() provider_config = self['config'].get(provider, {}) package_source = PackageSource( version=self['flocker-version'], branch=self['branch'], build_server=self['build-server'], ) try: get_runner = getattr(self, "_runner_" + provider.upper()) except AttributeError: raise UsageError( "Provider {!r} not supported. Available providers: {}".format( provider, ', '.join( name.lower() for name in self._get_provider_names() ) ) ) else: self.runner = get_runner( package_source=package_source, dataset_backend=self.dataset_backend(), provider_config=provider_config, ) def _make_cluster_identity(self, dataset_backend): """ Build a cluster identity based on the parameters. """ cluster_id = make_cluster_id( TestTypes.ACCEPTANCE, _provider_for_cluster_id(dataset_backend), ) return ClusterIdentity( purpose=u'acceptance-testing', prefix=u'acceptance-test', name=b'acceptance-cluster', id=cluster_id, ) def _provider_config_missing(self, provider): """ :param str provider: The name of the missing provider. :raise: ``UsageError`` indicating which provider configuration was missing. """ raise UsageError( "Configuration file must include a " "{!r} config stanza.".format(provider) ) def _runner_VAGRANT(self, package_source, dataset_backend, provider_config): """ :param PackageSource package_source: The source of omnibus packages. :param DatasetBackend dataset_backend: A ``DatasetBackend`` constant. :param provider_config: The ``vagrant`` section of the acceptance testing configuration file. Since the Vagrant runner accepts no configuration, this is ignored. :returns: ``VagrantRunner`` """ return VagrantRunner( config=self['config'], top_level=self.top_level, distribution=self['distribution'], package_source=package_source, variants=self['variants'], dataset_backend=dataset_backend, dataset_backend_configuration=self.dataset_backend_configuration() ) def _runner_MANAGED(self, package_source, dataset_backend, provider_config): """ :param PackageSource package_source: The source of omnibus packages. :param DatasetBackend dataset_backend: A ``DatasetBackend`` constant. :param provider_config: The ``managed`` section of the acceptance testing configuration file. The section of the configuration file should look something like: managed: addresses: - "172.16.255.240" - "172.16.255.241" distribution: "centos-7" :returns: ``ManagedRunner``. """ if provider_config is None: self._provider_config_missing("managed") if not provider_config.get("upgrade"): package_source = None return ManagedRunner( node_addresses=provider_config['addresses'], package_source=package_source, # TODO LATER Might be nice if this were part of # provider_config. See FLOC-2078. distribution=self['distribution'], dataset_backend=dataset_backend, dataset_backend_configuration=self.dataset_backend_configuration(), identity=self._make_cluster_identity(dataset_backend), cert_path=self['cert-directory'], logging_config=self['config'].get('logging'), ) def _libcloud_runner(self, package_source, dataset_backend, provider, provider_config): """ Run some nodes using ``libcloud``. By default, two nodes are run. This can be overridden by using the ``--number-of-nodes`` command line option. :param PackageSource package_source: The source of omnibus packages. :param DatasetBackend dataset_backend: A ``DatasetBackend`` constant. :param provider: The name of the cloud provider of nodes for the tests. :param provider_config: The ``managed`` section of the acceptance :returns: ``LibcloudRunner``. """ if provider_config is None: self._provider_config_missing(provider) provisioner = CLOUD_PROVIDERS[provider](**provider_config) return LibcloudRunner( config=self['config'], top_level=self.top_level, distribution=self['distribution'], package_source=package_source, provisioner=provisioner, dataset_backend=dataset_backend, dataset_backend_configuration=self.dataset_backend_configuration(), variants=self['variants'], num_nodes=self['number-of-nodes'], identity=self._make_cluster_identity(dataset_backend), cert_path=self['cert-directory'], ) def _runner_RACKSPACE(self, package_source, dataset_backend, provider_config): """ :param PackageSource package_source: The source of omnibus packages. :param DatasetBackend dataset_backend: A ``DatasetBackend`` constant. :param dict provider_config: The ``rackspace`` section of the acceptance testing configuration file. See the linked documentation for the form of that section. :see: :ref:`acceptance-testing-rackspace-config` """ return self._libcloud_runner( package_source, dataset_backend, "rackspace", provider_config ) def _runner_AWS(self, package_source, dataset_backend, provider_config): """ :param PackageSource package_source: The source of omnibus packages. :param DatasetBackend dataset_backend: A ``DatasetBackend`` constant. :param dict provider_config: The ``aws`` section of the acceptance testing configuration file. See the linked documentation for the form of that section. :see: :ref:`acceptance-testing-aws-config` """ return self._libcloud_runner( package_source, dataset_backend, "aws", provider_config )
class RunOptions(Options): description = "Run the acceptance tests." optParameters = [ [ 'distribution', None, None, 'The target distribution. ' 'One of {}.'.format(', '.join(DISTRIBUTIONS)) ], [ 'provider', None, 'vagrant', 'The compute-resource provider to test against. ' 'One of {}.' ], [ 'dataset-backend', None, 'zfs', 'The dataset backend to test against. ' 'One of {}'.format(', '.join( backend.name for backend in DatasetBackend.iterconstants())) ], [ 'config-file', None, None, 'Configuration for compute-resource providers and dataset backends.' ], ['branch', None, None, 'Branch to grab packages from'], [ 'flocker-version', None, flocker.__version__, 'Version of flocker to install' ], [ 'build-server', None, 'http://build.clusterhq.com/', 'Base URL of build server for package downloads' ], ] optFlags = [ ["keep", "k", "Keep VMs around, if the tests fail."], [ "no-pull", None, "Do not pull any Docker images when provisioning nodes." ], ] synopsis = ('Usage: run-acceptance-tests --distribution <distribution> ' '[--provider <provider>] [<test-cases>]') def __init__(self, top_level): """ :param FilePath top_level: The top-level of the flocker repository. """ Options.__init__(self) self.docs['provider'] = self.docs['provider'].format( self._get_provider_names()) self.top_level = top_level self['variants'] = [] def _get_provider_names(self): """ Find the names of all supported "providers" (eg Vagrant, Rackspace). :return: A ``list`` of ``str`` giving all such names. """ return prefixedMethodNames(self.__class__, "_runner_") def opt_variant(self, arg): """ Specify a variant of the provisioning to run. Supported variants: distro-testing, docker-head, zfs-testing. """ self['variants'].append(Variants.lookupByValue(arg)) def parseArgs(self, *trial_args): self['trial-args'] = trial_args def dataset_backend_configuration(self): """ Get the configuration corresponding to storage driver chosen by the command line options. """ drivers = self['config'].get('storage-drivers', {}) configuration = drivers.get(self['dataset-backend'], {}) return configuration def dataset_backend(self): """ Get the storage driver the acceptance testing nodes will use. :return: A constant from ``DatasetBackend`` matching the name of the backend chosen by the command-line options. """ configuration = self.dataset_backend_configuration() # Avoid requiring repetition of the backend name when it is the same as # the name of the configuration section. But allow it so that there # can be "great-openstack-provider" and "better-openstack-provider" # sections side-by-side that both use "openstack" backend but configure # it slightly differently. dataset_backend_name = configuration.get("backend", self["dataset-backend"]) try: return DatasetBackend.lookupByName(dataset_backend_name) except ValueError: raise UsageError( "Unknown dataset backend: {}".format(dataset_backend_name)) def postOptions(self): if self['distribution'] is None: raise UsageError("Distribution required.") if self['config-file'] is not None: config_file = FilePath(self['config-file']) self['config'] = yaml.safe_load(config_file.getContent()) else: self['config'] = {} provider = self['provider'].lower() provider_config = self['config'].get(provider, {}) if self['flocker-version']: rpm_version = make_rpm_version(self['flocker-version']) os_version = "%s-%s" % (rpm_version.version, rpm_version.release) if os_version.endswith('.dirty'): os_version = os_version[:-len('.dirty')] else: os_version = None package_source = PackageSource( version=self['flocker-version'], os_version=os_version, branch=self['branch'], build_server=self['build-server'], ) try: get_runner = getattr(self, "_runner_" + provider.upper()) except AttributeError: raise UsageError( "Provider {!r} not supported. Available providers: {}".format( provider, ', '.join(name.lower() for name in self._get_provider_names()))) else: self.runner = get_runner( package_source=package_source, dataset_backend=self.dataset_backend(), provider_config=provider_config, ) def _provider_config_missing(self, provider): """ :param str provider: The name of the missing provider. :raise: ``UsageError`` indicating which provider configuration was missing. """ raise UsageError("Configuration file must include a " "{!r} config stanza.".format(provider)) def _runner_VAGRANT(self, package_source, dataset_backend, provider_config): """ :param PackageSource package_source: The source of omnibus packages. :param DatasetBackend dataset_backend: A ``DatasetBackend`` constant. :param provider_config: The ``vagrant`` section of the acceptance testing configuration file. Since the Vagrant runner accepts no configuration, this is ignored. :returns: ``VagrantRunner`` """ return VagrantRunner( config=self['config'], top_level=self.top_level, distribution=self['distribution'], package_source=package_source, variants=self['variants'], dataset_backend=dataset_backend, dataset_backend_configuration=self.dataset_backend_configuration()) def _runner_MANAGED(self, package_source, dataset_backend, provider_config): """ :param PackageSource package_source: The source of omnibus packages. :param DatasetBackend dataset_backend: A ``DatasetBackend`` constant. :param provider_config: The ``managed`` section of the acceptance testing configuration file. The section of the configuration file should look something like: managed: addresses: - "172.16.255.240" - "172.16.255.241" distribution: "centos-7" :returns: ``ManagedRunner``. """ if provider_config is None: self._provider_config_missing("managed") if not provider_config.get("upgrade"): package_source = None return ManagedRunner( node_addresses=provider_config['addresses'], package_source=package_source, # TODO LATER Might be nice if this were part of # provider_config. See FLOC-2078. distribution=self['distribution'], dataset_backend=dataset_backend, dataset_backend_configuration=self.dataset_backend_configuration(), ) def _libcloud_runner(self, package_source, dataset_backend, provider, provider_config): """ Run some nodes using ``libcloud``. :param PackageSource package_source: The source of omnibus packages. :param DatasetBackend dataset_backend: A ``DatasetBackend`` constant. :param provider: The name of the cloud provider of nodes for the tests. :param provider_config: The ``managed`` section of the acceptance :returns: ``LibcloudRunner``. """ if provider_config is None: self._provider_config_missing(provider) provisioner = CLOUD_PROVIDERS[provider](**provider_config) return LibcloudRunner( config=self['config'], top_level=self.top_level, distribution=self['distribution'], package_source=package_source, provisioner=provisioner, dataset_backend=dataset_backend, dataset_backend_configuration=self.dataset_backend_configuration(), variants=self['variants'], ) def _runner_RACKSPACE(self, package_source, dataset_backend, provider_config): """ :param PackageSource package_source: The source of omnibus packages. :param DatasetBackend dataset_backend: A ``DatasetBackend`` constant. :param provider_config: The ``rackspace`` section of the acceptance testing configuration file. The section of the configuration file should look something like: rackspace: region: <rackspace region, e.g. "iad"> username: <rackspace username> key: <access key> keyname: <ssh-key-name> :see: :ref:`acceptance-testing-rackspace-config` """ return self._libcloud_runner(package_source, dataset_backend, "rackspace", provider_config) def _runner_AWS(self, package_source, dataset_backend, provider_config): """ :param PackageSource package_source: The source of omnibus packages. :param DatasetBackend dataset_backend: A ``DatasetBackend`` constant. :param provider_config: The ``aws`` section of the acceptance testing configuration file. The section of the configuration file should look something like: aws: region: <aws region, e.g. "us-west-2"> zone: <aws zone, e.g. "us-west-2a"> access_key: <aws access key> secret_access_token: <aws secret access token> keyname: <ssh-key-name> security_groups: ["<permissive security group>"] :see: :ref:`acceptance-testing-aws-config` """ return self._libcloud_runner(package_source, dataset_backend, "aws", provider_config)
def postOptions(self): if self['distribution'] is None: raise UsageError("Distribution required.") try: self.dataset_backend = DatasetBackend.lookupByName( self['dataset-backend']) except ValueError: raise UsageError("Unknown dataset backend: %s" % (self['dataset-backend'])) if self['config-file'] is not None: config_file = FilePath(self['config-file']) self['config'] = yaml.safe_load(config_file.getContent()) else: self['config'] = {} if self['flocker-version']: rpm_version = make_rpm_version(self['flocker-version']) os_version = "%s-%s" % (rpm_version.version, rpm_version.release) if os_version.endswith('.dirty'): os_version = os_version[:-len('.dirty')] else: os_version = None package_source = PackageSource( version=self['flocker-version'], os_version=os_version, branch=self['branch'], build_server=self['build-server'], ) if self['provider'] not in PROVIDERS: raise UsageError( "Provider %r not supported. Available providers: %s" % (self['provider'], ', '.join(PROVIDERS))) if self['provider'] in CLOUD_PROVIDERS: # Configuration must include credentials etc for cloud providers. try: provider_config = self['config'][self['provider']] except KeyError: raise UsageError( "Configuration file must include a " "{!r} config stanza.".format(self['provider']) ) provisioner = CLOUD_PROVIDERS[self['provider']](**provider_config) self.runner = LibcloudRunner( config=self['config'], top_level=self.top_level, distribution=self['distribution'], package_source=package_source, provisioner=provisioner, dataset_backend=self.dataset_backend, variants=self['variants'], ) else: self.runner = VagrantRunner( config=self['config'], top_level=self.top_level, distribution=self['distribution'], package_source=package_source, variants=self['variants'], )