def deploy(self, openstack_auth: Dict[str, str], reset=False):
'Deploying a Rally environment'
logging.info(f"Deploy: Rally environment {self.env_name}")
with elib.play_on(roles=self.rsc, gather_facts=False) as yaml:
# Creates the rally database if it does not exist
yaml.command(
f'{VENV}/bin/rally db ensure',
**title('ensure database exists'))
if not self.env_exists() or reset:
# Set the tracker of done tasks to 0
self._tasks = []
# Creates the rally environment and check that provided
# `openstack_auth` are correct credentials
yaml.command(
f"{VENV}/bin/rally env delete"
f" --env='{self.env_name}' --force",
**title(f'delete environment {self.env_name}'),
ignore_errors=True)
yaml.command(
f"{VENV}/bin/rally env create"
" --from-sysenv --no-use"
f" --name='{self.env_name}'",
**title(f'create environment {self.env_name}'),
environment=openstack_auth)
yaml.command(
f"{VENV}/bin/rally env check "
f" --env='{self.env_name}'",
**title('ensure OpenStack credentials are correct'))
def pull(agents: List[elib_t.Host]):
'Pulling the docker image of Shaker '
logging.info("Pull: get docker image for Shaker")
with elib.play_on(roles={'all': agents}, gather_facts=False) as yaml:
yaml.docker_image(**title(f'pulling docker image {IMG}'),
name=IMG,
source='pull',
state='present')
def run_scenario(self,
scenario: Path,
arguments: Dict[str, Any],
plugin: Optional[Path] = None,
pattern_hosts: str = "all"):
'Execute the Rally local `scenario` with `arguments`.'
logging.info(f"Running rally {scenario} in env {self.env_name}")
scenario_name = scenario.name
scenario_local_path = str(scenario)
scenario_remote_path = f'~/{scenario_name}'
plugin_remote_path = '~/plugin'
_tag = scenario_name + '-' + str(uuid.uuid4())
# Executing the scenario
logging.debug(f'Executing scenario {scenario_name} with tag {_tag}...')
with elib.play_on(roles=self.rsc,
pattern_hosts=pattern_hosts,
gather_facts=False) as yaml:
# Setup the scenario
yaml.copy(
**title(f'copy {scenario_name}'),
src=scenario_local_path,
dest=scenario_remote_path)
# Copy plugin if any
if plugin:
yaml.copy(**title(f'copy rally plugin {plugin}'),
src=str(plugin),
dest=plugin_remote_path)
# Run the scenario
yaml.command(
(f'{VENV}/bin/rally'
+ (f' --plugin-paths {plugin_remote_path}' if plugin else '')
+ f' task start {scenario_remote_path}'
+ f' --task-args={shlex.quote(json.dumps(arguments))}'
+ f' --tag="{_tag}"'
+ f' --deployment="{self.env_name}"'),
ignore_errors=True,
**title(f'execute {scenario_name} (may take a while...)'))
# Get the uuid and mark the scenario done in the tasks tracker
uuid_by_hosts = {
host: values['stdout']
for host, values
in elib.run_command(
f'{VENV}/bin/rally task list --uuids-only'
f' --deployment="{self.env_name}"'
f' --tag {_tag}',
roles=self.rsc,
pattern_hosts=pattern_hosts).get('ok', {}).items()}
logging.info(f"Scenario finished with uuid {uuid_by_hosts}")
self._tasks.append((scenario_name, uuid_by_hosts))
def env_exists(self) -> bool:
'Test whether the Rally environment exists or not'
try:
with elib.play_on(roles=self.rsc,
gather_facts=False,
on_error_continue=False) as yaml:
yaml.raw(f"{VENV}/bin/rally env show '{self.env_name}'")
except elib.errors.EnosFailedHostsError:
logging.error('...ignoring')
return False
else:
return True
def _resolve_openstack_auth(
self, globals_values: Dict[str, Any]) -> Dict[str, Any]:
"Compute and returns the value of `globals_values['openstack_auth']`"
# Get the former system paths. We latter load kolla (required by the
# `put_address_in_context` filter) and change that path.
old_sys_paths = sys.path.copy()
# Temporary file to later store the result of the rendered
# `openstack_auth` variable by Ansible.
_, osauth_path = tempfile.mkstemp()
try:
# Load kolla-ansible specific filters `KOLLA_FILTERS` since the
# `{{openstack_auth}}` variable relies on the
# `put_address_in_context` filter to manage IPv6.
#
# Note(rcherrueau): we also have to load kolla_ansible because the
# filter in `KOLLA_FILTERS` does something like `form kolla_ansible
# import filters`. We load kolla_ansible from the virtual_env in
# the system path. In that case, pbr may complain with: Versioning
# for this project requires either an sdist tarball, or access
# to an upstream git repository. We set pbr to version '1.2.3' to
# disable all version calculation logic by pbr [0].
# [0]https://docs.openstack.org/pbr/latest/user/packagers.html#versioning
ansible_filter_loader.add_directory(
str(self.venv_path / KOLLA_FILTERS))
sys.path.append(
str(self.venv_path / 'lib' / PY_VERSION / 'site-packages'))
os.environ['PBR_VERSION'] = '1.2.3'
# Render `openstack_auth` into `osauth_path`
with elib.play_on(roles={},
pattern_hosts="localhost",
extra_vars=globals_values) as yaml:
yaml.local_action(**title(
'Compute values of `openstack_auth`'),
module="copy",
content="{{ openstack_auth }}",
dest=osauth_path)
# Read and return the rendered values from `osauth_path`
with open(osauth_path, 'r') as rc_yaml:
return json.load(rc_yaml)
finally:
# Delete temporary `osauth_path` file
os.unlink(osauth_path)
# Reset system paths
sys.path = old_sys_paths
def backup(self, destination: Path, pattern_hosts: str = "all"):
'Backup Shaker HOME'
logging.info(f"Backup Shaker reports for home {self.home}")
with elib.play_on(roles=self.rsc,
pattern_hosts=pattern_hosts,
gather_facts=False) as yaml:
yaml.archive(**title(f"archive HOME {self.home}"),
path=self.home,
dest=self.home + '.tar.gz',
format='gz')
yaml.fetch(**title(f"fetch HOME {self.home}"),
src=self.home + '.tar.gz',
dest=str(destination /
'{{inventory_hostname}}-shaker.tar.gz'),
flat=True)
def deploy(self, openstack_auth: Dict[str, str], reset=False):
'Deploying a Shaker environment'
logging.info(f"Deploy: make shaker environment {self.home}")
self._openstack_auth = openstack_auth.copy()
with elib.play_on(roles=self.rsc, gather_facts=False) as yaml:
if reset: # Reset the environment
yaml.file(**title(f"delete HOME {self.home}"),
path=self.home,
state="absent")
# Create the environment
yaml.file(**title(f"create HOME {self.home}"),
path=self.home,
state='directory')
def run_scenario(self, scenario: str, pattern_hosts: str = "all"):
'Execute the Shaker `scenario`'
logging.info(f"Running Shaker {scenario}")
with elib.play_on(roles=self.rsc,
pattern_hosts=pattern_hosts,
gather_facts=False) as yaml:
yaml.docker_container(
**title(f"run {scenario} (may take a while...)"),
name=str(uuid.uuid4()),
image=IMG,
state='started',
ports=["11234:11234"],
detach=False,
volumes=[f"{self.home}:/artifacts"],
env=self._openstack_auth,
command=(' --flavor-name m1.medium'
' --server-endpoint {{ network_interface_ip }}:11234'
f' --scenario {scenario}'))
def backup(self, destination: Path):
'Backup kolla-ansible logs and conf'
logging.info('Backup kolla-ansible logs and conf')
with elib.play_on(inventory_path=str(self._inventory),
extra_vars=self.globals_values) as yaml:
yaml.archive(
**title('Archive kolla-ansible logs and conf'),
format='gz',
path=[
# kolla-ansible logs
'/var/lib/docker/volumes/kolla_logs/_data',
# kolla-ansible conf
'/etc/kolla'
],
dest='/tmp/kolla-log+conf.tar.gz')
yaml.fetch(**title('Fetch kolla-ansible logs and conf'),
flat=True,
src='/tmp/kolla-log+conf.tar.gz',
dest=(
str(destination) +
'/{{ inventory_hostname }}-kolla-log+conf.tar.gz'))
def pull(agents: List[elib_t.Host]):
'Installs Rally OpenStack'
logging.info("Pull: installing rally in a virtual environment")
with elib.play_on(roles={'all': agents},
priors=[elib.api.__python3__],
gather_facts=False) as yaml:
yaml.pip(**title(f'install virtualenv {VENV}'), name='virtualenv')
# XXX: https://cryptography.io/en/3.4.7/installation.html#rust
yaml.pip(**title('upgrade pip for cryptography dependency'),
name='pip',
state='latest',
virtualenv=VENV,
virtualenv_python='python3')
# XXX: We fix the version of decorator because of a bug when doing
# `rally task ...`. See
# https://bugs.launchpad.net/rally/+bug/1922707
yaml.pip(**title(f'install {PKG} in {VENV}'),
name=[PKG, 'decorator==4.4.2'],
state='present',
virtualenv=VENV)
def pull(pip_package: str, config_dir: Path) -> Path:
'''Install kolla-ansible in a virtual environment at `config_dir`.
The name of the virtual environment is computed based on the
`pip_package`, so calling that method with two different `pip_package`
values results in two different installations.
Args:
pip_package: The kolla-ansible pip package to install. Package
could be specified using the pip package syntax. For instance,
a PyPi package 'kolla-ansible==2.9.0', a git repository
'git+https://github.com/openstack/kolla-ansible.git@stable/ussuri',
or a local editable directory '-e ~/path/to/loca/kolla-ansible'.
config_dir: Directory to install kolla-ansible in.
'''
logging.info("Installing kolla-ansible and dependencies...")
# Generates a path for the virtual environment computing a
# deterministic hash, See https://stackoverflow.com/a/42089311
pip_ref_hash = int(
hashlib.sha256(pip_package.encode('utf-8')).hexdigest(),
16) % 10**8
venv = (config_dir / f'kolla-ansible-venv-{pip_ref_hash}').resolve()
# Install kolla-ansible and its dependencies
with elib.play_on(roles={}, pattern_hosts="localhost") as yaml:
yaml.local_action(
**title(f'Install {pip_package} in {venv}'),
module="pip",
# Pin Jinja2 version to fix the renaming of `contextfilter`
# into `pass_context.evalcontextfilter`.
# See https://github.com/BeyondTheClouds/enos/pull/346#issuecomment-1080851796 # noqa
name=[ANSIBLE_PKG, 'influxdb', 'Jinja2==3.0.3', pip_package],
virtualenv=str(venv),
virtualenv_python=PY_VERSION)
return venv
def backup(self, destination: Path, pattern_hosts: str = "all"):
'Generates json/html reports and backup them'
logging.info(f"Backup Rally reports for tasks {self._tasks}")
# Index the list of uuids by hosts
hosts_uuids = {}
for _, uuid_by_hosts in self._tasks:
for host, _uuid in uuid_by_hosts.items():
uuids = hosts_uuids.setdefault(host, [])
uuids.append(_uuid)
with elib.play_on(roles=self.rsc,
pattern_hosts=pattern_hosts,
gather_facts=False) as yaml:
# Generate html and json reports for all tasks
for host, uuids in hosts_uuids.items():
yaml.command(
f'{VENV}/bin/rally task report '
f' --uuid {" ".join(uuids)} --html-static'
f' --out rally-report.html',
**title(f'generate html report for {uuids}'),
when=f'inventory_hostname == "{host}"')
yaml.command(
f'{VENV}/bin/rally task report '
f' --uuid {" ".join(uuids)} --json'
f' --out rally-report.json',
**title(f'generate json report for {uuids}'),
when=f'inventory_hostname == "{host}"')
# Brings reports back
for ext in ('html', 'json'):
yaml.fetch(
**title(f'fetch the {ext} rally report'),
src=f'rally-report.{ext}', flat=True,
dest=(str(destination)
+ '/{{ inventory_hostname }}-rally-report.' + ext))
def prepare(env=None, **kwargs):
roles = env["roles"]
with en.play_on(roles=roles) as p:
p.debug(msg="Hello World !")