def disrupt_once(disrupt, ip_or_hostname, username, password, start_cmd,
stop_cmd, up_check_cmd, down_check_cmd, down_time_min,
down_time_max, cool_down_min, cool_down_max):
LOGGER.debug("Disrupt once {0}".format(ip_or_hostname))
down_time = random.randint(down_time_min, down_time_max)
cool_down_time = random.randint(cool_down_min, cool_down_max)
ssh = SSH(ip_or_hostname, username, password)
ssh.exec_command(stop_cmd)
ssh.exec_command(down_check_cmd) # TODO wait for down
if start_cmd: # TODO if service is down
time.sleep(down_time)
ssh.exec_command(start_cmd)
else:
disruption_finished = False
elapsed = 0
while not disruption_finished:
elapsed += cool_down_min
result, error = ssh.exec_command(up_check_cmd)
LOGGER.debug("Up check result is '{0}'. Error is {1}".format(
result.strip(), error.strip()))
if result is not None:
result = int(result.strip())
disruption_finished = (result == 0)
# TODO separate param for down_time_max
if not disruption_finished and elapsed > down_time_max:
LOGGER.exception(
"{disrupt} on {host} is not up in {timeout}".format(
disrupt=disrupt,
host=ip_or_hostname,
timeout=down_time_max))
break
time.sleep(1)
time.sleep(cool_down_time)
def disrupt_once(disrupt, ip_or_hostname, username, password, start_cmd,
stop_cmd, up_check_cmd, down_check_cmd, down_time_min,
down_time_max, cool_down_min, cool_down_max):
LOGGER.debug("Disrupt once {0}".format(ip_or_hostname))
down_time = random.randint(down_time_min, down_time_max)
cool_down_time = random.randint(cool_down_min, cool_down_max)
ssh = SSH(ip_or_hostname, username, password)
ssh.exec_command(stop_cmd)
ssh.exec_command(down_check_cmd) # TODO wait for down
if start_cmd: # TODO if service is down
time.sleep(down_time)
ssh.exec_command(start_cmd)
else:
disruption_finished = False
elapsed = 0
while not disruption_finished:
elapsed += cool_down_min
result, error = ssh.exec_command(up_check_cmd)
LOGGER.debug("Up check result is '{0}'. Error is {1}".format(
result.strip(), error.strip()))
if result is not None:
result = int(result.strip())
disruption_finished = (result == 0)
# TODO separate param for down_time_max
if not disruption_finished and elapsed > down_time_max:
LOGGER.exception(
"{disrupt} on {host} is not up in {timeout}".format(
disrupt=disrupt, host=ip_or_hostname,
timeout=down_time_max))
break
time.sleep(1)
time.sleep(cool_down_time)
def _ping_check(self):
for host_name in self.inventory.keys():
host = self.inventory[host_name]
results = ansible_runner.Runner(module_name="ping",
host_list=[host["ip_or_hostname"]],
remote_user=host["username"],
remote_pass=host["password"],
forks=2).run()
LOGGER.debug(results)
def _validate(ent_type, ent_list):
LOGGER.debug("Validating {0} config".format(ent_type))
for entity in ent_list:
if type(entity) == str:
continue
for class_name, config in entity.items():
actor_class = ACTOR_CLASSES.get(class_name, None)
if not actor_class:
raise Invalid("Unknown {0} {1}".format(ent_type, class_name))
actor_class.schema(config)
def validate_inventory(inventory):
inventory_schema = Schema({
Required("ip_or_hostname"): str,
Required("username"): str,
Required("password"): str,
Required("role"): str,
})
LOGGER.debug("Validating inventory config")
for host, config in inventory.items():
inventory_schema(config)
def _rabbitmq_check(self):
module_args = "sudo rabbitmqctl status"
for host in self._get_hosts_by_role("controller"):
results = ansible_runner.Runner(host_list=[host["ip_or_hostname"]],
remote_user=host["username"],
remote_pass=host["password"],
module_args=module_args,
module_name="shell",
forks=2).run()
LOGGER.debug(results)
def _mariadb_check(self):
module_args = "mysql -u{} -p{} -e \"show databases;\"" \
"| grep nova".format(self.db_user, self.db_password)
for host_name in self.inventory.keys():
host_data = self.inventory[host_name]
results = ansible_runner.Runner(
host_list=[host_data["ip_or_hostname"]],
remote_user=host_data["username"],
remote_pass=host_data["password"],
module_args=module_args, module_name="shell", forks=2).run()
LOGGER.debug(results)
def _process_check(self):
for role_name, services in self.services_to_monitor.iteritems():
for host in self._get_hosts_by_role(role_name):
for service in services:
module_args = self.monitor_command.format(
service_name=service)
results = ansible_runner.Runner(
host_list=[host["ip_or_hostname"]],
remote_user=host["username"],
remote_pass=host["password"], module_args=module_args,
module_name="shell", forks=2).run()
LOGGER.debug(results)
def validate_openrc(openrc):
openrc_schema = Schema({
Required("auth_url"): str,
Required("username"): str,
Required("password"): str,
Required("region_name"): str,
Required("tenant_name"): str,
Required("https_cacert"): str,
Optional("https_insecure"): bool,
})
LOGGER.debug("Validating openrc config")
openrc_schema(openrc)
def _mariadb_check(self):
module_args = "mysql -u{} -p{} -e \"show databases;\"" \
"| grep nova".format(self.db_user, self.db_password)
for host_name in self.inventory.keys():
host_data = self.inventory[host_name]
results = ansible_runner.Runner(
host_list=[host_data["ip_or_hostname"]],
remote_user=host_data["username"],
remote_pass=host_data["password"],
module_args=module_args,
module_name="shell",
forks=2).run()
LOGGER.debug(results)
def load(self):
count = 0
while True:
if self.task_status == TaskStatus.ABORTED:
self.times = count
break
count += 1
if count >= self.times:
self.task_status = TaskStatus.FINISHED
break
LOGGER.debug("Iteration {count} of {times}".format(
count=count, times=self.times
))
time.sleep(DummyLoader.COOL_DOWN)
def execute(self, params=None):
if params is None:
params = {}
for k, v in params.items():
for name in self.scenario_config.keys():
self.scenario_config[name][0]['runner'].update({k: v})
time.sleep(self.start_delay)
self.rally_task = rally_api.Task.create(self.deployment_name,
"cloud99")
self.runner_thread = threading.Thread(name=__name__, target=self.load)
self.checker_thread = threading.Thread(name=__name__,
target=self.check)
LOGGER.debug("Starting task {task_id}".format(
task_id=self.rally_task.task["uuid"]))
self.runner_thread.start()
self.checker_thread.start()
def round_robin_disruption(disrupt, nodes, start_cmd, stop_cmd, up_check_cmd,
down_check_cmd, down_time_min, down_time_max,
cool_down_min, cool_down_max, total, disrupt_func):
iteration = 1
while iteration <= total:
for host_name, host_info in nodes.items():
LOGGER.debug(
"HOST: {host} Iteration {iteration} of {total}".format(
host=host_info["ip_or_hostname"],
iteration=iteration,
total=total))
disrupt_func(disrupt, host_info["ip_or_hostname"],
host_info["username"], host_info["password"],
start_cmd, stop_cmd, up_check_cmd, down_check_cmd,
down_time_min, down_time_max, cool_down_min,
cool_down_max)
iteration += 1
def execute(self, params=None):
if params is None:
params = {}
for k, v in params.items():
for name in self.scenario_config.keys():
self.scenario_config[name][0]['runner'].update({k: v})
time.sleep(self.start_delay)
self.rally_task = rally_api.Task.create(self.deployment_name,
"cloud99")
self.runner_thread = threading.Thread(name=__name__,
target=self.load)
self.checker_thread = threading.Thread(name=__name__,
target=self.check)
LOGGER.debug("Starting task {task_id}".format(
task_id=self.rally_task.task["uuid"]))
self.runner_thread.start()
self.checker_thread.start()
def keystone_check(self):
status, message, service_list = self.keystone.keystone_service_list()
LOGGER.debug("{} reply '{}' Message '{}' Service list '{}'".format(
"Keystone", status, message, service_list))
def neutron_check(self):
status, message, service_list = self.neutron.neutron_agent_list()
LOGGER.debug("{} reply '{}' Message '{}' Agents list '{}'".format(
"Neutron", status, message, service_list))
def glance_check(self):
status, message, image_list = self.glance.glance_image_list()
LOGGER.debug("{} reply '{}' Message '{}' Image list '{}'".format(
"Glance", status, message, image_list))
def cinder_check(self):
status, message, cinder_list = self.cinder.cinder_list()
LOGGER.debug("{} reply '{}' Message '{}' Cinder list '{}'"
.format("Cinder", status, message, cinder_list))
def glance_check(self):
status, message, image_list = self.glance.glance_image_list()
LOGGER.debug("{} reply '{}' Message '{}' Image list '{}'"
.format("Glance", status, message, image_list))
def nova_check(self):
status, message, service_list = self.nova.nova_service_list()
LOGGER.debug("{} reply '{}' Message '{}' Service list '{}'"
.format("Nova", status, message, service_list))
def nova_check(self):
status, message, service_list = self.nova.nova_service_list()
LOGGER.debug("{} reply '{}' Message '{}' Service list '{}'".format(
"Nova", status, message, service_list))
def keystone_check(self):
status, message, service_list = self.keystone.keystone_service_list()
LOGGER.debug("{} reply '{}' Message '{}' Service list '{}'"
.format("Keystone", status, message, service_list))
def cinder_check(self):
status, message, cinder_list = self.cinder.cinder_list()
LOGGER.debug("{} reply '{}' Message '{}' Cinder list '{}'".format(
"Cinder", status, message, cinder_list))
def neutron_check(self):
status, message, service_list = self.neutron.neutron_agent_list()
LOGGER.debug("{} reply '{}' Message '{}' Agents list '{}'"
.format("Neutron", status, message, service_list))