def cli(ctx, debug, rally_debug):
if debug:
LOGGER.setLevel(logging.DEBUG)
if rally_debug:
for name in logging.Logger.manager.loggerDict.viewkeys():
if name.startswith("rally"):
logging.getLogger(name).setLevel(logging.DEBUG)
logging.getLogger(name).addHandler(LOG_FILE_HANDLER)
def cli(ctx, debug, rally_debug):
if debug:
LOGGER.setLevel(logging.DEBUG)
if rally_debug:
for name in logging.Logger.manager.loggerDict.viewkeys():
if name.startswith("rally"):
logging.getLogger(name).setLevel(logging.DEBUG)
logging.getLogger(name).addHandler(LOG_FILE_HANDLER)
def _ping_check(self):
for host_name in self.inventory.keys():
host = self.inventory[host_name]
results = ansible_runner.Runner(module_name="ping",
host_list=[host["ip_or_hostname"]],
remote_user=host["username"],
remote_pass=host["password"],
forks=2).run()
LOGGER.debug(results)
def validate_inventory(inventory):
inventory_schema = Schema({
Required("ip_or_hostname"): str,
Required("username"): str,
Required("password"): str,
Required("role"): str,
})
LOGGER.debug("Validating inventory config")
for host, config in inventory.items():
inventory_schema(config)
def _validate(ent_type, ent_list):
LOGGER.debug("Validating {0} config".format(ent_type))
for entity in ent_list:
if type(entity) == str:
continue
for class_name, config in entity.items():
actor_class = ACTOR_CLASSES.get(class_name, None)
if not actor_class:
raise Invalid("Unknown {0} {1}".format(ent_type, class_name))
actor_class.schema(config)
def validate_config(self):
try:
rally_api.Task.validate(self.deployment_name,
self.scenario_config,
task_instance=self.rally_task)
except Exception as e:
print(e)
LOGGER.exception(e)
self.observer.tell({'msg': 'validation_complete', 'valid': False})
self.observer.tell({'msg': 'validation_complete', 'valid': True})
def validate_config(self):
try:
rally_api.Task.validate(self.deployment_name,
self.scenario_config,
task_instance=self.rally_task)
except Exception as e:
print(e)
LOGGER.exception(e)
self.observer.tell({'msg': 'validation_complete', 'valid': False})
self.observer.tell({'msg': 'validation_complete', 'valid': True})
def _rabbitmq_check(self):
module_args = "sudo rabbitmqctl status"
for host in self._get_hosts_by_role("controller"):
results = ansible_runner.Runner(host_list=[host["ip_or_hostname"]],
remote_user=host["username"],
remote_pass=host["password"],
module_args=module_args,
module_name="shell",
forks=2).run()
LOGGER.debug(results)
def _mariadb_check(self):
module_args = "mysql -u{} -p{} -e \"show databases;\"" \
"| grep nova".format(self.db_user, self.db_password)
for host_name in self.inventory.keys():
host_data = self.inventory[host_name]
results = ansible_runner.Runner(
host_list=[host_data["ip_or_hostname"]],
remote_user=host_data["username"],
remote_pass=host_data["password"],
module_args=module_args, module_name="shell", forks=2).run()
LOGGER.debug(results)
def validate_openrc(openrc):
openrc_schema = Schema({
Required("auth_url"): str,
Required("username"): str,
Required("password"): str,
Required("region_name"): str,
Required("tenant_name"): str,
Required("https_cacert"): str,
Optional("https_insecure"): bool,
})
LOGGER.debug("Validating openrc config")
openrc_schema(openrc)
def _process_check(self):
for role_name, services in self.services_to_monitor.iteritems():
for host in self._get_hosts_by_role(role_name):
for service in services:
module_args = self.monitor_command.format(
service_name=service)
results = ansible_runner.Runner(
host_list=[host["ip_or_hostname"]],
remote_user=host["username"],
remote_pass=host["password"], module_args=module_args,
module_name="shell", forks=2).run()
LOGGER.debug(results)
def exec_command(self, cmd, get_pty=False):
result = None
error = None
try:
self.client.connect(hostname=self.host,
username=self.user,
password=self.password)
_, stdout, stderr = self.client.exec_command(cmd, get_pty)
except paramiko.AuthenticationException, ex:
LOGGER.exception("Authentication failed while connecting {}@{}."
" Exception {}".format(self.user, self.host, ex))
result = ''
error = ex
def _mariadb_check(self):
module_args = "mysql -u{} -p{} -e \"show databases;\"" \
"| grep nova".format(self.db_user, self.db_password)
for host_name in self.inventory.keys():
host_data = self.inventory[host_name]
results = ansible_runner.Runner(
host_list=[host_data["ip_or_hostname"]],
remote_user=host_data["username"],
remote_pass=host_data["password"],
module_args=module_args,
module_name="shell",
forks=2).run()
LOGGER.debug(results)
def load(self):
count = 0
while True:
if self.task_status == TaskStatus.ABORTED:
self.times = count
break
count += 1
if count >= self.times:
self.task_status = TaskStatus.FINISHED
break
LOGGER.debug("Iteration {count} of {times}".format(
count=count, times=self.times
))
time.sleep(DummyLoader.COOL_DOWN)
def abort(self):
try:
rally_api.Task.abort(self.rally_task.task["uuid"])
except RallyException as e:
LOGGER.exception(e)
finally:
self.runner_thread.join()
self.checker_thread.join()
res = rally_api.Task.get_detailed(self.rally_task.task["uuid"])
self.times = len(res["results"][0]["data"]["raw"])
# This will print standard rally report
task_cli.TaskCommands().detailed(task_id=self.rally_task.task['uuid'])
self.observer.tell({'msg': 'loader_finished', "times": self.times})
def abort(self):
try:
rally_api.Task.abort(self.rally_task.task["uuid"])
except RallyException as e:
LOGGER.exception(e)
finally:
self.runner_thread.join()
self.checker_thread.join()
res = rally_api.Task.get_detailed(self.rally_task.task["uuid"])
self.times = len(res["results"][0]["data"]["raw"])
# This will print standard rally report
task_cli.TaskCommands().detailed(
task_id=self.rally_task.task['uuid'])
self.observer.tell({'msg': 'loader_finished', "times": self.times})
def execute(self, params=None):
if params is None:
params = {}
for k, v in params.items():
for name in self.scenario_config.keys():
self.scenario_config[name][0]['runner'].update({k: v})
time.sleep(self.start_delay)
self.rally_task = rally_api.Task.create(self.deployment_name,
"cloud99")
self.runner_thread = threading.Thread(name=__name__, target=self.load)
self.checker_thread = threading.Thread(name=__name__,
target=self.check)
LOGGER.debug("Starting task {task_id}".format(
task_id=self.rally_task.task["uuid"]))
self.runner_thread.start()
self.checker_thread.start()
def round_robin_disruption(disrupt, nodes, start_cmd, stop_cmd, up_check_cmd,
down_check_cmd, down_time_min, down_time_max,
cool_down_min, cool_down_max, total, disrupt_func):
iteration = 1
while iteration <= total:
for host_name, host_info in nodes.items():
LOGGER.debug(
"HOST: {host} Iteration {iteration} of {total}".format(
host=host_info["ip_or_hostname"],
iteration=iteration,
total=total))
disrupt_func(disrupt, host_info["ip_or_hostname"],
host_info["username"], host_info["password"],
start_cmd, stop_cmd, up_check_cmd, down_check_cmd,
down_time_min, down_time_max, cool_down_min,
cool_down_max)
iteration += 1
def execute(self, params=None):
if params is None:
params = {}
for k, v in params.items():
for name in self.scenario_config.keys():
self.scenario_config[name][0]['runner'].update({k: v})
time.sleep(self.start_delay)
self.rally_task = rally_api.Task.create(self.deployment_name,
"cloud99")
self.runner_thread = threading.Thread(name=__name__,
target=self.load)
self.checker_thread = threading.Thread(name=__name__,
target=self.check)
LOGGER.debug("Starting task {task_id}".format(
task_id=self.rally_task.task["uuid"]))
self.runner_thread.start()
self.checker_thread.start()
def disrupt_once(disrupt, ip_or_hostname, username, password, start_cmd,
stop_cmd, up_check_cmd, down_check_cmd, down_time_min,
down_time_max, cool_down_min, cool_down_max):
# TODO (dratushnyy) named params in format
check_cmd = up_check_cmd.format(username=username, host=ip_or_hostname)
check_cmd = check_cmd.split(" ")
ssh = SSH(ip_or_hostname, username, password)
ssh.exec_command(stop_cmd)
disruption_finished = False
elapsed = 0
while not disruption_finished:
time.sleep(HostDisruptor.COOL_DOWN)
elapsed += HostDisruptor.COOL_DOWN
disruption_finished = (subprocess.call(check_cmd) == 0)
if not disruption_finished and elapsed > down_time_max:
LOGGER.exception("Host {host} not up in {timeout}".format(
host=ip_or_hostname, timeout=down_time_max))
break
time.sleep(cool_down_max)
def disrupt_once(disrupt, ip_or_hostname, username, password, start_cmd,
stop_cmd, up_check_cmd, down_check_cmd, down_time_min,
down_time_max, cool_down_min, cool_down_max):
LOGGER.debug("Disrupt once {0}".format(ip_or_hostname))
down_time = random.randint(down_time_min, down_time_max)
cool_down_time = random.randint(cool_down_min, cool_down_max)
ssh = SSH(ip_or_hostname, username, password)
ssh.exec_command(stop_cmd)
ssh.exec_command(down_check_cmd) # TODO wait for down
if start_cmd: # TODO if service is down
time.sleep(down_time)
ssh.exec_command(start_cmd)
else:
disruption_finished = False
elapsed = 0
while not disruption_finished:
elapsed += cool_down_min
result, error = ssh.exec_command(up_check_cmd)
LOGGER.debug("Up check result is '{0}'. Error is {1}".format(
result.strip(), error.strip()))
if result is not None:
result = int(result.strip())
disruption_finished = (result == 0)
# TODO separate param for down_time_max
if not disruption_finished and elapsed > down_time_max:
LOGGER.exception(
"{disrupt} on {host} is not up in {timeout}".format(
disrupt=disrupt, host=ip_or_hostname,
timeout=down_time_max))
break
time.sleep(1)
time.sleep(cool_down_time)
def disrupt_once(disrupt, ip_or_hostname, username, password, start_cmd,
stop_cmd, up_check_cmd, down_check_cmd, down_time_min,
down_time_max, cool_down_min, cool_down_max):
LOGGER.debug("Disrupt once {0}".format(ip_or_hostname))
down_time = random.randint(down_time_min, down_time_max)
cool_down_time = random.randint(cool_down_min, cool_down_max)
ssh = SSH(ip_or_hostname, username, password)
ssh.exec_command(stop_cmd)
ssh.exec_command(down_check_cmd) # TODO wait for down
if start_cmd: # TODO if service is down
time.sleep(down_time)
ssh.exec_command(start_cmd)
else:
disruption_finished = False
elapsed = 0
while not disruption_finished:
elapsed += cool_down_min
result, error = ssh.exec_command(up_check_cmd)
LOGGER.debug("Up check result is '{0}'. Error is {1}".format(
result.strip(), error.strip()))
if result is not None:
result = int(result.strip())
disruption_finished = (result == 0)
# TODO separate param for down_time_max
if not disruption_finished and elapsed > down_time_max:
LOGGER.exception(
"{disrupt} on {host} is not up in {timeout}".format(
disrupt=disrupt,
host=ip_or_hostname,
timeout=down_time_max))
break
time.sleep(1)
time.sleep(cool_down_time)
def cinder_check(self):
status, message, cinder_list = self.cinder.cinder_list()
LOGGER.debug("{} reply '{}' Message '{}' Cinder list '{}'"
.format("Cinder", status, message, cinder_list))
def glance_check(self):
status, message, image_list = self.glance.glance_image_list()
LOGGER.debug("{} reply '{}' Message '{}' Image list '{}'"
.format("Glance", status, message, image_list))
def nova_check(self):
status, message, service_list = self.nova.nova_service_list()
LOGGER.debug("{} reply '{}' Message '{}' Service list '{}'"
.format("Nova", status, message, service_list))
def neutron_check(self):
status, message, service_list = self.neutron.neutron_agent_list()
LOGGER.debug("{} reply '{}' Message '{}' Agents list '{}'".format(
"Neutron", status, message, service_list))
def __init__(self, observer, openrc, inventory, **params):
super(RallyLoader, self).__init__(observer, openrc, inventory,
**params)
self.scenario_file = os.path.abspath(
os.path.join(RallyLoader.scenarios_path, params['scenario_file']))
# TODO (dratushnyy) fallback to default path only if file not found
self.scenario_args_file = params.get('scenario_args_file', None)
if self.scenario_args_file:
self.scenario_args_file = os.path.abspath(
os.path.join(RallyLoader.scenarios_path,
self.scenario_args_file))
self.start_delay = params['start_delay']
self.deployment_name = params['deployment_name']
self.deployment_config = {
"type": "ExistingCloud",
"admin": {
"username": openrc["username"],
"password": openrc["password"],
"tenant_name": openrc["tenant_name"]
},
"auth_url": openrc["auth_url"],
"region_name": openrc["region_name"],
"https_insecure": openrc['https_insecure'],
"https_cacert": openrc["https_cacert"]
}
self.scenario_args = params.get('scenario_args', None)
# Need to be set to None to avoid exception in stop() method
self.rally_task = None
load_rally_plugins()
if params.get('db'):
db_connection = RallyLoader.conn_template.format(
user=params["db"]["user"],
passwd=params["db"]["pass"],
host=params["db"]["host"],
db_name=params["db"]["name"])
db_options.set_defaults(CONF, connection=db_connection)
try:
rally_api.Deployment.get(self.deployment_name)
except DBNonExistentTable as e:
db.schema_create()
except DeploymentNotFound as e:
try:
rally_api.Deployment.create(config=self.deployment_config,
name=self.deployment_name)
except ValidationError as e:
LOGGER.exception(e)
raise e
except OperationalError as e:
LOGGER.exception(e)
raise e
# Since there is no api method to do this - using cli
deployment_cli.DeploymentCommands().use(self.deployment_name)
# Using rally task cli to load and validate task
# TODO check is API support this?
try:
self.scenario_config = task_cli.TaskCommands().\
_load_and_validate_task(self.scenario_file,
json.dumps(self.scenario_args),
self.scenario_args_file,
self.deployment_name)
except Exception as e:
LOGGER.exception(e)
raise e
def glance_check(self):
status, message, image_list = self.glance.glance_image_list()
LOGGER.debug("{} reply '{}' Message '{}' Image list '{}'".format(
"Glance", status, message, image_list))
def __init__(self, observer, openrc, inventory, **params):
super(RallyLoader, self).__init__(observer, openrc, inventory,
**params)
self.scenario_file = os.path.abspath(os.path.join(
RallyLoader.scenarios_path, params['scenario_file']))
# TODO (dratushnyy) fallback to default path only if file not found
self.scenario_args_file = params.get('scenario_args_file', None)
if self.scenario_args_file:
self.scenario_args_file = os.path.abspath(os.path.join(
RallyLoader.scenarios_path, self.scenario_args_file))
self.start_delay = params['start_delay']
self.deployment_name = params['deployment_name']
self.deployment_config = {
"type": "ExistingCloud",
"admin": {
"username": openrc["username"],
"password": openrc["password"],
"tenant_name": openrc["tenant_name"]
},
"auth_url": openrc["auth_url"],
"region_name": openrc["region_name"],
"https_insecure": openrc['https_insecure'],
"https_cacert": openrc["https_cacert"]
}
self.scenario_args = params.get('scenario_args', None)
# Need to be set to None to avoid exception in stop() method
self.rally_task = None
load_rally_plugins()
if params.get('db'):
db_connection = RallyLoader.conn_template.format(
user=params["db"]["user"],
passwd=params["db"]["pass"],
host=params["db"]["host"],
db_name=params["db"]["name"])
db_options.set_defaults(CONF, connection=db_connection)
try:
rally_api.Deployment.get(self.deployment_name)
except DBNonExistentTable as e:
db.schema_create()
except DeploymentNotFound as e:
try:
rally_api.Deployment.create(config=self.deployment_config,
name=self.deployment_name)
except ValidationError as e:
LOGGER.exception(e)
raise e
except OperationalError as e:
LOGGER.exception(e)
raise e
# Since there is no api method to do this - using cli
deployment_cli.DeploymentCommands().use(self.deployment_name)
# Using rally task cli to load and validate task
# TODO check is API support this?
try:
self.scenario_config = task_cli.TaskCommands().\
_load_and_validate_task(self.scenario_file,
json.dumps(self.scenario_args),
self.scenario_args_file,
self.deployment_name)
except Exception as e:
LOGGER.exception(e)
raise e
def keystone_check(self):
status, message, service_list = self.keystone.keystone_service_list()
LOGGER.debug("{} reply '{}' Message '{}' Service list '{}'".format(
"Keystone", status, message, service_list))
def keystone_check(self):
status, message, service_list = self.keystone.keystone_service_list()
LOGGER.debug("{} reply '{}' Message '{}' Service list '{}'"
.format("Keystone", status, message, service_list))
def nova_check(self):
status, message, service_list = self.nova.nova_service_list()
LOGGER.debug("{} reply '{}' Message '{}' Service list '{}'".format(
"Nova", status, message, service_list))
def neutron_check(self):
status, message, service_list = self.neutron.neutron_agent_list()
LOGGER.debug("{} reply '{}' Message '{}' Agents list '{}'"
.format("Neutron", status, message, service_list))
def cinder_check(self):
status, message, cinder_list = self.cinder.cinder_list()
LOGGER.debug("{} reply '{}' Message '{}' Cinder list '{}'".format(
"Cinder", status, message, cinder_list))
def on_failure(self, exception_type, exception_value, traceback):
LOGGER.error(exception_type, exception_value, traceback)