def cli(ctx, debug, rally_debug): if debug: LOGGER.setLevel(logging.DEBUG) if rally_debug: for name in logging.Logger.manager.loggerDict.viewkeys(): if name.startswith("rally"): logging.getLogger(name).setLevel(logging.DEBUG) logging.getLogger(name).addHandler(LOG_FILE_HANDLER)
def cli(ctx, debug, rally_debug): if debug: LOGGER.setLevel(logging.DEBUG) if rally_debug: for name in logging.Logger.manager.loggerDict.viewkeys(): if name.startswith("rally"): logging.getLogger(name).setLevel(logging.DEBUG) logging.getLogger(name).addHandler(LOG_FILE_HANDLER)
def _ping_check(self): for host_name in self.inventory.keys(): host = self.inventory[host_name] results = ansible_runner.Runner(module_name="ping", host_list=[host["ip_or_hostname"]], remote_user=host["username"], remote_pass=host["password"], forks=2).run() LOGGER.debug(results)
def validate_inventory(inventory): inventory_schema = Schema({ Required("ip_or_hostname"): str, Required("username"): str, Required("password"): str, Required("role"): str, }) LOGGER.debug("Validating inventory config") for host, config in inventory.items(): inventory_schema(config)
def _validate(ent_type, ent_list): LOGGER.debug("Validating {0} config".format(ent_type)) for entity in ent_list: if type(entity) == str: continue for class_name, config in entity.items(): actor_class = ACTOR_CLASSES.get(class_name, None) if not actor_class: raise Invalid("Unknown {0} {1}".format(ent_type, class_name)) actor_class.schema(config)
def validate_config(self): try: rally_api.Task.validate(self.deployment_name, self.scenario_config, task_instance=self.rally_task) except Exception as e: print(e) LOGGER.exception(e) self.observer.tell({'msg': 'validation_complete', 'valid': False}) self.observer.tell({'msg': 'validation_complete', 'valid': True})
def validate_config(self): try: rally_api.Task.validate(self.deployment_name, self.scenario_config, task_instance=self.rally_task) except Exception as e: print(e) LOGGER.exception(e) self.observer.tell({'msg': 'validation_complete', 'valid': False}) self.observer.tell({'msg': 'validation_complete', 'valid': True})
def _rabbitmq_check(self): module_args = "sudo rabbitmqctl status" for host in self._get_hosts_by_role("controller"): results = ansible_runner.Runner(host_list=[host["ip_or_hostname"]], remote_user=host["username"], remote_pass=host["password"], module_args=module_args, module_name="shell", forks=2).run() LOGGER.debug(results)
def _mariadb_check(self): module_args = "mysql -u{} -p{} -e \"show databases;\"" \ "| grep nova".format(self.db_user, self.db_password) for host_name in self.inventory.keys(): host_data = self.inventory[host_name] results = ansible_runner.Runner( host_list=[host_data["ip_or_hostname"]], remote_user=host_data["username"], remote_pass=host_data["password"], module_args=module_args, module_name="shell", forks=2).run() LOGGER.debug(results)
def validate_openrc(openrc): openrc_schema = Schema({ Required("auth_url"): str, Required("username"): str, Required("password"): str, Required("region_name"): str, Required("tenant_name"): str, Required("https_cacert"): str, Optional("https_insecure"): bool, }) LOGGER.debug("Validating openrc config") openrc_schema(openrc)
def _process_check(self): for role_name, services in self.services_to_monitor.iteritems(): for host in self._get_hosts_by_role(role_name): for service in services: module_args = self.monitor_command.format( service_name=service) results = ansible_runner.Runner( host_list=[host["ip_or_hostname"]], remote_user=host["username"], remote_pass=host["password"], module_args=module_args, module_name="shell", forks=2).run() LOGGER.debug(results)
def exec_command(self, cmd, get_pty=False): result = None error = None try: self.client.connect(hostname=self.host, username=self.user, password=self.password) _, stdout, stderr = self.client.exec_command(cmd, get_pty) except paramiko.AuthenticationException, ex: LOGGER.exception("Authentication failed while connecting {}@{}." " Exception {}".format(self.user, self.host, ex)) result = '' error = ex
def _mariadb_check(self): module_args = "mysql -u{} -p{} -e \"show databases;\"" \ "| grep nova".format(self.db_user, self.db_password) for host_name in self.inventory.keys(): host_data = self.inventory[host_name] results = ansible_runner.Runner( host_list=[host_data["ip_or_hostname"]], remote_user=host_data["username"], remote_pass=host_data["password"], module_args=module_args, module_name="shell", forks=2).run() LOGGER.debug(results)
def load(self): count = 0 while True: if self.task_status == TaskStatus.ABORTED: self.times = count break count += 1 if count >= self.times: self.task_status = TaskStatus.FINISHED break LOGGER.debug("Iteration {count} of {times}".format( count=count, times=self.times )) time.sleep(DummyLoader.COOL_DOWN)
def abort(self): try: rally_api.Task.abort(self.rally_task.task["uuid"]) except RallyException as e: LOGGER.exception(e) finally: self.runner_thread.join() self.checker_thread.join() res = rally_api.Task.get_detailed(self.rally_task.task["uuid"]) self.times = len(res["results"][0]["data"]["raw"]) # This will print standard rally report task_cli.TaskCommands().detailed(task_id=self.rally_task.task['uuid']) self.observer.tell({'msg': 'loader_finished', "times": self.times})
def abort(self): try: rally_api.Task.abort(self.rally_task.task["uuid"]) except RallyException as e: LOGGER.exception(e) finally: self.runner_thread.join() self.checker_thread.join() res = rally_api.Task.get_detailed(self.rally_task.task["uuid"]) self.times = len(res["results"][0]["data"]["raw"]) # This will print standard rally report task_cli.TaskCommands().detailed( task_id=self.rally_task.task['uuid']) self.observer.tell({'msg': 'loader_finished', "times": self.times})
def execute(self, params=None): if params is None: params = {} for k, v in params.items(): for name in self.scenario_config.keys(): self.scenario_config[name][0]['runner'].update({k: v}) time.sleep(self.start_delay) self.rally_task = rally_api.Task.create(self.deployment_name, "cloud99") self.runner_thread = threading.Thread(name=__name__, target=self.load) self.checker_thread = threading.Thread(name=__name__, target=self.check) LOGGER.debug("Starting task {task_id}".format( task_id=self.rally_task.task["uuid"])) self.runner_thread.start() self.checker_thread.start()
def round_robin_disruption(disrupt, nodes, start_cmd, stop_cmd, up_check_cmd, down_check_cmd, down_time_min, down_time_max, cool_down_min, cool_down_max, total, disrupt_func): iteration = 1 while iteration <= total: for host_name, host_info in nodes.items(): LOGGER.debug( "HOST: {host} Iteration {iteration} of {total}".format( host=host_info["ip_or_hostname"], iteration=iteration, total=total)) disrupt_func(disrupt, host_info["ip_or_hostname"], host_info["username"], host_info["password"], start_cmd, stop_cmd, up_check_cmd, down_check_cmd, down_time_min, down_time_max, cool_down_min, cool_down_max) iteration += 1
def execute(self, params=None): if params is None: params = {} for k, v in params.items(): for name in self.scenario_config.keys(): self.scenario_config[name][0]['runner'].update({k: v}) time.sleep(self.start_delay) self.rally_task = rally_api.Task.create(self.deployment_name, "cloud99") self.runner_thread = threading.Thread(name=__name__, target=self.load) self.checker_thread = threading.Thread(name=__name__, target=self.check) LOGGER.debug("Starting task {task_id}".format( task_id=self.rally_task.task["uuid"])) self.runner_thread.start() self.checker_thread.start()
def disrupt_once(disrupt, ip_or_hostname, username, password, start_cmd, stop_cmd, up_check_cmd, down_check_cmd, down_time_min, down_time_max, cool_down_min, cool_down_max): # TODO (dratushnyy) named params in format check_cmd = up_check_cmd.format(username=username, host=ip_or_hostname) check_cmd = check_cmd.split(" ") ssh = SSH(ip_or_hostname, username, password) ssh.exec_command(stop_cmd) disruption_finished = False elapsed = 0 while not disruption_finished: time.sleep(HostDisruptor.COOL_DOWN) elapsed += HostDisruptor.COOL_DOWN disruption_finished = (subprocess.call(check_cmd) == 0) if not disruption_finished and elapsed > down_time_max: LOGGER.exception("Host {host} not up in {timeout}".format( host=ip_or_hostname, timeout=down_time_max)) break time.sleep(cool_down_max)
def disrupt_once(disrupt, ip_or_hostname, username, password, start_cmd, stop_cmd, up_check_cmd, down_check_cmd, down_time_min, down_time_max, cool_down_min, cool_down_max): LOGGER.debug("Disrupt once {0}".format(ip_or_hostname)) down_time = random.randint(down_time_min, down_time_max) cool_down_time = random.randint(cool_down_min, cool_down_max) ssh = SSH(ip_or_hostname, username, password) ssh.exec_command(stop_cmd) ssh.exec_command(down_check_cmd) # TODO wait for down if start_cmd: # TODO if service is down time.sleep(down_time) ssh.exec_command(start_cmd) else: disruption_finished = False elapsed = 0 while not disruption_finished: elapsed += cool_down_min result, error = ssh.exec_command(up_check_cmd) LOGGER.debug("Up check result is '{0}'. Error is {1}".format( result.strip(), error.strip())) if result is not None: result = int(result.strip()) disruption_finished = (result == 0) # TODO separate param for down_time_max if not disruption_finished and elapsed > down_time_max: LOGGER.exception( "{disrupt} on {host} is not up in {timeout}".format( disrupt=disrupt, host=ip_or_hostname, timeout=down_time_max)) break time.sleep(1) time.sleep(cool_down_time)
def disrupt_once(disrupt, ip_or_hostname, username, password, start_cmd, stop_cmd, up_check_cmd, down_check_cmd, down_time_min, down_time_max, cool_down_min, cool_down_max): LOGGER.debug("Disrupt once {0}".format(ip_or_hostname)) down_time = random.randint(down_time_min, down_time_max) cool_down_time = random.randint(cool_down_min, cool_down_max) ssh = SSH(ip_or_hostname, username, password) ssh.exec_command(stop_cmd) ssh.exec_command(down_check_cmd) # TODO wait for down if start_cmd: # TODO if service is down time.sleep(down_time) ssh.exec_command(start_cmd) else: disruption_finished = False elapsed = 0 while not disruption_finished: elapsed += cool_down_min result, error = ssh.exec_command(up_check_cmd) LOGGER.debug("Up check result is '{0}'. Error is {1}".format( result.strip(), error.strip())) if result is not None: result = int(result.strip()) disruption_finished = (result == 0) # TODO separate param for down_time_max if not disruption_finished and elapsed > down_time_max: LOGGER.exception( "{disrupt} on {host} is not up in {timeout}".format( disrupt=disrupt, host=ip_or_hostname, timeout=down_time_max)) break time.sleep(1) time.sleep(cool_down_time)
def cinder_check(self): status, message, cinder_list = self.cinder.cinder_list() LOGGER.debug("{} reply '{}' Message '{}' Cinder list '{}'" .format("Cinder", status, message, cinder_list))
def glance_check(self): status, message, image_list = self.glance.glance_image_list() LOGGER.debug("{} reply '{}' Message '{}' Image list '{}'" .format("Glance", status, message, image_list))
def nova_check(self): status, message, service_list = self.nova.nova_service_list() LOGGER.debug("{} reply '{}' Message '{}' Service list '{}'" .format("Nova", status, message, service_list))
def neutron_check(self): status, message, service_list = self.neutron.neutron_agent_list() LOGGER.debug("{} reply '{}' Message '{}' Agents list '{}'".format( "Neutron", status, message, service_list))
def __init__(self, observer, openrc, inventory, **params): super(RallyLoader, self).__init__(observer, openrc, inventory, **params) self.scenario_file = os.path.abspath( os.path.join(RallyLoader.scenarios_path, params['scenario_file'])) # TODO (dratushnyy) fallback to default path only if file not found self.scenario_args_file = params.get('scenario_args_file', None) if self.scenario_args_file: self.scenario_args_file = os.path.abspath( os.path.join(RallyLoader.scenarios_path, self.scenario_args_file)) self.start_delay = params['start_delay'] self.deployment_name = params['deployment_name'] self.deployment_config = { "type": "ExistingCloud", "admin": { "username": openrc["username"], "password": openrc["password"], "tenant_name": openrc["tenant_name"] }, "auth_url": openrc["auth_url"], "region_name": openrc["region_name"], "https_insecure": openrc['https_insecure'], "https_cacert": openrc["https_cacert"] } self.scenario_args = params.get('scenario_args', None) # Need to be set to None to avoid exception in stop() method self.rally_task = None load_rally_plugins() if params.get('db'): db_connection = RallyLoader.conn_template.format( user=params["db"]["user"], passwd=params["db"]["pass"], host=params["db"]["host"], db_name=params["db"]["name"]) db_options.set_defaults(CONF, connection=db_connection) try: rally_api.Deployment.get(self.deployment_name) except DBNonExistentTable as e: db.schema_create() except DeploymentNotFound as e: try: rally_api.Deployment.create(config=self.deployment_config, name=self.deployment_name) except ValidationError as e: LOGGER.exception(e) raise e except OperationalError as e: LOGGER.exception(e) raise e # Since there is no api method to do this - using cli deployment_cli.DeploymentCommands().use(self.deployment_name) # Using rally task cli to load and validate task # TODO check is API support this? try: self.scenario_config = task_cli.TaskCommands().\ _load_and_validate_task(self.scenario_file, json.dumps(self.scenario_args), self.scenario_args_file, self.deployment_name) except Exception as e: LOGGER.exception(e) raise e
def glance_check(self): status, message, image_list = self.glance.glance_image_list() LOGGER.debug("{} reply '{}' Message '{}' Image list '{}'".format( "Glance", status, message, image_list))
def __init__(self, observer, openrc, inventory, **params): super(RallyLoader, self).__init__(observer, openrc, inventory, **params) self.scenario_file = os.path.abspath(os.path.join( RallyLoader.scenarios_path, params['scenario_file'])) # TODO (dratushnyy) fallback to default path only if file not found self.scenario_args_file = params.get('scenario_args_file', None) if self.scenario_args_file: self.scenario_args_file = os.path.abspath(os.path.join( RallyLoader.scenarios_path, self.scenario_args_file)) self.start_delay = params['start_delay'] self.deployment_name = params['deployment_name'] self.deployment_config = { "type": "ExistingCloud", "admin": { "username": openrc["username"], "password": openrc["password"], "tenant_name": openrc["tenant_name"] }, "auth_url": openrc["auth_url"], "region_name": openrc["region_name"], "https_insecure": openrc['https_insecure'], "https_cacert": openrc["https_cacert"] } self.scenario_args = params.get('scenario_args', None) # Need to be set to None to avoid exception in stop() method self.rally_task = None load_rally_plugins() if params.get('db'): db_connection = RallyLoader.conn_template.format( user=params["db"]["user"], passwd=params["db"]["pass"], host=params["db"]["host"], db_name=params["db"]["name"]) db_options.set_defaults(CONF, connection=db_connection) try: rally_api.Deployment.get(self.deployment_name) except DBNonExistentTable as e: db.schema_create() except DeploymentNotFound as e: try: rally_api.Deployment.create(config=self.deployment_config, name=self.deployment_name) except ValidationError as e: LOGGER.exception(e) raise e except OperationalError as e: LOGGER.exception(e) raise e # Since there is no api method to do this - using cli deployment_cli.DeploymentCommands().use(self.deployment_name) # Using rally task cli to load and validate task # TODO check is API support this? try: self.scenario_config = task_cli.TaskCommands().\ _load_and_validate_task(self.scenario_file, json.dumps(self.scenario_args), self.scenario_args_file, self.deployment_name) except Exception as e: LOGGER.exception(e) raise e
def keystone_check(self): status, message, service_list = self.keystone.keystone_service_list() LOGGER.debug("{} reply '{}' Message '{}' Service list '{}'".format( "Keystone", status, message, service_list))
def keystone_check(self): status, message, service_list = self.keystone.keystone_service_list() LOGGER.debug("{} reply '{}' Message '{}' Service list '{}'" .format("Keystone", status, message, service_list))
def nova_check(self): status, message, service_list = self.nova.nova_service_list() LOGGER.debug("{} reply '{}' Message '{}' Service list '{}'".format( "Nova", status, message, service_list))
def neutron_check(self): status, message, service_list = self.neutron.neutron_agent_list() LOGGER.debug("{} reply '{}' Message '{}' Agents list '{}'" .format("Neutron", status, message, service_list))
def cinder_check(self): status, message, cinder_list = self.cinder.cinder_list() LOGGER.debug("{} reply '{}' Message '{}' Cinder list '{}'".format( "Cinder", status, message, cinder_list))
def on_failure(self, exception_type, exception_value, traceback): LOGGER.error(exception_type, exception_value, traceback)