def disrupt_once(disrupt, ip_or_hostname, username, password, start_cmd, stop_cmd, up_check_cmd, down_check_cmd, down_time_min, down_time_max, cool_down_min, cool_down_max): LOGGER.debug("Disrupt once {0}".format(ip_or_hostname)) down_time = random.randint(down_time_min, down_time_max) cool_down_time = random.randint(cool_down_min, cool_down_max) ssh = SSH(ip_or_hostname, username, password) ssh.exec_command(stop_cmd) ssh.exec_command(down_check_cmd) # TODO wait for down if start_cmd: # TODO if service is down time.sleep(down_time) ssh.exec_command(start_cmd) else: disruption_finished = False elapsed = 0 while not disruption_finished: elapsed += cool_down_min result, error = ssh.exec_command(up_check_cmd) LOGGER.debug("Up check result is '{0}'. Error is {1}".format( result.strip(), error.strip())) if result is not None: result = int(result.strip()) disruption_finished = (result == 0) # TODO separate param for down_time_max if not disruption_finished and elapsed > down_time_max: LOGGER.exception( "{disrupt} on {host} is not up in {timeout}".format( disrupt=disrupt, host=ip_or_hostname, timeout=down_time_max)) break time.sleep(1) time.sleep(cool_down_time)
def disrupt_once(disrupt, ip_or_hostname, username, password, start_cmd, stop_cmd, up_check_cmd, down_check_cmd, down_time_min, down_time_max, cool_down_min, cool_down_max): LOGGER.debug("Disrupt once {0}".format(ip_or_hostname)) down_time = random.randint(down_time_min, down_time_max) cool_down_time = random.randint(cool_down_min, cool_down_max) ssh = SSH(ip_or_hostname, username, password) ssh.exec_command(stop_cmd) ssh.exec_command(down_check_cmd) # TODO wait for down if start_cmd: # TODO if service is down time.sleep(down_time) ssh.exec_command(start_cmd) else: disruption_finished = False elapsed = 0 while not disruption_finished: elapsed += cool_down_min result, error = ssh.exec_command(up_check_cmd) LOGGER.debug("Up check result is '{0}'. Error is {1}".format( result.strip(), error.strip())) if result is not None: result = int(result.strip()) disruption_finished = (result == 0) # TODO separate param for down_time_max if not disruption_finished and elapsed > down_time_max: LOGGER.exception( "{disrupt} on {host} is not up in {timeout}".format( disrupt=disrupt, host=ip_or_hostname, timeout=down_time_max)) break time.sleep(1) time.sleep(cool_down_time)
def _ping_check(self): for host_name in self.inventory.keys(): host = self.inventory[host_name] results = ansible_runner.Runner(module_name="ping", host_list=[host["ip_or_hostname"]], remote_user=host["username"], remote_pass=host["password"], forks=2).run() LOGGER.debug(results)
def _validate(ent_type, ent_list): LOGGER.debug("Validating {0} config".format(ent_type)) for entity in ent_list: if type(entity) == str: continue for class_name, config in entity.items(): actor_class = ACTOR_CLASSES.get(class_name, None) if not actor_class: raise Invalid("Unknown {0} {1}".format(ent_type, class_name)) actor_class.schema(config)
def validate_inventory(inventory): inventory_schema = Schema({ Required("ip_or_hostname"): str, Required("username"): str, Required("password"): str, Required("role"): str, }) LOGGER.debug("Validating inventory config") for host, config in inventory.items(): inventory_schema(config)
def _rabbitmq_check(self): module_args = "sudo rabbitmqctl status" for host in self._get_hosts_by_role("controller"): results = ansible_runner.Runner(host_list=[host["ip_or_hostname"]], remote_user=host["username"], remote_pass=host["password"], module_args=module_args, module_name="shell", forks=2).run() LOGGER.debug(results)
def _mariadb_check(self): module_args = "mysql -u{} -p{} -e \"show databases;\"" \ "| grep nova".format(self.db_user, self.db_password) for host_name in self.inventory.keys(): host_data = self.inventory[host_name] results = ansible_runner.Runner( host_list=[host_data["ip_or_hostname"]], remote_user=host_data["username"], remote_pass=host_data["password"], module_args=module_args, module_name="shell", forks=2).run() LOGGER.debug(results)
def _process_check(self): for role_name, services in self.services_to_monitor.iteritems(): for host in self._get_hosts_by_role(role_name): for service in services: module_args = self.monitor_command.format( service_name=service) results = ansible_runner.Runner( host_list=[host["ip_or_hostname"]], remote_user=host["username"], remote_pass=host["password"], module_args=module_args, module_name="shell", forks=2).run() LOGGER.debug(results)
def validate_openrc(openrc): openrc_schema = Schema({ Required("auth_url"): str, Required("username"): str, Required("password"): str, Required("region_name"): str, Required("tenant_name"): str, Required("https_cacert"): str, Optional("https_insecure"): bool, }) LOGGER.debug("Validating openrc config") openrc_schema(openrc)
def _mariadb_check(self): module_args = "mysql -u{} -p{} -e \"show databases;\"" \ "| grep nova".format(self.db_user, self.db_password) for host_name in self.inventory.keys(): host_data = self.inventory[host_name] results = ansible_runner.Runner( host_list=[host_data["ip_or_hostname"]], remote_user=host_data["username"], remote_pass=host_data["password"], module_args=module_args, module_name="shell", forks=2).run() LOGGER.debug(results)
def load(self): count = 0 while True: if self.task_status == TaskStatus.ABORTED: self.times = count break count += 1 if count >= self.times: self.task_status = TaskStatus.FINISHED break LOGGER.debug("Iteration {count} of {times}".format( count=count, times=self.times )) time.sleep(DummyLoader.COOL_DOWN)
def execute(self, params=None): if params is None: params = {} for k, v in params.items(): for name in self.scenario_config.keys(): self.scenario_config[name][0]['runner'].update({k: v}) time.sleep(self.start_delay) self.rally_task = rally_api.Task.create(self.deployment_name, "cloud99") self.runner_thread = threading.Thread(name=__name__, target=self.load) self.checker_thread = threading.Thread(name=__name__, target=self.check) LOGGER.debug("Starting task {task_id}".format( task_id=self.rally_task.task["uuid"])) self.runner_thread.start() self.checker_thread.start()
def round_robin_disruption(disrupt, nodes, start_cmd, stop_cmd, up_check_cmd, down_check_cmd, down_time_min, down_time_max, cool_down_min, cool_down_max, total, disrupt_func): iteration = 1 while iteration <= total: for host_name, host_info in nodes.items(): LOGGER.debug( "HOST: {host} Iteration {iteration} of {total}".format( host=host_info["ip_or_hostname"], iteration=iteration, total=total)) disrupt_func(disrupt, host_info["ip_or_hostname"], host_info["username"], host_info["password"], start_cmd, stop_cmd, up_check_cmd, down_check_cmd, down_time_min, down_time_max, cool_down_min, cool_down_max) iteration += 1
def execute(self, params=None): if params is None: params = {} for k, v in params.items(): for name in self.scenario_config.keys(): self.scenario_config[name][0]['runner'].update({k: v}) time.sleep(self.start_delay) self.rally_task = rally_api.Task.create(self.deployment_name, "cloud99") self.runner_thread = threading.Thread(name=__name__, target=self.load) self.checker_thread = threading.Thread(name=__name__, target=self.check) LOGGER.debug("Starting task {task_id}".format( task_id=self.rally_task.task["uuid"])) self.runner_thread.start() self.checker_thread.start()
def keystone_check(self): status, message, service_list = self.keystone.keystone_service_list() LOGGER.debug("{} reply '{}' Message '{}' Service list '{}'".format( "Keystone", status, message, service_list))
def neutron_check(self): status, message, service_list = self.neutron.neutron_agent_list() LOGGER.debug("{} reply '{}' Message '{}' Agents list '{}'".format( "Neutron", status, message, service_list))
def glance_check(self): status, message, image_list = self.glance.glance_image_list() LOGGER.debug("{} reply '{}' Message '{}' Image list '{}'".format( "Glance", status, message, image_list))
def cinder_check(self): status, message, cinder_list = self.cinder.cinder_list() LOGGER.debug("{} reply '{}' Message '{}' Cinder list '{}'" .format("Cinder", status, message, cinder_list))
def glance_check(self): status, message, image_list = self.glance.glance_image_list() LOGGER.debug("{} reply '{}' Message '{}' Image list '{}'" .format("Glance", status, message, image_list))
def nova_check(self): status, message, service_list = self.nova.nova_service_list() LOGGER.debug("{} reply '{}' Message '{}' Service list '{}'" .format("Nova", status, message, service_list))
def nova_check(self): status, message, service_list = self.nova.nova_service_list() LOGGER.debug("{} reply '{}' Message '{}' Service list '{}'".format( "Nova", status, message, service_list))
def keystone_check(self): status, message, service_list = self.keystone.keystone_service_list() LOGGER.debug("{} reply '{}' Message '{}' Service list '{}'" .format("Keystone", status, message, service_list))
def cinder_check(self): status, message, cinder_list = self.cinder.cinder_list() LOGGER.debug("{} reply '{}' Message '{}' Cinder list '{}'".format( "Cinder", status, message, cinder_list))
def neutron_check(self): status, message, service_list = self.neutron.neutron_agent_list() LOGGER.debug("{} reply '{}' Message '{}' Agents list '{}'" .format("Neutron", status, message, service_list))