def disrupt_once(disrupt, ip_or_hostname, username, password, start_cmd,
stop_cmd, up_check_cmd, down_check_cmd, down_time_min,
down_time_max, cool_down_min, cool_down_max):
LOGGER.debug("Disrupt once {0}".format(ip_or_hostname))
down_time = random.randint(down_time_min, down_time_max)
cool_down_time = random.randint(cool_down_min, cool_down_max)
ssh = SSH(ip_or_hostname, username, password)
ssh.exec_command(stop_cmd)
ssh.exec_command(down_check_cmd) # TODO wait for down
if start_cmd: # TODO if service is down
time.sleep(down_time)
ssh.exec_command(start_cmd)
else:
disruption_finished = False
elapsed = 0
while not disruption_finished:
elapsed += cool_down_min
result, error = ssh.exec_command(up_check_cmd)
LOGGER.debug("Up check result is '{0}'. Error is {1}".format(
result.strip(), error.strip()))
if result is not None:
result = int(result.strip())
disruption_finished = (result == 0)
# TODO separate param for down_time_max
if not disruption_finished and elapsed > down_time_max:
LOGGER.exception(
"{disrupt} on {host} is not up in {timeout}".format(
disrupt=disrupt,
host=ip_or_hostname,
timeout=down_time_max))
break
time.sleep(1)
time.sleep(cool_down_time)
def disrupt_once(disrupt, ip_or_hostname, username, password, start_cmd,
stop_cmd, up_check_cmd, down_check_cmd, down_time_min,
down_time_max, cool_down_min, cool_down_max):
LOGGER.debug("Disrupt once {0}".format(ip_or_hostname))
down_time = random.randint(down_time_min, down_time_max)
cool_down_time = random.randint(cool_down_min, cool_down_max)
ssh = SSH(ip_or_hostname, username, password)
ssh.exec_command(stop_cmd)
ssh.exec_command(down_check_cmd) # TODO wait for down
if start_cmd: # TODO if service is down
time.sleep(down_time)
ssh.exec_command(start_cmd)
else:
disruption_finished = False
elapsed = 0
while not disruption_finished:
elapsed += cool_down_min
result, error = ssh.exec_command(up_check_cmd)
LOGGER.debug("Up check result is '{0}'. Error is {1}".format(
result.strip(), error.strip()))
if result is not None:
result = int(result.strip())
disruption_finished = (result == 0)
# TODO separate param for down_time_max
if not disruption_finished and elapsed > down_time_max:
LOGGER.exception(
"{disrupt} on {host} is not up in {timeout}".format(
disrupt=disrupt, host=ip_or_hostname,
timeout=down_time_max))
break
time.sleep(1)
time.sleep(cool_down_time)
def validate_config(self):
try:
rally_api.Task.validate(self.deployment_name,
self.scenario_config,
task_instance=self.rally_task)
except Exception as e:
print(e)
LOGGER.exception(e)
self.observer.tell({'msg': 'validation_complete', 'valid': False})
self.observer.tell({'msg': 'validation_complete', 'valid': True})
def validate_config(self):
try:
rally_api.Task.validate(self.deployment_name,
self.scenario_config,
task_instance=self.rally_task)
except Exception as e:
print(e)
LOGGER.exception(e)
self.observer.tell({'msg': 'validation_complete', 'valid': False})
self.observer.tell({'msg': 'validation_complete', 'valid': True})
def exec_command(self, cmd, get_pty=False):
result = None
error = None
try:
self.client.connect(hostname=self.host,
username=self.user,
password=self.password)
_, stdout, stderr = self.client.exec_command(cmd, get_pty)
except paramiko.AuthenticationException, ex:
LOGGER.exception("Authentication failed while connecting {}@{}."
" Exception {}".format(self.user, self.host, ex))
result = ''
error = ex
def abort(self):
try:
rally_api.Task.abort(self.rally_task.task["uuid"])
except RallyException as e:
LOGGER.exception(e)
finally:
self.runner_thread.join()
self.checker_thread.join()
res = rally_api.Task.get_detailed(self.rally_task.task["uuid"])
self.times = len(res["results"][0]["data"]["raw"])
# This will print standard rally report
task_cli.TaskCommands().detailed(task_id=self.rally_task.task['uuid'])
self.observer.tell({'msg': 'loader_finished', "times": self.times})
def abort(self):
try:
rally_api.Task.abort(self.rally_task.task["uuid"])
except RallyException as e:
LOGGER.exception(e)
finally:
self.runner_thread.join()
self.checker_thread.join()
res = rally_api.Task.get_detailed(self.rally_task.task["uuid"])
self.times = len(res["results"][0]["data"]["raw"])
# This will print standard rally report
task_cli.TaskCommands().detailed(
task_id=self.rally_task.task['uuid'])
self.observer.tell({'msg': 'loader_finished', "times": self.times})
def disrupt_once(disrupt, ip_or_hostname, username, password, start_cmd,
stop_cmd, up_check_cmd, down_check_cmd, down_time_min,
down_time_max, cool_down_min, cool_down_max):
# TODO (dratushnyy) named params in format
check_cmd = up_check_cmd.format(username=username, host=ip_or_hostname)
check_cmd = check_cmd.split(" ")
ssh = SSH(ip_or_hostname, username, password)
ssh.exec_command(stop_cmd)
disruption_finished = False
elapsed = 0
while not disruption_finished:
time.sleep(HostDisruptor.COOL_DOWN)
elapsed += HostDisruptor.COOL_DOWN
disruption_finished = (subprocess.call(check_cmd) == 0)
if not disruption_finished and elapsed > down_time_max:
LOGGER.exception("Host {host} not up in {timeout}".format(
host=ip_or_hostname, timeout=down_time_max))
break
time.sleep(cool_down_max)
def __init__(self, observer, openrc, inventory, **params):
super(RallyLoader, self).__init__(observer, openrc, inventory,
**params)
self.scenario_file = os.path.abspath(
os.path.join(RallyLoader.scenarios_path, params['scenario_file']))
# TODO (dratushnyy) fallback to default path only if file not found
self.scenario_args_file = params.get('scenario_args_file', None)
if self.scenario_args_file:
self.scenario_args_file = os.path.abspath(
os.path.join(RallyLoader.scenarios_path,
self.scenario_args_file))
self.start_delay = params['start_delay']
self.deployment_name = params['deployment_name']
self.deployment_config = {
"type": "ExistingCloud",
"admin": {
"username": openrc["username"],
"password": openrc["password"],
"tenant_name": openrc["tenant_name"]
},
"auth_url": openrc["auth_url"],
"region_name": openrc["region_name"],
"https_insecure": openrc['https_insecure'],
"https_cacert": openrc["https_cacert"]
}
self.scenario_args = params.get('scenario_args', None)
# Need to be set to None to avoid exception in stop() method
self.rally_task = None
load_rally_plugins()
if params.get('db'):
db_connection = RallyLoader.conn_template.format(
user=params["db"]["user"],
passwd=params["db"]["pass"],
host=params["db"]["host"],
db_name=params["db"]["name"])
db_options.set_defaults(CONF, connection=db_connection)
try:
rally_api.Deployment.get(self.deployment_name)
except DBNonExistentTable as e:
db.schema_create()
except DeploymentNotFound as e:
try:
rally_api.Deployment.create(config=self.deployment_config,
name=self.deployment_name)
except ValidationError as e:
LOGGER.exception(e)
raise e
except OperationalError as e:
LOGGER.exception(e)
raise e
# Since there is no api method to do this - using cli
deployment_cli.DeploymentCommands().use(self.deployment_name)
# Using rally task cli to load and validate task
# TODO check is API support this?
try:
self.scenario_config = task_cli.TaskCommands().\
_load_and_validate_task(self.scenario_file,
json.dumps(self.scenario_args),
self.scenario_args_file,
self.deployment_name)
except Exception as e:
LOGGER.exception(e)
raise e
def __init__(self, observer, openrc, inventory, **params):
super(RallyLoader, self).__init__(observer, openrc, inventory,
**params)
self.scenario_file = os.path.abspath(os.path.join(
RallyLoader.scenarios_path, params['scenario_file']))
# TODO (dratushnyy) fallback to default path only if file not found
self.scenario_args_file = params.get('scenario_args_file', None)
if self.scenario_args_file:
self.scenario_args_file = os.path.abspath(os.path.join(
RallyLoader.scenarios_path, self.scenario_args_file))
self.start_delay = params['start_delay']
self.deployment_name = params['deployment_name']
self.deployment_config = {
"type": "ExistingCloud",
"admin": {
"username": openrc["username"],
"password": openrc["password"],
"tenant_name": openrc["tenant_name"]
},
"auth_url": openrc["auth_url"],
"region_name": openrc["region_name"],
"https_insecure": openrc['https_insecure'],
"https_cacert": openrc["https_cacert"]
}
self.scenario_args = params.get('scenario_args', None)
# Need to be set to None to avoid exception in stop() method
self.rally_task = None
load_rally_plugins()
if params.get('db'):
db_connection = RallyLoader.conn_template.format(
user=params["db"]["user"],
passwd=params["db"]["pass"],
host=params["db"]["host"],
db_name=params["db"]["name"])
db_options.set_defaults(CONF, connection=db_connection)
try:
rally_api.Deployment.get(self.deployment_name)
except DBNonExistentTable as e:
db.schema_create()
except DeploymentNotFound as e:
try:
rally_api.Deployment.create(config=self.deployment_config,
name=self.deployment_name)
except ValidationError as e:
LOGGER.exception(e)
raise e
except OperationalError as e:
LOGGER.exception(e)
raise e
# Since there is no api method to do this - using cli
deployment_cli.DeploymentCommands().use(self.deployment_name)
# Using rally task cli to load and validate task
# TODO check is API support this?
try:
self.scenario_config = task_cli.TaskCommands().\
_load_and_validate_task(self.scenario_file,
json.dumps(self.scenario_args),
self.scenario_args_file,
self.deployment_name)
except Exception as e:
LOGGER.exception(e)
raise e