def is_docker_available():
LOGGER.info("Checking that docker is available...")
result = LocalCmdRunner().run('docker ps',
ignore_status=True,
verbose=False)
if result.ok:
LOGGER.info('Docker is available')
return True
else:
LOGGER.warning(
'Docker is not available on your computer. Please install docker software before continue'
)
return False
def stop_rsyslog():
global RSYSLOG_DOCKER_ID, RSYSLOG_CONF_PATH
if RSYSLOG_DOCKER_ID:
local_runner = LocalCmdRunner()
local_runner.run("docker kill {id}".format(id=RSYSLOG_DOCKER_ID), ignore_status=True)
if RSYSLOG_CONF_PATH:
try:
os.remove(RSYSLOG_CONF_PATH)
except Exception:
pass
RSYSLOG_CONF_PATH = None
RSYSLOG_DOCKER_ID = None
def verify_dockers_are_running():
result = LocalCmdRunner().run("docker ps --format '{{.Names}}'",
ignore_status=True) # pylint: disable=invalid-name
docker_names = result.stdout.strip().split()
if result.ok and docker_names:
if f"{GRAFANA_DOCKER_NAME}-{GRAFANA_DOCKER_PORT}" in docker_names \
and f"{PROMETHEUS_DOCKER_NAME}-{PROMETHEUS_DOCKER_PORT}" in docker_names:
LOGGER.info("Monitoring stack docker containers are running.\n%s",
result.stdout)
return True
LOGGER.error(
"Monitoring stack containers are not running\nStdout:\n%s\nstderr:%s",
result.stdout, result.stderr)
return False
def start_rsyslog(docker_name, log_dir, port="514"):
"""
Start rsyslog in a docker, for getting logs from db-nodes
:param docker_name: name of the docker instance
:param log_dir: directory where to store the logs
:param port: [Optional] the port binding for the docker run
:return: the listening port
"""
global RSYSLOG_DOCKER_ID, RSYSLOG_CONF_PATH # pylint: disable=global-statement
log_dir = os.path.abspath(log_dir)
# cause of docker-in-docker, we need to capture the host log dir for mounting it
# _SCT_BASE_DIR is set in hydra.sh
base_dir = os.environ.get("_SCT_BASE_DIR", None)
if base_dir:
mount_log_dir = os.path.join(base_dir, os.path.basename(log_dir))
else:
mount_log_dir = log_dir
conf_path = generate_conf_file()
RSYSLOG_CONF_PATH = conf_path
local_runner = LocalCmdRunner()
res = local_runner.run('''
mkdir -p {log_dir};
docker run --rm -d \
-v /etc/passwd:/etc/passwd:ro \
-v /etc/group:/etc/group:ro \
-v {mount_log_dir}:/logs \
-v {conf_path}:/etc/rsyslog.conf \
-p {port} \
--name {docker_name}-rsyslogd rsyslog/syslog_appliance_alpine
'''.format(log_dir=log_dir,
mount_log_dir=mount_log_dir,
conf_path=conf_path,
port=port,
docker_name=docker_name))
RSYSLOG_DOCKER_ID = res.stdout.strip()
LOGGER.info("Rsyslog started. Container id: %s", RSYSLOG_DOCKER_ID)
atexit.register(stop_rsyslog)
res = local_runner.run('docker port {0} 514'.format(RSYSLOG_DOCKER_ID))
listening_port = res.stdout.strip().split(':')[1]
return listening_port
def create_monitoring_stack_dir(base_dir, archive):
cmd = dedent("""
cd {data_dir}
cp {archive} ./
tar -xvf {archive_name}
chmod -R 777 {data_dir}
""".format(data_dir=base_dir,
archive_name=os.path.basename(archive),
archive=archive))
result = LocalCmdRunner().run(cmd, ignore_status=True)
if result.exited > 0:
LOGGER.error("Error during extracting monitoring stack")
return False
return get_monitoring_stack_dir(base_dir)
def _get_shared_snapshot_link(self, grafana_url):
result = LocalCmdRunner().run(
"cd {0.phantomjs_dir} && bin/phantomjs {0.sct_base_path}/data_dir/share_snapshot.js \"{1}\""
.format(self, grafana_url),
ignore_status=True)
# since there is only one monitoring node returning here
output = result.stdout.strip()
if "Error" in output:
LOGGER.error(output)
return ""
else:
matched = re.search(
r"https://snapshot.raintank.io/dashboard/snapshot/\w+", output)
LOGGER.info("Shared grafana snapshot: {}".format(matched.group()))
return matched.group()
def stop_auto_ssh(docker_name, node):
"""
stops an autossh docker instance
:param docker_name: prefix of the docker name (cluster.Setup.test_id() usually would be used)
:param node: an instance of a class derived from BaseNode that has ssh_login_info
:return: None
"""
# pylint: disable=protected-access
host_name = node.ssh_login_info['hostname']
container_name = f"{docker_name}-{host_name}-autossh"
local_runner = LocalCmdRunner()
LOGGER.debug("Saving autossh container logs")
local_runner.run(f"docker logs {container_name} &> {node.logdir}/autossh.log", ignore_status=True)
LOGGER.debug(f"Killing {container_name}")
local_runner.run(f"docker rm -f {container_name}", ignore_status=True)
def start_dockers(monitoring_dockers_dir, monitoring_stack_data_dir, scylla_version): # pylint: disable=unused-argument
graf_port = GRAFANA_DOCKER_PORT
alert_port = ALERT_DOCKER_PORT
prom_port = PROMETHEUS_DOCKER_PORT
lr = LocalCmdRunner() # pylint: disable=invalid-name
lr.run('cd {monitoring_dockers_dir}; ./kill-all.sh -g {graf_port} -m {alert_port} -p {prom_port}'.format(**locals()),
ignore_status=True, verbose=False)
cmd = dedent("""cd {monitoring_dockers_dir};
./start-all.sh \
-g {graf_port} -m {alert_port} -p {prom_port} \
-s {monitoring_dockers_dir}/config/scylla_servers.yml \
-n {monitoring_dockers_dir}/config/node_exporter_servers.yml \
-d {monitoring_stack_data_dir} -v {scylla_version}""".format(**locals()))
res = lr.run(cmd)
if res.ok:
LOGGER.info("Docker containers for monitoring stack are started")
def create_monitoring_data_dir(base_dir, archive):
monitoring_data_base_dir = os.path.join(base_dir, 'monitoring_data_dir')
cmd = dedent("""
mkdir -p {data_dir}
cd {data_dir}
cp {archive} ./
tar -xvf {archive_name}
chmod -R 777 {data_dir}
""".format(data_dir=monitoring_data_base_dir,
archive=archive,
archive_name=os.path.basename(archive)))
result = LocalCmdRunner().run(cmd, timeout=COMMAND_TIMEOUT, ignore_status=True)
if result.exited > 0:
LOGGER.error("Error during extracting prometheus snapshot. Switch to next archive")
return False
return get_monitoring_data_dir(monitoring_data_base_dir)
def start_dockers(monitoring_dockers_dir, monitoring_stack_data_dir,
scylla_version): # pylint: disable=unused-argument
lr = LocalCmdRunner() # pylint: disable=invalid-name
lr.run('cd {}; ./kill-all.sh'.format(monitoring_dockers_dir))
cmd = dedent("""cd {monitoring_dockers_dir};
./start-all.sh \
-s {monitoring_dockers_dir}/config/scylla_servers.yml \
-n {monitoring_dockers_dir}/config/node_exporter_servers.yml \
-d {monitoring_stack_data_dir} -v {scylla_version}""".format(
**locals()))
res = lr.run(cmd, ignore_status=True)
if res.ok:
r = lr.run('docker ps') # pylint: disable=invalid-name
LOGGER.info(r.stdout.encode('utf-8'))
return True
else:
raise Exception('dockers start failed. {}'.format(res))
def __init__(self,
name,
ssh_login_info=None,
instance=None,
global_ip=None,
grafana_ip=None): # pylint: disable=too-many-arguments
self.name = name
if ssh_login_info is None:
self.remoter = LocalCmdRunner()
else:
self.remoter = RemoteCmdRunner(**ssh_login_info)
self.ssh_login_info = ssh_login_info
self._instance = instance
self.external_address = global_ip
if grafana_ip is None:
self.grafana_address = global_ip
else:
self.grafana_address = grafana_ip
def is_auto_ssh_running(docker_name, node):
"""
check it auto_ssh docker running
:param docker_name: docker name
:type docker_name: str
:param node: Node instance where tunnel was up
:type node: BaseNode
:returns: true if running, false otherwise
:rtype: {boolean}
"""
local_runner = LocalCmdRunner()
host_name = node.ssh_login_info['hostname']
container_name = f"{docker_name}-{host_name}-autossh"
result = local_runner.run('docker ps', ignore_status=True)
return container_name in result.stdout
def test_create_and_run_in_separate_thread( # pylint: disable=too-many-arguments
self, remoter_type, host: str, stmt: str, verbose: bool,
ignore_status: bool, new_session: bool, retry: int,
timeout: Union[float, None]):
kwargs = {
'verbose': verbose,
'ignore_status': ignore_status,
'new_session': new_session,
'retry': retry,
'timeout': timeout
}
self.log.info(repr({stmt: stmt, **kwargs}))
try:
expected = LocalCmdRunner().run(stmt, **kwargs)
except Exception as exc: # pylint: disable=broad-except
expected = exc
# Paramiko fails too often when it is invoked like that, that is why it is not in the test
if issubclass(remoter_type, (RemoteCmdRunner, RemoteLibSSH2CmdRunner)):
remoter = remoter_type(hostname=host,
user=getpass.getuser(),
key_file=self.key_file)
else:
remoter = KubernetesCmdRunner(FakeKluster('http://127.0.0.1:8001'),
pod='sct-cluster-dc-1-kind-0',
container="scylla",
namespace="scylla")
libssh2_thread_results = []
self._run_parallel(3,
thread_body=self._create_and_run_in_separate_thread,
args=(remoter, stmt, kwargs,
libssh2_thread_results),
kwargs={})
for libssh2_result in libssh2_thread_results:
self.log.error(str(libssh2_result))
self._compare_results(expected,
libssh2_result,
stmt=stmt,
kwargs=kwargs)
def start_auto_ssh(docker_name, node, local_port, remote_port, ssh_mode="-R"):
"""
Starts a reverse port forwarding with autossh inside a docker container
:param docker_name: prefix of the docker name (cluster.Setup.test_id() usually would be used)
:param node: an instance of a class derived from BaseNode that has ssh_login_info
:param local_port: the destination port on local machine
:param remote_port: the source port on the remote
:param ssh_mode: define tunnel mode: -R reverse mode, -L direct mode
:return: None
"""
# pylint: disable=protected-access
host_name = node.ssh_login_info['hostname']
container_name = set_container_name(host_name=host_name,
docker_name=docker_name)
user_name = node.ssh_login_info['user']
key_path = node.ssh_login_info['key_file']
local_runner = LocalCmdRunner()
res = local_runner.run('''
docker run -d --network=host \
-e SSH_HOSTNAME={host_name} \
-e SSH_HOSTUSER={user_name} \
-e SSH_TUNNEL_HOST=127.0.0.1 \
-e SSH_MODE={ssh_mode} \
-e SSH_TUNNEL_LOCAL={local_port} \
-e SSH_TUNNEL_REMOTE={remote_port} \
-e AUTOSSH_GATETIME=0 \
-v {key_path}:/id_rsa \
--restart always \
--name {container_name} jnovack/autossh
'''.format(container_name=container_name,
user_name=user_name,
ssh_mode=ssh_mode,
local_port=local_port,
remote_port=remote_port,
key_path=key_path,
host_name=host_name))
atexit.register(stop_auto_ssh, docker_name, node)
LOGGER.debug('{container_name} {res.stdout}'.format(
container_name=container_name, res=res))
def test_run_in_mainthread( # pylint: disable=too-many-arguments
self, host: str, stmt: str, verbose: bool, ignore_status: bool, new_session: bool, retry: int,
timeout: Union[float, None]):
kwargs = {
'verbose': verbose,
'ignore_status': ignore_status,
'new_session': new_session,
'retry': retry,
'timeout': timeout}
try:
expected = LocalCmdRunner().run(stmt, **kwargs)
except Exception as exc: # pylint: disable=broad-except
expected = exc
remoter = RemoteCmdRunner(
hostname=host, user=getpass.getuser(), key_file=self.key_file)
try:
paramiko_result = remoter.run(stmt, **kwargs)
except Exception as exc: # pylint: disable=broad-except
paramiko_result = exc
remoter._reconnect()
try:
paramiko_result2 = remoter.run(stmt, **kwargs)
except Exception as exc: # pylint: disable=broad-except
paramiko_result2 = exc
remoter.stop()
remoter = RemoteLibSSH2CmdRunner(
hostname='127.0.0.1', user=getpass.getuser(), key_file=self.key_file)
try:
lib2ssh_result = remoter.run(stmt, **kwargs)
except Exception as exc: # pylint: disable=broad-except
lib2ssh_result = exc
remoter._reconnect()
try:
lib2ssh_result2 = remoter.run(stmt, **kwargs)
except Exception as exc: # pylint: disable=broad-except
lib2ssh_result2 = exc
remoter.stop()
self._compare_results(expected, paramiko_result, stmt=stmt, kwargs=kwargs)
self._compare_results(expected, paramiko_result2, stmt=stmt, kwargs=kwargs)
self._compare_results(expected, lib2ssh_result, stmt=stmt, kwargs=kwargs)
self._compare_results(expected, lib2ssh_result2, stmt=stmt, kwargs=kwargs)
def start_dockers(monitoring_dockers_dir, monitoring_stack_data_dir,
scylla_version): # pylint: disable=unused-argument
graf_port = GRAFANA_DOCKER_PORT
alert_port = ALERT_DOCKER_PORT
prom_port = PROMETHEUS_DOCKER_PORT
lr = LocalCmdRunner() # pylint: disable=invalid-name
lr.run(
'cd {monitoring_dockers_dir}; ./kill-all.sh -g {graf_port} -m {alert_port} -p {prom_port}'
.format(**locals()),
ignore_status=True,
verbose=False)
# clear scylla nodes from configuration
servers_yaml = Path(
monitoring_dockers_dir) / 'config' / 'scylla_servers.yml'
servers_yaml.write_text("- targets: []")
# clear SCT scrape configurations
prom_tmpl_file = Path(
monitoring_dockers_dir) / 'prometheus' / 'prometheus.yml.template'
templ_yaml = yaml.safe_load(prom_tmpl_file.read_text())
def remove_sct_metrics(metric):
return '_metrics' not in metric['job_name']
templ_yaml["scrape_configs"] = list(
filter(remove_sct_metrics, templ_yaml["scrape_configs"]))
prom_tmpl_file.write_text(yaml.safe_dump(templ_yaml))
cmd = dedent("""cd {monitoring_dockers_dir};
echo "" > UA.sh
./start-all.sh \
$(grep -q -- --no-renderer ./start-all.sh && echo "--no-renderer") \
$(grep -q -- --no-loki ./start-all.sh && echo "--no-loki") \
-g {graf_port} -m {alert_port} -p {prom_port} \
-s {monitoring_dockers_dir}/config/scylla_servers.yml \
-d {monitoring_stack_data_dir} -v {scylla_version} \
-b '-storage.tsdb.retention.time=100y' \
-c 'GF_USERS_DEFAULT_THEME=dark'""".format(**locals()))
res = lr.run(cmd)
if res.ok:
LOGGER.info("Docker containers for monitoring stack are started")
def get_docker_instances_by_testid(self):
instances = list_instances_gce({"TestId": self.test_id}, running=True)
filtered_instances = filter_gce_instances_by_type(instances)
for instance in filtered_instances['db_nodes']:
self.db_cluster.append(
CollectingNode(name=instance.name,
ssh_login_info={
"hostname": instance.public_ips[0],
"user": '******',
"key_file":
self.params['user_credentials_path']
},
instance=instance,
global_ip=instance.public_ips[0],
tags={
**self.tags,
"NodeType": "scylla-db",
}))
self.monitor_set.append(
CollectingNode(name=f"monitor-node-{self.test_id}-0",
global_ip='127.0.0.1',
grafana_ip=get_docker_bridge_gateway(
LocalCmdRunner()),
tags={
**self.tags,
"NodeType": "monitor",
}))
for instance in filtered_instances['loader_nodes']:
self.loader_set.append(
CollectingNode(name=instance.name,
ssh_login_info={
"hostname": instance.public_ips[0],
"user": '******',
"key_file":
self.params['user_credentials_path']
},
instance=instance,
global_ip=instance.public_ips[0],
tags={
**self.tags,
"NodeType": "loader",
}))
def test_run_in_mainthread( # pylint: disable=too-many-arguments
self, remoter_type, host: str, stmt: str, verbose: bool,
ignore_status: bool, new_session: bool, retry: int,
timeout: Union[float, None]):
kwargs = {
'verbose': verbose,
'ignore_status': ignore_status,
'new_session': new_session,
'retry': retry,
'timeout': timeout
}
try:
expected = LocalCmdRunner().run(stmt, **kwargs)
except Exception as exc: # pylint: disable=broad-except
expected = exc
if issubclass(remoter_type, (RemoteCmdRunner, RemoteLibSSH2CmdRunner)):
remoter = remoter_type(hostname=host,
user=getpass.getuser(),
key_file=self.key_file)
else:
remoter = KubernetesCmdRunner(FakeKluster('http://127.0.0.1:8001'),
pod='sct-cluster-dc-1-kind-0',
container="scylla",
namespace="scylla")
try:
result = remoter.run(stmt, **kwargs)
except Exception as exc: # pylint: disable=broad-except
result = exc
remoter._reconnect() # pylint: disable=protected-access
try:
result2 = remoter.run(stmt, **kwargs)
except Exception as exc: # pylint: disable=broad-except
result2 = exc
remoter.stop()
self._compare_results(expected, result, stmt=stmt, kwargs=kwargs)
self._compare_results(expected, result2, stmt=stmt, kwargs=kwargs)
def install_phantom_js(self):
"""Install phantom_js to sct root dir
If sct runs outside the docker container,
sct will install the phantomjs to
sct root dir
"""
localrunner = LocalCmdRunner()
if not self.phantomjs_installed:
LOGGER.debug("Installing phantomjs to sct root dir")
# pylint: disable=unused-variable
phantomjs_tar = "{0.phantomjs_base}.tar.bz2".format(self)
phantomjs_url = "https://bitbucket.org/ariya/phantomjs/downloads/{phantomjs_tar}".format(
**locals())
install_phantom_js_script = dedent("""
curl {phantomjs_url} -o {phantomjs_tar} -L
tar xvfj {phantomjs_tar}
""".format(**locals()))
localrunner.run("bash -ce '%s'" % install_phantom_js_script)
self.phantomjs_dir = os.path.join(self.sct_base_path,
self.phantomjs_base)
else:
LOGGER.debug("PhantomJS is already installed!")
def test_load_1000_threads(self, remoter_type, stmt: str):
kwargs = {
'verbose': True,
'ignore_status': False,
'new_session': True,
'retry': 2
}
self.log.info(repr({stmt: stmt, **kwargs}))
try:
expected = LocalCmdRunner().run(stmt, **kwargs)
except Exception as exc: # pylint: disable=broad-except
expected = exc
libssh2_thread_results = []
self._run_parallel(
1000,
thread_body=self._create_and_run_in_same_thread,
args=(remoter_type, self.key_file, stmt, kwargs, libssh2_thread_results),
kwargs={})
for libssh2_result in libssh2_thread_results:
self.log.error(str(libssh2_result))
self._compare_results(expected, libssh2_result, stmt=stmt, kwargs=kwargs)
def test_create_and_run_in_same_thread( # pylint: disable=too-many-arguments,too-many-locals
self, remoter_type, host: str, stmt: str, verbose: bool, ignore_status: bool, new_session: bool,
retry: int, timeout: Union[float, None]):
kwargs = {
'verbose': verbose,
'ignore_status': ignore_status,
'new_session': new_session,
'retry': retry,
'timeout': timeout}
self.log.info(repr({stmt: stmt, **kwargs}))
try:
expected = LocalCmdRunner().run(stmt, **kwargs)
except Exception as exc: # pylint: disable=broad-except
expected = exc
paramiko_thread_results = []
self._run_parallel(
3,
thread_body=self._create_and_run_in_same_thread,
args=(remoter_type, host, self.key_file, stmt, kwargs, paramiko_thread_results),
kwargs={})
for paramiko_result in paramiko_thread_results:
self._compare_results(expected, paramiko_result, stmt=stmt, kwargs=kwargs)
def __init__(self, name, parent_cluster, ssh_login_info=None, base_logdir=None, node_prefix=None, dc_idx=0):
super().__init__(name, parent_cluster)
self.remoter = LocalCmdRunner()
self.logdir = os.path.dirname(__file__)
def kill_running_monitoring_stack_services():
lr = LocalCmdRunner() # pylint: disable=invalid-name
for docker in get_monitoring_stack_services():
LOGGER.info("Killing %s", docker['service'])
lr.run('docker rm -f {name}-{port}'.format(name=docker['name'], port=docker['port']),
ignore_status=True)
import subprocess
import time
import os
import shutil
import threading
import uuid
import Queue
import xml.etree.cElementTree as etree
import atexit
from sdcm import wait
from sdcm.remote import LocalCmdRunner
from sdcm import cluster
localrunner = LocalCmdRunner()
class LibvirtNode(cluster.BaseNode):
"""
Wraps a domain object, so that we can also control the it through SSH.
"""
def __init__(self,
domain,
hypervisor,
parent_cluster,
node_prefix='node',
node_index=1,
domain_username='******',
domain_password='',
base_logdir=None):
name = '%s-%s' % (node_prefix, node_index)
import re
import json
import time
import logging
import os
import types
from sdcm import cluster
from sdcm.utils.common import retrying
from sdcm.remote import LocalCmdRunner
from sdcm.log import SDCMAdapter
from sdcm.utils.common import makedirs
LOGGER = logging.getLogger(__name__)
LOCALRUNNER = LocalCmdRunner()
class DockerCommandError(Exception):
pass
class DockerContainerNotExists(Exception):
pass
class DockerContainerNotRunning(Exception):
pass
class CannotFindContainers(Exception):
pass
def __init__(self):
self.remoter = LocalCmdRunner()
self.ip_address = "127.0.0.1"
self.logdir = os.path.dirname(__file__)
class TestHydraSh(unittest.TestCase):
cmd_runner = LocalCmdRunner()
@staticmethod
def prepare_environment(env):
for name in os.environ:
if any(
name.startswith(prefix)
for prefix in ['SCT_', 'AWS_', 'GOOGLE_']):
del os.environ[name]
for name, value in env.items():
os.environ[name] = value
@staticmethod
@contextlib.contextmanager
def environ():
old_environment = os.environ.copy()
yield
os.environ.clear()
for name, value in old_environment.items():
os.environ[name] = value
@staticmethod
def validate_result(
result,
expected_status: int,
expected: Sequence[Union[str, re.Pattern]],
not_expected: Sequence[Union[str, re.Pattern]],
):
errors = []
if expected_status is not None:
if result.return_code != expected_status:
errors.append(
f'Returned status {result.return_code}, while expected {expected_status}'
)
for pattern_expected in expected:
if isinstance(pattern_expected, re.Pattern):
if not pattern_expected.search(result.stdout):
errors.append(
f"Can't find regex {pattern_expected.pattern}")
elif isinstance(pattern_expected, str):
if pattern_expected not in result.stdout:
errors.append(f"Can't find {pattern_expected}")
for pattern_not_expected in not_expected:
if isinstance(pattern_not_expected, re.Pattern):
if pattern_not_expected.search(result.stdout):
errors.append(
f"Found pattern that should not be there: {pattern_not_expected}"
)
elif isinstance(pattern_not_expected, str):
if pattern_not_expected in result.stdout:
errors.append(
f"Found pattern that should not be there: {pattern_not_expected}"
)
return errors
@parameterized.expand(
LongevityPipelineTest(
backend='aws', runner=False, aws_creds=True,
gce_creds=True).hydra_test_cases + LongevityPipelineTest(
backend='aws', runner=True, aws_creds=True,
gce_creds=True).hydra_test_cases +
LongevityPipelineTest(
backend='gce', runner=False, aws_creds=True,
gce_creds=True).hydra_test_cases + LongevityPipelineTest(
backend='gce', runner=True, aws_creds=True,
gce_creds=True).hydra_test_cases +
LongevityPipelineTest(
backend='gce-siren', runner=False, aws_creds=True,
gce_creds=True).hydra_test_cases +
LongevityPipelineTest(
backend='gce-siren', runner=True, aws_creds=True,
gce_creds=True).hydra_test_cases)
def test_run_test(self, test_case_params: HydraTestCaseParams,
tmp_dir: HydraTestCaseTmpDir):
with tmp_dir, self.environ():
cmd, expected, not_expected, expected_status, env = test_case_params.as_tuple
self.prepare_environment(env)
result = self.cmd_runner.run(sct_abs_path('docker/env/hydra.sh') +
' --dry-run-hydra ' + cmd,
ignore_status=True)
errors = self.validate_result(
result=result,
expected_status=expected_status,
expected=expected,
not_expected=not_expected,
)
assert not errors, f'Case: {cmd}\nReturned:\n{result}\nFound following errors:\n' + (
'\n'.join(errors))
from sdcm.sct_events.system import SpotTerminationEvent
from sdcm.utils.aws_utils import tags_as_ec2_tags, ec2_instance_wait_public_ip
from sdcm.utils.common import list_instances_aws, get_ami_tags, MAX_SPOT_DURATION_TIME
from sdcm.utils.decorators import retrying
from sdcm.wait import exponential_retry
LOGGER = logging.getLogger(__name__)
INSTANCE_PROVISION_ON_DEMAND = 'on_demand'
INSTANCE_PROVISION_SPOT_FLEET = 'spot_fleet'
INSTANCE_PROVISION_SPOT_LOW_PRICE = 'spot_low_price'
INSTANCE_PROVISION_SPOT_DURATION = 'spot_duration'
SPOT_CNT_LIMIT = 20
SPOT_FLEET_LIMIT = 50
SPOT_TERMINATION_CHECK_OVERHEAD = 15
LOCAL_CMD_RUNNER = LocalCmdRunner()
EBS_VOLUME = "attached"
INSTANCE_STORE = "instance_store"
P = ParamSpec("P") # pylint: disable=invalid-name
R = TypeVar("R") # pylint: disable=invalid-name
# pylint: disable=too-many-lines
class AWSCluster(cluster.BaseCluster): # pylint: disable=too-many-instance-attributes,abstract-method,
"""
Cluster of Node objects, started on Amazon EC2.
"""
def restore_monitoring_stack(test_id):
from sdcm.remote import LocalCmdRunner
lr = LocalCmdRunner()
logger.info("Checking that docker is available...")
result = lr.run('docker ps', ignore_status=True, verbose=False)
if result.ok:
logger.info('Docker is available')
else:
logger.warning(
'Docker is not available on your computer. Please install docker software before continue'
)
return False
monitor_stack_base_dir = tempfile.mkdtemp()
stored_files_by_test_id = list_logs_by_test_id(test_id)
monitor_stack_archives = []
for f in stored_files_by_test_id:
if f['type'] in ['monitoring_data_stack', 'prometheus']:
monitor_stack_archives.append(f)
if not monitor_stack_archives or len(monitor_stack_archives) < 2:
logger.warning(
'There is no available archive files for monitoring data stack restoring for test id : {}'
.format(test_id))
return False
for arch in monitor_stack_archives:
logger.info('Download file {} to directory {}'.format(
arch['link'], monitor_stack_base_dir))
local_path_monitor_stack = S3Storage().download_file(
arch['link'], dst_dir=monitor_stack_base_dir)
monitor_stack_workdir = os.path.dirname(local_path_monitor_stack)
monitoring_stack_archive_file = os.path.basename(
local_path_monitor_stack)
logger.info('Extracting data from archive {}'.format(
arch['file_path']))
if arch['type'] == 'prometheus':
monitoring_stack_data_dir = os.path.join(monitor_stack_workdir,
'monitor_data_dir')
cmd = dedent("""
mkdir -p {data_dir}
cd {data_dir}
cp ../{archive} ./
tar -xvf {archive}
chmod -R 777 {data_dir}
""".format(data_dir=monitoring_stack_data_dir,
archive=monitoring_stack_archive_file))
result = lr.run(cmd, ignore_status=True)
else:
branches = re.search(
'(?P<monitoring_branch>branch-[\d]+\.[\d]+?)_(?P<scylla_version>[\d]+\.[\d]+?)',
monitoring_stack_archive_file)
monitoring_branch = branches.group('monitoring_branch')
scylla_version = branches.group('scylla_version')
cmd = dedent("""
cd {workdir}
tar -xvf {archive}
""".format(workdir=monitor_stack_workdir,
archive=monitoring_stack_archive_file))
result = lr.run(cmd, ignore_status=True)
if not result.ok:
logger.warning(
"During restoring file {} next errors occured:\n {}".format(
arch['link'], result))
return False
logger.info("Extracting data finished")
logger.info(
'Monitoring stack files available {}'.format(monitor_stack_workdir))
monitoring_dockers_dir = os.path.join(
monitor_stack_workdir,
'scylla-monitoring-{}'.format(monitoring_branch))
def upload_sct_dashboards():
sct_dashboard_file_name = "scylla-dash-per-server-nemesis.{}.json".format(
scylla_version)
sct_dashboard_file = os.path.join(monitoring_dockers_dir,
'sct_monitoring_addons',
sct_dashboard_file_name)
if not os.path.exists(sct_dashboard_file):
logger.info('There is no dashboard {}. Skip load dashboard'.format(
sct_dashboard_file_name))
return False
dashboard_url = 'http://*****:*****@retrying(n=3, sleep_time=1, message='Start docker containers')
def start_dockers(monitoring_dockers_dir, monitoring_stack_data_dir,
scylla_version):
lr.run('cd {}; ./kill-all.sh'.format(monitoring_dockers_dir))
cmd = dedent("""cd {monitoring_dockers_dir};
./start-all.sh \
-s {monitoring_dockers_dir}/config/scylla_servers.yml \
-n {monitoring_dockers_dir}/config/node_exporter_servers.yml \
-d {monitoring_stack_data_dir} -v {scylla_version}""".format(
**locals()))
res = lr.run(cmd, ignore_status=True)
if res.ok:
r = lr.run('docker ps')
logger.info(r.stdout.encode('utf-8'))
return True
else:
raise Exception('dockers start failed. {}'.format(res))
status = False
status = start_dockers(monitoring_dockers_dir, monitoring_stack_data_dir,
scylla_version)
upload_sct_dashboards()
upload_annotations()
return status