def test_10_longevity(self):
os.environ['SCT_CLUSTER_BACKEND'] = 'aws'
os.environ[
'SCT_CONFIG_FILES'] = 'internal_test_data/complex_test_case_with_version.yaml'
os.environ['SCT_AMI_ID_DB_SCYLLA_DESC'] = 'master'
conf = SCTConfiguration()
conf.verify_configuration()
self.assertEqual(conf.get('user_prefix'),
'longevity-50gb-4d-not-jenkins-maste')
def _get_node_amounts(config: SCTConfiguration) -> tuple[int, int]:
num_db_node = config.get("n_db_nodes")
num_db_node = sum([int(i) for i in num_db_node.split()]) if isinstance(
num_db_node, str) else num_db_node
num_loaders = config.get("n_loaders")
num_loaders = sum([int(i) for i in num_loaders.split()]) if isinstance(
num_loaders, str) else num_loaders
return num_db_node, num_loaders
def test_12_scylla_version_ami(self):
os.environ['SCT_CLUSTER_BACKEND'] = 'aws'
os.environ['SCT_SCYLLA_VERSION'] = '3.0.3'
os.environ['SCT_CONFIG_FILES'] = 'internal_test_data/multi_region_dc_test_case.yaml'
conf = SCTConfiguration()
conf.verify_configuration()
self.assertEqual(conf.get('ami_id_db_scylla'), 'ami-0f1aa8afb878fed2b ami-027c1337dcb46da50')
def test_12_scylla_version_ami_case1(): # pylint: disable=invalid-name
os.environ['SCT_CLUSTER_BACKEND'] = 'aws'
os.environ['SCT_SCYLLA_VERSION'] = '3.0.3'
os.environ['SCT_AMI_ID_DB_SCYLLA'] = 'ami-06f919eb ami-eae4f795'
os.environ[
'SCT_CONFIG_FILES'] = 'internal_test_data/multi_region_dc_test_case.yaml'
conf = SCTConfiguration()
conf.verify_configuration()
def _prepare_k8s_gce_minikube_resource_setup(sct_config: SCTConfiguration):
cloud_setup = _prepare_gce_resource_setup(sct_config)
image_id = sct_config.get("scylla_version")
cloud_setup.db_node.image_id = f"scylladb/scylladb:{image_id}"
cloud_setup.db_node.instance_type = sct_config.get(
"gce_instance_type_minikube")
return cloud_setup
def conf_docs(output_format):
add_file_logger()
config_logger = logging.getLogger('sdcm.sct_config')
config_logger.setLevel(logging.ERROR)
if output_format == 'markdown':
click.secho(SCTConfiguration().dump_help_config_markdown())
elif output_format == 'yaml':
click.secho(SCTConfiguration().dump_help_config_yaml())
def _prepare_k8s_gke_resource_setup(
sct_config: SCTConfiguration) -> BaseCloudSetupDetails:
cloud_setup = _prepare_gce_resource_setup(sct_config)
image_id = sct_config.get("scylla_version")
cloud_setup.db_node.image_id = f"scylladb/scylladb:{image_id}"
cloud_setup.monitor_node.image_id = sct_config.get("mgmt_docker_image")
cloud_setup.loader_node.image_id = f"scylladb/scylladb:{image_id}"
return cloud_setup
def test_09_unknown_env(self):
os.environ['SCT_CLUSTER_BACKEND'] = 'docker'
os.environ['SCT_CONFIG_FILES'] = 'internal_test_data/unknown_param_in_config.yaml'
os.environ['SCT_WHAT_IS_THAT'] = 'just_made_this_up'
os.environ['SCT_WHAT_IS_THAT_2'] = 'what is this ?'
conf = SCTConfiguration()
with self.assertRaises(ValueError) as cm:
conf.verify_configuration()
self.assertEquals(cm.exception.message, 'Unsupported environment variables were used:\n\t - SCT_WHAT_IS_THAT=just_made_this_up\n\t - SCT_WHAT_IS_THAT_2=what is this ?')
def test_13_scylla_version_ami_branch(self): # pylint: disable=invalid-name
os.environ['SCT_CLUSTER_BACKEND'] = 'aws'
os.environ['SCT_SCYLLA_VERSION'] = 'branch-3.1:9'
os.environ[
'SCT_CONFIG_FILES'] = 'internal_test_data/multi_region_dc_test_case.yaml'
conf = SCTConfiguration()
conf.verify_configuration()
self.assertEqual(conf.get('ami_id_db_scylla'),
'ami-00b5ac462978479c9 ami-0d911e53781d8a542')
def test_13_scylla_version_ami_branch(self):
os.environ['SCT_CLUSTER_BACKEND'] = 'aws'
os.environ['SCT_SCYLLA_VERSION'] = 'branch-3.1:60'
os.environ[
'SCT_CONFIG_FILES'] = 'internal_test_data/multi_region_dc_test_case.yaml'
conf = SCTConfiguration()
conf.verify_configuration()
self.assertEqual(conf.get('ami_id_db_scylla'),
'ami-0fd79a6f4ca83b0d6 ami-03d6caf668ad59911')
def output_conf(config_files, backend):
add_file_logger()
if backend:
os.environ['SCT_CLUSTER_BACKEND'] = backend
if config_files:
os.environ['SCT_CONFIG_FILES'] = config_files
config = SCTConfiguration()
click.secho(config.dump_config(), fg='green')
sys.exit(0)
def test_13_scylla_version_ami_branch(self): # pylint: disable=invalid-name
os.environ['SCT_CLUSTER_BACKEND'] = 'aws'
os.environ['SCT_SCYLLA_VERSION'] = 'branch-4.2:100'
os.environ[
'SCT_CONFIG_FILES'] = 'internal_test_data/multi_region_dc_test_case.yaml'
conf = SCTConfiguration()
conf.verify_configuration()
self.assertEqual(conf.get('ami_id_db_scylla'),
'ami-07d3138defbd9a2cf ami-0f703fdc8e06723f0')
def test_14_(self):
os.environ['SCT_CLUSTER_BACKEND'] = 'aws'
os.environ['SCT_REGION_NAME'] = 'us-east-1'
os.environ['SCT_N_DB_NODES'] = '2'
os.environ['SCT_INSTANCE_TYPE_DB'] = 'i3.large'
os.environ['SCT_AMI_ID_DB_SCYLLA'] = 'ami-b4f8b4cb'
conf = SCTConfiguration()
conf.verify_configuration()
self.assertEqual(conf.get('security_group_ids'), 'sg-c5e1f7a0')
def test_09_unknown_env(self):
os.environ['SCT_CLUSTER_BACKEND'] = 'docker'
os.environ['SCT_CONFIG_FILES'] = 'internal_test_data/unknown_param_in_config.yaml'
os.environ['SCT_WHAT_IS_THAT'] = 'just_made_this_up'
os.environ['SCT_WHAT_IS_THAT_2'] = 'what is this ?'
conf = SCTConfiguration()
with self.assertRaises(ValueError) as context:
conf.verify_configuration()
self.assertIn('SCT_WHAT_IS_THAT_2=what is this ?', str(context.exception))
self.assertIn('SCT_WHAT_IS_THAT=just_made_this_up', str(context.exception))
def conf_docs(output_format):
add_file_logger()
os.environ['SCT_CLUSTER_BACKEND'] = "aws" # just to pass SCTConfiguration() verification.
config_logger = logging.getLogger('sdcm.sct_config')
config_logger.setLevel(logging.ERROR)
if output_format == 'markdown':
click.secho(SCTConfiguration().dump_help_config_markdown())
elif output_format == 'yaml':
click.secho(SCTConfiguration().dump_help_config_yaml())
def test_15_new_scylla_repo(self):
centos_repo = 'https://s3.amazonaws.com/downloads.scylladb.com/enterprise/rpm/unstable/centos/9f724fedb93b4734fcfaec1156806921ff46e956-2bdfa9f7ef592edaf15e028faf3b7f695f39ebc1-525a0255f73d454f8f97f32b8bdd71c8dec35d3d-a6b2b2355c666b1893f702a587287da978aeec22/71/scylla.repo'
os.environ['SCT_CLUSTER_BACKEND'] = 'gce'
os.environ['SCT_SCYLLA_REPO'] = centos_repo
os.environ['SCT_NEW_SCYLLA_REPO'] = centos_repo
os.environ['SCT_USER_PREFIX'] = 'testing'
conf = SCTConfiguration()
conf.verify_configuration()
self.assertEqual(conf.get('target_upgrade_version'), '2019.1.1')
def test_16_oracle_scylla_version_eu_west_1(self):
ami_3_0_11 = "ami-0535c88b85b914499"
os.environ['SCT_CLUSTER_BACKEND'] = 'aws'
os.environ['SCT_ORACLE_SCYLLA_VERSION'] = "3.0.11"
os.environ['SCT_REGION_NAME'] = 'eu-west-1'
os.environ['SCT_CONFIG_FILES'] = 'internal_test_data/minimal_test_case.yaml'
conf = SCTConfiguration()
conf.verify_configuration()
self.assertEqual(conf.get('ami_id_db_oracle'), ami_3_0_11)
def test_can_create_basic_scylla_instance_definition_from_sct_config():
"""Test for azure_region_definition_builder"""
EnvConfig = namedtuple('EnvConfig', [
"SCT_CLUSTER_BACKEND", "SCT_TEST_ID", "SCT_CONFIG_FILES",
"SCT_AZURE_REGION_NAME", "SCT_N_DB_NODES", "SCT_USER_PREFIX",
"SCT_AZURE_IMAGE_DB", "SCT_N_LOADERS", "SCT_N_MONITORS_NODES"
])
env_config = EnvConfig(
SCT_CLUSTER_BACKEND="azure",
SCT_TEST_ID="example_test_id",
SCT_CONFIG_FILES=
f'["{Path(__file__).parent.absolute()}/azure_default_config.yaml"]',
SCT_AZURE_REGION_NAME="['eastus', 'easteu']",
SCT_N_DB_NODES="3 1",
SCT_USER_PREFIX="unit-test",
SCT_AZURE_IMAGE_DB="some_image_id",
SCT_N_LOADERS="2 0",
SCT_N_MONITORS_NODES="1")
os.environ.update(env_config._asdict())
config = SCTConfiguration()
tags = TestConfig.common_tags()
ssh_key = KeyStore().get_gce_ssh_key_pair()
prefix = config.get('user_prefix')
test_config = TestConfig()
builder = region_definition_builder.get_builder(params=config,
test_config=test_config)
region_definitions = builder.build_all_region_definitions()
instance_definition = InstanceDefinition(
name=f"{prefix}-db-node-eastus-1",
image_id=env_config.SCT_AZURE_IMAGE_DB,
type="Standard_L8s_v2",
user_name="scyllaadm",
root_disk_size=30,
tags=tags | {
"NodeType": "scylla-db",
"keep_action": "",
'NodeIndex': '1'
},
ssh_key=ssh_key)
assert len(region_definitions) == 2
actual_region_definition = region_definitions[0]
assert actual_region_definition.test_id == env_config.SCT_TEST_ID
assert actual_region_definition.backend == "azure"
assert actual_region_definition.region == "eastus"
# ignoring user_data in this validation
actual_region_definition.definitions[
0].user_data = instance_definition.user_data
# ssh_key is not shown, if actual looks the same as expected possibly ssh_key differ
assert instance_definition == actual_region_definition.definitions[0]
def test_16_oracle_scylla_version_us_east_1(self):
ami_3_0_11 = "ami-0a49c99b529429c18"
os.environ['SCT_CLUSTER_BACKEND'] = 'aws'
os.environ['SCT_ORACLE_SCYLLA_VERSION'] = "3.0.11"
os.environ['SCT_REGION_NAME'] = 'us-east-1'
os.environ['SCT_AMI_ID_DB_SCYLLA'] = 'ami-eae4f795'
os.environ['SCT_CONFIG_FILES'] = 'internal_test_data/minimal_test_case.yaml'
conf = SCTConfiguration()
conf.verify_configuration()
self.assertEqual(conf.get('ami_id_db_oracle'), ami_3_0_11)
def conf(config_file, backend):
if backend:
os.environ['SCT_CLUSTER_BACKEND'] = backend
os.environ['SCT_CONFIG_FILES'] = config_file
config = SCTConfiguration()
try:
config.verify_configuration()
except Exception as ex: # pylint: disable=broad-except
click.secho(str(ex), fg='red')
sys.exit(1)
else:
click.secho(config.dump_config(), fg='green')
sys.exit(0)
def collect_logs(test_id=None, logdir=None, backend=None, config_file=None):
add_file_logger()
from sdcm.logcollector import Collector
logging.getLogger("paramiko").setLevel(logging.CRITICAL)
if backend is None:
if os.environ.get('SCT_CLUSTER_BACKEND', None) is None:
os.environ['SCT_CLUSTER_BACKEND'] = 'aws'
else:
os.environ['SCT_CLUSTER_BACKEND'] = backend
if config_file and not os.environ.get('SCT_CONFIG_FILES', None):
os.environ['SCT_CONFIG_FILES'] = config_file
config = SCTConfiguration()
collector = Collector(test_id=test_id, params=config, test_dir=logdir)
collected_logs = collector.run()
table = PrettyTable(['Cluster set', 'Link'])
table.align = 'l'
for cluster_type, s3_link in collected_logs.items():
table.add_row([cluster_type, s3_link])
click.echo(table.get_string(title="Collected logs by test-id: {}".format(collector.test_id)))
def test_13_bool(self):
os.environ['SCT_CLUSTER_BACKEND'] = 'aws'
os.environ['SCT_STORE_RESULTS_IN_ELASTICSEARCH'] = 'False'
os.environ['SCT_CONFIG_FILES'] = 'internal_test_data/multi_region_dc_test_case.yaml'
conf = SCTConfiguration()
self.assertEqual(conf['store_results_in_elasticsearch'], False)
def get_builder(self, params: SCTConfiguration,
test_config: TestConfig) -> DefinitionBuilder:
"""Creates RegionDefinition for each region based on SCTConfiguration.
Prior use, must register builder for given backend."""
backend = params.get("cluster_backend")
return self._builder_classes[backend](params, test_config)
def test_13_bool(self):
os.environ['SCT_CLUSTER_BACKEND'] = 'aws'
os.environ['SCT_STORE_PERF_RESULTS'] = 'False'
os.environ[
'SCT_CONFIG_FILES'] = 'internal_test_data/multi_region_dc_test_case.yaml'
conf = SCTConfiguration()
self.assertEqual(conf['store_perf_results'], False)
def _prepare_docker_resource_setup(sct_config: SCTConfiguration):
db_node_setup = CloudNodesInfo(image_id=sct_config.get('docker_image'),
instance_type="docker",
node_amount=sct_config.get("n_db_nodes"),
post_behaviour=sct_config.get("post_behavior_db_nodes"))
loader_node_setup = CloudNodesInfo(image_id=sct_config.get('docker_image'),
instance_type="docker",
node_amount=sct_config.get("n_loaders"),
post_behaviour=sct_config.get("post_behavior_loader_nodes"))
monitor_node_setup = CloudNodesInfo(image_id=sct_config.get('docker_image'),
instance_type="docker",
node_amount=sct_config.get("n_monitor_nodes"),
post_behaviour=sct_config.get("post_behavior_monitor_nodes"))
cloud_setup = BaseCloudSetupDetails(db_node=db_node_setup, loader_node=loader_node_setup,
monitor_node=monitor_node_setup, backend=sct_config.get("cluster_backend"))
return cloud_setup
def sct_option(name, sct_name, **kwargs):
sct_opt = SCTConfiguration.get_config_option(sct_name)
multimple_use = kwargs.pop('multiple', False)
sct_opt.update(kwargs)
return click.option(name,
type=sct_opt['type'],
help=sct_opt['help'],
multiple=multimple_use)
def from_sct_config(cls, test_id: UUID, test_module_path: str,
sct_config: SCTConfiguration) -> TestRunWithHeartbeat:
# pylint: disable=too-many-locals
if cls.TESTRUN_INSTANCE:
raise ArgusTestRunError("Instance already initialized")
release_name = os.getenv("GIT_BRANCH", get_git_current_branch()).split("/")[-1]
if release_name not in cls.AVAILABLE_RELEASES:
raise ArgusTestRunError("Refusing to track a non-whitelisted branch", release_name, cls.AVAILABLE_RELEASES)
LOGGER.info("Preparing Test Details...")
test_group, *_ = test_module_path.split(".")
config_files = sct_config.get("config_files")
job_name = get_job_name()
job_url = get_job_url()
started_by = get_username()
details = TestDetails(name=get_test_name(), scm_revision_id=get_git_commit_id(), started_by=started_by,
build_job_name=job_name, build_job_url=job_url,
yaml_test_duration=sct_config.get("test_duration"), start_time=int(time.time()),
config_files=config_files, packages=[])
LOGGER.info("Preparing Resource Setup...")
backend = sct_config.get("cluster_backend")
raw_regions = sct_config.get("region_name") or sct_config.get("gce_datacenter") or "undefined_region"
regions = raw_regions.split()
primary_region = regions[0]
sct_runner_info = CloudInstanceDetails(public_ip=get_sct_runner_ip(), provider=backend,
region=primary_region, private_ip=get_my_ip())
cloud_setup = cls.BACKEND_MAP.get(backend, _prepare_unknown_resource_setup)(sct_config)
setup_details = TestResourcesSetup(sct_runner_host=sct_runner_info, region_name=regions,
cloud_setup=cloud_setup)
logs = TestLogs()
resources = TestResources()
results = TestResults(status=TestStatus.CREATED)
run_info = TestRunInfo(details=details, setup=setup_details, resources=resources, logs=logs, results=results)
LOGGER.info("Initializing TestRun...")
cls.TESTRUN_INSTANCE = TestRunWithHeartbeat(test_id=test_id, group=test_group, release_name=release_name,
assignee="",
run_info=run_info,
config=cls.config())
return cls.TESTRUN_INSTANCE
def _prepare_gce_resource_setup(sct_config: SCTConfiguration):
num_db_nodes, n_loaders = _get_node_amounts(sct_config)
db_node_setup = CloudNodesInfo(image_id=sct_config.get("gce_image_db"),
instance_type=sct_config.get("gce_instance_type_db"),
node_amount=num_db_nodes,
post_behaviour=sct_config.get("post_behavior_db_nodes"))
loader_node_setup = CloudNodesInfo(image_id=sct_config.get("gce_image_loader"),
instance_type=sct_config.get("gce_instance_type_loader"),
node_amount=n_loaders,
post_behaviour=sct_config.get("post_behavior_loader_nodes"))
monitor_node_setup = CloudNodesInfo(image_id=sct_config.get("gce_image_monitor"),
instance_type=sct_config.get("gce_instance_type_monitor"),
node_amount=sct_config.get("n_monitor_nodes"),
post_behaviour=sct_config.get("post_behavior_monitor_nodes"))
cloud_setup = GCESetupDetails(db_node=db_node_setup, loader_node=loader_node_setup,
monitor_node=monitor_node_setup)
return cloud_setup
def _prepare_unknown_resource_setup(sct_config: SCTConfiguration):
LOGGER.error("Unknown backend encountered: %s", sct_config.get("cluster_backend"))
db_node_setup = CloudNodesInfo(image_id="UNKNOWN",
instance_type="UNKNOWN",
node_amount=-1,
post_behaviour="UNKNOWN")
loader_node_setup = CloudNodesInfo(image_id="UNKNOWN",
instance_type="UNKNOWN",
node_amount=-1,
post_behaviour="UNKNOWN")
monitor_node_setup = CloudNodesInfo(image_id="UNKNOWN",
instance_type="UNKNOWN",
node_amount=-1,
post_behaviour="UNKNOWN")
cloud_setup = BaseCloudSetupDetails(db_node=db_node_setup, loader_node=loader_node_setup,
monitor_node=monitor_node_setup, backend=sct_config.get("cluster_backend"))
return cloud_setup
def __new__(cls, params: SCTConfiguration):
backend = params.get('cluster_backend')
try:
importlib.import_module(cls.__module__.replace(
'.common.', f'.{backend}.'),
package=None)
except ModuleNotFoundError:
pass
target_class = cls._cluster_backend_mapping.get(
backend, SCTProvisionLayout)
return object.__new__(target_class)
