def get_docker_instances_by_testid(self):
instances = list_instances_gce({"TestId": self.test_id}, running=True)
filtered_instances = filter_gce_instances_by_type(instances)
for instance in filtered_instances['db_nodes']:
self.db_cluster.append(
CollectingNode(name=instance.name,
ssh_login_info={
"hostname": instance.public_ips[0],
"user": '******',
"key_file":
self.params['user_credentials_path']
},
instance=instance,
global_ip=instance.public_ips[0]))
self.monitor_set.append(
CollectingNode(name=f"monitor-node-{self.test_id}-0",
global_ip='127.0.0.1',
grafana_ip=get_docker_bridge_gateway(
LocalCmdRunner())))
for instance in filtered_instances['loader_nodes']:
self.loader_set.append(
CollectingNode(name=instance.name,
ssh_login_info={
"hostname": instance.public_ips[0],
"user": '******',
"key_file":
self.params['user_credentials_path']
},
instance=instance,
global_ip=instance.public_ips[0]))
def _get_instances(self, dc_idx):
test_id = cluster.Setup.test_id()
if not test_id:
raise ValueError(
"test_id should be configured for using reuse_cluster")
instances_by_nodetype = list_instances_gce(tags_dict={
'TestId': test_id,
'NodeType': self.node_type
})
instances_by_zone = self._get_instances_by_prefix(dc_idx)
instances = []
attr_name = 'public_ips' if self._node_public_ips else 'private_ips'
for node_zone in instances_by_zone:
# Filter nodes by zone and by ip addresses
if not getattr(node_zone, attr_name):
continue
for node_nodetype in instances_by_nodetype:
if node_zone.uuid == node_nodetype.uuid:
instances.append(node_zone)
def sort_by_index(node):
metadata = gce_meta_to_dict(node.extra['metadata'])
return metadata.get('NodeIndex', 0)
instances = sorted(instances, key=sort_by_index)
return instances
def list_sct_runners(cls) -> list[SctRunnerInfo]:
sct_runners = []
for instance in list_instances_gce(
tags_dict={"NodeType": cls.NODE_TYPE}, verbose=True):
tags = gce_meta_to_dict(instance.extra["metadata"])
region = instance.extra["zone"].name
if launch_time := tags.get("launch_time"):
try:
launch_time = datetime_from_formatted(
date_string=launch_time)
except ValueError as exc:
LOGGER.warning("Value of `launch_time' tag is invalid: %s",
exc)
launch_time = None
if not launch_time:
create_time = instance.extra["creationTimestamp"]
LOGGER.info(
"`launch_time' tag is empty or invalid, fallback to creation time: %s",
create_time)
launch_time = datetime.datetime.fromisoformat(create_time)
sct_runners.append(
SctRunnerInfo(
sct_runner_class=cls,
cloud_service_instance=None, # we don't need it for GCE
region_az=region,
instance=instance,
instance_name=instance.name,
public_ips=instance.public_ips,
launch_time=launch_time,
keep=tags.get("keep"),
keep_action=tags.get("keep_action"),
))
def get_running_instances_for_email_report(test_id: str, ip_filter: str = None): # pylint: disable=too-many-locals
"""Get running instances left after testrun
Get all running instances leff after testrun is done.
If ip_filter is provided, the instance with that ip will be
filtered out of the results.
:param test_id: testrun test id
:type test_id: str
:param ip_filter: the ip of the sct test runner to be excluded
from the report, since it will be terminated on test end
:type ip_filter: str
:returns: list of instances left running after test run
in format:
[
["name", "public ip addrs", "state", "cloud", "region"]
]
:rtype: {list}
"""
nodes = []
tags = {"TestId": test_id, }
instances = list_instances_aws(tags_dict=tags, group_as_region=True, running=True)
for region in instances:
for instance in instances[region]:
name = [tag['Value'] for tag in instance['Tags'] if tag['Key'] == 'Name']
public_ip_addr = instance.get('PublicIpAddress', 'N/A')
if public_ip_addr != ip_filter:
nodes.append([name[0],
instance.get('PublicIpAddress', 'N/A'),
instance['State']['Name'],
"aws",
region])
instances = list_instances_gce(tags_dict=tags, running=True)
for instance in instances:
public_ips = instance.public_ips
if ip_filter not in public_ips:
nodes.append([instance.name,
", ".join(public_ips) if None not in instance.public_ips else "N/A",
instance.state,
"gce",
instance.extra["zone"].name])
resources = list_resources_docker(tags_dict=tags, running=True, group_as_builder=True)
for builder_name, containers in resources.get("containers", {}).items():
for container in containers:
container.reload()
nodes.append([container.name,
container.attrs["NetworkSettings"]["IPAddress"],
container.status,
"docker container",
builder_name])
for builder_name, images in resources.get("images", {}).items():
for image in images:
nodes.append([", ".join(image.tags),
"N/A",
"N/A",
"docker image",
builder_name])
return nodes
def get_running_instances_for_email_report(test_id):
"""Get running instances left after testrun
Get all running instances leff after testrun is done
:param test_id: testrun test id
:type test_id: str
:returns: list of instances left running after test run
in format:
[
["name", "public ip addrs", "state", "cloud", "region"]
]
:rtype: {list}
"""
nodes = []
tags = {
"TestId": test_id,
}
instances = list_instances_aws(tags_dict=tags,
group_as_region=True,
running=True)
for region in instances:
for instance in instances[region]:
name = [
tag['Value'] for tag in instance['Tags']
if tag['Key'] == 'Name'
]
nodes.append([
name[0],
instance.get('PublicIpAddress', 'N/A'),
instance['State']['Name'], "aws", region
])
instances = list_instances_gce(tags_dict=tags, running=True)
for instance in instances:
nodes.append([
instance.name, ", ".join(instance.public_ips)
if None not in instance.public_ips else "N/A", instance.state,
"gce", instance.extra["zone"].name
])
resources = list_resources_docker(tags_dict=tags,
running=True,
group_as_builder=True)
for builder_name, containers in resources.get("containers", {}).items():
for container in containers:
container.reload()
nodes.append([
container.name,
container.attrs["NetworkSettings"]["IPAddress"],
container.status, "docker container", builder_name
])
for builder_name, images in resources.get("images", {}).items():
for image in images:
nodes.append([
", ".join(image.tags), "N/A", "N/A", "docker image",
builder_name
])
return nodes
def get_gce_instances_by_testid(self):
instances = list_instances_gce({"TestId": self.test_id}, running=True)
filtered_instances = filter_gce_instances_by_type(instances)
for instance in filtered_instances['db_nodes']:
self.db_cluster.append(
CollectingNode(name=instance.name,
ssh_login_info={
"hostname": instance.public_ips[0],
"user": self.params['gce_image_username'],
"key_file":
self.params['user_credentials_path']
},
instance=instance,
global_ip=instance.public_ips[0],
tags={
**self.tags,
"NodeType": "scylla-db",
}))
for instance in filtered_instances['monitor_nodes']:
self.monitor_set.append(
CollectingNode(name=instance.name,
ssh_login_info={
"hostname": instance.public_ips[0],
"user": self.params['gce_image_username'],
"key_file":
self.params['user_credentials_path']
},
instance=instance,
global_ip=instance.public_ips[0],
tags={
**self.tags,
"NodeType": "monitor",
}))
for instance in filtered_instances['loader_nodes']:
self.loader_set.append(
CollectingNode(name=instance.name,
ssh_login_info={
"hostname": instance.public_ips[0],
"user": self.params['gce_image_username'],
"key_file":
self.params['user_credentials_path']
},
instance=instance,
global_ip=instance.public_ips[0],
tags={
**self.tags,
"NodeType": "loader",
}))
def get_gce_instances(self):
gce_instances = list_instances_gce(verbose=True)
for instance in gce_instances:
tags = gce_meta_to_dict(instance.extra['metadata'])
cloud_instance = CloudInstance(
cloud="gce",
name=instance.name,
region_az=instance.extra["zone"].name,
state=instance.state,
lifecycle="spot" if instance.extra["scheduling"]["preemptible"] else "on-demand",
instance_type=instance.size,
owner=tags.get('RunByUser', 'N/A') if tags else "N/A",
create_time=instance.extra['creationTimestamp'],
)
self.instances["gce"].append(cloud_instance)
self.all_instances += self.instances["gce"]
def _get_instances(self, dc_idx):
if not self.monitor_id:
raise ValueError("'monitor_id' must exist")
instances_by_nodetype = list_instances_gce(
tags_dict={'MonitorId': self.monitor_id, 'NodeType': self.node_type})
instances_by_zone = self._get_instances_by_prefix(dc_idx)
instances = []
attr_name = 'public_ips' if self._node_public_ips else 'private_ips'
for node_zone in instances_by_zone:
# Filter nodes by zone and by ip addresses
if not getattr(node_zone, attr_name):
continue
for node_nodetype in instances_by_nodetype:
if node_zone.uuid == node_nodetype.uuid:
instances.append(node_zone)
def sort_by_index(node):
metadata = gce_meta_to_dict(node.extra['metadata'])
return metadata.get('NodeIndex', 0)
instances = sorted(instances, key=sort_by_index)
return instances
def get_running_instances_for_email_report(test_id):
"""Get running instances left after testrun
Get all running instances leff after testrun is done
:param test_id: testrun test id
:type test_id: str
:returns: list of instances left running after test run
in format:
[
["name", "public ip addrs", "state", "cloud", "region"]
]
:rtype: {list}
"""
nodes = []
instances = list_instances_aws(tags_dict={'TestId': test_id},
group_as_region=True,
running=True)
for region in instances:
for instance in instances[region]:
name = [
tag['Value'] for tag in instance['Tags']
if tag['Key'] == 'Name'
]
nodes.append([
name[0],
instance.get('PublicIpAddress', 'N/A'),
instance['State']['Name'], "aws", region
])
instances = list_instances_gce(tags_dict={"TestId": test_id}, running=True)
for instance in instances:
nodes.append([
instance.name, ", ".join(instance.public_ips)
if None not in instance.public_ips else "N/A", instance.state,
"gce", instance.extra["zone"].name
])
return nodes
def list_resources(ctx, user, test_id, get_all, get_all_running, verbose):
# pylint: disable=too-many-locals,too-many-arguments,too-many-branches,too-many-statements
params = dict()
if user:
params['RunByUser'] = user
if test_id:
params['TestId'] = test_id
if all([not get_all, not get_all_running, not user, not test_id]):
click.echo(list_resources.get_help(ctx))
if get_all_running:
table_header = ["Name", "Region-AZ", "PublicIP", "TestId", "RunByUser", "LaunchTime"]
else:
table_header = ["Name", "Region-AZ", "State", "TestId", "RunByUser", "LaunchTime"]
click.secho("Checking AWS EC2...", fg='green')
aws_instances = list_instances_aws(tags_dict=params, running=get_all_running, verbose=verbose)
if aws_instances:
aws_table = PrettyTable(table_header)
aws_table.align = "l"
aws_table.sortby = 'LaunchTime'
for instance in aws_instances:
tags = aws_tags_to_dict(instance.get('Tags'))
name = tags.get("Name", "N/A")
test_id = tags.get("TestId", "N/A")
run_by_user = tags.get("RunByUser", "N/A")
aws_table.add_row([
name,
instance['Placement']['AvailabilityZone'],
instance.get('PublicIpAddress', 'N/A') if get_all_running else instance['State']['Name'],
test_id,
run_by_user,
instance['LaunchTime'].ctime()])
click.echo(aws_table.get_string(title="Instances used on AWS"))
else:
click.secho("Nothing found for selected filters in AWS!", fg="yellow")
click.secho("Checking AWS Elastic IPs...", fg='green')
elastic_ips_aws = list_elastic_ips_aws(tags_dict=params, verbose=verbose)
if elastic_ips_aws:
aws_table = PrettyTable(["AllocationId", "PublicIP", "TestId", "RunByUser", "InstanceId (attached to)"])
aws_table.align = "l"
aws_table.sortby = 'AllocationId'
for eip in elastic_ips_aws:
tags = aws_tags_to_dict(eip.get('Tags'))
test_id = tags.get("TestId", "N/A")
run_by_user = tags.get("RunByUser", "N/A")
aws_table.add_row([
eip['AllocationId'],
eip['PublicIp'],
test_id,
run_by_user,
eip.get('InstanceId', 'N/A')])
click.echo(aws_table.get_string(title="EIPs used on AWS"))
else:
click.secho("No elastic ips found for selected filters in AWS!", fg="yellow")
click.secho("Checking GCE...", fg='green')
gce_instances = list_instances_gce(tags_dict=params, running=get_all_running, verbose=verbose)
if gce_instances:
gce_table = PrettyTable(table_header)
gce_table.align = "l"
gce_table.sortby = 'LaunchTime'
for instance in gce_instances:
tags = gce_meta_to_dict(instance.extra['metadata'])
public_ips = ", ".join(instance.public_ips) if None not in instance.public_ips else "N/A"
gce_table.add_row([instance.name,
instance.extra["zone"].name,
public_ips if get_all_running else instance.state,
tags.get('TestId', 'N/A') if tags else "N/A",
tags.get('RunByUser', 'N/A') if tags else "N/A",
instance.extra['creationTimestamp'],
])
click.echo(gce_table.get_string(title="Resources used on GCE"))
else:
click.secho("Nothing found for selected filters in GCE!", fg="yellow")
def get_gce_instances(self):
gce_instances = list_instances_gce(verbose=True)
self["gce"] = [GCEInstance(instance) for instance in gce_instances]
self.all.extend(self["gce"])
def list_resources(ctx, user, test_id, get_all, get_all_running, verbose):
# pylint: disable=too-many-locals,too-many-arguments,too-many-branches,too-many-statements
add_file_logger()
params = dict()
if user:
params['RunByUser'] = user
if test_id:
params['TestId'] = test_id
if all([not get_all, not get_all_running, not user, not test_id]):
click.echo(list_resources.get_help(ctx))
if get_all_running:
table_header = ["Name", "Region-AZ", "PublicIP", "TestId", "RunByUser", "LaunchTime"]
else:
table_header = ["Name", "Region-AZ", "State", "TestId", "RunByUser", "LaunchTime"]
click.secho("Checking AWS EC2...", fg='green')
aws_instances = list_instances_aws(tags_dict=params, running=get_all_running, verbose=verbose)
if aws_instances:
aws_table = PrettyTable(table_header)
aws_table.align = "l"
aws_table.sortby = 'LaunchTime'
for instance in aws_instances:
tags = aws_tags_to_dict(instance.get('Tags'))
name = tags.get("Name", "N/A")
test_id = tags.get("TestId", "N/A")
run_by_user = tags.get("RunByUser", "N/A")
aws_table.add_row([
name,
instance['Placement']['AvailabilityZone'],
instance.get('PublicIpAddress', 'N/A') if get_all_running else instance['State']['Name'],
test_id,
run_by_user,
instance['LaunchTime'].ctime()])
click.echo(aws_table.get_string(title="Instances used on AWS"))
else:
click.secho("Nothing found for selected filters in AWS!", fg="yellow")
click.secho("Checking AWS Elastic IPs...", fg='green')
elastic_ips_aws = list_elastic_ips_aws(tags_dict=params, verbose=verbose)
if elastic_ips_aws:
aws_table = PrettyTable(["AllocationId", "PublicIP", "TestId", "RunByUser", "InstanceId (attached to)"])
aws_table.align = "l"
aws_table.sortby = 'AllocationId'
for eip in elastic_ips_aws:
tags = aws_tags_to_dict(eip.get('Tags'))
test_id = tags.get("TestId", "N/A")
run_by_user = tags.get("RunByUser", "N/A")
aws_table.add_row([
eip['AllocationId'],
eip['PublicIp'],
test_id,
run_by_user,
eip.get('InstanceId', 'N/A')])
click.echo(aws_table.get_string(title="EIPs used on AWS"))
else:
click.secho("No elastic ips found for selected filters in AWS!", fg="yellow")
click.secho("Checking GCE...", fg='green')
gke_clusters = list_clusters_gke(tags_dict=params, verbose=verbose)
if gke_clusters:
gke_table = PrettyTable(["Name", "Region-AZ", "TestId", "RunByUser", "CreateTime"])
gke_table.align = "l"
gke_table.sortby = 'CreateTime'
for cluster in gke_clusters:
tags = gce_meta_to_dict(cluster.extra['metadata'])
gke_table.add_row([cluster.name,
cluster.zone,
tags.get('TestId', 'N/A') if tags else "N/A",
tags.get('RunByUser', 'N/A') if tags else "N/A",
cluster.cluster_info['createTime'],
])
click.echo(gke_table.get_string(title="GKE clusters"))
else:
click.secho("Nothing found for selected filters in GKE!", fg="yellow")
gce_instances = list_instances_gce(tags_dict=params, running=get_all_running, verbose=verbose)
if gce_instances:
gce_table = PrettyTable(table_header)
gce_table.align = "l"
gce_table.sortby = 'LaunchTime'
for instance in gce_instances:
tags = gce_meta_to_dict(instance.extra['metadata'])
public_ips = ", ".join(instance.public_ips) if None not in instance.public_ips else "N/A"
gce_table.add_row([instance.name,
instance.extra["zone"].name,
public_ips if get_all_running else instance.state,
tags.get('TestId', 'N/A') if tags else "N/A",
tags.get('RunByUser', 'N/A') if tags else "N/A",
instance.extra['creationTimestamp'],
])
click.echo(gce_table.get_string(title="Resources used on GCE"))
else:
click.secho("Nothing found for selected filters in GCE!", fg="yellow")
click.secho("Checking Docker...", fg="green")
docker_resources = \
list_resources_docker(tags_dict=params, running=get_all_running, group_as_builder=True, verbose=verbose)
if any(docker_resources.values()):
if docker_resources.get("containers"):
docker_table = PrettyTable(["Name", "Builder", "Public IP" if get_all_running else "Status",
"TestId", "RunByUser", "Created"])
docker_table.align = "l"
docker_table.sortby = "Created"
for builder_name, docker_containers in docker_resources["containers"].items():
for container in docker_containers:
container.reload()
docker_table.add_row([
container.name,
builder_name,
container.attrs["NetworkSettings"]["IPAddress"] if get_all_running else container.status,
container.labels.get("TestId", "N/A"),
container.labels.get("RunByUser", "N/A"),
container.attrs.get("Created", "N/A"),
])
click.echo(docker_table.get_string(title="Containers used on Docker"))
if docker_resources.get("images"):
docker_table = PrettyTable(["Name", "Builder", "TestId", "RunByUser", "Created"])
docker_table.align = "l"
docker_table.sortby = "Created"
for builder_name, docker_images in docker_resources["images"].items():
for image in docker_images:
image.reload()
for tag in image.tags:
docker_table.add_row([
tag,
builder_name,
image.labels.get("TestId", "N/A"),
image.labels.get("RunByUser", "N/A"),
image.attrs.get("Created", "N/A"),
])
click.echo(docker_table.get_string(title="Images used on Docker"))
else:
click.secho("Nothing found for selected filters in Docker!", fg="yellow")
def list_resources(ctx, user, test_id, get_all, get_all_running):
params = dict()
if get_all or get_all_running:
params = None
elif user:
params['RunByUser'] = user
elif test_id:
params['TestId'] = test_id
else:
click.echo(list_resources.get_help(ctx))
if get_all_running:
table_header = [
"Name", "Region-AZ", "PublicIP", "TestId", "RunByUser",
"LaunchTime"
]
else:
table_header = [
"Name", "Region-AZ", "State", "TestId", "RunByUser", "LaunchTime"
]
click.secho("Checking EC2...", fg='green')
aws_instances = list_instances_aws(tags_dict=params,
running=get_all_running)
click.secho("Checking AWS EC2...", fg='green')
if aws_instances:
aws_table = PrettyTable(table_header)
aws_table.align = "l"
aws_table.sortby = 'LaunchTime'
for instance in aws_instances:
tags = aws_tags_to_dict(instance.get('Tags'))
name = tags.get("Name", "N/A")
test_id = tags.get("TestId", "N/A")
run_by_user = tags.get("RunByUser", "N/A")
aws_table.add_row([
name, instance['Placement']['AvailabilityZone'],
instance.get('PublicIpAddress', 'N/A')
if get_all_running else instance['State']['Name'], test_id,
run_by_user, instance['LaunchTime'].ctime()
])
click.echo(aws_table.get_string(title="Resources used on AWS"))
else:
click.secho("Nothing found for selected filters in AWS!", fg="yellow")
click.secho("Checking GCE...", fg='green')
gce_instances = list_instances_gce(tags_dict=params,
running=get_all_running)
if gce_instances:
gce_table = PrettyTable(table_header)
gce_table.align = "l"
gce_table.sortby = 'LaunchTime'
for instance in gce_instances:
tags = gce_meta_to_dict(instance.extra['metadata'])
public_ips = ", ".join(
instance.public_ips
) if None not in instance.public_ips else "N/A"
gce_table.add_row([
instance.name,
instance.extra["zone"].name,
public_ips if get_all_running else instance.state,
tags.get('TestId', 'N/A') if tags else "N/A",
tags.get('RunByUser', 'N/A') if tags else "N/A",
instance.extra['creationTimestamp'],
])
click.echo(gce_table.get_string(title="Resources used on GCE"))
else:
click.secho("Nothing found for selected filters in GCE!", fg="yellow")
