def get_aws_instances_by_testid(self):
instances = list_instances_aws({"TestId": self.test_id}, running=True)
filtered_instances = filter_aws_instances_by_type(instances)
for instance in filtered_instances['db_nodes']:
name = [
tag['Value'] for tag in instance['Tags']
if tag['Key'] == 'Name'
]
self.db_cluster.append(
CollectingNode(name=name[0],
ssh_login_info={
"hostname": instance['PublicIpAddress'],
"user": self.params['ami_db_scylla_user'],
"key_file":
self.params['user_credentials_path']
},
instance=instance,
global_ip=instance['PublicIpAddress'],
tags={
**self.tags,
"NodeType": "scylla-db",
}))
for instance in filtered_instances['monitor_nodes']:
name = [
tag['Value'] for tag in instance['Tags']
if tag['Key'] == 'Name'
]
self.monitor_set.append(
CollectingNode(name=name[0],
ssh_login_info={
"hostname": instance['PublicIpAddress'],
"user": self.params['ami_monitor_user'],
"key_file":
self.params['user_credentials_path']
},
instance=instance,
global_ip=instance['PublicIpAddress'],
tags={
**self.tags,
"NodeType": "monitor",
}))
for instance in filtered_instances['loader_nodes']:
name = [
tag['Value'] for tag in instance['Tags']
if tag['Key'] == 'Name'
]
self.loader_set.append(
CollectingNode(name=name[0],
ssh_login_info={
"hostname": instance['PublicIpAddress'],
"user": self.params['ami_loader_user'],
"key_file":
self.params['user_credentials_path']
},
instance=instance,
global_ip=instance['PublicIpAddress'],
tags={
**self.tags,
"NodeType": "loader",
}))
def _get_instances(self, dc_idx):
test_id = cluster.Setup.test_id()
if not test_id:
raise ValueError(
"test_id should be configured for using reuse_cluster")
ec2 = ec2_client.EC2Client(region_name=self.region_names[dc_idx],
spot_max_price_percentage=self.params.get(
'spot_max_price', default=0.60))
results = list_instances_aws(tags_dict={
'TestId': test_id,
'NodeType': self.node_type
},
region_name=self.region_names[dc_idx],
group_as_region=True)
instances = results[self.region_names[dc_idx]]
def sort_by_index(item):
for tag in item['Tags']:
if tag['Key'] == 'NodeIndex':
return tag['Value']
else:
return '0'
instances = sorted(instances, key=sort_by_index)
return [
ec2.get_instance(instance['InstanceId']) for instance in instances
]
def list_sct_runners(cls) -> list[SctRunnerInfo]:
all_instances = list_instances_aws(
tags_dict={"NodeType": cls.NODE_TYPE},
group_as_region=True,
verbose=True)
sct_runners = []
for region_name, instances in all_instances.items():
client = boto3.client("ec2", region_name=region_name)
for instance in instances:
tags = aws_tags_to_dict(instance.get("Tags"))
instance_name = instance["InstanceId"]
if "Name" in tags:
instance_name = f"{tags['Name']} ({instance_name})"
sct_runners.append(
SctRunnerInfo(
sct_runner_class=cls,
cloud_service_instance=client,
region_az=instance["Placement"]["AvailabilityZone"],
instance=instance,
instance_name=instance_name,
public_ips=[
instance.get("PublicIpAddress"),
],
launch_time=instance["LaunchTime"],
keep=tags.get("keep"),
keep_action=tags.get("keep_action"),
))
return sct_runners
def _get_instances(self, dc_idx):
if not self.monitor_id:
raise ValueError("'monitor_id' must exist")
ec2 = ec2_client.EC2ClientWrapper(
region_name=self.region_names[dc_idx],
spot_max_price_percentage=self.params.get('spot_max_price'))
results = list_instances_aws(
tags_dict={
'MonitorId': self.monitor_id,
'NodeType': self.node_type
},
region_name=self.region_names[dc_idx],
group_as_region=True,
)
instances = results[self.region_names[dc_idx]]
def sort_by_index(item):
for tag in item['Tags']:
if tag['Key'] == 'NodeIndex':
return tag['Value']
return '0'
instances = sorted(instances, key=sort_by_index)
return [
ec2.get_instance(instance['InstanceId']) for instance in instances
]
def get_running_instances_for_email_report(test_id: str, ip_filter: str = None): # pylint: disable=too-many-locals
"""Get running instances left after testrun
Get all running instances leff after testrun is done.
If ip_filter is provided, the instance with that ip will be
filtered out of the results.
:param test_id: testrun test id
:type test_id: str
:param ip_filter: the ip of the sct test runner to be excluded
from the report, since it will be terminated on test end
:type ip_filter: str
:returns: list of instances left running after test run
in format:
[
["name", "public ip addrs", "state", "cloud", "region"]
]
:rtype: {list}
"""
nodes = []
tags = {"TestId": test_id, }
instances = list_instances_aws(tags_dict=tags, group_as_region=True, running=True)
for region in instances:
for instance in instances[region]:
name = [tag['Value'] for tag in instance['Tags'] if tag['Key'] == 'Name']
public_ip_addr = instance.get('PublicIpAddress', 'N/A')
if public_ip_addr != ip_filter:
nodes.append([name[0],
instance.get('PublicIpAddress', 'N/A'),
instance['State']['Name'],
"aws",
region])
instances = list_instances_gce(tags_dict=tags, running=True)
for instance in instances:
public_ips = instance.public_ips
if ip_filter not in public_ips:
nodes.append([instance.name,
", ".join(public_ips) if None not in instance.public_ips else "N/A",
instance.state,
"gce",
instance.extra["zone"].name])
resources = list_resources_docker(tags_dict=tags, running=True, group_as_builder=True)
for builder_name, containers in resources.get("containers", {}).items():
for container in containers:
container.reload()
nodes.append([container.name,
container.attrs["NetworkSettings"]["IPAddress"],
container.status,
"docker container",
builder_name])
for builder_name, images in resources.get("images", {}).items():
for image in images:
nodes.append([", ".join(image.tags),
"N/A",
"N/A",
"docker image",
builder_name])
return nodes
def get_running_instances_for_email_report(test_id):
"""Get running instances left after testrun
Get all running instances leff after testrun is done
:param test_id: testrun test id
:type test_id: str
:returns: list of instances left running after test run
in format:
[
["name", "public ip addrs", "state", "cloud", "region"]
]
:rtype: {list}
"""
nodes = []
tags = {
"TestId": test_id,
}
instances = list_instances_aws(tags_dict=tags,
group_as_region=True,
running=True)
for region in instances:
for instance in instances[region]:
name = [
tag['Value'] for tag in instance['Tags']
if tag['Key'] == 'Name'
]
nodes.append([
name[0],
instance.get('PublicIpAddress', 'N/A'),
instance['State']['Name'], "aws", region
])
instances = list_instances_gce(tags_dict=tags, running=True)
for instance in instances:
nodes.append([
instance.name, ", ".join(instance.public_ips)
if None not in instance.public_ips else "N/A", instance.state,
"gce", instance.extra["zone"].name
])
resources = list_resources_docker(tags_dict=tags,
running=True,
group_as_builder=True)
for builder_name, containers in resources.get("containers", {}).items():
for container in containers:
container.reload()
nodes.append([
container.name,
container.attrs["NetworkSettings"]["IPAddress"],
container.status, "docker container", builder_name
])
for builder_name, images in resources.get("images", {}).items():
for image in images:
nodes.append([
", ".join(image.tags), "N/A", "N/A", "docker image",
builder_name
])
return nodes
def get_aws_instances(self):
aws_instances = list_instances_aws(verbose=True)
for instance in aws_instances:
tags = aws_tags_to_dict(instance.get('Tags'))
cloud_instance = CloudInstance(
cloud="aws",
name=tags.get("Name", "N/A"),
region_az=instance["Placement"]["AvailabilityZone"],
state=instance["State"]["Name"],
lifecycle="spot" if instance.get("SpotInstanceRequestId") else "on-demand",
instance_type=instance["InstanceType"],
owner=tags.get("RunByUser", tags.get("Owner", "N/A")),
create_time=instance['LaunchTime'].ctime(),
)
self.instances["aws"].append(cloud_instance)
self.all_instances += self.instances["aws"]
def get_running_instances_for_email_report(test_id):
"""Get running instances left after testrun
Get all running instances leff after testrun is done
:param test_id: testrun test id
:type test_id: str
:returns: list of instances left running after test run
in format:
[
["name", "public ip addrs", "state", "cloud", "region"]
]
:rtype: {list}
"""
nodes = []
instances = list_instances_aws(tags_dict={'TestId': test_id},
group_as_region=True,
running=True)
for region in instances:
for instance in instances[region]:
name = [
tag['Value'] for tag in instance['Tags']
if tag['Key'] == 'Name'
]
nodes.append([
name[0],
instance.get('PublicIpAddress', 'N/A'),
instance['State']['Name'], "aws", region
])
instances = list_instances_gce(tags_dict={"TestId": test_id}, running=True)
for instance in instances:
nodes.append([
instance.name, ", ".join(instance.public_ips)
if None not in instance.public_ips else "N/A", instance.state,
"gce", instance.extra["zone"].name
])
return nodes
def list_resources(ctx, user, test_id, get_all, get_all_running, verbose):
# pylint: disable=too-many-locals,too-many-arguments,too-many-branches,too-many-statements
params = dict()
if user:
params['RunByUser'] = user
if test_id:
params['TestId'] = test_id
if all([not get_all, not get_all_running, not user, not test_id]):
click.echo(list_resources.get_help(ctx))
if get_all_running:
table_header = ["Name", "Region-AZ", "PublicIP", "TestId", "RunByUser", "LaunchTime"]
else:
table_header = ["Name", "Region-AZ", "State", "TestId", "RunByUser", "LaunchTime"]
click.secho("Checking AWS EC2...", fg='green')
aws_instances = list_instances_aws(tags_dict=params, running=get_all_running, verbose=verbose)
if aws_instances:
aws_table = PrettyTable(table_header)
aws_table.align = "l"
aws_table.sortby = 'LaunchTime'
for instance in aws_instances:
tags = aws_tags_to_dict(instance.get('Tags'))
name = tags.get("Name", "N/A")
test_id = tags.get("TestId", "N/A")
run_by_user = tags.get("RunByUser", "N/A")
aws_table.add_row([
name,
instance['Placement']['AvailabilityZone'],
instance.get('PublicIpAddress', 'N/A') if get_all_running else instance['State']['Name'],
test_id,
run_by_user,
instance['LaunchTime'].ctime()])
click.echo(aws_table.get_string(title="Instances used on AWS"))
else:
click.secho("Nothing found for selected filters in AWS!", fg="yellow")
click.secho("Checking AWS Elastic IPs...", fg='green')
elastic_ips_aws = list_elastic_ips_aws(tags_dict=params, verbose=verbose)
if elastic_ips_aws:
aws_table = PrettyTable(["AllocationId", "PublicIP", "TestId", "RunByUser", "InstanceId (attached to)"])
aws_table.align = "l"
aws_table.sortby = 'AllocationId'
for eip in elastic_ips_aws:
tags = aws_tags_to_dict(eip.get('Tags'))
test_id = tags.get("TestId", "N/A")
run_by_user = tags.get("RunByUser", "N/A")
aws_table.add_row([
eip['AllocationId'],
eip['PublicIp'],
test_id,
run_by_user,
eip.get('InstanceId', 'N/A')])
click.echo(aws_table.get_string(title="EIPs used on AWS"))
else:
click.secho("No elastic ips found for selected filters in AWS!", fg="yellow")
click.secho("Checking GCE...", fg='green')
gce_instances = list_instances_gce(tags_dict=params, running=get_all_running, verbose=verbose)
if gce_instances:
gce_table = PrettyTable(table_header)
gce_table.align = "l"
gce_table.sortby = 'LaunchTime'
for instance in gce_instances:
tags = gce_meta_to_dict(instance.extra['metadata'])
public_ips = ", ".join(instance.public_ips) if None not in instance.public_ips else "N/A"
gce_table.add_row([instance.name,
instance.extra["zone"].name,
public_ips if get_all_running else instance.state,
tags.get('TestId', 'N/A') if tags else "N/A",
tags.get('RunByUser', 'N/A') if tags else "N/A",
instance.extra['creationTimestamp'],
])
click.echo(gce_table.get_string(title="Resources used on GCE"))
else:
click.secho("Nothing found for selected filters in GCE!", fg="yellow")
def get_aws_instances(self):
aws_instances = list_instances_aws(verbose=True)
self["aws"] = [AWSInstance(instance) for instance in aws_instances]
self.all.extend(self["aws"])
def list_resources(ctx, user, test_id, get_all, get_all_running, verbose):
# pylint: disable=too-many-locals,too-many-arguments,too-many-branches,too-many-statements
add_file_logger()
params = dict()
if user:
params['RunByUser'] = user
if test_id:
params['TestId'] = test_id
if all([not get_all, not get_all_running, not user, not test_id]):
click.echo(list_resources.get_help(ctx))
if get_all_running:
table_header = ["Name", "Region-AZ", "PublicIP", "TestId", "RunByUser", "LaunchTime"]
else:
table_header = ["Name", "Region-AZ", "State", "TestId", "RunByUser", "LaunchTime"]
click.secho("Checking AWS EC2...", fg='green')
aws_instances = list_instances_aws(tags_dict=params, running=get_all_running, verbose=verbose)
if aws_instances:
aws_table = PrettyTable(table_header)
aws_table.align = "l"
aws_table.sortby = 'LaunchTime'
for instance in aws_instances:
tags = aws_tags_to_dict(instance.get('Tags'))
name = tags.get("Name", "N/A")
test_id = tags.get("TestId", "N/A")
run_by_user = tags.get("RunByUser", "N/A")
aws_table.add_row([
name,
instance['Placement']['AvailabilityZone'],
instance.get('PublicIpAddress', 'N/A') if get_all_running else instance['State']['Name'],
test_id,
run_by_user,
instance['LaunchTime'].ctime()])
click.echo(aws_table.get_string(title="Instances used on AWS"))
else:
click.secho("Nothing found for selected filters in AWS!", fg="yellow")
click.secho("Checking AWS Elastic IPs...", fg='green')
elastic_ips_aws = list_elastic_ips_aws(tags_dict=params, verbose=verbose)
if elastic_ips_aws:
aws_table = PrettyTable(["AllocationId", "PublicIP", "TestId", "RunByUser", "InstanceId (attached to)"])
aws_table.align = "l"
aws_table.sortby = 'AllocationId'
for eip in elastic_ips_aws:
tags = aws_tags_to_dict(eip.get('Tags'))
test_id = tags.get("TestId", "N/A")
run_by_user = tags.get("RunByUser", "N/A")
aws_table.add_row([
eip['AllocationId'],
eip['PublicIp'],
test_id,
run_by_user,
eip.get('InstanceId', 'N/A')])
click.echo(aws_table.get_string(title="EIPs used on AWS"))
else:
click.secho("No elastic ips found for selected filters in AWS!", fg="yellow")
click.secho("Checking GCE...", fg='green')
gke_clusters = list_clusters_gke(tags_dict=params, verbose=verbose)
if gke_clusters:
gke_table = PrettyTable(["Name", "Region-AZ", "TestId", "RunByUser", "CreateTime"])
gke_table.align = "l"
gke_table.sortby = 'CreateTime'
for cluster in gke_clusters:
tags = gce_meta_to_dict(cluster.extra['metadata'])
gke_table.add_row([cluster.name,
cluster.zone,
tags.get('TestId', 'N/A') if tags else "N/A",
tags.get('RunByUser', 'N/A') if tags else "N/A",
cluster.cluster_info['createTime'],
])
click.echo(gke_table.get_string(title="GKE clusters"))
else:
click.secho("Nothing found for selected filters in GKE!", fg="yellow")
gce_instances = list_instances_gce(tags_dict=params, running=get_all_running, verbose=verbose)
if gce_instances:
gce_table = PrettyTable(table_header)
gce_table.align = "l"
gce_table.sortby = 'LaunchTime'
for instance in gce_instances:
tags = gce_meta_to_dict(instance.extra['metadata'])
public_ips = ", ".join(instance.public_ips) if None not in instance.public_ips else "N/A"
gce_table.add_row([instance.name,
instance.extra["zone"].name,
public_ips if get_all_running else instance.state,
tags.get('TestId', 'N/A') if tags else "N/A",
tags.get('RunByUser', 'N/A') if tags else "N/A",
instance.extra['creationTimestamp'],
])
click.echo(gce_table.get_string(title="Resources used on GCE"))
else:
click.secho("Nothing found for selected filters in GCE!", fg="yellow")
click.secho("Checking Docker...", fg="green")
docker_resources = \
list_resources_docker(tags_dict=params, running=get_all_running, group_as_builder=True, verbose=verbose)
if any(docker_resources.values()):
if docker_resources.get("containers"):
docker_table = PrettyTable(["Name", "Builder", "Public IP" if get_all_running else "Status",
"TestId", "RunByUser", "Created"])
docker_table.align = "l"
docker_table.sortby = "Created"
for builder_name, docker_containers in docker_resources["containers"].items():
for container in docker_containers:
container.reload()
docker_table.add_row([
container.name,
builder_name,
container.attrs["NetworkSettings"]["IPAddress"] if get_all_running else container.status,
container.labels.get("TestId", "N/A"),
container.labels.get("RunByUser", "N/A"),
container.attrs.get("Created", "N/A"),
])
click.echo(docker_table.get_string(title="Containers used on Docker"))
if docker_resources.get("images"):
docker_table = PrettyTable(["Name", "Builder", "TestId", "RunByUser", "Created"])
docker_table.align = "l"
docker_table.sortby = "Created"
for builder_name, docker_images in docker_resources["images"].items():
for image in docker_images:
image.reload()
for tag in image.tags:
docker_table.add_row([
tag,
builder_name,
image.labels.get("TestId", "N/A"),
image.labels.get("RunByUser", "N/A"),
image.attrs.get("Created", "N/A"),
])
click.echo(docker_table.get_string(title="Images used on Docker"))
else:
click.secho("Nothing found for selected filters in Docker!", fg="yellow")
def list_resources(ctx, user, test_id, get_all, get_all_running):
params = dict()
if get_all or get_all_running:
params = None
elif user:
params['RunByUser'] = user
elif test_id:
params['TestId'] = test_id
else:
click.echo(list_resources.get_help(ctx))
if get_all_running:
table_header = [
"Name", "Region-AZ", "PublicIP", "TestId", "RunByUser",
"LaunchTime"
]
else:
table_header = [
"Name", "Region-AZ", "State", "TestId", "RunByUser", "LaunchTime"
]
click.secho("Checking EC2...", fg='green')
aws_instances = list_instances_aws(tags_dict=params,
running=get_all_running)
click.secho("Checking AWS EC2...", fg='green')
if aws_instances:
aws_table = PrettyTable(table_header)
aws_table.align = "l"
aws_table.sortby = 'LaunchTime'
for instance in aws_instances:
tags = aws_tags_to_dict(instance.get('Tags'))
name = tags.get("Name", "N/A")
test_id = tags.get("TestId", "N/A")
run_by_user = tags.get("RunByUser", "N/A")
aws_table.add_row([
name, instance['Placement']['AvailabilityZone'],
instance.get('PublicIpAddress', 'N/A')
if get_all_running else instance['State']['Name'], test_id,
run_by_user, instance['LaunchTime'].ctime()
])
click.echo(aws_table.get_string(title="Resources used on AWS"))
else:
click.secho("Nothing found for selected filters in AWS!", fg="yellow")
click.secho("Checking GCE...", fg='green')
gce_instances = list_instances_gce(tags_dict=params,
running=get_all_running)
if gce_instances:
gce_table = PrettyTable(table_header)
gce_table.align = "l"
gce_table.sortby = 'LaunchTime'
for instance in gce_instances:
tags = gce_meta_to_dict(instance.extra['metadata'])
public_ips = ", ".join(
instance.public_ips
) if None not in instance.public_ips else "N/A"
gce_table.add_row([
instance.name,
instance.extra["zone"].name,
public_ips if get_all_running else instance.state,
tags.get('TestId', 'N/A') if tags else "N/A",
tags.get('RunByUser', 'N/A') if tags else "N/A",
instance.extra['creationTimestamp'],
])
click.echo(gce_table.get_string(title="Resources used on GCE"))
else:
click.secho("Nothing found for selected filters in GCE!", fg="yellow")
