def list_sct_runners(cls) -> list[SctRunnerInfo]: sct_runners = [] for instance in list_instances_gce( tags_dict={"NodeType": cls.NODE_TYPE}, verbose=True): tags = gce_meta_to_dict(instance.extra["metadata"]) region = instance.extra["zone"].name if launch_time := tags.get("launch_time"): try: launch_time = datetime_from_formatted( date_string=launch_time) except ValueError as exc: LOGGER.warning("Value of `launch_time' tag is invalid: %s", exc) launch_time = None if not launch_time: create_time = instance.extra["creationTimestamp"] LOGGER.info( "`launch_time' tag is empty or invalid, fallback to creation time: %s", create_time) launch_time = datetime.datetime.fromisoformat(create_time) sct_runners.append( SctRunnerInfo( sct_runner_class=cls, cloud_service_instance=None, # we don't need it for GCE region_az=region, instance=instance, instance_name=instance.name, public_ips=instance.public_ips, launch_time=launch_time, keep=tags.get("keep"), keep_action=tags.get("keep_action"), ))
def __init__(self, instance): tags = gce_meta_to_dict(instance.extra['metadata']) is_preemptible = instance.extra["scheduling"]["preemptible"] super(GCEInstance, self).__init__( cloud="gce", name=instance.name, instance_id=instance.id, region_az=instance.extra["zone"].name, state=instance.state, lifecycle=InstanceLifecycle.SPOT if is_preemptible else InstanceLifecycle.ON_DEMAND, instance_type=instance.size, owner=tags.get("RunByUser", NA) if tags else NA, create_time=datetime.fromisoformat(instance.extra['creationTimestamp']), keep=self.get_keep_alive_gce_instance(instance), )
def get_gce_instances(self): gce_instances = list_instances_gce(verbose=True) for instance in gce_instances: tags = gce_meta_to_dict(instance.extra['metadata']) cloud_instance = CloudInstance( cloud="gce", name=instance.name, region_az=instance.extra["zone"].name, state=instance.state, lifecycle="spot" if instance.extra["scheduling"]["preemptible"] else "on-demand", instance_type=instance.size, owner=tags.get('RunByUser', 'N/A') if tags else "N/A", create_time=instance.extra['creationTimestamp'], ) self.instances["gce"].append(cloud_instance) self.all_instances += self.instances["gce"]
def sort_by_index(node): metadata = gce_meta_to_dict(node.extra['metadata']) return metadata.get('NodeIndex', 0)
def list_resources(ctx, user, test_id, get_all, get_all_running, verbose): # pylint: disable=too-many-locals,too-many-arguments,too-many-branches,too-many-statements params = dict() if user: params['RunByUser'] = user if test_id: params['TestId'] = test_id if all([not get_all, not get_all_running, not user, not test_id]): click.echo(list_resources.get_help(ctx)) if get_all_running: table_header = ["Name", "Region-AZ", "PublicIP", "TestId", "RunByUser", "LaunchTime"] else: table_header = ["Name", "Region-AZ", "State", "TestId", "RunByUser", "LaunchTime"] click.secho("Checking AWS EC2...", fg='green') aws_instances = list_instances_aws(tags_dict=params, running=get_all_running, verbose=verbose) if aws_instances: aws_table = PrettyTable(table_header) aws_table.align = "l" aws_table.sortby = 'LaunchTime' for instance in aws_instances: tags = aws_tags_to_dict(instance.get('Tags')) name = tags.get("Name", "N/A") test_id = tags.get("TestId", "N/A") run_by_user = tags.get("RunByUser", "N/A") aws_table.add_row([ name, instance['Placement']['AvailabilityZone'], instance.get('PublicIpAddress', 'N/A') if get_all_running else instance['State']['Name'], test_id, run_by_user, instance['LaunchTime'].ctime()]) click.echo(aws_table.get_string(title="Instances used on AWS")) else: click.secho("Nothing found for selected filters in AWS!", fg="yellow") click.secho("Checking AWS Elastic IPs...", fg='green') elastic_ips_aws = list_elastic_ips_aws(tags_dict=params, verbose=verbose) if elastic_ips_aws: aws_table = PrettyTable(["AllocationId", "PublicIP", "TestId", "RunByUser", "InstanceId (attached to)"]) aws_table.align = "l" aws_table.sortby = 'AllocationId' for eip in elastic_ips_aws: tags = aws_tags_to_dict(eip.get('Tags')) test_id = tags.get("TestId", "N/A") run_by_user = tags.get("RunByUser", "N/A") aws_table.add_row([ eip['AllocationId'], eip['PublicIp'], test_id, run_by_user, eip.get('InstanceId', 'N/A')]) click.echo(aws_table.get_string(title="EIPs used on AWS")) else: click.secho("No elastic ips found for selected filters in AWS!", fg="yellow") click.secho("Checking GCE...", fg='green') gce_instances = list_instances_gce(tags_dict=params, running=get_all_running, verbose=verbose) if gce_instances: gce_table = PrettyTable(table_header) gce_table.align = "l" gce_table.sortby = 'LaunchTime' for instance in gce_instances: tags = gce_meta_to_dict(instance.extra['metadata']) public_ips = ", ".join(instance.public_ips) if None not in instance.public_ips else "N/A" gce_table.add_row([instance.name, instance.extra["zone"].name, public_ips if get_all_running else instance.state, tags.get('TestId', 'N/A') if tags else "N/A", tags.get('RunByUser', 'N/A') if tags else "N/A", instance.extra['creationTimestamp'], ]) click.echo(gce_table.get_string(title="Resources used on GCE")) else: click.secho("Nothing found for selected filters in GCE!", fg="yellow")
def list_resources(ctx, user, test_id, get_all, get_all_running, verbose): # pylint: disable=too-many-locals,too-many-arguments,too-many-branches,too-many-statements add_file_logger() params = dict() if user: params['RunByUser'] = user if test_id: params['TestId'] = test_id if all([not get_all, not get_all_running, not user, not test_id]): click.echo(list_resources.get_help(ctx)) if get_all_running: table_header = ["Name", "Region-AZ", "PublicIP", "TestId", "RunByUser", "LaunchTime"] else: table_header = ["Name", "Region-AZ", "State", "TestId", "RunByUser", "LaunchTime"] click.secho("Checking AWS EC2...", fg='green') aws_instances = list_instances_aws(tags_dict=params, running=get_all_running, verbose=verbose) if aws_instances: aws_table = PrettyTable(table_header) aws_table.align = "l" aws_table.sortby = 'LaunchTime' for instance in aws_instances: tags = aws_tags_to_dict(instance.get('Tags')) name = tags.get("Name", "N/A") test_id = tags.get("TestId", "N/A") run_by_user = tags.get("RunByUser", "N/A") aws_table.add_row([ name, instance['Placement']['AvailabilityZone'], instance.get('PublicIpAddress', 'N/A') if get_all_running else instance['State']['Name'], test_id, run_by_user, instance['LaunchTime'].ctime()]) click.echo(aws_table.get_string(title="Instances used on AWS")) else: click.secho("Nothing found for selected filters in AWS!", fg="yellow") click.secho("Checking AWS Elastic IPs...", fg='green') elastic_ips_aws = list_elastic_ips_aws(tags_dict=params, verbose=verbose) if elastic_ips_aws: aws_table = PrettyTable(["AllocationId", "PublicIP", "TestId", "RunByUser", "InstanceId (attached to)"]) aws_table.align = "l" aws_table.sortby = 'AllocationId' for eip in elastic_ips_aws: tags = aws_tags_to_dict(eip.get('Tags')) test_id = tags.get("TestId", "N/A") run_by_user = tags.get("RunByUser", "N/A") aws_table.add_row([ eip['AllocationId'], eip['PublicIp'], test_id, run_by_user, eip.get('InstanceId', 'N/A')]) click.echo(aws_table.get_string(title="EIPs used on AWS")) else: click.secho("No elastic ips found for selected filters in AWS!", fg="yellow") click.secho("Checking GCE...", fg='green') gke_clusters = list_clusters_gke(tags_dict=params, verbose=verbose) if gke_clusters: gke_table = PrettyTable(["Name", "Region-AZ", "TestId", "RunByUser", "CreateTime"]) gke_table.align = "l" gke_table.sortby = 'CreateTime' for cluster in gke_clusters: tags = gce_meta_to_dict(cluster.extra['metadata']) gke_table.add_row([cluster.name, cluster.zone, tags.get('TestId', 'N/A') if tags else "N/A", tags.get('RunByUser', 'N/A') if tags else "N/A", cluster.cluster_info['createTime'], ]) click.echo(gke_table.get_string(title="GKE clusters")) else: click.secho("Nothing found for selected filters in GKE!", fg="yellow") gce_instances = list_instances_gce(tags_dict=params, running=get_all_running, verbose=verbose) if gce_instances: gce_table = PrettyTable(table_header) gce_table.align = "l" gce_table.sortby = 'LaunchTime' for instance in gce_instances: tags = gce_meta_to_dict(instance.extra['metadata']) public_ips = ", ".join(instance.public_ips) if None not in instance.public_ips else "N/A" gce_table.add_row([instance.name, instance.extra["zone"].name, public_ips if get_all_running else instance.state, tags.get('TestId', 'N/A') if tags else "N/A", tags.get('RunByUser', 'N/A') if tags else "N/A", instance.extra['creationTimestamp'], ]) click.echo(gce_table.get_string(title="Resources used on GCE")) else: click.secho("Nothing found for selected filters in GCE!", fg="yellow") click.secho("Checking Docker...", fg="green") docker_resources = \ list_resources_docker(tags_dict=params, running=get_all_running, group_as_builder=True, verbose=verbose) if any(docker_resources.values()): if docker_resources.get("containers"): docker_table = PrettyTable(["Name", "Builder", "Public IP" if get_all_running else "Status", "TestId", "RunByUser", "Created"]) docker_table.align = "l" docker_table.sortby = "Created" for builder_name, docker_containers in docker_resources["containers"].items(): for container in docker_containers: container.reload() docker_table.add_row([ container.name, builder_name, container.attrs["NetworkSettings"]["IPAddress"] if get_all_running else container.status, container.labels.get("TestId", "N/A"), container.labels.get("RunByUser", "N/A"), container.attrs.get("Created", "N/A"), ]) click.echo(docker_table.get_string(title="Containers used on Docker")) if docker_resources.get("images"): docker_table = PrettyTable(["Name", "Builder", "TestId", "RunByUser", "Created"]) docker_table.align = "l" docker_table.sortby = "Created" for builder_name, docker_images in docker_resources["images"].items(): for image in docker_images: image.reload() for tag in image.tags: docker_table.add_row([ tag, builder_name, image.labels.get("TestId", "N/A"), image.labels.get("RunByUser", "N/A"), image.attrs.get("Created", "N/A"), ]) click.echo(docker_table.get_string(title="Images used on Docker")) else: click.secho("Nothing found for selected filters in Docker!", fg="yellow")
def list_resources(ctx, user, test_id, get_all, get_all_running): params = dict() if get_all or get_all_running: params = None elif user: params['RunByUser'] = user elif test_id: params['TestId'] = test_id else: click.echo(list_resources.get_help(ctx)) if get_all_running: table_header = [ "Name", "Region-AZ", "PublicIP", "TestId", "RunByUser", "LaunchTime" ] else: table_header = [ "Name", "Region-AZ", "State", "TestId", "RunByUser", "LaunchTime" ] click.secho("Checking EC2...", fg='green') aws_instances = list_instances_aws(tags_dict=params, running=get_all_running) click.secho("Checking AWS EC2...", fg='green') if aws_instances: aws_table = PrettyTable(table_header) aws_table.align = "l" aws_table.sortby = 'LaunchTime' for instance in aws_instances: tags = aws_tags_to_dict(instance.get('Tags')) name = tags.get("Name", "N/A") test_id = tags.get("TestId", "N/A") run_by_user = tags.get("RunByUser", "N/A") aws_table.add_row([ name, instance['Placement']['AvailabilityZone'], instance.get('PublicIpAddress', 'N/A') if get_all_running else instance['State']['Name'], test_id, run_by_user, instance['LaunchTime'].ctime() ]) click.echo(aws_table.get_string(title="Resources used on AWS")) else: click.secho("Nothing found for selected filters in AWS!", fg="yellow") click.secho("Checking GCE...", fg='green') gce_instances = list_instances_gce(tags_dict=params, running=get_all_running) if gce_instances: gce_table = PrettyTable(table_header) gce_table.align = "l" gce_table.sortby = 'LaunchTime' for instance in gce_instances: tags = gce_meta_to_dict(instance.extra['metadata']) public_ips = ", ".join( instance.public_ips ) if None not in instance.public_ips else "N/A" gce_table.add_row([ instance.name, instance.extra["zone"].name, public_ips if get_all_running else instance.state, tags.get('TestId', 'N/A') if tags else "N/A", tags.get('RunByUser', 'N/A') if tags else "N/A", instance.extra['creationTimestamp'], ]) click.echo(gce_table.get_string(title="Resources used on GCE")) else: click.secho("Nothing found for selected filters in GCE!", fg="yellow")