def action_processor(self, message: Dict) -> None: """Process incoming action messages""" if not isinstance(message, dict): log.error(f"Invalid message: {message}") return kind = message.get("kind") message_type = message.get("message_type") data = message.get("data") log.debug( f"Received message of kind {kind}, type {message_type}, data: {data}" ) if kind == "action": try: if message_type == self.action: start_time = time.time() self.do_action(data) run_time = int(time.time() - start_time) log.debug(f"{self.action} ran for {run_time} seconds") else: raise ValueError(f"Unknown message type {message_type}") except Exception as e: log.exception(f"Failed to {message_type}: {e}") reply_kind = "action_error" else: reply_kind = "action_done" reply_message = { "kind": reply_kind, "message_type": message_type, "data": data, } return reply_message
def __delitem__(self, key): if self.parent_resource and isinstance(self.parent_resource, BaseResource): log.debug(f"Calling parent resource to delete tag {key} in cloud") try: if self.parent_resource.delete_tag(key): log_msg = f"Successfully deleted tag {key} in cloud" self.parent_resource._changes.add("tags") self.parent_resource.log(log_msg) log.info((f"{log_msg} for {self.parent_resource.kind}" f" {self.parent_resource.id}")) return super().__delitem__(key) else: log_msg = f"Error deleting tag {key} in cloud" self.parent_resource.log(log_msg) log.error((f"{log_msg} for {self.parent_resource.kind}" f" {self.parent_resource.id}")) except Exception as e: log_msg = f"Unhandled exception while trying to delete tag {key} in cloud:" f" {type(e)} {e}" self.parent_resource.log(log_msg, exception=e) if self.parent_resource._raise_tags_exceptions: raise else: log.exception(log_msg) else: return super().__delitem__(key)
def send_graph( self, graph_export_iterator: GraphExportIterator, resotocore_base_uri: str, resotocore_graph: str, task_id: str, ) -> None: merge_uri = f"{resotocore_base_uri}/graph/{resotocore_graph}/merge" log.debug(f"Sending graph via {merge_uri}") headers = { "Content-Type": "application/x-ndjson", "Resoto-Worker-Nodes": str(graph_export_iterator.number_of_nodes), "Resoto-Worker-Edges": str(graph_export_iterator.number_of_edges), "Resoto-Worker-Task-Id": task_id, } if getattr(ArgumentParser.args, "psk", None): encode_jwt_to_headers(headers, {}, ArgumentParser.args.psk) request = requests.Request(method="POST", url=merge_uri, data=graph_export_iterator, headers=headers) r = self._send_request(request) if r.status_code != 200: log.error(r.content) raise RuntimeError(f"Failed to send graph: {r.content}") log.debug(f"resotocore reply: {r.content.decode()}") log.debug( f"Sent {graph_export_iterator.total_lines} items to resotocore")
def core_actions_processor(metrics: Metrics, search_uri: str, tls_data: TLSData, message: dict) -> None: if not isinstance(message, dict): log.error(f"Invalid message: {message}") return kind = message.get("kind") message_type = message.get("message_type") data = message.get("data") log.debug( f"Received message of kind {kind}, type {message_type}, data: {data}") if kind == "action": try: if message_type == "generate_metrics": start_time = time.time() update_metrics(metrics, search_uri, tls_data) run_time = time.time() - start_time log.debug(f"Updated metrics for {run_time:.2f} seconds") else: raise ValueError(f"Unknown message type {message_type}") except Exception as e: log.exception(f"Failed to {message_type}: {e}") reply_kind = "action_error" else: reply_kind = "action_done" reply_message = { "kind": reply_kind, "message_type": message_type, "data": data, } return reply_message
def add_event_listener( event_type: EventType, listener: Callable, blocking: bool = False, timeout: int = 900, one_shot: bool = False, ) -> bool: """Add an Event Listener""" if not callable(listener): log.error( f"Error registering {listener} of type {type(listener)} with event" f" {event_type.name}") return False log.debug(f"Registering {listener} with event {event_type.name}" f" (blocking: {blocking}, one-shot: {one_shot})") with _events_lock.write_access: if not event_listener_registered(event_type, listener): _events[event_type][listener] = { "blocking": blocking, "timeout": timeout, "one-shot": one_shot, "lock": Lock(), "pid": os.getpid(), } return True return False
def graph(self, search: str) -> Graph: def process_data_line(data: dict, graph: Graph): """Process a single line of resotocore graph data""" if data.get("type") == "node": node_id = data.get("id") node = node_from_dict(data) node_mapping[node_id] = node log.debug(f"Adding node {node} to the graph") graph.add_node(node) if node.kind == "graph_root": log.debug(f"Setting graph root {node}") graph.root = node elif data.get("type") == "edge": node_from = data.get("from") node_to = data.get("to") edge_type = EdgeType.from_value(data.get("edge_type")) if node_from not in node_mapping or node_to not in node_mapping: raise ValueError( f"One of {node_from} -> {node_to} unknown") graph.add_edge(node_mapping[node_from], node_mapping[node_to], edge_type=edge_type) graph = Graph() node_mapping = {} for data in self.search(search): try: process_data_line(data, graph) except ValueError as e: log.error(e) continue sanitize(graph) return graph
def dispatch_event(event: Event, blocking: bool = False) -> None: """Dispatch an Event""" waiting_str = "" if blocking else "not " log.debug( f"Dispatching event {event.event_type.name} and {waiting_str}waiting for" " listeners to return") if event.event_type not in _events.keys(): return with _events_lock.read_access: # Event listeners might unregister themselves during event dispatch # so we will work on a shallow copy while processing the current event. listeners = dict(_events[event.event_type]) threads = {} for listener, listener_data in listeners.items(): try: if listener_data["pid"] != os.getpid(): continue if listener_data["one-shot"] and not listener_data["lock"].acquire( blocking=False): log.error(f"Not calling one-shot listener {listener} of type" f" {type(listener)} - can't acquire lock") continue log.debug(f"Calling listener {listener} of type {type(listener)}" f" (blocking: {listener_data['blocking']})") thread_name = f"{event.event_type.name.lower()}_event" f"-{getattr(listener, '__name__', 'anonymous')}" t = Thread(target=listener, args=[event], name=thread_name) if blocking or listener_data["blocking"]: threads[t] = listener t.start() except Exception: log.exception("Caught unhandled event callback exception") finally: if listener_data["one-shot"]: log.debug( f"One-shot specified for event {event.event_type.name} " f"listener {listener} - removing event listener") remove_event_listener(event.event_type, listener) listener_data["lock"].release() start_time = time.time() for thread, listener in threads.items(): timeout = start_time + listeners[listener]["timeout"] - time.time() if timeout < 1: timeout = 1 log.debug( f"Waiting up to {timeout:.2f}s for event listener {thread.name} to finish" ) thread.join(timeout) log.debug( f"Event listener {thread.name} finished (timeout: {thread.is_alive()})" )
def run(self) -> None: self.name = "eventbus-listener" add_event_listener(EventType.SHUTDOWN, self.shutdown) while not self.shutdown_event.is_set(): log.debug("Connecting to resotocore event bus") try: self.connect() except Exception as e: log.error(e) time.sleep(1)
def update_age(self) -> None: try: self.age = parse_delta( Config.plugin_cleanup_aws_loadbalancers.min_age) log.debug(f"Cleanup AWS Load balancers minimum age is {self.age}") except ValueError: log.error( "Error while parsing Cleanup AWS Load balancers minimum age" f" {Config.plugin_cleanup_aws_loadbalancers.min_age}") raise
def collect(self) -> None: """Run by resoto during the global collect() run. This method kicks off code that adds GCP resources to `self.graph`. When collect() finishes the parent thread will take `self.graph` and merge it with the global production graph. """ log.debug("plugin: GCP collecting resources") credentials = Credentials.all() if len(Config.gcp.project) > 0: for project in list(credentials.keys()): if project not in Config.gcp.project: del credentials[project] if len(credentials) == 0: return max_workers = (len(credentials) if len(credentials) < Config.gcp.project_pool_size else Config.gcp.project_pool_size) pool_args = {"max_workers": max_workers} if Config.gcp.fork_process: pool_args["mp_context"] = multiprocessing.get_context("spawn") pool_args["initializer"] = resotolib.proc.initializer pool_executor = futures.ProcessPoolExecutor collect_args = { "args": ArgumentParser.args, "running_config": Config.running_config, "credentials": credentials if all(v is None for v in credentials.values()) else None, } else: pool_executor = futures.ThreadPoolExecutor collect_args = {} with pool_executor(**pool_args) as executor: wait_for = [ executor.submit( self.collect_project, project_id, **collect_args, ) for project_id in credentials.keys() ] for future in futures.as_completed(wait_for): project_graph = future.result() if not isinstance(project_graph, Graph): log.error( f"Skipping invalid project_graph {type(project_graph)}" ) continue self.graph.merge(project_graph)
def update_metrics(metrics: Metrics, search_uri: str, tls_data: Optional[TLSData] = None) -> None: metrics_descriptions = Config.resotometrics.metrics for _, data in metrics_descriptions.items(): if shutdown_event.is_set(): return metrics_search = data.search metric_type = data.type metric_help = data.help if metrics_search is None: continue if metric_type not in ("gauge", "counter"): log.error( f"Do not know how to handle metrics of type {metric_type}") continue try: for result in search(metrics_search, search_uri, tls_data=tls_data): labels = get_labels_from_result(result) label_values = get_label_values_from_result(result, labels) for metric_name, metric_value in get_metrics_from_result( result).items(): if metric_name not in metrics.staging: log.debug( f"Adding metric {metric_name} of type {metric_type}" ) if metric_type == "gauge": metrics.staging[metric_name] = GaugeMetricFamily( f"resoto_{metric_name}", metric_help, labels=labels, ) elif metric_type == "counter": metrics.staging[metric_name] = CounterMetricFamily( f"resoto_{metric_name}", metric_help, labels=labels, ) if metric_type == "counter" and metric_name in metrics.live: current_metric = metrics.live[metric_name] for sample in current_metric.samples: if sample.labels == result.get("group"): metric_value += sample.value break metrics.staging[metric_name].add_metric( label_values, metric_value) except RuntimeError as e: log.error(e) continue metrics.swap()
def delete( self, graph: Graph, snapshot_before_delete: bool = False, snapshot_timeout: int = 3600, ) -> bool: ec2 = aws_resource(self, "ec2", graph) volume = ec2.Volume(self.id) if snapshot_before_delete or self.snapshot_before_delete: log_msg = "Creating snapshot before deletion" self.log(log_msg) log.debug(f"{log_msg} of {self.kind} {self.dname}") snapshot = volume.create_snapshot( Description=f"resoto created snapshot for volume {self.id}", TagSpecifications=[ { "ResourceType": "snapshot", "Tags": [ {"Key": "Name", "Value": f"CK snap of {self.id}"}, {"Key": "owner", "Value": "resoto"}, ], }, ], ) start_utime = time.time() while snapshot.state == "pending": if time.time() > start_utime + snapshot_timeout: raise TimeoutError( ( f"AWS EC2 Volume Snapshot {self.dname} tag update timed out after " f"{snapshot_timeout} seconds with status {snapshot.state} ({snapshot.state_message})" ) ) time.sleep(10) log.debug( ( f"Waiting for snapshot {snapshot.id} to finish before deletion of " f"{self.kind} {self.dname} - progress {snapshot.progress}" ) ) snapshot = ec2.Snapshot(snapshot.id) if snapshot.state != "completed": log_msg = f"Failed to create snapshot - status {snapshot.state} ({snapshot.state_message})" self.log(log_msg) log.error( ( f"{log_msg} for {self.kind} {self.dname} in " f"account {self.account(graph).dname} region {self.region(graph).name}" ) ) return False volume.delete() return True
def wrapper(self, *args, **kwargs): if not isinstance(self, BaseResource): raise ValueError( "unless_protected() only supports BaseResource type objects") if self.protected: log.error( f"Resource {self.rtdname} is protected - refusing modification" ) self.log( ("Modification was requested even though resource is protected" " - refusing")) return False return f(self, *args, **kwargs)
def collect(self) -> None: log.debug("plugin: AWS collecting resources") if not self.authenticated: log.error("Failed to authenticate - skipping collection") return if Config.aws.assume_current and not Config.aws.do_not_scrape_current: log.warning( "You specified assume_current but not do_not_scrape_current! " "This will result in the same account being scraped twice and is likely not what you want." ) if Config.aws.role and Config.aws.scrape_org: accounts = [ AWSAccount(aws_account_id, {}, role=Config.aws.role) for aws_account_id in get_org_accounts(filter_current_account=not Config.aws.assume_current) if aws_account_id not in Config.aws.scrape_exclude_account ] if not Config.aws.do_not_scrape_current: accounts.append(AWSAccount(current_account_id(), {})) elif Config.aws.role and Config.aws.account: accounts = [AWSAccount(aws_account_id, {}, role=Config.aws.role) for aws_account_id in Config.aws.account] else: accounts = [AWSAccount(current_account_id(), {})] max_workers = len(accounts) if len(accounts) < Config.aws.account_pool_size else Config.aws.account_pool_size pool_args = {"max_workers": max_workers} if Config.aws.fork_process: pool_args["mp_context"] = multiprocessing.get_context("spawn") pool_args["initializer"] = resotolib.proc.initializer pool_executor = futures.ProcessPoolExecutor else: pool_executor = futures.ThreadPoolExecutor with pool_executor(**pool_args) as executor: wait_for = [ executor.submit( collect_account, account, self.regions, ArgumentParser.args, Config.running_config, ) for account in accounts ] for future in futures.as_completed(wait_for): account_graph = future.result() if not isinstance(account_graph, Graph): log.error(f"Returned account graph has invalid type {type(account_graph)}") continue self.graph.merge(account_graph)
def pre_cleanup(self, graph=None) -> bool: if not hasattr(self, "pre_delete"): return True if graph is None: graph = self._graph if self.phantom: raise RuntimeError( f"Can't cleanup phantom resource {self.rtdname}") if self.cleaned: log.debug(f"Resource {self.rtdname} has already been cleaned up") return True account = self.account(graph) region = self.region(graph) if not isinstance(account, BaseAccount) or not isinstance( region, BaseRegion): log.error( ("Could not determine account or region for pre cleanup of" f" {self.rtdname}")) return False log_suffix = f" in account {account.dname} region {region.name}" self.log("Trying to run pre clean up") log.debug(f"Trying to run pre clean up {self.rtdname}{log_suffix}") try: if not getattr(self, "pre_delete")(graph): self.log("Failed to run pre clean up") log.error( f"Failed to run pre clean up {self.rtdname}{log_suffix}") return False self.log("Successfully ran pre clean up") log.info( f"Successfully ran pre clean up {self.rtdname}{log_suffix}") except Exception as e: self.log("An error occurred during pre clean up", exception=e) log.exception( f"An error occurred during pre clean up {self.rtdname}{log_suffix}" ) cloud = self.cloud(graph) metrics_resource_pre_cleanup_exceptions.labels( cloud=cloud.name, account=account.dname, region=region.name, kind=self.kind, ).inc() return False return True
def patch_nodes(self, graph: Graph): headers = {"Content-Type": "application/x-ndjson"} if getattr(ArgumentParser.args, "psk", None): encode_jwt_to_headers(headers, {}, ArgumentParser.args.psk) r = requests.patch( f"{self.graph_uri}/nodes", data=GraphChangeIterator(graph), headers=headers, verify=self.verify, ) if r.status_code != 200: err = r.content.decode("utf-8") log.error(err) raise RuntimeError(f"Failed to patch nodes: {err}")
def run(self) -> None: self.name = self.identifier add_event_listener(EventType.SHUTDOWN, self.shutdown) for i in range(self.max_workers): threading.Thread(target=self.worker, daemon=True, name=f"worker-{i}").start() while not self.shutdown_event.is_set(): log.debug("Connecting to resotocore task queue") try: self.connect() except Exception as e: log.error(e) time.sleep(1)
def update_config_model( model: List, resotocore_uri: str = None, psk: str = None, verify: Optional[str] = None, ) -> bool: headers = {"Content-Type": "application/json"} resotocore_uri, psk, headers = default_args(resotocore_uri, psk, headers=headers) model_uri = f"{resotocore_uri}/configs/model" model_json = json.dumps(model, indent=4) log.debug("Updating config model") r = requests.patch(model_uri, data=model_json, headers=headers, verify=verify) if r.status_code != 200: log.error(r.content) raise RuntimeError(f"Failed to update model: {r.content}")
def load_config(self, reload: bool = False) -> None: if len(Config.running_config.classes) == 0: raise RuntimeError("No config added") with self._config_lock: try: config, new_config_revision = get_config(self.config_name, self.resotocore_uri, verify=self.verify) if len(config) == 0: if self._initial_load: raise ConfigNotFoundError( "Empty config returned - loading defaults") else: raise ValueError("Empty config returned") except ConfigNotFoundError: pass else: log.info( f"Loaded config {self.config_name} revision {new_config_revision}" ) new_config = {} for config_id, config_data in config.items(): if config_id in Config.running_config.classes: log.debug(f"Loading config section {config_id}") new_config[config_id] = jsons.load( config_data, Config.running_config.classes[config_id]) else: log.warning(f"Unknown config section {config_id}") if reload and self.restart_required(new_config): restart() Config.running_config.data = new_config Config.running_config.revision = new_config_revision self.init_default_config() if self._initial_load: # Try to store the generated config. Handle failure gracefully. try: self.save_config() except RuntimeError as e: log.error(f"Failed to save config: {e}") self.override_config(Config.running_config) self._initial_load = False if not self._ce.is_alive(): log.debug("Starting config event listener") self._ce.start()
def create_graph(self, resotocore_base_uri: str, resotocore_graph: str): graph_uri = f"{resotocore_base_uri}/graph/{resotocore_graph}" log.debug(f"Creating graph {resotocore_graph} via {graph_uri}") headers = { "accept": "application/json", "Content-Type": "application/json", } if getattr(ArgumentParser.args, "psk", None): encode_jwt_to_headers(headers, {}, ArgumentParser.args.psk) request = requests.Request(method="POST", url=graph_uri, data="", headers=headers) r = self._send_request(request) if r.status_code != 200: log.error(r.content) raise RuntimeError(f"Failed to create graph: {r.content}")
def collect(collectors: List[BaseCollectorPlugin]) -> Graph: graph = Graph(root=GraphRoot("root", {})) max_workers = (len(collectors) if len(collectors) < self._config.resotoworker.pool_size else self._config.resotoworker.pool_size) if max_workers == 0: log.error( "No workers configured or no collector plugins loaded - skipping collect" ) return pool_args = {"max_workers": max_workers} if self._config.resotoworker.fork_process: pool_args["mp_context"] = multiprocessing.get_context("spawn") pool_args["initializer"] = resotolib.proc.initializer pool_executor = futures.ProcessPoolExecutor collect_args = { "args": ArgumentParser.args, "running_config": self._config.running_config, } else: pool_executor = futures.ThreadPoolExecutor collect_args = {} with pool_executor(**pool_args) as executor: wait_for = [ executor.submit( collect_plugin_graph, collector, **collect_args, ) for collector in collectors ] for future in futures.as_completed(wait_for): cluster_graph = future.result() if not isinstance(cluster_graph, Graph): log.error( f"Skipping invalid cluster_graph {type(cluster_graph)}" ) continue graph.merge(cluster_graph) sanitize(graph) return graph
def post(uri, data, headers, verify: Optional[str] = None): if getattr(ArgumentParser.args, "psk", None): encode_jwt_to_headers(headers, {}, ArgumentParser.args.psk) r = requests.post(uri, data=data, headers=headers, stream=True, verify=verify) if r.status_code != 200: log.error(r.content.decode()) raise RuntimeError(f"Failed to search graph: {r.content.decode()}") for line in r.iter_lines(): if not line: continue try: data = json.loads(line.decode("utf-8")) yield data except TypeError as e: log.error(e) continue
def core_actions_processor(plugin_loader: PluginLoader, tls_data: TLSData, collector: Collector, message: Dict) -> None: collectors: List[BaseCollectorPlugin] = plugin_loader.plugins(PluginType.COLLECTOR) if not isinstance(message, dict): log.error(f"Invalid message: {message}") return kind = message.get("kind") message_type = message.get("message_type") data = message.get("data") task_id = data.get("task") log.debug(f"Received message of kind {kind}, type {message_type}, data: {data}") if kind == "action": try: if message_type == "collect": start_time = time.time() collector.collect_and_send(collectors, task_id=task_id) run_time = int(time.time() - start_time) log.info(f"Collect ran for {run_time} seconds") elif message_type == "cleanup": if not Config.resotoworker.cleanup: log.info("Cleanup called but disabled in config" " (resotoworker.cleanup) - skipping") else: if Config.resotoworker.cleanup_dry_run: log.info("Cleanup called with dry run configured" " (resotoworker.cleanup_dry_run)") start_time = time.time() cleanup(tls_data=tls_data) run_time = int(time.time() - start_time) log.info(f"Cleanup ran for {run_time} seconds") else: raise ValueError(f"Unknown message type {message_type}") except Exception as e: log.exception(f"Failed to {message_type}: {e}") reply_kind = "action_error" else: reply_kind = "action_done" reply_message = { "kind": reply_kind, "message_type": message_type, "data": data, } return reply_message
def get_org_accounts(filter_current_account=False): session = aws_session() client = session.client("organizations") accounts = [] try: response = client.list_accounts() accounts = response.get("Accounts", []) while response.get("NextToken") is not None: response = client.list_accounts(NextToken=response["NextToken"]) accounts.extend(response.get("Accounts", [])) except botocore.exceptions.ClientError as e: if e.response["Error"]["Code"] == "AccessDeniedException": log.error("AWS error - missing permissions to list organization accounts") else: raise filter_account_id = current_account_id() if filter_current_account else -1 accounts = [aws_account["Id"] for aws_account in accounts if aws_account["Id"] != filter_account_id] for account in accounts: log.debug(f"AWS found org account {account}") log.info(f"AWS found a total of {len(accounts)} org accounts") return accounts
def update_model( self, graph: Graph, resotocore_base_uri: str, dump_json: bool = False, tempdir: Optional[str] = None, ) -> None: model_uri = f"{resotocore_base_uri}/model" log.debug(f"Updating model via {model_uri}") model_json = json.dumps(graph.export_model(), indent=4) if dump_json: ts = datetime.now().strftime("%Y-%m-%d-%H-%M") with tempfile.NamedTemporaryFile( prefix=f"resoto-model-{ts}-", suffix=".json", delete=not dump_json, dir=tempdir, ) as model_outfile: log.info(f"Writing model json to file {model_outfile.name}") model_outfile.write(model_json.encode()) headers = { "Content-Type": "application/json", } if getattr(ArgumentParser.args, "psk", None): encode_jwt_to_headers(headers, {}, ArgumentParser.args.psk) request = requests.Request(method="PATCH", url=model_uri, data=model_json, headers=headers) r = self._send_request(request) if r.status_code != 200: log.error(r.content) raise RuntimeError(f"Failed to create model: {r.content}")
def authenticated(self) -> bool: try: _ = current_account_id() except botocore.exceptions.NoCredentialsError: log.error("No AWS credentials found") return False except botocore.exceptions.ClientError as e: if e.response["Error"]["Code"] == "AuthFailure": log.error("AWS was unable to validate the provided access credentials") elif e.response["Error"]["Code"] == "InvalidClientTokenId": log.error("AWS was unable to validate the provided security token") elif e.response["Error"]["Code"] == "ExpiredToken": log.error("AWS security token included in the request is expired") else: raise return False return True
def collect_plugin_graph( collector_plugin: BaseCollectorPlugin, args: Namespace = None, running_config: RunningConfig = None, ) -> Optional[Graph]: collector: BaseCollectorPlugin = collector_plugin() collector_name = f"collector_{collector.cloud}" resotolib.proc.set_thread_name(collector_name) if args is not None: ArgumentParser.args = args setup_logger("resotoworker") if running_config is not None: Config.running_config.apply(running_config) log.debug(f"Starting new collect process for {collector.cloud}") start_time = time() collector.start() collector.join(Config.resotoworker.timeout) elapsed = time() - start_time if not collector.is_alive(): # The plugin has finished its work if not collector.finished: log.error(f"Plugin {collector.cloud} did not finish collection" " - ignoring plugin results") return None if not collector.graph.is_dag_per_edge_type(): log.error(f"Graph of plugin {collector.cloud} is not acyclic" " - ignoring plugin results") return None log.info( f"Collector of plugin {collector.cloud} finished in {elapsed:.4f}s" ) return collector.graph else: log.error( f"Plugin {collector.cloud} timed out - discarding Plugin graph") return None
def cleanup(self, graph=None) -> bool: log.error( f"Resource {self.rtdname} is a phantom resource and can't be cleaned up" ) return False
def force_shutdown(delay: int = 10) -> None: time.sleep(delay) log_stats() log.error(("Some child process or thread timed out during shutdown" " - forcing shutdown completion")) os._exit(0)
def vpc_cleanup(self, graph: Graph): log.info("AWS VPC cleanup called") for node in graph.nodes: if node.protected or not node.clean or not isinstance( node, AWSVPC): continue cloud = node.cloud(graph) account = node.account(graph) region = node.region(graph) log_prefix = ( f"Found AWS VPC {node.dname} in cloud {cloud.name} account {account.dname} " f"region {region.name} marked for cleanup.") if self.config and len(self.config) > 0: if cloud.id not in self.config or account.id not in self.config[ cloud.id]: log.debug(( f"{log_prefix} Account not found in config - ignoring dependent resources." )) continue vpc_instances = [ i for i in node.descendants(graph, edge_type=EdgeType.delete) if isinstance(i, AWSEC2Instance) and i.instance_status not in ( "shutting-down", "terminated") and not i.clean ] if len(vpc_instances) > 0: log_msg = "VPC contains active EC2 instances - not cleaning VPC." log.debug(f"{log_prefix} {log_msg}") node.log(log_msg) node.clean = False continue log.debug( f"{log_prefix} Marking dependent resources for cleanup as well." ) for descendant in node.descendants(graph, edge_type=EdgeType.delete): log.debug( f"Found descendant {descendant.rtdname} of VPC {node.dname}" ) if isinstance( descendant, ( AWSVPCPeeringConnection, AWSEC2NetworkAcl, AWSEC2NetworkInterface, AWSELB, AWSALB, AWSALBTargetGroup, AWSEC2Subnet, AWSEC2SecurityGroup, AWSEC2InternetGateway, AWSEC2NATGateway, AWSEC2RouteTable, AWSVPCEndpoint, AWSEC2ElasticIP, ), ): descendant.log(( f"Marking for cleanup because resource is a descendant of VPC {node.dname} " f"which is set to be cleaned")) node.log( f"Marking {descendant.rtdname} for cleanup because resource is a descendant" ) descendant.clean = True else: if descendant.clean: log.debug(( f"Descendant {descendant.rtdname} of VPC {node.dname} is not targeted but " f"already marked for cleaning")) else: log.error(( f"Descendant {descendant.rtdname} of VPC {node.dname} is not targeted and " f"not marked for cleaning - VPC cleanup will likely fail" )) node.log(( f"Descendant {descendant.rtdname} is not targeted and not marked for cleaning " f"- cleanup will likely fail"))