def action_processor(self, message: Dict) -> None:
"""Process incoming action messages"""
if not isinstance(message, dict):
log.error(f"Invalid message: {message}")
return
kind = message.get("kind")
message_type = message.get("message_type")
data = message.get("data")
log.debug(
f"Received message of kind {kind}, type {message_type}, data: {data}"
)
if kind == "action":
try:
if message_type == self.action:
start_time = time.time()
self.do_action(data)
run_time = int(time.time() - start_time)
log.debug(f"{self.action} ran for {run_time} seconds")
else:
raise ValueError(f"Unknown message type {message_type}")
except Exception as e:
log.exception(f"Failed to {message_type}: {e}")
reply_kind = "action_error"
else:
reply_kind = "action_done"
reply_message = {
"kind": reply_kind,
"message_type": message_type,
"data": data,
}
return reply_message
def __delitem__(self, key):
if self.parent_resource and isinstance(self.parent_resource,
BaseResource):
log.debug(f"Calling parent resource to delete tag {key} in cloud")
try:
if self.parent_resource.delete_tag(key):
log_msg = f"Successfully deleted tag {key} in cloud"
self.parent_resource._changes.add("tags")
self.parent_resource.log(log_msg)
log.info((f"{log_msg} for {self.parent_resource.kind}"
f" {self.parent_resource.id}"))
return super().__delitem__(key)
else:
log_msg = f"Error deleting tag {key} in cloud"
self.parent_resource.log(log_msg)
log.error((f"{log_msg} for {self.parent_resource.kind}"
f" {self.parent_resource.id}"))
except Exception as e:
log_msg = f"Unhandled exception while trying to delete tag {key} in cloud:" f" {type(e)} {e}"
self.parent_resource.log(log_msg, exception=e)
if self.parent_resource._raise_tags_exceptions:
raise
else:
log.exception(log_msg)
else:
return super().__delitem__(key)
def send_graph(
self,
graph_export_iterator: GraphExportIterator,
resotocore_base_uri: str,
resotocore_graph: str,
task_id: str,
) -> None:
merge_uri = f"{resotocore_base_uri}/graph/{resotocore_graph}/merge"
log.debug(f"Sending graph via {merge_uri}")
headers = {
"Content-Type": "application/x-ndjson",
"Resoto-Worker-Nodes": str(graph_export_iterator.number_of_nodes),
"Resoto-Worker-Edges": str(graph_export_iterator.number_of_edges),
"Resoto-Worker-Task-Id": task_id,
}
if getattr(ArgumentParser.args, "psk", None):
encode_jwt_to_headers(headers, {}, ArgumentParser.args.psk)
request = requests.Request(method="POST",
url=merge_uri,
data=graph_export_iterator,
headers=headers)
r = self._send_request(request)
if r.status_code != 200:
log.error(r.content)
raise RuntimeError(f"Failed to send graph: {r.content}")
log.debug(f"resotocore reply: {r.content.decode()}")
log.debug(
f"Sent {graph_export_iterator.total_lines} items to resotocore")
def core_actions_processor(metrics: Metrics, search_uri: str,
tls_data: TLSData, message: dict) -> None:
if not isinstance(message, dict):
log.error(f"Invalid message: {message}")
return
kind = message.get("kind")
message_type = message.get("message_type")
data = message.get("data")
log.debug(
f"Received message of kind {kind}, type {message_type}, data: {data}")
if kind == "action":
try:
if message_type == "generate_metrics":
start_time = time.time()
update_metrics(metrics, search_uri, tls_data)
run_time = time.time() - start_time
log.debug(f"Updated metrics for {run_time:.2f} seconds")
else:
raise ValueError(f"Unknown message type {message_type}")
except Exception as e:
log.exception(f"Failed to {message_type}: {e}")
reply_kind = "action_error"
else:
reply_kind = "action_done"
reply_message = {
"kind": reply_kind,
"message_type": message_type,
"data": data,
}
return reply_message
def add_event_listener(
event_type: EventType,
listener: Callable,
blocking: bool = False,
timeout: int = 900,
one_shot: bool = False,
) -> bool:
"""Add an Event Listener"""
if not callable(listener):
log.error(
f"Error registering {listener} of type {type(listener)} with event"
f" {event_type.name}")
return False
log.debug(f"Registering {listener} with event {event_type.name}"
f" (blocking: {blocking}, one-shot: {one_shot})")
with _events_lock.write_access:
if not event_listener_registered(event_type, listener):
_events[event_type][listener] = {
"blocking": blocking,
"timeout": timeout,
"one-shot": one_shot,
"lock": Lock(),
"pid": os.getpid(),
}
return True
return False
def graph(self, search: str) -> Graph:
def process_data_line(data: dict, graph: Graph):
"""Process a single line of resotocore graph data"""
if data.get("type") == "node":
node_id = data.get("id")
node = node_from_dict(data)
node_mapping[node_id] = node
log.debug(f"Adding node {node} to the graph")
graph.add_node(node)
if node.kind == "graph_root":
log.debug(f"Setting graph root {node}")
graph.root = node
elif data.get("type") == "edge":
node_from = data.get("from")
node_to = data.get("to")
edge_type = EdgeType.from_value(data.get("edge_type"))
if node_from not in node_mapping or node_to not in node_mapping:
raise ValueError(
f"One of {node_from} -> {node_to} unknown")
graph.add_edge(node_mapping[node_from],
node_mapping[node_to],
edge_type=edge_type)
graph = Graph()
node_mapping = {}
for data in self.search(search):
try:
process_data_line(data, graph)
except ValueError as e:
log.error(e)
continue
sanitize(graph)
return graph
def dispatch_event(event: Event, blocking: bool = False) -> None:
"""Dispatch an Event"""
waiting_str = "" if blocking else "not "
log.debug(
f"Dispatching event {event.event_type.name} and {waiting_str}waiting for"
" listeners to return")
if event.event_type not in _events.keys():
return
with _events_lock.read_access:
# Event listeners might unregister themselves during event dispatch
# so we will work on a shallow copy while processing the current event.
listeners = dict(_events[event.event_type])
threads = {}
for listener, listener_data in listeners.items():
try:
if listener_data["pid"] != os.getpid():
continue
if listener_data["one-shot"] and not listener_data["lock"].acquire(
blocking=False):
log.error(f"Not calling one-shot listener {listener} of type"
f" {type(listener)} - can't acquire lock")
continue
log.debug(f"Calling listener {listener} of type {type(listener)}"
f" (blocking: {listener_data['blocking']})")
thread_name = f"{event.event_type.name.lower()}_event" f"-{getattr(listener, '__name__', 'anonymous')}"
t = Thread(target=listener, args=[event], name=thread_name)
if blocking or listener_data["blocking"]:
threads[t] = listener
t.start()
except Exception:
log.exception("Caught unhandled event callback exception")
finally:
if listener_data["one-shot"]:
log.debug(
f"One-shot specified for event {event.event_type.name} "
f"listener {listener} - removing event listener")
remove_event_listener(event.event_type, listener)
listener_data["lock"].release()
start_time = time.time()
for thread, listener in threads.items():
timeout = start_time + listeners[listener]["timeout"] - time.time()
if timeout < 1:
timeout = 1
log.debug(
f"Waiting up to {timeout:.2f}s for event listener {thread.name} to finish"
)
thread.join(timeout)
log.debug(
f"Event listener {thread.name} finished (timeout: {thread.is_alive()})"
)
def run(self) -> None:
self.name = "eventbus-listener"
add_event_listener(EventType.SHUTDOWN, self.shutdown)
while not self.shutdown_event.is_set():
log.debug("Connecting to resotocore event bus")
try:
self.connect()
except Exception as e:
log.error(e)
time.sleep(1)
def update_age(self) -> None:
try:
self.age = parse_delta(
Config.plugin_cleanup_aws_loadbalancers.min_age)
log.debug(f"Cleanup AWS Load balancers minimum age is {self.age}")
except ValueError:
log.error(
"Error while parsing Cleanup AWS Load balancers minimum age"
f" {Config.plugin_cleanup_aws_loadbalancers.min_age}")
raise
def collect(self) -> None:
"""Run by resoto during the global collect() run.
This method kicks off code that adds GCP resources to `self.graph`.
When collect() finishes the parent thread will take `self.graph` and merge
it with the global production graph.
"""
log.debug("plugin: GCP collecting resources")
credentials = Credentials.all()
if len(Config.gcp.project) > 0:
for project in list(credentials.keys()):
if project not in Config.gcp.project:
del credentials[project]
if len(credentials) == 0:
return
max_workers = (len(credentials)
if len(credentials) < Config.gcp.project_pool_size else
Config.gcp.project_pool_size)
pool_args = {"max_workers": max_workers}
if Config.gcp.fork_process:
pool_args["mp_context"] = multiprocessing.get_context("spawn")
pool_args["initializer"] = resotolib.proc.initializer
pool_executor = futures.ProcessPoolExecutor
collect_args = {
"args":
ArgumentParser.args,
"running_config":
Config.running_config,
"credentials":
credentials if all(v is None
for v in credentials.values()) else None,
}
else:
pool_executor = futures.ThreadPoolExecutor
collect_args = {}
with pool_executor(**pool_args) as executor:
wait_for = [
executor.submit(
self.collect_project,
project_id,
**collect_args,
) for project_id in credentials.keys()
]
for future in futures.as_completed(wait_for):
project_graph = future.result()
if not isinstance(project_graph, Graph):
log.error(
f"Skipping invalid project_graph {type(project_graph)}"
)
continue
self.graph.merge(project_graph)
def update_metrics(metrics: Metrics,
search_uri: str,
tls_data: Optional[TLSData] = None) -> None:
metrics_descriptions = Config.resotometrics.metrics
for _, data in metrics_descriptions.items():
if shutdown_event.is_set():
return
metrics_search = data.search
metric_type = data.type
metric_help = data.help
if metrics_search is None:
continue
if metric_type not in ("gauge", "counter"):
log.error(
f"Do not know how to handle metrics of type {metric_type}")
continue
try:
for result in search(metrics_search, search_uri,
tls_data=tls_data):
labels = get_labels_from_result(result)
label_values = get_label_values_from_result(result, labels)
for metric_name, metric_value in get_metrics_from_result(
result).items():
if metric_name not in metrics.staging:
log.debug(
f"Adding metric {metric_name} of type {metric_type}"
)
if metric_type == "gauge":
metrics.staging[metric_name] = GaugeMetricFamily(
f"resoto_{metric_name}",
metric_help,
labels=labels,
)
elif metric_type == "counter":
metrics.staging[metric_name] = CounterMetricFamily(
f"resoto_{metric_name}",
metric_help,
labels=labels,
)
if metric_type == "counter" and metric_name in metrics.live:
current_metric = metrics.live[metric_name]
for sample in current_metric.samples:
if sample.labels == result.get("group"):
metric_value += sample.value
break
metrics.staging[metric_name].add_metric(
label_values, metric_value)
except RuntimeError as e:
log.error(e)
continue
metrics.swap()
def delete(
self,
graph: Graph,
snapshot_before_delete: bool = False,
snapshot_timeout: int = 3600,
) -> bool:
ec2 = aws_resource(self, "ec2", graph)
volume = ec2.Volume(self.id)
if snapshot_before_delete or self.snapshot_before_delete:
log_msg = "Creating snapshot before deletion"
self.log(log_msg)
log.debug(f"{log_msg} of {self.kind} {self.dname}")
snapshot = volume.create_snapshot(
Description=f"resoto created snapshot for volume {self.id}",
TagSpecifications=[
{
"ResourceType": "snapshot",
"Tags": [
{"Key": "Name", "Value": f"CK snap of {self.id}"},
{"Key": "owner", "Value": "resoto"},
],
},
],
)
start_utime = time.time()
while snapshot.state == "pending":
if time.time() > start_utime + snapshot_timeout:
raise TimeoutError(
(
f"AWS EC2 Volume Snapshot {self.dname} tag update timed out after "
f"{snapshot_timeout} seconds with status {snapshot.state} ({snapshot.state_message})"
)
)
time.sleep(10)
log.debug(
(
f"Waiting for snapshot {snapshot.id} to finish before deletion of "
f"{self.kind} {self.dname} - progress {snapshot.progress}"
)
)
snapshot = ec2.Snapshot(snapshot.id)
if snapshot.state != "completed":
log_msg = f"Failed to create snapshot - status {snapshot.state} ({snapshot.state_message})"
self.log(log_msg)
log.error(
(
f"{log_msg} for {self.kind} {self.dname} in "
f"account {self.account(graph).dname} region {self.region(graph).name}"
)
)
return False
volume.delete()
return True
def wrapper(self, *args, **kwargs):
if not isinstance(self, BaseResource):
raise ValueError(
"unless_protected() only supports BaseResource type objects")
if self.protected:
log.error(
f"Resource {self.rtdname} is protected - refusing modification"
)
self.log(
("Modification was requested even though resource is protected"
" - refusing"))
return False
return f(self, *args, **kwargs)
def collect(self) -> None:
log.debug("plugin: AWS collecting resources")
if not self.authenticated:
log.error("Failed to authenticate - skipping collection")
return
if Config.aws.assume_current and not Config.aws.do_not_scrape_current:
log.warning(
"You specified assume_current but not do_not_scrape_current! "
"This will result in the same account being scraped twice and is likely not what you want."
)
if Config.aws.role and Config.aws.scrape_org:
accounts = [
AWSAccount(aws_account_id, {}, role=Config.aws.role)
for aws_account_id in get_org_accounts(filter_current_account=not Config.aws.assume_current)
if aws_account_id not in Config.aws.scrape_exclude_account
]
if not Config.aws.do_not_scrape_current:
accounts.append(AWSAccount(current_account_id(), {}))
elif Config.aws.role and Config.aws.account:
accounts = [AWSAccount(aws_account_id, {}, role=Config.aws.role) for aws_account_id in Config.aws.account]
else:
accounts = [AWSAccount(current_account_id(), {})]
max_workers = len(accounts) if len(accounts) < Config.aws.account_pool_size else Config.aws.account_pool_size
pool_args = {"max_workers": max_workers}
if Config.aws.fork_process:
pool_args["mp_context"] = multiprocessing.get_context("spawn")
pool_args["initializer"] = resotolib.proc.initializer
pool_executor = futures.ProcessPoolExecutor
else:
pool_executor = futures.ThreadPoolExecutor
with pool_executor(**pool_args) as executor:
wait_for = [
executor.submit(
collect_account,
account,
self.regions,
ArgumentParser.args,
Config.running_config,
)
for account in accounts
]
for future in futures.as_completed(wait_for):
account_graph = future.result()
if not isinstance(account_graph, Graph):
log.error(f"Returned account graph has invalid type {type(account_graph)}")
continue
self.graph.merge(account_graph)
def pre_cleanup(self, graph=None) -> bool:
if not hasattr(self, "pre_delete"):
return True
if graph is None:
graph = self._graph
if self.phantom:
raise RuntimeError(
f"Can't cleanup phantom resource {self.rtdname}")
if self.cleaned:
log.debug(f"Resource {self.rtdname} has already been cleaned up")
return True
account = self.account(graph)
region = self.region(graph)
if not isinstance(account, BaseAccount) or not isinstance(
region, BaseRegion):
log.error(
("Could not determine account or region for pre cleanup of"
f" {self.rtdname}"))
return False
log_suffix = f" in account {account.dname} region {region.name}"
self.log("Trying to run pre clean up")
log.debug(f"Trying to run pre clean up {self.rtdname}{log_suffix}")
try:
if not getattr(self, "pre_delete")(graph):
self.log("Failed to run pre clean up")
log.error(
f"Failed to run pre clean up {self.rtdname}{log_suffix}")
return False
self.log("Successfully ran pre clean up")
log.info(
f"Successfully ran pre clean up {self.rtdname}{log_suffix}")
except Exception as e:
self.log("An error occurred during pre clean up", exception=e)
log.exception(
f"An error occurred during pre clean up {self.rtdname}{log_suffix}"
)
cloud = self.cloud(graph)
metrics_resource_pre_cleanup_exceptions.labels(
cloud=cloud.name,
account=account.dname,
region=region.name,
kind=self.kind,
).inc()
return False
return True
def patch_nodes(self, graph: Graph):
headers = {"Content-Type": "application/x-ndjson"}
if getattr(ArgumentParser.args, "psk", None):
encode_jwt_to_headers(headers, {}, ArgumentParser.args.psk)
r = requests.patch(
f"{self.graph_uri}/nodes",
data=GraphChangeIterator(graph),
headers=headers,
verify=self.verify,
)
if r.status_code != 200:
err = r.content.decode("utf-8")
log.error(err)
raise RuntimeError(f"Failed to patch nodes: {err}")
def run(self) -> None:
self.name = self.identifier
add_event_listener(EventType.SHUTDOWN, self.shutdown)
for i in range(self.max_workers):
threading.Thread(target=self.worker,
daemon=True,
name=f"worker-{i}").start()
while not self.shutdown_event.is_set():
log.debug("Connecting to resotocore task queue")
try:
self.connect()
except Exception as e:
log.error(e)
time.sleep(1)
def update_config_model(
model: List,
resotocore_uri: str = None,
psk: str = None,
verify: Optional[str] = None,
) -> bool:
headers = {"Content-Type": "application/json"}
resotocore_uri, psk, headers = default_args(resotocore_uri, psk, headers=headers)
model_uri = f"{resotocore_uri}/configs/model"
model_json = json.dumps(model, indent=4)
log.debug("Updating config model")
r = requests.patch(model_uri, data=model_json, headers=headers, verify=verify)
if r.status_code != 200:
log.error(r.content)
raise RuntimeError(f"Failed to update model: {r.content}")
def load_config(self, reload: bool = False) -> None:
if len(Config.running_config.classes) == 0:
raise RuntimeError("No config added")
with self._config_lock:
try:
config, new_config_revision = get_config(self.config_name,
self.resotocore_uri,
verify=self.verify)
if len(config) == 0:
if self._initial_load:
raise ConfigNotFoundError(
"Empty config returned - loading defaults")
else:
raise ValueError("Empty config returned")
except ConfigNotFoundError:
pass
else:
log.info(
f"Loaded config {self.config_name} revision {new_config_revision}"
)
new_config = {}
for config_id, config_data in config.items():
if config_id in Config.running_config.classes:
log.debug(f"Loading config section {config_id}")
new_config[config_id] = jsons.load(
config_data,
Config.running_config.classes[config_id])
else:
log.warning(f"Unknown config section {config_id}")
if reload and self.restart_required(new_config):
restart()
Config.running_config.data = new_config
Config.running_config.revision = new_config_revision
self.init_default_config()
if self._initial_load:
# Try to store the generated config. Handle failure gracefully.
try:
self.save_config()
except RuntimeError as e:
log.error(f"Failed to save config: {e}")
self.override_config(Config.running_config)
self._initial_load = False
if not self._ce.is_alive():
log.debug("Starting config event listener")
self._ce.start()
def create_graph(self, resotocore_base_uri: str, resotocore_graph: str):
graph_uri = f"{resotocore_base_uri}/graph/{resotocore_graph}"
log.debug(f"Creating graph {resotocore_graph} via {graph_uri}")
headers = {
"accept": "application/json",
"Content-Type": "application/json",
}
if getattr(ArgumentParser.args, "psk", None):
encode_jwt_to_headers(headers, {}, ArgumentParser.args.psk)
request = requests.Request(method="POST",
url=graph_uri,
data="",
headers=headers)
r = self._send_request(request)
if r.status_code != 200:
log.error(r.content)
raise RuntimeError(f"Failed to create graph: {r.content}")
def collect(collectors: List[BaseCollectorPlugin]) -> Graph:
graph = Graph(root=GraphRoot("root", {}))
max_workers = (len(collectors) if len(collectors) <
self._config.resotoworker.pool_size else
self._config.resotoworker.pool_size)
if max_workers == 0:
log.error(
"No workers configured or no collector plugins loaded - skipping collect"
)
return
pool_args = {"max_workers": max_workers}
if self._config.resotoworker.fork_process:
pool_args["mp_context"] = multiprocessing.get_context("spawn")
pool_args["initializer"] = resotolib.proc.initializer
pool_executor = futures.ProcessPoolExecutor
collect_args = {
"args": ArgumentParser.args,
"running_config": self._config.running_config,
}
else:
pool_executor = futures.ThreadPoolExecutor
collect_args = {}
with pool_executor(**pool_args) as executor:
wait_for = [
executor.submit(
collect_plugin_graph,
collector,
**collect_args,
) for collector in collectors
]
for future in futures.as_completed(wait_for):
cluster_graph = future.result()
if not isinstance(cluster_graph, Graph):
log.error(
f"Skipping invalid cluster_graph {type(cluster_graph)}"
)
continue
graph.merge(cluster_graph)
sanitize(graph)
return graph
def post(uri, data, headers, verify: Optional[str] = None):
if getattr(ArgumentParser.args, "psk", None):
encode_jwt_to_headers(headers, {}, ArgumentParser.args.psk)
r = requests.post(uri,
data=data,
headers=headers,
stream=True,
verify=verify)
if r.status_code != 200:
log.error(r.content.decode())
raise RuntimeError(f"Failed to search graph: {r.content.decode()}")
for line in r.iter_lines():
if not line:
continue
try:
data = json.loads(line.decode("utf-8"))
yield data
except TypeError as e:
log.error(e)
continue
def core_actions_processor(plugin_loader: PluginLoader, tls_data: TLSData, collector: Collector, message: Dict) -> None:
collectors: List[BaseCollectorPlugin] = plugin_loader.plugins(PluginType.COLLECTOR)
if not isinstance(message, dict):
log.error(f"Invalid message: {message}")
return
kind = message.get("kind")
message_type = message.get("message_type")
data = message.get("data")
task_id = data.get("task")
log.debug(f"Received message of kind {kind}, type {message_type}, data: {data}")
if kind == "action":
try:
if message_type == "collect":
start_time = time.time()
collector.collect_and_send(collectors, task_id=task_id)
run_time = int(time.time() - start_time)
log.info(f"Collect ran for {run_time} seconds")
elif message_type == "cleanup":
if not Config.resotoworker.cleanup:
log.info("Cleanup called but disabled in config" " (resotoworker.cleanup) - skipping")
else:
if Config.resotoworker.cleanup_dry_run:
log.info("Cleanup called with dry run configured" " (resotoworker.cleanup_dry_run)")
start_time = time.time()
cleanup(tls_data=tls_data)
run_time = int(time.time() - start_time)
log.info(f"Cleanup ran for {run_time} seconds")
else:
raise ValueError(f"Unknown message type {message_type}")
except Exception as e:
log.exception(f"Failed to {message_type}: {e}")
reply_kind = "action_error"
else:
reply_kind = "action_done"
reply_message = {
"kind": reply_kind,
"message_type": message_type,
"data": data,
}
return reply_message
def get_org_accounts(filter_current_account=False):
session = aws_session()
client = session.client("organizations")
accounts = []
try:
response = client.list_accounts()
accounts = response.get("Accounts", [])
while response.get("NextToken") is not None:
response = client.list_accounts(NextToken=response["NextToken"])
accounts.extend(response.get("Accounts", []))
except botocore.exceptions.ClientError as e:
if e.response["Error"]["Code"] == "AccessDeniedException":
log.error("AWS error - missing permissions to list organization accounts")
else:
raise
filter_account_id = current_account_id() if filter_current_account else -1
accounts = [aws_account["Id"] for aws_account in accounts if aws_account["Id"] != filter_account_id]
for account in accounts:
log.debug(f"AWS found org account {account}")
log.info(f"AWS found a total of {len(accounts)} org accounts")
return accounts
def update_model(
self,
graph: Graph,
resotocore_base_uri: str,
dump_json: bool = False,
tempdir: Optional[str] = None,
) -> None:
model_uri = f"{resotocore_base_uri}/model"
log.debug(f"Updating model via {model_uri}")
model_json = json.dumps(graph.export_model(), indent=4)
if dump_json:
ts = datetime.now().strftime("%Y-%m-%d-%H-%M")
with tempfile.NamedTemporaryFile(
prefix=f"resoto-model-{ts}-",
suffix=".json",
delete=not dump_json,
dir=tempdir,
) as model_outfile:
log.info(f"Writing model json to file {model_outfile.name}")
model_outfile.write(model_json.encode())
headers = {
"Content-Type": "application/json",
}
if getattr(ArgumentParser.args, "psk", None):
encode_jwt_to_headers(headers, {}, ArgumentParser.args.psk)
request = requests.Request(method="PATCH",
url=model_uri,
data=model_json,
headers=headers)
r = self._send_request(request)
if r.status_code != 200:
log.error(r.content)
raise RuntimeError(f"Failed to create model: {r.content}")
def authenticated(self) -> bool:
try:
_ = current_account_id()
except botocore.exceptions.NoCredentialsError:
log.error("No AWS credentials found")
return False
except botocore.exceptions.ClientError as e:
if e.response["Error"]["Code"] == "AuthFailure":
log.error("AWS was unable to validate the provided access credentials")
elif e.response["Error"]["Code"] == "InvalidClientTokenId":
log.error("AWS was unable to validate the provided security token")
elif e.response["Error"]["Code"] == "ExpiredToken":
log.error("AWS security token included in the request is expired")
else:
raise
return False
return True
def collect_plugin_graph(
collector_plugin: BaseCollectorPlugin,
args: Namespace = None,
running_config: RunningConfig = None,
) -> Optional[Graph]:
collector: BaseCollectorPlugin = collector_plugin()
collector_name = f"collector_{collector.cloud}"
resotolib.proc.set_thread_name(collector_name)
if args is not None:
ArgumentParser.args = args
setup_logger("resotoworker")
if running_config is not None:
Config.running_config.apply(running_config)
log.debug(f"Starting new collect process for {collector.cloud}")
start_time = time()
collector.start()
collector.join(Config.resotoworker.timeout)
elapsed = time() - start_time
if not collector.is_alive(): # The plugin has finished its work
if not collector.finished:
log.error(f"Plugin {collector.cloud} did not finish collection"
" - ignoring plugin results")
return None
if not collector.graph.is_dag_per_edge_type():
log.error(f"Graph of plugin {collector.cloud} is not acyclic"
" - ignoring plugin results")
return None
log.info(
f"Collector of plugin {collector.cloud} finished in {elapsed:.4f}s"
)
return collector.graph
else:
log.error(
f"Plugin {collector.cloud} timed out - discarding Plugin graph")
return None
def cleanup(self, graph=None) -> bool:
log.error(
f"Resource {self.rtdname} is a phantom resource and can't be cleaned up"
)
return False
def force_shutdown(delay: int = 10) -> None:
time.sleep(delay)
log_stats()
log.error(("Some child process or thread timed out during shutdown" " - forcing shutdown completion"))
os._exit(0)
def vpc_cleanup(self, graph: Graph):
log.info("AWS VPC cleanup called")
for node in graph.nodes:
if node.protected or not node.clean or not isinstance(
node, AWSVPC):
continue
cloud = node.cloud(graph)
account = node.account(graph)
region = node.region(graph)
log_prefix = (
f"Found AWS VPC {node.dname} in cloud {cloud.name} account {account.dname} "
f"region {region.name} marked for cleanup.")
if self.config and len(self.config) > 0:
if cloud.id not in self.config or account.id not in self.config[
cloud.id]:
log.debug((
f"{log_prefix} Account not found in config - ignoring dependent resources."
))
continue
vpc_instances = [
i for i in node.descendants(graph, edge_type=EdgeType.delete)
if isinstance(i, AWSEC2Instance) and i.instance_status not in (
"shutting-down", "terminated") and not i.clean
]
if len(vpc_instances) > 0:
log_msg = "VPC contains active EC2 instances - not cleaning VPC."
log.debug(f"{log_prefix} {log_msg}")
node.log(log_msg)
node.clean = False
continue
log.debug(
f"{log_prefix} Marking dependent resources for cleanup as well."
)
for descendant in node.descendants(graph,
edge_type=EdgeType.delete):
log.debug(
f"Found descendant {descendant.rtdname} of VPC {node.dname}"
)
if isinstance(
descendant,
(
AWSVPCPeeringConnection,
AWSEC2NetworkAcl,
AWSEC2NetworkInterface,
AWSELB,
AWSALB,
AWSALBTargetGroup,
AWSEC2Subnet,
AWSEC2SecurityGroup,
AWSEC2InternetGateway,
AWSEC2NATGateway,
AWSEC2RouteTable,
AWSVPCEndpoint,
AWSEC2ElasticIP,
),
):
descendant.log((
f"Marking for cleanup because resource is a descendant of VPC {node.dname} "
f"which is set to be cleaned"))
node.log(
f"Marking {descendant.rtdname} for cleanup because resource is a descendant"
)
descendant.clean = True
else:
if descendant.clean:
log.debug((
f"Descendant {descendant.rtdname} of VPC {node.dname} is not targeted but "
f"already marked for cleaning"))
else:
log.error((
f"Descendant {descendant.rtdname} of VPC {node.dname} is not targeted and "
f"not marked for cleaning - VPC cleanup will likely fail"
))
node.log((
f"Descendant {descendant.rtdname} is not targeted and not marked for cleaning "
f"- cleanup will likely fail"))
