def core_actions_processor(metrics: Metrics, search_uri: str,
tls_data: TLSData, message: dict) -> None:
if not isinstance(message, dict):
log.error(f"Invalid message: {message}")
return
kind = message.get("kind")
message_type = message.get("message_type")
data = message.get("data")
log.debug(
f"Received message of kind {kind}, type {message_type}, data: {data}")
if kind == "action":
try:
if message_type == "generate_metrics":
start_time = time.time()
update_metrics(metrics, search_uri, tls_data)
run_time = time.time() - start_time
log.debug(f"Updated metrics for {run_time:.2f} seconds")
else:
raise ValueError(f"Unknown message type {message_type}")
except Exception as e:
log.exception(f"Failed to {message_type}: {e}")
reply_kind = "action_error"
else:
reply_kind = "action_done"
reply_message = {
"kind": reply_kind,
"message_type": message_type,
"data": data,
}
return reply_message
def on_message(self, ws, message):
try:
message: Dict = json.loads(message)
except json.JSONDecodeError:
log.exception(f"Unable to decode received message {message}")
return
self.queue.put(message)
def action_processor(self, message: Dict) -> None:
"""Process incoming action messages"""
if not isinstance(message, dict):
log.error(f"Invalid message: {message}")
return
kind = message.get("kind")
message_type = message.get("message_type")
data = message.get("data")
log.debug(
f"Received message of kind {kind}, type {message_type}, data: {data}"
)
if kind == "action":
try:
if message_type == self.action:
start_time = time.time()
self.do_action(data)
run_time = int(time.time() - start_time)
log.debug(f"{self.action} ran for {run_time} seconds")
else:
raise ValueError(f"Unknown message type {message_type}")
except Exception as e:
log.exception(f"Failed to {message_type}: {e}")
reply_kind = "action_error"
else:
reply_kind = "action_done"
reply_message = {
"kind": reply_kind,
"message_type": message_type,
"data": data,
}
return reply_message
def __delitem__(self, key):
if self.parent_resource and isinstance(self.parent_resource,
BaseResource):
log.debug(f"Calling parent resource to delete tag {key} in cloud")
try:
if self.parent_resource.delete_tag(key):
log_msg = f"Successfully deleted tag {key} in cloud"
self.parent_resource._changes.add("tags")
self.parent_resource.log(log_msg)
log.info((f"{log_msg} for {self.parent_resource.kind}"
f" {self.parent_resource.id}"))
return super().__delitem__(key)
else:
log_msg = f"Error deleting tag {key} in cloud"
self.parent_resource.log(log_msg)
log.error((f"{log_msg} for {self.parent_resource.kind}"
f" {self.parent_resource.id}"))
except Exception as e:
log_msg = f"Unhandled exception while trying to delete tag {key} in cloud:" f" {type(e)} {e}"
self.parent_resource.log(log_msg, exception=e)
if self.parent_resource._raise_tags_exceptions:
raise
else:
log.exception(log_msg)
else:
return super().__delitem__(key)
def collect_account(
account: AWSAccount,
regions: List,
args: Namespace = None,
running_config: RunningConfig = None,
) -> Graph:
collector_name = f"aws_{account.id}"
resotolib.proc.set_thread_name(collector_name)
if args is not None:
ArgumentParser.args = args
setup_logger("resotoworker-aws")
if running_config is not None:
Config.running_config.apply(running_config)
log.debug(f"Starting new collect process for account {account.dname}")
aac = AWSAccountCollector(regions, account)
try:
aac.collect()
except botocore.exceptions.ClientError as e:
log.exception(f"An AWS {e.response['Error']['Code']} error occurred while collecting account {account.dname}")
metrics_unhandled_account_exceptions.labels(account=account.dname).inc()
except Exception:
log.exception(f"An unhandled error occurred while collecting AWS account {account.dname}")
metrics_unhandled_account_exceptions.labels(account=account.dname).inc()
return aac.graph
def core_tag_tasks_processor(message: dict) -> None:
task_id = message.get("task_id")
# task_name = message.get("task_name")
# task_attrs = message.get("attrs", {})
task_data = message.get("data", {})
delete_tags = task_data.get("delete", [])
update_tags = task_data.get("update", {})
node_data = task_data.get("node")
result = "done"
extra_data = {}
try:
node = node_from_dict(node_data, include_select_ancestors=True)
for delete_tag in delete_tags:
del node.tags[delete_tag]
for k, v in update_tags.items():
node.tags[k] = v
node_dict = node_to_dict(node)
extra_data.update({"data": node_dict})
except Exception as e:
log.exception("Error while updating tags")
result = "error"
extra_data["error"] = repr(e)
reply_message = {
"task_id": task_id,
"result": result,
}
reply_message.update(extra_data)
return reply_message
def run(self) -> None:
try:
self.go()
except Exception:
metrics_unhandled_plugin_exceptions.labels(plugin=self.name).inc()
log.exception(f"Caught unhandled plugin exception in {self.name}")
else:
self.finished = True
def dispatch_event(event: Event, blocking: bool = False) -> None:
"""Dispatch an Event"""
waiting_str = "" if blocking else "not "
log.debug(
f"Dispatching event {event.event_type.name} and {waiting_str}waiting for"
" listeners to return")
if event.event_type not in _events.keys():
return
with _events_lock.read_access:
# Event listeners might unregister themselves during event dispatch
# so we will work on a shallow copy while processing the current event.
listeners = dict(_events[event.event_type])
threads = {}
for listener, listener_data in listeners.items():
try:
if listener_data["pid"] != os.getpid():
continue
if listener_data["one-shot"] and not listener_data["lock"].acquire(
blocking=False):
log.error(f"Not calling one-shot listener {listener} of type"
f" {type(listener)} - can't acquire lock")
continue
log.debug(f"Calling listener {listener} of type {type(listener)}"
f" (blocking: {listener_data['blocking']})")
thread_name = f"{event.event_type.name.lower()}_event" f"-{getattr(listener, '__name__', 'anonymous')}"
t = Thread(target=listener, args=[event], name=thread_name)
if blocking or listener_data["blocking"]:
threads[t] = listener
t.start()
except Exception:
log.exception("Caught unhandled event callback exception")
finally:
if listener_data["one-shot"]:
log.debug(
f"One-shot specified for event {event.event_type.name} "
f"listener {listener} - removing event listener")
remove_event_listener(event.event_type, listener)
listener_data["lock"].release()
start_time = time.time()
for thread, listener in threads.items():
timeout = start_time + listeners[listener]["timeout"] - time.time()
if timeout < 1:
timeout = 1
log.debug(
f"Waiting up to {timeout:.2f}s for event listener {thread.name} to finish"
)
thread.join(timeout)
log.debug(
f"Event listener {thread.name} finished (timeout: {thread.is_alive()})"
)
def on_config_event(self, message: Dict[str, Any]) -> None:
if (message.get("message_type") == "config-updated"
and message.get("data", {}).get("id") == self.config_name
and message.get("data", {}).get("revision") !=
Config.running_config.revision):
try:
log.debug(f"Config {self.config_name} has changed - reloading")
self.load_config(reload=True)
except Exception:
log.exception("Failed to reload config")
def pre_cleanup(self, graph=None) -> bool:
if not hasattr(self, "pre_delete"):
return True
if graph is None:
graph = self._graph
if self.phantom:
raise RuntimeError(
f"Can't cleanup phantom resource {self.rtdname}")
if self.cleaned:
log.debug(f"Resource {self.rtdname} has already been cleaned up")
return True
account = self.account(graph)
region = self.region(graph)
if not isinstance(account, BaseAccount) or not isinstance(
region, BaseRegion):
log.error(
("Could not determine account or region for pre cleanup of"
f" {self.rtdname}"))
return False
log_suffix = f" in account {account.dname} region {region.name}"
self.log("Trying to run pre clean up")
log.debug(f"Trying to run pre clean up {self.rtdname}{log_suffix}")
try:
if not getattr(self, "pre_delete")(graph):
self.log("Failed to run pre clean up")
log.error(
f"Failed to run pre clean up {self.rtdname}{log_suffix}")
return False
self.log("Successfully ran pre clean up")
log.info(
f"Successfully ran pre clean up {self.rtdname}{log_suffix}")
except Exception as e:
self.log("An error occurred during pre clean up", exception=e)
log.exception(
f"An error occurred during pre clean up {self.rtdname}{log_suffix}"
)
cloud = self.cloud(graph)
metrics_resource_pre_cleanup_exceptions.labels(
cloud=cloud.name,
account=account.dname,
region=region.name,
kind=self.kind,
).inc()
return False
return True
def on_message(self, ws, message):
try:
message: Dict = json.loads(message)
except json.JSONDecodeError:
log.exception(f"Unable to decode received message {message}")
return
log.debug(f"Received event: {message}")
if self.message_processor is not None and callable(
self.message_processor):
try:
self.message_processor(message)
except Exception:
log.exception(
f"Something went wrong while processing {message}")
def worker(self) -> None:
while not self.shutdown_event.is_set():
message = self.queue.get()
log.debug(f"{self.identifier} received: {message}")
if self.message_processor is not None and callable(
self.message_processor):
try:
result = self.message_processor(message)
log.debug(f"Sending reply {result}")
self.ws.send(json.dumps(result))
except Exception:
log.exception(
f"Something went wrong while processing {message}")
self.queue.task_done()
def collect_team(self, client: StreamingWrapper) -> Optional[Graph]:
"""Collects an individual team."""
team_id = client.get_team_id()
team = DigitalOceanTeam(id=team_id, tags={}, urn=f"do:team:{team_id}")
try:
dopc = DigitalOceanTeamCollector(team, client)
dopc.collect()
except Exception:
log.exception(
f"An unhandled error occurred while collecting team {team_id}")
return None
else:
return dopc.graph
def run(self) -> None:
try:
# ArgumentParser.args = self._args
# resotolib.config._config = self._config
# setup_logger("resotoworker")
# resotolib.proc.initializer()
current_thread().name = self.name
if self.bootstrap():
self.go()
except Exception:
metrics_unhandled_plugin_exceptions.labels(plugin=self.name).inc()
log.exception(f"Caught unhandled plugin exception in {self.name}")
else:
self.finished = True
def clean(self, node: BaseResource) -> None:
log_prefix = f"Resource {node.rtdname} is marked for removal"
if Config.resotoworker.cleanup_dry_run:
log.info(
f"{log_prefix}, not calling cleanup method because of dry run flag"
)
return
log.info(f"{log_prefix}, calling cleanup method")
try:
node.cleanup(self.graph)
except Exception:
log.exception(
f"An exception occurred when running resource cleanup on {node.rtdname}"
)
def pre_clean(self, node: BaseResource) -> None:
if not hasattr(node, "pre_delete"):
return
log_prefix = f"Resource {node.rtdname} is marked for removal"
if Config.resotoworker.cleanup_dry_run:
log.info(
f"{log_prefix}, not calling pre cleanup method because of dry run flag"
)
return
log.info(f"{log_prefix}, calling pre cleanup method")
try:
node.pre_cleanup(self.graph)
except Exception:
log.exception(
("An exception occurred when running resource pre cleanup on"
f" {node.rtdname}"))
def core_actions_processor(plugin_loader: PluginLoader, tls_data: TLSData, collector: Collector, message: Dict) -> None:
collectors: List[BaseCollectorPlugin] = plugin_loader.plugins(PluginType.COLLECTOR)
if not isinstance(message, dict):
log.error(f"Invalid message: {message}")
return
kind = message.get("kind")
message_type = message.get("message_type")
data = message.get("data")
task_id = data.get("task")
log.debug(f"Received message of kind {kind}, type {message_type}, data: {data}")
if kind == "action":
try:
if message_type == "collect":
start_time = time.time()
collector.collect_and_send(collectors, task_id=task_id)
run_time = int(time.time() - start_time)
log.info(f"Collect ran for {run_time} seconds")
elif message_type == "cleanup":
if not Config.resotoworker.cleanup:
log.info("Cleanup called but disabled in config" " (resotoworker.cleanup) - skipping")
else:
if Config.resotoworker.cleanup_dry_run:
log.info("Cleanup called with dry run configured" " (resotoworker.cleanup_dry_run)")
start_time = time.time()
cleanup(tls_data=tls_data)
run_time = int(time.time() - start_time)
log.info(f"Cleanup ran for {run_time} seconds")
else:
raise ValueError(f"Unknown message type {message_type}")
except Exception as e:
log.exception(f"Failed to {message_type}: {e}")
reply_kind = "action_error"
else:
reply_kind = "action_done"
reply_message = {
"kind": reply_kind,
"message_type": message_type,
"data": data,
}
return reply_message
def collect_project(
project_id: str,
args: Namespace = None,
running_config: RunningConfig = None,
credentials=None,
) -> Optional[Dict]:
"""Collects an individual project.
Is being called in collect() and either run within a thread or a spawned
process. Depending on whether `gcp.fork_process` was specified or not.
Because the spawned process does not inherit any of our memory or file
descriptors we are passing the already parsed `args` Namespace() to this
method.
"""
project = GCPProject(project_id, {})
collector_name = f"gcp_{project.id}"
resotolib.proc.set_thread_name(collector_name)
if args is not None:
ArgumentParser.args = args
setup_logger("resotoworker-gcp")
if running_config is not None:
Config.running_config.apply(running_config)
if credentials is not None:
Credentials._credentials = credentials
Credentials._initialized = True
log.debug(f"Starting new collect process for project {project.dname}")
try:
gpc = GCPProjectCollector(project)
gpc.collect()
except Exception:
log.exception(
f"An unhandled error occurred while collecting {project.rtdname}"
)
else:
return gpc.graph
def restart() -> None:
python_args = []
if not getattr(sys, "frozen", False):
python_args = subprocess._args_from_interpreter_flags()
args = python_args + sys.argv
path_prefix = "." + os.pathsep
python_path = os.environ.get("PYTHONPATH", "")
if sys.path[0] == "" and not python_path.startswith(path_prefix):
os.environ["PYTHONPATH"] = path_prefix + python_path
try:
close_fds()
except Exception:
log.exception("Failed to FD_CLOEXEC all file descriptors")
kill_children(SIGTERM, ensure_death=True)
os.chdir(initial_dir)
os.execv(sys.executable, [sys.executable] + args)
log.fatal("Failed to restart - exiting")
os._exit(1)
def main() -> None:
setup_logger("resotoworker")
# Try to run in a new process group and
# ignore if not possible for whatever reason
try:
os.setpgid(0, 0)
except Exception:
pass
resotolib.proc.parent_pid = os.getpid()
arg_parser = ArgumentParser(
description="resoto worker",
env_args_prefix="RESOTOWORKER_",
)
add_args(arg_parser)
jwt_add_args(arg_parser)
logging_add_args(arg_parser)
core_add_args(arg_parser)
Config.add_args(arg_parser)
TLSData.add_args(arg_parser)
# Find resoto Plugins in the resoto.plugins module
plugin_loader = PluginLoader()
plugin_loader.add_plugin_args(arg_parser)
# At this point the CLI, all Plugins as well as the WebServer have
# added their args to the arg parser
arg_parser.parse_args()
try:
wait_for_resotocore(resotocore.http_uri)
except TimeoutError as e:
log.fatal(f"Failed to connect to resotocore: {e}")
sys.exit(1)
tls_data = None
if resotocore.is_secure:
tls_data = TLSData(
common_name=ArgumentParser.args.subscriber_id,
resotocore_uri=resotocore.http_uri,
)
tls_data.start()
config = Config(
ArgumentParser.args.subscriber_id,
resotocore_uri=resotocore.http_uri,
tls_data=tls_data,
)
add_config(config)
plugin_loader.add_plugin_config(config)
config.load_config()
def send_request(request: requests.Request) -> requests.Response:
prepared = request.prepare()
s = requests.Session()
verify = None
if tls_data:
verify = tls_data.verify
return s.send(request=prepared, verify=verify)
core = Resotocore(send_request, config)
collector = Collector(core.send_to_resotocore, config)
# Handle Ctrl+c and other means of termination/shutdown
resotolib.proc.initializer()
add_event_listener(EventType.SHUTDOWN, shutdown, blocking=False)
# Try to increase nofile and nproc limits
increase_limits()
web_server_args = {}
if tls_data:
web_server_args = {
"ssl_cert": tls_data.cert_path,
"ssl_key": tls_data.key_path,
}
web_server = WebServer(
WebApp(mountpoint=Config.resotoworker.web_path),
web_host=Config.resotoworker.web_host,
web_port=Config.resotoworker.web_port,
**web_server_args,
)
web_server.daemon = True
web_server.start()
core_actions = CoreActions(
identifier=f"{ArgumentParser.args.subscriber_id}-collector",
resotocore_uri=resotocore.http_uri,
resotocore_ws_uri=resotocore.ws_uri,
actions={
"collect": {
"timeout": Config.resotoworker.timeout,
"wait_for_completion": True,
},
"cleanup": {
"timeout": Config.resotoworker.timeout,
"wait_for_completion": True,
},
},
message_processor=partial(core_actions_processor, plugin_loader, tls_data, collector),
tls_data=tls_data,
)
task_queue_filter = {}
if len(Config.resotoworker.collector) > 0:
task_queue_filter = {"cloud": list(Config.resotoworker.collector)}
core_tasks = CoreTasks(
identifier=f"{ArgumentParser.args.subscriber_id}-tagger",
resotocore_ws_uri=resotocore.ws_uri,
tasks=["tag"],
task_queue_filter=task_queue_filter,
message_processor=core_tag_tasks_processor,
tls_data=tls_data,
)
core_actions.start()
core_tasks.start()
for Plugin in plugin_loader.plugins(PluginType.ACTION):
try:
log.debug(f"Starting action plugin {Plugin}")
plugin = Plugin(tls_data=tls_data)
plugin.start()
except Exception as e:
log.exception(f"Caught unhandled persistent Plugin exception {e}")
# We wait for the shutdown Event to be set() and then end the program
# While doing so we print the list of active threads once per 15 minutes
shutdown_event.wait()
web_server.shutdown()
time.sleep(1) # everything gets 1000ms to shutdown gracefully before we force it
resotolib.proc.kill_children(resotolib.proc.SIGTERM, ensure_death=True)
log.info("Shutdown complete")
os._exit(0)
def override_config(running_config: RunningConfig) -> None:
if getattr(ArgumentParser.args, "config_override", None) is None:
return
for override in getattr(ArgumentParser.args, "config_override", []):
try:
if "=" not in override:
log.error(f"Invalid config override {override}")
continue
config_key, config_value = override.split("=", 1)
if "." not in config_key:
log.error(f"Invalid config override {config_key}")
continue
config_keys = config_key.split(".")
num_keys = len(config_keys)
config_part = running_config.data
set_value = False
# By default we cast the override value to the type of the current
# value. This works for most cases including dictionary values.
# Should the current value be None we see if there was a type specified
# for the dataclass field and use it as a fallback.
# This only works for dataclass fields.
config_section = config_keys[0]
top_config_key = config_keys[1]
fallback_target_type = None
if (config_section in Config.running_config.types
and top_config_key
in Config.running_config.types[config_section]):
fallback_target_type = Config.running_config.types[
config_section][top_config_key]
for num_key, key in enumerate(config_keys):
if num_key == num_keys - 1:
set_value = True
log.debug(f"Overriding config key {config_key}")
if hasattr(config_part, key):
attr_value = getattr(config_part, key)
if set_value:
config_value = Config.cast_target_type(
config_value, attr_value, fallback_target_type)
setattr(config_part, key, config_value)
else:
config_part = attr_value
elif isinstance(config_part, dict) and key in config_part:
attr_value = config_part[key]
if set_value:
config_value = Config.cast_target_type(
config_value, attr_value, fallback_target_type)
config_part[key] = config_value
else:
config_part = attr_value
else:
log.error(
f"Override key {config_key} is unknown - skipping")
break
target_type = str
if target_type in (list, tuple, set):
config_value = target_type(config_value.split(","))
config_value = convert(config_value, target_type)
except Exception:
log.exception(f"Failed to override config {override}")