def test_config():
config = Config("dummy", "dummy")
OneLoginPlugin.add_config(config)
config.init_default_config()
assert Config.onelogin.region == "us"
assert Config.onelogin.client_id is None
assert Config.onelogin.client_secret is None
def test_config():
config = Config("dummy", "dummy")
CleanupAWSVPCsPlugin.add_config(config)
Config.init_default_config()
assert Config.plugin_cleanup_aws_vpcs.enabled is False
assert Config.plugin_cleanup_aws_vpcs.validate(
Config.plugin_cleanup_aws_vpcs) is True
def test_config():
config = Config("dummy", "dummy")
CleanupUntaggedPlugin.add_config(config)
Config.init_default_config()
assert Config.plugin_cleanup_untagged.enabled is False
assert Config.plugin_cleanup_untagged.validate(
Config.plugin_cleanup_untagged) is True
def test_config():
config = Config("dummy", "dummy")
SlackCollectorPlugin.add_config(config)
SlackBotPlugin.add_config(config)
Config.init_default_config()
assert Config.slack.bot_token is None
assert Config.slack.include_archived is False
assert Config.slack.do_not_verify_ssl is False
def test_config():
config = Config("dummy", "dummy")
TagValidatorPlugin.add_config(config)
Config.init_default_config()
assert Config.plugin_tagvalidator.enabled is False
assert Config.plugin_tagvalidator.dry_run is False
assert Config.plugin_tagvalidator.validate(
Config.plugin_tagvalidator) is True
def test_config():
config = Config("dummy", "dummy")
VSphereCollectorPlugin.add_config(config)
Config.init_default_config()
assert Config.vsphere.user is None
assert Config.vsphere.password is None
assert Config.vsphere.host is None
assert Config.vsphere.port == 443
assert Config.vsphere.insecure is True
def test_args():
config = Config("dummy", "dummy")
GCPCollectorPlugin.add_config(config)
Config.init_default_config()
assert len(Config.gcp.service_account) == 0
assert len(Config.gcp.project) == 0
assert len(Config.gcp.collect) == 0
assert len(Config.gcp.no_collect) == 0
assert Config.gcp.project_pool_size == num_default_threads()
assert Config.gcp.fork_process is True
def test_config():
config = Config("dummy", "dummy")
OnpremCollectorPlugin.add_config(config)
config.init_default_config()
assert Config.onprem.location == "Default location"
assert Config.onprem.region == "Default region"
assert Config.onprem.ssh_user == "root"
assert Config.onprem.ssh_key is None
assert len(Config.onprem.server) == 0
assert Config.onprem.pool_size == 5
assert Config.onprem.fork_process is True
def test_config():
arg_parser = get_arg_parser()
core_add_args(arg_parser)
arg_parser.parse_known_args()
cfg = Config("test")
cfg.add_config(ConfigTest)
cfg.init_default_config()
assert Config.dict() == {
"configtest": {
"testvar1": "testing123",
"testvar2": 12345,
"testvar3": {
"mydict": {
"foo": "bar",
"abc": {
"def": "ghi"
}
},
"myint": 0,
"mystr": "Hello",
},
}
}
cfg2 = Config("test2")
assert cfg.configtest.testvar1 == cfg2.configtest.testvar1
assert cfg.configtest.testvar1 == Config.configtest.testvar1
assert cfg.configtest.testvar2 == cfg2.configtest.testvar2
assert cfg.configtest.testvar3 == cfg2.configtest.testvar3
Config.configtest.testvar2 += 1
assert cfg.configtest.testvar2 == 12346
with pytest.raises(ConfigNotFoundError):
Config.does_not_exist.foo = "bar"
with pytest.raises(ConfigNotFoundError):
cfg.does_not_exist.foo = "bar"
def test_config():
config = Config("dummy", "dummy")
KubernetesCollectorPlugin.add_config(config)
config.init_default_config()
assert len(Config.k8s.context) == 0
assert Config.k8s.config is None
assert len(Config.k8s.cluster) == 0
assert len(Config.k8s.apiserver) == 0
assert len(Config.k8s.token) == 0
assert len(Config.k8s.cacert) == 0
assert len(Config.k8s.collect) == 0
assert len(Config.k8s.no_collect) == 0
assert Config.k8s.pool_size == num_default_threads()
assert Config.k8s.fork_process is False
assert Config.k8s.all_contexts is False
def test_config_override():
arg_parser = get_arg_parser()
core_add_args(arg_parser)
arg_parser.parse_known_args()
cfg = Config("test")
cfg.add_config(ConfigTest)
cfg.init_default_config()
assert Config.dict() == {
"configtest": {
"testvar1": "testing123",
"testvar2": 12346,
"testvar3": {
"mydict": {
"foo": "bar",
"abc": {
"def": "ghi"
}
},
"myint": 0,
"mystr": "Hello",
},
}
}
ArgumentParser.args.config_override = [
"configtest.testvar1=testing124",
"configtest.testvar3.myint=1",
"configtest.testvar3.mystr=World",
"configtest.testvar3.mydict.foo=baz",
"configtest.testvar3.mydict.abc.def=jkl",
]
cfg.override_config(cfg.running_config)
assert Config.dict() == {
"configtest": {
"testvar1": "testing124",
"testvar2": 12346,
"testvar3": {
"mydict": {
"foo": "baz",
"abc": {
"def": "jkl"
}
},
"myint": 1,
"mystr": "World",
},
}
}
def test_args():
config = Config("dummy", "dummy")
AWSCollectorPlugin.add_config(config)
Config.init_default_config()
assert Config.aws.access_key_id is None
assert Config.aws.secret_access_key is None
assert Config.aws.role is None
assert Config.aws.role_override is False
assert Config.aws.account is None
assert Config.aws.region is None
assert Config.aws.scrape_org is False
assert Config.aws.fork_process is True
assert Config.aws.scrape_exclude_account == []
assert Config.aws.assume_current is False
assert Config.aws.do_not_scrape_current is False
assert Config.aws.account_pool_size == num_default_threads()
assert Config.aws.region_pool_size == 20
assert Config.aws.parallel_api_requests == 10
assert len(Config.aws.collect) == 0
assert len(Config.aws.no_collect) == 0
def test_config():
config = Config("dummy", "dummy")
CleanupExpiredPlugin.add_config(config)
Config.init_default_config()
assert Config.plugin_cleanup_expired.enabled is False
def test_config() -> None:
config = Config("dummy", "dummy")
DigitalOceanCollectorPlugin.add_config(config)
Config.init_default_config()
assert len(Config.digitalocean.api_tokens) == 0
assert len(Config.digitalocean.spaces_access_keys) == 0
def add_config(config: Config) -> None:
config.add_config(SlackConfig)
def test_config():
config = Config("dummy", "dummy")
CleanupAWSLoadbalancersPlugin.add_config(config)
Config.init_default_config()
assert Config.plugin_cleanup_aws_loadbalancers.enabled is False
assert Config.plugin_cleanup_aws_loadbalancers.min_age == "7 days"
def add_config(config: Config) -> None:
config.add_config(CleanupAWSVPCsConfig)
def add_config(config: Config) -> None:
config.add_config(TagValidatorConfig)
def add_config(config: Config) -> None:
config.add_config(CleanupUntaggedConfig)
def add_config(config: Config) -> None:
config.add_config(CleanupAWSAlarmsConfig)
def add_config(config: Config) -> None:
config.add_config(DigitalOceanCollectorConfig)
def test_config():
config = Config("dummy", "dummy")
GithubCollectorPlugin.add_config(config)
config.init_default_config()
assert Config.github.access_token is None
def add_config(config: Config) -> None:
config.add_config(CleanupAWSLoadbalancersConfig)
def add_config(config: Config) -> None:
config.add_config(OneloginConfig)
def main() -> None:
setup_logger("resotoworker")
# Try to run in a new process group and
# ignore if not possible for whatever reason
try:
os.setpgid(0, 0)
except Exception:
pass
resotolib.proc.parent_pid = os.getpid()
arg_parser = ArgumentParser(
description="resoto worker",
env_args_prefix="RESOTOWORKER_",
)
add_args(arg_parser)
jwt_add_args(arg_parser)
logging_add_args(arg_parser)
core_add_args(arg_parser)
Config.add_args(arg_parser)
TLSData.add_args(arg_parser)
# Find resoto Plugins in the resoto.plugins module
plugin_loader = PluginLoader()
plugin_loader.add_plugin_args(arg_parser)
# At this point the CLI, all Plugins as well as the WebServer have
# added their args to the arg parser
arg_parser.parse_args()
try:
wait_for_resotocore(resotocore.http_uri)
except TimeoutError as e:
log.fatal(f"Failed to connect to resotocore: {e}")
sys.exit(1)
tls_data = None
if resotocore.is_secure:
tls_data = TLSData(
common_name=ArgumentParser.args.subscriber_id,
resotocore_uri=resotocore.http_uri,
)
tls_data.start()
config = Config(
ArgumentParser.args.subscriber_id,
resotocore_uri=resotocore.http_uri,
tls_data=tls_data,
)
add_config(config)
plugin_loader.add_plugin_config(config)
config.load_config()
def send_request(request: requests.Request) -> requests.Response:
prepared = request.prepare()
s = requests.Session()
verify = None
if tls_data:
verify = tls_data.verify
return s.send(request=prepared, verify=verify)
core = Resotocore(send_request, config)
collector = Collector(core.send_to_resotocore, config)
# Handle Ctrl+c and other means of termination/shutdown
resotolib.proc.initializer()
add_event_listener(EventType.SHUTDOWN, shutdown, blocking=False)
# Try to increase nofile and nproc limits
increase_limits()
web_server_args = {}
if tls_data:
web_server_args = {
"ssl_cert": tls_data.cert_path,
"ssl_key": tls_data.key_path,
}
web_server = WebServer(
WebApp(mountpoint=Config.resotoworker.web_path),
web_host=Config.resotoworker.web_host,
web_port=Config.resotoworker.web_port,
**web_server_args,
)
web_server.daemon = True
web_server.start()
core_actions = CoreActions(
identifier=f"{ArgumentParser.args.subscriber_id}-collector",
resotocore_uri=resotocore.http_uri,
resotocore_ws_uri=resotocore.ws_uri,
actions={
"collect": {
"timeout": Config.resotoworker.timeout,
"wait_for_completion": True,
},
"cleanup": {
"timeout": Config.resotoworker.timeout,
"wait_for_completion": True,
},
},
message_processor=partial(core_actions_processor, plugin_loader, tls_data, collector),
tls_data=tls_data,
)
task_queue_filter = {}
if len(Config.resotoworker.collector) > 0:
task_queue_filter = {"cloud": list(Config.resotoworker.collector)}
core_tasks = CoreTasks(
identifier=f"{ArgumentParser.args.subscriber_id}-tagger",
resotocore_ws_uri=resotocore.ws_uri,
tasks=["tag"],
task_queue_filter=task_queue_filter,
message_processor=core_tag_tasks_processor,
tls_data=tls_data,
)
core_actions.start()
core_tasks.start()
for Plugin in plugin_loader.plugins(PluginType.ACTION):
try:
log.debug(f"Starting action plugin {Plugin}")
plugin = Plugin(tls_data=tls_data)
plugin.start()
except Exception as e:
log.exception(f"Caught unhandled persistent Plugin exception {e}")
# We wait for the shutdown Event to be set() and then end the program
# While doing so we print the list of active threads once per 15 minutes
shutdown_event.wait()
web_server.shutdown()
time.sleep(1) # everything gets 1000ms to shutdown gracefully before we force it
resotolib.proc.kill_children(resotolib.proc.SIGTERM, ensure_death=True)
log.info("Shutdown complete")
os._exit(0)
def add_config(config: Config) -> None:
config.add_config(GithubConfig)
def add_config(config: Config) -> None:
config.add_config(ResotoWorkerConfig)
def add_config(config: Config) -> None:
config.add_config(ProtectorConfig)
def add_config(config: Config) -> None:
config.add_config(K8sConfig)
def add_config(config: Config) -> None:
config.add_config(VSphereConfig)
