async def cli_deps( filled_graph_db: ArangoGraphDB, message_bus: MessageBus, event_sender: InMemoryEventSender, foo_model: Model, task_queue: WorkerTaskQueue, worker: Tuple[WorkerTaskDescription, WorkerTaskDescription, WorkerTaskDescription], expander: TemplateExpander, config_handler: ConfigHandler, ) -> AsyncIterator[CLIDependencies]: db_access = DbAccess(filled_graph_db.db.db, event_sender, NoAdjust()) model_handler = ModelHandlerStatic(foo_model) args = parse_args([ "--graphdb-database", "test", "--graphdb-username", "test", "--graphdb-password", "test" ]) deps = CLIDependencies( message_bus=message_bus, event_sender=event_sender, db_access=db_access, model_handler=model_handler, worker_task_queue=task_queue, args=args, template_expander=expander, forked_tasks=Queue(), config_handler=config_handler, ) yield deps await deps.stop()
def test_parse_broken(config_json: Json) -> None: # config_json is a valid parsable config cfg = deepcopy(config_json) # adjust the config: rename web_hosts -> hosts, and web_port -> port hosts = cfg["resotocore"]["api"]["web_hosts"] port = cfg["resotocore"]["api"]["web_port"] cfg["resotocore"]["api"]["hosts"] = hosts cfg["resotocore"]["api"]["port"] = port del cfg["resotocore"]["api"]["web_hosts"] del cfg["resotocore"]["api"]["web_port"] # parse this configuration parsed = parse_config(parse_args(["--analytics-opt-out"]), cfg) parsed_json = to_js(parsed.editable, strip_attr="kind") # web_hosts and web_port were not available and are reverted to the default values default = EditableConfig() assert parsed.api.web_hosts != hosts assert parsed.api.web_hosts == default.api.web_hosts assert parsed.api.web_port != port assert parsed.api.web_port == default.api.web_port # other config values are still unchanged assert parsed_json["cli"] == config_json["resotocore"]["cli"] assert parsed_json["runtime"] == config_json["resotocore"]["runtime"] assert parsed_json["graph_update"] == config_json["resotocore"][ "graph_update"]
def test_not_existing(system_db: StandardDatabase, test_db: StandardDatabase) -> None: access = DbAccess(test_db, NoEventSender(), NoAdjust(), empty_config()) # foo db and pass does not exist foodb = ["--graphdb-username", "foo", "--graphdb-password", "test", "--graphdb-database", "foo"] system_db.delete_user("foo", ignore_missing=True) system_db.delete_database("foo", ignore_missing=True) access.connect(parse_args(foodb), timedelta(seconds=5), sleep_time=0.1) assert system_db.has_user("foo") assert system_db.has_database("foo")
def test_already_existing(test_db: StandardDatabase) -> None: access = DbAccess(test_db, NoEventSender(), NoAdjust()) # test db and user already exist testdb = [ "--graphdb-username", "test", "--graphdb-password", "test", "--graphdb-database", "test" ] access.connect(parse_args(testdb), timedelta(seconds=0.01), sleep_time=0.01)
def default_config() -> CoreConfig: ed = EditableConfig() return CoreConfig( api=ed.api, cli=ed.cli, db=DatabaseConfig(), graph_update=ed.graph_update, # We use this flag explicitly - otherwise it is picked up by env vars runtime=RuntimeConfig(usage_metrics=False), workflows=ed.workflows, custom_commands=CustomCommandsConfig(), args=parse_args(["--analytics-opt-out"]), run=RunConfig(), )
def test_not_existing_and_default_root_account( local_client: ArangoClient, system_db: StandardDatabase, test_db: StandardDatabase ) -> None: access = DbAccess(test_db, NoEventSender(), NoAdjust(), empty_config()) # foo db and pass does not exist foodb = ["--graphdb-username", "foo", "--graphdb-password", "bombproof", "--graphdb-database", "foo"] system_db.delete_user("foo", ignore_missing=True) system_db.delete_database("foo", ignore_missing=True) access.connect(parse_args(foodb), timedelta(seconds=5), sleep_time=0.1) # The default root account is used and a valid password is given -> also the root account uses this password changed_root = local_client.db(username="******", password="******") # Rest the password to the default one, to reset the state before the test changed_root.replace_user("root", "", True)
def test_bootstrap(test_db: StandardDatabase) -> None: sd = test_db.collection("system_data") args = parse_args() # Delete any existing entry, so a new certificate needs to be created sd.delete("ca", ignore_missing=True) handler = CertificateHandler.lookup(args, test_db) ca = sd.get("ca") assert ca is not None # ensure the certificate in the database is the same as exposed by the handler ca_bytes, fingerprint = handler.authority_certificate assert ca_bytes == ca["certificate"].encode("utf-8") # a new handler will use the existing certificate handler2 = CertificateHandler.lookup(args, test_db) assert handler.authority_certificate == handler2.authority_certificate # but the host certificate will be different assert handler.host_certificate != handler2.host_certificate
async def test_merge_process( event_sender: AnalyticsEventSender, graph_db: ArangoGraphDB, foo_kinds: List[Kind] ) -> None: # set explicitly (is done in main explicitly as well) set_start_method("spawn") # wipe any existing data await graph_db.wipe() # store the model in db, so it can be loaded by the sub process graph_db.db.collection("model").insert_many([to_js(a) for a in foo_kinds]) # define args to parse for the sub process args = parse_args(["--graphdb-username", "test", "--graphdb-password", "test", "--graphdb-database", "test"]) # create sample graph data to insert graph = create_graph("test") async def iterator() -> AsyncGenerator[bytes, None]: for node in graph.nodes(): yield bytes(json.dumps(graph.nodes[node]), "utf-8") for from_node, to_node, data in graph.edges(data=True): yield bytes(json.dumps({"from": from_node, "to": to_node, "edge_type": data["edge_type"]}), "utf-8") result = await merge_graph_process(graph_db, event_sender, args, iterator(), timedelta(seconds=30), None) assert result == GraphUpdate(112, 1, 0, 212, 0, 0)
def run(arguments: List[str]) -> None: """ Run application. When this method returns, the process is done. :param arguments: the arguments provided to this process. Note: this method is used in tests to specify arbitrary arguments. """ args = parse_args(arguments) setup_process(args) # after setup, logging is possible info = system_info() log.info( f"Starting up version={info.version} on system with cpus={info.cpus}, " f"available_mem={info.mem_available}, total_mem={info.mem_total}") # The loop is here to restart the process in case of RestartService exceptions. while True: try: run_process(args) break # This line should never be reached. In case it does, break the loop. except RestartService as ex: message = f"Restarting Service. Reason: {ex.reason}" line = "-" * len(message) print(f"\n{line}\n{message}\n{line}\n")
def test_override_via_cmd_line(default_config: CoreConfig) -> None: config = {"runtime": {"debug": False}} parsed = parse_config(parse_args(["--debug"]), config) assert parsed.runtime.debug == True
def test_read_config(config_json: Json) -> None: parsed = parse_config(parse_args(["--analytics-opt-out"]), config_json) assert parsed.json() == config_json
def test_parse_empty(default_config: CoreConfig) -> None: result = parse_config(parse_args(["--analytics-opt-out"]), {}) assert result == default_config
def run(arguments: List[str]) -> None: """ Run application. When this method returns, the process is done. :param arguments: the arguments provided to this process. Note: this method is used in tests to specify arbitrary arguments. """ args = parse_args(arguments) setup_process(args) # after setup, logging is possible info = system_info() log.info( f"Starting up version={info.version} on system with cpus={info.cpus}, " f"available_mem={info.mem_available}, total_mem={info.mem_total}") # wait here for an initial connection to the database before we continue. blocking! created, system_data, sdb = DbAccess.connect(args, timedelta(seconds=60)) event_sender = NoEventSender( ) if args.analytics_opt_out else PostHogEventSender(system_data) db = db_access(args, sdb, event_sender) cert_handler = CertificateHandler.lookup(args, sdb) message_bus = MessageBus() scheduler = Scheduler() worker_task_queue = WorkerTaskQueue() model = ModelHandlerDB(db.get_model_db(), args.plantuml_server) template_expander = DBTemplateExpander(db.template_entity_db) config_handler = ConfigHandlerService( db.config_entity_db, db.config_validation_entity_db, db.configs_model_db, worker_task_queue, message_bus, ) cli_deps = CLIDependencies( message_bus=message_bus, event_sender=event_sender, db_access=db, model_handler=model, worker_task_queue=worker_task_queue, args=args, template_expander=template_expander, config_handler=config_handler, ) default_env = { "graph": args.cli_default_graph, "section": args.cli_default_section } cli = CLI(cli_deps, all_commands(cli_deps), default_env, aliases()) subscriptions = SubscriptionHandler(db.subscribers_db, message_bus) task_handler = TaskHandlerService(db.running_task_db, db.job_db, message_bus, event_sender, subscriptions, scheduler, cli, args) cli_deps.extend(task_handler=task_handler) api = Api( db, model, subscriptions, task_handler, message_bus, event_sender, worker_task_queue, cert_handler, config_handler, cli, template_expander, args, ) event_emitter = emit_recurrent_events(event_sender, model, subscriptions, worker_task_queue, message_bus, timedelta(hours=1), timedelta(hours=1)) async def on_start() -> None: # queue must be created inside an async function! cli_deps.extend(forked_tasks=Queue()) await db.start() await event_sender.start() await subscriptions.start() await scheduler.start() await worker_task_queue.start() await event_emitter.start() await cli.start() await task_handler.start() await api.start() if created: await event_sender.core_event(CoreEvent.SystemInstalled) await event_sender.core_event( CoreEvent.SystemStarted, { "version": version(), "created_at": to_json(system_data.created_at), "system": platform.system(), "platform": platform.platform(), "inside_docker": info.inside_docker, }, cpu_count=info.cpus, mem_total=info.mem_total, mem_available=info.mem_available, ) async def on_stop() -> None: duration = utc() - info.started_at await api.stop() await task_handler.stop() await cli.stop() await event_sender.core_event(CoreEvent.SystemStopped, total_seconds=int( duration.total_seconds())) await event_emitter.stop() await worker_task_queue.stop() await scheduler.stop() await subscriptions.stop() await db.stop() await event_sender.stop() async def async_initializer() -> Application: async def on_start_stop(_: Application) -> AsyncIterator[None]: await on_start() log.info("Initialization done. Starting API.") yield log.info("Shutdown initiated. Stop all tasks.") await on_stop() api.app.cleanup_ctx.append(on_start_stop) return api.app tls_context: Optional[SSLContext] = None if args.tls_cert: tls_context = SSLContext(ssl.PROTOCOL_TLS) tls_context.load_cert_chain(args.tls_cert, args.tls_key, args.tls_password) runner.run_app(async_initializer(), api.stop, host=args.host, port=args.port, ssl_context=tls_context)
def cert_handler() -> CertificateHandler: args = parse_args() key, certificate = bootstrap_ca() return CertificateHandler(args, key, certificate)
def task_handler_args() -> Namespace: return parse_args()