async def cli_deps(
filled_graph_db: ArangoGraphDB,
message_bus: MessageBus,
event_sender: InMemoryEventSender,
foo_model: Model,
task_queue: WorkerTaskQueue,
worker: Tuple[WorkerTaskDescription, WorkerTaskDescription,
WorkerTaskDescription],
expander: TemplateExpander,
config_handler: ConfigHandler,
) -> AsyncIterator[CLIDependencies]:
db_access = DbAccess(filled_graph_db.db.db, event_sender, NoAdjust())
model_handler = ModelHandlerStatic(foo_model)
args = parse_args([
"--graphdb-database", "test", "--graphdb-username", "test",
"--graphdb-password", "test"
])
deps = CLIDependencies(
message_bus=message_bus,
event_sender=event_sender,
db_access=db_access,
model_handler=model_handler,
worker_task_queue=task_queue,
args=args,
template_expander=expander,
forked_tasks=Queue(),
config_handler=config_handler,
)
yield deps
await deps.stop()
def test_parse_broken(config_json: Json) -> None:
# config_json is a valid parsable config
cfg = deepcopy(config_json)
# adjust the config: rename web_hosts -> hosts, and web_port -> port
hosts = cfg["resotocore"]["api"]["web_hosts"]
port = cfg["resotocore"]["api"]["web_port"]
cfg["resotocore"]["api"]["hosts"] = hosts
cfg["resotocore"]["api"]["port"] = port
del cfg["resotocore"]["api"]["web_hosts"]
del cfg["resotocore"]["api"]["web_port"]
# parse this configuration
parsed = parse_config(parse_args(["--analytics-opt-out"]), cfg)
parsed_json = to_js(parsed.editable, strip_attr="kind")
# web_hosts and web_port were not available and are reverted to the default values
default = EditableConfig()
assert parsed.api.web_hosts != hosts
assert parsed.api.web_hosts == default.api.web_hosts
assert parsed.api.web_port != port
assert parsed.api.web_port == default.api.web_port
# other config values are still unchanged
assert parsed_json["cli"] == config_json["resotocore"]["cli"]
assert parsed_json["runtime"] == config_json["resotocore"]["runtime"]
assert parsed_json["graph_update"] == config_json["resotocore"][
"graph_update"]
def test_not_existing(system_db: StandardDatabase, test_db: StandardDatabase) -> None:
access = DbAccess(test_db, NoEventSender(), NoAdjust(), empty_config())
# foo db and pass does not exist
foodb = ["--graphdb-username", "foo", "--graphdb-password", "test", "--graphdb-database", "foo"]
system_db.delete_user("foo", ignore_missing=True)
system_db.delete_database("foo", ignore_missing=True)
access.connect(parse_args(foodb), timedelta(seconds=5), sleep_time=0.1)
assert system_db.has_user("foo")
assert system_db.has_database("foo")
def test_already_existing(test_db: StandardDatabase) -> None:
access = DbAccess(test_db, NoEventSender(), NoAdjust())
# test db and user already exist
testdb = [
"--graphdb-username", "test", "--graphdb-password", "test",
"--graphdb-database", "test"
]
access.connect(parse_args(testdb),
timedelta(seconds=0.01),
sleep_time=0.01)
def default_config() -> CoreConfig:
ed = EditableConfig()
return CoreConfig(
api=ed.api,
cli=ed.cli,
db=DatabaseConfig(),
graph_update=ed.graph_update,
# We use this flag explicitly - otherwise it is picked up by env vars
runtime=RuntimeConfig(usage_metrics=False),
workflows=ed.workflows,
custom_commands=CustomCommandsConfig(),
args=parse_args(["--analytics-opt-out"]),
run=RunConfig(),
)
def test_not_existing_and_default_root_account(
local_client: ArangoClient, system_db: StandardDatabase, test_db: StandardDatabase
) -> None:
access = DbAccess(test_db, NoEventSender(), NoAdjust(), empty_config())
# foo db and pass does not exist
foodb = ["--graphdb-username", "foo", "--graphdb-password", "bombproof", "--graphdb-database", "foo"]
system_db.delete_user("foo", ignore_missing=True)
system_db.delete_database("foo", ignore_missing=True)
access.connect(parse_args(foodb), timedelta(seconds=5), sleep_time=0.1)
# The default root account is used and a valid password is given -> also the root account uses this password
changed_root = local_client.db(username="******", password="******")
# Rest the password to the default one, to reset the state before the test
changed_root.replace_user("root", "", True)
def test_bootstrap(test_db: StandardDatabase) -> None:
sd = test_db.collection("system_data")
args = parse_args()
# Delete any existing entry, so a new certificate needs to be created
sd.delete("ca", ignore_missing=True)
handler = CertificateHandler.lookup(args, test_db)
ca = sd.get("ca")
assert ca is not None
# ensure the certificate in the database is the same as exposed by the handler
ca_bytes, fingerprint = handler.authority_certificate
assert ca_bytes == ca["certificate"].encode("utf-8")
# a new handler will use the existing certificate
handler2 = CertificateHandler.lookup(args, test_db)
assert handler.authority_certificate == handler2.authority_certificate
# but the host certificate will be different
assert handler.host_certificate != handler2.host_certificate
async def test_merge_process(
event_sender: AnalyticsEventSender, graph_db: ArangoGraphDB, foo_kinds: List[Kind]
) -> None:
# set explicitly (is done in main explicitly as well)
set_start_method("spawn")
# wipe any existing data
await graph_db.wipe()
# store the model in db, so it can be loaded by the sub process
graph_db.db.collection("model").insert_many([to_js(a) for a in foo_kinds])
# define args to parse for the sub process
args = parse_args(["--graphdb-username", "test", "--graphdb-password", "test", "--graphdb-database", "test"])
# create sample graph data to insert
graph = create_graph("test")
async def iterator() -> AsyncGenerator[bytes, None]:
for node in graph.nodes():
yield bytes(json.dumps(graph.nodes[node]), "utf-8")
for from_node, to_node, data in graph.edges(data=True):
yield bytes(json.dumps({"from": from_node, "to": to_node, "edge_type": data["edge_type"]}), "utf-8")
result = await merge_graph_process(graph_db, event_sender, args, iterator(), timedelta(seconds=30), None)
assert result == GraphUpdate(112, 1, 0, 212, 0, 0)
def run(arguments: List[str]) -> None:
"""
Run application. When this method returns, the process is done.
:param arguments: the arguments provided to this process.
Note: this method is used in tests to specify arbitrary arguments.
"""
args = parse_args(arguments)
setup_process(args)
# after setup, logging is possible
info = system_info()
log.info(
f"Starting up version={info.version} on system with cpus={info.cpus}, "
f"available_mem={info.mem_available}, total_mem={info.mem_total}")
# The loop is here to restart the process in case of RestartService exceptions.
while True:
try:
run_process(args)
break # This line should never be reached. In case it does, break the loop.
except RestartService as ex:
message = f"Restarting Service. Reason: {ex.reason}"
line = "-" * len(message)
print(f"\n{line}\n{message}\n{line}\n")
def test_override_via_cmd_line(default_config: CoreConfig) -> None:
config = {"runtime": {"debug": False}}
parsed = parse_config(parse_args(["--debug"]), config)
assert parsed.runtime.debug == True
def test_read_config(config_json: Json) -> None:
parsed = parse_config(parse_args(["--analytics-opt-out"]), config_json)
assert parsed.json() == config_json
def test_parse_empty(default_config: CoreConfig) -> None:
result = parse_config(parse_args(["--analytics-opt-out"]), {})
assert result == default_config
def run(arguments: List[str]) -> None:
"""
Run application. When this method returns, the process is done.
:param arguments: the arguments provided to this process.
Note: this method is used in tests to specify arbitrary arguments.
"""
args = parse_args(arguments)
setup_process(args)
# after setup, logging is possible
info = system_info()
log.info(
f"Starting up version={info.version} on system with cpus={info.cpus}, "
f"available_mem={info.mem_available}, total_mem={info.mem_total}")
# wait here for an initial connection to the database before we continue. blocking!
created, system_data, sdb = DbAccess.connect(args, timedelta(seconds=60))
event_sender = NoEventSender(
) if args.analytics_opt_out else PostHogEventSender(system_data)
db = db_access(args, sdb, event_sender)
cert_handler = CertificateHandler.lookup(args, sdb)
message_bus = MessageBus()
scheduler = Scheduler()
worker_task_queue = WorkerTaskQueue()
model = ModelHandlerDB(db.get_model_db(), args.plantuml_server)
template_expander = DBTemplateExpander(db.template_entity_db)
config_handler = ConfigHandlerService(
db.config_entity_db,
db.config_validation_entity_db,
db.configs_model_db,
worker_task_queue,
message_bus,
)
cli_deps = CLIDependencies(
message_bus=message_bus,
event_sender=event_sender,
db_access=db,
model_handler=model,
worker_task_queue=worker_task_queue,
args=args,
template_expander=template_expander,
config_handler=config_handler,
)
default_env = {
"graph": args.cli_default_graph,
"section": args.cli_default_section
}
cli = CLI(cli_deps, all_commands(cli_deps), default_env, aliases())
subscriptions = SubscriptionHandler(db.subscribers_db, message_bus)
task_handler = TaskHandlerService(db.running_task_db, db.job_db,
message_bus, event_sender, subscriptions,
scheduler, cli, args)
cli_deps.extend(task_handler=task_handler)
api = Api(
db,
model,
subscriptions,
task_handler,
message_bus,
event_sender,
worker_task_queue,
cert_handler,
config_handler,
cli,
template_expander,
args,
)
event_emitter = emit_recurrent_events(event_sender, model, subscriptions,
worker_task_queue, message_bus,
timedelta(hours=1),
timedelta(hours=1))
async def on_start() -> None:
# queue must be created inside an async function!
cli_deps.extend(forked_tasks=Queue())
await db.start()
await event_sender.start()
await subscriptions.start()
await scheduler.start()
await worker_task_queue.start()
await event_emitter.start()
await cli.start()
await task_handler.start()
await api.start()
if created:
await event_sender.core_event(CoreEvent.SystemInstalled)
await event_sender.core_event(
CoreEvent.SystemStarted,
{
"version": version(),
"created_at": to_json(system_data.created_at),
"system": platform.system(),
"platform": platform.platform(),
"inside_docker": info.inside_docker,
},
cpu_count=info.cpus,
mem_total=info.mem_total,
mem_available=info.mem_available,
)
async def on_stop() -> None:
duration = utc() - info.started_at
await api.stop()
await task_handler.stop()
await cli.stop()
await event_sender.core_event(CoreEvent.SystemStopped,
total_seconds=int(
duration.total_seconds()))
await event_emitter.stop()
await worker_task_queue.stop()
await scheduler.stop()
await subscriptions.stop()
await db.stop()
await event_sender.stop()
async def async_initializer() -> Application:
async def on_start_stop(_: Application) -> AsyncIterator[None]:
await on_start()
log.info("Initialization done. Starting API.")
yield
log.info("Shutdown initiated. Stop all tasks.")
await on_stop()
api.app.cleanup_ctx.append(on_start_stop)
return api.app
tls_context: Optional[SSLContext] = None
if args.tls_cert:
tls_context = SSLContext(ssl.PROTOCOL_TLS)
tls_context.load_cert_chain(args.tls_cert, args.tls_key,
args.tls_password)
runner.run_app(async_initializer(),
api.stop,
host=args.host,
port=args.port,
ssl_context=tls_context)
def cert_handler() -> CertificateHandler:
args = parse_args()
key, certificate = bootstrap_ca()
return CertificateHandler(args, key, certificate)
def task_handler_args() -> Namespace:
return parse_args()
