async def merge_graph(self, db: DbAccess) -> GraphUpdate: # type: ignore
model = Model.from_kinds([kind async for kind in db.model_db.all()])
builder = GraphBuilder(model)
nxt = self.next_action()
while isinstance(nxt, ReadElement):
for element in nxt.jsons():
builder.add_from_json(element)
log.debug(f"Read {int(BatchSize / 1000)}K elements in process")
nxt = self.next_action()
if isinstance(nxt, PoisonPill):
log.debug("Got poison pill - going to die.")
shutdown_process(0)
elif isinstance(nxt, MergeGraph):
log.debug("Graph read into memory")
builder.check_complete()
graphdb = db.get_graph_db(nxt.graph)
outer_edge_db = db.get_pending_outer_edge_db()
_, result = await graphdb.merge_graph(builder.graph, model,
nxt.change_id, nxt.is_batch)
if nxt.task_id and builder.deferred_edges:
await outer_edge_db.update(
PendingDeferredEdges(nxt.task_id, nxt.graph,
builder.deferred_edges))
log.debug(
f"Updated {len(builder.deferred_edges)} pending outer edges for collect task {nxt.task_id}"
)
return result
def client(args: Namespace) -> StandardDatabase:
if args.graphdb_type not in "arangodb":
log.fatal(f"Unknown Graph DB type {args.graphdb_type}")
shutdown_process(1)
http_client = ArangoHTTPClient(args.graphdb_request_timeout,
not args.graphdb_no_ssl_verify)
client = ArangoClient(hosts=args.graphdb_server,
http_client=http_client)
return client.db(args.graphdb_database,
username=args.graphdb_username,
password=args.graphdb_password)
def main() -> None:
"""
Application entrypoint - no arguments are allowed.
"""
try:
run(sys.argv[1:])
log.info("Process finished.")
except (KeyboardInterrupt, SystemExit):
log.info("Stopping resoto graph core.")
shutdown_process(0)
except Exception as ex:
print(f"resotocore stopped. Reason {class_fqn(ex)}: {ex}",
file=sys.stderr)
shutdown_process(1)
def run(self) -> None:
try:
# Entrypoint of the new service
setup_process(self.args, f"merge_update_{self.pid}")
log.info(f"Import process started: {self.pid}")
result = asyncio.run(self.setup_and_merge())
self.write_queue.put(Result(result))
log.info(f"Update process done: {self.pid} Exit.")
shutdown_process(0)
except Exception as ex:
# not all exceptions can be pickled. Use string representation.
self.write_queue.put(Result(repr(ex)))
log.error(f"Update process interrupted. Preemptive Exit. {ex}",
exc_info=ex)
shutdown_process(1)
def connect(
cls,
args: Namespace,
timeout: timedelta,
sleep_time: float = 5,
verify: Union[str, bool, None] = None
) -> Tuple[bool, SystemData, StandardDatabase]:
deadline = utc() + timeout
db = cls.client(args, verify)
def create_database() -> None:
try:
# try to access the system database with default credentials.
# this only works if arango has been started with default settings.
http_client = ArangoHTTPClient(args.graphdb_request_timeout,
not args.graphdb_no_ssl_verify)
root_pw = args.graphdb_root_password
secure_root = not args.graphdb_bootstrap_do_not_secure
root_db = ArangoClient(
hosts=args.graphdb_server,
http_client=http_client).db(password=root_pw)
root_db.echo(
) # this call will fail, if we are not allowed to access the system db
user = args.graphdb_username
passwd = args.graphdb_password
database = args.graphdb_database
change = False
if not root_db.has_user(user):
log.info(
"Configured graph db user does not exist. Create it.")
root_db.create_user(user, passwd, active=True)
change = True
if not root_db.has_database(database):
log.info(
"Configured graph db database does not exist. Create it."
)
root_db.create_database(
database,
[{
"username": user,
"password": passwd,
"active": True,
"extra": {
"generated": "resoto"
}
}],
)
change = True
if change and secure_root and root_pw == "" and passwd != "" and passwd not in {
"test"
}:
root_db.replace_user("root", passwd, True)
log.info(
"Database is using an empty password. "
"Secure the root account with the provided user password. "
"Login to the Resoto database via provided username and password. "
"Login to the System database via `root` and provided password!"
)
if not change:
log.info(
"Not allowed to access database, while user and database exist. Wrong password?"
)
except Exception as ex:
log.error(
"Database or user does not exist or does not have enough permissions. "
f"Attempt to create user/database via default system account is not possible. Reason: {ex}. "
"You can provide the password of the root user via --graphdb-root-password to setup "
"a Resoto user and database automatically.")
def system_data() -> Tuple[bool, SystemData]:
def insert_system_data() -> SystemData:
system = SystemData(uuid_str(), utc(), 1)
log.info(f"Create new system data entry: {system}")
db.insert_document("system_data", {
"_key": "system",
**to_js(system)
},
overwrite=True)
return system
if not db.has_collection("system_data"):
db.create_collection("system_data")
sys_js = db.collection("system_data").get("system")
return (True, insert_system_data()) if not sys_js else (
False, from_js(sys_js, SystemData))
while True:
try:
db.echo()
try:
db_version = int(db.required_db_version())
except Exception as ex:
log.warning(
f"Not able to retrieve version of arangodb. Reason: {ex}. Continue."
)
else:
if db_version < 30802:
raise RequiredDependencyMissingError(
"Need arangodb in version 3.8.2 or later")
created, sys_data = system_data()
return created, sys_data, db
except ArangoServerError as ex:
if utc() > deadline:
log.error("Can not connect to database. Giving up.")
shutdown_process(1)
elif ex.error_code in (11, 1228, 1703):
# https://www.arangodb.com/docs/stable/appendix-error-codes.html
# This means we can reach the database, but are either not allowed to access it
# or the related user and or database could not be found.
# We assume the database does not exist and try to create it.
create_database()
else:
log.warning(
f"Problem accessing the graph database: {ex}. Trying again in 5 seconds."
)
# Retry directly after the first attempt
sleep(sleep_time)
except (RequestException, ConnectionError) as ex:
log.warning(
f"Can not access database. Trying again in 5 seconds: {ex}"
)
sleep(sleep_time)