def create_indices(db: StandardDatabase):
db_collection = db.collection(COLLECTION_PARALLELS)
db_collection.add_hash_index(["root_filename"], unique=False)
db_collection.add_hash_index(["src_lang"], unique=False)
db_collection = db.collection(COLLECTION_FILES)
db_collection.add_hash_index(["language"], unique=False)
db_collection.add_hash_index(["category"], unique=False)
async def post_users_password_reset(
change_data: PasswordReset,
user: User = Depends(get_current_confirmed_user),
db: StandardDatabase = Depends(database.get),
):
"""
Changes the password given a `request_token`.
The `request_token` is a token sent to the user's email address as a link
(something like `https://app.schoolsyst.com/password-reset/{request_token}`) after
performing a `POST /users/password-reset-request`.
"""
analysis = get_password_analysis(
change_data.new_password, user.email, user.username
)
if analysis and not is_password_strong_enough(analysis):
return HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail="This password is not strong enough",
headers={"X-See": "GET /password_analysis/"},
)
# validate the token
if not verify_jwt_token(change_data.request_token, JWT_SUB_FORMAT, user.username):
return HTTPException(
status.HTTP_403_FORBIDDEN,
detail=post_users_password_reset_responses[403]["description"],
)
# change the password
db.collection("users").update(
{"_key": user.key, "password_hash": hash_password(change_data.new_password)}
)
return
def reset_all_settings(
db: StandardDatabase = Depends(database.get),
current_user: User = Depends(get_current_confirmed_user),
) -> Settings:
# instead of deleting and re-inserting, update with a completely new object.
settings = Settings(_key=current_user.key, updated_at=datetime.now())
db.collection("settings").update(json.loads(settings.json(by_alias=True)))
return settings
def insert_mocks(
db: StandardDatabase,
collection_name: str,
):
mock_objects: list[BaseModel] = [
getattr(getattr(tests.mocks, collection_name), m)
for m in dir(getattr(tests.mocks, collection_name))
if not m.startswith("__")
]
for mock in mock_objects:
db.collection(collection_name).insert(mock.json(by_alias=True))
def delete(db: StandardDatabase, game: Game, requester=None):
"""
delete all objects associated with a game
:param db: connection
:param game: target
:param requester: player requesting the delete
"""
logger.info(f'delete called for {game.title}')
db_npcs = db.collection('npcs')
db_dialogs = db.collection('dialogs')
tasks = db.collection('tasks')
games = db.collection('games')
db_game = next(games.find({'_key': game.title}), None)
if not db_game:
logger.warning(f'game {game.title} not in metadata')
raise GameStateException(f'game {game.title} not in metadata')
if not db.has_graph(f'game_{game.title}'):
logger.warning(f'game {game.title} not in metadata')
raise GameStateException(f'game {game.title} does not exist')
if db_game['creator'] and not db_game['creator'] == requester:
raise GameStateException(f'cannot delete game {game.title}, you are not the owner')
for db_npc in db_npcs.find({'game': f'game_{game.title}'}):
db_dialog = next(db_dialogs.find({'_key': db_npc['dialog']}), None)
if not db.has_graph(f"dialog_{db_npc['dialog']}"):
logger.warning(f"dialog {db_npc['dialog']} does not exist")
raise GameStateException(f"dialog {db_npc['dialog']} does not exist")
for k, v in dict(db.graph(f"dialog_{db_npc['dialog']}").traverse(start_vertex=f"interactions/{db_dialog['start']}",
direction='outbound',
strategy='dfs',
edge_uniqueness='global',
vertex_uniqueness='global')).items():
if k == 'vertices':
for interaction in v:
if 'task' in interaction and interaction['task']:
db_task = next(tasks.find({'_key': interaction['task']}))
if db_task:
db.delete_document(db_task)
db.delete_document(db_dialog)
db.delete_graph(f"dialog_{db_npc['dialog']}", drop_collections=True)
db.delete_document(db_npc)
for node in nx.dfs_tree(game.graph):
for task in node.tasks:
db_task = next(tasks.find({'_key': task.id.hex}), None)
if db_task:
db.delete_document(db_task)
db.delete_document(db_game)
db.delete_graph(f'game_{game.title}', drop_collections=True)
def validate(db: StandardDatabase, token, email=None):
"""
validate player token
:param db: connection
:param token: jwt token
:param email: mail
:return Player or None
"""
if not db.has_collection('players'):
logger.error('cannot validate token if players do not exist')
raise PlayerStateException('players collection does not exist')
col = db.collection('players')
if email:
db_player = next(col.find({'mail': email}), None)
if not db_player:
logger.error(f'could not resolve player {email}')
raise PlayerStateException(f'player {email} does not exist')
if token in db_player['tokens']:
player = Player(db_player['mail'], '')
player.id = UUID(db_player['_key'])
player.password = db_player['password']
return player
else:
for db_player in col.all():
if 'tokens' in db_player and token in db_player['tokens']:
player = Player(db_player['mail'], '')
player.id = UUID(db_player['_key'])
player.password = db_player['password']
return player
return None
def load_parallels_sorted(json_parallels: [Parallel], db: StandardDatabase,
filename: str) -> None:
"""
Given an array of parallel objects, load them all into the `sorted parallels` collection presorted.
:param json_parallels: Array of parallel objects to be loaded as-they-are.
:param db: ArangoDB connection object
"""
db_collection_sorted = db.collection(COLLECTION_PARALLELS_SORTED_BY_FILE)
# I wonder if this can be done more efficiently, a lot of spaghetti code...
parallels_sorted_by_src_position = sorted(json_parallels,
key=lambda k: k['root_pos_beg'])
ids_sorted_by_src_position = list(
map(lambda parallel: parallel['id'], parallels_sorted_by_src_position))
parallels_sorted_by_tgt_position = sorted(
json_parallels, key=lambda k: natural_keys(k['par_segnr'][0]))
ids_sorted_by_tgt_position = list(
map(lambda parallel: parallel['id'], parallels_sorted_by_tgt_position))
parallels_sorted_by_length_par = sorted(json_parallels,
key=lambda k: k['par_length'])
ids_sorted_by_length_par = list(
map(lambda parallel: parallel['id'], parallels_sorted_by_length_par))
parallels_sorted_by_length_root = sorted(json_parallels,
key=lambda k: k['root_length'])
ids_sorted_by_length_root = list(
map(lambda parallel: parallel['id'], parallels_sorted_by_length_root))
ids_sorted_by_length_par.reverse()
ids_sorted_by_length_root.reverse()
ids_sorted_random = ids_sorted_by_length_root
random.shuffle(ids_sorted_random)
src_lang = json_parallels[0]['src_lang']
try:
db_collection_sorted.insert({
'_key':
filename,
'filename':
filename,
'lang':
src_lang,
'parallels_sorted_by_src_pos':
ids_sorted_by_src_position,
'parallels_sorted_by_tgt_pos':
ids_sorted_by_tgt_position,
'parallels_sorted_by_length_src':
ids_sorted_by_length_root,
'parallels_randomized':
ids_sorted_random,
'parallels_sorted_by_length_tgt':
ids_sorted_by_length_par
})
except (DocumentInsertError, IndexCreateError) as e:
print(f"Could not save sorted parallel for {filename}. Error: ", e)
def load_file_parallel_counts(
file: MenuItem,
total_length_count: list, # TODO: this is not typed correctly
total_file_length_count: list, # TODO: same as above
db: StandardDatabase,
):
db_collection = db.collection(COLLECTION_FILES_PARALLEL_COUNT)
sorted_total_file_length_count = sorted(total_file_length_count.items(),
key=lambda kv: kv[1],
reverse=True)
doc = {
"_key": file["filename"],
"category": file["category"],
"language": get_language_from_filename(file["filename"]),
"filenr": file["filenr"],
"totallengthcount": total_length_count,
"totalfilelengthcount": OrderedDict(sorted_total_file_length_count),
"totallength": sum(total_length_count.values()),
}
try:
db_collection.add_hash_index(["category"], unique=False)
if db_collection.get(file['filename']):
db_collection.delete(file['filename'])
db_collection.insert(doc)
except (DocumentInsertError, IndexCreateError) as e:
print("Could not load file. Error: ", e)
def load_segment(json_segment: Segment, count: int, parallel_ids: list,
parallel_ids_limited: list, db: StandardDatabase) -> str:
"""
Given a single segment object, load it into the `segments` collection.
:param json_segment: Segment JSON data
:param count: Incrementing number of segments
:param parallel_ids: Array of IDs of parallels of which segment is root
:param db: ArangoDB database object
:return: Segment nr
"""
collection = db.collection(COLLECTION_SEGMENTS)
try:
doc = {
"_key": json_segment["segnr"],
"_id": f'segments/{json_segment["segnr"]}',
"segnr": json_segment["segnr"],
"segtext": json_segment["segtext"],
"lang": json_segment["lang"],
"position": json_segment["position"],
"count": count,
"parallel_ids": parallel_ids,
"parallel_ids_limited": parallel_ids_limited
}
collection.insert(doc)
except (KeyError, AttributeError) as e:
print("Could not load segment. Error: ", e)
except DocumentInsertError as e:
print(f"Could not save segment {json_segment['segnr']}. Error: ", e)
return json_segment["segnr"]
def get_all_dialogs(db: StandardDatabase) -> [str]:
"""
get all dialog ids
:param db: connection
:return: list of all dialogs in the system or empty list
"""
return [d['_key'] for d in db.collection('dialogs') if '_key' in d]
def get_all_games(db: StandardDatabase) -> [str]:
"""
get all game titles
:param db: connection
:return: list of all game names or empty list
"""
return [d['_key'] for d in db.collection('games')]
def lookup(config: CoreConfig, db: StandardDatabase, temp_dir: Path) -> CertificateHandler:
args = config.args
# if we get a ca certificate from the command line, use it
if args.ca_cert and args.ca_cert_key:
ca_key = load_key_from_file(args.ca_cert_key, args.ca_cert_key_pass)
ca_cert = load_cert_from_file(args.ca_cert_cert)
log.info(f"Using CA certificate from command line. fingerprint:{cert_fingerprint(ca_cert)}")
return CertificateHandler(config, ca_key, ca_cert, temp_dir)
# otherwise, load from database or create it
sd = db.collection("system_data")
maybe_ca = sd.get("ca")
if maybe_ca and isinstance(maybe_ca.get("key"), str) and isinstance(maybe_ca.get("certificate"), str):
log.debug("Found existing certificate in data store.")
key = load_key_from_bytes(maybe_ca["key"].encode("utf-8"))
certificate = load_cert_from_bytes(maybe_ca["certificate"].encode("utf-8"))
log.info(f"Using CA certificate from database. fingerprint:{cert_fingerprint(certificate)}")
return CertificateHandler(config, key, certificate, temp_dir)
else:
wo = "with" if args.ca_cert_key_pass else "without"
key, certificate = bootstrap_ca()
log.info(
f"No ca certificate found - create new one {wo} passphrase. fingerprint:{cert_fingerprint(certificate)}"
)
key_string = key_to_bytes(key, args.ca_cert_key_pass).decode("utf-8")
certificate_string = cert_to_bytes(certificate).decode("utf-8")
sd.insert({"_key": "ca", "key": key_string, "certificate": certificate_string})
return CertificateHandler(config, key, certificate, temp_dir)
def create_user_account(
user_in: InUser, db: StandardDatabase = Depends(database.get)) -> User:
"""
Create a user account.
Emails and usernames are unique, usernames are _not_ case-sensitive.
The password must be strong enough. See GET /password_analysis/
"""
# Check if the username is not disallowed
if is_username_disallowed(user_in.username):
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail="This username is not allowed.",
)
# Check if the username is not already taken
if is_username_taken(db, user_in.username):
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail="This username is already taken",
)
# Check if the email is not already taken
if is_email_taken(db, user_in.email):
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail="This email is already taken",
)
# Check if password is strong enough
password_analysis = get_password_analysis(**user_in.dict())
if password_analysis and not is_password_strong_enough(password_analysis):
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail={
"message": "The password is not strong enough",
"analysis": password_analysis,
},
)
# Create the DBUser
db_user = DBUser(
joined_at=datetime.utcnow(),
email_is_confirmed=False,
password_hash=hash_password(user_in.password),
**user_in.dict(),
)
db.collection("users").insert(db_user.json(by_alias=True))
# Return a regular User
return User(**db_user.dict(by_alias=True))
async def get_personal_data_archive(
user: User = Depends(get_current_confirmed_user),
db: StandardDatabase = Depends(database.get),
) -> dict:
"""
Get an archive of all of the data linked to the user.
"""
data = {}
# The user's data
data["user"] = db.collection("users").get(user.key)
# the data of which the user is the owner for every collection
for c in COLLECTIONS:
if c == "users":
continue
data[c] = [
batch for batch in db.collection(c).find({"owner_key": user.key})
]
return data
def create(self, db: StandardDatabase, current_user: User, data):
return self.model_out(
**db.collection(self.name_pl).insert(
self.model_out(**data.dict(), owner_key=current_user.key).json(
by_alias=True
),
return_new=True,
)["new"]
)
def load_all_menu_categories(db: StandardDatabase):
categories_db_collection = db.collection(COLLECTION_MENU_CATEGORIES)
category_has_files_edge_db_collection = db.collection(
EDGE_COLLECTION_CATEGORY_HAS_FILES)
for language in DEFAULT_LANGS:
with open(f"../data/{language}-categories.json") as f:
print(f"Loading menu categories in {language}...")
categories = json.load(f)
category_count = 0
for category in categories:
load_menu_category(
category,
category_count,
language,
categories_db_collection,
category_has_files_edge_db_collection,
)
category_count += 1
print("✓")
def delete(self, db: StandardDatabase, current_user: User, key: ObjectBareKey):
full_key = OBJECT_KEY_FORMAT.format(object=key, owner=current_user.key)
resource = db.collection(self.name_pl).get(full_key)
if resource is None:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail=f"No {self.name_sg} with key {full_key} found",
)
resource = self.model_out(**resource)
if resource.owner_key != current_user.key:
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN,
detail="Currently logged-in user does not own the specified subject",
)
db.collection(self.name_pl).delete(full_key)
return Response(status_code=status.HTTP_204_NO_CONTENT)
def get_all_player_emails(db: StandardDatabase) -> [str]:
"""
return all player email addresses
:param db: connection
:return: list of known mail addresses
"""
if not db.has_collection('players'):
logger.error('cannot resolve player if players do not exist')
raise PlayerStateException('players collection does not exist')
col = db.collection('players')
return [p['mail'] for p in col.all()]
def load_search_index_chn(path, db: StandardDatabase):
with gzip.open(path) as f:
print(f"\nLoading file index data Chinese...")
index_data = json.load(f)
print(f"\nInserting file index data Chinese into DB...")
collection = db.collection(COLLECTION_SEARCH_INDEX_CHN)
chunksize = 10000
for i in tqdm(range(0, len(index_data), chunksize)):
collection.insert_many(index_data[i : i + chunksize])
print(f"\nDone loading index data Chn...")
print("\nDone creating View for Chinese")
def get_user(db: StandardDatabase, username: str) -> Optional[DBUser]:
"""
Get a user by username from the DB.
Returns `None` if the user is not found.
"""
# Usernames are not case-sensitive
user_dict = db.collection("users").find({
"username": username.lower()
}).batch()
if not user_dict:
return None
return DBUser(**user_dict[0])
def load_sources(db: StandardDatabase, root_url):
source_json_path = root_url + "sources.json"
db_file_collection = db.collection(COLLECTION_FILES)
with open(source_json_path, "rb") as f:
source_list = json.load(f)
for entry in source_list:
filename = entry['filename']
current_file = db_file_collection.get(filename)
if current_file:
current_file['source_string'] = entry['source']
print(current_file)
db_file_collection.update(current_file)
def load_search_index_tib(path, db: StandardDatabase):
with gzip.open(path) as f:
print(f"\nLoading file index data Tibetan...")
index_data = json.load(f)
print(f"\nInserting file index data Tibetan into DB...")
collection = db.collection(COLLECTION_SEARCH_INDEX_TIB)
# we have to do this in chunk, otherwise it will fail with broken_pipe
chunksize = 10000
for i in tqdm(range(0, len(index_data), chunksize)):
collection.insert_many(index_data[i : i + chunksize])
print(f"\nDone loading Tibetan index data...")
print("\nDone creating View")
def update(
self, db: StandardDatabase, current_user: User, key: ObjectBareKey, changes
):
resource = self.get(db, current_user, key)
updated_resource = {
**json.loads(resource.json()),
**json.loads(changes.json(exclude_unset=True)),
"updated_at": datetime.now().isoformat(sep="T"),
}
new_resource = db.collection(self.name_pl).update(
updated_resource, return_new=True
)["new"]
return self.model_out(**new_resource)
def load_parallels(json_parallels: [Parallel], db: StandardDatabase) -> None:
"""
Given an array of parallel objects, load them all into the `parallels` collection
:param json_parallels: Array of parallel objects to be loaded as-they-are.
:param db: ArangoDB connection object
"""
db_collection = db.collection(COLLECTION_PARALLELS)
db_collection_sorted = db.collection(COLLECTION_PARALLELS_SORTED_BY_FILE)
parallels_to_be_inserted = []
root_file_parallel_edges_to_be_inserted = []
for parallel in json_parallels:
if isinstance(parallel, dict):
root_filename = parallel["root_segnr"][0].split(":")[0]
root_filename = re.sub("_[0-9][0-9][0-9]", "", root_filename)
parallel_id = f"parallels/{parallel['id']}"
parallel["_key"] = parallel["id"]
parallel["_id"] = parallel_id
# here we delete some things that we don't need in the DB:
del parallel["par_pos_end"]
del parallel["root_pos_end"]
del parallel["par_segtext"]
del parallel["root_segtext"]
del parallel["par_string"]
del parallel["root_string"]
# todo: delete the root_filename key after it's not needed anymore
parallel["root_filename"] = root_filename
parallels_to_be_inserted.append(parallel)
chunksize = 10000
for i in range(0, len(parallels_to_be_inserted), chunksize):
try:
db_collection.insert_many(parallels_to_be_inserted[i:i +
chunksize])
except (DocumentInsertError, IndexCreateError) as e:
print(f"Could not save parallel {parallel}. Error: ", e)
def update_settings(
changes: InSettings,
db: StandardDatabase = Depends(database.get),
settings: Settings = Depends(settings.get),
) -> Settings:
updated_settings = {
**json.loads(settings.json(by_alias=True)),
**json.loads(changes.json(exclude_unset=True)),
"updated_at":
datetime.now().isoformat(sep="T"),
}
new_settings = db.collection("settings").update(updated_settings,
return_new=True)["new"]
return Settings(**new_settings)
def get(
db: StandardDatabase = Depends(database.get),
current_user: User = Depends(get_current_confirmed_user),
) -> Settings:
"""
Gets the settings for the current user.
If the settings are created when getting them (if the user has no settings tied to him),
the return value is:
(settings, True)
and else:
(settings, False)
"""
doc = db.collection("settings").get(current_user.key)
# If the user has no settings tied to him, create them with the default values.
if doc is None:
doc = db.collection("settings").insert(
Settings(_key=current_user.key).json(by_alias=True), return_new=True
)["new"]
return Settings(**doc)
async def delete_current_user(
user: User = Depends(get_current_user),
really_delete: bool = False,
db: StandardDatabase = Depends(database.get),
):
"""
Deletes the currently-logged-in user, and all of the associated resources.
This action does not require the user to have confirmed its email address.
"""
if not really_delete:
return HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail="Set really_delete to True to confirm deletion",
)
db.collection("users").delete(user.key)
for c in COLLECTIONS:
if c == "users":
continue
db.collection(c).delete_match({"owner_key": user.key})
return Response(status_code=status.HTTP_204_NO_CONTENT)
def reset_setting(
setting_key: SettingKey,
db: StandardDatabase = Depends(database.get),
current_user: User = Depends(get_current_confirmed_user),
) -> Settings:
default_settings = InSettings()
new_settings = db.collection("settings").update(
{
"_key": current_user.key,
setting_key: json.loads(default_settings.json())[setting_key],
},
return_new=True,
)["new"]
return Settings(**new_settings)
def delete(db: StandardDatabase, player: Player):
"""
remove existing player
:param db: connection
:param player: target
"""
if not db.has_collection('players'):
logger.error('cannot remove player if players do not exist')
raise PlayerStateException(f'players collection does not exist')
col = db.collection('players')
db_player = next(col.find({'mail': player.email}), None)
if not db_player:
logger.error(f'cannot find player {player.email}')
raise PlayerStateException(f'could not find player {player.email}')
col.delete(db_player)
def read(db: StandardDatabase, player_id):
"""
find existing player
:param db: connection
:param player_id: id
:return: Player or None
"""
if not db.has_collection('players'):
logger.error('cannot resolve player if players do not exist')
raise PlayerStateException('players collection does not exist')
col = db.collection('players')
db_player = next(col.find({'_key': player_id}), None)
if not db_player:
return None
return Player(db_player['email'], db_player['password'])
def __init__(self, adb: StandardDatabase):
"""
:param adb: connection to arangodb
"""
self.adb = adb
self.collection = adb.collection("leases")
