def create_indices(db: StandardDatabase): db_collection = db.collection(COLLECTION_PARALLELS) db_collection.add_hash_index(["root_filename"], unique=False) db_collection.add_hash_index(["src_lang"], unique=False) db_collection = db.collection(COLLECTION_FILES) db_collection.add_hash_index(["language"], unique=False) db_collection.add_hash_index(["category"], unique=False)
async def post_users_password_reset( change_data: PasswordReset, user: User = Depends(get_current_confirmed_user), db: StandardDatabase = Depends(database.get), ): """ Changes the password given a `request_token`. The `request_token` is a token sent to the user's email address as a link (something like `https://app.schoolsyst.com/password-reset/{request_token}`) after performing a `POST /users/password-reset-request`. """ analysis = get_password_analysis( change_data.new_password, user.email, user.username ) if analysis and not is_password_strong_enough(analysis): return HTTPException( status_code=status.HTTP_400_BAD_REQUEST, detail="This password is not strong enough", headers={"X-See": "GET /password_analysis/"}, ) # validate the token if not verify_jwt_token(change_data.request_token, JWT_SUB_FORMAT, user.username): return HTTPException( status.HTTP_403_FORBIDDEN, detail=post_users_password_reset_responses[403]["description"], ) # change the password db.collection("users").update( {"_key": user.key, "password_hash": hash_password(change_data.new_password)} ) return
def reset_all_settings( db: StandardDatabase = Depends(database.get), current_user: User = Depends(get_current_confirmed_user), ) -> Settings: # instead of deleting and re-inserting, update with a completely new object. settings = Settings(_key=current_user.key, updated_at=datetime.now()) db.collection("settings").update(json.loads(settings.json(by_alias=True))) return settings
def insert_mocks( db: StandardDatabase, collection_name: str, ): mock_objects: list[BaseModel] = [ getattr(getattr(tests.mocks, collection_name), m) for m in dir(getattr(tests.mocks, collection_name)) if not m.startswith("__") ] for mock in mock_objects: db.collection(collection_name).insert(mock.json(by_alias=True))
def delete(db: StandardDatabase, game: Game, requester=None): """ delete all objects associated with a game :param db: connection :param game: target :param requester: player requesting the delete """ logger.info(f'delete called for {game.title}') db_npcs = db.collection('npcs') db_dialogs = db.collection('dialogs') tasks = db.collection('tasks') games = db.collection('games') db_game = next(games.find({'_key': game.title}), None) if not db_game: logger.warning(f'game {game.title} not in metadata') raise GameStateException(f'game {game.title} not in metadata') if not db.has_graph(f'game_{game.title}'): logger.warning(f'game {game.title} not in metadata') raise GameStateException(f'game {game.title} does not exist') if db_game['creator'] and not db_game['creator'] == requester: raise GameStateException(f'cannot delete game {game.title}, you are not the owner') for db_npc in db_npcs.find({'game': f'game_{game.title}'}): db_dialog = next(db_dialogs.find({'_key': db_npc['dialog']}), None) if not db.has_graph(f"dialog_{db_npc['dialog']}"): logger.warning(f"dialog {db_npc['dialog']} does not exist") raise GameStateException(f"dialog {db_npc['dialog']} does not exist") for k, v in dict(db.graph(f"dialog_{db_npc['dialog']}").traverse(start_vertex=f"interactions/{db_dialog['start']}", direction='outbound', strategy='dfs', edge_uniqueness='global', vertex_uniqueness='global')).items(): if k == 'vertices': for interaction in v: if 'task' in interaction and interaction['task']: db_task = next(tasks.find({'_key': interaction['task']})) if db_task: db.delete_document(db_task) db.delete_document(db_dialog) db.delete_graph(f"dialog_{db_npc['dialog']}", drop_collections=True) db.delete_document(db_npc) for node in nx.dfs_tree(game.graph): for task in node.tasks: db_task = next(tasks.find({'_key': task.id.hex}), None) if db_task: db.delete_document(db_task) db.delete_document(db_game) db.delete_graph(f'game_{game.title}', drop_collections=True)
def validate(db: StandardDatabase, token, email=None): """ validate player token :param db: connection :param token: jwt token :param email: mail :return Player or None """ if not db.has_collection('players'): logger.error('cannot validate token if players do not exist') raise PlayerStateException('players collection does not exist') col = db.collection('players') if email: db_player = next(col.find({'mail': email}), None) if not db_player: logger.error(f'could not resolve player {email}') raise PlayerStateException(f'player {email} does not exist') if token in db_player['tokens']: player = Player(db_player['mail'], '') player.id = UUID(db_player['_key']) player.password = db_player['password'] return player else: for db_player in col.all(): if 'tokens' in db_player and token in db_player['tokens']: player = Player(db_player['mail'], '') player.id = UUID(db_player['_key']) player.password = db_player['password'] return player return None
def load_parallels_sorted(json_parallels: [Parallel], db: StandardDatabase, filename: str) -> None: """ Given an array of parallel objects, load them all into the `sorted parallels` collection presorted. :param json_parallels: Array of parallel objects to be loaded as-they-are. :param db: ArangoDB connection object """ db_collection_sorted = db.collection(COLLECTION_PARALLELS_SORTED_BY_FILE) # I wonder if this can be done more efficiently, a lot of spaghetti code... parallels_sorted_by_src_position = sorted(json_parallels, key=lambda k: k['root_pos_beg']) ids_sorted_by_src_position = list( map(lambda parallel: parallel['id'], parallels_sorted_by_src_position)) parallels_sorted_by_tgt_position = sorted( json_parallels, key=lambda k: natural_keys(k['par_segnr'][0])) ids_sorted_by_tgt_position = list( map(lambda parallel: parallel['id'], parallels_sorted_by_tgt_position)) parallels_sorted_by_length_par = sorted(json_parallels, key=lambda k: k['par_length']) ids_sorted_by_length_par = list( map(lambda parallel: parallel['id'], parallels_sorted_by_length_par)) parallels_sorted_by_length_root = sorted(json_parallels, key=lambda k: k['root_length']) ids_sorted_by_length_root = list( map(lambda parallel: parallel['id'], parallels_sorted_by_length_root)) ids_sorted_by_length_par.reverse() ids_sorted_by_length_root.reverse() ids_sorted_random = ids_sorted_by_length_root random.shuffle(ids_sorted_random) src_lang = json_parallels[0]['src_lang'] try: db_collection_sorted.insert({ '_key': filename, 'filename': filename, 'lang': src_lang, 'parallels_sorted_by_src_pos': ids_sorted_by_src_position, 'parallels_sorted_by_tgt_pos': ids_sorted_by_tgt_position, 'parallels_sorted_by_length_src': ids_sorted_by_length_root, 'parallels_randomized': ids_sorted_random, 'parallels_sorted_by_length_tgt': ids_sorted_by_length_par }) except (DocumentInsertError, IndexCreateError) as e: print(f"Could not save sorted parallel for {filename}. Error: ", e)
def load_file_parallel_counts( file: MenuItem, total_length_count: list, # TODO: this is not typed correctly total_file_length_count: list, # TODO: same as above db: StandardDatabase, ): db_collection = db.collection(COLLECTION_FILES_PARALLEL_COUNT) sorted_total_file_length_count = sorted(total_file_length_count.items(), key=lambda kv: kv[1], reverse=True) doc = { "_key": file["filename"], "category": file["category"], "language": get_language_from_filename(file["filename"]), "filenr": file["filenr"], "totallengthcount": total_length_count, "totalfilelengthcount": OrderedDict(sorted_total_file_length_count), "totallength": sum(total_length_count.values()), } try: db_collection.add_hash_index(["category"], unique=False) if db_collection.get(file['filename']): db_collection.delete(file['filename']) db_collection.insert(doc) except (DocumentInsertError, IndexCreateError) as e: print("Could not load file. Error: ", e)
def load_segment(json_segment: Segment, count: int, parallel_ids: list, parallel_ids_limited: list, db: StandardDatabase) -> str: """ Given a single segment object, load it into the `segments` collection. :param json_segment: Segment JSON data :param count: Incrementing number of segments :param parallel_ids: Array of IDs of parallels of which segment is root :param db: ArangoDB database object :return: Segment nr """ collection = db.collection(COLLECTION_SEGMENTS) try: doc = { "_key": json_segment["segnr"], "_id": f'segments/{json_segment["segnr"]}', "segnr": json_segment["segnr"], "segtext": json_segment["segtext"], "lang": json_segment["lang"], "position": json_segment["position"], "count": count, "parallel_ids": parallel_ids, "parallel_ids_limited": parallel_ids_limited } collection.insert(doc) except (KeyError, AttributeError) as e: print("Could not load segment. Error: ", e) except DocumentInsertError as e: print(f"Could not save segment {json_segment['segnr']}. Error: ", e) return json_segment["segnr"]
def get_all_dialogs(db: StandardDatabase) -> [str]: """ get all dialog ids :param db: connection :return: list of all dialogs in the system or empty list """ return [d['_key'] for d in db.collection('dialogs') if '_key' in d]
def get_all_games(db: StandardDatabase) -> [str]: """ get all game titles :param db: connection :return: list of all game names or empty list """ return [d['_key'] for d in db.collection('games')]
def lookup(config: CoreConfig, db: StandardDatabase, temp_dir: Path) -> CertificateHandler: args = config.args # if we get a ca certificate from the command line, use it if args.ca_cert and args.ca_cert_key: ca_key = load_key_from_file(args.ca_cert_key, args.ca_cert_key_pass) ca_cert = load_cert_from_file(args.ca_cert_cert) log.info(f"Using CA certificate from command line. fingerprint:{cert_fingerprint(ca_cert)}") return CertificateHandler(config, ca_key, ca_cert, temp_dir) # otherwise, load from database or create it sd = db.collection("system_data") maybe_ca = sd.get("ca") if maybe_ca and isinstance(maybe_ca.get("key"), str) and isinstance(maybe_ca.get("certificate"), str): log.debug("Found existing certificate in data store.") key = load_key_from_bytes(maybe_ca["key"].encode("utf-8")) certificate = load_cert_from_bytes(maybe_ca["certificate"].encode("utf-8")) log.info(f"Using CA certificate from database. fingerprint:{cert_fingerprint(certificate)}") return CertificateHandler(config, key, certificate, temp_dir) else: wo = "with" if args.ca_cert_key_pass else "without" key, certificate = bootstrap_ca() log.info( f"No ca certificate found - create new one {wo} passphrase. fingerprint:{cert_fingerprint(certificate)}" ) key_string = key_to_bytes(key, args.ca_cert_key_pass).decode("utf-8") certificate_string = cert_to_bytes(certificate).decode("utf-8") sd.insert({"_key": "ca", "key": key_string, "certificate": certificate_string}) return CertificateHandler(config, key, certificate, temp_dir)
def create_user_account( user_in: InUser, db: StandardDatabase = Depends(database.get)) -> User: """ Create a user account. Emails and usernames are unique, usernames are _not_ case-sensitive. The password must be strong enough. See GET /password_analysis/ """ # Check if the username is not disallowed if is_username_disallowed(user_in.username): raise HTTPException( status_code=status.HTTP_400_BAD_REQUEST, detail="This username is not allowed.", ) # Check if the username is not already taken if is_username_taken(db, user_in.username): raise HTTPException( status_code=status.HTTP_400_BAD_REQUEST, detail="This username is already taken", ) # Check if the email is not already taken if is_email_taken(db, user_in.email): raise HTTPException( status_code=status.HTTP_400_BAD_REQUEST, detail="This email is already taken", ) # Check if password is strong enough password_analysis = get_password_analysis(**user_in.dict()) if password_analysis and not is_password_strong_enough(password_analysis): raise HTTPException( status_code=status.HTTP_400_BAD_REQUEST, detail={ "message": "The password is not strong enough", "analysis": password_analysis, }, ) # Create the DBUser db_user = DBUser( joined_at=datetime.utcnow(), email_is_confirmed=False, password_hash=hash_password(user_in.password), **user_in.dict(), ) db.collection("users").insert(db_user.json(by_alias=True)) # Return a regular User return User(**db_user.dict(by_alias=True))
async def get_personal_data_archive( user: User = Depends(get_current_confirmed_user), db: StandardDatabase = Depends(database.get), ) -> dict: """ Get an archive of all of the data linked to the user. """ data = {} # The user's data data["user"] = db.collection("users").get(user.key) # the data of which the user is the owner for every collection for c in COLLECTIONS: if c == "users": continue data[c] = [ batch for batch in db.collection(c).find({"owner_key": user.key}) ] return data
def create(self, db: StandardDatabase, current_user: User, data): return self.model_out( **db.collection(self.name_pl).insert( self.model_out(**data.dict(), owner_key=current_user.key).json( by_alias=True ), return_new=True, )["new"] )
def load_all_menu_categories(db: StandardDatabase): categories_db_collection = db.collection(COLLECTION_MENU_CATEGORIES) category_has_files_edge_db_collection = db.collection( EDGE_COLLECTION_CATEGORY_HAS_FILES) for language in DEFAULT_LANGS: with open(f"../data/{language}-categories.json") as f: print(f"Loading menu categories in {language}...") categories = json.load(f) category_count = 0 for category in categories: load_menu_category( category, category_count, language, categories_db_collection, category_has_files_edge_db_collection, ) category_count += 1 print("✓")
def delete(self, db: StandardDatabase, current_user: User, key: ObjectBareKey): full_key = OBJECT_KEY_FORMAT.format(object=key, owner=current_user.key) resource = db.collection(self.name_pl).get(full_key) if resource is None: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail=f"No {self.name_sg} with key {full_key} found", ) resource = self.model_out(**resource) if resource.owner_key != current_user.key: raise HTTPException( status_code=status.HTTP_403_FORBIDDEN, detail="Currently logged-in user does not own the specified subject", ) db.collection(self.name_pl).delete(full_key) return Response(status_code=status.HTTP_204_NO_CONTENT)
def get_all_player_emails(db: StandardDatabase) -> [str]: """ return all player email addresses :param db: connection :return: list of known mail addresses """ if not db.has_collection('players'): logger.error('cannot resolve player if players do not exist') raise PlayerStateException('players collection does not exist') col = db.collection('players') return [p['mail'] for p in col.all()]
def load_search_index_chn(path, db: StandardDatabase): with gzip.open(path) as f: print(f"\nLoading file index data Chinese...") index_data = json.load(f) print(f"\nInserting file index data Chinese into DB...") collection = db.collection(COLLECTION_SEARCH_INDEX_CHN) chunksize = 10000 for i in tqdm(range(0, len(index_data), chunksize)): collection.insert_many(index_data[i : i + chunksize]) print(f"\nDone loading index data Chn...") print("\nDone creating View for Chinese")
def get_user(db: StandardDatabase, username: str) -> Optional[DBUser]: """ Get a user by username from the DB. Returns `None` if the user is not found. """ # Usernames are not case-sensitive user_dict = db.collection("users").find({ "username": username.lower() }).batch() if not user_dict: return None return DBUser(**user_dict[0])
def load_sources(db: StandardDatabase, root_url): source_json_path = root_url + "sources.json" db_file_collection = db.collection(COLLECTION_FILES) with open(source_json_path, "rb") as f: source_list = json.load(f) for entry in source_list: filename = entry['filename'] current_file = db_file_collection.get(filename) if current_file: current_file['source_string'] = entry['source'] print(current_file) db_file_collection.update(current_file)
def load_search_index_tib(path, db: StandardDatabase): with gzip.open(path) as f: print(f"\nLoading file index data Tibetan...") index_data = json.load(f) print(f"\nInserting file index data Tibetan into DB...") collection = db.collection(COLLECTION_SEARCH_INDEX_TIB) # we have to do this in chunk, otherwise it will fail with broken_pipe chunksize = 10000 for i in tqdm(range(0, len(index_data), chunksize)): collection.insert_many(index_data[i : i + chunksize]) print(f"\nDone loading Tibetan index data...") print("\nDone creating View")
def update( self, db: StandardDatabase, current_user: User, key: ObjectBareKey, changes ): resource = self.get(db, current_user, key) updated_resource = { **json.loads(resource.json()), **json.loads(changes.json(exclude_unset=True)), "updated_at": datetime.now().isoformat(sep="T"), } new_resource = db.collection(self.name_pl).update( updated_resource, return_new=True )["new"] return self.model_out(**new_resource)
def load_parallels(json_parallels: [Parallel], db: StandardDatabase) -> None: """ Given an array of parallel objects, load them all into the `parallels` collection :param json_parallels: Array of parallel objects to be loaded as-they-are. :param db: ArangoDB connection object """ db_collection = db.collection(COLLECTION_PARALLELS) db_collection_sorted = db.collection(COLLECTION_PARALLELS_SORTED_BY_FILE) parallels_to_be_inserted = [] root_file_parallel_edges_to_be_inserted = [] for parallel in json_parallels: if isinstance(parallel, dict): root_filename = parallel["root_segnr"][0].split(":")[0] root_filename = re.sub("_[0-9][0-9][0-9]", "", root_filename) parallel_id = f"parallels/{parallel['id']}" parallel["_key"] = parallel["id"] parallel["_id"] = parallel_id # here we delete some things that we don't need in the DB: del parallel["par_pos_end"] del parallel["root_pos_end"] del parallel["par_segtext"] del parallel["root_segtext"] del parallel["par_string"] del parallel["root_string"] # todo: delete the root_filename key after it's not needed anymore parallel["root_filename"] = root_filename parallels_to_be_inserted.append(parallel) chunksize = 10000 for i in range(0, len(parallels_to_be_inserted), chunksize): try: db_collection.insert_many(parallels_to_be_inserted[i:i + chunksize]) except (DocumentInsertError, IndexCreateError) as e: print(f"Could not save parallel {parallel}. Error: ", e)
def update_settings( changes: InSettings, db: StandardDatabase = Depends(database.get), settings: Settings = Depends(settings.get), ) -> Settings: updated_settings = { **json.loads(settings.json(by_alias=True)), **json.loads(changes.json(exclude_unset=True)), "updated_at": datetime.now().isoformat(sep="T"), } new_settings = db.collection("settings").update(updated_settings, return_new=True)["new"] return Settings(**new_settings)
def get( db: StandardDatabase = Depends(database.get), current_user: User = Depends(get_current_confirmed_user), ) -> Settings: """ Gets the settings for the current user. If the settings are created when getting them (if the user has no settings tied to him), the return value is: (settings, True) and else: (settings, False) """ doc = db.collection("settings").get(current_user.key) # If the user has no settings tied to him, create them with the default values. if doc is None: doc = db.collection("settings").insert( Settings(_key=current_user.key).json(by_alias=True), return_new=True )["new"] return Settings(**doc)
async def delete_current_user( user: User = Depends(get_current_user), really_delete: bool = False, db: StandardDatabase = Depends(database.get), ): """ Deletes the currently-logged-in user, and all of the associated resources. This action does not require the user to have confirmed its email address. """ if not really_delete: return HTTPException( status_code=status.HTTP_400_BAD_REQUEST, detail="Set really_delete to True to confirm deletion", ) db.collection("users").delete(user.key) for c in COLLECTIONS: if c == "users": continue db.collection(c).delete_match({"owner_key": user.key}) return Response(status_code=status.HTTP_204_NO_CONTENT)
def reset_setting( setting_key: SettingKey, db: StandardDatabase = Depends(database.get), current_user: User = Depends(get_current_confirmed_user), ) -> Settings: default_settings = InSettings() new_settings = db.collection("settings").update( { "_key": current_user.key, setting_key: json.loads(default_settings.json())[setting_key], }, return_new=True, )["new"] return Settings(**new_settings)
def delete(db: StandardDatabase, player: Player): """ remove existing player :param db: connection :param player: target """ if not db.has_collection('players'): logger.error('cannot remove player if players do not exist') raise PlayerStateException(f'players collection does not exist') col = db.collection('players') db_player = next(col.find({'mail': player.email}), None) if not db_player: logger.error(f'cannot find player {player.email}') raise PlayerStateException(f'could not find player {player.email}') col.delete(db_player)
def read(db: StandardDatabase, player_id): """ find existing player :param db: connection :param player_id: id :return: Player or None """ if not db.has_collection('players'): logger.error('cannot resolve player if players do not exist') raise PlayerStateException('players collection does not exist') col = db.collection('players') db_player = next(col.find({'_key': player_id}), None) if not db_player: return None return Player(db_player['email'], db_player['password'])
def __init__(self, adb: StandardDatabase): """ :param adb: connection to arangodb """ self.adb = adb self.collection = adb.collection("leases")