def init_db(config, ensure_indexes=True): global connection, db, fs connection = AsyncIOMotorClient(config.mongodb.uri.value, tz_aware=True) db = connection[config.mongodb.database.value] fs = AsyncIOMotorGridFSBucket(db) sync_connection = MongoClient(config.mongodb.uri.value, tz_aware=True) sync_db = sync_connection[config.mongodb.database.value] from aleph.model.messages import CappedMessage CappedMessage.create(sync_db) if ensure_indexes: LOGGER.info("Inserting indexes") from aleph.model.messages import Message Message.ensure_indexes(sync_db) from aleph.model.pending import PendingMessage, PendingTX PendingMessage.ensure_indexes(sync_db) PendingTX.ensure_indexes(sync_db) from aleph.model.chains import Chain Chain.ensure_indexes(sync_db) from aleph.model.p2p import Peer Peer.ensure_indexes(sync_db) PermanentPin.ensure_indexes(sync_db) # from aleph.model.hashes import Hash # Hash.ensure_indexes(sync_db) from aleph.model.messages import Message Message.fix_message_confirmations(sync_db)
def __init__(self, host, port, database_name, indexttl=None): logger.info( f"Connecting to mongodb, using {database_name} as database") self.dbname = database_name self.fsname = f"{database_name}fs" self.indexttl = indexttl self.conn = AsyncIOMotorClient(host, port) self.db = self.conn[self.dbname] self.fs = AsyncIOMotorGridFSBucket(self.conn[self.fsname])
async def test_stream_to_handler(self): # Sort of Tornado-specific, but it does work with asyncio. fs = AsyncIOMotorGridFSBucket(self.db) content_length = 1000 await fs.delete(1) await fs.upload_from_stream_with_id( 1, 'filename', source=b'a' * content_length) gridout = await fs.open_download_stream(1) handler = test.MockRequestHandler() await gridout.stream_to_handler(handler) self.assertEqual(content_length, handler.n_written) await fs.delete(1)
async def connect_to_mongo(self, uri=None): info("Connecting to the database... (new-extension)") if uri: self.client = AsyncIOMotorClient(uri) else: self.client = AsyncIOMotorClient() self.db = self.client[f"{self.app_string}_{self.environment}"] self.fs = AsyncIOMotorGridFSBucket(self.db) if self.environment == "development": from .mongo_testing_database import init_testing_database await init_testing_database() info("Database connection succeeded!")
async def findlargemp4fileffmpeg(starttime, endtime): #print("begin findlargemp4fileffmpeg") mp4list = [] client = AsyncIOMotorClient(ServerParameters.mongodbpath) db = client.jt808 bucket = AsyncIOMotorGridFSBucket(db, "eventuploadvideos") cursor = bucket.find({ "uploadDate": { '$gt': starttime, '$lte': endtime }, "filename": { "$regex": ".mp4$" } }) filelist = await cursor.to_list(100000) ccount = 0 for fi in filelist: if fi["length"] > 1000000: print(fi) if os.path.exists(fi["filename"]): os.remove(fi["filename"]) ds = await bucket.open_download_stream(fi["_id"]) f = open("input" + fi["filename"], 'wb') bbb = await ds.read() f.write(bbb) f.close() ds.close() converttstoh264("input" + fi["filename"], fi["filename"]) if os.path.exists("input" + fi["filename"]): os.remove("input" + fi["filename"]) # 保存到bucket try: if os.path.exists(fi["filename"]): uf = open(fi["filename"], "rb") ubbb = uf.read() uf.close() os.remove(fi["filename"]) bucket.delete(fi["_id"]) uds = bucket.open_upload_stream_with_id( fi["_id"], fi["filename"]) await uds.write(ubbb) uds.close() ccount = ccount + 1 logging.info("convert %s %s", fi["_id"], fi["filename"]) except BaseException as e: logging.error(e) logging.info("end findlargemp4fileffmpeg total %s convert %s", len(filelist), ccount) return
async def load(self, force=False) -> None: """Loads index from MongoDB It is safe to call `index.load()` in each request handler because whole data is fetched only if local and remote md5 hash doesn't match """ fs = AsyncIOMotorGridFSBucket(db()) grid_out = await fs.open_download_stream_by_name("index") if force or grid_out.md5 != self.md5: data = pickle.loads(await grid_out.read()) self.index, self.dictionary, self.corpus, self.model, self.documents = data self.md5 = grid_out.md5 self._id = grid_out._id
async def gridfs_delete(gridfsId): try: from motor.motor_asyncio import (AsyncIOMotorGridFSBucket) lazy_instance = app.config.get('LAZY_UMONGO', None) if lazy_instance is not None: database = lazy_instance.db fs = AsyncIOMotorGridFSBucket(database) await fs.delete(ObjectId(gridfsId)) return True except: traceback.print_exc() raise Exception("Erro download gridfs") return False
def test_init(self): name = "bucket" wc = WriteConcern(w="majority", wtimeout=1000) rp = ReadPreference.SECONDARY size = 8 bucket = AsyncIOMotorGridFSBucket( self.db, name, chunk_size_bytes=size, write_concern=wc, read_preference=rp ) self.assertEqual(name, bucket.collection.name) self.assertEqual(wc, bucket.collection.write_concern) self.assertEqual(rp, bucket.collection.read_preference) self.assertEqual(wc, bucket.delegate._chunks.write_concern) self.assertEqual(rp, bucket.delegate._chunks.read_preference) self.assertEqual(size, bucket.delegate._chunk_size_bytes)
async def gridfs_download(gridfsId): try: from motor.motor_asyncio import (AsyncIOMotorGridFSBucket) lazy_instance = app.config.get('LAZY_UMONGO', None) if lazy_instance is not None: database = lazy_instance.db fs = AsyncIOMotorGridFSBucket(database) grid_out = await fs.open_download_stream(ObjectId(gridfsId)) contents = await grid_out.read() return contents except: traceback.print_exc() raise Exception("Erro download gridfs") return None
async def put_gridfile(): with tempfile.TemporaryFile() as tmp: with gzip.GzipFile(mode='wb', fileobj=tmp) as gzfile: for _ in range(10): gzfile.write(b'Nonesuch nonsense\n') gfs = AsyncIOMotorGridFSBucket(client.my_database) tmp.seek(0) await gfs.upload_from_stream(filename='my_file', source=tmp, metadata={ 'contentType': 'text', 'compressed': True })
async def put_gridfile(): with tempfile.TemporaryFile() as tmp: with gzip.GzipFile(mode="wb", fileobj=tmp) as gzfile: for _ in range(10): gzfile.write(b"Nonesuch nonsense\n") gfs = AsyncIOMotorGridFSBucket(client.my_database) tmp.seek(0) await gfs.upload_from_stream(filename="my_file", source=tmp, metadata={ "contentType": "text", "compressed": True })
async def dump(self) -> None: """Pickles index and loads data to MongoDB""" data = pickle.dumps((self.index, self.dictionary, self.corpus, self.model, self.documents)) fs = AsyncIOMotorGridFSBucket(db()) try: grid_out = await fs.open_download_stream_by_name("index") await fs.delete(grid_out._id) except NoFile: pass grid_in = fs.open_upload_stream("index") await grid_in.write(data) await grid_in.close() self.md5 = grid_in.md5 self._id = str(grid_in._id)
def __init__(self, database, root_collection='fs', get_gridfs_file=get_gridfs_file, get_cache_time=get_cache_time, set_extra_headers=set_extra_headers): if not isinstance(database, AsyncIOMotorDatabase): raise TypeError("First argument to AIOHTTPGridFS must be " "AsyncIOMotorDatabase, not %r" % database) self._database = database self._bucket = AsyncIOMotorGridFSBucket(self._database, root_collection) self._get_gridfs_file = get_gridfs_file self._get_cache_time = get_cache_time self._set_extra_headers = set_extra_headers
async def image_endpoint(image_id: str): """Возвращает фотографию по её айди в mongoDB""" grid_filesystem = AsyncIOMotorGridFSBucket(database.files_conn(), collection="images") try: _id = ObjectId(image_id) grid_out = await grid_filesystem.open_download_stream(_id) contents = await grid_out.read() except Exception: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="Такой картинки не существует", ) return Response(content=contents, media_type=grid_out.content_type)
async def load_database(): current_app.client = AsyncIOMotorClient("db", 27017) current_app.priv_db = current_app.client['fenetre'] current_app.shared_db = current_app.client['shared'] current_app.gridfs_shared = AsyncIOMotorGridFSBucket( current_app.shared_db) _shared_instance.set_db(shared_db()) _private_instance.set_db(private_db()) await User.ensure_indexes() await SignupProvider.ensure_indexes() await Course.ensure_indexes() await Form.ensure_indexes()
async def _set_table(self, app, loop): app._client = AsyncIOMotorClient(app.config["MONGO_URI"], io_loop=loop) db = app._client[app.config["MONGO_DB"]] app._table = db[app.config.get("MONGO_TABLE", app.config["MONGO_DB"])] if app.config.get("MONGO_GRIDFS", False): app._gridfs = AsyncIOMotorGridFSBucket(db) app._table.create_index("created_at", expireAfterSeconds=1800) app._table.create_index([("path", ASCENDING), ("slug", ASCENDING)], unique=True) root = await app._root_model.get(app._table, path="") if root: await root._rebuild_sec(app)
async def startup(): global client, db, fs, accepted_extensions accepted_extensions = {x: 1 for x in accepted_extensions_list} try: client = AsyncIOMotorClient(uri) db = client["users"] fs = AsyncIOMotorGridFSBucket(db) # below isn't a good indication of connection because fs isn't created --> lazy creation I think --> so possibly do one operation on fs? logger.error( "MONGO CONNECTED" ) # uhhh error level log because it shows, but doesn't actually break stuff kinda hacky except: logger.error( "ERROR: mongo connection failed, are your environment variables set?" ) raise Exception("ERROR: mongo connection failed")
async def gridfs_upload(fileData, filename, contentType): try: from motor.motor_asyncio import (AsyncIOMotorGridFSBucket) lazy_instance = app.config.get('LAZY_UMONGO', None) if lazy_instance is not None: database = lazy_instance.db fs = AsyncIOMotorGridFSBucket(database) file_id = await fs.upload_from_stream( filename, fileData, chunk_size_bytes=16770000, metadata={"contentType": contentType}) # print(file_id) return file_id except: traceback.print_exc() raise Exception("Erro com gridfs") return None
async def setup(self, _, loop): self.mongo = AsyncIOMotorClient() self.db = self.mongo.dclub self.file_bucket = AsyncIOMotorGridFSBucket(self.db, bucket_name="files", chunk_size_bytes=10**6) await self.db.files.files.create_index([("md5", pymongo.ASCENDING)], unique=True) await self.db.messages.create_index([("user_id", pymongo.ASCENDING)]) await self.db.messages.create_index([("last_updated", pymongo.ASCENDING)]) self.session = aiohttp.ClientSession(loop=loop) self.redis = await aioredis.create_redis_pool("redis://localhost", loop=loop) await routes.setup(self)
async def connect_to_mongo(self, uri=None, db_name=None) -> None: """Open an asynchronous connection to a mongo database. :param uri: Where to find the database; defaults to None, in which case localhost:27027 will be used :type uri: str, optional :param db_name: Name of the database to use; if None, a name will be generated based on the title of the FastAPI app provided to the init_app method :type db_name: str, optional """ info("Connecting to the database...") if uri: self.client = AsyncIOMotorClient(uri) else: self.client = AsyncIOMotorClient() if db_name: self.db = self.client[db_name] else: self.db = self.client[f"{self.app_string}_{self.environment}"] self.fs = AsyncIOMotorGridFSBucket(self.db) info(f"Database {db_name} connection succeeded!")
def init_db(config, ensure_indexes=True): global connection, db, fs connection = AsyncIOMotorClient(config.mongodb.uri.value, tz_aware=True) db = connection[config.mongodb.database.value] fs = AsyncIOMotorGridFSBucket(db) sync_connection = MongoClient(config.mongodb.uri.value, tz_aware=True) sync_db = sync_connection[config.mongodb.database.value] if ensure_indexes: LOGGER.info('Inserting indexes') from aleph.model.messages import Message Message.ensure_indexes(sync_db) from aleph.model.pending import PendingMessage, PendingTX PendingMessage.ensure_indexes(sync_db) PendingTX.ensure_indexes(sync_db) from aleph.model.chains import Chain Chain.ensure_indexes(sync_db) from aleph.model.p2p import Peer Peer.ensure_indexes(sync_db)
async def test_iter_gridfs(self): gfs = AsyncIOMotorGridFSBucket(self.db) async def cleanup(): await self.db.fs.files.delete_many({}) await self.db.fs.chunks.delete_many({}) await cleanup() # Empty iterator. async for _ in gfs.find({'_id': 1}): self.fail() data = b'data' for n_files in 1, 2, 10: for i in range(n_files): async with gfs.open_upload_stream(filename='filename') as f: await f.write(data) # Force extra batches to test iteration. j = 0 async for _ in gfs.find({'filename': 'filename'}).batch_size(3): j += 1 self.assertEqual(j, n_files) await cleanup() await gfs.upload_from_stream_with_id(1, 'filename', source=data, chunk_size_bytes=1) cursor = gfs.find({'_id': 1}) await cursor.fetch_next gout = cursor.next_object() chunks = [] async for chunk in gout: chunks.append(chunk) self.assertEqual(len(chunks), len(data)) self.assertEqual(b''.join(chunks), data)
async def startup(): await db.connect_to_database(path=DATABASE_URL) disc_db = db.client[DATABASE_NAME] global fs fs = AsyncIOMotorGridFSBucket(disc_db) global song_db song_db = disc_db.get_collection(SONGS_DB) global mix_db mix_db = disc_db.get_collection(MIX_DB) user_col = disc_db[USER_DB] global user_db user_db = MongoDBUserDatabase(UserDB, user_col) global fastapi_users fastapi_users = FastAPIUsers(user_db, [jwt_authentication], User, UserCreate, UserUpdate, UserDB) app.include_router(fastapi_users.get_auth_router(jwt_authentication), prefix="/auth/jwt", tags=["auth"]) app.include_router( fastapi_users.get_register_router(after_register=on_after_register), prefix="/auth", tags=["auth"]) app.include_router( fastapi_users.get_reset_password_router( SECRET, after_forgot_password=on_after_forgot_password), prefix="/auth", tags=["auth"], ) app.include_router( fastapi_users.get_verify_router( SECRET, after_verification_request=after_verification_request), prefix="/auth", tags=["auth"], ) app.include_router(fastapi_users.get_users_router(), prefix="/users", tags=["users"])
def __init__(self, uri: str, ioloop=None, collection="fs", only_db=False): mongo = MongoConnection(uri=uri, ioloop=ioloop, only_db=only_db) fs = AsyncIOMotorGridFSBucket(mongo.db, collection=collection) #super().__init__(mongo.db,collection=collection) setattr(fs, "client", mongo.client) self.bucket = fs
def test_collection_param(self): bucket = AsyncIOMotorGridFSBucket(self.db, collection='collection') self.assertEqual('collection', bucket.collection.name)
def setUp(self): super(TestAsyncIOGridFSBucket, self).setUp() self.loop.run_until_complete(self._reset()) self.bucket = AsyncIOMotorGridFSBucket(self.db)
def __init__(self): super().__init__() self._db = get_db() self.fs = AsyncIOMotorGridFSBucket(self._db)
async def init(): global _client, _gfs _client = AsyncIOMotorClient(uri) await setup_indexes() _gfs = AsyncIOMotorGridFSBucket(db())
def __init__(self, host: str, port: int): # Set up fernet # Read from base64 encoded key if os.environ.get("LOCKBOX_CREDENTIAL_KEY"): key = os.environ.get("LOCKBOX_CREDENTIAL_KEY") # Read from key file elif os.environ.get("LOCKBOX_CREDENTIAL_KEY_FILE"): try: with open(os.environ.get("LOCKBOX_CREDENTIAL_KEY_FILE"), "rb") as f: key = base64.b64encode(f.read()) except IOError as e: raise ValueError( "Cannot read password encryption key file") from e else: raise ValueError( "Encryption key for passwords must be provided! Set LOCKBOX_CREDENTIAL_KEY or LOCKBOX_CREDENTIAL_KEY_FILE." ) # Should raise ValueError if key is invalid self.fernet = Fernet(key) if os.environ.get("LOCKBOX_SCHOOL"): try: self.school_code = int(os.environ["LOCKBOX_SCHOOL"]) except ValueError as e: logger.error(f"Invalid school code: {e}") self.school_code = None else: self.school_code = None self.client = AsyncIOMotorClient(host, port) self._private_db = self.client["lockbox"] self._shared_db = self.client["shared"] self._private_instance = MotorAsyncIOInstance(self._private_db) self._shared_instance = MotorAsyncIOInstance(self._shared_db) self._shared_gridfs = AsyncIOMotorGridFSBucket(self._shared_db) self.LockboxFailureImpl = self._private_instance.register( documents.LockboxFailure) self.FillFormResultImpl = self._private_instance.register( documents.FillFormResult) self.UserImpl = self._private_instance.register(documents.User) self.FormGeometryEntryImpl = self._private_instance.register( documents.FormGeometryEntry) self.CachedFormGeometryImpl = self._private_instance.register( documents.CachedFormGeometry) self.TaskImpl = self._private_instance.register(documents.Task) self.FormFieldImpl = self._shared_instance.register( documents.FormField) self.FormImpl = self._shared_instance.register(documents.Form) self.CourseImpl = self._shared_instance.register(documents.Course) self.FormFillingTestImpl = self._shared_instance.register( documents.FormFillingTest) self.LockboxFailureImplShared = self._shared_instance.register( documents.LockboxFailure) self.FillFormResultImplShared = self._shared_instance.register( documents.FillFormResult) self._scheduler = scheduler.Scheduler(self) tasks.set_task_handlers(self._scheduler) # Current school day, set by the check day task # Used as a fallback & indicator of whether the day's been checked # None when the day has not been checked self.current_day = None