def init_db(config, ensure_indexes=True):
global connection, db, fs
connection = AsyncIOMotorClient(config.mongodb.uri.value, tz_aware=True)
db = connection[config.mongodb.database.value]
fs = AsyncIOMotorGridFSBucket(db)
sync_connection = MongoClient(config.mongodb.uri.value, tz_aware=True)
sync_db = sync_connection[config.mongodb.database.value]
from aleph.model.messages import CappedMessage
CappedMessage.create(sync_db)
if ensure_indexes:
LOGGER.info("Inserting indexes")
from aleph.model.messages import Message
Message.ensure_indexes(sync_db)
from aleph.model.pending import PendingMessage, PendingTX
PendingMessage.ensure_indexes(sync_db)
PendingTX.ensure_indexes(sync_db)
from aleph.model.chains import Chain
Chain.ensure_indexes(sync_db)
from aleph.model.p2p import Peer
Peer.ensure_indexes(sync_db)
PermanentPin.ensure_indexes(sync_db)
# from aleph.model.hashes import Hash
# Hash.ensure_indexes(sync_db)
from aleph.model.messages import Message
Message.fix_message_confirmations(sync_db)
def __init__(self, host, port, database_name, indexttl=None):
logger.info(
f"Connecting to mongodb, using {database_name} as database")
self.dbname = database_name
self.fsname = f"{database_name}fs"
self.indexttl = indexttl
self.conn = AsyncIOMotorClient(host, port)
self.db = self.conn[self.dbname]
self.fs = AsyncIOMotorGridFSBucket(self.conn[self.fsname])
async def test_stream_to_handler(self):
# Sort of Tornado-specific, but it does work with asyncio.
fs = AsyncIOMotorGridFSBucket(self.db)
content_length = 1000
await fs.delete(1)
await fs.upload_from_stream_with_id(
1, 'filename', source=b'a' * content_length)
gridout = await fs.open_download_stream(1)
handler = test.MockRequestHandler()
await gridout.stream_to_handler(handler)
self.assertEqual(content_length, handler.n_written)
await fs.delete(1)
async def connect_to_mongo(self, uri=None):
info("Connecting to the database... (new-extension)")
if uri:
self.client = AsyncIOMotorClient(uri)
else:
self.client = AsyncIOMotorClient()
self.db = self.client[f"{self.app_string}_{self.environment}"]
self.fs = AsyncIOMotorGridFSBucket(self.db)
if self.environment == "development":
from .mongo_testing_database import init_testing_database
await init_testing_database()
info("Database connection succeeded！")
async def findlargemp4fileffmpeg(starttime, endtime):
#print("begin findlargemp4fileffmpeg")
mp4list = []
client = AsyncIOMotorClient(ServerParameters.mongodbpath)
db = client.jt808
bucket = AsyncIOMotorGridFSBucket(db, "eventuploadvideos")
cursor = bucket.find({
"uploadDate": {
'$gt': starttime,
'$lte': endtime
},
"filename": {
"$regex": ".mp4$"
}
})
filelist = await cursor.to_list(100000)
ccount = 0
for fi in filelist:
if fi["length"] > 1000000:
print(fi)
if os.path.exists(fi["filename"]):
os.remove(fi["filename"])
ds = await bucket.open_download_stream(fi["_id"])
f = open("input" + fi["filename"], 'wb')
bbb = await ds.read()
f.write(bbb)
f.close()
ds.close()
converttstoh264("input" + fi["filename"], fi["filename"])
if os.path.exists("input" + fi["filename"]):
os.remove("input" + fi["filename"])
# 保存到bucket
try:
if os.path.exists(fi["filename"]):
uf = open(fi["filename"], "rb")
ubbb = uf.read()
uf.close()
os.remove(fi["filename"])
bucket.delete(fi["_id"])
uds = bucket.open_upload_stream_with_id(
fi["_id"], fi["filename"])
await uds.write(ubbb)
uds.close()
ccount = ccount + 1
logging.info("convert %s %s", fi["_id"], fi["filename"])
except BaseException as e:
logging.error(e)
logging.info("end findlargemp4fileffmpeg total %s convert %s",
len(filelist), ccount)
return
async def load(self, force=False) -> None:
"""Loads index from MongoDB
It is safe to call `index.load()` in each request handler because
whole data is fetched only if local and remote md5 hash doesn't match
"""
fs = AsyncIOMotorGridFSBucket(db())
grid_out = await fs.open_download_stream_by_name("index")
if force or grid_out.md5 != self.md5:
data = pickle.loads(await grid_out.read())
self.index, self.dictionary, self.corpus, self.model, self.documents = data
self.md5 = grid_out.md5
self._id = grid_out._id
async def gridfs_delete(gridfsId):
try:
from motor.motor_asyncio import (AsyncIOMotorGridFSBucket)
lazy_instance = app.config.get('LAZY_UMONGO', None)
if lazy_instance is not None:
database = lazy_instance.db
fs = AsyncIOMotorGridFSBucket(database)
await fs.delete(ObjectId(gridfsId))
return True
except:
traceback.print_exc()
raise Exception("Erro download gridfs")
return False
def test_init(self):
name = "bucket"
wc = WriteConcern(w="majority", wtimeout=1000)
rp = ReadPreference.SECONDARY
size = 8
bucket = AsyncIOMotorGridFSBucket(
self.db, name, chunk_size_bytes=size, write_concern=wc, read_preference=rp
)
self.assertEqual(name, bucket.collection.name)
self.assertEqual(wc, bucket.collection.write_concern)
self.assertEqual(rp, bucket.collection.read_preference)
self.assertEqual(wc, bucket.delegate._chunks.write_concern)
self.assertEqual(rp, bucket.delegate._chunks.read_preference)
self.assertEqual(size, bucket.delegate._chunk_size_bytes)
async def gridfs_download(gridfsId):
try:
from motor.motor_asyncio import (AsyncIOMotorGridFSBucket)
lazy_instance = app.config.get('LAZY_UMONGO', None)
if lazy_instance is not None:
database = lazy_instance.db
fs = AsyncIOMotorGridFSBucket(database)
grid_out = await fs.open_download_stream(ObjectId(gridfsId))
contents = await grid_out.read()
return contents
except:
traceback.print_exc()
raise Exception("Erro download gridfs")
return None
async def put_gridfile():
with tempfile.TemporaryFile() as tmp:
with gzip.GzipFile(mode='wb', fileobj=tmp) as gzfile:
for _ in range(10):
gzfile.write(b'Nonesuch nonsense\n')
gfs = AsyncIOMotorGridFSBucket(client.my_database)
tmp.seek(0)
await gfs.upload_from_stream(filename='my_file',
source=tmp,
metadata={
'contentType': 'text',
'compressed': True
})
async def put_gridfile():
with tempfile.TemporaryFile() as tmp:
with gzip.GzipFile(mode="wb", fileobj=tmp) as gzfile:
for _ in range(10):
gzfile.write(b"Nonesuch nonsense\n")
gfs = AsyncIOMotorGridFSBucket(client.my_database)
tmp.seek(0)
await gfs.upload_from_stream(filename="my_file",
source=tmp,
metadata={
"contentType": "text",
"compressed": True
})
async def dump(self) -> None:
"""Pickles index and loads data to MongoDB"""
data = pickle.dumps((self.index, self.dictionary, self.corpus,
self.model, self.documents))
fs = AsyncIOMotorGridFSBucket(db())
try:
grid_out = await fs.open_download_stream_by_name("index")
await fs.delete(grid_out._id)
except NoFile:
pass
grid_in = fs.open_upload_stream("index")
await grid_in.write(data)
await grid_in.close()
self.md5 = grid_in.md5
self._id = str(grid_in._id)
def __init__(self,
database,
root_collection='fs',
get_gridfs_file=get_gridfs_file,
get_cache_time=get_cache_time,
set_extra_headers=set_extra_headers):
if not isinstance(database, AsyncIOMotorDatabase):
raise TypeError("First argument to AIOHTTPGridFS must be "
"AsyncIOMotorDatabase, not %r" % database)
self._database = database
self._bucket = AsyncIOMotorGridFSBucket(self._database, root_collection)
self._get_gridfs_file = get_gridfs_file
self._get_cache_time = get_cache_time
self._set_extra_headers = set_extra_headers
async def image_endpoint(image_id: str):
"""Возвращает фотографию по её айди в mongoDB"""
grid_filesystem = AsyncIOMotorGridFSBucket(database.files_conn(),
collection="images")
try:
_id = ObjectId(image_id)
grid_out = await grid_filesystem.open_download_stream(_id)
contents = await grid_out.read()
except Exception:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail="Такой картинки не существует",
)
return Response(content=contents, media_type=grid_out.content_type)
async def load_database():
current_app.client = AsyncIOMotorClient("db", 27017)
current_app.priv_db = current_app.client['fenetre']
current_app.shared_db = current_app.client['shared']
current_app.gridfs_shared = AsyncIOMotorGridFSBucket(
current_app.shared_db)
_shared_instance.set_db(shared_db())
_private_instance.set_db(private_db())
await User.ensure_indexes()
await SignupProvider.ensure_indexes()
await Course.ensure_indexes()
await Form.ensure_indexes()
async def _set_table(self, app, loop):
app._client = AsyncIOMotorClient(app.config["MONGO_URI"], io_loop=loop)
db = app._client[app.config["MONGO_DB"]]
app._table = db[app.config.get("MONGO_TABLE", app.config["MONGO_DB"])]
if app.config.get("MONGO_GRIDFS", False):
app._gridfs = AsyncIOMotorGridFSBucket(db)
app._table.create_index("created_at", expireAfterSeconds=1800)
app._table.create_index([("path", ASCENDING), ("slug", ASCENDING)],
unique=True)
root = await app._root_model.get(app._table, path="")
if root:
await root._rebuild_sec(app)
async def startup():
global client, db, fs, accepted_extensions
accepted_extensions = {x: 1 for x in accepted_extensions_list}
try:
client = AsyncIOMotorClient(uri)
db = client["users"]
fs = AsyncIOMotorGridFSBucket(db)
# below isn't a good indication of connection because fs isn't created --> lazy creation I think --> so possibly do one operation on fs?
logger.error(
"MONGO CONNECTED"
) # uhhh error level log because it shows, but doesn't actually break stuff kinda hacky
except:
logger.error(
"ERROR: mongo connection failed, are your environment variables set?"
)
raise Exception("ERROR: mongo connection failed")
async def gridfs_upload(fileData, filename, contentType):
try:
from motor.motor_asyncio import (AsyncIOMotorGridFSBucket)
lazy_instance = app.config.get('LAZY_UMONGO', None)
if lazy_instance is not None:
database = lazy_instance.db
fs = AsyncIOMotorGridFSBucket(database)
file_id = await fs.upload_from_stream(
filename,
fileData,
chunk_size_bytes=16770000,
metadata={"contentType": contentType})
# print(file_id)
return file_id
except:
traceback.print_exc()
raise Exception("Erro com gridfs")
return None
async def setup(self, _, loop):
self.mongo = AsyncIOMotorClient()
self.db = self.mongo.dclub
self.file_bucket = AsyncIOMotorGridFSBucket(self.db,
bucket_name="files",
chunk_size_bytes=10**6)
await self.db.files.files.create_index([("md5", pymongo.ASCENDING)],
unique=True)
await self.db.messages.create_index([("user_id", pymongo.ASCENDING)])
await self.db.messages.create_index([("last_updated",
pymongo.ASCENDING)])
self.session = aiohttp.ClientSession(loop=loop)
self.redis = await aioredis.create_redis_pool("redis://localhost",
loop=loop)
await routes.setup(self)
async def connect_to_mongo(self, uri=None, db_name=None) -> None:
"""Open an asynchronous connection to a mongo database.
:param uri: Where to find the database; defaults to None, in which case localhost:27027 will be used
:type uri: str, optional
:param db_name: Name of the database to use; if None, a name will be generated based on the title of the FastAPI app provided to the init_app method
:type db_name: str, optional
"""
info("Connecting to the database...")
if uri:
self.client = AsyncIOMotorClient(uri)
else:
self.client = AsyncIOMotorClient()
if db_name:
self.db = self.client[db_name]
else:
self.db = self.client[f"{self.app_string}_{self.environment}"]
self.fs = AsyncIOMotorGridFSBucket(self.db)
info(f"Database {db_name} connection succeeded！")
def init_db(config, ensure_indexes=True):
global connection, db, fs
connection = AsyncIOMotorClient(config.mongodb.uri.value, tz_aware=True)
db = connection[config.mongodb.database.value]
fs = AsyncIOMotorGridFSBucket(db)
sync_connection = MongoClient(config.mongodb.uri.value, tz_aware=True)
sync_db = sync_connection[config.mongodb.database.value]
if ensure_indexes:
LOGGER.info('Inserting indexes')
from aleph.model.messages import Message
Message.ensure_indexes(sync_db)
from aleph.model.pending import PendingMessage, PendingTX
PendingMessage.ensure_indexes(sync_db)
PendingTX.ensure_indexes(sync_db)
from aleph.model.chains import Chain
Chain.ensure_indexes(sync_db)
from aleph.model.p2p import Peer
Peer.ensure_indexes(sync_db)
async def test_iter_gridfs(self):
gfs = AsyncIOMotorGridFSBucket(self.db)
async def cleanup():
await self.db.fs.files.delete_many({})
await self.db.fs.chunks.delete_many({})
await cleanup()
# Empty iterator.
async for _ in gfs.find({'_id': 1}):
self.fail()
data = b'data'
for n_files in 1, 2, 10:
for i in range(n_files):
async with gfs.open_upload_stream(filename='filename') as f:
await f.write(data)
# Force extra batches to test iteration.
j = 0
async for _ in gfs.find({'filename': 'filename'}).batch_size(3):
j += 1
self.assertEqual(j, n_files)
await cleanup()
await gfs.upload_from_stream_with_id(1,
'filename',
source=data,
chunk_size_bytes=1)
cursor = gfs.find({'_id': 1})
await cursor.fetch_next
gout = cursor.next_object()
chunks = []
async for chunk in gout:
chunks.append(chunk)
self.assertEqual(len(chunks), len(data))
self.assertEqual(b''.join(chunks), data)
async def startup():
await db.connect_to_database(path=DATABASE_URL)
disc_db = db.client[DATABASE_NAME]
global fs
fs = AsyncIOMotorGridFSBucket(disc_db)
global song_db
song_db = disc_db.get_collection(SONGS_DB)
global mix_db
mix_db = disc_db.get_collection(MIX_DB)
user_col = disc_db[USER_DB]
global user_db
user_db = MongoDBUserDatabase(UserDB, user_col)
global fastapi_users
fastapi_users = FastAPIUsers(user_db, [jwt_authentication], User,
UserCreate, UserUpdate, UserDB)
app.include_router(fastapi_users.get_auth_router(jwt_authentication),
prefix="/auth/jwt",
tags=["auth"])
app.include_router(
fastapi_users.get_register_router(after_register=on_after_register),
prefix="/auth",
tags=["auth"])
app.include_router(
fastapi_users.get_reset_password_router(
SECRET, after_forgot_password=on_after_forgot_password),
prefix="/auth",
tags=["auth"],
)
app.include_router(
fastapi_users.get_verify_router(
SECRET, after_verification_request=after_verification_request),
prefix="/auth",
tags=["auth"],
)
app.include_router(fastapi_users.get_users_router(),
prefix="/users",
tags=["users"])
def __init__(self, uri: str, ioloop=None, collection="fs", only_db=False):
mongo = MongoConnection(uri=uri, ioloop=ioloop, only_db=only_db)
fs = AsyncIOMotorGridFSBucket(mongo.db, collection=collection)
#super().__init__(mongo.db,collection=collection)
setattr(fs, "client", mongo.client)
self.bucket = fs
def test_collection_param(self):
bucket = AsyncIOMotorGridFSBucket(self.db, collection='collection')
self.assertEqual('collection', bucket.collection.name)
def setUp(self):
super(TestAsyncIOGridFSBucket, self).setUp()
self.loop.run_until_complete(self._reset())
self.bucket = AsyncIOMotorGridFSBucket(self.db)
def __init__(self):
super().__init__()
self._db = get_db()
self.fs = AsyncIOMotorGridFSBucket(self._db)
async def init():
global _client, _gfs
_client = AsyncIOMotorClient(uri)
await setup_indexes()
_gfs = AsyncIOMotorGridFSBucket(db())
def __init__(self, host: str, port: int):
# Set up fernet
# Read from base64 encoded key
if os.environ.get("LOCKBOX_CREDENTIAL_KEY"):
key = os.environ.get("LOCKBOX_CREDENTIAL_KEY")
# Read from key file
elif os.environ.get("LOCKBOX_CREDENTIAL_KEY_FILE"):
try:
with open(os.environ.get("LOCKBOX_CREDENTIAL_KEY_FILE"),
"rb") as f:
key = base64.b64encode(f.read())
except IOError as e:
raise ValueError(
"Cannot read password encryption key file") from e
else:
raise ValueError(
"Encryption key for passwords must be provided! Set LOCKBOX_CREDENTIAL_KEY or LOCKBOX_CREDENTIAL_KEY_FILE."
)
# Should raise ValueError if key is invalid
self.fernet = Fernet(key)
if os.environ.get("LOCKBOX_SCHOOL"):
try:
self.school_code = int(os.environ["LOCKBOX_SCHOOL"])
except ValueError as e:
logger.error(f"Invalid school code: {e}")
self.school_code = None
else:
self.school_code = None
self.client = AsyncIOMotorClient(host, port)
self._private_db = self.client["lockbox"]
self._shared_db = self.client["shared"]
self._private_instance = MotorAsyncIOInstance(self._private_db)
self._shared_instance = MotorAsyncIOInstance(self._shared_db)
self._shared_gridfs = AsyncIOMotorGridFSBucket(self._shared_db)
self.LockboxFailureImpl = self._private_instance.register(
documents.LockboxFailure)
self.FillFormResultImpl = self._private_instance.register(
documents.FillFormResult)
self.UserImpl = self._private_instance.register(documents.User)
self.FormGeometryEntryImpl = self._private_instance.register(
documents.FormGeometryEntry)
self.CachedFormGeometryImpl = self._private_instance.register(
documents.CachedFormGeometry)
self.TaskImpl = self._private_instance.register(documents.Task)
self.FormFieldImpl = self._shared_instance.register(
documents.FormField)
self.FormImpl = self._shared_instance.register(documents.Form)
self.CourseImpl = self._shared_instance.register(documents.Course)
self.FormFillingTestImpl = self._shared_instance.register(
documents.FormFillingTest)
self.LockboxFailureImplShared = self._shared_instance.register(
documents.LockboxFailure)
self.FillFormResultImplShared = self._shared_instance.register(
documents.FillFormResult)
self._scheduler = scheduler.Scheduler(self)
tasks.set_task_handlers(self._scheduler)
# Current school day, set by the check day task
# Used as a fallback & indicator of whether the day's been checked
# None when the day has not been checked
self.current_day = None
