class MongoHandler: ''' mongo异步操作 ''' def __init__(self, collection_name='default', host=MONGODB['host'], port=MONGODB['port'], user=MONGODB['user'], pwd=MONGODB['pwd'], db=MONGODB['db']): self.client = AsyncIOMotorClient( 'mongodb://{user}:{pwd}@{host}:{port}/'.format(user=user, pwd=pwd, host=host, port=port)) self.db = self.client[db] self.collection_name = collection_name # for logging self.collection = self.db[collection_name] # self.loop = asyncio.new_event_loop() # error: different loop self.loop = asyncio.get_event_loop() async def process(self, item, repet_key, need_repet, repet_value): if isinstance(item, Item): item_data = dict(item) elif isinstance(item, dict): item_data = item else: try: item_data = json.loads(item) except: raise ('******************** item must dict ') repet = await self.find(repet_key, need_repet, repet_value) if not repet: if await self.collection.insert_one(item_data): logging.info('********************save to mongodb_{collection} ==> success:\n{item}'.format( collection=self.collection_name, item=item_data)) else: raise ('********************save to mongodb_{collection} ==> fail:\n{item}'.format( collection=self.collection_name, item=item_data)) else: logging.info( '********************mongodb_{collection} ==> repet :\n{item}'.format(collection=self.collection_name, item=item_data)) async def find(self, repet_key, need_repet, repet_value): '''可选的mongo查询去重 need_repet: 默认False不去重 repet_key: 查询key repet_value: 对应的value ''' if not need_repet: return False repet_data = await self.collection.find_one({repet_key: repet_value}, {'_id': 0, repet_key: 1}) return repet_data def run(self, item, repet_key, need_repet=False, repet_value=''): # asyncio.set_event_loop(self.loop) self.loop.run_until_complete(self.process(item, repet_key, need_repet, repet_value)) def close(self): self.client.close() self.loop.close()
class AddUrl: def __init__(self): self.client = AsyncIOMotorClient(host='127.0.0.1', port=27017) self.col_read = self.client['DianPing']['dp_shop_error'] self.set_name = 'dp_shop' async def start(self): self.rd = await aioredis.create_redis_pool(('localhost', 6379), encoding='utf-8') cursor = self.col_read.find({}) tasks = [ asyncio.create_task(self.add(shop['url'])) for shop in await cursor.to_list(length=None) ] # tasks = [asyncio.create_task(self.get_all())] await asyncio.wait(tasks) result = [task.result() for task in tasks] print(len(result)) self.rd.close() await self.rd.wait_closed() self.client.close() async def add(self, url): await self.rd.sadd(self.set_name, url) async def get_all(self): result = await self.rd.smembers(self.set_name) return result
class UrlDB: def __init__(self, uri: str, size: int = 5) -> None: self.client = AsyncIOMotorClient(uri) self.db = self.client.url self.size = size def close(self) -> None: # Kills the connection with the database self.client.close() async def get_url(self, collection: str, url_id: str): find = {"_id": url_id} result: dict = await self.db[collection].find_one(find) return result async def id_gen(self, collection: str): # Generates a valid Base62 url id that isn't in the DB base = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" while True: url_id = "".join(random.choices(base, k=self.size)) available = await self.get_url(collection, url_id) if available is None: return url_id async def post_url(self, collection: str, url: str): if used := await self.db[collection].find_one({"url": url}): # Checks if the url is not in use # True if the there is data False otherwise return models.UrlID(_id=used["_id"]) # Creates a new id and saves it to the DB url_id = await self.id_gen(collection) url_data = {"_id": url_id, "url": url} await self.db[collection].insert_one(url_data) return models.UrlID(_id=url_data["_id"])
class MongoManager(DatabaseManager): client: AsyncIOMotorClient = None db: AsyncIOMotorDatabase = None async def connect_to_database(self, path: str): logging.info("Connecting to MongoDB.") self.client = AsyncIOMotorClient(path, maxPoolSize=10, minPoolSize=10) self.db = self.client.main_db logging.info("Connected to MongoDB.") async def close_database_connection(self): logging.info("Closing connection with MongoDB.") self.client.close() logging.info("Closed connection with MongoDB.") async def get_models(self) -> List[Model]: models_list = [] models_q = self.db.models.find() async for post in models_q: models_list.append(Model(**post, id=post["model_url"])) return models_list async def get_model(self, model_url: str) -> Model: if (model_q := await self.db.models.find_one({"model_url": model_url})) is not None: return Model(**model_q, id=model_q["_id"]) else:
async def get(self, document_id): # caution : 이벤트 루프가 충돌할 수 있기 때문에, 넣을 때마다 매번 client를 선언해 주어야 함 client = AsyncIOMotorClient(self.uri) collection = client[self.dbname][self.collection] res = await collection.find_one({"id": document_id}) if res: res.pop('_id', None) client.close() return res
async def test_db(): """Creates a temporary database, sets a TTL index and returns a pymongo client. """ client = AsyncIOMotorClient(MONGO_URL) client.test_db.secrets.create_index('delete_at', expireAfterSeconds=0) yield client.test_db client.drop_database('test_db') client.close()
class DataBaseConnection: @retry(wait=wait_random_exponential(multiplier=1, max=60)) def connect(self) -> None: hostname = os.getenv("DB_HOSTNAME", "db") self.client = AsyncIOMotorClient(host=hostname) self.db = DataBase(self.client) print("Connected to database") def disconnect(self) -> None: self.client.close() print("Disconnected from database")
async def put(self, document: dict): # caution : 이벤트 루프가 충돌할 수 있기 때문에, 넣을 때마다 매번 client를 선언해 주어야 함 client = AsyncIOMotorClient(self.uri) collection = client[self.dbname][self.collection] if 'id' not in document: raise ValueError("document should contain id") await collection.replace_one({"id": document["id"]}, document, upsert=True) client.close()
def db(): test_db_name = f'test_{DB_NAME}' try: client = AsyncIOMotorClient(str(MONGODB_URL), maxPoolSize=MAX_CONNECTIONS_COUNT, minPoolSize=MIN_CONNECTIONS_COUNT) db = client[test_db_name] yield db finally: client.drop_database(test_db_name) client.close()
class MongoDBConnection: _client: Optional[Union[SyncMongoClient, AsyncMongoClient]] _uri_connection: str _database_name: str def __init__(self, uri_connection: str, database_name: str) -> None: self._client = None self._uri_connection = uri_connection self._database_name = database_name def default_database_name(self) -> str: return self._database_name def client( self, io_loop: Optional[AbstractEventLoop] = None ) -> Union[SyncMongoClient, AsyncMongoClient]: self.open(io_loop) return self._client def database( self, database: Optional[str] = None ) -> Union[SyncMongoDatabase, AsyncMongoDatabase]: return self.client().get_database( self._database_name if not database else database) def collection( self, collection: str, database: Optional[str] = None, ) -> Union[SyncMongoCollection, AsyncMongoCollection]: return self.database(database).get_collection(collection) def open( self, io_loop: Optional[AbstractEventLoop] = None ) -> 'MongoDBConnection': if not self._client: self._client = AsyncMongoClient( self._uri_connection, uuidRepresentation='standard', io_loop=io_loop if io_loop else get_event_loop(), ) return self def close(self) -> None: if self._client: self._client.close() async def drop_database(self, database: Optional[str] = None) -> None: await self.client().drop_database(database or self._database_name)
class MongoMotorManager: def __init__(self) -> None: self.MONGO_DETAILS = "mongodb://*****:*****@localhost:27017" self.database = None self.client = None def startup(self, database: str) -> AsyncIOMotorClient: self.client = AsyncIOMotorClient(self.MONGO_DETAILS) self.database = self.client.get_database(database) return self.database def shutdown(self): self.client.close()
class MotorComponent(BaseComponent): required_settings = {"mongo_dsn": (MongoDsn, ...)} mongo: Optional[AsyncIOMotorClient] async def startup(self, state: State) -> State: self.mongo = AsyncIOMotorClient(state.settings.mongo_dsn) state.mongo = self.mongo return state async def shutdown(self) -> None: if self.mongo: self.mongo.close()
class DatabaseService(ServiceCog): def __init__(self, bot, *args, **kwargs): super().__init__(bot) database_name = kwargs.pop("database_name", "bot") self.mongo_client = AsyncIOMotorClient(*args, **kwargs) self.db = self.mongo_client[database_name] self.bot.db = self.db def cog_unload(self): if hasattr(self.bot, "db"): del self.bot.db self.mongo_client.close()
class Connector(AbstractConnector): default_mongo_uri = "mongodb://localhost:27017/" # List of methods that will be bind to connector from AsyncIOMotorClient __bind_methods = ( "list_databases", "list_database_names", "server_info", "start_session", "end_session", "drop_database", "get_database", "close", ) def __init__(self, config=None, *, context=None, loop=None): super().__init__(config, context=context, loop=loop) self._client = None self._client_config = None def set_config(self, config: ValueExtractor) -> None: cfg = config.new_parent(logger=__package__) super().set_config(cfg) client_config = dict(self.config.get("client", {})) if client_config: self._client_config = client_config return # Support old uri only config. self._client_config = { "host": self.config.get("uri", self.default_mongo_uri), } async def connect(self): if self._client is None: self._client = AsyncIOMotorClient(**self._client_config) for method_name in self.__bind_methods: f = getattr(self._client, method_name) if f: setattr(self, method_name, f) async def disconnect(self): self._client.close() def __getattr__(self, item): return self._client[item] def __getitem__(self, item): return self._client[item]
async def mongodb_client(): client = AsyncIOMotorClient( "mongodb://localhost:27017", serverSelectionTimeoutMS=100, uuidRepresentation="standard", ) try: await client.server_info() yield client client.close() except pymongo.errors.ServerSelectionTimeoutError: pytest.skip("MongoDB not available", allow_module_level=True) return
class TestChildren(AioTestCase): async def setUp(self): self.client = AsyncIOMotorClient(MONGO_URI) self.table = self.client.tests.tests self.papers = [ {"type": "RealMongoTree", "path": "/testchildren/paper", "name": "Child 1", "slug": "child-1"}, {"type": "RealMongoTree", "path": "/testchildren/paper", "name": "Child 2", "slug": "child-2"}, {"type": "RealMongoTree", "path": "/testchildren/paper", "name": "Child 3", "slug": "child-3"}, {"type": "User", "path": "/testchildren/paper", "name": "User 1", "slug": "user-1"}, {"type": "User", "path": "/testchildren/paper", "name": "User 2", "slug": "user-2"}, {"type": "RealMongoTree", "path": "/testchildren", "name": "Paper", "slug": "paper", "members": [], "elements": ["child-1", "child-2", "child-3"]} ] for paper in self.papers: result = await self.table.insert_one(paper) paper["_id"] = result.inserted_id if paper["type"] == "User": self.papers[-1]["members"].append(paper["_id"]) async def tearDown(self): for paper in self.papers: await self.table.delete_one({"_id": paper["_id"]}) self.client.close() async def testNoTable(self): model = models.RealMongoTree() with self.assertRaises(InvalidOperation): await model.children("elements", None) async def test(self): model = models.RealMongoTree(self.table) await model.get(_id = self.papers[-1]["_id"]) children = await model.children("elements", models) ids = [child["_id"] for child in children.get_data()] for child in self.papers[:2]: self.assertIn(child["_id"], ids) async def testMembers(self): model = models.RealMongoTree(self.table) await model.get(_id = self.papers[-1]["_id"]) children = await model.children("members", models) ids = [child["_id"] for child in children.get_data()] for user in self.papers[3:5]: self.assertIn(user["_id"], ids)
class DataBase: @retry(wait=wait_random_exponential(multiplier=1, max=60)) def connect(self): hostname = os.getenv("DB_HOSTNAME", "db") self.client = AsyncIOMotorClient(host=hostname) self.db: AsyncIOMotorDatabase = self.client["database"] print("Connected to database") def disconnect(self): self.client.close() print("Disconnected from database") async def insert(self, collection: str, doc: dict): await self.db[collection].insert_one(doc)
class MongoDBConnection(): """MongoDB Connection""" def __init__(self, host='127.0.0.1', port=27017): """ be sure to use the ip address not name for local windows""" self.host = host self.port = port self.connection = None def __enter__(self): self.connection = AsyncIOMotorClient(self.host, self.port) #self.connection = MongoClient(self.host, self.port) return self def __exit__(self, exc_type, exc_val, exc_tb): self.connection.close()
class Database(metaclass=SingletonMeta): _client: AsyncIOMotorClient = None def __init__(self, connectionString, maxPoolSize, minPoolSize): self._connectionString = connectionString self._maxPoolSize = maxPoolSize self._minPoolSize = minPoolSize async def connect(self): self._client = AsyncIOMotorClient(str(self._connectionString), maxPoolSize=self._maxPoolSize, minPoolSize=self._minPoolSize) async def disconnect(self): self._client.close()
class AsyncIoMotor: app_string: str = None client: AsyncIOMotorClient = None db: AsyncIOMotorDatabase = None fs: AsyncIOMotorGridFSBucket = None environment: str = "development" def __getitem__(self, collection_name: str) -> AsyncIOMotorCollection: if not hasattr(self, collection_name): setattr(self, collection_name, self.db.get_collection(collection_name)) return getattr(self, collection_name) def init_app(self, app: FastAPI, uri: str = None, env: str = "development") -> None: async def connect(): await self.connect_to_mongo(uri=uri) async def close(): await self.close_mongo_connection() self.app_string = ''.join(c if c.isalpha() else "_" for c in list(app.title.lower())) self.environment = env app.add_event_handler("startup", connect) app.add_event_handler("shutdown", close) async def connect_to_mongo(self, uri=None): info("Connecting to the database... (new-extension)") if uri: self.client = AsyncIOMotorClient(uri) else: self.client = AsyncIOMotorClient() self.db = self.client[f"{self.app_string}_{self.environment}"] self.fs = AsyncIOMotorGridFSBucket(self.db) if self.environment == "development": from .mongo_testing_database import init_testing_database await init_testing_database() info("Database connection succeeded!") async def close_mongo_connection(self): info("Closing database connection...") self.client.close() info("Database connection closed!")
class TestAncestors(AioTestCase): async def setUp(self): self.client = AsyncIOMotorClient(MONGO_URI) self.table = self.client.tests.tests self.papers = [ {"type": "MinimalMongoTree", "path": "/", "name": "Parent 1", "slug": "parent-1"}, {"type": "MinimalMongoTree", "path": "/parent-1", "name": "Parent 2", "slug": "parent-2"}, {"type": "MinimalMongoTree", "path": "/parent-1/parent-2", "name": "Paper", "slug": "paper"} ] for paper in self.papers: result = await self.table.insert_one(paper) paper["_id"] = result.inserted_id async def tearDown(self): for paper in self.papers: await self.table.delete_one({"_id": paper["_id"]}) self.client.close() async def testNoTable(self): model = models.MinimalMongoTree() model.load({"type": "MinimalMongoTree", "path": "/parent-1/parent-2", "name": "Paper no table", "slug": "paper-no-table"}) with self.assertRaises(InvalidOperation): await model.ancestors(models) async def test(self): model = models.MinimalMongoTree(self.table) await model.get(_id = self.papers[-1]["_id"]) ancestors = await model.ancestors(models) self.assertEqual(len(ancestors), 2) self.assertIsInstance(ancestors[0], models.MinimalMongoTree) self.assertEqual(ancestors[0].get_data()["_id"], self.papers[0]["_id"]) self.assertIsInstance(ancestors[1], models.MinimalMongoTree) self.assertEqual(ancestors[1].get_data()["_id"], self.papers[1]["_id"]) async def testParentOnly(self): model = models.MinimalMongoTree(self.table) await model.get(_id = self.papers[-1]["_id"]) parent = await model.ancestors(models, True) self.assertIsInstance(parent, models.MinimalMongoTree) self.assertEqual(parent.get_data()["_id"], self.papers[-2]["_id"])
class DataBase(Base): db: AsyncIOMotorDatabase db_client: AsyncIOMotorClient def __init__(self: "Bot", **kwargs: Any): self._init_db() self.db = self.db_client.get_database("caligo") super().__init__(**kwargs) def _init_db(self) -> None: self.db_client = AsyncIOMotorClient(self.getConfig["db_uri"], connect=False) async def close_db(self) -> None: self.db_client.close() def get_db(self: "Bot", name: str) -> AgnosticCollection: return self.db.get_collection(name)
class MongoPipelineABC(abc.ABC): def __init__(self, mongo_uri): self.mongo_uri = mongo_uri @classmethod def from_crawler(cls, crawler): return cls(mongo_uri=crawler.settings.get('MONGO_URI')) def open_spider(self, spider): self.client = AsyncIOMotorClient(self.mongo_uri) self.db = self.client.get_default_database() def close_spider(self, spider): self.client.close() @abc.abstractmethod def process_item(self, item, spider): pass
class TestDeleteTree(AioTestCase): async def setUp(self): self.client = AsyncIOMotorClient(MONGO_URI) self.table = self.client.tests.tests self.papers = [ {"type": "RealMongoTree", "path": "/testdelete/paper", "name": "Child 1", "slug": "child-1"}, {"type": "RealMongoTree", "path": "/testdelete/paper", "name": "Child 2", "slug": "child-2"}, {"type": "RealMongoTree", "path": "/testdelete/paper", "name": "Child 3", "slug": "child-3"}, {"type": "User", "path": "/testdelete/paper", "name": "User 1", "slug": "user-1"}, {"type": "User", "path": "/testdelete/paper", "name": "User 2", "slug": "user-2"}, {"type": "RealMongoTree", "path": "/testdelete", "name": "Paper", "slug": "paper", "members": [], "elements": ["child-1", "child-2", "child-3"]} ] for paper in self.papers: result = await self.table.insert_one(paper) paper["_id"] = result.inserted_id if paper["type"] == "User": self.papers[-1]["members"].append(paper["_id"]) async def tearDown(self): for paper in self.papers: await self.table.delete_one({"_id": paper["_id"]}) self.client.close() async def testNoTable(self): model = models.RealMongoTree() with self.assertRaises(InvalidOperation): await model.delete() async def test(self): model = models.RealMongoTree(self.table) await model.get(_id = self.papers[-1]["_id"]) url = model.get_url() await model.delete(models) children = await self.table.find({"path": {"$regex": "^{}".format(url)}}).to_list(None) self.assertFalse(children)
class HF1(commands.Bot): def __init__(self): super().__init__( command_prefix='!', description= "A simple yet powerful moderation bot. Written in discord.py", intents=intents) self.config = Config() self.db: AsyncIOMotorDatabase self.functions = Functions() async def on_ready(self): print(f"{self.user.name} ({self.user.id}) is online") print("______________") pstatus = f"github.com/iphonediscord/hf1" await self.change_presence(activity=discord.Game(name=pstatus), status=discord.Status.online) print(f'\n Bot presence set to "{pstatus}"') print("______________") print("Loading extensions...") for cog in cogs: self.load_extension(cog) print("Loaded extensions") print("______________") async def init_http(self): self.session = ClientSession() async def init_mongo(self) -> None: self.mongo = AsyncIOMotorClient(MONGO_URI) await self.mongo.admin.command("ismaster") self.db = self.mongo.HF1 print("Connected to mongo") print("______________") async def close(self): await super().close() await self.session.close() self.mongo.close()
class TestUpdateField(AioTestCase): async def setUp(self): self.client = AsyncIOMotorClient(MONGO_URI) self.table = self.client.tests.tests self.papers = [ {"type": "RealMongoTree", "path": "/testupdatefield/paper", "name": "Child 1", "slug": "child-1"}, {"type": "RealMongoTree", "path": "/testupdatefield/paper", "name": "Child 2", "slug": "child-2"}, {"type": "RealMongoTree", "path": "/testupdatefield/paper", "name": "Child 3", "slug": "child-3"}, {"type": "User", "path": "/testupdatefield/paper", "name": "User 1", "slug": "user-1"}, {"type": "User", "path": "/testupdatefield/paper", "name": "User 2", "slug": "user-2"}, {"type": "RealMongoTree", "path": "/testupdatefield", "name": "Paper", "slug": "paper", "members": [], "elements": ["child-1", "child-2", "child-3"]} ] for paper in self.papers: result = await self.table.insert_one(paper) paper["_id"] = result.inserted_id if paper["type"] == "User": self.papers[-1]["members"].append(paper["_id"]) async def tearDown(self): for paper in self.papers: await self.table.delete_one({"_id": paper["_id"]}) self.client.close() async def testNoTable(self): model = models.RealMongoTree() with self.assertRaises(InvalidOperation): await model.update({"name": "This will fail"}, models) async def testValidationError(self): model = models.RealMongoTree(self.table) await model.get(_id = self.papers[-1]["_id"]) with self.assertRaises(ValidationError): await model.update({"path": 25}, models)
async def mongo_engine(app): mongo_conf = app.config.mongo mongo_uri = f"mongodb://{mongo_conf.host}:{mongo_conf.port}" db_name = mongo_conf.database conn = AsyncIOMotorClient(mongo_uri, username=mongo_conf.user, password=mongo_conf.password, authSource=mongo_conf.auth_source, maxPoolSize=mongo_conf.max_pool_size, io_loop=app.loop, connect=True, connectTimeoutMS=5000) app.mongo = conn[db_name] await ManageIndexes(app)() yield conn.close()
class AMAbot(commands.Bot): def __init__(self): super().__init__( command_prefix = Config.prefix, description = "Host crowd sources Q&As directly in your Discord guild. https://github.com/wwwaffles/amabot" ) self.config = Config() self.db: AsyncIOMotorDatabase self.functions = Functions() async def on_ready(self): print(f"{self.user.name} ({self.user.id}) is online") print("______________") # Sets playing status to "watching your questions" await self.change_presence(activity = discord.Activity(name = "your questions | github.com/wwwaffles/amabot", type = 3)) # Attempts to load extensions print("Loading extensions...") for cog in cogs: self.load_extension(cog) print("Loaded extensions") print("______________") async def init_http(self): self.session = ClientSession() async def init_mongo(self) -> None: self.mongo = AsyncIOMotorClient(MONGO_URI) await self.mongo.admin.command("ismaster") self.db = self.mongo.amabot print("Connected to database") print("______________") async def close(self): await super().close() await self.session.close() self.mongo.close()
class MongoManager(object): client: AsyncIOMotorClient = None db: AsyncIOMotorDatabase = None async def connect_to_database(self): logging.info("Connecting to MongoDB.") self.client = AsyncIOMotorClient('mongodb://localhost:27017', maxPoolSize=10, minPoolSize=10) self.db = self.client.main_db logging.info("Connected to MongoDB.") async def close_database_connection(self): logging.info("Closing connection with MongoDB.") self.client.close() logging.info("Closed connection with MongoDB.") async def get_posts(self) -> List[PostDB]: posts_list = [] posts_q = self.db.posts.find() async for post in posts_q: posts_list.append(PostDB(**post, id=post['_id'])) return posts_list async def get_post(self, post_id: OID) -> PostDB: post_q = await self.db.posts.find_one({'_id': ObjectId(post_id)}) if post_q: return PostDB(**post_q, id=post_q['_id']) async def delete_post(self, post_id: OID): await self.db.posts.delete_one({'_id': ObjectId(post_id)}) async def update_post(self, post_id: OID, post: PostDB): await self.db.posts.update_one({'_id': ObjectId(post_id)}, {'$set': post.dict(exclude={'id'})}) async def add_post(self, post: PostDB): await self.db.posts.insert_one(post.dict(exclude={'id'}))
class Genius(commands.Bot): def __init__(self): super().__init__( command_prefix = "//", description = "" ) self.config = Config() self.db: AsyncIOMotorDatabase self.functions = Functions() async def on_ready(self): print(f"{self.user.name} ({self.user.id}) is online") print("______________") # Sets playing status to "watching your questions" await self.change_presence(activity = discord.Activity(name = "you", type = 3)) # Attempts to load extensions print("Loading extensions...") for cog in cogs: self.load_extension(cog) print("Loaded extensions") print("______________") async def init_http(self): self.session = ClientSession() async def init_mongo(self) -> None: self.mongo = AsyncIOMotorClient(MONGO_URI) await self.mongo.admin.command("ismaster") self.db = self.mongo.genius print("Connected to database") print("______________") async def close(self): await super().close() await self.session.close() self.mongo.close()