def default_open_connection(app, loop):
log.info('opening motor connection')
client = AsyncIOMotorClient(app.config.MOTOR_URI, io_loop=loop)
db = client.get_default_database()
app.motor_client = client
BaseModel.__motor_client__ = client
BaseModel.__motor_db__ = db
async def on_start(app):
client = AsyncIOMotorClient(config.mongodb_uri)
app['db_client'] = client
app['db'] = client.get_default_database() # defined in mongodb_uri
app['websockets'] = []
await app['db'].data.create_index([('x', ASCENDING), ('y', ASCENDING)])
asyncio.ensure_future(ping(app))
async def upsert_user_config(client: AsyncIOMotorClient,
user_config: UserConfig) -> str:
collection: AsyncIOMotorCollection = client.get_default_database(
)[_USER_CONFIG_COLLECTION]
result = await collection.replace_one({},
user_config.to_dict(),
upsert=True)
return result.upserted_id
async def get_user_config_optional(
client: AsyncIOMotorClient) -> Optional[UserConfig]:
collection = client.get_default_database()[_USER_CONFIG_COLLECTION]
config = await collection.find_one(None)
if config is None:
return None
return UserConfig.from_dict(config)
async def main() -> None:
client = ApiClient(METABOT_URL)
api = AsyncApis(client).metabot_api
motor = AsyncIOMotorClient(MONGODB_URI)
db = motor.get_default_database()
history = db[HISTORY_COLLECTION]
await announce(api, history)
async def main() -> None:
client = ApiClient(METABOT_URL)
api = AsyncApis(client).metabot_api
motor = AsyncIOMotorClient(MONGODB_URI)
db = motor.get_default_database()
users = db[USERS_COLLECTION]
async for members in list_users(api):
users_list = [member['id'] for member in members]
print(f'Adding more leave days to users: {users_list}') # noqa T001
await increase_days_by_config(users, users_list)
async def on_start(app):
config = Config()
client = AsyncIOMotorClient(config.mongodb_uri)
app['db_client'] = client
app['db'] = client.get_default_database() # defined in mongodb_uri
app['config'] = config
app['ping_task'] = asyncio.ensure_future(ping(app))
app['online'] = set()
app['rooms'] = set()
app['waiting'] = None # slot for waiting user
await app['db'].users.create_index('name', unique=True)
def default_open_connection(app, loop, name=None, uri=None):
if not name:
name = app.name
logger.info('opening motor connection for [{}]'.format(name))
client = AsyncIOMotorClient(uri or app.config.MOTOR_URI, io_loop=loop)
db = client.get_default_database()
app.motor_client = client
BaseModel.__motor_client__ = client
BaseModel.__motor_db__ = db
if not hasattr(app, 'motor_clients'):
app.motor_clients = {}
app.motor_clients[name] = client
BaseModel.__motor_clients__[name] = client
BaseModel.__motor_dbs__[name] = db
class MongoPipelineABC(abc.ABC):
def __init__(self, mongo_uri):
self.mongo_uri = mongo_uri
@classmethod
def from_crawler(cls, crawler):
return cls(mongo_uri=crawler.settings.get('MONGO_URI'))
def open_spider(self, spider):
self.client = AsyncIOMotorClient(self.mongo_uri)
self.db = self.client.get_default_database()
def close_spider(self, spider):
self.client.close()
@abc.abstractmethod
def process_item(self, item, spider):
pass
class Storage:
db = None
mongo_client = None
def __init__(self, mongo_uri):
self.mongo_client = AsyncIOMotorClient(mongo_uri)
self.db = self.mongo_client.get_default_database()
def find(self, collection):
pass
async def update(self, collection, query, update, options):
return await self.db[collection].update(query, update, **options)
def insert(self, collection, document):
pass
def remove(self, collection):
pass
class Mongo(object):
"""MongoDB driver for MongoDB Atlas or local server."""
def __init__(self):
"""Load credentials and initialize client."""
URI = env.str('MONGO_URI', default='mongodb://localhost:27017/nazurin')
self.client = AsyncIOMotorClient(URI)
self.db = self.client.get_default_database()
def collection(self, key: str):
self._collection = self.db[key]
return self
def document(self, key: Union[str, int]):
self._document = key
return self
async def get(self) -> Optional[dict]:
return await self._collection.find_one({'_id': self._document})
async def exists(self) -> bool:
count = await self._collection.count_documents({'_id': self._document},
limit=1)
return count > 0
async def insert(self, key: Optional[Union[str, int]], data: dict) -> bool:
if key:
data['_id'] = key
try:
result = await self._collection.insert_one(data)
return result.acknowledged
except DuplicateKeyError as error:
raise NazurinError('Already exists in database.') from error
async def update(self, data: dict) -> bool:
result = await self._collection.update_one({'_id': self._document},
{'$set': data})
return result.modified_count == 1
async def delete(self) -> bool:
result = await self._collection.delete_one({'_id': self._document})
return result.deleted_count == 1
async def on_start(app):
client = AsyncIOMotorClient(config.mongodb_uri)
app['db_client'] = client
app['db'] = client.get_default_database() # defined in mongodb_uri
app['websockets'] = []
app['websockets_colors'] = {}
app['colors_websocket'] = {}
app['users'] = {}
app['limiter'] = RateLimiter(config.rate_limit)
try:
async with ClientSession() as session:
async with session.get(config.jwt_certs_url) as res:
app['jwt_cers'] = await res.json()
logger.info("Load jwt certs: %s", app['jwt_cers'])
except:
logger.error('Get jwt certs error')
raise
await app['db'].data.create_index([('x', ASCENDING), ('y', ASCENDING)])
asyncio.ensure_future(ping(app))
import asyncio
import os
from motor.motor_asyncio import AsyncIOMotorClient
from datetime import datetime
client = AsyncIOMotorClient(os.getenv('MONGO_URI'))
db = client.get_default_database()
async def update_fan_page_from_fb_user():
cursor = db.fb_user.find(
{
'action_path': {
'$ne': "/a/mobile/friends/add_friend.php",
'$exists': 1
}
}
)
futures = [
db.fan_page.update_one(
{'page_id': user['uid']},
{
'$set': {
'link': 'https://www.facebook.com' + user['profile_link']
}
},
upsert=True
)
async for user in cursor
class CachedMarket(MarketBase):
def __init__(self, market: MarketBase, mongodb_uri: Optional[str] = None):
self._env = env = Env()
env.read_env()
self._mongodb_uri = mongodb_uri if mongodb_uri else env.str(
'MONGODB_URI')
self._market = market
self._mongo_client = AsyncIOMotorClient(self._mongodb_uri)
self._database = self._mongo_client.get_default_database('markets')
self._hist_mongo_client = None
self._initied = False
def connect(self):
self._market.connect()
def disconnect(self):
self._mongo_client.close()
self._market.disconnect()
def _collection(self, symbol: str):
db = self._database
if self._market.__class__.__market_name__ is None or self._market.__class__.__timeframe__ is None:
raise Exception(
'Initialize market cache error (without market_name or timeframe)'
)
collection_name = '%s_%s_%s' % (self._market.__class__.__market_name__,
symbol,
self._market.__class__.__timeframe__)
collection = db[collection_name]
return collection
async def _insert_db(self, symbol: str, klines: List[dict]):
collection = self._collection(symbol)
await collection.bulk_write([
ReplaceOne({'datetime': kline['datetime']}, kline, upsert=True)
for kline in klines
])
async def _initial_cache(self, symbol: str):
collection = self._collection(symbol)
if await collection.count_documents({}) > 0:
rs = await (collection.find().sort([('datetime', -1)
]).limit(1).to_list(1))
last_kline = rs[0]
history_klines = await self._market.get_kline_histories(
symbol, from_ts=last_kline['datetime'])
else:
await collection.create_index('datetime', unique=True)
history_klines = await self._market.get_kline_histories(symbol,
limit=5000)
await self._insert_db(symbol, history_klines)
async def watch_klines(self, symbol: str):
if not self._initied:
self.logger.debug('init cache from watch_klines')
await self._initial_cache(symbol)
self._initied = True
self.logger.debug('do watch klines from delegate')
async for kline in self._market.watch_klines(symbol):
await self._insert_db(symbol, [kline])
yield kline
async def get_kline_histories(self,
symbol: str,
from_ts: Optional[int] = None,
limit: Optional[int] = None):
if not self._initied:
await self._initial_cache(symbol)
self._initied = True
collection = self._collection(symbol)
criteria = {}
if from_ts:
criteria = {'datetime': {'$gte': from_ts}}
cursor = collection.find(criteria).sort([('datetime', -1)])
if limit:
cursor = cursor.limit(limit)
count = limit
else:
count = await collection.count_documents(criteria)
return reversed(
list(
map(
lambda x:
{k: v
for k, v in iteritems(x)
if not k.startswith('_')}, await cursor.to_list(count))))
