def save_capped_db(ops, coll): try: result = coll.bulk_write(ops) except Exception as e: log.exception("Error saving CMC tickers. %s", str(e)) db = get_db() stats = db.command("collstats",coll.name) if stats['capped'] == False: return False max_size = stats['maxSize'] # Capped collection full. Drop and re-create w/ indexes. if stats['size'] / max_size > 0.9: from pymongo import IndexModel, ASCENDING, DESCENDING log.info("Capped collection > 90% full. Dropping and recreating.") name = coll.name coll.drop() db.create_collection(name, capped=True, size=max_size) idx1 = IndexModel( [("symbol", ASCENDING)], name="symbol") idx2 = IndexModel( [("date", DESCENDING)], name="date_-1") db[name].create_indexes([idx1, idx2]) log.info("Retrying bulk_write") try: result = db[name].bulk_write(ops) except Exception as e: log.exception("Error saving CMC tickers. %s", str(e)) return False else: log.error("Size is <90% max. Unknown error.") return True
def create_indexes() -> None: ticker_ts_index = IndexModel([('ticker', ASCENDING), ('time_series.time', ASCENDING)], unique=True) name_index = IndexModel([('name', ASCENDING)], unique=True) cusip_index = IndexModel([('cusip', ASCENDING)], unique=True) isin_index = IndexModel([('isin', ASCENDING)], unique=True) sedol_index = IndexModel([('sedol', ASCENDING)], unique=True) _collection.create_indexes([ticker_ts_index, name_index, cusip_index, isin_index, sedol_index])
class DbReservation(BaseDocument): __collection_name__ = 'reservation' __indexes__ = [ IndexModel([('end', ASCENDING), ('start', ASCENDING)]), IndexModel([('items', ASCENDING), ('end', ASCENDING), ('start', ASCENDING)]), IndexModel([('user_id', ASCENDING), ('end', ASCENDING), ('start', ASCENDING)]), IndexModel([('user_id', ASCENDING), ('items', ASCENDING), ('end', ASCENDING), ('start', ASCENDING)]), ] id: UUID = Field(..., alias='_id') type: ReservationType = Field(...) name: str = Field(...) start: date = Field(...) end: date = Field(...) user_id: str = Field(...) team_id: Optional[str] = None contact: str = Field(...) items: List[UUID] = Field(...)
def _create_indexes(self): uid_unique_index = IndexModel("uid", unique=True) creator_index = IndexModel("splash_md.creator") sort_index = IndexModel([("splash_md.last_edit", DESCENDING), ("uid", DESCENDING)]) self._collection.create_indexes( [uid_unique_index, creator_index, sort_index])
def __create_index(self, mongo_db_name, col_name): self.client[mongo_db_name][col_name].create_indexes([ IndexModel([("insertion_timestamp", pymongo.ASCENDING)]), IndexModel([("level", pymongo.ASCENDING)]), IndexModel([("caller", pymongo.ASCENDING)]), IndexModel([("pid", pymongo.ASCENDING)]) ])
def build_index(self): indexes_model = [ IndexModel([("date_report", ASCENDING)]), IndexModel([("media", ASCENDING)]), IndexModel([("success", ASCENDING)]) ] self.collection.create_indexes(indexes_model)
def create_index_models(coll_name_ext, ind_field_names): ''' Функция не индексирует, а лишь готовит руководящие индексацией IndexModel-объекты. В случае db-VCF и db-BED, независимо от того, указал исследователь имена индексируемых полей или нет, модели индексации формируются для полей с локациями и основополагающими идентификаторами. А то, что облигатные индексы полей с хромосами и позициями проектируются составными, позволит потом парсерам эффективно сортировать свои результаты стандартным для вычислительной генетики образом - по геномной локации. Для обозначенных исследователем полей планируется построение одиночных индексов. Если они придутся на хромосому или позицию db-VCF/db-BED, то будут сосуществовать с обязательным компаундным индексом. Обновить набор индексов впоследствии можно будет отдельной программой из состава проекта, а также с помощью MongoDB Shell или Compass. ''' if coll_name_ext == 'vcf': index_models = [ IndexModel([('#CHROM', ASCENDING), ('POS', ASCENDING)]), IndexModel([('ID', ASCENDING)]) ] elif coll_name_ext == 'bed': index_models = [ IndexModel([('chrom', ASCENDING), ('start', ASCENDING), ('end', ASCENDING)]), IndexModel([('name', ASCENDING)]) ] else: index_models = [] if ind_field_names is not None: index_models += [ IndexModel([(ind_field_name, ASCENDING)]) for ind_field_name in ind_field_names ] return index_models
class AuthorizationCode(BaseDocument, AuthorizationCodeMixin): __indexes__ = [ IndexModel([('expiration_time', ASCENDING)], expireAfterSeconds=0), IndexModel([('user_id', ASCENDING)]), ] __collection_name__ = 'authorization_code' code: str = Field(..., alias='_id') user_id: str = ... client_id: str redirect_uri: str response_type: Optional[str] scope: Optional[str] nonce: Optional[str] auth_time: int = ... expiration_time: datetime = ... code_challenge: Optional[str] code_challenge_method: Optional[str] def is_expired(self): return self.expiration_time < datetime.utcnow() def get_redirect_uri(self): return self.redirect_uri def get_scope(self): return self.scope def get_auth_time(self): return self.auth_time def get_nonce(self): return self.nonce
def get_content_collection(chat_id): logger.info('Start copying data.') mongo_client = pymongo.MongoClient() db = mongo_client['tg_backup'] # Move chat data to separate collection # First of all: create index! content_collection = db['content'] chat_id_index_name = 'chat_id' chat_id_index = IndexModel([('chat_id', ASCENDING)], name=chat_id_index_name) if chat_id_index_name not in content_collection.index_information(): content_collection.create_indexes([chat_id_index]) # Second: create new collection tmp_collection = db['tmp_content'] cursor_for_copy = content_collection.find({'chat_id': chat_id, **MONGO_HAS_SOME_TEXT}) # Third: move all docs with any text if not already if tmp_collection.count() != cursor_for_copy.count(): tmp_collection.remove() copy_bar = tqdm(cursor_for_copy, total=cursor_for_copy.count(), unit='msg', smoothing=0.01, desc='Moving data') for doc in copy_bar: tmp_collection.insert(doc) logger.info('Copied successfuly.') # Fourth: create new indexes msg_date_index_name = 'date_index' msg_date_index = IndexModel([('date', DESCENDING)], name=msg_date_index_name) if msg_date_index_name not in tmp_collection.index_information(): tmp_collection.create_indexes([msg_date_index]) else: logger.info('Data already copied.') return tmp_collection
def __init__(self, mongodb_url: str, db_name: str, collection_name: str): self.collection_name = collection_name self.db_name = db_name self._conn: MongoClient = MongoClient(mongodb_url) self._get_collection().create_indexes([ IndexModel([("image_hash", HASHED)]), IndexModel([("fid", ASCENDING)], unique=True), ])
class Meta: indexes = [ IndexModel([('userName', 1)], unique=True), IndexModel([('email', 1)], unique=True) ] # unique field 지정은 이렇게 collection_name = 'users' # 지정 안해주면 "User" collection을 따로 만들어버림 final = True # _cls 필드 저장 안하도록 설정
def set_meal_indexes(self): self.db.meals.create_indexes([ IndexModel([("product_ids", ASCENDING)], name='product_ids_', unique=True), IndexModel([("status", ASCENDING)], name='status_'), IndexModel([("amount", ASCENDING)], name='amount_') ])
def set_couriers_indexes(self): self.db.couriers.create_indexes([ IndexModel([("position_id", ASCENDING)], name='position_id_', unique=True), IndexModel([("status", ASCENDING)], name='status_'), IndexModel([("age", DESCENDING)], name='age_') ])
def set_feedback_indexes(self): self.db.feedback.create_indexes([ IndexModel([("customer_id", ASCENDING)], name='customer_id_'), IndexModel([("order_id", ASCENDING)], name='order_id_'), IndexModel([("restaurant_id", ASCENDING)], name='restaurant_id_'), IndexModel([("courier_id", ASCENDING)], name='courier_id_'), IndexModel([("rating", ASCENDING)], name='rating_') ])
class Calls: calls: pymotyc.Collection[CallModel] = pymotyc.Collection( indexes=[ IndexModel('call_id', unique=True), IndexModel('call_guid', unique=True), 'phone', 'topic' ], )
def create_indexes(): conn = MongoClient(conf.MONGO_URL) conn.main.static.create_indexes([ IndexModel([("id", ASCENDING)], unique=True), IndexModel([("expire", ASCENDING)], expireAfterSeconds=600) ]) conn.main.teams.create_indexes( [IndexModel([("ip", ASCENDING)], unique=True)]) conn.close()
def set_restaurants_indexes(self): self.db.restaurants.create_indexes([ IndexModel([("position_id", ASCENDING)], name='position_id_', unique=True), IndexModel([("meal_ids", ASCENDING)], name='meal_ids_'), IndexModel([("order_ids", ASCENDING)], name='order_ids_'), IndexModel([("status", ASCENDING)], name='status_') ])
def __init__(self): index_keywords = IndexModel([('keywords', pymongo.TEXT)], name=index_name, default_language='english') index_sha1=IndexModel([('sha1', pymongo.ASCENDING)], name="index_sha1") client = pymongo.MongoClient(dbstr) self.collection = client['crawlerdb']['news'] if not index_name in self.collection.index_information(): self.collection.create_indexes([index_keywords]) if not "index_sha1" in self.collection.index_information(): self.collection.create_indexes([index_sha1])
def _init_collection(self): indexes = [ IndexModel([('gold', ASCENDING)]), IndexModel([('subject', ASCENDING)]) ] logger.debug('inserting %d indexes', len(indexes)) self.collection.create_indexes(indexes) logger.debug('done')
class Peer(BaseClass): """Holds information about the chains state.""" COLLECTION = "peers" INDEXES = [ IndexModel([("type", ASCENDING)]), IndexModel([("address", ASCENDING)]), IndexModel([("last_seen", DESCENDING)]) ]
def __init__(self, function_result_status_persistance_conf, queue_name): self.function_result_status_persistance_conf = function_result_status_persistance_conf if self.function_result_status_persistance_conf.is_save_status: task_status_col = self.mongo_db_task_status.get_collection(queue_name) task_status_col.create_indexes([IndexModel([("insert_time_str", -1)]), IndexModel([("insert_time", -1)]), IndexModel([("params_str", -1)]), IndexModel([("success", 1)]) ], ) task_status_col.create_index([("utime", 1)], expireAfterSeconds=function_result_status_persistance_conf.expire_seconds) # 只保留7天。 self._mongo_bulk_write_helper = MongoBulkWriteHelper(task_status_col, 100, 2)
def _model(index): """Converts to IndexModel""" if isinstance(index, list): index = IndexModel([_tuple(x) for x in index]) elif not isinstance(index, IndexModel): index = IndexModel([_tuple(index)]) index.document['background'] = True return index
class Meta: """Post meta.""" collection = 'posts' indexes = [ IndexModel([('title', ASCENDING)], unique=True, name='title_index'), IndexModel([('author', ASCENDING), ('created', DESCENDING)], name='author_created_index') ]
def __init__(self, db_url: str): self._client = MongoClient(db_url) self._database = self._client[urlparse(db_url).path[1:]] self.data1 = self._init_collection( "data1", [IndexModel("created_at")], ) self.data2 = self._init_collection( "data2", [IndexModel("created_at")], )
async def setup_db(app): user_indexes = [ IndexModel([("username", ASCENDING)], name="username_index", unique=True), IndexModel([("cards", ASCENDING)], name="cards_index", sparse=True), IndexModel([("tokens.token", ASCENDING)], name="token_index", sparse=True), ] await app["db"].users.create_indexes(user_indexes)
def _make_indexes(self, collection: str) -> None: """Makes indexes used by Corintick. Metadata is not used for querying and therefore not indexed. Making `ix1` unique is meant to avoid accidentally inserting duplicate documents. """ ix1 = IndexModel([('uid', 1), ('start', -1), ('end', -1)], unique=True, name='default') ix2 = IndexModel([('uid', 1), ('end', -1), ('start', -1)], name='reverse') self.db.get_collection(collection).create_indexes([ix1, ix2])
def _init_collection(self): indexes = [ IndexModel([('subject_id', ASCENDING)]), IndexModel([('user_id', ASCENDING)]), IndexModel([('subject_id', ASCENDING), ('user_name', ASCENDING)]), IndexModel([('seen_before', ASCENDING), ('classification_id', ASCENDING)])] logger.debug('inserting %d indexes', len(indexes)) self.collection.create_indexes(indexes) logger.debug('done')
class TournamentDAO(DataStore): __metaclass__ = Singleton index_1 = IndexModel([('tournament_id', pymongo.ASCENDING)], unique=True, sparse=True) index_2 = IndexModel([('name', pymongo.ASCENDING)], unique=True) def __init__(self): super().__init__('Tournament') self._create_indexes([self.index_1, self.index_2])
def create_indexes(collection): while True: try: id = IndexModel([("id", ASCENDING)], unique=True) hashtags = IndexModel([("hashtags", ASCENDING)]) created = IndexModel([("created", ASCENDING)]) collection.create_indexes([id, hashtags, created]) except NotMasterError: print("Waiting for mongodb-server to become primary...") time.sleep(1) continue break
def parse_indexes(cls, indexes=[]): """only used for create_indexes""" indexes_ = [] for item in indexes or cls.__dict__.get("__indexes__", []): if isinstance(item, str): indexes_.append(IndexModel(get_sort(item, for_index=True))) else: indexes_.append( IndexModel(get_sort(item[0], for_index=True), **item[1])) return indexes_