class DataDriver(storage.DataDriverBase): BASE_CAPABILITIES = tuple(storage.Capabilities) _DRIVER_OPTIONS = options._config_options() _COL_SUFIX = "_messages_p" def __init__(self, conf, cache, control_driver): super(DataDriver, self).__init__(conf, cache, control_driver) self.mongodb_conf = self.conf[options.MESSAGE_MONGODB_GROUP] conn = self.connection server_info = conn.server_info()['version'] self.server_version = tuple(map(int, server_info.split('.'))) if self.server_version < (2, 2): raise RuntimeError( _('The mongodb driver requires mongodb>=2.2, ' '%s found') % server_info) if not len(conn.nodes) > 1 and not conn.is_mongos: if not self.conf.unreliable: raise RuntimeError( _('Either a replica set or a mongos is ' 'required to guarantee message delivery')) else: _mongo_wc = conn.write_concern.document.get('w') durable = (_mongo_wc == 'majority' or _mongo_wc >= 2) if not self.conf.unreliable and not durable: raise RuntimeError( _('Using a write concern other than ' '`majority` or > 2 makes the service ' 'unreliable. Please use a different ' 'write concern or set `unreliable` ' 'to True in the config file.')) # FIXME(flaper87): Make this dynamic self._capabilities = self.BASE_CAPABILITIES @property def capabilities(self): return self._capabilities def is_alive(self): try: # NOTE(zyuan): Requires admin access to mongodb return 'ok' in self.connection.admin.command('ping') except pymongo.errors.PyMongoError: return False def _health(self): KPI = {} KPI['storage_reachable'] = self.is_alive() KPI['operation_status'] = self._get_operation_status() message_volume = {'free': 0, 'claimed': 0, 'total': 0} for msg_col in [db.messages for db in self.message_databases]: msg_count_claimed = msg_col.find({'c.id': {'$ne': None}}).count() message_volume['claimed'] += msg_count_claimed msg_count_total = msg_col.find().count() message_volume['total'] += msg_count_total message_volume['free'] = (message_volume['total'] - message_volume['claimed']) KPI['message_volume'] = message_volume return KPI @decorators.lazy_property(write=False) def message_databases(self): """List of message databases, ordered by partition number.""" kwargs = {} if not self.server_version < (2, 6): # NOTE(flaper87): Skip mongodb versions below 2.6 when # setting the write concern on the database. pymongo 3.0 # fails with norepl when creating indexes. doc = self.connection.write_concern.document.copy() doc.setdefault('w', 'majority') doc.setdefault('j', False) kwargs['write_concern'] = pymongo.WriteConcern(**doc) name = self.mongodb_conf.database partitions = self.mongodb_conf.partitions databases = [] for p in range(partitions): db_name = name + self._COL_SUFIX + str(p) databases.append(self.connection.get_database(db_name, **kwargs)) return databases @decorators.lazy_property(write=False) def subscriptions_database(self): """Database dedicated to the "subscription" collection.""" name = self.mongodb_conf.database + '_subscriptions' return self.connection[name] @decorators.lazy_property(write=False) def connection(self): """MongoDB client connection instance.""" return _connection(self.mongodb_conf) @decorators.lazy_property(write=False) def message_controller(self): return controllers.MessageController(self) @decorators.lazy_property(write=False) def claim_controller(self): return controllers.ClaimController(self) @decorators.lazy_property(write=False) def subscription_controller(self): return controllers.SubscriptionController(self)
class DataDriver(storage.DataDriverBase): BASE_CAPABILITIES = tuple(storage.Capabilities) _DRIVER_OPTIONS = options._config_options() def __init__(self, conf, cache): super(DataDriver, self).__init__(conf, cache) self.mongodb_conf = self.conf[options.MESSAGE_MONGODB_GROUP] conn = self.connection server_version = conn.server_info()['version'] if tuple(map(int, server_version.split('.'))) < (2, 2): raise RuntimeError( _('The mongodb driver requires mongodb>=2.2, ' '%s found') % server_version) if not len(conn.nodes) > 1 and not conn.is_mongos: if not self.conf.unreliable: raise RuntimeError( _('Either a replica set or a mongos is ' 'required to guarantee message delivery')) else: wc = conn.write_concern.get('w') majority = (wc == 'majority' or wc >= 2) if not wc: # NOTE(flaper87): No write concern specified, use majority # and don't count journal as a replica. Use `update` to avoid # overwriting `wtimeout` conn.write_concern.update({'w': 'majority'}) elif not self.conf.unreliable and not majority: raise RuntimeError( _('Using a write concern other than ' '`majority` or > 2 makes the service ' 'unreliable. Please use a different ' 'write concern or set `unreliable` ' 'to True in the config file.')) conn.write_concern['j'] = False # FIXME(flaper87): Make this dynamic self._capabilities = self.BASE_CAPABILITIES @property def capabilities(self): return self._capabilities def is_alive(self): try: # NOTE(zyuan): Requires admin access to mongodb return 'ok' in self.connection.admin.command('ping') except pymongo.errors.PyMongoError: return False def _health(self): KPI = {} KPI['storage_reachable'] = self.is_alive() KPI['operation_status'] = self._get_operation_status() message_volume = {'free': 0, 'claimed': 0, 'total': 0} for msg_col in [db.messages for db in self.message_databases]: msg_count_claimed = msg_col.find({'c.id': {'$ne': None}}).count() message_volume['claimed'] += msg_count_claimed msg_count_total = msg_col.find().count() message_volume['total'] += msg_count_total message_volume['free'] = (message_volume['total'] - message_volume['claimed']) KPI['message_volume'] = message_volume return KPI @decorators.lazy_property(write=False) def queues_database(self): """Database dedicated to the "queues" collection. The queues collection is separated out into its own database to avoid writer lock contention with the messages collections. """ name = self.mongodb_conf.database + '_queues' return self.connection[name] @decorators.lazy_property(write=False) def message_databases(self): """List of message databases, ordered by partition number.""" name = self.mongodb_conf.database partitions = self.mongodb_conf.partitions # NOTE(kgriffs): Partition names are zero-based, and # the list is ordered by partition, which means that a # caller can, e.g., get zaqar_mp0 by simply indexing # the first element in the list of databases: # # self.driver.message_databases[0] # return [ self.connection[name + '_messages_p' + str(p)] for p in range(partitions) ] @decorators.lazy_property(write=False) def subscriptions_database(self): """Database dedicated to the "subscription" collection.""" name = self.mongodb_conf.database + '_subscriptions' return self.connection[name] @decorators.lazy_property(write=False) def connection(self): """MongoDB client connection instance.""" return _connection(self.mongodb_conf) @decorators.lazy_property(write=False) def queue_controller(self): return controllers.QueueController(self) @decorators.lazy_property(write=False) def message_controller(self): return controllers.MessageController(self) @decorators.lazy_property(write=False) def claim_controller(self): return controllers.ClaimController(self) @decorators.lazy_property(write=False) def subscription_controller(self): return controllers.SubscriptionController(self)