class DataDriver(storage.DataDriverBase):
BASE_CAPABILITIES = tuple(storage.Capabilities)
_DRIVER_OPTIONS = options._config_options()
_COL_SUFIX = "_messages_p"
def __init__(self, conf, cache, control_driver):
super(DataDriver, self).__init__(conf, cache, control_driver)
self.mongodb_conf = self.conf[options.MESSAGE_MONGODB_GROUP]
conn = self.connection
server_info = conn.server_info()['version']
self.server_version = tuple(map(int, server_info.split('.')))
if self.server_version < (2, 2):
raise RuntimeError(
_('The mongodb driver requires mongodb>=2.2, '
'%s found') % server_info)
if not len(conn.nodes) > 1 and not conn.is_mongos:
if not self.conf.unreliable:
raise RuntimeError(
_('Either a replica set or a mongos is '
'required to guarantee message delivery'))
else:
_mongo_wc = conn.write_concern.document.get('w')
durable = (_mongo_wc == 'majority' or _mongo_wc >= 2)
if not self.conf.unreliable and not durable:
raise RuntimeError(
_('Using a write concern other than '
'`majority` or > 2 makes the service '
'unreliable. Please use a different '
'write concern or set `unreliable` '
'to True in the config file.'))
# FIXME(flaper87): Make this dynamic
self._capabilities = self.BASE_CAPABILITIES
@property
def capabilities(self):
return self._capabilities
def is_alive(self):
try:
# NOTE(zyuan): Requires admin access to mongodb
return 'ok' in self.connection.admin.command('ping')
except pymongo.errors.PyMongoError:
return False
def _health(self):
KPI = {}
KPI['storage_reachable'] = self.is_alive()
KPI['operation_status'] = self._get_operation_status()
message_volume = {'free': 0, 'claimed': 0, 'total': 0}
for msg_col in [db.messages for db in self.message_databases]:
msg_count_claimed = msg_col.find({'c.id': {'$ne': None}}).count()
message_volume['claimed'] += msg_count_claimed
msg_count_total = msg_col.find().count()
message_volume['total'] += msg_count_total
message_volume['free'] = (message_volume['total'] -
message_volume['claimed'])
KPI['message_volume'] = message_volume
return KPI
@decorators.lazy_property(write=False)
def message_databases(self):
"""List of message databases, ordered by partition number."""
kwargs = {}
if not self.server_version < (2, 6):
# NOTE(flaper87): Skip mongodb versions below 2.6 when
# setting the write concern on the database. pymongo 3.0
# fails with norepl when creating indexes.
doc = self.connection.write_concern.document.copy()
doc.setdefault('w', 'majority')
doc.setdefault('j', False)
kwargs['write_concern'] = pymongo.WriteConcern(**doc)
name = self.mongodb_conf.database
partitions = self.mongodb_conf.partitions
databases = []
for p in range(partitions):
db_name = name + self._COL_SUFIX + str(p)
databases.append(self.connection.get_database(db_name, **kwargs))
return databases
@decorators.lazy_property(write=False)
def subscriptions_database(self):
"""Database dedicated to the "subscription" collection."""
name = self.mongodb_conf.database + '_subscriptions'
return self.connection[name]
@decorators.lazy_property(write=False)
def connection(self):
"""MongoDB client connection instance."""
return _connection(self.mongodb_conf)
@decorators.lazy_property(write=False)
def message_controller(self):
return controllers.MessageController(self)
@decorators.lazy_property(write=False)
def claim_controller(self):
return controllers.ClaimController(self)
@decorators.lazy_property(write=False)
def subscription_controller(self):
return controllers.SubscriptionController(self)
class DataDriver(storage.DataDriverBase):
BASE_CAPABILITIES = tuple(storage.Capabilities)
_DRIVER_OPTIONS = options._config_options()
def __init__(self, conf, cache):
super(DataDriver, self).__init__(conf, cache)
self.mongodb_conf = self.conf[options.MESSAGE_MONGODB_GROUP]
conn = self.connection
server_version = conn.server_info()['version']
if tuple(map(int, server_version.split('.'))) < (2, 2):
raise RuntimeError(
_('The mongodb driver requires mongodb>=2.2, '
'%s found') % server_version)
if not len(conn.nodes) > 1 and not conn.is_mongos:
if not self.conf.unreliable:
raise RuntimeError(
_('Either a replica set or a mongos is '
'required to guarantee message delivery'))
else:
wc = conn.write_concern.get('w')
majority = (wc == 'majority' or wc >= 2)
if not wc:
# NOTE(flaper87): No write concern specified, use majority
# and don't count journal as a replica. Use `update` to avoid
# overwriting `wtimeout`
conn.write_concern.update({'w': 'majority'})
elif not self.conf.unreliable and not majority:
raise RuntimeError(
_('Using a write concern other than '
'`majority` or > 2 makes the service '
'unreliable. Please use a different '
'write concern or set `unreliable` '
'to True in the config file.'))
conn.write_concern['j'] = False
# FIXME(flaper87): Make this dynamic
self._capabilities = self.BASE_CAPABILITIES
@property
def capabilities(self):
return self._capabilities
def is_alive(self):
try:
# NOTE(zyuan): Requires admin access to mongodb
return 'ok' in self.connection.admin.command('ping')
except pymongo.errors.PyMongoError:
return False
def _health(self):
KPI = {}
KPI['storage_reachable'] = self.is_alive()
KPI['operation_status'] = self._get_operation_status()
message_volume = {'free': 0, 'claimed': 0, 'total': 0}
for msg_col in [db.messages for db in self.message_databases]:
msg_count_claimed = msg_col.find({'c.id': {'$ne': None}}).count()
message_volume['claimed'] += msg_count_claimed
msg_count_total = msg_col.find().count()
message_volume['total'] += msg_count_total
message_volume['free'] = (message_volume['total'] -
message_volume['claimed'])
KPI['message_volume'] = message_volume
return KPI
@decorators.lazy_property(write=False)
def queues_database(self):
"""Database dedicated to the "queues" collection.
The queues collection is separated out into its own database
to avoid writer lock contention with the messages collections.
"""
name = self.mongodb_conf.database + '_queues'
return self.connection[name]
@decorators.lazy_property(write=False)
def message_databases(self):
"""List of message databases, ordered by partition number."""
name = self.mongodb_conf.database
partitions = self.mongodb_conf.partitions
# NOTE(kgriffs): Partition names are zero-based, and
# the list is ordered by partition, which means that a
# caller can, e.g., get zaqar_mp0 by simply indexing
# the first element in the list of databases:
#
# self.driver.message_databases[0]
#
return [
self.connection[name + '_messages_p' + str(p)]
for p in range(partitions)
]
@decorators.lazy_property(write=False)
def subscriptions_database(self):
"""Database dedicated to the "subscription" collection."""
name = self.mongodb_conf.database + '_subscriptions'
return self.connection[name]
@decorators.lazy_property(write=False)
def connection(self):
"""MongoDB client connection instance."""
return _connection(self.mongodb_conf)
@decorators.lazy_property(write=False)
def queue_controller(self):
return controllers.QueueController(self)
@decorators.lazy_property(write=False)
def message_controller(self):
return controllers.MessageController(self)
@decorators.lazy_property(write=False)
def claim_controller(self):
return controllers.ClaimController(self)
@decorators.lazy_property(write=False)
def subscription_controller(self):
return controllers.SubscriptionController(self)