def lookup(self, queue, project=None):
"""Lookup a pool driver for the given queue and project.
:param queue: Name of the queue for which to find a pool
:param project: Project to which the queue belongs, or
None to specify the "global" or "generic" project.
:returns: A storage driver instance for the appropriate pool. If
the driver does not exist yet, it is created and cached. If the
queue is not mapped, returns None.
:rtype: Maybe DataDriver
"""
try:
pool_id = self._pool_id(queue, project)
except errors.QueueNotMapped as ex:
LOG.debug(ex)
if not self._catalog_conf.enable_virtual_pool:
return None
conf_section = ('drivers:message_store:%s' %
self._conf.drivers.message_store)
try:
# NOTE(flaper87): Try to load the driver to check
# whether it can be used as the default store for
# the default pool.
utils.load_storage_driver(self._conf, self._cache,
control_driver=self.control)
except cerrors.InvalidDriver:
# NOTE(kgriffs): Return `None`, rather than letting the
# exception bubble up, so that the higher layer doesn't
# have to duplicate the try..except..log code all over
# the place.
return None
if conf_section not in self._conf:
# NOTE(flaper87): If there's no config section for this storage
# skip the pool registration entirely since we won't know how
# to connect to it.
return None
# NOTE(flaper87): This assumes the storage driver type is the
# same as the management.
pool_conf = {'uri': self._conf[conf_section].uri,
'options': {}}
# NOTE(flaper87): This will be using the config
# storage configuration as the default one if no
# default storage has been registered in the pool
# store.
return self.get_driver(None, pool_conf)
return self.get_driver(pool_id)
def setUp(self):
super(PoolCatalogTest, self).setUp()
cache = oslo_cache.get_cache()
control = utils.load_storage_driver(self.conf, cache,
control_mode=True)
self.pools_ctrl = control.pools_controller
self.flavors_ctrl = control.flavors_controller
self.catalogue_ctrl = control.catalogue_controller
# NOTE(cpp-cabrera): populate catalogue
self.pool = str(uuid.uuid1())
self.pool2 = str(uuid.uuid1())
self.pool_group = 'pool-group'
self.queue = str(uuid.uuid1())
self.flavor = str(uuid.uuid1())
self.project = str(uuid.uuid1())
self.pools_ctrl.create(self.pool, 100, 'sqlite://:memory:')
self.pools_ctrl.create(self.pool2, 100,
'sqlite://:memory:',
group=self.pool_group)
self.catalogue_ctrl.insert(self.project, self.queue, self.pool)
self.catalog = pooling.Catalog(self.conf, cache, control)
self.flavors_ctrl.create(self.flavor, self.pool_group,
project=self.project)
def setUp(self):
super(PoolCatalogTest, self).setUp()
oslo_cache.register_config(self.conf)
cache = oslo_cache.get_cache(self.conf)
control = utils.load_storage_driver(self.conf, cache, control_mode=True)
self.pools_ctrl = control.pools_controller
self.flavors_ctrl = control.flavors_controller
self.catalogue_ctrl = control.catalogue_controller
# NOTE(cpp-cabrera): populate catalogue
self.pool = str(uuid.uuid1())
self.pool2 = str(uuid.uuid1())
self.pool_group = "pool-group"
self.queue = str(uuid.uuid1())
self.flavor = str(uuid.uuid1())
self.project = str(uuid.uuid1())
# FIXME(therve) This is horrible, we need to manage duplication in a
# nicer way
if "localhost" in self.mongodb_url:
other_url = self.mongodb_url.replace("localhost", "127.0.0.1")
elif "127.0.0.1" in self.mongodb_url:
other_url = self.mongodb_url.replace("127.0.0.1", "localhost")
else:
self.skipTest("Can't build a dummy mongo URL.")
self.pools_ctrl.create(self.pool, 100, self.mongodb_url)
self.pools_ctrl.create(self.pool2, 100, other_url, group=self.pool_group)
self.catalogue_ctrl.insert(self.project, self.queue, self.pool)
self.catalog = pooling.Catalog(self.conf, cache, control)
self.flavors_ctrl.create(self.flavor, self.pool_group, project=self.project)
def setUp(self):
super(PoolCatalogTest, self).setUp()
cache = oslo_cache.get_cache()
control = utils.load_storage_driver(self.conf, cache,
control_mode=True)
self.pools_ctrl = control.pools_controller
self.flavors_ctrl = control.flavors_controller
self.catalogue_ctrl = control.catalogue_controller
# NOTE(cpp-cabrera): populate catalogue
self.pool = str(uuid.uuid1())
self.pool2 = str(uuid.uuid1())
self.pool_group = 'pool-group'
self.queue = str(uuid.uuid1())
self.flavor = str(uuid.uuid1())
self.project = str(uuid.uuid1())
self.pools_ctrl.create(self.pool, 100, 'mongodb://localhost:27017')
self.pools_ctrl.create(self.pool2, 100,
'mongodb://127.0.0.1:27017',
group=self.pool_group)
self.catalogue_ctrl.insert(self.project, self.queue, self.pool)
self.catalog = pooling.Catalog(self.conf, cache, control)
self.flavors_ctrl.create(self.flavor, self.pool_group,
project=self.project)
def storage(self):
LOG.debug(u'Loading storage driver')
if self.conf.pooling:
LOG.debug(u'Storage pooling enabled')
storage_driver = pooling.DataDriver(self.conf, self.cache,
self.control)
else:
storage_driver = storage_utils.load_storage_driver(
self.conf, self.cache, control_driver=self.control)
LOG.debug(u'Loading storage pipeline')
return pipeline.DataDriver(self.conf, storage_driver, self.control)
def _init_driver(self, pool_id):
"""Given a pool name, returns a storage driver.
:param pool_id: The name of a pool.
:type pool_id: six.text_type
:returns: a storage driver
:rtype: zaqar.storage.base.DataDriverBase
"""
pool = self._pools_ctrl.get(pool_id, detailed=True)
conf = utils.dynamic_conf(pool['uri'], pool['options'],
conf=self._conf)
return utils.load_storage_driver(conf, self._cache)
def get_default_pool(self, use_listing=True):
if use_listing:
cursor = self._pools_ctrl.list(limit=0)
pools_list = list(next(cursor))
if pools_list:
return self.get_driver(pools_list[0]['name'])
if self._catalog_conf.enable_virtual_pool:
conf_section = ('drivers:message_store:%s' %
self._conf.drivers.message_store)
try:
# NOTE(flaper87): Try to load the driver to check
# whether it can be used as the default store for
# the default pool.
utils.load_storage_driver(self._conf, self._cache,
control_driver=self.control)
except cerrors.InvalidDriver:
# NOTE(kgriffs): Return `None`, rather than letting the
# exception bubble up, so that the higher layer doesn't
# have to duplicate the try..except..log code all over
# the place.
return None
if conf_section not in self._conf:
# NOTE(flaper87): If there's no config section for this storage
# skip the pool registration entirely since we won't know how
# to connect to it.
return None
# NOTE(flaper87): This assumes the storage driver type is the
# same as the management.
pool_conf = {'uri': self._conf[conf_section].uri,
'options': {}}
# NOTE(flaper87): This will be using the config
# storage configuration as the default one if no
# default storage has been registered in the pool
# store.
return self.get_driver(None, pool_conf)
def get_default_pool(self, use_listing=True):
if use_listing:
cursor = self._pools_ctrl.list(limit=0)
pools_list = list(next(cursor))
if pools_list:
return self.get_driver(pools_list[0]['name'])
if self._catalog_conf.enable_virtual_pool:
conf_section = ('drivers:message_store:%s' %
self._conf.drivers.message_store)
try:
# NOTE(flaper87): Try to load the driver to check
# whether it can be used as the default store for
# the default pool.
utils.load_storage_driver(self._conf, self._cache,
control_driver=self.control)
except cerrors.InvalidDriver:
# NOTE(kgriffs): Return `None`, rather than letting the
# exception bubble up, so that the higher layer doesn't
# have to duplicate the try..except..log code all over
# the place.
return None
if conf_section not in self._conf:
# NOTE(flaper87): If there's no config section for this storage
# skip the pool registration entirely since we won't know how
# to connect to it.
return None
# NOTE(flaper87): This assumes the storage driver type is the
# same as the management.
pool_conf = {'uri': self._conf[conf_section].uri,
'options': {}}
# NOTE(flaper87): This will be using the config
# storage configuration as the default one if no
# default storage has been registered in the pool
# store.
return self.get_driver(None, pool_conf)
def _init_driver(self, pool_id):
"""Given a pool name, returns a storage driver.
:param pool_id: The name of a pool.
:type pool_id: six.text_type
:returns: a storage driver
:rtype: zaqar.storage.base.DataDriverBase
"""
pool = self._pools_ctrl.get(pool_id, detailed=True)
conf = utils.dynamic_conf(pool['uri'],
pool['options'],
conf=self._conf)
return utils.load_storage_driver(conf, self._cache)
def storage(self):
LOG.debug(u'Loading storage driver')
if self.conf.pooling:
LOG.debug(u'Storage pooling enabled')
storage_driver = pooling.DataDriver(self.conf, self.cache,
self.control)
else:
storage_driver = storage_utils.load_storage_driver(
self.conf, self.cache, control_driver=self.control)
LOG.debug(u'Loading storage pipeline')
return pipeline.DataDriver(self.conf, storage_driver,
self.control)
def setUp(self):
super(PoolQueuesTest, self).setUp()
cache = oslo_cache.get_cache()
control = utils.load_storage_driver(self.conf,
cache,
control_mode=True)
self.pools_ctrl = control.pools_controller
self.driver = pooling.DataDriver(self.conf, cache, control)
self.controller = self.driver.queue_controller
# fake two pools
for _ in six.moves.xrange(2):
self.pools_ctrl.create(str(uuid.uuid1()), 100, 'sqlite://:memory:')
def _init_driver(self, pool_id, pool_conf=None):
"""Given a pool name, returns a storage driver.
:param pool_id: The name of a pool.
:type pool_id: six.text_type
:returns: a storage driver
:rtype: zaqar.storage.base.DataDriverBase
"""
if pool_id is not None:
pool = self._pools_ctrl.get(pool_id, detailed=True)
else:
pool = pool_conf
conf = utils.dynamic_conf(pool["uri"], pool["options"], conf=self._conf)
storage = utils.load_storage_driver(conf, self._cache, control_driver=self.control)
return pipeline.DataDriver(conf, storage, self.control)
def storage(self):
LOG.debug(u'Loading storage driver')
if self.conf.pooling:
LOG.debug(u'Storage pooling enabled')
storage_driver = pooling.DataDriver(self.conf, self.cache,
self.control)
if self.conf.profiler.enabled:
storage_driver = profiler.trace_cls("pooling_data_"
"driver")(storage_driver)
else:
storage_driver = storage_utils.load_storage_driver(
self.conf, self.cache, control_driver=self.control)
LOG.debug(u'Loading storage pipeline')
return pipeline.DataDriver(self.conf, storage_driver,
self.control)
def storage(self):
LOG.debug(u'Loading storage driver')
if self.conf.pooling:
LOG.debug(u'Storage pooling enabled')
storage_driver = pooling.DataDriver(self.conf, self.cache,
self.control)
if self.conf.profiler.enabled:
storage_driver = profiler.trace_cls("pooling_data_"
"driver")(storage_driver)
else:
storage_driver = storage_utils.load_storage_driver(
self.conf, self.cache, control_driver=self.control)
LOG.debug(u'Loading storage pipeline')
return pipeline.DataDriver(self.conf, storage_driver,
self.control)
def setUp(self):
super(PoolQueuesTest, self).setUp()
self.conf.register_opts([cfg.StrOpt('storage')],
group='drivers')
cache = oslo_cache.get_cache()
control = utils.load_storage_driver(self.conf, cache,
control_mode=True)
self.pools_ctrl = control.pools_controller
self.driver = pooling.DataDriver(self.conf, cache, control)
self.controller = self.driver.queue_controller
# fake two pools
for _ in six.moves.xrange(2):
self.pools_ctrl.create(str(uuid.uuid1()), 100,
'sqlite://:memory:')
def _init_driver(self, pool_id, pool_conf=None):
"""Given a pool name, returns a storage driver.
:param pool_id: The name of a pool.
:type pool_id: six.text_type
:returns: a storage driver
:rtype: zaqar.storage.base.DataDriverBase
"""
if pool_id is not None:
pool = self._pools_ctrl.get(pool_id, detailed=True)
else:
pool = pool_conf
conf = utils.dynamic_conf(pool['uri'], pool['options'],
conf=self._conf)
storage = utils.load_storage_driver(conf,
self._cache,
control_driver=self.control)
return pipeline.DataDriver(conf, storage, self.control)
def setUp(self):
super(PoolCatalogTest, self).setUp()
oslo_cache.register_config(self.conf)
cache = oslo_cache.get_cache(self.conf)
control = utils.load_storage_driver(self.conf,
cache,
control_mode=True)
self.pools_ctrl = control.pools_controller
self.flavors_ctrl = control.flavors_controller
self.catalogue_ctrl = control.catalogue_controller
# NOTE(cpp-cabrera): populate catalogue
self.pool = str(uuid.uuid1())
self.pool2 = str(uuid.uuid1())
self.pool_group = 'pool-group'
self.queue = str(uuid.uuid1())
self.flavor = str(uuid.uuid1())
self.project = str(uuid.uuid1())
# FIXME(therve) This is horrible, we need to manage duplication in a
# nicer way
if 'localhost' in self.mongodb_url:
other_url = self.mongodb_url.replace('localhost', '127.0.0.1')
elif '127.0.0.1' in self.mongodb_url:
other_url = self.mongodb_url.replace('127.0.0.1', 'localhost')
else:
self.skipTest("Can't build a dummy mongo URL.")
self.pools_ctrl.create(self.pool, 100, self.mongodb_url)
self.pools_ctrl.create(self.pool2,
100,
other_url,
group=self.pool_group)
self.catalogue_ctrl.insert(self.project, self.queue, self.pool)
self.catalog = pooling.Catalog(self.conf, cache, control)
self.flavors_ctrl.create(self.flavor,
self.pool_group,
project=self.project)
def control(self):
LOG.debug(u'Loading storage control driver')
return storage_utils.load_storage_driver(self.conf, self.cache,
control_mode=True)
def control(self):
LOG.debug(u'Loading storage control driver')
return storage_utils.load_storage_driver(self.conf, self.cache,
control_mode=True)