def rpc_client(self):
if self._client is None:
if not rpc.initialized():
rpc.init(CONF)
target = messaging.Target(topic=CONF.volume_topic)
self._client = rpc.get_client(target)
return self._client
def __init__(self, host, binary, topic, manager, report_interval=None,
periodic_interval=None, periodic_fuzzy_delay=None,
service_name=None, *args, **kwargs):
super(Service, self).__init__()
if not rpc.initialized():
rpc.init(CONF)
self.host = host
self.binary = binary
self.topic = topic
self.manager_class_name = manager
manager_class = importutils.import_class(self.manager_class_name)
if CONF.profiler.enabled:
manager_class = profiler.trace_cls("rpc")(manager_class)
self.manager = manager_class(host=self.host,
service_name=service_name,
*args, **kwargs)
self.report_interval = report_interval
self.periodic_interval = periodic_interval
self.periodic_fuzzy_delay = periodic_fuzzy_delay
self.basic_config_check()
self.saved_args, self.saved_kwargs = args, kwargs
self.timers = []
setup_profiler(binary, host)
self.rpcserver = None
def test_init_no_notifications(self, driver, serializer_mock):
"""Test short-circuiting notifications with default and noop driver."""
self.override_config('driver', driver,
group='oslo_messaging_notifications')
rpc.init(test.CONF)
self.assertEqual(rpc.utils.DO_NOTHING, rpc.NOTIFIER)
serializer_mock.assert_not_called()
def main():
objects.register_all()
gmr_opts.set_defaults(CONF)
CONF(sys.argv[1:], project="cinder", version=version.version_string())
config.set_middleware_defaults()
logging.setup(CONF, "cinder")
LOG = logging.getLogger("cinder.all")
versionutils.report_deprecated_feature(LOG, _("cinder-all is deprecated in Newton and will be removed in Ocata."))
utils.monkey_patch()
gmr.TextGuruMeditation.setup_autorun(version, conf=CONF)
rpc.init(CONF)
launcher = service.process_launcher()
# cinder-api
try:
server = service.WSGIService("osapi_volume")
launcher.launch_service(server, workers=server.workers or 1)
except (Exception, SystemExit):
LOG.exception(_LE("Failed to load osapi_volume"))
for binary in ["cinder-scheduler", "cinder-backup"]:
try:
launcher.launch_service(service.Service.create(binary=binary))
except (Exception, SystemExit):
LOG.exception(_LE("Failed to load %s"), binary)
# cinder-volume
try:
if CONF.enabled_backends:
for backend in CONF.enabled_backends:
CONF.register_opt(volume_cmd.host_opt, group=backend)
backend_host = getattr(CONF, backend).backend_host
host = "%s@%s" % (backend_host or CONF.host, backend)
server = service.Service.create(
host=host, service_name=backend, binary="cinder-volume", coordination=True
)
# Dispose of the whole DB connection pool here before
# starting another process. Otherwise we run into cases
# where child processes share DB connections which results
# in errors.
session.dispose_engine()
launcher.launch_service(server)
else:
LOG.warning(
_LW(
"Configuration for cinder-volume does not specify "
'"enabled_backends", using DEFAULT as backend. '
"Support for DEFAULT section to configure drivers "
"will be removed in the next release."
)
)
server = service.Service.create(binary="cinder-volume", coordination=True)
launcher.launch_service(server)
except (Exception, SystemExit):
LOG.exception(_LE("Failed to load cinder-volume"))
launcher.wait()
def initialize_application():
objects.register_all()
CONF(sys.argv[1:], project='cinder',
version=version.version_string())
logging.setup(CONF, "cinder")
rpc.init(CONF)
return wsgi.Loader(CONF).load_app(name='osapi_volume')
def initialize_application():
objects.register_all()
CONF(sys.argv[1:], project="cinder", version=version.version_string())
logging.setup(CONF, "cinder")
config.set_middleware_defaults()
rpc.init(CONF)
return wsgi.Loader(CONF).load_app(name="osapi_volume")
def _rpc_client(self):
if self._client is None:
if not rpc.initialized():
rpc.init(CONF)
target = messaging.Target(topic=constants.VOLUME_TOPIC)
serializer = objects.base.CinderObjectSerializer()
self._client = rpc.get_client(target, serializer=serializer)
return self._client
def rpc_client(self):
if self._client is None:
if not rpc.initialized():
rpc.init(CONF)
target = messaging.Target(topic=CONF.volume_topic)
serializer = objects.base.CinderObjectSerializer()
self._client = rpc.get_client(target, serializer=serializer)
return self._client
def main():
objects.register_all()
gmr_opts.set_defaults(CONF)
CONF(sys.argv[1:], project='cinder',
version=version.version_string())
config.set_middleware_defaults()
logging.setup(CONF, "cinder")
LOG = logging.getLogger('cinder.all')
versionutils.report_deprecated_feature(LOG, _(
'cinder-all is deprecated in Newton and will be removed in Ocata.'))
utils.monkey_patch()
gmr.TextGuruMeditation.setup_autorun(version, conf=CONF)
rpc.init(CONF)
launcher = service.process_launcher()
# cinder-api
try:
server = service.WSGIService('osapi_volume')
launcher.launch_service(server, workers=server.workers or 1)
except (Exception, SystemExit):
LOG.exception(_LE('Failed to load osapi_volume'))
for binary in ['cinder-scheduler', 'cinder-backup']:
try:
launcher.launch_service(service.Service.create(binary=binary))
except (Exception, SystemExit):
LOG.exception(_LE('Failed to load %s'), binary)
# cinder-volume
try:
if CONF.enabled_backends:
for backend in CONF.enabled_backends:
CONF.register_opt(volume_cmd.host_opt, group=backend)
backend_host = getattr(CONF, backend).backend_host
host = "%s@%s" % (backend_host or CONF.host, backend)
server = service.Service.create(host=host,
service_name=backend,
binary='cinder-volume')
# Dispose of the whole DB connection pool here before
# starting another process. Otherwise we run into cases
# where child processes share DB connections which results
# in errors.
session.dispose_engine()
launcher.launch_service(server)
else:
server = service.Service.create(binary='cinder-volume')
launcher.launch_service(server)
except (Exception, SystemExit):
LOG.exception(_LE('Failed to load conder-volume'))
launcher.wait()
def main():
CONF(sys.argv[1:], project='cinder',
version=version.version_string())
logging.setup("cinder")
utils.monkey_patch()
rpc.init(CONF)
launcher = service.process_launcher()
server = service.WSGIService('osapi_volume')
launcher.launch_service(server, workers=server.workers)
launcher.wait()
def initialize_application():
objects.register_all()
CONF(sys.argv[1:], project='cinder',
version=version.version_string())
logging.setup(CONF, "cinder")
config.set_middleware_defaults()
rpc.init(CONF)
service.setup_profiler(constants.API_BINARY, CONF.host)
return wsgi.Loader(CONF).load_app(name='osapi_volume')
def main():
objects.register_all()
admin_context = context.get_admin_context()
CONF(sys.argv[1:], project='cinder',
version=version.version_string())
logging.setup(CONF, "cinder")
LOG = logging.getLogger("cinder")
rpc.init(CONF)
begin, end = utils.last_completed_audit_period()
begin, end = _time_error(LOG, begin, end)
LOG.info("Starting volume usage audit")
LOG.info("Creating usages for %(begin_period)s until %(end_period)s",
{"begin_period": begin, "end_period": end})
extra_info = {
'audit_period_beginning': str(begin),
'audit_period_ending': str(end),
}
volumes = objects.VolumeList.get_all_active_by_window(admin_context,
begin,
end)
LOG.info("Found %d volumes", len(volumes))
for volume_ref in volumes:
_obj_ref_action(_vol_notify_usage, LOG, volume_ref, extra_info,
admin_context, begin, end,
cinder.volume.utils.notify_about_volume_usage,
"volume_id", "volume")
snapshots = objects.SnapshotList.get_all_active_by_window(admin_context,
begin, end)
LOG.info("Found %d snapshots", len(snapshots))
for snapshot_ref in snapshots:
_obj_ref_action(_snap_notify_usage, LOG, snapshot_ref, extra_info,
admin_context, begin,
end, cinder.volume.utils.notify_about_snapshot_usage,
"snapshot_id", "snapshot")
backups = objects.BackupList.get_all_active_by_window(admin_context,
begin, end)
LOG.info("Found %d backups", len(backups))
for backup_ref in backups:
_obj_ref_action(_backup_notify_usage, LOG, backup_ref, extra_info,
admin_context, begin,
end, cinder.volume.utils.notify_about_backup_usage,
"backup_id", "backup")
LOG.info("Volume usage audit completed")
def main():
objects.register_all()
CONF(sys.argv[1:], project='cinder',
version=version.version_string())
logging.setup(CONF, "cinder")
utils.monkey_patch()
gmr.TextGuruMeditation.setup_autorun(version)
rpc.init(CONF)
launcher = service.process_launcher()
server = service.WSGIService('osapi_volume')
launcher.launch_service(server, workers=server.workers)
launcher.wait()
def main():
objects.register_all()
gmr_opts.set_defaults(CONF)
CONF(sys.argv[1:], project='cinder', version=version.version_string())
config.set_middleware_defaults()
logging.setup(CONF, "cinder")
python_logging.captureWarnings(True)
utils.monkey_patch()
gmr.TextGuruMeditation.setup_autorun(version, conf=CONF)
rpc.init(CONF)
launcher = service.process_launcher()
server = service.WSGIService('osapi_volume')
launcher.launch_service(server, workers=server.workers)
launcher.wait()
def main():
objects.register_all()
gmr_opts.set_defaults(CONF)
CONF(sys.argv[1:], project="cinder", version=version.version_string())
config.set_middleware_defaults()
logging.setup(CONF, "cinder")
python_logging.captureWarnings(True)
utils.monkey_patch()
gmr.TextGuruMeditation.setup_autorun(version, conf=CONF)
rpc.init(CONF)
launcher = service.process_launcher()
server = service.WSGIService("osapi_volume")
launcher.launch_service(server, workers=server.workers)
launcher.wait()
def _get_non_shared_target_hosts(ctxt):
hosts = []
numvols_needing_update = 0
rpc.init(CONF)
rpcapi = volume_rpcapi.VolumeAPI()
services = objects.ServiceList.get_all_by_topic(ctxt, 'cinder-volume')
for service in services:
capabilities = rpcapi.get_capabilities(ctxt, service.host, True)
if not capabilities.get('shared_targets', True):
hosts.append(service.host)
numvols_needing_update += db_api.model_query(
ctxt,
models.Volume).filter_by(shared_targets=True,
service_uuid=service.uuid).count()
return hosts, numvols_needing_update
def _get_non_shared_target_hosts(ctxt):
hosts = []
numvols_needing_update = 0
rpc.init(CONF)
rpcapi = volume_rpcapi.VolumeAPI()
services = objects.ServiceList.get_all_by_topic(ctxt,
constants.VOLUME_TOPIC)
for service in services:
capabilities = rpcapi.get_capabilities(ctxt, service.host, True)
if not capabilities.get('shared_targets', True):
hosts.append(service.host)
numvols_needing_update += db_api.model_query(
ctxt, models.Volume).filter_by(
shared_targets=True,
service_uuid=service.uuid).count()
return hosts, numvols_needing_update
def __init__(self, host, binary, topic, manager, report_interval=None,
periodic_interval=None, periodic_fuzzy_delay=None,
service_name=None, coordination=False, *args, **kwargs):
super(Service, self).__init__()
if not rpc.initialized():
rpc.init(CONF)
self.host = host
self.binary = binary
self.topic = topic
self.manager_class_name = manager
self.coordination = coordination
manager_class = importutils.import_class(self.manager_class_name)
if CONF.profiler.enabled:
manager_class = profiler.trace_cls("rpc")(manager_class)
# NOTE(geguileo): We need to create the Service DB entry before we
# create the manager, otherwise capped versions for serializer and rpc
# client would used existing DB entries not including us, which could
# result in us using None (if it's the first time the service is run)
# or an old version (if this is a normal upgrade of a single service).
ctxt = context.get_admin_context()
try:
service_ref = objects.Service.get_by_args(ctxt, host, binary)
service_ref.rpc_current_version = manager_class.RPC_API_VERSION
obj_version = objects_base.OBJ_VERSIONS.get_current()
service_ref.object_current_version = obj_version
service_ref.save()
self.service_id = service_ref.id
except exception.NotFound:
self._create_service_ref(ctxt, manager_class.RPC_API_VERSION)
self.manager = manager_class(host=self.host,
service_name=service_name,
*args, **kwargs)
self.report_interval = report_interval
self.periodic_interval = periodic_interval
self.periodic_fuzzy_delay = periodic_fuzzy_delay
self.basic_config_check()
self.saved_args, self.saved_kwargs = args, kwargs
self.timers = []
setup_profiler(binary, host)
self.rpcserver = None
def _get_non_shared_target_hosts(ctxt):
hosts = []
numvols_needing_update = 0
rpc.init(CONF)
rpcapi = volume_rpcapi.VolumeAPI()
services = objects.ServiceList.get_all_by_topic(ctxt,
constants.VOLUME_TOPIC)
for service in services:
capabilities = rpcapi.get_capabilities(ctxt, service.host, True)
# Select only non iSCSI connections and iSCSI that are explicit
if (capabilities.get('storage_protocol') != 'iSCSI'
or not capabilities.get('shared_targets', True)):
hosts.append(service.host)
numvols_needing_update += db_api.model_query(
ctxt,
models.Volume).filter_by(shared_targets=True,
service_uuid=service.uuid).count()
return hosts, numvols_needing_update
def _get_non_shared_target_hosts(ctxt):
hosts = []
numvols_needing_update = 0
rpc.init(CONF)
rpcapi = volume_rpcapi.VolumeAPI()
services = objects.ServiceList.get_all_by_topic(ctxt,
constants.VOLUME_TOPIC)
for service in services:
capabilities = rpcapi.get_capabilities(ctxt, service.host, True)
# Select only non iSCSI connections and iSCSI that are explicit
if (capabilities.get('storage_protocol') != 'iSCSI' or
not capabilities.get('shared_targets', True)):
hosts.append(service.host)
numvols_needing_update += db_api.model_query(
ctxt, models.Volume).filter_by(
shared_targets=True,
service_uuid=service.uuid).count()
return hosts, numvols_needing_update
def delete(self, volume_id):
"""Delete a volume, bypassing the check that it must be available."""
ctxt = context.get_admin_context()
volume = objects.Volume.get_by_id(ctxt, volume_id)
host = vutils.extract_host(volume.host) if volume.host else None
if not host:
print(_("Volume not yet assigned to host."))
print(_("Deleting volume from database and skipping rpc."))
volume.destroy()
return
if volume.status == 'in-use':
print(_("Volume is in-use."))
print(_("Detach volume from instance and then try again."))
return
rpc.init(CONF)
rpcapi = volume_rpcapi.VolumeAPI()
rpcapi.delete_volume(ctxt, volume)
def delete(self, volume_id):
"""Delete a volume, bypassing the check that it must be available."""
ctxt = context.get_admin_context()
volume = objects.Volume.get_by_id(ctxt, volume_id)
host = vutils.extract_host(volume.host) if volume.host else None
if not host:
print(_("Volume not yet assigned to host."))
print(_("Deleting volume from database and skipping rpc."))
volume.destroy()
return
if volume.status == 'in-use':
print(_("Volume is in-use."))
print(_("Detach volume from instance and then try again."))
return
rpc.init(CONF)
rpcapi = volume_rpcapi.VolumeAPI()
rpcapi.delete_volume(ctxt, volume)
def init_app():
"""initialize application"""
cfg.CONF(sys.argv[1:], project='cinder', version=version.version_string())
# choose the first in backend list
backend = cfg.CONF.enabled_backends[0]
cfg.CONF.register_opt(host_opt, backend)
# get backend host
backend_host = getattr(cfg.CONF, backend).backend_host
if not backend_host:
backend_host = socket.gethostname()
# form a host
host = "%s@%s" % (backend_host, backend)
rpc.init(cfg.CONF)
return (backend, host, None)
def __init__(self,
host,
binary,
topic,
manager,
report_interval=None,
periodic_interval=None,
periodic_fuzzy_delay=None,
service_name=None,
*args,
**kwargs):
super(Service, self).__init__()
if not rpc.initialized():
rpc.init(CONF)
self.host = host
self.binary = binary
self.topic = topic
self.manager_class_name = manager
manager_class = importutils.import_class(self.manager_class_name)
#modify by yanghc mantis:0000300 begin
#manager_class = profiler.trace_cls("rpc")(manager_class)
if CONF.profiler.profiler_enabled:
manager_class = profiler.trace_cls("rpc")(manager_class)
#modify by yanghc mantis:0000300 begin end
self.manager = manager_class(host=self.host,
service_name=service_name,
*args,
**kwargs)
self.report_interval = report_interval
self.periodic_interval = periodic_interval
self.periodic_fuzzy_delay = periodic_fuzzy_delay
self.basic_config_check()
self.saved_args, self.saved_kwargs = args, kwargs
self.timers = []
setup_profiler(binary, host)
def setUp(self):
"""Run before each test method to initialize test environment."""
super(TestCase, self).setUp()
# Create default notifier
self.notifier = fake_notifier.get_fake_notifier()
# Mock rpc get notifier with fake notifier method that joins all
# notifications with the default notifier
p = mock.patch("cinder.rpc.get_notifier", side_effect=self._get_joined_notifier)
p.start()
# Unit tests do not need to use lazy gettext
i18n.enable_lazy(False)
test_timeout = os.environ.get("OS_TEST_TIMEOUT", 0)
try:
test_timeout = int(test_timeout)
except ValueError:
# If timeout value is invalid do not set a timeout.
test_timeout = 0
if test_timeout > 0:
self.useFixture(fixtures.Timeout(test_timeout, gentle=True))
self.useFixture(fixtures.NestedTempfile())
self.useFixture(fixtures.TempHomeDir())
environ_enabled = lambda var_name: strutils.bool_from_string(os.environ.get(var_name))
if environ_enabled("OS_STDOUT_CAPTURE"):
stdout = self.useFixture(fixtures.StringStream("stdout")).stream
self.useFixture(fixtures.MonkeyPatch("sys.stdout", stdout))
if environ_enabled("OS_STDERR_CAPTURE"):
stderr = self.useFixture(fixtures.StringStream("stderr")).stream
self.useFixture(fixtures.MonkeyPatch("sys.stderr", stderr))
if environ_enabled("OS_LOG_CAPTURE"):
log_format = "%(levelname)s [%(name)s] %(message)s"
if environ_enabled("OS_DEBUG"):
level = logging.DEBUG
else:
level = logging.INFO
self.useFixture(fixtures.LoggerFixture(nuke_handlers=False, format=log_format, level=level))
rpc.add_extra_exmods("cinder.tests.unit")
self.addCleanup(rpc.clear_extra_exmods)
self.addCleanup(rpc.cleanup)
self.messaging_conf = messaging_conffixture.ConfFixture(CONF)
self.messaging_conf.transport_driver = "fake"
self.messaging_conf.response_timeout = 15
self.useFixture(self.messaging_conf)
rpc.init(CONF)
conf_fixture.set_defaults(CONF)
CONF([], default_config_files=[])
# NOTE(vish): We need a better method for creating fixtures for tests
# now that we have some required db setup for the system
# to work properly.
self.start = timeutils.utcnow()
CONF.set_default("connection", "sqlite://", "database")
CONF.set_default("sqlite_synchronous", False, "database")
global _DB_CACHE
if not _DB_CACHE:
_DB_CACHE = Database(
sqla_api,
migration,
sql_connection=CONF.database.connection,
sqlite_db=CONF.database.sqlite_db,
sqlite_clean_db=CONF.sqlite_clean_db,
)
self.useFixture(_DB_CACHE)
# NOTE(danms): Make sure to reset us back to non-remote objects
# for each test to avoid interactions. Also, backup the object
# registry.
objects_base.CinderObject.indirection_api = None
self._base_test_obj_backup = copy.copy(objects_base.CinderObjectRegistry._registry._obj_classes)
self.addCleanup(self._restore_obj_registry)
# emulate some of the mox stuff, we can't use the metaclass
# because it screws with our generators
mox_fixture = self.useFixture(moxstubout.MoxStubout())
self.mox = mox_fixture.mox
self.stubs = mox_fixture.stubs
self.addCleanup(CONF.reset)
self.addCleanup(self._common_cleanup)
self.injected = []
self._services = []
fake_notifier.stub_notifier(self.stubs)
self.override_config("fatal_exception_format_errors", True)
# This will be cleaned up by the NestedTempfile fixture
lock_path = self.useFixture(fixtures.TempDir()).path
self.fixture = self.useFixture(config_fixture.Config(lockutils.CONF))
self.fixture.config(lock_path=lock_path, group="oslo_concurrency")
lockutils.set_defaults(lock_path)
self.override_config(
"policy_file",
os.path.join(
os.path.abspath(os.path.join(os.path.dirname(__file__), "..")), "cinder/tests/unit/policy.json"
),
group="oslo_policy",
)
self._disable_osprofiler()
def setUp(self):
"""Run before each test method to initialize test environment."""
super(TestCase, self).setUp()
# Create default notifier
self.notifier = fake_notifier.get_fake_notifier()
# Mock rpc get notifier with fake notifier method that joins all
# notifications with the default notifier
p = mock.patch('cinder.rpc.get_notifier',
side_effect=self._get_joined_notifier)
p.start()
# Unit tests do not need to use lazy gettext
i18n.enable_lazy(False)
test_timeout = os.environ.get('OS_TEST_TIMEOUT', 0)
try:
test_timeout = int(test_timeout)
except ValueError:
# If timeout value is invalid do not set a timeout.
test_timeout = 0
if test_timeout > 0:
self.useFixture(fixtures.Timeout(test_timeout, gentle=True))
self.useFixture(fixtures.NestedTempfile())
self.useFixture(fixtures.TempHomeDir())
environ_enabled = (lambda var_name:
strutils.bool_from_string(os.environ.get(var_name)))
if environ_enabled('OS_STDOUT_CAPTURE'):
stdout = self.useFixture(fixtures.StringStream('stdout')).stream
self.useFixture(fixtures.MonkeyPatch('sys.stdout', stdout))
if environ_enabled('OS_STDERR_CAPTURE'):
stderr = self.useFixture(fixtures.StringStream('stderr')).stream
self.useFixture(fixtures.MonkeyPatch('sys.stderr', stderr))
self.useFixture(log_fixture.get_logging_handle_error_fixture())
self.useFixture(cinder_fixtures.StandardLogging())
rpc.add_extra_exmods("cinder.tests.unit")
self.addCleanup(rpc.clear_extra_exmods)
self.addCleanup(rpc.cleanup)
self.messaging_conf = messaging_conffixture.ConfFixture(CONF)
self.messaging_conf.transport_driver = 'fake'
self.messaging_conf.response_timeout = 15
self.useFixture(self.messaging_conf)
rpc.init(CONF)
conf_fixture.set_defaults(CONF)
CONF([], default_config_files=[])
# NOTE(vish): We need a better method for creating fixtures for tests
# now that we have some required db setup for the system
# to work properly.
self.start = timeutils.utcnow()
CONF.set_default('connection', 'sqlite://', 'database')
CONF.set_default('sqlite_synchronous', False, 'database')
global _DB_CACHE
if not _DB_CACHE:
_DB_CACHE = Database(sqla_api, migration,
sql_connection=CONF.database.connection,
sqlite_db=CONF.database.sqlite_db,
sqlite_clean_db=CONF.sqlite_clean_db)
self.useFixture(_DB_CACHE)
# NOTE(danms): Make sure to reset us back to non-remote objects
# for each test to avoid interactions. Also, backup the object
# registry.
objects_base.CinderObject.indirection_api = None
self._base_test_obj_backup = copy.copy(
objects_base.CinderObjectRegistry._registry._obj_classes)
self.addCleanup(self._restore_obj_registry)
# emulate some of the mox stuff, we can't use the metaclass
# because it screws with our generators
mox_fixture = self.useFixture(moxstubout.MoxStubout())
self.mox = mox_fixture.mox
self.stubs = mox_fixture.stubs
self.addCleanup(CONF.reset)
self.addCleanup(self._common_cleanup)
self.injected = []
self._services = []
fake_notifier.stub_notifier(self.stubs)
self.override_config('fatal_exception_format_errors', True)
# This will be cleaned up by the NestedTempfile fixture
lock_path = self.useFixture(fixtures.TempDir()).path
self.fixture = self.useFixture(
config_fixture.Config(lockutils.CONF))
self.fixture.config(lock_path=lock_path,
group='oslo_concurrency')
lockutils.set_defaults(lock_path)
self.override_config('policy_file',
os.path.join(
os.path.abspath(
os.path.join(
os.path.dirname(__file__),
'..',
)
),
'cinder/tests/unit/policy.json'),
group='oslo_policy')
self._disable_osprofiler()
# NOTE(geguileo): This is required because common get_by_id method in
# cinder.db.sqlalchemy.api caches get methods and if we use a mocked
# get method in one test it would carry on to the next test. So we
# clear out the cache.
sqla_api._GET_METHODS = {}
def __init__(self,
host,
binary,
topic,
manager,
report_interval=None,
periodic_interval=None,
periodic_fuzzy_delay=None,
service_name=None,
coordination=False,
cluster=None,
*args,
**kwargs):
super(Service, self).__init__()
if not rpc.initialized():
rpc.init(CONF)
self.cluster = cluster
self.host = host
self.binary = binary
self.topic = topic
self.manager_class_name = manager
self.coordination = coordination
manager_class = importutils.import_class(self.manager_class_name)
if CONF.profiler.enabled:
manager_class = profiler.trace_cls("rpc")(manager_class)
self.service = None
self.manager = manager_class(host=self.host,
cluster=self.cluster,
service_name=service_name,
*args,
**kwargs)
self.availability_zone = self.manager.availability_zone
# NOTE(geguileo): We need to create the Service DB entry before we
# create the manager, otherwise capped versions for serializer and rpc
# client would use existing DB entries not including us, which could
# result in us using None (if it's the first time the service is run)
# or an old version (if this is a normal upgrade of a single service).
ctxt = context.get_admin_context()
try:
service_ref = objects.Service.get_by_args(ctxt, host, binary)
service_ref.rpc_current_version = manager_class.RPC_API_VERSION
obj_version = objects_base.OBJ_VERSIONS.get_current()
service_ref.object_current_version = obj_version
# added_to_cluster attribute marks when we consider that we have
# just added a host to a cluster so we can include resources into
# that cluster. We consider that we have added the host when we
# didn't have data in the cluster DB field and our current
# configuration has a cluster value. We don't want to do anything
# automatic if the cluster is changed, in those cases we'll want
# to use cinder manage command and to it manually.
self.added_to_cluster = (not service_ref.cluster_name and cluster)
if service_ref.cluster_name != cluster:
LOG.info(
'This service has been moved from cluster '
'%(cluster_svc)s to %(cluster_cfg)s. Resources '
'will %(opt_no)sbe moved to the new cluster', {
'cluster_svc': service_ref.cluster_name,
'cluster_cfg': cluster,
'opt_no': '' if self.added_to_cluster else 'NO '
})
if self.added_to_cluster:
# We pass copy service's disable status in the cluster if we
# have to create it.
self._ensure_cluster_exists(ctxt, service_ref)
service_ref.cluster_name = cluster
service_ref.save()
Service.service_id = service_ref.id
self.origin_service_id = service_ref.id
except exception.NotFound:
self._create_service_ref(ctxt, manager_class.RPC_API_VERSION)
# Service entry Entry didn't exist because it was manually removed
# or it's the first time running, to be on the safe side we say we
# were added if we are clustered.
self.added_to_cluster = bool(cluster)
self.report_interval = report_interval
self.periodic_interval = periodic_interval
self.periodic_fuzzy_delay = periodic_fuzzy_delay
self.basic_config_check()
self.saved_args, self.saved_kwargs = args, kwargs
setup_profiler(binary, host)
self.rpcserver = None
self.backend_rpcserver = None
self.cluster_rpcserver = None
def setUp(self):
"""Run before each test method to initialize test environment."""
super(TestCase, self).setUp()
# Import cinder objects for test cases
objects.register_all()
# Unit tests do not need to use lazy gettext
i18n.enable_lazy(False)
test_timeout = os.environ.get('OS_TEST_TIMEOUT', 0)
try:
test_timeout = int(test_timeout)
except ValueError:
# If timeout value is invalid do not set a timeout.
test_timeout = 0
if test_timeout > 0:
self.useFixture(fixtures.Timeout(test_timeout, gentle=True))
self.useFixture(fixtures.NestedTempfile())
self.useFixture(fixtures.TempHomeDir())
environ_enabled = (lambda var_name:
strutils.bool_from_string(os.environ.get(var_name)))
if environ_enabled('OS_STDOUT_CAPTURE'):
stdout = self.useFixture(fixtures.StringStream('stdout')).stream
self.useFixture(fixtures.MonkeyPatch('sys.stdout', stdout))
if environ_enabled('OS_STDERR_CAPTURE'):
stderr = self.useFixture(fixtures.StringStream('stderr')).stream
self.useFixture(fixtures.MonkeyPatch('sys.stderr', stderr))
if environ_enabled('OS_LOG_CAPTURE'):
log_format = '%(levelname)s [%(name)s] %(message)s'
if environ_enabled('OS_DEBUG'):
level = logging.DEBUG
else:
level = logging.INFO
self.useFixture(fixtures.LoggerFixture(nuke_handlers=False,
format=log_format,
level=level))
rpc.add_extra_exmods("cinder.tests.unit")
self.addCleanup(rpc.clear_extra_exmods)
self.addCleanup(rpc.cleanup)
self.messaging_conf = messaging_conffixture.ConfFixture(CONF)
self.messaging_conf.transport_driver = 'fake'
self.messaging_conf.response_timeout = 15
self.useFixture(self.messaging_conf)
rpc.init(CONF)
conf_fixture.set_defaults(CONF)
CONF([], default_config_files=[])
# NOTE(vish): We need a better method for creating fixtures for tests
# now that we have some required db setup for the system
# to work properly.
self.start = timeutils.utcnow()
CONF.set_default('connection', 'sqlite://', 'database')
CONF.set_default('sqlite_synchronous', False, 'database')
global _DB_CACHE
if not _DB_CACHE:
_DB_CACHE = Database(sqla_api, migration,
sql_connection=CONF.database.connection,
sqlite_db=CONF.database.sqlite_db,
sqlite_clean_db=CONF.sqlite_clean_db)
self.useFixture(_DB_CACHE)
# emulate some of the mox stuff, we can't use the metaclass
# because it screws with our generators
self.mox = mox.Mox()
self.stubs = stubout.StubOutForTesting()
self.addCleanup(CONF.reset)
self.addCleanup(self.mox.UnsetStubs)
self.addCleanup(self.stubs.UnsetAll)
self.addCleanup(self.stubs.SmartUnsetAll)
self.addCleanup(self.mox.VerifyAll)
self.addCleanup(self._common_cleanup)
self.injected = []
self._services = []
fake_notifier.stub_notifier(self.stubs)
self.override_config('fatal_exception_format_errors', True)
# This will be cleaned up by the NestedTempfile fixture
lock_path = self.useFixture(fixtures.TempDir()).path
self.fixture = self.useFixture(
config_fixture.Config(lockutils.CONF))
self.fixture.config(lock_path=lock_path,
group='oslo_concurrency')
lockutils.set_defaults(lock_path)
self.override_config('policy_file',
os.path.join(
os.path.abspath(
os.path.join(
os.path.dirname(__file__),
'..',
)
),
'cinder/tests/unit/policy.json'))
def __init__(self,
host,
binary,
topic,
manager,
report_interval=None,
periodic_interval=None,
periodic_fuzzy_delay=None,
service_name=None,
coordination=False,
cluster=None,
*args,
**kwargs):
super(Service, self).__init__()
if not rpc.initialized():
rpc.init(CONF)
self.cluster = cluster
self.host = host
self.binary = binary
self.topic = topic
self.manager_class_name = manager
self.coordination = coordination
manager_class = importutils.import_class(self.manager_class_name)
if CONF.profiler.enabled:
manager_class = profiler.trace_cls("rpc")(manager_class)
# NOTE(geguileo): We need to create the Service DB entry before we
# create the manager, otherwise capped versions for serializer and rpc
# client would use existing DB entries not including us, which could
# result in us using None (if it's the first time the service is run)
# or an old version (if this is a normal upgrade of a single service).
ctxt = context.get_admin_context()
self.is_upgrading_to_n = self.is_svc_upgrading_to_n(binary)
try:
service_ref = objects.Service.get_by_args(ctxt, host, binary)
service_ref.rpc_current_version = manager_class.RPC_API_VERSION
obj_version = objects_base.OBJ_VERSIONS.get_current()
service_ref.object_current_version = obj_version
# TODO(geguileo): In O we can remove the service upgrading part on
# the next equation, because by then all our services will be
# properly setting the cluster during volume migrations since
# they'll have the new Volume ORM model. But until then we can
# only set the cluster in the DB and pass added_to_cluster to
# init_host when we have completed the rolling upgrade from M to N.
# added_to_cluster attribute marks when we consider that we have
# just added a host to a cluster so we can include resources into
# that cluster. We consider that we have added the host when we
# didn't have data in the cluster DB field and our current
# configuration has a cluster value. We don't want to do anything
# automatic if the cluster is changed, in those cases we'll want
# to use cinder manage command and to it manually.
self.added_to_cluster = (not service_ref.cluster_name and cluster
and not self.is_upgrading_to_n)
# TODO(geguileo): In O - Remove self.is_upgrading_to_n part
if (service_ref.cluster_name != cluster
and not self.is_upgrading_to_n):
LOG.info(
_LI('This service has been moved from cluster '
'%(cluster_svc)s to %(cluster_cfg)s. Resources '
'will %(opt_no)sbe moved to the new cluster'), {
'cluster_svc': service_ref.cluster_name,
'cluster_cfg': cluster,
'opt_no': '' if self.added_to_cluster else 'NO '
})
if self.added_to_cluster:
# We pass copy service's disable status in the cluster if we
# have to create it.
self._ensure_cluster_exists(ctxt, service_ref.disabled)
service_ref.cluster_name = cluster
service_ref.save()
Service.service_id = service_ref.id
except exception.NotFound:
# We don't want to include cluster information on the service or
# create the cluster entry if we are upgrading.
self._create_service_ref(ctxt, manager_class.RPC_API_VERSION)
# TODO(geguileo): In O set added_to_cluster to True
# We don't want to include resources in the cluster during the
# start while we are still doing the rolling upgrade.
self.added_to_cluster = not self.is_upgrading_to_n
self.manager = manager_class(host=self.host,
cluster=self.cluster,
service_name=service_name,
*args,
**kwargs)
self.report_interval = report_interval
self.periodic_interval = periodic_interval
self.periodic_fuzzy_delay = periodic_fuzzy_delay
self.basic_config_check()
self.saved_args, self.saved_kwargs = args, kwargs
self.timers = []
setup_profiler(binary, host)
self.rpcserver = None
self.cluster_rpcserver = None
return list_response
def GetCapacity(self, req, context):
pass
def ControllerGetCapabilities(self, req, context):
pass
def serve():
server = grpc.server(futures.ThreadPoolExecutor(max_workers=10))
csi_pb2_grpc.add_ControllerServicer_to_server(CinderServicer(), server)
server.add_insecure_port('[::]:50051')
server.start()
print("now serving...")
try:
while True:
time.sleep(_ONE_DAY_IN_SECONDS)
except KeyboardInterrupt:
server.stop(0)
if __name__ == '__main__':
objects.register_all()
CONF(sys.argv[1:], project='cinder', version=version.version_string())
config.set_middleware_defaults()
#utils.monkey_patch()
rpc.init(CONF)
serve()
def main():
admin_context = context.get_admin_context()
CONF(sys.argv[1:], project='cinder',
version=version.version_string())
logging.setup("cinder")
LOG = logging.getLogger("cinder")
rpc.init(CONF)
begin, end = utils.last_completed_audit_period()
if CONF.start_time:
begin = datetime.strptime(CONF.start_time, "%Y-%m-%d %H:%M:%S")
if CONF.end_time:
end = datetime.strptime(CONF.end_time, "%Y-%m-%d %H:%M:%S")
if not end > begin:
msg = _("The end time (%(end)s) must be after the start "
"time (%(start)s).") % {'start': begin,
'end': end}
print(msg)
LOG.error(msg)
sys.exit(-1)
print(_("Starting volume usage audit"))
msg = _("Creating usages for %(begin_period)s until %(end_period)s")
print(msg % {"begin_period": str(begin), "end_period": str(end)})
extra_info = {
'audit_period_beginning': str(begin),
'audit_period_ending': str(end),
}
volumes = db.volume_get_active_by_window(admin_context,
begin,
end)
print(_("Found %d volumes") % len(volumes))
for volume_ref in volumes:
try:
LOG.debug("Send exists notification for <volume_id: "
"%(volume_id)s> <project_id %(project_id)s> "
"<%(extra_info)s>" %
{'volume_id': volume_ref.id,
'project_id': volume_ref.project_id,
'extra_info': extra_info})
cinder.volume.utils.notify_about_volume_usage(
admin_context,
volume_ref,
'exists', extra_usage_info=extra_info)
except Exception as e:
LOG.error(_LE("Failed to send exists notification"
" for volume %s.") %
volume_ref.id)
print(traceback.format_exc(e))
if (CONF.send_actions and
volume_ref.created_at > begin and
volume_ref.created_at < end):
try:
local_extra_info = {
'audit_period_beginning': str(volume_ref.created_at),
'audit_period_ending': str(volume_ref.created_at),
}
LOG.debug("Send create notification for "
"<volume_id: %(volume_id)s> "
"<project_id %(project_id)s> <%(extra_info)s>" %
{'volume_id': volume_ref.id,
'project_id': volume_ref.project_id,
'extra_info': local_extra_info})
cinder.volume.utils.notify_about_volume_usage(
admin_context,
volume_ref,
'create.start', extra_usage_info=local_extra_info)
cinder.volume.utils.notify_about_volume_usage(
admin_context,
volume_ref,
'create.end', extra_usage_info=local_extra_info)
except Exception as e:
LOG.error(_LE("Failed to send create notification for "
"volume %s.") % volume_ref.id)
print(traceback.format_exc(e))
if (CONF.send_actions and volume_ref.deleted_at and
volume_ref.deleted_at > begin and
volume_ref.deleted_at < end):
try:
local_extra_info = {
'audit_period_beginning': str(volume_ref.deleted_at),
'audit_period_ending': str(volume_ref.deleted_at),
}
LOG.debug("Send delete notification for "
"<volume_id: %(volume_id)s> "
"<project_id %(project_id)s> <%(extra_info)s>" %
{'volume_id': volume_ref.id,
'project_id': volume_ref.project_id,
'extra_info': local_extra_info})
cinder.volume.utils.notify_about_volume_usage(
admin_context,
volume_ref,
'delete.start', extra_usage_info=local_extra_info)
cinder.volume.utils.notify_about_volume_usage(
admin_context,
volume_ref,
'delete.end', extra_usage_info=local_extra_info)
except Exception as e:
LOG.error(_LE("Failed to send delete notification for volume "
"%s.") % volume_ref.id)
print(traceback.format_exc(e))
snapshots = db.snapshot_get_active_by_window(admin_context,
begin,
end)
print(_("Found %d snapshots") % len(snapshots))
for snapshot_ref in snapshots:
try:
LOG.debug("Send notification for <snapshot_id: %(snapshot_id)s> "
"<project_id %(project_id)s> <%(extra_info)s>" %
{'snapshot_id': snapshot_ref.id,
'project_id': snapshot_ref.project_id,
'extra_info': extra_info})
cinder.volume.utils.notify_about_snapshot_usage(admin_context,
snapshot_ref,
'exists',
extra_info)
except Exception as e:
LOG.error(_LE("Failed to send exists notification "
"for snapshot %s.")
% snapshot_ref.id)
print(traceback.format_exc(e))
if (CONF.send_actions and
snapshot_ref.created_at > begin and
snapshot_ref.created_at < end):
try:
local_extra_info = {
'audit_period_beginning': str(snapshot_ref.created_at),
'audit_period_ending': str(snapshot_ref.created_at),
}
LOG.debug("Send create notification for "
"<snapshot_id: %(snapshot_id)s> "
"<project_id %(project_id)s> <%(extra_info)s>" %
{'snapshot_id': snapshot_ref.id,
'project_id': snapshot_ref.project_id,
'extra_info': local_extra_info})
cinder.volume.utils.notify_about_snapshot_usage(
admin_context,
snapshot_ref,
'create.start', extra_usage_info=local_extra_info)
cinder.volume.utils.notify_about_snapshot_usage(
admin_context,
snapshot_ref,
'create.end', extra_usage_info=local_extra_info)
except Exception as e:
LOG.error(_LE("Failed to send create notification for snapshot"
"%s.") % snapshot_ref.id)
print(traceback.format_exc(e))
if (CONF.send_actions and snapshot_ref.deleted_at and
snapshot_ref.deleted_at > begin and
snapshot_ref.deleted_at < end):
try:
local_extra_info = {
'audit_period_beginning': str(snapshot_ref.deleted_at),
'audit_period_ending': str(snapshot_ref.deleted_at),
}
LOG.debug("Send delete notification for "
"<snapshot_id: %(snapshot_id)s> "
"<project_id %(project_id)s> <%(extra_info)s>" %
{'snapshot_id': snapshot_ref.id,
'project_id': snapshot_ref.project_id,
'extra_info': local_extra_info})
cinder.volume.utils.notify_about_snapshot_usage(
admin_context,
snapshot_ref,
'delete.start', extra_usage_info=local_extra_info)
cinder.volume.utils.notify_about_snapshot_usage(
admin_context,
snapshot_ref,
'delete.end', extra_usage_info=local_extra_info)
except Exception as e:
LOG.error(_LE("Failed to send delete notification for snapshot"
"%s.") % snapshot_ref.id)
print(traceback.format_exc(e))
print(_("Volume usage audit completed"))
def setUp(self):
"""Run before each test method to initialize test environment."""
super(TestCase, self).setUp()
# Create default notifier
self.notifier = fake_notifier.get_fake_notifier()
# Mock rpc get notifier with fake notifier method that joins all
# notifications with the default notifier
self.patch('cinder.rpc.get_notifier',
side_effect=self._get_joined_notifier)
if self.MOCK_WORKER:
# Mock worker creation for all tests that don't care about it
clean_path = 'cinder.objects.cleanable.CinderCleanableObject.%s'
for method in ('create_worker', 'set_worker', 'unset_worker'):
self.patch(clean_path % method, return_value=None)
# Unit tests do not need to use lazy gettext
i18n.enable_lazy(False)
test_timeout = os.environ.get('OS_TEST_TIMEOUT', 0)
try:
test_timeout = int(test_timeout)
except ValueError:
# If timeout value is invalid do not set a timeout.
test_timeout = 0
if test_timeout > 0:
self.useFixture(fixtures.Timeout(test_timeout, gentle=True))
self.useFixture(fixtures.NestedTempfile())
self.useFixture(fixtures.TempHomeDir())
environ_enabled = (lambda var_name: strutils.bool_from_string(
os.environ.get(var_name)))
if environ_enabled('OS_STDOUT_CAPTURE'):
stdout = self.useFixture(fixtures.StringStream('stdout')).stream
self.useFixture(fixtures.MonkeyPatch('sys.stdout', stdout))
if environ_enabled('OS_STDERR_CAPTURE'):
stderr = self.useFixture(fixtures.StringStream('stderr')).stream
self.useFixture(fixtures.MonkeyPatch('sys.stderr', stderr))
self.useFixture(log_fixture.get_logging_handle_error_fixture())
self.useFixture(cinder_fixtures.StandardLogging())
rpc.add_extra_exmods("cinder.tests.unit")
self.addCleanup(rpc.clear_extra_exmods)
self.addCleanup(rpc.cleanup)
self.messaging_conf = messaging_conffixture.ConfFixture(CONF)
self.messaging_conf.transport_driver = 'fake'
self.messaging_conf.response_timeout = 15
self.useFixture(self.messaging_conf)
# Load oslo_messaging_notifications config group so we can set an
# override to prevent notifications from being ignored due to the
# short-circuit mechanism.
oslo_messaging.get_notification_transport(CONF)
# We need to use a valid driver for the notifications, so we use test.
self.override_config('driver', ['test'],
group='oslo_messaging_notifications')
rpc.init(CONF)
# NOTE(geguileo): This is required because _determine_obj_version_cap
# and _determine_rpc_version_cap functions in cinder.rpc.RPCAPI cache
# versions in LAST_RPC_VERSIONS and LAST_OBJ_VERSIONS so we may have
# weird interactions between tests if we don't clear them before each
# test.
rpc.LAST_OBJ_VERSIONS = {}
rpc.LAST_RPC_VERSIONS = {}
conf_fixture.set_defaults(CONF)
CONF([], default_config_files=[])
# NOTE(vish): We need a better method for creating fixtures for tests
# now that we have some required db setup for the system
# to work properly.
self.start = timeutils.utcnow()
CONF.set_default('connection', 'sqlite://', 'database')
CONF.set_default('sqlite_synchronous', False, 'database')
global _DB_CACHE
if not _DB_CACHE:
_DB_CACHE = Database(sqla_api,
migration,
sql_connection=CONF.database.connection)
self.useFixture(_DB_CACHE)
# NOTE(danms): Make sure to reset us back to non-remote objects
# for each test to avoid interactions. Also, backup the object
# registry.
objects_base.CinderObject.indirection_api = None
self._base_test_obj_backup = copy.copy(
objects_base.CinderObjectRegistry._registry._obj_classes)
self.addCleanup(self._restore_obj_registry)
# emulate some of the mox stuff, we can't use the metaclass
# because it screws with our generators
mox_fixture = self.useFixture(moxstubout.MoxStubout())
self.mox = mox_fixture.mox
self.stubs = mox_fixture.stubs
self.addCleanup(CONF.reset)
self.addCleanup(self._common_cleanup)
self.injected = []
self._services = []
fake_notifier.mock_notifier(self)
self.override_config('fatal_exception_format_errors', True)
# This will be cleaned up by the NestedTempfile fixture
lock_path = self.useFixture(fixtures.TempDir()).path
self.fixture = self.useFixture(config_fixture.Config(lockutils.CONF))
self.fixture.config(lock_path=lock_path, group='oslo_concurrency')
lockutils.set_defaults(lock_path)
self.override_config('policy_file',
os.path.join(
os.path.abspath(
os.path.join(
os.path.dirname(__file__),
'..',
)), self.POLICY_PATH),
group='oslo_policy')
self._disable_osprofiler()
self._disallow_invalid_uuids()
# NOTE(geguileo): This is required because common get_by_id method in
# cinder.db.sqlalchemy.api caches get methods and if we use a mocked
# get method in one test it would carry on to the next test. So we
# clear out the cache.
sqla_api._GET_METHODS = {}
self.override_config('backend_url',
'file://' + lock_path,
group='coordination')
coordination.COORDINATOR.start()
self.addCleanup(coordination.COORDINATOR.stop)
def setUp(self):
"""Run before each test method to initialize test environment."""
super(TestCase, self).setUp()
# Create default notifier
self.notifier = fake_notifier.get_fake_notifier()
# Mock rpc get notifier with fake notifier method that joins all
# notifications with the default notifier
self.patch('cinder.rpc.get_notifier',
side_effect=self._get_joined_notifier)
if self.MOCK_WORKER:
# Mock worker creation for all tests that don't care about it
clean_path = 'cinder.objects.cleanable.CinderCleanableObject.%s'
for method in ('create_worker', 'set_worker', 'unset_worker'):
self.patch(clean_path % method, return_value=None)
if self.MOCK_TOOZ:
self.patch('cinder.coordination.Coordinator.start')
self.patch('cinder.coordination.Coordinator.stop')
self.patch('cinder.coordination.Coordinator.get_lock')
# Unit tests do not need to use lazy gettext
i18n.enable_lazy(False)
test_timeout = os.environ.get('OS_TEST_TIMEOUT', 0)
try:
test_timeout = int(test_timeout)
except ValueError:
# If timeout value is invalid do not set a timeout.
test_timeout = 0
if test_timeout > 0:
self.useFixture(fixtures.Timeout(test_timeout, gentle=True))
self.useFixture(fixtures.NestedTempfile())
self.useFixture(fixtures.TempHomeDir())
environ_enabled = (lambda var_name:
strutils.bool_from_string(os.environ.get(var_name)))
if environ_enabled('OS_STDOUT_CAPTURE'):
stdout = self.useFixture(fixtures.StringStream('stdout')).stream
self.useFixture(fixtures.MonkeyPatch('sys.stdout', stdout))
if environ_enabled('OS_STDERR_CAPTURE'):
stderr = self.useFixture(fixtures.StringStream('stderr')).stream
self.useFixture(fixtures.MonkeyPatch('sys.stderr', stderr))
self.useFixture(log_fixture.get_logging_handle_error_fixture())
self.useFixture(cinder_fixtures.StandardLogging())
rpc.add_extra_exmods("cinder.tests.unit")
self.addCleanup(rpc.clear_extra_exmods)
self.addCleanup(rpc.cleanup)
self.messaging_conf = messaging_conffixture.ConfFixture(CONF)
self.messaging_conf.transport_driver = 'fake'
self.messaging_conf.response_timeout = 15
self.useFixture(self.messaging_conf)
# Load oslo_messaging_notifications config group so we can set an
# override to prevent notifications from being ignored due to the
# short-circuit mechanism.
oslo_messaging.get_notification_transport(CONF)
# We need to use a valid driver for the notifications, so we use test.
self.override_config('driver', ['test'],
group='oslo_messaging_notifications')
rpc.init(CONF)
# NOTE(geguileo): This is required because _determine_obj_version_cap
# and _determine_rpc_version_cap functions in cinder.rpc.RPCAPI cache
# versions in LAST_RPC_VERSIONS and LAST_OBJ_VERSIONS so we may have
# weird interactions between tests if we don't clear them before each
# test.
rpc.LAST_OBJ_VERSIONS = {}
rpc.LAST_RPC_VERSIONS = {}
conf_fixture.set_defaults(CONF)
CONF([], default_config_files=[])
# NOTE(vish): We need a better method for creating fixtures for tests
# now that we have some required db setup for the system
# to work properly.
self.start = timeutils.utcnow()
CONF.set_default('connection', 'sqlite://', 'database')
CONF.set_default('sqlite_synchronous', False, 'database')
global _DB_CACHE
if not _DB_CACHE:
_DB_CACHE = Database(sqla_api, migration,
sql_connection=CONF.database.connection)
self.useFixture(_DB_CACHE)
# NOTE(blk-u): WarningsFixture must be after the Database fixture
# because sqlalchemy-migrate messes with the warnings filters.
self.useFixture(cinder_fixtures.WarningsFixture())
# NOTE(danms): Make sure to reset us back to non-remote objects
# for each test to avoid interactions. Also, backup the object
# registry.
objects_base.CinderObject.indirection_api = None
self._base_test_obj_backup = copy.copy(
objects_base.CinderObjectRegistry._registry._obj_classes)
self.addCleanup(self._restore_obj_registry)
self.addCleanup(CONF.reset)
self.addCleanup(self._common_cleanup)
self.injected = []
self._services = []
fake_notifier.mock_notifier(self)
# This will be cleaned up by the NestedTempfile fixture
lock_path = self.useFixture(fixtures.TempDir()).path
self.fixture = self.useFixture(
config_fixture.Config(lockutils.CONF))
self.fixture.config(lock_path=lock_path,
group='oslo_concurrency')
lockutils.set_defaults(lock_path)
self.override_config('policy_file',
os.path.join(
os.path.abspath(
os.path.join(
os.path.dirname(__file__),
'..',
)
),
self.POLICY_PATH),
group='oslo_policy')
self.override_config('resource_query_filters_file',
os.path.join(
os.path.abspath(
os.path.join(
os.path.dirname(__file__),
'..',
)
),
self.RESOURCE_FILTER_PATH))
self._disable_osprofiler()
# NOTE(geguileo): This is required because common get_by_id method in
# cinder.db.sqlalchemy.api caches get methods and if we use a mocked
# get method in one test it would carry on to the next test. So we
# clear out the cache.
sqla_api._GET_METHODS = {}
self.override_config('backend_url', 'file://' + lock_path,
group='coordination')
coordination.COORDINATOR.start()
self.addCleanup(coordination.COORDINATOR.stop)
if six.PY3:
# TODO(smcginnis) Python 3 deprecates assertRaisesRegexp to
# assertRaisesRegex, but Python 2 does not have the new name. This
# can be removed once we stop supporting py2 or the new name is
# added.
self.assertRaisesRegexp = self.assertRaisesRegex
# Ensure we have the default tpool size value and we don't carry
# threads from other test runs.
tpool.killall()
tpool._nthreads = 20
def setUp(self):
"""Run before each test method to initialize test environment."""
super(TestCase, self).setUp()
test_timeout = os.environ.get('OS_TEST_TIMEOUT', 0)
try:
test_timeout = int(test_timeout)
except ValueError:
# If timeout value is invalid do not set a timeout.
test_timeout = 0
if test_timeout > 0:
self.useFixture(fixtures.Timeout(test_timeout, gentle=True))
self.useFixture(fixtures.NestedTempfile())
self.useFixture(fixtures.TempHomeDir())
if (os.environ.get('OS_STDOUT_CAPTURE') == 'True' or
os.environ.get('OS_STDOUT_CAPTURE') == '1'):
stdout = self.useFixture(fixtures.StringStream('stdout')).stream
self.useFixture(fixtures.MonkeyPatch('sys.stdout', stdout))
if (os.environ.get('OS_STDERR_CAPTURE') == 'True' or
os.environ.get('OS_STDERR_CAPTURE') == '1'):
stderr = self.useFixture(fixtures.StringStream('stderr')).stream
self.useFixture(fixtures.MonkeyPatch('sys.stderr', stderr))
self.log_fixture = self.useFixture(fixtures.FakeLogger(
level=logging.DEBUG))
rpc.add_extra_exmods("cinder.tests")
self.addCleanup(rpc.clear_extra_exmods)
self.addCleanup(rpc.cleanup)
self.messaging_conf = messaging_conffixture.ConfFixture(CONF)
self.messaging_conf.transport_driver = 'fake'
self.messaging_conf.response_timeout = 15
self.useFixture(self.messaging_conf)
rpc.init(CONF)
conf_fixture.set_defaults(CONF)
CONF([], default_config_files=[])
# NOTE(vish): We need a better method for creating fixtures for tests
# now that we have some required db setup for the system
# to work properly.
self.start = timeutils.utcnow()
CONF.set_default('connection', 'sqlite://', 'database')
CONF.set_default('sqlite_synchronous', False)
self.log_fixture = self.useFixture(fixtures.FakeLogger())
global _DB_CACHE
if not _DB_CACHE:
_DB_CACHE = Database(session, migration,
sql_connection=CONF.database.connection,
sqlite_db=CONF.sqlite_db,
sqlite_clean_db=CONF.sqlite_clean_db)
self.useFixture(_DB_CACHE)
# emulate some of the mox stuff, we can't use the metaclass
# because it screws with our generators
self.mox = mox.Mox()
self.stubs = stubout.StubOutForTesting()
self.addCleanup(CONF.reset)
self.addCleanup(self.mox.UnsetStubs)
self.addCleanup(self.stubs.UnsetAll)
self.addCleanup(self.stubs.SmartUnsetAll)
self.addCleanup(self.mox.VerifyAll)
self.addCleanup(self._common_cleanup)
self.injected = []
self._services = []
fake_notifier.stub_notifier(self.stubs)
CONF.set_override('fatal_exception_format_errors', True)
# This will be cleaned up by the NestedTempfile fixture
CONF.set_override('lock_path', tempfile.mkdtemp())
def test_init_notifications(self, messaging_mock):
rpc.init(test.CONF)
self.assertTrue(messaging_mock.JsonPayloadSerializer.called)
self.assertTrue(messaging_mock.Notifier.called)
self.assertEqual(rpc.NOTIFIER, messaging_mock.Notifier.return_value)
def __init__(self, host, binary, topic, manager, report_interval=None,
periodic_interval=None, periodic_fuzzy_delay=None,
service_name=None, coordination=False, cluster=None, *args,
**kwargs):
super(Service, self).__init__()
if not rpc.initialized():
rpc.init(CONF)
self.cluster = cluster
self.host = host
self.binary = binary
self.topic = topic
self.manager_class_name = manager
self.coordination = coordination
manager_class = importutils.import_class(self.manager_class_name)
if CONF.profiler.enabled:
manager_class = profiler.trace_cls("rpc")(manager_class)
# NOTE(geguileo): We need to create the Service DB entry before we
# create the manager, otherwise capped versions for serializer and rpc
# client would use existing DB entries not including us, which could
# result in us using None (if it's the first time the service is run)
# or an old version (if this is a normal upgrade of a single service).
ctxt = context.get_admin_context()
self.is_upgrading_to_n = self.is_svc_upgrading_to_n(binary)
try:
service_ref = objects.Service.get_by_args(ctxt, host, binary)
service_ref.rpc_current_version = manager_class.RPC_API_VERSION
obj_version = objects_base.OBJ_VERSIONS.get_current()
service_ref.object_current_version = obj_version
# TODO(geguileo): In O we can remove the service upgrading part on
# the next equation, because by then all our services will be
# properly setting the cluster during volume migrations since
# they'll have the new Volume ORM model. But until then we can
# only set the cluster in the DB and pass added_to_cluster to
# init_host when we have completed the rolling upgrade from M to N.
# added_to_cluster attribute marks when we consider that we have
# just added a host to a cluster so we can include resources into
# that cluster. We consider that we have added the host when we
# didn't have data in the cluster DB field and our current
# configuration has a cluster value. We don't want to do anything
# automatic if the cluster is changed, in those cases we'll want
# to use cinder manage command and to it manually.
self.added_to_cluster = (not service_ref.cluster_name and cluster
and not self.is_upgrading_to_n)
# TODO(geguileo): In O - Remove self.is_upgrading_to_n part
if (service_ref.cluster_name != cluster and
not self.is_upgrading_to_n):
LOG.info(_LI('This service has been moved from cluster '
'%(cluster_svc)s to %(cluster_cfg)s. Resources '
'will %(opt_no)sbe moved to the new cluster'),
{'cluster_svc': service_ref.cluster_name,
'cluster_cfg': cluster,
'opt_no': '' if self.added_to_cluster else 'NO '})
if self.added_to_cluster:
# We pass copy service's disable status in the cluster if we
# have to create it.
self._ensure_cluster_exists(ctxt, service_ref.disabled)
service_ref.cluster_name = cluster
service_ref.save()
Service.service_id = service_ref.id
except exception.NotFound:
# We don't want to include cluster information on the service or
# create the cluster entry if we are upgrading.
self._create_service_ref(ctxt, manager_class.RPC_API_VERSION)
# TODO(geguileo): In O set added_to_cluster to True
# We don't want to include resources in the cluster during the
# start while we are still doing the rolling upgrade.
self.added_to_cluster = not self.is_upgrading_to_n
self.manager = manager_class(host=self.host,
cluster=self.cluster,
service_name=service_name,
*args, **kwargs)
self.report_interval = report_interval
self.periodic_interval = periodic_interval
self.periodic_fuzzy_delay = periodic_fuzzy_delay
self.basic_config_check()
self.saved_args, self.saved_kwargs = args, kwargs
self.timers = []
setup_profiler(binary, host)
self.rpcserver = None
self.cluster_rpcserver = None
def test_init_notifications(self, messaging_mock):
rpc.init(test.CONF)
self.assertTrue(messaging_mock.JsonPayloadSerializer.called)
self.assertTrue(messaging_mock.Notifier.called)
self.assertEqual(rpc.NOTIFIER, messaging_mock.Notifier.return_value)
def __init__(self, host, binary, topic, manager, report_interval=None,
periodic_interval=None, periodic_fuzzy_delay=None,
service_name=None, coordination=False, cluster=None, *args,
**kwargs):
super(Service, self).__init__()
if not rpc.initialized():
rpc.init(CONF)
self.cluster = cluster
self.host = host
self.binary = binary
self.topic = topic
self.manager_class_name = manager
self.coordination = coordination
manager_class = importutils.import_class(self.manager_class_name)
if CONF.profiler.enabled:
manager_class = profiler.trace_cls("rpc")(manager_class)
self.service = None
self.manager = manager_class(host=self.host,
cluster=self.cluster,
service_name=service_name,
*args, **kwargs)
self.availability_zone = self.manager.availability_zone
# NOTE(geguileo): We need to create the Service DB entry before we
# create the manager, otherwise capped versions for serializer and rpc
# client would use existing DB entries not including us, which could
# result in us using None (if it's the first time the service is run)
# or an old version (if this is a normal upgrade of a single service).
ctxt = context.get_admin_context()
try:
service_ref = objects.Service.get_by_args(ctxt, host, binary)
service_ref.rpc_current_version = manager_class.RPC_API_VERSION
obj_version = objects_base.OBJ_VERSIONS.get_current()
service_ref.object_current_version = obj_version
# added_to_cluster attribute marks when we consider that we have
# just added a host to a cluster so we can include resources into
# that cluster. We consider that we have added the host when we
# didn't have data in the cluster DB field and our current
# configuration has a cluster value. We don't want to do anything
# automatic if the cluster is changed, in those cases we'll want
# to use cinder manage command and to it manually.
self.added_to_cluster = (not service_ref.cluster_name and cluster)
if service_ref.cluster_name != cluster:
LOG.info('This service has been moved from cluster '
'%(cluster_svc)s to %(cluster_cfg)s. Resources '
'will %(opt_no)sbe moved to the new cluster',
{'cluster_svc': service_ref.cluster_name,
'cluster_cfg': cluster,
'opt_no': '' if self.added_to_cluster else 'NO '})
if self.added_to_cluster:
# We pass copy service's disable status in the cluster if we
# have to create it.
self._ensure_cluster_exists(ctxt, service_ref)
service_ref.cluster_name = cluster
service_ref.save()
Service.service_id = service_ref.id
self.origin_service_id = service_ref.id
except exception.NotFound:
self._create_service_ref(ctxt, manager_class.RPC_API_VERSION)
# Service entry Entry didn't exist because it was manually removed
# or it's the first time running, to be on the safe side we say we
# were added if we are clustered.
self.added_to_cluster = bool(cluster)
self.report_interval = report_interval
self.periodic_interval = periodic_interval
self.periodic_fuzzy_delay = periodic_fuzzy_delay
self.basic_config_check()
self.saved_args, self.saved_kwargs = args, kwargs
setup_profiler(binary, host)
self.rpcserver = None
self.backend_rpcserver = None
self.cluster_rpcserver = None
def main():
objects.register_all()
admin_context = context.get_admin_context()
CONF(sys.argv[1:], project='cinder',
version=version.version_string())
logging.setup(CONF, "cinder")
LOG = logging.getLogger("cinder")
rpc.init(CONF)
begin, end = utils.last_completed_audit_period()
if CONF.start_time:
begin = datetime.datetime.strptime(CONF.start_time,
"%Y-%m-%d %H:%M:%S")
if CONF.end_time:
end = datetime.datetime.strptime(CONF.end_time,
"%Y-%m-%d %H:%M:%S")
if not end > begin:
msg = _("The end time (%(end)s) must be after the start "
"time (%(start)s).") % {'start': begin,
'end': end}
LOG.error(msg)
sys.exit(-1)
LOG.debug("Starting volume usage audit")
msg = _("Creating usages for %(begin_period)s until %(end_period)s")
LOG.debug(msg, {"begin_period": str(begin), "end_period": str(end)})
extra_info = {
'audit_period_beginning': str(begin),
'audit_period_ending': str(end),
}
volumes = db.volume_get_active_by_window(admin_context,
begin,
end)
LOG.debug("Found %d volumes"), len(volumes)
for volume_ref in volumes:
try:
LOG.debug("Send exists notification for <volume_id: "
"%(volume_id)s> <project_id %(project_id)s> "
"<%(extra_info)s>",
{'volume_id': volume_ref.id,
'project_id': volume_ref.project_id,
'extra_info': extra_info})
cinder.volume.utils.notify_about_volume_usage(
admin_context,
volume_ref,
'exists', extra_usage_info=extra_info)
except Exception as exc_msg:
LOG.exception(_LE("Exists volume notification failed: %s"),
exc_msg, resource=volume_ref)
if (CONF.send_actions and
volume_ref.created_at > begin and
volume_ref.created_at < end):
try:
local_extra_info = {
'audit_period_beginning': str(volume_ref.created_at),
'audit_period_ending': str(volume_ref.created_at),
}
LOG.debug("Send create notification for "
"<volume_id: %(volume_id)s> "
"<project_id %(project_id)s> <%(extra_info)s>",
{'volume_id': volume_ref.id,
'project_id': volume_ref.project_id,
'extra_info': local_extra_info})
cinder.volume.utils.notify_about_volume_usage(
admin_context,
volume_ref,
'create.start', extra_usage_info=local_extra_info)
cinder.volume.utils.notify_about_volume_usage(
admin_context,
volume_ref,
'create.end', extra_usage_info=local_extra_info)
except Exception as exc_msg:
LOG.exception(_LE("Create volume notification failed: %s"),
exc_msg, resource=volume_ref)
if (CONF.send_actions and volume_ref.deleted_at and
volume_ref.deleted_at > begin and
volume_ref.deleted_at < end):
try:
local_extra_info = {
'audit_period_beginning': str(volume_ref.deleted_at),
'audit_period_ending': str(volume_ref.deleted_at),
}
LOG.debug("Send delete notification for "
"<volume_id: %(volume_id)s> "
"<project_id %(project_id)s> <%(extra_info)s>",
{'volume_id': volume_ref.id,
'project_id': volume_ref.project_id,
'extra_info': local_extra_info})
cinder.volume.utils.notify_about_volume_usage(
admin_context,
volume_ref,
'delete.start', extra_usage_info=local_extra_info)
cinder.volume.utils.notify_about_volume_usage(
admin_context,
volume_ref,
'delete.end', extra_usage_info=local_extra_info)
except Exception as exc_msg:
LOG.exception(_LE("Delete volume notification failed: %s"),
exc_msg, resource=volume_ref)
snapshots = objects.SnapshotList.get_active_by_window(admin_context,
begin, end)
LOG.debug("Found %d snapshots"), len(snapshots)
for snapshot_ref in snapshots:
try:
LOG.debug("Send notification for <snapshot_id: %(snapshot_id)s> "
"<project_id %(project_id)s> <%(extra_info)s>",
{'snapshot_id': snapshot_ref.id,
'project_id': snapshot_ref.project_id,
'extra_info': extra_info})
cinder.volume.utils.notify_about_snapshot_usage(admin_context,
snapshot_ref,
'exists',
extra_info)
except Exception as exc_msg:
LOG.exception(_LE("Exists snapshot notification failed: %s"),
exc_msg, resource=snapshot_ref)
if (CONF.send_actions and
snapshot_ref.created_at > begin and
snapshot_ref.created_at < end):
try:
local_extra_info = {
'audit_period_beginning': str(snapshot_ref.created_at),
'audit_period_ending': str(snapshot_ref.created_at),
}
LOG.debug("Send create notification for "
"<snapshot_id: %(snapshot_id)s> "
"<project_id %(project_id)s> <%(extra_info)s>",
{'snapshot_id': snapshot_ref.id,
'project_id': snapshot_ref.project_id,
'extra_info': local_extra_info})
cinder.volume.utils.notify_about_snapshot_usage(
admin_context,
snapshot_ref,
'create.start', extra_usage_info=local_extra_info)
cinder.volume.utils.notify_about_snapshot_usage(
admin_context,
snapshot_ref,
'create.end', extra_usage_info=local_extra_info)
except Exception as exc_msg:
LOG.exception(_LE("Create snapshot notification failed: %s"),
exc_msg, resource=snapshot_ref)
if (CONF.send_actions and snapshot_ref.deleted_at and
snapshot_ref.deleted_at > begin and
snapshot_ref.deleted_at < end):
try:
local_extra_info = {
'audit_period_beginning': str(snapshot_ref.deleted_at),
'audit_period_ending': str(snapshot_ref.deleted_at),
}
LOG.debug("Send delete notification for "
"<snapshot_id: %(snapshot_id)s> "
"<project_id %(project_id)s> <%(extra_info)s>",
{'snapshot_id': snapshot_ref.id,
'project_id': snapshot_ref.project_id,
'extra_info': local_extra_info})
cinder.volume.utils.notify_about_snapshot_usage(
admin_context,
snapshot_ref,
'delete.start', extra_usage_info=local_extra_info)
cinder.volume.utils.notify_about_snapshot_usage(
admin_context,
snapshot_ref,
'delete.end', extra_usage_info=local_extra_info)
except Exception as exc_msg:
LOG.exception(_LE("Delete snapshot notification failed: %s"),
exc_msg, resource=snapshot_ref)
LOG.debug("Volume usage audit completed")
def setUp(self):
"""Run before each test method to initialize test environment."""
super(TestCase, self).setUp()
test_timeout = os.environ.get('OS_TEST_TIMEOUT', 0)
try:
test_timeout = int(test_timeout)
except ValueError:
# If timeout value is invalid do not set a timeout.
test_timeout = 0
if test_timeout > 0:
self.useFixture(fixtures.Timeout(test_timeout, gentle=True))
self.useFixture(fixtures.NestedTempfile())
self.useFixture(fixtures.TempHomeDir())
if (os.environ.get('OS_STDOUT_CAPTURE') == 'True'
or os.environ.get('OS_STDOUT_CAPTURE') == '1'):
stdout = self.useFixture(fixtures.StringStream('stdout')).stream
self.useFixture(fixtures.MonkeyPatch('sys.stdout', stdout))
if (os.environ.get('OS_STDERR_CAPTURE') == 'True'
or os.environ.get('OS_STDERR_CAPTURE') == '1'):
stderr = self.useFixture(fixtures.StringStream('stderr')).stream
self.useFixture(fixtures.MonkeyPatch('sys.stderr', stderr))
self.log_fixture = self.useFixture(
fixtures.FakeLogger(level=logging.DEBUG))
rpc.add_extra_exmods("cinder.tests")
self.addCleanup(rpc.clear_extra_exmods)
self.addCleanup(rpc.cleanup)
self.messaging_conf = messaging_conffixture.ConfFixture(CONF)
self.messaging_conf.transport_driver = 'fake'
self.messaging_conf.response_timeout = 15
self.useFixture(self.messaging_conf)
rpc.init(CONF)
conf_fixture.set_defaults(CONF)
CONF([], default_config_files=[])
# NOTE(vish): We need a better method for creating fixtures for tests
# now that we have some required db setup for the system
# to work properly.
self.start = timeutils.utcnow()
CONF.set_default('connection', 'sqlite://', 'database')
CONF.set_default('sqlite_synchronous', False)
self.log_fixture = self.useFixture(fixtures.FakeLogger())
global _DB_CACHE
if not _DB_CACHE:
_DB_CACHE = Database(session,
migration,
sql_connection=CONF.database.connection,
sqlite_db=CONF.sqlite_db,
sqlite_clean_db=CONF.sqlite_clean_db)
self.useFixture(_DB_CACHE)
# emulate some of the mox stuff, we can't use the metaclass
# because it screws with our generators
self.mox = mox.Mox()
self.stubs = stubout.StubOutForTesting()
self.addCleanup(CONF.reset)
self.addCleanup(self.mox.UnsetStubs)
self.addCleanup(self.stubs.UnsetAll)
self.addCleanup(self.stubs.SmartUnsetAll)
self.addCleanup(self.mox.VerifyAll)
self.addCleanup(self._common_cleanup)
self.injected = []
self._services = []
fake_notifier.stub_notifier(self.stubs)
CONF.set_override('fatal_exception_format_errors', True)
# This will be cleaned up by the NestedTempfile fixture
CONF.set_override('lock_path', tempfile.mkdtemp())
def setUp(self):
"""Run before each test method to initialize test environment."""
super(TestCase, self).setUp()
# Create default notifier
self.notifier = fake_notifier.get_fake_notifier()
# Mock rpc get notifier with fake notifier method that joins all
# notifications with the default notifier
self.patch('cinder.rpc.get_notifier',
side_effect=self._get_joined_notifier)
if self.MOCK_WORKER:
# Mock worker creation for all tests that don't care about it
clean_path = 'cinder.objects.cleanable.CinderCleanableObject.%s'
for method in ('create_worker', 'set_worker', 'unset_worker'):
self.patch(clean_path % method, return_value=None)
if self.MOCK_TOOZ:
self.patch('cinder.coordination.Coordinator.start')
self.patch('cinder.coordination.Coordinator.stop')
self.patch('cinder.coordination.Coordinator.get_lock')
# Unit tests do not need to use lazy gettext
i18n.enable_lazy(False)
test_timeout = os.environ.get('OS_TEST_TIMEOUT', 0)
try:
test_timeout = int(test_timeout)
except ValueError:
# If timeout value is invalid do not set a timeout.
test_timeout = 0
if test_timeout > 0:
self.useFixture(fixtures.Timeout(test_timeout, gentle=True))
self.useFixture(fixtures.NestedTempfile())
self.useFixture(fixtures.TempHomeDir())
environ_enabled = (lambda var_name: strutils.bool_from_string(
os.environ.get(var_name)))
if environ_enabled('OS_STDOUT_CAPTURE'):
stdout = self.useFixture(fixtures.StringStream('stdout')).stream
self.useFixture(fixtures.MonkeyPatch('sys.stdout', stdout))
if environ_enabled('OS_STDERR_CAPTURE'):
stderr = self.useFixture(fixtures.StringStream('stderr')).stream
self.useFixture(fixtures.MonkeyPatch('sys.stderr', stderr))
self.useFixture(log_fixture.get_logging_handle_error_fixture())
self.useFixture(cinder_fixtures.StandardLogging())
rpc.add_extra_exmods("cinder.tests.unit")
self.addCleanup(rpc.clear_extra_exmods)
self.addCleanup(rpc.cleanup)
self.messaging_conf = messaging_conffixture.ConfFixture(CONF)
self.messaging_conf.transport_url = 'fake:/'
self.messaging_conf.response_timeout = 15
self.useFixture(self.messaging_conf)
# Load oslo_messaging_notifications config group so we can set an
# override to prevent notifications from being ignored due to the
# short-circuit mechanism.
oslo_messaging.get_notification_transport(CONF)
# We need to use a valid driver for the notifications, so we use test.
self.override_config('driver', ['test'],
group='oslo_messaging_notifications')
rpc.init(CONF)
# NOTE(geguileo): This is required because _determine_obj_version_cap
# and _determine_rpc_version_cap functions in cinder.rpc.RPCAPI cache
# versions in LAST_RPC_VERSIONS and LAST_OBJ_VERSIONS so we may have
# weird interactions between tests if we don't clear them before each
# test.
rpc.LAST_OBJ_VERSIONS = {}
rpc.LAST_RPC_VERSIONS = {}
# Init AuthProtocol to register some base options first, such as
# auth_url.
auth_token.AuthProtocol('fake_app', {
'auth_type': 'password',
'auth_url': 'fake_url'
})
conf_fixture.set_defaults(CONF)
CONF([], default_config_files=[])
# NOTE(vish): We need a better method for creating fixtures for tests
# now that we have some required db setup for the system
# to work properly.
self.start = timeutils.utcnow()
CONF.set_default('connection', 'sqlite://', 'database')
CONF.set_default('sqlite_synchronous', False, 'database')
global _DB_CACHE
if not _DB_CACHE:
_DB_CACHE = Database(sqla_api,
migration,
sql_connection=CONF.database.connection)
self.useFixture(_DB_CACHE)
# NOTE(blk-u): WarningsFixture must be after the Database fixture
# because sqlalchemy-migrate messes with the warnings filters.
self.useFixture(cinder_fixtures.WarningsFixture())
# NOTE(danms): Make sure to reset us back to non-remote objects
# for each test to avoid interactions. Also, backup the object
# registry.
objects_base.CinderObject.indirection_api = None
self._base_test_obj_backup = copy.copy(
objects_base.CinderObjectRegistry._registry._obj_classes)
self.addCleanup(self._restore_obj_registry)
self.addCleanup(CONF.reset)
self.addCleanup(self._common_cleanup)
self.injected = []
self._services = []
fake_notifier.mock_notifier(self)
# This will be cleaned up by the NestedTempfile fixture
lock_path = self.useFixture(fixtures.TempDir()).path
self.fixture = self.useFixture(config_fixture.Config(lockutils.CONF))
self.fixture.config(lock_path=lock_path, group='oslo_concurrency')
lockutils.set_defaults(lock_path)
self.override_config('policy_file',
os.path.join(
os.path.abspath(
os.path.join(
os.path.dirname(__file__),
'..',
)), self.POLICY_PATH),
group='oslo_policy')
self.override_config(
'resource_query_filters_file',
os.path.join(
os.path.abspath(os.path.join(
os.path.dirname(__file__),
'..',
)), self.RESOURCE_FILTER_PATH))
self._disable_osprofiler()
# NOTE(geguileo): This is required because common get_by_id method in
# cinder.db.sqlalchemy.api caches get methods and if we use a mocked
# get method in one test it would carry on to the next test. So we
# clear out the cache.
sqla_api._GET_METHODS = {}
self.override_config('backend_url',
'file://' + lock_path,
group='coordination')
coordination.COORDINATOR.start()
self.addCleanup(coordination.COORDINATOR.stop)
if six.PY3:
# TODO(smcginnis) Python 3 deprecates assertRaisesRegexp to
# assertRaisesRegex, but Python 2 does not have the new name. This
# can be removed once we stop supporting py2 or the new name is
# added.
self.assertRaisesRegexp = self.assertRaisesRegex
# Ensure we have the default tpool size value and we don't carry
# threads from other test runs.
tpool.killall()
tpool._nthreads = 20
# NOTE(mikal): make sure we don't load a privsep helper accidentally
self.useFixture(cinder_fixtures.PrivsepNoHelperFixture())
def main():
objects.register_all()
admin_context = context.get_admin_context()
CONF(sys.argv[1:], project='cinder', version=version.version_string())
logging.setup(CONF, "cinder")
LOG = logging.getLogger("cinder")
rpc.init(CONF)
begin, end = utils.last_completed_audit_period()
if CONF.start_time:
begin = datetime.datetime.strptime(CONF.start_time,
"%Y-%m-%d %H:%M:%S")
if CONF.end_time:
end = datetime.datetime.strptime(CONF.end_time, "%Y-%m-%d %H:%M:%S")
if not end > begin:
msg = _("The end time (%(end)s) must be after the start "
"time (%(start)s).") % {
'start': begin,
'end': end
}
LOG.error(msg)
sys.exit(-1)
LOG.debug("Starting volume usage audit")
msg = _("Creating usages for %(begin_period)s until %(end_period)s")
LOG.debug(msg, {"begin_period": str(begin), "end_period": str(end)})
extra_info = {
'audit_period_beginning': str(begin),
'audit_period_ending': str(end),
}
volumes = db.volume_get_active_by_window(admin_context, begin, end)
LOG.debug("Found %d volumes", len(volumes))
for volume_ref in volumes:
try:
LOG.debug(
"Send exists notification for <volume_id: "
"%(volume_id)s> <project_id %(project_id)s> "
"<%(extra_info)s>", {
'volume_id': volume_ref.id,
'project_id': volume_ref.project_id,
'extra_info': extra_info
})
cinder.volume.utils.notify_about_volume_usage(
admin_context,
volume_ref,
'exists',
extra_usage_info=extra_info)
except Exception as exc_msg:
LOG.exception(_LE("Exists volume notification failed: %s"),
exc_msg,
resource=volume_ref)
if (CONF.send_actions and volume_ref.created_at > begin
and volume_ref.created_at < end):
try:
local_extra_info = {
'audit_period_beginning': str(volume_ref.created_at),
'audit_period_ending': str(volume_ref.created_at),
}
LOG.debug(
"Send create notification for "
"<volume_id: %(volume_id)s> "
"<project_id %(project_id)s> <%(extra_info)s>", {
'volume_id': volume_ref.id,
'project_id': volume_ref.project_id,
'extra_info': local_extra_info
})
cinder.volume.utils.notify_about_volume_usage(
admin_context,
volume_ref,
'create.start',
extra_usage_info=local_extra_info)
cinder.volume.utils.notify_about_volume_usage(
admin_context,
volume_ref,
'create.end',
extra_usage_info=local_extra_info)
except Exception as exc_msg:
LOG.exception(_LE("Create volume notification failed: %s"),
exc_msg,
resource=volume_ref)
if (CONF.send_actions and volume_ref.deleted_at
and volume_ref.deleted_at > begin
and volume_ref.deleted_at < end):
try:
local_extra_info = {
'audit_period_beginning': str(volume_ref.deleted_at),
'audit_period_ending': str(volume_ref.deleted_at),
}
LOG.debug(
"Send delete notification for "
"<volume_id: %(volume_id)s> "
"<project_id %(project_id)s> <%(extra_info)s>", {
'volume_id': volume_ref.id,
'project_id': volume_ref.project_id,
'extra_info': local_extra_info
})
cinder.volume.utils.notify_about_volume_usage(
admin_context,
volume_ref,
'delete.start',
extra_usage_info=local_extra_info)
cinder.volume.utils.notify_about_volume_usage(
admin_context,
volume_ref,
'delete.end',
extra_usage_info=local_extra_info)
except Exception as exc_msg:
LOG.exception(_LE("Delete volume notification failed: %s"),
exc_msg,
resource=volume_ref)
snapshots = objects.SnapshotList.get_active_by_window(
admin_context, begin, end)
LOG.debug("Found %d snapshots", len(snapshots))
for snapshot_ref in snapshots:
try:
LOG.debug(
"Send notification for <snapshot_id: %(snapshot_id)s> "
"<project_id %(project_id)s> <%(extra_info)s>", {
'snapshot_id': snapshot_ref.id,
'project_id': snapshot_ref.project_id,
'extra_info': extra_info
})
cinder.volume.utils.notify_about_snapshot_usage(
admin_context, snapshot_ref, 'exists', extra_info)
except Exception as exc_msg:
LOG.exception(_LE("Exists snapshot notification failed: %s"),
exc_msg,
resource=snapshot_ref)
if (CONF.send_actions and snapshot_ref.created_at > begin
and snapshot_ref.created_at < end):
try:
local_extra_info = {
'audit_period_beginning': str(snapshot_ref.created_at),
'audit_period_ending': str(snapshot_ref.created_at),
}
LOG.debug(
"Send create notification for "
"<snapshot_id: %(snapshot_id)s> "
"<project_id %(project_id)s> <%(extra_info)s>", {
'snapshot_id': snapshot_ref.id,
'project_id': snapshot_ref.project_id,
'extra_info': local_extra_info
})
cinder.volume.utils.notify_about_snapshot_usage(
admin_context,
snapshot_ref,
'create.start',
extra_usage_info=local_extra_info)
cinder.volume.utils.notify_about_snapshot_usage(
admin_context,
snapshot_ref,
'create.end',
extra_usage_info=local_extra_info)
except Exception as exc_msg:
LOG.exception(_LE("Create snapshot notification failed: %s"),
exc_msg,
resource=snapshot_ref)
if (CONF.send_actions and snapshot_ref.deleted_at
and snapshot_ref.deleted_at > begin
and snapshot_ref.deleted_at < end):
try:
local_extra_info = {
'audit_period_beginning': str(snapshot_ref.deleted_at),
'audit_period_ending': str(snapshot_ref.deleted_at),
}
LOG.debug(
"Send delete notification for "
"<snapshot_id: %(snapshot_id)s> "
"<project_id %(project_id)s> <%(extra_info)s>", {
'snapshot_id': snapshot_ref.id,
'project_id': snapshot_ref.project_id,
'extra_info': local_extra_info
})
cinder.volume.utils.notify_about_snapshot_usage(
admin_context,
snapshot_ref,
'delete.start',
extra_usage_info=local_extra_info)
cinder.volume.utils.notify_about_snapshot_usage(
admin_context,
snapshot_ref,
'delete.end',
extra_usage_info=local_extra_info)
except Exception as exc_msg:
LOG.exception(_LE("Delete snapshot notification failed: %s"),
exc_msg,
resource=snapshot_ref)
LOG.debug("Volume usage audit completed")
def setUp(self):
"""Run before each test method to initialize test environment."""
super(TestCase, self).setUp()
# Create default notifier
self.notifier = fake_notifier.get_fake_notifier()
# Mock rpc get notifier with fake notifier method that joins all
# notifications with the default notifier
p = mock.patch('cinder.rpc.get_notifier',
side_effect=self._get_joined_notifier)
p.start()
# Unit tests do not need to use lazy gettext
i18n.enable_lazy(False)
test_timeout = os.environ.get('OS_TEST_TIMEOUT', 0)
try:
test_timeout = int(test_timeout)
except ValueError:
# If timeout value is invalid do not set a timeout.
test_timeout = 0
if test_timeout > 0:
self.useFixture(fixtures.Timeout(test_timeout, gentle=True))
self.useFixture(fixtures.NestedTempfile())
self.useFixture(fixtures.TempHomeDir())
environ_enabled = (lambda var_name:
strutils.bool_from_string(os.environ.get(var_name)))
if environ_enabled('OS_STDOUT_CAPTURE'):
stdout = self.useFixture(fixtures.StringStream('stdout')).stream
self.useFixture(fixtures.MonkeyPatch('sys.stdout', stdout))
if environ_enabled('OS_STDERR_CAPTURE'):
stderr = self.useFixture(fixtures.StringStream('stderr')).stream
self.useFixture(fixtures.MonkeyPatch('sys.stderr', stderr))
self.useFixture(log_fixture.get_logging_handle_error_fixture())
self.useFixture(cinder_fixtures.StandardLogging())
rpc.add_extra_exmods("cinder.tests.unit")
self.addCleanup(rpc.clear_extra_exmods)
self.addCleanup(rpc.cleanup)
self.messaging_conf = messaging_conffixture.ConfFixture(CONF)
self.messaging_conf.transport_driver = 'fake'
self.messaging_conf.response_timeout = 15
self.useFixture(self.messaging_conf)
rpc.init(CONF)
# NOTE(geguileo): This is required because _determine_obj_version_cap
# and _determine_rpc_version_cap functions in cinder.rpc.RPCAPI cache
# versions in LAST_RPC_VERSIONS and LAST_OBJ_VERSIONS so we may have
# weird interactions between tests if we don't clear them before each
# test.
rpc.LAST_OBJ_VERSIONS = {}
rpc.LAST_RPC_VERSIONS = {}
conf_fixture.set_defaults(CONF)
CONF([], default_config_files=[])
# NOTE(vish): We need a better method for creating fixtures for tests
# now that we have some required db setup for the system
# to work properly.
self.start = timeutils.utcnow()
CONF.set_default('connection', 'sqlite://', 'database')
CONF.set_default('sqlite_synchronous', False, 'database')
global _DB_CACHE
if not _DB_CACHE:
_DB_CACHE = Database(sqla_api, migration,
sql_connection=CONF.database.connection,
sqlite_db=CONF.database.sqlite_db,
sqlite_clean_db='clean.sqlite')
self.useFixture(_DB_CACHE)
# NOTE(danms): Make sure to reset us back to non-remote objects
# for each test to avoid interactions. Also, backup the object
# registry.
objects_base.CinderObject.indirection_api = None
self._base_test_obj_backup = copy.copy(
objects_base.CinderObjectRegistry._registry._obj_classes)
self.addCleanup(self._restore_obj_registry)
# emulate some of the mox stuff, we can't use the metaclass
# because it screws with our generators
mox_fixture = self.useFixture(moxstubout.MoxStubout())
self.mox = mox_fixture.mox
self.stubs = mox_fixture.stubs
self.addCleanup(CONF.reset)
self.addCleanup(self._common_cleanup)
self.injected = []
self._services = []
fake_notifier.stub_notifier(self.stubs)
self.override_config('fatal_exception_format_errors', True)
# This will be cleaned up by the NestedTempfile fixture
lock_path = self.useFixture(fixtures.TempDir()).path
self.fixture = self.useFixture(
config_fixture.Config(lockutils.CONF))
self.fixture.config(lock_path=lock_path,
group='oslo_concurrency')
lockutils.set_defaults(lock_path)
self.override_config('policy_file',
os.path.join(
os.path.abspath(
os.path.join(
os.path.dirname(__file__),
'..',
)
),
'cinder/tests/unit/policy.json'),
group='oslo_policy')
self._disable_osprofiler()
# NOTE(geguileo): This is required because common get_by_id method in
# cinder.db.sqlalchemy.api caches get methods and if we use a mocked
# get method in one test it would carry on to the next test. So we
# clear out the cache.
sqla_api._GET_METHODS = {}
def main():
objects.register_all()
gmr_opts.set_defaults(CONF)
CONF(sys.argv[1:], project='cinder', version=version.version_string())
config.set_middleware_defaults()
logging.setup(CONF, "cinder")
LOG = logging.getLogger('cinder.all')
versionutils.report_deprecated_feature(
LOG,
_('cinder-all is deprecated in Newton and will be removed in Ocata.'))
utils.monkey_patch()
gmr.TextGuruMeditation.setup_autorun(version, conf=CONF)
rpc.init(CONF)
launcher = service.process_launcher()
# cinder-api
try:
server = service.WSGIService('osapi_volume')
launcher.launch_service(server, workers=server.workers or 1)
except (Exception, SystemExit):
LOG.exception(_LE('Failed to load osapi_volume'))
for binary in ['cinder-scheduler', 'cinder-backup']:
try:
launcher.launch_service(service.Service.create(binary=binary))
except (Exception, SystemExit):
LOG.exception(_LE('Failed to load %s'), binary)
# cinder-volume
try:
if CONF.enabled_backends:
for backend in filter(None, CONF.enabled_backends):
CONF.register_opt(volume_cmd.host_opt, group=backend)
backend_host = getattr(CONF, backend).backend_host
host = "%s@%s" % (backend_host or CONF.host, backend)
server = service.Service.create(host=host,
service_name=backend,
binary='cinder-volume',
coordination=True)
# Dispose of the whole DB connection pool here before
# starting another process. Otherwise we run into cases
# where child processes share DB connections which results
# in errors.
session.dispose_engine()
launcher.launch_service(server)
else:
LOG.warning(
_LW('Configuration for cinder-volume does not specify '
'"enabled_backends", using DEFAULT as backend. '
'Support for DEFAULT section to configure drivers '
'will be removed in the next release.'))
server = service.Service.create(binary='cinder-volume',
coordination=True)
launcher.launch_service(server)
except (Exception, SystemExit):
LOG.exception(_LE('Failed to load cinder-volume'))
launcher.wait()
