def _assess_db_object_and_api_performance(mock_log, mock_request):
print('Phase - Assess DB & Object conversion Performance')
_add_a_line()
# Just mock it to silence it since getting the logger to update
# config seems like not a thing once started. :\
mock_log.debug = mock.Mock()
# Internal logic requires major/minor versions and a context to
# proceed. This is just to make the NodesController respond properly.
mock_request.context = context.get_admin_context()
mock_request.version.major = 1
mock_request.version.minor = 71
start = time.time()
node_api_controller = node_api.NodesController()
node_api_controller.context = context.get_admin_context()
fields = ("uuid,power_state,target_power_state,provision_state,"
"target_provision_state,last_error,maintenance,properties,"
"instance_uuid,traits,resource_class")
total_nodes = 0
res = node_api_controller._get_nodes_collection(
chassis_uuid=None,
instance_uuid=None,
associated=None,
maintenance=None,
retired=None,
provision_state=None,
marker=None,
limit=None,
sort_key="id",
sort_dir="asc",
fields=fields.split(','))
total_nodes = len(res['nodes'])
while len(res['nodes']) != 1:
print(" ** Getting nodes ** %s Elapsed: %s seconds." %
(total_nodes, _calculate_delta(start, time.time())))
res = node_api_controller._get_nodes_collection(
chassis_uuid=None,
instance_uuid=None,
associated=None,
maintenance=None,
retired=None,
provision_state=None,
marker=res['nodes'][-1]['uuid'],
limit=None,
sort_key="id",
sort_dir="asc",
fields=fields.split(','))
new_nodes = len(res['nodes'])
if new_nodes == 0:
break
total_nodes = total_nodes + new_nodes
delta = _calculate_delta(start, time.time())
print('Took %s seconds to return all %s nodes via '
'nodes API call pattern.\n' % (delta, total_nodes))
def run(self):
while not self.stop:
try:
# Set timeout to check self.stop periodically
(node_id, params) = QUEUE.get(block=True,
timeout=self.queue_timeout)
except Queue.Empty:
pass
else:
# Requests comes here from BareMetalDeploy.post()
LOG.info(_('start deployment for node %(node_id)s, '
'params %(params)s') %
{'node_id': node_id, 'params': params})
context = ironic_context.get_admin_context()
try:
db.bm_node_update(context, node_id,
{'task_state': states.DEPLOYING})
deploy(**params)
except Exception:
LOG.error(_('deployment to node %s failed') % node_id)
db.bm_node_update(context, node_id,
{'task_state': states.DEPLOYFAIL})
else:
LOG.info(_('deployment to node %s done') % node_id)
db.bm_node_update(context, node_id,
{'task_state': states.DEPLOYDONE})
def start(self):
super(RPCService, self).start()
admin_context = context.get_admin_context()
serializer = objects_base.IronicObjectSerializer(is_server=True)
# Perform preparatory actions before starting the RPC listener
self.manager.prepare_host()
if CONF.rpc_transport == 'json-rpc':
self.rpcserver = json_rpc.WSGIService(
self.manager, serializer, context.RequestContext.from_dict)
else:
target = messaging.Target(topic=self.topic, server=self.host)
endpoints = [self.manager]
self.rpcserver = rpc.get_server(target, endpoints, serializer)
self.rpcserver.start()
self.handle_signal()
self.manager.init_host(admin_context)
LOG.info(
'Created RPC server for service %(service)s on host '
'%(host)s.', {
'service': self.topic,
'host': self.host
})
def attach_volume(self, connection_info, instance, mountpoint):
node = _get_baremetal_node_by_instance_uuid(instance['uuid'])
ctx = nova_context.get_admin_context()
pxe_ip = db.bm_pxe_ip_get_by_bm_node_id(ctx, node['id'])
if not pxe_ip:
if not CONF.use_unsafe_iscsi:
raise exception.NovaException(_(
'No fixed PXE IP is associated to %s') % instance['uuid'])
mount_device = mountpoint.rpartition("/")[2]
self._volume_driver_method('connect_volume',
connection_info,
mount_device)
device_path = connection_info['data']['device_path']
iqn = _get_iqn(instance['name'], mountpoint)
tid = _get_next_tid()
_create_iscsi_export_tgtadm(device_path, tid, iqn)
if pxe_ip:
_allow_iscsi_tgtadm(tid, pxe_ip['address'])
else:
# NOTE(NTTdocomo): Since nova-compute does not know the
# instance's initiator ip, it allows any initiators
# to connect to the volume. This means other bare-metal
# instances that are not attached the volume can connect
# to the volume. Do not set CONF.use_unsafe_iscsi
# out of dev/test environments.
# TODO(NTTdocomo): support CHAP
_allow_iscsi_tgtadm(tid, 'ALL')
def _assess_db_and_object_performance():
print('Phase - Assess DB & Object conversion Performance')
_add_a_line()
start = time.time()
node_list = node.Node().list(context.get_admin_context())
got_list = time.time()
delta = _calculate_delta(start, got_list)
print('Obtained list of node objects in %s seconds.' % delta)
count = 0
tbl_size = 0
# In a sense, this helps provide a relative understanding if the
# database is the bottleneck, or the objects post conversion.
# converting completely to json and then measuring the size helps
# ensure that everything is "assessed" while not revealing too
# much detail.
for node_obj in node_list:
# Just looping through the entire set to count should be
# enough to ensure that the entry is loaded from the db
# and then converted to an object.
tbl_size = tbl_size + sys.getsizeof(node_obj.as_dict(secure=True))
count = count + 1
delta = _calculate_delta(got_list, time.time())
print('Took %s seconds to iterate through %s node objects.' %
(delta, count))
print('Nodes table is roughly %s bytes of JSON.\n' % tbl_size)
observed_vendors = []
for node_obj in node_list:
vendor = node_obj.driver_internal_info.get('vendor')
if vendor:
observed_vendors.append(vendor)
def setUp(self):
super(TestPatch, self).setUp()
ndict = dbutils.get_test_node()
self.context = context.get_admin_context()
self.node = self.dbapi.create_node(ndict)
self.mox.StubOutWithMock(rpcapi.ConductorAPI, 'update_node')
self.mox.StubOutWithMock(rpcapi.ConductorAPI, 'start_state_change')
def setUp(self):
super(BackfillVersionTestCase, self).setUp()
self.context = context.get_admin_context()
self.dbapi = db_api.get_instance()
obj_mapping = release_mappings.RELEASE_MAPPING['ocata']['objects']
self.node_ver = obj_mapping['Node'][0]
self.chassis_ver = obj_mapping['Chassis'][0]
def _report_conductors():
print('Phase - identifying conductors/drivers')
_add_a_line()
conductors = conductor.Conductor().list(
context.get_admin_context(),
)
drivers = []
groups = []
online_count = 0
online_by = timeutils.utcnow(with_timezone=True) - \
datetime.timedelta(seconds=90)
for conductor_obj in conductors:
if conductor_obj.conductor_group:
groups.append(conductor_obj.conductor_group)
if conductor_obj.updated_at > online_by:
online_count = online_count + 1
for driver in conductor_obj.drivers:
drivers.append(driver)
conductor_count = len(conductors)
print('Conductor count: %s' % conductor_count)
print('Online conductor count: %s' % online_count)
running_with_groups = len(groups)
print('Conductors with conductor_groups: %s' % running_with_groups)
group_count = len(set(groups))
print('Conductor group count: %s' % group_count)
driver_list = list(set(drivers))
print('Presently supported drivers: %s' % driver_list)
def attach_volume(self, connection_info, instance, mountpoint):
node = _get_baremetal_node_by_instance_uuid(instance['uuid'])
ctx = nova_context.get_admin_context()
pxe_ip = db.bm_pxe_ip_get_by_bm_node_id(ctx, node['id'])
if not pxe_ip:
if not CONF.use_unsafe_iscsi:
raise exception.NovaException(
_('No fixed PXE IP is associated to %s') %
instance['uuid'])
mount_device = mountpoint.rpartition("/")[2]
self._volume_driver_method('connect_volume', connection_info,
mount_device)
device_path = connection_info['data']['device_path']
iqn = _get_iqn(instance['name'], mountpoint)
tid = _get_next_tid()
_create_iscsi_export_tgtadm(device_path, tid, iqn)
if pxe_ip:
_allow_iscsi_tgtadm(tid, pxe_ip['address'])
else:
# NOTE(NTTdocomo): Since nova-compute does not know the
# instance's initiator ip, it allows any initiators
# to connect to the volume. This means other bare-metal
# instances that are not attached the volume can connect
# to the volume. Do not set CONF.use_unsafe_iscsi
# out of dev/test environments.
# TODO(NTTdocomo): support CHAP
_allow_iscsi_tgtadm(tid, 'ALL')
def run(self):
while not self.stop:
try:
# Set timeout to check self.stop periodically
(node_id, params) = QUEUE.get(block=True,
timeout=self.queue_timeout)
except Queue.Empty:
pass
else:
# Requests comes here from BareMetalDeploy.post()
LOG.info(_('start deployment for node %(node_id)s, '
'params %(params)s') % locals())
context = ironic_context.get_admin_context()
try:
db.bm_node_update(context, node_id,
{'task_state': states.DEPLOYING})
deploy(**params)
except Exception:
LOG.error(_('deployment to node %s failed') % node_id)
db.bm_node_update(context, node_id,
{'task_state': states.DEPLOYFAIL})
else:
LOG.info(_('deployment to node %s done') % node_id)
db.bm_node_update(context, node_id,
{'task_state': states.DEPLOYDONE})
def plug(self, instance, vif):
LOG.debug(
_("plug: instance_uuid=%(uuid)s vif=%(vif)s") % {
'uuid': instance['uuid'],
'vif': vif
})
network, mapping = vif
vif_uuid = mapping['vif_uuid']
ctx = nova_context.get_admin_context()
node = db.bm_node_get_by_instance_uuid(ctx, instance['uuid'])
# TODO(deva): optimize this database query
# this is just searching for a free physical interface
pifs = db.bm_interface_get_all_by_bm_node_id(ctx, node['id'])
for pif in pifs:
if not pif['vif_uuid']:
db.bm_interface_set_vif_uuid(ctx, pif['id'], vif_uuid)
LOG.debug(
_("pif:%(id)s is plugged (vif_uuid=%(vif_uuid)s)") % {
'id': pif['id'],
'vif_uuid': vif_uuid
})
self._after_plug(instance, network, mapping, pif)
return
# NOTE(deva): should this really be raising an exception
# when there are no physical interfaces left?
raise exception.NovaException(
_("Baremetal node: %(id)s has no available physical interface"
" for virtual interface %(vif_uuid)s") % {
'id': node['id'],
'vif_uuid': vif_uuid
})
def plug(self, instance, vif):
LOG.debug(_("plug: instance_uuid=%(uuid)s vif=%(vif)s")
% {'uuid': instance['uuid'], 'vif': vif})
network, mapping = vif
vif_uuid = mapping['vif_uuid']
ctx = nova_context.get_admin_context()
node = db.bm_node_get_by_instance_uuid(ctx, instance['uuid'])
# TODO(deva): optimize this database query
# this is just searching for a free physical interface
pifs = db.bm_interface_get_all_by_bm_node_id(ctx, node['id'])
for pif in pifs:
if not pif['vif_uuid']:
db.bm_interface_set_vif_uuid(ctx, pif['id'], vif_uuid)
LOG.debug(_("pif:%(id)s is plugged (vif_uuid=%(vif_uuid)s)")
% {'id': pif['id'], 'vif_uuid': vif_uuid})
self._after_plug(instance, network, mapping, pif)
return
# NOTE(deva): should this really be raising an exception
# when there are no physical interfaces left?
raise exception.NovaException(_(
"Baremetal node: %(id)s has no available physical interface"
" for virtual interface %(vif_uuid)s")
% {'id': node['id'], 'vif_uuid': vif_uuid})
def setUp(self):
"""Run before each test method to initialize test environment."""
super(TestCase, self).setUp()
self.context = ironic_context.get_admin_context()
test_timeout = os.environ.get('OS_TEST_TIMEOUT', 0)
try:
test_timeout = int(test_timeout)
except ValueError:
# If timeout value is invalid do not set a timeout.
test_timeout = 0
if test_timeout > 0:
self.useFixture(fixtures.Timeout(test_timeout, gentle=True))
self.useFixture(fixtures.NestedTempfile())
self.useFixture(fixtures.TempHomeDir())
if (os.environ.get('OS_STDOUT_CAPTURE') == 'True' or
os.environ.get('OS_STDOUT_CAPTURE') == '1'):
stdout = self.useFixture(fixtures.StringStream('stdout')).stream
self.useFixture(fixtures.MonkeyPatch('sys.stdout', stdout))
if (os.environ.get('OS_STDERR_CAPTURE') == 'True' or
os.environ.get('OS_STDERR_CAPTURE') == '1'):
stderr = self.useFixture(fixtures.StringStream('stderr')).stream
self.useFixture(fixtures.MonkeyPatch('sys.stderr', stderr))
self.log_fixture = self.useFixture(fixtures.FakeLogger())
self._set_config()
# NOTE(danms): Make sure to reset us back to non-remote objects
# for each test to avoid interactions. Also, backup the object
# registry
objects_base.IronicObject.indirection_api = None
self._base_test_obj_backup = copy.copy(
objects_base.IronicObjectRegistry.obj_classes())
self.addCleanup(self._restore_obj_registry)
self.addCleanup(self._clear_attrs)
self.addCleanup(hash_ring.HashRingManager().reset)
self.useFixture(fixtures.EnvironmentVariable('http_proxy'))
self.policy = self.useFixture(policy_fixture.PolicyFixture())
driver_factory.DriverFactory._extension_manager = None
driver_factory.HardwareTypesFactory._extension_manager = None
for factory in driver_factory._INTERFACE_LOADERS.values():
factory._extension_manager = None
# Block access to utils.execute() and related functions.
# NOTE(bigjools): Not using a decorator on tests because I don't
# want to force every test method to accept a new arg. Instead, they
# can override or examine this self._exec_patch Mock as needed.
if self.block_execute:
self._exec_patch = mock.Mock()
self._exec_patch.side_effect = Exception(
"Don't call ironic_lib.utils.execute() / "
"processutils.execute() or similar functions in tests!")
self.patch(processutils, 'execute', self._exec_patch)
self.patch(subprocess, 'Popen', self._exec_patch)
self.patch(subprocess, 'call', self._exec_patch)
self.patch(subprocess, 'check_call', self._exec_patch)
self.patch(subprocess, 'check_output', self._exec_patch)
self.patch(utils, 'execute', self._exec_patch)
def _run_online_data_migrations(self, max_count=None, options=None):
"""Perform online data migrations for the release.
Online data migrations are done by running all the data migration
functions in the ONLINE_MIGRATIONS list. If max_count is None, all
the functions will be run in batches of 50 objects, until the
migrations are done. Otherwise, this will run (some of) the functions
until max_count objects have been migrated.
:param: max_count: the maximum number of individual object migrations
or modified rows, a value >= 1. If None, migrations are run in a
loop in batches of 50, until completion.
:param: options: options to pass to migrations. List of values in the
form of <migration name>.<option>=<value>
:raises: SystemExit. With exit code of:
0: when all migrations are complete.
1: when objects were migrated and the command needs to be
re-run (because there might be more objects to be migrated)
127: if max_count is < 1 or any option is invalid
:raises: Exception from a migration function
"""
parsed_options = {}
if options:
for option in options:
try:
migration, key_value = option.split('.', 1)
key, value = key_value.split('=', 1)
except ValueError:
print(_("Malformed option %s") % option)
sys.exit(127)
else:
parsed_options.setdefault(migration, {})[key] = value
admin_context = context.get_admin_context()
finished_migrating = False
if max_count is None:
max_count = 50
print(
_('Running batches of %i until migrations have been '
'completed.') % max_count)
while not finished_migrating:
finished_migrating = self._run_migration_functions(
admin_context, max_count, parsed_options)
print(_('Data migrations have completed.'))
sys.exit(0)
if max_count < 1:
print(_('"max-count" must be a positive value.'), file=sys.stderr)
sys.exit(127)
finished_migrating = self._run_migration_functions(
admin_context, max_count, parsed_options)
if finished_migrating:
print(_('Data migrations have completed.'))
sys.exit(0)
else:
print(_('Data migrations have not completed. Please re-run.'))
sys.exit(1)
def _run_online_data_migrations(self, max_count=None, options=None):
"""Perform online data migrations for the release.
Online data migrations are done by running all the data migration
functions in the ONLINE_MIGRATIONS list. If max_count is None, all
the functions will be run in batches of 50 objects, until the
migrations are done. Otherwise, this will run (some of) the functions
until max_count objects have been migrated.
:param: max_count: the maximum number of individual object migrations
or modified rows, a value >= 1. If None, migrations are run in a
loop in batches of 50, until completion.
:param: options: options to pass to migrations. List of values in the
form of <migration name>.<option>=<value>
:raises: SystemExit. With exit code of:
0: when all migrations are complete.
1: when objects were migrated and the command needs to be
re-run (because there might be more objects to be migrated)
127: if max_count is < 1 or any option is invalid
:raises: Exception from a migration function
"""
parsed_options = {}
if options:
for option in options:
try:
migration, key_value = option.split('.', 1)
key, value = key_value.split('=', 1)
except ValueError:
print(_("Malformed option %s") % option)
sys.exit(127)
else:
parsed_options.setdefault(migration, {})[key] = value
admin_context = context.get_admin_context()
finished_migrating = False
if max_count is None:
max_count = 50
print(_('Running batches of %i until migrations have been '
'completed.') % max_count)
while not finished_migrating:
finished_migrating = self._run_migration_functions(
admin_context, max_count, parsed_options)
print(_('Data migrations have completed.'))
sys.exit(0)
if max_count < 1:
print(_('"max-count" must be a positive value.'), file=sys.stderr)
sys.exit(127)
finished_migrating = self._run_migration_functions(admin_context,
max_count,
parsed_options)
if finished_migrating:
print(_('Data migrations have completed.'))
sys.exit(0)
else:
print(_('Data migrations have not completed. Please re-run.'))
sys.exit(1)
def test_load(self):
ctxt = context.get_admin_context()
uuid = self.fake_port["uuid"]
self.mox.StubOutWithMock(self.dbapi, "get_port")
self.dbapi.get_port(uuid).AndReturn(self.fake_port)
self.mox.ReplayAll()
objects.Port.get_by_uuid(ctxt, uuid)
self.mox.VerifyAll()
def test_load(self):
ctxt = context.get_admin_context()
uuid = self.fake_node['uuid']
self.mox.StubOutWithMock(self.dbapi, 'get_node')
self.dbapi.get_node(uuid).AndReturn(self.fake_node)
self.mox.ReplayAll()
objects.Node.get_by_uuid(ctxt, uuid)
self.mox.VerifyAll()
def _fail_transient_state(self, state, last_error):
"""Apply "fail" transition to nodes in a transient state.
If the conductor server dies abruptly mid deployment or cleaning
(OMM Killer, power outage, etc...) we can not resume the process even
if the conductor comes back online. Cleaning the reservation of
the nodes (dbapi.clear_node_reservations_for_conductor) is not enough
to unstick it, so let's gracefully fail the process.
"""
filters = {'reserved': False, 'provision_state': state}
self._fail_if_in_state(ironic_context.get_admin_context(), filters,
state, 'provision_updated_at',
last_error=last_error)
def _fail_transient_state(self, state, last_error):
"""Apply "fail" transition to nodes in a transient state.
If the conductor server dies abruptly mid deployment or cleaning
(OMM Killer, power outage, etc...) we can not resume the process even
if the conductor comes back online. Cleaning the reservation of
the nodes (dbapi.clear_node_reservations_for_conductor) is not enough
to unstick it, so let's gracefully fail the process.
"""
filters = {'reserved': False, 'provision_state': state}
self._fail_if_in_state(ironic_context.get_admin_context(), filters,
state, 'provision_updated_at',
last_error=last_error)
def unplug(self, instance, vif):
LOG.debug(_("unplug: instance_uuid=%(uuid)s vif=%(vif)s"),
{'uuid': instance['uuid'], 'vif': vif})
network, mapping = vif
vif_uuid = mapping['vif_uuid']
ctx = nova_context.get_admin_context()
try:
pif = db.bm_interface_get_by_vif_uuid(ctx, vif_uuid)
db.bm_interface_set_vif_uuid(ctx, pif['id'], None)
LOG.debug(_("pif:%(id)s is unplugged (vif_uuid=%(vif_uuid)s)")
% {'id': pif['id'], 'vif_uuid': vif_uuid})
self._after_unplug(instance, network, mapping, pif)
except exception.NovaException:
LOG.warn(_("no pif for vif_uuid=%s") % vif_uuid)
def test_save(self):
ctxt = context.get_admin_context()
uuid = self.fake_node['uuid']
self.mox.StubOutWithMock(self.dbapi, 'get_node')
self.mox.StubOutWithMock(self.dbapi, 'update_node')
self.dbapi.get_node(uuid).AndReturn(self.fake_node)
self.dbapi.update_node(uuid, {'properties': {"fake": "property"}})
self.mox.ReplayAll()
n = objects.Node.get_by_uuid(ctxt, uuid)
n.properties = {"fake": "property"}
n.save()
self.mox.VerifyAll()
def test_save(self):
ctxt = context.get_admin_context()
uuid = self.fake_port["uuid"]
self.mox.StubOutWithMock(self.dbapi, "get_port")
self.mox.StubOutWithMock(self.dbapi, "update_port")
self.dbapi.get_port(uuid).AndReturn(self.fake_port)
self.dbapi.update_port(uuid, {"address": "b2:54:00:cf:2d:40"})
self.mox.ReplayAll()
p = objects.Port.get_by_uuid(ctxt, uuid)
p.address = "b2:54:00:cf:2d:40"
p.save()
self.mox.VerifyAll()
def start(self):
super(RPCService, self).start()
admin_context = context.get_admin_context()
target = messaging.Target(topic=self.topic, server=self.host)
endpoints = [self.manager]
serializer = objects_base.IronicObjectSerializer()
self.rpcserver = rpc.get_server(target, endpoints, serializer)
self.rpcserver.start()
self.handle_signal()
self.manager.init_host(admin_context)
LOG.info(_LI('Created RPC server for service %(service)s on host '
'%(host)s.'),
{'service': self.topic, 'host': self.host})
def test_refresh(self):
ctxt = context.get_admin_context()
uuid = self.fake_node['uuid']
self.mox.StubOutWithMock(self.dbapi, 'get_node')
self.dbapi.get_node(uuid).AndReturn(
dict(self.fake_node, properties={"fake": "first"}))
self.dbapi.get_node(uuid).AndReturn(
dict(self.fake_node, properties={"fake": "second"}))
self.mox.ReplayAll()
n = objects.Node.get_by_uuid(ctxt, uuid)
self.assertEqual(n.properties, {"fake": "first"})
n.refresh()
self.assertEqual(n.properties, {"fake": "second"})
self.mox.VerifyAll()
def start(self):
super(RPCService, self).start()
admin_context = context.get_admin_context()
target = messaging.Target(topic=self.topic, server=self.host)
endpoints = [self.manager]
serializer = objects_base.IronicObjectSerializer()
self.rpcserver = rpc.get_server(target, endpoints, serializer)
self.rpcserver.start()
self.handle_signal()
self.manager.init_host(admin_context)
LOG.info(_LI('Created RPC server for service %(service)s on host '
'%(host)s.'),
{'service': self.topic, 'host': self.host})
def post(self, environ, start_response):
LOG.info("post: environ=%s", environ)
inpt = environ['wsgi.input']
length = int(environ.get('CONTENT_LENGTH', 0))
x = inpt.read(length)
q = dict(cgi.parse_qsl(x))
try:
node_id = q['i']
deploy_key = q['k']
address = q['a']
port = q.get('p', '3260')
iqn = q['n']
lun = q.get('l', '1')
err_msg = q.get('e')
except KeyError as e:
start_response('400 Bad Request', [('Content-type', 'text/plain')])
return "parameter '%s' is not defined" % e
if err_msg:
LOG.error(_('Deploy agent error message: %s'), err_msg)
context = ironic_context.get_admin_context()
d = db.bm_node_get(context, node_id)
if d['deploy_key'] != deploy_key:
start_response('400 Bad Request', [('Content-type', 'text/plain')])
return 'key is not match'
params = {'address': address,
'port': port,
'iqn': iqn,
'lun': lun,
'image_path': d['image_path'],
'pxe_config_path': d['pxe_config_path'],
'root_mb': int(d['root_mb']),
'swap_mb': int(d['swap_mb']),
}
# Restart worker, if needed
if not self.worker.isAlive():
self.worker = Worker()
self.worker.start()
LOG.info("request is queued: node %s, params %s", node_id, params)
QUEUE.put((node_id, params))
# Requests go to Worker.run()
start_response('200 OK', [('Content-type', 'text/plain')])
return ''
def test_refresh(self):
ctxt = context.get_admin_context()
uuid = self.fake_port["uuid"]
self.mox.StubOutWithMock(self.dbapi, "get_port")
self.dbapi.get_port(uuid).AndReturn(self.fake_port)
self.dbapi.get_port(uuid).AndReturn(utils.get_test_port(address="c3:54:00:cf:2d:40"))
self.mox.ReplayAll()
p = objects.Port.get_by_uuid(ctxt, uuid)
self.assertEqual(p.address, "52:54:00:cf:2d:31")
p.refresh()
self.assertEqual(p.address, "c3:54:00:cf:2d:40")
self.mox.VerifyAll()
def _run_online_data_migrations(self, max_count=None):
"""Perform online data migrations for the release.
Online data migrations are done by running all the data migration
functions in the ONLINE_MIGRATIONS list. If max_count is None, all
the functions will be run in batches of 50 objects, until the
migrations are done. Otherwise, this will run (some of) the functions
until max_count objects have been migrated.
:param: max_count: the maximum number of individual object migrations
or modified rows, a value >= 1. If None, migrations are run in a
loop in batches of 50, until completion.
:raises: SystemExit. With exit code of:
0: when all migrations are complete.
1: when objects were migrated and the command needs to be
re-run (because there might be more objects to be migrated)
127: if max_count is < 1
:raises: Exception from a migration function
"""
admin_context = context.get_admin_context()
finished_migrating = False
if max_count is None:
max_count = 50
print(
_('Running batches of %i until migrations have been '
'completed.') % max_count)
while not finished_migrating:
finished_migrating = self._run_migration_functions(
admin_context, max_count)
print(_('Data migrations have completed.'))
sys.exit(0)
if max_count < 1:
print(_('"max-count" must be a positive value.'), file=sys.stderr)
sys.exit(127)
finished_migrating = self._run_migration_functions(
admin_context, max_count)
if finished_migrating:
print(_('Data migrations have completed.'))
sys.exit(0)
else:
print(_('Data migrations have not completed. Please re-run.'))
sys.exit(1)
def _run_online_data_migrations(self, max_count=None):
"""Perform online data migrations for the release.
Online data migrations are done by running all the data migration
functions in the ONLINE_MIGRATIONS list. If max_count is None, all
the functions will be run in batches of 50 objects, until the
migrations are done. Otherwise, this will run (some of) the functions
until max_count objects have been migrated.
:param: max_count: the maximum number of individual object migrations
or modified rows, a value >= 1. If None, migrations are run in a
loop in batches of 50, until completion.
:raises: SystemExit. With exit code of:
0: when all migrations are complete.
1: when objects were migrated and the command needs to be
re-run (because there might be more objects to be migrated)
127: if max_count is < 1
:raises: Exception from a migration function
"""
admin_context = context.get_admin_context()
finished_migrating = False
if max_count is None:
max_count = 50
print(_('Running batches of %i until migrations have been '
'completed.') % max_count)
while not finished_migrating:
finished_migrating = self._run_migration_functions(
admin_context, max_count)
print(_('Data migrations have completed.'))
sys.exit(0)
if max_count < 1:
print(_('"max-count" must be a positive value.'), file=sys.stderr)
sys.exit(127)
finished_migrating = self._run_migration_functions(admin_context,
max_count)
if finished_migrating:
print(_('Data migrations have completed.'))
sys.exit(0)
else:
print(_('Data migrations have not completed. Please re-run.'))
sys.exit(1)
def post(self, environ, start_response):
LOG.info("post: environ=%s", environ)
inpt = environ['wsgi.input']
length = int(environ.get('CONTENT_LENGTH', 0))
x = inpt.read(length)
q = dict(cgi.parse_qsl(x))
try:
node_id = q['i']
deploy_key = q['k']
address = q['a']
port = q.get('p', '3260')
iqn = q['n']
lun = q.get('l', '1')
except KeyError as e:
start_response('400 Bad Request', [('Content-type', 'text/plain')])
return "parameter '%s' is not defined" % e
context = ironic_context.get_admin_context()
d = db.bm_node_get(context, node_id)
if d['deploy_key'] != deploy_key:
start_response('400 Bad Request', [('Content-type', 'text/plain')])
return 'key is not match'
params = {'address': address,
'port': port,
'iqn': iqn,
'lun': lun,
'image_path': d['image_path'],
'pxe_config_path': d['pxe_config_path'],
'root_mb': int(d['root_mb']),
'swap_mb': int(d['swap_mb']),
}
# Restart worker, if needed
if not self.worker.isAlive():
self.worker = Worker()
self.worker.start()
LOG.info("request is queued: node %s, params %s", node_id, params)
QUEUE.put((node_id, params))
# Requests go to Worker.run()
start_response('200 OK', [('Content-type', 'text/plain')])
return ''
def unplug(self, instance, vif):
LOG.debug(_("unplug: instance_uuid=%(uuid)s vif=%(vif)s"), {
'uuid': instance['uuid'],
'vif': vif
})
network, mapping = vif
vif_uuid = mapping['vif_uuid']
ctx = nova_context.get_admin_context()
try:
pif = db.bm_interface_get_by_vif_uuid(ctx, vif_uuid)
db.bm_interface_set_vif_uuid(ctx, pif['id'], None)
LOG.debug(
_("pif:%(id)s is unplugged (vif_uuid=%(vif_uuid)s)") % {
'id': pif['id'],
'vif_uuid': vif_uuid
})
self._after_unplug(instance, network, mapping, pif)
except exception.NovaException:
LOG.warn(_("no pif for vif_uuid=%s") % vif_uuid)
def setUp(self):
"""Run before each test method to initialize test environment."""
super(TestCase, self).setUp()
self.context = ironic_context.get_admin_context()
test_timeout = os.environ.get('OS_TEST_TIMEOUT', 0)
try:
test_timeout = int(test_timeout)
except ValueError:
# If timeout value is invalid do not set a timeout.
test_timeout = 0
if test_timeout > 0:
self.useFixture(fixtures.Timeout(test_timeout, gentle=True))
self.useFixture(fixtures.NestedTempfile())
self.useFixture(fixtures.TempHomeDir())
if (os.environ.get('OS_STDOUT_CAPTURE') == 'True' or
os.environ.get('OS_STDOUT_CAPTURE') == '1'):
stdout = self.useFixture(fixtures.StringStream('stdout')).stream
self.useFixture(fixtures.MonkeyPatch('sys.stdout', stdout))
if (os.environ.get('OS_STDERR_CAPTURE') == 'True' or
os.environ.get('OS_STDERR_CAPTURE') == '1'):
stderr = self.useFixture(fixtures.StringStream('stderr')).stream
self.useFixture(fixtures.MonkeyPatch('sys.stderr', stderr))
self.log_fixture = self.useFixture(fixtures.FakeLogger())
self._set_config()
# NOTE(danms): Make sure to reset us back to non-remote objects
# for each test to avoid interactions. Also, backup the object
# registry
objects_base.IronicObject.indirection_api = None
self._base_test_obj_backup = copy.copy(
objects_base.IronicObjectRegistry.obj_classes())
self.addCleanup(self._restore_obj_registry)
self.addCleanup(self._clear_attrs)
self.addCleanup(hash_ring.HashRingManager().reset)
self.useFixture(fixtures.EnvironmentVariable('http_proxy'))
self.policy = self.useFixture(policy_fixture.PolicyFixture())
driver_factory.DriverFactory._extension_manager = None
driver_factory.HardwareTypesFactory._extension_manager = None
for factory in driver_factory._INTERFACE_LOADERS.values():
factory._extension_manager = None
def setUp(self):
"""Run before each test method to initialize test environment."""
super(TestCase, self).setUp()
self.context = ironic_context.get_admin_context()
test_timeout = os.environ.get('OS_TEST_TIMEOUT', 0)
try:
test_timeout = int(test_timeout)
except ValueError:
# If timeout value is invalid do not set a timeout.
test_timeout = 0
if test_timeout > 0:
self.useFixture(fixtures.Timeout(test_timeout, gentle=True))
self.useFixture(fixtures.NestedTempfile())
self.useFixture(fixtures.TempHomeDir())
if (os.environ.get('OS_STDOUT_CAPTURE') == 'True' or
os.environ.get('OS_STDOUT_CAPTURE') == '1'):
stdout = self.useFixture(fixtures.StringStream('stdout')).stream
self.useFixture(fixtures.MonkeyPatch('sys.stdout', stdout))
if (os.environ.get('OS_STDERR_CAPTURE') == 'True' or
os.environ.get('OS_STDERR_CAPTURE') == '1'):
stderr = self.useFixture(fixtures.StringStream('stderr')).stream
self.useFixture(fixtures.MonkeyPatch('sys.stderr', stderr))
self.log_fixture = self.useFixture(fixtures.FakeLogger())
self._set_config()
# NOTE(danms): Make sure to reset us back to non-remote objects
# for each test to avoid interactions. Also, backup the object
# registry
objects_base.IronicObject.indirection_api = None
self._base_test_obj_backup = copy.copy(
objects_base.IronicObjectRegistry.obj_classes())
self.addCleanup(self._restore_obj_registry)
self.addCleanup(self._clear_attrs)
self.addCleanup(hash_ring.HashRingManager().reset)
self.useFixture(fixtures.EnvironmentVariable('http_proxy'))
self.policy = self.useFixture(policy_fixture.PolicyFixture())
driver_factory.DriverFactory._extension_manager = None
driver_factory.HardwareTypesFactory._extension_manager = None
for factory in driver_factory._INTERFACE_LOADERS.values():
factory._extension_manager = None
def __init__(self, **kwargs):
global _conn
global _virtual_power_settings
global _cmds
if _cmds is None:
LOG.debug("Setting up %s commands." % CONF.virtual_power_type)
_vpc = "ironic.virtual_power_driver_settings.%s" % CONF.virtual_power_type
_cmds = importutils.import_class(_vpc)
self._vp_cmd = _cmds()
self.connection_data = _conn
node = kwargs.pop("node", {})
instance = kwargs.pop("instance", {})
self._node_name = instance.get("hostname", "")
context = nova_context.get_admin_context()
ifs = db.bm_interface_get_all_by_bm_node_id(context, node["id"])
self._mac_addresses = [_normalize_mac(i["address"]) for i in ifs]
self._connection = None
self._matched_name = ""
self.state = None
def start(self):
super(RPCService, self).start()
admin_context = context.get_admin_context()
serializer = objects_base.IronicObjectSerializer(is_server=True)
if CONF.rpc_transport == 'json-rpc':
self.rpcserver = json_rpc.WSGIService(self.manager,
serializer)
else:
target = messaging.Target(topic=self.topic, server=self.host)
endpoints = [self.manager]
self.rpcserver = rpc.get_server(target, endpoints, serializer)
self.rpcserver.start()
self.handle_signal()
self.manager.init_host(admin_context)
LOG.info('Created RPC server for service %(service)s on host '
'%(host)s.',
{'service': self.topic, 'host': self.host})
def setUp(self):
"""Run before each test method to initialize test environment."""
super(TestCase, self).setUp()
self.context = ironic_context.get_admin_context()
self._set_config()
# NOTE(danms): Make sure to reset us back to non-remote objects
# for each test to avoid interactions. Also, backup the object
# registry
self._base_test_obj_backup = copy.copy(
objects_base.IronicObjectRegistry.obj_classes())
self.addCleanup(self._restore_obj_registry)
self.addCleanup(self._clear_attrs)
self.addCleanup(hash_ring.HashRingManager().reset)
self.useFixture(fixtures.EnvironmentVariable('http_proxy'))
self.policy = self.useFixture(policy_fixture.PolicyFixture())
driver_factory.HardwareTypesFactory._extension_manager = None
for factory in driver_factory._INTERFACE_LOADERS.values():
factory._extension_manager = None
# Ban running external processes via 'execute' like functions. If the
# patched function is called, an exception is raised to warn the
# tester.
if self.block_execute:
# NOTE(jlvillal): Intentionally not using mock as if you mock a
# mock it causes things to not work correctly. As doing an
# autospec=True causes strangeness. By using a simple function we
# can then mock it without issue.
self.patch(processutils, 'execute', do_not_call)
self.patch(subprocess, 'call', do_not_call)
self.patch(subprocess, 'check_call', do_not_call)
self.patch(subprocess, 'check_output', do_not_call)
self.patch(utils, 'execute', do_not_call)
# subprocess.Popen is a class
self.patch(subprocess, 'Popen', DoNotCallPopen)
if sys.version_info < (3, 7):
_patch_mock_callable._old_func = mock._callable
mock._callable = _patch_mock_callable
def __init__(self, **kwargs):
global _conn
global _virtual_power_settings
global _cmds
if _cmds is None:
LOG.debug("Setting up %s commands." %
CONF.virtual_power_type)
_vpc = 'ironic.virtual_power_driver_settings.%s' % \
CONF.virtual_power_type
_cmds = importutils.import_class(_vpc)
self._vp_cmd = _cmds()
self.connection_data = _conn
node = kwargs.pop('node', {})
instance = kwargs.pop('instance', {})
self._node_name = instance.get('hostname', "")
context = nova_context.get_admin_context()
ifs = db.bm_interface_get_all_by_bm_node_id(context, node['id'])
self._mac_addresses = [_normalize_mac(i['address']) for i in ifs]
self._connection = None
self._matched_name = ''
self.state = None
def setUp(self):
super(UpdateToLatestVersionsTestCase, self).setUp()
self.context = context.get_admin_context()
self.dbapi = db_api.get_instance()
obj_versions = release_mappings.get_object_versions(
objects=['Node', 'Chassis'])
master_objs = release_mappings.RELEASE_MAPPING['master']['objects']
self.node_ver = master_objs['Node'][0]
self.chassis_ver = master_objs['Chassis'][0]
self.node_old_ver = self._get_old_object_version(
self.node_ver, obj_versions['Node'])
self.chassis_old_ver = self._get_old_object_version(
self.chassis_ver, obj_versions['Chassis'])
self.node_version_same = self.node_old_ver == self.node_ver
self.chassis_version_same = self.chassis_old_ver == self.chassis_ver
# number of objects with different versions
self.num_diff_objs = 2
if self.node_version_same:
self.num_diff_objs -= 1
if self.chassis_version_same:
self.num_diff_objs -= 1
def setUp(self):
"""Run before each test method to initialize test environment."""
super(TestCase, self).setUp()
self.context = ironic_context.get_admin_context()
self._set_config()
# NOTE(danms): Make sure to reset us back to non-remote objects
# for each test to avoid interactions. Also, backup the object
# registry
objects_base.IronicObject.indirection_api = None
self._base_test_obj_backup = copy.copy(
objects_base.IronicObjectRegistry.obj_classes())
self.addCleanup(self._restore_obj_registry)
self.addCleanup(self._clear_attrs)
self.addCleanup(hash_ring.HashRingManager().reset)
self.useFixture(fixtures.EnvironmentVariable('http_proxy'))
self.policy = self.useFixture(policy_fixture.PolicyFixture())
driver_factory.DriverFactory._extension_manager = None
driver_factory.HardwareTypesFactory._extension_manager = None
for factory in driver_factory._INTERFACE_LOADERS.values():
factory._extension_manager = None
# Block access to utils.execute() and related functions.
# NOTE(bigjools): Not using a decorator on tests because I don't
# want to force every test method to accept a new arg. Instead, they
# can override or examine this self._exec_patch Mock as needed.
if self.block_execute:
self._exec_patch = mock.Mock()
self._exec_patch.side_effect = Exception(
"Don't call ironic_lib.utils.execute() / "
"processutils.execute() or similar functions in tests!")
self.patch(processutils, 'execute', self._exec_patch)
self.patch(subprocess, 'Popen', self._exec_patch)
self.patch(subprocess, 'call', self._exec_patch)
self.patch(subprocess, 'check_call', self._exec_patch)
self.patch(subprocess, 'check_output', self._exec_patch)
self.patch(utils, 'execute', self._exec_patch)
def setUp(self):
super(UpdateToLatestVersionsTestCase, self).setUp()
self.context = context.get_admin_context()
self.dbapi = db_api.get_instance()
obj_versions = release_mappings.get_object_versions(
objects=['Node', 'Chassis'])
master_objs = release_mappings.RELEASE_MAPPING['master']['objects']
self.node_ver = master_objs['Node'][0]
self.chassis_ver = master_objs['Chassis'][0]
self.node_old_ver = self._get_old_object_version(
self.node_ver, obj_versions['Node'])
self.chassis_old_ver = self._get_old_object_version(
self.chassis_ver, obj_versions['Chassis'])
self.node_version_same = self.node_old_ver == self.node_ver
self.chassis_version_same = self.chassis_old_ver == self.chassis_ver
# number of objects with different versions
self.num_diff_objs = 2
if self.node_version_same:
self.num_diff_objs -= 1
if self.chassis_version_same:
self.num_diff_objs -= 1
def setup(name, host='0.0.0.0'):
"""Setup OSprofiler notifier and enable profiling.
:param name: name of the service that will be profiled
:param host: hostname or host IP address that the service will be
running on. By default host will be set to 0.0.0.0, but
specifying host name / address usage is highly recommended.
:raises TypeError: in case of invalid connection string for
a notifier backend, which is set in
osprofiler.initializer.init_from_conf.
"""
if not CONF.profiler.enabled:
return
admin_context = context.get_admin_context()
initializer.init_from_conf(conf=CONF,
context=admin_context.to_dict(),
project="ironic",
service=name,
host=host)
LOG.info("OSProfiler is enabled. Trace is generated using "
"[profiler]/hmac_keys specified in ironic.conf. "
"To disable, set [profiler]/enabled=false")
def setUp(self):
"""Run before each test method to initialize test environment."""
super(TestCase, self).setUp()
self.context = ironic_context.get_admin_context()
self._set_config()
# NOTE(danms): Make sure to reset us back to non-remote objects
# for each test to avoid interactions. Also, backup the object
# registry
self._base_test_obj_backup = copy.copy(
objects_base.IronicObjectRegistry.obj_classes())
self.addCleanup(self._restore_obj_registry)
self.addCleanup(self._clear_attrs)
self.addCleanup(hash_ring.HashRingManager().reset)
self.useFixture(fixtures.EnvironmentVariable('http_proxy'))
self.policy = self.useFixture(policy_fixture.PolicyFixture())
driver_factory.HardwareTypesFactory._extension_manager = None
for factory in driver_factory._INTERFACE_LOADERS.values():
factory._extension_manager = None
# Ban running external processes via 'execute' like functions. If the
# patched function is called, an exception is raised to warn the
# tester.
if self.block_execute:
# NOTE(jlvillal): Intentionally not using mock as if you mock a
# mock it causes things to not work correctly. As doing an
# autospec=True causes strangeness. By using a simple function we
# can then mock it without issue.
self.patch(processutils, 'execute', do_not_call)
self.patch(subprocess, 'call', do_not_call)
self.patch(subprocess, 'check_call', do_not_call)
self.patch(subprocess, 'check_output', do_not_call)
self.patch(utils, 'execute', do_not_call)
# subprocess.Popen is a class
self.patch(subprocess, 'Popen', DoNotCallPopen)
def setUp(self):
super(TestNodePayloads, self).setUp()
self.ctxt = context.get_admin_context()
self.fake_node = db_utils.get_test_node()
self.node = obj_utils.get_test_node(self.ctxt, **self.fake_node)
def setUp(self):
super(TestTraitObject, self).setUp()
self.ctxt = context.get_admin_context()
self.fake_trait = db_utils.get_test_node_trait()
self.node_id = self.fake_trait['node_id']
def setUp(self):
super(NovaApiTestCase, self).setUp()
self.api = nova
self.ctx = context.get_admin_context()
def setUp(self):
super(TestConvertToVersion, self).setUp()
self.ctxt = context.get_admin_context()
self.fake_node = db_utils.get_test_node(driver='fake-hardware')
def init_host(self, admin_context=None):
"""Initialize the conductor host.
:param admin_context: the admin context to pass to periodic tasks.
:raises: RuntimeError when conductor is already running.
:raises: NoDriversLoaded when no drivers are enabled on the conductor.
:raises: DriverNotFound if a driver is enabled that does not exist.
:raises: DriverLoadError if an enabled driver cannot be loaded.
:raises: DriverNameConflict if a classic driver and a dynamic driver
are both enabled and have the same name.
"""
if self._started:
raise RuntimeError(_('Attempt to start an already running '
'conductor manager'))
self._shutdown = False
self.dbapi = dbapi.get_instance()
self._keepalive_evt = threading.Event()
"""Event for the keepalive thread."""
# TODO(dtantsur): make the threshold configurable?
rejection_func = rejection.reject_when_reached(
CONF.conductor.workers_pool_size)
self._executor = futurist.GreenThreadPoolExecutor(
max_workers=CONF.conductor.workers_pool_size,
check_and_reject=rejection_func)
"""Executor for performing tasks async."""
# TODO(jroll) delete the use_groups argument and use the default
# in Stein.
self.ring_manager = hash_ring.HashRingManager(
use_groups=self._use_groups())
"""Consistent hash ring which maps drivers to conductors."""
# TODO(dtantsur): remove in Stein
if CONF.enabled_drivers:
raise RuntimeError("The enabled_drivers configuration option "
"no longer has any effect and must be empty")
_check_enabled_interfaces()
# NOTE(deva): these calls may raise DriverLoadError or DriverNotFound
# NOTE(vdrok): Instantiate network and storage interface factory on
# startup so that all the interfaces are loaded at the very
# beginning, and failures prevent the conductor from starting.
hardware_types = driver_factory.hardware_types()
driver_factory.NetworkInterfaceFactory()
driver_factory.StorageInterfaceFactory()
# NOTE(jroll) this is passed to the dbapi, which requires a list, not
# a generator (which keys() returns in py3)
hardware_type_names = list(hardware_types)
# check that at least one driver is loaded, whether classic or dynamic
if not hardware_type_names:
msg = ("Conductor %s cannot be started because no hardware types "
"were specified in the 'enabled_hardware_types' config "
"option.")
LOG.error(msg, self.host)
raise exception.NoDriversLoaded(conductor=self.host)
self._collect_periodic_tasks(admin_context)
# clear all target_power_state with locks by this conductor
self.dbapi.clear_node_target_power_state(self.host)
# clear all locks held by this conductor before registering
self.dbapi.clear_node_reservations_for_conductor(self.host)
try:
# Register this conductor with the cluster
self.conductor = objects.Conductor.register(
admin_context, self.host, hardware_type_names,
CONF.conductor.conductor_group)
except exception.ConductorAlreadyRegistered:
# This conductor was already registered and did not shut down
# properly, so log a warning and update the record.
LOG.warning("A conductor with hostname %(hostname)s was "
"previously registered. Updating registration",
{'hostname': self.host})
self.conductor = objects.Conductor.register(
admin_context, self.host, hardware_type_names,
CONF.conductor.conductor_group, update_existing=True)
# register hardware types and interfaces supported by this conductor
# and validate them against other conductors
try:
self._register_and_validate_hardware_interfaces(hardware_types)
except (exception.DriverLoadError, exception.DriverNotFound,
exception.ConductorHardwareInterfacesAlreadyRegistered,
exception.InterfaceNotFoundInEntrypoint,
exception.NoValidDefaultForInterface) as e:
with excutils.save_and_reraise_exception():
LOG.error('Failed to register hardware types. %s', e)
self.del_host()
# Start periodic tasks
self._periodic_tasks_worker = self._executor.submit(
self._periodic_tasks.start, allow_empty=True)
self._periodic_tasks_worker.add_done_callback(
self._on_periodic_tasks_stop)
for state in states.STUCK_STATES_TREATED_AS_FAIL:
self._fail_transient_state(
state,
_("The %(state)s state can't be resumed by conductor "
"%(host)s. Moving to fail state.") %
{'state': state, 'host': self.host})
# Start consoles if it set enabled in a greenthread.
try:
self._spawn_worker(self._start_consoles,
ironic_context.get_admin_context())
except exception.NoFreeConductorWorker:
LOG.warning('Failed to start worker for restarting consoles.')
# Spawn a dedicated greenthread for the keepalive
try:
self._spawn_worker(self._conductor_service_record_keepalive)
LOG.info('Successfully started conductor with hostname '
'%(hostname)s.',
{'hostname': self.host})
except exception.NoFreeConductorWorker:
with excutils.save_and_reraise_exception():
LOG.critical('Failed to start keepalive')
self.del_host()
# Resume allocations that started before the restart.
try:
self._spawn_worker(self._resume_allocations,
ironic_context.get_admin_context())
except exception.NoFreeConductorWorker:
LOG.warning('Failed to start worker for resuming allocations.')
if CONF.conductor.enable_mdns:
self._publish_endpoint()
self._started = True
def setUp(self):
super(TestBIOSSettingObject, self).setUp()
self.ctxt = context.get_admin_context()
self.bios_setting = db_utils.get_test_bios_setting()
self.node_id = self.bios_setting['node_id']
def init_host(self, admin_context=None):
"""Initialize the conductor host.
:param admin_context: the admin context to pass to periodic tasks.
:raises: RuntimeError when conductor is already running.
:raises: NoDriversLoaded when no drivers are enabled on the conductor.
:raises: DriverNotFound if a driver is enabled that does not exist.
:raises: DriverLoadError if an enabled driver cannot be loaded.
:raises: DriverNameConflict if a classic driver and a dynamic driver
are both enabled and have the same name.
:raises: ConfigInvalid if required config options for connection with
radosgw are missing while storing config drive.
"""
if self._started:
raise RuntimeError(
_('Attempt to start an already running '
'conductor manager'))
self._shutdown = False
self.dbapi = dbapi.get_instance()
self._keepalive_evt = threading.Event()
"""Event for the keepalive thread."""
# TODO(dtantsur): make the threshold configurable?
rejection_func = rejection.reject_when_reached(
CONF.conductor.workers_pool_size)
self._executor = futurist.GreenThreadPoolExecutor(
max_workers=CONF.conductor.workers_pool_size,
check_and_reject=rejection_func)
"""Executor for performing tasks async."""
self.ring_manager = hash_ring.HashRingManager()
"""Consistent hash ring which maps drivers to conductors."""
_check_enabled_interfaces()
# NOTE(deva): these calls may raise DriverLoadError or DriverNotFound
# NOTE(vdrok): Instantiate network and storage interface factory on
# startup so that all the interfaces are loaded at the very
# beginning, and failures prevent the conductor from starting.
drivers = driver_factory.drivers()
hardware_types = driver_factory.hardware_types()
driver_factory.NetworkInterfaceFactory()
driver_factory.StorageInterfaceFactory()
# NOTE(jroll) this is passed to the dbapi, which requires a list, not
# a generator (which keys() returns in py3)
driver_names = list(drivers)
hardware_type_names = list(hardware_types)
# check that at least one driver is loaded, whether classic or dynamic
if not driver_names and not hardware_type_names:
msg = ("Conductor %s cannot be started because no drivers "
"were loaded. This could be because no classic drivers "
"were specified in the 'enabled_drivers' config option "
"and no dynamic drivers were specified in the "
"'enabled_hardware_types' config option.")
LOG.error(msg, self.host)
raise exception.NoDriversLoaded(conductor=self.host)
# check for name clashes between classic and dynamic drivers
name_clashes = set(driver_names).intersection(hardware_type_names)
if name_clashes:
name_clashes = ', '.join(name_clashes)
msg = ("Conductor %(host)s cannot be started because there is "
"one or more name conflicts between classic drivers and "
"dynamic drivers (%(names)s). Check any external driver "
"plugins and the 'enabled_drivers' and "
"'enabled_hardware_types' config options.")
LOG.error(msg, {'host': self.host, 'names': name_clashes})
raise exception.DriverNameConflict(names=name_clashes)
# Collect driver-specific periodic tasks.
# Conductor periodic tasks accept context argument, driver periodic
# tasks accept this manager and context. We have to ensure that the
# same driver interface class is not traversed twice, otherwise
# we'll have several instances of the same task.
LOG.debug('Collecting periodic tasks')
self._periodic_task_callables = []
periodic_task_classes = set()
self._collect_periodic_tasks(self, (admin_context, ))
for driver_obj in drivers.values():
for iface_name in driver_obj.all_interfaces:
iface = getattr(driver_obj, iface_name, None)
if iface and iface.__class__ not in periodic_task_classes:
self._collect_periodic_tasks(iface, (self, admin_context))
periodic_task_classes.add(iface.__class__)
if (len(self._periodic_task_callables) >
CONF.conductor.workers_pool_size):
LOG.warning(
'This conductor has %(tasks)d periodic tasks '
'enabled, but only %(workers)d task workers '
'allowed by [conductor]workers_pool_size option', {
'tasks': len(self._periodic_task_callables),
'workers': CONF.conductor.workers_pool_size
})
self._periodic_tasks = periodics.PeriodicWorker(
self._periodic_task_callables,
executor_factory=periodics.ExistingExecutor(self._executor))
# Check for required config options if object_store_endpoint_type is
# radosgw
if (CONF.deploy.configdrive_use_object_store
and CONF.deploy.object_store_endpoint_type == "radosgw"):
if (None in (CONF.swift.auth_url, CONF.swift.username,
CONF.swift.password)):
msg = _("Parameters missing to make a connection with "
"radosgw. Ensure that [swift]/auth_url, "
"[swift]/username, and [swift]/password are all "
"configured.")
raise exception.ConfigInvalid(msg)
# clear all target_power_state with locks by this conductor
self.dbapi.clear_node_target_power_state(self.host)
# clear all locks held by this conductor before registering
self.dbapi.clear_node_reservations_for_conductor(self.host)
try:
# Register this conductor with the cluster
self.conductor = objects.Conductor.register(
admin_context, self.host, driver_names)
except exception.ConductorAlreadyRegistered:
# This conductor was already registered and did not shut down
# properly, so log a warning and update the record.
LOG.warning(
"A conductor with hostname %(hostname)s was "
"previously registered. Updating registration",
{'hostname': self.host})
self.conductor = objects.Conductor.register(admin_context,
self.host,
driver_names,
update_existing=True)
# register hardware types and interfaces supported by this conductor
# and validate them against other conductors
try:
self._register_and_validate_hardware_interfaces(hardware_types)
except (exception.DriverLoadError, exception.DriverNotFound,
exception.ConductorHardwareInterfacesAlreadyRegistered,
exception.InterfaceNotFoundInEntrypoint,
exception.NoValidDefaultForInterface) as e:
with excutils.save_and_reraise_exception():
LOG.error('Failed to register hardware types. %s', e)
self.del_host()
# Start periodic tasks
self._periodic_tasks_worker = self._executor.submit(
self._periodic_tasks.start, allow_empty=True)
self._periodic_tasks_worker.add_done_callback(
self._on_periodic_tasks_stop)
# NOTE(lucasagomes): If the conductor server dies abruptly
# mid deployment (OMM Killer, power outage, etc...) we
# can not resume the deployment even if the conductor
# comes back online. Cleaning the reservation of the nodes
# (dbapi.clear_node_reservations_for_conductor) is not enough to
# unstick it, so let's gracefully fail the deployment so the node
# can go through the steps (deleting & cleaning) to make itself
# available again.
filters = {'reserved': False, 'provision_state': states.DEPLOYING}
last_error = (_("The deployment can't be resumed by conductor "
"%s. Moving to fail state.") % self.host)
self._fail_if_in_state(ironic_context.get_admin_context(),
filters,
states.DEPLOYING,
'provision_updated_at',
last_error=last_error)
# Start consoles if it set enabled in a greenthread.
try:
self._spawn_worker(self._start_consoles,
ironic_context.get_admin_context())
except exception.NoFreeConductorWorker:
LOG.warning('Failed to start worker for restarting consoles.')
# Spawn a dedicated greenthread for the keepalive
try:
self._spawn_worker(self._conductor_service_record_keepalive)
LOG.info(
'Successfully started conductor with hostname '
'%(hostname)s.', {'hostname': self.host})
except exception.NoFreeConductorWorker:
with excutils.save_and_reraise_exception():
LOG.critical('Failed to start keepalive')
self.del_host()
self._started = True
def setUp(self):
super(BackfillVersionTestCase, self).setUp()
self.context = context.get_admin_context()
self.dbapi = db_api.get_instance()
obj_mapping = release_mappings.RELEASE_MAPPING['pike']['objects']
self.conductor_ver = obj_mapping['Conductor'][0]
def setUp(self):
super(DbTestCase, self).setUp()
self.context = ironic_context.get_admin_context()
def init_host(self, admin_context=None):
"""Initialize the conductor host.
:param admin_context: the admin context to pass to periodic tasks.
:raises: RuntimeError when conductor is already running.
:raises: NoDriversLoaded when no drivers are enabled on the conductor.
:raises: DriverNotFound if a driver is enabled that does not exist.
:raises: DriverLoadError if an enabled driver cannot be loaded.
:raises: DriverNameConflict if a classic driver and a dynamic driver
are both enabled and have the same name.
:raises: ConfigInvalid if required config options for connection with
radosgw are missing while storing config drive.
"""
if self._started:
raise RuntimeError(_('Attempt to start an already running '
'conductor manager'))
self._shutdown = False
self.dbapi = dbapi.get_instance()
self._keepalive_evt = threading.Event()
"""Event for the keepalive thread."""
# TODO(dtantsur): make the threshold configurable?
rejection_func = rejection.reject_when_reached(
CONF.conductor.workers_pool_size)
self._executor = futurist.GreenThreadPoolExecutor(
max_workers=CONF.conductor.workers_pool_size,
check_and_reject=rejection_func)
"""Executor for performing tasks async."""
self.ring_manager = hash_ring.HashRingManager()
"""Consistent hash ring which maps drivers to conductors."""
_check_enabled_interfaces()
# NOTE(deva): these calls may raise DriverLoadError or DriverNotFound
# NOTE(vdrok): Instantiate network and storage interface factory on
# startup so that all the interfaces are loaded at the very
# beginning, and failures prevent the conductor from starting.
drivers = driver_factory.drivers()
hardware_types = driver_factory.hardware_types()
driver_factory.NetworkInterfaceFactory()
driver_factory.StorageInterfaceFactory()
# NOTE(jroll) this is passed to the dbapi, which requires a list, not
# a generator (which keys() returns in py3)
driver_names = list(drivers)
hardware_type_names = list(hardware_types)
# check that at least one driver is loaded, whether classic or dynamic
if not driver_names and not hardware_type_names:
msg = ("Conductor %s cannot be started because no drivers "
"were loaded. This could be because no classic drivers "
"were specified in the 'enabled_drivers' config option "
"and no dynamic drivers were specified in the "
"'enabled_hardware_types' config option.")
LOG.error(msg, self.host)
raise exception.NoDriversLoaded(conductor=self.host)
# check for name clashes between classic and dynamic drivers
name_clashes = set(driver_names).intersection(hardware_type_names)
if name_clashes:
name_clashes = ', '.join(name_clashes)
msg = ("Conductor %(host)s cannot be started because there is "
"one or more name conflicts between classic drivers and "
"dynamic drivers (%(names)s). Check any external driver "
"plugins and the 'enabled_drivers' and "
"'enabled_hardware_types' config options.")
LOG.error(msg, {'host': self.host, 'names': name_clashes})
raise exception.DriverNameConflict(names=name_clashes)
# Collect driver-specific periodic tasks.
# Conductor periodic tasks accept context argument, driver periodic
# tasks accept this manager and context. We have to ensure that the
# same driver interface class is not traversed twice, otherwise
# we'll have several instances of the same task.
LOG.debug('Collecting periodic tasks')
self._periodic_task_callables = []
periodic_task_classes = set()
self._collect_periodic_tasks(self, (admin_context,))
for driver_obj in drivers.values():
for iface_name in driver_obj.all_interfaces:
iface = getattr(driver_obj, iface_name, None)
if iface and iface.__class__ not in periodic_task_classes:
self._collect_periodic_tasks(iface, (self, admin_context))
periodic_task_classes.add(iface.__class__)
if (len(self._periodic_task_callables) >
CONF.conductor.workers_pool_size):
LOG.warning('This conductor has %(tasks)d periodic tasks '
'enabled, but only %(workers)d task workers '
'allowed by [conductor]workers_pool_size option',
{'tasks': len(self._periodic_task_callables),
'workers': CONF.conductor.workers_pool_size})
self._periodic_tasks = periodics.PeriodicWorker(
self._periodic_task_callables,
executor_factory=periodics.ExistingExecutor(self._executor))
# Check for required config options if object_store_endpoint_type is
# radosgw
if (CONF.deploy.configdrive_use_object_store and
CONF.deploy.object_store_endpoint_type == "radosgw"):
if (None in (CONF.swift.auth_url, CONF.swift.username,
CONF.swift.password)):
msg = _("Parameters missing to make a connection with "
"radosgw. Ensure that [swift]/auth_url, "
"[swift]/username, and [swift]/password are all "
"configured.")
raise exception.ConfigInvalid(msg)
# clear all target_power_state with locks by this conductor
self.dbapi.clear_node_target_power_state(self.host)
# clear all locks held by this conductor before registering
self.dbapi.clear_node_reservations_for_conductor(self.host)
try:
# Register this conductor with the cluster
self.conductor = objects.Conductor.register(
admin_context, self.host, driver_names)
except exception.ConductorAlreadyRegistered:
# This conductor was already registered and did not shut down
# properly, so log a warning and update the record.
LOG.warning("A conductor with hostname %(hostname)s was "
"previously registered. Updating registration",
{'hostname': self.host})
self.conductor = objects.Conductor.register(
admin_context, self.host, driver_names, update_existing=True)
# register hardware types and interfaces supported by this conductor
# and validate them against other conductors
try:
self._register_and_validate_hardware_interfaces(hardware_types)
except (exception.DriverLoadError, exception.DriverNotFound,
exception.ConductorHardwareInterfacesAlreadyRegistered,
exception.InterfaceNotFoundInEntrypoint,
exception.NoValidDefaultForInterface) as e:
with excutils.save_and_reraise_exception():
LOG.error('Failed to register hardware types. %s', e)
self.del_host()
# Start periodic tasks
self._periodic_tasks_worker = self._executor.submit(
self._periodic_tasks.start, allow_empty=True)
self._periodic_tasks_worker.add_done_callback(
self._on_periodic_tasks_stop)
# NOTE(lucasagomes): If the conductor server dies abruptly
# mid deployment (OMM Killer, power outage, etc...) we
# can not resume the deployment even if the conductor
# comes back online. Cleaning the reservation of the nodes
# (dbapi.clear_node_reservations_for_conductor) is not enough to
# unstick it, so let's gracefully fail the deployment so the node
# can go through the steps (deleting & cleaning) to make itself
# available again.
filters = {'reserved': False,
'provision_state': states.DEPLOYING}
last_error = (_("The deployment can't be resumed by conductor "
"%s. Moving to fail state.") % self.host)
self._fail_if_in_state(ironic_context.get_admin_context(), filters,
states.DEPLOYING, 'provision_updated_at',
last_error=last_error)
# Start consoles if it set enabled in a greenthread.
try:
self._spawn_worker(self._start_consoles,
ironic_context.get_admin_context())
except exception.NoFreeConductorWorker:
LOG.warning('Failed to start worker for restarting consoles.')
# Spawn a dedicated greenthread for the keepalive
try:
self._spawn_worker(self._conductor_service_record_keepalive)
LOG.info('Successfully started conductor with hostname '
'%(hostname)s.',
{'hostname': self.host})
except exception.NoFreeConductorWorker:
with excutils.save_and_reraise_exception():
LOG.critical('Failed to start keepalive')
self.del_host()
self._started = True
def setUp(self):
super(TestDeployTemplateObject, self).setUp()
self.ctxt = context.get_admin_context()
self.fake_template = db_utils.get_test_deploy_template()
def test_get_admin_context(self):
admin_context = context.get_admin_context()
self.assertTrue(admin_context.is_admin)
def _get_baremetal_node_by_instance_uuid(instance_uuid):
context = nova_context.get_admin_context()
return db.bm_node_get_by_instance_uuid(context, instance_uuid)
def test_get_admin_context(self):
admin_context = context.get_admin_context()
self.assertTrue(admin_context.is_admin)
def setUp(self):
super(TestBIOSSettingObject, self).setUp()
self.ctxt = context.get_admin_context()
self.bios_setting = db_utils.get_test_bios_setting()
self.node_id = self.bios_setting['node_id']