def _process_node(node_info, node, introspection_data):
# NOTE(dtantsur): repeat the check in case something changed
keep_power_on = ir_utils.check_provision_state(node)
_run_post_hooks(node_info, introspection_data)
store_introspection_data(node_info.uuid, introspection_data)
ironic = ir_utils.get_client()
pxe_filter.driver().sync(ironic)
node_info.invalidate_cache()
rules.apply(node_info, introspection_data)
resp = {'uuid': node.id}
# determine how to handle power
if keep_power_on or not node_info.manage_boot:
power_action = False
else:
power_action = CONF.processing.power_off
utils.executor().submit(_finish,
node_info,
ironic,
introspection_data,
power_off=power_action)
return resp
def _abort(node_info, ironic):
# runs in background
LOG.debug('Forcing power-off', node_info=node_info)
if node_info.manage_boot:
try:
ironic.node.set_power_state(node_info.uuid, 'off')
except Exception as exc:
LOG.warning('Failed to power off node: %s',
exc,
node_info=node_info)
node_info.finished(istate.Events.abort_end,
error=_('Canceled by operator'))
# block this node from PXE Booting the introspection image
try:
pxe_filter.driver().sync(ironic)
except Exception as exc:
# Note(mkovacik): this will be retried in the PXE filter sync
# periodic task; we continue aborting
LOG.warning('Failed to sync the PXE filter: %s',
exc,
node_info=node_info)
LOG.info('Introspection aborted', node_info=node_info)
def shutdown(self, error=None):
"""Stop serving API, clean up.
:returns: None
"""
# TODO(aarefiev): move shutdown code to WorkerService
if not self._shutting_down.acquire(blocking=False):
LOG.warning('Attempted to shut down while already shutting down')
return
LOG.debug('Shutting down')
self.rpc_server.stop()
if self._periodics_worker is not None:
try:
self._periodics_worker.stop()
self._periodics_worker.wait()
except Exception as e:
LOG.exception(
'Service error occurred when stopping '
'periodic workers. Error: %s', e)
self._periodics_worker = None
if utils.executor().alive:
utils.executor().shutdown(wait=True)
pxe_filter.driver().tear_down_filter()
self._shutting_down.release()
LOG.info('Shut down successfully')
sys.exit(error)
def periodic_clean_up(): # pragma: no cover
try:
if node_cache.clean_up():
pxe_filter.driver().sync(ir_utils.get_client())
sync_with_ironic()
except Exception:
LOG.exception('Periodic clean up of node cache failed')
def del_host(self):
if not self._shutting_down.acquire(blocking=False):
LOG.warning('Attempted to shut down while already shutting down')
return
pxe_filter.driver().tear_down_filter()
if self._periodics_worker is not None:
try:
self._periodics_worker.stop()
self._periodics_worker.wait()
except Exception as e:
LOG.exception(
'Service error occurred when stopping '
'periodic workers. Error: %s', e)
self._periodics_worker = None
if utils.executor().alive:
utils.executor().shutdown(wait=True)
if self._zeroconf is not None:
self._zeroconf.close()
self._zeroconf = None
self._shutting_down.release()
LOG.info('Shut down successfully')
def periodic_clean_up(): # pragma: no cover
try:
if node_cache.clean_up():
pxe_filter.driver().sync(ir_utils.get_client())
except Exception:
LOG.exception('Periodic clean up of node cache failed')
try:
sync_with_ironic()
except Exception:
LOG.exception('Periodic sync of node list with ironic failed')
def _init_host(self):
"""Initialize Worker host
Init db connection, load and validate processing
hooks, runs periodic tasks.
:returns None
"""
db.init()
try:
hooks = plugins_base.validate_processing_hooks()
except Exception as exc:
LOG.critical(str(exc))
sys.exit(1)
LOG.info('Enabled processing hooks: %s', [h.name for h in hooks])
driver = pxe_filter.driver()
driver.init_filter()
periodic_clean_up_ = periodics.periodic(
spacing=CONF.clean_up_period)(periodic_clean_up)
self._periodics_worker = periodics.PeriodicWorker(
callables=[(driver.get_periodic_sync_task(), None, None),
(periodic_clean_up_, None, None)],
executor_factory=periodics.ExistingExecutor(utils.executor()),
on_failure=self._periodics_watchdog)
utils.executor().submit(self._periodics_worker.start)
def _background_introspect_locked(node_info, ironic):
# TODO(dtantsur): pagination
macs = list(node_info.ports())
if macs:
node_info.add_attribute(node_cache.MACS_ATTRIBUTE, macs)
LOG.info('Whitelisting MAC\'s %s for a PXE boot',
macs,
node_info=node_info)
pxe_filter.driver().sync(ironic)
attrs = node_info.attributes
if CONF.processing.node_not_found_hook is None and not attrs:
raise utils.Error(_(
'No lookup attributes were found, inspector won\'t '
'be able to find it after introspection, consider creating '
'ironic ports or providing an IPMI address'),
node_info=node_info)
LOG.info('The following attributes will be used for look up: %s',
attrs,
node_info=node_info)
if node_info.manage_boot:
try:
ironic.node.set_boot_device(node_info.uuid,
'pxe',
persistent=False)
except Exception as exc:
LOG.warning('Failed to set boot device to PXE: %s',
exc,
node_info=node_info)
try:
ironic.node.set_power_state(node_info.uuid, 'reboot')
except Exception as exc:
raise utils.Error(_('Failed to power on the node, check it\'s '
'power management configuration: %s'),
exc,
node_info=node_info)
LOG.info('Introspection started successfully', node_info=node_info)
else:
LOG.info(
'Introspection environment is ready, external power on '
'is required within %d seconds',
CONF.timeout,
node_info=node_info)
def init_host(self):
"""Initialize Worker host
Init db connection, load and validate processing
hooks, runs periodic tasks.
:returns None
"""
if CONF.processing.store_data == 'none':
LOG.warning('Introspection data will not be stored. Change '
'"[processing] store_data" option if this is not '
'the desired behavior')
else:
LOG.info('Introspection data will be stored in the %s backend',
CONF.processing.store_data)
db.init()
try:
hooks = plugins_base.validate_processing_hooks()
except Exception as exc:
LOG.critical(str(exc))
sys.exit(1)
LOG.info('Enabled processing hooks: %s', [h.name for h in hooks])
driver = pxe_filter.driver()
driver.init_filter()
periodic_clean_up_ = periodics.periodic(
spacing=CONF.clean_up_period)(periodic_clean_up)
self._periodics_worker = periodics.PeriodicWorker(
callables=[(driver.get_periodic_sync_task(), None, None),
(periodic_clean_up_, None, None)],
executor_factory=periodics.ExistingExecutor(utils.executor()),
on_failure=self._periodics_watchdog)
utils.executor().submit(self._periodics_worker.start)
if CONF.enable_mdns:
endpoint = keystone.get_endpoint('service_catalog')
self._zeroconf = mdns.Zeroconf()
self._zeroconf.register_service('baremetal-introspection',
endpoint)
if not CONF.standalone:
try:
coordinator = coordination.get_coordinator(prefix='conductor')
coordinator.start(heartbeat=True)
coordinator.join_group()
except tooz.ToozError:
with excutils.save_and_reraise_exception():
LOG.critical('Failed when connecting to coordination '
'backend.')
self.del_host()
else:
LOG.info('Successfully connected to coordination backend.')
def del_host(self):
if not self._shutting_down.acquire(blocking=False):
LOG.warning('Attempted to shut down while already shutting down')
return
pxe_filter.driver().tear_down_filter()
if self._periodics_worker is not None:
try:
self._periodics_worker.stop()
self._periodics_worker.wait()
except Exception as e:
LOG.exception('Service error occurred when stopping '
'periodic workers. Error: %s', e)
self._periodics_worker = None
if utils.executor().alive:
utils.executor().shutdown(wait=True)
self._shutting_down.release()
LOG.info('Shut down successfully')
def _process_node(node_info, node, introspection_data):
# NOTE(dtantsur): repeat the check in case something changed
ir_utils.check_provision_state(node)
_run_post_hooks(node_info, introspection_data)
_store_data(node_info.uuid, introspection_data)
ironic = ir_utils.get_client()
pxe_filter.driver().sync(ironic)
node_info.invalidate_cache()
rules.apply(node_info, introspection_data)
resp = {'uuid': node.uuid}
utils.executor().submit(_finish,
node_info,
ironic,
introspection_data,
power_off=CONF.processing.power_off)
return resp
def _background_introspect_locked(node_info, ironic):
# TODO(dtantsur): pagination
macs = list(node_info.ports())
if macs:
node_info.add_attribute(node_cache.MACS_ATTRIBUTE, macs)
LOG.info('Whitelisting MAC\'s %s for a PXE boot', macs,
node_info=node_info)
pxe_filter.driver().sync(ironic)
attrs = node_info.attributes
if CONF.processing.node_not_found_hook is None and not attrs:
raise utils.Error(
_('No lookup attributes were found, inspector won\'t '
'be able to find it after introspection, consider creating '
'ironic ports or providing an IPMI address'),
node_info=node_info)
LOG.info('The following attributes will be used for look up: %s',
attrs, node_info=node_info)
if node_info.manage_boot:
try:
ironic.node.set_boot_device(node_info.uuid, 'pxe',
persistent=False)
except Exception as exc:
raise utils.Error(_('Failed to set boot device to PXE: %s') % exc,
node_info=node_info)
try:
ironic.node.set_power_state(node_info.uuid, 'reboot')
except Exception as exc:
raise utils.Error(_('Failed to power on the node, check it\'s '
'power management configuration: %s') % exc,
node_info=node_info)
LOG.info('Introspection started successfully',
node_info=node_info)
else:
LOG.info('Introspection environment is ready, external power on '
'is required within %d seconds', CONF.timeout,
node_info=node_info)
def _abort(node_info, ironic):
# runs in background
LOG.debug('Forcing power-off', node_info=node_info)
if node_info.manage_boot:
try:
ironic.node.set_power_state(node_info.uuid, 'off')
except Exception as exc:
LOG.warning('Failed to power off node: %s', exc,
node_info=node_info)
node_info.finished(istate.Events.abort_end,
error=_('Canceled by operator'))
# block this node from PXE Booting the introspection image
try:
pxe_filter.driver().sync(ironic)
except Exception as exc:
# Note(mkovacik): this will be retried in the PXE filter sync
# periodic task; we continue aborting
LOG.warning('Failed to sync the PXE filter: %s', exc,
node_info=node_info)
LOG.info('Introspection aborted', node_info=node_info)
def del_host(self):
"""Shutdown the ironic inspector conductor service."""
if not CONF.standalone:
try:
coordinator = coordination.get_coordinator(prefix='conductor')
if coordinator.started:
coordinator.leave_group()
coordinator.stop()
except tooz.ToozError:
LOG.exception('Failed to stop coordinator')
if not self._shutting_down.acquire(blocking=False):
LOG.warning('Attempted to shut down while already shutting down')
return
pxe_filter.driver().tear_down_filter()
if self._periodics_worker is not None:
try:
self._periodics_worker.stop()
self._periodics_worker.wait()
except Exception as e:
LOG.exception(
'Service error occurred when stopping '
'periodic workers. Error: %s', e)
self._periodics_worker = None
if utils.executor().alive:
utils.executor().shutdown(wait=True)
if self._zeroconf is not None:
self._zeroconf.close()
self._zeroconf = None
self._shutting_down.release()
LOG.info('Shut down successfully')
def init_host(self):
"""Initialize Worker host
Init db connection, load and validate processing
hooks, runs periodic tasks.
:returns None
"""
if CONF.processing.store_data == 'none':
LOG.warning('Introspection data will not be stored. Change '
'"[processing] store_data" option if this is not '
'the desired behavior')
elif CONF.processing.store_data == 'swift':
LOG.info(
'Introspection data will be stored in Swift in the '
'container %s', CONF.swift.container)
db.init()
try:
hooks = plugins_base.validate_processing_hooks()
except Exception as exc:
LOG.critical(str(exc))
sys.exit(1)
LOG.info('Enabled processing hooks: %s', [h.name for h in hooks])
driver = pxe_filter.driver()
driver.init_filter()
periodic_clean_up_ = periodics.periodic(
spacing=CONF.clean_up_period)(periodic_clean_up)
self._periodics_worker = periodics.PeriodicWorker(
callables=[(driver.get_periodic_sync_task(), None, None),
(periodic_clean_up_, None, None)],
executor_factory=periodics.ExistingExecutor(utils.executor()),
on_failure=self._periodics_watchdog)
utils.executor().submit(self._periodics_worker.start)
if CONF.enable_mdns:
endpoint = keystone.get_endpoint('service_catalog')
self._zeroconf = mdns.Zeroconf()
self._zeroconf.register_service('baremetal-introspection',
endpoint)
def init_host(self):
"""Initialize Worker host
Init db connection, load and validate processing
hooks, runs periodic tasks.
:returns None
"""
if CONF.processing.store_data == 'none':
LOG.warning('Introspection data will not be stored. Change '
'"[processing] store_data" option if this is not '
'the desired behavior')
elif CONF.processing.store_data == 'swift':
LOG.info('Introspection data will be stored in Swift in the '
'container %s', CONF.swift.container)
db.init()
try:
hooks = plugins_base.validate_processing_hooks()
except Exception as exc:
LOG.critical(str(exc))
sys.exit(1)
LOG.info('Enabled processing hooks: %s', [h.name for h in hooks])
driver = pxe_filter.driver()
driver.init_filter()
periodic_clean_up_ = periodics.periodic(
spacing=CONF.clean_up_period
)(periodic_clean_up)
self._periodics_worker = periodics.PeriodicWorker(
callables=[(driver.get_periodic_sync_task(), None, None),
(periodic_clean_up_, None, None)],
executor_factory=periodics.ExistingExecutor(utils.executor()),
on_failure=self._periodics_watchdog)
utils.executor().submit(self._periodics_worker.start)
def test_driver(self):
ret = pxe_filter.driver()
self.assertIs(self.mock_driver, ret)
self.mock__driver_manager.assert_called_once_with()
def periodic_clean_up(): # pragma: no cover
if node_cache.clean_up():
pxe_filter.driver().sync(ir_utils.get_client())
def init_host(self):
"""Initialize Worker host
Init db connection, load and validate processing
hooks, runs periodic tasks.
:returns None
"""
if CONF.processing.store_data == 'none':
LOG.warning('Introspection data will not be stored. Change '
'"[processing] store_data" option if this is not '
'the desired behavior')
else:
LOG.info('Introspection data will be stored in the %s backend',
CONF.processing.store_data)
db.init()
self.coordinator = None
try:
self.coordinator = coordination.get_coordinator(prefix='conductor')
self.coordinator.start(heartbeat=True)
self.coordinator.join_group()
except Exception as exc:
if CONF.standalone:
LOG.info(
'Coordination backend cannot be started, assuming '
'no other instances are running. Error: %s', exc)
self.coordinator = None
else:
with excutils.save_and_reraise_exception():
LOG.critical(
'Failure when connecting to coordination '
'backend',
exc_info=True)
self.del_host()
else:
LOG.info('Successfully connected to coordination backend.')
try:
hooks = plugins_base.validate_processing_hooks()
except Exception as exc:
LOG.critical(str(exc))
sys.exit(1)
LOG.info('Enabled processing hooks: %s', [h.name for h in hooks])
driver = pxe_filter.driver()
driver.init_filter()
periodic_clean_up_ = periodics.periodic(
spacing=CONF.clean_up_period,
enabled=(CONF.clean_up_period != 0))(periodic_clean_up)
sync_with_ironic_ = periodics.periodic(
spacing=CONF.clean_up_period,
enabled=(CONF.clean_up_period != 0))(sync_with_ironic)
callables = [(periodic_clean_up_, None, None),
(sync_with_ironic_, (self, ), None)]
driver_task = driver.get_periodic_sync_task()
if driver_task is not None:
callables.append((driver_task, None, None))
# run elections periodically if we have a coordinator
# that we were able to start
if (self.coordinator and self.coordinator.started):
periodic_leader_election_ = periodics.periodic(
spacing=CONF.leader_election_interval)(
periodic_leader_election)
callables.append((periodic_leader_election_, (self, ), None))
self._periodics_worker = periodics.PeriodicWorker(
callables=callables,
executor_factory=periodics.ExistingExecutor(utils.executor()),
on_failure=self._periodics_watchdog)
utils.executor().submit(self._periodics_worker.start)
if CONF.enable_mdns:
endpoint = keystone.get_endpoint('service_catalog')
self._zeroconf = mdns.Zeroconf()
self._zeroconf.register_service('baremetal-introspection',
endpoint)
