def _process_node(node, introspection_data, node_info):
# NOTE(dtantsur): repeat the check in case something changed
ir_utils.check_provision_state(node)
node_info.create_ports(introspection_data.get('macs') or ())
_run_post_hooks(node_info, introspection_data)
_store_data(node_info, introspection_data)
ironic = ir_utils.get_client()
firewall.update_filters(ironic)
node_info.invalidate_cache()
rules.apply(node_info, introspection_data)
resp = {'uuid': node.uuid}
if node_info.options.get('new_ipmi_credentials'):
new_username, new_password = (
node_info.options.get('new_ipmi_credentials'))
utils.executor().submit(_finish_set_ipmi_credentials,
ironic, node, node_info, introspection_data,
new_username, new_password)
resp['ipmi_setup_credentials'] = True
resp['ipmi_username'] = new_username
resp['ipmi_password'] = new_password
else:
utils.executor().submit(_finish, ironic, node_info, introspection_data)
return resp
def introspect(node_id, manage_boot=True, token=None):
"""Initiate hardware properties introspection for a given node.
:param node_id: node UUID or name
:param manage_boot: whether to manage boot for this node
:param token: authentication token
:raises: Error
"""
ironic = ir_utils.get_client(token)
node = ir_utils.get_node(node_id, ironic=ironic)
ir_utils.check_provision_state(node)
if manage_boot:
validation = ironic.node.validate(node.uuid)
if not validation.power['result']:
msg = _('Failed validation of power interface, reason: %s')
raise utils.Error(msg % validation.power['reason'],
node_info=node)
bmc_address, bmc_ipv4, bmc_ipv6 = ir_utils.get_ipmi_address(node)
lookup_attrs = list(filter(None, [bmc_ipv4, bmc_ipv6]))
node_info = node_cache.start_introspection(node.uuid,
bmc_address=lookup_attrs,
manage_boot=manage_boot,
ironic=ironic)
utils.executor().submit(_background_introspect, node_info, ironic)
def introspect(node_id, manage_boot=True, token=None):
"""Initiate hardware properties introspection for a given node.
:param node_id: node UUID or name
:param manage_boot: whether to manage boot for this node
:param token: authentication token
:raises: Error
"""
ironic = ir_utils.get_client(token)
node = ir_utils.get_node(node_id, ironic=ironic)
ir_utils.check_provision_state(node)
if manage_boot:
try:
ironic.validate_node(node.id, required='power')
except os_exc.ValidationException as exc:
msg = _('Failed validation of power interface: %s')
raise utils.Error(msg % exc, node_info=node)
bmc_address, bmc_ipv4, bmc_ipv6 = ir_utils.get_ipmi_address(node)
lookup_attrs = list(filter(None, [bmc_ipv4, bmc_ipv6]))
node_info = node_cache.start_introspection(node.id,
bmc_address=lookup_attrs,
manage_boot=manage_boot,
ironic=ironic)
if manage_boot:
try:
utils.executor().submit(_do_introspect, node_info, ironic)
except Exception as exc:
msg = _('Failed to submit introspection job: %s')
raise utils.Error(msg % exc, node_info=node)
else:
_do_introspect(node_info, ironic)
def introspect(node_id, manage_boot=True, token=None):
"""Initiate hardware properties introspection for a given node.
:param node_id: node UUID or name
:param manage_boot: whether to manage boot for this node
:param token: authentication token
:raises: Error
"""
ironic = ir_utils.get_client(token)
node = ir_utils.get_node(node_id, ironic=ironic)
ir_utils.check_provision_state(node)
if manage_boot:
validation = ironic.node.validate(node.uuid)
if not validation.power['result']:
msg = _('Failed validation of power interface, reason: %s')
raise utils.Error(msg % validation.power['reason'], node_info=node)
bmc_address = ir_utils.get_ipmi_address(node)
node_info = node_cache.start_introspection(node.uuid,
bmc_address=bmc_address,
manage_boot=manage_boot,
ironic=ironic)
utils.executor().submit(_background_introspect, node_info, ironic)
def _init_host(self):
"""Initialize Worker host
Init db connection, load and validate processing
hooks, runs periodic tasks.
:returns None
"""
db.init()
try:
hooks = plugins_base.validate_processing_hooks()
except Exception as exc:
LOG.critical(str(exc))
sys.exit(1)
LOG.info('Enabled processing hooks: %s', [h.name for h in hooks])
if CONF.firewall.manage_firewall:
firewall.init()
periodic_update_ = periodics.periodic(
spacing=CONF.firewall.firewall_update_period,
enabled=CONF.firewall.manage_firewall)(periodic_update)
periodic_clean_up_ = periodics.periodic(
spacing=CONF.clean_up_period)(periodic_clean_up)
self._periodics_worker = periodics.PeriodicWorker(
callables=[(periodic_update_, None, None),
(periodic_clean_up_, None, None)],
executor_factory=periodics.ExistingExecutor(utils.executor()))
utils.executor().submit(self._periodics_worker.start)
def shutdown(self, error=None):
"""Stop serving API, clean up.
:returns: None
"""
# TODO(aarefiev): move shutdown code to WorkerService
if not self._shutting_down.acquire(blocking=False):
LOG.warning('Attempted to shut down while already shutting down')
return
LOG.debug('Shutting down')
self.rpc_server.stop()
if self._periodics_worker is not None:
try:
self._periodics_worker.stop()
self._periodics_worker.wait()
except Exception as e:
LOG.exception(
'Service error occurred when stopping '
'periodic workers. Error: %s', e)
self._periodics_worker = None
if utils.executor().alive:
utils.executor().shutdown(wait=True)
pxe_filter.driver().tear_down_filter()
self._shutting_down.release()
LOG.info('Shut down successfully')
sys.exit(error)
def _process_node(node_info, node, introspection_data):
# NOTE(dtantsur): repeat the check in case something changed
keep_power_on = ir_utils.check_provision_state(node)
_run_post_hooks(node_info, introspection_data)
store_introspection_data(node_info.uuid, introspection_data)
ironic = ir_utils.get_client()
pxe_filter.driver().sync(ironic)
node_info.invalidate_cache()
rules.apply(node_info, introspection_data)
resp = {'uuid': node.id}
# determine how to handle power
if keep_power_on or not node_info.manage_boot:
power_action = False
else:
power_action = CONF.processing.power_off
utils.executor().submit(_finish,
node_info,
ironic,
introspection_data,
power_off=power_action)
return resp
def del_host(self):
if not self._shutting_down.acquire(blocking=False):
LOG.warning('Attempted to shut down while already shutting down')
return
pxe_filter.driver().tear_down_filter()
if self._periodics_worker is not None:
try:
self._periodics_worker.stop()
self._periodics_worker.wait()
except Exception as e:
LOG.exception(
'Service error occurred when stopping '
'periodic workers. Error: %s', e)
self._periodics_worker = None
if utils.executor().alive:
utils.executor().shutdown(wait=True)
if self._zeroconf is not None:
self._zeroconf.close()
self._zeroconf = None
self._shutting_down.release()
LOG.info('Shut down successfully')
def _init_host(self):
"""Initialize Worker host
Init db connection, load and validate processing
hooks, runs periodic tasks.
:returns None
"""
db.init()
try:
hooks = plugins_base.validate_processing_hooks()
except Exception as exc:
LOG.critical(str(exc))
sys.exit(1)
LOG.info('Enabled processing hooks: %s', [h.name for h in hooks])
driver = pxe_filter.driver()
driver.init_filter()
periodic_clean_up_ = periodics.periodic(
spacing=CONF.clean_up_period)(periodic_clean_up)
self._periodics_worker = periodics.PeriodicWorker(
callables=[(driver.get_periodic_sync_task(), None, None),
(periodic_clean_up_, None, None)],
executor_factory=periodics.ExistingExecutor(utils.executor()),
on_failure=self._periodics_watchdog)
utils.executor().submit(self._periodics_worker.start)
def _process_node(node_info, node, introspection_data):
# NOTE(dtantsur): repeat the check in case something changed
ir_utils.check_provision_state(node)
interfaces = introspection_data.get('interfaces')
node_info.create_ports(list(interfaces.values()))
_run_post_hooks(node_info, introspection_data)
_store_data(node_info, introspection_data)
ironic = ir_utils.get_client()
firewall.update_filters(ironic)
node_info.invalidate_cache()
rules.apply(node_info, introspection_data)
resp = {'uuid': node.uuid}
if node_info.options.get('new_ipmi_credentials'):
new_username, new_password = (
node_info.options.get('new_ipmi_credentials'))
utils.executor().submit(_finish_set_ipmi_credentials,
node_info, ironic, node, introspection_data,
new_username, new_password)
resp['ipmi_setup_credentials'] = True
resp['ipmi_username'] = new_username
resp['ipmi_password'] = new_password
else:
utils.executor().submit(_finish, node_info, ironic, introspection_data,
power_off=CONF.processing.power_off)
return resp
def process(introspection_data):
"""Process data from the ramdisk.
This function heavily relies on the hooks to do the actual data processing.
"""
unprocessed_data = copy.deepcopy(introspection_data)
failures = []
_run_pre_hooks(introspection_data, failures)
node_info = _find_node_info(introspection_data, failures)
if node_info:
# Locking is already done in find_node() but may be not done in a
# node_not_found hook
node_info.acquire_lock()
if failures or node_info is None:
msg = _('The following failures happened during running '
'pre-processing hooks:\n%s') % '\n'.join(failures)
if node_info is not None:
node_info.finished(error='\n'.join(failures))
raise utils.Error(msg, node_info=node_info, data=introspection_data)
LOG.info(_LI('Matching node is %s'), node_info.uuid,
node_info=node_info, data=introspection_data)
if node_info.finished_at is not None:
# race condition or introspection canceled
raise utils.Error(_('Node processing already finished with '
'error: %s') % node_info.error,
node_info=node_info, code=400)
# Note(mkovacik): store data now when we're sure that a background
# thread won't race with other process() or introspect.abort()
# call
utils.executor().submit(_store_unprocessed_data, node_info,
unprocessed_data)
try:
node = node_info.node()
except exceptions.NotFound:
msg = _('Node was found in cache, but is not found in Ironic')
node_info.finished(error=msg)
raise utils.Error(msg, code=404, node_info=node_info,
data=introspection_data)
try:
return _process_node(node, introspection_data, node_info)
except utils.Error as exc:
node_info.finished(error=str(exc))
raise
except Exception as exc:
LOG.exception(_LE('Unexpected exception during processing'))
msg = _('Unexpected exception %(exc_class)s during processing: '
'%(error)s') % {'exc_class': exc.__class__.__name__,
'error': exc}
node_info.finished(error=msg)
raise utils.Error(msg, node_info=node_info, data=introspection_data)
def init_host(self):
"""Initialize Worker host
Init db connection, load and validate processing
hooks, runs periodic tasks.
:returns None
"""
if CONF.processing.store_data == 'none':
LOG.warning('Introspection data will not be stored. Change '
'"[processing] store_data" option if this is not '
'the desired behavior')
else:
LOG.info('Introspection data will be stored in the %s backend',
CONF.processing.store_data)
db.init()
try:
hooks = plugins_base.validate_processing_hooks()
except Exception as exc:
LOG.critical(str(exc))
sys.exit(1)
LOG.info('Enabled processing hooks: %s', [h.name for h in hooks])
driver = pxe_filter.driver()
driver.init_filter()
periodic_clean_up_ = periodics.periodic(
spacing=CONF.clean_up_period)(periodic_clean_up)
self._periodics_worker = periodics.PeriodicWorker(
callables=[(driver.get_periodic_sync_task(), None, None),
(periodic_clean_up_, None, None)],
executor_factory=periodics.ExistingExecutor(utils.executor()),
on_failure=self._periodics_watchdog)
utils.executor().submit(self._periodics_worker.start)
if CONF.enable_mdns:
endpoint = keystone.get_endpoint('service_catalog')
self._zeroconf = mdns.Zeroconf()
self._zeroconf.register_service('baremetal-introspection',
endpoint)
if not CONF.standalone:
try:
coordinator = coordination.get_coordinator(prefix='conductor')
coordinator.start(heartbeat=True)
coordinator.join_group()
except tooz.ToozError:
with excutils.save_and_reraise_exception():
LOG.critical('Failed when connecting to coordination '
'backend.')
self.del_host()
else:
LOG.info('Successfully connected to coordination backend.')
def shutdown(self):
LOG.debug('Shutting down')
firewall.clean_up()
if self._periodics_worker is not None:
self._periodics_worker.stop()
self._periodics_worker.wait()
self._periodics_worker = None
if utils.executor().alive:
utils.executor().shutdown(wait=True)
LOG.info(_LI('Shut down successfully'))
def shutdown(self):
LOG.debug('Shutting down')
firewall.clean_up()
if self._periodics_worker is not None:
self._periodics_worker.stop()
self._periodics_worker.wait()
self._periodics_worker = None
if utils.executor().alive:
utils.executor().shutdown(wait=True)
LOG.info(_LI('Shut down successfully'))
def introspect(node_id, token=None):
"""Initiate hardware properties introspection for a given node.
:param node_id: node UUID or name
:param token: authentication token
:raises: Error
"""
ironic = ir_utils.get_client(token)
node = ir_utils.get_node(node_id, ironic=ironic)
ir_utils.check_provision_state(node)
validation = ironic.node.validate(node.uuid)
if not validation.power['result']:
msg = _('Failed validation of power interface, reason: %s')
raise utils.Error(msg % validation.power['reason'], node_info=node)
bmc_address = ir_utils.get_ipmi_address(node)
node_info = node_cache.start_introspection(node.uuid,
bmc_address=bmc_address,
ironic=ironic)
def _handle_exceptions(fut):
try:
fut.result()
except utils.Error as exc:
# Logging has already happened in Error.__init__
node_info.finished(error=str(exc))
except Exception as exc:
msg = _('Unexpected exception in background introspection thread')
LOG.exception(msg, node_info=node_info)
node_info.finished(error=msg)
future = utils.executor().submit(_background_introspect, ironic, node_info)
future.add_done_callback(_handle_exceptions)
def init(self):
if utils.get_auth_strategy() != 'noauth':
utils.add_auth_middleware(app)
else:
LOG.warning(_LW('Starting unauthenticated, please check'
' configuration'))
if CONF.processing.store_data == 'none':
LOG.warning(_LW('Introspection data will not be stored. Change '
'"[processing] store_data" option if this is not '
'the desired behavior'))
elif CONF.processing.store_data == 'swift':
LOG.info(_LI('Introspection data will be stored in Swift in the '
'container %s'), CONF.swift.container)
utils.add_cors_middleware(app)
db.init()
try:
hooks = [ext.name for ext in
plugins_base.processing_hooks_manager()]
except KeyError as exc:
# callback function raises MissingHookError derived from KeyError
# on missing hook
LOG.critical(_LC('Hook(s) %s failed to load or was not found'),
str(exc))
sys.exit(1)
LOG.info(_LI('Enabled processing hooks: %s'), hooks)
if CONF.firewall.manage_firewall:
firewall.init()
periodic_update_ = periodics.periodic(
spacing=CONF.firewall.firewall_update_period,
enabled=CONF.firewall.manage_firewall
)(periodic_update)
periodic_clean_up_ = periodics.periodic(
spacing=CONF.clean_up_period
)(periodic_clean_up)
self._periodics_worker = periodics.PeriodicWorker(
callables=[(periodic_update_, None, None),
(periodic_clean_up_, None, None)],
executor_factory=periodics.ExistingExecutor(utils.executor()))
utils.executor().submit(self._periodics_worker.start)
def init(self):
if CONF.auth_strategy != 'noauth':
utils.add_auth_middleware(app)
else:
LOG.warning('Starting unauthenticated, please check'
' configuration')
if CONF.processing.store_data == 'none':
LOG.warning('Introspection data will not be stored. Change '
'"[processing] store_data" option if this is not '
'the desired behavior')
elif CONF.processing.store_data == 'swift':
LOG.info(
'Introspection data will be stored in Swift in the '
'container %s', CONF.swift.container)
utils.add_cors_middleware(app)
db.init()
try:
hooks = [
ext.name for ext in plugins_base.processing_hooks_manager()
]
except KeyError as exc:
# callback function raises MissingHookError derived from KeyError
# on missing hook
LOG.critical('Hook(s) %s failed to load or was not found',
str(exc))
sys.exit(1)
LOG.info('Enabled processing hooks: %s', hooks)
if CONF.firewall.manage_firewall:
firewall.init()
periodic_update_ = periodics.periodic(
spacing=CONF.firewall.firewall_update_period,
enabled=CONF.firewall.manage_firewall)(periodic_update)
periodic_clean_up_ = periodics.periodic(
spacing=CONF.clean_up_period)(periodic_clean_up)
self._periodics_worker = periodics.PeriodicWorker(
callables=[(periodic_update_, None, None),
(periodic_clean_up_, None, None)],
executor_factory=periodics.ExistingExecutor(utils.executor()))
utils.executor().submit(self._periodics_worker.start)
def _process_node(node, introspection_data, node_info):
# NOTE(dtantsur): repeat the check in case something changed
ir_utils.check_provision_state(node)
node_info.create_ports(introspection_data.get('macs') or ())
_run_post_hooks(node_info, introspection_data)
if CONF.processing.store_data == 'swift':
stored_data = {k: v for k, v in introspection_data.items()
if k not in _STORAGE_EXCLUDED_KEYS}
swift_object_name = swift.store_introspection_data(stored_data,
node_info.uuid)
LOG.info(_LI('Introspection data was stored in Swift in object %s'),
swift_object_name,
node_info=node_info, data=introspection_data)
if CONF.processing.store_data_location:
node_info.patch([{'op': 'add', 'path': '/extra/%s' %
CONF.processing.store_data_location,
'value': swift_object_name}])
else:
LOG.debug('Swift support is disabled, introspection data '
'won\'t be stored',
node_info=node_info, data=introspection_data)
ironic = ir_utils.get_client()
firewall.update_filters(ironic)
node_info.invalidate_cache()
rules.apply(node_info, introspection_data)
resp = {'uuid': node.uuid}
if node_info.options.get('new_ipmi_credentials'):
new_username, new_password = (
node_info.options.get('new_ipmi_credentials'))
utils.executor().submit(_finish_set_ipmi_credentials,
ironic, node, node_info, introspection_data,
new_username, new_password)
resp['ipmi_setup_credentials'] = True
resp['ipmi_username'] = new_username
resp['ipmi_password'] = new_password
else:
utils.executor().submit(_finish, ironic, node_info, introspection_data)
return resp
def init_host(self):
"""Initialize Worker host
Init db connection, load and validate processing
hooks, runs periodic tasks.
:returns None
"""
if CONF.processing.store_data == 'none':
LOG.warning('Introspection data will not be stored. Change '
'"[processing] store_data" option if this is not '
'the desired behavior')
elif CONF.processing.store_data == 'swift':
LOG.info(
'Introspection data will be stored in Swift in the '
'container %s', CONF.swift.container)
db.init()
try:
hooks = plugins_base.validate_processing_hooks()
except Exception as exc:
LOG.critical(str(exc))
sys.exit(1)
LOG.info('Enabled processing hooks: %s', [h.name for h in hooks])
driver = pxe_filter.driver()
driver.init_filter()
periodic_clean_up_ = periodics.periodic(
spacing=CONF.clean_up_period)(periodic_clean_up)
self._periodics_worker = periodics.PeriodicWorker(
callables=[(driver.get_periodic_sync_task(), None, None),
(periodic_clean_up_, None, None)],
executor_factory=periodics.ExistingExecutor(utils.executor()),
on_failure=self._periodics_watchdog)
utils.executor().submit(self._periodics_worker.start)
if CONF.enable_mdns:
endpoint = keystone.get_endpoint('service_catalog')
self._zeroconf = mdns.Zeroconf()
self._zeroconf.register_service('baremetal-introspection',
endpoint)
def abort(node_id, token=None):
"""Abort running introspection.
:param node_id: node UUID or name
:param token: authentication token
:raises: Error
"""
LOG.debug('Aborting introspection for node %s', node_id)
ironic = ir_utils.get_client(token)
node_info = node_cache.get_node(node_id, ironic=ironic)
# check pending operations
locked = node_info.acquire_lock(blocking=False)
if not locked:
# Node busy --- cannot abort atm
raise utils.Error(_('Node is locked, please, retry later'),
node_info=node_info, code=409)
utils.executor().submit(_abort, node_info, ironic)
def abort(uuid, token=None):
"""Abort running introspection.
:param uuid: node uuid
:param token: authentication token
:raises: Error
"""
LOG.debug('Aborting introspection for node %s', uuid)
ironic = ir_utils.get_client(token)
node_info = node_cache.get_node(uuid, ironic=ironic, locked=False)
# check pending operations
locked = node_info.acquire_lock(blocking=False)
if not locked:
# Node busy --- cannot abort atm
raise utils.Error(_('Node is locked, please, retry later'),
node_info=node_info, code=409)
utils.executor().submit(_abort, node_info, ironic)
def init(self):
if utils.get_auth_strategy() != 'noauth':
utils.add_auth_middleware(app)
else:
LOG.warning(
_LW('Starting unauthenticated, please check'
' configuration'))
if CONF.processing.store_data == 'none':
LOG.warning(
_LW('Introspection data will not be stored. Change '
'"[processing] store_data" option if this is not '
'the desired behavior'))
elif CONF.processing.store_data == 'swift':
LOG.info(
_LI('Introspection data will be stored in Swift in the '
'container %s'), CONF.swift.container)
utils.add_cors_middleware(app)
db.init()
try:
hooks = [
ext.name for ext in plugins_base.processing_hooks_manager()
]
except KeyError as exc:
# stevedore raises KeyError on missing hook
LOG.critical(_LC('Hook %s failed to load or was not found'),
str(exc))
sys.exit(1)
LOG.info(_LI('Enabled processing hooks: %s'), hooks)
if CONF.firewall.manage_firewall:
firewall.init()
self._periodics_worker = periodics.PeriodicWorker(
callables=[(periodic_update, None, None),
(periodic_clean_up, None, None)],
executor_factory=periodics.ExistingExecutor(utils.executor()))
utils.executor().submit(self._periodics_worker.start)
def del_host(self):
if not self._shutting_down.acquire(blocking=False):
LOG.warning('Attempted to shut down while already shutting down')
return
pxe_filter.driver().tear_down_filter()
if self._periodics_worker is not None:
try:
self._periodics_worker.stop()
self._periodics_worker.wait()
except Exception as e:
LOG.exception('Service error occurred when stopping '
'periodic workers. Error: %s', e)
self._periodics_worker = None
if utils.executor().alive:
utils.executor().shutdown(wait=True)
self._shutting_down.release()
LOG.info('Shut down successfully')
def _process_node(node_info, node, introspection_data):
# NOTE(dtantsur): repeat the check in case something changed
ir_utils.check_provision_state(node)
_run_post_hooks(node_info, introspection_data)
_store_data(node_info.uuid, introspection_data)
ironic = ir_utils.get_client()
pxe_filter.driver().sync(ironic)
node_info.invalidate_cache()
rules.apply(node_info, introspection_data)
resp = {'uuid': node.uuid}
utils.executor().submit(_finish,
node_info,
ironic,
introspection_data,
power_off=CONF.processing.power_off)
return resp
def introspect(uuid, new_ipmi_credentials=None, token=None):
"""Initiate hardware properties introspection for a given node.
:param uuid: node uuid
:param new_ipmi_credentials: tuple (new username, new password) or None
:param token: authentication token
:raises: Error
"""
ironic = ir_utils.get_client(token)
try:
node = ironic.node.get(uuid)
except exceptions.NotFound:
raise utils.Error(_("Cannot find node %s") % uuid, code=404)
except exceptions.HttpError as exc:
raise utils.Error(
_("Cannot get node %(node)s: %(exc)s") % {
'node': uuid,
'exc': exc
})
ir_utils.check_provision_state(node, with_credentials=new_ipmi_credentials)
if new_ipmi_credentials:
new_ipmi_credentials = (_validate_ipmi_credentials(
node, new_ipmi_credentials))
else:
validation = ironic.node.validate(node.uuid)
if not validation.power['result']:
msg = _('Failed validation of power interface, reason: %s')
raise utils.Error(msg % validation.power['reason'], node_info=node)
bmc_address = ir_utils.get_ipmi_address(node)
node_info = node_cache.add_node(node.uuid,
bmc_address=bmc_address,
ironic=ironic)
node_info.set_option('new_ipmi_credentials', new_ipmi_credentials)
def _handle_exceptions(fut):
try:
fut.result()
except utils.Error as exc:
# Logging has already happened in Error.__init__
node_info.finished(error=str(exc))
except Exception as exc:
msg = _('Unexpected exception in background introspection thread')
LOG.exception(msg, node_info=node_info)
node_info.finished(error=msg)
future = utils.executor().submit(_background_introspect, ironic, node_info)
future.add_done_callback(_handle_exceptions)
def init_host(self):
"""Initialize Worker host
Init db connection, load and validate processing
hooks, runs periodic tasks.
:returns None
"""
if CONF.processing.store_data == 'none':
LOG.warning('Introspection data will not be stored. Change '
'"[processing] store_data" option if this is not '
'the desired behavior')
elif CONF.processing.store_data == 'swift':
LOG.info('Introspection data will be stored in Swift in the '
'container %s', CONF.swift.container)
db.init()
try:
hooks = plugins_base.validate_processing_hooks()
except Exception as exc:
LOG.critical(str(exc))
sys.exit(1)
LOG.info('Enabled processing hooks: %s', [h.name for h in hooks])
driver = pxe_filter.driver()
driver.init_filter()
periodic_clean_up_ = periodics.periodic(
spacing=CONF.clean_up_period
)(periodic_clean_up)
self._periodics_worker = periodics.PeriodicWorker(
callables=[(driver.get_periodic_sync_task(), None, None),
(periodic_clean_up_, None, None)],
executor_factory=periodics.ExistingExecutor(utils.executor()),
on_failure=self._periodics_watchdog)
utils.executor().submit(self._periodics_worker.start)
def reapply(node_ident):
"""Re-apply introspection steps.
Re-apply preprocessing, postprocessing and introspection rules on
stored data.
:param node_ident: node UUID or name
:raises: utils.Error
"""
LOG.debug('Processing re-apply introspection request for node '
'UUID: %s', node_ident)
node_info = node_cache.get_node(node_ident, locked=False)
if not node_info.acquire_lock(blocking=False):
# Note (mkovacik): it should be sufficient to check data
# presence & locking. If either introspection didn't start
# yet, was in waiting state or didn't finish yet, either data
# won't be available or locking would fail
raise utils.Error(_('Node locked, please, try again later'),
node_info=node_info, code=409)
utils.executor().submit(_reapply, node_info)
def reapply(uuid):
"""Re-apply introspection steps.
Re-apply preprocessing, postprocessing and introspection rules on
stored data.
:param uuid: node uuid to use
:raises: utils.Error
"""
LOG.debug('Processing re-apply introspection request for node '
'UUID: %s', uuid)
node_info = node_cache.get_node(uuid, locked=False)
if not node_info.acquire_lock(blocking=False):
# Note (mkovacik): it should be sufficient to check data
# presence & locking. If either introspection didn't start
# yet, was in waiting state or didn't finish yet, either data
# won't be available or locking would fail
raise utils.Error(_('Node locked, please, try again later'),
node_info=node_info, code=409)
utils.executor().submit(_reapply, node_info)
def del_host(self):
"""Shutdown the ironic inspector conductor service."""
if not CONF.standalone:
try:
coordinator = coordination.get_coordinator(prefix='conductor')
if coordinator.started:
coordinator.leave_group()
coordinator.stop()
except tooz.ToozError:
LOG.exception('Failed to stop coordinator')
if not self._shutting_down.acquire(blocking=False):
LOG.warning('Attempted to shut down while already shutting down')
return
pxe_filter.driver().tear_down_filter()
if self._periodics_worker is not None:
try:
self._periodics_worker.stop()
self._periodics_worker.wait()
except Exception as e:
LOG.exception(
'Service error occurred when stopping '
'periodic workers. Error: %s', e)
self._periodics_worker = None
if utils.executor().alive:
utils.executor().shutdown(wait=True)
if self._zeroconf is not None:
self._zeroconf.close()
self._zeroconf = None
self._shutting_down.release()
LOG.info('Shut down successfully')
def introspect(uuid, new_ipmi_credentials=None, token=None):
"""Initiate hardware properties introspection for a given node.
:param uuid: node uuid
:param new_ipmi_credentials: tuple (new username, new password) or None
:param token: authentication token
:raises: Error
"""
ironic = ir_utils.get_client(token)
try:
node = ironic.node.get(uuid)
except exceptions.NotFound:
raise utils.Error(_("Cannot find node %s") % uuid, code=404)
except exceptions.HttpError as exc:
raise utils.Error(_("Cannot get node %(node)s: %(exc)s") %
{'node': uuid, 'exc': exc})
ir_utils.check_provision_state(node, with_credentials=new_ipmi_credentials)
if new_ipmi_credentials:
new_ipmi_credentials = (
_validate_ipmi_credentials(node, new_ipmi_credentials))
else:
validation = ironic.node.validate(node.uuid)
if not validation.power['result']:
msg = _('Failed validation of power interface, reason: %s')
raise utils.Error(msg % validation.power['reason'],
node_info=node)
bmc_address = ir_utils.get_ipmi_address(node)
node_info = node_cache.add_node(node.uuid,
bmc_address=bmc_address,
ironic=ironic)
node_info.set_option('new_ipmi_credentials', new_ipmi_credentials)
def _handle_exceptions(fut):
try:
fut.result()
except utils.Error as exc:
# Logging has already happened in Error.__init__
node_info.finished(error=str(exc))
except Exception as exc:
msg = _('Unexpected exception in background introspection thread')
LOG.exception(msg, node_info=node_info)
node_info.finished(error=msg)
future = utils.executor().submit(_background_introspect, ironic, node_info)
future.add_done_callback(_handle_exceptions)
def shutdown(self):
"""Stop serving API, clean up.
:returns: None
"""
# TODO(aarefiev): move shutdown code to WorkerService
LOG.debug('Shutting down')
firewall.clean_up()
if self._periodics_worker is not None:
try:
self._periodics_worker.stop()
self._periodics_worker.wait()
except Exception as e:
LOG.exception(
'Service error occurred when stopping '
'periodic workers. Error: %s', e)
self._periodics_worker = None
if utils.executor().alive:
utils.executor().shutdown(wait=True)
LOG.info('Shut down successfully')
def process(introspection_data):
"""Process data from the ramdisk.
This function heavily relies on the hooks to do the actual data processing.
"""
unprocessed_data = copy.deepcopy(introspection_data)
failures = []
_run_pre_hooks(introspection_data, failures)
node_info = _find_node_info(introspection_data, failures)
if node_info:
# Locking is already done in find_node() but may be not done in a
# node_not_found hook
node_info.acquire_lock()
if failures or node_info is None:
msg = _('The following failures happened during running '
'pre-processing hooks:\n%s') % '\n'.join(failures)
if node_info is not None:
node_info.finished(error='\n'.join(failures))
_store_logs(introspection_data, node_info)
raise utils.Error(msg, node_info=node_info, data=introspection_data)
LOG.info(_LI('Matching node is %s'), node_info.uuid,
node_info=node_info, data=introspection_data)
if node_info.finished_at is not None:
# race condition or introspection canceled
raise utils.Error(_('Node processing already finished with '
'error: %s') % node_info.error,
node_info=node_info, code=400)
# Note(mkovacik): store data now when we're sure that a background
# thread won't race with other process() or introspect.abort()
# call
utils.executor().submit(_store_unprocessed_data, node_info,
unprocessed_data)
try:
node = node_info.node()
except ir_utils.NotFound as exc:
with excutils.save_and_reraise_exception():
node_info.finished(error=str(exc))
_store_logs(introspection_data, node_info)
try:
result = _process_node(node_info, node, introspection_data)
except utils.Error as exc:
node_info.finished(error=str(exc))
with excutils.save_and_reraise_exception():
_store_logs(introspection_data, node_info)
except Exception as exc:
LOG.exception(_LE('Unexpected exception during processing'))
msg = _('Unexpected exception %(exc_class)s during processing: '
'%(error)s') % {'exc_class': exc.__class__.__name__,
'error': exc}
node_info.finished(error=msg)
_store_logs(introspection_data, node_info)
raise utils.Error(msg, node_info=node_info, data=introspection_data,
code=500)
if CONF.processing.always_store_ramdisk_logs:
_store_logs(introspection_data, node_info)
return result
def init_host(self):
"""Initialize Worker host
Init db connection, load and validate processing
hooks, runs periodic tasks.
:returns None
"""
if CONF.processing.store_data == 'none':
LOG.warning('Introspection data will not be stored. Change '
'"[processing] store_data" option if this is not '
'the desired behavior')
else:
LOG.info('Introspection data will be stored in the %s backend',
CONF.processing.store_data)
db.init()
self.coordinator = None
try:
self.coordinator = coordination.get_coordinator(prefix='conductor')
self.coordinator.start(heartbeat=True)
self.coordinator.join_group()
except Exception as exc:
if CONF.standalone:
LOG.info(
'Coordination backend cannot be started, assuming '
'no other instances are running. Error: %s', exc)
self.coordinator = None
else:
with excutils.save_and_reraise_exception():
LOG.critical(
'Failure when connecting to coordination '
'backend',
exc_info=True)
self.del_host()
else:
LOG.info('Successfully connected to coordination backend.')
try:
hooks = plugins_base.validate_processing_hooks()
except Exception as exc:
LOG.critical(str(exc))
sys.exit(1)
LOG.info('Enabled processing hooks: %s', [h.name for h in hooks])
driver = pxe_filter.driver()
driver.init_filter()
periodic_clean_up_ = periodics.periodic(
spacing=CONF.clean_up_period,
enabled=(CONF.clean_up_period != 0))(periodic_clean_up)
sync_with_ironic_ = periodics.periodic(
spacing=CONF.clean_up_period,
enabled=(CONF.clean_up_period != 0))(sync_with_ironic)
callables = [(periodic_clean_up_, None, None),
(sync_with_ironic_, (self, ), None)]
driver_task = driver.get_periodic_sync_task()
if driver_task is not None:
callables.append((driver_task, None, None))
# run elections periodically if we have a coordinator
# that we were able to start
if (self.coordinator and self.coordinator.started):
periodic_leader_election_ = periodics.periodic(
spacing=CONF.leader_election_interval)(
periodic_leader_election)
callables.append((periodic_leader_election_, (self, ), None))
self._periodics_worker = periodics.PeriodicWorker(
callables=callables,
executor_factory=periodics.ExistingExecutor(utils.executor()),
on_failure=self._periodics_watchdog)
utils.executor().submit(self._periodics_worker.start)
if CONF.enable_mdns:
endpoint = keystone.get_endpoint('service_catalog')
self._zeroconf = mdns.Zeroconf()
self._zeroconf.register_service('baremetal-introspection',
endpoint)
