def _process_node(node, introspection_data, node_info): # NOTE(dtantsur): repeat the check in case something changed ir_utils.check_provision_state(node) node_info.create_ports(introspection_data.get('macs') or ()) _run_post_hooks(node_info, introspection_data) _store_data(node_info, introspection_data) ironic = ir_utils.get_client() firewall.update_filters(ironic) node_info.invalidate_cache() rules.apply(node_info, introspection_data) resp = {'uuid': node.uuid} if node_info.options.get('new_ipmi_credentials'): new_username, new_password = ( node_info.options.get('new_ipmi_credentials')) utils.executor().submit(_finish_set_ipmi_credentials, ironic, node, node_info, introspection_data, new_username, new_password) resp['ipmi_setup_credentials'] = True resp['ipmi_username'] = new_username resp['ipmi_password'] = new_password else: utils.executor().submit(_finish, ironic, node_info, introspection_data) return resp
def introspect(node_id, manage_boot=True, token=None): """Initiate hardware properties introspection for a given node. :param node_id: node UUID or name :param manage_boot: whether to manage boot for this node :param token: authentication token :raises: Error """ ironic = ir_utils.get_client(token) node = ir_utils.get_node(node_id, ironic=ironic) ir_utils.check_provision_state(node) if manage_boot: validation = ironic.node.validate(node.uuid) if not validation.power['result']: msg = _('Failed validation of power interface, reason: %s') raise utils.Error(msg % validation.power['reason'], node_info=node) bmc_address, bmc_ipv4, bmc_ipv6 = ir_utils.get_ipmi_address(node) lookup_attrs = list(filter(None, [bmc_ipv4, bmc_ipv6])) node_info = node_cache.start_introspection(node.uuid, bmc_address=lookup_attrs, manage_boot=manage_boot, ironic=ironic) utils.executor().submit(_background_introspect, node_info, ironic)
def introspect(node_id, manage_boot=True, token=None): """Initiate hardware properties introspection for a given node. :param node_id: node UUID or name :param manage_boot: whether to manage boot for this node :param token: authentication token :raises: Error """ ironic = ir_utils.get_client(token) node = ir_utils.get_node(node_id, ironic=ironic) ir_utils.check_provision_state(node) if manage_boot: try: ironic.validate_node(node.id, required='power') except os_exc.ValidationException as exc: msg = _('Failed validation of power interface: %s') raise utils.Error(msg % exc, node_info=node) bmc_address, bmc_ipv4, bmc_ipv6 = ir_utils.get_ipmi_address(node) lookup_attrs = list(filter(None, [bmc_ipv4, bmc_ipv6])) node_info = node_cache.start_introspection(node.id, bmc_address=lookup_attrs, manage_boot=manage_boot, ironic=ironic) if manage_boot: try: utils.executor().submit(_do_introspect, node_info, ironic) except Exception as exc: msg = _('Failed to submit introspection job: %s') raise utils.Error(msg % exc, node_info=node) else: _do_introspect(node_info, ironic)
def introspect(node_id, manage_boot=True, token=None): """Initiate hardware properties introspection for a given node. :param node_id: node UUID or name :param manage_boot: whether to manage boot for this node :param token: authentication token :raises: Error """ ironic = ir_utils.get_client(token) node = ir_utils.get_node(node_id, ironic=ironic) ir_utils.check_provision_state(node) if manage_boot: validation = ironic.node.validate(node.uuid) if not validation.power['result']: msg = _('Failed validation of power interface, reason: %s') raise utils.Error(msg % validation.power['reason'], node_info=node) bmc_address = ir_utils.get_ipmi_address(node) node_info = node_cache.start_introspection(node.uuid, bmc_address=bmc_address, manage_boot=manage_boot, ironic=ironic) utils.executor().submit(_background_introspect, node_info, ironic)
def _init_host(self): """Initialize Worker host Init db connection, load and validate processing hooks, runs periodic tasks. :returns None """ db.init() try: hooks = plugins_base.validate_processing_hooks() except Exception as exc: LOG.critical(str(exc)) sys.exit(1) LOG.info('Enabled processing hooks: %s', [h.name for h in hooks]) if CONF.firewall.manage_firewall: firewall.init() periodic_update_ = periodics.periodic( spacing=CONF.firewall.firewall_update_period, enabled=CONF.firewall.manage_firewall)(periodic_update) periodic_clean_up_ = periodics.periodic( spacing=CONF.clean_up_period)(periodic_clean_up) self._periodics_worker = periodics.PeriodicWorker( callables=[(periodic_update_, None, None), (periodic_clean_up_, None, None)], executor_factory=periodics.ExistingExecutor(utils.executor())) utils.executor().submit(self._periodics_worker.start)
def shutdown(self, error=None): """Stop serving API, clean up. :returns: None """ # TODO(aarefiev): move shutdown code to WorkerService if not self._shutting_down.acquire(blocking=False): LOG.warning('Attempted to shut down while already shutting down') return LOG.debug('Shutting down') self.rpc_server.stop() if self._periodics_worker is not None: try: self._periodics_worker.stop() self._periodics_worker.wait() except Exception as e: LOG.exception( 'Service error occurred when stopping ' 'periodic workers. Error: %s', e) self._periodics_worker = None if utils.executor().alive: utils.executor().shutdown(wait=True) pxe_filter.driver().tear_down_filter() self._shutting_down.release() LOG.info('Shut down successfully') sys.exit(error)
def _process_node(node_info, node, introspection_data): # NOTE(dtantsur): repeat the check in case something changed keep_power_on = ir_utils.check_provision_state(node) _run_post_hooks(node_info, introspection_data) store_introspection_data(node_info.uuid, introspection_data) ironic = ir_utils.get_client() pxe_filter.driver().sync(ironic) node_info.invalidate_cache() rules.apply(node_info, introspection_data) resp = {'uuid': node.id} # determine how to handle power if keep_power_on or not node_info.manage_boot: power_action = False else: power_action = CONF.processing.power_off utils.executor().submit(_finish, node_info, ironic, introspection_data, power_off=power_action) return resp
def del_host(self): if not self._shutting_down.acquire(blocking=False): LOG.warning('Attempted to shut down while already shutting down') return pxe_filter.driver().tear_down_filter() if self._periodics_worker is not None: try: self._periodics_worker.stop() self._periodics_worker.wait() except Exception as e: LOG.exception( 'Service error occurred when stopping ' 'periodic workers. Error: %s', e) self._periodics_worker = None if utils.executor().alive: utils.executor().shutdown(wait=True) if self._zeroconf is not None: self._zeroconf.close() self._zeroconf = None self._shutting_down.release() LOG.info('Shut down successfully')
def _init_host(self): """Initialize Worker host Init db connection, load and validate processing hooks, runs periodic tasks. :returns None """ db.init() try: hooks = plugins_base.validate_processing_hooks() except Exception as exc: LOG.critical(str(exc)) sys.exit(1) LOG.info('Enabled processing hooks: %s', [h.name for h in hooks]) driver = pxe_filter.driver() driver.init_filter() periodic_clean_up_ = periodics.periodic( spacing=CONF.clean_up_period)(periodic_clean_up) self._periodics_worker = periodics.PeriodicWorker( callables=[(driver.get_periodic_sync_task(), None, None), (periodic_clean_up_, None, None)], executor_factory=periodics.ExistingExecutor(utils.executor()), on_failure=self._periodics_watchdog) utils.executor().submit(self._periodics_worker.start)
def _process_node(node_info, node, introspection_data): # NOTE(dtantsur): repeat the check in case something changed ir_utils.check_provision_state(node) interfaces = introspection_data.get('interfaces') node_info.create_ports(list(interfaces.values())) _run_post_hooks(node_info, introspection_data) _store_data(node_info, introspection_data) ironic = ir_utils.get_client() firewall.update_filters(ironic) node_info.invalidate_cache() rules.apply(node_info, introspection_data) resp = {'uuid': node.uuid} if node_info.options.get('new_ipmi_credentials'): new_username, new_password = ( node_info.options.get('new_ipmi_credentials')) utils.executor().submit(_finish_set_ipmi_credentials, node_info, ironic, node, introspection_data, new_username, new_password) resp['ipmi_setup_credentials'] = True resp['ipmi_username'] = new_username resp['ipmi_password'] = new_password else: utils.executor().submit(_finish, node_info, ironic, introspection_data, power_off=CONF.processing.power_off) return resp
def process(introspection_data): """Process data from the ramdisk. This function heavily relies on the hooks to do the actual data processing. """ unprocessed_data = copy.deepcopy(introspection_data) failures = [] _run_pre_hooks(introspection_data, failures) node_info = _find_node_info(introspection_data, failures) if node_info: # Locking is already done in find_node() but may be not done in a # node_not_found hook node_info.acquire_lock() if failures or node_info is None: msg = _('The following failures happened during running ' 'pre-processing hooks:\n%s') % '\n'.join(failures) if node_info is not None: node_info.finished(error='\n'.join(failures)) raise utils.Error(msg, node_info=node_info, data=introspection_data) LOG.info(_LI('Matching node is %s'), node_info.uuid, node_info=node_info, data=introspection_data) if node_info.finished_at is not None: # race condition or introspection canceled raise utils.Error(_('Node processing already finished with ' 'error: %s') % node_info.error, node_info=node_info, code=400) # Note(mkovacik): store data now when we're sure that a background # thread won't race with other process() or introspect.abort() # call utils.executor().submit(_store_unprocessed_data, node_info, unprocessed_data) try: node = node_info.node() except exceptions.NotFound: msg = _('Node was found in cache, but is not found in Ironic') node_info.finished(error=msg) raise utils.Error(msg, code=404, node_info=node_info, data=introspection_data) try: return _process_node(node, introspection_data, node_info) except utils.Error as exc: node_info.finished(error=str(exc)) raise except Exception as exc: LOG.exception(_LE('Unexpected exception during processing')) msg = _('Unexpected exception %(exc_class)s during processing: ' '%(error)s') % {'exc_class': exc.__class__.__name__, 'error': exc} node_info.finished(error=msg) raise utils.Error(msg, node_info=node_info, data=introspection_data)
def init_host(self): """Initialize Worker host Init db connection, load and validate processing hooks, runs periodic tasks. :returns None """ if CONF.processing.store_data == 'none': LOG.warning('Introspection data will not be stored. Change ' '"[processing] store_data" option if this is not ' 'the desired behavior') else: LOG.info('Introspection data will be stored in the %s backend', CONF.processing.store_data) db.init() try: hooks = plugins_base.validate_processing_hooks() except Exception as exc: LOG.critical(str(exc)) sys.exit(1) LOG.info('Enabled processing hooks: %s', [h.name for h in hooks]) driver = pxe_filter.driver() driver.init_filter() periodic_clean_up_ = periodics.periodic( spacing=CONF.clean_up_period)(periodic_clean_up) self._periodics_worker = periodics.PeriodicWorker( callables=[(driver.get_periodic_sync_task(), None, None), (periodic_clean_up_, None, None)], executor_factory=periodics.ExistingExecutor(utils.executor()), on_failure=self._periodics_watchdog) utils.executor().submit(self._periodics_worker.start) if CONF.enable_mdns: endpoint = keystone.get_endpoint('service_catalog') self._zeroconf = mdns.Zeroconf() self._zeroconf.register_service('baremetal-introspection', endpoint) if not CONF.standalone: try: coordinator = coordination.get_coordinator(prefix='conductor') coordinator.start(heartbeat=True) coordinator.join_group() except tooz.ToozError: with excutils.save_and_reraise_exception(): LOG.critical('Failed when connecting to coordination ' 'backend.') self.del_host() else: LOG.info('Successfully connected to coordination backend.')
def shutdown(self): LOG.debug('Shutting down') firewall.clean_up() if self._periodics_worker is not None: self._periodics_worker.stop() self._periodics_worker.wait() self._periodics_worker = None if utils.executor().alive: utils.executor().shutdown(wait=True) LOG.info(_LI('Shut down successfully'))
def shutdown(self): LOG.debug('Shutting down') firewall.clean_up() if self._periodics_worker is not None: self._periodics_worker.stop() self._periodics_worker.wait() self._periodics_worker = None if utils.executor().alive: utils.executor().shutdown(wait=True) LOG.info(_LI('Shut down successfully'))
def introspect(node_id, token=None): """Initiate hardware properties introspection for a given node. :param node_id: node UUID or name :param token: authentication token :raises: Error """ ironic = ir_utils.get_client(token) node = ir_utils.get_node(node_id, ironic=ironic) ir_utils.check_provision_state(node) validation = ironic.node.validate(node.uuid) if not validation.power['result']: msg = _('Failed validation of power interface, reason: %s') raise utils.Error(msg % validation.power['reason'], node_info=node) bmc_address = ir_utils.get_ipmi_address(node) node_info = node_cache.start_introspection(node.uuid, bmc_address=bmc_address, ironic=ironic) def _handle_exceptions(fut): try: fut.result() except utils.Error as exc: # Logging has already happened in Error.__init__ node_info.finished(error=str(exc)) except Exception as exc: msg = _('Unexpected exception in background introspection thread') LOG.exception(msg, node_info=node_info) node_info.finished(error=msg) future = utils.executor().submit(_background_introspect, ironic, node_info) future.add_done_callback(_handle_exceptions)
def init(self): if utils.get_auth_strategy() != 'noauth': utils.add_auth_middleware(app) else: LOG.warning(_LW('Starting unauthenticated, please check' ' configuration')) if CONF.processing.store_data == 'none': LOG.warning(_LW('Introspection data will not be stored. Change ' '"[processing] store_data" option if this is not ' 'the desired behavior')) elif CONF.processing.store_data == 'swift': LOG.info(_LI('Introspection data will be stored in Swift in the ' 'container %s'), CONF.swift.container) utils.add_cors_middleware(app) db.init() try: hooks = [ext.name for ext in plugins_base.processing_hooks_manager()] except KeyError as exc: # callback function raises MissingHookError derived from KeyError # on missing hook LOG.critical(_LC('Hook(s) %s failed to load or was not found'), str(exc)) sys.exit(1) LOG.info(_LI('Enabled processing hooks: %s'), hooks) if CONF.firewall.manage_firewall: firewall.init() periodic_update_ = periodics.periodic( spacing=CONF.firewall.firewall_update_period, enabled=CONF.firewall.manage_firewall )(periodic_update) periodic_clean_up_ = periodics.periodic( spacing=CONF.clean_up_period )(periodic_clean_up) self._periodics_worker = periodics.PeriodicWorker( callables=[(periodic_update_, None, None), (periodic_clean_up_, None, None)], executor_factory=periodics.ExistingExecutor(utils.executor())) utils.executor().submit(self._periodics_worker.start)
def init(self): if CONF.auth_strategy != 'noauth': utils.add_auth_middleware(app) else: LOG.warning('Starting unauthenticated, please check' ' configuration') if CONF.processing.store_data == 'none': LOG.warning('Introspection data will not be stored. Change ' '"[processing] store_data" option if this is not ' 'the desired behavior') elif CONF.processing.store_data == 'swift': LOG.info( 'Introspection data will be stored in Swift in the ' 'container %s', CONF.swift.container) utils.add_cors_middleware(app) db.init() try: hooks = [ ext.name for ext in plugins_base.processing_hooks_manager() ] except KeyError as exc: # callback function raises MissingHookError derived from KeyError # on missing hook LOG.critical('Hook(s) %s failed to load or was not found', str(exc)) sys.exit(1) LOG.info('Enabled processing hooks: %s', hooks) if CONF.firewall.manage_firewall: firewall.init() periodic_update_ = periodics.periodic( spacing=CONF.firewall.firewall_update_period, enabled=CONF.firewall.manage_firewall)(periodic_update) periodic_clean_up_ = periodics.periodic( spacing=CONF.clean_up_period)(periodic_clean_up) self._periodics_worker = periodics.PeriodicWorker( callables=[(periodic_update_, None, None), (periodic_clean_up_, None, None)], executor_factory=periodics.ExistingExecutor(utils.executor())) utils.executor().submit(self._periodics_worker.start)
def _process_node(node, introspection_data, node_info): # NOTE(dtantsur): repeat the check in case something changed ir_utils.check_provision_state(node) node_info.create_ports(introspection_data.get('macs') or ()) _run_post_hooks(node_info, introspection_data) if CONF.processing.store_data == 'swift': stored_data = {k: v for k, v in introspection_data.items() if k not in _STORAGE_EXCLUDED_KEYS} swift_object_name = swift.store_introspection_data(stored_data, node_info.uuid) LOG.info(_LI('Introspection data was stored in Swift in object %s'), swift_object_name, node_info=node_info, data=introspection_data) if CONF.processing.store_data_location: node_info.patch([{'op': 'add', 'path': '/extra/%s' % CONF.processing.store_data_location, 'value': swift_object_name}]) else: LOG.debug('Swift support is disabled, introspection data ' 'won\'t be stored', node_info=node_info, data=introspection_data) ironic = ir_utils.get_client() firewall.update_filters(ironic) node_info.invalidate_cache() rules.apply(node_info, introspection_data) resp = {'uuid': node.uuid} if node_info.options.get('new_ipmi_credentials'): new_username, new_password = ( node_info.options.get('new_ipmi_credentials')) utils.executor().submit(_finish_set_ipmi_credentials, ironic, node, node_info, introspection_data, new_username, new_password) resp['ipmi_setup_credentials'] = True resp['ipmi_username'] = new_username resp['ipmi_password'] = new_password else: utils.executor().submit(_finish, ironic, node_info, introspection_data) return resp
def init_host(self): """Initialize Worker host Init db connection, load and validate processing hooks, runs periodic tasks. :returns None """ if CONF.processing.store_data == 'none': LOG.warning('Introspection data will not be stored. Change ' '"[processing] store_data" option if this is not ' 'the desired behavior') elif CONF.processing.store_data == 'swift': LOG.info( 'Introspection data will be stored in Swift in the ' 'container %s', CONF.swift.container) db.init() try: hooks = plugins_base.validate_processing_hooks() except Exception as exc: LOG.critical(str(exc)) sys.exit(1) LOG.info('Enabled processing hooks: %s', [h.name for h in hooks]) driver = pxe_filter.driver() driver.init_filter() periodic_clean_up_ = periodics.periodic( spacing=CONF.clean_up_period)(periodic_clean_up) self._periodics_worker = periodics.PeriodicWorker( callables=[(driver.get_periodic_sync_task(), None, None), (periodic_clean_up_, None, None)], executor_factory=periodics.ExistingExecutor(utils.executor()), on_failure=self._periodics_watchdog) utils.executor().submit(self._periodics_worker.start) if CONF.enable_mdns: endpoint = keystone.get_endpoint('service_catalog') self._zeroconf = mdns.Zeroconf() self._zeroconf.register_service('baremetal-introspection', endpoint)
def abort(node_id, token=None): """Abort running introspection. :param node_id: node UUID or name :param token: authentication token :raises: Error """ LOG.debug('Aborting introspection for node %s', node_id) ironic = ir_utils.get_client(token) node_info = node_cache.get_node(node_id, ironic=ironic) # check pending operations locked = node_info.acquire_lock(blocking=False) if not locked: # Node busy --- cannot abort atm raise utils.Error(_('Node is locked, please, retry later'), node_info=node_info, code=409) utils.executor().submit(_abort, node_info, ironic)
def abort(uuid, token=None): """Abort running introspection. :param uuid: node uuid :param token: authentication token :raises: Error """ LOG.debug('Aborting introspection for node %s', uuid) ironic = ir_utils.get_client(token) node_info = node_cache.get_node(uuid, ironic=ironic, locked=False) # check pending operations locked = node_info.acquire_lock(blocking=False) if not locked: # Node busy --- cannot abort atm raise utils.Error(_('Node is locked, please, retry later'), node_info=node_info, code=409) utils.executor().submit(_abort, node_info, ironic)
def init(self): if utils.get_auth_strategy() != 'noauth': utils.add_auth_middleware(app) else: LOG.warning( _LW('Starting unauthenticated, please check' ' configuration')) if CONF.processing.store_data == 'none': LOG.warning( _LW('Introspection data will not be stored. Change ' '"[processing] store_data" option if this is not ' 'the desired behavior')) elif CONF.processing.store_data == 'swift': LOG.info( _LI('Introspection data will be stored in Swift in the ' 'container %s'), CONF.swift.container) utils.add_cors_middleware(app) db.init() try: hooks = [ ext.name for ext in plugins_base.processing_hooks_manager() ] except KeyError as exc: # stevedore raises KeyError on missing hook LOG.critical(_LC('Hook %s failed to load or was not found'), str(exc)) sys.exit(1) LOG.info(_LI('Enabled processing hooks: %s'), hooks) if CONF.firewall.manage_firewall: firewall.init() self._periodics_worker = periodics.PeriodicWorker( callables=[(periodic_update, None, None), (periodic_clean_up, None, None)], executor_factory=periodics.ExistingExecutor(utils.executor())) utils.executor().submit(self._periodics_worker.start)
def del_host(self): if not self._shutting_down.acquire(blocking=False): LOG.warning('Attempted to shut down while already shutting down') return pxe_filter.driver().tear_down_filter() if self._periodics_worker is not None: try: self._periodics_worker.stop() self._periodics_worker.wait() except Exception as e: LOG.exception('Service error occurred when stopping ' 'periodic workers. Error: %s', e) self._periodics_worker = None if utils.executor().alive: utils.executor().shutdown(wait=True) self._shutting_down.release() LOG.info('Shut down successfully')
def _process_node(node_info, node, introspection_data): # NOTE(dtantsur): repeat the check in case something changed ir_utils.check_provision_state(node) _run_post_hooks(node_info, introspection_data) _store_data(node_info.uuid, introspection_data) ironic = ir_utils.get_client() pxe_filter.driver().sync(ironic) node_info.invalidate_cache() rules.apply(node_info, introspection_data) resp = {'uuid': node.uuid} utils.executor().submit(_finish, node_info, ironic, introspection_data, power_off=CONF.processing.power_off) return resp
def introspect(uuid, new_ipmi_credentials=None, token=None): """Initiate hardware properties introspection for a given node. :param uuid: node uuid :param new_ipmi_credentials: tuple (new username, new password) or None :param token: authentication token :raises: Error """ ironic = ir_utils.get_client(token) try: node = ironic.node.get(uuid) except exceptions.NotFound: raise utils.Error(_("Cannot find node %s") % uuid, code=404) except exceptions.HttpError as exc: raise utils.Error( _("Cannot get node %(node)s: %(exc)s") % { 'node': uuid, 'exc': exc }) ir_utils.check_provision_state(node, with_credentials=new_ipmi_credentials) if new_ipmi_credentials: new_ipmi_credentials = (_validate_ipmi_credentials( node, new_ipmi_credentials)) else: validation = ironic.node.validate(node.uuid) if not validation.power['result']: msg = _('Failed validation of power interface, reason: %s') raise utils.Error(msg % validation.power['reason'], node_info=node) bmc_address = ir_utils.get_ipmi_address(node) node_info = node_cache.add_node(node.uuid, bmc_address=bmc_address, ironic=ironic) node_info.set_option('new_ipmi_credentials', new_ipmi_credentials) def _handle_exceptions(fut): try: fut.result() except utils.Error as exc: # Logging has already happened in Error.__init__ node_info.finished(error=str(exc)) except Exception as exc: msg = _('Unexpected exception in background introspection thread') LOG.exception(msg, node_info=node_info) node_info.finished(error=msg) future = utils.executor().submit(_background_introspect, ironic, node_info) future.add_done_callback(_handle_exceptions)
def init_host(self): """Initialize Worker host Init db connection, load and validate processing hooks, runs periodic tasks. :returns None """ if CONF.processing.store_data == 'none': LOG.warning('Introspection data will not be stored. Change ' '"[processing] store_data" option if this is not ' 'the desired behavior') elif CONF.processing.store_data == 'swift': LOG.info('Introspection data will be stored in Swift in the ' 'container %s', CONF.swift.container) db.init() try: hooks = plugins_base.validate_processing_hooks() except Exception as exc: LOG.critical(str(exc)) sys.exit(1) LOG.info('Enabled processing hooks: %s', [h.name for h in hooks]) driver = pxe_filter.driver() driver.init_filter() periodic_clean_up_ = periodics.periodic( spacing=CONF.clean_up_period )(periodic_clean_up) self._periodics_worker = periodics.PeriodicWorker( callables=[(driver.get_periodic_sync_task(), None, None), (periodic_clean_up_, None, None)], executor_factory=periodics.ExistingExecutor(utils.executor()), on_failure=self._periodics_watchdog) utils.executor().submit(self._periodics_worker.start)
def reapply(node_ident): """Re-apply introspection steps. Re-apply preprocessing, postprocessing and introspection rules on stored data. :param node_ident: node UUID or name :raises: utils.Error """ LOG.debug('Processing re-apply introspection request for node ' 'UUID: %s', node_ident) node_info = node_cache.get_node(node_ident, locked=False) if not node_info.acquire_lock(blocking=False): # Note (mkovacik): it should be sufficient to check data # presence & locking. If either introspection didn't start # yet, was in waiting state or didn't finish yet, either data # won't be available or locking would fail raise utils.Error(_('Node locked, please, try again later'), node_info=node_info, code=409) utils.executor().submit(_reapply, node_info)
def reapply(uuid): """Re-apply introspection steps. Re-apply preprocessing, postprocessing and introspection rules on stored data. :param uuid: node uuid to use :raises: utils.Error """ LOG.debug('Processing re-apply introspection request for node ' 'UUID: %s', uuid) node_info = node_cache.get_node(uuid, locked=False) if not node_info.acquire_lock(blocking=False): # Note (mkovacik): it should be sufficient to check data # presence & locking. If either introspection didn't start # yet, was in waiting state or didn't finish yet, either data # won't be available or locking would fail raise utils.Error(_('Node locked, please, try again later'), node_info=node_info, code=409) utils.executor().submit(_reapply, node_info)
def del_host(self): """Shutdown the ironic inspector conductor service.""" if not CONF.standalone: try: coordinator = coordination.get_coordinator(prefix='conductor') if coordinator.started: coordinator.leave_group() coordinator.stop() except tooz.ToozError: LOG.exception('Failed to stop coordinator') if not self._shutting_down.acquire(blocking=False): LOG.warning('Attempted to shut down while already shutting down') return pxe_filter.driver().tear_down_filter() if self._periodics_worker is not None: try: self._periodics_worker.stop() self._periodics_worker.wait() except Exception as e: LOG.exception( 'Service error occurred when stopping ' 'periodic workers. Error: %s', e) self._periodics_worker = None if utils.executor().alive: utils.executor().shutdown(wait=True) if self._zeroconf is not None: self._zeroconf.close() self._zeroconf = None self._shutting_down.release() LOG.info('Shut down successfully')
def introspect(uuid, new_ipmi_credentials=None, token=None): """Initiate hardware properties introspection for a given node. :param uuid: node uuid :param new_ipmi_credentials: tuple (new username, new password) or None :param token: authentication token :raises: Error """ ironic = ir_utils.get_client(token) try: node = ironic.node.get(uuid) except exceptions.NotFound: raise utils.Error(_("Cannot find node %s") % uuid, code=404) except exceptions.HttpError as exc: raise utils.Error(_("Cannot get node %(node)s: %(exc)s") % {'node': uuid, 'exc': exc}) ir_utils.check_provision_state(node, with_credentials=new_ipmi_credentials) if new_ipmi_credentials: new_ipmi_credentials = ( _validate_ipmi_credentials(node, new_ipmi_credentials)) else: validation = ironic.node.validate(node.uuid) if not validation.power['result']: msg = _('Failed validation of power interface, reason: %s') raise utils.Error(msg % validation.power['reason'], node_info=node) bmc_address = ir_utils.get_ipmi_address(node) node_info = node_cache.add_node(node.uuid, bmc_address=bmc_address, ironic=ironic) node_info.set_option('new_ipmi_credentials', new_ipmi_credentials) def _handle_exceptions(fut): try: fut.result() except utils.Error as exc: # Logging has already happened in Error.__init__ node_info.finished(error=str(exc)) except Exception as exc: msg = _('Unexpected exception in background introspection thread') LOG.exception(msg, node_info=node_info) node_info.finished(error=msg) future = utils.executor().submit(_background_introspect, ironic, node_info) future.add_done_callback(_handle_exceptions)
def shutdown(self): """Stop serving API, clean up. :returns: None """ # TODO(aarefiev): move shutdown code to WorkerService LOG.debug('Shutting down') firewall.clean_up() if self._periodics_worker is not None: try: self._periodics_worker.stop() self._periodics_worker.wait() except Exception as e: LOG.exception( 'Service error occurred when stopping ' 'periodic workers. Error: %s', e) self._periodics_worker = None if utils.executor().alive: utils.executor().shutdown(wait=True) LOG.info('Shut down successfully')
def process(introspection_data): """Process data from the ramdisk. This function heavily relies on the hooks to do the actual data processing. """ unprocessed_data = copy.deepcopy(introspection_data) failures = [] _run_pre_hooks(introspection_data, failures) node_info = _find_node_info(introspection_data, failures) if node_info: # Locking is already done in find_node() but may be not done in a # node_not_found hook node_info.acquire_lock() if failures or node_info is None: msg = _('The following failures happened during running ' 'pre-processing hooks:\n%s') % '\n'.join(failures) if node_info is not None: node_info.finished(error='\n'.join(failures)) _store_logs(introspection_data, node_info) raise utils.Error(msg, node_info=node_info, data=introspection_data) LOG.info(_LI('Matching node is %s'), node_info.uuid, node_info=node_info, data=introspection_data) if node_info.finished_at is not None: # race condition or introspection canceled raise utils.Error(_('Node processing already finished with ' 'error: %s') % node_info.error, node_info=node_info, code=400) # Note(mkovacik): store data now when we're sure that a background # thread won't race with other process() or introspect.abort() # call utils.executor().submit(_store_unprocessed_data, node_info, unprocessed_data) try: node = node_info.node() except ir_utils.NotFound as exc: with excutils.save_and_reraise_exception(): node_info.finished(error=str(exc)) _store_logs(introspection_data, node_info) try: result = _process_node(node_info, node, introspection_data) except utils.Error as exc: node_info.finished(error=str(exc)) with excutils.save_and_reraise_exception(): _store_logs(introspection_data, node_info) except Exception as exc: LOG.exception(_LE('Unexpected exception during processing')) msg = _('Unexpected exception %(exc_class)s during processing: ' '%(error)s') % {'exc_class': exc.__class__.__name__, 'error': exc} node_info.finished(error=msg) _store_logs(introspection_data, node_info) raise utils.Error(msg, node_info=node_info, data=introspection_data, code=500) if CONF.processing.always_store_ramdisk_logs: _store_logs(introspection_data, node_info) return result
def init_host(self): """Initialize Worker host Init db connection, load and validate processing hooks, runs periodic tasks. :returns None """ if CONF.processing.store_data == 'none': LOG.warning('Introspection data will not be stored. Change ' '"[processing] store_data" option if this is not ' 'the desired behavior') else: LOG.info('Introspection data will be stored in the %s backend', CONF.processing.store_data) db.init() self.coordinator = None try: self.coordinator = coordination.get_coordinator(prefix='conductor') self.coordinator.start(heartbeat=True) self.coordinator.join_group() except Exception as exc: if CONF.standalone: LOG.info( 'Coordination backend cannot be started, assuming ' 'no other instances are running. Error: %s', exc) self.coordinator = None else: with excutils.save_and_reraise_exception(): LOG.critical( 'Failure when connecting to coordination ' 'backend', exc_info=True) self.del_host() else: LOG.info('Successfully connected to coordination backend.') try: hooks = plugins_base.validate_processing_hooks() except Exception as exc: LOG.critical(str(exc)) sys.exit(1) LOG.info('Enabled processing hooks: %s', [h.name for h in hooks]) driver = pxe_filter.driver() driver.init_filter() periodic_clean_up_ = periodics.periodic( spacing=CONF.clean_up_period, enabled=(CONF.clean_up_period != 0))(periodic_clean_up) sync_with_ironic_ = periodics.periodic( spacing=CONF.clean_up_period, enabled=(CONF.clean_up_period != 0))(sync_with_ironic) callables = [(periodic_clean_up_, None, None), (sync_with_ironic_, (self, ), None)] driver_task = driver.get_periodic_sync_task() if driver_task is not None: callables.append((driver_task, None, None)) # run elections periodically if we have a coordinator # that we were able to start if (self.coordinator and self.coordinator.started): periodic_leader_election_ = periodics.periodic( spacing=CONF.leader_election_interval)( periodic_leader_election) callables.append((periodic_leader_election_, (self, ), None)) self._periodics_worker = periodics.PeriodicWorker( callables=callables, executor_factory=periodics.ExistingExecutor(utils.executor()), on_failure=self._periodics_watchdog) utils.executor().submit(self._periodics_worker.start) if CONF.enable_mdns: endpoint = keystone.get_endpoint('service_catalog') self._zeroconf = mdns.Zeroconf() self._zeroconf.register_service('baremetal-introspection', endpoint)