def _process_node(node_info, node, introspection_data): # NOTE(dtantsur): repeat the check in case something changed keep_power_on = ir_utils.check_provision_state(node) _run_post_hooks(node_info, introspection_data) store_introspection_data(node_info.uuid, introspection_data) ironic = ir_utils.get_client() pxe_filter.driver().sync(ironic) node_info.invalidate_cache() rules.apply(node_info, introspection_data) resp = {'uuid': node.id} # determine how to handle power if keep_power_on or not node_info.manage_boot: power_action = False else: power_action = CONF.processing.power_off utils.executor().submit(_finish, node_info, ironic, introspection_data, power_off=power_action) return resp
def _abort(node_info, ironic): # runs in background LOG.debug('Forcing power-off', node_info=node_info) if node_info.manage_boot: try: ironic.node.set_power_state(node_info.uuid, 'off') except Exception as exc: LOG.warning('Failed to power off node: %s', exc, node_info=node_info) node_info.finished(istate.Events.abort_end, error=_('Canceled by operator')) # block this node from PXE Booting the introspection image try: pxe_filter.driver().sync(ironic) except Exception as exc: # Note(mkovacik): this will be retried in the PXE filter sync # periodic task; we continue aborting LOG.warning('Failed to sync the PXE filter: %s', exc, node_info=node_info) LOG.info('Introspection aborted', node_info=node_info)
def shutdown(self, error=None): """Stop serving API, clean up. :returns: None """ # TODO(aarefiev): move shutdown code to WorkerService if not self._shutting_down.acquire(blocking=False): LOG.warning('Attempted to shut down while already shutting down') return LOG.debug('Shutting down') self.rpc_server.stop() if self._periodics_worker is not None: try: self._periodics_worker.stop() self._periodics_worker.wait() except Exception as e: LOG.exception( 'Service error occurred when stopping ' 'periodic workers. Error: %s', e) self._periodics_worker = None if utils.executor().alive: utils.executor().shutdown(wait=True) pxe_filter.driver().tear_down_filter() self._shutting_down.release() LOG.info('Shut down successfully') sys.exit(error)
def periodic_clean_up(): # pragma: no cover try: if node_cache.clean_up(): pxe_filter.driver().sync(ir_utils.get_client()) sync_with_ironic() except Exception: LOG.exception('Periodic clean up of node cache failed')
def del_host(self): if not self._shutting_down.acquire(blocking=False): LOG.warning('Attempted to shut down while already shutting down') return pxe_filter.driver().tear_down_filter() if self._periodics_worker is not None: try: self._periodics_worker.stop() self._periodics_worker.wait() except Exception as e: LOG.exception( 'Service error occurred when stopping ' 'periodic workers. Error: %s', e) self._periodics_worker = None if utils.executor().alive: utils.executor().shutdown(wait=True) if self._zeroconf is not None: self._zeroconf.close() self._zeroconf = None self._shutting_down.release() LOG.info('Shut down successfully')
def periodic_clean_up(): # pragma: no cover try: if node_cache.clean_up(): pxe_filter.driver().sync(ir_utils.get_client()) except Exception: LOG.exception('Periodic clean up of node cache failed') try: sync_with_ironic() except Exception: LOG.exception('Periodic sync of node list with ironic failed')
def _init_host(self): """Initialize Worker host Init db connection, load and validate processing hooks, runs periodic tasks. :returns None """ db.init() try: hooks = plugins_base.validate_processing_hooks() except Exception as exc: LOG.critical(str(exc)) sys.exit(1) LOG.info('Enabled processing hooks: %s', [h.name for h in hooks]) driver = pxe_filter.driver() driver.init_filter() periodic_clean_up_ = periodics.periodic( spacing=CONF.clean_up_period)(periodic_clean_up) self._periodics_worker = periodics.PeriodicWorker( callables=[(driver.get_periodic_sync_task(), None, None), (periodic_clean_up_, None, None)], executor_factory=periodics.ExistingExecutor(utils.executor()), on_failure=self._periodics_watchdog) utils.executor().submit(self._periodics_worker.start)
def _background_introspect_locked(node_info, ironic): # TODO(dtantsur): pagination macs = list(node_info.ports()) if macs: node_info.add_attribute(node_cache.MACS_ATTRIBUTE, macs) LOG.info('Whitelisting MAC\'s %s for a PXE boot', macs, node_info=node_info) pxe_filter.driver().sync(ironic) attrs = node_info.attributes if CONF.processing.node_not_found_hook is None and not attrs: raise utils.Error(_( 'No lookup attributes were found, inspector won\'t ' 'be able to find it after introspection, consider creating ' 'ironic ports or providing an IPMI address'), node_info=node_info) LOG.info('The following attributes will be used for look up: %s', attrs, node_info=node_info) if node_info.manage_boot: try: ironic.node.set_boot_device(node_info.uuid, 'pxe', persistent=False) except Exception as exc: LOG.warning('Failed to set boot device to PXE: %s', exc, node_info=node_info) try: ironic.node.set_power_state(node_info.uuid, 'reboot') except Exception as exc: raise utils.Error(_('Failed to power on the node, check it\'s ' 'power management configuration: %s'), exc, node_info=node_info) LOG.info('Introspection started successfully', node_info=node_info) else: LOG.info( 'Introspection environment is ready, external power on ' 'is required within %d seconds', CONF.timeout, node_info=node_info)
def init_host(self): """Initialize Worker host Init db connection, load and validate processing hooks, runs periodic tasks. :returns None """ if CONF.processing.store_data == 'none': LOG.warning('Introspection data will not be stored. Change ' '"[processing] store_data" option if this is not ' 'the desired behavior') else: LOG.info('Introspection data will be stored in the %s backend', CONF.processing.store_data) db.init() try: hooks = plugins_base.validate_processing_hooks() except Exception as exc: LOG.critical(str(exc)) sys.exit(1) LOG.info('Enabled processing hooks: %s', [h.name for h in hooks]) driver = pxe_filter.driver() driver.init_filter() periodic_clean_up_ = periodics.periodic( spacing=CONF.clean_up_period)(periodic_clean_up) self._periodics_worker = periodics.PeriodicWorker( callables=[(driver.get_periodic_sync_task(), None, None), (periodic_clean_up_, None, None)], executor_factory=periodics.ExistingExecutor(utils.executor()), on_failure=self._periodics_watchdog) utils.executor().submit(self._periodics_worker.start) if CONF.enable_mdns: endpoint = keystone.get_endpoint('service_catalog') self._zeroconf = mdns.Zeroconf() self._zeroconf.register_service('baremetal-introspection', endpoint) if not CONF.standalone: try: coordinator = coordination.get_coordinator(prefix='conductor') coordinator.start(heartbeat=True) coordinator.join_group() except tooz.ToozError: with excutils.save_and_reraise_exception(): LOG.critical('Failed when connecting to coordination ' 'backend.') self.del_host() else: LOG.info('Successfully connected to coordination backend.')
def del_host(self): if not self._shutting_down.acquire(blocking=False): LOG.warning('Attempted to shut down while already shutting down') return pxe_filter.driver().tear_down_filter() if self._periodics_worker is not None: try: self._periodics_worker.stop() self._periodics_worker.wait() except Exception as e: LOG.exception('Service error occurred when stopping ' 'periodic workers. Error: %s', e) self._periodics_worker = None if utils.executor().alive: utils.executor().shutdown(wait=True) self._shutting_down.release() LOG.info('Shut down successfully')
def _process_node(node_info, node, introspection_data): # NOTE(dtantsur): repeat the check in case something changed ir_utils.check_provision_state(node) _run_post_hooks(node_info, introspection_data) _store_data(node_info.uuid, introspection_data) ironic = ir_utils.get_client() pxe_filter.driver().sync(ironic) node_info.invalidate_cache() rules.apply(node_info, introspection_data) resp = {'uuid': node.uuid} utils.executor().submit(_finish, node_info, ironic, introspection_data, power_off=CONF.processing.power_off) return resp
def _background_introspect_locked(node_info, ironic): # TODO(dtantsur): pagination macs = list(node_info.ports()) if macs: node_info.add_attribute(node_cache.MACS_ATTRIBUTE, macs) LOG.info('Whitelisting MAC\'s %s for a PXE boot', macs, node_info=node_info) pxe_filter.driver().sync(ironic) attrs = node_info.attributes if CONF.processing.node_not_found_hook is None and not attrs: raise utils.Error( _('No lookup attributes were found, inspector won\'t ' 'be able to find it after introspection, consider creating ' 'ironic ports or providing an IPMI address'), node_info=node_info) LOG.info('The following attributes will be used for look up: %s', attrs, node_info=node_info) if node_info.manage_boot: try: ironic.node.set_boot_device(node_info.uuid, 'pxe', persistent=False) except Exception as exc: raise utils.Error(_('Failed to set boot device to PXE: %s') % exc, node_info=node_info) try: ironic.node.set_power_state(node_info.uuid, 'reboot') except Exception as exc: raise utils.Error(_('Failed to power on the node, check it\'s ' 'power management configuration: %s') % exc, node_info=node_info) LOG.info('Introspection started successfully', node_info=node_info) else: LOG.info('Introspection environment is ready, external power on ' 'is required within %d seconds', CONF.timeout, node_info=node_info)
def del_host(self): """Shutdown the ironic inspector conductor service.""" if not CONF.standalone: try: coordinator = coordination.get_coordinator(prefix='conductor') if coordinator.started: coordinator.leave_group() coordinator.stop() except tooz.ToozError: LOG.exception('Failed to stop coordinator') if not self._shutting_down.acquire(blocking=False): LOG.warning('Attempted to shut down while already shutting down') return pxe_filter.driver().tear_down_filter() if self._periodics_worker is not None: try: self._periodics_worker.stop() self._periodics_worker.wait() except Exception as e: LOG.exception( 'Service error occurred when stopping ' 'periodic workers. Error: %s', e) self._periodics_worker = None if utils.executor().alive: utils.executor().shutdown(wait=True) if self._zeroconf is not None: self._zeroconf.close() self._zeroconf = None self._shutting_down.release() LOG.info('Shut down successfully')
def init_host(self): """Initialize Worker host Init db connection, load and validate processing hooks, runs periodic tasks. :returns None """ if CONF.processing.store_data == 'none': LOG.warning('Introspection data will not be stored. Change ' '"[processing] store_data" option if this is not ' 'the desired behavior') elif CONF.processing.store_data == 'swift': LOG.info( 'Introspection data will be stored in Swift in the ' 'container %s', CONF.swift.container) db.init() try: hooks = plugins_base.validate_processing_hooks() except Exception as exc: LOG.critical(str(exc)) sys.exit(1) LOG.info('Enabled processing hooks: %s', [h.name for h in hooks]) driver = pxe_filter.driver() driver.init_filter() periodic_clean_up_ = periodics.periodic( spacing=CONF.clean_up_period)(periodic_clean_up) self._periodics_worker = periodics.PeriodicWorker( callables=[(driver.get_periodic_sync_task(), None, None), (periodic_clean_up_, None, None)], executor_factory=periodics.ExistingExecutor(utils.executor()), on_failure=self._periodics_watchdog) utils.executor().submit(self._periodics_worker.start) if CONF.enable_mdns: endpoint = keystone.get_endpoint('service_catalog') self._zeroconf = mdns.Zeroconf() self._zeroconf.register_service('baremetal-introspection', endpoint)
def init_host(self): """Initialize Worker host Init db connection, load and validate processing hooks, runs periodic tasks. :returns None """ if CONF.processing.store_data == 'none': LOG.warning('Introspection data will not be stored. Change ' '"[processing] store_data" option if this is not ' 'the desired behavior') elif CONF.processing.store_data == 'swift': LOG.info('Introspection data will be stored in Swift in the ' 'container %s', CONF.swift.container) db.init() try: hooks = plugins_base.validate_processing_hooks() except Exception as exc: LOG.critical(str(exc)) sys.exit(1) LOG.info('Enabled processing hooks: %s', [h.name for h in hooks]) driver = pxe_filter.driver() driver.init_filter() periodic_clean_up_ = periodics.periodic( spacing=CONF.clean_up_period )(periodic_clean_up) self._periodics_worker = periodics.PeriodicWorker( callables=[(driver.get_periodic_sync_task(), None, None), (periodic_clean_up_, None, None)], executor_factory=periodics.ExistingExecutor(utils.executor()), on_failure=self._periodics_watchdog) utils.executor().submit(self._periodics_worker.start)
def test_driver(self): ret = pxe_filter.driver() self.assertIs(self.mock_driver, ret) self.mock__driver_manager.assert_called_once_with()
def periodic_clean_up(): # pragma: no cover if node_cache.clean_up(): pxe_filter.driver().sync(ir_utils.get_client())
def init_host(self): """Initialize Worker host Init db connection, load and validate processing hooks, runs periodic tasks. :returns None """ if CONF.processing.store_data == 'none': LOG.warning('Introspection data will not be stored. Change ' '"[processing] store_data" option if this is not ' 'the desired behavior') else: LOG.info('Introspection data will be stored in the %s backend', CONF.processing.store_data) db.init() self.coordinator = None try: self.coordinator = coordination.get_coordinator(prefix='conductor') self.coordinator.start(heartbeat=True) self.coordinator.join_group() except Exception as exc: if CONF.standalone: LOG.info( 'Coordination backend cannot be started, assuming ' 'no other instances are running. Error: %s', exc) self.coordinator = None else: with excutils.save_and_reraise_exception(): LOG.critical( 'Failure when connecting to coordination ' 'backend', exc_info=True) self.del_host() else: LOG.info('Successfully connected to coordination backend.') try: hooks = plugins_base.validate_processing_hooks() except Exception as exc: LOG.critical(str(exc)) sys.exit(1) LOG.info('Enabled processing hooks: %s', [h.name for h in hooks]) driver = pxe_filter.driver() driver.init_filter() periodic_clean_up_ = periodics.periodic( spacing=CONF.clean_up_period, enabled=(CONF.clean_up_period != 0))(periodic_clean_up) sync_with_ironic_ = periodics.periodic( spacing=CONF.clean_up_period, enabled=(CONF.clean_up_period != 0))(sync_with_ironic) callables = [(periodic_clean_up_, None, None), (sync_with_ironic_, (self, ), None)] driver_task = driver.get_periodic_sync_task() if driver_task is not None: callables.append((driver_task, None, None)) # run elections periodically if we have a coordinator # that we were able to start if (self.coordinator and self.coordinator.started): periodic_leader_election_ = periodics.periodic( spacing=CONF.leader_election_interval)( periodic_leader_election) callables.append((periodic_leader_election_, (self, ), None)) self._periodics_worker = periodics.PeriodicWorker( callables=callables, executor_factory=periodics.ExistingExecutor(utils.executor()), on_failure=self._periodics_watchdog) utils.executor().submit(self._periodics_worker.start) if CONF.enable_mdns: endpoint = keystone.get_endpoint('service_catalog') self._zeroconf = mdns.Zeroconf() self._zeroconf.register_service('baremetal-introspection', endpoint)