def pip_install_requirements(requirements_path): """ Installs packages specified in a requirements.txt file, using the tortuga package repo in addition to the standard python repos. This function returns nothing, and does nothing if the requirements.txt file is not found. :param requirements_path: the path to the requirements.txt file """ cm = ConfigManager() if not os.path.exists(requirements_path): logger.debug('Requirements not found: {}'.format(requirements_path)) return if is_requirements_empty(requirements_path): logger.debug('Requirements empty: {}'.format(requirements_path)) return pip_cmd = [ '{}/pip'.format(cm.getBinDir()), 'install', ] installer = cm.getInstaller() int_webroot = cm.getIntWebRootUrl(installer) installer_repo = '{}/python-tortuga/simple/'.format(int_webroot) if cm.is_offline_installation(): # add tortuga distribution repo pip_cmd.append('--index-url') pip_cmd.append(installer_repo) # add offline dependencies repo pip_cmd.append('--extra-index-url') pip_cmd.append('{}/offline-deps/python/simple/'.format(int_webroot)) else: pip_cmd.append('--extra-index-url') pip_cmd.append(installer_repo) pip_cmd.extend(['--trusted-host', installer, '-r', requirements_path]) logger.debug(' '.join(pip_cmd)) proc = subprocess.Popen(pip_cmd) proc.wait() if proc.returncode: raise Exception(proc.stderr)
def pip_install_requirements(requirements_path): """ Installs packages specified in a requirements.txt file, using the kit package repo in addition to the standard python repos. This function returns nothing, and does nothing if the requirements.txt file is not found. :param kit_installer: an instance of KitInstallerBase, which will be searched for a local python package repo :param requirements_path: the path to the requirements.txt file """ # # In the kit directory: # # /opt/tortuga/kits/kit-x.y.z/tortuga_kits/kit_x_y_z # # if there is a python_packages directory, with a simple subdirectory # in it, it is assumed that the simple subdirectory is a PEP 503 # compliant Python package repository. If found, this directory is # added to the list of directories searched for Python packages via # pip when installing the requirements.txt file. # # These directories can easily be created using the py2pi utility. # cm = ConfigManager() if not os.path.exists(requirements_path): logger.debug('Requirements not found: {}'.format(requirements_path)) return if is_requirements_empty(requirements_path): logger.debug('Requirements empty: {}'.format(requirements_path)) return installer = cm.getInstaller() int_webroot = cm.getIntWebRootUrl(installer) installer_repo = '{}/python-tortuga/simple/'.format(int_webroot) pip_cmd = [ '{}/pip'.format(cm.getBinDir()), 'install', '--extra-index-url', installer_repo, '--trusted-host', installer, '-r', requirements_path ] logger.debug(' '.join(pip_cmd)) subprocess.Popen(pip_cmd).wait()
class SyncManager(TortugaObjectManager): """Class for cluster sync management""" __instanceLock = threading.RLock() # update delay increase (seconds) CLUSTER_UPDATE_DELAY_INCREASE = 30 # after this limit is reached, warning will be logged CLUSTER_UPDATE_WARNING_LIMIT = 10 def __init__(self): super(SyncManager, self).__init__() self._isUpdateScheduled = False self._isUpdateRunning = False self._cm = ConfigManager() self._logger = logging.getLogger(SYNC_NAMESPACE) def __runClusterUpdate(self, opts={}): """ Run cluster update. """ self._logger.debug('Update timer running, opts={}'.format(opts)) updateCmd = os.path.join(self._cm.getBinDir(), 'run_cluster_update.sh') delay = 0 updateCnt = 0 while self.__resetIsUpdateScheduled(): self._isUpdateRunning = True self._logger.debug('New cluster update delay: %s seconds' % (delay)) time.sleep(delay) delay += SyncManager.CLUSTER_UPDATE_DELAY_INCREASE # Log warning if timer has been running for too many times. updateCnt += 1 self._logger.debug('Cluster update timer count: %s' % (updateCnt)) if updateCnt > SyncManager.CLUSTER_UPDATE_WARNING_LIMIT: self._logger.warning( 'Cluster updated more than %s times using the same' ' timer (possible configuration problem)' % (SyncManager.CLUSTER_UPDATE_WARNING_LIMIT)) self._logger.debug('Starting cluster update using: %s' % (updateCmd)) # Since we might sleep for a while, we need to # reset update flag just before we run update to avoid # unnecessary syncs. self.__resetIsUpdateScheduled() if 'node' in opts: node_update = opts['node'] env = { **os.environ, 'FACTER_node_tags_update': json.dumps(node_update) } self._logger.debug('FACTER_node_tags_update={}'.format( env['FACTER_node_tags_update'])) p = TortugaSubprocess(updateCmd, env=env) elif 'software_profile' in opts: sp_update = opts['software_profile'] env = { **os.environ, 'FACTER_softwareprofile_tags_update': json.dumps(sp_update) } self._logger.debug( 'FACTER_softwareprofile_tags_update={}'.format( env['FACTER_softwareprofile_tags_update'])) p = TortugaSubprocess(updateCmd, env=env) else: p = TortugaSubprocess(updateCmd) try: p.run() self._logger.debug('Cluster update successful') self._logger.debug('stdout: {}'.format( p.getStdOut().decode().rstrip())) self._logger.debug('stderr: {}'.format( p.getStdErr().decode().rstrip())) except CommandFailed: if p.getExitStatus() == tortugaStatus.\ TORTUGA_ANOTHER_INSTANCE_OWNS_LOCK_ERROR: self._logger.debug( 'Another cluster update is already running, will' ' try to reschedule it') self._isUpdateRunning = False self.scheduleClusterUpdate( updateReason='another update already running', delay=60, opts=opts) break else: self._logger.error( 'Update command "%s" failed (exit status: %s):' ' %s' % (updateCmd, p.getExitStatus(), p.getStdErr())) self._logger.debug('Done with cluster update') self._isUpdateRunning = False self._logger.debug('Update timer exiting') def __resetIsUpdateScheduled(self): """ Reset cluster update flag, return old flag value. """ SyncManager.__instanceLock.acquire() try: flag = self._isUpdateScheduled self._isUpdateScheduled = False return flag finally: SyncManager.__instanceLock.release() def scheduleClusterUpdate(self, updateReason=None, delay=5, opts={}): """ Schedule cluster update. """ SyncManager.__instanceLock.acquire() try: if self._isUpdateScheduled: # Already scheduled. return # Start update timer if needed. self._isUpdateScheduled = True if not self._isUpdateRunning: self._logger.debug('Scheduling cluster update in %s seconds,' ' reason: %s, opts: %s' % (delay, updateReason, opts)) t = threading.Timer(delay, self.__runClusterUpdate, kwargs=dict(opts=opts)) t.start() else: self._logger.debug( 'Will not schedule new update timer while the old' ' timer is running') finally: SyncManager.__instanceLock.release() def getUpdateStatus(self): # pylint: disable=no-self-use """ Check cluster update flag. """ return RunManager().checkLock('cfmsync')
class KitInstallerBase(ConfigurableMixin, metaclass=KitInstallerMeta): """ Base class for kit installers. """ config_type = 'kit' # # The kit installation directory # install_path = None # # Metadata, loaded via the load_meta class method. # name = None version = None iteration = None spec = (None, None, None) meta = {} # # Loader state # ws_controllers_loaded = False db_tables_loaded = False # # Attributes, provided by instances of this class # puppet_modules = [] task_modules = [] def __init__(self): self.config_manager = ConfigManager() # # Setup paths # self.kit_path = os.path.dirname(inspect.getfile(self.__class__)) self.puppet_modules_path = os.path.join(self.kit_path, 'puppet_modules') self.files_path = os.path.join(self.kit_path, 'files') # # Initialize configuration # super().__init__() # # Load components and resource adapters # self._component_installers = {} self._component_installers_loaded = False # # Web service controller classes # self._ws_controller_classes = [] self.session = None def get_config_base(self): return self.config_manager.getKitConfigBase() @classmethod def load_meta(cls, meta_dict): """ Loads the meta data for the kit into the class. :param meta_dict: A dict containing the metadata, as specified by the KitMetadataSchema class. """ errors = KitMetadataSchema().validate(meta_dict) if errors: raise Exception('Kit metadata validation error: {}'.format(errors)) requires_core = meta_dict.get('requires_core', VERSION) if not version_is_compatible(requires_core): raise Exception('The {} kit requires tortuga core >= {}'.format( meta_dict['name'], requires_core)) meta_dict = copy.deepcopy(meta_dict) cls.name = meta_dict.pop('name') cls.version = meta_dict.pop('version') cls.iteration = meta_dict.pop('iteration') cls.spec = (cls.name, cls.version, cls.iteration) cls.meta = meta_dict def _load_component_installers(self): """ Load component installers for this kit. """ if self._component_installers_loaded: return kit_pkg_name = inspect.getmodule(self).__package__ comp_pkg_name = '{}.components'.format(kit_pkg_name) logger.debug('Searching for component installers in package: %s', comp_pkg_name) # # Look for the components sub-package # try: comp_pkg = importlib.import_module(comp_pkg_name) except ModuleNotFoundError: logger.warning('No component installers found for kit: %s', kit_pkg_name) return # # Walk the components sub-package, looking for component installers # for loader, name, ispkg in pkgutil.walk_packages(comp_pkg.__path__): if not ispkg: continue full_pkg_path = '{}.{}'.format(comp_pkg_name, name) try: # # Look for the component module in the package # comp_inst_mod = importlib.import_module( '{}.component'.format(full_pkg_path)) # # Look for the ComponentInstaller class in the module # if not hasattr(comp_inst_mod, 'ComponentInstaller'): logger.warning('ComponentInstaller class not found: %s', full_pkg_path) # # Initialize the ComponentInstaller class and register # it with the KitInstaller # comp_inst_class = comp_inst_mod.ComponentInstaller comp_inst = comp_inst_class(self) comp_inst.session = self.session self._component_installers[comp_inst_class.name] = \ comp_inst logger.debug('Component installer registered: %s', comp_inst.spec) except ModuleNotFoundError: logger.debug('Package not a component: %s', full_pkg_path) self._component_installers_loaded = True def is_installable(self): """ Determines whether or not this kit is installable under the given conditions/circumstances. Override this in your implementations as necessary. :return: True if it is installable, False otherwise. """ return True def run_action(self, action_name, *args, **kwargs): """ Runs the specified action. :param action_name: the name of the action to run """ try: action = getattr(self, 'action_{}'.format(action_name)) return action(*args, **kwargs) except KeyError: raise Exception('Unknown action: {}'.format(action_name)) def get_kit(self): """ Gets the Kit instance for this kit. :return: a Kit instance """ kit = Kit(name=self.name, version=self.version, iteration=self.iteration) kit.setDescription(self.meta.get('description', None)) for component_installer in self.get_all_component_installers(): kit.addComponent(component_installer.get_component()) return kit def get_eula(self): """ Gets the EULA for this kit, if it exists. :return: a Eula instance if there is a EULA file, otherwise None. """ eula = None eula_path = os.path.join(self.install_path, EULA_FILE) if os.path.exists(eula_path) and os.path.isfile(eula_path): eula_fp = open(eula_path) text = eula_fp.read() eula_fp.close() eula = Eula(text=text) else: logger.debug('EULA not found: %s', eula_path) return eula def get_component_installer(self, component_name: str): self._load_component_installers() return self._component_installers.get(component_name) def get_all_component_installers(self): self._load_component_installers() return [ci for ci in self._component_installers.values()] def register_database_tables(self): """ Register database table mappers for this kit. """ # # If another kit of the same name already exists, and has loaded # the database tables, then we don't need to do it a second time # for ki in get_all_kit_installers(): if ki.spec[0] == self.spec[0] and ki.db_tables_loaded: self.__class__.db_tables_loaded = True return kit_pkg_name = inspect.getmodule(self).__package__ db_table_pkg_name = '{}.db.models'.format(kit_pkg_name) logger.debug('Searching for database table mappers in package: %s', db_table_pkg_name) try: importlib.import_module(db_table_pkg_name) self.__class__.db_tables_loaded = True except ModuleNotFoundError: logger.debug('No database table mappers found for kit: %s', self.spec) def register_web_service_controllers(self): """ Register web service controllers for this kit. """ # # If another kit of the same name already exists, and has loaded # the ws controllers, then we don't need to do it a second time # for ki in get_all_kit_installers(): if ki.spec[0] == self.spec[0] and ki.ws_controllers_loaded: self.__class__.ws_controllers_loaded = True return kit_pkg_name = inspect.getmodule(self).__package__ ws_pkg_name = '{}.web_service.controllers'.format(kit_pkg_name) logger.debug('Searching for web service controllers in package: %s', ws_pkg_name) try: importlib.import_module(ws_pkg_name) self.__class__.ws_controllers_loaded = True except ModuleNotFoundError: logger.debug('No web service controllers found for kit: %s', self.spec) def register_event_listeners(self): """ Register event listeners for this kit. """ kit_pkg_name = inspect.getmodule(self).__package__ listener_pkg_name = '{}.events.listeners'.format(kit_pkg_name) logger.debug('Searching for event listeners in package: %s', listener_pkg_name) try: importlib.import_module(listener_pkg_name) except ModuleNotFoundError: logger.debug('No event listeners found for kit: %s', self.spec) def action_install_puppet_modules(self, *args, **kwargs): # # Prevent circular import # from .actions import UninstallPuppetModulesAction, \ InstallPuppetModulesAction # # Do an uninstall first, just in case there is an old version of # the module still hanging around. This should fail silently if the # module is not installed. # UninstallPuppetModulesAction(self)(*args, **kwargs) # # Do the actual install # return InstallPuppetModulesAction(self)(*args, **kwargs) def action_pre_install(self): pass def action_pre_uninstall(self): pass def action_post_install(self): # # Check for python packages to install # pkg_dir = os.path.join(self.install_path, 'python_packages') if os.path.exists(pkg_dir): self._update_python_repo(pkg_dir) # # Install required python packages from requirements.txt # requirements_path = os.path.join(self.kit_path, 'requirements.txt') pip_install_requirements(requirements_path) def _update_python_repo(self, pkg_dir: str): """ Updates the Tortuga Python repo with packages from the kit. :param pkg_dir: the source directory from which the packages will be copied """ # # Copy the files from the pkg_dir to the Tortuga repo # whl_path = os.path.join(pkg_dir, '*.whl') repo_path = os.path.join(self.config_manager.getTortugaIntWebRoot(), 'python-tortuga') cmd = 'rsync -a {} {}'.format(whl_path, repo_path) logger.debug(cmd) executeCommand(cmd) # # Re-build the package index # dir2pi = os.path.join(self.config_manager.getBinDir(), 'dir2pi') cmd = '{} {}'.format(dir2pi, repo_path) logger.debug(cmd) executeCommand(cmd) def action_post_uninstall(self): pass def action_uninstall_puppet_modules(self, *args, **kwargs): # # Prevent circular import # from .actions import UninstallPuppetModulesAction return UninstallPuppetModulesAction(self)(*args, **kwargs) def action_get_metadata(self, hardware_profile_name: Optional[str] = None, software_profile_name: Optional[str] = None, node_name: Optional[str] = None) -> dict: pass
class KitActions(ActionsBase): ''' A kit is a group of components that constitute a complete application. ''' def __init__(self, moduledir=None): ''' Arguments: moduledir Path to the module's directory. Defaults to CWD. E.g: "/opt/tortuga/kits/kit-ganglia-1.2.3" Attributes: name Kit name. version Kit version. moduledir Fully-qualified path to the root of the kit as installed on the filesystem. For example, "/opt/tortuga/kits/kit-ganglia-1.2.3" components a list of ComponentActions() objects _logger A logger instance for creating log messages _config configManager instance _root $TORTUGA_ROOT; e.g: "/opt/tortuga" ''' super(KitActions, self).__init__() self.name = self.__class__.__name__.lower() self.version = None if moduledir: self.moduledir = moduledir else: self.moduledir = os.getcwd() self.components = [] # Most kits need these things; including here for convenience. self._config = ConfigManager() self._root = self._config.getRoot() @property def config(self): return self._config def getLogger(self): return self._logger def getConfigManager(self): return self._config def getRoot(self): return self._root # Overridden form ActionsBase def getConfigFile(self): return "%s/%s-kit.conf" % (self.getConfigBase(), self.name.lower()) def getConfigBase(self): return "%s/%s" % (self.getConfigManager().getKitConfigBase(), self.name.lower()) def pre_install(self): ''' Pre-installation kit hook. ''' pass def post_install(self): ''' Post-installation kit hook. ''' pass def pre_uninstall(self): ''' Post-uninstallation kit hook. ''' pass def post_uninstall(self): ''' Post-uninstallation kit hook. ''' pass def add_component(self, component): '''Add the given component to the kit's list of components''' # Point the component to its parent component.kit = self self.components.append(component) def lookup_cname(self, cname): ''' Return the ComponentActions object from the KitActions whose name is "cname" Raises: ComponentNotFound ''' for c in self.components: if c.__component_name__ == cname: return c(self) raise ComponentNotFound("Can't find component [%s] in kit [%s]" % (cname, self.__class__.__name__)) def is_puppet_module_installed(self, name): cmd = '/opt/puppetlabs/bin/puppet module list --render-as=json' p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) puppet_module_list = json.load(p.stdout) retval = p.wait() if retval != 0: return None for modules in \ puppet_module_list['modules_by_path'].values(): for module in modules: if module.startswith('Module %s(' % (name)): return True return False def installPuppetModule(self, modulePath): """ Install "standard" Puppet module using "puppet module install --force" Raises: ConfigurationError """ if not os.path.exists(modulePath): errmsg = ('Error: unable to install puppet module [%s].' ' Module does not exist' % (modulePath)) self.getLogger().error(errmsg) raise ConfigurationError(errmsg) cmd = ('/opt/puppetlabs/bin/puppet module install --color false' ' --force %s' % (modulePath)) tortugaSubprocess.executeCommand(cmd) def uninstallPuppetModule(self, moduleName): cmd = ('/opt/puppetlabs/bin/puppet module uninstall' ' --color false --ignore-changes %s' % (moduleName)) tortugaSubprocess.executeCommandAndIgnoreFailure(cmd) def install_wheel_matching_filespec(self, whl_pathspec): # Find an whl matching the filespec whl_files = glob.glob(whl_pathspec) if not whl_files: raise FileNotFound('No files found matching spec %s' % (whl_pathspec)) # Use the first whl file found cmd = '%s/pip install %s' % (self._config.getBinDir(), whl_files[0]) tortugaSubprocess.executeCommandAndIgnoreFailure(cmd) def uninstall_wheel(self, wheel_name): cmd = 'pip uninstall %s' % (wheel_name) tortugaSubprocess.executeCommandAndIgnoreFailure(cmd)
class ResourceAdapter(object): \ # pylint: disable=too-many-public-methods ''' This is the base class for all resource adapters to derive from. The default actions simply print a debug message to show that the subclass did not implement the action. ''' def __init__(self, addHostSession=None): if '__adaptername__' not in self.__class__.__dict__: raise NotImplementedError( 'Subclasses of ResourceAdapter must have __adaptername__' ' defined') self._logger = logging.getLogger('tortuga.resourceAdapter.%s' % (self.__adaptername__)) self._logger.addHandler(logging.NullHandler()) self.__installer_public_hostname = None self.__installer_public_ipaddress = None self.__private_dns_zone = None # Initialize caches self.__addHostApi = None self.__nodeApi = None self.__osObject = None self.__sanApi = None # Initialize abort flag (to "not" aborting) self.__isAborted = False self._cm = ConfigManager() self._addHostSession = addHostSession @property def addHostSession(self): return self._addHostSession @property def cacheCfgFilePath(self): return os.path.join(self._cm.getRoot(), 'var', '%s-instance.conf' % (self.__adaptername__)) @property def cfgFileName(self): return os.path.join( self._cm.getKitConfigBase(), 'adapter-defaults-%s.conf' % (self.__adaptername__)) def hookAction(self, action, nodes, args=None): # Only the 'default' resource adapter overrides the hookAction() # method. pass def start(self, addNodesRequest, dbSession, dbHardwareProfile, dbSoftwareProfile=None): \ # pylint: disable=unused-argument self.__trace(addNodesRequest, dbSession, dbHardwareProfile, dbSoftwareProfile) def validate_start_arguments(self, addNodesRequest, dbHardwareProfile, dbSoftwareProfile): self.__trace(addNodesRequest, dbHardwareProfile, dbSoftwareProfile) def stop(self, hardwareProfileName, deviceName): self.__trace(hardwareProfileName, deviceName) def updateNode(self, session, node, updateNodeRequest): \ # pylint: disable=unused-argument self.__trace(session, node, updateNodeRequest) def suspendActiveNode(self, nodeId): '''Change the given active node to an idle node''' self.__trace(nodeId) def idleActiveNode(self, nodeIds): '''Change the given active node to an idle node''' self.__trace(nodeIds) def activateIdleNode(self, node, softwareProfileName, softwareProfileChanged): '''Change the given idle node to an active node''' self.__trace(node, softwareProfileName, softwareProfileChanged) def deleteNode(self, nodeIds): '''Remove the given node (active or idle) from the system''' self.__trace(nodeIds) def _async_delete_nodes(self, nodes): """ Asynchronously delete nodes; calls "ResourceAdapter._delete_node()" method for each deleted nodes :param dbNodes: list of Nodes objects :return: None """ greenlets = [] for node in nodes: greenlets.append(gevent.spawn(self._delete_node, node)) # TODO: implement timeout gevent.joinall(greenlets) def _delete_node(self, node): """ Abstract method called to delete node from "ResourceAdapter._async_delete_nodes()" :param node: Nodes object """ def transferNode(self, nodeIdSoftwareProfileTuples, newSoftwareProfileName): '''Transfer the given idle node''' self.__trace(nodeIdSoftwareProfileTuples, newSoftwareProfileName) def startupNode(self, nodeIds, remainingNodeList=None, tmpBootMethod='n'): \ # pylint: disable=unused-argument '''Start the given node''' # By default raise unsupported operation raise UnsupportedOperation('Node does not support starting') def shutdownNode(self, nodes, bSoftReset=False): \ # pylint: disable=unused-argument '''Shutdown the given node''' # By default raise unsupported operation raise UnsupportedOperation('Node does not support shutdown') def rebootNode(self, nodes, bSoftReset=False): \ # pylint: disable=unused-argument '''Reboot the given node''' # By default raise unsupported operation raise UnsupportedOperation('Node does not support rebooting') def checkpointNode(self, nodeId): \ # pylint: disable=unused-argument '''Checkpoint the given node''' # By default raise unsupported operation raise UnsupportedOperation('Node does not support checkpointing') def revertNodeToCheckpoint(self, nodeId): \ # pylint: disable=unused-argument '''Revert the given node to the checkpoint''' # By default raise unsupported operation raise UnsupportedOperation('Node does not support checkpointing') def migrateNode(self, nodeId, remainingNodeList, liveMigrate): \ # pylint: disable=unused-argument '''Migrate the given node''' # By default raise unsupported operation raise UnsupportedOperation('Node does not support migrating') def addVolumeToNode(self, node, volume, isDirect): \ # pylint: disable=unused-argument '''Add a disk to a node''' # By default raise unsupported operation raise UnsupportedOperation( 'Node does not support dynamic disk addition') def removeVolumeFromNode(self, node, volume): \ # pylint: disable=unused-argument '''Remove a disk from a node''' # By default raise unsupported operation raise UnsupportedOperation( 'Node does not support dynamic disk deletion' % (node)) def abort(self): '''abort node addition''' self._logger.debug('Setting abort flag') self.__isAborted = True def isAborted(self): '''Returns status of abort flag''' return self.__isAborted def __trace(self, *pargs, **kargs): stack = traceback.extract_stack() funcname = stack[-2][2] self._logger.debug('-- (pass) %s::%s %s %s' % (self.__adaptername__, funcname, pargs, kargs)) def getLogger(self): return self._logger def getResourceAdapterConfig(self, sectionName=None): """ Raises: ResourceNotFound """ self.getLogger().debug( 'getResourceAdapterConfig(sectionName=[{0}])'.format( sectionName if sectionName else '(none)')) try: # Load default values defaultResourceAdapterConfigDict = self._loadConfigDict() if sectionName is None or sectionName == 'default': return defaultResourceAdapterConfigDict except ResourceNotFound: defaultResourceAdapterConfigDict = {} overrideConfigDict = self._loadConfigDict(sectionName) # Override defaults with hardware profile specific settings return dict( list(defaultResourceAdapterConfigDict.items()) + list(overrideConfigDict.items())) def _loadConfigDict(self, sectionName=None): """ Raises: ResourceNotFound """ if sectionName is None: sectionName = 'default' session = DbManager().openSession() try: self.getLogger().debug('_loadConfigDict()') result = ResourceAdapterCredentialsDbHandler().get( session, self.__adaptername__, sectionName) configDict = {} for entry in result['configuration']: configDict[entry['key']] = entry['value'] finally: DbManager().closeSession() return configDict def getResourceAdapterConfigProfileByNodeName(self, name): """Get resource adapter configuration for existing node""" self.getLogger().debug( 'getResourceAdapterConfigByNodeName(): name=[{0}]'.format(name)) instance_cache = self.instanceCacheRefresh() return instance_cache.get(name, 'resource_adapter_configuration') \ if instance_cache.has_section(name) and instance_cache.has_option( name, 'resource_adapter_configuration') else None def __getAddHostApi(self): '''Get and cache the Add Host API''' if self.__addHostApi is None: from tortuga.addhost.addHostServerLocal \ import AddHostServerLocal self.__addHostApi = AddHostServerLocal() return self.__addHostApi def __getNodeApi(self): '''Get and cache the Node API''' if self.__nodeApi is None: from tortuga.node.nodeApi import NodeApi self.__nodeApi = NodeApi() return self.__nodeApi def __getOsObject(self): '''Get and cache the OS Object Factory''' if self.__osObject is None: from tortuga.os_utility import osUtility self.__osObject = osUtility.getOsObjectFactory() return self.__osObject def __getSanApi(self): '''Internal: Get and cache the SAN API''' if self.__sanApi is None: from tortuga.san import san self.__sanApi = san.San() return self.__sanApi # Properties for this object addHostApi = property(__getAddHostApi, None, None, None) nodeApi = property(__getNodeApi, None, None, None) osObject = property(__getOsObject, None, None, None) sanApi = property(__getSanApi, None, None, None) def statusMessage(self, msg): if self._addHostSession: AddHostManager().updateStatus(self._addHostSession, msg) else: # Just print out the message...this is a stop gap for resource # adapters running outside of the addHostManager framework sys.stdout.write(msg + '\n') sys.stdout.flush() def getOptions(self, dbSoftwareProfile, dbHardwareProfile): \ # pylint: disable=unused-argument return {} def instanceCacheWrite(self, cfg): # Write the instance cache back to disk self.getLogger().debug('instanceCacheWrite()') with open(self.cacheCfgFilePath, 'w') as fp: cfg.write(fp) def instanceCacheRefresh(self): self.getLogger().debug('instanceCacheRefresh()') cfg = configparser.ConfigParser() cfg.read(self.cacheCfgFilePath) return cfg def instanceCacheSet(self, name, metadata=None): self.getLogger().debug('instanceCacheSet(node=[%s], metadata=[%s])' % (name, metadata)) cfg = self.instanceCacheRefresh() if not cfg.has_section(name): cfg.add_section(name) # Write metadata to node section if metadata: for key, value in metadata.items(): cfg.set(name, key, value) self.instanceCacheWrite(cfg) def instanceCacheSetBulk(self, instance_ids, nodes=None): self.getLogger().debug( 'instanceCacheSetBulk(instance_ids=[%s], nodes=[%s])' % (' '.join(instance_ids), ' '.join( [node.name for node in nodes or []]))) cfg = self.instanceCacheRefresh() if not nodes: if not cfg.has_section('unassigned'): cfg.add_section('unassigned') instances = set() else: val = cfg.get('unassigned', 'instances') instances = set(val.split(' ')) instances |= set(instance_ids) cfg.set('unassigned', 'instances', ' '.join(instances)) self.instanceCacheWrite(cfg) def instanceCacheGet(self, nodeName): self.getLogger().debug('instanceCacheGet(nodeName=[%s])' % (nodeName)) cfg = self.instanceCacheRefresh() if not cfg.has_section(nodeName): raise ResourceNotFound( 'No instance cache entry for [{0}]'.format(nodeName)) # Read entire section into a dict result = {} for key, value in cfg.items(nodeName): result[key] = value return result def instanceCacheDelete(self, name): # Clear instance from configuration config = self.instanceCacheRefresh() if not config.has_section(name): self.getLogger().debug( 'Cache clear: node [{0}] not found, no action' ' taken'.format(name)) return self.getLogger().debug('Cache clear: node [{0}]'.format(name)) config.remove_section(name) self.instanceCacheWrite(config) def instanceCacheUpdate(self, name, added=None, deleted=None): """ 'added' is a list of key-value tuples to be added 'deleted' is a list of keys to be removed from the instance cache """ self.getLogger().debug( 'instanceCacheUpdate(): name=[{0}]'.format(name)) config = self.instanceCacheRefresh() if not config.has_section(name): config.add_section(name) for key, value in added or []: config.set(name, key, value) for key in deleted or []: config.remove_option(name, key) self.instanceCacheWrite(config) def __findNicForProvisioningNetwork(self, nics, prov_network): """ TODO: move this elsewhere Raises: NicNotFound """ nics = [nic for nic in nics if nic.network == prov_network] if not nics: raise NicNotFound( 'Unable to find NIC on provisioning network [%s]' % (prov_network.address + '/' + prov_network.netmask)) return nics[0] def writeLocalBootConfiguration(self, node, hardwareprofile, softwareprofile): """ Raises: NicNotFound """ if not hardwareprofile.nics: # Hardware profile has no provisioning NICs defined. This # shouldn't happen... self.getLogger().debug( 'No provisioning nics defined in hardware profile %s' % (hardwareprofile.name)) return # Determine the provisioning nic for the hardware profile hwProfileProvisioningNic = hardwareprofile.nics[0] nic = None if hwProfileProvisioningNic.network: # Find the nic attached to the newly added node that is on # the same network as the provisioning nic. nic = self.__findNicForProvisioningNetwork( node.nics, hwProfileProvisioningNic.network) if not nic or not nic.mac: self.getLogger().warning( 'MAC address not defined for nic (ip=[%s]) on node [%s]' % (nic.ip, node.name)) return # Set up DHCP/PXE for newly addded node bhm = getOsObjectFactory().getOsBootHostManager() # Write out the PXE file bhm.writePXEFile(node, hardwareprofile=hardwareprofile, softwareprofile=softwareprofile, localboot=False) # Add a DHCP lease bhm.addDhcpLease(node, nic) def removeLocalBootConfiguration(self, node): bhm = self.osObject.getOsBootHostManager() bhm.rmPXEFile(node) bhm.removeDhcpLease(node) def _pre_add_host(self, name, hwprofilename, swprofilename, ip): \ # pylint: disable=unused-argument # Perform "pre-add-host" operation command = ('sudo %s/pre-add-host' ' --hardware-profile %s' ' --software-profile %s' ' --host-name %s' % (self._cm.getBinDir(), hwprofilename, swprofilename, name)) if ip: command += ' --ip %s' % (ip) self.getLogger().debug('calling command= [%s]' % (command)) p = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, close_fds=True) p.communicate() p.wait() @property def installer_public_hostname(self): if self.__installer_public_hostname is None: cmd = '/opt/puppetlabs/bin/facter fqdn' with open(os.devnull, 'w') as devnull: p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=devnull) stdout, _ = p.communicate() retval = p.wait() if retval == 0: self.__installer_public_hostname = stdout.decode().rstrip() self.getLogger().debug('using installerName [%s] from Facter' % (self.__installer_public_hostname)) else: self.__installer_public_hostname = self._cm.getHost() self.getLogger().debug('using installerName [%s] from system' % (self.__installer_public_hostname)) return self.__installer_public_hostname @property def installer_public_ipaddress(self): # Get installer IP if self.__installer_public_ipaddress is None: self.getLogger().debug('Looking up installer IP using DNS') aiInfo = socket.getaddrinfo(self.installer_public_hostname, None, socket.AF_INET, socket.SOCK_STREAM) self.__installer_public_ipaddress = aiInfo[0][4][0] return self.__installer_public_ipaddress @property def private_dns_zone(self): if self.__private_dns_zone is None: self.__private_dns_zone = \ ParameterApi().getParameter('DNSZone').getValue() return self.__private_dns_zone def get_node_vcpus(self, name): \ # pylint: disable=unused-argument return 1 def get_instance_size_mapping(self, value): """ Helper method for matching the first field (instance size) in the resource adapter specific CSV file :return: instance type/size to vcpus mapping :returntype int: """ fn = os.path.join( self._cm.getKitConfigBase(), '{0}-instance-sizes.csv'.format(self.__adaptername__)) if not os.path.exists(fn): return 1 try: with open(fn) as fp: reader = csv.reader(fp) for row in reader: if row[0] == value: return int(row[1]) return 1 except Exception as exc: self.getLogger().error( 'Error processing instance type mapping' ' [{0}] (exc=[{1}]). Using default value'.format(fn, exc)) return 1