class SyncManager(TortugaObjectManager, Singleton): """Singleton class for cluster sync management""" # Singleton. __instanceLock = threading.RLock() # update delay increase (seconds) CLUSTER_UPDATE_DELAY_INCREASE = 30 # after this limit is reached, warning will be logged CLUSTER_UPDATE_WARNING_LIMIT = 10 def __init__(self): super(SyncManager, self).__init__() self._isUpdateScheduled = False self._isUpdateRunning = False self._sudoCmd = \ osUtility.getOsObjectFactory().getOsSysManager().getSudoCommand() self._cm = ConfigManager() def __runClusterUpdate(self): """ Run cluster update. """ self.getLogger().debug('Update timer running') updateCmd = '%s %s' % (self._sudoCmd, os.path.join(self._cm.getRoot(), 'bin/run_cluster_update.sh')) delay = 0 updateCnt = 0 while self.__resetIsUpdateScheduled(): self._isUpdateRunning = True self.getLogger().debug('New cluster update delay: %s seconds' % (delay)) time.sleep(delay) delay += SyncManager.CLUSTER_UPDATE_DELAY_INCREASE # Log warning if timer has been running for too many times. updateCnt += 1 self.getLogger().debug('Cluster update timer count: %s' % (updateCnt)) if updateCnt > SyncManager.CLUSTER_UPDATE_WARNING_LIMIT: self.getLogger().warn( 'Cluster updated more than %s times using the same' ' timer (possible configuration problem)' % (SyncManager.CLUSTER_UPDATE_WARNING_LIMIT)) self.getLogger().debug('Starting cluster update using: %s' % (updateCmd)) # Since we might sleep for a while, we need to # reset update flag just before we run update to avoid # unnecessary syncs. self.__resetIsUpdateScheduled() p = TortugaSubprocess(updateCmd) try: p.run() self.getLogger().debug('Cluster update successful') except CommandFailed: if p.getExitStatus() == tortugaStatus.\ TORTUGA_ANOTHER_INSTANCE_OWNS_LOCK_ERROR: self.getLogger().debug( 'Another cluster update is already running, will' ' try to reschedule it') self._isUpdateRunning = False self.scheduleClusterUpdate( updateReason='another update already running', delay=60) break else: self.getLogger().error( 'Update command "%s" failed (exit status: %s):' ' %s' % (updateCmd, p.getExitStatus(), p.getStdErr())) self.getLogger().debug('Done with cluster update') self._isUpdateRunning = False self.getLogger().debug('Update timer exiting') def __resetIsUpdateScheduled(self): """ Reset cluster update flag, return old flag value. """ SyncManager.__instanceLock.acquire() try: flag = self._isUpdateScheduled self._isUpdateScheduled = False return flag finally: SyncManager.__instanceLock.release() def scheduleClusterUpdate(self, updateReason=None, delay=5): """ Schedule cluster update. """ SyncManager.__instanceLock.acquire() try: if self._isUpdateScheduled: # Already scheduled. return # Start update timer if needed. self._isUpdateScheduled = True if not self._isUpdateRunning: self.getLogger().debug( 'Scheduling cluster update in %s seconds,' ' reason: %s' % (delay, updateReason)) t = threading.Timer(delay, self.__runClusterUpdate) t.start() else: self.getLogger().debug( 'Will not schedule new update timer while the old' ' timer is running') finally: SyncManager.__instanceLock.release() def getUpdateStatus(self): # pylint: disable=no-self-use """ Check cluster update flag. """ return RunManager().checkLock('cfmsync')
class KitInstallerBase(ConfigurableMixin, metaclass=KitInstallerMeta): """ Base class for kit installers. """ config_type = 'kit' # # The kit installation directory # install_path = None # # Metadata, loaded via the load_meta class method. # name = None version = None iteration = None spec = (None, None, None) meta = {} # # Attributes, provided by instances of this class # puppet_modules = [] def __init__(self): self.config_manager = ConfigManager() # # Setup paths # self.kit_path = os.path.dirname(inspect.getfile(self.__class__)) self.puppet_modules_path = os.path.join(self.kit_path, 'puppet_modules') self.files_path = os.path.join(self.kit_path, 'files') # # Initialize configuration # super().__init__() # # Load components and resource adapters # self._component_installers = {} self._component_installers_loaded = False # # Web service controller classes # self._ws_controller_classes = [] self._ws_controller_classes_loaded = False def get_config_base(self): return self.config_manager.getKitConfigBase() @classmethod def load_meta(cls, meta_dict): """ Loads the meta data for the kit into the class. :param meta_dict: A dict containing the metadata, as specified by the KitMetadataSchema class. """ errors = KitMetadataSchema().validate(meta_dict) if errors: raise Exception('Kit metadata validation error: {}'.format(errors)) meta_dict = copy.deepcopy(meta_dict) cls.name = meta_dict.pop('name') cls.version = meta_dict.pop('version') cls.iteration = meta_dict.pop('iteration') cls.spec = (cls.name, cls.version, cls.iteration) cls.meta = meta_dict def _load_component_installers(self): """ Load component installers for this kit. """ if self._component_installers_loaded: return kit_pkg_name = inspect.getmodule(self).__package__ comp_pkg_name = '{}.components'.format(kit_pkg_name) logger.debug( 'Searching for component installers in package: {}'.format( comp_pkg_name)) # # Look for the components sub-package # try: comp_pkg = importlib.import_module(comp_pkg_name) except ModuleNotFoundError: logger.warning('No component installers found for kit: {}'.format( kit_pkg_name)) return # # Walk the components sub-package, looking for component installers # for loader, name, ispkg in pkgutil.walk_packages(comp_pkg.__path__): if not ispkg: continue full_pkg_path = '{}.{}'.format(comp_pkg_name, name) try: # # Look for the component module in the package # comp_inst_mod = importlib.import_module( '{}.component'.format(full_pkg_path)) # # Look for the ComponentInstaller class in the module # if not hasattr(comp_inst_mod, 'ComponentInstaller'): logger.warning( 'ComponentInstaller class not found: {}'.format( full_pkg_path)) # # Initialize the ComponentInstaller class and register # it with the KitInstaller # comp_inst_class = comp_inst_mod.ComponentInstaller comp_inst = comp_inst_class(self) self._component_installers[comp_inst_class.name] = \ comp_inst logger.debug('Component installer registered: {}'.format( comp_inst.spec)) except ModuleNotFoundError: logger.debug( 'Package not a component: {}'.format(full_pkg_path)) self._component_installers_loaded = True def is_installable(self): """ Determines whether or not this kit is installable under the given conditions/circumstances. Override this in your implementations as necessary. :return: True if it is installable, False otherwise. """ return True def run_action(self, action_name, *args, **kwargs): """ Runs the specified action. :param action_name: the name of the action to run """ try: logger.debug('Calling kit action: {} with arguments {}, {}'.format( action_name, args, kwargs)) action = getattr(self, 'action_{}'.format(action_name)) return action(*args, **kwargs) except KeyError: raise Exception('Unknown action: {}'.format(action_name)) def get_kit(self): """ Gets the Kit instance for this kit. :return: a Kit instance """ kit = Kit(name=self.name, version=self.version, iteration=self.iteration) kit.setDescription(self.meta.get('description', None)) for component_installer in self.get_all_component_installers(): kit.addComponent(component_installer.get_component()) return kit def get_eula(self): """ Gets the EULA for this kit, if it exists. :return: a Eula instance if there is a EULA file, otherwise None. """ eula = None eula_path = os.path.join(self.install_path, EULA_FILE) if os.path.exists(eula_path) and os.path.isfile(eula_path): eula_fp = open(eula_path) text = eula_fp.read() eula_fp.close() eula = Eula(text=text) else: logger.debug('EULA not found: {}'.format(eula_path)) return eula def get_component_installer(self, component_name): self._load_component_installers() return self._component_installers[component_name] def get_all_component_installers(self): self._load_component_installers() return [ci for ci in self._component_installers.values()] def register_database_table_mappers(self): """ Register database table mappers for this kit. """ kit_pkg_name = inspect.getmodule(self).__package__ db_table_pkg_name = '{}.db.tables'.format(kit_pkg_name) logger.debug( 'Searching for database table mappers in package: {}'.format( db_table_pkg_name)) try: importlib.import_module(db_table_pkg_name) except ModuleNotFoundError: logger.debug('No database table mappers found for kit: {}'.format( self.spec)) def register_web_service_controllers(self): """ Register web service controllers for this kit. """ kit_pkg_name = inspect.getmodule(self).__package__ ws_pkg_name = '{}.web_service.controllers'.format(kit_pkg_name) logger.debug( 'Searching for web service controllers in package: {}'.format( ws_pkg_name)) try: importlib.import_module(ws_pkg_name) except ModuleNotFoundError: logger.debug('No web service controllers found for kit: {}'.format( self.spec)) def register_web_service_worker_actions(self): """ Register web service worker actions for this kit. """ kit_pkg_name = inspect.getmodule(self).__package__ ws_pkg_name = '{}.web_service.worker'.format(kit_pkg_name) logger.debug( 'Searching for web service worker actions in package: {}'.format( ws_pkg_name)) try: importlib.import_module(ws_pkg_name) except ModuleNotFoundError: logger.debug( 'No web service worker actions found for kit: {}'.format( self.spec)) def action_install_puppet_modules(self, *args, **kwargs): # # Prevent circular import # from .actions import InstallPuppetModulesAction return InstallPuppetModulesAction(self)(*args, **kwargs) def action_pre_install(self): pass def action_pre_uninstall(self): pass def action_post_install(self): # # Install required python packages from requirements.txt # requirements_path = os.path.join(self.kit_path, 'requirements.txt') pip_install_requirements(self, requirements_path) def action_post_uninstall(self): pass def action_uninstall_puppet_modules(self, *args, **kwargs): # # Prevent circular import # from .actions import UninstallPuppetModulesAction return UninstallPuppetModulesAction(self)(*args, **kwargs)
def main(verbose, debug, no_autodetect, ignore_iam, unattended, region, profile): ec2_metadata = get_ec2_metadata() print('Configuring Tortuga AWS resource adapter') if unattended: ignore_iam = False verbose = True if not region: region = ec2_metadata['placement']['availability-zone'][:-1] \ if ec2_metadata and not no_autodetect else None if verbose and region: print_statement('Region [{0}] obtained from EC2 instance metadata', region) if region: print_statement('Detected AWS region: [{0}]', region) else: if not no_autodetect: error_message('Error: unable to determine current AWS region') if unattended: sys.exit(1) response = input(colorama.Style.BRIGHT + 'AWS region [{}]: '.format(DEFAULT_AWS_REGION) + colorama.Style.RESET_ALL) if not response: region = DEFAULT_AWS_REGION else: region = response try: # validate region session = boto3.session.Session() if region not in session.get_available_regions('ec2'): error_message('Error: invalid AWS region [{0}]', region) sys.exit(1) except botocore.exceptions.EndpointConnectionError: error_message( 'Error connecting to EC2 endpoint (invalid region?)') sys.exit(1) creds = False iam_profile_name = None if not ignore_iam: # if IAM profile is not in use, query access/secret keys client = boto3.client('sts') response = None print(colorama.Fore.GREEN + colorama.Style.BRIGHT + 'Checking for IAM profile...' + colorama.Style.RESET_ALL, end=' ') try: response = client.get_caller_identity() iam_arn = response['Arn'] iam_user_policy = iam_arn.split(':')[5] assumed_role = False if iam_user_policy.startswith('assumed-role/'): _, iam_profile_name, _ = iam_user_policy.split('/', 2) assumed_role = True else: _, iam_profile_name = iam_user_policy.split('/', 1) print(iam_profile_name) if not assumed_role: print(colorama.Style.BRIGHT + colorama.Fore.YELLOW + '*' + colorama.Style.RESET_ALL + format_string_with_arg( ' Ensure IAM profile [{0}] is used to launch' ' Grid Engine/Tortuga instance', iam_profile_name)) creds = True except botocore.exceptions.NoCredentialsError: print(colorama.Fore.YELLOW + colorama.Style.BRIGHT + 'not found' + colorama.Style.RESET_ALL) except Exception as exc: # noqa pylint: disable=broad-except print(colorama.Style.DIM + '[debug] Error querying IAM profile name:' ' {0}'.format(exc) + colorama.Style.RESET_ALL) else: if debug: print(colorama.Style.DIM + '[debug] Ignoring IAM profile (--ignore-iam argument' ' specified)' + colorama.Style.RESET_ALL) access_key = None secret_key = None if not creds: if unattended: # cannot query for credentials in unattended mode print('Unable to query user for credentials in unattended mode') print('Exiting.') sys.exit(1) if os.getenv('AWS_ACCESS_KEY') and os.getenv('AWS_SECRET_ACCESS_KEY'): print(colorama.Fore.GREEN + colorama.Style.BRIGHT + 'Using AWS_ACCESS_KEY/AWS_SECRET_ACCESS_KEY from' ' environment' + colorama.Style.RESET_ALL) # use envvars for credentials access_key = os.getenv('AWS_ACCESS_KEY') secret_key = os.getenv('AWS_SECRET_ACCESS_KEY') if not access_key: print( format_string_with_arg( 'IAM profile not detected. Using' ' AWS access and secret access keys.', forecolour=colorama.Fore.YELLOW)) access_key = input(colorama.Style.BRIGHT + 'AWS access key: ' + colorama.Style.RESET_ALL) if not access_key: print('Aborted by user.') sys.exit(1) secret_key = getpass.getpass(colorama.Style.BRIGHT + 'AWS secret key: ' + colorama.Style.RESET_ALL) if not secret_key: print('Aborted by user.') sys.exit(1) # validate AWS credentials print('Validating AWS access credentials...', end=' ') ec2 = boto3.client('ec2', region_name=region, aws_access_key_id=access_key, aws_secret_access_key=secret_key) try: if debug: print() print(colorama.Style.DIM + '[debug] Calling \'describe_images()\' to ' 'validate credentials' + colorama.Style.RESET_ALL) ec2.describe_images(Owners=['self']) print(colorama.Fore.GREEN + colorama.Style.BRIGHT + 'ok.' + colorama.Style.RESET_ALL) except botocore.exceptions.ClientError as exc: print(colorama.Fore.RED + colorama.Style.BRIGHT + 'failed.' + colorama.Style.RESET_ALL) errmsg = 'Error validating provided access credentials' if exc.response['Error']['Code'] != 'AuthFailure': error_message(errmsg + ': {0}', exc) else: error_message(errmsg) sys.exit(1) else: # using available IAM profile ec2 = boto3.client('ec2', region_name=region) # Write/update "~/.aws/credentials" update_aws_credentials(region, access_key, secret_key) # keypair keypair = None try: # Extract keypair name from ssh public key metadata for _, key_values in ec2_metadata['public-keys'].items(): keypair = key_values['openssh-key'].split(' ')[-1] break except Exception: # noqa pylint: disable=broad-except pass while not keypair: response = input('Keypair [? for list]: ') if not response: break if response.startswith('?'): result = ec2.describe_key_pairs() for tmp_keypair in result['KeyPairs']: print(' ' + colorama.Fore.YELLOW + colorama.Style.BRIGHT + tmp_keypair['KeyName'] + colorama.Style.RESET_ALL + colorama.Style.DIM + ' (' + tmp_keypair['KeyFingerprint'] + ')' + colorama.Style.RESET_ALL) continue # validate keypair try: ec2.describe_key_pairs(KeyNames=[response]) keypair = response break except botocore.exceptions.ClientError as exc: if exc.response['Error']['Code'] == 'InvalidKeyPair.NotFound': print('Keypair invalid or inaccessible.') continue print('{0}'.format(exc.message)) continue if not keypair: print('Aborted by user.') sys.exit(1) # get values from EC2 metadata subnet_id = None group_id = None vpc_id = None if ec2_metadata: for _, values in \ ec2_metadata['network']['interfaces']['macs'].items(): subnet_id = values['subnet-id'] group_id = values['security-group-ids'] vpc_id = values['vpc-id'] print(colorama.Style.BRIGHT + colorama.Fore.GREEN + 'Detected subnet [' + colorama.Style.RESET_ALL + subnet_id + colorama.Style.BRIGHT + colorama.Fore.GREEN + '] (VPC [' + colorama.Style.RESET_ALL + vpc_id + colorama.Style.BRIGHT + colorama.Fore.GREEN + '])') print_statement('Detected security group [{}]', group_id) break # subnet_id if not subnet_id: subnets = ec2.describe_subnets() while not subnet_id: response = input('Subnet ID [? for list]: ') if not response: continue if response.startswith('?'): for subnet in subnets['Subnets']: name = get_resource_name_from_tag(subnet) buf = colorama.Fore.YELLOW + colorama.Style.BRIGHT + \ subnet['SubnetId'] + colorama.Style.RESET_ALL + \ colorama.Style.DIM if name: buf += ' ' + name buf += ' ({0}) (VPC ID: {1})'.format(subnet['CidrBlock'], subnet['VpcId']) + \ colorama.Style.RESET_ALL print(' ' + buf) continue for subnet in subnets['Subnets']: if subnet['SubnetId'] == response: subnet_id = response vpc_id = subnet['VpcId'] break else: # print('Error: invalid subnet ID') error_message('Error: invalid subnet ID') continue break # security group(s) while not group_id: response = input('Security group ID (? for list): ') if not response: break if response.startswith('?'): result = ec2.describe_security_groups(Filters=[{ 'Name': 'vpc-id', 'Values': [vpc_id] }]) for secgroup in result['SecurityGroups']: print(' ' + colorama.Fore.YELLOW + colorama.Style.BRIGHT + secgroup['GroupId'] + ' ' + colorama.Fore.WHITE + colorama.Style.DIM + secgroup['GroupName'] + colorama.Style.RESET_ALL) continue try: result = ec2.describe_security_groups(GroupIds=[response], Filters=[{ 'Name': 'vpc-id', 'Values': [vpc_id] }]) if result['SecurityGroups']: group_id = result['SecurityGroups'][0]['GroupId'] break except botocore.exceptions.ClientError: pass print('Invalid security group ID') if not group_id: print('Aborted by user.') sys.exit(1) ami_id = ec2_metadata['ami-id'] if ec2_metadata else None while not ami_id: response = input('UGE/Tortuga AMI ID: ') if not response: print('Aborted by user.') sys.exit(1) try: result = ec2.describe_images(ImageIds=[response]) if result['Images']: ami_id = response break except botocore.exceptions.ClientError as exc: if exc.response['Error']['Code'] == 'InvalidAMIID.Malformed': print('Malformed AMI ID') continue elif exc.response['Error']['Code'] == 'InvalidAMIID.NotFound': print('AMI not found or inaccessible') continue print('Invalid AMI ID: {0}'.format(exc.message)) continue break # query default compute instance type for instance_type in (DEFAULT_INSTANCE_TYPE, 'm4.large', 'm3.large'): if instance_type != DEFAULT_INSTANCE_TYPE: print(colorama.Style.BRIGHT + colorama.Fore.YELLOW + 'Falling back to [' + colorama.Style.RESET_ALL + instance_type + colorama.Style.BRIGHT + colorama.Fore.YELLOW + ']...') sys.stdout.flush() print(colorama.Style.BRIGHT + colorama.Fore.GREEN + 'Attempting to validate instance type [' + colorama.Style.RESET_ALL + instance_type + colorama.Style.BRIGHT + colorama.Fore.GREEN + ']... ' + colorama.Style.RESET_ALL, end='') sys.stdout.flush() # validate user-provided instance type result = validate_instance_type(ec2, instance_type, ami_id, subnet_id, debug=debug) if result: print(colorama.Style.BRIGHT + colorama.Fore.GREEN + 'done.' + colorama.Style.RESET_ALL) break print(colorama.Style.BRIGHT + colorama.Fore.RED + 'failed.' + colorama.Style.RESET_ALL) else: # unable to determine valid instance type error_message('\nUnable to determine valid instance type') sys.exit(1) # determine which bootstrap/cloud-init script template to use cloud_init_script_template = None user_data_script_template = None aws_adapter_cfg = os.path.join(ConfigManager().getKitConfigBase(), 'aws', 'adapter.ini') tags = '' if os.path.exists(aws_adapter_cfg): cfg = configparser.ConfigParser() cfg.read(aws_adapter_cfg) if cfg.has_section('aws'): if cfg.has_option('aws', 'cloud_init_script_template'): cloud_init_script_template = cfg.get( 'aws', 'cloud_init_script_template') elif cfg.has_option('aws', 'user_data_script_template'): user_data_script_template = cfg.get( 'aws', 'user_data_script_template') if cfg.has_option('aws', 'tags'): tags = cfg.get('aws', 'tags') if not user_data_script_template and not cloud_init_script_template: user_data_script_template = 'bootstrap.tmpl' adapter_cfg = { 'associate_public_ip_address': 'true', } if user_data_script_template: adapter_cfg['user_data_script_template'] = user_data_script_template elif cloud_init_script_template: adapter_cfg['cloud_init_script_template'] = cloud_init_script_template if access_key and secret_key: adapter_cfg['awsaccesskey'] = access_key adapter_cfg['awssecretkey'] = secret_key # parse tags to determine if 'Name' has been defined tag_parser = settings.TagListSetting() parsed_tags = tag_parser.dump(tags) if 'Name' not in parsed_tags.keys(): parsed_tags['Name'] = 'Tortuga compute node' parsed_tag_list = [] for k, v in parsed_tags.items(): parsed_tag_list.append('{}={}'.format(k, v)) tags = ','.join(parsed_tags) override_adapter_cfg = { 'keypair': keypair, 'ami': ami_id, 'instancetype': instance_type, 'securitygroup': ','.join(group_id.split('\n')), 'subnet_id': subnet_id, 'tags': tags, 'region': region, } adapter_cfg.update(override_adapter_cfg) _update_resource_adapter_configuration(adapter_cfg, profile) print_statement('Resource adapter configuration completed successfully.')
class WebsocketClient: """ Websocket client class. """ def __init__(self, token: Optional[str] = None, username: Optional[str] = None, password: Optional[str] = None, url: Optional[str] = None, verify: bool = True): self._token = token self._username = username self._password = password self._url = url self._verify = verify self._websocket = None self._cm = ConfigManager() async def start(self): """ Initializes the websocket and starts the event loop. """ if self._url.startswith('wss:'): ssl_context = ssl.SSLContext() if self._verify: ssl_context.load_verify_locations(self._cm.getCaBundle()) else: ssl_context.check_hostname = False ssl_context.verify_mode = ssl.CERT_NONE else: ssl_context = None async with websockets.connect(self._url, ssl=ssl_context) as ws: await self.send_recieve(ws) async def send_recieve(self, ws: websockets.WebSocketClientProtocol): """ The main loop that sends/receives data. :param ws: the web socket client """ while True: msg = await ws.recv() data = json.loads(msg) pretty_print(data) if data['type'] == 'message': if data['name'] == 'authentication-required': await self.send_auth(ws) if data['name'] == 'authentication-succeeded': await self.send_subscribe(ws) async def send_auth(self, ws: websockets.WebSocketClientProtocol): """ Sends an authentication request. :param ws: the web socket client """ if self._token: data = { 'action': 'authenticate', 'method': 'jwt', 'data': { 'token': self._token } } else: data = { 'action': 'authenticate', 'method': 'password', 'data': { 'username': self._username, 'password': self._password } } await ws.send(json.dumps(data)) async def send_subscribe(self, ws: websockets.WebSocketClientProtocol): """ Sends a subscription request. :param ws: the web socket client """ data = {'action': 'subscribe'} await ws.send(json.dumps(data))
class SoftwareProfileManager(TortugaObjectManager): \ # pylint: disable=too-many-public-methods BASE_KIT_NAME = 'base' def __init__(self): super(SoftwareProfileManager, self).__init__() self._sp_db_api = SoftwareProfileDbApi() self._node_db_api = NodeDbApi() self._component_db_api = ComponentDbApi() self._global_param_db_api = GlobalParameterDbApi() self._kit_db_api = KitDbApi() self._config_manager = ConfigManager() self._logger = logging.getLogger(SOFTWARE_PROFILE_NAMESPACE) def getSoftwareProfileList(self, session: Session, tags=None): """Return all of the softwareprofiles with referenced components in this softwareprofile """ results = self._sp_db_api.getSoftwareProfileList(session, tags=tags) for software_profile_obj in results: # load any available software profile metadata software_profile_obj.setMetadata( self.get_software_profile_metadata( session, software_profile_obj.getName())) return results def addAdmin(self, session: Session, softwareProfileName, adminUsername): """ Add an admin as an authorized user. Returns: None Throws: TortugaException AdminNotFound SoftwareProfileNotFound """ return self._sp_db_api.addAdmin(session, softwareProfileName, adminUsername) def deleteAdmin(self, session: Session, softwareProfileName, adminUsername): """ Remove an admin as an authorized user. Returns: None Throws: TortugaException AdminNotFound SoftwareProfileNotFound """ return self._sp_db_api.deleteAdmin(session, softwareProfileName, adminUsername) def updateSoftwareProfile(self, session: Session, softwareProfileObject): self._logger.debug('Updating software profile: %s' % (softwareProfileObject.getName())) # # First get the object from the db we are updating # existing_swp = self.getSoftwareProfileById( session, softwareProfileObject.getId()) # # Set parameters that we will not allow updating # softwareProfileObject.setOsInfo(existing_swp.getOsInfo()) softwareProfileObject.setOsId(existing_swp.getOsId()) softwareProfileObject.setType(existing_swp.getType()) # # Do the DB update # self._sp_db_api.updateSoftwareProfile(session, softwareProfileObject) # # Get the new version # new_swp = self.getSoftwareProfileById(session, softwareProfileObject.getId()) # # If the tags have changed, fire the tags changed event # if existing_swp.getTags() != new_swp.getTags(): SoftwareProfileTagsChanged.fire( softwareprofile_id=str(new_swp.getId()), softwareprofile_name=new_swp.getName(), tags=new_swp.getTags(), previous_tags=existing_swp.getTags()) def getSoftwareProfile( self, session: Session, name: str, optionDict: Optional[Dict[str, bool]] = None) -> SoftwareProfile: """ Retrieve software profile by name """ software_profile_obj: SoftwareProfile = \ self._sp_db_api.getSoftwareProfile( session, name, optionDict=optionDict) # load any available software profile metadata software_profile_obj.setMetadata( self.get_software_profile_metadata(session, name)) return software_profile_obj def getSoftwareProfileById( self, session: Session, id_: int, optionDict: Optional[Dict[str, bool]] = None) -> SoftwareProfile: """ Retrieve software profile by id """ software_profile_obj: SoftwareProfile = \ self._sp_db_api.getSoftwareProfileById( session, id_, optionDict=optionDict) # load any available software profile metadata software_profile_obj.setMetadata( self.get_software_profile_metadata(session, software_profile_obj.getName())) return software_profile_obj def _getCoreComponentForOsInfo(self, session: Session, osInfo): # Find core component baseKit = None for baseKit in self._kit_db_api.getKitList(session): if not baseKit.getName() == self.BASE_KIT_NAME: continue break else: raise KitNotFound('Kit [%s] not found.' % (self.BASE_KIT_NAME)) baseComp = None for baseComp in baseKit.getComponentList(): if baseComp.getName() != 'core': continue break else: raise ComponentNotFound('Component [%s] not found in kit [%s]' % ('core', baseKit.getName())) comp = self._component_db_api.getBestMatchComponent( session, baseComp.getName(), baseComp.getVersion(), osInfo, baseKit.getId()) comp.setKit(baseKit) return comp def _getOsInfo(self, session: Session, bOsMediaRequired: bool): if not bOsMediaRequired: # As a placeholder, use the same OS as the installer # Find installer node entry node = self._node_db_api.getNode(session, ConfigManager().getInstaller(), {'softwareprofile': True}) return node.getSoftwareProfile().getOsInfo() # Use available operating system kit; raise exception if # multiple available os_kits = self._kit_db_api.getKitList(session, os_kits_only=True) if not os_kits: raise KitNotFound('No operating system kit installed') if len(os_kits) > 1: raise KitNotFound( 'Multiple OS kits defined; use --os option to specify' ' operating system') kit = self._kit_db_api.getKit(session, os_kits[0].getName(), os_kits[0].getVersion(), '0') components = kit.getComponentList() if not components: raise ComponentNotFound('Malformed operating system kit [%s]' % (os_kits)) osinfo_list = components[0].getOsInfoList() if len(osinfo_list) > 1: raise ComponentNotFound( 'Multiple operating system components for kit [%s];' ' use --os argument to specify operating system' % (os_kits[0])) return osinfo_list[0] def createSoftwareProfile(self, session: Session, swProfileSpec, settingsDict=None): """ Exceptions: ConfigurationError NetworkNotFound ComponentNotFound KitNotFound OSError """ if settingsDict == None: settingsDict = {} bOsMediaRequired = settingsDict.get('bOsMediaRequired', True) unmanagedProfile = settingsDict.get('unmanagedProfile', False) # Validate software profile name validation.validateProfileName(swProfileSpec.getName()) # Insert default description for software profile if swProfileSpec.getDescription() is None: swProfileSpec.setDescription('%s Nodes' % (swProfileSpec.getName())) self._logger.debug('Creating software profile [%s]' % (swProfileSpec)) osInfo = swProfileSpec.getOsInfo() \ if swProfileSpec.getOsInfo() else self._getOsInfo( session, bOsMediaRequired) # If we're creating an unmanaged software profile (no # DHCP/PXE/kickstart/OS) just create it now and we're done if unmanagedProfile: self._sp_db_api.addSoftwareProfile(session, swProfileSpec) else: if bOsMediaRequired and swProfileSpec.getOsInfo(): try: self._kit_db_api.getKit( session, swProfileSpec.getOsInfo().getName(), swProfileSpec.getOsInfo().getVersion(), '0') except KitNotFound: self._logger.error('OS kit for [%s] not found' % (swProfileSpec.getOsInfo())) raise else: swProfileSpec.setOsInfo(osInfo) # Get component manager for appropriate OS family osConfig = osHelper.getOsInfo(osInfo.getName(), osInfo.getVersion(), osInfo.getArch()) osObjFactory = osUtility.getOsObjectFactory( osConfig.getOsFamilyInfo().getName()) # Need to be fancy with components spComponents = swProfileSpec.getComponents() swProfileSpec.setComponents(TortugaObjectList()) bFoundOsComponent = False bFoundCoreComponent = False components = [] # Iterate over components, adding them to the software profile for c in spComponents: cobj = self._component_db_api.getBestMatchComponent( session, c.getName(), c.getVersion(), osInfo, c.getKit().getId()) k = cobj.getKit() if k.getIsOs(): # This component is a member of the OS kit, set the flag bFoundOsComponent = True elif k.getName() == 'base' and c.getName() == 'core': # Found the 'core' component in 'base' kit bFoundCoreComponent = True components.append(cobj) # If the operating system is undefined for this software # profile, use the same OS as the installer. if bOsMediaRequired and not bFoundOsComponent: # Find OS component osCompName = '%s-%s-%s' % ( osInfo.getName(), osInfo.getVersion(), osInfo.getArch()) self._logger.debug('Automatically adding OS component [%s]' ' (not specified in template)' % (osCompName)) try: osComponent = self._component_db_api.getComponent( session, osCompName, osInfo.getVersion(), osInfo, {'kit': True}) components.append(osComponent) except ComponentNotFound: # Cannot find OS component, don't freak out pass # Ensure 'core' component is enabled if not bFoundCoreComponent: # Attempt to automatically add the core component, only # if one exists for this OS try: comp = self._getCoreComponentForOsInfo(session, osInfo) self._logger.debug('Automatically adding [core] component' ' (not specified in template)') components.append(comp) except ComponentNotFound: self._logger.warning( 'OS [{}] does not have a compatible \'core\'' ' component'.format(osInfo)) # Initialize values for kernel, kernelParams, and initrd if not swProfileSpec.getKernel(): swProfileSpec.setKernel( osObjFactory.getOsSysManager().getKernel(osInfo)) if not swProfileSpec.getInitrd(): swProfileSpec.setInitrd( osObjFactory.getOsSysManager().getInitrd(osInfo)) # Add the software profile self._sp_db_api.addSoftwareProfile(session, swProfileSpec) # Enable components in one fell swoop for comp in components: self._logger.debug('Enabling component [%s]' % (comp.getName())) if comp.getKit().getIsOs(): # Don't use enableComponent() on OS kit self._component_db_api.addComponentToSoftwareProfile( session, comp.getId(), swProfileSpec.getId()) continue self.enableComponent(session, swProfileSpec.getName(), comp.getKit().getName(), comp.getKit().getVersion(), comp.getKit().getIteration(), comp.getName(), comp.getVersion()) # # Fire the tags changed event for all creates that have tags # # Get the latest version from the db in case the create method # added some embellishments # swp = self.getSoftwareProfile(session, swProfileSpec.getName()) if swp.getTags(): SoftwareProfileTagsChanged.fire(softwareprofile_id=str( swp.getId()), softwareprofile_name=swp.getName(), tags=swp.getTags(), previous_tags={}) def _getComponent(self, kit, compName, compVersion): \ # pylint: disable=no-self-use # Iterate over component list, looking for a match comp = None for comp in kit.getComponentList(): if comp.getName() == compName and \ comp.getVersion() == compVersion: break else: raise ComponentNotFound("Component [%s-%s] not found in kit [%s]" % (compName, compVersion, kit)) return comp def _get_kit_by_component(self, session: Session, comp_name, comp_version=None): """ Gets a kit by compoent name/version. :param comp_name: the name of the component :param comp_version: the version of the component :raises KitNotFound: :raises ComponentNotFound: """ kit_list = self._kit_db_api.getKitList(session) kits = [ kit for kit in kit_list for component in kit.getComponentList() if component.getName() == comp_name and ( comp_version is None or component.getVersion() == comp_version) ] if not kits: raise KitNotFound('Kit containing component [%s] not found' % (comp_name)) if len(kits) > 1: raise ComponentNotFound( 'Kit name must be specified, multiple kits contain ' 'component: {}'.format(comp_name)) return kits[0] def enableComponent(self, session: Session, software_profile_name: str, kit_name: str, kit_version: str, kit_iteration: str, comp_name: str, comp_version: Optional[str] = None): """ Enable a component on a software profile. :param software_profile_name: the name of the software profile :param kit_name: the name of the kit :param kit_version: the version of the kit :param kit_iteration: the iteration of the kit :param comp_name: the name of the component :param comp_version: the version of the component :raises KitNotFound: :raises SoftwareProfileNotFound: :raises ComponentNotFound: """ kit, comp_version = self._get_kit_and_component_version( session, kit_name, kit_version, kit_iteration, comp_name, comp_version) software_profile = self.getSoftwareProfile(session, software_profile_name, {'os': True}) if kit.getIsOs(): best_match_component = self._enable_os_kit_component( session, kit, comp_name, comp_version, software_profile) else: best_match_component = self._enable_kit_component( session, kit, comp_name, comp_version, software_profile) if not best_match_component: self._logger.info('Component not enabled: {}'.format(comp_name)) else: self._logger.info( 'Enabled component on software profile: {} -> {}'.format( best_match_component, software_profile)) def _get_kit_and_component_version(self, session: Session, kit_name, kit_version, kit_iteration, comp_name, comp_version=None): """ Gets a Kit instance and component version. :param kit_name: the name of the kit :param kit_version: the version of the kit :param kit_iteration: the iteration of the kit :param comp_name: the component name :param comp_version: the component version (optional) :return: a tuple, consisting of (Kit, component_version) """ kit = None if kit_name is None: kit = self._get_kit_by_component(session, comp_name, comp_version=comp_version) # # Get component version if required # if comp_version is None: for component in kit.getComponentList(): if component.getName() == comp_name: comp_version = component.getVersion() break elif kit_version is None or kit_iteration is None: kits_found = 0 for k in self._kit_db_api.getKitList(session): if k.getName() == kit_name and \ (kit_version is None or k.getVersion() == kit_version) and \ (kit_iteration is None or k.getIteration() == kit_iteration): kit = k kits_found += 1 if kits_found > 1: if kit_version is not None: raise KitNotFound('Multiple kits found: {}-{}'.format( kit_name, kit_version)) else: raise KitNotFound( 'Multiple kits found {}'.format(kit_name)) else: kit = self._kit_db_api.getKit(session, kit_name, kit_version, kit_iteration) if kit is None: raise KitNotFound('Kit [%s] not found' % (Kit(kit_name, kit_version, kit_iteration))) return kit, comp_version def _enable_kit_component(self, session: Session, kit, comp_name, comp_version, software_profile): """ Enables a regular kit component on a specific software profile. :param kit: the Kit instance, whose component is being enabled :param comp_name: the name of the component to enable :param comp_version: the version of the component to enable :param software_profile: the software profile on which the component will be enabled :return: the Component instance that was enabled """ kit_spec = (kit.getName(), kit.getVersion(), kit.getIteration()) installer = get_kit_installer(kit_spec)() installer.session = session comp_installer = installer.get_component_installer(comp_name) if comp_installer is None: raise ComponentNotFound('Component [%s] not found in kit [%s]' % (comp_name, kit)) if not comp_installer.is_enableable(software_profile): self._logger.warning('Component cannot be enabled: {}'.format( comp_installer.spec)) return None comp_installer.run_action('pre_enable', software_profile.getName()) best_match_component = self._add_component_to_software_profile( session, kit, comp_name, comp_version, software_profile) comp_installer.run_action('enable', software_profile.getName()) comp_installer.run_action('post_enable', software_profile.getName()) return best_match_component def _enable_os_kit_component(self, session: Session, kit, comp_name, comp_version, software_profile): """ Enables an OS kit component on a specific software profile. :param kit: the OS Kit instance, whose component is being enabled :param comp_name: the name of the component to enable :param comp_version: the version of the component to enable :param software_profile: the software profile on which the component will be enabled :return: the Component instance that was enabled """ return self._add_component_to_software_profile(session, kit, comp_name, comp_version, software_profile) def _add_component_to_software_profile(self, session: Session, kit, comp_name, comp_version, software_profile): """ Adds a kit to a software profile. This is a data-only operation, as no pre/post enable actions are called. :param kit: the OS Kit instance, whose component is being added :param comp_name: the name of the component to add :param comp_version: the version of the component to add :param software_profile: the software profile to which the component will be added :return: the Component instance that was added """ best_match_component = \ self._component_db_api.getBestMatchComponent( session, comp_name, comp_version, software_profile.getOsInfo(), kit.getId()) self._component_db_api.addComponentToSoftwareProfile( session, best_match_component.getId(), software_profile.getId()) return best_match_component def disableComponent(self, session: Session, software_profile_name, kit_name, kit_version, kit_iteration, comp_name, comp_version=None): \ # pylint: disable=unused-argument """ Disables a component on a software profile. :param software_profile_name: the name of the software profile :param kit_name: the name of the kit :param kit_version: the version of the kit :param kit_iteration: the iteration of the kit :param comp_name: the name of the component :param comp_version: the version of the component :raises KitNotFound: :raises SoftwareProfileNotFound: :raises ComponentNotFound: """ kit, comp_version = self._get_kit_and_component_version( session, kit_name, kit_version, kit_iteration, comp_name) software_profile = self.getSoftwareProfile(session, software_profile_name, {'os': True}) if kit.getIsOs(): best_match_component = self._disable_os_kit_component( session, kit, comp_name, comp_version, software_profile) else: best_match_component = self._disable_kit_component( session, kit, comp_name, comp_version, software_profile) self._logger.info( 'Disabled component on software profile: {} -> {}'.format( best_match_component, software_profile)) def _disable_kit_component(self, session, kit, comp_name, comp_version, software_profile): """ Disables a regular kit component on a specific software profile. :param kit: the Kit instance, whose component is being disabled :param comp_name: the name of the component to disable :param comp_version: the version of the component to disable :param software_profile: the software profile on which the component will be disable :return: the Component instance that was disabled """ kit_spec = (kit.getName(), kit.getVersion(), kit.getIteration()) installer = get_kit_installer(kit_spec)() installer.session = session comp_installer = installer.get_component_installer(comp_name) if comp_installer is None: raise ComponentNotFound('Component [%s] not found in kit [%s]' % (comp_name, kit)) comp_installer.run_action('pre_disable', software_profile.getName()) comp_installer.run_action('disable', software_profile.getName()) best_match_component = \ self._remove_component_from_software_profile( session, kit, comp_name, comp_version, software_profile) comp_installer.run_action('post_disable', software_profile.getName()) return best_match_component def _disable_os_kit_component(self, session, kit, comp_name, comp_version, software_profile): """ Enables an OS kit component on a specific software profile. :param kit: the OS Kit instance, whose component is being disabled :param comp_name: the name of the component to disable :param comp_version: the version of the component to disable :param software_profile: the software profile on which the component will be disabled :return: the Component instance that was disabled """ return self._remove_component_from_software_profile( session, kit, comp_name, comp_version, software_profile) def _remove_component_from_software_profile(self, session: Session, kit, comp_name, comp_version, software_profile): """ Removes a kit to a software profile. This is a data-only operation, as no pre/post disable actions are called. :param kit: the OS Kit instance, whose component is being removed :param comp_name: the name of the component to remove :param comp_version: the version of the component to remove :param software_profile: the software profile to which the component will be removed :return: the Component instance that was removed """ best_match_component = self._component_db_api.getBestMatchComponent( session, comp_name, comp_version, software_profile.getOsInfo(), kit.getId()) self._component_db_api.deleteComponentFromSoftwareProfile( session, best_match_component.getId(), software_profile.getId()) return best_match_component def deleteSoftwareProfile(self, session: Session, name): """ Delete software profile by name Raises: SoftwareProfileNotFound """ self._sp_db_api.deleteSoftwareProfile(session, name) # Remove all flags for software profile swProfileFlagPath = os.path.join(self._config_manager.getRoot(), 'var/run/actions/%s' % (name)) if os.path.exists(swProfileFlagPath): shutil.rmtree(swProfileFlagPath) self._logger.info('Deleted software profile [%s]' % (name)) def getNodeList(self, session: Session, softwareProfileName): return self._sp_db_api.getNodeList(session, softwareProfileName) def getEnabledComponentList(self, session: Session, name): """ Get the list of enabled components """ return self._sp_db_api.getEnabledComponentList(session, name) def getPartitionList(self, session: Session, softwareProfileName): """ Get list of partitions. """ return self._sp_db_api.getPartitionList(session, softwareProfileName) def addUsableHardwareProfileToSoftwareProfile( self, session: Session, hardwareProfileName: str, softwareProfileName: str) -> None: """ Map software profile to hardware profile """ self._logger.info( 'Mapping hardware profile [%s] to software profile [%s]', hardwareProfileName, softwareProfileName) self._sp_db_api.addUsableHardwareProfileToSoftwareProfile( session, hardwareProfileName, softwareProfileName) def deleteUsableHardwareProfileFromSoftwareProfile(self, session: Session, hardwareProfileName, softwareProfileName): return self._sp_db_api.deleteUsableHardwareProfileFromSoftwareProfile( session, hardwareProfileName, softwareProfileName) def copySoftwareProfile(self, session: Session, srcSoftwareProfileName, dstSoftwareProfileName): validation.validateProfileName(dstSoftwareProfileName) self._logger.info('Copying software profile [%s] to [%s]', srcSoftwareProfileName, dstSoftwareProfileName) self._sp_db_api.copySoftwareProfile(session, srcSoftwareProfileName, dstSoftwareProfileName) # # Fire the tags changed event for all copies that have tags # swp = self.getSoftwareProfile(session, dstSoftwareProfileName) if swp.getTags(): SoftwareProfileTagsChanged.fire(softwareprofile_id=str( swp.getId()), softwareprofile_name=swp.getName(), tags=swp.getTags(), previous_tags={}) def getUsableNodes(self, session: Session, softwareProfileName): return self._sp_db_api.getUsableNodes(session, softwareProfileName) def get_software_profile_metadata(self, session: Session, name: str) -> Dict[str, str]: """ Call action_get_metadata() method for all kits """ self._logger.debug('Retrieving metadata for software profile [%s]', name) metadata: Dict[str, str] = {} for kit in self._kit_db_api.getKitList(session): if kit.getIsOs(): # ignore OS kits continue kit_installer = get_kit_installer( (kit.getName(), kit.getVersion(), kit.getIteration()))() kit_installer.session = session # we are only interested in software profile metadata item = kit_installer.action_get_metadata( software_profile_name=name) if item: metadata.update(item) return metadata
class ComponentInstaller(ComponentInstallerBase): """ Tortuga DHCP component. """ name = 'dhcpd' version = '7.0.1' os_list = [ { 'family': 'rhel', 'version': '6', 'arch': 'x86_64' }, { 'family': 'rhel', 'version': '7', 'arch': 'x86_64' }, ] installer_only = True def __init__(self, kit): """ Initialise parent class. """ super().__init__(kit) self._provider = DhcpdDhcpProvider(self) self._manager = self._get_os_dhcpd_manager('dhcpd') self._config = ConfigManager() def _get_os_dhcpd_manager(self, name): """ Get dhcpd manager for the appropriate os. :param name: the name of the dhcpd manager to get :returns: the dhcpd manager instance """ dir_name = '{}/util'.format(self.kit_installer.kit_path) dhcpd_manager = \ getOsObjectFactory().getOsKitApplicationManager(name, dir_name) return dhcpd_manager def _get_provisioning_networks(self): """ Get provisioning networks. :returns: Generator provisioning networks """ for network in NetworksDbHandler().getNetworkList(self.session): if network.type == 'provision': yield network def _get_provisioning_nics(self, node): """ Get provisioning nics. :param node: Node object :returns: Generator nics """ for nic in node.getNics(): if nic.getNetwork().getType() == 'provision': yield nic def _get_provisioning_nics_ip(self, node): """ Get provisioning nics IP addresses. :param node: Node object :returns: Generator IPv4Address """ for nic in self._get_provisioning_nics(node): yield ipaddress.IPv4Address(nic.getIp()) @staticmethod def _get_local_nics(nics): """ Get valid NICs. :returns: Generator nics """ for nic in nics: if nic.boot and nic.mac: yield nic def _get_installer_ip(self, network_id): """ Return IP address of provisioning interface on installer :raises NicNotFound: """ installer_node = NodeApi().getInstallerNode(self.session) prov_nics = self._get_provisioning_nics(installer_node) for prov_nic in prov_nics: if prov_nic.getNetwork().getId() == network_id: return ipaddress.IPv4Address(prov_nic.getIp()) raise NicNotFound( 'Network has no corresponding provisioning NIC on installer') def _dhcp_subnets(self): """ DHCP subnet dictionary. :returns: Dictionary IPv4Network network address IPv4Network subnet """ subnets = {} for network in self._get_provisioning_networks(): subnet = {'nodes': []} installer_ip = self._get_installer_ip(network.id) subnet['installerIp'] = installer_ip if not network.gateway: logger.info( '[dhcpd] Gateway not defined for network [{}/{}], using' ' IP [{}]'.format(network.address, network.netmask, installer_ip)) subnet['gateway'] = installer_ip else: subnet['gateway'] = network.gateway for nic in self._get_local_nics(network.nics): node = nic.node if node.hardwareprofile.location != 'local' \ or node.state == 'Deleted' \ or node.name == self._config.getInstaller(): continue node = { 'ip': nic.ip, 'mac': nic.mac, 'fqdn': node.name, 'hostname': node.name.split('.', 1)[0], 'unmanaged': False } subnet['nodes'].append(node) subnet_address = ipaddress.IPv4Network('{}/{}'.format( network.address, network.netmask)) subnets[subnet_address] = subnet return subnets @property def _get_kit_settings_dictionary(self): """ :returns: Dictionary """ settings = {} config = configparser.ConfigParser() config.read( os.path.join(self._config.getKitConfigBase(), 'tortuga.ini')) if config.has_section('tortuga_kit_base'): if config.has_option('tortuga_kit_base', 'disable_services'): settings['disable_services'] = \ config.get('tortuga_kit_base', 'disable_services') \ .split(' ') return settings def _configure(self, softwareProfileName, fd, *args, **kwargs): """ Shim for unused arguments. :param softwareProfileName: :param fd: :param *args: :param **kwargs: :returns: None """ self.action_configure(softwareProfileName, *args, **kwargs) def action_configure(self, _, *args, **kwargs): """ Configure. :param _: Unused :param *args: Unused :param **kwargs: Unused :returns: None """ try: result = GlobalParameterDbApi().getParameter( self.session, 'DHCPLeaseTime') dhcp_lease_time = int(result.getValue()) except ParameterNotFound: dhcp_lease_time = 2400 try: result = GlobalParameterDbApi().getParameter( self.session, 'DNSZone') dns_zone = result.getValue() except ParameterNotFound: dns_zone = '' installer_node = NodeApi().getInstallerNode(self.session) self._manager.configure(dhcp_lease_time, dns_zone, self._get_provisioning_nics_ip(installer_node), self._dhcp_subnets(), installerNode=installer_node, bUpdateSysconfig=kwargs.get( 'bUpdateSysconfig', True), kit_settings=self._get_kit_settings_dictionary) def action_post_install(self, *args, **kwargs): """ Triggered post install. :param *args: List Objects :param **kwargs: Dictionary Objects :returns: None """ self._provider.write() self.action_configure(None, args, kwargs, bUpdateSysconfig=True) def action_add_host(self, hardware_profile_name, software_profile_name, nodes, *args, **kwargs): """ Triggerd at add host. :returns: None """ self.action_configure(software_profile_name, None, args, kwargs, bUpdateSysconfig=False) def action_delete_host(self, hardware_profile_name, software_profile_name, nodes, *args, **kwargs): """ Triggered delete host. :returns: None """ self.action_configure(software_profile_name, None, args, kwargs, bUpdateSysconfig=False)
class SyncManager(TortugaObjectManager): """Class for cluster sync management""" __instanceLock = threading.RLock() # update delay increase (seconds) CLUSTER_UPDATE_DELAY_INCREASE = 30 # after this limit is reached, warning will be logged CLUSTER_UPDATE_WARNING_LIMIT = 10 def __init__(self): super(SyncManager, self).__init__() self._isUpdateScheduled = False self._isUpdateRunning = False self._cm = ConfigManager() self._logger = logging.getLogger(SYNC_NAMESPACE) def __runClusterUpdate(self, opts={}): """ Run cluster update. """ self._logger.debug('Update timer running, opts={}'.format(opts)) updateCmd = os.path.join(self._cm.getBinDir(), 'run_cluster_update.sh') delay = 0 updateCnt = 0 while self.__resetIsUpdateScheduled(): self._isUpdateRunning = True self._logger.debug('New cluster update delay: %s seconds' % (delay)) time.sleep(delay) delay += SyncManager.CLUSTER_UPDATE_DELAY_INCREASE # Log warning if timer has been running for too many times. updateCnt += 1 self._logger.debug('Cluster update timer count: %s' % (updateCnt)) if updateCnt > SyncManager.CLUSTER_UPDATE_WARNING_LIMIT: self._logger.warning( 'Cluster updated more than %s times using the same' ' timer (possible configuration problem)' % (SyncManager.CLUSTER_UPDATE_WARNING_LIMIT)) self._logger.debug('Starting cluster update using: %s' % (updateCmd)) # Since we might sleep for a while, we need to # reset update flag just before we run update to avoid # unnecessary syncs. self.__resetIsUpdateScheduled() if 'node' in opts: node_update = opts['node'] env = { **os.environ, 'FACTER_node_tags_update': json.dumps(node_update) } self._logger.debug('FACTER_node_tags_update={}'.format( env['FACTER_node_tags_update'])) p = TortugaSubprocess(updateCmd, env=env) elif 'software_profile' in opts: sp_update = opts['software_profile'] env = { **os.environ, 'FACTER_softwareprofile_tags_update': json.dumps(sp_update) } self._logger.debug( 'FACTER_softwareprofile_tags_update={}'.format( env['FACTER_softwareprofile_tags_update'])) p = TortugaSubprocess(updateCmd, env=env) else: p = TortugaSubprocess(updateCmd) try: p.run() self._logger.debug('Cluster update successful') self._logger.debug('stdout: {}'.format( p.getStdOut().decode().rstrip())) self._logger.debug('stderr: {}'.format( p.getStdErr().decode().rstrip())) except CommandFailed: if p.getExitStatus() == tortugaStatus.\ TORTUGA_ANOTHER_INSTANCE_OWNS_LOCK_ERROR: self._logger.debug( 'Another cluster update is already running, will' ' try to reschedule it') self._isUpdateRunning = False self.scheduleClusterUpdate( updateReason='another update already running', delay=60, opts=opts) break else: self._logger.error( 'Update command "%s" failed (exit status: %s):' ' %s' % (updateCmd, p.getExitStatus(), p.getStdErr())) self._logger.debug('Done with cluster update') self._isUpdateRunning = False self._logger.debug('Update timer exiting') def __resetIsUpdateScheduled(self): """ Reset cluster update flag, return old flag value. """ SyncManager.__instanceLock.acquire() try: flag = self._isUpdateScheduled self._isUpdateScheduled = False return flag finally: SyncManager.__instanceLock.release() def scheduleClusterUpdate(self, updateReason=None, delay=5, opts={}): """ Schedule cluster update. """ SyncManager.__instanceLock.acquire() try: if self._isUpdateScheduled: # Already scheduled. return # Start update timer if needed. self._isUpdateScheduled = True if not self._isUpdateRunning: self._logger.debug('Scheduling cluster update in %s seconds,' ' reason: %s, opts: %s' % (delay, updateReason, opts)) t = threading.Timer(delay, self.__runClusterUpdate, kwargs=dict(opts=opts)) t.start() else: self._logger.debug( 'Will not schedule new update timer while the old' ' timer is running') finally: SyncManager.__instanceLock.release() def getUpdateStatus(self): # pylint: disable=no-self-use """ Check cluster update flag. """ return RunManager().checkLock('cfmsync')
class RuleEngine(RuleEngineInterface): def __init__(self, minTriggerInterval=60): self._cm = ConfigManager() self._lock = threading.RLock() self._processingLock = threading.RLock() self._minTriggerInterval = minTriggerInterval self._ruleDict = {} self._disabledRuleDict = {} # Used for rules in the disabled state self._eventRuleDict = {} # used for "event" type monitoring self._pollTimerDict = {} # used for "poll" monitoring self._receiveRuleDict = {} # used for "receive" type monitoring self._receiveQ = queue.Queue(0) # infinite size FIFO queue self._rulesDir = self._cm.getRulesDir() self._logger = logging.getLogger('tortuga.rule.%s' % self.__class__.__name__) self._logger.addHandler(logging.NullHandler()) self.__initRules() # the following are used for "receive" type monitoring self._processingTimer = None self._processingTimerRunning = False def __getRuleDirName(self, applicationName): return '%s/%s' % (self._rulesDir, applicationName) def __getRuleFileName(self, applicationName, ruleName): return '%s/%s.xml' % (self.__getRuleDirName(applicationName), ruleName) def __readRuleFile(self, applicationName, ruleName): with open(self.__getRuleFileName(applicationName, ruleName)) as ruleFile: content = ruleFile.read() return content def __writeRuleFile(self, rule): ruleDir = self.__getRuleDirName(rule.getApplicationName()) if not os.path.exists(ruleDir): os.makedirs(ruleDir) with open( self.__getRuleFileName(rule.getApplicationName(), rule.getName()), 'w') as ruleFile: ruleFile.write('%s\n' % (rule.getXmlRep())) def __getRuleId(self, applicationName, ruleName): # pylint: disable=no-self-use return '%s/%s' % (applicationName, ruleName) def __checkRuleExists(self, ruleId): if ruleId not in self._ruleDict: raise RuleNotFound('Rule [%s] not found.' % ruleId) def __checkRuleDoesNotExist(self, ruleId): """ Raises: RuleAlreadyExists """ if ruleId in self._ruleDict: raise RuleAlreadyExists('Rule [%s] already exists.' % ruleId) def __checkRuleEnabled(self, ruleId): """ Raises: RuleNotFound RuleDisabled """ self.__checkRuleExists(ruleId) if ruleId in self._disabledRuleDict: raise RuleDisabled('Rule [%s] is disabled.' % ruleId) def __initRules(self): """ Initialize all known rules. """ self._logger.debug('[%s] Initializing known rules' % (self.__class__.__name__)) fileList = osUtility.findFiles(self._rulesDir) parser = RuleXmlParser() for f in fileList: try: rule = parser.parse(f) ruleId = self.__getRuleId(rule.getApplicationName(), rule.getName()) self._logger.debug('[%s] Found rule [%s]' % (self.__class__.__name__, ruleId)) self.addRule(rule) except Exception as ex: self._logger.error('[%s] Invalid rule file [%s] (Error: %s)' % (self.__class__.__name__, f, ex)) def __evaluateNumbers(self, metric, operator, triggerValue): \ # pylint: disable=exec-used trigger = None try: triggerString = '%s %s %s' % (metric, operator, triggerValue) self._logger.debug('[%s] Evaluating as numbers: %s' % (self.__class__.__name__, triggerString)) exec('trigger = %s' % triggerString) return trigger except Exception as ex: self._logger.debug('[%s] Could not evaluate as numbers: %s' % (self.__class__.__name__, ex)) return None def __evaluateStrings(self, metric, operator, triggerValue): \ # pylint: disable=exec-used trigger = None try: triggerString = '"%s" %s "%s"' % (metric, operator, triggerValue) self._logger.debug('[%s] Evaluating as strings: %s' % (self.__class__.__name__, triggerString)) exec('trigger = %s' % triggerString) return trigger except Exception as ex: self._logger.debug('[%s] Could not evaluate as strings: %s' % (self.__class__.__name__, ex)) return None def __parseMonitorData(self, monitorData=''): if not monitorData: return None self._logger.debug('[%s] Parsing data: %s' % (self.__class__.__name__, monitorData)) try: return libxml2.parseDoc(monitorData) except Exception as ex: self._logger.error('[%s] Could not parse data: %s' % (self.__class__.__name__, ex)) return None def __evaluateConditions(self, rule, monitorXmlDoc=None, xPathReplacementDict=None): # Return True if all rule conditions were satisfied. triggerAction = False try: if monitorXmlDoc is not None: triggerAction = True for condition in rule.getConditionList(): self._logger.debug('[%s] Evaluating: [%s]' % (self.__class__.__name__, condition)) metricXPath = condition.getMetricXPath() metric = self.__replaceXPathVariables( metricXPath, xPathReplacementDict or {}) if metric == metricXPath: # No replacement was done, try to evaluate xpath. metric = monitorXmlDoc.xpathEval('%s' % metricXPath) self._logger.debug('[%s] Got metric: [%s]' % (self.__class__.__name__, metric)) if metric == "" or metric == "nan": self._logger.debug( '[%s] Metric is not defined, will not trigger' ' action' % (self.__class__.__name__)) triggerAction = False break operator = condition.getEvaluationOperator() triggerValue = self.__replaceXPathVariables( condition.getTriggerValue(), xPathReplacementDict or {}) trigger = self.__evaluateNumbers(metric, operator, triggerValue) if trigger is None: trigger = self.__evaluateStrings( metric, operator, triggerValue) self._logger.debug('[%s] Evaluation result: [%s]' % (self.__class__.__name__, trigger)) if not trigger: triggerAction = False break else: self._logger.debug( '[%s] No monitor xml doc, will not trigger action' % (self.__class__.__name__)) except Exception as ex: self._logger.error('[%s] Could not evaluate data: %s' % (self.__class__.__name__, ex)) self._logger.debug('[%s] Will not trigger action' % (self.__class__.__name__)) self._logger.debug('[%s] Returning trigger action flag: [%s]' % (self.__class__.__name__, triggerAction)) return triggerAction def __evaluateXPathVariables(self, xmlDoc, xPathVariableList): resultDict = {} if not xmlDoc: return resultDict self._logger.debug('[%s] xPath variable list: %s' % (self.__class__.__name__, xPathVariableList)) for v in xPathVariableList: name = v.getName() value = '' try: self._logger.debug( '[%s] Evaluating xPath variable %s: %s' % (self.__class__.__name__, name, v.getXPath())) value = xmlDoc.xpathEval('%s' % v.getXPath()) except Exception as ex: self._logger.error( '[%s] Could not evaluate xPath variable [%s]: %s' % (self.__class__.__name__, name, ex)) self._logger.debug('[%s] Will replace it with empty string' % (self.__class__.__name__)) resultDict[name] = value self._logger.debug('[%s] XPath variable replacement dictionary: %s' % (self.__class__.__name__, resultDict)) return resultDict def __replaceXPathVariables(self, inputString, xPathReplacementDict): \ # pylint: disable=no-self-use outputString = inputString #self._logger.debug('Original string: %s' % inputString) for key in xPathReplacementDict: #self._logger.debug('Replacing: %s' % key) outputString = outputString.replace( key, '%s' % xPathReplacementDict[key]) #self._logger.debug('New string: %s' % outputString) return outputString def __poll(self, rule): ruleId = self.__getRuleId(rule.getApplicationName(), rule.getName()) self._logger.debug('[%s] Begin poll timer for [%s]' % (self.__class__.__name__, ruleId)) if not self.hasRule(ruleId): self._logger.debug('[%s] Timer execution cancelled for [%s]' % (self.__class__.__name__, ruleId)) return rule.ruleInvoked() self._logger.debug('[%s] Timer execution started for [%s]' % (self.__class__.__name__, ruleId)) appMonitor = rule.getApplicationMonitor() queryCmd = appMonitor.getQueryCommand() self._logger.debug('[%s] Query command: %s' % (self.__class__.__name__, queryCmd)) actionCmd = appMonitor.getActionCommand() self._logger.debug('[%s] Action command: %s' % (self.__class__.__name__, actionCmd)) xPathReplacementDict = {} try: invokeAction = True queryStdOut = None if queryCmd: self._logger.debug('[%s] About to invoke: [%s]' % (self.__class__.__name__, queryCmd)) try: p = tortugaSubprocess.executeCommand( 'source %s/tortuga.sh && ' % (self._cm.getEtcDir()) + queryCmd) queryStdOut = p.getStdOut() appMonitor.queryInvocationSucceeded() except Exception as ex: appMonitor.queryInvocationFailed() raise monitorXmlDoc = self.__parseMonitorData(queryStdOut) xPathReplacementDict = self.__evaluateXPathVariables( monitorXmlDoc, rule.getXPathVariableList()) invokeAction = self.__evaluateConditions( rule, monitorXmlDoc, xPathReplacementDict) if invokeAction: try: actionCmd = self.__replaceXPathVariables( actionCmd, xPathReplacementDict) self._logger.debug('[%s] About to invoke: [%s]' % (self.__class__.__name__, actionCmd)) p = tortugaSubprocess.executeCommand( 'source %s/tortuga.sh && ' % (self._cm.getEtcDir()) + actionCmd) appMonitor.actionInvocationSucceeded() self._logger.debug('[%s] Done with command: [%s]' % (self.__class__.__name__, actionCmd)) except Exception as ex: appMonitor.actionInvocationFailed() raise else: self._logger.debug('[%s] Will skip action: [%s]' % (self.__class__.__name__, actionCmd)) except TortugaException as ex: self._logger.error('[%s] %s' % (self.__class__.__name__, ex)) scheduleTimer = True if self.hasRule(ruleId): # Check if we need to stop invoking this rule. maxActionInvocations = appMonitor.getMaxActionInvocations() successfulActionInvocations = \ appMonitor.getSuccessfulActionInvocations() if maxActionInvocations: if int(maxActionInvocations) <= successfulActionInvocations: # Rule must be disabled. self._logger.debug( '[%s] Max. number of successful invocations (%s)' ' reached for rule [%s]' % (self.__class__.__name__, maxActionInvocations, ruleId)) scheduleTimer = False self.disableRule(rule.getApplicationName(), rule.getName()) else: # Rule is already deleted. scheduleTimer = False if scheduleTimer: pollPeriod = float(appMonitor.getPollPeriod()) # Make sure we do not fire too often. lastSuccessfulActionTime = \ appMonitor.getLastSuccessfulActionInvocationTime() if lastSuccessfulActionTime: now = time.time() possibleNewSuccessfulActionTime = \ now + pollPeriod - lastSuccessfulActionTime if possibleNewSuccessfulActionTime < self._minTriggerInterval: pollPeriod = self._minTriggerInterval self._logger.debug( '[%s] Increasing poll period to [%s] for' ' rule [%s]' % (self.__class__.__name__, pollPeriod, ruleId)) self._logger.debug('[%s] Scheduling new timer for rule [%s] in' ' [%s] seconds' % (self.__class__.__name__, ruleId, pollPeriod)) t = threading.Timer(pollPeriod, self.__poll, args=[rule]) t.daemon = True self.__runPollTimer(ruleId, t) else: self._logger.debug( '[%s] Will not schedule new timer for rule [%s]' % (self.__class__.__name__, rule)) def __runPollTimer(self, ruleId, pollTimer): self._pollTimerDict[ruleId] = pollTimer self._logger.debug('[%s] Starting poll timer for [%s]' % (self.__class__.__name__, ruleId)) pollTimer.start() def __cancelPollTimer(self, ruleId): if ruleId not in self._pollTimerDict: self._logger.debug('[%s] No poll timer for [%s]' % (self.__class__.__name__, ruleId)) return pollTimer = self._pollTimerDict[ruleId] self._logger.debug('[%s] Stopping poll timer for [%s]' % (self.__class__.__name__, ruleId)) pollTimer.cancel() del self._pollTimerDict[ruleId] def __process(self): self._logger.debug('[%s] Begin processing timer' % (self.__class__.__name__)) while True: qSize = self._receiveQ.qsize() self._logger.debug('[%s] Current receive Q size: %s' % (self.__class__.__name__, qSize)) if qSize == 0: break applicationName, applicationData = self._receiveQ.get() self._logger.debug('[%s] Processing data for [%s]' % (self.__class__.__name__, applicationName)) monitorXmlDoc = self.__parseMonitorData(applicationData) for ruleId in self._receiveRuleDict.keys(): rule = self._receiveRuleDict.get(ruleId) # Rule might have been cancelled before we use it. if not rule: continue # Check if this is appropriate for the data. if rule.getApplicationName() != applicationName: continue self._logger.debug('[%s] Processing data using rule [%s]' % (self.__class__.__name__, ruleId)) rule.ruleInvoked() appMonitor = rule.getApplicationMonitor() actionCmd = appMonitor.getActionCommand() self._logger.debug('[%s] Action command: [%s]' % (self.__class__.__name__, actionCmd)) try: xPathReplacementDict = self.__evaluateXPathVariables( monitorXmlDoc, rule.getXPathVariableList()) invokeAction = self.__evaluateConditions( rule, monitorXmlDoc, xPathReplacementDict) if invokeAction: try: actionCmd = self.__replaceXPathVariables( actionCmd, xPathReplacementDict) self._logger.debug( '[%s] About to invoke: [%s]' % (self.__class__.__name__, actionCmd)) tortugaSubprocess.executeCommand( 'source %s/tortuga.sh && ' % (self._cm.getEtcDir()) + actionCmd) appMonitor.actionInvocationSucceeded() self._logger.debug( '[%s] Done with command: [%s]' % (self.__class__.__name__, actionCmd)) maxActionInvocations = \ appMonitor.getMaxActionInvocations() successfulActionInvocations = \ appMonitor.getSuccessfulActionInvocations() if maxActionInvocations: if int(maxActionInvocations) <= \ successfulActionInvocations: # Rule must be disabled. self._logger.debug( '[%s] Max. number of successful' ' invocations (%s) reached for' ' rule [%s]' % (self.__class__.__name__, maxActionInvocations, ruleId)) self.disableRule(rule.getApplicationName(), rule.getName()) except Exception as ex: appMonitor.actionInvocationFailed() else: self._logger.debug( '[%s] Will skip action: [%s]' % (self.__class__.__name__, actionCmd)) except TortugaException as ex: self._logger.error('[%s] %s' % (self.__class__.__name__, ex)) self._logger.debug('[%s] No more rules appropriate for [%s]' % (self.__class__.__name__, applicationName)) # No more data to process, exit timer. self._logger.debug('[%s] No more data to process' % (self.__class__.__name__)) self.__cancelProcessingTimer() def __runProcessingTimer(self): self._processingLock.acquire() try: if not self._processingTimerRunning: self._logger.debug('[%s] Starting processing timer' % (self.__class__.__name__)) self._processingTimer = threading.Timer(5, self.__process) self._processingTimer.daemon = True self._processingTimer.start() self._processingTimerRunning = True else: self._logger.debug('[%s] Processing timer already running' % (self.__class__.__name__)) finally: self._processingLock.release() def __cancelProcessingTimer(self): self._processingLock.acquire() try: self._processingTimerRunning = False self._logger.debug('[%s] Processing timer stopped' % (self.__class__.__name__)) finally: self._processingLock.release() def hasRule(self, ruleId): return ruleId in self._ruleDict def addRule(self, rule): self._lock.acquire() try: return self.__addRule(rule) finally: self._lock.release() def __addRule(self, rule): ruleId = self.__getRuleId(rule.getApplicationName(), rule.getName()) self._logger.debug('[%s] Adding rule: [%s]' % (self.__class__.__name__, ruleId)) self.__checkRuleDoesNotExist(ruleId) # Write rule file. self.__writeRuleFile(rule) rule.decode() self._ruleDict[ruleId] = rule if rule.isStatusEnabled(): self.__enableRule(rule) else: # Rule is disabled, just put it in the 'disabled' dict self._disabledRuleDict[ruleId] = rule return ruleId def __enableRule(self, rule): ruleId = self.__getRuleId(rule.getApplicationName(), rule.getName()) self._logger.debug('[%s] Enabling rule: [%s]' % (self.__class__.__name__, ruleId)) appMonitor = rule.getApplicationMonitor() monitorType = appMonitor.getType() rule.setStatusEnabled() if monitorType == 'poll': self._logger.debug('[%s] [%s] is poll rule' % (self.__class__.__name__, ruleId)) pollPeriod = appMonitor.getPollPeriod() if not pollPeriod: pollPeriod = self._minTriggerInterval self._logger.debug( '[%s] Preparing poll timer with period %s second(s)' % (self.__class__.__name__, pollPeriod)) t = threading.Timer(float(pollPeriod), self.__poll, args=[rule]) t.daemon = True self.__runPollTimer(ruleId, t) elif monitorType == 'receive': self._logger.debug('[%s] [%s] is receive rule' % (self.__class__.__name__, ruleId)) self._receiveRuleDict[ruleId] = rule else: # assume this is 'event' rule self._logger.debug('[%s] [%s] is event rule' % (self.__class__.__name__, ruleId)) self._eventRuleDict[ruleId] = rule if ruleId in self._disabledRuleDict: del self._disabledRuleDict[ruleId] def enableRule(self, applicationName, ruleName): """ Raises: RuleAlreadyEnabled """ self._lock.acquire() try: ruleId = self.__getRuleId(applicationName, ruleName) self.__checkRuleExists(ruleId) if ruleId not in self._disabledRuleDict: raise RuleAlreadyEnabled('Rule [%s] is already enabled.' % (ruleId)) rule = self._ruleDict[ruleId] self.__enableRule(rule) rule.encode() self.__writeRuleFile(rule) rule.decode() finally: self._lock.release() def deleteRule(self, applicationName, ruleName): self._lock.acquire() try: return self.__deleteRule(applicationName, ruleName) finally: self._lock.release() def __deleteRule(self, applicationName, ruleName): ruleId = self.__getRuleId(applicationName, ruleName) self._logger.debug('[%s] Deleting rule %s' % (self.__class__.__name__, ruleId)) self.__checkRuleExists(ruleId) rule = self._ruleDict[ruleId] if rule.isStatusEnabled(): self.__disableRule(rule) del self._disabledRuleDict[ruleId] del self._ruleDict[ruleId] osUtility.removeFile(self.__getRuleFileName(applicationName, ruleName)) # Put rule in the 'disabled' state. def disableRule(self, applicationName, ruleName): """ Raises: RuleAlreadyDisabled """ self._lock.acquire() try: ruleId = self.__getRuleId(applicationName, ruleName) self.__checkRuleExists(ruleId) if ruleId in self._disabledRuleDict: raise RuleAlreadyDisabled('Rule [%s] is already disabled.' % ruleId) rule = self._ruleDict[ruleId] self.__disableRule(rule) rule.encode() self.__writeRuleFile(rule) rule.decode() finally: self._lock.release() def __disableRule(self, rule, status='disabled by administrator'): ruleId = self.__getRuleId(rule.getApplicationName(), rule.getName()) self._logger.debug('[%s] Disabling rule [%s]' % (self.__class__.__name__, ruleId)) appMonitor = rule.getApplicationMonitor() monitorType = appMonitor.getType() rule.setStatus(status) if monitorType == 'poll': self.__cancelPollTimer(ruleId) elif monitorType == 'receive': del self._receiveRuleDict[ruleId] else: del self._eventRuleDict[ruleId] self._disabledRuleDict[ruleId] = rule def getRule(self, applicationName, ruleName): self._lock.acquire() try: return self.__getRule(applicationName, ruleName) finally: self._lock.release() def __getRule(self, applicationName, ruleName): ruleId = self.__getRuleId(applicationName, ruleName) self.__checkRuleExists(ruleId) return copy.deepcopy(self._ruleDict[ruleId]) def getRuleList(self): self._lock.acquire() try: return self.__getRuleList() finally: self._lock.release() def __getRuleList(self): ruleList = TortugaObjectList() for ruleId in self._ruleDict.keys(): ruleList.append(copy.deepcopy(self._ruleDict[ruleId])) return ruleList def receiveApplicationData(self, applicationName, applicationData): self._lock.acquire() try: return self.__receiveApplicationData(applicationName, applicationData) finally: self._lock.release() def __receiveApplicationData(self, applicationName, applicationData): self._logger.debug('[%s] Received data for [%s]' % (self.__class__.__name__, applicationName)) self._receiveQ.put((applicationName, applicationData)) self.__runProcessingTimer() def executeRule(self, applicationName, ruleName, applicationData): self._lock.acquire() try: return self.__executeRule(applicationName, ruleName, applicationData) finally: self._lock.release() def __executeRule(self, applicationName, ruleName, applicationData): """ Raises: RuleDisabled """ ruleId = self.__getRuleId(applicationName, ruleName) self._logger.debug('[%s] Received request to execute rule [%s]' % (self.__class__.__name__, ruleId)) self.__checkRuleExists(ruleId) if ruleId in self._disabledRuleDict: raise RuleDisabled('Rule [%s] is disabled.' % (ruleId)) rule = self._ruleDict[ruleId] appMonitor = rule.getApplicationMonitor() monitorType = appMonitor.getType() if monitorType == 'poll': self._logger.debug('[%s] [%s] is poll rule' % (self.__class__.__name__, ruleId)) self.__cancelPollTimer(ruleId) self.__poll(rule) elif monitorType == 'receive': self._logger.debug('[%s] [%s] is receive rule' % (self.__class__.__name__, ruleId)) self._receiveQ.put((applicationName, applicationData)) self.__runProcessingTimer() else: # assume this is 'event' rule self._logger.debug('[%s] [%s] is event rule' % (self.__class__.__name__, ruleId)) self.__execute(rule) def __execute(self, rule): ruleId = self.__getRuleId(rule.getApplicationName(), rule.getName()) self._logger.debug('[%s] Begin execution for [%s]' % (self.__class__.__name__, ruleId)) rule.ruleInvoked() appMonitor = rule.getApplicationMonitor() queryCmd = appMonitor.getQueryCommand() self._logger.debug('[%s] Query command: [%s]' % (self.__class__.__name__, queryCmd)) actionCmd = appMonitor.getActionCommand() self._logger.debug('[%s] Action command: [%s]' % (self.__class__.__name__, actionCmd)) xPathReplacementDict = {} try: invokeAction = True queryStdOut = None if queryCmd: self._logger.debug('[%s] About to invoke: [%s]' % (self.__class__.__name__, queryCmd)) try: p = tortugaSubprocess.executeCommand( 'source %s/tortuga.sh && ' % (self._cm.getEtcDir()) + queryCmd) queryStdOut = p.getStdOut() appMonitor.queryInvocationSucceeded() except Exception as ex: appMonitor.queryInvocationFailed() raise monitorXmlDoc = self.__parseMonitorData(queryStdOut) xPathReplacementDict = self.__evaluateXPathVariables( monitorXmlDoc, rule.getXPathVariableList()) invokeAction = self.__evaluateConditions( rule, monitorXmlDoc, xPathReplacementDict) if invokeAction: try: actionCmd = self.__replaceXPathVariables( actionCmd, xPathReplacementDict) self._logger.debug('[%s] About to invoke: [%s]' % (self.__class__.__name__, actionCmd)) p = tortugaSubprocess.executeCommand( 'source %s/tortuga.sh && ' % (self._cm.getEtcDir()) + actionCmd) appMonitor.actionInvocationSucceeded() self._logger.debug('[%s] Done with command: [%s]' % (self.__class__.__name__, actionCmd)) except Exception as ex: appMonitor.actionInvocationFailed() raise else: self._logger.debug('[%s] Will skip action: [%s]' % (self.__class__.__name__, actionCmd)) except TortugaException as ex: self._logger.error('[%s] %s' % (self.__class__.__name__, ex)) if self.hasRule(ruleId): # Check if we need to stop invoking this rule. maxActionInvocations = appMonitor.getMaxActionInvocations() successfulActionInvocations = \ appMonitor.getSuccessfulActionInvocations() if maxActionInvocations: if int(maxActionInvocations) <= successfulActionInvocations: # Rule must be disabled. self._logger.debug( '[%s] Max. number of successful invocations (%s)' ' reached for rule [%s]' % (self.__class__.__name__, maxActionInvocations, ruleId)) self.disableRule(rule.getApplicationName(), rule.getName())
class TortugaScriptConfig(Config): AUTH_METHOD_PASSWORD = '******' AUTH_METHOD_TOKEN = 'token' NAVOPS_CLI = '/opt/navops-launch/bin/navopsctl' DEFAULT_FILENAME = os.path.join(os.path.expanduser('~'), '.tortuga', 'config') def __init__(self, **kwargs): # # Internal properties # self._filename = None self._cm = ConfigManager() # # Determine the username/password to use as default # default_username = self._cm.getCfmUser() default_password = self._cm.getCfmPassword() if default_password == 'not-set': default_username = None default_password = None # # Check for default navops cli location # default_navops_cli = self.NAVOPS_CLI if not os.path.exists(default_navops_cli): default_navops_cli = None # # Configuration settings # self.url = kwargs.get('url', self._cm.getInstallerUrl()) self.token = kwargs.get('token', None) self.navops_cli = kwargs.get('navops_cli', default_navops_cli) self.username = kwargs.get('username', default_username) self.password = kwargs.get('password', default_password) self.verify = kwargs.get('verify', True) def _load_from_environment(self): if os.getenv('TORTUGA_WS_URL'): self.url = os.getenv('TORTUGA_WS_URL') if os.getenv('TORTUGA_WS_USERNAME'): self.username = os.getenv('TORTUGA_WS_USERNAME') if os.getenv('TORTUGA_WS_PASSWORD'): self.password = os.getenv('TORTUGA_WS_PASSWORD') if os.getenv('TORTUGA_WS_TOKEN'): self.token = os.getenv('TORTUGA_WS_TOKEN') if os.getenv('TORTUGA_WS_NO_VERIFY'): self.verify = False @classmethod def load(cls, filename: str = None) -> 'TortugaScriptConfig': # # If a file name is provided, then we try to load that first # if filename: config = cls._load_from_file(filename) # # If no filename is provided, then we have to figure out where to # get a configuration # else: # # First, check if the user has a config in their home directory # if os.path.exists(cls.DEFAULT_FILENAME): config = cls._load_from_file(cls.DEFAULT_FILENAME) # # Otherwise, create a new config from scratch # else: config = cls() # # Override the config with any settings provided from the # environment # config._load_from_environment() return config @classmethod def _load_from_file(cls, filename) -> 'TortugaScriptConfig': if not os.path.exists(filename): raise ConfigFileNotFoundException( 'Config file not found: {}'.format(filename)) with open(filename) as fp: try: config_data = json.load(fp) except json.JSONDecodeError: raise ConfigException( 'Invalid config file: {}'.format(filename)) try: unmarshalled = TortugaScriptConfigSchema().load(config_data) except ValidationError: raise ConfigException('Invalid config file: {}'.format(filename)) return TortugaScriptConfig(**unmarshalled.data) def save(self, filename: str = None): if not filename: if self._filename: filename = filename else: filename = TortugaScriptConfig.DEFAULT_FILENAME if not os.path.exists(filename): os.makedirs(os.path.dirname(filename), exist_ok=True, mode=0o700) marshalled = TortugaScriptConfigSchema().dump(self) with open(filename, 'w') as fp: json.dump(marshalled.data, fp, indent=4) def get_auth_method(self) -> str: """ Gets the authentication method that should be used. :return str: token or password :raises ConfigException: if no auth method is configured """ # # For the CFM user, always use password authentication # if self.username == self._cm.getCfmUser() and self.password: return self.AUTH_METHOD_PASSWORD # # For all other cases, if the navops CLI is present, or there # is a token, use token-based authentication # if self.navops_cli or self.token: return self.AUTH_METHOD_TOKEN # # Otherwise, fall back to password authentication # if self.username and self.password: return self.AUTH_METHOD_PASSWORD raise ConfigException('Authentication required. Use "tortuga login".') def get_token(self) -> str: """ Gets the current authentication token. :return str: the token :raises ConfigException: if token is unavailable """ if self.navops_cli: try: return self._get_navops_token() except ConfigException: pass if self.token: return self.token raise ConfigException('Authentication required. Use "tortuga login".') def _get_navops_token(self) -> str: cmd = '{} token'.format(self.navops_cli) try: p = executeCommand(cmd) except Exception as ex: logger.info(str(ex)) raise ConfigException(str(ex)) if p.getExitStatus() != 0: raise ConfigException(p.getStdErr()) return p.getStdOut().decode()
def createHardwareProfile(self, session: Session, hwProfileSpec: HardwareProfile, settingsDict: Optional[Union[dict, None]] = None): bUseDefaults = settingsDict['defaults'] \ if settingsDict and 'defaults' in settingsDict else False osInfo = settingsDict['osInfo'] \ if settingsDict and \ settingsDict and 'osInfo' in settingsDict else None validation.validateProfileName(hwProfileSpec.getName()) if hwProfileSpec.getDescription() is None: hwProfileSpec.setDescription('%s Nodes' % (hwProfileSpec.getName())) installerNode = self._nodeDbApi.getNode(session, ConfigManager().getInstaller(), {'softwareprofile': True}) if bUseDefaults: if not hwProfileSpec.getNetworks(): # No <network>...</network> entries found in the template, # use the default provisioning interface from the primary # installer. # Find first provisioning network and use it for nic in installerNode.getNics(): network = nic.getNetwork() if network.getType() == 'provision': # for now set the default interface to be index 0 # with the same device networkDevice = fixNetworkDeviceName( nic.getNetworkDevice().getName()) network.setNetworkDevice( NetworkDevice(name=networkDevice)) hwProfileSpec.getNetworks().append(network) break else: raise NetworkNotFound( 'Unable to find provisioning network') else: # Ensure network device is defined installerNic = None for network in hwProfileSpec.getNetworks(): for installerNic in installerNode.getNics(): installerNetwork = installerNic.getNetwork() if network.getId() and \ network.getId() == installerNetwork.getId(): break elif network.getAddress() and \ network.getAddress() == \ installerNetwork.getAddress() and \ network.getNetmask() and \ network.getNetmask() == \ installerNetwork.getNetmask(): break else: # Unable to find network matching specification in # template. raise NetworkNotFound( 'Unable to find provisioning network [%s]' % (network)) networkDevice = fixNetworkDeviceName( installerNic.getNetworkDevice().getName()) network.setNetworkDevice(NetworkDevice(name=networkDevice)) if not osInfo: osInfo = installerNode.getSoftwareProfile().getOsInfo() osObjFactory = osUtility.getOsObjectFactory(osInfo.getName()) if not hwProfileSpec.getKernel(): hwProfileSpec.setKernel( osObjFactory.getOsSysManager().getKernel(osInfo)) if not hwProfileSpec.getInitrd(): hwProfileSpec.setInitrd( osObjFactory.getOsSysManager().getInitrd(osInfo)) self._hpDbApi.addHardwareProfile(session, hwProfileSpec) # Iterate over all networks in the newly defined hardware profile # and build assocations to provisioning NICs if bUseDefaults: for network in \ [network for network in hwProfileSpec.getNetworks() if network.getType() == 'provision']: # Get provisioning nic for network try: provisioningNic = self.getProvisioningNicForNetwork( session, network.getAddress(), network.getNetmask()) except NicNotFound: # There is currently no provisioning NIC defined for # this network. This is not a fatal error. continue self.setProvisioningNic(session, hwProfileSpec.getName(), provisioningNic.getId()) # # Fire the tags changed event for all creates that have tags # # Get the latest version from the db in case the create method # added some embellishments # hwp = self.getHardwareProfile(session, hwProfileSpec.getName()) if hwp.getTags(): HardwareProfileTagsChanged.fire(hardwareprofile_id=str( hwp.getId()), hardwareprofile_name=hwp.getName(), tags=hwp.getTags(), previous_tags={})
class TortugaWsApi: """ Base tortuga ws api class. """ def __init__(self, username=None, password=None): self._logger = logging.getLogger('tortuga.wsapi.{0}'.format( self.__class__.__name__)) self._logger.addHandler(logging.NullHandler()) self._cm = ConfigManager() if username is None and password is None: self._logger.debug('[%s] Using built-in user credentials' % (self.__module__)) username = self._cm.getCfmUser() password = self._cm.getCfmPassword() self._username = username self._password = password self._sm = None def _getWsUrl(self, url): """Extract scheme and net location from provided url. Use defaults if none exist.""" result = urlparse(url) scheme = result.scheme if result.scheme else \ self._cm.getAdminScheme() netloc = result.netloc if result.netloc else \ '{0}:{1}'.format(self._cm.getInstaller(), self._cm.getAdminPort()) return '{0}://{1}'.format(scheme, netloc) def _getSessionManager(self): if not self._sm: self._sm = sessionManager.createSession() return self._sm def getLogger(self): """ Get logger for this class. """ return self._logger def getConfigManager(self): """ Return configmanager reference """ return self._cm def sendSessionRequest(self, url, method='GET', contentType='application/json', data='', acceptType='application/json'): """ Send authorized session request Raises: UserNotAuthorized """ sm = self._getSessionManager() if not sm.hasSession(): if self._username is None: raise UserNotAuthorized('Username not supplied') if self._password is None: raise UserNotAuthorized('Password not supplied') wsUrl = self._getWsUrl(url) # establishSession() sets the 'wsUrl' so the explicit call # to setHost() is not required sm.establishSession(wsUrl, self._username, self._password) return sm.sendRequest(url, method, contentType, data, acceptType=acceptType) def sendRequest(self, url, method='GET', contentType='application/json', data='', acceptType='application/json'): """ Send unauthorized request. """ sm = self._getSessionManager() # Because there's no call to establishSession(), explicitly call # setHost() sm.setHost(self._getWsUrl(url)) return self._getSessionManager().sendRequest(url, method, contentType, data, acceptType)
def __init__(self, osFamilyInfo): self._osFamilyInfo = osFamilyInfo self._logger = logging.getLogger('tortuga.os') self._logger.addHandler(logging.NullHandler()) self._cm = ConfigManager()
class OsBootHostManagerCommon(OsObjectManager): """Methods for manipulating PXE files""" def __init__(self): OsObjectManager.__init__(self) # Cache this for later try: self.passdata = pwd.getpwnam('apache') except KeyError: self.passdata = pwd.getpwnam(os.getenv('USER')) self._cm = ConfigManager() def deletePuppetNodeCert(self, nodeName): # Remove the Puppet certificate when the node is reinstalled self.getLogger().debug('deletePuppetNodeCert(node=[%s])' % (nodeName)) puppetSslDir = '/etc/puppetlabs/puppet/ssl' puppetReportDir = '/var/lib/puppet/reports' puppetYamlDir = '/var/lib/puppet/yaml' filesToRemove = [ os.path.join(puppetSslDir, 'public_keys/%s.pem' % (nodeName)), os.path.join(puppetSslDir, 'ca/signed/%s.pem' % (nodeName)), os.path.join(puppetSslDir, 'private_keys/%s.pem' % (nodeName)), os.path.join(puppetSslDir, 'certs/%s.pem' % (nodeName)), os.path.join(puppetYamlDir, 'node/%s.yaml' % (nodeName)), os.path.join(puppetYamlDir, 'facts/%s.yaml' % (nodeName)), ] for fn in filesToRemove: try: os.unlink(fn) except OSError as exc: if exc.errno != 2: self.getLogger().error( 'Error attempting to remove %s (reason: %s)' % (fn, exc)) fn = os.path.join(puppetReportDir, nodeName) try: shutil.rmtree(fn) except OSError as exc: if exc.errno != 2: self.getLogger().error( 'Error attempting to remove %s (reason: %s)' % (fn, exc)) def nodeCleanup(self, nodeName): """ Remove files related to the node """ # Remove 'private' directory private_dir = os.path.join(self._cm.getRoot(), 'private', nodeName) if os.path.exists(private_dir): shutil.rmtree(private_dir) def addDhcpLease(self, node, nic): # Add DHCP lease to DHCP server pass def removeDhcpLease(self, nodeName): # Remove the DHCP lease from the DHCP server. This will be # a no-op on any platform that doesn't support the operation # (ie. any platform not running ISC DHCPD) pass def setNodeForNetworkBoot(self, dbNode): # Update node status to "Expired" and boot from network dbNode.state = 'Expired' dbNode.bootFrom = 0 self.deletePuppetNodeCert(dbNode.name) # Write the updated file self.writePXEFile(dbNode)
def main(): cm = ConfigManager() p = argparse.ArgumentParser() p.add_argument('-f', '--force', dest='force', action='store_true', default=False) p.add_argument('name', help='Software profile name') p.add_argument('kit', help='Kit descriptor (NAME-VERSION-ITERATION)') p.add_argument('component', help='Component descriptor (NAME-VERSION)') args = p.parse_args() kitNameAndVersion, kitIteration = args.kit.rsplit('-', 1) kitName, kitVersion = kitNameAndVersion.split('-', 1) compName, _ = args.component.split('-', 2) flagFile = os.path.join( cm.getRoot(), 'var/run/actions/%s/component_%s_%s_post_install' % (args.name, args.kit, args.component)) if os.path.exists(flagFile): if not args.force: sys.stderr.write( 'post-install component action for [%s] already run\n' % (compName)) sys.exit(0) # Remove the existing flag file, we're forcing a run os.unlink(flagFile) load_kits() kit_spec = (kitName, kitVersion, kitIteration) try: with DbManager().session() as session: kit_installer = get_kit_installer(kit_spec)() kit_installer.session = session c = kit_installer.get_component_installer(compName) if c is None: raise ComponentNotFound( 'Component [%s] not found in kit [%s]' % (compName, kitName)) c.run_action('post_install') logger.debug( 'post_install component action run for [%s] from kit [%s]' % (args.component, args.kit)) # Ensure destination directory exists if not os.path.exists(os.path.dirname(flagFile)): os.makedirs(os.path.dirname(flagFile)) # touch flagFile open(flagFile, 'w').close() except Exception as exc: # noqa pylint: disable=broad-except print('Error: {}'.format(exc), file=sys.stderr) sys.exit(0)
def __init__(self, osFamilyInfo: OsFamilyInfo) -> None: self._osFamilyInfo = osFamilyInfo self._logger = logging.getLogger(OS_NAMESPACE) self._cm = ConfigManager()
class RepoManager: """ Class for repository management. """ def __init__(self): """ Initialize repository manager instance. """ self._logger = logging.getLogger(REPO_NAMESPACE) self._kitArchiveDir = None self._cm = ConfigManager() self._repoRoot = self._cm.getReposDir() self._repoMap = {} self.__configure() def __configureRepo(self, osInfo, localPath, remoteUrl=None): \ # pylint: disable=unused-argument """ Configure repo for a given OS""" repo = None osName = osInfo.getName() factoryModule = '%sObjectFactory' % osName factoryClass = '%sObjectFactory' % osName.capitalize() _temp = __import__('tortuga.os_utility.{0}'.format(factoryModule), globals(), locals(), [factoryClass], 0) Factory = getattr(_temp, factoryClass) repo = Factory().getRepo(osInfo, localPath, remoteUrl) repo.createRoot() self._logger.debug('Configured repo [%s] for [%s]' % (repo, osInfo)) return repo def __configure(self): """ Configure all repositories from the config file. """ self._kitArchiveDir = self._cm.getKitDir() configFile = self._cm.getRepoConfigFile() if not os.path.exists(configFile): self._logger.debug('Repo configuration file [%s] not found' % (configFile)) else: self._logger.debug('Reading repo configuration file [%s]' % (configFile)) configParser = configparser.ConfigParser() configParser.read(configFile) try: osKeyList = configParser.sections() if osKeyList: self._logger.debug('Found OS sections [%s]' % (' '.join(osKeyList))) for osKey in osKeyList: self._logger.debug('Parsing OS section [%s]' % (osKey)) osData = osKey.split('__') osName = osData[0] osVersion = osData[1] osArch = osData[2] osInfo = OsInfo(osName, osVersion, osArch) # Ignore all repos that weren't enabled bEnabled = True if configParser.has_option(osKey, 'enabled'): value = configParser.get(osKey, 'enabled') bEnabled = value.lower() == 'true' if not bEnabled: self._logger.debug('Repo for [%s] is disabled' % (osInfo)) continue localPath = None if configParser.has_option(osKey, 'localPath'): localPath = configParser.get(osKey, 'localPath', True) if not localPath: localPath = self._repoRoot remoteUrl = None if configParser.has_option(osKey, 'remoteUrl'): remoteUrl = configParser.get(osKey, 'remoteUrl', True) repo = self.__configureRepo(osInfo, localPath, remoteUrl) self._repoMap[osKey] = repo if not os.path.exists(self._kitArchiveDir): self._logger.debug('Creating kit archive directory [%s]' % (self._kitArchiveDir)) os.makedirs(self._kitArchiveDir) osInfo = osUtility.getNativeOsInfo() repo = self.__configureRepo(osInfo, self._repoRoot) osKey = '%s__%s__%s' % (osInfo.getName(), osInfo.getVersion(), osInfo.getArch()) # Configure repo for native os if there are no repos configured. if osKey not in self._repoMap: self._repoMap[osKey] = repo except ConfigurationError: raise except Exception as ex: raise ConfigurationError(exception=ex) def getRepo(self, osInfo=None): """ Return repo given os info. """ if not osInfo: osInfo = osUtility.getNativeOsInfo() osKey = '%s__%s__%s' % (osInfo.getName(), osInfo.getVersion(), osInfo.getArch()) return self._repoMap.get(osKey) def getRepoList(self): """ Return all repos. """ return list(self._repoMap.values()) def getKitArchiveDir(self): """ Return kit archive directory. """ return self._kitArchiveDir
def __init__(self): super(ActionManager, self).__init__() self._actionBase = "%s%s" % (ConfigManager().getRoot(), ConfigManager().getActionLog())
class NodeManager(TortugaObjectManager): \ # pylint: disable=too-many-public-methods def __init__(self): super(NodeManager, self).__init__() self._nodeDbApi = NodeDbApi() self._hardwareProfileDbApi = HardwareProfileDbApi() self._cm = ConfigManager() self._san = san.San() def __validateHostName(self, hostname: str, name_format: str) -> NoReturn: """ Raises: ConfigurationError """ bWildcardNameFormat = (name_format == '*') if hostname and not bWildcardNameFormat: # Host name specified, but hardware profile does not # allow setting the host name raise ConfigurationError( 'Hardware profile does not allow setting host names' ' of imported nodes') elif not hostname and bWildcardNameFormat: # Host name not specified but hardware profile expects it raise ConfigurationError( 'Hardware profile requires host names to be set') def createNewNode(self, session: Session, addNodeRequest: dict, dbHardwareProfile: HardwareProfiles, dbSoftwareProfile: Optional[SoftwareProfiles] = None, validateIp: bool = True, bGenerateIp: bool = True, dns_zone: Optional[str] = None) -> Nodes: """ Convert the addNodeRequest into a Nodes object Raises: NicNotFound """ self.getLogger().debug( 'createNewNode(): session=[%s], addNodeRequest=[%s],' ' dbHardwareProfile=[%s], dbSoftwareProfile=[%s],' ' validateIp=[%s], bGenerateIp=[%s]' % (id(session), addNodeRequest, dbHardwareProfile.name, dbSoftwareProfile.name if dbSoftwareProfile else '(none)', validateIp, bGenerateIp)) # This is where the Nodes() object is first created. node = Nodes() # Set the default node state node.state = 'Discovered' if 'rack' in addNodeRequest: node.rack = addNodeRequest['rack'] node.addHostSession = addNodeRequest['addHostSession'] hostname = addNodeRequest['name'] \ if 'name' in addNodeRequest else None # Ensure no conflicting options (ie. specifying host name for # hardware profile in which host names are generated) self.__validateHostName(hostname, dbHardwareProfile.nameFormat) node.name = hostname # Complete initialization of new node record nic_defs = addNodeRequest['nics'] \ if 'nics' in addNodeRequest else [] AddHostServerLocal().initializeNode(session, node, dbHardwareProfile, dbSoftwareProfile, nic_defs, bValidateIp=validateIp, bGenerateIp=bGenerateIp, dns_zone=dns_zone) # Set hardware profile of new node node.hardwareProfileId = dbHardwareProfile.id # Set software profile of new node; if the software profile is None, # attempt to set the software profile to the idle software profile # of the associated hardware profile. This may also be None, in # which case the software profile is undefined. node.softwareprofile = dbSoftwareProfile \ if dbSoftwareProfile else dbHardwareProfile.idlesoftwareprofile node.isIdle = dbSoftwareProfile.isIdle \ if dbSoftwareProfile else True # Return the new node return node def getNode(self, name, optionDict=None): """Get node by name""" optionDict_ = optionDict.copy() if optionDict else {} optionDict_.update({'hardwareprofile': True}) node = self._nodeDbApi.getNode(name, optionDict_) hwprofile = self._hardwareProfileDbApi.getHardwareProfile( node.getHardwareProfile().getName(), {'resourceadapter': True}) adapter_name = hwprofile.getResourceAdapter().getName() \ if hwprofile.getResourceAdapter() else 'default' # Query vcpus from resource adapter ResourceAdapterClass = resourceAdapterFactory.getResourceAdapterClass( adapter_name) # Update Node object node.setVcpus(ResourceAdapterClass().get_node_vcpus(node.getName())) return node def getNodeById(self, nodeId, optionDict=None): """ Get node by node id Raises: NodeNotFound """ return self._nodeDbApi.getNodeById(int(nodeId), optionDict) def getNodeByIp(self, ip): """ Get node by IP address Raises: NodeNotFound """ return self._nodeDbApi.getNodeByIp(ip) def getNodeList(self, tags=None): """Return all nodes""" return self._nodeDbApi.getNodeList(tags=tags) def updateNode(self, nodeName, updateNodeRequest): self.getLogger().debug('updateNode(): name=[{0}]'.format(nodeName)) session = DbManager().openSession() try: node = NodesDbHandler().getNode(session, nodeName) if 'nics' in updateNodeRequest: nic = updateNodeRequest['nics'][0] if 'ip' in nic: node.nics[0].ip = nic['ip'] node.nics[0].boot = True # Call resource adapter NodesDbHandler().updateNode(session, node, updateNodeRequest) run_post_install = False if 'state' in updateNodeRequest: run_post_install = node.state == 'Allocated' and \ updateNodeRequest['state'] == 'Provisioned' node.state = updateNodeRequest['state'] session.commit() if run_post_install: self.getLogger().debug( 'updateNode(): run-post-install for node [{0}]'.format( node.name)) self.__scheduleUpdate() except Exception: session.rollback() self.getLogger().exception( 'Exception updating node [{0}]'.format(nodeName)) finally: DbManager().closeSession() def updateNodeStatus(self, nodeName, state=None, bootFrom=None): """Update node status If neither 'state' nor 'bootFrom' are not None, this operation will update only the 'lastUpdated' timestamp. Returns: bool indicating whether state and/or bootFrom differed from current value """ value = 'None' if bootFrom is None else \ '1 (disk)' if int(bootFrom) == 1 else '0 (network)' self.getLogger().debug( 'updateNodeStatus(): node=[%s], state=[%s], bootFrom=[%s]' % (nodeName, state, value)) session = DbManager().openSession() try: node = NodesDbHandler().getNode(session, nodeName) result = self._updateNodeStatus(node, state=state, bootFrom=bootFrom) session.commit() return result finally: DbManager().closeSession() def _updateNodeStatus(self, dbNode, state=None, bootFrom=None): """ Internal method which takes a 'Nodes' object instead of a node name. """ result = NodesDbHandler().updateNodeStatus(dbNode, state, bootFrom) # Only change local boot configuration if the hardware profile is # not marked as 'remote' and we're not acting on the installer node. if dbNode.softwareprofile and \ dbNode.softwareprofile.type != 'installer' and \ dbNode.hardwareprofile.location not in \ ('remote', 'remote-vpn'): osUtility.getOsObjectFactory().getOsBootHostManager().\ writePXEFile(dbNode, localboot=bootFrom) return result def __process_nodeErrorDict(self, nodeErrorDict): result = {} nodes_deleted = [] for key, nodeList in nodeErrorDict.items(): result[key] = [dbNode.name for dbNode in nodeList] if key == 'NodesDeleted': for node in nodeList: node_deleted = { 'name': node.name, 'hardwareprofile': node.hardwareprofile.name, 'addHostSession': node.addHostSession, } if node.softwareprofile: node_deleted['softwareprofile'] = \ node.softwareprofile.name nodes_deleted.append(node_deleted) return result, nodes_deleted def deleteNode(self, nodespec): """ Delete node by nodespec Raises: NodeNotFound """ installer_hostname = socket.getfqdn().split('.', 1)[0] session = DbManager().openSession() try: nodes = [] for node in self.__expand_nodespec(session, nodespec): if node.name.split('.', 1)[0] == installer_hostname: self.getLogger().info( 'Ignoring request to delete installer node' ' ([{0}])'.format(node.name)) continue nodes.append(node) if not nodes: raise NodeNotFound('No nodes matching nodespec [%s]' % (nodespec)) self.__preDeleteHost(nodes) nodeErrorDict = NodesDbHandler().deleteNode(session, nodes) # REALLY!?!? Convert a list of Nodes objects into a list of # node names so we can report the list back to the end-user. # This needs to be FIXED! result, nodes_deleted = self.__process_nodeErrorDict(nodeErrorDict) session.commit() # ============================================================ # Perform actions *after* node deletion(s) have been committed # to database. # ============================================================ self.__postDeleteHost(nodes_deleted) addHostSessions = set( [tmpnode['addHostSession'] for tmpnode in nodes_deleted]) if addHostSessions: AddHostManager().delete_sessions(addHostSessions) bhm = osUtility.getOsObjectFactory().getOsBootHostManager() for nodeName in result['NodesDeleted']: # Remove the Puppet cert bhm.deletePuppetNodeCert(nodeName) bhm.nodeCleanup(nodeName) self.getLogger().info('Node [%s] deleted' % (nodeName)) # Schedule a cluster update self.__scheduleUpdate() return result except TortugaException: session.rollback() raise except Exception: session.rollback() self.getLogger().exception('Exception in NodeManager.deleteNode()') raise finally: DbManager().closeSession() def __process_delete_node_result(self, nodeErrorDict): # REALLY!?!? Convert a list of Nodes objects into a list of # node names so we can report the list back to the end-user. # This needs to be FIXED! result = {} nodes_deleted = [] for key, nodeList in nodeErrorDict.items(): result[key] = [dbNode.name for dbNode in nodeList] if key == 'NodesDeleted': for node in nodeList: node_deleted = { 'name': node.name, 'hardwareprofile': node.hardwareprofile.name, } if node.softwareprofile: node_deleted['softwareprofile'] = \ node.softwareprofile.name nodes_deleted.append(node_deleted) return result, nodes_deleted def __preDeleteHost(self, nodes): self.getLogger().debug('__preDeleteHost(): nodes=[%s]' % (' '.join([node.name for node in nodes]))) if not nodes: self.getLogger().debug('No nodes deleted in this operation') return kitmgr = KitActionsManager() for node in nodes: kitmgr.pre_delete_host( node.hardwareprofile.name, node.softwareprofile.name if node.softwareprofile else None, nodes=[node.name]) def __postDeleteHost(self, nodes_deleted): # 'nodes_deleted' is a list of dicts of the following format: # # { # 'name': 'compute-01', # 'softwareprofile': 'Compute', # 'hardwareprofile': 'LocalIron', # } # # if the node does not have an associated software profile, the # dict does not contain the key 'softwareprofile'. self.getLogger().debug('__postDeleteHost(): nodes_deleted=[%s]' % (nodes_deleted)) if not nodes_deleted: self.getLogger().debug('No nodes deleted in this operation') return kitmgr = KitActionsManager() for node_dict in nodes_deleted: kitmgr.post_delete_host(node_dict['hardwareprofile'], node_dict['softwareprofile'] if 'softwareprofile' in node_dict else None, nodes=[node_dict['name']]) def __scheduleUpdate(self): tortugaSubprocess.executeCommand( os.path.join(self._cm.getRoot(), 'bin/schedule-update')) def getInstallerNode(self, optionDict=None): return self._nodeDbApi.getNode(self._cm.getInstaller(), optionDict=optionDict) def getProvisioningInfo(self, nodeName): return self._nodeDbApi.getProvisioningInfo(nodeName) def getKickstartFile(self, node, hardwareprofile, softwareprofile): """ Generate kickstart file for specified node Raises: OsNotSupported """ osFamilyName = softwareprofile.os.family.name try: osSupportModule = __import__('tortuga.os.%s.osSupport' % (osFamilyName), fromlist=['OSSupport']) except ImportError: raise OsNotSupported('Operating system family [%s] not supported' % (osFamilyName)) OSSupport = osSupportModule.OSSupport tmpOsFamilyInfo = OsFamilyInfo(softwareprofile.os.family.name, softwareprofile.os.family.version, softwareprofile.os.family.arch) return OSSupport(tmpOsFamilyInfo).getKickstartFileContents( node, hardwareprofile, softwareprofile) def __transferNodeCommon(self, session, dbDstSoftwareProfile, results): \ # pylint: disable=no-self-use # Aggregate list of transferred nodes based on hardware profile # to call resource adapter minimal number of times. hwProfileMap = {} for transferResultDict in results: dbNode = transferResultDict['node'] dbHardwareProfile = dbNode.hardwareprofile if dbHardwareProfile not in hwProfileMap: hwProfileMap[dbHardwareProfile] = [transferResultDict] else: hwProfileMap[dbHardwareProfile].append(transferResultDict) session.commit() nodeTransferDict = {} # Kill two birds with one stone... do the resource adapter # action as well as populate the nodeTransferDict. This saves # having to iterate twice on the same result data. for dbHardwareProfile, nodesDict in hwProfileMap.items(): adapter = resourceAdapterFactory.getApi( dbHardwareProfile.resourceadapter.name) dbNodeTuples = [] for nodeDict in nodesDict: dbNode = nodeDict['node'] dbSrcSoftwareProfile = nodeDict['prev_softwareprofile'] if dbSrcSoftwareProfile.name not in nodeTransferDict: nodeTransferDict[dbSrcSoftwareProfile.name] = { 'added': [], 'removed': [dbNode], } else: nodeTransferDict[dbSrcSoftwareProfile.name]['removed'].\ append(dbNode) if dbDstSoftwareProfile.name not in nodeTransferDict: nodeTransferDict[dbDstSoftwareProfile.name] = { 'added': [dbNode], 'removed': [], } else: nodeTransferDict[dbDstSoftwareProfile.name]['added'].\ append(dbNode) # The destination software profile is available through # node relationship. dbNodeTuples.append((dbNode, dbSrcSoftwareProfile)) adapter.transferNode(dbNodeTuples, dbDstSoftwareProfile) session.commit() # Now call the 'refresh' action to all participatory components KitActionsManager().refresh(nodeTransferDict) return results def transferNode(self, nodespec, dstSoftwareProfileName, bForce=False): """ Transfer nodes defined by 'nodespec' to 'dstSoftwareProfile' Raises: NodeNotFound SoftwareProfileNotFound NodeTransferNotValid """ session = DbManager().openSession() try: nodes = self.__expand_nodespec(session, nodespec) if not nodes: raise NodeNotFound('No nodes matching nodespec [%s]' % (nodespec)) dbDstSoftwareProfile = SoftwareProfilesDbHandler().\ getSoftwareProfile(session, dstSoftwareProfileName) results = NodesDbHandler().transferNode(session, nodes, dbDstSoftwareProfile, bForce=bForce) return self.__transferNodeCommon(session, dbDstSoftwareProfile, results) finally: DbManager().closeSession() def transferNodes(self, srcSoftwareProfileName, dstSoftwareProfileName, count, bForce=False): """ Transfer 'count' nodes from 'srcSoftwareProfile' to 'dstSoftwareProfile' Raises: SoftwareProfileNotFound """ session = DbManager().openSession() try: # It is not necessary to specify a source software profile. If # not specified, pick any eligible nodes in the hardware profile # mapped to the destination software profile. Don't ask me who # uses this capability, but it's here if you need it... dbSrcSoftwareProfile = SoftwareProfilesDbHandler().\ getSoftwareProfile( session, srcSoftwareProfileName) \ if srcSoftwareProfileName else None dbDstSoftwareProfile = SoftwareProfilesDbHandler().\ getSoftwareProfile(session, dstSoftwareProfileName) results = NodesDbHandler().transferNodes(session, dbSrcSoftwareProfile, dbDstSoftwareProfile, int(float(count)), bForce=bForce) return self.__transferNodeCommon(session, dbDstSoftwareProfile, results) finally: DbManager().closeSession() def idleNode(self, nodespec): """ Raises: NodeNotFound """ session = DbManager().openSession() try: nodes = self.__expand_nodespec(session, nodespec) if not nodes: raise NodeNotFound('No nodes matching nodespec [%s]' % (nodespec)) result = NodesDbHandler().idleNode(session, nodes) # Convert list of Nodes to list of node names for providing # user feedback. result_dict = {} for key, dbNodes in result.items(): result_dict[key] = [dbNode.name for dbNode in dbNodes] session.commit() # Remove Puppet certificate(s) for idled node(s) for node_name in result_dict['success']: # Remove Puppet certificate for idled node bhm = osUtility.getOsObjectFactory().getOsBootHostManager() bhm.deletePuppetNodeCert(node_name) # Schedule a cluster update self.__scheduleUpdate() return result_dict except TortugaException as ex: session.rollback() raise except Exception as ex: session.rollback() self.getLogger().exception('[%s] %s' % (self.__class__.__name__, ex)) raise finally: DbManager().closeSession() def __process_activateNode_results(self, tmp_results, dstswprofilename): results = {} for key, values in tmp_results.items(): # With the exception of the "ProfileMappingNotAllowed" dict # item, all items in the dict are lists of nodes. if key != 'ProfileMappingNotAllowed': results[key] = [dbNode.name for dbNode in values] else: results[key] = \ [(value[0].name, value[1], value[2]) for value in values] if tmp_results['success']: # Iterate over activated nodes, creating dict keyed on # 'addHostSession' addHostSessions = {} for node in tmp_results['success']: if node.addHostSession not in addHostSessions: addHostSessions[node.addHostSession] = [] addHostSessions[node.addHostSession] = \ node.hardwareprofile.name # For each 'addHostSession', call postAddHost() for addHostSession, hwprofile in addHostSessions.items(): AddHostManager().postAddHost(hwprofile, dstswprofilename, addHostSession) return results def activateNode(self, nodespec, softwareProfileName): """ Raises: SoftwareProfileNotFound NodeNotFound TortugaException """ session = DbManager().openSession() try: dbSoftwareProfile = SoftwareProfilesDbHandler().\ getSoftwareProfile(session, softwareProfileName) \ if softwareProfileName else None dbNodes = self.__expand_nodespec(session, nodespec) if not dbNodes: raise NodeNotFound('No nodes matching nodespec [%s]' % (nodespec)) tmp_results = NodesDbHandler().activateNode( session, dbNodes, dbSoftwareProfile) results = self.__process_activateNode_results( tmp_results, softwareProfileName) session.commit() # Schedule a cluster update self.__scheduleUpdate() return results except TortugaException as ex: session.rollback() raise except Exception as ex: session.rollback() self.getLogger().exception('%s' % ex) raise finally: DbManager().closeSession() def startupNode(self, nodespec, remainingNodeList=None, bootMethod='n'): """ Raises: NodeNotFound """ return self._nodeDbApi.startupNode(nodespec, remainingNodeList=remainingNodeList or [], bootMethod=bootMethod) def shutdownNode(self, nodespec, bSoftShutdown=False): """ Raises: NodeNotFound """ return self._nodeDbApi.shutdownNode(nodespec, bSoftShutdown) def build_node_filterspec(self, nodespec): filter_spec = [] for nodespec_token in nodespec.split(','): # Convert shell-style wildcards into SQL wildcards if '*' in nodespec_token or '?' in nodespec_token: filter_spec.append( nodespec_token.replace('*', '%').replace('?', '_')) continue if '.' not in nodespec_token: filter_spec.append(nodespec_token) filter_spec.append(nodespec_token + '.%') continue # Add nodespec "AS IS" filter_spec.append(nodespec_token) return filter_spec def __expand_nodespec(self, session, nodespec): \ # pylint: disable=no-self-use # Expand wildcards in nodespec. Each token in the nodespec can # be wildcard that expands into one or more nodes. return NodesDbHandler().getNodesByNameFilter( session, self.build_node_filterspec(nodespec)) def rebootNode(self, nodespec, bSoftReset=False, bReinstall=False): """ Raises: NodeNotFound """ session = DbManager().openSession() try: nodes = self.__expand_nodespec(session, nodespec) if not nodes: raise NodeNotFound('No nodes matching nodespec [%s]' % (nodespec)) bhm = osUtility.getOsObjectFactory().getOsBootHostManager() if bReinstall: for dbNode in nodes: bhm.setNodeForNetworkBoot(dbNode) results = NodesDbHandler().rebootNode(session, nodes, bSoftReset) session.commit() return results finally: DbManager().closeSession() def checkpointNode(self, nodeName): return self._nodeDbApi.checkpointNode(nodeName) def revertNodeToCheckpoint(self, nodeName): return self._nodeDbApi.revertNodeToCheckpoint(nodeName) def migrateNode(self, nodeName, remainingNodeList, liveMigrate): return self._nodeDbApi.migrateNode(nodeName, remainingNodeList, liveMigrate) def evacuateChildren(self, nodeName): self._nodeDbApi.evacuateChildren(nodeName) def getChildrenList(self, nodeName): return self._nodeDbApi.getChildrenList(nodeName) def setParentNode(self, nodeName, parentNodeName): self._nodeDbApi.setParentNode(nodeName, parentNodeName) def addStorageVolume(self, nodeName, volume, isDirect="DEFAULT"): """ Raises: VolumeDoesNotExist UnsupportedOperation """ node = self.getNode(nodeName, {'hardwareprofile': True}) # Only allow persistent volumes to be attached... vol = self._san.getVolume(volume) if vol is None: raise VolumeDoesNotExist('Volume [%s] does not exist' % (volume)) if not vol.getPersistent(): raise UnsupportedOperation( 'Only persistent volumes can be attached') api = resourceAdapterFactory.getApi( node.getHardwareProfile().getResourceAdapter().getName()) if isDirect == "DEFAULT": return api.addVolumeToNode(node, volume) return api.addVolumeToNode(node, volume, isDirect) def removeStorageVolume(self, nodeName, volume): """ Raises: VolumeDoesNotExist UnsupportedOperation """ node = self.getNode(nodeName, {'hardwareprofile': True}) api = resourceAdapterFactory.getApi( node.getHardwareProfile().getResourceAdapter().getName()) vol = self._san.getVolume(volume) if vol is None: raise VolumeDoesNotExist('The volume [%s] does not exist' % (volume)) if not vol.getPersistent(): raise UnsupportedOperation( 'Only persistent volumes can be detached') return api.removeVolumeFromNode(node, volume) def getStorageVolumes(self, nodeName): return self._san.getNodeVolumes(self.getNode(nodeName).getName()) def getNodesByNodeState(self, state): return self._nodeDbApi.getNodesByNodeState(state) def getNodesByNameFilter(self, _filter): return self._nodeDbApi.getNodesByNameFilter(_filter)
def __init__(self, osFamilyInfo: OsFamilyInfo) -> None: self._osFamilyInfo = osFamilyInfo self._logger = logging.getLogger('tortuga.os') self._cm = ConfigManager()
def __init__(self): self._logger = logging.\ getLogger('tortuga.%s' % (self.__class__.__name__)) self._logger.addHandler(logging.NullHandler()) self._cm = ConfigManager()
class TortugaCli(metaclass=ABCMeta): """ Base tortuga command line interface class. """ def __init__(self, validArgCount=0): self._logger = logging.getLogger(CLI_NAMESPACE) self._parser = argparse.ArgumentParser() self._args = [] self._validArgCount = validArgCount self._url = None self._username = None self._password = None self._verify = True self._optionGroupDict = {} self._cm = ConfigManager() self.__initializeLocale() def __initializeLocale(self): """Initialize the gettext domain """ langdomain = 'tortugaStrings' # Locate the Internationalization stuff localedir = '../share/locale' \ if os.path.exists('../share/locale') else \ os.path.join(self._cm.getRoot(), 'share/locale') gettext.install(langdomain, localedir) def getParser(self): """ Get parser for this class. """ return self._parser def addOption(self, *args, **kwargs): """ Add option. """ self._parser.add_argument(*args, **kwargs) def addOptionToGroup(self, groupName, *args, **kwargs): """ Add option for the given group name. Group should be created using addOptionGroup(). """ group = self._optionGroupDict.get(groupName) group.add_argument(*args, **kwargs) def addOptionGroup(self, groupName, desc): """ Add option group. """ group = self._parser.add_argument_group(groupName, desc) self._optionGroupDict[groupName] = group return group def parseArgs(self, usage=None): """ Parse args Raises: InvalidArgument """ common_group = _('Common Tortuga Options') self.addOptionGroup(common_group, None) self.addOptionToGroup(common_group, '-V', action='store_true', dest='cmdVersion', default=False, help=_('print version and exit')) self.addOptionToGroup(common_group, '-d', '--debug', dest='consoleLogLevel', default='warning', help=_('set debug level; valid values are: ' 'critical, error, warning, info, debug')) self.addOptionToGroup(common_group, '--url', help=_('Tortuga web service URL')) self.addOptionToGroup(common_group, '--username', dest='username', help=_('Tortuga web service user name')) self.addOptionToGroup(common_group, '--password', dest='password', help=_('Tortuga web service password')) self.addOptionToGroup(common_group, '--no-verify', dest='verify', action='store_false', default=True, help=_("Don't verify the API SSL certificate")) if usage: self._parser.description = usage try: self._args = self._parser.parse_args() except SystemExit as rc: sys.stdout.flush() sys.stderr.flush() sys.exit(int(str(rc))) if self._args.cmdVersion: print( _('{0} version: {1}'.format(os.path.basename(sys.argv[0]), self._cm.getTortugaRelease()))) sys.exit(0) self._setup_logging(self._args.consoleLogLevel) self._url, self._username, self._password, self._verify = \ self._get_web_service_options() return self._args def _setup_logging(self, log_level_name: str): """ Setup logging for the specified log level. :param str log_level_name: the name of the log level to use """ log_level_name = log_level_name.upper() if log_level_name not in [ 'CRITICAL', 'ERROR', 'WARNING', 'INFO', 'DEBUG' ]: print('Invalid debug level: {}'.format(log_level_name)) sys.exit(0) log_level = getattr(logging, log_level_name) logger = logging.getLogger(ROOT_NAMESPACE) logger.setLevel(log_level) ch = logging.StreamHandler() ch.setLevel(log_level) formatter = logging.Formatter( '%(asctime)s - %(name)s - %(levelname)s - %(message)s') ch.setFormatter(formatter) logger.addHandler(ch) def _get_web_service_options(self): """ Read Tortuga web service credentials from config file, environment, or command-line. Command-line overrides either config file or environment. :return: tuple of (url, username, password) """ username = password = url = None cfg_file = os.path.join(os.path.expanduser('~'), '.local', 'tortuga', 'credentials') if os.path.exists(cfg_file): cfg = configparser.ConfigParser() cfg.read(cfg_file) username = cfg.get('default', 'username') \ if cfg.has_section('default') and \ cfg.has_option('default', 'username') else None password = cfg.get('default', 'password') \ if cfg.has_section('default') and \ cfg.has_option('default', 'password') else None url = cfg.get('default', 'url') \ if cfg.has_section('default') and \ cfg.has_option('default', 'url') else None # TORTUGA_WS_URL if self._args.url: # Command-line "--server" argument overrides env var and # setting contained within '/etc/profile.nii' url = self._args.url elif os.getenv('TORTUGA_WS_URL'): url = os.getenv('TORTUGA_WS_URL') # TORTUGA_WS_USERNAME if self._args.username: username = self._args.username elif os.getenv('TORTUGA_WS_USERNAME'): username = os.getenv('TORTUGA_WS_USERNAME') # TORTUGA_WS_PASSWORD if self._args.password: password = self._args.password elif os.getenv('TORTUGA_WS_PASSWORD'): password = os.getenv('TORTUGA_WS_PASSWORD') # # CLI arguments should override the environment variable # if os.getenv('TORTUGA_WS_NO_VERIFY'): verify = False else: verify = self._args.verify return url, username, password, verify def usage(self, s=None): """ Print usage information """ if s: sys.stderr.write(_('Error: {0}').format(s) + '\n') self._parser.print_help() sys.exit(1) def getArgs(self): '''Returns the command line argument list''' return self._args def getUrl(self): return self._url def getUsername(self): """ Get user name. """ return self._username def getPassword(self): """ Get password. """ return self._password @abstractmethod def runCommand(self): \ # pylint: disable=no-self-use """ This method must be implemented by the derived class. """ def run(self): """ Invoke runCommand() in derivative class and handle exceptions. """ try: self.runCommand() except TortugaException as ex: print(ex.getErrorMessage()) raise SystemExit(ex.getErrorCode()) except SystemExit: raise except Exception as ex: print(str(ex)) raise SystemExit(-1) def _parseDiskSize(self, diskSizeParam): \ # pylint: disable=no-self-use """ Parses diskSizeParam, returns an int value representing number of megabytes Raises: ValueError """ if diskSizeParam.endswith('TB'): return int(float(diskSizeParam[:-2]) * 1000000) if diskSizeParam.endswith('GB'): return int(float(diskSizeParam[:-2]) * 1000) elif diskSizeParam.endswith('MB'): # Must be an integer return int(diskSizeParam[:-2]) return int(diskSizeParam) def _getDiskSizeDisplayStr(self, volSize): \ # pylint: disable=no-self-use if volSize < 1000: result = '%s MB' % (volSize) elif volSize < 1000000: result = '%.3f GB' % (float(volSize) / 1000) else: result = '%.3f TB' % (float(volSize) / 1000000) return result
class NodeManager(TortugaObjectManager): \ # pylint: disable=too-many-public-methods def __init__(self): super(NodeManager, self).__init__() self._nodeDbApi = NodeDbApi() self._cm = ConfigManager() self._bhm = osUtility.getOsObjectFactory().getOsBootHostManager( self._cm) self._syncApi = SyncApi() self._nodesDbHandler = NodesDbHandler() self._addHostManager = AddHostManager() self._logger = logging.getLogger(NODE_NAMESPACE) def __validateHostName(self, hostname: str, name_format: str) -> None: """ Raises: ConfigurationError """ bWildcardNameFormat = (name_format == '*') if hostname and not bWildcardNameFormat: # Host name specified, but hardware profile does not # allow setting the host name raise ConfigurationError( 'Hardware profile does not allow setting host names' ' of imported nodes') elif not hostname and bWildcardNameFormat: # Host name not specified but hardware profile expects it raise ConfigurationError( 'Hardware profile requires host names to be set') def createNewNode(self, session: Session, addNodeRequest: dict, dbHardwareProfile: HardwareProfileModel, dbSoftwareProfile: Optional[SoftwareProfileModel] = None, validateIp: bool = True, bGenerateIp: bool = True, dns_zone: Optional[str] = None) -> NodeModel: """ Convert the addNodeRequest into a Nodes object Raises: NicNotFound """ self._logger.debug( 'createNewNode(): session=[%s], addNodeRequest=[%s],' ' dbHardwareProfile=[%s], dbSoftwareProfile=[%s],' ' validateIp=[%s], bGenerateIp=[%s]' % (id(session), addNodeRequest, dbHardwareProfile.name, dbSoftwareProfile.name if dbSoftwareProfile else '(none)', validateIp, bGenerateIp)) hostname = addNodeRequest['name'] \ if 'name' in addNodeRequest else None # Ensure no conflicting options (ie. specifying host name for # hardware profile in which host names are generated) self.__validateHostName(hostname, dbHardwareProfile.nameFormat) node = NodeModel(name=hostname) if 'rack' in addNodeRequest: node.rack = addNodeRequest['rack'] node.addHostSession = addNodeRequest['addHostSession'] # Complete initialization of new node record nic_defs = addNodeRequest['nics'] \ if 'nics' in addNodeRequest else [] AddHostServerLocal().initializeNode(session, node, dbHardwareProfile, dbSoftwareProfile, nic_defs, bValidateIp=validateIp, bGenerateIp=bGenerateIp, dns_zone=dns_zone) # Set hardware profile of new node node.hardwareprofile = dbHardwareProfile node.softwareprofile = dbSoftwareProfile # Return the new node return node def getNode(self, session: Session, name, optionDict: OptionDict = None) \ -> Node: """ Get node by name Raises: NodeNotFound """ return self.__populate_nodes(session, [ self._nodeDbApi.getNode( session, name, optionDict=get_default_relations(optionDict)) ])[0] def getNodeById(self, session: Session, nodeId: int, optionDict: OptionDict = None) -> Node: """ Get node by node id Raises: NodeNotFound """ return self.__populate_nodes(session, [ self._nodeDbApi.getNodeById( session, int(nodeId), optionDict=get_default_relations(optionDict)) ])[0] def getNodeByIp(self, session: Session, ip: str, optionDict: Dict[str, bool] = None) -> Node: """ Get node by IP address Raises: NodeNotFound """ return self.__populate_nodes(session, [ self._nodeDbApi.getNodeByIp( session, ip, optionDict=get_default_relations(optionDict)) ])[0] def getNodeList(self, session, tags=None, optionDict: Optional[OptionDict] = None) -> List[Node]: """ Return all nodes """ return self.__populate_nodes( session, self._nodeDbApi.getNodeList( session, tags=tags, optionDict=get_default_relations(optionDict))) def __populate_nodes(self, session: Session, nodes: List[Node]) -> List[Node]: """ Expand non-database fields in Node objects """ class SoftwareProfileMetadataCache(defaultdict): def __missing__(self, key): metadata = \ SoftwareProfileManager().get_software_profile_metadata( session, key ) self[key] = metadata return metadata swprofile_map = SoftwareProfileMetadataCache() for node in nodes: if not node.getSoftwareProfile(): continue node.getSoftwareProfile().setMetadata( swprofile_map[node.getSoftwareProfile().getName()]) return nodes def updateNode(self, session: Session, nodeName: str, updateNodeRequest: dict) -> None: """ Calls updateNode() method of resource adapter """ self._logger.debug('updateNode(): name=[{0}]'.format(nodeName)) try: node = self._nodesDbHandler.getNode(session, nodeName) if 'nics' in updateNodeRequest: nic = updateNodeRequest['nics'][0] if 'ip' in nic: node.nics[0].ip = nic['ip'] node.nics[0].boot = True # Call resource adapter # self._nodesDbHandler.updateNode(session, node, updateNodeRequest) adapter = self.__getResourceAdapter(node.hardwareprofile) adapter.updateNode(session, node, updateNodeRequest) run_post_install = False # # Capture previous state and node data as dict for firing the # event later on # previous_state = node.state node_dict = Node.getFromDbDict(node.__dict__).getCleanDict() if 'state' in updateNodeRequest: run_post_install = \ node.state == state.NODE_STATE_ALLOCATED and \ updateNodeRequest['state'] == state.NODE_STATE_PROVISIONED node.state = updateNodeRequest['state'] node_dict['state'] = updateNodeRequest['state'] session.commit() # # If the node state has changed, then fire the node state changed # event # if node_dict['state'] != previous_state: NodeStateChanged.fire(node=node_dict, previous_state=previous_state) if run_post_install: self._logger.debug( 'updateNode(): run-post-install for node [{0}]'.format( node.name)) self.__scheduleUpdate() except Exception: session.rollback() raise def updateNodeStatus(self, session: Session, nodeName: str, node_state: Optional[str] = None, bootFrom: int = None): """Update node status If neither 'state' nor 'bootFrom' are not None, this operation will update only the 'lastUpdated' timestamp. Returns: bool indicating whether state and/or bootFrom differed from current value """ value = 'None' if bootFrom is None else \ '1 (disk)' if int(bootFrom) == 1 else '0 (network)' self._logger.debug( 'updateNodeStatus(): node=[%s], node_state=[{%s}],' ' bootFrom=[{%s}]', nodeName, node_state, value) dbNode = self._nodesDbHandler.getNode(session, nodeName) # # Capture previous state and node data in dict form for the # event later on # previous_state = dbNode.state node_dict = Node.getFromDbDict(dbNode.__dict__).getCleanDict() # Bitfield representing node changes (0 = state change, # 1 = bootFrom # change) changed = 0 if node_state is not None and node_state != dbNode.state: # 'state' changed changed |= 1 if bootFrom is not None and bootFrom != dbNode.bootFrom: # 'bootFrom' changed changed |= 2 if changed: # Create custom log message msg = 'Node [%s] state change:' % (dbNode.name) if changed & 1: msg += ' state: [%s] -> [%s]' % (dbNode.state, node_state) dbNode.state = node_state node_dict['state'] = node_state if changed & 2: msg += ' bootFrom: [%d] -> [%d]' % (dbNode.bootFrom, bootFrom) dbNode.bootFrom = bootFrom self._logger.info(msg) else: self._logger.info('Updated timestamp for node [%s]' % (dbNode.name)) dbNode.lastUpdate = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time())) result = bool(changed) # Only change local boot configuration if the hardware profile is # not marked as 'remote' and we're not acting on the installer # node. if dbNode.softwareprofile and \ dbNode.softwareprofile.type != 'installer' and \ dbNode.hardwareprofile.location != 'remote': # update local boot configuration for on-premise nodes self._bhm.writePXEFile(session, dbNode, localboot=bootFrom) session.commit() # # If the node state has changed, fire the node state changed # event # if state and (previous_state != state): NodeStateChanged.fire(node=node_dict, previous_state=previous_state) return result def __process_nodeErrorDict(self, nodeErrorDict): result = {} nodes_deleted = [] for key, nodeList in nodeErrorDict.items(): result[key] = [dbNode.name for dbNode in nodeList] if key == 'NodesDeleted': for node in nodeList: node_deleted = { 'name': node.name, 'hardwareprofile': node.hardwareprofile.name, 'addHostSession': node.addHostSession, } if node.softwareprofile: node_deleted['softwareprofile'] = \ node.softwareprofile.name nodes_deleted.append(node_deleted) return result, nodes_deleted def deleteNode(self, session, nodespec: str, force: bool = False): """ Delete node by nodespec Raises: NodeNotFound """ kitmgr = KitActionsManager() kitmgr.session = session try: nodes = self._nodesDbHandler.expand_nodespec( session, nodespec, include_installer=False) if not nodes: raise NodeNotFound('No nodes matching nodespec [%s]' % (nodespec)) self.__validate_delete_nodes_request(nodes, force) self.__preDeleteHost(kitmgr, nodes) nodeErrorDict = self.__delete_node(session, nodes) # REALLY!?!? Convert a list of Nodes objects into a list of # node names so we can report the list back to the end-user. # This needs to be FIXED! result, nodes_deleted = self.__process_nodeErrorDict(nodeErrorDict) session.commit() # ============================================================ # Perform actions *after* node deletion(s) have been committed # to database. # ============================================================ self.__postDeleteHost(kitmgr, nodes_deleted) addHostSessions = set( [tmpnode['addHostSession'] for tmpnode in nodes_deleted]) if addHostSessions: self._addHostManager.delete_sessions(addHostSessions) for nodeName in result['NodesDeleted']: # Remove the Puppet cert self._bhm.deletePuppetNodeCert(nodeName) self._bhm.nodeCleanup(nodeName) self._logger.info('Node [%s] deleted' % (nodeName)) # Schedule a cluster update self.__scheduleUpdate() return result except Exception: session.rollback() raise def __validate_delete_nodes_request(self, nodes: List[NodeModel], force: bool): """ Raises: DeleteNodeFailed """ swprofile_distribution: Dict[SoftwareProfileModel, int] = {} for node in nodes: if node.softwareprofile not in swprofile_distribution: swprofile_distribution[node.softwareprofile] = 0 swprofile_distribution[node.softwareprofile] += 1 errors: List[str] = [] for software_profile, num_nodes_deleted in \ swprofile_distribution.items(): if software_profile.lockedState == 'HardLocked': errors.append( f'Nodes cannot be deleted from hard locked software' ' profile [{software_profile.name}]') continue if software_profile.minNodes and \ len(software_profile.nodes) - num_nodes_deleted < \ software_profile.minNodes: if force and software_profile.lockedState == 'SoftLocked': # allow deletion of nodes when force is set and profile # is soft locked continue # do not allow number of software profile nodes to drop # below configured minimum errors.append( 'Software profile [{}] requires minimum of {} nodes;' ' denied request to delete {} node(s)'.format( software_profile.name, software_profile.minNodes, num_nodes_deleted)) continue if software_profile.lockedState == 'SoftLocked' and not force: errors.append( 'Nodes cannot be deleted from soft locked software' f' profile [{software_profile.name}]') if errors: raise OperationFailed('\n'.join(errors)) def __delete_node(self, session: Session, dbNodes: List[NodeModel]) \ -> Dict[str, List[NodeModel]]: """ Raises: DeleteNodeFailed """ result: Dict[str, list] = { 'NodesDeleted': [], 'DeleteNodeFailed': [], 'SoftwareProfileLocked': [], 'SoftwareProfileHardLocked': [], } nodes: Dict[HardwareProfileModel, List[NodeModel]] = {} events_to_fire: List[dict] = [] # # Mark node states as deleted in the database # for dbNode in dbNodes: # # Capture previous state and node data as a dict for firing # the event later on # event_data = { 'previous_state': dbNode.state, 'node': Node.getFromDbDict(dbNode.__dict__).getCleanDict() } dbNode.state = state.NODE_STATE_DELETED event_data['node']['state'] = 'Deleted' if dbNode.hardwareprofile not in nodes: nodes[dbNode.hardwareprofile] = [dbNode] else: nodes[dbNode.hardwareprofile].append(dbNode) session.commit() # # Fire node state change events # for event in events_to_fire: NodeStateChanged.fire(node=event['node'], previous_state=event['previous_state']) # # Call resource adapter with batch(es) of node lists keyed on # hardware profile. # for hwprofile, hwprofile_nodes in nodes.items(): # Get the ResourceAdapter adapter = self.__get_resource_adapter(session, hwprofile) # Call the resource adapter adapter.deleteNode(hwprofile_nodes) # Iterate over all nodes in hardware profile, completing the # delete operation. for dbNode in hwprofile_nodes: for tag in dbNode.tags: if len(tag.nodes) == 1 and \ not tag.softwareprofiles and \ not tag.hardwareprofiles: session.delete(tag) # Delete the Node self._logger.debug('Deleting node [%s]' % (dbNode.name)) session.delete(dbNode) result['NodesDeleted'].append(dbNode) return result def __get_resource_adapter(self, session: Session, hardwareProfile: HardwareProfileModel): """ Raises: OperationFailed """ if not hardwareProfile.resourceadapter: raise OperationFailed( 'Hardware profile [%s] does not have an associated' ' resource adapter' % (hardwareProfile.name)) adapter = resourceAdapterFactory.get_api( hardwareProfile.resourceadapter.name) adapter.session = session return adapter def __process_delete_node_result(self, nodeErrorDict): # REALLY!?!? Convert a list of Nodes objects into a list of # node names so we can report the list back to the end-user. # This needs to be FIXED! result = {} nodes_deleted = [] for key, nodeList in nodeErrorDict.items(): result[key] = [dbNode.name for dbNode in nodeList] if key == 'NodesDeleted': for node in nodeList: node_deleted = { 'name': node.name, 'hardwareprofile': node.hardwareprofile.name, } if node.softwareprofile: node_deleted['softwareprofile'] = \ node.softwareprofile.name nodes_deleted.append(node_deleted) return result, nodes_deleted def __preDeleteHost(self, kitmgr: KitActionsManager, nodes): self._logger.debug('__preDeleteHost(): nodes=[%s]' % (' '.join([node.name for node in nodes]))) for node in nodes: kitmgr.pre_delete_host( node.hardwareprofile.name, node.softwareprofile.name if node.softwareprofile else None, nodes=[node.name]) def __postDeleteHost(self, kitmgr, nodes_deleted): # 'nodes_deleted' is a list of dicts of the following format: # # { # 'name': 'compute-01', # 'softwareprofile': 'Compute', # 'hardwareprofile': 'LocalIron', # } # # if the node does not have an associated software profile, the # dict does not contain the key 'softwareprofile'. self._logger.debug('__postDeleteHost(): nodes_deleted=[%s]' % (nodes_deleted)) if not nodes_deleted: self._logger.debug('No nodes deleted in this operation') return for node_dict in nodes_deleted: kitmgr.post_delete_host(node_dict['hardwareprofile'], node_dict['softwareprofile'] if 'softwareprofile' in node_dict else None, nodes=[node_dict['name']]) def __scheduleUpdate(self): self._syncApi.scheduleClusterUpdate() def getInstallerNode(self, session, optionDict: Optional[OptionDict] = None): return self._nodeDbApi.getNode( session, self._cm.getInstaller(), optionDict=get_default_relations(optionDict)) def getProvisioningInfo(self, session: Session, nodeName): return self._nodeDbApi.getProvisioningInfo(session, nodeName) def startupNode(self, session, nodespec: str, remainingNodeList: List[NodeModel] = None, bootMethod: str = 'n') -> None: """ Raises: NodeNotFound """ try: nodes = self._nodesDbHandler.expand_nodespec(session, nodespec) if not nodes: raise NodeNotFound('No matching nodes for nodespec [%s]' % (nodespec)) # Break list of nodes into dict keyed on hardware profile nodes_dict = self.__processNodeList(nodes) for dbHardwareProfile, detailsDict in nodes_dict.items(): # Get the ResourceAdapter adapter = self.__getResourceAdapter(dbHardwareProfile) # Call startup action extension adapter.startupNode(detailsDict['nodes'], remainingNodeList=remainingNodeList or [], tmpBootMethod=bootMethod) session.commit() except TortugaException: session.rollback() raise except Exception as ex: session.rollback() self._logger.exception(str(ex)) raise def shutdownNode(self, session, nodespec: str, bSoftShutdown: bool = False) \ -> None: """ Raises: NodeNotFound """ try: nodes = self._nodesDbHandler.expand_nodespec(session, nodespec) if not nodes: raise NodeNotFound('No matching nodes for nodespec [%s]' % (nodespec)) d = self.__processNodeList(nodes) for dbHardwareProfile, detailsDict in d.items(): # Get the ResourceAdapter adapter = self.__getResourceAdapter(dbHardwareProfile) # Call shutdown action extension adapter.shutdownNode(detailsDict['nodes'], bSoftShutdown) session.commit() except TortugaException: session.rollback() raise except Exception as ex: session.rollback() self._logger.exception(str(ex)) raise def rebootNode(self, session, nodespec: str, bSoftReset: bool = False, bReinstall: bool = False) -> None: """ Raises: NodeNotFound """ nodes = self._nodesDbHandler.expand_nodespec(session, nodespec) if not nodes: raise NodeNotFound('No nodes matching nodespec [%s]' % (nodespec)) if bReinstall: for dbNode in nodes: self._bhm.setNodeForNetworkBoot(session, dbNode) for dbHardwareProfile, detailsDict in \ self.__processNodeList(nodes).items(): # iterate over hardware profile/nodes dict to reboot each # node adapter = self.__getResourceAdapter(dbHardwareProfile) # Call reboot action extension adapter.rebootNode(detailsDict['nodes'], bSoftReset) session.commit() def getNodesByNodeState(self, session, node_state: str, optionDict: Optional[OptionDict] = None) \ -> TortugaObjectList: """ Get nodes by state """ return self.__populate_nodes( session, self._nodeDbApi.getNodesByNodeState( session, node_state, optionDict=get_default_relations(optionDict))) def getNodesByNameFilter(self, session, nodespec: str, optionDict: OptionDict = None, include_installer: Optional[bool] = True) \ -> TortugaObjectList: """ Return TortugaObjectList of Node objects matching nodespec """ return self.__populate_nodes( session, self._nodeDbApi.getNodesByNameFilter( session, nodespec, optionDict=get_default_relations(optionDict), include_installer=include_installer)) def getNodesByAddHostSession(self, session, addHostSession: str, optionDict: OptionDict = None) \ -> TortugaObjectList: """ Return TortugaObjectList of Node objects matching add host session """ return self.__populate_nodes( session, self._nodeDbApi.getNodesByAddHostSession( session, addHostSession, optionDict=get_default_relations(optionDict))) def __processNodeList(self, dbNodes: List[NodeModel]) \ -> Dict[HardwareProfileModel, Dict[str, list]]: """ Returns dict indexed by hardware profile, each with a list of nodes in the hardware profile """ d: Dict[HardwareProfileModel, Dict[str, list]] = {} for dbNode in dbNodes: if dbNode.hardwareprofile not in d: d[dbNode.hardwareprofile] = { 'nodes': [], } d[dbNode.hardwareprofile]['nodes'].append(dbNode) return d def __getResourceAdapter(self, hardwareProfile: HardwareProfileModel): """ Raises: OperationFailed """ if not hardwareProfile.resourceadapter: raise OperationFailed( 'Hardware profile [%s] does not have an associated' ' resource adapter' % (hardwareProfile.name)) return resourceAdapterFactory.get_api( hardwareProfile.resourceadapter.name) \ if hardwareProfile.resourceadapter else None
class TortugaDeployer: \ # pylint: disable=too-many-public-methods def __init__(self, logger, cmdline_options=None): self._cm = ConfigManager() self._logger = logger self._osObjectFactory = osUtility.getOsObjectFactory() self._settings = self.__load_settings(cmdline_options) self._settings['installer_software_profile'] = 'Installer' self._settings['installer_hardware_profile'] = 'Installer' self._settings['eulaAccepted'] = False self._settings['fqdn'] = getfqdn() self._settings['osInfo'] = getOsInfo() self._forceCleaning = False self._depotCreated = False fsManager = self._osObjectFactory.getOsFileSystemManager() self._lockFilePath = os.path.join(fsManager.getOsLockFilePath(), 'tortuga-setup') langdomain = 'tortuga-config' localedir = os.path.join(self._cm.getRoot(), 'share', 'locale') if not os.path.exists(localedir): # Try the system path localedir = '/usr/share/locale' gettext.bindtextdomain(langdomain, localedir) gettext.textdomain(langdomain) self.gettext = gettext.gettext self._ = self.gettext self._logger.info('Detected OS: [%s]', self._settings['osInfo']) def __load_settings(self, cmdline_options): settings = dict(list(cmdline_options.items())) default_cfgfile = os.path.join(self._cm.getKitConfigBase(), 'tortuga.ini') if 'inifile' in cmdline_options and \ cmdline_options['inifile'] != default_cfgfile: # Copy configuration specified on command-line to # $TORTUGA_ROOT/config/tortuga.ini self._logger.info('Using configuration file [%s]' % (settings['inifile'])) self._logger.info('Copying configuration to [%s]' % (default_cfgfile)) if os.path.exists(default_cfgfile): # Back up existing 'tortuga.ini' shutil.move(default_cfgfile, default_cfgfile + '.orig') shutil.copyfile(cmdline_options['inifile'], default_cfgfile) settings['inifile'] = default_cfgfile cfg = configparser.ConfigParser() cfg.read(settings['inifile']) settings['timezone'] = '' settings['utc'] = False settings['keyboard'] = 'us' settings['language'] = 'en_US.UTF-8' # Get database setting value = cfg.get('database', 'engine') \ if cfg.has_section('database') and \ cfg.has_option('database', 'engine') else None if value and value not in ('mysql', 'sqlite'): raise InvalidArgument('Unsupported database engine [%s]' % (value)) settings['database'] = {'engine': value if value else 'sqlite'} # Get depot directory if cfg.has_section('installer') and \ cfg.has_option('installer', 'depotpath'): settings['depotpath'] = cfg.get('installer', 'depotpath') # For consistency's sake... self._cm.setDepotDir(settings['depotpath']) else: settings['depotpath'] = self._cm.getDepotDir() # Internal web port settings['intWebPort'] = cfg.getint('installer', 'intWebPort') \ if cfg.has_section('installer') and \ cfg.has_option('installer', 'intWebPort') else \ self._cm.getIntWebPort() self._cm.setIntWebPort(settings['intWebPort']) # Admin port settings['adminPort'] = cfg.getint('installer', 'adminPort') \ if cfg.has_section('installer') and \ cfg.has_option('installer', 'adminPort') else \ self._cm.getAdminPort() self._cm.setAdminPort(settings['adminPort']) # IntWebServicePort settings['intWebServicePort'] = cfg.getint( 'installer', 'intWebServicePort') \ if cfg.has_section('installer') and \ cfg.has_option('installer', 'intWebServicePort') else \ self._cm.getIntWebServicePort() self._cm.setIntWebServicePort(settings['intWebServicePort']) return settings def _get_setting(self, name, section=None): if section and section in self._settings: return self._settings[section][name] \ if name in self._settings[section] else None return self._settings[name] if name in self._settings else None def eout(self, message, *args): """ Output messages to STDERR with Internationalization. Additional arguments will be used to substitute variables in the message output """ if args: mesg = self.gettext(message) % args else: mesg = self.gettext(message) sys.stderr.write(mesg) def out(self, message, *args): """ Output messages to STDOUT with Internationalization. Additional arguments will be used to substitute variables in the message output """ if args: mesg = self.gettext(message) % args else: mesg = self.gettext(message) sys.stdout.write(mesg) def prompt(self, default_value, auto_answer_default_value, text_list, question, tag=None, section=None, isPassword=False): """Generic user prompting routine""" resp_value = None bDefaults = self._settings['defaults'] if tag: resp_value = self._get_setting(tag, section=section) if not resp_value and bDefaults: # Use the default value default_value = auto_answer_default_value elif bDefaults: default_value = auto_answer_default_value if text_list: self.out('\n') for line in text_list: self.out(line + '\n') if default_value and not isPassword: self.out('\n%s [%s]: ' % (question, default_value)) else: self.out('\n%s: ' % (question)) if bDefaults or resp_value: if resp_value: value = resp_value else: value = auto_answer_default_value if not isPassword: self.out('%s\n' % value) else: if isPassword: import getpass value = getpass.getpass('').strip() else: value = input('').strip() if not value: value = default_value return value def checkPreInstallConfiguration(self): # pylint: disable=no-self-use """ Raises: InvalidMachineConfiguration """ # Check for existence of /etc/hosts if not os.path.exists('/etc/hosts'): raise InvalidMachineConfiguration( '/etc/hosts file is missing. Unable to proceed with' ' installation') def preInstallPrep(self): bAcceptEula = self._settings['acceptEula'] license_file = ' %s/LICENSE' % (self._cm.getEtcDir()) print() if bAcceptEula: cmd = 'cat %s\n' % (license_file) os.system(cmd) else: cmd = 'more %s\n' % (license_file) print("To install Tortuga you must read and agree to " "the following EULA.") print("Press 'Enter' to continue...") input('') os.system(cmd) print() while True: print('Do you agree? [Yes / No]', end=' ') answer = input('').lower() if answer not in ['yes', 'no', 'y', 'n']: print('Invalid response. Please respond \'Yes\'' ' or \'No\'') continue break if answer[0] == 'n': raise EulaAcceptanceRequired( 'You must accept the EULA to install Tortuga') self._settings['eulaAccepted'] = \ 'Accepted on: %s local machine time' % (time.ctime()) # Restore resolv.conf if we have a backup if osUtility.haveBackupFile('/etc/resolv.conf'): osUtility.restoreFile('/etc/resolv.conf') def _runCommandWithSpinner(self, cmd, statusMsg, logFileName): self._logger.debug( '_runCommandWithSpinner(cmd=[%s], logFileName=[%s])' % (cmd, logFileName)) self.out(statusMsg + ' ') # Open the log file in unbuffered mode fpOut = open(logFileName, 'wb', 0) p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, bufsize=1, close_fds=True) for i in itertools.cycle(['/', '-', '\\', '|']): buf = p.stdout.readline() sys.stdout.write('') sys.stdout.flush() if not buf: break fpOut.write(buf) sys.stdout.write(i) sys.stdout.flush() sys.stdout.write(' ') self.out('done.\n') retval = p.wait() fpOut.close() return retval def puppetApply(self): ''' Complete the installer configuration by running against the previously installed Puppet master. Display a spinner while Puppet runs. ''' self._logger.info('Running Puppet for post-configuration') logFileName = '/tmp/tortuga_setup.log' cmd = ('/opt/puppetlabs/bin/puppet agent --color false --onetime' ' --no-daemonize --detailed-exitcodes --verbose 2>&1') retval = self._runCommandWithSpinner( cmd, statusMsg=('\nCompleting installer configuration.' ' Please wait...'), logFileName=logFileName) if retval not in (0, 2): # Puppet can return a non-zero return code, even if it was # successful. errmsg = 'Puppet post-configuration failed (see log file %s)' % ( logFileName) self._logger.error(errmsg) self.out(errmsg + '\n') raise Exception(errmsg) self._logger.info('Puppet post-configuration completed') def startSetup(self): # If force was specified clean first and then run... bForce = self._settings['force'] if bForce: self._forceCleaning = True self.out('--force option specified. Cleaning previous' ' installation.\n') self.cleanup() self._forceCleaning = False if os.path.exists(self._lockFilePath): raise SoftwareAlreadyDeployed( "\ntortuga-setup has already been run.\n\n" "Use --force option to force reinstallation.") open(self._lockFilePath, 'w').close() self.out('Tortuga Setup\n') def getClusterConfig(self): sysManager = self._osObjectFactory.getOsSysManager() self._settings['timezone'], self._settings['utc'] = \ sysManager.findTimeInfo() self._settings['keyboard'] = sysManager.findKeyboard() self._settings['language'] = sysManager.findLanguage() self.out(_('\nStarting Tortuga setup...\n')) # Ports configuration if not self._settings['defaults']: intWebPort, adminPort, intWebServicePort = self.configurePorts() self._cm.setIntWebPort(intWebPort) self._cm.setAdminPort(adminPort) self._cm.setIntWebServicePort(intWebServicePort) self._settings['intWebPort'] = intWebPort self._settings['adminPort'] = adminPort self._settings['intWebServicePort'] = intWebServicePort # Admin username and password self._settings['adminUsername'], \ self._settings['adminPassword'] = self.promptForAdminCredentials() def prepDepot(self): depotpath = None if not self._settings['defaults']: self.out( _('Tortuga requires a directory for storage of OS' ' distribution media and other files required for' ' node provisioning.\n\n')) while not depotpath: if self._settings['defaults']: response = self._settings['depotpath'] else: try: response = input( 'Please enter a depot path (Ctrl-C to interrupt)' ' [%s]: ' % (self._settings['depotpath'])) except KeyboardInterrupt: raise InvalidArgument(_('Aborted by user.')) if not response: response = self._settings['depotpath'] if not response.startswith('/'): errmsg = 'Depot path must be fully-qualified' if not self._settings['defaults']: self.out('Error: %s\n' % (errmsg)) continue raise InvalidArgument(errmsg) if response == '/': errmsg = 'Depot path cannot be system root directory' if not self._settings['defaults']: self.out(_('Error: %s\n' % (errmsg))) continue raise InvalidArgument(errmsg) if os.path.exists(response): if not self._settings['force']: if not self._settings['defaults']: self.out( _('Directory [%s] already exists. Do you wish to' ' remove it [N/y]? ') % (response)) remove_response = input('') if not remove_response or \ remove_response[0].lower() == 'n': continue_response = input( 'Do you wish to continue [N/y]? ') if continue_response and \ continue_response[0].lower() == 'y': continue raise InvalidArgument(_('Aborted by user.')) else: raise InvalidArgument( _('Existing depot directory [%s] will not be' ' removed.') % (response)) else: self.out( _('\nRemoving existing depot directory [%s]... ') % (response)) depotpath = response tortugaSubprocess.executeCommand('rm -rf %s/*' % (depotpath)) self.out(_('done.\n')) else: depotpath = response self._settings['depotpath'] = depotpath self._cm.setDepotDir(self._settings['depotpath']) def _portPrompt(self, promptStr, defaultValue): while True: tmpPort = self.prompt(defaultValue, defaultValue, None, promptStr) try: tmpPort = int(tmpPort) if tmpPort <= 0 or tmpPort > 65535: raise ValueError('Port must be between 1 and 65535') # Success break except ValueError as ex: self.out('Error: ' + str(ex) + '\n') return tmpPort def configurePorts(self): reconfigurePorts = self.prompt( 'N', 'N', [ 'The following ports will be used by Tortuga:' '', ' +-----------------------------+-------+', ' | Description | Port |', ' +-----------------------------+-------+', ' | Internal webserver | %5d |' % (self._settings['intWebPort']), ' | SSL webservice daemon | %5d |' % (self._settings['adminPort']), ' | Local webservice daemon | %5d |' % (self._settings['intWebServicePort']), ' +-----------------------------+-------+' ], 'Do you wish to change the default configuration [N/y]?') if not reconfigurePorts or reconfigurePorts[0].lower() == 'n': return self._settings['intWebPort'], \ self._settings['adminPort'], \ self._settings['intWebServicePort'] # Internal web server port intWebPort = self._portPrompt('Enter port for internal webserver', self._settings['intWebPort']) # SSL webservice daemon port adminPort = self._portPrompt('Enter port for SSL webservice daemon', self._settings['adminPort']) # Local webservice daemon port intWebServicePort = self._portPrompt( 'Enter port for local webservice daemon', self._settings['intWebServicePort']) return intWebPort, adminPort, intWebServicePort def _removePackageSources(self): pkgManager = self._osObjectFactory.getOsPackageManager() for pkgSrcName in pkgManager.getPackageSourceNames(): self._logger.info('Removing package source [%s]' % (pkgSrcName)) pkgManager.removePackageSource(pkgSrcName) def _disableTortugaws(self): self.out(' * Disabling Tortuga webservice\n') _tortugaWsManager = self._osObjectFactory.getTortugawsManager() serviceName = _tortugaWsManager.getServiceName() _osServiceManager = getOsObjectFactory().getOsServiceManager() try: _osServiceManager.stop(serviceName) except CommandFailed: pass def cleanup(self): # If possible, remove any package sources we added self._removePackageSources() osUtility.removeFile(self._lockFilePath) osUtility.removeFile(self._cm.getProfileNiiFile()) # Turn off the webservice daemon self._disableTortugaws() # Restore resolv.conf if osUtility.haveBackupFile('/etc/resolv.conf'): osUtility.restoreFile('/etc/resolv.conf') # Drop database dbManager = self._osObjectFactory.getOsApplicationManager( self._settings['database']['engine']) try: dbSchema = self._cm.getDbSchema() self.out(' * Removing database [%s]\n' % (dbSchema)) dbManager.destroyDb(dbSchema) except Exception as ex: # pylint: disable=broad-except self._logger.exception( 'Could not destroy existing db: {}'.format(ex)) # Remove DB password file osUtility.removeFile(self._cm.getDbPasswordFile()) # Remove CFM secret cfmSecretFile = self._cm.getCfmSecretFile() if os.path.exists(cfmSecretFile): osUtility.removeFile(self._cm.getCfmSecretFile()) # Generic cleanup osUtility.removeLink('/etc/tortuga-release') # Cleanup or remove depot directory errmsg = 'Removing contents of [%s]' % (self._settings['depotpath']) self._logger.debug(errmsg) if self._depotCreated: self.out(' * %s\n' % (errmsg)) osUtility.removeDir(self._settings['depotpath']) else: if self._settings['depotpath']: self.out(' * %s\n' % (errmsg)) tortugaSubprocess.executeCommand('rm -rf %s/*' % (self._settings['depotpath'])) self.out('\n') if not self._forceCleaning: self.out('Consult log(s) for further details.\n') self._logger.error('Installation failed') def runSetup(self): """ Installer setup. """ self.checkPreInstallConfiguration() # Do not run cleanup if this fails. self.startSetup() try: self.preInstallPrep() self.getClusterConfig() self.prepDepot() self.preConfig() self.pre_init_db() self.puppetBootstrap() dbm, session = self.initDatabase() try: self.createAdminUser(session, self._settings['adminUsername'], self._settings['adminPassword']) self.installKits(dbm) self.enableComponents(session) finally: dbm.closeSession() self.puppetApply() self.out('\nTortuga installation completed successfully!\n\n') print('Run \"exec -l $SHELL\" to initialize Tortuga environment\n') except Exception: # pylint: disable=broad-except self._logger.exception('Fatal error occurred during setup') raise TortugaException('Installation failed') def _generate_db_password(self): """ Generate a database password. """ # # Because Apache httpd server is not installed at the time this # runs, we cannot set the ownership of this file to be 'apache' # (which is necessary for the Tortuga webservice). # # Set ownership of file to root:puppet. # # When the Puppet bootstrap runs, it changes the ownership to # 'apache:puppet' and everybody is happy! # puppet_user = pwd.getpwnam('puppet') gid = puppet_user[3] self._generate_password_file(self._cm.getDbPasswordFile(), gid=gid) def _generate_redis_password(self): """ Generate a password for Redis. """ # # Puppet needs read access to this file so that it can use it for # writing the redis config file. # puppet_user = pwd.getpwnam('puppet') gid = puppet_user[3] self._generate_password_file(self._cm.getRedisPasswordFile(), gid=gid) def _generate_password_file(self, file_name: str, password_length: int = 32, uid: int = 0, gid: int = 0, mode: int = 0o440): """ Generate a password in a file. :param file_name: the name of the file in which the password will be stored :param password_length: the length of the password, default = 32 :param uid: the uid (owner) of the file, default = 0 :param gid: the gid (group) of the file, default = 0 :param mode: the file perms, default 0440 """ password = self._generate_password(password_length) with open(file_name, 'w') as fp: fp.write(password) os.chown(file_name, uid, gid) os.chmod(file_name, mode) def _generate_password(self, length: int = 8) -> str: """ Generate a password. :param length: the length of the password :return: the generated password """ chars = string.ascii_letters + string.digits return ''.join([random.choice(chars) for _ in range(length)]) def preConfig(self): # Create default hieradata directory hieraDataDir = '/etc/puppetlabs/code/environments/production/data' if not os.path.exists(hieraDataDir): os.makedirs(hieraDataDir) # Derive host name of puppet master from FQDN fqdn = self._settings['fqdn'] configDict = { 'version': 5, 'DNSZone': 'private', 'puppet_server': fqdn, 'depot': self._settings['depotpath'], } with open(os.path.join(hieraDataDir, 'tortuga-common.yaml'), 'wb') as fp: fp.write( yaml.safe_dump(configDict, explicit_start=True, default_flow_style=False).encode()) self._generate_db_password() self._generate_redis_password() def pre_init_db(self): # If using 'mysql' as the database backend, we need to install the # puppetlabs-mysql Puppet module prior to bootstrapping. This used # to be done in 'install-tortuga.sh' if self._settings['database']['engine'] == 'mysql': print('\nUsing MySQL as backing database.') puppet_module = 'puppetlabs-mysql' logmsg = f'Installing \'{puppet_module}\' module' self._logger.debug(logmsg) print(f'\n{logmsg}...', end='') cmd = ('/opt/puppetlabs/bin/puppet module install' f' --color false {puppet_module}') tortugaSubprocess.executeCommand(cmd) print('done.') def puppetBootstrap(self): localPuppetRoot = os.path.join(self._cm.getEtcDir(), 'puppet') logFileName = '/tmp/bootstrap.log' puppet_server = self._settings['fqdn'] # Bootstrap using Puppet cmd = ('/opt/puppetlabs/bin/puppet apply --verbose' ' --detailed-exitcodes' ' --execute "class { \'tortuga::installer\':' ' puppet_server => \'%s\',' '}"' % (puppet_server)) retval = self._runCommandWithSpinner( cmd, '\nPerforming pre-configuration... Please wait...', logFileName=logFileName) if retval not in (0, 2): # Puppet can return a non-zero return code, even if it was # successful. self._logger.debug('Puppet pre-configuration returned non-zero' ' return code [%d]' % (retval)) errmsg = 'Puppet bootstrap failed (see log file %s)' % ( logFileName) self._logger.error(errmsg) raise Exception(errmsg) self._logger.debug('Puppet pre-configuration completed') def initDatabase(self) -> Tuple[Any, Session]: msg = _('Initializing database') self._logger.info(msg) print_('\n' + msg + '... ', end='') # This cannot be a global import since the database configuration # may be set in this script. from tortuga.db.dbManager import DbManager dbm = DbManager() # create database dbm.init_database() session = dbm.openSession() # Prime the database previously created as part of the bootstrap try: dbUtility.primeDb(session, self._settings) dbUtility.init_global_parameters(session, self._settings) print_(_('done')) session.commit() except Exception as exc: # pylint: disable=broad-except session.rollback() print_(_('failed.')) print_(_('Exception raised initializing database:') + ' {0}'.format(exc), file=sys.stderr) self._logger.debug('Done initializing database') return dbm, session def installKits(self, dbm): self._logger.info('Installing kits') self.out('\n' + _('Installing kits') + '...\n') kitApi = KitApi() # Iterate over the glob of 'kits-*.tar.bz2' kitFileGlob = '%s/kits/kit-*.tar.bz2' % (self._cm.getRoot()) # Split comma-separated list of kits to skip installing. Sorry, you # cannot skip installing the base kit. val = self._settings['skip_kits'] \ if 'skip_kits' in self._settings else '' skip_kits = set([ item for item in val.split(',') if item != 'base']) \ if val else set() for kitPackage in glob.glob(kitFileGlob): try: kit = get_metadata_from_archive(kitPackage) except KitNotFound: msg = 'Kit [%s] is malformed/invalid. Skipping.' % ( os.path.basename(kitPackage)) self._logger.error(msg) self.out(' %s\n' % (msg)) continue if kit['name'] in skip_kits: msg = 'Kit [%s] installation skipped.' % (kit['name']) self.out(' %s\n' % (msg)) self._logger.info(msg) continue try: kitApi.installKitPackage(dbm, kitPackage) except EulaAcceptanceRequired: msg = 'Kit [%s] requires EULA acceptance. Skipping.' % ( kitPackage) self.out(' %s\n' % (msg)) self._logger.info(msg) continue self.out(' - %s installed.\n' % (kit['name'])) self._logger.info('Kit [%s] installed' % (kit['name'])) self._logger.info('Done installing kits') load_kits() def enableComponents(self, session: Session): """ Raises: ConfigurationError """ self._logger.info('Enabling \'installer\' component') base_kit = KitApi().getKit(session, 'base') enabledComponents = ['installer'] # get list of components from 'base' kit components = [ c for c in base_kit.getComponentList() if c.getName() in enabledComponents ] installerNode = NodeApi().getInstallerNode(session) for component in components: SoftwareProfileApi().enableComponent( session, installerNode.getSoftwareProfile().getName(), base_kit.getName(), base_kit.getVersion(), base_kit.getIteration(), component.getName(), compVersion=component.getVersion(), ) def promptForAdminCredentials(self): # Get admin username and password for use with web service if self._settings['defaults']: self.out(_('\nUsing default Tortuga admin user name/password.\n')) return 'admin', 'password' username = password = None # Administrator username while True: username = self.prompt('admin', 'admin', [ 'Enter name for Tortuga admin user.', 'This user is not associated with any system user.' ], 'Admin user name') if len(username) > 3: break self.out('Admin user name must be at least 4 characters.') # Administrator password while True: password = self.prompt('', 'password', ['Enter password for Tortuga admin user.'], 'Admin password', None, None, True) if len(password) < 4: self.out('Admin password must be at least 4 characters.') continue confirmPassword = self.prompt('', 'password', ['Confirm admin password.'], 'Confirm password', None, None, True) if confirmPassword == password: self.out('\n') break self.out('Passwords did not match.') return username, password def createAdminUser(self, session: Session, username, password): msg = _('Adding administrative user') self._logger.info(msg) self.out('\n' + msg + '... ') AdminApi().addAdmin(session, username, password, False, description='Added by tortuga-setup') self.out(_('done.') + '\n')
class OsBootHostManagerCommon(OsObjectManager): """Methods for manipulating PXE files""" def __init__(self): OsObjectManager.__init__(self) # Cache this for later try: self.passdata = pwd.getpwnam('apache') except KeyError: self.passdata = pwd.getpwnam(os.getenv('USER')) self.hardwareProfileDbApi = HardwareProfileDbApi() self.softwareProfileDbApi = SoftwareProfileDbApi() self._nodeApi = nodeApi.NodeApi() self._cm = ConfigManager() def __getActualSoftwareProfile(self, node, softwareProfileName): """ Determine the actual software profile. For example, """ self.getLogger().debug('__getActualSoftwareProfile(): node=[%s],' ' softwareProfileName=[%s]' % (node.name, softwareProfileName)) softwareProfile = None if node.isIdle: # Use the software profile that was passed in if there is one, if softwareProfileName: softwareProfile = self.softwareProfileDbApi.\ getSoftwareProfile(softwareProfileName, {'os': True}) else: # ELSE use the default idle software profile hardwareProfile = node.getHardwareProfile() idleSoftwareProfileId = hardwareProfile.\ getIdleSoftwareProfileId() if idleSoftwareProfileId: softwareProfile = self.softwareProfileDbApi.\ getSoftwareProfileById(idleSoftwareProfileId, {'os': True}) else: # Use active software profile if softwareProfileName is None: softwareProfile = node.getSoftwareProfile() else: softwareProfile = self.softwareProfileDbApi.\ getSoftwareProfile(softwareProfileName, {'os': True}) return softwareProfile def deletePuppetNodeCert(self, nodeName): # Remove the Puppet certificate when the node is reinstalled self.getLogger().debug('deletePuppetNodeCert(node=[%s])' % (nodeName)) puppetSslDir = '/etc/puppetlabs/puppet/ssl' puppetReportDir = '/var/lib/puppet/reports' puppetYamlDir = '/var/lib/puppet/yaml' filesToRemove = [ os.path.join(puppetSslDir, 'public_keys/%s.pem' % (nodeName)), os.path.join(puppetSslDir, 'ca/signed/%s.pem' % (nodeName)), os.path.join(puppetSslDir, 'private_keys/%s.pem' % (nodeName)), os.path.join(puppetSslDir, 'certs/%s.pem' % (nodeName)), os.path.join(puppetYamlDir, 'node/%s.yaml' % (nodeName)), os.path.join(puppetYamlDir, 'facts/%s.yaml' % (nodeName)), ] for fn in filesToRemove: try: os.unlink(fn) except OSError as exc: if exc.errno != 2: self.getLogger().error( 'Error attempting to remove %s (reason: %s)' % (fn, exc)) fn = os.path.join(puppetReportDir, nodeName) try: shutil.rmtree(fn) except OSError as exc: if exc.errno != 2: self.getLogger().error( 'Error attempting to remove %s (reason: %s)' % (fn, exc)) def nodeCleanup(self, nodeName): """ Remove files related to the node """ # Remove 'private' directory private_dir = os.path.join(self._cm.getRoot(), 'private', nodeName) if os.path.exists(private_dir): shutil.rmtree(private_dir) def addDhcpLease(self, node, nic): # Add DHCP lease to DHCP server pass def removeDhcpLease(self, nodeName): # Remove the DHCP lease from the DHCP server. This will be # a no-op on any platform that doesn't support the operation # (ie. any platform not running ISC DHCPD) pass def setNodeForNetworkBoot(self, dbNode): # Update node status to "Expired" and boot from network dbNode.state = 'Expired' dbNode.bootFrom = 0 self.deletePuppetNodeCert(dbNode.name) # Write the updated file self.writePXEFile(dbNode)
class TortugaCli(object): """ Base tortuga command line interface class. """ def __init__(self, validArgCount=0): self._logger = logging.getLogger('tortuga.cli.%s' % (self.__class__.__name__)) self._logger.addHandler(logging.NullHandler()) self._parser = OptionParser(add_help_option=False) self._options = None self._args = [] self._validArgCount = validArgCount self._username = None self._password = None self._optionGroupDict = {} self._cm = ConfigManager() self.__initializeLocale() commonGroup = _('Common Tortuga Options') self.addOptionGroup(commonGroup, None) self.addOptionToGroup(commonGroup, '-h', '--help', action='help', help=_('show this help message and exit')) self.addOptionToGroup(commonGroup, '-?', '', action='help', help=_('show this help message and exit')) self.addOptionToGroup(commonGroup, '-V', '', action='store_true', dest='cmdVersion', default=False, help=_('print version and exit')) self.addOptionToGroup( commonGroup, '-d', '--debug', dest='consoleLogLevel', help=_('set debug level; valid values are: critical, error,' ' warning, info, debug')) self.addOptionToGroup( commonGroup, '--username', dest='username', help=_('Credential to use when not running as root on the' ' installer.')) self.addOptionToGroup( commonGroup, '--password', dest='password', help=_('Credential to use when not running as root on the' ' installer.')) def getLogger(self): """ Get logger for this class. """ return self._logger def __initializeLocale(self): """Initialize the gettext domain """ langdomain = 'tortugaStrings' # Locate the Internationalization stuff localedir = '../share/locale' \ if os.path.exists('../share/locale') else \ os.path.join(self._cm.getRoot(), 'share/locale') gettext.install(langdomain, localedir) def getParser(self): """ Get parser for this class. """ return self._parser def addOption(self, *args, **kwargs): """ Add option. """ self._parser.add_option(*args, **kwargs) def addOptionToGroup(self, groupName, *args, **kwargs): """ Add option for the given group name. Group should be created using addOptionGroup(). """ group = self._optionGroupDict.get(groupName) group.add_option(*args, **kwargs) def addOptionGroup(self, groupName, desc): """ Add option group. """ group = OptionGroup(self._parser, groupName, desc) self._parser.add_option_group(group) self._optionGroupDict[groupName] = group def parseArgs(self, usage=None): """ Parse args Raises: InvalidArgument """ if usage: self._parser.usage = usage try: self._options, self._args = self._parser.parse_args() except SystemExit as rc: sys.stdout.flush() sys.stderr.flush() sys.exit(int(str(rc))) if self._validArgCount < len(self._args): # Postitional args are not enabled and we have some msg = _("Invalid Argument(s):") for arg in self._args[self._validArgCount:]: msg += " " + arg raise InvalidArgument(msg) optDict = self._options.__dict__ if optDict.get('cmdVersion'): print( _('{0} version: {1}'.format(os.path.basename(sys.argv[0]), self._cm.getTortugaRelease()))) sys.exit(0) # Log level. consoleLogLevel = optDict.get('consoleLogLevel', None) if consoleLogLevel: # logManager.setConsoleLogLevel(consoleLogLevel) logger = logging.getLogger('tortuga') logger.setLevel(logging.DEBUG) # create console handler and set level to debug ch = logging.StreamHandler() ch.setLevel(logging.DEBUG) # create formatter formatter = logging.Formatter( '%(asctime)s - %(name)s - %(levelname)s - %(message)s') # add formatter to ch ch.setFormatter(formatter) # add ch to logger logger.addHandler(ch) # Promote options to attributes self._username = self._options.username self._password = self._options.password return self._options, self._args def usage(self, s=None): '''Print the help provided by optparse''' if s: sys.stderr.write(_('Error: {0}').format(s) + '\n') self._parser.print_help() sys.exit(1) def getOptions(self): '''Returns the command line options''' return self._options def getNArgs(self): '''Returns the number of command line arguments''' return len(self._args) def getArgs(self): '''Returns the command line argument list''' return self._args def getArg(self, i): '''Returns the i-th command line argument''' return self._args[i] def getUsername(self): """ Get user name. """ return self._username def getPassword(self): """ Get password. """ return self._password def runCommand(self): \ # pylint: disable=no-self-use """ This method must be implemented by the derived class. """ raise AbstractMethod( _('runCommand() has to be overriden in the derived class.')) def run(self): """ Invoke runCommand() in derivative class and handle exceptions. """ try: self.runCommand() except TortugaException as ex: print('%s' % (ex.getErrorMessage())) raise SystemExit(ex.getErrorCode()) except SystemExit as ex: raise except Exception as ex: print('%s' % (ex)) raise SystemExit(-1) def getParam(self, xtype, options, oname, config, section, cname, default=None): ''' Get the value of a configurable parameter. First look at command line options. Return it if there. Then look in the configFile. Return it if there. Otherwise return the default. ''' value = self.__getParam2(options, oname, config, section, cname, default) if xtype == int: if not value: value = 0 elif type(value) != int: value = int(value) elif xtype == bool: if type(value) == str: value = value.lower() == 'true' elif type(value) == int: value = bool(value) return value def __getParam2(self, options, oname, config, section, cname, default): \ # pylint: disable=no-self-use # Command line option takes precedence if options and oname in options.__dict__ and \ options.__dict__[oname] is not None: return options.__dict__[oname] # Config file is next if config and config.has_section(section) and \ config.has_option(section, cname): return config.get(section, cname) # Last resort return default def _parseDiskSize(self, diskSizeParam): \ # pylint: disable=no-self-use """ Parses diskSizeParam, returns an int value representing number of megabytes Raises: ValueError """ if diskSizeParam.endswith('TB'): return int(float(diskSizeParam[:-2]) * 1000000) if diskSizeParam.endswith('GB'): return int(float(diskSizeParam[:-2]) * 1000) elif diskSizeParam.endswith('MB'): # Must be an integer return int(diskSizeParam[:-2]) return int(diskSizeParam) def _getDiskSizeDisplayStr(self, volSize): \ # pylint: disable=no-self-use if volSize < 1000: result = '%s MB' % (volSize) elif volSize < 1000000: result = '%.3f GB' % (float(volSize) / 1000) else: result = '%.3f TB' % (float(volSize) / 1000000) return result
class TortugaProxyConfig(TortugaCli): def __init__(self): super(TortugaProxyConfig, self).__init__(validArgCount=4) self._cm = ConfigManager() self._kitApi = KitApi() def parseArgs(self, usage=None): # TODO: add stuff here self.addOption('-f', '--force', action='store_true', default=False, dest='bForce', help='Override built-in sanity checks') self.addOption('-n', '--dry-run', action='store_true', dest='bDryRun', default=False, help='Do not write anything to disk') super().parseArgs(usage=usage) def runCommand(self): self.parseArgs() if self.getNArgs() < 1: self.usage() return action = self.getArgs()[0] if action == 'list': self._listProxies() elif action == 'add': if self.getNArgs() != 3: self.usage() return self._addProxy() elif action == 'delete': self._deleteProxy() else: raise InvalidArgument('Unknown directive [%s]' % (action)) def _getProxyCfg(self): \ # pylint: disable=no-self-use cfg = configparser.ConfigParser() cfg.read('/opt/tortuga/config/base/apache-component.conf') return cfg def __get_proxy_set(self, cfg): \ # pylint: disable=no-self-use return set(cfg.get('proxy', 'proxy_list').split(' ')) \ if cfg.has_option('proxy', 'proxy_list') else set() def _getProxyMap(self, cfg): proxyMap = {} if not cfg.has_section('proxy'): return proxyMap proxy_option_list = self.__get_proxy_set(cfg) for opt in proxy_option_list: if not cfg.has_option('proxy', opt): continue proxyMap[opt] = cfg.get('proxy', opt) return proxyMap def _writeProxyMap(self, cfg, proxyMap): if not cfg.has_section('proxy'): cfg.add_section('proxy') # Determine differences between what exists on disk and what has # just been removed. for deleted_option in self.__get_proxy_set(cfg) - set(proxyMap.keys()): if not cfg.has_option('proxy', deleted_option): continue cfg.remove_option('proxy', deleted_option) cfg.set('proxy', 'proxy_list', ' '.join(list(proxyMap.keys()))) for key, value in proxyMap.items(): cfg.set('proxy', key, value) proxyDict = dict(cfg.items('proxy')) if self.getArgs().bDryRun: print('[dryrun] %s' % (pprint.pformat(proxyDict))) return with open('/opt/tortuga/config/base/apache-component.conf', 'w') as fp: cfg.write(fp) def _addProxy(self): proxy_from = self.getArgs()[1] proxy_to = self.getArgs()[2] cfg = self._getProxyCfg() proxyMap = self._getProxyMap(cfg) if proxy_from in proxyMap: if proxy_to == proxyMap[proxy_from]: print('Proxy already mapped') sys.exit(1) if not self.getArgs().bForce: print('URI [%s] is already proxied to [%s]' % (proxy_from, proxyMap[proxy_from])) sys.exit(1) proxyMap[proxy_from] = proxy_to self._writeProxyMap(cfg, proxyMap) def __find_kit_by_name_and_version(self, os_name, os_version): """ Iterate over list of all installed kits looking for a name and version match only. Returns Kit object, otherwise None. """ kit = None for kit in self._kitApi.getKitList(): if kit.getName() == os_name and \ kit.getVersion() == os_version: break else: return None return kit def __get_existing_kit_by_url(self, proxy_uri): """ Given a proxy URI, determine if the path matches that of an installed kit. Returns Kit object or None. """ uri_parts = proxy_uri.split('/') if len(uri_parts) != 5: # Short-circuit any check if the URI is longer/shorter than # a properly formatted Tortuga kit URL. return None # Check if this URI is formatted like valid OS kit URL os_name = uri_parts[2] os_version = uri_parts[3] os_arch = uri_parts[4] fake_url = self._cm.getYumRootUrl( 'INSTALLER') + '/%s/%s/%s' % (os_name, os_version, os_arch) o = urllib.parse.urlparse(fake_url) if o.path != proxy_uri: # The paths don't start with the requisite Tortuga path return None version = os_version.split('-') # Check if supplied 'version' element of the path matches the # Tortuga convention. if len(version) == 1: # Possibly an OS kit # bOsKit = True if os_arch == 'x86_64' else None pass elif len(version) == 2: # Possibly a non-OS kit. Non-OS kits must have the 'arch' # set to 'noarch' # bOsKit = False if os_arch == 'noarch' else None pass else: # version element doesn't match Tortuga format return None return self.__find_kit_by_name_and_version(os_name, os_version) def _deleteProxy(self): proxy_path = self.getArgs()[1] cfg = self._getProxyCfg() proxyMap = self._getProxyMap(cfg) if proxy_path in proxyMap: existingKit = self.__get_existing_kit_by_url(proxy_path) if existingKit and not self.getArgs().bForce: print('WARNING: an installed %s [%s] matches this URL.' ' Use \'--force\' to override this sanity check.' % ('OS kit' if existingKit.getIsOs() else 'kit', existingKit)) sys.exit(1) if proxy_path not in proxyMap: print('Error: proxy path [%s] not found' % (proxy_path)) sys.exit(1) del proxyMap[proxy_path] self._writeProxyMap(cfg, proxyMap) def _listProxies(self): cfg = self._getProxyCfg() for key, value in self._getProxyMap(cfg).items(): print('%s -> %s' % (key, value))
class DbManager(TortugaObjectManager): """ Class for db management. :param engine: a SQLAlchemy database engine instance :param init: a flag that is set when the database has not yet been initialized. If this flag is set, not attempts will be made to load/map kit tables. This flag is cleared once the database has been initialized. """ def __init__(self, engine=None): super().__init__() if not engine: self._cm = ConfigManager() self._dbConfig = self._refreshDbConfig() engineURI = self.__getDbEngineURI() if self._dbConfig['engine'] == 'sqlite' and \ not os.path.exists(self._dbConfig['path']): # Ensure SQLite database file is created with proper permissions fd = os.open( self._dbConfig['path'], os.O_CREAT, mode=0o600) os.close(fd) self._engine = sqlalchemy.create_engine(engineURI) else: self._engine = engine self.Session = sqlalchemy.orm.scoped_session( sqlalchemy.orm.sessionmaker(bind=self.engine)) def _register_database_tables(self): for kit_installer_class in get_all_kit_installers(): kit_installer = kit_installer_class() kit_installer.register_database_tables() @property def engine(self): """ SQLAlchemy Engine object property """ self._register_database_tables() return self._engine def session(self): """ Database session context manager """ return SessionContextManager(self) def init_database(self): # # Create tables # self._register_database_tables() try: ModelBase.metadata.create_all(self.engine) except Exception: self._logger.exception('SQLAlchemy raised exception') raise DbError('Check database settings or credentials') @property def metadata(self): return self._metadata def __getDbEngineURI(self): dbPort = self._dbConfig['port'] dbHost = self._dbConfig['host'] engine = self._dbConfig['engine'] dbUser = self._dbConfig['username'] dbPassword = self._dbConfig['password'] if engine == 'sqlite': engineURI = 'sqlite:///%s' % (self._dbConfig['path']) else: if dbUser is not None: if dbPassword is not None: userspec = '%s:%s' % (dbUser, dbPassword) else: userspec = dbUser else: userspec = None if dbPort is not None: hostspec = '%s:%s' % (dbHost, dbPort) else: hostspec = dbHost engineURI = f'{engine}+pymysql' if engine == 'mysql' else engine engineURI += '://' if userspec is not None: engineURI += f'{userspec}@' engineURI += f'{hostspec}' + '/{}'.format(self._cm.getDbSchema()) return engineURI def _getDefaultDbEngine(self): \ # pylint: disable=no-self-use return 'sqlite' def _getDefaultDbHost(self): \ # pylint: disable=no-self-use return 'localhost' def _getDefaultDbPort(self, engine): \ # pylint: disable=no-self-use # MySQL default port if engine == 'mysql': return 3306 return None def _getDefaultDbUserName(self): return self._cm.getDbUser() def _getDefaultDbPassword(self): if os.path.exists(self._cm.getDbPasswordFile()): with open(self._cm.getDbPasswordFile()) as fp: dbPassword = fp.read() else: dbPassword = None return dbPassword def _refreshDbConfig(self, cfg=None): dbConfig = {} if cfg is None: cfg = configparser.ConfigParser() cfg.read(os.path.join(self._cm.getKitConfigBase(), 'tortuga.ini')) # Database engine val = cfg.get('database', 'engine').strip().lower() \ if cfg.has_option('database', 'engine') else \ self._getDefaultDbEngine() dbConfig['engine'] = val if dbConfig['engine'] == 'sqlite': # If database is sqlite, read the path dbConfig['path'] = cfg.get('database', 'path') \ if cfg.has_section('database') and \ cfg.has_option('database', 'path') else \ os.path.join(self._cm.getEtcDir(), self._cm.getDbSchema() + '.sqlite') # Database host val = cfg.get('database', 'host') \ if cfg.has_option('database', 'host') else \ self._getDefaultDbHost() dbConfig['host'] = val # Database port val = cfg.get('database', 'port') \ if cfg.has_option('database', 'port') else None dbConfig['port'] = val if val else self._getDefaultDbPort( engine=dbConfig['engine']) # Database username val = cfg.get('database', 'username') \ if cfg.has_option('database', 'username') \ else self._getDefaultDbUserName() dbConfig['username'] = val # Database password val = cfg.get('database', 'password') \ if cfg.has_option('database', 'password') \ else self._getDefaultDbPassword() dbConfig['password'] = val return dbConfig def get_backend_opts(self): \ # pylint: disable=no-self-use return { 'mysql_engine': 'InnoDB', } def getMetadataTable(self, table): return self._metadata.tables[table] def openSession(self): """ Open db session. """ return self.Session() def closeSession(self): """Close scoped_session.""" self.Session.remove()
def __init__(self): super(TortugaProxyConfig, self).__init__(validArgCount=4) self._cm = ConfigManager() self._kitApi = KitApi()
def __getInstallerNode(self, session): return self._nodesDbHandler.getNode(session, ConfigManager().getInstaller())
def get_puppet_node_yaml(session, nodeName): _cm = ConfigManager() publicInstallerFQDN = _cm.getInstaller().lower() primaryInstallerHostName = publicInstallerFQDN.split('.', 1)[0] try: dnsZone = GlobalParametersDbHandler().getParameter( session, 'DNSZone').value.lower() except ParameterNotFound: dnsZone = None try: depot_path = GlobalParametersDbHandler().getParameter( session, 'depot').value.lower() _cm.setDepotDir(depot_path) except ParameterNotFound: pass bInstaller = primaryInstallerHostName == nodeName.split('.', 1)[0] try: dbNode = NodesDbHandler().getNode(session, nodeName) except NodeNotFound: sys.exit(1) data = None try: from tortuga.db.dataRequestsDbHandler import DataRequestsDbHandler dbDataRequest = DataRequestsDbHandler().get_by_addHostSession( session, dbNode.addHostSession) if dbDataRequest: data = dbDataRequest.request except Exception as e: pass if dbNode.hardwareprofile.nics: privateInstallerFQDN = '%s%s%s' % (primaryInstallerHostName, get_installer_hostname_suffix( dbNode.hardwareprofile.nics[0], enable_interface_aliases=None), '.%s' % (dnsZone) if dnsZone else '') else: privateInstallerFQDN = '%s%s' % (primaryInstallerHostName, '.%s' % (dnsZone) if dnsZone else '') if not bInstaller and dbNode.hardwareprofile.location == 'local': # If the hardware profile does not have an associated provisioning # NIC, use the public installer FQDN by default. This can happen if # the user has added their own "public" nodes to a local hardware # profile. if not dbNode.hardwareprofile.nics: installerHostName = publicInstallerFQDN else: installerHostName = privateInstallerFQDN else: # If the specified node is the installer itself or a node # accessing the installer through it's public interface, use the # public host name. installerHostName = publicInstallerFQDN puppet_classes = {} enabledKits = set() if dbNode.softwareprofile: for dbComponent in dbNode.softwareprofile.components: if not dbComponent.kit.isOs: # # Load the kit and component installers # kit_spec = (dbComponent.kit.name, dbComponent.kit.version, dbComponent.kit.iteration) kit_installer = get_kit_installer(kit_spec)() kit_installer.session = session _component = kit_installer.get_component_installer( dbComponent.name) # # Get the puppet args for the component # try: puppet_class_args = _component.run_action( 'get_puppet_args', dbNode.softwareprofile, dbNode.hardwareprofile, data=data) if puppet_class_args is not None: puppet_classes[_component.puppet_class] = \ puppet_class_args except Exception: # noqa pylint: disable=broad-except # suppress exception if unable to get Puppet args puppet_classes[_component.puppet_class] = {} else: # # OS kit component is omitted on installer. The installer # is assumed to have a pre-existing OS repository # configuration. # if bInstaller: continue enabledKits.add(dbComponent.kit) dataDict = {} if puppet_classes: dataDict['classes'] = puppet_classes parametersDict = {} dataDict['parameters'] = parametersDict # software profile if dbNode.softwareprofile: parametersDict['swprofilename'] = dbNode.softwareprofile.name # hardware profile parametersDict['hwprofilename'] = dbNode.hardwareprofile.name # installer hostname parametersDict['primary_installer_hostname'] = installerHostName # Local repos directory repodir = os.path.join(_cm.getDepotDir(), 'kits') # Build YUM repository entries only if we have kits associated with # the software profile. if enabledKits: repourl = _cm.getIntWebRootUrl(installerHostName) + '/repos' \ if not bInstaller else 'file://{0}'.format(repodir) repo_type = None if dbNode.softwareprofile.os.family.name == 'rhel': repo_type = 'yum' # elif dbNode.softwareprofile.os.family == 'ubuntu': # repo_type = 'apt' if repo_type: # Only add 'repos' entries for supported operating system # families. repos_dict = {} for kit in enabledKits: if kit.isOs: verstr = str(kit.version) arch = kit.components[0].os[0].arch else: verstr = '%s-%s' % (kit.version, kit.iteration) arch = 'noarch' for dbKitSource in dbNode.softwareprofile.kitsources: if dbKitSource in kit.sources: baseurl = dbKitSource.url break else: subpath = '%s/%s/%s' % (kit.name, verstr, arch) if not kit.isOs and not os.path.exists( os.path.join(repodir, subpath, 'repodata/repomd.xml')): continue baseurl = '%s/%s' % (repourl, subpath) # [TODO] temporary workaround for handling RHEL media # path. # # This code is duplicated from tortuga.boot.distro if kit.isOs and \ dbNode.softwareprofile.os.name == 'rhel' and \ dbNode.softwareprofile.os.family.version != '7': subpath += '/Server' if repo_type == 'yum': if dbNode.hardwareprofile.location == 'remote': cost = 1200 else: cost = 1000 repos_dict['uc-kit-%s' % (kit.name)] = { 'type': repo_type, 'baseurl': baseurl, 'cost': cost, } if repos_dict: parametersDict['repos'] = repos_dict # Enable '3rdparty' repo if dbNode.softwareprofile: third_party_repo_subpath = '3rdparty/%s/%s/%s' % ( dbNode.softwareprofile.os.family.name, dbNode.softwareprofile.os.family.version, dbNode.softwareprofile.os.arch) local_repos_path = os.path.join(repodir, third_party_repo_subpath) # Check for existence of repository metadata to validate existence if enabledKits and os.path.exists( os.path.join(local_repos_path, 'repodata', 'repomd.xml')): third_party_repo_dict = { 'tortuga-third-party': { 'type': 'yum', 'baseurl': os.path.join(repourl, third_party_repo_subpath), }, } if 'repos' not in parametersDict: parametersDict['repos'] = third_party_repo_dict else: parametersDict['repos'] = dict( list(parametersDict['repos'].items()) + list(third_party_repo_dict.items())) # environment dataDict['environment'] = 'production' sys.stdout.write( yaml.safe_dump(dataDict, default_flow_style=False, explicit_start=True))