def _readFile(self, fileName): """ Read credentials from possibly encrypted credentials file """ credsFile = fileName creds = None if self._unencrypted: creds = super()._readFile(credsFile) else: if not pathlib.Path(credsFile).is_file(): logging.info(self._noFileMsg) else: credsRead = False while not credsRead: # Repeat until read and decrypt were successful # (i.e. until GPG agent responded or correct password was supplied) try: with open(credsFile, 'rb') as credsFh: credsDec = self._gpg.decrypt_file( credsFh, passphrase=self._passphrase) credsRead = credsDec.ok except IOError: fail(f"Error reading from {credsFile}") self._setRecipient(credsDec) creds = str(credsDec) # logging.debug(f'creds >>>{creds}<<<') return creds
def __init__(self, ctx, hname, user): self._userFull = f'{user.name}@{hname}' self._authKeys = None # Prepare command execution self._cmdShell = CmdShell() self._cmdSsh = CmdSsh(ctx, hname, user) self._rsyncSsh, self._rsyncSshSecrets = self._cmdSsh.getSshCmdAndSecrets( withLogin=False) # Get the path to the remote authorized_keys file homeDir = getHomeDir(ctx, hname, user) if not homeDir: fail( f"Could not determine the home directory of '{self._userFull}'" ) self._akPath = f'{homeDir}/.ssh/authorized_keys' # Read the authorized_keys file info = _PublicKeyInformation(hname, user, self._akPath) self._read(info) # Sets self._authKeys logging.debug( f'self._authKeys >>>\n{self._authKeys}\n<<< self._authKeys')
def getContext(args, withCreds=True, withConfig=True, failOnDiscoveryError=True): """ Get context """ setupLogging(args) ctx = types.SimpleNamespace() ctx.ar = args ctx.cr = None ctx.cf = None ctx.cs = getConstants() if withCreds: ctx.creds = Creds(ctx) ctx.cr = ctx.creds.get() if withConfig: ctx.config = Config(ctx, failOnDiscoveryError=failOnDiscoveryError) ctx.cf = ctx.config.getFull() elif withConfig: fail("Can't get configuration without credentials") if args.dump_context: _printHeader('COMMAND LINE ARGUMENTS') print(yaml.dump({'ar': nestedNsToObj(ctx.ar)})) _printHeader('CREDENTIALS') print(yaml.dump({'cr': nestedNsToObj(ctx.cr)})) _printHeader('CONFIGURATION') print(yaml.dump({'cf': nestedNsToObj(ctx.cf)})) sys.exit(0) return ctx
def getValidNfsServerAddress(ctx): """ Get valid NFS server address serving the OCP cluster net """ # Get all IP addresses for NFS server allIps = getAllNfsServerIpAddresses(ctx) # Need an OC login to get worker node address ocp = Ocp(ctx, login="******", verify=True) worker = ocp.getWorkerNodeList()[0] # Run Python script tools/modules/nfs-ping-test on helper node (in-line) cmdShell = CmdShell() host = ctx.cf.ocp.helper.host.name user = ctx.cr.ocp.helper.user repo = ctx.cf.build.repo.root toolCmd = f'python3 - <{repo}/tools/modules/nfs-ping-test {worker} {allIps}' cmdSsh = CmdSsh(ctx, host, user, reuseCon=False) runCmd, secr = cmdSsh.getSshCmdAndSecrets(withLogin=True) ipAddr = cmdShell.run(f'{runCmd} {toolCmd}', secrets=secr).out logging.debug( f"Running shell cmd: '{runCmd} {toolCmd}' returns '{ipAddr}'") if ipAddr == 'None': message = "Could not identify valid IP address for the NFS server" message += f"on worker node {worker}.\n" fail(message) del ocp return ipAddr
def _writeFile(self, fileName, contents): try: # pylint: disable=unspecified-encoding with open(fileName, 'w') as wFh: wFh.write(contents) except IOError: fail(f"Error writing to file {fileName}")
def _discoverAndCache(self, configCacheFile, configCacheTimeout, failOnDiscoveryError): logging.debug('Running configuration discovery') # self._checkRequiredOptional() try: self._discover() self._config['expiryTime'] = str(self._getCurrentTime() + configCacheTimeout) logging.debug(f"Writing config to cache file '{configCacheFile}'") # pylint: disable=unspecified-encoding with open(configCacheFile, 'w') as ccfh: yaml.dump(self._config, stream=ccfh) except _DiscoveryError as derr: message = '\nThe following problem occured during configuration discovery:\n\n' message += f"{'-'*65}\n" message += f'{derr}\n' message += f"{'-'*65}\n" if failOnDiscoveryError: fail(message) else: message += '\nThe configuration cache file was not written. Proceeding with\n' message += 'possibly incomplete configuration. This may lead to runtime\n' message += 'errors. Check your configuration file\n\n' message += f" {self._instanceFile }\n\n" message += 'and correct the error.\n\n' warn(message)
def _genBuildContextFlavor(self, sidU, dirs, sapadm, sidadm, sapsysGid, host, filterFilePath): # Flavor dependent actions for build context generation # pylint: disable=unused-argument,too-many-arguments fail( 'This function must be overwritten by derived flavor specific builder class.' )
def genFileFromTemplate(templatePath, outFilePath, params): """ Generate file from template replacing parameters enclosed in "{{...}} """ content = instantiateTemplate(templatePath, params) # pylint: disable=invalid-name, unspecified-encoding try: with open(outFilePath, 'w') as fh: print(content, file=fh) except IOError: fail(f"Error writing to file {outFilePath}")
def create(self, hideDescriptions): """ Create new configuration from template """ try: # pylint: disable=unspecified-encoding with open(self._templateFile, 'r') as tmplFh: self._instance = yaml.load(tmplFh.read(), Loader=yaml.Loader) except IOError: fail(f"Error reading from file {self._templateFile}") self.edit(hideDescriptions)
def getOverlayUuid(ctx, overlayUuid): """ Get overlay UUID from parameter or cli argument. Exit, if no overlay UUID can be determined """ if not overlayUuid: if ctx.ar.overlay_uuid: overlayUuid = ctx.ar.overlay_uuid else: fail("Please specify an overlay share UUID via option '-u'") return Overlays(ctx).find(overlayUuid).uuid
def _runCmd(cmd): result = CmdShell().run(cmd) if result.rc != 0: msg = '' msg += f"Command '{cmd}' failed" msg += f'\n stdout: >>>{result.out}<<<' msg += f'\n stderr: >>>{result.err}<<<' msg += f'\n rc: {result.rc}' fail(msg) return result
def _genContainerfile(self, sidU, dirs, image, sapadm, sidadm, sapsysGid): # Generate containerfile from template depending on flavor # MUST RUN AFTER BUILD CONTEXT SETUP # pylint: disable=too-many-arguments logging.info("##### Generating Containerfile #####") sidL = sidU.lower() # Common parameters if dirs.usrSapReal != '/usr/sap': usrSapLinkCmd = f'ln -s {dirs.usrSapReal} /usr/sap' else: usrSapLinkCmd = 'true' # get optional packages packages = getattr(self._ctx.cf.images, self._flavor).packages pkgParams = self._getOptionalPackageParams(packages, dirs) params = { 'IMAGE_BRANCH': image.branch, 'IMAGE_COMMIT': image.commit, 'IMAGE_DATE': image.date, 'IMAGE_DESCRIPTION': image.description, 'IMAGE_VERSION': image.version, 'SAPADM_COMMENT': sapadm.comment, 'SAPADM_HOME': sapadm.home, 'SAPADM_SHELL': sapadm.shell, 'SAPADM_UID': sapadm.uid, 'SAPMNT': dirs.sapmnt, 'SAPSYS_GID': sapsysGid, 'sid': sidL, 'SID': sidU, 'SIDADM_COMMENT': sidadm.comment, 'SIDADM_HOME': sidadm.home, 'SIDADM_SHELL': sidadm.shell, 'SIDADM_UID': sidadm.uid, 'USR_SAP_REAL': dirs.usrSapReal, 'USR_SAP_LINK_CMD': usrSapLinkCmd, 'INSTALL_OPT_PACKAGES': pkgParams.installOptPackagesDnf, 'COPY_OPT_PACKAGE_FILES': pkgParams.copyOptPackageFiles, 'INSTALL_OPT_PACKAGE_FILES': pkgParams.installOptPackageFiles } params.update(self._getContainerfileParams(sidU, dirs)) containerfile = f'{dirs.tmp}/containerfile' template = f'{dirs.repoRoot}/openshift/images/{self._flavor}/containerfile.template' genFileFromTemplate(template, containerfile, params) try: # pylint: disable=invalid-name, unspecified-encoding with open(containerfile) as fh: logging.debug( f"Contents of '{containerfile}': >>>\n{fh.read()}<<<") except IOError: fail(f"Error reading from {containerfile}") return containerfile
def _read(self, info): """ Read the contents of the authorized_keys file """ res = self._cmdSsh.run(f'cat {self._akPath}') if res.rc != 0: fail( f"Could not get the authorized keys '{self._akPath}' file of '{self._userFull}'" ) self._authKeys = _PublicKeys(res.out, info, keepAll=True)
def podmanOcpRegistryLogin(self): """ Log into the default registry of an OpenShift cluster """ out = CmdShell().run( 'podman login' ' --tls-verify=false' ' -u $(oc whoami) -p $(oc whoami --show-token)' f' default-route-openshift-image-registry.apps.{self._domain}').out if not out.startswith('Login Succeeded!'): fail('podman login failed')
def getBuilder(ctx): # pylint: disable=inconsistent-return-statements """ Get image builder for specific image flavor """ if ctx.ar.image_flavor == 'nws4': return BuilderNws4(ctx) if ctx.ar.image_flavor == 'hdb': return BuilderHdb(ctx) # This should never be reached fail(f"Unknown image flavor '{ctx.ar.image_flavor}'")
def getExecPath(execName, doFail=True): """ Get path of executable """ execPath = shutil.which(execName) if not execPath: if doFail: fail(f"Did not find executable '{execName}'") else: logging.warning(f"Did not find executable '{execName}'") else: logging.info(f"Using '{execPath}' as '{execName}'") return execPath
def instantiateTemplate(templatePath, params): """ Instantiate a template file replacing parameters enclosed in "{{...}} """ # pylint: disable=invalid-name, unspecified-encoding try: with open(templatePath, 'r') as fh: content = fh.read() for (k, v) in params.items(): content = content.replace('{{'+k+'}}', v) except IOError: fail(f"Error reading from file {templatePath}") return content
def _readFile(self, fileName): contents = None if not pathlib.Path(fileName).is_file(): logging.info(f"File '{fileName}' does not exist") else: try: # pylint: disable=unspecified-encoding with open(fileName, 'r') as ctFh: contents = ctFh.read() except IOError: fail(f"Error reading from file {fileName}") return contents
def start(self): """ start a deployment """ deploymentFile = self._ctx.ar.deployment_file if not deploymentFile: self._appName = self._deployments.getValidAppName() # check if deployment is already started if self._deployments.isDeployed(self._appName): fail( f"Deployment with app name '{self._appName}' already started." ) # start the deployment deploymentFile = self._deployments.getDeploymentFile(self._appName) startDeployment(self._ctx, deploymentFile=deploymentFile)
def _genBuildContext(self, sidU, dirs, sapadm, sidadm, sapsysGid, host, remoteOs): # Generate podman build context # pylint: disable=too-many-arguments filterFilePath = f'{dirs.tmp}/rsync-filter' logging.debug(f"filterFilePath: {filterFilePath}") try: # pylint: disable=invalid-name, unspecified-encoding with open(filterFilePath, 'w') as fh: print(self._getRsyncFilter(sidU, dirs, remoteOs), file=fh) except IOError: fail(f"Error writing to file {filterFilePath}") self._genBuildContextFlavor(sidU, dirs, sapadm, sidadm, sapsysGid, host, filterFilePath)
def containerLogin(self): """ Login to a container """ podName = self.getPodName() if not podName: fail( "Cannot get the name of the pod - check if the pod is started." ) containerName = self.getContainerName(self._ctx.ar.container_flavor) self._printSwitchUserMsg(switchLogin=False) print(f"Logging into container '{containerName}' of pod '{podName}'", file=sys.stderr) os.execlp('oc', 'oc', 'exec', '-it', podName, '-c', containerName, '--', 'bash')
def verify(self): """ Verify various configuration settings """ success, msg = self._checkSsh() if not success: fail(msg) showMsgOk("SSH key setup is valid.") if self._ctx.ar.function in self._functions: return self._functions[self._ctx.ar.function]() success = self._verifyOcp() and success success = self._verifyImages() and success success = self._verifyNws4() and success success = self._verifyHdb() and success success = self._verifyNfs() and success success = self._verifySapSystem() and success return success
def genYaml(self, overlayUuid): """ generate the deployment description file """ deployment = Deployment(self._ctx, overlayUuid=overlayUuid).get() if not ocpMemoryResourcesValid(self._ctx): fail("Fatal error. Stopping the deployment.") parms = getParmsForDeploymentYamlFile(self._ctx, deployment) templatePath = f'{self._ctx.cf.build.repo.root}/openshift/' serviceTemplate = f'{templatePath}/service-nodeport.yaml.template' deploymentTemplate = f'{templatePath}/deployment.yaml.template' serviceYamlPart = instantiateYamlTemplate(serviceTemplate, parms) deploymentYamlPart = instantiateYamlTemplate(deploymentTemplate, parms) if refSystemIsStandard(self._ctx): # If the reference system is a standard system no OCP secret definition # for the HDB connect user is required # -> # Remove all OCP secret definition related environment variables # to avoid problems at deployment time in case no OCP secret was defined delEnvVars = ('SOOS_DI_DBUSER', 'SOOS_DI_DBUSERPWD') initContSpec = deploymentYamlPart['spec']['template']['spec'][ 'initContainers'][0] initContSpec['env'] = [ e for e in initContSpec['env'] if e['name'] not in delEnvVars ] # Write deployment file try: # pylint: disable=unspecified-encoding with open(deployment.file, 'w') as oFh: print(yaml.dump(serviceYamlPart), file=oFh, end='') print('---', file=oFh) print(yaml.dump(deploymentYamlPart), file=oFh, end='') print(deployment.file) except IOError: fail(f"Error writing to file {deployment.file}")
def find(self, uuidPrefix): """ Find an existing overlay which matches a given UUID prefix """ found = [ ovl for ovl in self._overlays if ovl.uuid.startswith(uuidPrefix) ] if len(found) < 1: fail( f"Found no matching overlay uuid for uuid prefix '{uuidPrefix}'" ) elif len(found) > 1: msg = f"Overlay uuid prefix '{uuidPrefix}' matches more than one uuid:\n" for ovl in found: msg += f' {ovl}\n' fail(msg) return found[0]
def _writeFile(self, fileName, contents): # See also https://pythonhosted.org/python-gnupg/#encryption credsFile = fileName creds = contents if self._unencrypted: super()._writeFile(credsFile, contents) else: if self._recipient: # Recipient specified -> encrypt for recipient using asymmetric encryption print(f"Encrypting for recipient '{self._recipient}'", file=sys.stderr) credsEnc = self._gpg.encrypt(creds, recipients=[self._recipient], passphrase=self._passphrase) else: # No recipient specified -> use symmetric AES256 encryption print( 'No recipient specified - using symmetric AES256 encryption', file=sys.stderr) credsEnc = self._gpg.encrypt(creds, symmetric='AES256', recipients=None, passphrase=self._passphrase) if not credsEnc.ok: # pylint: disable=no-member fail( f"Encryption failed\n" f" Status: '{credsEnc.status}'\n Stderr: '{credsEnc.stderr}'" ) try: # pylint: disable=unspecified-encoding with open(credsFile, 'w') as credsFh: credsFh.write(str(credsEnc)) except IOError: fail(f"Error writing to {credsFile}")
def __init__(self, ctx, login="******", verify=False, setProject=True, logout=True, appName=None): self._ctx = ctx self._ocp = ctx.cf.ocp self._domain = ctx.cf.ocp.domain self._admin = ctx.cr.ocp.admin self._user = ctx.cr.ocp.user self._project = ctx.cf.ocp.project self._appName = appName self._verify = {"domain": True, "creds": True, "project": True} self._logout = logout self._loginuser = self._user if login == "admin": self._loginuser = self._admin self._orgUser, self._userSwitch = self._setOrgUser() self._result = self.ocLogin(self._loginuser) if self._result.rc > 0: if not verify: fail(self._result.err, exitCode=4) else: self._verify["domain"] = self._isDomainValid(self._result) self._verify["creds"] = self._isCredentialsValid(self._result) # Change to project result = self.setProject() if result.rc > 0: if not verify: if setProject: fail(result.err) else: warn(f"Could not set project '{self._project}'") else: self._verify["project"] = False
def _getPodProperty(self, propertyName, propertySelector): """ Get a property of the pod in which our current deployment is running """ podProperty = '' if not self._appName: fail("Internal error: appName not set") res = CmdShell().run(f'oc get pods --selector="app={self._appName}"' ' -o template --template "{{range .items}}{{' + propertySelector + '}}{{end}}"') if res.rc == 0: podProperty = res.out.strip() logging.debug(f"Pod property '{propertyName}': '{podProperty}'") else: logging.debug(f"Could not get pod property '{propertyName}'" f" for app '{self._appName}' (reason: {res.err})") return podProperty
def __init__(self, ctx, hostname, user, sshId=None, check=True, reuseCon=True): # pylint: disable=too-many-arguments super().__init__() self._cmdShell = CmdShell() self._sshId = None if not sshId: sshId = ctx.cr.build.user.sshid self._sshId = sshId if sshId: logging.debug(f"Using SSH ID '{sshId}'") else: logging.debug('SSH ID is not set') sshCmd, sshLogin, sshCmdSecrets = CmdSsh._getSshCmdAndSecrets( hostname, user, sshId, reuseCon) self._sshCmd = sshCmd self._sshLogin = sshLogin self._sshCmdSecrets = sshCmdSecrets # Check the connection and create the socket for connection # reuse in case the socket does not exist yet if check: logging.debug(f"Connecting to host '{hostname}'") res = self.run('true') if res.rc != 0: msg = self.formatSshError(res, hostname, user) fail(msg)
def getValidAppName(self): """ returns valid application name """ appNames = self.getAppNames() msg = self._getErrorMsg(appNames) if not self._ctx.ar.app_name: if self._deploymentType == self._ctx.cs.deployAll: return None if len(appNames) == 1: return appNames[0] fail(msg) # parts or full app-name specified as parameter match = None for appName in appNames: if self._ctx.ar.app_name in appName: if match: fail(msg) match = appName if not match: fail(msg) return match
def stop(self, ignore=False): """ Stop a deployment """ deploymentFile = self._ctx.ar.deployment_file if deploymentFile: self._appName = self._deployments.getAppNameFromFile( deploymentFile) else: self._appName = self._deployments.getValidAppName() # check if deployment is already stopped if not ignore: if self._deployments.isNotDeployed(self._appName): fail( f"Deployment with app name '{self._appName}' not deployed." ) # stop the deployment deploymentFile = self._deployments.getDeploymentFile(self._appName) stopDeployment(self._ctx, deploymentFile=deploymentFile) # wait for stopped self._waitForStopped(self._appName)