def _readFile(self, fileName):
""" Read credentials from possibly encrypted credentials file """
credsFile = fileName
creds = None
if self._unencrypted:
creds = super()._readFile(credsFile)
else:
if not pathlib.Path(credsFile).is_file():
logging.info(self._noFileMsg)
else:
credsRead = False
while not credsRead:
# Repeat until read and decrypt were successful
# (i.e. until GPG agent responded or correct password was supplied)
try:
with open(credsFile, 'rb') as credsFh:
credsDec = self._gpg.decrypt_file(
credsFh, passphrase=self._passphrase)
credsRead = credsDec.ok
except IOError:
fail(f"Error reading from {credsFile}")
self._setRecipient(credsDec)
creds = str(credsDec)
# logging.debug(f'creds >>>{creds}<<<')
return creds
def __init__(self, ctx, hname, user):
self._userFull = f'{user.name}@{hname}'
self._authKeys = None
# Prepare command execution
self._cmdShell = CmdShell()
self._cmdSsh = CmdSsh(ctx, hname, user)
self._rsyncSsh, self._rsyncSshSecrets = self._cmdSsh.getSshCmdAndSecrets(
withLogin=False)
# Get the path to the remote authorized_keys file
homeDir = getHomeDir(ctx, hname, user)
if not homeDir:
fail(
f"Could not determine the home directory of '{self._userFull}'"
)
self._akPath = f'{homeDir}/.ssh/authorized_keys'
# Read the authorized_keys file
info = _PublicKeyInformation(hname, user, self._akPath)
self._read(info) # Sets self._authKeys
logging.debug(
f'self._authKeys >>>\n{self._authKeys}\n<<< self._authKeys')
def getContext(args,
withCreds=True,
withConfig=True,
failOnDiscoveryError=True):
""" Get context """
setupLogging(args)
ctx = types.SimpleNamespace()
ctx.ar = args
ctx.cr = None
ctx.cf = None
ctx.cs = getConstants()
if withCreds:
ctx.creds = Creds(ctx)
ctx.cr = ctx.creds.get()
if withConfig:
ctx.config = Config(ctx, failOnDiscoveryError=failOnDiscoveryError)
ctx.cf = ctx.config.getFull()
elif withConfig:
fail("Can't get configuration without credentials")
if args.dump_context:
_printHeader('COMMAND LINE ARGUMENTS')
print(yaml.dump({'ar': nestedNsToObj(ctx.ar)}))
_printHeader('CREDENTIALS')
print(yaml.dump({'cr': nestedNsToObj(ctx.cr)}))
_printHeader('CONFIGURATION')
print(yaml.dump({'cf': nestedNsToObj(ctx.cf)}))
sys.exit(0)
return ctx
def getValidNfsServerAddress(ctx):
""" Get valid NFS server address serving the OCP cluster net """
# Get all IP addresses for NFS server
allIps = getAllNfsServerIpAddresses(ctx)
# Need an OC login to get worker node address
ocp = Ocp(ctx, login="******", verify=True)
worker = ocp.getWorkerNodeList()[0]
# Run Python script tools/modules/nfs-ping-test on helper node (in-line)
cmdShell = CmdShell()
host = ctx.cf.ocp.helper.host.name
user = ctx.cr.ocp.helper.user
repo = ctx.cf.build.repo.root
toolCmd = f'python3 - <{repo}/tools/modules/nfs-ping-test {worker} {allIps}'
cmdSsh = CmdSsh(ctx, host, user, reuseCon=False)
runCmd, secr = cmdSsh.getSshCmdAndSecrets(withLogin=True)
ipAddr = cmdShell.run(f'{runCmd} {toolCmd}', secrets=secr).out
logging.debug(
f"Running shell cmd: '{runCmd} {toolCmd}' returns '{ipAddr}'")
if ipAddr == 'None':
message = "Could not identify valid IP address for the NFS server"
message += f"on worker node {worker}.\n"
fail(message)
del ocp
return ipAddr
def _writeFile(self, fileName, contents):
try:
# pylint: disable=unspecified-encoding
with open(fileName, 'w') as wFh:
wFh.write(contents)
except IOError:
fail(f"Error writing to file {fileName}")
def _discoverAndCache(self, configCacheFile, configCacheTimeout,
failOnDiscoveryError):
logging.debug('Running configuration discovery')
# self._checkRequiredOptional()
try:
self._discover()
self._config['expiryTime'] = str(self._getCurrentTime() +
configCacheTimeout)
logging.debug(f"Writing config to cache file '{configCacheFile}'")
# pylint: disable=unspecified-encoding
with open(configCacheFile, 'w') as ccfh:
yaml.dump(self._config, stream=ccfh)
except _DiscoveryError as derr:
message = '\nThe following problem occured during configuration discovery:\n\n'
message += f"{'-'*65}\n"
message += f'{derr}\n'
message += f"{'-'*65}\n"
if failOnDiscoveryError:
fail(message)
else:
message += '\nThe configuration cache file was not written. Proceeding with\n'
message += 'possibly incomplete configuration. This may lead to runtime\n'
message += 'errors. Check your configuration file\n\n'
message += f" {self._instanceFile }\n\n"
message += 'and correct the error.\n\n'
warn(message)
def _genBuildContextFlavor(self, sidU, dirs, sapadm, sidadm, sapsysGid,
host, filterFilePath):
# Flavor dependent actions for build context generation
# pylint: disable=unused-argument,too-many-arguments
fail(
'This function must be overwritten by derived flavor specific builder class.'
)
def genFileFromTemplate(templatePath, outFilePath, params):
""" Generate file from template replacing parameters enclosed in "{{...}} """
content = instantiateTemplate(templatePath, params)
# pylint: disable=invalid-name, unspecified-encoding
try:
with open(outFilePath, 'w') as fh:
print(content, file=fh)
except IOError:
fail(f"Error writing to file {outFilePath}")
def create(self, hideDescriptions):
""" Create new configuration from template """
try:
# pylint: disable=unspecified-encoding
with open(self._templateFile, 'r') as tmplFh:
self._instance = yaml.load(tmplFh.read(), Loader=yaml.Loader)
except IOError:
fail(f"Error reading from file {self._templateFile}")
self.edit(hideDescriptions)
def getOverlayUuid(ctx, overlayUuid):
""" Get overlay UUID from parameter or cli argument.
Exit, if no overlay UUID can be determined """
if not overlayUuid:
if ctx.ar.overlay_uuid:
overlayUuid = ctx.ar.overlay_uuid
else:
fail("Please specify an overlay share UUID via option '-u'")
return Overlays(ctx).find(overlayUuid).uuid
def _runCmd(cmd):
result = CmdShell().run(cmd)
if result.rc != 0:
msg = ''
msg += f"Command '{cmd}' failed"
msg += f'\n stdout: >>>{result.out}<<<'
msg += f'\n stderr: >>>{result.err}<<<'
msg += f'\n rc: {result.rc}'
fail(msg)
return result
def _genContainerfile(self, sidU, dirs, image, sapadm, sidadm, sapsysGid):
# Generate containerfile from template depending on flavor
# MUST RUN AFTER BUILD CONTEXT SETUP
# pylint: disable=too-many-arguments
logging.info("##### Generating Containerfile #####")
sidL = sidU.lower()
# Common parameters
if dirs.usrSapReal != '/usr/sap':
usrSapLinkCmd = f'ln -s {dirs.usrSapReal} /usr/sap'
else:
usrSapLinkCmd = 'true'
# get optional packages
packages = getattr(self._ctx.cf.images, self._flavor).packages
pkgParams = self._getOptionalPackageParams(packages, dirs)
params = {
'IMAGE_BRANCH': image.branch,
'IMAGE_COMMIT': image.commit,
'IMAGE_DATE': image.date,
'IMAGE_DESCRIPTION': image.description,
'IMAGE_VERSION': image.version,
'SAPADM_COMMENT': sapadm.comment,
'SAPADM_HOME': sapadm.home,
'SAPADM_SHELL': sapadm.shell,
'SAPADM_UID': sapadm.uid,
'SAPMNT': dirs.sapmnt,
'SAPSYS_GID': sapsysGid,
'sid': sidL,
'SID': sidU,
'SIDADM_COMMENT': sidadm.comment,
'SIDADM_HOME': sidadm.home,
'SIDADM_SHELL': sidadm.shell,
'SIDADM_UID': sidadm.uid,
'USR_SAP_REAL': dirs.usrSapReal,
'USR_SAP_LINK_CMD': usrSapLinkCmd,
'INSTALL_OPT_PACKAGES': pkgParams.installOptPackagesDnf,
'COPY_OPT_PACKAGE_FILES': pkgParams.copyOptPackageFiles,
'INSTALL_OPT_PACKAGE_FILES': pkgParams.installOptPackageFiles
}
params.update(self._getContainerfileParams(sidU, dirs))
containerfile = f'{dirs.tmp}/containerfile'
template = f'{dirs.repoRoot}/openshift/images/{self._flavor}/containerfile.template'
genFileFromTemplate(template, containerfile, params)
try:
# pylint: disable=invalid-name, unspecified-encoding
with open(containerfile) as fh:
logging.debug(
f"Contents of '{containerfile}': >>>\n{fh.read()}<<<")
except IOError:
fail(f"Error reading from {containerfile}")
return containerfile
def _read(self, info):
""" Read the contents of the authorized_keys file """
res = self._cmdSsh.run(f'cat {self._akPath}')
if res.rc != 0:
fail(
f"Could not get the authorized keys '{self._akPath}' file of '{self._userFull}'"
)
self._authKeys = _PublicKeys(res.out, info, keepAll=True)
def podmanOcpRegistryLogin(self):
""" Log into the default registry of an OpenShift cluster """
out = CmdShell().run(
'podman login'
' --tls-verify=false'
' -u $(oc whoami) -p $(oc whoami --show-token)'
f' default-route-openshift-image-registry.apps.{self._domain}').out
if not out.startswith('Login Succeeded!'):
fail('podman login failed')
def getBuilder(ctx): # pylint: disable=inconsistent-return-statements
""" Get image builder for specific image flavor """
if ctx.ar.image_flavor == 'nws4':
return BuilderNws4(ctx)
if ctx.ar.image_flavor == 'hdb':
return BuilderHdb(ctx)
# This should never be reached
fail(f"Unknown image flavor '{ctx.ar.image_flavor}'")
def getExecPath(execName, doFail=True):
""" Get path of executable """
execPath = shutil.which(execName)
if not execPath:
if doFail:
fail(f"Did not find executable '{execName}'")
else:
logging.warning(f"Did not find executable '{execName}'")
else:
logging.info(f"Using '{execPath}' as '{execName}'")
return execPath
def instantiateTemplate(templatePath, params):
""" Instantiate a template file replacing parameters enclosed in "{{...}} """
# pylint: disable=invalid-name, unspecified-encoding
try:
with open(templatePath, 'r') as fh:
content = fh.read()
for (k, v) in params.items():
content = content.replace('{{'+k+'}}', v)
except IOError:
fail(f"Error reading from file {templatePath}")
return content
def _readFile(self, fileName):
contents = None
if not pathlib.Path(fileName).is_file():
logging.info(f"File '{fileName}' does not exist")
else:
try:
# pylint: disable=unspecified-encoding
with open(fileName, 'r') as ctFh:
contents = ctFh.read()
except IOError:
fail(f"Error reading from file {fileName}")
return contents
def start(self):
""" start a deployment """
deploymentFile = self._ctx.ar.deployment_file
if not deploymentFile:
self._appName = self._deployments.getValidAppName()
# check if deployment is already started
if self._deployments.isDeployed(self._appName):
fail(
f"Deployment with app name '{self._appName}' already started."
)
# start the deployment
deploymentFile = self._deployments.getDeploymentFile(self._appName)
startDeployment(self._ctx, deploymentFile=deploymentFile)
def _genBuildContext(self, sidU, dirs, sapadm, sidadm, sapsysGid, host,
remoteOs):
# Generate podman build context
# pylint: disable=too-many-arguments
filterFilePath = f'{dirs.tmp}/rsync-filter'
logging.debug(f"filterFilePath: {filterFilePath}")
try:
# pylint: disable=invalid-name, unspecified-encoding
with open(filterFilePath, 'w') as fh:
print(self._getRsyncFilter(sidU, dirs, remoteOs), file=fh)
except IOError:
fail(f"Error writing to file {filterFilePath}")
self._genBuildContextFlavor(sidU, dirs, sapadm, sidadm, sapsysGid,
host, filterFilePath)
def containerLogin(self):
""" Login to a container """
podName = self.getPodName()
if not podName:
fail(
"Cannot get the name of the pod - check if the pod is started."
)
containerName = self.getContainerName(self._ctx.ar.container_flavor)
self._printSwitchUserMsg(switchLogin=False)
print(f"Logging into container '{containerName}' of pod '{podName}'",
file=sys.stderr)
os.execlp('oc', 'oc', 'exec', '-it', podName, '-c', containerName,
'--', 'bash')
def verify(self):
""" Verify various configuration settings """
success, msg = self._checkSsh()
if not success:
fail(msg)
showMsgOk("SSH key setup is valid.")
if self._ctx.ar.function in self._functions:
return self._functions[self._ctx.ar.function]()
success = self._verifyOcp() and success
success = self._verifyImages() and success
success = self._verifyNws4() and success
success = self._verifyHdb() and success
success = self._verifyNfs() and success
success = self._verifySapSystem() and success
return success
def genYaml(self, overlayUuid):
""" generate the deployment description file """
deployment = Deployment(self._ctx, overlayUuid=overlayUuid).get()
if not ocpMemoryResourcesValid(self._ctx):
fail("Fatal error. Stopping the deployment.")
parms = getParmsForDeploymentYamlFile(self._ctx, deployment)
templatePath = f'{self._ctx.cf.build.repo.root}/openshift/'
serviceTemplate = f'{templatePath}/service-nodeport.yaml.template'
deploymentTemplate = f'{templatePath}/deployment.yaml.template'
serviceYamlPart = instantiateYamlTemplate(serviceTemplate, parms)
deploymentYamlPart = instantiateYamlTemplate(deploymentTemplate, parms)
if refSystemIsStandard(self._ctx):
# If the reference system is a standard system no OCP secret definition
# for the HDB connect user is required
# ->
# Remove all OCP secret definition related environment variables
# to avoid problems at deployment time in case no OCP secret was defined
delEnvVars = ('SOOS_DI_DBUSER', 'SOOS_DI_DBUSERPWD')
initContSpec = deploymentYamlPart['spec']['template']['spec'][
'initContainers'][0]
initContSpec['env'] = [
e for e in initContSpec['env'] if e['name'] not in delEnvVars
]
# Write deployment file
try:
# pylint: disable=unspecified-encoding
with open(deployment.file, 'w') as oFh:
print(yaml.dump(serviceYamlPart), file=oFh, end='')
print('---', file=oFh)
print(yaml.dump(deploymentYamlPart), file=oFh, end='')
print(deployment.file)
except IOError:
fail(f"Error writing to file {deployment.file}")
def find(self, uuidPrefix):
""" Find an existing overlay which matches a given UUID prefix """
found = [
ovl for ovl in self._overlays if ovl.uuid.startswith(uuidPrefix)
]
if len(found) < 1:
fail(
f"Found no matching overlay uuid for uuid prefix '{uuidPrefix}'"
)
elif len(found) > 1:
msg = f"Overlay uuid prefix '{uuidPrefix}' matches more than one uuid:\n"
for ovl in found:
msg += f' {ovl}\n'
fail(msg)
return found[0]
def _writeFile(self, fileName, contents):
# See also https://pythonhosted.org/python-gnupg/#encryption
credsFile = fileName
creds = contents
if self._unencrypted:
super()._writeFile(credsFile, contents)
else:
if self._recipient:
# Recipient specified -> encrypt for recipient using asymmetric encryption
print(f"Encrypting for recipient '{self._recipient}'",
file=sys.stderr)
credsEnc = self._gpg.encrypt(creds,
recipients=[self._recipient],
passphrase=self._passphrase)
else:
# No recipient specified -> use symmetric AES256 encryption
print(
'No recipient specified - using symmetric AES256 encryption',
file=sys.stderr)
credsEnc = self._gpg.encrypt(creds,
symmetric='AES256',
recipients=None,
passphrase=self._passphrase)
if not credsEnc.ok:
# pylint: disable=no-member
fail(
f"Encryption failed\n"
f" Status: '{credsEnc.status}'\n Stderr: '{credsEnc.stderr}'"
)
try:
# pylint: disable=unspecified-encoding
with open(credsFile, 'w') as credsFh:
credsFh.write(str(credsEnc))
except IOError:
fail(f"Error writing to {credsFile}")
def __init__(self,
ctx,
login="******",
verify=False,
setProject=True,
logout=True,
appName=None):
self._ctx = ctx
self._ocp = ctx.cf.ocp
self._domain = ctx.cf.ocp.domain
self._admin = ctx.cr.ocp.admin
self._user = ctx.cr.ocp.user
self._project = ctx.cf.ocp.project
self._appName = appName
self._verify = {"domain": True, "creds": True, "project": True}
self._logout = logout
self._loginuser = self._user
if login == "admin":
self._loginuser = self._admin
self._orgUser, self._userSwitch = self._setOrgUser()
self._result = self.ocLogin(self._loginuser)
if self._result.rc > 0:
if not verify:
fail(self._result.err, exitCode=4)
else:
self._verify["domain"] = self._isDomainValid(self._result)
self._verify["creds"] = self._isCredentialsValid(self._result)
# Change to project
result = self.setProject()
if result.rc > 0:
if not verify:
if setProject:
fail(result.err)
else:
warn(f"Could not set project '{self._project}'")
else:
self._verify["project"] = False
def _getPodProperty(self, propertyName, propertySelector):
""" Get a property of the pod in which our current deployment is running """
podProperty = ''
if not self._appName:
fail("Internal error: appName not set")
res = CmdShell().run(f'oc get pods --selector="app={self._appName}"'
' -o template --template "{{range .items}}{{' +
propertySelector + '}}{{end}}"')
if res.rc == 0:
podProperty = res.out.strip()
logging.debug(f"Pod property '{propertyName}': '{podProperty}'")
else:
logging.debug(f"Could not get pod property '{propertyName}'"
f" for app '{self._appName}' (reason: {res.err})")
return podProperty
def __init__(self,
ctx,
hostname,
user,
sshId=None,
check=True,
reuseCon=True): # pylint: disable=too-many-arguments
super().__init__()
self._cmdShell = CmdShell()
self._sshId = None
if not sshId:
sshId = ctx.cr.build.user.sshid
self._sshId = sshId
if sshId:
logging.debug(f"Using SSH ID '{sshId}'")
else:
logging.debug('SSH ID is not set')
sshCmd, sshLogin, sshCmdSecrets = CmdSsh._getSshCmdAndSecrets(
hostname, user, sshId, reuseCon)
self._sshCmd = sshCmd
self._sshLogin = sshLogin
self._sshCmdSecrets = sshCmdSecrets
# Check the connection and create the socket for connection
# reuse in case the socket does not exist yet
if check:
logging.debug(f"Connecting to host '{hostname}'")
res = self.run('true')
if res.rc != 0:
msg = self.formatSshError(res, hostname, user)
fail(msg)
def getValidAppName(self):
""" returns valid application name """
appNames = self.getAppNames()
msg = self._getErrorMsg(appNames)
if not self._ctx.ar.app_name:
if self._deploymentType == self._ctx.cs.deployAll:
return None
if len(appNames) == 1:
return appNames[0]
fail(msg)
# parts or full app-name specified as parameter
match = None
for appName in appNames:
if self._ctx.ar.app_name in appName:
if match:
fail(msg)
match = appName
if not match:
fail(msg)
return match
def stop(self, ignore=False):
""" Stop a deployment """
deploymentFile = self._ctx.ar.deployment_file
if deploymentFile:
self._appName = self._deployments.getAppNameFromFile(
deploymentFile)
else:
self._appName = self._deployments.getValidAppName()
# check if deployment is already stopped
if not ignore:
if self._deployments.isNotDeployed(self._appName):
fail(
f"Deployment with app name '{self._appName}' not deployed."
)
# stop the deployment
deploymentFile = self._deployments.getDeploymentFile(self._appName)
stopDeployment(self._ctx, deploymentFile=deploymentFile)
# wait for stopped
self._waitForStopped(self._appName)
