def getValidNfsServerAddress(ctx):
""" Get valid NFS server address serving the OCP cluster net """
# Get all IP addresses for NFS server
allIps = getAllNfsServerIpAddresses(ctx)
# Need an OC login to get worker node address
ocp = Ocp(ctx, login="******", verify=True)
worker = ocp.getWorkerNodeList()[0]
# Run Python script tools/modules/nfs-ping-test on helper node (in-line)
cmdShell = CmdShell()
host = ctx.cf.ocp.helper.host.name
user = ctx.cr.ocp.helper.user
repo = ctx.cf.build.repo.root
toolCmd = f'python3 - <{repo}/tools/modules/nfs-ping-test {worker} {allIps}'
cmdSsh = CmdSsh(ctx, host, user, reuseCon=False)
runCmd, secr = cmdSsh.getSshCmdAndSecrets(withLogin=True)
ipAddr = cmdShell.run(f'{runCmd} {toolCmd}', secrets=secr).out
logging.debug(
f"Running shell cmd: '{runCmd} {toolCmd}' returns '{ipAddr}'")
if ipAddr == 'None':
message = "Could not identify valid IP address for the NFS server"
message += f"on worker node {worker}.\n"
fail(message)
del ocp
return ipAddr
class AuthorizedKeys():
""" Representation of an authorized_keys file of a specific user at a specific host """
def __init__(self, ctx, hname, user):
self._userFull = f'{user.name}@{hname}'
self._authKeys = None
# Prepare command execution
self._cmdShell = CmdShell()
self._cmdSsh = CmdSsh(ctx, hname, user)
self._rsyncSsh, self._rsyncSshSecrets = self._cmdSsh.getSshCmdAndSecrets(
withLogin=False)
# Get the path to the remote authorized_keys file
homeDir = getHomeDir(ctx, hname, user)
if not homeDir:
fail(
f"Could not determine the home directory of '{self._userFull}'"
)
self._akPath = f'{homeDir}/.ssh/authorized_keys'
# Read the authorized_keys file
info = _PublicKeyInformation(hname, user, self._akPath)
self._read(info) # Sets self._authKeys
logging.debug(
f'self._authKeys >>>\n{self._authKeys}\n<<< self._authKeys')
def __str__(self):
return str(self._authKeys)
def _read(self, info):
""" Read the contents of the authorized_keys file """
res = self._cmdSsh.run(f'cat {self._akPath}')
if res.rc != 0:
fail(
f"Could not get the authorized keys '{self._akPath}' file of '{self._userFull}'"
)
self._authKeys = _PublicKeys(res.out, info, keepAll=True)
def write(self):
""" Write the contents of the authorized_keys file """
# Write the contents to a temporary local file and transfer the file to
# the remote user's .ssh directory using rsync
# Keep a backup of the original authorized_keys file on the remote side
with tempfile.NamedTemporaryFile(mode='w') as akFh:
print(self._authKeys, file=akFh, flush=True)
source = akFh.name
target = self._akPath
backupSuffix = '.bak'
rsyncCmd = f'rsync -av -e "{self._rsyncSsh}"'
rsyncCmd += f' --backup --suffix "{backupSuffix}"'
rsyncCmd += f' "{source}" "{self._userFull}:{target}"'
res = self._cmdShell.run(rsyncCmd, self._rsyncSshSecrets)
if res.rc != 0:
fail(f"Could not write the authorized keys file '{target}'"
f" of '{self._userFull}\n({res.err})")
def add(self, keys):
""" Add keys in a key list to the internal authorized_keys list """
return self._authKeys.addKeys(keys)
def remove(self, keys):
""" Remove keys in a key list from internal authorized_keys list """
return self._authKeys.removeKeys(keys)
def numKeys(self):
""" Return the number of keys in the internal authorized_keys key list """
return self._authKeys.numKeys()
class RemoteCopy():
""" Perform selective remote copy with correct symlink preservation """
def __init__(self, ctx, host, user, filterFilePath='/dev/null'):
"""Perform selective remote copy with correct symlink preservation
Parameters:
host: host of source directory; also ssh and rsync host
user: ssh and rsync user
filterFilePath: optional; path to rsync filter file
"""
self._host = host
self._user = user
self._filterFilePath = filterFilePath
# Initialize ssh connection
self._cmdSsh = CmdSsh(ctx, host, user)
self._rsyncSsh, self._rsyncSshSecrets = self._cmdSsh.getSshCmdAndSecrets(withLogin=False)
def _runRsync(self, source, filterFilePath, verbose, dryRun):
logging.debug(f'source: >>>{source}<<<')
cmdShell = CmdShell()
cmd = 'rsync -a --relative'
cmd += f' -e "{self._rsyncSsh}"'
if verbose > 0:
cmd += ' -'+'v'*verbose
cmd += f' -f "merge {filterFilePath}"'
if dryRun:
cmd += ' -n'
if not isinstance(source, list):
cmd += f' {self._user.name}@{self._host}:{source} ./'
cmdShell.run(cmd, self._rsyncSshSecrets)
else:
with tempfile.NamedTemporaryFile(mode='w') as tfh:
tfh.write("\n".join(str(fn) for fn in source))
tfh.flush()
logging.debug(f"Contents of file '{tfh.name}':")
logging.debug('>>>')
# pylint: disable=unspecified-encoding
with open(tfh.name) as rfh:
logging.debug(rfh.read())
logging.debug('<<<')
cmd += f' -r --files-from={tfh.name}'
cmd += f' {self._user.name}@{self._host}:/ ./'
cmdShell.run(cmd, self._rsyncSshSecrets)
def _getRealPathAndSymlinks(self, path, targets):
logging.debug(f"path '{path}', targets '{targets}'")
curPath = ''
for component in list(filter(lambda x: x != '', path.split('/'))):
curDir = curPath
curPath = curDir + '/' + component
logging.debug(f"Current path '{curPath}'")
if curPath not in targets.keys():
logging.debug(f"Visiting new path '{curPath}'")
target = self._cmdSsh.run(f'readlink {curPath}').out
if not target:
targets[curPath] = None
else:
logging.debug(f"found symlink '{curPath}', target '{target}'")
if not target.startswith('/'):
relTarget = target
target = os.path.normpath(curDir + '/' + target)
logging.debug(f"Converted relative target '{relTarget}'"
f" to absolute target '{target}'")
(targets[curPath], targets) = self._getRealPathAndSymlinks(target, targets)
if targets[curPath]:
curPath = targets[curPath]
logging.debug(f"returning path '{curPath}', targets >>>{targets}<<<")
return (curPath, targets)
def _symlinkConvertRelToAbs(self, symlink, linkTarget):
if not linkTarget.startswith('/'):
relTarget = linkTarget
linkTarget = os.path.join(os.path.dirname(symlink), linkTarget)[1:] # skip leading '.'
logging.debug(f"Converted relative target '{relTarget}' "
f"to absolute target '{linkTarget}'")
return linkTarget
def copy(self, source, filterFilePath=None, verbose=1, dryRun=False):
""" Perform remote copy
Parameters:
source : root of source directorytree; must be an absolute path
filterFilePath: optional: path to file containing rsync filters;
if not supplied filter file path supplied to constructor will be used
verbose : optional: set rsync verbose level;
choices: [0, 1, 2, 3], corresponding to rsync verbose levels
[<none>, '-v', '-vv', '-vvv']
dryRun : optional: if set to True, perform a trial rsync run with no changes made'
"""
if not filterFilePath:
filterFilePath = self._filterFilePath
logging.info(f"Remote copy of '{source}' started.")
logging.debug(f'source >>>{source }<<<')
logging.debug(f'filterFilePath >>>{filterFilePath}<<<')
logging.debug(f'verbose >>>{verbose }<<<')
logging.debug(f'dryRun >>>{dryRun }<<<')
cmdShell = CmdShell()
symlinksVisited = []
existingSymlinks = cmdShell.run('find ./ -type l').out
if existingSymlinks:
# Do not follow local existing symlinks
symlinksVisited = existingSymlinks.strip().split('\n')
logging.debug(f'symlinksVisited >>>{symlinksVisited}<<<')
# rsync root of source directory supplied on command line
(realPath, targets) = self._getRealPathAndSymlinks(source, {})
self._runRsync(realPath, filterFilePath, verbose, dryRun)
# If root of source directory tree is a symlink itself append it to list of visited links
logging.debug(f"source : '{source}'")
logging.debug(f"realPath: '{realPath}'")
if realPath != source:
# cmdShell.run(f'ln -s {realPath} .{source}')
symlinksVisited.append(f'.{source}')
# Recursively detect all symlinks and rsync their targets
finished = False
while not finished:
finished = True
symlinksFound = cmdShell.run('find ./ -type l').out
logging.debug(f'symlinksFound >>>{symlinksFound}<<<')
logging.debug(f'symlinksVisited >>>{symlinksVisited}<<<')
if symlinksFound:
symlinksFound = symlinksFound.strip().split('\n')
realPaths = []
for symlink in symlinksFound:
if (symlink not in symlinksVisited
and symlink[1:] not in targets.keys()): # skip leading '.'
logging.debug(f'symlink >>>{symlink}<<<')
linkTarget = os.readlink(symlink)
logging.debug(f'linkTarget >>>{linkTarget}<<<')
linkTarget = self._symlinkConvertRelToAbs(symlink, linkTarget)
(realPath, targets) = self._getRealPathAndSymlinks(linkTarget, targets)
if realPath not in realPaths:
realPaths.append(realPath)
logging.debug(f'realPaths: >>>{realPaths}<<<')
symlinksVisited.append(symlink)
finished = False
if realPaths:
self._runRsync(realPaths, filterFilePath, verbose, dryRun)
# Copy all symlinks that were not yet copied
logging.debug('Final rsync call')
self._runRsync([s for (s, t) in targets.items() if t],
filterFilePath, verbose, dryRun)
logging.info(f"Remote copy of '{source}' finished.")
