def post_install(self):
"""Post installation steps."""
logger.debug("post_install")
if self.do_server:
self._set_cell_config(self.dirs['AFS_CONF_DIR'], self.cell, self.cellhosts)
self._set_krb_config(self.dirs['AFS_CONF_DIR'], self.cell, self.realm)
if self.do_client:
# Create the CellServDB and ThisCell files. Remove any symlinks that
# may have been created by the bosserver. Combine the optional CellServDB.dist
# to access other cells.
csdb = os.path.join(self.dirs['AFS_DATA_DIR'], "CellServDB")
local = os.path.join(self.dirs['AFS_DATA_DIR'], "CellServDB.local")
dist = os.path.join(self.dirs['AFS_DATA_DIR'], "CellServDB.dist")
self._set_cell_config(self.dirs['AFS_DATA_DIR'], self.cell, self.cellhosts, ext=".local")
if self.csdb:
shutil.copyfile(self.csdb, dist)
else:
touch(dist)
cat([local, dist], csdb)
# Set the cache info parameters.
# XXX: size should be calculated from partition, if mounted.
cache_size = 102000 # k blocks
self._set_cache_info(
self.dirs['AFS_DATA_DIR'],
self.dirs['AFS_MOUNT_DIR'],
self.dirs['AFS_CACHE_DIR'],
cache_size)
if self.scripts['post_install']:
logger.info("Running post-install script.")
args = shlex.split(self.scripts['post_install'])
sh(*args, output=False)
def pre_remove(self):
"""Pre remove steps."""
logger.debug("pre_remove")
if self.scripts['pre_remove']:
logger.info("Running pre-remove script.")
args = shlex.split(self.scripts['pre_remove'])
sh(*args, output=False)
def configure(options=None, srcdir='.', force=False):
if options is None:
options = []
if not force and os.path.exists('config.status'):
logger.warning("Skipping configure; config.status already exists")
return 0
sh('%s/configure' % srcdir, *options, output=False)
def make(jobs=1, target='all', program='make'):
args = [program]
if jobs > 1:
# Requires a `make` whichsupports the -j option. If your `make` does
# not, then specify the path to `gmake` or set jobs to 1.
args.append('-j')
args.append('{0}'.format(jobs))
args.append(target)
sh(*args, output=False)
def _install_openafs_ko(self, kmod):
"""Install the openafs.ko file and run depmod."""
release = os.uname()[2]
src = kmod
dst = path_join("/lib/modules", release, "extra/openafs/openafs.ko")
logger.info("Installing kernel module from '%s' to '%s'.", src, dst)
mkdirp(os.path.dirname(dst))
shutil.copy2(src, dst)
sh('/sbin/depmod', '-a')
def ssh(hostname, args, ident=None, dryrun=False):
"""Execute the remote command with ssh."""
cmdline = subprocess.list2cmdline(args)
args = ['ssh', '-q', '-t', '-o', 'PasswordAuthentication=no']
if ident:
args.append('-i')
args.append(ident)
args.append(hostname)
args.append(cmdline)
sh(*args, prefix=hostname, quiet=False, output=False, dryrun=dryrun)
def post_remove(self):
"""Post remove steps."""
logger.debug("post_remove")
if self.purge:
if self.do_server:
remove_files("/usr/afs/")
self._purge_volumes()
if self.do_client:
self._purge_cache()
if self.scripts['post_remove']:
logger.info("Running post-remove script.")
args = shlex.split(self.scripts['post_remove'])
sh(*args, output=False)
def create(config, keyfile, keytype='rsa', **kwargs):
"""Create a public/private key pair.
Run ssh-keygen to create the ssh ident files."""
if os.access(keyfile, os.F_OK):
# Do not clobber the existing key file; it will confuse
# the ssh-agents.
logger.error("Skipping ssh-keygen; file %s already exists." % (keyfile))
return 1
logger.info("Creating ssh key file %s." % (keyfile))
args = ['ssh-keygen', '-t', keytype, '-f', keyfile]
sh(*args, quiet=False, output=False)
config.set_value('ssh', 'keyfile', keyfile)
config.save()
def test_is_loaded(self):
mount = which('mount', extra_paths=['/usr/sbin'])
output = "\n".join(sh(mount))
if "AFS on /afs" in output:
self.assertTrue(is_loaded('openafs'))
else:
self.assertFalse(is_loaded('openafs'))
def pkg_install(packages, dryrun=False, update_all=True):
# Recent versions of solarisstudio fail to install due to dependency
# conflicts unless the system is updated first.
if update_all:
sh('pkg', 'update', '-v', 'entire@latest', dryrun=dryrun, output=False)
# Ignore package already installed errors.
ERROR_ALREADY_INSTALLED = 4
try:
sh('pkg',
'install',
'--accept',
*packages,
dryrun=dryrun,
output=False)
except CommandFailed as e:
if e.code != ERROR_ALREADY_INSTALLED:
logger.error("pkg install failed: %s" % e)
raise e
def _afs_driver_511(self):
output = sh('/bin/isalist')[0]
if 'amd64' in output:
driver = '/kernel/drv/amd64/afs'
elif 'sparcv9' in output:
driver = '/kernel/drv/sparcv9/afs'
else:
driver = '/kernel/drv/afs'
return driver
def is_git_clean_enabled(gitdir, git='git'):
enabled = False
try:
output = sh(git, '--git-dir', gitdir, 'config', '--bool', '--get',
'afsutil.clean')
if output[0] == 'true':
enabled = True
except CommandFailed as e:
if e.code != 1:
raise e
return enabled
def pre_install(self):
"""Pre installation steps."""
# Get name and IP address of the cell hosts. Use the local hostname
# and local interface if the cell hosts are not specified. Do this
# before we start installing to catch errors early.
logger.debug("pre_install")
if self.scripts['pre_install']:
logger.info("Running pre-install script.")
args = shlex.split(self.scripts['pre_install'])
sh(*args, output=False)
if self.cellhosts is None:
if self.hostnames:
self.cellhosts = self._lookup_cellhosts(self.hostnames)
else:
self.cellhosts = self._detect_cellhosts()
logger.info("Cell hosts are: %s", pprint.pformat(self.cellhosts))
if self.do_server:
self._make_vice_dirs()
if self.do_client:
if self.csdb is not None:
file_should_exist(self.csdb)
def copy(config, source, dest, quiet=False, exclude='', sudo=False, local=False, **kwargs):
"""Copy a file to the remote hosts."""
keyfile = config.optstr('ssh', 'keyfile', required=True)
script = "/bin/sh -c 'cat > %s'" % (dest)
if sudo:
script = 'sudo -n %s' % (script)
try:
with open(source, 'r') as f:
data = f.read()
except Exception as e:
logger.error("Cannot read file %s: %s." % (source, e))
return 1
code = 0
for hostname in config.opthostnames():
if hostname in exclude:
continue
if islocal(hostname):
if local:
if not quiet:
logger.info("Copying file %s to localhost:%s" % (source, dest))
if sudo:
sh('sudo', '-n', 'cp', source, dest)
else:
sh('cp', source, dest, quiet=True)
else:
if not quiet:
logger.info("Copying file %s to %s:%s" % (source, hostname, dest))
args = ['ssh', '-q', '-t', '-o', 'PasswordAuthentication=no', '-i', keyfile, hostname, script]
p = subprocess.Popen(args, stdin=subprocess.PIPE)
p.stdin.write(data)
p.stdin.close()
code = p.wait()
if code != 0:
logger.error("Failed to copy file to host '%s'; exit code %d." % (hostname, code))
break
return code
def execute(config, command, exclude='', quiet=False, sudo=False, local=False, **kwargs):
"""Run a command on each remote host."""
keyfile = config.optstr('ssh', 'keyfile', required=True)
if not command:
logger.error("Missing command")
return 1
exclude = exclude.split(',')
args = shlex.split(command) # Note: shlex handles quoting properly.
if sudo:
args = _sudo(args)
if not keyfile:
logger.error("Missing value for keyfile.")
return 1
if not os.access(keyfile, os.F_OK):
logger.error("Cannot access keyfile %s." % (keyfile))
return 1
code = 0
for hostname in config.opthostnames():
if hostname in exclude:
continue
if islocal(hostname):
if local:
try:
sh(*args, quiet=False, output=False)
except CommandFailed as e:
logger.error("local command failed: %s" % (e.out))
code = e.code
break
else:
try:
ssh(hostname, args, ident=keyfile)
except CommandFailed as e:
logger.error("remote command failed; host=%s: %s" % (hostname, e.out))
code = e.code
break
return code
def _rc(component, action):
# Try systemd if a unit file is present, otherwise fallback
# to our init script.
name = 'openafs-%s' % (component)
unit_file = '/usr/lib/systemd/system/%s.service' % (name)
init_script = "/etc/init.d/%s" % (name)
if os.path.isfile(unit_file):
sh('systemctl', action, name, output=False)
elif os.path.isfile(init_script):
if logger.getEffectiveLevel() == logging.DEBUG:
sh('/bin/bash', '-x', init_script, action, output=False)
else:
sh(init_script, action, output=False)
else:
raise AssertionError("Init script is missing for %s!" % (name))
def _run(cmd, args=None, quiet=False, retry=0, wait=1, cleanup=None):
"""Execute a command and return the output as a string.
cmd: command to be executed
args: list of command line arguments
quiet: do not log command and output
retry: number of retry attempts, 0 for none
wait: delay between retry attempts
cleanup: cleanup function to run before retry
returns: command output as a string
Raises a CommandFailed exception if the command exits with
a non-zero exit code."""
count = 0 # retry counter
if args is None:
args = []
elif not isinstance(args, list):
args = list(args)
cmd = _cmdpath.get(cmd, cmd)
args.insert(0, which(cmd, raise_errors=True, extra_paths=PATHS))
while True:
try:
lines = sh(*args, quite=quiet)
break
except CommandFailed as cf:
if count < retry:
count += 1
logger.info("Retrying %s command in %d seconds; retry %d of %d.",
cmd, wait, count, retry)
time.sleep(wait)
if cleanup:
cleanup() # Try to cleanup the mess from the last failure.
else:
raise cf
return "\n".join(lines)
def zypper_install(packages, dryrun=False):
sh('zypper', 'install', '-y', *packages, dryrun=dryrun, output=False)
def dnf_install(packages, dryrun=False):
sh('dnf', 'install', '-y', *packages, dryrun=dryrun, output=False)
def yum_install(packages, dryrun=False):
sh('yum', 'install', '-y', *packages, dryrun=dryrun, output=False)
def apt_get_install(packages, dryrun=False):
sh('apt-get', '-y', 'install', *packages, dryrun=dryrun, output=False)
def test_sh(self):
self.assertIsNotNone(sh("/bin/ls", "/bin"))
self.assertIn("sh", sh("/bin/ls", "/bin"))
def build(**kwargs):
"""Build the OpenAFS binaries.
Build the transarc-path compatible bins by default, which are
deprecated, but old habits die hard.
"""
cf = kwargs.get('cf', cfopts())
xcf = flatten(kwargs.get('xcf', []))
target = kwargs.get('target', 'all')
clean = kwargs.get('clean', False)
jobs = kwargs.get('jobs', 1)
srcdir = kwargs.get('srcdir', '.')
tarball = kwargs.get('tarball', None)
paths = kwargs.get('paths', [])
log = kwargs.get('log', None)
paths = lists2dict(paths)
_make = paths.get('make', 'make')
_git = paths.get('git', 'git')
_tar = paths.get('tar', None) # tar or gtar, depending on OS
cf = shlex.split(cf) # Note: shlex handles quoting properly.
for x in xcf:
for y in shlex.split(x):
_cfadd(cf, y)
# The legacy Transarc-style distribution requires a different top level target.
if '--enable-transarc-paths' in cf and target == 'all':
target = 'dest'
if not os.path.exists(srcdir):
raise AssertionError("srcdir not found: %s" % (srcdir))
if srcdir == '.':
_sanity_check_dir()
if clean:
gitdir = os.path.abspath(os.path.join(srcdir, '.git'))
if not os.path.isdir(gitdir):
logger.error("Unable to run git clean; not a git directory: '%s'",
gitdir)
return 1
if not is_git_clean_enabled(gitdir, git=_git):
logger.warning(
"Unable to run git clean; not enabled in the git config.")
logger.warning("To enable git clean before builds:")
logger.warning(" git config --local afsutil.clean true")
return 1
args = [
_git, '--git-dir', gitdir, '--work-tree', srcdir, 'clean', '-f',
'-d', '-x', '-q'
]
if log:
args.append('--exclude')
args.append(log)
sh(*args, output=False)
_setenv()
regen(srcdir=srcdir)
configure(options=cf, srcdir=srcdir)
make(jobs=jobs, target=target, program=_make)
if target == 'dest':
_create_tarball(tarball, program=_tar)
def execute(self, args):
"""Run the command."""
sh(*args, prefix=self.name, quiet=False, output=False, dryrun=self.dryrun)
def lookup_solarisstudio(creds='/root/creds',
key='pkg.oracle.com.key.pem',
cert='pkg.oracle.com.certificate.pem',
**kwargs):
"""
Lookup the Solaris Studio package.
Before running this function, create an account on the Oracle Technology Network
and follow the instructions to create and download the key and certificate files.
Place the key and cert file in a local directory or at a private url. The
default location of the key and cert files is /root/creds. Set the 'creds'
keyword argurment to specify a different path or a url.
"""
path = None
tmpdir = None
quiet = True
def download(baseurl, filename, path):
url = os.path.join(baseurl, filename)
dst = os.path.join(path, filename)
rsp = urllib2.urlopen(url)
logger.info("Downloading '%s' to '%s'.", url, dst)
with open(dst, 'wb') as f:
f.write(rsp.read())
def normalize(s):
return [int(x) for x in s.split('.')]
def compare_versions(a, b):
return cmp(normalize(a), normalize(b))
if creds.startswith('http://') or creds.startswith('https://'):
try:
path = tmpdir = tempfile.mkdtemp()
download(creds, key, tmpdir)
download(creds, cert, tmpdir)
except urllib2.HTTPError as e:
logger.error("Unable to download files from url '%s', %s'." %
(creds, e))
return None
finally:
logger.info("Cleaning up temporary files in '%s'.", tmpdir)
if os.path.exists(os.path.join(tmpdir, key)):
os.remove(os.path.join(tmpdir, key))
if os.path.exists(os.path.join(tmpdir, cert)):
os.remove(os.path.join(tmpdir, cert))
if os.path.exists(tmpdir) and tmpdir != "/":
os.rmdir(tmpdir)
elif os.path.exists(creds):
logger.info("Using credentials in path '%s'.", creds)
path = creds
else:
logger.error("creds path '%s' not found.", creds)
return None
logger.info("Setting publisher for solarisstudio.")
sh(
'pkg',
'set-publisher', # Will refresh if already set.
'-k',
os.path.join(path, key),
'-c',
os.path.join(path, cert),
'-G',
'*',
'-g',
'https://pkg.oracle.com/solarisstudio/release',
'solarisstudio',
output=False,
quiet=quiet)
logger.info("Getting available solaris studio packages.")
packages = {}
output = sh('pkg',
'list',
'-H',
'-a',
'-v',
'--no-refresh',
'pkg://solarisstudio/*',
quiet=quiet)
for line in output:
# Extract the root package name, version, and install state.
fmri, ifo = line.split()
pkg, vt = fmri.split('@')
v, t = vt.split(':')
version = v.split(',')[0]
name = pkg.replace('pkg://solarisstudio/developer/', '')
istate = ifo[0] == 'i'
if '/' in name:
pass # Skip non-root ones, e.g. 'developerstudio-125/cc'.
else:
logger.info("Found package %s, version %s, installed %s.", name,
version, istate)
packages[name] = {'version': version, 'installed': istate}
# Find the most recent version.
solarisstudio = None
vers = '0.0'
for name in packages.keys():
if compare_versions(packages[name]['version'], vers) > 0:
vers = packages[name]['version']
solarisstudio = 'pkg://solarisstudio/developer/{0}'.format(name)
if solarisstudio:
logger.info("Most recent version found is '%s'.", solarisstudio)
else:
logger.info("Could not find solaris studio package.")
return solarisstudio
def regen(srcdir='.', force=False):
if not force and os.path.exists('%s/configure' % srcdir):
logger.warning("Skipping regen.sh; configure already exists")
return 0
sh('/bin/sh', '-c', 'cd %s && ./regen.sh' % srcdir, output=False)
def yum(*args):
"""Helper to run the yum command."""
return sh('yum', *args, quiet=False)
def rpm(*args):
"""Helper to run the rpm command."""
return sh('rpm', *args, quiet=True)