def chown_app_dir(self, workdir, homedir):
cmd = "chown -R %s:%s %s" % (self.config.apps.uid,
self.config.apps.gid, homedir)
with Chroot(workdir):
try:
commands.execute(cmd, timeout=self.config.commands.timelimit,
output_loglevel=logging.INFO, strip_envs=True)
except commands.CommandTimeout:
log.error("chown is taking too long to execute, aborting")
return False
except commands.CommandFailed:
log.error("chown failed")
return False
return True
def umount_filesystems(workdir, timeout=60):
mounts = []
if os.path.isfile("/proc/mounts"):
with open("/proc/mounts") as mtab:
for line in mtab:
try:
mount = line.split()[1]
except IndexError:
pass
else:
if mount.startswith(workdir.rstrip('/') + '/'):
mounts.append(mount)
for mount in mounts:
log.info("Found mounted filesystem at '%s', unmounting" % mount)
commands.execute("umount %s" % mount, timeout=timeout)
def bootstrap_os(self):
"""
Bootstrap base OS image.
"""
log.info("Bootstrapping new OS image")
# directory is encoded into string to prevent unicode errors
directory = tempfile.mkdtemp(dir=self.config.paths.workdir,
prefix="upaas_bootstrap_")
log.debug("Created temporary directory for bootstrap at "
"'%s'" % directory)
for cmd in self.config.bootstrap.commands:
cmd = cmd.replace("%workdir%", directory)
try:
commands.execute(cmd, timeout=self.config.bootstrap.timelimit,
cwd=directory, env=self.config.bootstrap.env,
strip_envs=True)
except commands.CommandTimeout as e:
log.error("Bootstrap was taking too long and it was killed")
kill_and_remove_dir(directory)
raise exceptions.OSBootstrapError(e)
except commands.CommandFailed as e:
log.error("Bootstrap command failed")
kill_and_remove_dir(directory)
raise exceptions.OSBootstrapError(e)
log.info("All commands completed, installing packages")
self.install_packages(directory, self.config.bootstrap.packages)
log.info("Bootstrap done, packing image")
archive_path = os.path.join(directory, "image.tar.gz")
if not tar.pack_tar(directory, archive_path,
timeout=self.config.bootstrap.timelimit):
kill_and_remove_dir(directory)
raise exceptions.OSBootstrapError("Tar error")
else:
log.info("Image packed, uploading")
try:
self.storage.put(archive_path, distro.distro_image_filename())
except Exception as e:
log.error("Upload failed: %s" % e)
raise
log.info("Image uploaded")
kill_and_remove_dir(directory)
log.info("All done")
def directory_pids(directory):
"""
List pid of all processes running inside given directory.
List will contain integers and will use ascending sorting.
:param directory: Directory to scan for running processes.
:type directory: str
:returns: list of int -- [134, 245, 673, 964]
"""
log.debug("Scanning for processes running in %s" % directory)
ret = set()
if os.path.exists(directory):
(rcode, output) = execute('lsof -t +d %s' % directory,
valid_retcodes=[0, 1])
if output:
for line in output:
try:
ret.add(int(line))
except ValueError:
log.debug("Could not convert PID value to int: "
"'%s'" % line)
return sorted(list(ret))
else:
log.debug("No such directory: %s" % directory)
return []
def run_actions(self, actions, workdir, homedir='/'):
for name in actions:
log.info("Executing '%s' setup actions" % name)
for cmd in self.actions[name]:
with Chroot(workdir, workdir=homedir):
try:
commands.execute(
cmd, timeout=self.config.commands.timelimit,
env=self.envs, output_loglevel=logging.INFO,
strip_envs=True)
except commands.CommandTimeout:
log.error("Command is taking too long to execute, "
"aborting")
return False
except commands.CommandFailed as e:
log.error("Execution failed: %s" % e)
return False
return True
def update(self, workdir, homedir):
log.info("Updating repository in '%s'" % homedir)
with Chroot(workdir, workdir=homedir):
for cmd in self.metadata.repository.update:
cmd = cmd.replace("%destination%", homedir)
try:
commands.execute(cmd,
timeout=self.config.commands.timelimit,
env=self.metadata.repository.env,
output_loglevel=logging.INFO,
strip_envs=True)
except commands.CommandTimeout:
log.error("Command is taking too long, aborting")
return False
except commands.CommandFailed:
log.error("Command failed")
return False
return True
def install_packages(self, workdir, packages):
with Chroot(workdir):
for name in packages:
cmd = self.config.commands.install.cmd.replace("%package%",
name)
try:
commands.execute(cmd,
timeout=self.config.commands.timelimit,
env=self.config.commands.install.env,
output_loglevel=logging.INFO,
strip_envs=True)
except commands.CommandTimeout:
log.error("Installing package '%s' is taking to long, "
"aborting" % name)
return False
except commands.CommandFailed:
log.error("Installing package '%s' failed" % name)
return False
return True
def unpack_tar(archive_path, destination, timeout=None):
"""
Unpack tar archive in destination directory.
:param archive_path: Path to tar file.
:param destination: Destination directory in which we will unpack tar file.
:param timeout: Timeout in seconds.
"""
try:
commands.execute("tar -xzpf %s" % archive_path, timeout=timeout,
cwd=destination)
except commands.CommandTimeout:
log.error("Tar command was taking too long and it was killed")
return False
except commands.CommandFailed:
log.error("Tar command failed")
return False
else:
return True
def unpack_os(self, directory, workdir, system_filename=None):
empty_os_image = False
if not system_filename:
system_filename = distro.distro_image_filename()
empty_os_image = True
os_image_path = os.path.join(directory, "os.image")
log.info("Fetching OS image '%s'" % system_filename)
try:
self.storage.get(system_filename, os_image_path)
except StorageError:
log.error("Storage error while fetching OS image")
return False
else:
log.info("Unpacking OS image")
if not tar.unpack_tar(os_image_path, workdir):
log.error("Error while unpacking OS image to '%s'" % workdir)
return False
# verify if os is working
log.info("Checking if OS image is working (will execute /bin/true)")
try:
with Chroot(workdir):
commands.execute('/bin/true',
timeout=self.config.commands.timelimit,
output_loglevel=logging.INFO)
except Exception as e:
log.error("Broken OS image! /bin/true failed: %s" % e)
if empty_os_image:
try:
self.storage.delete(system_filename)
except StorageError as e:
log.error("Storage error while deleting OS image: %s" % e)
else:
log.info("Deleted broken OS image from storage")
self.bootstrap_os()
return self.unpack_os(directory, workdir,
system_filename=system_filename)
return False
else:
return True
def pack_tar(source, archive_path, timeout=None):
"""
Pack files at given directory into tar archive.
:param source: Directory which content should be packed.
:param archive_path: Path at which tar archive file will be created.
:param timeout: Timeout in seconds.
"""
def _cleanup(archive_path):
try:
log.debug("Removing incomplete tar archive '%s' if "
"present" % archive_path)
os.remove(archive_path)
except OSError:
pass
cmd = "tar -czpf %s *" % archive_path
# check if pigz is installed
try:
commands.execute("pigz -V", timeout=10)
except commands.CommandError:
pass
else:
cmd = "tar --use-compress-program=pigz -cpf %s *" % archive_path
log.info("Using pigz for parallel compression")
try:
commands.execute(cmd, timeout=timeout, cwd=source)
except commands.CommandTimeout:
log.error("Tar command was taking too long and it was killed")
_cleanup(archive_path)
return False
except commands.CommandFailed:
log.error("Tar command failed")
_cleanup(archive_path)
return False
else:
return True
def vcs_cmd(name, cmd, replace=None):
name = 'repository.revision.%s' % name
if hasattr(cmd, '__call__'):
cmd = cmd()
if replace:
for (rold, rnew) in replace:
cmd = cmd.replace(rold, rnew)
env = self.metadata.repository.env.copy()
env['LANG'] = 'C.UTF-8'
env['LC_ALL'] = 'C.UTF-8'
try:
_, output = commands.execute(
cmd, timeout=self.config.commands.timelimit, env=env,
output_loglevel=logging.INFO, strip_envs=True)
except commands.CommandTimeout:
log.error("%s command is taking too long, aborting" % name)
except commands.CommandFailed:
log.error("%s command failed" % name)
else:
return ''.encode('utf-8').join(output).rstrip(
'\n'.encode('utf-8'))
def test_env_and_output():
_, output = commands.execute("echo $MYENV", env={"MYENV": "MYVALUE"})
assert output == ["MYVALUE\n"]
def test_return_code():
rcode, _ = commands.execute("exit 123", valid_retcodes=[123])
assert rcode == 123
def test_timeout():
with pytest.raises(commands.CommandTimeout):
commands.execute("sleep 2", timeout=1)
def test_cwd_and_output(empty_dir):
_, output = commands.execute("pwd", cwd=empty_dir)
assert output == [empty_dir + "\n"]
def test_failed_command():
with pytest.raises(commands.CommandFailed):
commands.execute("non existing command")