def test_03_uuid(self):
"""Test03 Unique.uuid()."""
uniq = Unique()
rand = uniq.uuid("zxcvbnm")
self.assertEqual(len(rand), 36)
rand = uniq.uuid(789)
self.assertEqual(len(rand), 36)
def test_07_filename(self):
"""Test07 Unique.filename()."""
uniq = Unique()
rand = uniq.filename("zxcvbnmasdf")
self.assertTrue(rand.endswith("zxcvbnmasdf"))
self.assertTrue(rand.startswith("udocker"))
self.assertGreater(len(rand.strip()), 56)
rand = uniq.filename(12345)
self.assertTrue(rand.endswith("12345"))
self.assertTrue(rand.startswith("udocker"))
self.assertGreater(len(rand.strip()), 50)
def create_fromlayer(self, imagerepo, tag, layer_file, container_json):
"""Create a container from a layer file exported by Docker.
"""
self.imagerepo = imagerepo
self.tag = tag
if not self.container_id:
self.container_id = Unique().uuid(os.path.basename(self.imagerepo))
if not container_json:
Msg().err("Error: create container: getting json")
return False
container_dir = self.localrepo.setup_container(self.imagerepo,
self.tag,
self.container_id)
if not container_dir:
Msg().err("Error: create container: setting up")
return False
fjson = container_dir + "/container.json"
self.localrepo.save_json(fjson, container_json)
status = self._untar_layers([
layer_file,
], container_dir + "/ROOT")
if not status:
Msg().err("Error: creating container:", self.container_id)
elif not self._chk_container_root():
Msg().out("Warning: check container content:",
self.container_id,
l=Msg.WAR)
return self.container_id
def clone(self):
"""Clone a container by creating a complete copy
"""
source_container_dir = self.localrepo.cd_container(self.container_id)
if not source_container_dir:
Msg().err("Error: source container not found:", self.container_id)
return False
dest_container_id = Unique().uuid(os.path.basename(self.imagerepo))
dest_container_dir = self.localrepo.setup_container(
"CLONING", "inprogress", dest_container_id)
if not dest_container_dir:
Msg().err("Error: create destination container: setting up")
return False
status = FileUtil(source_container_dir).copydir(dest_container_dir)
if not status:
Msg().err("Error: creating container:", dest_container_id)
return False
if not self._chk_container_root(dest_container_id):
Msg().out("Warning: check container content:",
dest_container_id,
l=Msg.WAR)
return dest_container_id
def mktmp(self):
"""Generate a temporary filename"""
while True:
tmp_file = self._tmpdir + '/' + \
Unique().filename(self.basename)
if not os.path.exists(tmp_file):
FileUtil.tmptrash[tmp_file] = True
self.filename = tmp_file
return tmp_file
def load(self, tmp_imagedir, imagerepo=None):
"""Load a Docker file created with docker save, mimic Docker
load. The file is a tarball containing several layers, each
layer has metadata and data content (directory tree) stored
as a tar file.
"""
self._imagerepo = imagerepo
structure = self._load_structure(tmp_imagedir)
if not structure:
Msg().err("Error: failed to load image structure")
return []
if "repositories" in structure and structure["repositories"]:
repositories = self._load_repositories(structure)
else:
if not imagerepo:
imagerepo = Unique().imagename()
tag = Unique().imagetag()
repositories = self._load_image(structure, imagerepo, tag)
return repositories
def _load_manifest(self, structure, manifest):
"""Load OCI manifest"""
try:
tag = manifest["annotations"]["org.opencontainers.image.ref.name"]
except KeyError:
tag = Unique().imagetag()
if '/' in tag and ':' in tag:
(imagerepo, tag) = tag.split(':', 1)
else:
imagerepo = Unique().imagename()
if self._imagerepo:
imagerepo = self._imagerepo
imagetag = imagerepo + ':' + tag
structure["manifest"][imagetag] = dict()
structure["manifest"][imagetag]["json"] = \
self.localrepo.load_json(structure["repolayers"]\
[manifest["digest"]]["layer_f"])
structure["manifest"][imagetag]["json_f"] = \
structure["repolayers"][manifest["digest"]]["layer_f"]
return self._load_image(structure, imagerepo, tag)
def create_fromimage(self, imagerepo, tag):
"""Create a container from an image in the repository.
Since images are stored as layers in tar files, this
step consists in extracting those layers into a ROOT
directory in the appropriate sequence.
first argument: imagerepo
second argument: image tag in that repo
"""
self.imagerepo = imagerepo
self.tag = tag
image_dir = self.localrepo.cd_imagerepo(self.imagerepo, self.tag)
if not image_dir:
Msg().err("Error: create container: imagerepo is invalid")
return False
(container_json, layer_files) = self.localrepo.get_image_attributes()
if not container_json:
Msg().err("Error: create container: getting layers or json")
return False
if not self.container_id:
self.container_id = Unique().uuid(os.path.basename(self.imagerepo))
container_dir = self.localrepo.setup_container(self.imagerepo,
self.tag,
self.container_id)
if not container_dir:
Msg().err("Error: create container: setting up container")
return False
self.localrepo.save_json(container_dir + "/container.json",
container_json)
status = self._untar_layers(layer_files, container_dir + "/ROOT")
if not status:
Msg().err("Error: creating container:", self.container_id)
elif not self._chk_container_root():
Msg().out("Warning: check container content:",
self.container_id,
l=Msg.WAR)
return self.container_id
def import_tocontainer(self, tarfile, imagerepo, tag, container_name):
"""Import a tar file containing a simple directory tree possibly
created with Docker export and create local container ready to use"""
if not imagerepo:
imagerepo = "IMPORTED"
tag = "latest"
if not os.path.exists(tarfile) and tarfile != '-':
Msg().err("Error: tar file does not exist:", tarfile)
return False
if container_name:
if self.localrepo.get_container_id(container_name):
Msg().err("Error: container name already exists:",
container_name)
return False
layer_id = Unique().layer_v1()
container_json = self.create_container_meta(layer_id)
container_id = ContainerStructure(self.localrepo).create_fromlayer(
imagerepo, tag, tarfile, container_json)
if container_name:
self.localrepo.set_container_name(container_id, container_name)
return container_id
def import_toimage(self, tarfile, imagerepo, tag, move_tarball=True):
"""Import a tar file containing a simple directory tree possibly
created with Docker export and create local image"""
if not os.path.exists(tarfile) and tarfile != '-':
Msg().err("Error: tar file does not exist: ", tarfile)
return False
self.localrepo.setup_imagerepo(imagerepo)
tag_dir = self.localrepo.cd_imagerepo(imagerepo, tag)
if tag_dir:
Msg().err("Error: tag already exists in repo:", tag)
return False
tag_dir = self.localrepo.setup_tag(tag)
if not tag_dir:
Msg().err("Error: creating repo and tag")
return False
if not self.localrepo.set_version("v1"):
Msg().err("Error: setting repository version")
return False
layer_id = Unique().layer_v1()
layer_file = self.localrepo.layersdir + '/' + layer_id + ".layer"
json_file = self.localrepo.layersdir + '/' + layer_id + ".json"
if move_tarball:
try:
os.rename(tarfile, layer_file)
except (IOError, OSError):
pass
if not os.path.exists(layer_file):
if not FileUtil(tarfile).copyto(layer_file):
Msg().err("Error: in move/copy file", tarfile)
return False
self.localrepo.add_image_layer(layer_file)
self.localrepo.save_json("ancestry", [layer_id])
container_json = self.create_container_meta(layer_id)
self.localrepo.save_json(json_file, container_json)
self.localrepo.add_image_layer(json_file)
Msg().out("Info: added layer", layer_id, l=Msg.INF)
return layer_id
def clone_fromfile(self, clone_file):
"""Create a cloned container from a file containing a cloned container
exported by udocker.
"""
if not self.container_id:
self.container_id = Unique().uuid(os.path.basename(self.imagerepo))
container_dir = self.localrepo.setup_container("CLONING", "inprogress",
self.container_id)
if not container_dir:
Msg().err("Error: create container: setting up")
return False
status = self._untar_layers([
clone_file,
], container_dir)
if not status:
Msg().err("Error: creating container clone:", self.container_id)
elif not self._chk_container_root():
Msg().out("Warning: check container content:",
self.container_id,
l=Msg.WAR)
return self.container_id
def test_05_imagetag(self):
"""Test05 Unique.imagetag()."""
uniq = Unique()
rand = uniq.imagetag()
self.assertEqual(len(rand), 10)
def run(self, container_id):
"""Execute a Docker container using singularity.
This is the main method invoked to run a container with singularity.
* argument: container_id or name
* options: many via self.opt see the help
"""
if os.path.isdir(
FileBind(self.localrepo, container_id).container_orig_dir):
FileBind(self.localrepo, container_id).restore() # legacy 1.1.3
Config.conf['sysdirs_list'] = (
# "/dev", "/proc", "/sys",
"/etc/resolv.conf", "/etc/host.conf",
"/lib/modules",
)
# setup execution
exec_path = self._run_init(container_id)
if not exec_path:
return 2
self.opt["cmd"][0] = exec_path.replace(self.container_root + "/", "")
self._run_invalid_options()
self._make_container_directories()
self.select_singularity()
self._run_as_root()
# set environment variables
self._run_env_set()
if Msg.level >= Msg.DBG:
singularity_debug = ["--debug", "-v", ]
elif self._has_option("--silent"):
singularity_debug = ["--silent", ]
elif self._has_option("--quiet"):
singularity_debug = ["--quiet", ]
else:
singularity_debug = []
if self.executable.startswith(self.localrepo.bindir):
Config.conf['singularity_options'].extend(["-u", ])
#if FileUtil("nvidia-smi").find_exec():
# Config.conf['singularity_options'].extend(["--nv", ])
singularity_vol_list = self._get_volume_bindings()
# build the actual command
self.execution_id = Unique().uuid(self.container_id)
cmd_l = self._set_cpu_affinity()
cmd_l.append(self.executable)
cmd_l.extend(singularity_debug)
cmd_l.append("exec")
cmd_l.extend(Config.conf['singularity_options'])
if self.opt["cwd"]:
cmd_l.extend(["--pwd", self.opt["cwd"], ])
cmd_l.extend(singularity_vol_list)
cmd_l.append(self.container_root)
cmd_l.extend(self.opt["cmd"])
Msg().out("CMD =", cmd_l, l=Msg.VER)
# if not --hostenv clean the environment
self._run_env_cleanup_dict()
# execute
self._run_banner(self.opt["cmd"][0], '/')
status = subprocess.call(cmd_l, shell=False, close_fds=True, \
env=os.environ.update(self._singularity_env_get()))
return status
def run(self, container_id):
"""Execute a Docker container using runc. This is the main method
invoked to run the a container with runc.
* argument: container_id or name
* options: many via self.opt see the help
"""
Config.conf['sysdirs_list'] = (
"/etc/resolv.conf",
"/etc/host.conf",
# "/etc/passwd", "/etc/group",
)
# setup execution
if not self._run_init(container_id):
return 2
self._run_invalid_options()
self._container_specfile = "config.json"
if self.container_dir:
if self.localrepo.iswriteable_container(container_id):
self._container_specdir = self.container_dir
else:
self._container_specdir = FileUtil("SPECDIR").mktmpdir()
FileUtil(self._container_specdir).register_prefix()
self._container_specfile = \
self._container_specdir + '/' + self._container_specfile
self._filebind = FileBind(self.localrepo, container_id)
self._filebind.setup()
self.select_runc()
# create new OCI spec file
if not self._load_spec(new=True):
return 4
self._uid_check()
# if not --hostenv clean the environment
self._run_env_cleanup_list()
# set environment variables
self._run_env_set()
self._set_spec()
if (Config.conf['runc_nomqueue']
or (Config.conf['runc_nomqueue'] is None
and not HostInfo().oskernel_isgreater([4, 8, 0]))):
self._del_mount_spec("mqueue", "/dev/mqueue")
self._del_mount_spec("cgroup", "/sys/fs/cgroup")
self._del_namespace_spec("network")
self._set_id_mappings()
self._add_volume_bindings()
self._add_devices()
self._add_capabilities_spec()
self._mod_mount_spec("shm", "/dev/shm", {"options": ["size=2g"]})
self._proot_overlay()
self._save_spec()
if Msg.level >= Msg.DBG:
runc_debug = [
"--debug",
]
Msg().out(
json.dumps(self._container_specjson, indent=4, sort_keys=True))
else:
runc_debug = []
# build the actual command
self.execution_id = Unique().uuid(self.container_id)
cmd_l = self._set_cpu_affinity()
cmd_l.append(self.executable)
cmd_l.extend(runc_debug)
cmd_l.extend(["--root", self._container_specdir, "run"])
cmd_l.extend(["--bundle", self._container_specdir, self.execution_id])
Msg().err("CMD =", cmd_l, l=Msg.VER)
self._run_banner(self.opt["cmd"][0], '%')
if sys.stdout.isatty():
return self.run_pty(cmd_l)
return self.run_nopty(cmd_l)
def test_01_init(self):
"""Test01 Unique() constructor."""
uniq = Unique()
self.assertEqual(uniq.def_name, "udocker")
def test_06_layer_v1(self):
"""Test06 Unique.layer_v1()."""
uniq = Unique()
rand = uniq.layer_v1()
self.assertEqual(len(rand), 64)
def test_02__rnd(self):
"""Test02 Unique._rnd()."""
uniq = Unique()
rand = uniq._rnd(64)
self.assertIsInstance(rand, str)
self.assertEqual(len(rand), 64)
def test_04_imagename(self):
"""Test04 Unique.imagename()."""
uniq = Unique()
rand = uniq.imagename()
self.assertEqual(len(rand), 16)