def build_from_spec(spec_file=None,
build_dir=None,
size=None,
sudopw=None,
build_folder=False,
debug=False):
'''build_from_spec will build a "spec" file in a "build_dir" and return the directory
:param spec_file: the spec file, called "Singuarity"
:param sudopw: the sudopw for Singularity, root should provide ''
:param build_dir: the directory to build in. If not defined, will use tmpdir.
:param size: the size of the image
:param build_folder: "build" the image into a folder instead. Default False
:param debug: ask for verbose output from builder
'''
if spec_file == None:
spec_file = "Singularity"
if build_dir == None:
build_dir = tempfile.mkdtemp()
bot.logger.debug("Building in directory %s", build_dir)
# Copy the spec to a temporary directory
bot.logger.debug("Spec file set to %s", spec_file)
spec_path = "%s/%s" % (build_dir, os.path.basename(spec_file))
bot.logger.debug("Spec file for build should be in %s", spec_path)
# If it's not already there
if not os.path.exists(spec_path):
shutil.copyfile(spec_file, spec_path)
image_path = "%s/image" % (build_dir)
# Run create image and bootstrap with Singularity command line tool.
if sudopw is not None:
cli = Singularity(sudopw=sudopw, debug=debug)
else:
cli = Singularity(debug=debug)
print("\nCreating and bootstrapping image...")
# Does the user want to "build" into a folder or image?
if build_folder == True:
bot.logger.debug("build_folder is true, creating %s", image_path)
os.mkdir(image_path)
else:
cli.create(image_path, size=size, sudo=True)
result = cli.bootstrap(image_path=image_path, spec_path=spec_path)
print(result)
# If image, rename based on hash
if build_folder == False:
version = get_image_file_hash(image_path)
final_path = "%s/%s" % (build_dir, version)
os.rename(image_path, final_path)
image_path = final_path
bot.logger.debug("Built image: %s", image_path)
return image_path
def create_container(container=None, do_import=False):
'''supporting function to create empty container
'''
cli = Singularity()
tmpdir = tempfile.mkdtemp()
if container is None:
container = "%s/container.img" % (tmpdir)
if do_import is True:
cli.importcmd(container, 'docker://ubuntu')
return cli.create(container)
def unpack_node(image_path, name=None, output_folder=None, size=None):
'''unpackage node is intended to unpackage a node that was packaged with
package_node. The image should be a .tgz file. The general steps are to:
1. Package the node using the package_node function
2. Transfer the package somewhere that Singularity is installed'''
if not image_path.endswith(".tgz"):
bot.error(
"The image_path should end with .tgz. Did you create with package_node?"
)
sys.exit(1)
if output_folder is None:
output_folder = os.path.dirname(os.path.abspath(image_path))
image_name = os.path.basename(image_path)
if name is None:
name = image_name.replace('.tgz', '.img')
if not name.endswith('.img'):
name = "%s.img" % (name)
bot.debug("Preparing to unpack %s to %s." % (image_name, name))
unpacked_image = "%s/%s" % (output_folder, name)
S = Singularity(sudo=True)
S.create(image_path=unpacked_image, size=size)
cmd = [
"gunzip", "-dc", image_path, "|", "sudo", "singularity", "import",
unpacked_image
]
output = run_command(cmd)
# TODO: singularity mount the container, cleanup files (/etc/fstab,...)
# and add your custom singularity files.
return unpacked_image
def test_test_container(self):
'''the retry function should return None if result is None
'''
from singularity.build.utils import test_container
from singularity.cli import Singularity
cli = Singularity()
unfinished_container = cli.create("%s/container.img" % self.tmpdir)
print("Case 1: Testing that errored container does not run")
result = test_container(unfinished_container)
self.assertEqual(result["return_code"], 255)
print("Case 2: Testing that finished container does run")
finished_container = cli.importcmd(unfinished_container,
'docker://ubuntu')
result = test_container(finished_container)
self.assertEqual(result["return_code"], 0)
class TestClient(unittest.TestCase):
def setUp(self):
self.pwd = get_installdir()
self.cli = Singularity()
self.tmpdir = tempfile.mkdtemp()
def tearDown(self):
shutil.rmtree(self.tmpdir)
def test_commands(self):
print('Testing client.create command')
container = "%s/container.img" % (self.tmpdir)
created_container = self.cli.create(container)
self.assertEqual(created_container, container)
self.assertTrue(os.path.exists(created_container))
os.remove(container)
print("Testing client.pull command")
print("...Case 1: Testing naming pull by image name")
image = self.cli.pull("shub://vsoch/singularity-images",
pull_folder=self.tmpdir)
self.assertTrue(os.path.exists(image))
self.assertTrue('vsoch-singularity-images-master' in image)
print(image)
os.remove(image)
print("...Case 3: Testing docker pull")
container = self.cli.pull("docker://ubuntu:14.04",
pull_folder=self.tmpdir)
self.assertTrue("ubuntu:14.04" in container)
print(container)
self.assertTrue(os.path.exists(container))
print('Testing client.execute command')
result = self.cli.execute(container, 'ls /')
print(result)
self.assertTrue('bin\nboot\ndev' in result)
print("Testing client.inspect command")
result = self.cli.inspect(container, quiet=True)
labels = json.loads(result)
self.assertTrue('data' in labels)
os.remove(container)
From: ubuntu:latest
%runscript
exec "Hello World!"
'''
os.chdir(replicates)
with open('Singularity','w') as filey:
filey.writelines(runscript)
from singularity.cli import Singularity
cli = Singularity()
for num in range(0,100):
container_name = 'ubuntu-hello-world-%s.img' %(num)
cli.create(container_name)
cli.bootstrap(container_name,'Singularity')
# ALL SINGULARITY HUB
containers = shub.get_containers()
os.chdir(hub)
for container_name,container in containers.items():
for branch, manifest in container.items():
name = manifest['name'].replace('/','-')
uncompressed = "%s-%s.img" %(name,branch)
if not os.path.exists(uncompressed):
try:
image = shub.pull_container(manifest,
download_folder=hub,
# Get general help:
S.help()
# These are the defaults, which can be specified
S = Singularity(sudo=True,verbose=False)
# Let's define a path to an image
# wget http://www.vbmis.com/bmi/project/singularity/package_image/ubuntu:latest-2016-04-06.img
image_path = 'ubuntu:latest-2016-04-06.img'
# Run singularity --exec
S.execute(image_path=image_path,command='ls')
# $'docker2singularity.sh\nget_docker_container_id.sh\nget_docker_meta.py\nmakeBases.py\nsingularity\nubuntu:latest-2016-04-06.img\n'
# These are the defaults, which can be specified
# For any function you can get the docs:
S.help(command="exec")
# or return as string
help = S.help(command="exec",stdout=False)
# export an image, default export_type="tar" , pipe=False , output_file = None will produce file in tmp
tmptar = S.export(image_path=image_path)
# create an empty image
S.create(image_path='test.img')
# import a docker image - no need to specify the file, the image name works
# still under development
docker_image = S.docker2singularity("ubuntu:latest")
# Create a client
S = Singularity()
# Get general help:
S.help()
# These are the defaults, which can be specified
S = Singularity(sudo=False, debug=False)
# Note that the "create" and "import" workflow is deprecated in favor of build
# https://singularityware.github.io/docs-build
image = S.build('myimage.simg', 'docker://ubuntu:latest') # requires sudo
# (Deprecated) create an image and import into it
image = S.create('myimage.simg')
S.importcmd(image, 'docker://ubuntu:latest')
# Execute command to container
result = S.execute(image, command='cat /singularity')
print(result)
'''
'#!/bin/sh\n\nexec "/bin/bash"\n'
'''
# For any function you can get the docs:
S.help(command="exec")
# export an image to tar
tar = S.export(image)
class TestPackage(unittest.TestCase):
def setUp(self):
self.pwd = get_installdir()
self.cli = Singularity()
self.tmpdir = tempfile.mkdtemp()
self.image1 = "%s/tests/data/busybox-2016-02-16.img" % (self.pwd)
self.image2 = "%s/tests/data/cirros-2016-01-04.img" % (self.pwd)
# We can't test creating the packages, because requires sudo :/
self.pkg1 = "%s/tests/data/busybox-2016-02-16.img.zip" % (self.pwd)
self.pkg2 = "%s/tests/data/cirros-2016-01-04.img.zip" % (self.pwd)
def tearDown(self):
shutil.rmtree(self.tmpdir)
def test_packaging(self):
# Check content of packages
print("TESTING content extraction of packages")
self.pkg1_includes = list_package(self.pkg1)
self.pkg2_includes = list_package(self.pkg2)
includes = ['files.txt', 'VERSION', 'NAME', 'folders.txt']
for pkg_includes in [self.pkg1_includes, self.pkg2_includes]:
[self.assertTrue(x in pkg_includes) for x in includes]
self.assertTrue(os.path.basename(self.image1) in self.pkg1_includes)
self.assertTrue(os.path.basename(self.image2) in self.pkg2_includes)
print("TESTING loading packages...")
pkg1_loaded = load_package(self.pkg1)
self.assertTrue(len(pkg1_loaded["files.txt"]) == 12)
self.assertTrue(len(pkg1_loaded["folders.txt"]) == 18)
# Did it extract successfully?
image1_extraction = pkg1_loaded[os.path.basename(self.image1)]
self.assertTrue(os.path.exists(image1_extraction))
shutil.rmtree(os.path.dirname(image1_extraction))
'''
def test_estimate_from_size(self):
"""test estimate from size will ensure that we correctly estimate the size
of a container build plus some optional padding
Note: we currently can't test this in CI due to needing
sudo password for bootstrap
"""
from singularity.package import estimate_image_size
spec = "From: ubuntu:16.04\nBootstrap: docker"
spec_file = "%s/Singularity" %(self.tmpdir)
spec_file = write_file(spec_file,spec)
print("Case 1: Testing that no specification of padding uses default 200")
image_size = estimate_image_size(spec_file)
'''
def test_package(self):
'''test package will ensure that we can generate an image package'''
from singularity.package import package
container = self.cli.create("%s/container.img" % self.tmpdir)
container = self.cli.importcmd(container, "docker://ubuntu")
image_package = package(image_path=container,
output_folder=self.tmpdir,
S=self.cli)
self.assertTrue(os.path.exists(image_package))
