def main(args, parser, subparser):
# We can only continue if singularity is installed
if check_install() is not True:
bot.error("Cannot find Singularity! Is it installed?")
sys.exit(1)
# Output folder will be pwd if not specified
output_folder = os.getcwd()
if args.outfolder is not None:
output_folder = os.getcwd()
# If we are given an image, ensure full path
if args.image is not None:
if not os.path.exists(args.image):
cli = Singularity(debug=args.debug)
image = cli.pull(args.image)
if image is None:
bot.error("Cannot find image. Exiting.")
sys.exit(1)
# the user wants to make a tree
if args.tree is True:
from singularity.cli.app import make_tree
make_tree(image)
clean_up(image, existed)
else:
print("Please specify an image to inspect with --image")
subparser.print_help()
def package(image_path,
spec_path=None,
output_folder=None,
remove_image=False,
verbose=False,
S=None):
'''generate a zip (including the image) to a user specified output_folder.
:param image_path: full path to singularity image file
:param runscript: if True, will extract runscript to include in package as runscript
:param software: if True, will extract files.txt and folders.txt to package
:param remove_image: if True, will not include original image in package (default,False)
:param verbose: be verbose when using singularity --export (default,False)
:param S: the Singularity object (optional) will be created if not required.
'''
# Run create image and bootstrap with Singularity command line tool.
S = Singularity(debug=verbose)
file_obj, tar = get_image_tar(image_path, S=S)
members = tar.getmembers()
image_name, ext = os.path.splitext(os.path.basename(image_path))
zip_name = "%s.zip" % (image_name.replace(" ", "_"))
# Include the image in the package?
to_package = dict()
if not remove_image:
to_package["files"] = [image_path]
# If the specfile is provided, it should also be packaged
if spec_path is not None:
singularity_spec = "".join(read_file(spec_path))
to_package['Singularity'] = singularity_spec
to_package["VERSION"] = get_image_file_hash(image_path)
try:
inspection = S.inspect(image_path)
to_package["inspect.json"] = inspection
inspection = json.loads(inspection)
to_package['runscript'] = inspection['data']['attributes']['runscript']
except:
bot.warning("Trouble extracting container metadata with inspect.")
bot.info("Adding software list to package.")
files = [x.path for x in members if x.isfile()]
folders = [x.path for x in members if x.isdir()]
to_package["files.txt"] = files
to_package["folders.txt"] = folders
# Do zip up here - let's start with basic structures
zipfile = zip_up(to_package,
zip_name=zip_name,
output_folder=output_folder)
bot.debug("Package created at %s" % (zipfile))
if not delete_image_tar(file_obj, tar):
bot.warning("Could not clean up temporary tarfile.")
# return package to user
return zipfile
def main(args, parser, subparser):
# We can only continue if singularity is installed
if check_install() is not True:
bot.error("Cannot find Singularity! Is it installed?")
sys.exit(1)
# Output folder will be pwd if not specified
output_folder = os.getcwd()
if args.outfolder is not None:
output_folder = os.getcwd()
# If we are given an image, ensure full path
if args.image is not None:
image = args.image
if not os.path.exists(image):
cli = Singularity(debug=args.debug)
image = cli.pull(image)
if image is None:
bot.error("Cannot find image. Exiting.")
sys.exit(1)
remove_image = not args.include_image
package(image_path=image,
output_folder=output_folder,
remove_image=remove_image)
else:
print("Please specify an image to package with --image")
subparser.print_help()
def extract_apps(image_path, app_names, S=None, verbose=True):
''' extract app will extract metadata for one or more apps
Parameters
==========
image_path: the absolute path to the image
app_name: the name of the app under /scif/apps
'''
if S is None:
S = Singularity(debug=verbose, sudo=True)
if not isinstance(app_names, list):
app_names = [app_names]
file_obj, tar = get_image_tar(image_path, S=S)
members = tar.getmembers()
apps = dict()
for app_name in app_names:
metadata = dict()
# Inspect: labels, env, runscript, tests, help
try:
inspection = json.loads(S.inspect(image_path, app=app_name))
del inspection['data']['attributes']['deffile']
metadata['inspect'] = inspection
# If illegal characters prevent load, not much we can do
except:
pass
base = '/scif/apps/%s' % app_name
metadata['files'] = [x.path for x in members if base in x.path]
apps[app_name] = metadata
return apps
def package(image_path,output_folder=None,runscript=True,software=True,remove_image=False,verbose=False):
'''package will take an image and generate a zip (including the image
to a user specified output_folder.
:param image_path: full path to singularity image file
:param runscript: if True, will extract runscript to include in package as runscript
:param software: if True, will extract files.txt and folders.txt to package
:param remove_image: if True, will not include original image in package (default,False)
:param verbose: be verbose when using singularity --export (default,False)
'''
S = Singularity(verbose=verbose)
tmptar = S.export(image_path=image_path,pipe=False)
tar = tarfile.open(tmptar)
members = tar.getmembers()
image_name = os.path.basename(image_path)
zip_name = "%s.zip" %(image_name.replace(" ","_"))
# Include the image in the package?
if remove_image:
to_package = dict()
else:
to_package = {image_name:image_path}
# Package the image with an md5 sum as VERSION
version = get_image_hash(image_path)
to_package["VERSION"] = version
# Look for runscript
if runscript == True:
try:
runscript_member = tar.getmember("./singularity")
runscript_file = tar.extractfile("./singularity")
runscript = runscript_file.read()
to_package["runscript"] = runscript
print("Found runscript!")
# Try to extract input args, only python supported, will return None otherwise
params_json = get_runscript_parameters(runscript=runscript,
name=image_name,
version=version)
if params_json != None:
print('Extracted runscript params!')
to_package['%s.json' %(image_name)] = params_json
except KeyError:
print("No runscript found in image!")
if software == True:
print("Adding software list to package!")
files = [x.path for x in members if x.isfile()]
folders = [x.path for x in members if x.isdir()]
to_package["files.txt"] = files
to_package["folders.txt"] = folders
# Do zip up here - let's start with basic structures
zipfile = zip_up(to_package,zip_name=zip_name,output_folder=output_folder)
print("Package created at %s" %(zipfile))
# return package to user
return zipfile
def setUp(self):
self.tmpdir = tempfile.mkdtemp()
self.cli = Singularity()
self.container = self.cli.pull('docker://ubuntu:16.04',
pull_folder=self.tmpdir)
self.comparator = self.cli.pull('docker://ubuntu:12.04',
pull_folder=self.tmpdir)
def get_memory_tar(image_path):
'''get an in memory tar of an image. Use carefully, not as reliable
as get_image_tar
'''
cli = Singularity()
byte_array = cli.export(image_path)
file_object = io.BytesIO(byte_array)
tar = tarfile.open(mode="r|*", fileobj=file_object)
return (file_object, tar)
def get_memory_tar(image_path):
'''get an in memory tar of an image (does not require sudo!)'''
cli = Singularity()
if "pancakes" in os.environ:
del os.environ['pancakes']
byte_array = cli.export(image_path)
file_object = io.BytesIO(byte_array)
tar = tarfile.open(mode="r|*", fileobj=file_object)
return (file_object, tar)
def setUp(self):
self.pwd = get_installdir()
self.cli = Singularity()
self.tmpdir = tempfile.mkdtemp()
self.image1 = "%s/tests/data/busybox-2016-02-16.img" % (self.pwd)
self.image2 = "%s/tests/data/cirros-2016-01-04.img" % (self.pwd)
# We can't test creating the packages, because requires sudo :/
self.pkg1 = "%s/tests/data/busybox-2016-02-16.img.zip" % (self.pwd)
self.pkg2 = "%s/tests/data/cirros-2016-01-04.img.zip" % (self.pwd)
def create_container(container=None, do_import=False):
'''supporting function to create empty container
'''
cli = Singularity()
tmpdir = tempfile.mkdtemp()
if container is None:
container = "%s/container.img" % (tmpdir)
if do_import is True:
cli.importcmd(container, 'docker://ubuntu')
return cli.create(container)
def main(args, parser, subparser):
# We can only continue if singularity is installed
if check_install() is not True:
bot.error("Cannot find Singularity! Is it installed?")
sys.exit(1)
# Output folder will be pwd if not specified
output_folder = os.getcwd()
if args.outfolder is not None:
output_folder = os.getcwd()
# If we are given an image, ensure full path
image = args.image
if image is not None:
existed = True
if not os.path.exists(image):
cli = Singularity(debug=args.debug)
image = cli.pull(image)
existed = False
if image is None:
bot.error("Cannot find image. Exiting.")
sys.exit(1)
# The user wants to estimate the os
if args.os is True:
from singularity.analysis.classify import estimate_os
estimated_os = estimate_os(container=image)
print(estimated_os)
# The user wants to get a list of all os
elif args.oscalc is True:
from singularity.analysis.classify import estimate_os
estimated_os = estimate_os(container=image, return_top=False)
print(estimated_os["SCORE"].to_dict())
# The user wants to get a list of tags
elif args.tags is True:
from singularity.analysis.classify import get_tags
tags = get_tags(container=image)
print(tags)
# The user wants to plot image vs. the docker os
elif args.osplot is True:
from singularity.app import plot_os_sims
plot_os_sims(image)
clean_up(image, existed)
else:
print("Please specify an image to analyze with --image")
subparser.print_help()
class TestAnalysisCompare(unittest.TestCase):
def setUp(self):
self.tmpdir = tempfile.mkdtemp()
self.cli = Singularity()
self.container = self.cli.pull('docker://ubuntu:16.04',
pull_folder=self.tmpdir)
self.comparator = self.cli.pull('docker://ubuntu:12.04',
pull_folder=self.tmpdir)
def tearDown(self):
os.remove(self.container)
os.remove(self.comparator)
shutil.rmtree(self.tmpdir)
def test_container_similarity(self):
print(
"Testing singularity.analysis.compare.container_similarity_vector")
import pandas
from singularity.analysis.compare import container_similarity_vector
from singularity.package import get_packages
packages_set = get_packages('docker-os')[0:2]
vector = container_similarity_vector(container1=self.container,
custom_set=packages_set)
self.assertTrue('files.txt' in vector)
self.assertTrue(isinstance(vector['files.txt'], pandas.DataFrame))
print(
"Testing singularity.analysis.compare.compare_singularity_images")
from singularity.analysis.compare import compare_singularity_images
sim = compare_singularity_images(self.container, self.comparator)
self.assertTrue(isinstance(sim, pandas.DataFrame))
self.assertTrue(sim.loc[self.container, self.comparator] -
0.4803262269280298 < 0.01)
print("Testitng singularity.analysis.compare.compare_containers")
from singularity.analysis.compare import compare_containers
comparison = compare_containers(self.container, self.comparator)
self.assertTrue('files.txt' in comparison)
for key in ['total1', 'total2', 'intersect', 'unique2', 'unique1']:
self.assertTrue(key in comparison['files.txt'])
print("Testing singularity.analysis.compare.calculate_similarity")
from singularity.analysis.compare import calculate_similarity
sim = calculate_similarity(self.container, self.comparator)
self.assertTrue(sim['files.txt'] - 0.4921837537163134 < 0.01)
def test_information_coefficient(self):
print("Testing singularity.analysis.metrics.information_coefficient")
from singularity.analysis.metrics import information_coefficient
self.assertEqual(information_coefficient(100, 100, range(0, 50)), 0.5)
self.assertEqual(information_coefficient(100, 100, range(0, 100)), 1.0)
def docker2singularity(docker_image,output_folder=None):
'''docker2singulrity is a wrapper for the Singularity.docker2singularity
client function. Does not currently include runscript (/singularity) in image,
but does export full docker image spec under /singularity.json
:param docker_image: the full docker repo/image,eg "ubuntu:latest"
:param output_folder: the output folder to create the image in. If not
specified, will use pwd.
'''
S = Singularity()
docker_image = S.docker2singularity(docker_image=docker_image,
output_dir=output_folder)
return docker_image
def main(args, parser, subparser):
# We can only continue if singularity is installed
if check_install() is not True:
bot.error("Cannot find Singularity! Is it installed?")
sys.exit(1)
# Output folder will be pwd if not specified
output_folder = os.getcwd()
if args.outfolder is not None:
output_folder = os.getcwd()
if args.images is not None:
image1, image2 = args.images.split(',')
bot.debug("Image1: %s" % image1)
bot.debug("Image2: %s" % image2)
images = dict()
cli = Singularity(debug=args.debug)
for image in [image1, image2]:
existed = True
if not os.path.exists(image):
image = cli.pull(image)
existed = False
images[image] = existed
# Just for clarity
image1, image2 = list(images.keys())
# the user wants to make a similarity tree
if args.simtree is True:
from singularity.cli.app import make_sim_tree
make_sim_tree(image1, image2)
# the user wants to make a difference tree
elif args.subtract is True:
from singularity.cli.app import make_diff_tree
make_diff_tree(image1, image2)
else: # If none specified, just print score
from singularity.analysis.compare import calculate_similarity
score = calculate_similarity(image1, image2, by="files.txt")
print(score["files.txt"])
for image, existed in images.items():
clean_up(image, existed)
else:
print("Please specify images to compare with --images")
subparser.print_help()
def build_from_spec(spec_file=None,
build_dir=None,
build_folder=False,
sandbox=False,
isolated=False,
debug=False):
'''build_from_spec will build a "spec" file in a "build_dir" and return the directory
:param spec_file: the spec file, called "Singuarity"
:param build_dir: the directory to build in. If not defined, will use tmpdir.
:param isolated: "build" the image inside an isolated environment (>2.4)
:param sandbox: ask for a sandbox build
:param debug: ask for verbose output from builder
'''
if spec_file == None:
spec_file = "Singularity"
if build_dir == None:
build_dir = tempfile.mkdtemp()
bot.debug("Building in directory %s" %build_dir)
# Copy the spec to a temporary directory
bot.debug("Spec file set to %s" % spec_file)
spec_path = os.path.abspath(spec_file)
bot.debug("Spec file for build should be in %s" %spec_path)
image_path = "%s/build.simg" %(build_dir)
# Run create image and bootstrap with Singularity command line tool.
cli = Singularity(debug=debug)
print("\nBuilding image...")
# Does the user want to "build" into a folder or image?
result = cli.build(image_path=image_path,
spec_path=spec_path,
sandbox=sandbox,
isolated=isolated)
print(result)
# If image, rename based on hash
if sandbox is False:
version = get_image_file_hash(image_path)
final_path = "%s/%s.simg" %(build_dir,version)
os.rename(image_path,final_path)
image_path = final_path
bot.debug("Built image: %s" %image_path)
return image_path
def get_image_tar(image_path, S=None):
'''get an image tar, either written in memory or to
the file system. file_obj will either be the file object,
or the file itself.
'''
bot.debug('Generate file system tar...')
if S is None:
S = Singularity()
file_obj = S.export(image_path=image_path)
if file_obj is None:
bot.error("Error generating tar, exiting.")
sys.exit(1)
tar = tarfile.open(file_obj)
return file_obj, tar
def pull(self, images, file_name=None, decompress=True):
bot.debug('Execution of PULL for %s images' % len(images))
for image in images:
# If we need to decompress, it's old ext3 format
if decompress is True:
ext = 'img.gz'
else:
ext = 'simg' # squashfs
q = parse_image_name(image, ext=ext)
# Verify image existence, and obtain id
url = "%s/container/%s/%s:%s" % (self.base, q['collection'],
q['image'], q['tag'])
bot.debug('Retrieving manifest at %s' % url)
manifest = self.get(url)
bot.debug(manifest)
if file_name is None:
file_name = q['storage'].replace('/', '-')
image_file = self.download(url=manifest['image'],
file_name=file_name,
show_progress=True)
bot.debug('Retrieved image file %s' % image_file)
if os.path.exists(image_file) and decompress is True:
# If compressed, decompress
try:
cli = Singularity()
sys.stdout.write('Decompressing image ')
bot.spinner.start()
image_file = cli.decompress(image_file, quiet=True)
except KeyboardInterrupt:
bot.warning('Decompression cancelled.')
except:
bot.info('Image is not compressed.')
image_name = image_file.replace('.gz', '')
image_file = shutil.move(image_file, image_name)
pass
bot.spinner.stop()
bot.custom(prefix="Success!", message=image_file)
def get_image_hash(image_path,
level=None,
level_filter=None,
include_files=None,
skip_files=None,
version=None):
'''get_image_hash will generate a sha1 hash of an image, depending on a level
of reproducibility specified by the user. (see function get_levels for descriptions)
the user can also provide a level_filter manually with level_filter (for custom levels)
:param level: the level of reproducibility to use, which maps to a set regular
expression to match particular files/folders in the image. Choices are in notes.
:param skip_files: an optional list of files to skip
:param include_files: an optional list of files to keep (only if level not defined)
:param version: the version to use. If not defined, default is 2.3
::notes
LEVEL DEFINITIONS
The level definitions come down to including folders/files in the comparison. For files
that Singularity produces on the fly that might be different (timestamps) but equal content
(eg for a replication) we hash the content ("assess_content") instead of the file.
'''
# First get a level dictionary, with description and regexp
if level_filter is not None:
file_filter = level_filter
elif level is None:
file_filter = get_level("RECIPE",
version=version,
include_files=include_files,
skip_files=skip_files)
else:
file_filter = get_level(level,
version=version,
skip_files=skip_files,
include_files=include_files)
cli = Singularity()
file_obj, tar = get_memory_tar(image_path)
hasher = hashlib.md5()
for member in tar:
member_name = member.name.replace('.', '', 1)
# For files, we either assess content, or include the file
if member.isdir() or member.issym():
continue
elif assess_content(member, file_filter):
content = extract_content(image_path, member.name, cli)
hasher.update(content)
elif include_file(member, file_filter):
buf = member.tobuf()
hasher.update(buf)
digest = hasher.hexdigest()
file_obj.close()
return digest
def setUp(self):
self.pwd = get_installdir()
self.cli = Singularity()
self.tmpdir = tempfile.mkdtemp()
self.image1 = "%s/tests/data/busybox-2017-10-21.simg" %(self.pwd)
self.image2 = "%s/tests/data/cirros-2017-10-21.simg" %(self.pwd)
self.pkg1 = "%s/tests/data/busybox-2017-10-21.zip" %(self.pwd)
self.pkg2 = "%s/tests/data/cirros-2017-10-21.zip" %(self.pwd)
def test_test_container(self):
'''the retry function should return None if result is None
'''
from singularity.build.utils import test_container
from singularity.cli import Singularity
cli = Singularity()
unfinished_container = cli.create("%s/container.img" % self.tmpdir)
print("Case 1: Testing that errored container does not run")
result = test_container(unfinished_container)
self.assertEqual(result["return_code"], 255)
print("Case 2: Testing that finished container does run")
finished_container = cli.importcmd(unfinished_container,
'docker://ubuntu')
result = test_container(finished_container)
self.assertEqual(result["return_code"], 0)
def get_singularity_client(**kwargs):
try:
import singularity
from singularity.cli import Singularity
except ImportError:
raise ImportError(
"the singularity python package is required for this")
return Singularity(**kwargs)
def build_from_spec(spec_file=None,
build_dir=None,
size=None,
sudopw=None,
build_folder=False,
debug=False):
'''build_from_spec will build a "spec" file in a "build_dir" and return the directory
:param spec_file: the spec file, called "Singuarity"
:param sudopw: the sudopw for Singularity, root should provide ''
:param build_dir: the directory to build in. If not defined, will use tmpdir.
:param size: the size of the image
:param build_folder: "build" the image into a folder instead. Default False
:param debug: ask for verbose output from builder
'''
if spec_file == None:
spec_file = "Singularity"
if build_dir == None:
build_dir = tempfile.mkdtemp()
bot.logger.debug("Building in directory %s", build_dir)
# Copy the spec to a temporary directory
bot.logger.debug("Spec file set to %s", spec_file)
spec_path = "%s/%s" % (build_dir, os.path.basename(spec_file))
bot.logger.debug("Spec file for build should be in %s", spec_path)
# If it's not already there
if not os.path.exists(spec_path):
shutil.copyfile(spec_file, spec_path)
image_path = "%s/image" % (build_dir)
# Run create image and bootstrap with Singularity command line tool.
if sudopw is not None:
cli = Singularity(sudopw=sudopw, debug=debug)
else:
cli = Singularity(debug=debug)
print("\nCreating and bootstrapping image...")
# Does the user want to "build" into a folder or image?
if build_folder == True:
bot.logger.debug("build_folder is true, creating %s", image_path)
os.mkdir(image_path)
else:
cli.create(image_path, size=size, sudo=True)
result = cli.bootstrap(image_path=image_path, spec_path=spec_path)
print(result)
# If image, rename based on hash
if build_folder == False:
version = get_image_file_hash(image_path)
final_path = "%s/%s" % (build_dir, version)
os.rename(image_path, final_path)
image_path = final_path
bot.logger.debug("Built image: %s", image_path)
return image_path
class TestClient(unittest.TestCase):
def setUp(self):
self.pwd = get_installdir()
self.cli = Singularity()
self.tmpdir = tempfile.mkdtemp()
def tearDown(self):
shutil.rmtree(self.tmpdir)
def test_commands(self):
print('Testing client.create command')
container = "%s/container.img" % (self.tmpdir)
created_container = self.cli.create(container)
self.assertEqual(created_container, container)
self.assertTrue(os.path.exists(created_container))
os.remove(container)
print("Testing client.pull command")
print("...Case 1: Testing naming pull by image name")
image = self.cli.pull("shub://vsoch/singularity-images",
pull_folder=self.tmpdir)
self.assertTrue(os.path.exists(image))
self.assertTrue('vsoch-singularity-images-master' in image)
print(image)
os.remove(image)
print("...Case 3: Testing docker pull")
container = self.cli.pull("docker://ubuntu:14.04",
pull_folder=self.tmpdir)
self.assertTrue("ubuntu:14.04" in container)
print(container)
self.assertTrue(os.path.exists(container))
print('Testing client.execute command')
result = self.cli.execute(container, 'ls /')
print(result)
self.assertTrue('bin\nboot\ndev' in result)
print("Testing client.inspect command")
result = self.cli.inspect(container, quiet=True)
labels = json.loads(result)
self.assertTrue('data' in labels)
os.remove(container)
def extract_content(image_path, member_name, cli=None, return_hash=False):
'''extract_content will extract content from an image using cat.
If hash=True, a hash sum is returned instead
'''
if member_name.startswith('./'):
member_name = member_name.replace('.', '', 1)
if return_hash:
hashy = hashlib.md5()
if cli == None:
cli = Singularity()
content = cli.execute(image_path, 'cat %s' % (member_name))
if not isinstance(content, bytes):
content = bytes(content)
# If permissions don't allow read, return None
if len(content) == 0:
return None
if return_hash:
hashy.update(content)
return hashy.hexdigest()
return content
def extract_guts(image_path,
tar,
file_filter,
tag_root=True,
include_sizes=True):
'''extract the file guts from an in memory tarfile. The file is not closed.
This should not be done for large images.
:TODO: this should have a switch for a function to decide if we should
read the memory in tar, or from fileystem
'''
cli = Singularity()
results = dict()
digest = dict()
if tag_root:
roots = dict()
if include_sizes:
sizes = dict()
for member in tar:
member_name = member.name.replace('.', '', 1)
included = False
if member.isdir() or member.issym():
continue
elif assess_content(member, file_filter):
digest[member_name] = extract_content(image_path,
member.name,
cli,
return_hash=True)
included = True
elif include_file(member, file_filter):
hasher = hashlib.md5()
buf = member.tobuf()
hasher.update(buf)
digest[member_name] = hasher.hexdigest()
included = True
if included:
if include_sizes:
sizes[member_name] = member.size
if tag_root:
roots[member_name] = is_root_owned(member)
results['hashes'] = digest
if include_sizes:
results['sizes'] = sizes
if tag_root:
results['root_owned'] = roots
return results
def unpack_node(image_path, name=None, output_folder=None, size=None):
'''unpackage node is intended to unpackage a node that was packaged with
package_node. The image should be a .tgz file. The general steps are to:
1. Package the node using the package_node function
2. Transfer the package somewhere that Singularity is installed'''
if not image_path.endswith(".tgz"):
bot.error(
"The image_path should end with .tgz. Did you create with package_node?"
)
sys.exit(1)
if output_folder is None:
output_folder = os.path.dirname(os.path.abspath(image_path))
image_name = os.path.basename(image_path)
if name is None:
name = image_name.replace('.tgz', '.img')
if not name.endswith('.img'):
name = "%s.img" % (name)
bot.debug("Preparing to unpack %s to %s." % (image_name, name))
unpacked_image = "%s/%s" % (output_folder, name)
S = Singularity(sudo=True)
S.create(image_path=unpacked_image, size=size)
cmd = [
"gunzip", "-dc", image_path, "|", "sudo", "singularity", "import",
unpacked_image
]
output = run_command(cmd)
# TODO: singularity mount the container, cleanup files (/etc/fstab,...)
# and add your custom singularity files.
return unpacked_image
#!/usr/bin/env python # This shows how to use the Singularity (python wrapper) client from singularity.cli import Singularity # The default will ask for your sudo password, and then not ask again to # run commands. It is not stored anywhere, however you should not save / pickle # the object as it will expose your password. S = Singularity() # Get general help: S.help() # These are the defaults, which can be specified S = Singularity(sudo=True,verbose=False) # Let's define a path to an image # wget http://www.vbmis.com/bmi/project/singularity/package_image/ubuntu:latest-2016-04-06.img image_path = 'ubuntu:latest-2016-04-06.img' # Run singularity --exec S.execute(image_path=image_path,command='ls') # $'docker2singularity.sh\nget_docker_container_id.sh\nget_docker_meta.py\nmakeBases.py\nsingularity\nubuntu:latest-2016-04-06.img\n' # These are the defaults, which can be specified # For any function you can get the docs: S.help(command="exec") # or return as string help = S.help(command="exec",stdout=False)
def setUp(self):
self.pwd = get_installdir()
self.cli = Singularity()
self.tmpdir = tempfile.mkdtemp()
def assess_differences(image_file1,
image_file2,
levels=None,
version=None,
size_heuristic=False,
guts1=None,
guts2=None):
'''assess_differences will compare two images on each level of
reproducibility, returning for each level a dictionary with files
that are the same, different, and an overall score.
:param size_heuristic: if True, assess root owned files based on size
:param guts1,guts2: the result (dict with sizes,roots,etc) from get_content_hashes
'''
if levels is None:
levels = get_levels(version=version)
reports = dict()
scores = dict()
for level_name, level_filter in levels.items():
contenders = []
different = []
setdiff = []
same = 0
# Compare the dictionary of file:hash between two images, and get root owned lookup
if guts1 is None:
guts1 = get_content_hashes(image_path=image_file1,
level_filter=level_filter,
tag_root=True,
include_sizes=True)
if guts2 is None:
guts2 = get_content_hashes(image_path=image_file2,
level_filter=level_filter,
tag_root=True,
include_sizes=True)
files = list(
set(list(guts1['hashes'].keys()) + list(guts2['hashes'].keys())))
for file_name in files:
# If it's not in one or the other
if file_name not in guts1['hashes'] or file_name not in guts2[
'hashes']:
setdiff.append(file_name)
else:
if guts1['hashes'][file_name] == guts2['hashes'][file_name]:
same += 1
else:
# If the file is root owned, we compare based on size
if size_heuristic == True:
if guts1['root_owned'][file_name] or guts2[
'root_owned'][file_name]:
if guts1['sizes'][file_name] == guts2['sizes'][
file_name]:
same += 1
else:
different.append(file_name)
else:
# Otherwise, we can assess the bytes content by reading it
contenders.append(file_name)
else:
contenders.append(file_name)
# If the user wants identical (meaning extraction order and timestamps)
if level_name == "IDENTICAL":
different = different + contenders
# Otherwise we need to check based on byte content
else:
if len(contenders) > 0:
cli = Singularity()
for rogue in contenders:
hashy1 = extract_content(image_file1,
rogue,
cli,
return_hash=True)
hashy2 = extract_content(image_file2,
rogue,
cli,
return_hash=True)
# If we can't compare, we use size as a heuristic
if hashy1 is None or hashy2 is None: # if one is symlink, could be None
different.append(file_name)
elif len(hashy1) == 0 or len(hashy2) == 0:
if guts1['sizes'][file_name] == guts2['sizes'][
file_name]:
same += 1
else:
different.append(file_name)
elif hashy1 != hashy2:
different.append(rogue)
else:
same += 1
# We use a similar Jacaard coefficient, twice the shared information in the numerator
# (the intersection, same), as a proportion of the total summed files
union = len(guts1['hashes']) + len(guts2['hashes'])
report = {
'difference': setdiff,
'intersect_different': different,
'same': same,
'union': union
}
if union == 0:
scores[level_name] = 0
else:
scores[level_name] = 2 * (same) / union
reports[level_name] = report
gc.collect()
reports['scores'] = scores
return reports
class TestClient(unittest.TestCase):
def setUp(self):
self.pwd = get_installdir()
self.cli = Singularity()
self.tmpdir = tempfile.mkdtemp()
self.image1 = "%s/tests/data/busybox-2016-02-16.img" % (self.pwd)
self.image2 = "%s/tests/data/cirros-2016-01-04.img" % (self.pwd)
def tearDown(self):
shutil.rmtree(self.tmpdir)
def test_create(self):
print('Testing client.create command')
container = "%s/container.img" % (self.tmpdir)
created_container = create_container(container)
self.assertEqual(created_container, container)
self.assertTrue(os.path.exists(container))
def test_import(self):
from singularity.build.utils import test_container
print("Testing client.import command")
container = create_container()
# Container should not be valid
print("Case 1: Before import, container is not valid")
result = test_container(container)
self.assertEqual(result['return_code'], 255)
print("Case 2: After import, container is valid")
self.cli.importcmd(container, 'docker://ubuntu')
result = test_container(container)
self.assertEqual(result['return_code'], 0)
def test_run(self):
print("Testing client.run command")
container = create_container(do_import=True)
result = self.cli.run(container)
self.assertEqual(result, '')
def test_exec(self):
print('Testing client.execute command')
container = create_container(do_import=True)
result = self.cli.execute(container, 'ls /')
print(result)
#if isinstance(result,bytes):
# result = result.decode('utf-8')
#self.assertTrue(len(result)>0)
def test_pull(self):
print("Testing client.pull command")
print("Case 1: Testing naming pull by image name")
image = self.cli.pull("shub://vsoch/singularity-images")
print(image)
print("Case 2: Testing naming pull by image commit")
image = self.cli.pull("shub://vsoch/singularity-images",
name_by="commit")
print(image)
print("Case 3: Testing naming pull by image hash")
image = self.cli.pull("shub://vsoch/singularity-images",
name_by="hash")
print(image)
def test_get_image(self):
print("Testing singularity.cli.get_image")
from singularity.cli import get_image
from singularity.build.utils import test_container
tmpimg = get_image('docker://ubuntu')
self.assertTrue(os.path.exists(tmpimg))
result = test_container(tmpimg)
self.assertEqual(result['return_code'], 0)
# EXACT REPLICATES
runscript = '''Bootstrap:docker
From: ubuntu:latest
%runscript
exec "Hello World!"
'''
os.chdir(replicates)
with open('Singularity','w') as filey:
filey.writelines(runscript)
from singularity.cli import Singularity
cli = Singularity()
for num in range(0,100):
container_name = 'ubuntu-hello-world-%s.img' %(num)
cli.create(container_name)
cli.bootstrap(container_name,'Singularity')
# ALL SINGULARITY HUB
containers = shub.get_containers()
os.chdir(hub)
for container_name,container in containers.items():
for branch, manifest in container.items():
name = manifest['name'].replace('/','-')
uncompressed = "%s-%s.img" %(name,branch)
def pull_image(self,image_url):
"""Pull the singularity image to specified image path."""
image_path = self.get_image_path()
s = Singularity()
container_path = s.pull(image_url[0],image_name=image_path[0])
return Unicode(container_path)
def setUp(self):
self.pwd = get_installdir()
self.cli = Singularity()
self.tmpdir = tempfile.mkdtemp()
self.image1 = "%s/tests/data/busybox-2016-02-16.img" % (self.pwd)
self.image2 = "%s/tests/data/cirros-2016-01-04.img" % (self.pwd)
import re
import os
import sys
required_softwares = ['docker','singularity']
image_directory = '/home/vanessa/Documents/Work/SINGULARITY/docker2singularity'
os.chdir(image_directory)
print('Checking for required software...')
for required_software in required_softwares:
if check_install(required_software) != True:
print("You must have %s installed to use this script!" %(required_software.upper()))
sys.exit(32)
# Create a command line client
S = Singularity()
# Here is out list of images (note -some of these couldn't be found)
# from https://github.com/docker/docker/wiki/Public-docker-images
docker_images = ["ubuntu:latest","ubuntu:12.10","ubuntu:12.04",
"opensuse:13.1","centos","busybox","base/arch",
"tianon/debian:wheezy","tianon/debian:7.1",
"tianon/debian:jessie","tianon/debian-roll:stable",
"tianon/debian-roll:7.1","johncosta/redis",
"shykes/couchdb","jpetazzo/pgsql","samalba/hipache",
"creack/firefox-vnc","jbarbier/memcached",
"shykes/znc","shykes/dockerbuilder","shykes/node-opencv",
"audreyt/ethercalc"]
# Let's get the list of official docker from a request
url = "https://github.com/docker-library/official-images/tree/master/library"
