def main(args, parser, subparser): # We can only continue if singularity is installed if check_install() is not True: bot.error("Cannot find Singularity! Is it installed?") sys.exit(1) # Output folder will be pwd if not specified output_folder = os.getcwd() if args.outfolder is not None: output_folder = os.getcwd() # If we are given an image, ensure full path if args.image is not None: if not os.path.exists(args.image): cli = Singularity(debug=args.debug) image = cli.pull(args.image) if image is None: bot.error("Cannot find image. Exiting.") sys.exit(1) # the user wants to make a tree if args.tree is True: from singularity.cli.app import make_tree make_tree(image) clean_up(image, existed) else: print("Please specify an image to inspect with --image") subparser.print_help()
def package(image_path, spec_path=None, output_folder=None, remove_image=False, verbose=False, S=None): '''generate a zip (including the image) to a user specified output_folder. :param image_path: full path to singularity image file :param runscript: if True, will extract runscript to include in package as runscript :param software: if True, will extract files.txt and folders.txt to package :param remove_image: if True, will not include original image in package (default,False) :param verbose: be verbose when using singularity --export (default,False) :param S: the Singularity object (optional) will be created if not required. ''' # Run create image and bootstrap with Singularity command line tool. S = Singularity(debug=verbose) file_obj, tar = get_image_tar(image_path, S=S) members = tar.getmembers() image_name, ext = os.path.splitext(os.path.basename(image_path)) zip_name = "%s.zip" % (image_name.replace(" ", "_")) # Include the image in the package? to_package = dict() if not remove_image: to_package["files"] = [image_path] # If the specfile is provided, it should also be packaged if spec_path is not None: singularity_spec = "".join(read_file(spec_path)) to_package['Singularity'] = singularity_spec to_package["VERSION"] = get_image_file_hash(image_path) try: inspection = S.inspect(image_path) to_package["inspect.json"] = inspection inspection = json.loads(inspection) to_package['runscript'] = inspection['data']['attributes']['runscript'] except: bot.warning("Trouble extracting container metadata with inspect.") bot.info("Adding software list to package.") files = [x.path for x in members if x.isfile()] folders = [x.path for x in members if x.isdir()] to_package["files.txt"] = files to_package["folders.txt"] = folders # Do zip up here - let's start with basic structures zipfile = zip_up(to_package, zip_name=zip_name, output_folder=output_folder) bot.debug("Package created at %s" % (zipfile)) if not delete_image_tar(file_obj, tar): bot.warning("Could not clean up temporary tarfile.") # return package to user return zipfile
def main(args, parser, subparser): # We can only continue if singularity is installed if check_install() is not True: bot.error("Cannot find Singularity! Is it installed?") sys.exit(1) # Output folder will be pwd if not specified output_folder = os.getcwd() if args.outfolder is not None: output_folder = os.getcwd() # If we are given an image, ensure full path if args.image is not None: image = args.image if not os.path.exists(image): cli = Singularity(debug=args.debug) image = cli.pull(image) if image is None: bot.error("Cannot find image. Exiting.") sys.exit(1) remove_image = not args.include_image package(image_path=image, output_folder=output_folder, remove_image=remove_image) else: print("Please specify an image to package with --image") subparser.print_help()
def extract_apps(image_path, app_names, S=None, verbose=True): ''' extract app will extract metadata for one or more apps Parameters ========== image_path: the absolute path to the image app_name: the name of the app under /scif/apps ''' if S is None: S = Singularity(debug=verbose, sudo=True) if not isinstance(app_names, list): app_names = [app_names] file_obj, tar = get_image_tar(image_path, S=S) members = tar.getmembers() apps = dict() for app_name in app_names: metadata = dict() # Inspect: labels, env, runscript, tests, help try: inspection = json.loads(S.inspect(image_path, app=app_name)) del inspection['data']['attributes']['deffile'] metadata['inspect'] = inspection # If illegal characters prevent load, not much we can do except: pass base = '/scif/apps/%s' % app_name metadata['files'] = [x.path for x in members if base in x.path] apps[app_name] = metadata return apps
def package(image_path,output_folder=None,runscript=True,software=True,remove_image=False,verbose=False): '''package will take an image and generate a zip (including the image to a user specified output_folder. :param image_path: full path to singularity image file :param runscript: if True, will extract runscript to include in package as runscript :param software: if True, will extract files.txt and folders.txt to package :param remove_image: if True, will not include original image in package (default,False) :param verbose: be verbose when using singularity --export (default,False) ''' S = Singularity(verbose=verbose) tmptar = S.export(image_path=image_path,pipe=False) tar = tarfile.open(tmptar) members = tar.getmembers() image_name = os.path.basename(image_path) zip_name = "%s.zip" %(image_name.replace(" ","_")) # Include the image in the package? if remove_image: to_package = dict() else: to_package = {image_name:image_path} # Package the image with an md5 sum as VERSION version = get_image_hash(image_path) to_package["VERSION"] = version # Look for runscript if runscript == True: try: runscript_member = tar.getmember("./singularity") runscript_file = tar.extractfile("./singularity") runscript = runscript_file.read() to_package["runscript"] = runscript print("Found runscript!") # Try to extract input args, only python supported, will return None otherwise params_json = get_runscript_parameters(runscript=runscript, name=image_name, version=version) if params_json != None: print('Extracted runscript params!') to_package['%s.json' %(image_name)] = params_json except KeyError: print("No runscript found in image!") if software == True: print("Adding software list to package!") files = [x.path for x in members if x.isfile()] folders = [x.path for x in members if x.isdir()] to_package["files.txt"] = files to_package["folders.txt"] = folders # Do zip up here - let's start with basic structures zipfile = zip_up(to_package,zip_name=zip_name,output_folder=output_folder) print("Package created at %s" %(zipfile)) # return package to user return zipfile
def setUp(self): self.tmpdir = tempfile.mkdtemp() self.cli = Singularity() self.container = self.cli.pull('docker://ubuntu:16.04', pull_folder=self.tmpdir) self.comparator = self.cli.pull('docker://ubuntu:12.04', pull_folder=self.tmpdir)
def get_memory_tar(image_path): '''get an in memory tar of an image. Use carefully, not as reliable as get_image_tar ''' cli = Singularity() byte_array = cli.export(image_path) file_object = io.BytesIO(byte_array) tar = tarfile.open(mode="r|*", fileobj=file_object) return (file_object, tar)
def get_memory_tar(image_path): '''get an in memory tar of an image (does not require sudo!)''' cli = Singularity() if "pancakes" in os.environ: del os.environ['pancakes'] byte_array = cli.export(image_path) file_object = io.BytesIO(byte_array) tar = tarfile.open(mode="r|*", fileobj=file_object) return (file_object, tar)
def setUp(self): self.pwd = get_installdir() self.cli = Singularity() self.tmpdir = tempfile.mkdtemp() self.image1 = "%s/tests/data/busybox-2016-02-16.img" % (self.pwd) self.image2 = "%s/tests/data/cirros-2016-01-04.img" % (self.pwd) # We can't test creating the packages, because requires sudo :/ self.pkg1 = "%s/tests/data/busybox-2016-02-16.img.zip" % (self.pwd) self.pkg2 = "%s/tests/data/cirros-2016-01-04.img.zip" % (self.pwd)
def create_container(container=None, do_import=False): '''supporting function to create empty container ''' cli = Singularity() tmpdir = tempfile.mkdtemp() if container is None: container = "%s/container.img" % (tmpdir) if do_import is True: cli.importcmd(container, 'docker://ubuntu') return cli.create(container)
def main(args, parser, subparser): # We can only continue if singularity is installed if check_install() is not True: bot.error("Cannot find Singularity! Is it installed?") sys.exit(1) # Output folder will be pwd if not specified output_folder = os.getcwd() if args.outfolder is not None: output_folder = os.getcwd() # If we are given an image, ensure full path image = args.image if image is not None: existed = True if not os.path.exists(image): cli = Singularity(debug=args.debug) image = cli.pull(image) existed = False if image is None: bot.error("Cannot find image. Exiting.") sys.exit(1) # The user wants to estimate the os if args.os is True: from singularity.analysis.classify import estimate_os estimated_os = estimate_os(container=image) print(estimated_os) # The user wants to get a list of all os elif args.oscalc is True: from singularity.analysis.classify import estimate_os estimated_os = estimate_os(container=image, return_top=False) print(estimated_os["SCORE"].to_dict()) # The user wants to get a list of tags elif args.tags is True: from singularity.analysis.classify import get_tags tags = get_tags(container=image) print(tags) # The user wants to plot image vs. the docker os elif args.osplot is True: from singularity.app import plot_os_sims plot_os_sims(image) clean_up(image, existed) else: print("Please specify an image to analyze with --image") subparser.print_help()
class TestAnalysisCompare(unittest.TestCase): def setUp(self): self.tmpdir = tempfile.mkdtemp() self.cli = Singularity() self.container = self.cli.pull('docker://ubuntu:16.04', pull_folder=self.tmpdir) self.comparator = self.cli.pull('docker://ubuntu:12.04', pull_folder=self.tmpdir) def tearDown(self): os.remove(self.container) os.remove(self.comparator) shutil.rmtree(self.tmpdir) def test_container_similarity(self): print( "Testing singularity.analysis.compare.container_similarity_vector") import pandas from singularity.analysis.compare import container_similarity_vector from singularity.package import get_packages packages_set = get_packages('docker-os')[0:2] vector = container_similarity_vector(container1=self.container, custom_set=packages_set) self.assertTrue('files.txt' in vector) self.assertTrue(isinstance(vector['files.txt'], pandas.DataFrame)) print( "Testing singularity.analysis.compare.compare_singularity_images") from singularity.analysis.compare import compare_singularity_images sim = compare_singularity_images(self.container, self.comparator) self.assertTrue(isinstance(sim, pandas.DataFrame)) self.assertTrue(sim.loc[self.container, self.comparator] - 0.4803262269280298 < 0.01) print("Testitng singularity.analysis.compare.compare_containers") from singularity.analysis.compare import compare_containers comparison = compare_containers(self.container, self.comparator) self.assertTrue('files.txt' in comparison) for key in ['total1', 'total2', 'intersect', 'unique2', 'unique1']: self.assertTrue(key in comparison['files.txt']) print("Testing singularity.analysis.compare.calculate_similarity") from singularity.analysis.compare import calculate_similarity sim = calculate_similarity(self.container, self.comparator) self.assertTrue(sim['files.txt'] - 0.4921837537163134 < 0.01) def test_information_coefficient(self): print("Testing singularity.analysis.metrics.information_coefficient") from singularity.analysis.metrics import information_coefficient self.assertEqual(information_coefficient(100, 100, range(0, 50)), 0.5) self.assertEqual(information_coefficient(100, 100, range(0, 100)), 1.0)
def docker2singularity(docker_image,output_folder=None): '''docker2singulrity is a wrapper for the Singularity.docker2singularity client function. Does not currently include runscript (/singularity) in image, but does export full docker image spec under /singularity.json :param docker_image: the full docker repo/image,eg "ubuntu:latest" :param output_folder: the output folder to create the image in. If not specified, will use pwd. ''' S = Singularity() docker_image = S.docker2singularity(docker_image=docker_image, output_dir=output_folder) return docker_image
def main(args, parser, subparser): # We can only continue if singularity is installed if check_install() is not True: bot.error("Cannot find Singularity! Is it installed?") sys.exit(1) # Output folder will be pwd if not specified output_folder = os.getcwd() if args.outfolder is not None: output_folder = os.getcwd() if args.images is not None: image1, image2 = args.images.split(',') bot.debug("Image1: %s" % image1) bot.debug("Image2: %s" % image2) images = dict() cli = Singularity(debug=args.debug) for image in [image1, image2]: existed = True if not os.path.exists(image): image = cli.pull(image) existed = False images[image] = existed # Just for clarity image1, image2 = list(images.keys()) # the user wants to make a similarity tree if args.simtree is True: from singularity.cli.app import make_sim_tree make_sim_tree(image1, image2) # the user wants to make a difference tree elif args.subtract is True: from singularity.cli.app import make_diff_tree make_diff_tree(image1, image2) else: # If none specified, just print score from singularity.analysis.compare import calculate_similarity score = calculate_similarity(image1, image2, by="files.txt") print(score["files.txt"]) for image, existed in images.items(): clean_up(image, existed) else: print("Please specify images to compare with --images") subparser.print_help()
def build_from_spec(spec_file=None, build_dir=None, build_folder=False, sandbox=False, isolated=False, debug=False): '''build_from_spec will build a "spec" file in a "build_dir" and return the directory :param spec_file: the spec file, called "Singuarity" :param build_dir: the directory to build in. If not defined, will use tmpdir. :param isolated: "build" the image inside an isolated environment (>2.4) :param sandbox: ask for a sandbox build :param debug: ask for verbose output from builder ''' if spec_file == None: spec_file = "Singularity" if build_dir == None: build_dir = tempfile.mkdtemp() bot.debug("Building in directory %s" %build_dir) # Copy the spec to a temporary directory bot.debug("Spec file set to %s" % spec_file) spec_path = os.path.abspath(spec_file) bot.debug("Spec file for build should be in %s" %spec_path) image_path = "%s/build.simg" %(build_dir) # Run create image and bootstrap with Singularity command line tool. cli = Singularity(debug=debug) print("\nBuilding image...") # Does the user want to "build" into a folder or image? result = cli.build(image_path=image_path, spec_path=spec_path, sandbox=sandbox, isolated=isolated) print(result) # If image, rename based on hash if sandbox is False: version = get_image_file_hash(image_path) final_path = "%s/%s.simg" %(build_dir,version) os.rename(image_path,final_path) image_path = final_path bot.debug("Built image: %s" %image_path) return image_path
def get_image_tar(image_path, S=None): '''get an image tar, either written in memory or to the file system. file_obj will either be the file object, or the file itself. ''' bot.debug('Generate file system tar...') if S is None: S = Singularity() file_obj = S.export(image_path=image_path) if file_obj is None: bot.error("Error generating tar, exiting.") sys.exit(1) tar = tarfile.open(file_obj) return file_obj, tar
def pull(self, images, file_name=None, decompress=True): bot.debug('Execution of PULL for %s images' % len(images)) for image in images: # If we need to decompress, it's old ext3 format if decompress is True: ext = 'img.gz' else: ext = 'simg' # squashfs q = parse_image_name(image, ext=ext) # Verify image existence, and obtain id url = "%s/container/%s/%s:%s" % (self.base, q['collection'], q['image'], q['tag']) bot.debug('Retrieving manifest at %s' % url) manifest = self.get(url) bot.debug(manifest) if file_name is None: file_name = q['storage'].replace('/', '-') image_file = self.download(url=manifest['image'], file_name=file_name, show_progress=True) bot.debug('Retrieved image file %s' % image_file) if os.path.exists(image_file) and decompress is True: # If compressed, decompress try: cli = Singularity() sys.stdout.write('Decompressing image ') bot.spinner.start() image_file = cli.decompress(image_file, quiet=True) except KeyboardInterrupt: bot.warning('Decompression cancelled.') except: bot.info('Image is not compressed.') image_name = image_file.replace('.gz', '') image_file = shutil.move(image_file, image_name) pass bot.spinner.stop() bot.custom(prefix="Success!", message=image_file)
def get_image_hash(image_path, level=None, level_filter=None, include_files=None, skip_files=None, version=None): '''get_image_hash will generate a sha1 hash of an image, depending on a level of reproducibility specified by the user. (see function get_levels for descriptions) the user can also provide a level_filter manually with level_filter (for custom levels) :param level: the level of reproducibility to use, which maps to a set regular expression to match particular files/folders in the image. Choices are in notes. :param skip_files: an optional list of files to skip :param include_files: an optional list of files to keep (only if level not defined) :param version: the version to use. If not defined, default is 2.3 ::notes LEVEL DEFINITIONS The level definitions come down to including folders/files in the comparison. For files that Singularity produces on the fly that might be different (timestamps) but equal content (eg for a replication) we hash the content ("assess_content") instead of the file. ''' # First get a level dictionary, with description and regexp if level_filter is not None: file_filter = level_filter elif level is None: file_filter = get_level("RECIPE", version=version, include_files=include_files, skip_files=skip_files) else: file_filter = get_level(level, version=version, skip_files=skip_files, include_files=include_files) cli = Singularity() file_obj, tar = get_memory_tar(image_path) hasher = hashlib.md5() for member in tar: member_name = member.name.replace('.', '', 1) # For files, we either assess content, or include the file if member.isdir() or member.issym(): continue elif assess_content(member, file_filter): content = extract_content(image_path, member.name, cli) hasher.update(content) elif include_file(member, file_filter): buf = member.tobuf() hasher.update(buf) digest = hasher.hexdigest() file_obj.close() return digest
def setUp(self): self.pwd = get_installdir() self.cli = Singularity() self.tmpdir = tempfile.mkdtemp() self.image1 = "%s/tests/data/busybox-2017-10-21.simg" %(self.pwd) self.image2 = "%s/tests/data/cirros-2017-10-21.simg" %(self.pwd) self.pkg1 = "%s/tests/data/busybox-2017-10-21.zip" %(self.pwd) self.pkg2 = "%s/tests/data/cirros-2017-10-21.zip" %(self.pwd)
def test_test_container(self): '''the retry function should return None if result is None ''' from singularity.build.utils import test_container from singularity.cli import Singularity cli = Singularity() unfinished_container = cli.create("%s/container.img" % self.tmpdir) print("Case 1: Testing that errored container does not run") result = test_container(unfinished_container) self.assertEqual(result["return_code"], 255) print("Case 2: Testing that finished container does run") finished_container = cli.importcmd(unfinished_container, 'docker://ubuntu') result = test_container(finished_container) self.assertEqual(result["return_code"], 0)
def get_singularity_client(**kwargs): try: import singularity from singularity.cli import Singularity except ImportError: raise ImportError( "the singularity python package is required for this") return Singularity(**kwargs)
def build_from_spec(spec_file=None, build_dir=None, size=None, sudopw=None, build_folder=False, debug=False): '''build_from_spec will build a "spec" file in a "build_dir" and return the directory :param spec_file: the spec file, called "Singuarity" :param sudopw: the sudopw for Singularity, root should provide '' :param build_dir: the directory to build in. If not defined, will use tmpdir. :param size: the size of the image :param build_folder: "build" the image into a folder instead. Default False :param debug: ask for verbose output from builder ''' if spec_file == None: spec_file = "Singularity" if build_dir == None: build_dir = tempfile.mkdtemp() bot.logger.debug("Building in directory %s", build_dir) # Copy the spec to a temporary directory bot.logger.debug("Spec file set to %s", spec_file) spec_path = "%s/%s" % (build_dir, os.path.basename(spec_file)) bot.logger.debug("Spec file for build should be in %s", spec_path) # If it's not already there if not os.path.exists(spec_path): shutil.copyfile(spec_file, spec_path) image_path = "%s/image" % (build_dir) # Run create image and bootstrap with Singularity command line tool. if sudopw is not None: cli = Singularity(sudopw=sudopw, debug=debug) else: cli = Singularity(debug=debug) print("\nCreating and bootstrapping image...") # Does the user want to "build" into a folder or image? if build_folder == True: bot.logger.debug("build_folder is true, creating %s", image_path) os.mkdir(image_path) else: cli.create(image_path, size=size, sudo=True) result = cli.bootstrap(image_path=image_path, spec_path=spec_path) print(result) # If image, rename based on hash if build_folder == False: version = get_image_file_hash(image_path) final_path = "%s/%s" % (build_dir, version) os.rename(image_path, final_path) image_path = final_path bot.logger.debug("Built image: %s", image_path) return image_path
class TestClient(unittest.TestCase): def setUp(self): self.pwd = get_installdir() self.cli = Singularity() self.tmpdir = tempfile.mkdtemp() def tearDown(self): shutil.rmtree(self.tmpdir) def test_commands(self): print('Testing client.create command') container = "%s/container.img" % (self.tmpdir) created_container = self.cli.create(container) self.assertEqual(created_container, container) self.assertTrue(os.path.exists(created_container)) os.remove(container) print("Testing client.pull command") print("...Case 1: Testing naming pull by image name") image = self.cli.pull("shub://vsoch/singularity-images", pull_folder=self.tmpdir) self.assertTrue(os.path.exists(image)) self.assertTrue('vsoch-singularity-images-master' in image) print(image) os.remove(image) print("...Case 3: Testing docker pull") container = self.cli.pull("docker://ubuntu:14.04", pull_folder=self.tmpdir) self.assertTrue("ubuntu:14.04" in container) print(container) self.assertTrue(os.path.exists(container)) print('Testing client.execute command') result = self.cli.execute(container, 'ls /') print(result) self.assertTrue('bin\nboot\ndev' in result) print("Testing client.inspect command") result = self.cli.inspect(container, quiet=True) labels = json.loads(result) self.assertTrue('data' in labels) os.remove(container)
def extract_content(image_path, member_name, cli=None, return_hash=False): '''extract_content will extract content from an image using cat. If hash=True, a hash sum is returned instead ''' if member_name.startswith('./'): member_name = member_name.replace('.', '', 1) if return_hash: hashy = hashlib.md5() if cli == None: cli = Singularity() content = cli.execute(image_path, 'cat %s' % (member_name)) if not isinstance(content, bytes): content = bytes(content) # If permissions don't allow read, return None if len(content) == 0: return None if return_hash: hashy.update(content) return hashy.hexdigest() return content
def extract_guts(image_path, tar, file_filter, tag_root=True, include_sizes=True): '''extract the file guts from an in memory tarfile. The file is not closed. This should not be done for large images. :TODO: this should have a switch for a function to decide if we should read the memory in tar, or from fileystem ''' cli = Singularity() results = dict() digest = dict() if tag_root: roots = dict() if include_sizes: sizes = dict() for member in tar: member_name = member.name.replace('.', '', 1) included = False if member.isdir() or member.issym(): continue elif assess_content(member, file_filter): digest[member_name] = extract_content(image_path, member.name, cli, return_hash=True) included = True elif include_file(member, file_filter): hasher = hashlib.md5() buf = member.tobuf() hasher.update(buf) digest[member_name] = hasher.hexdigest() included = True if included: if include_sizes: sizes[member_name] = member.size if tag_root: roots[member_name] = is_root_owned(member) results['hashes'] = digest if include_sizes: results['sizes'] = sizes if tag_root: results['root_owned'] = roots return results
def unpack_node(image_path, name=None, output_folder=None, size=None): '''unpackage node is intended to unpackage a node that was packaged with package_node. The image should be a .tgz file. The general steps are to: 1. Package the node using the package_node function 2. Transfer the package somewhere that Singularity is installed''' if not image_path.endswith(".tgz"): bot.error( "The image_path should end with .tgz. Did you create with package_node?" ) sys.exit(1) if output_folder is None: output_folder = os.path.dirname(os.path.abspath(image_path)) image_name = os.path.basename(image_path) if name is None: name = image_name.replace('.tgz', '.img') if not name.endswith('.img'): name = "%s.img" % (name) bot.debug("Preparing to unpack %s to %s." % (image_name, name)) unpacked_image = "%s/%s" % (output_folder, name) S = Singularity(sudo=True) S.create(image_path=unpacked_image, size=size) cmd = [ "gunzip", "-dc", image_path, "|", "sudo", "singularity", "import", unpacked_image ] output = run_command(cmd) # TODO: singularity mount the container, cleanup files (/etc/fstab,...) # and add your custom singularity files. return unpacked_image
#!/usr/bin/env python # This shows how to use the Singularity (python wrapper) client from singularity.cli import Singularity # The default will ask for your sudo password, and then not ask again to # run commands. It is not stored anywhere, however you should not save / pickle # the object as it will expose your password. S = Singularity() # Get general help: S.help() # These are the defaults, which can be specified S = Singularity(sudo=True,verbose=False) # Let's define a path to an image # wget http://www.vbmis.com/bmi/project/singularity/package_image/ubuntu:latest-2016-04-06.img image_path = 'ubuntu:latest-2016-04-06.img' # Run singularity --exec S.execute(image_path=image_path,command='ls') # $'docker2singularity.sh\nget_docker_container_id.sh\nget_docker_meta.py\nmakeBases.py\nsingularity\nubuntu:latest-2016-04-06.img\n' # These are the defaults, which can be specified # For any function you can get the docs: S.help(command="exec") # or return as string help = S.help(command="exec",stdout=False)
def setUp(self): self.pwd = get_installdir() self.cli = Singularity() self.tmpdir = tempfile.mkdtemp()
def assess_differences(image_file1, image_file2, levels=None, version=None, size_heuristic=False, guts1=None, guts2=None): '''assess_differences will compare two images on each level of reproducibility, returning for each level a dictionary with files that are the same, different, and an overall score. :param size_heuristic: if True, assess root owned files based on size :param guts1,guts2: the result (dict with sizes,roots,etc) from get_content_hashes ''' if levels is None: levels = get_levels(version=version) reports = dict() scores = dict() for level_name, level_filter in levels.items(): contenders = [] different = [] setdiff = [] same = 0 # Compare the dictionary of file:hash between two images, and get root owned lookup if guts1 is None: guts1 = get_content_hashes(image_path=image_file1, level_filter=level_filter, tag_root=True, include_sizes=True) if guts2 is None: guts2 = get_content_hashes(image_path=image_file2, level_filter=level_filter, tag_root=True, include_sizes=True) files = list( set(list(guts1['hashes'].keys()) + list(guts2['hashes'].keys()))) for file_name in files: # If it's not in one or the other if file_name not in guts1['hashes'] or file_name not in guts2[ 'hashes']: setdiff.append(file_name) else: if guts1['hashes'][file_name] == guts2['hashes'][file_name]: same += 1 else: # If the file is root owned, we compare based on size if size_heuristic == True: if guts1['root_owned'][file_name] or guts2[ 'root_owned'][file_name]: if guts1['sizes'][file_name] == guts2['sizes'][ file_name]: same += 1 else: different.append(file_name) else: # Otherwise, we can assess the bytes content by reading it contenders.append(file_name) else: contenders.append(file_name) # If the user wants identical (meaning extraction order and timestamps) if level_name == "IDENTICAL": different = different + contenders # Otherwise we need to check based on byte content else: if len(contenders) > 0: cli = Singularity() for rogue in contenders: hashy1 = extract_content(image_file1, rogue, cli, return_hash=True) hashy2 = extract_content(image_file2, rogue, cli, return_hash=True) # If we can't compare, we use size as a heuristic if hashy1 is None or hashy2 is None: # if one is symlink, could be None different.append(file_name) elif len(hashy1) == 0 or len(hashy2) == 0: if guts1['sizes'][file_name] == guts2['sizes'][ file_name]: same += 1 else: different.append(file_name) elif hashy1 != hashy2: different.append(rogue) else: same += 1 # We use a similar Jacaard coefficient, twice the shared information in the numerator # (the intersection, same), as a proportion of the total summed files union = len(guts1['hashes']) + len(guts2['hashes']) report = { 'difference': setdiff, 'intersect_different': different, 'same': same, 'union': union } if union == 0: scores[level_name] = 0 else: scores[level_name] = 2 * (same) / union reports[level_name] = report gc.collect() reports['scores'] = scores return reports
class TestClient(unittest.TestCase): def setUp(self): self.pwd = get_installdir() self.cli = Singularity() self.tmpdir = tempfile.mkdtemp() self.image1 = "%s/tests/data/busybox-2016-02-16.img" % (self.pwd) self.image2 = "%s/tests/data/cirros-2016-01-04.img" % (self.pwd) def tearDown(self): shutil.rmtree(self.tmpdir) def test_create(self): print('Testing client.create command') container = "%s/container.img" % (self.tmpdir) created_container = create_container(container) self.assertEqual(created_container, container) self.assertTrue(os.path.exists(container)) def test_import(self): from singularity.build.utils import test_container print("Testing client.import command") container = create_container() # Container should not be valid print("Case 1: Before import, container is not valid") result = test_container(container) self.assertEqual(result['return_code'], 255) print("Case 2: After import, container is valid") self.cli.importcmd(container, 'docker://ubuntu') result = test_container(container) self.assertEqual(result['return_code'], 0) def test_run(self): print("Testing client.run command") container = create_container(do_import=True) result = self.cli.run(container) self.assertEqual(result, '') def test_exec(self): print('Testing client.execute command') container = create_container(do_import=True) result = self.cli.execute(container, 'ls /') print(result) #if isinstance(result,bytes): # result = result.decode('utf-8') #self.assertTrue(len(result)>0) def test_pull(self): print("Testing client.pull command") print("Case 1: Testing naming pull by image name") image = self.cli.pull("shub://vsoch/singularity-images") print(image) print("Case 2: Testing naming pull by image commit") image = self.cli.pull("shub://vsoch/singularity-images", name_by="commit") print(image) print("Case 3: Testing naming pull by image hash") image = self.cli.pull("shub://vsoch/singularity-images", name_by="hash") print(image) def test_get_image(self): print("Testing singularity.cli.get_image") from singularity.cli import get_image from singularity.build.utils import test_container tmpimg = get_image('docker://ubuntu') self.assertTrue(os.path.exists(tmpimg)) result = test_container(tmpimg) self.assertEqual(result['return_code'], 0)
# EXACT REPLICATES runscript = '''Bootstrap:docker From: ubuntu:latest %runscript exec "Hello World!" ''' os.chdir(replicates) with open('Singularity','w') as filey: filey.writelines(runscript) from singularity.cli import Singularity cli = Singularity() for num in range(0,100): container_name = 'ubuntu-hello-world-%s.img' %(num) cli.create(container_name) cli.bootstrap(container_name,'Singularity') # ALL SINGULARITY HUB containers = shub.get_containers() os.chdir(hub) for container_name,container in containers.items(): for branch, manifest in container.items(): name = manifest['name'].replace('/','-') uncompressed = "%s-%s.img" %(name,branch)
def pull_image(self,image_url): """Pull the singularity image to specified image path.""" image_path = self.get_image_path() s = Singularity() container_path = s.pull(image_url[0],image_name=image_path[0]) return Unicode(container_path)
def setUp(self): self.pwd = get_installdir() self.cli = Singularity() self.tmpdir = tempfile.mkdtemp() self.image1 = "%s/tests/data/busybox-2016-02-16.img" % (self.pwd) self.image2 = "%s/tests/data/cirros-2016-01-04.img" % (self.pwd)
import re import os import sys required_softwares = ['docker','singularity'] image_directory = '/home/vanessa/Documents/Work/SINGULARITY/docker2singularity' os.chdir(image_directory) print('Checking for required software...') for required_software in required_softwares: if check_install(required_software) != True: print("You must have %s installed to use this script!" %(required_software.upper())) sys.exit(32) # Create a command line client S = Singularity() # Here is out list of images (note -some of these couldn't be found) # from https://github.com/docker/docker/wiki/Public-docker-images docker_images = ["ubuntu:latest","ubuntu:12.10","ubuntu:12.04", "opensuse:13.1","centos","busybox","base/arch", "tianon/debian:wheezy","tianon/debian:7.1", "tianon/debian:jessie","tianon/debian-roll:stable", "tianon/debian-roll:7.1","johncosta/redis", "shykes/couchdb","jpetazzo/pgsql","samalba/hipache", "creack/firefox-vnc","jbarbier/memcached", "shykes/znc","shykes/dockerbuilder","shykes/node-opencv", "audreyt/ethercalc"] # Let's get the list of official docker from a request url = "https://github.com/docker-library/official-images/tree/master/library"