def test_check_install(self): '''check install is used to check if a particular software is installed. If no command is provided, singularity is assumed to be the test case''' from singularity.utils import check_install is_installed = check_install() self.assertTrue(is_installed) is_not_installed = check_install('fakesoftwarename') self.assertTrue(not is_not_installed)
def test_check_install(self): '''check install is used to check if a particular software is installed. If no command is provided, singularity is assumed to be the test case''' print("Testing utils.check_install") from singularity.utils import check_install is_installed = check_install() self.assertTrue(is_installed) is_not_installed = check_install('fakesoftwarename') self.assertTrue(not is_not_installed)
def main(args, parser, subparser): # We can only continue if singularity is installed if check_install() is not True: bot.error("Cannot find Singularity! Is it installed?") sys.exit(1) # Output folder will be pwd if not specified output_folder = os.getcwd() if args.outfolder is not None: output_folder = os.getcwd() # If we are given an image, ensure full path if args.image is not None: image = args.image if not os.path.exists(image): cli = Singularity(debug=args.debug) image = cli.pull(image) if image is None: bot.error("Cannot find image. Exiting.") sys.exit(1) remove_image = not args.include_image package(image_path=image, output_folder=output_folder, remove_image=remove_image) else: print("Please specify an image to package with --image") subparser.print_help()
def main(args, parser, subparser): # We can only continue if singularity is installed if check_install() is not True: bot.error("Cannot find Singularity! Is it installed?") sys.exit(1) # Output folder will be pwd if not specified output_folder = os.getcwd() if args.outfolder is not None: output_folder = os.getcwd() # If we are given an image, ensure full path if args.image is not None: if not os.path.exists(args.image): cli = Singularity(debug=args.debug) image = cli.pull(args.image) if image is None: bot.error("Cannot find image. Exiting.") sys.exit(1) # the user wants to make a tree if args.tree is True: from singularity.cli.app import make_tree make_tree(image) clean_up(image, existed) else: print("Please specify an image to inspect with --image") subparser.print_help()
def main(): parser = argparse.ArgumentParser( description="package Singularity containers for singularity hub.") parser.add_argument("--image", dest='image', help="full path to singularity image (for use with --package)", type=str, default=None) parser.add_argument("--docker2singularity", dest='docker', help="name of Docker image to export to Singularity (does not include runscript cmd)", type=str, default=None) parser.add_argument("--outfolder", dest='outfolder', help="full path to folder for output, if not specified, will go to pwd", type=str, default=None) parser.add_argument("--runscript", dest='runscript', help="specify extension to generate a runscript template in the PWD, or include --outfolder to change output directory. Currently supported types are py (python).", type=str, default=None) parser.add_argument('--package', help="package a singularity container for singularity hub", dest='package', default=False, action='store_true') parser.add_argument('--tree', help="view the guts of an image or package.", dest='tree', default=False, action='store_true') try: args = parser.parse_args() except: parser.print_help() sys.exit(0) # Output folder will be pwd if not specified if args.outfolder == None: output_folder = os.getcwd() else: output_folder = args.outfolder # If the user wants a runscript template. if args.runscript != None: get_runscript_template(output_folder=output_folder, script_name="singularity", language=args.runscript) # If the user wants to export docker2singularity! if args.docker != None: docker2singularity(output_folder=output_folder, docker_image=args.docker) # We can only continue if singularity is installed if check_install() == True: # We also must have an image to work with if args.image !=None: image = os.path.abspath(args.image) # Make sure the image exists! if os.path.exists(image): # the user wants to make a tree for an image if args.tree == True: make_tree(image) # The user wants to package the image elif args.package == True: package(image_path=image, output_folder=output_folder, runscript=True, software=True) else: print("Please specify a singularity image with --image.")
def main(args, parser, subparser): # We can only continue if singularity is installed if check_install() is not True: bot.error("Cannot find Singularity! Is it installed?") sys.exit(1) # Output folder will be pwd if not specified output_folder = os.getcwd() if args.outfolder is not None: output_folder = os.getcwd() # If we are given an image, ensure full path image = args.image if image is not None: existed = True if not os.path.exists(image): cli = Singularity(debug=args.debug) image = cli.pull(image) existed = False if image is None: bot.error("Cannot find image. Exiting.") sys.exit(1) # The user wants to estimate the os if args.os is True: from singularity.analysis.classify import estimate_os estimated_os = estimate_os(container=image) print(estimated_os) # The user wants to get a list of all os elif args.oscalc is True: from singularity.analysis.classify import estimate_os estimated_os = estimate_os(container=image, return_top=False) print(estimated_os["SCORE"].to_dict()) # The user wants to get a list of tags elif args.tags is True: from singularity.analysis.classify import get_tags tags = get_tags(container=image) print(tags) # The user wants to plot image vs. the docker os elif args.osplot is True: from singularity.app import plot_os_sims plot_os_sims(image) clean_up(image, existed) else: print("Please specify an image to analyze with --image") subparser.print_help()
def main(args, parser, subparser): # We can only continue if singularity is installed if check_install() is not True: bot.error("Cannot find Singularity! Is it installed?") sys.exit(1) # Output folder will be pwd if not specified output_folder = os.getcwd() if args.outfolder is not None: output_folder = os.getcwd() if args.images is not None: image1, image2 = args.images.split(',') bot.debug("Image1: %s" % image1) bot.debug("Image2: %s" % image2) images = dict() cli = Singularity(debug=args.debug) for image in [image1, image2]: existed = True if not os.path.exists(image): image = cli.pull(image) existed = False images[image] = existed # Just for clarity image1, image2 = list(images.keys()) # the user wants to make a similarity tree if args.simtree is True: from singularity.cli.app import make_sim_tree make_sim_tree(image1, image2) # the user wants to make a difference tree elif args.subtract is True: from singularity.cli.app import make_diff_tree make_diff_tree(image1, image2) else: # If none specified, just print score from singularity.analysis.compare import calculate_similarity score = calculate_similarity(image1, image2, by="files.txt") print(score["files.txt"]) for image, existed in images.items(): clean_up(image, existed) else: print("Please specify images to compare with --images") subparser.print_help()
from singularity.analysis.compare import (compare_packages) from singularity.analysis.utils import get_packages from singularity.utils import check_install from glob import glob import os import pandas import pickle import sys base = '/home/vanessa/Documents/Dropbox/Code/singularity/singularity-python' analysis_directory = "%s/examples/package_tree" % (base) # Check for Singularity installation if check_install() != True: print("You must have Singularity installed to use this script!") sys.exit(32) ############################################################################### # Get paths to your package(s) ############################################################################### # Option 1: Get a family manually package_set1 = get_packages(family='docker-library') # Option 2: Specify your own package directory (arg is packages=packages) package_directory = '%s/examples/package_image/packages' % (base) package_set1 = glob("%s/*.zip" % (package_directory)) # Option 3: provide no input args, and default (os) for package_set1 is used
from glob import glob import pandas import sys import os base = '/scratch/users/vsochat/DATA/SINGULARITY' package_folder = '%s/packages' %(base) analysis_folder = '%s/analysis' %(base) output_folder = '%s/scores' %(base) for folder in [analysis_folder,output_folder]: if not os.path.exists(folder): os.mkdir(folder) # Check for Singularity installation if check_install() != True: print("You must have Singularity installed to use this script!") sys.exit(32) # if you are on a cluster and it should be installed, try adding # module load singularity to your .bash_profile or .modules file # that is sourced packages = glob("%s/*.zip" %(package_folder)) print("Found %s packages!" %(len(packages))) for pkg1 in packages: print "Parsing %s" %(pkg1) pkg1_name = os.path.basename(pkg1) for pkg2 in packages: pkg2_name = os.path.basename(pkg2)
def docker2singularity(self,docker_image,output_dir=None): '''docker2singularity will convert a docker image to singularity :param docker_image: the name of the docker image, doesn't have to be on local machine (eg, ubuntu:latest) ''' if check_install('docker') == True: sudo = True cmd = ['docker','run','-d',docker_image,'tail','-f','/dev/null'] runningid = self.run_command(cmd,sudo=sudo) # sha256:d59bdb51bb5c4fb7b2c8d90ae445e0720c169c553bcf553f67cb9dd208a4ec15 # Take the first 12 characters to get id of container container_id=runningid[0:12] # Network address, if needed cmd = ['docker','inspect','--format="{{.NetworkSettings.IPAddress}}"',container_id] network_addr = self.run_command(cmd,sudo=sudo) # Get image name cmd = ['docker','inspect','--format="{{.Config.Image}}"',container_id] image_name = self.run_command(cmd,sudo=sudo) # Get creation date cmd = ['docker','inspect','--format="{{.Created}}"',docker_image] creation_date = self.run_command(cmd,sudo=sudo)[0:10] # Estimate image size cmd = ['docker','inspect','--format="{{.Size}}"',docker_image] size = int(self.run_command(cmd,sudo=sudo)) # convert size in MB (it seems too small for singularity containers ...?). Add 1MB and round up (minimum). size=int(round(size/1000000.0)+1.0) # adding half of the container size seems to work (do not know why exactly...?) # I think it would be Ok by adding 1/3 of the size. size=size+size/2 # Create the image tmpdir = tempfile.mkdtemp() new_container_name = "%s-%s.img" %(image_name,creation_date) json_file = "%s/singularity.json" %(tmpdir) runscript = "%s/singularity" %(tmpdir) if output_dir != None: new_container_name = "%s/%s" %(output_dir,new_container_name) self.create(image_path=new_container_name,size=size) # export running docker container cmd = ['docker','export',container_id,'|','sudo','singularity','import',new_container_name] self.run_command(cmd,sudo=sudo,suppress=True) # Add stuff about the container to it cmd = ['docker','inspect', container_id] json_data = json.loads(self.run_command(cmd,sudo=sudo)) write_json(json_data,json_file) #TODO: we might be able to extract boutique / make package here # Add singularity.json to container cmd = ['singularity','copy',new_container_name,json_file,'/'] self.run_command(cmd,sudo=sudo) invalid_commands = ["","null","none"] # Bootstrap the image to set up scripts for environemnt setup self.run_command(['singularity','bootstrap',new_container_name],sudo=sudo) self.run_command(['chmod','a+rw','-R',tmpdir],sudo=sudo) # Runscript command = self.run_command(['docker','inspect','--format="{{json .Config.Cmd}}"',docker_image],sudo=sudo) if command not in invalid_commands: command = "/bin/sh -c %s" %(command) # Remove quotes, commas, and braces command = re.sub('"|[[]|[]]',"",command) # Get the entrypoint entrypoint = self.run_command(['docker','inspect','--format="{{json .Config.Entrypoint}}"',docker_image],sudo=sudo) if entrypoint not in invalid_commands: entrypoint = "/bin/sh -c %s" %(entrypoint) # Remove quotes, commas, and braces entrypoint = re.sub('"|[[]|[]]',"",entrypoint) runscript = "%s/singularity" %(tmpdir) runscript_file = open(runscript,'w') runscript_file.writelines('#!/bin/sh\n') if entrypoint not in invalid_commands: runscript_file.writelines('%s $@' %(entrypoint)) else: if command not in invalid_commands: runscript_file.writelines('%s $@' %(command)) runscript_file.close() self.run_command(['chmod','+x',runscript],sudo=sudo) self.run_command(['singularity','copy',new_container_name,runscript,'/'],sudo=sudo) # Set up the environment docker_environment = "%s/docker_environment" %(tmpdir) environment = self.run_command(['docker','run','--rm','--entrypoint="/usr/bin/env"',docker_image],sudo=sudo).split('\n') # don't include HOME and HOSTNAME - they mess with local config environment = [x for x in environment if not re.search("^HOME",x) and not re.search("^HOSTNAME",x)] write_file(docker_environment,"\n".join(environment)) # Write to container self.run_command(["chmod","u+x",docker_environment],sudo=sudo) self.run_command(['singularity','copy',new_container_name,docker_environment,'/'],sudo=sudo) self.run_command(['singularity','exec','--writable',new_container_name,'/bin/sh -c ',docker_environment,'/'],sudo=sudo) cmd = """singularity exec --writable %s /bin/sh -c "echo '. /docker_environment' >> /environment" """ %(new_container_name) self.run_command([cmd.replace('\n','')],sudo=sudo,suppress=True) shutil.rmtree(tmpdir) # Permissions - make sure any user can execute everything in container cmd = """singularity exec --writable --contain %s /bin/sh -c "find /* -maxdepth 0 -not -path '/dev*' -not -path '/proc*' -not -path '/sys*' -exec chmod a+r -R '{}' \;" """ %(new_container_name) self.run_command([cmd.replace('\n','')],sudo=sudo,suppress=True) cmd = """singularity exec --writable --contain %s /bin/sh -c "find / -executable -perm -u+x,o-x -not -path '/dev*' -not -path '/proc*' -not -path '/sys*' -exec chmod a+x '{}' \;" """ %(new_container_name) self.run_command([cmd.replace('\n','')],sudo=sudo,suppress=True) print("Stopping container... please wait!") self.run_command(['docker','stop',container_id],sudo=sudo) return new_container_name else: return None
from singularity.utils import check_install from singularity.cli import Singularity from BeautifulSoup import BeautifulSoup import requests import re import os import sys required_softwares = ['docker', 'singularity'] image_directory = '/home/vanessa/Documents/Work/SINGULARITY/docker2singularity' os.chdir(image_directory) print('Checking for required software...') for required_software in required_softwares: if check_install(required_software) != True: print("You must have %s installed to use this script!" % (required_software.upper())) sys.exit(32) # Create a command line client S = Singularity() # Here is out list of images (note -some of these couldn't be found) # from https://github.com/docker/docker/wiki/Public-docker-images docker_images = [ "ubuntu:latest", "ubuntu:12.10", "ubuntu:12.04", "opensuse:13.1", "centos", "busybox", "base/arch", "tianon/debian:wheezy", "tianon/debian:7.1", "tianon/debian:jessie", "tianon/debian-roll:stable", "tianon/debian-roll:7.1", "johncosta/redis", "shykes/couchdb", "jpetazzo/pgsql", "samalba/hipache", "creack/firefox-vnc",
def docker2singularity(self,docker_image,output_dir=None): '''docker2singularity will convert a docker image to singularity :param docker_image: the name of the docker image, doesn't have to be on local machine (eg, ubuntu:latest) ''' if check_install('docker') == True: sudo = True cmd = ['docker','run','-d',docker_image,'tail','-f','/dev/null'] runningid = self.run_command(cmd,sudo=sudo) # sha256:d59bdb51bb5c4fb7b2c8d90ae445e0720c169c553bcf553f67cb9dd208a4ec15 # Take the first 12 characters to get id of container container_id=runningid[0:12] # Network address, if needed cmd = ['docker','inspect','--format="{{.NetworkSettings.IPAddress}}"',container_id] network_addr = self.run_command(cmd,sudo=sudo) # Get image name cmd = ['docker','inspect','--format="{{.Config.Image}}"',container_id] image_name = self.run_command(cmd,sudo=sudo) # Get creation date cmd = ['docker','inspect','--format="{{.Created}}"',docker_image] creation_date = self.run_command(cmd,sudo=sudo)[0:10] # Estimate image size cmd = ['docker','inspect','--format="{{.Size}}"',docker_image] size = int(self.run_command(cmd,sudo=sudo)) # convert size in MB (it seems too small for singularity containers ...?). Add 1MB and round up (minimum). size=int(round(size/1000000.0)+1.0) # adding half of the container size seems to work (do not know why exactly...?) # I think it would be Ok by adding 1/3 of the size. size=size+size/2 # Create the image new_container_name = "%s-%s.img" %(image_name,creation_date) json_file = "singularity.json" runscript = "singularity" if output_dir != None: new_container_name = "%s/%s" %(output_dir,new_container_name) json_file = "%s/%s" %(output_dir,json_file) runscript = "%s/%s" %(output_dir,runscript) self.create(image_path=new_container_name,size=size) # export running docker container cmd = ['docker','export',container_id,'|','sudo','singularity','import',new_container_name] self.run_command(cmd,sudo=sudo,suppress=True) # Add stuff about the container to it cmd = ['docker','inspect', container_id] json_data = json.loads(self.run_command(cmd,sudo=sudo)) write_json(json_data,json_file) #TODO: we might be able to extract boutique / make package here # Add singularity.json to container cmd = ['singularity','copy',new_container_name,json_file,'/'] self.run_command(cmd,sudo=sudo) # Merge the /etc/group file, and add to container self.run_command(['docker','cp',"%s:/etc/group" %(container_id),'grouphost'],sudo=sudo) self.run_command(['sort','/etc/group','grouphost','|','uniq','-u','>','group'],sudo=sudo) self.run_command(['singularity','copy',new_container_name,'group','/etc/group'],sudo=sudo) os.remove(json_file) os.remove('grouphost') os.remove('group') # Runscript! Currently disabling - having some bugs with this #cmd = ['docker','inspect',"--format='{{json .Config.Cmd}}'",docker_image] #command = self.run_command(cmd,sudo=sudo) #command = command.replace('["',"").replace('"]',"") #if command != "none": # write_file(runscript,command) # cmd = ['chmod','+x',runscript] # self.run_command(cmd,sudo=sudo) # cmd = ['singularity','copy',new_container_name,runscript,'/'] # self.run_command(cmd,sudo=sudo) # os.remove(runscript) print("Stopping container... please wait!") cmd = ['docker','stop',container_id] self.run_command(cmd,sudo=sudo) return new_container_name else: return None
def main(): parser = get_parser() try: args = parser.parse_args() except: sys.exit(0) # Not running in Singularity Hub environment os.environ['SINGULARITY_HUB'] = "False" # if environment logging variable not set, make silent if args.debug is False: os.environ['MESSAGELEVEL'] = "CRITICAL" if args.version is True: print(singularity.__version__) sys.exit(0) # Initialize the message bot, with level above from singularity.logman import bot from singularity.utils import check_install from singularity.cli import get_image # Output folder will be pwd if not specified if args.outfolder is None: output_folder = os.getcwd() else: output_folder = args.outfolder # We can only continue if singularity is installed if check_install() is True: # If we are given an image, ensure full path if args.image is not None: image,existed = get_image(args.image, return_existed=True, size=args.size) if image is None: bot.logger.error("Cannot find image. Exiting.") sys.exit(1) # the user wants to make a tree if args.tree is True: from singularity.app import make_tree make_tree(image) clean_up(image,existed) # The user wants to estimate the os elif args.os is True: from singularity.analysis.classify import estimate_os estimated_os = estimate_os(container=image) print(estimated_os) # The user wants to get a list of all os elif args.oscalc is True: from singularity.analysis.classify import estimate_os estimated_os = estimate_os(container=image,return_top=False) print(estimated_os["SCORE"].to_dict()) # The user wants to get a list of tags elif args.tags is True: from singularity.analysis.classify import get_tags tags = get_tags(container=image) print(tags) # The user wants to plot image vs. the docker os elif args.osplot is True: from singularity.app import plot_os_sims plot_os_sims(image) clean_up(image,existed) # The user wants to package the image elif args.package is True: from singularity.package import package remove_image = not args.include_image package(image_path=image, output_folder=output_folder, runscript=True, software=True, remove_image=remove_image) else: print("Not sure what to do?") parser.print_help() # If we are given two image, we probably want a similar tree elif args.images is not None: image1,image2 = args.images.split(',') bot.logger.debug("Image1: %s",image1) bot.logger.debug("Image2: %s",image2) image1,existed1 = get_image(image1, return_existed=True, size=args.size) image2,existed2 = get_image(image2, return_existed=True, size=args.size) if image1 is None or image2 is None: bot.logger.error("Cannot find image. Exiting.") sys.exit(1) # the user wants to make a similarity tree if args.simtree is True: from singularity.app import make_sim_tree make_sim_tree(image1,image2) # the user wants to make a difference tree if args.subtract is True: from singularity.app import make_diff_tree make_diff_tree(image1,image2) if args.simcalc is True: from singularity.analysis.compare import calculate_similarity score = calculate_similarity(image1,image2,by="files.txt") print(score["files.txt"]) clean_up(image1,existed1) clean_up(image2,existed2) else: print("Please specify one or more containers with --image(s)")
from singularity.utils import check_install from singularity.cli import Singularity from BeautifulSoup import BeautifulSoup import requests import re import os import sys required_softwares = ['docker','singularity'] image_directory = '/home/vanessa/Documents/Work/SINGULARITY/docker2singularity' os.chdir(image_directory) print('Checking for required software...') for required_software in required_softwares: if check_install(required_software) != True: print("You must have %s installed to use this script!" %(required_software.upper())) sys.exit(32) # Create a command line client S = Singularity() # Here is out list of images (note -some of these couldn't be found) # from https://github.com/docker/docker/wiki/Public-docker-images docker_images = ["ubuntu:latest","ubuntu:12.10","ubuntu:12.04", "opensuse:13.1","centos","busybox","base/arch", "tianon/debian:wheezy","tianon/debian:7.1", "tianon/debian:jessie","tianon/debian-roll:stable", "tianon/debian-roll:7.1","johncosta/redis", "shykes/couchdb","jpetazzo/pgsql","samalba/hipache", "creack/firefox-vnc","jbarbier/memcached",