def test_check_install(self):
'''check install is used to check if a particular software is installed.
If no command is provided, singularity is assumed to be the test case'''
from singularity.utils import check_install
is_installed = check_install()
self.assertTrue(is_installed)
is_not_installed = check_install('fakesoftwarename')
self.assertTrue(not is_not_installed)
def test_check_install(self):
'''check install is used to check if a particular software is installed.
If no command is provided, singularity is assumed to be the test case'''
print("Testing utils.check_install")
from singularity.utils import check_install
is_installed = check_install()
self.assertTrue(is_installed)
is_not_installed = check_install('fakesoftwarename')
self.assertTrue(not is_not_installed)
def main(args, parser, subparser):
# We can only continue if singularity is installed
if check_install() is not True:
bot.error("Cannot find Singularity! Is it installed?")
sys.exit(1)
# Output folder will be pwd if not specified
output_folder = os.getcwd()
if args.outfolder is not None:
output_folder = os.getcwd()
# If we are given an image, ensure full path
if args.image is not None:
image = args.image
if not os.path.exists(image):
cli = Singularity(debug=args.debug)
image = cli.pull(image)
if image is None:
bot.error("Cannot find image. Exiting.")
sys.exit(1)
remove_image = not args.include_image
package(image_path=image,
output_folder=output_folder,
remove_image=remove_image)
else:
print("Please specify an image to package with --image")
subparser.print_help()
def main(args, parser, subparser):
# We can only continue if singularity is installed
if check_install() is not True:
bot.error("Cannot find Singularity! Is it installed?")
sys.exit(1)
# Output folder will be pwd if not specified
output_folder = os.getcwd()
if args.outfolder is not None:
output_folder = os.getcwd()
# If we are given an image, ensure full path
if args.image is not None:
if not os.path.exists(args.image):
cli = Singularity(debug=args.debug)
image = cli.pull(args.image)
if image is None:
bot.error("Cannot find image. Exiting.")
sys.exit(1)
# the user wants to make a tree
if args.tree is True:
from singularity.cli.app import make_tree
make_tree(image)
clean_up(image, existed)
else:
print("Please specify an image to inspect with --image")
subparser.print_help()
def main():
parser = argparse.ArgumentParser(
description="package Singularity containers for singularity hub.")
parser.add_argument("--image", dest='image', help="full path to singularity image (for use with --package)", type=str, default=None)
parser.add_argument("--docker2singularity", dest='docker', help="name of Docker image to export to Singularity (does not include runscript cmd)", type=str, default=None)
parser.add_argument("--outfolder", dest='outfolder', help="full path to folder for output, if not specified, will go to pwd", type=str, default=None)
parser.add_argument("--runscript", dest='runscript', help="specify extension to generate a runscript template in the PWD, or include --outfolder to change output directory. Currently supported types are py (python).", type=str, default=None)
parser.add_argument('--package', help="package a singularity container for singularity hub", dest='package', default=False, action='store_true')
parser.add_argument('--tree', help="view the guts of an image or package.", dest='tree', default=False, action='store_true')
try:
args = parser.parse_args()
except:
parser.print_help()
sys.exit(0)
# Output folder will be pwd if not specified
if args.outfolder == None:
output_folder = os.getcwd()
else:
output_folder = args.outfolder
# If the user wants a runscript template.
if args.runscript != None:
get_runscript_template(output_folder=output_folder,
script_name="singularity",
language=args.runscript)
# If the user wants to export docker2singularity!
if args.docker != None:
docker2singularity(output_folder=output_folder,
docker_image=args.docker)
# We can only continue if singularity is installed
if check_install() == True:
# We also must have an image to work with
if args.image !=None:
image = os.path.abspath(args.image)
# Make sure the image exists!
if os.path.exists(image):
# the user wants to make a tree for an image
if args.tree == True:
make_tree(image)
# The user wants to package the image
elif args.package == True:
package(image_path=image,
output_folder=output_folder,
runscript=True,
software=True)
else:
print("Please specify a singularity image with --image.")
def main(args, parser, subparser):
# We can only continue if singularity is installed
if check_install() is not True:
bot.error("Cannot find Singularity! Is it installed?")
sys.exit(1)
# Output folder will be pwd if not specified
output_folder = os.getcwd()
if args.outfolder is not None:
output_folder = os.getcwd()
# If we are given an image, ensure full path
image = args.image
if image is not None:
existed = True
if not os.path.exists(image):
cli = Singularity(debug=args.debug)
image = cli.pull(image)
existed = False
if image is None:
bot.error("Cannot find image. Exiting.")
sys.exit(1)
# The user wants to estimate the os
if args.os is True:
from singularity.analysis.classify import estimate_os
estimated_os = estimate_os(container=image)
print(estimated_os)
# The user wants to get a list of all os
elif args.oscalc is True:
from singularity.analysis.classify import estimate_os
estimated_os = estimate_os(container=image, return_top=False)
print(estimated_os["SCORE"].to_dict())
# The user wants to get a list of tags
elif args.tags is True:
from singularity.analysis.classify import get_tags
tags = get_tags(container=image)
print(tags)
# The user wants to plot image vs. the docker os
elif args.osplot is True:
from singularity.app import plot_os_sims
plot_os_sims(image)
clean_up(image, existed)
else:
print("Please specify an image to analyze with --image")
subparser.print_help()
def main(args, parser, subparser):
# We can only continue if singularity is installed
if check_install() is not True:
bot.error("Cannot find Singularity! Is it installed?")
sys.exit(1)
# Output folder will be pwd if not specified
output_folder = os.getcwd()
if args.outfolder is not None:
output_folder = os.getcwd()
if args.images is not None:
image1, image2 = args.images.split(',')
bot.debug("Image1: %s" % image1)
bot.debug("Image2: %s" % image2)
images = dict()
cli = Singularity(debug=args.debug)
for image in [image1, image2]:
existed = True
if not os.path.exists(image):
image = cli.pull(image)
existed = False
images[image] = existed
# Just for clarity
image1, image2 = list(images.keys())
# the user wants to make a similarity tree
if args.simtree is True:
from singularity.cli.app import make_sim_tree
make_sim_tree(image1, image2)
# the user wants to make a difference tree
elif args.subtract is True:
from singularity.cli.app import make_diff_tree
make_diff_tree(image1, image2)
else: # If none specified, just print score
from singularity.analysis.compare import calculate_similarity
score = calculate_similarity(image1, image2, by="files.txt")
print(score["files.txt"])
for image, existed in images.items():
clean_up(image, existed)
else:
print("Please specify images to compare with --images")
subparser.print_help()
from singularity.analysis.compare import (compare_packages)
from singularity.analysis.utils import get_packages
from singularity.utils import check_install
from glob import glob
import os
import pandas
import pickle
import sys
base = '/home/vanessa/Documents/Dropbox/Code/singularity/singularity-python'
analysis_directory = "%s/examples/package_tree" % (base)
# Check for Singularity installation
if check_install() != True:
print("You must have Singularity installed to use this script!")
sys.exit(32)
###############################################################################
# Get paths to your package(s)
###############################################################################
# Option 1: Get a family manually
package_set1 = get_packages(family='docker-library')
# Option 2: Specify your own package directory (arg is packages=packages)
package_directory = '%s/examples/package_image/packages' % (base)
package_set1 = glob("%s/*.zip" % (package_directory))
# Option 3: provide no input args, and default (os) for package_set1 is used
from glob import glob
import pandas
import sys
import os
base = '/scratch/users/vsochat/DATA/SINGULARITY'
package_folder = '%s/packages' %(base)
analysis_folder = '%s/analysis' %(base)
output_folder = '%s/scores' %(base)
for folder in [analysis_folder,output_folder]:
if not os.path.exists(folder):
os.mkdir(folder)
# Check for Singularity installation
if check_install() != True:
print("You must have Singularity installed to use this script!")
sys.exit(32)
# if you are on a cluster and it should be installed, try adding
# module load singularity to your .bash_profile or .modules file
# that is sourced
packages = glob("%s/*.zip" %(package_folder))
print("Found %s packages!" %(len(packages)))
for pkg1 in packages:
print "Parsing %s" %(pkg1)
pkg1_name = os.path.basename(pkg1)
for pkg2 in packages:
pkg2_name = os.path.basename(pkg2)
def docker2singularity(self,docker_image,output_dir=None):
'''docker2singularity will convert a docker image to singularity
:param docker_image: the name of the docker image, doesn't have to be on local machine (eg, ubuntu:latest)
'''
if check_install('docker') == True:
sudo = True
cmd = ['docker','run','-d',docker_image,'tail','-f','/dev/null']
runningid = self.run_command(cmd,sudo=sudo)
# sha256:d59bdb51bb5c4fb7b2c8d90ae445e0720c169c553bcf553f67cb9dd208a4ec15
# Take the first 12 characters to get id of container
container_id=runningid[0:12]
# Network address, if needed
cmd = ['docker','inspect','--format="{{.NetworkSettings.IPAddress}}"',container_id]
network_addr = self.run_command(cmd,sudo=sudo)
# Get image name
cmd = ['docker','inspect','--format="{{.Config.Image}}"',container_id]
image_name = self.run_command(cmd,sudo=sudo)
# Get creation date
cmd = ['docker','inspect','--format="{{.Created}}"',docker_image]
creation_date = self.run_command(cmd,sudo=sudo)[0:10]
# Estimate image size
cmd = ['docker','inspect','--format="{{.Size}}"',docker_image]
size = int(self.run_command(cmd,sudo=sudo))
# convert size in MB (it seems too small for singularity containers ...?). Add 1MB and round up (minimum).
size=int(round(size/1000000.0)+1.0)
# adding half of the container size seems to work (do not know why exactly...?)
# I think it would be Ok by adding 1/3 of the size.
size=size+size/2
# Create the image
tmpdir = tempfile.mkdtemp()
new_container_name = "%s-%s.img" %(image_name,creation_date)
json_file = "%s/singularity.json" %(tmpdir)
runscript = "%s/singularity" %(tmpdir)
if output_dir != None:
new_container_name = "%s/%s" %(output_dir,new_container_name)
self.create(image_path=new_container_name,size=size)
# export running docker container
cmd = ['docker','export',container_id,'|','sudo','singularity','import',new_container_name]
self.run_command(cmd,sudo=sudo,suppress=True)
# Add stuff about the container to it
cmd = ['docker','inspect', container_id]
json_data = json.loads(self.run_command(cmd,sudo=sudo))
write_json(json_data,json_file)
#TODO: we might be able to extract boutique / make package here
# Add singularity.json to container
cmd = ['singularity','copy',new_container_name,json_file,'/']
self.run_command(cmd,sudo=sudo)
invalid_commands = ["","null","none"]
# Bootstrap the image to set up scripts for environemnt setup
self.run_command(['singularity','bootstrap',new_container_name],sudo=sudo)
self.run_command(['chmod','a+rw','-R',tmpdir],sudo=sudo)
# Runscript
command = self.run_command(['docker','inspect','--format="{{json .Config.Cmd}}"',docker_image],sudo=sudo)
if command not in invalid_commands:
command = "/bin/sh -c %s" %(command)
# Remove quotes, commas, and braces
command = re.sub('"|[[]|[]]',"",command)
# Get the entrypoint
entrypoint = self.run_command(['docker','inspect','--format="{{json .Config.Entrypoint}}"',docker_image],sudo=sudo)
if entrypoint not in invalid_commands:
entrypoint = "/bin/sh -c %s" %(entrypoint)
# Remove quotes, commas, and braces
entrypoint = re.sub('"|[[]|[]]',"",entrypoint)
runscript = "%s/singularity" %(tmpdir)
runscript_file = open(runscript,'w')
runscript_file.writelines('#!/bin/sh\n')
if entrypoint not in invalid_commands:
runscript_file.writelines('%s $@' %(entrypoint))
else:
if command not in invalid_commands:
runscript_file.writelines('%s $@' %(command))
runscript_file.close()
self.run_command(['chmod','+x',runscript],sudo=sudo)
self.run_command(['singularity','copy',new_container_name,runscript,'/'],sudo=sudo)
# Set up the environment
docker_environment = "%s/docker_environment" %(tmpdir)
environment = self.run_command(['docker','run','--rm','--entrypoint="/usr/bin/env"',docker_image],sudo=sudo).split('\n')
# don't include HOME and HOSTNAME - they mess with local config
environment = [x for x in environment if not re.search("^HOME",x) and not re.search("^HOSTNAME",x)]
write_file(docker_environment,"\n".join(environment))
# Write to container
self.run_command(["chmod","u+x",docker_environment],sudo=sudo)
self.run_command(['singularity','copy',new_container_name,docker_environment,'/'],sudo=sudo)
self.run_command(['singularity','exec','--writable',new_container_name,'/bin/sh -c ',docker_environment,'/'],sudo=sudo)
cmd = """singularity exec --writable %s /bin/sh -c "echo '. /docker_environment' >> /environment"
""" %(new_container_name)
self.run_command([cmd.replace('\n','')],sudo=sudo,suppress=True)
shutil.rmtree(tmpdir)
# Permissions - make sure any user can execute everything in container
cmd = """singularity exec --writable --contain %s /bin/sh -c "find /* -maxdepth 0 -not -path '/dev*' -not -path '/proc*' -not -path '/sys*' -exec chmod a+r -R '{}' \;"
""" %(new_container_name)
self.run_command([cmd.replace('\n','')],sudo=sudo,suppress=True)
cmd = """singularity exec --writable --contain %s /bin/sh -c "find / -executable -perm -u+x,o-x -not -path '/dev*' -not -path '/proc*' -not -path '/sys*' -exec chmod a+x '{}' \;"
""" %(new_container_name)
self.run_command([cmd.replace('\n','')],sudo=sudo,suppress=True)
print("Stopping container... please wait!")
self.run_command(['docker','stop',container_id],sudo=sudo)
return new_container_name
else:
return None
from singularity.utils import check_install
from singularity.cli import Singularity
from BeautifulSoup import BeautifulSoup
import requests
import re
import os
import sys
required_softwares = ['docker', 'singularity']
image_directory = '/home/vanessa/Documents/Work/SINGULARITY/docker2singularity'
os.chdir(image_directory)
print('Checking for required software...')
for required_software in required_softwares:
if check_install(required_software) != True:
print("You must have %s installed to use this script!" %
(required_software.upper()))
sys.exit(32)
# Create a command line client
S = Singularity()
# Here is out list of images (note -some of these couldn't be found)
# from https://github.com/docker/docker/wiki/Public-docker-images
docker_images = [
"ubuntu:latest", "ubuntu:12.10", "ubuntu:12.04", "opensuse:13.1", "centos",
"busybox", "base/arch", "tianon/debian:wheezy", "tianon/debian:7.1",
"tianon/debian:jessie", "tianon/debian-roll:stable",
"tianon/debian-roll:7.1", "johncosta/redis", "shykes/couchdb",
"jpetazzo/pgsql", "samalba/hipache", "creack/firefox-vnc",
def docker2singularity(self,docker_image,output_dir=None):
'''docker2singularity will convert a docker image to singularity
:param docker_image: the name of the docker image, doesn't have to be on local machine (eg, ubuntu:latest)
'''
if check_install('docker') == True:
sudo = True
cmd = ['docker','run','-d',docker_image,'tail','-f','/dev/null']
runningid = self.run_command(cmd,sudo=sudo)
# sha256:d59bdb51bb5c4fb7b2c8d90ae445e0720c169c553bcf553f67cb9dd208a4ec15
# Take the first 12 characters to get id of container
container_id=runningid[0:12]
# Network address, if needed
cmd = ['docker','inspect','--format="{{.NetworkSettings.IPAddress}}"',container_id]
network_addr = self.run_command(cmd,sudo=sudo)
# Get image name
cmd = ['docker','inspect','--format="{{.Config.Image}}"',container_id]
image_name = self.run_command(cmd,sudo=sudo)
# Get creation date
cmd = ['docker','inspect','--format="{{.Created}}"',docker_image]
creation_date = self.run_command(cmd,sudo=sudo)[0:10]
# Estimate image size
cmd = ['docker','inspect','--format="{{.Size}}"',docker_image]
size = int(self.run_command(cmd,sudo=sudo))
# convert size in MB (it seems too small for singularity containers ...?). Add 1MB and round up (minimum).
size=int(round(size/1000000.0)+1.0)
# adding half of the container size seems to work (do not know why exactly...?)
# I think it would be Ok by adding 1/3 of the size.
size=size+size/2
# Create the image
new_container_name = "%s-%s.img" %(image_name,creation_date)
json_file = "singularity.json"
runscript = "singularity"
if output_dir != None:
new_container_name = "%s/%s" %(output_dir,new_container_name)
json_file = "%s/%s" %(output_dir,json_file)
runscript = "%s/%s" %(output_dir,runscript)
self.create(image_path=new_container_name,size=size)
# export running docker container
cmd = ['docker','export',container_id,'|','sudo','singularity','import',new_container_name]
self.run_command(cmd,sudo=sudo,suppress=True)
# Add stuff about the container to it
cmd = ['docker','inspect', container_id]
json_data = json.loads(self.run_command(cmd,sudo=sudo))
write_json(json_data,json_file)
#TODO: we might be able to extract boutique / make package here
# Add singularity.json to container
cmd = ['singularity','copy',new_container_name,json_file,'/']
self.run_command(cmd,sudo=sudo)
# Merge the /etc/group file, and add to container
self.run_command(['docker','cp',"%s:/etc/group" %(container_id),'grouphost'],sudo=sudo)
self.run_command(['sort','/etc/group','grouphost','|','uniq','-u','>','group'],sudo=sudo)
self.run_command(['singularity','copy',new_container_name,'group','/etc/group'],sudo=sudo)
os.remove(json_file)
os.remove('grouphost')
os.remove('group')
# Runscript! Currently disabling - having some bugs with this
#cmd = ['docker','inspect',"--format='{{json .Config.Cmd}}'",docker_image]
#command = self.run_command(cmd,sudo=sudo)
#command = command.replace('["',"").replace('"]',"")
#if command != "none":
# write_file(runscript,command)
# cmd = ['chmod','+x',runscript]
# self.run_command(cmd,sudo=sudo)
# cmd = ['singularity','copy',new_container_name,runscript,'/']
# self.run_command(cmd,sudo=sudo)
# os.remove(runscript)
print("Stopping container... please wait!")
cmd = ['docker','stop',container_id]
self.run_command(cmd,sudo=sudo)
return new_container_name
else:
return None
def main():
parser = get_parser()
try:
args = parser.parse_args()
except:
sys.exit(0)
# Not running in Singularity Hub environment
os.environ['SINGULARITY_HUB'] = "False"
# if environment logging variable not set, make silent
if args.debug is False:
os.environ['MESSAGELEVEL'] = "CRITICAL"
if args.version is True:
print(singularity.__version__)
sys.exit(0)
# Initialize the message bot, with level above
from singularity.logman import bot
from singularity.utils import check_install
from singularity.cli import get_image
# Output folder will be pwd if not specified
if args.outfolder is None:
output_folder = os.getcwd()
else:
output_folder = args.outfolder
# We can only continue if singularity is installed
if check_install() is True:
# If we are given an image, ensure full path
if args.image is not None:
image,existed = get_image(args.image,
return_existed=True,
size=args.size)
if image is None:
bot.logger.error("Cannot find image. Exiting.")
sys.exit(1)
# the user wants to make a tree
if args.tree is True:
from singularity.app import make_tree
make_tree(image)
clean_up(image,existed)
# The user wants to estimate the os
elif args.os is True:
from singularity.analysis.classify import estimate_os
estimated_os = estimate_os(container=image)
print(estimated_os)
# The user wants to get a list of all os
elif args.oscalc is True:
from singularity.analysis.classify import estimate_os
estimated_os = estimate_os(container=image,return_top=False)
print(estimated_os["SCORE"].to_dict())
# The user wants to get a list of tags
elif args.tags is True:
from singularity.analysis.classify import get_tags
tags = get_tags(container=image)
print(tags)
# The user wants to plot image vs. the docker os
elif args.osplot is True:
from singularity.app import plot_os_sims
plot_os_sims(image)
clean_up(image,existed)
# The user wants to package the image
elif args.package is True:
from singularity.package import package
remove_image = not args.include_image
package(image_path=image,
output_folder=output_folder,
runscript=True,
software=True,
remove_image=remove_image)
else:
print("Not sure what to do?")
parser.print_help()
# If we are given two image, we probably want a similar tree
elif args.images is not None:
image1,image2 = args.images.split(',')
bot.logger.debug("Image1: %s",image1)
bot.logger.debug("Image2: %s",image2)
image1,existed1 = get_image(image1,
return_existed=True,
size=args.size)
image2,existed2 = get_image(image2,
return_existed=True,
size=args.size)
if image1 is None or image2 is None:
bot.logger.error("Cannot find image. Exiting.")
sys.exit(1)
# the user wants to make a similarity tree
if args.simtree is True:
from singularity.app import make_sim_tree
make_sim_tree(image1,image2)
# the user wants to make a difference tree
if args.subtract is True:
from singularity.app import make_diff_tree
make_diff_tree(image1,image2)
if args.simcalc is True:
from singularity.analysis.compare import calculate_similarity
score = calculate_similarity(image1,image2,by="files.txt")
print(score["files.txt"])
clean_up(image1,existed1)
clean_up(image2,existed2)
else:
print("Please specify one or more containers with --image(s)")
from singularity.utils import check_install
from singularity.cli import Singularity
from BeautifulSoup import BeautifulSoup
import requests
import re
import os
import sys
required_softwares = ['docker','singularity']
image_directory = '/home/vanessa/Documents/Work/SINGULARITY/docker2singularity'
os.chdir(image_directory)
print('Checking for required software...')
for required_software in required_softwares:
if check_install(required_software) != True:
print("You must have %s installed to use this script!" %(required_software.upper()))
sys.exit(32)
# Create a command line client
S = Singularity()
# Here is out list of images (note -some of these couldn't be found)
# from https://github.com/docker/docker/wiki/Public-docker-images
docker_images = ["ubuntu:latest","ubuntu:12.10","ubuntu:12.04",
"opensuse:13.1","centos","busybox","base/arch",
"tianon/debian:wheezy","tianon/debian:7.1",
"tianon/debian:jessie","tianon/debian-roll:stable",
"tianon/debian-roll:7.1","johncosta/redis",
"shykes/couchdb","jpetazzo/pgsql","samalba/hipache",
"creack/firefox-vnc","jbarbier/memcached",