def main(args, parser, subparser):
# We can only continue if singularity is installed
if check_install() is not True:
bot.error("Cannot find Singularity! Is it installed?")
sys.exit(1)
# Output folder will be pwd if not specified
output_folder = os.getcwd()
if args.outfolder is not None:
output_folder = os.getcwd()
# If we are given an image, ensure full path
image = args.image
if image is not None:
existed = True
if not os.path.exists(image):
cli = Singularity(debug=args.debug)
image = cli.pull(image)
existed = False
if image is None:
bot.error("Cannot find image. Exiting.")
sys.exit(1)
# The user wants to estimate the os
if args.os is True:
from singularity.analysis.classify import estimate_os
estimated_os = estimate_os(container=image)
print(estimated_os)
# The user wants to get a list of all os
elif args.oscalc is True:
from singularity.analysis.classify import estimate_os
estimated_os = estimate_os(container=image, return_top=False)
print(estimated_os["SCORE"].to_dict())
# The user wants to get a list of tags
elif args.tags is True:
from singularity.analysis.classify import get_tags
tags = get_tags(container=image)
print(tags)
# The user wants to plot image vs. the docker os
elif args.osplot is True:
from singularity.app import plot_os_sims
plot_os_sims(image)
clean_up(image, existed)
else:
print("Please specify an image to analyze with --image")
subparser.print_help()
def app_plot_os_sims():
if app.sims == None:
sims = estimate_os(container=app.image,
sudopw=app.sudopw,
return_top=False)
sims = sims.sort_values(by=['SCORE'])
app.sims = sims['SCORE'].to_dict()
container_name = os.path.basename(app.image).split(".")[0]
return render_template('similarity_scatter.html',sim_scores=app.sims,
container_name=container_name)
#!/usr/bin/env python # This is an example of generating image packages from within python from singularity.analysis.classify import estimate_os image_package = "python:3.6.0.img.zip" # We can obtain the estimated os (top match) estimated_os = estimate_os(image_package=image_package) # Most similar OS found to be %s debian:7.11 # We can also get the whole list and values os_similarity = estimate_os(image_package=image_package, return_top=False) # If you want to sort os_similarity.sort_values(by=['SCORE'])
def test_estimate_os(self):
print("Testing singularity.analysis.classify.estimate_os")
from singularity.analysis.classify import estimate_os
estimated_os = estimate_os(self.container)
self.assertTrue(estimated_os.startswith('ubuntu'))
def run_build(build_dir,params,verbose=True, compress_image=False):
'''run_build takes a build directory and params dictionary, and does the following:
- downloads repo to a temporary directory
- changes branch or commit, if needed
- creates and bootstraps singularity image from Singularity file
- returns a dictionary with:
image (path), image_package (path), metadata (dict)
The following must be included in params:
spec_file, repo_url, branch, commit
Optional parameters
size
'''
# Download the repo and image
download_repo(repo_url=params['repo_url'],
destination=build_dir)
os.chdir(build_dir)
if params['branch'] != None:
bot.info('Checking out branch %s' %params['branch'])
os.system('git checkout %s' %(params['branch']))
else:
params['branch'] = "master"
# Commit
if params['commit'] not in [None,'']:
bot.info('Checking out commit %s' %params['commit'])
os.system('git checkout %s .' %(params['commit']))
# From here on out commit is used as a unique id, if we don't have one, we use current
else:
params['commit'] = os.popen('git log -n 1 --pretty=format:"%H"').read()
bot.warning("commit not specified, setting to current %s" %params['commit'])
# Dump some params for the builder, in case it fails after this
passing_params = "/tmp/params.pkl"
pickle.dump(params,open(passing_params,'wb'))
# Now look for spec file
if os.path.exists(params['spec_file']):
bot.info("Found spec file %s in repository" %params['spec_file'])
# If the user has a symbolic link
if os.path.islink(params['spec_file']):
bot.info("%s is a symbolic link." %params['spec_file'])
params['spec_file'] = os.path.realpath(params['spec_file'])
# START TIMING
start_time = datetime.now()
image = build_from_spec(spec_file=params['spec_file'], # default will package the image
build_dir=build_dir,
isolated=True,
sandbox=False,
debug=params['debug'])
# Save has for metadata (also is image name)
version = get_image_file_hash(image)
params['version'] = version
pickle.dump(params,open(passing_params,'wb'))
final_time = (datetime.now() - start_time).seconds
bot.info("Final time of build %s seconds." %final_time)
# Did the container build successfully?
test_result = test_container(image)
if test_result['return_code'] != 0:
bot.error("Image failed to build, cancelling.")
sys.exit(1)
# Get singularity version
singularity_version = get_singularity_version()
# Package the image metadata (files, folders, etc)
image_package = package(image_path=image,
spec_path=params['spec_file'],
output_folder=build_dir,
remove_image=True,
verbose=True)
# Derive software tags by subtracting similar OS
diff = get_diff(image_package=image_package)
# Inspect to get labels and other metadata
cli = Singularity(debug=params['debug'])
inspect = cli.inspect(image_path=image)
# Get information on apps
app_names = cli.apps(image_path=image)
apps = extract_apps(image_path=image, app_names=app_names)
# Count file types, and extensions
counts = dict()
counts['readme'] = file_counts(diff=diff)
counts['copyright'] = file_counts(diff=diff,patterns=['copyright'])
counts['authors-thanks-credit'] = file_counts(diff=diff,
patterns=['authors','thanks','credit','contributors'])
counts['todo'] = file_counts(diff=diff,patterns=['todo'])
extensions = extension_counts(diff=diff)
os_sims = estimate_os(image_package=image_package,return_top=False)
most_similar = os_sims['SCORE'].values.argmax()
most_similar = os_sims['SCORE'].index.tolist()[most_similar]
metrics = {'build_time_seconds':final_time,
'singularity_version':singularity_version,
'singularity_python_version':singularity_python_version,
'estimated_os': most_similar,
'os_sims':os_sims['SCORE'].to_dict(),
'file_counts':counts,
'file_ext':extensions,
'inspect':inspect,
'version': version,
'apps': apps}
# Compress Image
if compress_image is True:
compressed_image = "%s.gz" %image
os.system('gzip -c -9 %s > %s' %(image,compressed_image))
image = compressed_image
output = {'image':image,
'image_package':image_package,
'metadata':metrics,
'params':params }
return output
else:
# Tell the user what is actually there
present_files = glob("*")
bot.error("Build file %s not found in repository" %params['spec_file'])
bot.info("Found files are %s" %"\n".join(present_files))
# Params have been exported, will be found by log
sys.exit(1)
def main():
parser = get_parser()
try:
args = parser.parse_args()
except:
sys.exit(0)
# Not running in Singularity Hub environment
os.environ['SINGULARITY_HUB'] = "False"
# if environment logging variable not set, make silent
if args.debug is False:
os.environ['MESSAGELEVEL'] = "CRITICAL"
if args.version is True:
print(singularity.__version__)
sys.exit(0)
# Initialize the message bot, with level above
from singularity.logman import bot
from singularity.utils import check_install
from singularity.cli import get_image
# Output folder will be pwd if not specified
if args.outfolder is None:
output_folder = os.getcwd()
else:
output_folder = args.outfolder
# We can only continue if singularity is installed
if check_install() is True:
# If we are given an image, ensure full path
if args.image is not None:
image,existed = get_image(args.image,
return_existed=True,
size=args.size)
if image is None:
bot.logger.error("Cannot find image. Exiting.")
sys.exit(1)
# the user wants to make a tree
if args.tree is True:
from singularity.app import make_tree
make_tree(image)
clean_up(image,existed)
# The user wants to estimate the os
elif args.os is True:
from singularity.analysis.classify import estimate_os
estimated_os = estimate_os(container=image)
print(estimated_os)
# The user wants to get a list of all os
elif args.oscalc is True:
from singularity.analysis.classify import estimate_os
estimated_os = estimate_os(container=image,return_top=False)
print(estimated_os["SCORE"].to_dict())
# The user wants to get a list of tags
elif args.tags is True:
from singularity.analysis.classify import get_tags
tags = get_tags(container=image)
print(tags)
# The user wants to plot image vs. the docker os
elif args.osplot is True:
from singularity.app import plot_os_sims
plot_os_sims(image)
clean_up(image,existed)
# The user wants to package the image
elif args.package is True:
from singularity.package import package
remove_image = not args.include_image
package(image_path=image,
output_folder=output_folder,
runscript=True,
software=True,
remove_image=remove_image)
else:
print("Not sure what to do?")
parser.print_help()
# If we are given two image, we probably want a similar tree
elif args.images is not None:
image1,image2 = args.images.split(',')
bot.logger.debug("Image1: %s",image1)
bot.logger.debug("Image2: %s",image2)
image1,existed1 = get_image(image1,
return_existed=True,
size=args.size)
image2,existed2 = get_image(image2,
return_existed=True,
size=args.size)
if image1 is None or image2 is None:
bot.logger.error("Cannot find image. Exiting.")
sys.exit(1)
# the user wants to make a similarity tree
if args.simtree is True:
from singularity.app import make_sim_tree
make_sim_tree(image1,image2)
# the user wants to make a difference tree
if args.subtract is True:
from singularity.app import make_diff_tree
make_diff_tree(image1,image2)
if args.simcalc is True:
from singularity.analysis.compare import calculate_similarity
score = calculate_similarity(image1,image2,by="files.txt")
print(score["files.txt"])
clean_up(image1,existed1)
clean_up(image2,existed2)
else:
print("Please specify one or more containers with --image(s)")
def run_build(build_dir, params, verbose=True):
'''run_build takes a build directory and params dictionary, and does the following:
- downloads repo to a temporary directory
- changes branch or commit, if needed
- creates and bootstraps singularity image from Singularity file
- returns a dictionary with:
image (path), image_package (path), metadata (dict)
The following must be included in params:
spec_file, repo_url, branch, commit
Optional parameters
size
'''
# Download the repo and image
download_repo(repo_url=params['repo_url'], destination=build_dir)
os.chdir(build_dir)
if params['branch'] != None:
bot.logger.info('Checking out branch %s', params['branch'])
os.system('git checkout %s' % (params['branch']))
# Commit
if params['commit'] not in [None, '']:
bot.logger.info('Checking out commit %s', params['commit'])
os.system('git checkout %s .' % (params['commit']))
# From here on out commit is used as a unique id, if we don't have one, we use current
else:
params['commit'] = os.popen('git log -n 1 --pretty=format:"%H"').read()
bot.logger.warning("commit not specified, setting to current %s",
params['commit'])
# Dump some params for the builder, in case it fails after this
passing_params = "/tmp/params.pkl"
pickle.dump(params, open(passing_params, 'wb'))
# If there is not a specfile, but is a Dockerfile, try building that
if not os.path.exists(
params['spec_file']) and os.path.exists('Dockerfile'):
bot.logger.warning("Build file %s not found in repository",
params['spec_file'])
bot.logger.warning(
"Dockerfile found in repository, will attempt build.")
dockerfile = dockerfile_to_singularity(dockerfile_path='Dockerfile',
output_dir=build_dir)
if dockerfile is not None:
bot.logger.info("""\n
--------------------------------------------------------------
Dockerfile
--------------------------------------------------------------
\n%s""" % (dockerfile))
# Now look for spec file
if os.path.exists(params['spec_file']):
bot.logger.info("Found spec file %s in repository",
params['spec_file'])
# If size is None, set default of 800
if params['size'] in [None, '']:
bot.logger.info("""\n
--------------------------------------------------------------
Size not detected for build. Will first try to estimate, and then
use default of 800MB padding. If your build still fails, you should
try setting the size manually under collection --> edit builder
---------------------------------------------------------------------
\n""")
# Testing estimation of size
try:
params['size'] = estimate_image_size(spec_file=os.path.abspath(
params['spec_file']),
sudopw='',
padding=params['padding'])
bot.logger.info("Size estimated as %s", params['size'])
except:
params['size'] = 800
bot.logger.info(
"Size estimation didn't work, using default %s",
params['size'])
# START TIMING
os.chdir(build_dir)
start_time = datetime.now()
image = build_from_spec(
spec_file=params['spec_file'], # default will package the image
size=params['size'],
sudopw='', # with root should not need sudo
build_dir=build_dir,
debug=params['debug'])
final_time = (datetime.now() - start_time).seconds
bot.logger.info("Final time of build %s seconds.", final_time)
# Did the container build successfully?
test_result = test_container(image)
if test_result['return_code'] == 255:
bot.logger.error("Image failed to bootstrap, cancelling build.")
sys.exit(1)
# Compress image
compressed_image = "%s.img.gz" % image
os.system('gzip -c -9 %s > %s' % (image, compressed_image))
# Get singularity version
singularity_version = get_singularity_version()
old_version = False
if singularity_version.startswith('2.2'):
old_version = True
# Package the image metadata (files, folders, etc)
image_package = package(image_path=image,
spec_path=params['spec_file'],
output_folder=build_dir,
sudopw='',
remove_image=True,
verbose=True,
old_version=old_version)
# Derive software tags by subtracting similar OS
diff = get_diff(image_package=image_package)
# Get tags for services, executables
interesting_folders = ['init', 'init.d', 'bin', 'systemd']
tags = get_tags(search_folders=interesting_folders, diff=diff)
# Count file types, and extensions
counts = dict()
counts['readme'] = file_counts(diff=diff)
counts['copyright'] = file_counts(diff=diff, patterns=['copyright'])
counts['authors-thanks-credit'] = file_counts(
diff=diff, patterns=['authors', 'thanks', 'credit'])
counts['todo'] = file_counts(diff=diff, patterns=['todo'])
extensions = extension_counts(diff=diff)
os_sims = estimate_os(image_package=image_package, return_top=False)
most_similar = os_sims['SCORE'].idxmax()
metrics = {
'size': params['size'],
'build_time_seconds': final_time,
'singularity_version': singularity_version,
'singularity_python_version': singularity_python_version,
'estimated_os': most_similar,
'os_sims': os_sims['SCORE'].to_dict(),
'tags': tags,
'file_counts': counts,
'file_ext': extensions
}
output = {
'image': compressed_image,
'image_package': image_package,
'metadata': metrics,
'params': params
}
return output
else:
# Tell the user what is actually there
present_files = glob("*")
bot.logger.error("Build file %s not found in repository",
params['spec_file'])
bot.logger.info("Found files are %s", "\n".join(present_files))
# Params have been exported, will be found by log
sys.exit(1)
