def main(args, parser, subparser): # We can only continue if singularity is installed if check_install() is not True: bot.error("Cannot find Singularity! Is it installed?") sys.exit(1) # Output folder will be pwd if not specified output_folder = os.getcwd() if args.outfolder is not None: output_folder = os.getcwd() # If we are given an image, ensure full path image = args.image if image is not None: existed = True if not os.path.exists(image): cli = Singularity(debug=args.debug) image = cli.pull(image) existed = False if image is None: bot.error("Cannot find image. Exiting.") sys.exit(1) # The user wants to estimate the os if args.os is True: from singularity.analysis.classify import estimate_os estimated_os = estimate_os(container=image) print(estimated_os) # The user wants to get a list of all os elif args.oscalc is True: from singularity.analysis.classify import estimate_os estimated_os = estimate_os(container=image, return_top=False) print(estimated_os["SCORE"].to_dict()) # The user wants to get a list of tags elif args.tags is True: from singularity.analysis.classify import get_tags tags = get_tags(container=image) print(tags) # The user wants to plot image vs. the docker os elif args.osplot is True: from singularity.app import plot_os_sims plot_os_sims(image) clean_up(image, existed) else: print("Please specify an image to analyze with --image") subparser.print_help()
def app_plot_os_sims(): if app.sims == None: sims = estimate_os(container=app.image, sudopw=app.sudopw, return_top=False) sims = sims.sort_values(by=['SCORE']) app.sims = sims['SCORE'].to_dict() container_name = os.path.basename(app.image).split(".")[0] return render_template('similarity_scatter.html',sim_scores=app.sims, container_name=container_name)
#!/usr/bin/env python # This is an example of generating image packages from within python from singularity.analysis.classify import estimate_os image_package = "python:3.6.0.img.zip" # We can obtain the estimated os (top match) estimated_os = estimate_os(image_package=image_package) # Most similar OS found to be %s debian:7.11 # We can also get the whole list and values os_similarity = estimate_os(image_package=image_package, return_top=False) # If you want to sort os_similarity.sort_values(by=['SCORE'])
def test_estimate_os(self): print("Testing singularity.analysis.classify.estimate_os") from singularity.analysis.classify import estimate_os estimated_os = estimate_os(self.container) self.assertTrue(estimated_os.startswith('ubuntu'))
def run_build(build_dir,params,verbose=True, compress_image=False): '''run_build takes a build directory and params dictionary, and does the following: - downloads repo to a temporary directory - changes branch or commit, if needed - creates and bootstraps singularity image from Singularity file - returns a dictionary with: image (path), image_package (path), metadata (dict) The following must be included in params: spec_file, repo_url, branch, commit Optional parameters size ''' # Download the repo and image download_repo(repo_url=params['repo_url'], destination=build_dir) os.chdir(build_dir) if params['branch'] != None: bot.info('Checking out branch %s' %params['branch']) os.system('git checkout %s' %(params['branch'])) else: params['branch'] = "master" # Commit if params['commit'] not in [None,'']: bot.info('Checking out commit %s' %params['commit']) os.system('git checkout %s .' %(params['commit'])) # From here on out commit is used as a unique id, if we don't have one, we use current else: params['commit'] = os.popen('git log -n 1 --pretty=format:"%H"').read() bot.warning("commit not specified, setting to current %s" %params['commit']) # Dump some params for the builder, in case it fails after this passing_params = "/tmp/params.pkl" pickle.dump(params,open(passing_params,'wb')) # Now look for spec file if os.path.exists(params['spec_file']): bot.info("Found spec file %s in repository" %params['spec_file']) # If the user has a symbolic link if os.path.islink(params['spec_file']): bot.info("%s is a symbolic link." %params['spec_file']) params['spec_file'] = os.path.realpath(params['spec_file']) # START TIMING start_time = datetime.now() image = build_from_spec(spec_file=params['spec_file'], # default will package the image build_dir=build_dir, isolated=True, sandbox=False, debug=params['debug']) # Save has for metadata (also is image name) version = get_image_file_hash(image) params['version'] = version pickle.dump(params,open(passing_params,'wb')) final_time = (datetime.now() - start_time).seconds bot.info("Final time of build %s seconds." %final_time) # Did the container build successfully? test_result = test_container(image) if test_result['return_code'] != 0: bot.error("Image failed to build, cancelling.") sys.exit(1) # Get singularity version singularity_version = get_singularity_version() # Package the image metadata (files, folders, etc) image_package = package(image_path=image, spec_path=params['spec_file'], output_folder=build_dir, remove_image=True, verbose=True) # Derive software tags by subtracting similar OS diff = get_diff(image_package=image_package) # Inspect to get labels and other metadata cli = Singularity(debug=params['debug']) inspect = cli.inspect(image_path=image) # Get information on apps app_names = cli.apps(image_path=image) apps = extract_apps(image_path=image, app_names=app_names) # Count file types, and extensions counts = dict() counts['readme'] = file_counts(diff=diff) counts['copyright'] = file_counts(diff=diff,patterns=['copyright']) counts['authors-thanks-credit'] = file_counts(diff=diff, patterns=['authors','thanks','credit','contributors']) counts['todo'] = file_counts(diff=diff,patterns=['todo']) extensions = extension_counts(diff=diff) os_sims = estimate_os(image_package=image_package,return_top=False) most_similar = os_sims['SCORE'].values.argmax() most_similar = os_sims['SCORE'].index.tolist()[most_similar] metrics = {'build_time_seconds':final_time, 'singularity_version':singularity_version, 'singularity_python_version':singularity_python_version, 'estimated_os': most_similar, 'os_sims':os_sims['SCORE'].to_dict(), 'file_counts':counts, 'file_ext':extensions, 'inspect':inspect, 'version': version, 'apps': apps} # Compress Image if compress_image is True: compressed_image = "%s.gz" %image os.system('gzip -c -9 %s > %s' %(image,compressed_image)) image = compressed_image output = {'image':image, 'image_package':image_package, 'metadata':metrics, 'params':params } return output else: # Tell the user what is actually there present_files = glob("*") bot.error("Build file %s not found in repository" %params['spec_file']) bot.info("Found files are %s" %"\n".join(present_files)) # Params have been exported, will be found by log sys.exit(1)
def main(): parser = get_parser() try: args = parser.parse_args() except: sys.exit(0) # Not running in Singularity Hub environment os.environ['SINGULARITY_HUB'] = "False" # if environment logging variable not set, make silent if args.debug is False: os.environ['MESSAGELEVEL'] = "CRITICAL" if args.version is True: print(singularity.__version__) sys.exit(0) # Initialize the message bot, with level above from singularity.logman import bot from singularity.utils import check_install from singularity.cli import get_image # Output folder will be pwd if not specified if args.outfolder is None: output_folder = os.getcwd() else: output_folder = args.outfolder # We can only continue if singularity is installed if check_install() is True: # If we are given an image, ensure full path if args.image is not None: image,existed = get_image(args.image, return_existed=True, size=args.size) if image is None: bot.logger.error("Cannot find image. Exiting.") sys.exit(1) # the user wants to make a tree if args.tree is True: from singularity.app import make_tree make_tree(image) clean_up(image,existed) # The user wants to estimate the os elif args.os is True: from singularity.analysis.classify import estimate_os estimated_os = estimate_os(container=image) print(estimated_os) # The user wants to get a list of all os elif args.oscalc is True: from singularity.analysis.classify import estimate_os estimated_os = estimate_os(container=image,return_top=False) print(estimated_os["SCORE"].to_dict()) # The user wants to get a list of tags elif args.tags is True: from singularity.analysis.classify import get_tags tags = get_tags(container=image) print(tags) # The user wants to plot image vs. the docker os elif args.osplot is True: from singularity.app import plot_os_sims plot_os_sims(image) clean_up(image,existed) # The user wants to package the image elif args.package is True: from singularity.package import package remove_image = not args.include_image package(image_path=image, output_folder=output_folder, runscript=True, software=True, remove_image=remove_image) else: print("Not sure what to do?") parser.print_help() # If we are given two image, we probably want a similar tree elif args.images is not None: image1,image2 = args.images.split(',') bot.logger.debug("Image1: %s",image1) bot.logger.debug("Image2: %s",image2) image1,existed1 = get_image(image1, return_existed=True, size=args.size) image2,existed2 = get_image(image2, return_existed=True, size=args.size) if image1 is None or image2 is None: bot.logger.error("Cannot find image. Exiting.") sys.exit(1) # the user wants to make a similarity tree if args.simtree is True: from singularity.app import make_sim_tree make_sim_tree(image1,image2) # the user wants to make a difference tree if args.subtract is True: from singularity.app import make_diff_tree make_diff_tree(image1,image2) if args.simcalc is True: from singularity.analysis.compare import calculate_similarity score = calculate_similarity(image1,image2,by="files.txt") print(score["files.txt"]) clean_up(image1,existed1) clean_up(image2,existed2) else: print("Please specify one or more containers with --image(s)")
def run_build(build_dir, params, verbose=True): '''run_build takes a build directory and params dictionary, and does the following: - downloads repo to a temporary directory - changes branch or commit, if needed - creates and bootstraps singularity image from Singularity file - returns a dictionary with: image (path), image_package (path), metadata (dict) The following must be included in params: spec_file, repo_url, branch, commit Optional parameters size ''' # Download the repo and image download_repo(repo_url=params['repo_url'], destination=build_dir) os.chdir(build_dir) if params['branch'] != None: bot.logger.info('Checking out branch %s', params['branch']) os.system('git checkout %s' % (params['branch'])) # Commit if params['commit'] not in [None, '']: bot.logger.info('Checking out commit %s', params['commit']) os.system('git checkout %s .' % (params['commit'])) # From here on out commit is used as a unique id, if we don't have one, we use current else: params['commit'] = os.popen('git log -n 1 --pretty=format:"%H"').read() bot.logger.warning("commit not specified, setting to current %s", params['commit']) # Dump some params for the builder, in case it fails after this passing_params = "/tmp/params.pkl" pickle.dump(params, open(passing_params, 'wb')) # If there is not a specfile, but is a Dockerfile, try building that if not os.path.exists( params['spec_file']) and os.path.exists('Dockerfile'): bot.logger.warning("Build file %s not found in repository", params['spec_file']) bot.logger.warning( "Dockerfile found in repository, will attempt build.") dockerfile = dockerfile_to_singularity(dockerfile_path='Dockerfile', output_dir=build_dir) if dockerfile is not None: bot.logger.info("""\n -------------------------------------------------------------- Dockerfile -------------------------------------------------------------- \n%s""" % (dockerfile)) # Now look for spec file if os.path.exists(params['spec_file']): bot.logger.info("Found spec file %s in repository", params['spec_file']) # If size is None, set default of 800 if params['size'] in [None, '']: bot.logger.info("""\n -------------------------------------------------------------- Size not detected for build. Will first try to estimate, and then use default of 800MB padding. If your build still fails, you should try setting the size manually under collection --> edit builder --------------------------------------------------------------------- \n""") # Testing estimation of size try: params['size'] = estimate_image_size(spec_file=os.path.abspath( params['spec_file']), sudopw='', padding=params['padding']) bot.logger.info("Size estimated as %s", params['size']) except: params['size'] = 800 bot.logger.info( "Size estimation didn't work, using default %s", params['size']) # START TIMING os.chdir(build_dir) start_time = datetime.now() image = build_from_spec( spec_file=params['spec_file'], # default will package the image size=params['size'], sudopw='', # with root should not need sudo build_dir=build_dir, debug=params['debug']) final_time = (datetime.now() - start_time).seconds bot.logger.info("Final time of build %s seconds.", final_time) # Did the container build successfully? test_result = test_container(image) if test_result['return_code'] == 255: bot.logger.error("Image failed to bootstrap, cancelling build.") sys.exit(1) # Compress image compressed_image = "%s.img.gz" % image os.system('gzip -c -9 %s > %s' % (image, compressed_image)) # Get singularity version singularity_version = get_singularity_version() old_version = False if singularity_version.startswith('2.2'): old_version = True # Package the image metadata (files, folders, etc) image_package = package(image_path=image, spec_path=params['spec_file'], output_folder=build_dir, sudopw='', remove_image=True, verbose=True, old_version=old_version) # Derive software tags by subtracting similar OS diff = get_diff(image_package=image_package) # Get tags for services, executables interesting_folders = ['init', 'init.d', 'bin', 'systemd'] tags = get_tags(search_folders=interesting_folders, diff=diff) # Count file types, and extensions counts = dict() counts['readme'] = file_counts(diff=diff) counts['copyright'] = file_counts(diff=diff, patterns=['copyright']) counts['authors-thanks-credit'] = file_counts( diff=diff, patterns=['authors', 'thanks', 'credit']) counts['todo'] = file_counts(diff=diff, patterns=['todo']) extensions = extension_counts(diff=diff) os_sims = estimate_os(image_package=image_package, return_top=False) most_similar = os_sims['SCORE'].idxmax() metrics = { 'size': params['size'], 'build_time_seconds': final_time, 'singularity_version': singularity_version, 'singularity_python_version': singularity_python_version, 'estimated_os': most_similar, 'os_sims': os_sims['SCORE'].to_dict(), 'tags': tags, 'file_counts': counts, 'file_ext': extensions } output = { 'image': compressed_image, 'image_package': image_package, 'metadata': metrics, 'params': params } return output else: # Tell the user what is actually there present_files = glob("*") bot.logger.error("Build file %s not found in repository", params['spec_file']) bot.logger.info("Found files are %s", "\n".join(present_files)) # Params have been exported, will be found by log sys.exit(1)