def package(image_path, spec_path=None, output_folder=None, remove_image=False, verbose=False, S=None): '''generate a zip (including the image) to a user specified output_folder. :param image_path: full path to singularity image file :param runscript: if True, will extract runscript to include in package as runscript :param software: if True, will extract files.txt and folders.txt to package :param remove_image: if True, will not include original image in package (default,False) :param verbose: be verbose when using singularity --export (default,False) :param S: the Singularity object (optional) will be created if not required. ''' # Run create image and bootstrap with Singularity command line tool. S = Singularity(debug=verbose) file_obj, tar = get_image_tar(image_path, S=S) members = tar.getmembers() image_name, ext = os.path.splitext(os.path.basename(image_path)) zip_name = "%s.zip" % (image_name.replace(" ", "_")) # Include the image in the package? to_package = dict() if not remove_image: to_package["files"] = [image_path] # If the specfile is provided, it should also be packaged if spec_path is not None: singularity_spec = "".join(read_file(spec_path)) to_package['Singularity'] = singularity_spec to_package["VERSION"] = get_image_file_hash(image_path) try: inspection = S.inspect(image_path) to_package["inspect.json"] = inspection inspection = json.loads(inspection) to_package['runscript'] = inspection['data']['attributes']['runscript'] except: bot.warning("Trouble extracting container metadata with inspect.") bot.info("Adding software list to package.") files = [x.path for x in members if x.isfile()] folders = [x.path for x in members if x.isdir()] to_package["files.txt"] = files to_package["folders.txt"] = folders # Do zip up here - let's start with basic structures zipfile = zip_up(to_package, zip_name=zip_name, output_folder=output_folder) bot.debug("Package created at %s" % (zipfile)) if not delete_image_tar(file_obj, tar): bot.warning("Could not clean up temporary tarfile.") # return package to user return zipfile
def remove(self, image, force=False): '''delete an image to Singularity Registry''' q = parse_image_name(image) url = '%s/container/%s/%s:%s' % (self.base, q["collection"], q["image"], q["tag"]) SREGISTRY_EVENT = self.authorize(request_type="delete", names=q) headers = {'Authorization': SREGISTRY_EVENT} self.update_headers(fields=headers) continue_delete = True if force is False: response = input("Are you sure you want to delete %s?" % q['uri']) while len(response) < 1 or response[0].lower().strip( ) not in "ynyesno": response = input("Please answer yes or no: ") if response[0].lower().strip() in "no": continue_delete = False if continue_delete is True: response = self.delete(url) message = self.read_response(response) bot.info("Response %s, %s" % (response.status_code, message)) else: bot.info("Delete cancelled.")
def get_singularity_version(singularity_version=None): '''get_singularity_version will determine the singularity version for a build first, an environmental variable is looked at, followed by using the system version. ''' if singularity_version is None: singularity_version = os.environ.get("SINGULARITY_VERSION") if singularity_version is None: try: cmd = ['singularity', '--version'] output = run_command(cmd) if isinstance(output['message'], bytes): output['message'] = output['message'].decode('utf-8') singularity_version = output['message'].strip('\n') bot.info("Singularity %s being used." % singularity_version) except: singularity_version = None bot.warning( "Singularity version not found, so it's likely not installed.") return singularity_version
def file_counts(container=None, patterns=None, image_package=None, diff=None): '''file counts will return a list of files that match one or more regular expressions. if no patterns is defined, a default of readme is used. All patterns and files are made case insensitive. :param container: if provided, will use container as image. Can also provide :param image_package: if provided, can be used instead of container :param patterns: one or more patterns (str or list) of files to search for. :param diff: the difference between a container and it's parent OS from get_diff if not provided, will be generated. ''' if diff == None: diff = get_diff(container=container, image_package=image_package) if patterns == None: patterns = 'readme' if not isinstance(patterns, list): patterns = [patterns] count = 0 for folder, items in diff.items(): for pattern in patterns: count += len( [x for x in items if re.search(pattern.lower(), x.lower())]) bot.info("Total files matching patterns is %s" % count) return count
def file_counts(container=None, patterns=None, image_package=None, file_list=None): '''file counts will return a list of files that match one or more regular expressions. if no patterns is defined, a default of readme is used. All patterns and files are made case insensitive. Parameters ========== :param container: if provided, will use container as image. Can also provide :param image_package: if provided, can be used instead of container :param patterns: one or more patterns (str or list) of files to search for. :param diff: the difference between a container and it's parent OS from get_diff if not provided, will be generated. ''' if file_list is None: file_list = get_container_contents(container, split_delim='\n')['all'] if patterns == None: patterns = 'readme' if not isinstance(patterns,list): patterns = [patterns] count = 0 for pattern in patterns: count += len([x for x in file_list if re.search(pattern.lower(),x.lower())]) bot.info("Total files matching patterns is %s" %count) return count
def clean_up(image, existed): '''clean up will remove an image file if existed is False (meaning it was created as temporary for the script ''' if existed == False: if os.path.exists(image): bot.info("%s created was temporary, removing" % image) os.remove(image)
def main(args, parser, subparser): if args.recipe is None: subparser.print_help() bot.newline() print("Please specify creating a recipe with --recipe") sys.exit(0) # Output folder will be pwd if not specified output_folder = os.getcwd() if args.outfolder is not None: output_folder = os.getcwd() bootstrap = '' if args.bootstrap is not None: bootstrap = args.bootstrap bot.debug("bootstrap: %s" % bootstrap) bootstrap_from = '' if args.bootstrap_from is not None: bootstrap_from = args.bootstrap_from bot.debug("from: %s" % bootstrap_from) template = "Singularity" output_file = template app = '' if args.app is not None: app = args.app.lower() template = "Singularity.app" output_file = "Singularity.%s" % app input_file = "%s/cli/app/templates/recipes/%s" % (get_installdir(), template) output_file = "%s/%s" % (output_folder, output_file) if os.path.exists(output_file): ext = str(uuid.uuid4())[0:4] output_file = "%s.%s" % (output_file, ext) # Read the file, make substitutions contents = read_file(input_file, readlines=False) # Replace all occurrences of app contents = contents.replace('{{ app }}', app) contents = contents.replace('{{ bootstrap }}', bootstrap) contents = contents.replace('{{ from }}', bootstrap_from) write_file(output_file, contents) bot.info("Output file written to %s" % output_file)
def sniff_extension(file_path, verbose=True): '''sniff_extension will attempt to determine the file type based on the extension, and return the proper mimetype :param file_path: the full path to the file to sniff :param verbose: print stuff out ''' mime_types = { "xls": 'application/vnd.ms-excel', "xlsx": 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', "xml": 'text/xml', "ods": 'application/vnd.oasis.opendocument.spreadsheet', "csv": 'text/plain', "tmpl": 'text/plain', "pdf": 'application/pdf', "php": 'application/x-httpd-php', "jpg": 'image/jpeg', "png": 'image/png', "gif": 'image/gif', "bmp": 'image/bmp', "txt": 'text/plain', "doc": 'application/msword', "js": 'text/js', "swf": 'application/x-shockwave-flash', "mp3": 'audio/mpeg', "zip": 'application/zip', "simg": 'application/zip', "rar": 'application/rar', "tar": 'application/tar', "arj": 'application/arj', "cab": 'application/cab', "html": 'text/html', "htm": 'text/html', "default": 'application/octet-stream', "folder": 'application/vnd.google-apps.folder', "img": "application/octet-stream" } ext = os.path.basename(file_path).split('.')[-1] mime_type = mime_types.get(ext, None) if mime_type == None: mime_type = mime_types['txt'] if verbose == True: bot.info("%s --> %s" % (file_path, mime_type)) return mime_type
def make_package_tree(matrix=None, labels=None, width=25, height=10, title=None, font_size=None): '''make package tree will make a dendrogram comparing a matrix of packages :param matrix: a pandas df of packages, with names in index and columns :param labels: a list of labels corresponding to row names, will be pulled from rows if not defined :param title: a title for the plot, if not defined, will be left out. :returns a plot that can be saved with savefig ''' from matplotlib import pyplot as plt from scipy.cluster.hierarchy import (dendrogram, linkage) if font_size is None: font_size = 8. from scipy.cluster.hierarchy import cophenet from scipy.spatial.distance import pdist if not isinstance(matrix, pandas.DataFrame): bot.info( "No pandas DataFrame (matrix) of similarities defined, will use default." ) matrix = compare_packages()['files.txt'] title = 'Docker Library Similarity to Base OS' Z = linkage(matrix, 'ward') c, coph_dists = cophenet(Z, pdist(matrix)) if labels == None: labels = matrix.index.tolist() plt.figure(figsize=(width, height)) if title != None: plt.title(title) plt.xlabel('image index') plt.ylabel('distance') dendrogram( Z, leaf_rotation=90., # rotates the x axis labels leaf_font_size=font_size, # font size for the x axis labels labels=labels) return plt
def pull(self, images, file_name=None, decompress=True): bot.debug('Execution of PULL for %s images' % len(images)) for image in images: # If we need to decompress, it's old ext3 format if decompress is True: ext = 'img.gz' else: ext = 'simg' # squashfs q = parse_image_name(image, ext=ext) # Verify image existence, and obtain id url = "%s/container/%s/%s:%s" % (self.base, q['collection'], q['image'], q['tag']) bot.debug('Retrieving manifest at %s' % url) manifest = self.get(url) bot.debug(manifest) if file_name is None: file_name = q['storage'].replace('/', '-') image_file = self.download(url=manifest['image'], file_name=file_name, show_progress=True) bot.debug('Retrieved image file %s' % image_file) if os.path.exists(image_file) and decompress is True: # If compressed, decompress try: cli = Singularity() sys.stdout.write('Decompressing image ') bot.spinner.start() image_file = cli.decompress(image_file, quiet=True) except KeyboardInterrupt: bot.warning('Decompression cancelled.') except: bot.info('Image is not compressed.') image_name = image_file.replace('.gz', '') image_file = shutil.move(image_file, image_name) pass bot.spinner.stop() bot.custom(prefix="Success!", message=image_file)
def sniff_extension(file_path,verbose=True): '''sniff_extension will attempt to determine the file type based on the extension, and return the proper mimetype :param file_path: the full path to the file to sniff :param verbose: print stuff out ''' mime_types = { "xls": 'application/vnd.ms-excel', "xlsx": 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', "xml": 'text/xml', "ods": 'application/vnd.oasis.opendocument.spreadsheet', "csv": 'text/plain', "tmpl": 'text/plain', "pdf": 'application/pdf', "php": 'application/x-httpd-php', "jpg": 'image/jpeg', "png": 'image/png', "gif": 'image/gif', "bmp": 'image/bmp', "txt": 'text/plain', "doc": 'application/msword', "js": 'text/js', "swf": 'application/x-shockwave-flash', "mp3": 'audio/mpeg', "zip": 'application/zip', "simg": 'application/zip', "rar": 'application/rar', "tar": 'application/tar', "arj": 'application/arj', "cab": 'application/cab', "html": 'text/html', "htm": 'text/html', "default": 'application/octet-stream', "folder": 'application/vnd.google-apps.folder', "img" : "application/octet-stream" } ext = os.path.basename(file_path).split('.')[-1] mime_type = mime_types.get(ext,None) if mime_type == None: mime_type = mime_types['txt'] if verbose==True: bot.info("%s --> %s" %(file_path, mime_type)) return mime_type
def main(): parser = get_parser() subparsers = get_subparsers(parser) try: args = parser.parse_args() except: sys.exit(0) # Not running in Singularity Hub environment os.environ['SINGULARITY_HUB'] = "False" # if environment logging variable not set, make silent if args.debug is False: os.environ['MESSAGELEVEL'] = "CRITICAL" # Always print the version from singularity.logger import bot import singularity bot.info("Singularity Python Version: %s" % singularity.__version__) if args.command == "create": from .create import main if args.command == "compare": from .compare import main elif args.command == "analysis": from .analysis import main elif args.command == "inspect": from .inspect import main elif args.command == "package": from .package import main # Pass on to the correct parser if args.command is not None: main(args=args, parser=parser, subparser=subparsers[args.command]) else: parser.print_help()
def check_install(software=None, quiet=True): '''check_install will attempt to run the singularity command, and return True if installed. The command line utils will not run without this check. ''' if software is None: software = "singularity" cmd = [software, '--version'] try: version = run_command(cmd,software) except: # FileNotFoundError return False if version is not None: if quiet is False and version['return_code'] == 0: version = version['message'] bot.info("Found %s version %s" % (software.upper(), version)) return True return False
def check_install(software=None, quiet=True): '''check_install will attempt to run the singularity command, and return True if installed. The command line utils will not run without this check. ''' if software is None: software = "singularity" cmd = [software, '--version'] try: version = run_command(cmd, software) except: # FileNotFoundError return False if version is not None: if quiet is False and version['return_code'] == 0: version = version['message'] bot.info("Found %s version %s" % (software.upper(), version)) return True return False
def label_search(self, key=None, value=None): '''search across labels''' if key is not None: key = key.lower() if value is not None: value = value.lower() show_details = True if key is None and value is None: url = '%s/labels/search' % (self.base) show_details = False elif key is not None and value is not None: url = '%s/labels/search/%s/key/%s/value' % (self.base, key, value) elif key is None: url = '%s/labels/search/%s/value' % (self.base, value) else: url = '%s/labels/search/%s/key' % (self.base, key) result = self.get(url) if len(result) == 0: bot.info("No labels found.") sys.exit(0) bot.info("Labels\n") rows = [] for l in result: if show_details is True: entry = [ "%s:%s" % (l['key'], l['value']), "\n%s\n\n" % "\n".join(l['containers']) ] else: entry = [ "N=%s" % len(l['containers']), "%s:%s" % (l['key'], l['value']) ] rows.append(entry) bot.table(rows)
def collection_search(self, query): '''collection search will list all containers for a specific collection. We assume query is the name of a collection''' query = query.lower().strip('/') url = '%s/collection/%s' % (self.base, query) result = self.get(url) if len(result) == 0: bot.info("No collections found.") sys.exit(1) bot.custom(prefix="COLLECTION", message=query) rows = [] for container in result['containers']: rows.append([container['uri'], container['detail']]) bot.table(rows)
def get_tags(container=None, image_package=None, search_folders=None, diff=None, return_unique=True): '''get tags will return a list of tags that describe the software in an image, meaning inside of a paricular folder. If search_folder is not defined, uses lib :param container: if provided, will use container as image. Can also provide :param image_package: if provided, can be used instead of container :param search_folders: specify one or more folders to look for tags :param diff: the difference between a container and it's parent OS from get_diff if None, will be derived. :param return_unique: return unique files in folders. Default True. Default is 'bin' ::notes The algorithm works as follows: 1) first compare package to set of base OS (provided with shub) 2) subtract the most similar os from image, leaving "custom" files 3) organize custom files into dict based on folder name 4) return search_folders as tags ''' if diff == None: diff = get_diff(container=container, image_package=image_package) if search_folders == None: search_folders = 'bin' if not isinstance(search_folders, list): search_folders = [search_folders] tags = [] for search_folder in search_folders: if search_folder in diff: bot.info("Adding tags for folder %s" % search_folder) tags = tags + diff[search_folder] else: bot.info("Did not find folder %s in difference." % search_folder) if return_unique == True: tags = list(set(tags)) return tags
def get_build_params(metadata): '''get_build_params uses get_build_metadata to retrieve corresponding meta data values for a build :param metadata: a list, each item a dictionary of metadata, in format: metadata = [{'key': 'repo_url', 'value': repo_url }, {'key': 'repo_id', 'value': repo_id }, {'key': 'credential', 'value': credential }, {'key': 'response_url', 'value': response_url }, {'key': 'token', 'value': token}, {'key': 'commit', 'value': commit }] ''' params = dict() for item in metadata: if item['value'] == None: response = get_build_metadata(key=item['key']) item['value'] = response params[item['key']] = item['value'] if item['key'] not in ['token', 'secret', 'credential']: bot.info('%s is set to %s' % (item['key'], item['value'])) return params
def search(self): '''a "show all" search that doesn't require a query''' url = '%s/collections/' % self.base results = self.paginate_get(url) if len(results) == 0: bot.info("No container collections found.") sys.exit(1) bot.info("Collections") rows = [] for result in results: if "containers" in result: for c in result['containers']: rows.append([c['uri'], c['detail']]) bot.table(rows)
def get_build_params(metadata): '''get_build_params uses get_build_metadata to retrieve corresponding meta data values for a build :param metadata: a list, each item a dictionary of metadata, in format: metadata = [{'key': 'repo_url', 'value': repo_url }, {'key': 'repo_id', 'value': repo_id }, {'key': 'credential', 'value': credential }, {'key': 'response_url', 'value': response_url }, {'key': 'token', 'value': token}, {'key': 'commit', 'value': commit }] ''' params = dict() for item in metadata: if item['value'] == None: response = get_build_metadata(key=item['key']) item['value'] = response params[item['key']] = item['value'] if item['key'] not in ['token', 'secret', 'credential']: bot.info('%s is set to %s' %(item['key'],item['value'])) return params
def get_image_name(manifest, extension='simg', use_commit=False, use_hash=False): '''get_image_name will return the image name for a manifest. The user can name based on a hash or commit, or a name with the collection, namespace, branch, and tag. ''' if use_hash: image_name = "%s.%s" % (manifest['version'], extension) elif use_commit: image_name = "%s.%s" % (manifest['commit'], extension) else: image_name = "%s-%s-%s.%s" % (manifest['name'].replace( '/', '-'), manifest['branch'].replace( '/', '-'), manifest['tag'].replace('/', ''), extension) bot.info("Singularity Hub Image: %s" % image_name) return image_name
def get_packages(family=None): '''get packages will return a list of packages (under some family) provided by singularity python.If no name is specified, the default (os) will be used. :param name: the name of the package family to load ''' package_base = get_package_base() package_folders = glob("%s/*" % (package_base)) package_families = [os.path.basename(x) for x in package_folders] if family == None: family = "docker-os" family = family.lower() if family in package_families: package_folder = "%s/%s" % (package_base, family) packages = glob("%s/*.zip" % (package_folder)) bot.info("Found %s packages in family %s" % (len(packages), family)) return packages bot.warning("Family %s not included. Options are %s" % (family, ", ".join(package_families))) return None
def get_singularity_version(singularity_version=None): '''get_singularity_version will determine the singularity version for a build first, an environmental variable is looked at, followed by using the system version. ''' if singularity_version is None: singularity_version = os.environ.get("SINGULARITY_VERSION") if singularity_version is None: try: cmd = ['singularity','--version'] output = run_command(cmd) if isinstance(output['message'],bytes): output['message'] = output['message'].decode('utf-8') singularity_version = output['message'].strip('\n') bot.info("Singularity %s being used." % singularity_version) except: singularity_version = None bot.warning("Singularity version not found, so it's likely not installed.") return singularity_version
def parse_container_name(image): '''parse_container_name will return a json structure with a repo name, tag, user. ''' container_name = image if not is_number(image): image = image.replace(' ', '') # If the user provided a number (unique id for an image), return it if is_number(image) == True: bot.info("Numeric image ID %s found.", image) return int(image) image = image.split('/') # If there are two parts, we have username with repo (and maybe tag) if len(image) >= 2: user = image[0] image = image[1] # Otherwise, we trigger error (not supported just usernames yet) else: bot.error( 'You must specify a repo name and username, %s is not valid' % container_name) sys.exit(1) # Now split the name by : in case there is a tag image = image.split(':') if len(image) == 2: repo_name = image[0] repo_tag = image[1] # Otherwise, assume latest of an image else: repo_name = image[0] repo_tag = "latest" bot.info("User: %s" % user) bot.info("Repo Name: %s" % repo_name) bot.info("Repo Tag: %s" % repo_tag) parsed = {'repo_name': repo_name, 'repo_tag': repo_tag, 'user': user} return parsed
def run_build(logfile='/tmp/.shub-log'): '''run_build will generate the Singularity build from a spec_file from a repo_url. If no arguments are required, the metadata api is queried for the values. :param build_dir: directory to do the build in. If not specified, will use temporary. :param spec_file: the spec_file name to use, assumed to be in git repo :param repo_url: the url to download the repo from :param repo_id: the repo_id to uniquely identify the repo (in case name changes) :param commit: the commit to checkout. If none provided, will use most recent. :param bucket_name: the name of the bucket to send files to :param verbose: print out extra details as we go (default True) :param token: a token to send back to the server to authenticate the collection :param secret: a secret to match to the correct container :param response_url: the build url to send the response back to. Should also come from metadata. If not specified, no response is sent :param branch: the branch to checkout for the build. :: note: this function is currently configured to work with Google Compute Engine metadata api, and should (will) be customized if needed to work elsewhere ''' # If we are building the image, this will not be set go = get_build_metadata(key='dobuild') if go == None: sys.exit(0) # If the user wants debug, this will be set debug = True enable_debug = get_build_metadata(key='debug') if enable_debug == None: debug = False bot.info('DEBUG %s' %debug) # Uaw /tmp for build directory build_dir = tempfile.mkdtemp() # Get variables from the instance metadata API metadata = [{'key': 'repo_url', 'value': None }, {'key': 'repo_id', 'value': None }, {'key': 'response_url', 'value': None }, {'key': 'bucket_name', 'value': None }, {'key': 'tag', 'value': None }, {'key': 'container_id', 'value': None }, {'key': 'commit', 'value': None }, {'key': 'token', 'value': None}, {'key': 'branch', 'value': None }, {'key': 'spec_file', 'value': None}, {'key': 'logging_url', 'value': None }, {'key': 'logfile', 'value': logfile }] # Obtain values from build bot.log("BUILD PARAMETERS:") params = get_build_params(metadata) params['debug'] = debug # Default spec file is Singularity if params['spec_file'] == None: params['spec_file'] = "Singularity" if params['bucket_name'] == None: params['bucket_name'] = "singularityhub" if params['tag'] == None: params['tag'] = "latest" output = run_build_main(build_dir=build_dir, params=params) # Output includes: finished_image = output['image'] metadata = output['metadata'] params = output['params'] # Upload image package files to Google Storage if os.path.exists(finished_image): bot.info("%s successfully built" %finished_image) dest_dir = tempfile.mkdtemp(prefix='build') # The path to the images on google drive will be the github url/commit folder trailing_path = "%s/%s" %(params['commit'], params['version']) image_path = get_image_path(params['repo_url'], trailing_path) # commits are no longer unique # storage is by commit build_files = [finished_image] bot.info("Sending image to storage:") bot.info('\n'.join(build_files)) # Start the storage service, retrieve the bucket storage_service = get_google_service() # default is "storage" "v1" bucket = get_bucket(storage_service,params["bucket_name"]) # For each file, upload to storage files = [] for build_file in build_files: bot.info("Uploading %s to storage..." %build_file) storage_file = upload_file(storage_service, bucket=bucket, bucket_path=image_path, file_name=build_file) files.append(storage_file) # Finally, package everything to send back to shub response = {"files": json.dumps(files), "repo_url": params['repo_url'], "commit": params['commit'], "repo_id": params['repo_id'], "branch": params['branch'], "tag": params['tag'], "container_id": params['container_id'], "spec_file":params['spec_file'], "token": params['token'], "metadata": json.dumps(metadata)} # Did the user specify a specific log file? custom_logfile = get_build_metadata('logfile') if custom_logfile is not None: logfile = custom_logfile response['logfile'] = logfile # Send final build data to instance send_build_data(build_dir=build_dir, response_url=params['response_url'], secret=params['token'], data=response) # Dump final params, for logger to retrieve passing_params = "/tmp/params.pkl" pickle.dump(params,open(passing_params,'wb'))
def run_build(logfile='/tmp/.shub-log'): '''run_build will generate the Singularity build from a spec_file from a repo_url. If no arguments are required, the metadata api is queried for the values. :param build_dir: directory to do the build in. If not specified, will use temporary. :param spec_file: the spec_file name to use, assumed to be in git repo :param repo_url: the url to download the repo from :param repo_id: the repo_id to uniquely identify the repo (in case name changes) :param commit: the commit to checkout. If none provided, will use most recent. :param bucket_name: the name of the bucket to send files to :param verbose: print out extra details as we go (default True) :param token: a token to send back to the server to authenticate the collection :param secret: a secret to match to the correct container :param response_url: the build url to send the response back to. Should also come from metadata. If not specified, no response is sent :param branch: the branch to checkout for the build. :: note: this function is currently configured to work with Google Compute Engine metadata api, and should (will) be customized if needed to work elsewhere ''' # If we are building the image, this will not be set go = get_build_metadata(key='dobuild') if go == None: sys.exit(0) # If the user wants debug, this will be set debug = True enable_debug = get_build_metadata(key='debug') if enable_debug == None: debug = False bot.info('DEBUG %s' % debug) # Uaw /tmp for build directory build_dir = tempfile.mkdtemp() # Get variables from the instance metadata API metadata = [{ 'key': 'repo_url', 'value': None }, { 'key': 'repo_id', 'value': None }, { 'key': 'response_url', 'value': None }, { 'key': 'bucket_name', 'value': None }, { 'key': 'tag', 'value': None }, { 'key': 'container_id', 'value': None }, { 'key': 'commit', 'value': None }, { 'key': 'token', 'value': None }, { 'key': 'branch', 'value': None }, { 'key': 'spec_file', 'value': None }, { 'key': 'logging_url', 'value': None }, { 'key': 'logfile', 'value': logfile }] # Obtain values from build bot.log("BUILD PARAMETERS:") params = get_build_params(metadata) params['debug'] = debug # Default spec file is Singularity if params['spec_file'] == None: params['spec_file'] = "Singularity" if params['bucket_name'] == None: params['bucket_name'] = "singularityhub" if params['tag'] == None: params['tag'] = "latest" output = run_build_main(build_dir=build_dir, params=params) # Output includes: image_package = output['image_package'] finished_image = output['image'] metadata = output['metadata'] params = output['params'] # Upload image package files to Google Storage if os.path.exists(image_package): bot.info("Package %s successfully built" % image_package) dest_dir = tempfile.mkdtemp(prefix='build') with zipfile.ZipFile(image_package) as zf: zf.extractall(dest_dir) # The path to the images on google drive will be the github url/commit folder trailing_path = "%s/%s" % (params['commit'], params['version']) image_path = get_image_path(params['repo_url'], trailing_path) # commits are no longer unique # storage is by commit build_files = glob("%s/*" % (dest_dir)) build_files.append(finished_image) bot.info("Sending build files to storage:") bot.info('\n'.join(build_files)) # Start the storage service, retrieve the bucket storage_service = get_google_service() # default is "storage" "v1" bucket = get_bucket(storage_service, params["bucket_name"]) # For each file, upload to storage files = [] for build_file in build_files: bot.info("Uploading %s to storage..." % build_file) storage_file = upload_file(storage_service, bucket=bucket, bucket_path=image_path, file_name=build_file) files.append(storage_file) # Finally, package everything to send back to shub response = { "files": json.dumps(files), "repo_url": params['repo_url'], "commit": params['commit'], "repo_id": params['repo_id'], "branch": params['branch'], "tag": params['tag'], "container_id": params['container_id'], "spec_file": params['spec_file'], "token": params['token'], "metadata": json.dumps(metadata) } # Did the user specify a specific log file? custom_logfile = get_build_metadata('logfile') if custom_logfile is not None: logfile = custom_logfile response['logfile'] = logfile # Send final build data to instance send_build_data(build_dir=build_dir, response_url=params['response_url'], secret=params['token'], data=response) # Dump final params, for logger to retrieve passing_params = "/tmp/params.pkl" pickle.dump(params, open(passing_params, 'wb'))
def run_build(build_dir, params, verbose=True): '''run_build takes a build directory and params dictionary, and does the following: - downloads repo to a temporary directory - changes branch or commit, if needed - creates and bootstraps singularity image from Singularity file - returns a dictionary with: image (path), metadata (dict) The following must be included in params: spec_file, repo_url, branch, commit ''' # Download the repository download_repo(repo_url=params['repo_url'], destination=build_dir) os.chdir(build_dir) if params['branch'] != None: bot.info('Checking out branch %s' %params['branch']) os.system('git checkout %s' %(params['branch'])) else: params['branch'] = "master" # Set the debug level Client.debug = params['debug'] # Commit if params['commit'] not in [None,'']: bot.info('Checking out commit %s' %params['commit']) os.system('git checkout %s .' %(params['commit'])) # From here on out commit is used as a unique id, if we don't have one, we use current else: params['commit'] = os.popen('git log -n 1 --pretty=format:"%H"').read() bot.warning("commit not specified, setting to current %s" %params['commit']) # Dump some params for the builder, in case it fails after this passing_params = "/tmp/params.pkl" pickle.dump(params, open(passing_params,'wb')) # Now look for spec file if os.path.exists(params['spec_file']): bot.info("Found spec file %s in repository" %params['spec_file']) # If the user has a symbolic link if os.path.islink(params['spec_file']): bot.info("%s is a symbolic link." %params['spec_file']) params['spec_file'] = os.path.realpath(params['spec_file']) # START TIMING start_time = datetime.now() # Secure Build image = Client.build(recipe=params['spec_file'], build_folder=build_dir, isolated=True) # Save has for metadata (also is image name) version = get_image_file_hash(image) params['version'] = version pickle.dump(params, open(passing_params,'wb')) # Rename image to be hash finished_image = "%s/%s.simg" %(os.path.dirname(image), version) image = shutil.move(image, finished_image) final_time = (datetime.now() - start_time).seconds bot.info("Final time of build %s seconds." %final_time) # Did the container build successfully? test_result = test_container(image) if test_result['return_code'] != 0: bot.error("Image failed to build, cancelling.") sys.exit(1) # Get singularity version singularity_version = Client.version() Client.debug = False inspect = Client.inspect(image) # this is a string Client.debug = params['debug'] # Get information on apps Client.debug = False app_names = Client.apps(image) Client.debug = params['debug'] apps = extract_apps(image, app_names) metrics = {'build_time_seconds': final_time, 'singularity_version': singularity_version, 'singularity_python_version': singularity_python_version, 'inspect': inspect, 'version': version, 'apps': apps} output = {'image':image, 'metadata':metrics, 'params':params } return output else: # Tell the user what is actually there present_files = glob("*") bot.error("Build file %s not found in repository" %params['spec_file']) bot.info("Found files are %s" %"\n".join(present_files)) # Params have been exported, will be found by log sys.exit(1)
def container_search(self, query, across_collections=False, environment=False, deffile=False, runscript=False, test=False): '''search for a specific container. If across collections is False, the query is parsed as a full container name and a specific container is returned. If across_collections is True, the container is searched for across collections. If across collections is True, details are not shown''' query = query.lower().strip('/') q = parse_image_name(query, defaults=False) if q['tag'] is not None: if across_collections is True: url = '%s/container/search/name/%s/tag/%s' % (self.base, q['image'], q['tag']) else: url = '%s/container/search/collection/%s/name/%s/tag/%s' % ( self.base, q['collection'], q['image'], q['tag']) elif q['tag'] is None: if across_collections is True: url = '%s/container/search/name/%s' % (self.base, q['image']) else: url = '%s/container/search/collection/%s/name/%s' % ( self.base, q['collection'], q['image']) result = self.get(url) if "containers" in result: result = result['containers'] if len(result) == 0: bot.info("No containers found.") sys.exit(1) bot.info("Containers %s" % query) rows = [] for c in result: # Convert date to readable thing datetime_object = parser.parse(c['add_date']) print_date = datetime_object.strftime('%b %d, %Y %I:%M%p') rows.append( ['%s/%s' % (c['collection'], c['name']), c['tag'], print_date]) bot.table(rows) # Finally, show metadata and other values if test is True or deffile is True or environment is True or runscript is True: bot.newline() for c in result: metadata = c['metadata'] if test is True: bot.custom(prefix='%test', message=metadata['test'], color="CYAN") bot.newline() if deffile is True: bot.custom(prefix='Singularity', message=metadata['deffile'], color="CYAN") bot.newline() if environment is True: bot.custom(prefix='%environment', message=metadata['environment'], color="CYAN") bot.newline() if runscript is True: bot.custom(prefix='%runscript', message=metadata['runscript'], color="CYAN") bot.newline()
def run_build(build_dir,params,verbose=True, compress_image=False): '''run_build takes a build directory and params dictionary, and does the following: - downloads repo to a temporary directory - changes branch or commit, if needed - creates and bootstraps singularity image from Singularity file - returns a dictionary with: image (path), image_package (path), metadata (dict) The following must be included in params: spec_file, repo_url, branch, commit Optional parameters size ''' # Download the repo and image download_repo(repo_url=params['repo_url'], destination=build_dir) os.chdir(build_dir) if params['branch'] != None: bot.info('Checking out branch %s' %params['branch']) os.system('git checkout %s' %(params['branch'])) else: params['branch'] = "master" # Commit if params['commit'] not in [None,'']: bot.info('Checking out commit %s' %params['commit']) os.system('git checkout %s .' %(params['commit'])) # From here on out commit is used as a unique id, if we don't have one, we use current else: params['commit'] = os.popen('git log -n 1 --pretty=format:"%H"').read() bot.warning("commit not specified, setting to current %s" %params['commit']) # Dump some params for the builder, in case it fails after this passing_params = "/tmp/params.pkl" pickle.dump(params,open(passing_params,'wb')) # Now look for spec file if os.path.exists(params['spec_file']): bot.info("Found spec file %s in repository" %params['spec_file']) # If the user has a symbolic link if os.path.islink(params['spec_file']): bot.info("%s is a symbolic link." %params['spec_file']) params['spec_file'] = os.path.realpath(params['spec_file']) # START TIMING start_time = datetime.now() image = build_from_spec(spec_file=params['spec_file'], # default will package the image build_dir=build_dir, isolated=True, sandbox=False, debug=params['debug']) # Save has for metadata (also is image name) version = get_image_file_hash(image) params['version'] = version pickle.dump(params,open(passing_params,'wb')) final_time = (datetime.now() - start_time).seconds bot.info("Final time of build %s seconds." %final_time) # Did the container build successfully? test_result = test_container(image) if test_result['return_code'] != 0: bot.error("Image failed to build, cancelling.") sys.exit(1) # Get singularity version singularity_version = get_singularity_version() # Package the image metadata (files, folders, etc) image_package = package(image_path=image, spec_path=params['spec_file'], output_folder=build_dir, remove_image=True, verbose=True) # Derive software tags by subtracting similar OS diff = get_diff(image_package=image_package) # Inspect to get labels and other metadata cli = Singularity(debug=params['debug']) inspect = cli.inspect(image_path=image) # Get information on apps app_names = cli.apps(image_path=image) apps = extract_apps(image_path=image, app_names=app_names) # Count file types, and extensions counts = dict() counts['readme'] = file_counts(diff=diff) counts['copyright'] = file_counts(diff=diff,patterns=['copyright']) counts['authors-thanks-credit'] = file_counts(diff=diff, patterns=['authors','thanks','credit','contributors']) counts['todo'] = file_counts(diff=diff,patterns=['todo']) extensions = extension_counts(diff=diff) os_sims = estimate_os(image_package=image_package,return_top=False) most_similar = os_sims['SCORE'].values.argmax() most_similar = os_sims['SCORE'].index.tolist()[most_similar] metrics = {'build_time_seconds':final_time, 'singularity_version':singularity_version, 'singularity_python_version':singularity_python_version, 'estimated_os': most_similar, 'os_sims':os_sims['SCORE'].to_dict(), 'file_counts':counts, 'file_ext':extensions, 'inspect':inspect, 'version': version, 'apps': apps} # Compress Image if compress_image is True: compressed_image = "%s.gz" %image os.system('gzip -c -9 %s > %s' %(image,compressed_image)) image = compressed_image output = {'image':image, 'image_package':image_package, 'metadata':metrics, 'params':params } return output else: # Tell the user what is actually there present_files = glob("*") bot.error("Build file %s not found in repository" %params['spec_file']) bot.info("Found files are %s" %"\n".join(present_files)) # Params have been exported, will be found by log sys.exit(1)