def main(args, parser, subparser): # We can only continue if singularity is installed if check_install() is not True: bot.error("Cannot find Singularity! Is it installed?") sys.exit(1) # Output folder will be pwd if not specified output_folder = os.getcwd() if args.outfolder is not None: output_folder = os.getcwd() # If we are given an image, ensure full path if args.image is not None: if not os.path.exists(args.image): cli = Singularity(debug=args.debug) image = cli.pull(args.image) if image is None: bot.error("Cannot find image. Exiting.") sys.exit(1) # the user wants to make a tree if args.tree is True: from singularity.cli.app import make_tree make_tree(image) clean_up(image, existed) else: print("Please specify an image to inspect with --image") subparser.print_help()
def get_container_contents(container=None, gets=None, split_delim=None, image_package=None): '''get_container_contents will return a list of folders and or files for a container. The environmental variable SINGULARITY_HUB being set means that container objects are referenced instead of packages :param container: the container to get content for :param gets: a list of file names to return, without parent folders :param split_delim: if defined, will split text by split delimiter :param image_package: if defined, user has provided an image_package ''' from singularity.package import package, load_package if container == None and image_package == None: bot.error("You must define an image package or container.") sys.exit(1) # Default returns are the list of files and folders if gets == None: gets = ['files.txt', 'folders.txt'] if not isinstance(gets, list): gets = [gets] # We will look for everything in guts, then return it guts = dict() SINGULARITY_HUB = os.environ.get('SINGULARITY_HUB', "False") # Visualization deployed local or elsewhere if SINGULARITY_HUB == "False": tmpdir = tempfile.mkdtemp() if image_package == None: image_package = package(image_path=container, output_folder=tmpdir, remove_image=True) guts = load_package(image_package, get=gets) shutil.rmtree(tmpdir) # Visualization deployed by singularity hub else: # user has provided a package, but not a container if container == None: guts = load_package(image_package, get=gets) # user has provided a container, but not a package else: for sfile in container.files: for gut_key in gets: if os.path.basename(sfile['name']) == gut_key: if split_delim == None: guts[gut_key] = requests.get( sfile['mediaLink']).text else: guts[gut_key] = requests.get( sfile['mediaLink']).text.split(split_delim) return guts
def run_command(self, cmd, sudo=False, quiet=False): '''run_command is a wrapper for the global run_command, checking first for sudo and exiting on error if needed :param cmd: the command to run :param sudo: does the command require sudo? On success, returns result. Otherwise, exists on error ''' result = run_command(cmd,sudo=sudo) message = result['message'] return_code = result['return_code'] if result['return_code'] == 0: if isinstance(message,bytes): try: message=message.decode('utf-8') except UnicodeDecodeError: try: message = unicode(message, errors='replace') except NameError: message = str(message, errors='replace') return message if quiet is False: bot.error("Return Code %s: %s" %(return_code, message)) sys.exit(1)
def main(args, parser, subparser): # We can only continue if singularity is installed if check_install() is not True: bot.error("Cannot find Singularity! Is it installed?") sys.exit(1) # Output folder will be pwd if not specified output_folder = os.getcwd() if args.outfolder is not None: output_folder = os.getcwd() # If we are given an image, ensure full path if args.image is not None: image = args.image if not os.path.exists(image): cli = Singularity(debug=args.debug) image = cli.pull(image) if image is None: bot.error("Cannot find image. Exiting.") sys.exit(1) remove_image = not args.include_image package(image_path=image, output_folder=output_folder, remove_image=remove_image) else: print("Please specify an image to package with --image") subparser.print_help()
def get_levels(version=None): '''get_levels returns a dictionary of levels (key) and values (dictionaries with descriptions and regular expressions for files) for the user. :param version: the version of singularity to use (default is 2.2) :param include_files: files to add to the level, only relvant if ''' valid_versions = ['2.3', '2.2'] if version is None: version = "2.3" version = str(version) if version not in valid_versions: bot.error("Unsupported version %s, valid versions are %s" % (version, ",".join(valid_versions))) levels_file = os.path.abspath( os.path.join(get_installdir(), 'analysis', 'reproduce', 'data', 'reproduce_levels.json')) levels = read_json(levels_file) if version == "2.2": # Labels not added until 2.3 del levels['LABELS'] levels = make_levels_set(levels) return levels
def get_levels(version=None): '''get_levels returns a dictionary of levels (key) and values (dictionaries with descriptions and regular expressions for files) for the user. :param version: the version of singularity to use (default is 2.2) :param include_files: files to add to the level, only relvant if ''' valid_versions = ['2.3','2.2'] if version is None: version = "2.3" version = str(version) if version not in valid_versions: bot.error("Unsupported version %s, valid versions are %s" %(version, ",".join(valid_versions))) levels_file = os.path.abspath(os.path.join(get_installdir(), 'analysis', 'reproduce', 'data', 'reproduce_levels.json')) levels = read_json(levels_file) if version == "2.2": # Labels not added until 2.3 del levels['LABELS'] levels = make_levels_set(levels) return levels
def unpack_node(image_path,name=None,output_folder=None,size=None): '''unpackage node is intended to unpackage a node that was packaged with package_node. The image should be a .tgz file. The general steps are to: 1. Package the node using the package_node function 2. Transfer the package somewhere that Singularity is installed''' if not image_path.endswith(".tgz"): bot.error("The image_path should end with .tgz. Did you create with package_node?") sys.exit(1) if output_folder is None: output_folder = os.path.dirname(os.path.abspath(image_path)) image_name = os.path.basename(image_path) if name is None: name = image_name.replace('.tgz','.img') if not name.endswith('.img'): name = "%s.img" %(name) bot.debug("Preparing to unpack %s to %s." %(image_name,name)) unpacked_image = "%s/%s" %(output_folder,name) if not os.path.exists(unpacked_image): os.mkdir(unpacked_image) cmd = ["gunzip","-dc",image_path,"|","sudo","singularity","import", unpacked_image] output = run_command(cmd) # TODO: singularity mount the container, cleanup files (/etc/fstab,...) # and add your custom singularity files. return unpacked_image
def pull(self,image_path,pull_folder='', name_by_hash=False, name_by_commit=False, image_name=None, size=None): '''pull will pull a singularity hub image :param image_path: full path to image / uris :param name_by: can be one of commit or hash, default is by image name ''' if image_name is not None: name_by_hash=False name_by_commit=False final_image = None if not image_path.startswith('shub://') and not image_path.startswith('docker://'): bot.error("pull is only valid for docker and shub, %s is invalid." %image_name) sys.exit(1) if self.debug is True: cmd = ['singularity','--debug','pull'] else: cmd = ['singularity','pull'] if pull_folder not in [None,'']: os.environ['SINGULARITY_PULLFOLDER'] = pull_folder pull_folder = "%s/" % pull_folder if image_path.startswith('shub://'): if image_name is not None: bot.debug("user specified naming pulled image %s" %image_name) cmd = cmd +["--name",image_name] elif name_by_commit is True: bot.debug("user specified naming by commit.") cmd.append("--commit") elif name_by_hash is True: bot.debug("user specified naming by hash.") cmd.append("--hash") # otherwise let the Singularity client determine own name elif image_path.startswith('docker://'): if size is not None: cmd = cmd + ["--size",size] if image_name is None: image_name = "%s" %image_path.replace("docker://","").replace("/","-") final_image = "%s%s.img" %(pull_folder,image_name) cmd = cmd + ["--name", image_name] cmd.append(image_path) bot.debug(' '.join(cmd)) output = self.run_command(cmd) self.println(output) if final_image is None: # shub final_image = output.split('Container is at:')[-1].strip('\n').strip() return final_image
def compress(self,image_path): '''compress will (properly) compress an image''' if os.path.exists(image_path): compressed_image = "%s.gz" %image_path os.system('gzip -c -6 %s > %s' %(image_path,compressed_image)) return compressed_image else: bot.error("Cannot find image %s" %image_path) sys.exit(1)
def decompress(self,image_path,quiet=True): '''decompress will (properly) decompress an image''' if not os.path.exists(image_path): bot.error("Cannot find image %s" %image_path) sys.exit(1) extracted_file = image_path.replace('.gz','') cmd = ['gzip','-d','-f', image_path] result = self.run_command(cmd, quiet=quiet) # exits if return code != 0 return extracted_file
def main(args, parser, subparser): # We can only continue if singularity is installed if check_install() is not True: bot.error("Cannot find Singularity! Is it installed?") sys.exit(1) # Output folder will be pwd if not specified output_folder = os.getcwd() if args.outfolder is not None: output_folder = os.getcwd() # If we are given an image, ensure full path image = args.image if image is not None: existed = True if not os.path.exists(image): cli = Singularity(debug=args.debug) image = cli.pull(image) existed = False if image is None: bot.error("Cannot find image. Exiting.") sys.exit(1) # The user wants to estimate the os if args.os is True: from singularity.analysis.classify import estimate_os estimated_os = estimate_os(container=image) print(estimated_os) # The user wants to get a list of all os elif args.oscalc is True: from singularity.analysis.classify import estimate_os estimated_os = estimate_os(container=image, return_top=False) print(estimated_os["SCORE"].to_dict()) # The user wants to get a list of tags elif args.tags is True: from singularity.analysis.classify import get_tags tags = get_tags(container=image) print(tags) # The user wants to plot image vs. the docker os elif args.osplot is True: from singularity.app import plot_os_sims plot_os_sims(image) clean_up(image, existed) else: print("Please specify an image to analyze with --image") subparser.print_help()
def mkdir_p(path): '''mkdir_p attempts to get the same functionality as mkdir -p :param path: the path to create. ''' try: os.makedirs(path) except OSError as e: if e.errno == errno.EEXIST and os.path.isdir(path): pass else: bot.error("Error creating path %s, exiting." % path) sys.exit(1)
def get_script(script_name): '''get_script will return a build script_name, if it is included in singularity/build/scripts, otherwise will alert the user and return None :param script_name: the name of the script to look for ''' install_dir = get_installdir() script_path = "%s/build/scripts/%s" %(install_dir,script_name) if os.path.exists(script_path): return script_path else: bot.error("Script %s is not included in singularity-python!" %script_path) return None
def mkdir_p(path): '''mkdir_p attempts to get the same functionality as mkdir -p :param path: the path to create. ''' try: os.makedirs(path) except OSError as e: if e.errno == errno.EEXIST and os.path.isdir(path): pass else: bot.error("Error creating path %s, exiting." % path) sys.exit(1)
def get_image_tar(image_path): '''get an image tar, either written in memory or to the file system. file_obj will either be the file object, or the file itself. ''' bot.debug('Generate file system tar...') file_obj = Client.image.export(image_path=image_path) if file_obj is None: bot.error("Error generating tar, exiting.") sys.exit(1) tar = tarfile.open(file_obj) return file_obj, tar
def get_script(script_name): '''get_script will return a build script_name, if it is included in singularity/build/scripts, otherwise will alert the user and return None :param script_name: the name of the script to look for ''' install_dir = get_installdir() script_path = "%s/build/scripts/%s" % (install_dir, script_name) if os.path.exists(script_path): return script_path else: bot.error("Script %s is not included in singularity-python!" % script_path) return None
def main(args, parser, subparser): # We can only continue if singularity is installed if check_install() is not True: bot.error("Cannot find Singularity! Is it installed?") sys.exit(1) # Output folder will be pwd if not specified output_folder = os.getcwd() if args.outfolder is not None: output_folder = os.getcwd() if args.images is not None: image1, image2 = args.images.split(',') bot.debug("Image1: %s" % image1) bot.debug("Image2: %s" % image2) images = dict() cli = Singularity(debug=args.debug) for image in [image1, image2]: existed = True if not os.path.exists(image): image = cli.pull(image) existed = False images[image] = existed # Just for clarity image1, image2 = list(images.keys()) # the user wants to make a similarity tree if args.simtree is True: from singularity.cli.app import make_sim_tree make_sim_tree(image1, image2) # the user wants to make a difference tree elif args.subtract is True: from singularity.cli.app import make_diff_tree make_diff_tree(image1, image2) else: # If none specified, just print score from singularity.analysis.compare import calculate_similarity score = calculate_similarity(image1, image2, by="files.txt") print(score["files.txt"]) for image, existed in images.items(): clean_up(image, existed) else: print("Please specify images to compare with --images") subparser.print_help()
def get_image_tar(image_path, S=None): '''get an image tar, either written in memory or to the file system. file_obj will either be the file object, or the file itself. ''' bot.debug('Generate file system tar...') if S is None: S = Singularity() file_obj = S.export(image_path=image_path) if file_obj is None: bot.error("Error generating tar, exiting.") sys.exit(1) tar = tarfile.open(file_obj) return file_obj, tar
def main(args, parser, subparser): # Does the user have a valid image? image = args.image[0] if not os.path.exists(image): subparser.print_help() bot.error("Please supply one or more paths to existing images.") sys.exit(1) # Authenticate sreg = Client(secrets=args.secrets) response = sreg.push(path=image, name=args.name, tag=args.tag, compress=args.compress)
def download(self, url, file_name, headers=None, show_progress=True): '''stream to a temporary file, rename on successful completion :param file_name: the file name to stream to :param url: the url to stream from :param headers: additional headers to add ''' fd, tmp_file = tempfile.mkstemp(prefix=("%s.tmp." % file_name)) os.close(fd) response = self.stream(url, headers=headers, stream_to=tmp_file) if isinstance(response, HTTPError): bot.error("Error downloading %s, exiting." % url) sys.exit(1) shutil.move(tmp_file, file_name) return file_name
def parse_container_name(image): '''parse_container_name will return a json structure with a repo name, tag, user. ''' container_name = image if not is_number(image): image = image.replace(' ', '') # If the user provided a number (unique id for an image), return it if is_number(image) == True: bot.info("Numeric image ID %s found.", image) return int(image) image = image.split('/') # If there are two parts, we have username with repo (and maybe tag) if len(image) >= 2: user = image[0] image = image[1] # Otherwise, we trigger error (not supported just usernames yet) else: bot.error( 'You must specify a repo name and username, %s is not valid' % container_name) sys.exit(1) # Now split the name by : in case there is a tag image = image.split(':') if len(image) == 2: repo_name = image[0] repo_tag = image[1] # Otherwise, assume latest of an image else: repo_name = image[0] repo_tag = "latest" bot.info("User: %s" % user) bot.info("Repo Name: %s" % repo_name) bot.info("Repo Tag: %s" % repo_tag) parsed = {'repo_name': repo_name, 'repo_tag': repo_tag, 'user': user} return parsed
def create(self,image_path,size=None,sudo=False): '''create will create a a new image :param image_path: full path to image :param size: image sizein MiB, default is 1024MiB :param filesystem: supported file systems ext3/ext4 (ext[2/3]: default ext3 ''' if size == None: size=1024 if self.debug == True: cmd = ['singularity','--debug','image.create','--size',str(size),image_path] else: cmd = ['singularity','image.create','--size',str(size),image_path] output = self.run_command(cmd,sudo=sudo) self.println(output) if not os.path.exists(image_path): bot.error("Could not create image %s" %image_path) sys.exit(1) return image_path
def getenv(variable_key, required=False, default=None, silent=False): '''getenv will attempt to get an environment variable. If the variable is not found, None is returned. :param variable_key: the variable name :param required: exit with error if not found :param silent: Do not print debugging information for variable ''' variable = os.environ.get(variable_key, default) if variable is None and required: bot.error("Cannot find environment variable %s, exiting." % variable_key) sys.exit(1) if silent: bot.verbose2("%s found" % variable_key) else: if variable is not None: bot.verbose2("%s found as %s" % (variable_key, variable)) else: bot.verbose2("%s not defined (None)" % variable_key) return variable
def read_client_secrets(secrets=None,required=True): # If token file not provided, check environment if secrets is None: secrets = os.environ.get("SREGISTRY_CLIENT_SECRETS") # Fall back to default if secrets is None: userhome = pwd.getpwuid(os.getuid())[5] secrets = "%s/.sregistry" % (userhome) if secrets is not None: if os.path.exists(secrets): return read_json(secrets) message = 'Client secrets file not found at %s or $SREGISTRY_CLIENT_SECRETS.' %secrets if required: bot.error(message) sys.exit(1) bot.warning(message) return None
def call(self, url, func, data=None, return_json=True, stream=False): '''call will issue the call, and issue a refresh token given a 401 response. :param func: the function (eg, post, get) to call :param url: the url to send file to :param data: additional data to add to the request :param return_json: return json if successful ''' if data is not None: if not isinstance(data, dict): data = json.dumps(data) response = func(url=url, headers=self.headers, data=data, stream=stream) # Errored response, try again with refresh if response.status_code in [500, 502]: bot.error("Beep boop! %s: %s" % (response.reason, response.status_code)) sys.exit(1) # Errored response, try again with refresh if response.status_code == 404: bot.error("Beep boop! %s: %s" % (response.reason, response.status_code)) sys.exit(1) # Errored response, try again with refresh if response.status_code == 401: bot.error("Your credentials are expired! %s: %s" % (response.reason, response.status_code)) sys.exit(1) elif response.status_code == 200: if return_json: try: response = response.json() except ValueError: bot.error("The server returned a malformed response.") sys.exit(1) return response
def stream(self, url, headers=None, stream_to=None): '''stream is a get that will stream to file_name ''' bot.debug("GET %s" % url) if headers == None: headers = self._init_headers() response = requests.get(url, headers=headers, stream=True) if response.status_code == 200: # Keep user updated with Progress Bar? content_size = None if 'Content-Length' in response.headers: progress = 0 content_size = int(response.headers['Content-Length']) bot.show_progress(progress, content_size, length=35) chunk_size = 1 << 20 with open(stream_to, 'wb') as filey: for chunk in response.iter_content(chunk_size=chunk_size): filey.write(chunk) if content_size is not None: progress += chunk_size bot.show_progress(iteration=progress, total=content_size, length=35, carriage_return=False) # Newline to finish download sys.stdout.write('\n') return stream_to bot.error("Problem with stream, response %s" % (response.status_code)) sys.exit(1)
def unpack_node(image_path, name=None, output_folder=None, size=None): '''unpackage node is intended to unpackage a node that was packaged with package_node. The image should be a .tgz file. The general steps are to: 1. Package the node using the package_node function 2. Transfer the package somewhere that Singularity is installed''' if not image_path.endswith(".tgz"): bot.error( "The image_path should end with .tgz. Did you create with package_node?" ) sys.exit(1) if output_folder is None: output_folder = os.path.dirname(os.path.abspath(image_path)) image_name = os.path.basename(image_path) if name is None: name = image_name.replace('.tgz', '.img') if not name.endswith('.img'): name = "%s.img" % (name) bot.debug("Preparing to unpack %s to %s." % (image_name, name)) unpacked_image = "%s/%s" % (output_folder, name) S = Singularity(sudo=True) S.create(image_path=unpacked_image, size=size) cmd = [ "gunzip", "-dc", image_path, "|", "sudo", "singularity", "import", unpacked_image ] output = run_command(cmd) # TODO: singularity mount the container, cleanup files (/etc/fstab,...) # and add your custom singularity files. return unpacked_image
def run_build(build_dir,params,verbose=True, compress_image=False): '''run_build takes a build directory and params dictionary, and does the following: - downloads repo to a temporary directory - changes branch or commit, if needed - creates and bootstraps singularity image from Singularity file - returns a dictionary with: image (path), image_package (path), metadata (dict) The following must be included in params: spec_file, repo_url, branch, commit Optional parameters size ''' # Download the repo and image download_repo(repo_url=params['repo_url'], destination=build_dir) os.chdir(build_dir) if params['branch'] != None: bot.info('Checking out branch %s' %params['branch']) os.system('git checkout %s' %(params['branch'])) else: params['branch'] = "master" # Commit if params['commit'] not in [None,'']: bot.info('Checking out commit %s' %params['commit']) os.system('git checkout %s .' %(params['commit'])) # From here on out commit is used as a unique id, if we don't have one, we use current else: params['commit'] = os.popen('git log -n 1 --pretty=format:"%H"').read() bot.warning("commit not specified, setting to current %s" %params['commit']) # Dump some params for the builder, in case it fails after this passing_params = "/tmp/params.pkl" pickle.dump(params,open(passing_params,'wb')) # Now look for spec file if os.path.exists(params['spec_file']): bot.info("Found spec file %s in repository" %params['spec_file']) # If the user has a symbolic link if os.path.islink(params['spec_file']): bot.info("%s is a symbolic link." %params['spec_file']) params['spec_file'] = os.path.realpath(params['spec_file']) # START TIMING start_time = datetime.now() image = build_from_spec(spec_file=params['spec_file'], # default will package the image build_dir=build_dir, isolated=True, sandbox=False, debug=params['debug']) # Save has for metadata (also is image name) version = get_image_file_hash(image) params['version'] = version pickle.dump(params,open(passing_params,'wb')) final_time = (datetime.now() - start_time).seconds bot.info("Final time of build %s seconds." %final_time) # Did the container build successfully? test_result = test_container(image) if test_result['return_code'] != 0: bot.error("Image failed to build, cancelling.") sys.exit(1) # Get singularity version singularity_version = get_singularity_version() # Package the image metadata (files, folders, etc) image_package = package(image_path=image, spec_path=params['spec_file'], output_folder=build_dir, remove_image=True, verbose=True) # Derive software tags by subtracting similar OS diff = get_diff(image_package=image_package) # Inspect to get labels and other metadata cli = Singularity(debug=params['debug']) inspect = cli.inspect(image_path=image) # Get information on apps app_names = cli.apps(image_path=image) apps = extract_apps(image_path=image, app_names=app_names) # Count file types, and extensions counts = dict() counts['readme'] = file_counts(diff=diff) counts['copyright'] = file_counts(diff=diff,patterns=['copyright']) counts['authors-thanks-credit'] = file_counts(diff=diff, patterns=['authors','thanks','credit','contributors']) counts['todo'] = file_counts(diff=diff,patterns=['todo']) extensions = extension_counts(diff=diff) os_sims = estimate_os(image_package=image_package,return_top=False) most_similar = os_sims['SCORE'].values.argmax() most_similar = os_sims['SCORE'].index.tolist()[most_similar] metrics = {'build_time_seconds':final_time, 'singularity_version':singularity_version, 'singularity_python_version':singularity_python_version, 'estimated_os': most_similar, 'os_sims':os_sims['SCORE'].to_dict(), 'file_counts':counts, 'file_ext':extensions, 'inspect':inspect, 'version': version, 'apps': apps} # Compress Image if compress_image is True: compressed_image = "%s.gz" %image os.system('gzip -c -9 %s > %s' %(image,compressed_image)) image = compressed_image output = {'image':image, 'image_package':image_package, 'metadata':metrics, 'params':params } return output else: # Tell the user what is actually there present_files = glob("*") bot.error("Build file %s not found in repository" %params['spec_file']) bot.info("Found files are %s" %"\n".join(present_files)) # Params have been exported, will be found by log sys.exit(1)
def push(self, path, name, tag=None, compress=False): '''push an image to Singularity Registry''' path = os.path.abspath(path) image = os.path.basename(path) bot.debug("PUSH %s" % path) if not os.path.exists(path): bot.error('%s does not exist.' %path) sys.exit(1) cli = Singularity() metadata = cli.inspect(image_path=path, quiet=True) metadata = json.loads(metadata) # Try to add the size try: image_size = os.path.getsize(path) >> 20 if metadata['data']['attributes']['labels'] is None: metadata['data']['attributes']['labels'] = {'SREGISTRY_SIZE_MB': image_size } else: metadata['data']['attributes']['labels']['SREGISTRY_SIZE_MB'] = image_size except: bot.warning("Cannot load metadata to add calculated size.") pass if "deffile" in metadata['data']['attributes']: if metadata['data']['attributes']['deffile'] is not None: fromimage = parse_header(metadata['data']['attributes']['deffile'], header="from", remove_header=True) metadata['data']['attributes']['labels']['SREGISTRY_FROM'] = fromimage bot.debug("%s was built from a definition file." % image) if compress is True: ext = 'img.gz' # ext3 format else: ext = 'simg' # ext3 format metadata = json.dumps(metadata) names = parse_image_name(name,tag=tag, ext=ext) url = '%s/push/' % self.base if compress is True: try: sys.stdout.write('Compressing image ') bot.spinner.start() upload_from = cli.compress(path) bot.spinner.stop() except KeyboardInterrupt: print('Upload cancelled') if os.path.exists("%s.gz" %path): os.remove("%s.gz" %path) sys.exit(1) else: upload_from = path upload_to = os.path.basename(names['storage']) SREGISTRY_EVENT = self.authorize(request_type="push", names=names) encoder = MultipartEncoder(fields={'collection': names['collection'], 'name':names['image'], 'metadata':metadata, 'tag': names['tag'], 'datafile': (upload_to, open(upload_from, 'rb'), 'text/plain')}) progress_callback = create_callback(encoder) monitor = MultipartEncoderMonitor(encoder, progress_callback) headers = {'Content-Type': monitor.content_type, 'Authorization': SREGISTRY_EVENT } try: r = requests.post(url, data=monitor, headers=headers) message = self.read_response(r) print('\n[Return status {0} {1}]'.format(r.status_code, message)) except KeyboardInterrupt: print('\nUpload cancelled.') # Clean up if compress is True: if os.path.exists("%s.gz" %path): os.remove("%s.gz" %path)
def run_build(build_dir, params, verbose=True): '''run_build takes a build directory and params dictionary, and does the following: - downloads repo to a temporary directory - changes branch or commit, if needed - creates and bootstraps singularity image from Singularity file - returns a dictionary with: image (path), metadata (dict) The following must be included in params: spec_file, repo_url, branch, commit ''' # Download the repository download_repo(repo_url=params['repo_url'], destination=build_dir) os.chdir(build_dir) if params['branch'] != None: bot.info('Checking out branch %s' %params['branch']) os.system('git checkout %s' %(params['branch'])) else: params['branch'] = "master" # Set the debug level Client.debug = params['debug'] # Commit if params['commit'] not in [None,'']: bot.info('Checking out commit %s' %params['commit']) os.system('git checkout %s .' %(params['commit'])) # From here on out commit is used as a unique id, if we don't have one, we use current else: params['commit'] = os.popen('git log -n 1 --pretty=format:"%H"').read() bot.warning("commit not specified, setting to current %s" %params['commit']) # Dump some params for the builder, in case it fails after this passing_params = "/tmp/params.pkl" pickle.dump(params, open(passing_params,'wb')) # Now look for spec file if os.path.exists(params['spec_file']): bot.info("Found spec file %s in repository" %params['spec_file']) # If the user has a symbolic link if os.path.islink(params['spec_file']): bot.info("%s is a symbolic link." %params['spec_file']) params['spec_file'] = os.path.realpath(params['spec_file']) # START TIMING start_time = datetime.now() # Secure Build image = Client.build(recipe=params['spec_file'], build_folder=build_dir, isolated=True) # Save has for metadata (also is image name) version = get_image_file_hash(image) params['version'] = version pickle.dump(params, open(passing_params,'wb')) # Rename image to be hash finished_image = "%s/%s.simg" %(os.path.dirname(image), version) image = shutil.move(image, finished_image) final_time = (datetime.now() - start_time).seconds bot.info("Final time of build %s seconds." %final_time) # Did the container build successfully? test_result = test_container(image) if test_result['return_code'] != 0: bot.error("Image failed to build, cancelling.") sys.exit(1) # Get singularity version singularity_version = Client.version() Client.debug = False inspect = Client.inspect(image) # this is a string Client.debug = params['debug'] # Get information on apps Client.debug = False app_names = Client.apps(image) Client.debug = params['debug'] apps = extract_apps(image, app_names) metrics = {'build_time_seconds': final_time, 'singularity_version': singularity_version, 'singularity_python_version': singularity_python_version, 'inspect': inspect, 'version': version, 'apps': apps} output = {'image':image, 'metadata':metrics, 'params':params } return output else: # Tell the user what is actually there present_files = glob("*") bot.error("Build file %s not found in repository" %params['spec_file']) bot.info("Found files are %s" %"\n".join(present_files)) # Params have been exported, will be found by log sys.exit(1)