def write_singularity_infos(base_dir, prefix, start_number, content, extension=None): '''write_singularity_infos will write some metadata object to a file in some base, starting at some default number. For example, we would want to write dockerN files with docker environment exports to some directory ENV_DIR and increase N until we find an available path :param base_dir: the directory base to write the file to :param prefix: the name of the file prefix (eg, docker) :param start_number: the number to start looking for available file at :param content: the content to write :param extension: the extension to use. If not defined, uses .sh ''' if extension is None: extension = "" else: extension = ".%s" % extension # if the base directory doesn't exist, exit with error. if not os.path.exists(base_dir): msg = "Cannot find required metadata directory" msg = "%s %s. Exiting!" % (msg, base_dir) bot.warning(msg) sys.exit(1) # Get the next available number output_file = get_next_infos(base_dir, prefix, start_number, extension) write_file(output_file, content) return output_file
def write_singularity_infos(base_dir,prefix,start_number,content,extension=None): '''write_singularity_infos will write some metadata object to a file in some base, starting at some default number. For example, we would want to write dockerN files with docker environment exports to some directory ENV_DIR and increase N until we find an available path :param base_dir: the directory base to write the file to :param prefix: the name of the file prefix (eg, docker) :param start_number: the number to start looking for available file at :param content: the content to write :param extension: the extension to use. If not defined, uses .sh ''' if extension == None: extension = "" else: extension = ".%s" %(extension) # if the base directory doesn't exist, exit with error. if not os.path.exists(base_dir): bot.warning("Cannot find required metadata directory %s. Exiting!" %base_dir) sys.exit(1) # Get the next available number output_file = get_next_infos(base_dir,prefix,start_number,extension) write_file(output_file,content) return output_file
def change_tar_permissions(tar_file, file_permission=None, folder_permission=None): '''change_tar_permissions changes a permission if any member in a tarfile file does not have it :param file_path the path to the file :param file_permission: stat permission to use for files :param folder_permission: stat permission to use for folders ''' tar = tarfile.open(tar_file, "r:gz") # Owner read, write (o+rw) if file_permission is None: file_permission = stat.S_IRUSR | stat.S_IWUSR # Owner read, write execute (o+rwx) if folder_permission is None: folder_permission = stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR # Add owner write permission to all, not symlinks members = tar.getmembers() if len(members) > 0: bot.verbose("Fixing permission for %s" % tar_file) # Add all content objects to file fd, tmp_tar = tempfile.mkstemp(prefix=("%s.fixperm." % tar_file)) os.close(fd) fixed_tar = tarfile.open(tmp_tar, "w:gz") for member in members: # add o+rwx for directories if member.isdir() and not member.issym(): member.mode = folder_permission | member.mode extracted = tar.extractfile(member) fixed_tar.addfile(member, extracted) # add o+rw for plain files elif member.isfile() and not member.issym(): member.mode = file_permission | member.mode extracted = tar.extractfile(member) fixed_tar.addfile(member, extracted) else: fixed_tar.addfile(member) fixed_tar.close() tar.close() # Rename the fixed tar to be the old name os.rename(tmp_tar, tar_file) else: tar.close() bot.warning("Tar file %s is empty, skipping." % tar_file) return tar_file
def extract_metadata_tar(manifest, image_name, include_env=True, include_labels=True, runscript=None): '''extract_metadata_tar will write a tarfile with the environment, labels, and runscript. include_env and include_labels should be booleans, and runscript should be None or a string to write to the runscript. ''' tar_file = None files = [] if include_env or include_labels: # Extract and add environment if include_env: environ = extract_env(manifest) if environ not in [None, ""]: bot.verbose3('Adding Docker environment to metadata tar') template = get_template('tarinfo') template['name'] = './%s/env/%s-%s.sh' % (METADATA_FOLDER_NAME, DOCKER_NUMBER, DOCKER_PREFIX) template['content'] = environ files.append(template) # Extract and add labels if include_labels: labels = extract_labels(manifest) if labels is not None: if isinstance(labels, dict): labels = print_json(labels) bot.verbose3('Adding Docker labels to metadata tar') template = get_template('tarinfo') template['name'] = "./%s/labels.json" % METADATA_FOLDER_NAME template['content'] = labels files.append(template) if runscript is not None: bot.verbose3('Adding Docker runscript to metadata tar') template = get_template('tarinfo') template['name'] = "./%s/runscript" % METADATA_FOLDER_NAME template['content'] = runscript files.append(template) if len(files) > 0: output_folder = get_cache(subfolder="metadata", quiet=True) tar_file = create_tar(files, output_folder) else: bot.warning("No metadata will be included.") return tar_file
def extract_metadata_tar(manifest, image_name, include_env=True, include_labels=True, runscript=None): '''extract_metadata_tar will write a tarfile with the environment, labels, and runscript. include_env and include_labels should be booleans, and runscript should be None or a string to write to the runscript. ''' tar_file = None files = [] if include_env or include_labels: # Extract and add environment if include_env: environ = extract_env(manifest) if environ not in [None, ""]: bot.verbose3('Adding Docker environment to metadata tar') template = get_template('tarinfo') template['name'] = './%s/env/%s-%s.sh' % ( METADATA_FOLDER_NAME, DOCKER_NUMBER, DOCKER_PREFIX) template['content'] = environ files.append(template) # Extract and add labels if include_labels: labels = extract_labels(manifest) if labels is not None: if isinstance(labels, dict): labels = print_json(labels) bot.verbose3('Adding Docker labels to metadata tar') template = get_template('tarinfo') template['name'] = "./%s/labels.json" % METADATA_FOLDER_NAME template['content'] = labels files.append(template) if runscript is not None: bot.verbose3('Adding Docker runscript to metadata tar') template = get_template('tarinfo') template['name'] = "./%s/runscript" % METADATA_FOLDER_NAME template['content'] = runscript files.append(template) if len(files) > 0: output_folder = get_cache(subfolder="metadata", quiet=True) tar_file = create_tar(files, output_folder) else: bot.warning("No metadata will be included.") return tar_file
def get_image_uri(image, quiet=False): '''get_image_uri will parse a uri sent from Singularity to determine if it's singularity (shub://) or docker (docker://) :param image: the complete image uri (example: docker://ubuntu:latest ''' image_uri = None image = image.replace(' ', '') match = re.findall('^[A-Za-z0-9-]+[:]//', image) if len(match) == 0: if not quiet: bot.warning("Could not detect any uri in %s" % image) else: image_uri = match[0].lower() if not quiet: bot.debug("Found uri %s" % image_uri) return image_uri
def get_image_uri(image,quiet=False): '''get_image_uri will parse a uri sent from Singularity to determine if it's singularity (shub://) or docker (docker://) :param image: the complete image uri (example: docker://ubuntu:latest ''' image_uri = None image = image.replace(' ','') match = re.findall('^[A-Za-z0-9-]+[:]//',image) if len(match) == 0: if not quiet: bot.warning("Could not detect any uri in %s" %image) else: image_uri = match[0].lower() if not quiet: bot.debug("Found uri %s" %image_uri) return image_uri
def get_fullpath(file_path, required=True): '''get_fullpath checks if a file exists, and returns the full path to it if it does. If required is true, an error is triggered. :param file_path: the path to check :param required: is the file required? If True, will exit with error ''' file_path = os.path.abspath(file_path) if os.path.exists(file_path): return file_path # If file is required, we exit if required is True: bot.error("Cannot find file %s, exiting." % file_path) sys.exit(1) # If file isn't required and doesn't exist, return None bot.warning("Cannot find file %s" % file_path) return None
def get_fullpath(file_path,required=True): '''get_fullpath checks if a file exists, and returns the full path to it if it does. If required is true, an error is triggered. :param file_path: the path to check :param required: is the file required? If True, will exit with error ''' file_path = os.path.abspath(file_path) if os.path.exists(file_path): return file_path # If file is required, we exit if required == True: bot.error("Cannot find file %s, exiting." %file_path) sys.exit(1) # If file isn't required and doesn't exist, return None bot.warning("Cannot find file %s" %file_path) return None
def get_template(template_name): '''get_template will return a default template for some function in Singularity Python. This is to reduce redundancy if data structures are used multiple times, etc. If there is no template, None is returned. ''' template_name = template_name.lower() templates = dict() templates['tarinfo'] = {"gid": 0, "uid": 0, "uname": "root", "gname": "root", "mode": 493} if template_name in templates: bot.debug("Found template for %s" % (template_name)) return templates[template_name] else: bot.warning("Cannot find template %s" % (template_name)) return None
def verify_layer(targz): '''check that a downloaded layer's sha256 checksum is OK correct checksum is in the filename: sha256:7d460157dea423c1e16c544fecad995439e12dd50c8db4a8e134fa245cd1846e.tar.gz ''' targz_basename = os.path.basename(targz) bot.debug("Verifying checksum for layer: %s" % targz_basename) if targz_basename[:6] != 'sha256': bot.warning( "Unknown hash function for layer (%s) - will not checksum" % targz_basename[:5]) return True expected = targz_basename[7:71] sha256 = hashlib.sha256() try: with open(targz, 'rb') as f: for block in iter(lambda: f.read(1048576), b''): sha256.update(block) except Exception as e: bot.error("Error computing checksum for layer (%s) - %s" % (targz_basename, str(e))) return False computed = sha256.hexdigest() bot.debug("Computed checksum %s, expected checksum %s" % (computed, expected)) if computed != expected: bot.error("Downloaded layer %s does not match checksum" % targz_basename) return False return True
def get_template(template_name): '''get_template will return a default template for some function in Singularity Python. This is to reduce redundancy if data structures are used multiple times, etc. If there is no template, None is returned. ''' template_name = template_name.lower() templates = dict() templates['tarinfo'] = { "gid": 0, "uid": 0, "uname": "root", "gname": "root", "mode": 493 } if template_name in templates: bot.debug("Found template for %s" % template_name) return templates[template_name] else: bot.warning("Cannot find template %s" % template_name) return None