def write_singularity_infos(base_dir,
prefix,
start_number,
content,
extension=None):
'''write_singularity_infos will write some metadata object
to a file in some base, starting at some default number. For example,
we would want to write dockerN files with docker environment exports to
some directory ENV_DIR and increase N until we find an available path
:param base_dir: the directory base to write the file to
:param prefix: the name of the file prefix (eg, docker)
:param start_number: the number to start looking for available file at
:param content: the content to write
:param extension: the extension to use. If not defined, uses .sh
'''
if extension is None:
extension = ""
else:
extension = ".%s" % extension
# if the base directory doesn't exist, exit with error.
if not os.path.exists(base_dir):
msg = "Cannot find required metadata directory"
msg = "%s %s. Exiting!" % (msg, base_dir)
bot.warning(msg)
sys.exit(1)
# Get the next available number
output_file = get_next_infos(base_dir, prefix, start_number, extension)
write_file(output_file, content)
return output_file
def write_singularity_infos(base_dir,prefix,start_number,content,extension=None):
'''write_singularity_infos will write some metadata object
to a file in some base, starting at some default number. For example,
we would want to write dockerN files with docker environment exports to
some directory ENV_DIR and increase N until we find an available path
:param base_dir: the directory base to write the file to
:param prefix: the name of the file prefix (eg, docker)
:param start_number: the number to start looking for available file at
:param content: the content to write
:param extension: the extension to use. If not defined, uses .sh
'''
if extension == None:
extension = ""
else:
extension = ".%s" %(extension)
# if the base directory doesn't exist, exit with error.
if not os.path.exists(base_dir):
bot.warning("Cannot find required metadata directory %s. Exiting!" %base_dir)
sys.exit(1)
# Get the next available number
output_file = get_next_infos(base_dir,prefix,start_number,extension)
write_file(output_file,content)
return output_file
def change_tar_permissions(tar_file,
file_permission=None,
folder_permission=None):
'''change_tar_permissions changes a permission if
any member in a tarfile file does not have it
:param file_path the path to the file
:param file_permission: stat permission to use for files
:param folder_permission: stat permission to use for folders
'''
tar = tarfile.open(tar_file, "r:gz")
# Owner read, write (o+rw)
if file_permission is None:
file_permission = stat.S_IRUSR | stat.S_IWUSR
# Owner read, write execute (o+rwx)
if folder_permission is None:
folder_permission = stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR
# Add owner write permission to all, not symlinks
members = tar.getmembers()
if len(members) > 0:
bot.verbose("Fixing permission for %s" % tar_file)
# Add all content objects to file
fd, tmp_tar = tempfile.mkstemp(prefix=("%s.fixperm." % tar_file))
os.close(fd)
fixed_tar = tarfile.open(tmp_tar, "w:gz")
for member in members:
# add o+rwx for directories
if member.isdir() and not member.issym():
member.mode = folder_permission | member.mode
extracted = tar.extractfile(member)
fixed_tar.addfile(member, extracted)
# add o+rw for plain files
elif member.isfile() and not member.issym():
member.mode = file_permission | member.mode
extracted = tar.extractfile(member)
fixed_tar.addfile(member, extracted)
else:
fixed_tar.addfile(member)
fixed_tar.close()
tar.close()
# Rename the fixed tar to be the old name
os.rename(tmp_tar, tar_file)
else:
tar.close()
bot.warning("Tar file %s is empty, skipping." % tar_file)
return tar_file
def change_tar_permissions(tar_file,
file_permission=None,
folder_permission=None):
'''change_tar_permissions changes a permission if
any member in a tarfile file does not have it
:param file_path the path to the file
:param file_permission: stat permission to use for files
:param folder_permission: stat permission to use for folders
'''
tar = tarfile.open(tar_file, "r:gz")
# Owner read, write (o+rw)
if file_permission is None:
file_permission = stat.S_IRUSR | stat.S_IWUSR
# Owner read, write execute (o+rwx)
if folder_permission is None:
folder_permission = stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR
# Add owner write permission to all, not symlinks
members = tar.getmembers()
if len(members) > 0:
bot.verbose("Fixing permission for %s" % tar_file)
# Add all content objects to file
fd, tmp_tar = tempfile.mkstemp(prefix=("%s.fixperm." % tar_file))
os.close(fd)
fixed_tar = tarfile.open(tmp_tar, "w:gz")
for member in members:
# add o+rwx for directories
if member.isdir() and not member.issym():
member.mode = folder_permission | member.mode
extracted = tar.extractfile(member)
fixed_tar.addfile(member, extracted)
# add o+rw for plain files
elif member.isfile() and not member.issym():
member.mode = file_permission | member.mode
extracted = tar.extractfile(member)
fixed_tar.addfile(member, extracted)
else:
fixed_tar.addfile(member)
fixed_tar.close()
tar.close()
# Rename the fixed tar to be the old name
os.rename(tmp_tar, tar_file)
else:
tar.close()
bot.warning("Tar file %s is empty, skipping." % tar_file)
return tar_file
def extract_metadata_tar(manifest,
image_name,
include_env=True,
include_labels=True,
runscript=None):
'''extract_metadata_tar will write a tarfile with the environment,
labels, and runscript. include_env and include_labels should be booleans,
and runscript should be None or a string to write to the runscript.
'''
tar_file = None
files = []
if include_env or include_labels:
# Extract and add environment
if include_env:
environ = extract_env(manifest)
if environ not in [None, ""]:
bot.verbose3('Adding Docker environment to metadata tar')
template = get_template('tarinfo')
template['name'] = './%s/env/%s-%s.sh' % (METADATA_FOLDER_NAME,
DOCKER_NUMBER,
DOCKER_PREFIX)
template['content'] = environ
files.append(template)
# Extract and add labels
if include_labels:
labels = extract_labels(manifest)
if labels is not None:
if isinstance(labels, dict):
labels = print_json(labels)
bot.verbose3('Adding Docker labels to metadata tar')
template = get_template('tarinfo')
template['name'] = "./%s/labels.json" % METADATA_FOLDER_NAME
template['content'] = labels
files.append(template)
if runscript is not None:
bot.verbose3('Adding Docker runscript to metadata tar')
template = get_template('tarinfo')
template['name'] = "./%s/runscript" % METADATA_FOLDER_NAME
template['content'] = runscript
files.append(template)
if len(files) > 0:
output_folder = get_cache(subfolder="metadata", quiet=True)
tar_file = create_tar(files, output_folder)
else:
bot.warning("No metadata will be included.")
return tar_file
def extract_metadata_tar(manifest,
image_name,
include_env=True,
include_labels=True,
runscript=None):
'''extract_metadata_tar will write a tarfile with the environment,
labels, and runscript. include_env and include_labels should be booleans,
and runscript should be None or a string to write to the runscript.
'''
tar_file = None
files = []
if include_env or include_labels:
# Extract and add environment
if include_env:
environ = extract_env(manifest)
if environ not in [None, ""]:
bot.verbose3('Adding Docker environment to metadata tar')
template = get_template('tarinfo')
template['name'] = './%s/env/%s-%s.sh' % (
METADATA_FOLDER_NAME, DOCKER_NUMBER, DOCKER_PREFIX)
template['content'] = environ
files.append(template)
# Extract and add labels
if include_labels:
labels = extract_labels(manifest)
if labels is not None:
if isinstance(labels, dict):
labels = print_json(labels)
bot.verbose3('Adding Docker labels to metadata tar')
template = get_template('tarinfo')
template['name'] = "./%s/labels.json" % METADATA_FOLDER_NAME
template['content'] = labels
files.append(template)
if runscript is not None:
bot.verbose3('Adding Docker runscript to metadata tar')
template = get_template('tarinfo')
template['name'] = "./%s/runscript" % METADATA_FOLDER_NAME
template['content'] = runscript
files.append(template)
if len(files) > 0:
output_folder = get_cache(subfolder="metadata", quiet=True)
tar_file = create_tar(files, output_folder)
else:
bot.warning("No metadata will be included.")
return tar_file
def get_image_uri(image, quiet=False):
'''get_image_uri will parse a uri sent from Singularity to determine if it's
singularity (shub://) or docker (docker://)
:param image: the complete image uri (example: docker://ubuntu:latest
'''
image_uri = None
image = image.replace(' ', '')
match = re.findall('^[A-Za-z0-9-]+[:]//', image)
if len(match) == 0:
if not quiet:
bot.warning("Could not detect any uri in %s" % image)
else:
image_uri = match[0].lower()
if not quiet:
bot.debug("Found uri %s" % image_uri)
return image_uri
def get_image_uri(image,quiet=False):
'''get_image_uri will parse a uri sent from Singularity to determine if it's
singularity (shub://) or docker (docker://)
:param image: the complete image uri (example: docker://ubuntu:latest
'''
image_uri = None
image = image.replace(' ','')
match = re.findall('^[A-Za-z0-9-]+[:]//',image)
if len(match) == 0:
if not quiet:
bot.warning("Could not detect any uri in %s" %image)
else:
image_uri = match[0].lower()
if not quiet:
bot.debug("Found uri %s" %image_uri)
return image_uri
def get_fullpath(file_path, required=True):
'''get_fullpath checks if a file exists, and returns the
full path to it if it does. If required is true, an error is triggered.
:param file_path: the path to check
:param required: is the file required? If True, will exit with error
'''
file_path = os.path.abspath(file_path)
if os.path.exists(file_path):
return file_path
# If file is required, we exit
if required is True:
bot.error("Cannot find file %s, exiting." % file_path)
sys.exit(1)
# If file isn't required and doesn't exist, return None
bot.warning("Cannot find file %s" % file_path)
return None
def get_fullpath(file_path,required=True):
'''get_fullpath checks if a file exists, and returns the
full path to it if it does. If required is true, an error is triggered.
:param file_path: the path to check
:param required: is the file required? If True, will exit with error
'''
file_path = os.path.abspath(file_path)
if os.path.exists(file_path):
return file_path
# If file is required, we exit
if required == True:
bot.error("Cannot find file %s, exiting." %file_path)
sys.exit(1)
# If file isn't required and doesn't exist, return None
bot.warning("Cannot find file %s" %file_path)
return None
def get_template(template_name):
'''get_template will return a default template for some function in
Singularity Python. This is to reduce redundancy if data structures
are used multiple times, etc. If there is no template, None is returned.
'''
template_name = template_name.lower()
templates = dict()
templates['tarinfo'] = {"gid": 0,
"uid": 0,
"uname": "root",
"gname": "root",
"mode": 493}
if template_name in templates:
bot.debug("Found template for %s" % (template_name))
return templates[template_name]
else:
bot.warning("Cannot find template %s" % (template_name))
return None
def verify_layer(targz):
'''check that a downloaded layer's sha256 checksum is OK
correct checksum is in the filename:
sha256:7d460157dea423c1e16c544fecad995439e12dd50c8db4a8e134fa245cd1846e.tar.gz
'''
targz_basename = os.path.basename(targz)
bot.debug("Verifying checksum for layer: %s" % targz_basename)
if targz_basename[:6] != 'sha256':
bot.warning(
"Unknown hash function for layer (%s) - will not checksum" %
targz_basename[:5])
return True
expected = targz_basename[7:71]
sha256 = hashlib.sha256()
try:
with open(targz, 'rb') as f:
for block in iter(lambda: f.read(1048576), b''):
sha256.update(block)
except Exception as e:
bot.error("Error computing checksum for layer (%s) - %s" %
(targz_basename, str(e)))
return False
computed = sha256.hexdigest()
bot.debug("Computed checksum %s, expected checksum %s" %
(computed, expected))
if computed != expected:
bot.error("Downloaded layer %s does not match checksum" %
targz_basename)
return False
return True
def get_template(template_name):
'''get_template will return a default template for some function in
Singularity Python. This is to reduce redundancy if data structures
are used multiple times, etc. If there is no template, None is returned.
'''
template_name = template_name.lower()
templates = dict()
templates['tarinfo'] = {
"gid": 0,
"uid": 0,
"uname": "root",
"gname": "root",
"mode": 493
}
if template_name in templates:
bot.debug("Found template for %s" % template_name)
return templates[template_name]
else:
bot.warning("Cannot find template %s" % template_name)
return None