def PULL(image, download_folder=None, layerfile=None):
'''PULL will retrieve a Singularity Hub image and
download to the local file system, to the variable
specified by SINGULARITY_PULLFOLDER.
:param image: the singularity hub image name
:param download folder: the folder to pull the image to.
:param layerfile: if defined, write pulled image to file
'''
client = SingularityApiConnection(image=image)
manifest = client.get_manifest()
if download_folder is None:
cache_base = get_cache(subfolder="shub")
else:
cache_base = os.path.abspath(download_folder)
bot.debug("Pull folder set to %s" % cache_base)
# The image name is the md5 hash, download if it's not there
image_name = get_image_name(manifest)
# Did the user specify an absolute path?
custom_folder = os.path.dirname(image_name)
if custom_folder not in [None, ""]:
cache_base = custom_folder
image_name = os.path.basename(image_name)
image_file = "%s/%s" % (cache_base, image_name)
bot.debug('Pulling to %s' % image_file)
if not os.path.exists(image_file):
image_file = client.download_image(manifest=manifest,
download_folder=cache_base,
image_name=image_name)
else:
if not bot.is_quiet(): # not --quiet
print("Image already exists at %s, skipping download" % image_file)
if not bot.is_quiet(): # not --quiet
print("Singularity Hub Image Download: %s" % image_file)
manifest = {
'image_file': image_file,
'manifest': manifest,
'cache_base': cache_base,
'image': image
}
if layerfile is not None:
bot.debug("Writing Singularity Hub image path to %s" % layerfile)
write_file(layerfile, image_file, mode="w")
return manifest
def extract_metadata_tar(manifest,
image_name,
include_env=True,
include_labels=True,
runscript=None):
'''extract_metadata_tar will write a tarfile with the environment,
labels, and runscript. include_env and include_labels should be booleans,
and runscript should be None or a string to write to the runscript.
'''
tar_file = None
files = []
if include_env or include_labels:
# Extract and add environment
if include_env:
environ = extract_env(manifest)
if environ not in [None, ""]:
bot.verbose3('Adding Docker environment to metadata tar')
template = get_template('tarinfo')
template['name'] = './%s/env/%s-%s.sh' % (METADATA_FOLDER_NAME,
DOCKER_NUMBER,
DOCKER_PREFIX)
template['content'] = environ
files.append(template)
# Extract and add labels
if include_labels:
labels = extract_labels(manifest)
if labels is not None:
if isinstance(labels, dict):
labels = print_json(labels)
bot.verbose3('Adding Docker labels to metadata tar')
template = get_template('tarinfo')
template['name'] = "./%s/labels.json" % METADATA_FOLDER_NAME
template['content'] = labels
files.append(template)
if runscript is not None:
bot.verbose3('Adding Docker runscript to metadata tar')
template = get_template('tarinfo')
template['name'] = "./%s/runscript" % METADATA_FOLDER_NAME
template['content'] = runscript
files.append(template)
if len(files) > 0:
output_folder = get_cache(subfolder="metadata", quiet=True)
tar_file = create_tar(files, output_folder)
else:
bot.warning("No metadata will be included.")
return tar_file
def PULL(image, download_folder=None, layerfile=None):
'''PULL will retrieve a Singularity Hub image and
download to the local file system, to the variable
specified by SINGULARITY_PULLFOLDER.
:param image: the singularity hub image name
:param download folder: the folder to pull the image to.
:param layerfile: if defined, write pulled image to file
'''
client = SingularityApiConnection(image=image)
manifest = client.get_manifest()
if download_folder is None:
cache_base = get_cache(subfolder="shub")
else:
cache_base = os.path.abspath(download_folder)
bot.debug("Pull folder set to %s" % cache_base)
# The image name is the md5 hash, download if it's not there
image_name = get_image_name(manifest)
# Did the user specify an absolute path?
custom_folder = os.path.dirname(image_name)
if custom_folder not in [None, ""]:
cache_base = custom_folder
image_name = os.path.basename(image_name)
image_file = "%s/%s" % (cache_base, image_name)
bot.debug('Pulling to %s' % image_file)
if not os.path.exists(image_file):
image_file = client.download_image(manifest=manifest,
download_folder=cache_base)
else:
if not bot.is_quiet(): # not --quiet
print("Image already exists at %s, skipping download" % image_file)
if not bot.is_quiet(): # not --quiet
print("Singularity Hub Image Download: %s" % image_file)
manifest = {'image_file': image_file,
'manifest': manifest,
'cache_base': cache_base,
'image': image}
if layerfile is not None:
bot.debug("Writing Singularity Hub image path to %s" % layerfile)
write_file(layerfile, image_file, mode="w")
return manifest
def extract_metadata_tar(manifest,
image_name,
include_env=True,
include_labels=True,
runscript=None):
'''extract_metadata_tar will write a tarfile with the environment,
labels, and runscript. include_env and include_labels should be booleans,
and runscript should be None or a string to write to the runscript.
'''
tar_file = None
files = []
if include_env or include_labels:
# Extract and add environment
if include_env:
environ = extract_env(manifest)
if environ not in [None, ""]:
bot.verbose3('Adding Docker environment to metadata tar')
template = get_template('tarinfo')
template['name'] = './%s/env/%s-%s.sh' % (
METADATA_FOLDER_NAME, DOCKER_NUMBER, DOCKER_PREFIX)
template['content'] = environ
files.append(template)
# Extract and add labels
if include_labels:
labels = extract_labels(manifest)
if labels is not None:
if isinstance(labels, dict):
labels = print_json(labels)
bot.verbose3('Adding Docker labels to metadata tar')
template = get_template('tarinfo')
template['name'] = "./%s/labels.json" % METADATA_FOLDER_NAME
template['content'] = labels
files.append(template)
if runscript is not None:
bot.verbose3('Adding Docker runscript to metadata tar')
template = get_template('tarinfo')
template['name'] = "./%s/runscript" % METADATA_FOLDER_NAME
template['content'] = runscript
files.append(template)
if len(files) > 0:
output_folder = get_cache(subfolder="metadata", quiet=True)
tar_file = create_tar(files, output_folder)
else:
bot.warning("No metadata will be included.")
return tar_file
def IMPORT(image, auth=None, layerfile=None):
'''IMPORT is the main script that will obtain docker layers,
runscript information (either entrypoint or cmd), and environment
and return a list of tarballs to extract into the image
:param auth: if needed, an authentication header (default None)
:param layerfile: The file to write layers to extract into
'''
bot.debug("Starting Docker IMPORT, includes env, runscript, and metadata.")
bot.verbose("Docker image: %s" % image)
# Does the user want to override default of using ENTRYPOINT?
if INCLUDE_CMD:
bot.verbose2("Specified Docker CMD as %runscript.")
else:
bot.verbose2("Specified Docker ENTRYPOINT as %runscript.")
# Input Parsing ----------------------------
# Parse image name, repo name, and namespace
client = DockerApiConnection(image=image, auth=auth)
docker_image_uri = "Docker image path: %s" % client.assemble_uri("/")
bot.info(docker_image_uri)
# IMAGE METADATA -------------------------------------------
# Use Docker Registry API (version 2.0) to get images ids, manifest
images = client.get_images()
# DOWNLOAD LAYERS -------------------------------------------
# Each is a .tar.gz file, obtained from registry with curl
# Get the cache (or temporary one) for docker
cache_base = get_cache(subfolder="docker")
download_client = MultiProcess()
# Generate a queue of tasks to run with MultiProcess
layers = []
tasks = []
for ii in range(len(images)):
image_id = images[ii]
targz = "%s/%s.tar.gz" % (cache_base, image_id)
if not os.path.exists(targz):
tasks.append((client, image_id, cache_base))
layers.append(targz)
# Does the user want to change permissions of tar?
func2 = None
if PLUGIN_FIXPERMS:
func2 = change_permissions
if len(tasks) > 0:
download_layers = download_client.run(func=download_layer,
func2=func2,
tasks=tasks)
# Get Docker runscript
runscript = extract_runscript(manifest=client.manifestv1,
includecmd=INCLUDE_CMD)
# Add the environment export
tar_file = extract_metadata_tar(client.manifestv1,
client.assemble_uri(),
runscript=runscript)
bot.verbose2('Tar file with Docker env and labels: %s' % tar_file)
# Write all layers to the layerfile
if layerfile is not None:
bot.verbose3("Writing Docker layers files to %s" % layerfile)
write_file(layerfile, "\n".join(layers), mode="w")
if tar_file is not None:
write_file(layerfile, "\n%s" % tar_file, mode="a")
# Return additions dictionary
additions = {"layers": layers,
"image": image,
"manifest": client.manifest,
"manifestv1": client.manifestv1,
"cache_base": cache_base,
"metadata": tar_file}
bot.debug("*** FINISHING DOCKER IMPORT PYTHON PORTION ****\n")
return additions
def IMPORT(image, auth=None, layerfile=None):
'''IMPORT is the main script that will obtain docker layers,
runscript information (either entrypoint or cmd), and environment
and return a list of tarballs to extract into the image
:param auth: if needed, an authentication header (default None)
:param layerfile: The file to write layers to extract into
'''
bot.debug("Starting Docker IMPORT, includes env, runscript, and metadata.")
bot.verbose("Docker image: %s" % image)
# Does the user want to override default of using ENTRYPOINT?
if INCLUDE_CMD:
bot.verbose2("Specified Docker CMD as %runscript.")
else:
bot.verbose2("Specified Docker ENTRYPOINT as %runscript.")
# Input Parsing ----------------------------
# Parse image name, repo name, and namespace
client = DockerApiConnection(image=image, auth=auth)
docker_image_uri = "Docker image path: %s" % client.assemble_uri("/")
bot.info(docker_image_uri)
# IMAGE METADATA -------------------------------------------
# Use Docker Registry API (version 2.0) to get images ids, manifest
images = client.get_images()
# DOWNLOAD LAYERS -------------------------------------------
# Each is a .tar.gz file, obtained from registry with curl
# Get the cache (or temporary one) for docker
cache_base = get_cache(subfolder="docker")
download_client = MultiProcess()
# Generate a queue of tasks to run with MultiProcess
layers = []
tasks = []
for ii in range(len(images)):
image_id = images[ii]
targz = "%s/%s.tar.gz" % (cache_base, image_id)
if not os.path.exists(targz):
tasks.append((client, image_id, cache_base))
layers.append(targz)
# Does the user want to change permissions of tar?
func2 = None
if PLUGIN_FIXPERMS:
func2 = change_permissions
if len(tasks) > 0:
download_layers = download_client.run(func=download_layer,
func2=func2,
tasks=tasks)
# Get Docker runscript
runscript = extract_runscript(manifest=client.manifestv1,
includecmd=INCLUDE_CMD)
# Add the environment export
tar_file = extract_metadata_tar(client.manifestv1,
client.assemble_uri(),
runscript=runscript)
bot.verbose2('Tar file with Docker env and labels: %s' % tar_file)
# Write all layers to the layerfile
if layerfile is not None:
bot.verbose3("Writing Docker layers files to %s" % layerfile)
write_file(layerfile, "\n".join(layers), mode="w")
if tar_file is not None:
write_file(layerfile, "\n%s" % tar_file, mode="a")
# Return additions dictionary
additions = {
"layers": layers,
"image": image,
"manifest": client.manifest,
"manifestv1": client.manifestv1,
"cache_base": cache_base,
"metadata": tar_file
}
bot.debug("*** FINISHING DOCKER IMPORT PYTHON PORTION ****\n")
return additions
