def get_config(manifest, spec="Entrypoint", delim=None):
'''get_config returns a particular spec (default is Entrypoint)
from a VERSION 1 manifest obtained with get_manifest.
:param manifest: the manifest obtained from get_manifest
:param spec: the key of the spec to return, default is "Entrypoint"
:param delim: Given a list, the delim to use to join the entries.
Default is newline
'''
cmd = None
if "history" in manifest:
history = manifest['history']
for entry in manifest['history']:
if 'v1Compatibility' in entry:
entry = json.loads(entry['v1Compatibility'])
if "config" in entry:
if spec in entry["config"]:
if entry["config"][spec] is not None:
cmd = entry["config"][spec]
break
# Standard is to include commands like ['/bin/sh']
if isinstance(cmd, list):
if delim is not None:
cmd = delim.join(cmd)
bot.verbose3("Found Docker command (%s) %s" % (spec, cmd))
return cmd
def get_config(manifest, spec="Entrypoint", delim=None):
'''get_config returns a particular spec (default is Entrypoint)
from a VERSION 1 manifest obtained with get_manifest.
:param manifest: the manifest obtained from get_manifest
:param spec: the key of the spec to return, default is "Entrypoint"
:param delim: Given a list, the delim to use to join the entries.
Default is newline
'''
cmd = None
if "history" in manifest:
history = manifest['history']
for entry in manifest['history']:
if 'v1Compatibility' in entry:
entry = json.loads(entry['v1Compatibility'])
if "config" in entry:
if spec in entry["config"]:
if entry["config"][spec] is not None:
cmd = entry["config"][spec]
break
# Standard is to include commands like ['/bin/sh']
if isinstance(cmd, list):
if delim is not None:
cmd = delim.join(cmd)
bot.verbose3("Found Docker command (%s) %s" % (spec, cmd))
return cmd
def update_token(self, response=None, auth=None):
'''update_token uses HTTP basic authentication to get a token for
Docker registry API V2 operations. We get here if a 401 is
returned for a request.
https://docs.docker.com/registry/spec/auth/token/
'''
if self.token_url is None:
if response is None:
response = self.get_tags(return_response=True)
if not isinstance(response, HTTPError):
bot.verbose3('Response on obtaining token is None.')
return None
not_asking_auth = "Www-Authenticate" not in response.headers
if response.code != 401 or not_asking_auth:
bot.error("Authentication error, exiting.")
sys.exit(1)
challenge = response.headers["Www-Authenticate"]
regexp = '^Bearer\s+realm="(.+)",service="(.+)",scope="(.+)",?'
match = re.match(regexp, challenge)
if not match:
bot.error("Unrecognized authentication challenge, exiting.")
sys.exit(1)
realm = match.group(1)
service = match.group(2)
scope = match.group(3).split(',')[0]
self.token_url = ("%s?service=%s&expires_in=9000&scope=%s" %
(realm, service, scope))
headers = dict()
# First priority comes to auth supplied directly to function
if auth is not None:
headers.update(auth)
# Second priority is default if supplied at init
elif self.auth is not None:
headers.update(self.auth)
response = self.get(self.token_url,
default_headers=False,
headers=headers)
try:
token = json.loads(response)["token"]
token = {"Authorization": "Bearer %s" % token}
self.token = token
self.update_headers(token)
except Exception:
msg = "Error getting token for repository "
msg += "%s/%s, exiting." % (self.namespace, self.repo_name)
bot.error(msg)
sys.exit(1)
def update_token(self, response=None, auth=None):
'''update_token uses HTTP basic authentication to get a token for
Docker registry API V2 operations. We get here if a 401 is
returned for a request.
https://docs.docker.com/registry/spec/auth/token/
'''
if self.token_url is None:
if response is None:
response = self.get_tags(return_response=True)
if not isinstance(response, HTTPError):
bot.verbose3('Response on obtaining token is None.')
return None
not_asking_auth = "Www-Authenticate" not in response.headers
if response.code != 401 or not_asking_auth:
bot.error("Authentication error, exiting.")
sys.exit(1)
challenge = response.headers["Www-Authenticate"]
regexp = '^Bearer\s+realm="(.+)",service="(.+)",scope="(.+)",?'
match = re.match(regexp, challenge)
if not match:
bot.error("Unrecognized authentication challenge, exiting.")
sys.exit(1)
realm = match.group(1)
service = match.group(2)
scope = match.group(3).split(',')[0]
self.token_url = ("%s?service=%s&expires_in=9000&scope=%s"
% (realm, service, scope))
headers = dict()
# First priority comes to auth supplied directly to function
if auth is not None:
headers.update(auth)
# Second priority is default if supplied at init
elif self.auth is not None:
headers.update(self.auth)
response = self.get(self.token_url,
default_headers=False,
headers=headers)
try:
token = json.loads(response)["token"]
token = {"Authorization": "Bearer %s" % token}
self.token = token
self.update_headers(token)
except Exception:
bot.error("Error getting token for repository %s, exiting."
% self.repo_name)
sys.exit(1)
def clean_up(files):
'''clean up will delete a list of files, only if they exist
'''
if not isinstance(files, list):
files = [files]
for f in files:
if os.path.exists(f):
bot.verbose3("Cleaning up %s" % f)
os.remove(f)
def clean_up(files):
'''clean up will delete a list of files, only if they exist
'''
if not isinstance(files, list):
files = [files]
for f in files:
if os.path.exists(f):
bot.verbose3("Cleaning up %s" % f)
os.remove(f)
def read_file(filename,mode="r",readlines=True):
'''write_file will open a file, "filename" and write content, "content"
and properly close the file
'''
bot.verbose3("Reading file %s with mode %s." %(filename,mode))
with open(filename,mode) as filey:
if readlines:
content = filey.readlines()
else:
content = filey.read()
return content
def read_file(filename, mode="r", readlines=True):
'''read_file will open a file, "filename" and
read content, "content" and properly close the file
'''
bot.verbose3("Reading file %s with mode %s." % (filename, mode))
with open(filename, mode) as filey:
if readlines:
content = filey.readlines()
else:
content = filey.read()
return content
def extract_env(manifest):
'''extract the environment from the manifest, or return None. Used by
functions env_extract_image, and env_extract_tar
'''
environ = get_config(manifest,'Env')
if environ is not None:
if isinstance(environ,list):
environ = "\n".join(environ)
environ = ["export %s" %x for x in environ.split('\n')]
environ = "\n".join(environ)
bot.verbose3("Found Docker container environment!")
return environ
def extract_env(manifest):
'''extract the environment from the manifest, or return None.
Used by functions env_extract_image, and env_extract_tar
'''
environ = get_config(manifest, 'Env')
if environ is not None:
if not isinstance(environ, list):
environ = [environ]
lines = []
for line in environ:
line = re.findall("(?P<var_name>.+?)=(?P<var_value>.+)", line)
line = ['export %s="%s"' % (x[0], x[1]) for x in line]
lines = lines + line
environ = "\n".join(lines)
bot.verbose3("Found Docker container environment!")
return environ
def extract_labels(manifest, labelfile=None, prefix=None):
'''extract_labels will write a file of key value pairs including
maintainer, and labels.
:param manifest: the manifest to use
:param labelfile: if defined, write to labelfile (json)
:param prefix: an optional prefix to add to the names
'''
if prefix is None:
prefix = ""
labels = get_config(manifest, 'Labels')
if labels is not None and len(labels) is not 0:
bot.verbose3("Found Docker container labels!")
if labelfile is not None:
for key, value in labels.items():
key = "%s%s" % (prefix, key)
value = ADD(key, value, labelfile, force=True)
return labels
def extract_env(manifest):
'''extract the environment from the manifest, or return None.
Used by functions env_extract_image, and env_extract_tar
'''
environ = get_config(manifest, 'Env')
if environ is not None:
if not isinstance(environ, list):
environ = [environ]
lines = []
for line in environ:
line = re.findall("(?P<var_name>.+?)=(?P<var_value>.+)", line)
line = ['export %s="%s"' % (x[0], x[1]) for x in line]
lines = lines + line
environ = "\n".join(lines)
bot.verbose3("Found Docker container environment!")
return environ
def extract_labels(manifest, labelfile=None, prefix=None):
'''extract_labels will write a file of key value pairs including
maintainer, and labels.
:param manifest: the manifest to use
:param labelfile: if defined, write to labelfile (json)
:param prefix: an optional prefix to add to the names
'''
if prefix is None:
prefix = ""
labels = get_config(manifest, 'Labels')
if labels is not None and len(labels) is not 0:
bot.verbose3("Found Docker container labels!")
if labelfile is not None:
for key, value in labels.items():
key = "%s%s" % (prefix, key)
value = ADD(key, value, labelfile, force=True)
return labels
def extract_runscript(manifest, includecmd=False):
'''create_runscript will write a bash script with default "ENTRYPOINT"
into the base_dir. If includecmd is True, CMD is used instead. For both.
if the result is found empty, the other is tried, and then a default used.
:param manifest: the manifest to use to get the runscript
:param includecmd: overwrite default command (ENTRYPOINT) default is False
'''
cmd = None
# Does the user want to use the CMD instead of ENTRYPOINT?
commands = ["Entrypoint", "Cmd"]
if includecmd is True:
commands.reverse()
configs = get_configs(manifest, commands)
# Look for non "None" command
for command in commands:
if configs[command] is not None:
cmd = configs[command]
break
if cmd is not None:
bot.verbose3("Adding Docker %s as Singularity runscript..." %
command.upper())
# If the command is a list, join. (eg ['/usr/bin/python','hello.py']
if not isinstance(cmd, list):
cmd = [cmd]
cmd = " ".join(['"%s"' % x for x in cmd])
if not RUNSCRIPT_COMMAND_ASIS:
cmd = 'exec %s "$@"' % cmd
cmd = "#!/bin/sh\n\n%s\n" % cmd
return cmd
bot.debug("CMD and ENTRYPOINT not found, skipping runscript.")
return cmd
def extract_runscript(manifest, includecmd=False):
'''create_runscript will write a bash script with default "ENTRYPOINT"
into the base_dir. If includecmd is True, CMD is used instead. For both.
if the result is found empty, the other is tried, and then a default used.
:param manifest: the manifest to use to get the runscript
:param includecmd: overwrite default command (ENTRYPOINT) default is False
'''
cmd = None
# Does the user want to use the CMD instead of ENTRYPOINT?
commands = ["Entrypoint", "Cmd"]
if includecmd is True:
commands.reverse()
configs = get_configs(manifest, commands)
# Look for non "None" command
for command in commands:
if configs[command] is not None:
cmd = configs[command]
break
if cmd is not None:
bot.verbose3("Adding Docker %s as Singularity runscript..."
% command.upper())
# If the command is a list, join. (eg ['/usr/bin/python','hello.py']
if not isinstance(cmd, list):
cmd = [cmd]
cmd = " ".join(['"%s"' % x for x in cmd])
if not RUNSCRIPT_COMMAND_ASIS:
cmd = 'exec %s "$@"' % cmd
cmd = "#!/bin/sh\n\n%s\n" % cmd
return cmd
bot.debug("CMD and ENTRYPOINT not found, skipping runscript.")
return cmd
def extract_metadata_tar(manifest,
image_name,
include_env=True,
include_labels=True,
runscript=None):
'''extract_metadata_tar will write a tarfile with the environment,
labels, and runscript. include_env and include_labels should be booleans,
and runscript should be None or a string to write to the runscript.
'''
tar_file = None
files = []
if include_env or include_labels:
# Extract and add environment
if include_env:
environ = extract_env(manifest)
if environ not in [None, ""]:
bot.verbose3('Adding Docker environment to metadata tar')
template = get_template('tarinfo')
template['name'] = './%s/env/%s-%s.sh' % (METADATA_FOLDER_NAME,
DOCKER_NUMBER,
DOCKER_PREFIX)
template['content'] = environ
files.append(template)
# Extract and add labels
if include_labels:
labels = extract_labels(manifest)
if labels is not None:
if isinstance(labels, dict):
labels = print_json(labels)
bot.verbose3('Adding Docker labels to metadata tar')
template = get_template('tarinfo')
template['name'] = "./%s/labels.json" % METADATA_FOLDER_NAME
template['content'] = labels
files.append(template)
if runscript is not None:
bot.verbose3('Adding Docker runscript to metadata tar')
template = get_template('tarinfo')
template['name'] = "./%s/runscript" % METADATA_FOLDER_NAME
template['content'] = runscript
files.append(template)
if len(files) > 0:
output_folder = get_cache(subfolder="metadata", quiet=True)
tar_file = create_tar(files, output_folder)
else:
bot.warning("No metadata will be included.")
return tar_file
def extract_metadata_tar(manifest,
image_name,
include_env=True,
include_labels=True,
runscript=None):
'''extract_metadata_tar will write a tarfile with the environment,
labels, and runscript. include_env and include_labels should be booleans,
and runscript should be None or a string to write to the runscript.
'''
tar_file = None
files = []
if include_env or include_labels:
# Extract and add environment
if include_env:
environ = extract_env(manifest)
if environ not in [None, ""]:
bot.verbose3('Adding Docker environment to metadata tar')
template = get_template('tarinfo')
template['name'] = './%s/env/%s-%s.sh' % (
METADATA_FOLDER_NAME, DOCKER_NUMBER, DOCKER_PREFIX)
template['content'] = environ
files.append(template)
# Extract and add labels
if include_labels:
labels = extract_labels(manifest)
if labels is not None:
if isinstance(labels, dict):
labels = print_json(labels)
bot.verbose3('Adding Docker labels to metadata tar')
template = get_template('tarinfo')
template['name'] = "./%s/labels.json" % METADATA_FOLDER_NAME
template['content'] = labels
files.append(template)
if runscript is not None:
bot.verbose3('Adding Docker runscript to metadata tar')
template = get_template('tarinfo')
template['name'] = "./%s/runscript" % METADATA_FOLDER_NAME
template['content'] = runscript
files.append(template)
if len(files) > 0:
output_folder = get_cache(subfolder="metadata", quiet=True)
tar_file = create_tar(files, output_folder)
else:
bot.warning("No metadata will be included.")
return tar_file
def IMPORT(image, auth=None, layerfile=None):
'''IMPORT is the main script that will obtain docker layers,
runscript information (either entrypoint or cmd), and environment
and return a list of tarballs to extract into the image
:param auth: if needed, an authentication header (default None)
:param layerfile: The file to write layers to extract into
'''
bot.debug("Starting Docker IMPORT, includes env, runscript, and metadata.")
bot.verbose("Docker image: %s" % image)
# Does the user want to override default of using ENTRYPOINT?
if INCLUDE_CMD:
bot.verbose2("Specified Docker CMD as %runscript.")
else:
bot.verbose2("Specified Docker ENTRYPOINT as %runscript.")
# Input Parsing ----------------------------
# Parse image name, repo name, and namespace
client = DockerApiConnection(image=image, auth=auth)
docker_image_uri = "Docker image path: %s" % client.assemble_uri("/")
bot.info(docker_image_uri)
# IMAGE METADATA -------------------------------------------
# Use Docker Registry API (version 2.0) to get images ids, manifest
images = client.get_images()
# DOWNLOAD LAYERS -------------------------------------------
# Each is a .tar.gz file, obtained from registry with curl
# Get the cache (or temporary one) for docker
cache_base = get_cache(subfolder="docker")
download_client = MultiProcess()
# Generate a queue of tasks to run with MultiProcess
layers = []
tasks = []
for ii in range(len(images)):
image_id = images[ii]
targz = "%s/%s.tar.gz" % (cache_base, image_id)
if not os.path.exists(targz):
tasks.append((client, image_id, cache_base))
layers.append(targz)
# Does the user want to change permissions of tar?
func2 = None
if PLUGIN_FIXPERMS:
func2 = change_permissions
if len(tasks) > 0:
download_layers = download_client.run(func=download_layer,
func2=func2,
tasks=tasks)
# Get Docker runscript
runscript = extract_runscript(manifest=client.manifestv1,
includecmd=INCLUDE_CMD)
# Add the environment export
tar_file = extract_metadata_tar(client.manifestv1,
client.assemble_uri(),
runscript=runscript)
bot.verbose2('Tar file with Docker env and labels: %s' % tar_file)
# Write all layers to the layerfile
if layerfile is not None:
bot.verbose3("Writing Docker layers files to %s" % layerfile)
write_file(layerfile, "\n".join(layers), mode="w")
if tar_file is not None:
write_file(layerfile, "\n%s" % tar_file, mode="a")
# Return additions dictionary
additions = {"layers": layers,
"image": image,
"manifest": client.manifest,
"manifestv1": client.manifestv1,
"cache_base": cache_base,
"metadata": tar_file}
bot.debug("*** FINISHING DOCKER IMPORT PYTHON PORTION ****\n")
return additions
def IMPORT(image, auth=None, layerfile=None):
'''IMPORT is the main script that will obtain docker layers,
runscript information (either entrypoint or cmd), and environment
and return a list of tarballs to extract into the image
:param auth: if needed, an authentication header (default None)
:param layerfile: The file to write layers to extract into
'''
bot.debug("Starting Docker IMPORT, includes env, runscript, and metadata.")
bot.verbose("Docker image: %s" % image)
# Does the user want to override default of using ENTRYPOINT?
if INCLUDE_CMD:
bot.verbose2("Specified Docker CMD as %runscript.")
else:
bot.verbose2("Specified Docker ENTRYPOINT as %runscript.")
# Input Parsing ----------------------------
# Parse image name, repo name, and namespace
client = DockerApiConnection(image=image, auth=auth)
docker_image_uri = "Docker image path: %s" % client.assemble_uri("/")
bot.info(docker_image_uri)
# IMAGE METADATA -------------------------------------------
# Use Docker Registry API (version 2.0) to get images ids, manifest
images = client.get_images()
# DOWNLOAD LAYERS -------------------------------------------
# Each is a .tar.gz file, obtained from registry with curl
# Get the cache (or temporary one) for docker
cache_base = get_cache(subfolder="docker")
download_client = MultiProcess()
# Generate a queue of tasks to run with MultiProcess
layers = []
tasks = []
for ii in range(len(images)):
image_id = images[ii]
targz = "%s/%s.tar.gz" % (cache_base, image_id)
if not os.path.exists(targz):
tasks.append((client, image_id, cache_base))
layers.append(targz)
# Does the user want to change permissions of tar?
func2 = None
if PLUGIN_FIXPERMS:
func2 = change_permissions
if len(tasks) > 0:
download_layers = download_client.run(func=download_layer,
func2=func2,
tasks=tasks)
# Get Docker runscript
runscript = extract_runscript(manifest=client.manifestv1,
includecmd=INCLUDE_CMD)
# Add the environment export
tar_file = extract_metadata_tar(client.manifestv1,
client.assemble_uri(),
runscript=runscript)
bot.verbose2('Tar file with Docker env and labels: %s' % tar_file)
# Write all layers to the layerfile
if layerfile is not None:
bot.verbose3("Writing Docker layers files to %s" % layerfile)
write_file(layerfile, "\n".join(layers), mode="w")
if tar_file is not None:
write_file(layerfile, "\n%s" % tar_file, mode="a")
# Return additions dictionary
additions = {
"layers": layers,
"image": image,
"manifest": client.manifest,
"manifestv1": client.manifestv1,
"cache_base": cache_base,
"metadata": tar_file
}
bot.debug("*** FINISHING DOCKER IMPORT PYTHON PORTION ****\n")
return additions
def update_token(self, response=None, auth=None, expires_in=300):
'''update_token uses HTTP basic authentication to get a token for
Docker registry API V2 operations. We get here if a 401 is
returned for a request.
https://docs.docker.com/registry/spec/auth/token/
'''
if self.tokenExpires and self.tokenExpires > (time.time() - 5):
bot.debug("Not renewing token - does not expire within 5s")
return
if self.token_url is None:
if response is None:
response = self.get_tags(return_response=True)
if not isinstance(response, HTTPError):
bot.verbose3('Response on obtaining token is None.')
return None
not_asking_auth = "Www-Authenticate" not in response.headers
if response.code != 401 or not_asking_auth:
bot.error("Authentication error, exiting.")
sys.exit(1)
challenge = response.headers["Www-Authenticate"]
self.token_url = self.get_token_url(challenge,
expires_in=expires_in)
headers = dict()
# First priority comes to auth supplied directly to function
if auth is not None:
headers.update(auth)
# Second priority is default if supplied at init
elif self.auth is not None:
headers.update(self.auth)
bot.debug("Requesting token from docker registry API")
response = self.get(self.token_url,
default_headers=False,
headers=headers,
updating_token=True)
try:
data = json.loads(response)
if 'access_token' in data:
access_token = data["access_token"]
else:
access_token = data["token"]
token = {"Authorization": "Bearer %s" % access_token}
self.token = token
# Default expiry from API is 60s
expires_in = 60
if 'expires_in' in data:
expires_in = data['expires_in']
# issued_at and expires_in are optional, so use the completion of
# token exchange (now)
self.tokenExpires = time.time() + expires_in
bot.debug("Received token valid for %d - expiring at %d" %
(expires_in, self.tokenExpires))
self.update_headers(token)
except Exception as e:
bot.error(
"Error getting token for repository %s, please check your credentials.\n%s\n" # noqa
% (self.repo_name, str(e)))
sys.exit(1)
