def download_atomically(self,
url,
file_name,
headers=None,
show_progress=False):
'''download stream atomically will stream to a temporary file, and
rename only upon successful completion. This is to ensure that
errored downloads are not found as complete in the cache
:param file_name: the file name to stream to
:param url: the url to stream from
:param headers: additional headers to add to the get (default None)
'''
try:
tmp_file = "%s.%s" % (file_name,
next(tempfile._get_candidate_names()))
response = self.stream(url,
file_name=tmp_file,
headers=headers,
show_progress=show_progress)
os.rename(tmp_file, file_name)
except Exception:
download_folder = os.path.dirname(os.path.abspath(file_name))
msg = "Error downloading %s. " % (url)
msg += "Do you have permission to write to %s?" % (download_folder)
bot.error(msg)
try:
os.remove(tmp_file)
except Exception:
pass
sys.exit(1)
return file_name
def download_atomically(self,
url,
file_name,
headers=None,
show_progress=False):
'''download stream atomically will stream to a temporary file, and
rename only upon successful completion. This is to ensure that
errored downloads are not found as complete in the cache
:param file_name: the file name to stream to
:param url: the url to stream from
:param headers: additional headers to add to the get (default None)
'''
try:
tmp_file = "%s.%s" % (file_name,
next(tempfile._get_candidate_names()))
response = self.stream(url,
file_name=tmp_file,
headers=headers,
show_progress=show_progress)
os.rename(tmp_file, file_name)
except Exception:
download_folder = os.path.dirname(os.path.abspath(file_name))
msg = "Error downloading %s. " % (url)
msg += "Do you have permission to write to %s?" % (download_folder)
bot.error(msg)
try:
os.remove(tmp_file)
except Exception:
pass
sys.exit(1)
return file_name
def run(self,func,tasks,func2=None):
'''run will send a list of tasks, a tuple with arguments, through a function.
the arguments should be ordered correctly.
:param func: the function to run with multiprocessing.pool
:param tasks: a list of tasks, each a tuple of arguments to process
:param func2: filter function to run result from func through (optional)
'''
# Keep track of some progress for the user
progress = 1
total = len(tasks)
# If two functions are run per task, double total jobs
if func2 is not None:
total = total * 2
finished = []
level1 = []
results = []
try:
prefix = "[%s/%s]" %(progress,total)
bot.show_progress(0,total,length=35,prefix=prefix)
pool = multiprocessing.Pool(self.workers,init_worker)
self.start()
for task in tasks:
result = pool.apply_async(multi_wrapper,
multi_package(func,[task]))
results.append(result)
level1.append(result._job)
while len(results) > 0:
result = results.pop()
result.wait()
bot.show_progress(progress,total,length=35,prefix=prefix)
progress+=1
prefix = "[%s/%s]" %(progress,total)
# Pass the result through a second function?
if func2 is not None and result._job in level1:
result = pool.apply_async(multi_wrapper,
multi_package(func2,[(result.get(),)]))
results.append(result)
else:
finished.append(result.get())
self.end()
pool.close()
pool.join()
except (KeyboardInterrupt, SystemExit):
bot.error("Keyboard interrupt detected, terminating workers!")
pool.terminate()
sys.exit(1)
except Exception as e:
bot.error(e)
return finished
def get_manifest(self,old_version=False):
'''get_manifest should return an image manifest for a particular repo and tag.
The image details are extracted when the client is generated.
:param old_version: return version 1 (for cmd/entrypoint), default False
'''
registry = self.registry
if registry == None:
registry = self.api_base
registry = add_http(registry) # make sure we have a complete url
base = "%s/%s/%s/%s/manifests" %(registry,self.api_version,self.namespace,self.repo_name)
if self.version is not None:
base = "%s/%s" %(base,self.version)
else:
base = "%s/%s" %(base,self.repo_tag)
bot.verbose("Obtaining manifest: %s" %base)
headers = self.headers
if old_version == True:
headers['Accept'] = 'application/json'
response = self.get(base,headers=self.headers)
try:
response = json.loads(response)
except:
# If the call fails, give the user a list of acceptable tags
tags = self.get_tags()
print("\n".join(tags))
repo_uri = "%s/%s:%s" %(self.namespace,self.repo_name,self.repo_tag)
bot.error("Error getting manifest for %s, exiting." %repo_uri)
sys.exit(1)
return response
def main():
'''main is a wrapper for the client to hand the parser
to the executable functions
This makes it possible to set up a parser in test cases
'''
bot.debug("\n*** STARTING SINGULARITY PYTHON PULL ****")
from defaults import LAYERFILE, DISABLE_CACHE, getenv
# What image is the user asking for?
container = getenv("SINGULARITY_CONTAINER", required=True)
pull_folder = getenv("SINGULARITY_PULLFOLDER")
image_uri = get_image_uri(container)
container = remove_image_uri(container, quiet=True)
if image_uri == "shub://":
from shub.main import PULL
manifest = PULL(image=container,
download_folder=pull_folder,
layerfile=LAYERFILE)
else:
bot.error("uri %s is not supported for pull. Exiting." % (image_uri))
sys.exit(1)
def get_tags(self, return_response=False):
'''get_tags will return the tags for a repo using the
Docker Version 2.0 Registry API
'''
registry = self.registry
if registry is None:
registry = self.api_base
registry = add_http(registry) # make sure we have a complete url
base = "%s/%s/%s/tags/list" % (registry, self.api_version,
self.repo_name)
bot.verbose("Obtaining tags: %s" % base)
# We use get_tags for a testing endpoint in update_token
response = self.get(base, return_response=return_response)
if return_response:
return response
try:
response = json.loads(response)
return response['tags']
except Exception:
bot.error("Error obtaining tags: %s" % base)
sys.exit(1)
def main():
'''main is a wrapper for the client to hand the parser
to the executable functions
This makes it possible to set up a parser in test cases
'''
bot.debug("\n*** STARTING SINGULARITY PYTHON PULL ****")
from defaults import LAYERFILE, DISABLE_CACHE, getenv
# What image is the user asking for?
container = getenv("SINGULARITY_CONTAINER", required=True)
pull_folder = getenv("SINGULARITY_PULLFOLDER")
image_uri = get_image_uri(container)
container = remove_image_uri(container, quiet=True)
if image_uri == "shub://":
from shub.main import PULL
manifest = PULL(image=container,
download_folder=pull_folder,
layerfile=LAYERFILE)
else:
bot.error("uri %s is not supported for pull. Exiting." % (image_uri))
sys.exit(1)
def get_tags(self, return_response=False):
'''get_tags will return the tags for a repo using the
Docker Version 2.0 Registry API
'''
registry = self.registry
if registry is None:
registry = self.api_base
registry = add_http(registry) # make sure we have a complete url
base = "%s/%s/%s/tags/list" % (registry,
self.api_version,
self.repo_name)
bot.verbose("Obtaining tags: %s" % base)
# We use get_tags for a testing endpoint in update_token
response = self.get(base,
return_response=return_response)
if return_response:
return response
try:
response = json.loads(response)
return response['tags']
except Exception:
bot.error("Error obtaining tags: %s" % base)
sys.exit(1)
def read_digests(manifest):
'''read_layers will return a list of layers from a manifest. The function is
intended to work with both version 1 and 2 of the schema
:param manifest: the manifest to read_layers from
'''
digests = []
# https://github.com/docker/distribution/blob/master/docs/spec/manifest-v2-2.md#image-manifest
if 'layers' in manifest:
layer_key = 'layers'
digest_key = 'digest'
bot.debug('Image manifest version 2.2 found.')
# https://github.com/docker/distribution/blob/master/docs/spec/manifest-v2-1.md#example-manifest
elif 'fsLayers' in manifest:
layer_key = 'fsLayers'
digest_key = 'blobSum'
bot.debug('Image manifest version 2.1 found.')
else:
bot.error(
'Improperly formed manifest, layers or fsLayers must be present')
sys.exit(1)
for layer in manifest[layer_key]:
if digest_key in layer:
if layer[digest_key] not in digests:
bot.debug("Adding digest %s" % layer[digest_key])
digests.append(layer[digest_key])
return digests
def get_tags(self, return_response=False):
'''get_tags will return the tags for a repo using the Docker Version 2.0 Registry API
:param namespace: the namespace (eg, "library")
:param repo_name: the name for the repo (eg, "ubuntu")
:param registry: the docker registry to use (default will use index.docker.io)
:param auth: authorization header (default None)
'''
registry = self.registry
if registry == None:
registry = self.api_base
registry = add_http(registry) # make sure we have a complete url
base = "%s/%s/%s/%s/tags/list" % (registry, self.api_version,
self.namespace, self.repo_name)
bot.verbose("Obtaining tags: %s" % base)
# We use get_tags for a testing endpoint in update_token
response = self.get(base, return_response=return_response)
if return_response:
return response
try:
response = json.loads(response)
return response['tags']
except:
bot.error("Error obtaining tags: %s" % base)
sys.exit(1)
def read_digests(manifest):
'''read_layers will return a list of layers from a manifest. The function is
intended to work with both version 1 and 2 of the schema
:param manifest: the manifest to read_layers from
'''
digests = []
# https://github.com/docker/distribution/blob/master/docs/spec/manifest-v2-2.md#image-manifest
if 'layers' in manifest:
layer_key = 'layers'
digest_key = 'digest'
bot.debug('Image manifest version 2.2 found.')
# https://github.com/docker/distribution/blob/master/docs/spec/manifest-v2-1.md#example-manifest
elif 'fsLayers' in manifest:
layer_key = 'fsLayers'
digest_key = 'blobSum'
bot.debug('Image manifest version 2.1 found.')
else:
bot.error('Improperly formed manifest, layers or fsLayers must be present')
sys.exit(1)
for layer in manifest[layer_key]:
if digest_key in layer:
if layer[digest_key] not in digests:
bot.debug("Adding digest %s" %layer[digest_key])
digests.append(layer[digest_key])
return digests
def get_tags(self,return_response=False):
'''get_tags will return the tags for a repo using the Docker Version 2.0 Registry API
:param namespace: the namespace (eg, "library")
:param repo_name: the name for the repo (eg, "ubuntu")
:param registry: the docker registry to use (default will use index.docker.io)
:param auth: authorization header (default None)
'''
registry = self.registry
if registry == None:
registry = self.api_base
registry = add_http(registry) # make sure we have a complete url
base = "%s/%s/%s/%s/tags/list" %(registry,self.api_version,self.namespace,self.repo_name)
bot.verbose("Obtaining tags: %s" %base)
# We use get_tags for a testing endpoint in update_token
response = self.get(base,
return_response=return_response)
if return_response:
return response
try:
response = json.loads(response)
return response['tags']
except:
bot.error("Error obtaining tags: %s" %base)
sys.exit(1)
def get_layer(self,
image_id,
download_folder=None,
change_perms=False,
return_tmp=False):
'''get_layer will download an image layer (.tar.gz)
to a specified download folder.
:param download_folder: if specified, download to folder.
Otherwise return response with raw data
:param change_perms: change permissions additionally
(default False to support multiprocessing)
:param return_tmp: If true, return the temporary file name (and
don't rename to the file's final name). Default
is False, should be True for multiprocessing
that requires extra permission changes
'''
registry = self.registry
if registry is None:
registry = self.api_base
# make sure we have a complete url
registry = add_http(registry)
# The <name> variable is the namespace/repo_name
base = "%s/%s/%s/%s/blobs/%s" % (registry, self.api_version,
self.namespace, self.repo_name,
image_id)
bot.verbose("Downloading layers from %s" % base)
if download_folder is None:
download_folder = tempfile.mkdtemp()
download_folder = "%s/%s.tar.gz" % (download_folder, image_id)
# Update user what we are doing
bot.debug("Downloading layer %s" % image_id)
# Step 1: Download the layer atomically
file_name = "%s.%s" % (download_folder,
next(tempfile._get_candidate_names()))
tar_download = self.download_atomically(url=base, file_name=file_name)
bot.debug('Download of raw file (pre permissions fix) is %s' %
tar_download)
# Step 2: Fix Permissions?
if change_perms:
tar_download = change_tar_permissions(tar_download)
if return_tmp is True:
return tar_download
try:
os.rename(tar_download, download_folder)
except Exception:
msg = "Cannot untar layer %s," % tar_download
msg += " was there a problem with download?"
bot.error(msg)
sys.exit(1)
return download_folder
def stream(self,url,file_name,data=None,headers=None,default_headers=True,
show_progress=False):
'''stream is a get that will stream to file_name
:param data: a dictionary of key:value items to add to the data args variable
:param url: the url to get
:param show_progress: if True, show a progress bar with the bot
:returns response: the requests response object, or stream
'''
bot.debug("GET (stream) %s" %url)
# If we use default headers, start with client's
request_headers = dict()
if default_headers and len(self.headers) > 0:
request_headers = self.headers
if headers is not None:
request_headers.update(headers)
request = self.prepare_request(headers=request_headers,
data=data,
url=url)
response = self.submit_request(request)
# Keep user updated with Progress Bar?
if show_progress:
content_size = None
if 'Content-Length' in response.headers and response.code not in [400,401]:
progress = 0
content_size = int(response.headers['Content-Length'])
bot.show_progress(progress,content_size,length=35)
chunk_size = 1 << 20
with open(file_name, 'wb') as filey:
while True:
chunk = response.read(chunk_size)
if not chunk:
break
try:
filey.write(chunk)
if show_progress:
if content_size is not None:
progress+=chunk_size
bot.show_progress(iteration=progress,
total=content_size,
length=35,
carriage_return=False)
except Exception as error:
bot.error("Error writing to %s: %s exiting" %(file_name,error))
sys.exit(1)
# Newline to finish download
if show_progress:
sys.stdout.write('\n')
return file_name
def load_image(self, image):
self.image = parse_image_uri(image=image,
uri='shub://',
default_registry=SHUB_API_BASE,
quiet=True)
# parse_image_uri may return an empty namespace cause that's allowed
# with docker://, but not with shub://
if len(self.image["namespace"]) == 0:
bot.error("Namespace cannot be empty for shub:// url!")
sys.exit(1)
def main():
'''this function will run a docker import, returning a list of layers
and environmental variables and metadata to the metadata base
'''
container = getenv("SINGULARITY_CONTAINER", required=True)
image_uri = get_image_uri(container, quiet=True)
container = remove_image_uri(container)
########################################################################
# Docker Image #########################################################
########################################################################
if image_uri == "docker://":
bot.debug("\n*** STARTING DOCKER IMPORT PYTHON ****")
from sutils import basic_auth_header
from defaults import LAYERFILE
bot.debug("Docker layers and metadata will be written to: %s"
% (LAYERFILE))
username = getenv("SINGULARITY_DOCKER_USERNAME")
password = getenv("SINGULARITY_DOCKER_PASSWORD",
silent=True)
auth = None
if username is not None and password is not None:
auth = basic_auth_header(username, password)
from docker.main import IMPORT
manifest = IMPORT(auth=auth,
image=container,
layerfile=LAYERFILE)
########################################################################
# Singularity Hub ######################################################
########################################################################
elif image_uri == "shub://":
bot.debug("\n*** STARTING SINGULARITY HUB IMPORT PYTHON ****")
from defaults import LAYERFILE, LABELFILE
from shub.main import IMPORT
IMPORT(image=container,
layerfile=LAYERFILE,
labelfile=LABELFILE)
else:
bot.error("uri %s is not supported for import. Exiting."
% (image_uri))
sys.exit(1)
def create_folders(path):
'''create_folders attempts to get the same functionality as mkdir -p
:param path: the path to create.
'''
try:
os.makedirs(path)
except OSError as e:
if e.errno == errno.EEXIST and os.path.isdir(path):
pass
else:
bot.error("Error creating path %s, exiting." % path)
sys.exit(1)
def create_folders(path):
'''create_folders attempts to get the same functionality as mkdir -p
:param path: the path to create.
'''
try:
os.makedirs(path)
except OSError as e:
if e.errno == errno.EEXIST and os.path.isdir(path):
pass
else:
bot.error("Error creating path %s, exiting." %path)
sys.exit(1)
def check_errors(self, response, exit=True):
'''take a response with errors key,
iterate through errors in expected format and exit upon completion
'''
if "errors" in response:
for error in response['errors']:
bot.error("%s: %s" % (error['code'], error['message']))
if error['code'] == "UNAUTHORIZED":
msg = "Check existence, naming, and permissions"
bot.error(msg)
if exit:
sys.exit(1)
return response
def update_manifests(self):
'''update manifests ensures that each of a version1 and version2
manifest are present
'''
if self.repo_name is None and self.namespace is None:
bot.error("Insufficient metadata to get manifest.")
sys.exit(1)
# Get full image manifest, using version 2.0 of Docker Registry API
if self.manifest is None:
self.manifest = self.get_manifest()
if self.manifestv1 is None:
self.manifestv1 = self.get_manifest(old_version=True)
def main():
'''this function will run a docker import, returning a list of layers
and environmental variables and metadata to the metadata base
'''
container = getenv("SINGULARITY_CONTAINER", required=True)
image_uri = get_image_uri(container, quiet=True)
container = remove_image_uri(container)
##############################################################################
# Docker Image ###############################################################
##############################################################################
if image_uri == "docker://":
bot.debug("\n*** STARTING DOCKER IMPORT PYTHON ****")
from sutils import basic_auth_header
from defaults import LAYERFILE
bot.debug(
"Docker layers and (env,labels,runscript) will be written to: %s" %
LAYERFILE)
username = getenv("SINGULARITY_DOCKER_USERNAME")
password = getenv("SINGULARITY_DOCKER_PASSWORD", silent=True)
auth = None
if username is not None and password is not None:
auth = basic_auth_header(username, password)
from docker.main import IMPORT
manifest = IMPORT(auth=auth, image=container, layerfile=LAYERFILE)
##############################################################################
# Singularity Hub ############################################################
##############################################################################
elif image_uri == "shub://":
bot.debug("\n*** STARTING SINGULARITY HUB IMPORT PYTHON ****")
from defaults import LAYERFILE, LABELFILE
from shub.main import IMPORT
IMPORT(image=container, layerfile=LAYERFILE, labelfile=LABELFILE)
else:
bot.error(
"uri %s is not a currently supported uri for import. Exiting." %
image_uri)
sys.exit(1)
def check_errors(self, response, exit=True):
'''take a response with errors key,
iterate through errors in expected format and exit upon completion
'''
if "errors" in response:
for error in response['errors']:
bot.error("%s: %s" % (error['code'],
error['message']))
if error['code'] == "UNAUTHORIZED":
msg = "Check existence, naming, and permissions"
bot.error(msg)
if exit:
sys.exit(1)
return response
def get_manifest(self, old_version=False, version=None):
'''get_manifest should return an image manifest
for a particular repo and tag. The image details
are extracted when the client is generated.
:param old_version: return version 1
(for cmd/entrypoint), default False
'''
registry = self.registry
if registry is None:
registry = self.api_base
# make sure we have a complete url
registry = add_http(registry)
base = "%s/%s/%s/%s/manifests" % (registry, self.api_version,
self.namespace, self.repo_name)
# First priority given to calling function
if version is not None:
base = "%s/%s" % (base, version)
elif self.version is not None:
base = "%s/%s" % (base, self.version)
else:
base = "%s/%s" % (base, self.repo_tag)
bot.verbose("Obtaining manifest: %s" % base)
headers = self.headers.copy()
if old_version is True:
headers['Accept'] = 'application/json'
response = self.get(base, headers=headers)
try:
response = json.loads(response)
except Exception:
# If the call fails, give the user a list of acceptable tags
tags = self.get_tags()
print("\n".join(tags))
repo_uri = "%s/%s:%s" % (self.namespace, self.repo_name,
self.repo_tag)
bot.error("Error getting manifest for %s, exiting." % repo_uri)
sys.exit(1)
# If we have errors, don't continue
return self.check_errors(response)
def main():
parser = get_parser()
try:
(args, options) = parser.parse_args()
except:
sys.exit(0)
if args.key is not None and args.file is not None:
value = GET(key=args.key, jsonfile=args.file)
else:
bot.error("--key and --file must be defined for GET. Exiting")
sys.exit(1)
def main():
parser = get_parser()
try:
(args, options) = parser.parse_args()
except Exception:
sys.exit(0)
if args.file is not None:
dump = DUMP(args.file)
else:
bot.error("--file must be defined for DUMP. Exiting")
sys.exit(1)
def run_command(cmd):
'''run_command uses subprocess to send a command to the terminal.
:param cmd: the command to send, should be a list for subprocess
'''
try:
bot.verbose2("Running command %s with subprocess" % " ".join(cmd))
process = subprocess.Popen(cmd, stdout=subprocess.PIPE)
except OSError as error:
bot.error("Error with subprocess: %s, returning None" % error)
return None
output = process.communicate()[0]
if process.returncode != 0:
return None
return output
def DUMP(jsonfile):
'''DUMP will return the entire layfile as text, key value pairs
:param jsonfile_path: the path to the jsonfile
'''
bot.debug("Reading %s to prepare dump to STDOUT" %jsonfile)
if not os.path.exists(jsonfile):
bot.error("Cannot find %s, exiting." %jsonfile)
sys.exit(1)
contents = read_json(jsonfile)
dump = ""
for key,value in contents.items():
dump = '%s%s:"%s"\n' %(dump,key,value)
dump = dump.strip('\n')
print(dump)
return dump
def main():
parser = get_parser()
try:
(args, options) = parser.parse_args()
except:
sys.exit(0)
if args.file is not None:
dump = DUMP(args.file)
else:
bot.error("--file must be defined for DUMP. Exiting")
sys.exit(1)
def run_command(cmd):
'''run_command uses subprocess to send a command to the terminal.
:param cmd: the command to send, should be a list for subprocess
'''
try:
bot.verbose2("Running command %s with subprocess" %" ".join(cmd))
process = subprocess.Popen(cmd,stdout=subprocess.PIPE)
except OSError as error:
bot.error("Error with subprocess: %s, returning None" %error)
return None
output = process.communicate()[0]
if process.returncode != 0:
return None
return output
def get_images(self, manifest=None):
'''get_images will return a list of layers from a manifest.
The function is intended to work with both version
1 and 2 of the schema. All layers (including redundant)
are returned.
For version 1 manifests: extraction is reversed
:param manifest: the manifest to read_layers from
'''
if manifest is None:
self.update_manifests()
manifest = self.manifest
digests = []
layer_key = 'layers'
digest_key = 'digest'
# Docker manifest-v2-2.md#image-manifest
if 'layers' in manifest:
bot.debug('Image manifest version 2.2 found.')
# Docker manifest-v2-1.md#example-manifest # noqa
elif 'fsLayers' in manifest:
layer_key = 'fsLayers'
digest_key = 'blobSum'
bot.debug('Image manifest version 2.1 found.')
else:
msg = "Improperly formed manifest, "
msg += "layers, manifests, or fsLayers must be present"
bot.error(msg)
sys.exit(1)
for layer in manifest[layer_key]:
if digest_key in layer:
bot.debug("Adding digest %s" % layer[digest_key])
digests.append(layer[digest_key])
# Reverse layer order for manifest version 1.0
if self.reverseLayers is True:
message = 'v%s manifest, reversing layers' % self.schemaVersion
bot.debug(message)
digests.reverse()
return digests
def get_images(self, manifest=None):
'''get_images will return a list of layers from a manifest.
The function is intended to work with both version
1 and 2 of the schema. All layers (including redundant)
are returned.
For version 1 manifests: extraction is reversed
:param manifest: the manifest to read_layers from
'''
if manifest is None:
self.update_manifests()
manifest = self.manifest
digests = []
layer_key = 'layers'
digest_key = 'digest'
# Docker manifest-v2-2.md#image-manifest
if 'layers' in manifest:
bot.debug('Image manifest version 2.2 found.')
# Docker manifest-v2-1.md#example-manifest # noqa
elif 'fsLayers' in manifest:
layer_key = 'fsLayers'
digest_key = 'blobSum'
bot.debug('Image manifest version 2.1 found.')
else:
msg = "Improperly formed manifest, "
msg += "layers, manifests, or fsLayers must be present"
bot.error(msg)
sys.exit(1)
for layer in manifest[layer_key]:
if digest_key in layer:
bot.debug("Adding digest %s" % layer[digest_key])
digests.append(layer[digest_key])
# Reverse layer order for manifest version 1.0
if self.reverseLayers is True:
message = 'v%s manifest, reversing layers' % self.schemaVersion
bot.debug(message)
digests.reverse()
return digests
def get_layer(self,image_id,download_folder=None,change_perms=False,return_tmp=False):
'''get_layer will download an image layer (.tar.gz) to a specified download folder.
:param download_folder: if specified, download to folder.
Otherwise return response with raw data (not recommended)
:param change_perms: change permissions additionally (default False to support
multiprocessing)
:param return_tmp: If true, return the temporary file name (and don't rename to the file's final
name). Default is False, should be True for multiprocessing that requires extra permission changes
'''
registry = self.registry
if registry == None:
registry = self.api_base
registry = add_http(registry) # make sure we have a complete url
# The <name> variable is the namespace/repo_name
base = "%s/%s/%s/%s/blobs/%s" %(registry,self.api_version,self.namespace,self.repo_name,image_id)
bot.verbose("Downloading layers from %s" %base)
if download_folder is None:
download_folder = tempfile.mkdtemp()
download_folder = "%s/%s.tar.gz" %(download_folder,image_id)
# Update user what we are doing
bot.debug("Downloading layer %s" %image_id)
# Step 1: Download the layer atomically
file_name = "%s.%s" %(download_folder,next(tempfile._get_candidate_names()))
tar_download = self.download_atomically(url=base,
file_name=file_name)
bot.debug('Download of raw file (pre permissions fix) is %s' %tar_download)
# Step 2: Fix Permissions?
if change_perms:
tar_download = change_tar_permissions(tar_download)
if return_tmp is True:
return tar_download
try:
os.rename(tar_download,download_folder)
except:
bot.error("Cannot untar layer %s, was there a problem with download?" %tar_download)
sys.exit(1)
return download_folder
def get_fullpath(file_path,required=True):
'''get_fullpath checks if a file exists, and returns the
full path to it if it does. If required is true, an error is triggered.
:param file_path: the path to check
:param required: is the file required? If True, will exit with error
'''
file_path = os.path.abspath(file_path)
if os.path.exists(file_path):
return file_path
# If file is required, we exit
if required == True:
bot.error("Cannot find file %s, exiting." %file_path)
sys.exit(1)
# If file isn't required and doesn't exist, return None
bot.warning("Cannot find file %s" %file_path)
return None
def GET(key,jsonfile):
'''GET will return a key from the jsonfile, if it exists. If it doesn't, returns None.
'''
key = format_keyname(key)
bot.debug("GET %s from %s" %(key,jsonfile))
if not os.path.exists(jsonfile):
bot.error("Cannot find %s, exiting." %jsonfile)
sys.exit(1)
contents = read_json(jsonfile)
if key in contents:
value = contents[key]
print(value)
bot.debug('%s is %s' %(key,value))
else:
bot.error("%s is not defined in file. Exiting" %key)
sys.exit(1)
return value
def get_fullpath(file_path, required=True):
'''get_fullpath checks if a file exists, and returns the
full path to it if it does. If required is true, an error is triggered.
:param file_path: the path to check
:param required: is the file required? If True, will exit with error
'''
file_path = os.path.abspath(file_path)
if os.path.exists(file_path):
return file_path
# If file is required, we exit
if required is True:
bot.error("Cannot find file %s, exiting." % file_path)
sys.exit(1)
# If file isn't required and doesn't exist, return None
bot.warning("Cannot find file %s" % file_path)
return None
def main():
parser = get_parser()
try:
(args,options) = parser.parse_args()
except:
sys.exit(0)
if args.key is not None and args.file is not None and args.value is not None:
value = ADD(key=args.key,
value=args.value,
jsonfile=args.file,
force=args.force)
else:
bot.error("--key and --file and --value must be defined for ADD. Exiting")
sys.exit(1)
def get_images(self):
'''get_images is a wrapper for get_manifest, but it additionally parses the repo_name and tag's
images and returns the complete ids
:param repo_name: the name of the repo, eg "ubuntu"
:param namespace: the namespace for the image, default is "library"
:param repo_tag: the repo tag, default is "latest"
:param registry: the docker registry url, default will use index.docker.io
'''
# Get full image manifest, using version 2.0 of Docker Registry API
if self.manifest is None:
if self.repo_name is not None and self.namespace is not None:
self.manifest = self.get_manifest()
else:
bot.error("No namespace and repo name OR manifest provided, exiting.")
sys.exit(1)
digests = read_digests(self.manifest)
return digests
def update_manifests(self):
'''update manifests ensures that each of a version1 and version2
manifest are present
'''
bot.debug('Updating manifests.')
if self.repo_name is None:
bot.error("Insufficient metadata to get manifest.")
sys.exit(1)
# Get full image manifest, using version 2.0 of Docker Registry API
if self.manifest is None:
bot.debug('MANIFEST (Primary): not found, making initial call.')
self.manifest = self.get_manifest()
# This is the primary manifest schema version, determines if we
# need to reverse layers
self.schemaVersion = self.manifest['schemaVersion']
if self.schemaVersion == 1:
self.reverseLayers = True
if self.manifestv1 is None:
bot.debug('MANIFEST (Metadata): not found, making initial call.')
self.manifestv1 = self.get_manifest(old_version=True)
# https://docs.docker.com/registry/spec/manifest-v2-2/#manifest-list
if "manifests" in self.manifest:
for entry in self.manifest['manifests']:
if entry['platform']['architecture'] == DOCKER_ARCHITECTURE:
if entry['platform']['os'] == DOCKER_OS:
digest = entry['digest']
bot.debug('Image manifest version 2.2 list found.')
bot.debug('Obtaining architecture: %s, OS: %s'
% (DOCKER_ARCHITECTURE, DOCKER_OS))
# Obtain specific os, architecture
self.manifest = self.get_manifest(version=digest)
break
# If we didn't get a new manifest, fall back to version 1
if "manifests" in self.manifest:
self.manifest = self.manifestv1
def update_manifests(self):
'''update manifests ensures that each of a version1 and version2
manifest are present
'''
bot.debug('Updating manifests.')
if self.repo_name is None:
bot.error("Insufficient metadata to get manifest.")
sys.exit(1)
# Get full image manifest, using version 2.0 of Docker Registry API
if self.manifest is None:
bot.debug('MANIFEST (Primary): not found, making initial call.')
self.manifest = self.get_manifest()
# This is the primary manifest schema version, determines if we
# need to reverse layers
self.schemaVersion = self.manifest['schemaVersion']
if self.schemaVersion == 1:
self.reverseLayers = True
if self.manifestv1 is None:
bot.debug('MANIFEST (Metadata): not found, making initial call.')
self.manifestv1 = self.get_manifest(old_version=True)
# https://docs.docker.com/registry/spec/manifest-v2-2/#manifest-list
if "manifests" in self.manifest:
for entry in self.manifest['manifests']:
if entry['platform']['architecture'] == DOCKER_ARCHITECTURE:
if entry['platform']['os'] == DOCKER_OS:
digest = entry['digest']
bot.debug('Image manifest version 2.2 list found.')
bot.debug('Obtaining architecture: %s, OS: %s' %
(DOCKER_ARCHITECTURE, DOCKER_OS))
# Obtain specific os, architecture
self.manifest = self.get_manifest(version=digest)
break
# If we didn't get a new manifest, fall back to version 1
if "manifests" in self.manifest:
self.manifest = self.manifestv1
def run_command(cmd, env=None, quiet=False):
'''run_command uses subprocess to send a command to the terminal.
:param cmd: the command to send, should be a list for subprocess
:param env: an optional environment to include, a dictionary of key/values
'''
try:
if quiet is False:
bot.verbose2("Running command %s with subprocess" % " ".join(cmd))
if env is None:
process = subprocess.Popen(cmd, stdout=subprocess.PIPE)
else:
process = subprocess.Popen(cmd, stdout=subprocess.PIPE, env=env)
except OSError as error:
bot.error("Error with subprocess: %s, returning None" % error)
return None
output = process.communicate()[0]
if process.returncode != 0:
return None
return output
def main():
parser = get_parser()
try:
(args, options) = parser.parse_args()
except Exception:
sys.exit(0)
if args.key is not None and args.file is not None:
if args.value is not None:
value = ADD(key=args.key,
value=args.value,
jsonfile=args.file,
force=args.force,
quiet=args.quiet)
else:
bot.error("--key and --file and --value must be defined for ADD.")
sys.exit(1)
def run_command(cmd, env=None, quiet=False):
'''run_command uses subprocess to send a command to the terminal.
:param cmd: the command to send, should be a list for subprocess
:param env: an optional environment to include, a dictionary of key/values
'''
try:
if quiet is False:
bot.verbose2("Running command %s with subprocess" % " ".join(cmd))
if env is None:
process = subprocess.Popen(cmd, stdout=subprocess.PIPE)
else:
process = subprocess.Popen(cmd, stdout=subprocess.PIPE, env=env)
except OSError as error:
bot.error("Error with subprocess: %s, returning None" % error)
return None
output = process.communicate()[0]
if process.returncode != 0:
return None
return output
def getenv(variable_key,required=False,default=None,silent=False):
'''getenv will attempt to get an environment variable. If the variable
is not found, None is returned.
:param variable_key: the variable name
:param required: exit with error if not found
:param silent: Do not print debugging information for variable
'''
variable = os.environ.get(variable_key, default)
if variable == None and required:
bot.error("Cannot find environment variable %s, exiting." %variable_key)
sys.exit(1)
if silent:
bot.verbose2("%s found" %variable_key)
else:
if variable is not None:
bot.verbose2("%s found as %s" %(variable_key,variable))
else:
bot.verbose2("%s not defined (None)" %variable_key)
return variable
def DELETE(key,jsonfile):
'''DELETE will remove a key from a json file
'''
key = format_keyname(key)
bot.debug("DELETE %s from %s" %(key,jsonfile))
if not os.path.exists(jsonfile):
bot.error("Cannot find %s, exiting." %jsonfile)
sys.exit(1)
contents = read_json(jsonfile)
if key in contents:
del contents[key]
if len(contents) > 0:
write_json(contents,jsonfile)
else:
bot.debug('%s is empty, deleting.' %jsonfile)
os.remove(jsonfile)
return True
else:
bot.debug('Warning, %s not found in %s' %(key,jsonfile))
return False
def main():
'''this function will run the main size functions and call shub clients
'''
container = getenv("SINGULARITY_CONTAINER",required=True)
image_uri = get_image_uri(container)
container = remove_image_uri(container)
from defaults import LAYERFILE
##############################################################################
# Singularity Hub ############################################################
##############################################################################
if image_uri == "shub://":
bot.debug("\n*** STARTING SINGULARITY HUB SIZE PYTHON ****")
from shub.main import SIZE
SIZE(image=container,
contentfile=LAYERFILE)
elif image_uri == "docker://":
from sutils import basic_auth_header
from docker.main import SIZE
bot.debug("Docker sizes will be written to: %s" %LAYERFILE)
username = getenv("SINGULARITY_DOCKER_USERNAME")
password = getenv("SINGULARITY_DOCKER_PASSWORD",silent=True)
auth = None
if username is not None and password is not None:
auth = basic_auth_header(username, password)
SIZE(image=container,
auth=auth,
contentfile=LAYERFILE)
else:
bot.error("uri %s is not a currently supported uri for size. Exiting." %image_uri)
sys.exit(1)
def update_token(self, response=None, auth=None):
'''update_token uses HTTP basic authentication to get a token for
Docker registry API V2 operations. We get here if a 401 is
returned for a request.
https://docs.docker.com/registry/spec/auth/token/
'''
if self.token_url is None:
if response is None:
response = self.get_tags(return_response=True)
if not isinstance(response, HTTPError):
bot.verbose3('Response on obtaining token is None.')
return None
not_asking_auth = "Www-Authenticate" not in response.headers
if response.code != 401 or not_asking_auth:
bot.error("Authentication error, exiting.")
sys.exit(1)
challenge = response.headers["Www-Authenticate"]
regexp = '^Bearer\s+realm="(.+)",service="(.+)",scope="(.+)",?'
match = re.match(regexp, challenge)
if not match:
bot.error("Unrecognized authentication challenge, exiting.")
sys.exit(1)
realm = match.group(1)
service = match.group(2)
scope = match.group(3).split(',')[0]
self.token_url = ("%s?service=%s&expires_in=9000&scope=%s" %
(realm, service, scope))
headers = dict()
# First priority comes to auth supplied directly to function
if auth is not None:
headers.update(auth)
# Second priority is default if supplied at init
elif self.auth is not None:
headers.update(self.auth)
response = self.get(self.token_url,
default_headers=False,
headers=headers)
try:
token = json.loads(response)["token"]
token = {"Authorization": "Bearer %s" % token}
self.token = token
self.update_headers(token)
except Exception:
msg = "Error getting token for repository "
msg += "%s/%s, exiting." % (self.namespace, self.repo_name)
bot.error(msg)
sys.exit(1)
def get_manifest(self):
'''get_image will return a json object with image metadata
based on a unique id.
Parameters
==========
:param image: the image name, either an id
or a repo name, tag, etc.
Returns
=======
manifest: a json manifest from the registry
'''
# make sure we have a complete url
registry = add_http(self.image['registry'])
base = "%s/api/container/%s/%s:%s" % (registry,
self.image['namespace'],
self.image['repo_name'],
self.image['repo_tag'])
# ------------------------------------------------------
# If we need to authenticate, will do it here
# ------------------------------------------------------
# If the Hub returns 404, the image name is likely wrong
response = self.get(base, return_response=True)
if response.code == 404:
msg = "Cannot find image."
msg += " Is your capitalization correct?"
bot.error(msg)
sys.exit(1)
try:
response = response.read().decode('utf-8')
response = json.loads(response)
except Exception:
print("Error getting image manifest using url %s" % base)
sys.exit(1)
return response
def get_manifest(self):
'''get_image will return a json object with image metadata
based on a unique id.
Parameters
==========
:param image: the image name, either an id
or a repo name, tag, etc.
Returns
=======
manifest: a json manifest from the registry
'''
# make sure we have a complete url
registry = add_http(self.image['registry'])
base = "%s/api/container/%s/%s:%s" % (registry,
self.image['namespace'],
self.image['repo_name'],
self.image['repo_tag'])
# ------------------------------------------------------
# If we need to authenticate, will do it here
# ------------------------------------------------------
# If the Hub returns 404, the image name is likely wrong
response = self.get(base, return_response=True)
if response.code == 404:
msg = "Cannot find image."
msg += " Is your capitalization correct?"
bot.error(msg)
sys.exit(1)
try:
response = response.read().decode('utf-8')
response = json.loads(response)
except Exception:
print("Error getting image manifest using url %s" % base)
sys.exit(1)
return response
def verify_layer(targz):
'''check that a downloaded layer's sha256 checksum is OK
correct checksum is in the filename:
sha256:7d460157dea423c1e16c544fecad995439e12dd50c8db4a8e134fa245cd1846e.tar.gz
'''
targz_basename = os.path.basename(targz)
bot.debug("Verifying checksum for layer: %s" % targz_basename)
if targz_basename[:6] != 'sha256':
bot.warning(
"Unknown hash function for layer (%s) - will not checksum" %
targz_basename[:5])
return True
expected = targz_basename[7:71]
sha256 = hashlib.sha256()
try:
with open(targz, 'rb') as f:
for block in iter(lambda: f.read(1048576), b''):
sha256.update(block)
except Exception as e:
bot.error("Error computing checksum for layer (%s) - %s" %
(targz_basename, str(e)))
return False
computed = sha256.hexdigest()
bot.debug("Computed checksum %s, expected checksum %s" %
(computed, expected))
if computed != expected:
bot.error("Downloaded layer %s does not match checksum" %
targz_basename)
return False
return True
def update_token(self, response=None, auth=None):
'''update_token uses HTTP basic authentication to get a token for
Docker registry API V2 operations. We get here if a 401 is
returned for a request.
https://docs.docker.com/registry/spec/auth/token/
'''
if self.token_url is None:
if response is None:
response = self.get_tags(return_response=True)
if not isinstance(response, HTTPError):
bot.verbose3('Response on obtaining token is None.')
return None
not_asking_auth = "Www-Authenticate" not in response.headers
if response.code != 401 or not_asking_auth:
bot.error("Authentication error, exiting.")
sys.exit(1)
challenge = response.headers["Www-Authenticate"]
regexp = '^Bearer\s+realm="(.+)",service="(.+)",scope="(.+)",?'
match = re.match(regexp, challenge)
if not match:
bot.error("Unrecognized authentication challenge, exiting.")
sys.exit(1)
realm = match.group(1)
service = match.group(2)
scope = match.group(3).split(',')[0]
self.token_url = ("%s?service=%s&expires_in=9000&scope=%s"
% (realm, service, scope))
headers = dict()
# First priority comes to auth supplied directly to function
if auth is not None:
headers.update(auth)
# Second priority is default if supplied at init
elif self.auth is not None:
headers.update(self.auth)
response = self.get(self.token_url,
default_headers=False,
headers=headers)
try:
token = json.loads(response)["token"]
token = {"Authorization": "Bearer %s" % token}
self.token = token
self.update_headers(token)
except Exception:
bot.error("Error getting token for repository %s, exiting."
% self.repo_name)
sys.exit(1)
def ADD(key,value,jsonfile,force=False):
'''ADD will write or update a key in a json file
'''
key = format_keyname(key)
bot.debug("Adding label: '%s' = '%s'" %(key, value))
bot.debug("ADD %s from %s" %(key,jsonfile))
if os.path.exists(jsonfile):
contents = read_json(jsonfile)
if key in contents:
bot.debug('Warning, %s is already set. Overwrite is set to %s' %(key,force))
if force == True:
contents[key] = value
else:
bot.error('%s found in %s and overwrite set to %s.' %(key,jsonfile,force))
sys.exit(1)
else:
contents[key] = value
else:
contents = {key:value}
bot.debug('%s is %s' %(key,value))
write_json(contents,jsonfile)
return value
def get_images(self):
'''get_images is a wrapper for get_manifest, but it additionally parses the repo_name and tag's
images and returns the complete ids
:param repo_name: the name of the repo, eg "ubuntu"
:param namespace: the namespace for the image, default is "library"
:param repo_tag: the repo tag, default is "latest"
:param registry: the docker registry url, default will use index.docker.io
'''
# Get full image manifest, using version 2.0 of Docker Registry API
if self.manifest is None:
if self.repo_name is not None and self.namespace is not None:
self.manifest = self.get_manifest()
else:
bot.error(
"No namespace and repo name OR manifest provided, exiting."
)
sys.exit(1)
digests = read_digests(self.manifest)
return digests
def getenv(variable_key, required=False, default=None, silent=False):
'''getenv will attempt to get an environment variable. If the
variable is not found, None is returned.
:param variable_key: the variable name
:param required: exit with error if not found
:param silent: Do not print debugging information for variable
'''
variable = os.environ.get(variable_key, default)
if variable is None and required:
bot.error("Cannot find environment variable %s, exiting."
% (variable_key))
sys.exit(1)
if silent:
bot.verbose2("%s found" % (variable_key))
else:
if variable is not None:
bot.verbose2("%s found as %s" % (variable_key, variable))
else:
bot.verbose2("%s not defined (None)" % (variable_key))
return variable
def main():
'''parse configuration options and produce
configuration output file
'''
bot.info("\n*** STARTING PYTHON CONFIGURATION HELPER ****")
parser = get_parser()
try:
(args, options) = parser.parse_args()
except Exception:
bot.error("Input args to %s improperly set, exiting."
% os.path.abspath(__file__))
parser.print_help()
sys.exit(1)
# Check for required args
[check_required(parser, arg) for arg in [args.defaults,
args.infile,
args.outfile]]
# Run the configuration
configure(args)