def download_image(self,manifest,download_folder=None,extract=True):
'''download_image will download a singularity image from singularity
hub to a download_folder, named based on the image version (commit id)
:param manifest: the manifest obtained with get_manifest
:param download_folder: the folder to download to, if None, will be pwd
:param extract: if True, will extract image to .img and return that.
'''
image_file = get_image_name(manifest)
if not bot.is_quiet():
print("Found image %s:%s" %(manifest['name'],
manifest['branch']))
print("Downloading image... %s" %(image_file))
if download_folder is not None:
image_file = "%s/%s" %(download_folder,image_file)
url = manifest['image']
# Download image file atomically, streaming
image_file = self.download_atomically(url=url,
file_name=image_file,
show_progress=True)
if extract == True:
if not bot.is_quiet():
print("Decompressing %s" %image_file)
os.system('gzip -d -f %s' %(image_file))
image_file = image_file.replace('.gz','')
return image_file
def download_image(self, manifest, download_folder=None, extract=True):
'''download_image will download a singularity image from singularity
hub to a download_folder, named based on the image version (commit id)
:param manifest: the manifest obtained with get_manifest
:param download_folder: the folder to download to, if None, will be pwd
:param extract: if True, will extract image to .img and return that.
'''
image_file = get_image_name(manifest)
if not bot.is_quiet():
print("Found image %s:%s" % (manifest['name'], manifest['branch']))
print("Downloading image... %s" % (image_file))
if download_folder is not None:
image_file = "%s/%s" % (download_folder, image_file)
url = manifest['image']
# Download image file atomically, streaming
image_file = self.download_atomically(url=url,
file_name=image_file,
show_progress=True)
if extract == True:
if not bot.is_quiet():
print("Decompressing %s" % image_file)
os.system('gzip -d -f %s' % (image_file))
image_file = image_file.replace('.gz', '')
return image_file
def download_image(self,
manifest,
image_name=None,
download_folder=None,
extract=True):
'''
download_image will download a singularity image from singularity
hub to a download_folder, named based on the image version (commit id)
Parameters
==========
:param manifest: the manifest obtained with get_manifest
:param download_folder: the folder to download to, if None, will be pwd
:param extract: if True, will extract image to .img and return that.
Returns
=======
image_file: the full path to the downloaded image
'''
if image_name is None:
image_name = get_image_name(manifest)
if not bot.is_quiet():
print("Found image %s:%s" % (manifest['name'], manifest['branch']))
print("Downloading image... %s" % image_name)
url = manifest['image']
if url is None:
bot.error("%s is not ready for download" % image_name)
bot.error("please try when build completed or specify tag.")
sys.exit(1)
if not image_name.endswith('.gz'):
image_name = "%s.gz" % image_name
if download_folder is not None:
image_name = "%s/%s" % (download_folder, image_name)
# Download image file atomically, streaming
image_file = self.download_atomically(url=url,
file_name=image_name,
show_progress=True)
if extract is True:
if not bot.is_quiet():
print("Decompressing %s" % image_file)
output = run_command(['gzip', '-d', '-f', image_file])
image_file = image_file.replace('.gz', '')
# Any error in extraction (return code not 0) will return None
if output is None:
bot.error('Error extracting image, cleaning up.')
clean_up([image_file, "%s.gz" % image_file])
return image_file
def PULL(image, download_folder=None, layerfile=None):
'''PULL will retrieve a Singularity Hub image and
download to the local file system, to the variable
specified by SINGULARITY_PULLFOLDER.
:param image: the singularity hub image name
:param download folder: the folder to pull the image to.
:param layerfile: if defined, write pulled image to file
'''
client = SingularityApiConnection(image=image)
manifest = client.get_manifest()
if download_folder is None:
cache_base = get_cache(subfolder="shub")
else:
cache_base = os.path.abspath(download_folder)
bot.debug("Pull folder set to %s" % cache_base)
# The image name is the md5 hash, download if it's not there
image_name = get_image_name(manifest)
# Did the user specify an absolute path?
custom_folder = os.path.dirname(image_name)
if custom_folder not in [None, ""]:
cache_base = custom_folder
image_name = os.path.basename(image_name)
image_file = "%s/%s" % (cache_base, image_name)
bot.debug('Pulling to %s' % image_file)
if not os.path.exists(image_file):
image_file = client.download_image(manifest=manifest,
download_folder=cache_base,
image_name=image_name)
else:
if not bot.is_quiet(): # not --quiet
print("Image already exists at %s, skipping download" % image_file)
if not bot.is_quiet(): # not --quiet
print("Singularity Hub Image Download: %s" % image_file)
manifest = {
'image_file': image_file,
'manifest': manifest,
'cache_base': cache_base,
'image': image
}
if layerfile is not None:
bot.debug("Writing Singularity Hub image path to %s" % layerfile)
write_file(layerfile, image_file, mode="w")
return manifest
def PULL(image, download_folder=None, layerfile=None):
'''PULL will retrieve a Singularity Hub image and
download to the local file system, to the variable
specified by SINGULARITY_PULLFOLDER.
:param image: the singularity hub image name
:param download folder: the folder to pull the image to.
:param layerfile: if defined, write pulled image to file
'''
client = SingularityApiConnection(image=image)
manifest = client.get_manifest()
if download_folder is None:
cache_base = get_cache(subfolder="shub")
else:
cache_base = os.path.abspath(download_folder)
bot.debug("Pull folder set to %s" % cache_base)
# The image name is the md5 hash, download if it's not there
image_name = get_image_name(manifest)
# Did the user specify an absolute path?
custom_folder = os.path.dirname(image_name)
if custom_folder not in [None, ""]:
cache_base = custom_folder
image_name = os.path.basename(image_name)
image_file = "%s/%s" % (cache_base, image_name)
bot.debug('Pulling to %s' % image_file)
if not os.path.exists(image_file):
image_file = client.download_image(manifest=manifest,
download_folder=cache_base)
else:
if not bot.is_quiet(): # not --quiet
print("Image already exists at %s, skipping download" % image_file)
if not bot.is_quiet(): # not --quiet
print("Singularity Hub Image Download: %s" % image_file)
manifest = {'image_file': image_file,
'manifest': manifest,
'cache_base': cache_base,
'image': image}
if layerfile is not None:
bot.debug("Writing Singularity Hub image path to %s" % layerfile)
write_file(layerfile, image_file, mode="w")
return manifest
def get_image_name(manifest,extension='img.gz'):
'''get_image_name will return the image name for a manifest
:param manifest: the image manifest with 'image' as key with download link
:param use_hash: use the image hash instead of name
'''
from defaults import ( SHUB_CONTAINERNAME,
SHUB_NAMEBYCOMMIT,
SHUB_NAMEBYHASH )
# First preference goes to a custom name
if SHUB_CONTAINERNAME is not None:
for replace in [" ",".gz",".img"]:
SHUB_CONTAINERNAME = SHUB_CONTAINERNAME.replace(replace,"")
image_name = "%s.%s" %(SHUB_CONTAINERNAME,extension)
# Second preference goes to commit
elif SHUB_NAMEBYCOMMIT is not None:
image_name = "%s.%s" %(manifest['version'],extension)
elif SHUB_NAMEBYHASH is not None:
image_url = os.path.basename(unquote(manifest['image']))
image_name = re.findall(".+[.]%s" %(extension),image_url)[0]
# Default uses the image name-branch
else:
image_name = "%s-%s.%s" %(manifest['name'].replace('/','-'),
manifest['branch'].replace('/','-'),
extension)
if not bot.is_quiet():
print("Singularity Hub Image: %s" %image_name)
return image_name
def get_default_name(manifest, source="Hub"):
''' get manifest name returns the default name that is discovered
via the image manifest. This is the fallback option in the case that
the user doesn't ask to name by commit, hash, or custom name
'''
# Singularity Hub v2.0
if "tag" in manifest and "branch" in manifest:
version = "%s-%s" % (manifest["branch"], manifest["tag"])
# Singularity Registry
elif "tag" in manifest:
version = manifest["tag"]
# Singularity Hub v1.0
else:
version = manifest['branch']
# Remove slashes
version = version.replace('/', '-')
# sregistry images store collection/name separately
name = manifest['name']
if 'frozen' in manifest:
source = "Registry"
name = '%s-%s' % (manifest['collection'], name)
image_name = "%s-%s" % (name.replace('/', '-'), version)
if not bot.is_quiet():
print("Singularity %s Image: %s" % (source, image_name))
return image_name
def get_image_name(manifest, extension='img.gz'):
'''get_image_name will return the image name for a manifest
:param manifest: the image manifest with 'image' as key with download link
:param use_hash: use the image hash instead of name
'''
from defaults import (SHUB_CONTAINERNAME, SHUB_NAMEBYCOMMIT,
SHUB_NAMEBYHASH)
# First preference goes to a custom name
if SHUB_CONTAINERNAME is not None:
for replace in [" ", ".gz", ".img"]:
SHUB_CONTAINERNAME = SHUB_CONTAINERNAME.replace(replace, "")
image_name = "%s.%s" % (SHUB_CONTAINERNAME, extension)
# Second preference goes to commit
elif SHUB_NAMEBYCOMMIT is not None:
image_name = "%s.%s" % (manifest['version'], extension)
elif SHUB_NAMEBYHASH is not None:
image_url = os.path.basename(unquote(manifest['image']))
image_name = re.findall(".+[.]%s" % (extension), image_url)[0]
# Default uses the image name-branch
else:
image_name = "%s-%s.%s" % (manifest['name'].replace(
'/', '-'), manifest['branch'].replace('/', '-'), extension)
if not bot.is_quiet():
print("Singularity Hub Image: %s" % image_name)
return image_name
def get_image_name(manifest, extension="simg"):
'''return the image name for a manifest. Estimates extension from file
:param manifest: the image manifest with 'image'
as key with download link
'''
from defaults import (SHUB_CONTAINERNAME,
SHUB_NAMEBYCOMMIT,
SHUB_NAMEBYHASH)
# First preference goes to a custom name
default_naming = True
if SHUB_CONTAINERNAME is not None:
for replace in [" ", ".gz", ".img", ".simg"]:
SHUB_CONTAINERNAME = SHUB_CONTAINERNAME.replace(replace, "")
image_name = "%s.%s" % (SHUB_CONTAINERNAME, extension)
default_naming = False
# Second preference goes to commit
elif SHUB_NAMEBYCOMMIT is not None and manifest['version'] is not None:
image_name = "%s.%s" % (manifest['version'], extension)
default_naming = False
elif SHUB_NAMEBYHASH is not None:
image_url = os.path.basename(unquote(manifest['image']))
image_name = re.findall(".+[.]%s" % (extension), image_url)[0]
default_naming = False
# Default uses the image name-branch
if default_naming is True:
# Tag is derived from branch for Shub, tag from sregistry
tag_source = 'branch'
if tag_source not in manifest:
tag_source = 'tag'
# sregistry images store collection/name separately
name = manifest['name']
source = "Hub"
if 'frozen' in manifest:
source = "Registry"
name = '%s-%s' % (manifest['collection'], name)
image_name = "%s-%s.%s" % (name.replace('/', '-'),
manifest[tag_source].replace('/', '-'),
extension)
if not bot.is_quiet():
print("Singularity %s Image: %s" % (source, image_name))
return image_name
def extract_tar(archive, output_folder):
'''extract a tar archive to a specified output folder
:param archive: the archive file to extract
:param output_folder: the output folder to extract to
'''
# If extension is .tar.gz, use -xzf
args = '-xf'
if archive.endswith(".tar.gz"):
args = '-xzf'
# Just use command line, more succinct.
command = ["tar", args, archive, "-C", output_folder, "--exclude=dev/*"]
if not bot.is_quiet():
print("Extracting %s" % archive)
return run_command(command)
def extract_tar(archive,output_folder):
'''extract_tar will extract a tar archive to a specified output folder
:param archive: the archive file to extract
:param output_folder: the output folder to extract to
'''
# If extension is .tar.gz, use -xzf
args = '-xf'
if archive.endswith(".tar.gz"):
args = '-xzf'
# Just use command line, more succinct.
command = ["tar", args, archive, "-C", output_folder, "--exclude=dev/*"]
if not bot.is_quiet():
print("Extracting %s" %archive)
return run_command(command)
def get_image_name(manifest, extension="simg"):
'''return the image name for a manifest. Estimates extension from file
:param manifest: the image manifest with 'image'
as key with download link
'''
from defaults import (SHUB_CONTAINERNAME, SHUB_NAMEBYCOMMIT,
SHUB_NAMEBYHASH)
# First preference goes to a custom name
default_naming = True
if SHUB_CONTAINERNAME is not None:
for replace in [" ", ".gz", ".img", ".simg"]:
SHUB_CONTAINERNAME = SHUB_CONTAINERNAME.replace(replace, "")
image_name = "%s.%s" % (SHUB_CONTAINERNAME, extension)
default_naming = False
# Second preference goes to commit
elif SHUB_NAMEBYCOMMIT is not None:
if manifest.get("commit") is not None:
commit = manifest['commit']
elif manifest['version'] is not None:
commit = manifest['version']
image_name = "%s.%s" % (commit, extension)
default_naming = False
elif SHUB_NAMEBYHASH is not None:
image_url = os.path.basename(unquote(manifest['image']))
image_name = re.findall(".+[.]%s" % (extension), image_url)
if len(image_name) == 0:
image_name = re.findall(".+[.]img.gz", image_url)
if len(image_name) > 0:
default_naming = False
image_name = image_name[0]
# Default uses the image name-branch
if default_naming is True:
# Singularity Hub v2.0
if "tag" in manifest and "branch" in manifest:
container_name = "%s-%s" % (manifest["branch"], manifest["tag"])
# Singularity Registry
elif "tag" in manifest:
container_name = manifest["tag"]
# Singularity Hub v1.0
else:
container_name = manifest['branch']
# Remove slashes
container_name = container_name.replace('/', '-')
# sregistry images store collection/name separately
name = manifest['name']
source = "Hub"
if 'frozen' in manifest:
source = "Registry"
name = '%s-%s' % (manifest['collection'], name)
image_name = "%s-%s.%s" % (name.replace(
'/', '-'), container_name, extension)
if not bot.is_quiet():
print("Singularity %s Image: %s" % (source, image_name))
return image_name
def download_image(self,
manifest,
download_folder=None,
extract=False):
'''
download_image will download a singularity image from singularity
hub to a download_folder, named based on the image version (commit id)
Parameters
==========
:param manifest: the manifest obtained with get_manifest
:param download_folder: the folder to download to, if None, will be pwd
:param extract: if True, will extract image to .img and return that.
Returns
=======
image_file: the full path to the downloaded image
'''
from defaults import SHUB_CONTAINERNAME
# Returns just basename, no extension
image_name = get_image_name(manifest)
if not bot.is_quiet():
print("Found image %s:%s" % (manifest['name'],
manifest['branch']))
print("Downloading image... %s" % image_name)
url = manifest['image']
if url is None:
bot.error("%s is not ready for download" % image_name)
bot.error("please try when build completes or specify tag.")
sys.exit(1)
if download_folder is not None:
image_name = "%s/%s" % (download_folder, image_name)
# Download image file atomically, streaming
image_file = self.download_atomically(url=url,
file_name=image_name,
show_progress=True)
# Compressed ext3 images need extraction
image_type = get_image_format(image_file)
extension = "simg"
if image_type == "GZIP" or extract is True:
extension = "img"
if not image_file.endswith('.gz'):
os.rename(image_file, "%s.gz" % image_file)
image_file = "%s.gz" % image_file
if not bot.is_quiet():
print("Decompressing %s" % image_file)
output = run_command(['gzip', '-d', '-f', image_file])
image_file = image_file.replace('.gz', '')
# Any error in extraction (return code not 0) will return None
if output is None:
bot.error('Error extracting image, cleaning up.')
clean_up([image_file, "%s.gz" % image_file])
# If the user has provided a default name, be true to it
if SHUB_CONTAINERNAME is not None:
folder = os.path.dirname(image_file)
image_name = "%s/%s" % (folder, SHUB_CONTAINERNAME)
os.rename(image_file, image_name)
image_file = image_name
# Otherwise rename to have extension matching image type
else:
image_name = "%s.%s" % (image_file, extension)
os.rename(image_file, image_name)
image_file = image_name
return image_file