def test_headers(self):
'''test_add_http ensures that http is added to a url
'''
print("Testing utils header functions...")
from utils import parse_headers
from utils import basic_auth_header
# If we don't give headers, and no default, should get {}
print("Case 1: Don't give headers, return empty dictionary")
empty_dict = parse_headers(default_header=False)
self.assertEqual(empty_dict,dict())
# If we ask for default, should get something back
print("Case 2: ask for default headers...")
headers = parse_headers(default_header=True)
for field in ["Accept","Content-Type"]:
self.assertTrue(field in headers)
# Can we add a header?
print("Case 3: add a custom header")
new_header = {"cookies":"nom"}
headers = parse_headers(default_header=True,
headers=new_header)
for field in ["Accept","Content-Type","cookies"]:
self.assertTrue(field in headers)
# Basic auth header
print("Case 4: basic_auth_header - ask for custom authentication header")
auth = basic_auth_header(username='******',
password='******')
self.assertEqual(auth['Authorization'],
'Basic dmFuZXNzYTpwYW5jYWtlcw==')
def test_headers(self):
'''test_add_http ensures that http is added to a url
'''
print("Testing utils header functions...")
from utils import parse_headers
from utils import basic_auth_header
# If we don't give headers, and no default, should get {}
print("Case 1: Don't give headers, return empty dictionary")
empty_dict = parse_headers(default_header=False)
self.assertEqual(empty_dict, dict())
# If we ask for default, should get something back
print("Case 2: ask for default headers...")
headers = parse_headers(default_header=True)
for field in ["Accept", "Content-Type"]:
self.assertTrue(field in headers)
# Can we add a header?
print("Case 3: add a custom header")
new_header = {"cookies": "nom"}
headers = parse_headers(default_header=True, headers=new_header)
for field in ["Accept", "Content-Type", "cookies"]:
self.assertTrue(field in headers)
# Basic auth header
print(
"Case 4: basic_auth_header - ask for custom authentication header")
auth = basic_auth_header(username='******', password='******')
self.assertEqual(auth['Authorization'],
'Basic dmFuZXNzYTpwYW5jYWtlcw==')
def run(args):
# Find root filesystem location
if args.rootfs != None:
singularity_rootfs = args.rootfs
else:
singularity_rootfs = os.environ.get("SINGULARITY_ROOTFS", None)
if singularity_rootfs == None and args.shub == None:
logger.error(
"root file system not specified OR defined as environmental variable, exiting!"
)
sys.exit(1)
if singularity_rootfs != None:
logger.info("Root file system defined as %s", singularity_rootfs)
# Does the registry require authentication?
auth = None
if args.username is not None and args.password is not None:
auth = basic_auth_header(args.username, args.password)
logger.info("Username for registry authentication: %s", args.username)
# Does the user want to download a Singularity image?
if args.shub != None:
image = args.shub
manifest = get_shub_manifest(image)
if args.pull_folder == None:
cache_base = get_cache(subfolder="shub",
disable_cache=args.disable_cache)
else:
cache_base = args.pull_folder
# The image name is the md5 hash, download if it's not there
image_name = get_image_name(manifest)
image_file = "%s/%s" % (cache_base, image_name)
if not os.path.exists(image_file):
image_file = download_image(manifest=manifest,
download_folder=cache_base)
else:
print("Image already exists at %s, skipping download." %
image_file)
logger.info("Singularity Hub Image Download: %s", image_file)
# If singularity_rootfs is provided, write metadata to it
if singularity_rootfs != None:
logger.debug(
"Writing SINGULARITY_RUNDIR and SINGULARITY_IMAGE to %s",
singularity_rootfs)
write_file("%s/SINGULARITY_RUNDIR" % singularity_rootfs,
os.path.dirname(image_file))
write_file("%s/SINGULARITY_IMAGE" % singularity_rootfs, image_file)
# Do we have a docker image specified?
elif args.docker != None:
# Does the user want to override default Entrypoint and use CMD as runscript?
includecmd = args.includecmd
logger.info("Including Docker command as Runscript? %s", includecmd)
image = args.docker
logger.info("Docker image: %s", image)
# Input Parsing ----------------------------
# Parse image name, repo name, and namespace
image = parse_image_uri(image=image, uri="docker://")
namespace = image['namespace']
repo_name = image['repo_name']
repo_tag = image['repo_tag']
# Tell the user the namespace, repo name and tag
logger.info("Docker image path: %s/%s:%s", namespace, repo_name,
repo_tag)
# IMAGE METADATA -------------------------------------------
# Use Docker Registry API (version 2.0) to get images ids, manifest
# Get an image manifest - has image ids to parse, and will be
# used later to get Cmd
manifest = get_manifest(repo_name=repo_name,
namespace=namespace,
repo_tag=repo_tag,
registry=args.registry,
auth=auth)
# Get images from manifest using version 2.0 of Docker Registry API
images = get_images(manifest=manifest)
# DOWNLOAD LAYERS -------------------------------------------
# Each is a .tar.gz file, obtained from registry with curl
# Get the cache (or temporary one) for docker
cache_base = get_cache(subfolder="docker",
disable_cache=args.disable_cache)
layers = []
for image_id in images:
# Download the layer, if we don't have it
targz = "%s/%s.tar.gz" % (cache_base, image_id)
if not os.path.exists(targz):
targz = get_layer(image_id=image_id,
namespace=namespace,
repo_name=repo_name,
download_folder=cache_base,
registry=args.registry,
auth=auth)
layers.append(targz) # in case we want a list at the end
# Extract image and remove tar
output = extract_tar(targz, singularity_rootfs)
if output is None:
logger.error("Error extracting image: %s", targz)
sys.exit(1)
if args.disable_cache == True:
os.remove(targz)
# If the user wants to include the CMD as runscript, generate it here
if includecmd == True:
spec = "Cmd"
else:
spec = "Entrypoint"
cmd = get_config(manifest, spec=spec)
# Only add runscript if command is defined
if cmd != None:
print("Adding Docker %s as Singularity runscript..." %
(spec.upper()))
print(cmd)
runscript = create_runscript(cmd=cmd, base_dir=singularity_rootfs)
# When we finish, clean up images
if args.disable_cache == True:
shutil.rmtree(cache_base)
logger.info("*** FINISHING DOCKER BOOTSTRAP PYTHON PORTION ****\n")
def run(args):
# Find root filesystem location
if args.rootfs != None:
singularity_rootfs = args.rootfs
else:
singularity_rootfs = os.environ.get("SINGULARITY_ROOTFS", None)
if singularity_rootfs == None and args.shub == None:
logger.error("root file system not specified OR defined as environmental variable, exiting!")
sys.exit(1)
if singularity_rootfs != None:
logger.info("Root file system defined as %s", singularity_rootfs)
# Does the registry require authentication?
auth = None
if args.username is not None and args.password is not None:
auth = basic_auth_header(args.username, args.password)
logger.info("Username for registry authentication: %s", args.username)
# Does the user want to download a Singularity image?
if args.shub != None:
image_id = int(args.shub)
manifest = get_shub_manifest(image_id)
cache_base = get_cache(subfolder="shub",
disable_cache = args.disable_cache)
# The image name is the md5 hash, download if it's not there
image_name = get_image_name(manifest)
image_file = "%s/%s" %(cache_base,image_name)
if not os.path.exists(image_file):
image_file = download_image(manifest=manifest,
download_folder=cache_base)
else:
print("Image already exists at %s, skipping download." %image_file)
logger.info("Singularity Hub Image Download: %s", image_file)
# If singularity_rootfs is provided, write metadata to it
if singularity_rootfs != None:
logger.debug("Writing SINGULARITY_RUNDIR and SINGULARITY_IMAGE to %s",singularity_rootfs)
write_file("%s/SINGULARITY_RUNDIR" %singularity_rootfs, os.path.dirname(image_file))
write_file("%s/SINGULARITY_IMAGE" %singularity_rootfs, image_file)
# Do we have a docker image specified?
elif args.docker != None:
# Does the user want to override default Entrypoint and use CMD as runscript?
includecmd = args.includecmd
logger.info("Including Docker command as Runscript? %s", includecmd)
image = args.docker
logger.info("Docker image: %s", image)
# Input Parsing ----------------------------
# Parse image name, repo name, and namespace
# First split the docker image name by /
image = image.split('/')
# If there are two parts, we have namespace with repo (and maybe tab)
if len(image) == 2:
namespace = image[0]
image = image[1]
# Otherwise, we must be using library namespace
else:
namespace = "library"
image = image[0]
# Now split the docker image name by :
image = image.split(':')
if len(image) == 2:
repo_name = image[0]
repo_tag = image[1]
# Otherwise, assume latest of an image
else:
repo_name = image[0]
repo_tag = "latest"
# Tell the user the namespace, repo name and tag
logger.info("Docker image path: %s/%s:%s", namespace,repo_name,repo_tag)
# IMAGE METADATA -------------------------------------------
# Use Docker Registry API (version 2.0) to get images ids, manifest
# Get an image manifest - has image ids to parse, and will be
# used later to get Cmd
manifest = get_manifest(repo_name=repo_name,
namespace=namespace,
repo_tag=repo_tag,
registry=args.registry,
auth=auth)
# Get images from manifest using version 2.0 of Docker Registry API
images = get_images(repo_name=repo_name,
namespace=namespace,
registry=args.registry,
auth=auth)
# DOWNLOAD LAYERS -------------------------------------------
# Each is a .tar.gz file, obtained from registry with curl
# Get the cache (or temporary one) for docker
cache_base = get_cache(subfolder="docker",
disable_cache = args.disable_cache)
layers = []
for image_id in images:
# Download the layer, if we don't have it
targz = "%s/%s.tar.gz" %(cache_base,image_id)
if not os.path.exists(targz):
targz = get_layer(image_id=image_id,
namespace=namespace,
repo_name=repo_name,
download_folder=cache_base,
registry=args.registry,
auth=auth)
layers.append(targz) # in case we want a list at the end
# Extract image and remove tar
output = extract_tar(targz,singularity_rootfs)
if output is None:
logger.error("Error extracting image: %s", targz)
sys.exit(1)
if args.disable_cache == True:
os.remove(targz)
# If the user wants to include the CMD as runscript, generate it here
if includecmd == True:
spec="Cmd"
else:
spec="Entrypoint"
cmd = get_config(manifest,spec=spec)
# Only add runscript if command is defined
if cmd != None:
print("Adding Docker %s as Singularity runscript..." %(spec.upper()))
print(cmd)
runscript = create_runscript(cmd=cmd,
base_dir=singularity_rootfs)
# When we finish, clean up images
if args.disable_cache == True:
shutil.rmtree(cache_base)
logger.info("*** FINISHING DOCKER BOOTSTRAP PYTHON PORTION ****\n")
def main():
logger.info("\n*** STARTING DOCKER BOOTSTRAP PYTHON PORTION ****")
parser = argparse.ArgumentParser(
description="bootstrap Docker images for Singularity containers")
# Name of the docker image, required
parser.add_argument(
"--docker",
dest='docker',
help=
"name of Docker image to bootstrap, in format library/ubuntu:latest",
type=str,
default=None)
# root file system of singularity image
parser.add_argument("--rootfs",
dest='rootfs',
help="the path for the root filesystem to extract to",
type=str,
default=None)
# Docker registry (default is registry-1.docker.io
parser.add_argument(
"--registry",
dest='registry',
help="the registry path to use, to replace registry-1.docker.io",
type=str,
default=None)
# Flag to add the Docker CMD as a runscript
parser.add_argument(
"--cmd",
dest='includecmd',
action="store_true",
help=
"boolean to specify that CMD should be used instead of ENTRYPOINT as the runscript.",
default=False)
parser.add_argument("--username",
dest='username',
help="username for registry authentication",
default=None)
parser.add_argument("--password",
dest='password',
help="password for registry authentication",
default=None)
# Flag to disable cache
parser.add_argument("--no-cache",
dest='disable_cache',
action="store_true",
help="boolean to specify disabling the cache.",
default=False)
try:
args = parser.parse_args()
except:
logger.error("Input args to %s improperly set, exiting.",
os.path.abspath(__file__))
parser.print_help()
sys.exit(0)
# Find root filesystem location
if args.rootfs != None:
singularity_rootfs = args.rootfs
logger.info("Root file system defined by command line variable as %s",
singularity_rootfs)
else:
singularity_rootfs = os.environ.get("SINGULARITY_ROOTFS", None)
if singularity_rootfs == None:
logger.error(
"root file system not specified OR defined as environmental variable, exiting!"
)
sys.exit(1)
logger.info("Root file system defined by env variable as %s",
singularity_rootfs)
# Does the registry require authentication?
auth = None
if args.username is not None and args.password is not None:
auth = basic_auth_header(args.username, args.password)
logger.info("Username for registry authentication: %s", args.username)
# Does the user want to override default Entrypoint and use CMD as runscript?
includecmd = args.includecmd
logger.info("Including Docker command as Runscript? %s", includecmd)
# Do we have a docker image specified?
if args.docker != None:
image = args.docker
logger.info("Docker image: %s", image)
# INPUT PARSING -------------------------------------------
# Parse image name, repo name, and namespace
# First split the docker image name by /
image = image.split('/')
# If there are two parts, we have namespace with repo (and maybe tab)
if len(image) == 2:
namespace = image[0]
image = image[1]
# Otherwise, we must be using library namespace
else:
namespace = "library"
image = image[0]
# Now split the docker image name by :
image = image.split(':')
if len(image) == 2:
repo_name = image[0]
repo_tag = image[1]
# Otherwise, assume latest of an image
else:
repo_name = image[0]
repo_tag = "latest"
# Tell the user the namespace, repo name and tag
logger.info("Docker image path: %s/%s:%s", namespace, repo_name,
repo_tag)
# IMAGE METADATA -------------------------------------------
# Use Docker Registry API (version 2.0) to get images ids, manifest
# Get an image manifest - has image ids to parse, and will be
# used later to get Cmd
manifest = get_manifest(repo_name=repo_name,
namespace=namespace,
repo_tag=repo_tag,
registry=args.registry,
auth=auth)
# Get images from manifest using version 2.0 of Docker Registry API
images = get_images(manifest=manifest,
registry=args.registry,
auth=auth)
# DOWNLOAD LAYERS -------------------------------------------
# Each is a .tar.gz file, obtained from registry with curl
# Get the cache (or temporary one) for docker
cache_base = get_cache(subfolder="docker",
disable_cache=args.disable_cache)
layers = []
for image_id in images:
# Download the layer, if we don't have it
targz = "%s/%s.tar.gz" % (cache_base, image_id)
if not os.path.exists(targz):
targz = get_layer(image_id=image_id,
namespace=namespace,
repo_name=repo_name,
download_folder=cache_base,
registry=args.registry,
auth=auth)
layers.append(targz) # in case we want a list at the end
# @chrisfilo suggestion to try compiling into one tar.gz
# Extract image and remove tar
extract_tar(targz, singularity_rootfs)
if args.disable_cache == True:
os.remove(targz)
# If the user wants to include the CMD as runscript, generate it here
if includecmd == True:
spec = "Cmd"
else:
spec = "Entrypoint"
cmd = get_config(manifest, spec=spec)
# Only add runscript if command is defined
if cmd != None:
print("Adding Docker %s as Singularity runscript..." %
(spec.upper()))
print(cmd)
runscript = create_runscript(cmd=cmd, base_dir=singularity_rootfs)
# When we finish, clean up images
if args.disable_cache == True:
shutil.rmtree(cache_base)
logger.info("*** FINISHING DOCKER BOOTSTRAP PYTHON PORTION ****\n")
