def test_write_read_files(self):
'''test_write_read_files will test the
functions write_file and read_file
'''
print("Testing utils.write_file...")
from sutils import write_file
import json
tmpfile = tempfile.mkstemp()[1]
os.remove(tmpfile)
write_file(tmpfile, "hello!")
self.assertTrue(os.path.exists(tmpfile))
print("Testing utils.read_file...")
from sutils import read_file
content = read_file(tmpfile)[0]
self.assertEqual("hello!", content)
from sutils import write_json
print("Testing utils.write_json...")
print("Case 1: Providing bad json")
bad_json = {"Wakkawakkawakka'}": [{True}, "2", 3]}
tmpfile = tempfile.mkstemp()[1]
os.remove(tmpfile)
with self.assertRaises(TypeError) as cm:
write_json(bad_json, tmpfile)
print("Case 2: Providing good json")
good_json = {"Wakkawakkawakka": [True, "2", 3]}
tmpfile = tempfile.mkstemp()[1]
os.remove(tmpfile)
write_json(good_json, tmpfile)
content = json.load(open(tmpfile, 'r'))
self.assertTrue(isinstance(content, dict))
self.assertTrue("Wakkawakkawakka" in content)
def test_write_read_files(self):
'''test_write_read_files will test the
functions write_file and read_file
'''
print("Testing utils.write_file...")
from sutils import write_file
import json
tmpfile = tempfile.mkstemp()[1]
os.remove(tmpfile)
write_file(tmpfile, "hello!")
self.assertTrue(os.path.exists(tmpfile))
print("Testing utils.read_file...")
from sutils import read_file
content = read_file(tmpfile)[0]
self.assertEqual("hello!", content)
from sutils import write_json
print("Testing utils.write_json...")
print("Case 1: Providing bad json")
bad_json = {"Wakkawakkawakka'}": [{True}, "2", 3]}
tmpfile = tempfile.mkstemp()[1]
os.remove(tmpfile)
with self.assertRaises(TypeError) as cm:
write_json(bad_json, tmpfile)
print("Case 2: Providing good json")
good_json = {"Wakkawakkawakka": [True, "2", 3]}
tmpfile = tempfile.mkstemp()[1]
os.remove(tmpfile)
write_json(good_json, tmpfile)
content = json.load(open(tmpfile, 'r'))
self.assertTrue(isinstance(content, dict))
self.assertTrue("Wakkawakkawakka" in content)
def SIZE(image,auth=None,contentfile=None):
'''size is intended to be run before an import, to return to the contentfile a list of sizes
(one per layer) corresponding with the layers that will be downloaded for image
'''
bot.debug("Starting Docker SIZE, will get size from manifest")
bot.verbose("Docker image: %s" %image)
client = DockerApiConnection(image=image,auth=auth)
size = client.get_size()
if contentfile is not None:
write_file(contentfile,str(size),mode="w")
return size
def PULL(image, download_folder=None, layerfile=None):
'''PULL will retrieve a Singularity Hub image and
download to the local file system, to the variable
specified by SINGULARITY_PULLFOLDER.
:param image: the singularity hub image name
:param download folder: the folder to pull the image to.
:param layerfile: if defined, write pulled image to file
'''
client = SingularityApiConnection(image=image)
manifest = client.get_manifest()
if download_folder is None:
cache_base = get_cache(subfolder="shub")
else:
cache_base = os.path.abspath(download_folder)
bot.debug("Pull folder set to %s" % cache_base)
# The image name is the md5 hash, download if it's not there
image_name = get_image_name(manifest)
# Did the user specify an absolute path?
custom_folder = os.path.dirname(image_name)
if custom_folder not in [None, ""]:
cache_base = custom_folder
image_name = os.path.basename(image_name)
image_file = "%s/%s" % (cache_base, image_name)
bot.debug('Pulling to %s' % image_file)
if not os.path.exists(image_file):
image_file = client.download_image(manifest=manifest,
download_folder=cache_base,
image_name=image_name)
else:
if not bot.is_quiet(): # not --quiet
print("Image already exists at %s, skipping download" % image_file)
if not bot.is_quiet(): # not --quiet
print("Singularity Hub Image Download: %s" % image_file)
manifest = {
'image_file': image_file,
'manifest': manifest,
'cache_base': cache_base,
'image': image
}
if layerfile is not None:
bot.debug("Writing Singularity Hub image path to %s" % layerfile)
write_file(layerfile, image_file, mode="w")
return manifest
def SIZE(image, contentfile=None):
'''size is intended to be run before an import, to return to the contentfile a list of sizes
(one per layer) corresponding with the layers that will be downloaded for image
'''
bot.debug("Starting Singularity Hub SIZE, will get size from manifest")
bot.debug("Singularity Hub image: %s" % image)
client = SingularityApiConnection(image=image)
manifest = client.get_manifest()
size = json.loads(manifest['metrics'].replace("'", '"'))['size']
if contentfile is not None:
write_file(contentfile, str(size), mode="w")
return size
def SIZE(image,contentfile=None):
'''size is intended to be run before an import, to return to the contentfile a list of sizes
(one per layer) corresponding with the layers that will be downloaded for image
'''
bot.debug("Starting Singularity Hub SIZE, will get size from manifest")
bot.debug("Singularity Hub image: %s" %image)
client = SingularityApiConnection(image=image)
manifest = client.get_manifest()
size = json.loads(manifest['metrics'].replace("'",'"'))['size']
if contentfile is not None:
write_file(contentfile,str(size),mode="w")
return size
def PULL(image, download_folder=None, layerfile=None):
'''PULL will retrieve a Singularity Hub image and
download to the local file system, to the variable
specified by SINGULARITY_PULLFOLDER.
:param image: the singularity hub image name
:param download folder: the folder to pull the image to.
:param layerfile: if defined, write pulled image to file
'''
client = SingularityApiConnection(image=image)
manifest = client.get_manifest()
if download_folder is None:
cache_base = get_cache(subfolder="shub")
else:
cache_base = os.path.abspath(download_folder)
bot.debug("Pull folder set to %s" % cache_base)
# The image name is the md5 hash, download if it's not there
image_name = get_image_name(manifest)
# Did the user specify an absolute path?
custom_folder = os.path.dirname(image_name)
if custom_folder not in [None, ""]:
cache_base = custom_folder
image_name = os.path.basename(image_name)
image_file = "%s/%s" % (cache_base, image_name)
bot.debug('Pulling to %s' % image_file)
if not os.path.exists(image_file):
image_file = client.download_image(manifest=manifest,
download_folder=cache_base)
else:
if not bot.is_quiet(): # not --quiet
print("Image already exists at %s, skipping download" % image_file)
if not bot.is_quiet(): # not --quiet
print("Singularity Hub Image Download: %s" % image_file)
manifest = {'image_file': image_file,
'manifest': manifest,
'cache_base': cache_base,
'image': image}
if layerfile is not None:
bot.debug("Writing Singularity Hub image path to %s" % layerfile)
write_file(layerfile, image_file, mode="w")
return manifest
def configure(args):
# Get fullpath to each file, and concurrently check that exists
defaultfile = get_fullpath(args.defaults) # ../src/lib/config_defaults.h
infile = get_fullpath(args.infile) # singularity.conf.in
# Find define statements
define_re = re.compile("#define ([A-Z_]+) (.*)")
# Read in input and default files
defaultfile = read_file(defaultfile)
data = "".join(read_file(infile))
# Lookup for values we want replaced
lookup = {'0': 'no',
'1': 'yes'}
defaults = {}
# Read in defaults to dictionary
for line in defaultfile:
match = define_re.match(line)
if match:
key, value = match.groups()
# Maintain the original default set by user
defaults[key] = value
# Use parsed value for final config
new_value = value.replace('"', '')
if new_value in lookup:
new_value = lookup[new_value]
data = data.replace("@" + key + "@", new_value)
# Write to output file
outfile = "%s.tmp" % args.outfile
write_file(outfile, data)
os.rename(outfile, args.outfile)
bot.info("*** FINISHED PYTHON CONFIGURATION HELPER ****\n")
def IMPORT(image, auth=None, layerfile=None):
'''IMPORT is the main script that will obtain docker layers,
runscript information (either entrypoint or cmd), and environment
and return a list of tarballs to extract into the image
:param auth: if needed, an authentication header (default None)
:param layerfile: The file to write layers to extract into
'''
bot.debug("Starting Docker IMPORT, includes env, runscript, and metadata.")
bot.verbose("Docker image: %s" % image)
# Does the user want to override default of using ENTRYPOINT?
if INCLUDE_CMD:
bot.verbose2("Specified Docker CMD as %runscript.")
else:
bot.verbose2("Specified Docker ENTRYPOINT as %runscript.")
# Input Parsing ----------------------------
# Parse image name, repo name, and namespace
client = DockerApiConnection(image=image, auth=auth)
docker_image_uri = "Docker image path: %s" % client.assemble_uri("/")
bot.info(docker_image_uri)
# IMAGE METADATA -------------------------------------------
# Use Docker Registry API (version 2.0) to get images ids, manifest
images = client.get_images()
# DOWNLOAD LAYERS -------------------------------------------
# Each is a .tar.gz file, obtained from registry with curl
# Get the cache (or temporary one) for docker
cache_base = get_cache(subfolder="docker")
download_client = MultiProcess()
# Generate a queue of tasks to run with MultiProcess
layers = []
tasks = []
for ii in range(len(images)):
image_id = images[ii]
targz = "%s/%s.tar.gz" % (cache_base, image_id)
if not os.path.exists(targz):
tasks.append((client, image_id, cache_base))
layers.append(targz)
# Does the user want to change permissions of tar?
func2 = None
if PLUGIN_FIXPERMS:
func2 = change_permissions
if len(tasks) > 0:
download_layers = download_client.run(func=download_layer,
func2=func2,
tasks=tasks)
# Get Docker runscript
runscript = extract_runscript(manifest=client.manifestv1,
includecmd=INCLUDE_CMD)
# Add the environment export
tar_file = extract_metadata_tar(client.manifestv1,
client.assemble_uri(),
runscript=runscript)
bot.verbose2('Tar file with Docker env and labels: %s' % tar_file)
# Write all layers to the layerfile
if layerfile is not None:
bot.verbose3("Writing Docker layers files to %s" % layerfile)
write_file(layerfile, "\n".join(layers), mode="w")
if tar_file is not None:
write_file(layerfile, "\n%s" % tar_file, mode="a")
# Return additions dictionary
additions = {"layers": layers,
"image": image,
"manifest": client.manifest,
"manifestv1": client.manifestv1,
"cache_base": cache_base,
"metadata": tar_file}
bot.debug("*** FINISHING DOCKER IMPORT PYTHON PORTION ****\n")
return additions
def IMPORT(image, auth=None, layerfile=None):
'''IMPORT is the main script that will obtain docker layers,
runscript information (either entrypoint or cmd), and environment
and return a list of tarballs to extract into the image
:param auth: if needed, an authentication header (default None)
:param layerfile: The file to write layers to extract into
'''
bot.debug("Starting Docker IMPORT, includes env, runscript, and metadata.")
bot.verbose("Docker image: %s" % image)
# Does the user want to override default of using ENTRYPOINT?
if INCLUDE_CMD:
bot.verbose2("Specified Docker CMD as %runscript.")
else:
bot.verbose2("Specified Docker ENTRYPOINT as %runscript.")
# Input Parsing ----------------------------
# Parse image name, repo name, and namespace
client = DockerApiConnection(image=image, auth=auth)
docker_image_uri = "Docker image path: %s" % client.assemble_uri("/")
bot.info(docker_image_uri)
# IMAGE METADATA -------------------------------------------
# Use Docker Registry API (version 2.0) to get images ids, manifest
images = client.get_images()
# DOWNLOAD LAYERS -------------------------------------------
# Each is a .tar.gz file, obtained from registry with curl
# Get the cache (or temporary one) for docker
cache_base = get_cache(subfolder="docker")
download_client = MultiProcess()
# Generate a queue of tasks to run with MultiProcess
layers = []
tasks = []
for ii in range(len(images)):
image_id = images[ii]
targz = "%s/%s.tar.gz" % (cache_base, image_id)
if not os.path.exists(targz):
tasks.append((client, image_id, cache_base))
layers.append(targz)
# Does the user want to change permissions of tar?
func2 = None
if PLUGIN_FIXPERMS:
func2 = change_permissions
if len(tasks) > 0:
download_layers = download_client.run(func=download_layer,
func2=func2,
tasks=tasks)
# Get Docker runscript
runscript = extract_runscript(manifest=client.manifestv1,
includecmd=INCLUDE_CMD)
# Add the environment export
tar_file = extract_metadata_tar(client.manifestv1,
client.assemble_uri(),
runscript=runscript)
bot.verbose2('Tar file with Docker env and labels: %s' % tar_file)
# Write all layers to the layerfile
if layerfile is not None:
bot.verbose3("Writing Docker layers files to %s" % layerfile)
write_file(layerfile, "\n".join(layers), mode="w")
if tar_file is not None:
write_file(layerfile, "\n%s" % tar_file, mode="a")
# Return additions dictionary
additions = {
"layers": layers,
"image": image,
"manifest": client.manifest,
"manifestv1": client.manifestv1,
"cache_base": cache_base,
"metadata": tar_file
}
bot.debug("*** FINISHING DOCKER IMPORT PYTHON PORTION ****\n")
return additions
