def extract_apps(image_path, app_names, S=None, verbose=True):
''' extract app will extract metadata for one or more apps
Parameters
==========
image_path: the absolute path to the image
app_name: the name of the app under /scif/apps
'''
if S is None:
S = Singularity(debug=verbose, sudo=True)
if not isinstance(app_names, list):
app_names = [app_names]
file_obj, tar = get_image_tar(image_path, S=S)
members = tar.getmembers()
apps = dict()
for app_name in app_names:
metadata = dict()
# Inspect: labels, env, runscript, tests, help
try:
inspection = json.loads(S.inspect(image_path, app=app_name))
del inspection['data']['attributes']['deffile']
metadata['inspect'] = inspection
# If illegal characters prevent load, not much we can do
except:
pass
base = '/scif/apps/%s' % app_name
metadata['files'] = [x.path for x in members if base in x.path]
apps[app_name] = metadata
return apps
def package(image_path,
spec_path=None,
output_folder=None,
remove_image=False,
verbose=False,
S=None):
'''generate a zip (including the image) to a user specified output_folder.
:param image_path: full path to singularity image file
:param runscript: if True, will extract runscript to include in package as runscript
:param software: if True, will extract files.txt and folders.txt to package
:param remove_image: if True, will not include original image in package (default,False)
:param verbose: be verbose when using singularity --export (default,False)
:param S: the Singularity object (optional) will be created if not required.
'''
# Run create image and bootstrap with Singularity command line tool.
S = Singularity(debug=verbose)
file_obj, tar = get_image_tar(image_path, S=S)
members = tar.getmembers()
image_name, ext = os.path.splitext(os.path.basename(image_path))
zip_name = "%s.zip" % (image_name.replace(" ", "_"))
# Include the image in the package?
to_package = dict()
if not remove_image:
to_package["files"] = [image_path]
# If the specfile is provided, it should also be packaged
if spec_path is not None:
singularity_spec = "".join(read_file(spec_path))
to_package['Singularity'] = singularity_spec
to_package["VERSION"] = get_image_file_hash(image_path)
try:
inspection = S.inspect(image_path)
to_package["inspect.json"] = inspection
inspection = json.loads(inspection)
to_package['runscript'] = inspection['data']['attributes']['runscript']
except:
bot.warning("Trouble extracting container metadata with inspect.")
bot.info("Adding software list to package.")
files = [x.path for x in members if x.isfile()]
folders = [x.path for x in members if x.isdir()]
to_package["files.txt"] = files
to_package["folders.txt"] = folders
# Do zip up here - let's start with basic structures
zipfile = zip_up(to_package,
zip_name=zip_name,
output_folder=output_folder)
bot.debug("Package created at %s" % (zipfile))
if not delete_image_tar(file_obj, tar):
bot.warning("Could not clean up temporary tarfile.")
# return package to user
return zipfile
def compare_singularity_images(image_paths1, image_paths2=None):
'''compare_singularity_images is a wrapper for compare_containers to compare
singularity containers. If image_paths2 is not defined, pairwise comparison is done
with image_paths1
'''
repeat = False
if image_paths2 is None:
image_paths2 = image_paths1
repeat = True
if not isinstance(image_paths1, list):
image_paths1 = [image_paths1]
if not isinstance(image_paths2, list):
image_paths2 = [image_paths2]
dfs = pandas.DataFrame(index=image_paths1, columns=image_paths2)
comparisons_done = []
for image1 in image_paths1:
fileobj1, tar1 = get_image_tar(image1)
members1 = [x.name for x in tar1]
for image2 in image_paths2:
comparison_id = [image1, image2]
comparison_id.sort()
comparison_id = "".join(comparison_id)
if comparison_id not in comparisons_done:
if image1 == image2:
sim = 1.0
else:
fileobj2, tar2 = get_image_tar(image2)
members2 = [x.name for x in tar2]
c = compare_lists(members1, members2)
sim = information_coefficient(c['total1'], c['total2'],
c['intersect'])
delete_image_tar(fileobj2, tar2)
dfs.loc[image1, image2] = sim
if repeat:
dfs.loc[image2, image1] = sim
comparisons_done.append(comparison_id)
delete_image_tar(fileobj1, tar1)
return dfs
def compare_singularity_images(image_paths1, image_paths2=None):
'''compare_singularity_images is a wrapper for compare_containers to compare
singularity containers. If image_paths2 is not defined, pairwise comparison is done
with image_paths1
'''
repeat = False
if image_paths2 is None:
image_paths2 = image_paths1
repeat = True
if not isinstance(image_paths1,list):
image_paths1 = [image_paths1]
if not isinstance(image_paths2,list):
image_paths2 = [image_paths2]
dfs = pandas.DataFrame(index=image_paths1,columns=image_paths2)
comparisons_done = []
for image1 in image_paths1:
fileobj1,tar1 = get_image_tar(image1)
members1 = [x.name for x in tar1]
for image2 in image_paths2:
comparison_id = [image1,image2]
comparison_id.sort()
comparison_id = "".join(comparison_id)
if comparison_id not in comparisons_done:
if image1 == image2:
sim = 1.0
else:
fileobj2,tar2 = get_image_tar(image2)
members2 = [x.name for x in tar2]
c = compare_lists(members1, members2)
sim = information_coefficient(c['total1'],c['total2'],c['intersect'])
delete_image_tar(fileobj2, tar2)
dfs.loc[image1,image2] = sim
if repeat:
dfs.loc[image2,image1] = sim
comparisons_done.append(comparison_id)
delete_image_tar(fileobj1, tar1)
return dfs
def test_extract_guts(self):
from singularity.analysis.reproduce.utils import extract_guts
from singularity.analysis.reproduce import (get_image_tar, get_levels)
print("Testing singularity.analysis.reproduce.extract_guts")
levels = get_levels()
file_obj, tar = get_image_tar(self.image1)
guts = extract_guts(image_path=self.image1,
tar=tar,
file_filter=levels['REPLICATE'])
for key in ['root_owned', 'sizes', 'hashes']:
self.assertTrue(key in guts)
tar.close()
def test_extract_guts(self):
from singularity.analysis.reproduce.utils import extract_guts
from singularity.analysis.reproduce import (
get_image_tar,
get_levels )
print("Testing singularity.analysis.reproduce.extract_guts")
levels = get_levels()
file_obj,tar = get_image_tar(self.image1)
guts = extract_guts(image_path=self.image1,
tar=tar,
file_filter=levels['REPLICATE'])
for key in ['root_owned','sizes','hashes']:
self.assertTrue(key in guts)
tar.close()
def get_container_contents(container, split_delim=None):
'''get_container_contents will return a list of folders and or files
for a container. The environmental variable SINGULARITY_HUB being set
means that container objects are referenced instead of packages
:param container: the container to get content for
:param gets: a list of file names to return, without parent folders
:param split_delim: if defined, will split text by split delimiter
'''
# We will look for everything in guts, then return it
guts = dict()
SINGULARITY_HUB = os.environ.get('SINGULARITY_HUB',"False")
# Visualization deployed local or elsewhere
if SINGULARITY_HUB == "False":
file_obj,tar = get_image_tar(container)
guts = extract_guts(image_path=container, tar=tar)
delete_image_tar(file_obj, tar)
# Visualization deployed by singularity hub
else:
# user has provided a package, but not a container
if container == None:
guts = load_package(image_package,get=gets)
# user has provided a container, but not a package
else:
for sfile in container.files:
for gut_key in gets:
if os.path.basename(sfile['name']) == gut_key:
if split_delim == None:
guts[gut_key] = requests.get(sfile['mediaLink']).text
else:
guts[gut_key] = requests.get(sfile['mediaLink']).text.split(split_delim)
return guts