def package(image_path,
spec_path=None,
output_folder=None,
remove_image=False,
verbose=False,
S=None):
'''generate a zip (including the image) to a user specified output_folder.
:param image_path: full path to singularity image file
:param runscript: if True, will extract runscript to include in package as runscript
:param software: if True, will extract files.txt and folders.txt to package
:param remove_image: if True, will not include original image in package (default,False)
:param verbose: be verbose when using singularity --export (default,False)
:param S: the Singularity object (optional) will be created if not required.
'''
# Run create image and bootstrap with Singularity command line tool.
S = Singularity(debug=verbose)
file_obj, tar = get_image_tar(image_path, S=S)
members = tar.getmembers()
image_name, ext = os.path.splitext(os.path.basename(image_path))
zip_name = "%s.zip" % (image_name.replace(" ", "_"))
# Include the image in the package?
to_package = dict()
if not remove_image:
to_package["files"] = [image_path]
# If the specfile is provided, it should also be packaged
if spec_path is not None:
singularity_spec = "".join(read_file(spec_path))
to_package['Singularity'] = singularity_spec
to_package["VERSION"] = get_image_file_hash(image_path)
try:
inspection = S.inspect(image_path)
to_package["inspect.json"] = inspection
inspection = json.loads(inspection)
to_package['runscript'] = inspection['data']['attributes']['runscript']
except:
bot.warning("Trouble extracting container metadata with inspect.")
bot.info("Adding software list to package.")
files = [x.path for x in members if x.isfile()]
folders = [x.path for x in members if x.isdir()]
to_package["files.txt"] = files
to_package["folders.txt"] = folders
# Do zip up here - let's start with basic structures
zipfile = zip_up(to_package,
zip_name=zip_name,
output_folder=output_folder)
bot.debug("Package created at %s" % (zipfile))
if not delete_image_tar(file_obj, tar):
bot.warning("Could not clean up temporary tarfile.")
# return package to user
return zipfile
def build_from_spec(spec_file=None,
build_dir=None,
build_folder=False,
sandbox=False,
isolated=False,
debug=False):
'''build_from_spec will build a "spec" file in a "build_dir" and return the directory
:param spec_file: the spec file, called "Singuarity"
:param build_dir: the directory to build in. If not defined, will use tmpdir.
:param isolated: "build" the image inside an isolated environment (>2.4)
:param sandbox: ask for a sandbox build
:param debug: ask for verbose output from builder
'''
if spec_file == None:
spec_file = "Singularity"
if build_dir == None:
build_dir = tempfile.mkdtemp()
bot.debug("Building in directory %s" %build_dir)
# Copy the spec to a temporary directory
bot.debug("Spec file set to %s" % spec_file)
spec_path = os.path.abspath(spec_file)
bot.debug("Spec file for build should be in %s" %spec_path)
image_path = "%s/build.simg" %(build_dir)
# Run create image and bootstrap with Singularity command line tool.
cli = Singularity(debug=debug)
print("\nBuilding image...")
# Does the user want to "build" into a folder or image?
result = cli.build(image_path=image_path,
spec_path=spec_path,
sandbox=sandbox,
isolated=isolated)
print(result)
# If image, rename based on hash
if sandbox is False:
version = get_image_file_hash(image_path)
final_path = "%s/%s.simg" %(build_dir,version)
os.rename(image_path,final_path)
image_path = final_path
bot.debug("Built image: %s" %image_path)
return image_path
def test_get_image_file_hash(self):
from singularity.analysis.reproduce import get_image_file_hash
print("Testing singularity.analysis.reproduce.get_image_file_hash")
hashy = get_image_file_hash(self.image1)
self.assertEqual('843929a1c92d53656fa744c6b831f59b',hashy)
def run_build(build_dir, params, verbose=True):
'''run_build takes a build directory and params dictionary, and does the following:
- downloads repo to a temporary directory
- changes branch or commit, if needed
- creates and bootstraps singularity image from Singularity file
- returns a dictionary with:
image (path), metadata (dict)
The following must be included in params:
spec_file, repo_url, branch, commit
'''
# Download the repository
download_repo(repo_url=params['repo_url'],
destination=build_dir)
os.chdir(build_dir)
if params['branch'] != None:
bot.info('Checking out branch %s' %params['branch'])
os.system('git checkout %s' %(params['branch']))
else:
params['branch'] = "master"
# Set the debug level
Client.debug = params['debug']
# Commit
if params['commit'] not in [None,'']:
bot.info('Checking out commit %s' %params['commit'])
os.system('git checkout %s .' %(params['commit']))
# From here on out commit is used as a unique id, if we don't have one, we use current
else:
params['commit'] = os.popen('git log -n 1 --pretty=format:"%H"').read()
bot.warning("commit not specified, setting to current %s" %params['commit'])
# Dump some params for the builder, in case it fails after this
passing_params = "/tmp/params.pkl"
pickle.dump(params, open(passing_params,'wb'))
# Now look for spec file
if os.path.exists(params['spec_file']):
bot.info("Found spec file %s in repository" %params['spec_file'])
# If the user has a symbolic link
if os.path.islink(params['spec_file']):
bot.info("%s is a symbolic link." %params['spec_file'])
params['spec_file'] = os.path.realpath(params['spec_file'])
# START TIMING
start_time = datetime.now()
# Secure Build
image = Client.build(recipe=params['spec_file'],
build_folder=build_dir,
isolated=True)
# Save has for metadata (also is image name)
version = get_image_file_hash(image)
params['version'] = version
pickle.dump(params, open(passing_params,'wb'))
# Rename image to be hash
finished_image = "%s/%s.simg" %(os.path.dirname(image), version)
image = shutil.move(image, finished_image)
final_time = (datetime.now() - start_time).seconds
bot.info("Final time of build %s seconds." %final_time)
# Did the container build successfully?
test_result = test_container(image)
if test_result['return_code'] != 0:
bot.error("Image failed to build, cancelling.")
sys.exit(1)
# Get singularity version
singularity_version = Client.version()
Client.debug = False
inspect = Client.inspect(image) # this is a string
Client.debug = params['debug']
# Get information on apps
Client.debug = False
app_names = Client.apps(image)
Client.debug = params['debug']
apps = extract_apps(image, app_names)
metrics = {'build_time_seconds': final_time,
'singularity_version': singularity_version,
'singularity_python_version': singularity_python_version,
'inspect': inspect,
'version': version,
'apps': apps}
output = {'image':image,
'metadata':metrics,
'params':params }
return output
else:
# Tell the user what is actually there
present_files = glob("*")
bot.error("Build file %s not found in repository" %params['spec_file'])
bot.info("Found files are %s" %"\n".join(present_files))
# Params have been exported, will be found by log
sys.exit(1)
def run_build(build_dir,params,verbose=True, compress_image=False):
'''run_build takes a build directory and params dictionary, and does the following:
- downloads repo to a temporary directory
- changes branch or commit, if needed
- creates and bootstraps singularity image from Singularity file
- returns a dictionary with:
image (path), image_package (path), metadata (dict)
The following must be included in params:
spec_file, repo_url, branch, commit
Optional parameters
size
'''
# Download the repo and image
download_repo(repo_url=params['repo_url'],
destination=build_dir)
os.chdir(build_dir)
if params['branch'] != None:
bot.info('Checking out branch %s' %params['branch'])
os.system('git checkout %s' %(params['branch']))
else:
params['branch'] = "master"
# Commit
if params['commit'] not in [None,'']:
bot.info('Checking out commit %s' %params['commit'])
os.system('git checkout %s .' %(params['commit']))
# From here on out commit is used as a unique id, if we don't have one, we use current
else:
params['commit'] = os.popen('git log -n 1 --pretty=format:"%H"').read()
bot.warning("commit not specified, setting to current %s" %params['commit'])
# Dump some params for the builder, in case it fails after this
passing_params = "/tmp/params.pkl"
pickle.dump(params,open(passing_params,'wb'))
# Now look for spec file
if os.path.exists(params['spec_file']):
bot.info("Found spec file %s in repository" %params['spec_file'])
# If the user has a symbolic link
if os.path.islink(params['spec_file']):
bot.info("%s is a symbolic link." %params['spec_file'])
params['spec_file'] = os.path.realpath(params['spec_file'])
# START TIMING
start_time = datetime.now()
image = build_from_spec(spec_file=params['spec_file'], # default will package the image
build_dir=build_dir,
isolated=True,
sandbox=False,
debug=params['debug'])
# Save has for metadata (also is image name)
version = get_image_file_hash(image)
params['version'] = version
pickle.dump(params,open(passing_params,'wb'))
final_time = (datetime.now() - start_time).seconds
bot.info("Final time of build %s seconds." %final_time)
# Did the container build successfully?
test_result = test_container(image)
if test_result['return_code'] != 0:
bot.error("Image failed to build, cancelling.")
sys.exit(1)
# Get singularity version
singularity_version = get_singularity_version()
# Package the image metadata (files, folders, etc)
image_package = package(image_path=image,
spec_path=params['spec_file'],
output_folder=build_dir,
remove_image=True,
verbose=True)
# Derive software tags by subtracting similar OS
diff = get_diff(image_package=image_package)
# Inspect to get labels and other metadata
cli = Singularity(debug=params['debug'])
inspect = cli.inspect(image_path=image)
# Get information on apps
app_names = cli.apps(image_path=image)
apps = extract_apps(image_path=image, app_names=app_names)
# Count file types, and extensions
counts = dict()
counts['readme'] = file_counts(diff=diff)
counts['copyright'] = file_counts(diff=diff,patterns=['copyright'])
counts['authors-thanks-credit'] = file_counts(diff=diff,
patterns=['authors','thanks','credit','contributors'])
counts['todo'] = file_counts(diff=diff,patterns=['todo'])
extensions = extension_counts(diff=diff)
os_sims = estimate_os(image_package=image_package,return_top=False)
most_similar = os_sims['SCORE'].values.argmax()
most_similar = os_sims['SCORE'].index.tolist()[most_similar]
metrics = {'build_time_seconds':final_time,
'singularity_version':singularity_version,
'singularity_python_version':singularity_python_version,
'estimated_os': most_similar,
'os_sims':os_sims['SCORE'].to_dict(),
'file_counts':counts,
'file_ext':extensions,
'inspect':inspect,
'version': version,
'apps': apps}
# Compress Image
if compress_image is True:
compressed_image = "%s.gz" %image
os.system('gzip -c -9 %s > %s' %(image,compressed_image))
image = compressed_image
output = {'image':image,
'image_package':image_package,
'metadata':metrics,
'params':params }
return output
else:
# Tell the user what is actually there
present_files = glob("*")
bot.error("Build file %s not found in repository" %params['spec_file'])
bot.info("Found files are %s" %"\n".join(present_files))
# Params have been exported, will be found by log
sys.exit(1)
# dict_keys(['IDENTICAL', 'BASE', 'REPLICATE', 'RECIPE', 'RUNSCRIPT', 'ENVIRONMENT', 'LABELS']) # or specify a different version: levels = get_levels(version=2.3) # We can, then generate an image hash, and by default the level "REPLICATION" will be used: get_image_hash(image_path) #'bf8e242931e25ae9496015868ab2e8cc8d156ffd' # But we can also specify a level that we want: get_image_hash(image_path, level="IDENTICAL") #'3f4dcf64b58d314ac9ef0c02f641cd2109a07d64' ######################################################## # Get Image Content Digest (One Container --> Multiple) ######################################################## # We might want to get a dictionary of content hashes for all files # of one container at one level! digest = get_content_hashes(image_path) digest['./usr/bin/pinky'] # 'ee2b438c278011bdac1a3a927e2d37519a8ed9c7' # We can also get a hash of the entire image file, this is done on the # binary file and not contents inside. file_hash = get_image_file_hash(image_path) # e'13775a83962ae60744d691eb7f7fd1e96599e656'