def extract_apps(image_path, app_names, S=None, verbose=True):
''' extract app will extract metadata for one or more apps
Parameters
==========
image_path: the absolute path to the image
app_name: the name of the app under /scif/apps
'''
if S is None:
S = Singularity(debug=verbose, sudo=True)
if not isinstance(app_names, list):
app_names = [app_names]
file_obj, tar = get_image_tar(image_path, S=S)
members = tar.getmembers()
apps = dict()
for app_name in app_names:
metadata = dict()
# Inspect: labels, env, runscript, tests, help
try:
inspection = json.loads(S.inspect(image_path, app=app_name))
del inspection['data']['attributes']['deffile']
metadata['inspect'] = inspection
# If illegal characters prevent load, not much we can do
except:
pass
base = '/scif/apps/%s' % app_name
metadata['files'] = [x.path for x in members if base in x.path]
apps[app_name] = metadata
return apps
def package(image_path,
spec_path=None,
output_folder=None,
remove_image=False,
verbose=False,
S=None):
'''generate a zip (including the image) to a user specified output_folder.
:param image_path: full path to singularity image file
:param runscript: if True, will extract runscript to include in package as runscript
:param software: if True, will extract files.txt and folders.txt to package
:param remove_image: if True, will not include original image in package (default,False)
:param verbose: be verbose when using singularity --export (default,False)
:param S: the Singularity object (optional) will be created if not required.
'''
# Run create image and bootstrap with Singularity command line tool.
S = Singularity(debug=verbose)
file_obj, tar = get_image_tar(image_path, S=S)
members = tar.getmembers()
image_name, ext = os.path.splitext(os.path.basename(image_path))
zip_name = "%s.zip" % (image_name.replace(" ", "_"))
# Include the image in the package?
to_package = dict()
if not remove_image:
to_package["files"] = [image_path]
# If the specfile is provided, it should also be packaged
if spec_path is not None:
singularity_spec = "".join(read_file(spec_path))
to_package['Singularity'] = singularity_spec
to_package["VERSION"] = get_image_file_hash(image_path)
try:
inspection = S.inspect(image_path)
to_package["inspect.json"] = inspection
inspection = json.loads(inspection)
to_package['runscript'] = inspection['data']['attributes']['runscript']
except:
bot.warning("Trouble extracting container metadata with inspect.")
bot.info("Adding software list to package.")
files = [x.path for x in members if x.isfile()]
folders = [x.path for x in members if x.isdir()]
to_package["files.txt"] = files
to_package["folders.txt"] = folders
# Do zip up here - let's start with basic structures
zipfile = zip_up(to_package,
zip_name=zip_name,
output_folder=output_folder)
bot.debug("Package created at %s" % (zipfile))
if not delete_image_tar(file_obj, tar):
bot.warning("Could not clean up temporary tarfile.")
# return package to user
return zipfile
class TestClient(unittest.TestCase):
def setUp(self):
self.pwd = get_installdir()
self.cli = Singularity()
self.tmpdir = tempfile.mkdtemp()
def tearDown(self):
shutil.rmtree(self.tmpdir)
def test_commands(self):
print('Testing client.create command')
container = "%s/container.img" % (self.tmpdir)
created_container = self.cli.create(container)
self.assertEqual(created_container, container)
self.assertTrue(os.path.exists(created_container))
os.remove(container)
print("Testing client.pull command")
print("...Case 1: Testing naming pull by image name")
image = self.cli.pull("shub://vsoch/singularity-images",
pull_folder=self.tmpdir)
self.assertTrue(os.path.exists(image))
self.assertTrue('vsoch-singularity-images-master' in image)
print(image)
os.remove(image)
print("...Case 3: Testing docker pull")
container = self.cli.pull("docker://ubuntu:14.04",
pull_folder=self.tmpdir)
self.assertTrue("ubuntu:14.04" in container)
print(container)
self.assertTrue(os.path.exists(container))
print('Testing client.execute command')
result = self.cli.execute(container, 'ls /')
print(result)
self.assertTrue('bin\nboot\ndev' in result)
print("Testing client.inspect command")
result = self.cli.inspect(container, quiet=True)
labels = json.loads(result)
self.assertTrue('data' in labels)
os.remove(container)
# Execute command to container
result = S.execute(image, command='cat /singularity')
print(result)
'''
'#!/bin/sh\n\nexec "/bin/bash"\n'
'''
# For any function you can get the docs:
S.help(command="exec")
# export an image to tar
tar = S.export(image)
# Show apps and inspect
S.apps(image)
S.inspect(image)
'''
{
"data": {
"attributes": {
"deffile": null,
"help": null,
"labels": null,
"environment": "# Custom environment shell code should follow\n\n",
"runscript": "#!/bin/sh\n\nexec \"/bin/bash\"\n",
"test": null
},
"type": "container"
}
}
'''
def push(self, path, name, tag=None, compress=False):
'''push an image to Singularity Registry'''
path = os.path.abspath(path)
image = os.path.basename(path)
bot.debug("PUSH %s" % path)
if not os.path.exists(path):
bot.error('%s does not exist.' %path)
sys.exit(1)
cli = Singularity()
metadata = cli.inspect(image_path=path, quiet=True)
metadata = json.loads(metadata)
# Try to add the size
try:
image_size = os.path.getsize(path) >> 20
if metadata['data']['attributes']['labels'] is None:
metadata['data']['attributes']['labels'] = {'SREGISTRY_SIZE_MB': image_size }
else:
metadata['data']['attributes']['labels']['SREGISTRY_SIZE_MB'] = image_size
except:
bot.warning("Cannot load metadata to add calculated size.")
pass
if "deffile" in metadata['data']['attributes']:
if metadata['data']['attributes']['deffile'] is not None:
fromimage = parse_header(metadata['data']['attributes']['deffile'],
header="from",
remove_header=True)
metadata['data']['attributes']['labels']['SREGISTRY_FROM'] = fromimage
bot.debug("%s was built from a definition file." % image)
if compress is True:
ext = 'img.gz' # ext3 format
else:
ext = 'simg' # ext3 format
metadata = json.dumps(metadata)
names = parse_image_name(name,tag=tag, ext=ext)
url = '%s/push/' % self.base
if compress is True:
try:
sys.stdout.write('Compressing image ')
bot.spinner.start()
upload_from = cli.compress(path)
bot.spinner.stop()
except KeyboardInterrupt:
print('Upload cancelled')
if os.path.exists("%s.gz" %path):
os.remove("%s.gz" %path)
sys.exit(1)
else:
upload_from = path
upload_to = os.path.basename(names['storage'])
SREGISTRY_EVENT = self.authorize(request_type="push",
names=names)
encoder = MultipartEncoder(fields={'collection': names['collection'],
'name':names['image'],
'metadata':metadata,
'tag': names['tag'],
'datafile': (upload_to, open(upload_from, 'rb'), 'text/plain')})
progress_callback = create_callback(encoder)
monitor = MultipartEncoderMonitor(encoder, progress_callback)
headers = {'Content-Type': monitor.content_type,
'Authorization': SREGISTRY_EVENT }
try:
r = requests.post(url, data=monitor, headers=headers)
message = self.read_response(r)
print('\n[Return status {0} {1}]'.format(r.status_code, message))
except KeyboardInterrupt:
print('\nUpload cancelled.')
# Clean up
if compress is True:
if os.path.exists("%s.gz" %path):
os.remove("%s.gz" %path)
def run_build(build_dir,params,verbose=True, compress_image=False):
'''run_build takes a build directory and params dictionary, and does the following:
- downloads repo to a temporary directory
- changes branch or commit, if needed
- creates and bootstraps singularity image from Singularity file
- returns a dictionary with:
image (path), image_package (path), metadata (dict)
The following must be included in params:
spec_file, repo_url, branch, commit
Optional parameters
size
'''
# Download the repo and image
download_repo(repo_url=params['repo_url'],
destination=build_dir)
os.chdir(build_dir)
if params['branch'] != None:
bot.info('Checking out branch %s' %params['branch'])
os.system('git checkout %s' %(params['branch']))
else:
params['branch'] = "master"
# Commit
if params['commit'] not in [None,'']:
bot.info('Checking out commit %s' %params['commit'])
os.system('git checkout %s .' %(params['commit']))
# From here on out commit is used as a unique id, if we don't have one, we use current
else:
params['commit'] = os.popen('git log -n 1 --pretty=format:"%H"').read()
bot.warning("commit not specified, setting to current %s" %params['commit'])
# Dump some params for the builder, in case it fails after this
passing_params = "/tmp/params.pkl"
pickle.dump(params,open(passing_params,'wb'))
# Now look for spec file
if os.path.exists(params['spec_file']):
bot.info("Found spec file %s in repository" %params['spec_file'])
# If the user has a symbolic link
if os.path.islink(params['spec_file']):
bot.info("%s is a symbolic link." %params['spec_file'])
params['spec_file'] = os.path.realpath(params['spec_file'])
# START TIMING
start_time = datetime.now()
image = build_from_spec(spec_file=params['spec_file'], # default will package the image
build_dir=build_dir,
isolated=True,
sandbox=False,
debug=params['debug'])
# Save has for metadata (also is image name)
version = get_image_file_hash(image)
params['version'] = version
pickle.dump(params,open(passing_params,'wb'))
final_time = (datetime.now() - start_time).seconds
bot.info("Final time of build %s seconds." %final_time)
# Did the container build successfully?
test_result = test_container(image)
if test_result['return_code'] != 0:
bot.error("Image failed to build, cancelling.")
sys.exit(1)
# Get singularity version
singularity_version = get_singularity_version()
# Package the image metadata (files, folders, etc)
image_package = package(image_path=image,
spec_path=params['spec_file'],
output_folder=build_dir,
remove_image=True,
verbose=True)
# Derive software tags by subtracting similar OS
diff = get_diff(image_package=image_package)
# Inspect to get labels and other metadata
cli = Singularity(debug=params['debug'])
inspect = cli.inspect(image_path=image)
# Get information on apps
app_names = cli.apps(image_path=image)
apps = extract_apps(image_path=image, app_names=app_names)
# Count file types, and extensions
counts = dict()
counts['readme'] = file_counts(diff=diff)
counts['copyright'] = file_counts(diff=diff,patterns=['copyright'])
counts['authors-thanks-credit'] = file_counts(diff=diff,
patterns=['authors','thanks','credit','contributors'])
counts['todo'] = file_counts(diff=diff,patterns=['todo'])
extensions = extension_counts(diff=diff)
os_sims = estimate_os(image_package=image_package,return_top=False)
most_similar = os_sims['SCORE'].values.argmax()
most_similar = os_sims['SCORE'].index.tolist()[most_similar]
metrics = {'build_time_seconds':final_time,
'singularity_version':singularity_version,
'singularity_python_version':singularity_python_version,
'estimated_os': most_similar,
'os_sims':os_sims['SCORE'].to_dict(),
'file_counts':counts,
'file_ext':extensions,
'inspect':inspect,
'version': version,
'apps': apps}
# Compress Image
if compress_image is True:
compressed_image = "%s.gz" %image
os.system('gzip -c -9 %s > %s' %(image,compressed_image))
image = compressed_image
output = {'image':image,
'image_package':image_package,
'metadata':metrics,
'params':params }
return output
else:
# Tell the user what is actually there
present_files = glob("*")
bot.error("Build file %s not found in repository" %params['spec_file'])
bot.info("Found files are %s" %"\n".join(present_files))
# Params have been exported, will be found by log
sys.exit(1)
