def test_get_singularity_version(self):
'''ensure that singularity --version returns a valid version string
'''
from singularity.build.utils import get_singularity_version
version = get_singularity_version()
self.assertTrue(len(version) > 0)
version = get_singularity_version(2.2)
self.assertEqual(version, 2.2)
os.environ['SINGULARITY_VERSION'] = "2.xx"
version = get_singularity_version()
self.assertEqual(version, "2.xx")
def version(self):
'''return the version of singularity
'''
from singularity.build.utils import get_singularity_version
return get_singularity_version()
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
'''
from singularity.build.utils import get_singularity_version
version = get_singularity_version()
# Ensure that correct converter is loaded depending on
# Singularity version
if not isinstance(version, str):
version = str(version)
if version.startswith('2.2'):
from .conversion.converter2pt2 import *
else:
from .conversion.converter2pt3 import *
def run_build(build_dir,params,verbose=True, compress_image=False):
'''run_build takes a build directory and params dictionary, and does the following:
- downloads repo to a temporary directory
- changes branch or commit, if needed
- creates and bootstraps singularity image from Singularity file
- returns a dictionary with:
image (path), image_package (path), metadata (dict)
The following must be included in params:
spec_file, repo_url, branch, commit
Optional parameters
size
'''
# Download the repo and image
download_repo(repo_url=params['repo_url'],
destination=build_dir)
os.chdir(build_dir)
if params['branch'] != None:
bot.info('Checking out branch %s' %params['branch'])
os.system('git checkout %s' %(params['branch']))
else:
params['branch'] = "master"
# Commit
if params['commit'] not in [None,'']:
bot.info('Checking out commit %s' %params['commit'])
os.system('git checkout %s .' %(params['commit']))
# From here on out commit is used as a unique id, if we don't have one, we use current
else:
params['commit'] = os.popen('git log -n 1 --pretty=format:"%H"').read()
bot.warning("commit not specified, setting to current %s" %params['commit'])
# Dump some params for the builder, in case it fails after this
passing_params = "/tmp/params.pkl"
pickle.dump(params,open(passing_params,'wb'))
# Now look for spec file
if os.path.exists(params['spec_file']):
bot.info("Found spec file %s in repository" %params['spec_file'])
# If the user has a symbolic link
if os.path.islink(params['spec_file']):
bot.info("%s is a symbolic link." %params['spec_file'])
params['spec_file'] = os.path.realpath(params['spec_file'])
# START TIMING
start_time = datetime.now()
image = build_from_spec(spec_file=params['spec_file'], # default will package the image
build_dir=build_dir,
isolated=True,
sandbox=False,
debug=params['debug'])
# Save has for metadata (also is image name)
version = get_image_file_hash(image)
params['version'] = version
pickle.dump(params,open(passing_params,'wb'))
final_time = (datetime.now() - start_time).seconds
bot.info("Final time of build %s seconds." %final_time)
# Did the container build successfully?
test_result = test_container(image)
if test_result['return_code'] != 0:
bot.error("Image failed to build, cancelling.")
sys.exit(1)
# Get singularity version
singularity_version = get_singularity_version()
# Package the image metadata (files, folders, etc)
image_package = package(image_path=image,
spec_path=params['spec_file'],
output_folder=build_dir,
remove_image=True,
verbose=True)
# Derive software tags by subtracting similar OS
diff = get_diff(image_package=image_package)
# Inspect to get labels and other metadata
cli = Singularity(debug=params['debug'])
inspect = cli.inspect(image_path=image)
# Get information on apps
app_names = cli.apps(image_path=image)
apps = extract_apps(image_path=image, app_names=app_names)
# Count file types, and extensions
counts = dict()
counts['readme'] = file_counts(diff=diff)
counts['copyright'] = file_counts(diff=diff,patterns=['copyright'])
counts['authors-thanks-credit'] = file_counts(diff=diff,
patterns=['authors','thanks','credit','contributors'])
counts['todo'] = file_counts(diff=diff,patterns=['todo'])
extensions = extension_counts(diff=diff)
os_sims = estimate_os(image_package=image_package,return_top=False)
most_similar = os_sims['SCORE'].values.argmax()
most_similar = os_sims['SCORE'].index.tolist()[most_similar]
metrics = {'build_time_seconds':final_time,
'singularity_version':singularity_version,
'singularity_python_version':singularity_python_version,
'estimated_os': most_similar,
'os_sims':os_sims['SCORE'].to_dict(),
'file_counts':counts,
'file_ext':extensions,
'inspect':inspect,
'version': version,
'apps': apps}
# Compress Image
if compress_image is True:
compressed_image = "%s.gz" %image
os.system('gzip -c -9 %s > %s' %(image,compressed_image))
image = compressed_image
output = {'image':image,
'image_package':image_package,
'metadata':metrics,
'params':params }
return output
else:
# Tell the user what is actually there
present_files = glob("*")
bot.error("Build file %s not found in repository" %params['spec_file'])
bot.info("Found files are %s" %"\n".join(present_files))
# Params have been exported, will be found by log
sys.exit(1)
def run_build(build_dir, params, verbose=True):
'''run_build takes a build directory and params dictionary, and does the following:
- downloads repo to a temporary directory
- changes branch or commit, if needed
- creates and bootstraps singularity image from Singularity file
- returns a dictionary with:
image (path), image_package (path), metadata (dict)
The following must be included in params:
spec_file, repo_url, branch, commit
Optional parameters
size
'''
# Download the repo and image
download_repo(repo_url=params['repo_url'], destination=build_dir)
os.chdir(build_dir)
if params['branch'] != None:
bot.logger.info('Checking out branch %s', params['branch'])
os.system('git checkout %s' % (params['branch']))
# Commit
if params['commit'] not in [None, '']:
bot.logger.info('Checking out commit %s', params['commit'])
os.system('git checkout %s .' % (params['commit']))
# From here on out commit is used as a unique id, if we don't have one, we use current
else:
params['commit'] = os.popen('git log -n 1 --pretty=format:"%H"').read()
bot.logger.warning("commit not specified, setting to current %s",
params['commit'])
# Dump some params for the builder, in case it fails after this
passing_params = "/tmp/params.pkl"
pickle.dump(params, open(passing_params, 'wb'))
# If there is not a specfile, but is a Dockerfile, try building that
if not os.path.exists(
params['spec_file']) and os.path.exists('Dockerfile'):
bot.logger.warning("Build file %s not found in repository",
params['spec_file'])
bot.logger.warning(
"Dockerfile found in repository, will attempt build.")
dockerfile = dockerfile_to_singularity(dockerfile_path='Dockerfile',
output_dir=build_dir)
if dockerfile is not None:
bot.logger.info("""\n
--------------------------------------------------------------
Dockerfile
--------------------------------------------------------------
\n%s""" % (dockerfile))
# Now look for spec file
if os.path.exists(params['spec_file']):
bot.logger.info("Found spec file %s in repository",
params['spec_file'])
# If size is None, set default of 800
if params['size'] in [None, '']:
bot.logger.info("""\n
--------------------------------------------------------------
Size not detected for build. Will first try to estimate, and then
use default of 800MB padding. If your build still fails, you should
try setting the size manually under collection --> edit builder
---------------------------------------------------------------------
\n""")
# Testing estimation of size
try:
params['size'] = estimate_image_size(spec_file=os.path.abspath(
params['spec_file']),
sudopw='',
padding=params['padding'])
bot.logger.info("Size estimated as %s", params['size'])
except:
params['size'] = 800
bot.logger.info(
"Size estimation didn't work, using default %s",
params['size'])
# START TIMING
os.chdir(build_dir)
start_time = datetime.now()
image = build_from_spec(
spec_file=params['spec_file'], # default will package the image
size=params['size'],
sudopw='', # with root should not need sudo
build_dir=build_dir,
debug=params['debug'])
final_time = (datetime.now() - start_time).seconds
bot.logger.info("Final time of build %s seconds.", final_time)
# Did the container build successfully?
test_result = test_container(image)
if test_result['return_code'] == 255:
bot.logger.error("Image failed to bootstrap, cancelling build.")
sys.exit(1)
# Compress image
compressed_image = "%s.img.gz" % image
os.system('gzip -c -9 %s > %s' % (image, compressed_image))
# Get singularity version
singularity_version = get_singularity_version()
old_version = False
if singularity_version.startswith('2.2'):
old_version = True
# Package the image metadata (files, folders, etc)
image_package = package(image_path=image,
spec_path=params['spec_file'],
output_folder=build_dir,
sudopw='',
remove_image=True,
verbose=True,
old_version=old_version)
# Derive software tags by subtracting similar OS
diff = get_diff(image_package=image_package)
# Get tags for services, executables
interesting_folders = ['init', 'init.d', 'bin', 'systemd']
tags = get_tags(search_folders=interesting_folders, diff=diff)
# Count file types, and extensions
counts = dict()
counts['readme'] = file_counts(diff=diff)
counts['copyright'] = file_counts(diff=diff, patterns=['copyright'])
counts['authors-thanks-credit'] = file_counts(
diff=diff, patterns=['authors', 'thanks', 'credit'])
counts['todo'] = file_counts(diff=diff, patterns=['todo'])
extensions = extension_counts(diff=diff)
os_sims = estimate_os(image_package=image_package, return_top=False)
most_similar = os_sims['SCORE'].idxmax()
metrics = {
'size': params['size'],
'build_time_seconds': final_time,
'singularity_version': singularity_version,
'singularity_python_version': singularity_python_version,
'estimated_os': most_similar,
'os_sims': os_sims['SCORE'].to_dict(),
'tags': tags,
'file_counts': counts,
'file_ext': extensions
}
output = {
'image': compressed_image,
'image_package': image_package,
'metadata': metrics,
'params': params
}
return output
else:
# Tell the user what is actually there
present_files = glob("*")
bot.logger.error("Build file %s not found in repository",
params['spec_file'])
bot.logger.info("Found files are %s", "\n".join(present_files))
# Params have been exported, will be found by log
sys.exit(1)