def singularity_build(self, path):
"""Builds an image from a recipefile.
"""
recipefile_path = os.path.join(path, 'Singularity')
log.info('{}[{}] singularity build {} {}'.format(
self.msg_prefix, self.action['name'], self.image_name,
recipefile_path))
if not self.dry_run:
sclient.build(recipefile_path, self.image_name)
def export_to_targz(image, tmpdir=None):
"""export a Singularity image to a .tar.gz file. If run within a docker
image, you should set via_build to false (as sudo will work under
priviledged). Outside of Docker as regular user, via_build works
better.
Parameters
==========
image: the full path to the Singularity image
tmpdir: a temporary directory to export to.
"""
print("Exporting %s to targz..." % image)
if tmpdir == None:
tmpdir = tempfile.mkdtemp()
# We will build into this directory (sandbox) to export without sudo
export_dir = get_temporary_name(tmpdir, "singularity-clair")
targz = "%s.gz" % export_dir
sandbox = Client.build(image, export_dir, sandbox=True, sudo=False)
# Write the tarfile
with tarfile.open(targz, "w:gz") as tar:
tar.add(sandbox, arcname="/")
shutil.rmtree(sandbox)
if os.path.exists(targz):
return targz
def action_build(builder={}):
'''build a container from a generated recipe.
'''
# Post indicates browsing the tree, with an additional path to parse
if request.method == "POST":
uri = request.form.get('uri')
recipe = request.form.get('recipe').replace('\r','')
app.logger.info('BUILD: %s' %uri)
# Clean up old recipes
cleanup('/tmp', "^Singularity.*")
recipe = write_temp_recipe(recipe)
if recipe is not None:
# When whiteout but is fixed with Singularity...
image, builder = Client.build(recipe=recipe,
robot_name=True,
sudo=False, stream=True)
app.logger.info('build %s' %image)
builder = itertools.chain(builder, [image])
return Response(builder, mimetype='text/plain')
def test_build_from_docker(tmp_path):
container = str(tmp_path / "container.sif")
created_container = Client.build(
"docker://busybox:1.30.1", image=container, sudo=False
)
assert created_container == container
assert os.path.exists(created_container)
def sandbox(tmp_path):
image = Client.build("docker://busybox:1.30.1",
image=str(tmp_path / 'sandbox'),
sandbox=True,
sudo=False)
assert os.path.exists(image)
config = os.path.join(get_installdir(), 'oci', 'config.json')
shutil.copyfile(config, os.path.join(image, 'config.json'))
return image
def build_to_singularity(definition_entry, container_location):
"""Builds a Singularity container from a Dockerfile or Singularity file
within the definition db.
Parameters:
definition_entry (str): Entry of definition db entry to build singularity container from.
container_location (str): Path to location to build the container.
Returns:
container_location: Returns the location of the Singularity container or None if it
fails to save.
"""
definition_id = definition_entry["definition_id"]
pull_s3_dir(definition_id)
Client.load(PROJECT_ROOT + definition_id)
Client.build(image=os.path.join(PROJECT_ROOT, container_location),
sudo=False)
shutil.rmtree(PROJECT_ROOT + definition_id)
#TODO Find a better way to error check
if os.path.exists(PROJECT_ROOT + container_location):
logging.info(f"Successfully built {container_location}")
return container_location
else:
return None
def _pull(self,
file_name,
names,
save=True,
force=False,
uri="docker://",
**kwargs):
"""pull an image from a docker hub. This is a (less than ideal) workaround
that actually does the following:
- creates a sandbox folder
- adds docker layers, metadata folder, and custom metadata to it
- converts to a squashfs image with build
the docker manifests are stored with registry metadata.
Parameters
==========
images: refers to the uri given by the user to pull in the format
<collection>/<namespace>. You should have an API that is able to
retrieve a container based on parsing this uri.
file_name: the user's requested name for the file. It can
optionally be None if the user wants a default.
save: if True, you should save the container to the database
using self.add()
Returns
=======
finished: a single container path, or list of paths
"""
# Use Singularity to build the image, based on user preference
if file_name is None:
file_name = self._get_storage_name(names)
# Determine if the user already has the image
if os.path.exists(file_name) and force is False:
bot.exit("Image exists! Remove first, or use --force to overwrite")
digest = names["version"] or names["tag"]
# Build from sandbox, prefix with sandbox
sandbox = get_tmpdir(prefix="sregistry-sandbox")
# First effort, get image via Sregistry
layers = self._download_layers(names["url"], digest)
# This is the url where the manifests were obtained
url = self._get_manifest_selfLink(names["url"], digest)
# Add environment to the layers
envtar = self._get_environment_tar()
layers = [envtar] + layers
# Create singularity image from an empty folder
for layer in layers:
bot.info("Exploding %s" % layer)
result = extract_tar(layer, sandbox, handle_whiteout=True)
if result["return_code"] != 0:
bot.exit(result["message"])
sudo = kwargs.get("sudo", False)
# Build from a sandbox (recipe) into the image_file (squashfs)
image_file = Singularity.build(image=file_name, recipe=sandbox, sudo=sudo)
# Fall back to using Singularity
if image_file is None:
bot.info("Downloading with native Singularity, please wait...")
image = file_name.replace("docker://", uri)
image_file = Singularity.pull(image, pull_folder=sandbox)
# Save to local storage
if save is True:
# Did we get the manifests?
manifests = {}
if hasattr(self, "manifests"):
manifests = self.manifests
container = self.add(image_path=image_file,
image_uri=names["uri"],
metadata=manifests,
url=url)
# When the container is created, this is the path to the image
image_file = container.image
if os.path.exists(image_file):
bot.debug("Retrieved image file %s" % image_file)
bot.custom(prefix="Success!", message=image_file)
# Clean up sandbox
shutil.rmtree(sandbox)
return image_file
def run_build(build_dir, params, verbose=True):
'''run_build takes a build directory and params dictionary, and does the following:
- downloads repo to a temporary directory
- changes branch or commit, if needed
- creates and bootstraps singularity image from Singularity file
- returns a dictionary with:
image (path), metadata (dict)
The following must be included in params:
spec_file, repo_url, branch, commit
'''
# Download the repository
download_repo(repo_url=params['repo_url'],
destination=build_dir)
os.chdir(build_dir)
if params['branch'] != None:
bot.info('Checking out branch %s' %params['branch'])
os.system('git checkout %s' %(params['branch']))
else:
params['branch'] = "master"
# Set the debug level
Client.debug = params['debug']
# Commit
if params['commit'] not in [None,'']:
bot.info('Checking out commit %s' %params['commit'])
os.system('git checkout %s .' %(params['commit']))
# From here on out commit is used as a unique id, if we don't have one, we use current
else:
params['commit'] = os.popen('git log -n 1 --pretty=format:"%H"').read()
bot.warning("commit not specified, setting to current %s" %params['commit'])
# Dump some params for the builder, in case it fails after this
passing_params = "/tmp/params.pkl"
pickle.dump(params, open(passing_params,'wb'))
# Now look for spec file
if os.path.exists(params['spec_file']):
bot.info("Found spec file %s in repository" %params['spec_file'])
# If the user has a symbolic link
if os.path.islink(params['spec_file']):
bot.info("%s is a symbolic link." %params['spec_file'])
params['spec_file'] = os.path.realpath(params['spec_file'])
# START TIMING
start_time = datetime.now()
# Secure Build
image = Client.build(recipe=params['spec_file'],
build_folder=build_dir,
isolated=True)
# Save has for metadata (also is image name)
version = get_image_file_hash(image)
params['version'] = version
pickle.dump(params, open(passing_params,'wb'))
# Rename image to be hash
finished_image = "%s/%s.simg" %(os.path.dirname(image), version)
image = shutil.move(image, finished_image)
final_time = (datetime.now() - start_time).seconds
bot.info("Final time of build %s seconds." %final_time)
# Did the container build successfully?
test_result = test_container(image)
if test_result['return_code'] != 0:
bot.error("Image failed to build, cancelling.")
sys.exit(1)
# Get singularity version
singularity_version = Client.version()
Client.debug = False
inspect = Client.inspect(image) # this is a string
Client.debug = params['debug']
# Get information on apps
Client.debug = False
app_names = Client.apps(image)
Client.debug = params['debug']
apps = extract_apps(image, app_names)
metrics = {'build_time_seconds': final_time,
'singularity_version': singularity_version,
'singularity_python_version': singularity_python_version,
'inspect': inspect,
'version': version,
'apps': apps}
output = {'image':image,
'metadata':metrics,
'params':params }
return output
else:
# Tell the user what is actually there
present_files = glob("*")
bot.error("Build file %s not found in repository" %params['spec_file'])
bot.info("Found files are %s" %"\n".join(present_files))
# Params have been exported, will be found by log
sys.exit(1)
def _pull(self, file_name, names, save=True, force=False, **kwargs):
'''pull an image from aws. This is a (less than ideal) workaround
that actually does the following:
- creates a sandbox folder
- adds docker layers from S3
- converts to a squashfs image with build
Parameters
==========
images: refers to the uri given by the user to pull in the format
<collection>/<namespace>. You should have an API that is able to
retrieve a container based on parsing this uri.
file_name: the user's requested name for the file. It can
optionally be None if the user wants a default.
save: if True, you should save the container to the database
using self.add()
Returns
=======
finished: a single container path, or list of paths
'''
# Use Singularity to build the image, based on user preference
if file_name is None:
file_name = self._get_storage_name(names)
# Determine if the user already has the image
if os.path.exists(file_name) and force is False:
bot.error('Image exists! Remove first, or use --force to overwrite')
sys.exit(1)
digest = names['version'] or names['tag']
# Build from sandbox
sandbox = get_tmpdir(prefix="sregistry-sandbox")
# First effort, get image via Sregistry
layers, url = self._download_layers(names['url'], digest)
# Add environment to the layers
envtar = self._get_environment_tar()
layers = [envtar] + layers
# Create singularity image from an empty folder
for layer in layers:
bot.info('Exploding %s' % layer)
result = extract_tar(layer, sandbox, handle_whiteout=True)
if result['return_code'] != 0:
bot.error(result['message'])
sys.exit(1)
sudo = kwargs.get('sudo', False)
# Build from a sandbox (recipe) into the image_file (squashfs)
image_file = Singularity.build(image=file_name, recipe=sandbox, sudo=sudo)
# Fall back to using Singularity
if image_file is None:
bot.info('Downloading with native Singularity, please wait...')
image = image.replace('aws://', 'docker://')
image_file = Singularity.pull(image, pull_folder=sandbox)
# Save to local storage
if save is True:
# Did we get the manifests?
manifests = {}
if hasattr(self, 'manifest'):
manifest = self.manifest
container = self.add(image_path=image_file,
image_uri=names['uri'],
metadata=manifest,
url=url)
# When the container is created, this is the path to the image
image_file = container.image
if os.path.exists(image_file):
bot.debug('Retrieved image file %s' % image_file)
bot.custom(prefix="Success!", message=image_file)
# Clean up sandbox
shutil.rmtree(sandbox)
return image_file
def singularity_build(self, path, image):
"""Builds an image from a recipefile.
"""
Client.build(os.path.join(
path, 'singularity.def'
), self.generate_image_name(image))
# remove the image, if it has been built before, including from local registry
#
try:
os.unlink(local_image)
except FileNotFoundError:
pass
if 'CIRCLECI' not in os.environ and use_sregistry is True:
sregistry_client.rm(registry_image)
#
# build the image
#
spython_client.quiet = False # circumvent problem with sregistry setting this attribute to True, which kills my local builds!
filename = spython_client.build(recipe='Singularity.' + image.name,
image=local_image,
sudo_options=sudo_options)
if filename is None:
print(("Error creating singularity image {}".format(local_image)))
sys.exit(1)
#
# add to local registry
#
if 'CIRCLECI' not in os.environ and use_sregistry is True:
sregistry_client.add(image_path=local_image,
image_uri=registry_image,
copy=True)
#
# push to GS bucket