def main(args, parser, subparser):
# We can only continue if singularity is installed
if check_install() is not True:
bot.error("Cannot find Singularity! Is it installed?")
sys.exit(1)
# Output folder will be pwd if not specified
output_folder = os.getcwd()
if args.outfolder is not None:
output_folder = os.getcwd()
# If we are given an image, ensure full path
if args.image is not None:
if not os.path.exists(args.image):
cli = Singularity(debug=args.debug)
image = cli.pull(args.image)
if image is None:
bot.error("Cannot find image. Exiting.")
sys.exit(1)
# the user wants to make a tree
if args.tree is True:
from singularity.cli.app import make_tree
make_tree(image)
clean_up(image, existed)
else:
print("Please specify an image to inspect with --image")
subparser.print_help()
def get_container_contents(container=None,
gets=None,
split_delim=None,
image_package=None):
'''get_container_contents will return a list of folders and or files
for a container. The environmental variable SINGULARITY_HUB being set
means that container objects are referenced instead of packages
:param container: the container to get content for
:param gets: a list of file names to return, without parent folders
:param split_delim: if defined, will split text by split delimiter
:param image_package: if defined, user has provided an image_package
'''
from singularity.package import package, load_package
if container == None and image_package == None:
bot.error("You must define an image package or container.")
sys.exit(1)
# Default returns are the list of files and folders
if gets == None:
gets = ['files.txt', 'folders.txt']
if not isinstance(gets, list):
gets = [gets]
# We will look for everything in guts, then return it
guts = dict()
SINGULARITY_HUB = os.environ.get('SINGULARITY_HUB', "False")
# Visualization deployed local or elsewhere
if SINGULARITY_HUB == "False":
tmpdir = tempfile.mkdtemp()
if image_package == None:
image_package = package(image_path=container,
output_folder=tmpdir,
remove_image=True)
guts = load_package(image_package, get=gets)
shutil.rmtree(tmpdir)
# Visualization deployed by singularity hub
else:
# user has provided a package, but not a container
if container == None:
guts = load_package(image_package, get=gets)
# user has provided a container, but not a package
else:
for sfile in container.files:
for gut_key in gets:
if os.path.basename(sfile['name']) == gut_key:
if split_delim == None:
guts[gut_key] = requests.get(
sfile['mediaLink']).text
else:
guts[gut_key] = requests.get(
sfile['mediaLink']).text.split(split_delim)
return guts
def run_command(self, cmd, sudo=False, quiet=False):
'''run_command is a wrapper for the global run_command, checking first
for sudo and exiting on error if needed
:param cmd: the command to run
:param sudo: does the command require sudo?
On success, returns result. Otherwise, exists on error
'''
result = run_command(cmd,sudo=sudo)
message = result['message']
return_code = result['return_code']
if result['return_code'] == 0:
if isinstance(message,bytes):
try:
message=message.decode('utf-8')
except UnicodeDecodeError:
try:
message = unicode(message, errors='replace')
except NameError:
message = str(message, errors='replace')
return message
if quiet is False:
bot.error("Return Code %s: %s" %(return_code,
message))
sys.exit(1)
def main(args, parser, subparser):
# We can only continue if singularity is installed
if check_install() is not True:
bot.error("Cannot find Singularity! Is it installed?")
sys.exit(1)
# Output folder will be pwd if not specified
output_folder = os.getcwd()
if args.outfolder is not None:
output_folder = os.getcwd()
# If we are given an image, ensure full path
if args.image is not None:
image = args.image
if not os.path.exists(image):
cli = Singularity(debug=args.debug)
image = cli.pull(image)
if image is None:
bot.error("Cannot find image. Exiting.")
sys.exit(1)
remove_image = not args.include_image
package(image_path=image,
output_folder=output_folder,
remove_image=remove_image)
else:
print("Please specify an image to package with --image")
subparser.print_help()
def get_levels(version=None):
'''get_levels returns a dictionary of levels (key) and values (dictionaries with
descriptions and regular expressions for files) for the user.
:param version: the version of singularity to use (default is 2.2)
:param include_files: files to add to the level, only relvant if
'''
valid_versions = ['2.3', '2.2']
if version is None:
version = "2.3"
version = str(version)
if version not in valid_versions:
bot.error("Unsupported version %s, valid versions are %s" %
(version, ",".join(valid_versions)))
levels_file = os.path.abspath(
os.path.join(get_installdir(), 'analysis', 'reproduce', 'data',
'reproduce_levels.json'))
levels = read_json(levels_file)
if version == "2.2":
# Labels not added until 2.3
del levels['LABELS']
levels = make_levels_set(levels)
return levels
def get_levels(version=None):
'''get_levels returns a dictionary of levels (key) and values (dictionaries with
descriptions and regular expressions for files) for the user.
:param version: the version of singularity to use (default is 2.2)
:param include_files: files to add to the level, only relvant if
'''
valid_versions = ['2.3','2.2']
if version is None:
version = "2.3"
version = str(version)
if version not in valid_versions:
bot.error("Unsupported version %s, valid versions are %s" %(version,
",".join(valid_versions)))
levels_file = os.path.abspath(os.path.join(get_installdir(),
'analysis',
'reproduce',
'data',
'reproduce_levels.json'))
levels = read_json(levels_file)
if version == "2.2":
# Labels not added until 2.3
del levels['LABELS']
levels = make_levels_set(levels)
return levels
def unpack_node(image_path,name=None,output_folder=None,size=None):
'''unpackage node is intended to unpackage a node that was packaged with
package_node. The image should be a .tgz file. The general steps are to:
1. Package the node using the package_node function
2. Transfer the package somewhere that Singularity is installed'''
if not image_path.endswith(".tgz"):
bot.error("The image_path should end with .tgz. Did you create with package_node?")
sys.exit(1)
if output_folder is None:
output_folder = os.path.dirname(os.path.abspath(image_path))
image_name = os.path.basename(image_path)
if name is None:
name = image_name.replace('.tgz','.img')
if not name.endswith('.img'):
name = "%s.img" %(name)
bot.debug("Preparing to unpack %s to %s." %(image_name,name))
unpacked_image = "%s/%s" %(output_folder,name)
if not os.path.exists(unpacked_image):
os.mkdir(unpacked_image)
cmd = ["gunzip","-dc",image_path,"|","sudo","singularity","import", unpacked_image]
output = run_command(cmd)
# TODO: singularity mount the container, cleanup files (/etc/fstab,...)
# and add your custom singularity files.
return unpacked_image
def pull(self,image_path,pull_folder='',
name_by_hash=False,
name_by_commit=False,
image_name=None,
size=None):
'''pull will pull a singularity hub image
:param image_path: full path to image / uris
:param name_by: can be one of commit or hash, default is by image name
'''
if image_name is not None:
name_by_hash=False
name_by_commit=False
final_image = None
if not image_path.startswith('shub://') and not image_path.startswith('docker://'):
bot.error("pull is only valid for docker and shub, %s is invalid." %image_name)
sys.exit(1)
if self.debug is True:
cmd = ['singularity','--debug','pull']
else:
cmd = ['singularity','pull']
if pull_folder not in [None,'']:
os.environ['SINGULARITY_PULLFOLDER'] = pull_folder
pull_folder = "%s/" % pull_folder
if image_path.startswith('shub://'):
if image_name is not None:
bot.debug("user specified naming pulled image %s" %image_name)
cmd = cmd +["--name",image_name]
elif name_by_commit is True:
bot.debug("user specified naming by commit.")
cmd.append("--commit")
elif name_by_hash is True:
bot.debug("user specified naming by hash.")
cmd.append("--hash")
# otherwise let the Singularity client determine own name
elif image_path.startswith('docker://'):
if size is not None:
cmd = cmd + ["--size",size]
if image_name is None:
image_name = "%s" %image_path.replace("docker://","").replace("/","-")
final_image = "%s%s.img" %(pull_folder,image_name)
cmd = cmd + ["--name", image_name]
cmd.append(image_path)
bot.debug(' '.join(cmd))
output = self.run_command(cmd)
self.println(output)
if final_image is None: # shub
final_image = output.split('Container is at:')[-1].strip('\n').strip()
return final_image
def compress(self,image_path):
'''compress will (properly) compress an image'''
if os.path.exists(image_path):
compressed_image = "%s.gz" %image_path
os.system('gzip -c -6 %s > %s' %(image_path,compressed_image))
return compressed_image
else:
bot.error("Cannot find image %s" %image_path)
sys.exit(1)
def decompress(self,image_path,quiet=True):
'''decompress will (properly) decompress an image'''
if not os.path.exists(image_path):
bot.error("Cannot find image %s" %image_path)
sys.exit(1)
extracted_file = image_path.replace('.gz','')
cmd = ['gzip','-d','-f', image_path]
result = self.run_command(cmd, quiet=quiet) # exits if return code != 0
return extracted_file
def main(args, parser, subparser):
# We can only continue if singularity is installed
if check_install() is not True:
bot.error("Cannot find Singularity! Is it installed?")
sys.exit(1)
# Output folder will be pwd if not specified
output_folder = os.getcwd()
if args.outfolder is not None:
output_folder = os.getcwd()
# If we are given an image, ensure full path
image = args.image
if image is not None:
existed = True
if not os.path.exists(image):
cli = Singularity(debug=args.debug)
image = cli.pull(image)
existed = False
if image is None:
bot.error("Cannot find image. Exiting.")
sys.exit(1)
# The user wants to estimate the os
if args.os is True:
from singularity.analysis.classify import estimate_os
estimated_os = estimate_os(container=image)
print(estimated_os)
# The user wants to get a list of all os
elif args.oscalc is True:
from singularity.analysis.classify import estimate_os
estimated_os = estimate_os(container=image, return_top=False)
print(estimated_os["SCORE"].to_dict())
# The user wants to get a list of tags
elif args.tags is True:
from singularity.analysis.classify import get_tags
tags = get_tags(container=image)
print(tags)
# The user wants to plot image vs. the docker os
elif args.osplot is True:
from singularity.app import plot_os_sims
plot_os_sims(image)
clean_up(image, existed)
else:
print("Please specify an image to analyze with --image")
subparser.print_help()
def mkdir_p(path):
'''mkdir_p attempts to get the same functionality as mkdir -p
:param path: the path to create.
'''
try:
os.makedirs(path)
except OSError as e:
if e.errno == errno.EEXIST and os.path.isdir(path):
pass
else:
bot.error("Error creating path %s, exiting." % path)
sys.exit(1)
def get_script(script_name):
'''get_script will return a build script_name, if it is included
in singularity/build/scripts, otherwise will alert the user and return None
:param script_name: the name of the script to look for
'''
install_dir = get_installdir()
script_path = "%s/build/scripts/%s" %(install_dir,script_name)
if os.path.exists(script_path):
return script_path
else:
bot.error("Script %s is not included in singularity-python!" %script_path)
return None
def mkdir_p(path):
'''mkdir_p attempts to get the same functionality as mkdir -p
:param path: the path to create.
'''
try:
os.makedirs(path)
except OSError as e:
if e.errno == errno.EEXIST and os.path.isdir(path):
pass
else:
bot.error("Error creating path %s, exiting." % path)
sys.exit(1)
def get_image_tar(image_path):
'''get an image tar, either written in memory or to
the file system. file_obj will either be the file object,
or the file itself.
'''
bot.debug('Generate file system tar...')
file_obj = Client.image.export(image_path=image_path)
if file_obj is None:
bot.error("Error generating tar, exiting.")
sys.exit(1)
tar = tarfile.open(file_obj)
return file_obj, tar
def get_script(script_name):
'''get_script will return a build script_name, if it is included
in singularity/build/scripts, otherwise will alert the user and return None
:param script_name: the name of the script to look for
'''
install_dir = get_installdir()
script_path = "%s/build/scripts/%s" % (install_dir, script_name)
if os.path.exists(script_path):
return script_path
else:
bot.error("Script %s is not included in singularity-python!" %
script_path)
return None
def main(args, parser, subparser):
# We can only continue if singularity is installed
if check_install() is not True:
bot.error("Cannot find Singularity! Is it installed?")
sys.exit(1)
# Output folder will be pwd if not specified
output_folder = os.getcwd()
if args.outfolder is not None:
output_folder = os.getcwd()
if args.images is not None:
image1, image2 = args.images.split(',')
bot.debug("Image1: %s" % image1)
bot.debug("Image2: %s" % image2)
images = dict()
cli = Singularity(debug=args.debug)
for image in [image1, image2]:
existed = True
if not os.path.exists(image):
image = cli.pull(image)
existed = False
images[image] = existed
# Just for clarity
image1, image2 = list(images.keys())
# the user wants to make a similarity tree
if args.simtree is True:
from singularity.cli.app import make_sim_tree
make_sim_tree(image1, image2)
# the user wants to make a difference tree
elif args.subtract is True:
from singularity.cli.app import make_diff_tree
make_diff_tree(image1, image2)
else: # If none specified, just print score
from singularity.analysis.compare import calculate_similarity
score = calculate_similarity(image1, image2, by="files.txt")
print(score["files.txt"])
for image, existed in images.items():
clean_up(image, existed)
else:
print("Please specify images to compare with --images")
subparser.print_help()
def get_image_tar(image_path, S=None):
'''get an image tar, either written in memory or to
the file system. file_obj will either be the file object,
or the file itself.
'''
bot.debug('Generate file system tar...')
if S is None:
S = Singularity()
file_obj = S.export(image_path=image_path)
if file_obj is None:
bot.error("Error generating tar, exiting.")
sys.exit(1)
tar = tarfile.open(file_obj)
return file_obj, tar
def main(args, parser, subparser):
# Does the user have a valid image?
image = args.image[0]
if not os.path.exists(image):
subparser.print_help()
bot.error("Please supply one or more paths to existing images.")
sys.exit(1)
# Authenticate
sreg = Client(secrets=args.secrets)
response = sreg.push(path=image,
name=args.name,
tag=args.tag,
compress=args.compress)
def download(self, url, file_name, headers=None, show_progress=True):
'''stream to a temporary file, rename on successful completion
:param file_name: the file name to stream to
:param url: the url to stream from
:param headers: additional headers to add
'''
fd, tmp_file = tempfile.mkstemp(prefix=("%s.tmp." % file_name))
os.close(fd)
response = self.stream(url, headers=headers, stream_to=tmp_file)
if isinstance(response, HTTPError):
bot.error("Error downloading %s, exiting." % url)
sys.exit(1)
shutil.move(tmp_file, file_name)
return file_name
def parse_container_name(image):
'''parse_container_name will return a json structure with a repo name, tag, user.
'''
container_name = image
if not is_number(image):
image = image.replace(' ', '')
# If the user provided a number (unique id for an image), return it
if is_number(image) == True:
bot.info("Numeric image ID %s found.", image)
return int(image)
image = image.split('/')
# If there are two parts, we have username with repo (and maybe tag)
if len(image) >= 2:
user = image[0]
image = image[1]
# Otherwise, we trigger error (not supported just usernames yet)
else:
bot.error(
'You must specify a repo name and username, %s is not valid' %
container_name)
sys.exit(1)
# Now split the name by : in case there is a tag
image = image.split(':')
if len(image) == 2:
repo_name = image[0]
repo_tag = image[1]
# Otherwise, assume latest of an image
else:
repo_name = image[0]
repo_tag = "latest"
bot.info("User: %s" % user)
bot.info("Repo Name: %s" % repo_name)
bot.info("Repo Tag: %s" % repo_tag)
parsed = {'repo_name': repo_name, 'repo_tag': repo_tag, 'user': user}
return parsed
def create(self,image_path,size=None,sudo=False):
'''create will create a a new image
:param image_path: full path to image
:param size: image sizein MiB, default is 1024MiB
:param filesystem: supported file systems ext3/ext4 (ext[2/3]: default ext3
'''
if size == None:
size=1024
if self.debug == True:
cmd = ['singularity','--debug','image.create','--size',str(size),image_path]
else:
cmd = ['singularity','image.create','--size',str(size),image_path]
output = self.run_command(cmd,sudo=sudo)
self.println(output)
if not os.path.exists(image_path):
bot.error("Could not create image %s" %image_path)
sys.exit(1)
return image_path
def getenv(variable_key, required=False, default=None, silent=False):
'''getenv will attempt to get an environment variable. If the variable
is not found, None is returned.
:param variable_key: the variable name
:param required: exit with error if not found
:param silent: Do not print debugging information for variable
'''
variable = os.environ.get(variable_key, default)
if variable is None and required:
bot.error("Cannot find environment variable %s, exiting." %
variable_key)
sys.exit(1)
if silent:
bot.verbose2("%s found" % variable_key)
else:
if variable is not None:
bot.verbose2("%s found as %s" % (variable_key, variable))
else:
bot.verbose2("%s not defined (None)" % variable_key)
return variable
def read_client_secrets(secrets=None,required=True):
# If token file not provided, check environment
if secrets is None:
secrets = os.environ.get("SREGISTRY_CLIENT_SECRETS")
# Fall back to default
if secrets is None:
userhome = pwd.getpwuid(os.getuid())[5]
secrets = "%s/.sregistry" % (userhome)
if secrets is not None:
if os.path.exists(secrets):
return read_json(secrets)
message = 'Client secrets file not found at %s or $SREGISTRY_CLIENT_SECRETS.' %secrets
if required:
bot.error(message)
sys.exit(1)
bot.warning(message)
return None
def call(self, url, func, data=None, return_json=True, stream=False):
'''call will issue the call, and issue a refresh token
given a 401 response.
:param func: the function (eg, post, get) to call
:param url: the url to send file to
:param data: additional data to add to the request
:param return_json: return json if successful
'''
if data is not None:
if not isinstance(data, dict):
data = json.dumps(data)
response = func(url=url,
headers=self.headers,
data=data,
stream=stream)
# Errored response, try again with refresh
if response.status_code in [500, 502]:
bot.error("Beep boop! %s: %s" %
(response.reason, response.status_code))
sys.exit(1)
# Errored response, try again with refresh
if response.status_code == 404:
bot.error("Beep boop! %s: %s" %
(response.reason, response.status_code))
sys.exit(1)
# Errored response, try again with refresh
if response.status_code == 401:
bot.error("Your credentials are expired! %s: %s" %
(response.reason, response.status_code))
sys.exit(1)
elif response.status_code == 200:
if return_json:
try:
response = response.json()
except ValueError:
bot.error("The server returned a malformed response.")
sys.exit(1)
return response
def stream(self, url, headers=None, stream_to=None):
'''stream is a get that will stream to file_name
'''
bot.debug("GET %s" % url)
if headers == None:
headers = self._init_headers()
response = requests.get(url, headers=headers, stream=True)
if response.status_code == 200:
# Keep user updated with Progress Bar?
content_size = None
if 'Content-Length' in response.headers:
progress = 0
content_size = int(response.headers['Content-Length'])
bot.show_progress(progress, content_size, length=35)
chunk_size = 1 << 20
with open(stream_to, 'wb') as filey:
for chunk in response.iter_content(chunk_size=chunk_size):
filey.write(chunk)
if content_size is not None:
progress += chunk_size
bot.show_progress(iteration=progress,
total=content_size,
length=35,
carriage_return=False)
# Newline to finish download
sys.stdout.write('\n')
return stream_to
bot.error("Problem with stream, response %s" % (response.status_code))
sys.exit(1)
def unpack_node(image_path, name=None, output_folder=None, size=None):
'''unpackage node is intended to unpackage a node that was packaged with
package_node. The image should be a .tgz file. The general steps are to:
1. Package the node using the package_node function
2. Transfer the package somewhere that Singularity is installed'''
if not image_path.endswith(".tgz"):
bot.error(
"The image_path should end with .tgz. Did you create with package_node?"
)
sys.exit(1)
if output_folder is None:
output_folder = os.path.dirname(os.path.abspath(image_path))
image_name = os.path.basename(image_path)
if name is None:
name = image_name.replace('.tgz', '.img')
if not name.endswith('.img'):
name = "%s.img" % (name)
bot.debug("Preparing to unpack %s to %s." % (image_name, name))
unpacked_image = "%s/%s" % (output_folder, name)
S = Singularity(sudo=True)
S.create(image_path=unpacked_image, size=size)
cmd = [
"gunzip", "-dc", image_path, "|", "sudo", "singularity", "import",
unpacked_image
]
output = run_command(cmd)
# TODO: singularity mount the container, cleanup files (/etc/fstab,...)
# and add your custom singularity files.
return unpacked_image
def run_build(build_dir,params,verbose=True, compress_image=False):
'''run_build takes a build directory and params dictionary, and does the following:
- downloads repo to a temporary directory
- changes branch or commit, if needed
- creates and bootstraps singularity image from Singularity file
- returns a dictionary with:
image (path), image_package (path), metadata (dict)
The following must be included in params:
spec_file, repo_url, branch, commit
Optional parameters
size
'''
# Download the repo and image
download_repo(repo_url=params['repo_url'],
destination=build_dir)
os.chdir(build_dir)
if params['branch'] != None:
bot.info('Checking out branch %s' %params['branch'])
os.system('git checkout %s' %(params['branch']))
else:
params['branch'] = "master"
# Commit
if params['commit'] not in [None,'']:
bot.info('Checking out commit %s' %params['commit'])
os.system('git checkout %s .' %(params['commit']))
# From here on out commit is used as a unique id, if we don't have one, we use current
else:
params['commit'] = os.popen('git log -n 1 --pretty=format:"%H"').read()
bot.warning("commit not specified, setting to current %s" %params['commit'])
# Dump some params for the builder, in case it fails after this
passing_params = "/tmp/params.pkl"
pickle.dump(params,open(passing_params,'wb'))
# Now look for spec file
if os.path.exists(params['spec_file']):
bot.info("Found spec file %s in repository" %params['spec_file'])
# If the user has a symbolic link
if os.path.islink(params['spec_file']):
bot.info("%s is a symbolic link." %params['spec_file'])
params['spec_file'] = os.path.realpath(params['spec_file'])
# START TIMING
start_time = datetime.now()
image = build_from_spec(spec_file=params['spec_file'], # default will package the image
build_dir=build_dir,
isolated=True,
sandbox=False,
debug=params['debug'])
# Save has for metadata (also is image name)
version = get_image_file_hash(image)
params['version'] = version
pickle.dump(params,open(passing_params,'wb'))
final_time = (datetime.now() - start_time).seconds
bot.info("Final time of build %s seconds." %final_time)
# Did the container build successfully?
test_result = test_container(image)
if test_result['return_code'] != 0:
bot.error("Image failed to build, cancelling.")
sys.exit(1)
# Get singularity version
singularity_version = get_singularity_version()
# Package the image metadata (files, folders, etc)
image_package = package(image_path=image,
spec_path=params['spec_file'],
output_folder=build_dir,
remove_image=True,
verbose=True)
# Derive software tags by subtracting similar OS
diff = get_diff(image_package=image_package)
# Inspect to get labels and other metadata
cli = Singularity(debug=params['debug'])
inspect = cli.inspect(image_path=image)
# Get information on apps
app_names = cli.apps(image_path=image)
apps = extract_apps(image_path=image, app_names=app_names)
# Count file types, and extensions
counts = dict()
counts['readme'] = file_counts(diff=diff)
counts['copyright'] = file_counts(diff=diff,patterns=['copyright'])
counts['authors-thanks-credit'] = file_counts(diff=diff,
patterns=['authors','thanks','credit','contributors'])
counts['todo'] = file_counts(diff=diff,patterns=['todo'])
extensions = extension_counts(diff=diff)
os_sims = estimate_os(image_package=image_package,return_top=False)
most_similar = os_sims['SCORE'].values.argmax()
most_similar = os_sims['SCORE'].index.tolist()[most_similar]
metrics = {'build_time_seconds':final_time,
'singularity_version':singularity_version,
'singularity_python_version':singularity_python_version,
'estimated_os': most_similar,
'os_sims':os_sims['SCORE'].to_dict(),
'file_counts':counts,
'file_ext':extensions,
'inspect':inspect,
'version': version,
'apps': apps}
# Compress Image
if compress_image is True:
compressed_image = "%s.gz" %image
os.system('gzip -c -9 %s > %s' %(image,compressed_image))
image = compressed_image
output = {'image':image,
'image_package':image_package,
'metadata':metrics,
'params':params }
return output
else:
# Tell the user what is actually there
present_files = glob("*")
bot.error("Build file %s not found in repository" %params['spec_file'])
bot.info("Found files are %s" %"\n".join(present_files))
# Params have been exported, will be found by log
sys.exit(1)
def push(self, path, name, tag=None, compress=False):
'''push an image to Singularity Registry'''
path = os.path.abspath(path)
image = os.path.basename(path)
bot.debug("PUSH %s" % path)
if not os.path.exists(path):
bot.error('%s does not exist.' %path)
sys.exit(1)
cli = Singularity()
metadata = cli.inspect(image_path=path, quiet=True)
metadata = json.loads(metadata)
# Try to add the size
try:
image_size = os.path.getsize(path) >> 20
if metadata['data']['attributes']['labels'] is None:
metadata['data']['attributes']['labels'] = {'SREGISTRY_SIZE_MB': image_size }
else:
metadata['data']['attributes']['labels']['SREGISTRY_SIZE_MB'] = image_size
except:
bot.warning("Cannot load metadata to add calculated size.")
pass
if "deffile" in metadata['data']['attributes']:
if metadata['data']['attributes']['deffile'] is not None:
fromimage = parse_header(metadata['data']['attributes']['deffile'],
header="from",
remove_header=True)
metadata['data']['attributes']['labels']['SREGISTRY_FROM'] = fromimage
bot.debug("%s was built from a definition file." % image)
if compress is True:
ext = 'img.gz' # ext3 format
else:
ext = 'simg' # ext3 format
metadata = json.dumps(metadata)
names = parse_image_name(name,tag=tag, ext=ext)
url = '%s/push/' % self.base
if compress is True:
try:
sys.stdout.write('Compressing image ')
bot.spinner.start()
upload_from = cli.compress(path)
bot.spinner.stop()
except KeyboardInterrupt:
print('Upload cancelled')
if os.path.exists("%s.gz" %path):
os.remove("%s.gz" %path)
sys.exit(1)
else:
upload_from = path
upload_to = os.path.basename(names['storage'])
SREGISTRY_EVENT = self.authorize(request_type="push",
names=names)
encoder = MultipartEncoder(fields={'collection': names['collection'],
'name':names['image'],
'metadata':metadata,
'tag': names['tag'],
'datafile': (upload_to, open(upload_from, 'rb'), 'text/plain')})
progress_callback = create_callback(encoder)
monitor = MultipartEncoderMonitor(encoder, progress_callback)
headers = {'Content-Type': monitor.content_type,
'Authorization': SREGISTRY_EVENT }
try:
r = requests.post(url, data=monitor, headers=headers)
message = self.read_response(r)
print('\n[Return status {0} {1}]'.format(r.status_code, message))
except KeyboardInterrupt:
print('\nUpload cancelled.')
# Clean up
if compress is True:
if os.path.exists("%s.gz" %path):
os.remove("%s.gz" %path)
def run_build(build_dir, params, verbose=True):
'''run_build takes a build directory and params dictionary, and does the following:
- downloads repo to a temporary directory
- changes branch or commit, if needed
- creates and bootstraps singularity image from Singularity file
- returns a dictionary with:
image (path), metadata (dict)
The following must be included in params:
spec_file, repo_url, branch, commit
'''
# Download the repository
download_repo(repo_url=params['repo_url'],
destination=build_dir)
os.chdir(build_dir)
if params['branch'] != None:
bot.info('Checking out branch %s' %params['branch'])
os.system('git checkout %s' %(params['branch']))
else:
params['branch'] = "master"
# Set the debug level
Client.debug = params['debug']
# Commit
if params['commit'] not in [None,'']:
bot.info('Checking out commit %s' %params['commit'])
os.system('git checkout %s .' %(params['commit']))
# From here on out commit is used as a unique id, if we don't have one, we use current
else:
params['commit'] = os.popen('git log -n 1 --pretty=format:"%H"').read()
bot.warning("commit not specified, setting to current %s" %params['commit'])
# Dump some params for the builder, in case it fails after this
passing_params = "/tmp/params.pkl"
pickle.dump(params, open(passing_params,'wb'))
# Now look for spec file
if os.path.exists(params['spec_file']):
bot.info("Found spec file %s in repository" %params['spec_file'])
# If the user has a symbolic link
if os.path.islink(params['spec_file']):
bot.info("%s is a symbolic link." %params['spec_file'])
params['spec_file'] = os.path.realpath(params['spec_file'])
# START TIMING
start_time = datetime.now()
# Secure Build
image = Client.build(recipe=params['spec_file'],
build_folder=build_dir,
isolated=True)
# Save has for metadata (also is image name)
version = get_image_file_hash(image)
params['version'] = version
pickle.dump(params, open(passing_params,'wb'))
# Rename image to be hash
finished_image = "%s/%s.simg" %(os.path.dirname(image), version)
image = shutil.move(image, finished_image)
final_time = (datetime.now() - start_time).seconds
bot.info("Final time of build %s seconds." %final_time)
# Did the container build successfully?
test_result = test_container(image)
if test_result['return_code'] != 0:
bot.error("Image failed to build, cancelling.")
sys.exit(1)
# Get singularity version
singularity_version = Client.version()
Client.debug = False
inspect = Client.inspect(image) # this is a string
Client.debug = params['debug']
# Get information on apps
Client.debug = False
app_names = Client.apps(image)
Client.debug = params['debug']
apps = extract_apps(image, app_names)
metrics = {'build_time_seconds': final_time,
'singularity_version': singularity_version,
'singularity_python_version': singularity_python_version,
'inspect': inspect,
'version': version,
'apps': apps}
output = {'image':image,
'metadata':metrics,
'params':params }
return output
else:
# Tell the user what is actually there
present_files = glob("*")
bot.error("Build file %s not found in repository" %params['spec_file'])
bot.info("Found files are %s" %"\n".join(present_files))
# Params have been exported, will be found by log
sys.exit(1)
