def package(image_path,
spec_path=None,
output_folder=None,
remove_image=False,
verbose=False,
S=None):
'''generate a zip (including the image) to a user specified output_folder.
:param image_path: full path to singularity image file
:param runscript: if True, will extract runscript to include in package as runscript
:param software: if True, will extract files.txt and folders.txt to package
:param remove_image: if True, will not include original image in package (default,False)
:param verbose: be verbose when using singularity --export (default,False)
:param S: the Singularity object (optional) will be created if not required.
'''
# Run create image and bootstrap with Singularity command line tool.
S = Singularity(debug=verbose)
file_obj, tar = get_image_tar(image_path, S=S)
members = tar.getmembers()
image_name, ext = os.path.splitext(os.path.basename(image_path))
zip_name = "%s.zip" % (image_name.replace(" ", "_"))
# Include the image in the package?
to_package = dict()
if not remove_image:
to_package["files"] = [image_path]
# If the specfile is provided, it should also be packaged
if spec_path is not None:
singularity_spec = "".join(read_file(spec_path))
to_package['Singularity'] = singularity_spec
to_package["VERSION"] = get_image_file_hash(image_path)
try:
inspection = S.inspect(image_path)
to_package["inspect.json"] = inspection
inspection = json.loads(inspection)
to_package['runscript'] = inspection['data']['attributes']['runscript']
except:
bot.warning("Trouble extracting container metadata with inspect.")
bot.info("Adding software list to package.")
files = [x.path for x in members if x.isfile()]
folders = [x.path for x in members if x.isdir()]
to_package["files.txt"] = files
to_package["folders.txt"] = folders
# Do zip up here - let's start with basic structures
zipfile = zip_up(to_package,
zip_name=zip_name,
output_folder=output_folder)
bot.debug("Package created at %s" % (zipfile))
if not delete_image_tar(file_obj, tar):
bot.warning("Could not clean up temporary tarfile.")
# return package to user
return zipfile
def remove(self, image, force=False):
'''delete an image to Singularity Registry'''
q = parse_image_name(image)
url = '%s/container/%s/%s:%s' % (self.base, q["collection"], q["image"],
q["tag"])
SREGISTRY_EVENT = self.authorize(request_type="delete", names=q)
headers = {'Authorization': SREGISTRY_EVENT}
self.update_headers(fields=headers)
continue_delete = True
if force is False:
response = input("Are you sure you want to delete %s?" % q['uri'])
while len(response) < 1 or response[0].lower().strip(
) not in "ynyesno":
response = input("Please answer yes or no: ")
if response[0].lower().strip() in "no":
continue_delete = False
if continue_delete is True:
response = self.delete(url)
message = self.read_response(response)
bot.info("Response %s, %s" % (response.status_code, message))
else:
bot.info("Delete cancelled.")
def get_singularity_version(singularity_version=None):
'''get_singularity_version will determine the singularity version for a build
first, an environmental variable is looked at, followed by using the system
version.
'''
if singularity_version is None:
singularity_version = os.environ.get("SINGULARITY_VERSION")
if singularity_version is None:
try:
cmd = ['singularity', '--version']
output = run_command(cmd)
if isinstance(output['message'], bytes):
output['message'] = output['message'].decode('utf-8')
singularity_version = output['message'].strip('\n')
bot.info("Singularity %s being used." % singularity_version)
except:
singularity_version = None
bot.warning(
"Singularity version not found, so it's likely not installed.")
return singularity_version
def file_counts(container=None, patterns=None, image_package=None, diff=None):
'''file counts will return a list of files that match one or more regular expressions.
if no patterns is defined, a default of readme is used. All patterns and files are made
case insensitive.
:param container: if provided, will use container as image. Can also provide
:param image_package: if provided, can be used instead of container
:param patterns: one or more patterns (str or list) of files to search for.
:param diff: the difference between a container and it's parent OS from get_diff
if not provided, will be generated.
'''
if diff == None:
diff = get_diff(container=container, image_package=image_package)
if patterns == None:
patterns = 'readme'
if not isinstance(patterns, list):
patterns = [patterns]
count = 0
for folder, items in diff.items():
for pattern in patterns:
count += len(
[x for x in items if re.search(pattern.lower(), x.lower())])
bot.info("Total files matching patterns is %s" % count)
return count
def file_counts(container=None,
patterns=None,
image_package=None,
file_list=None):
'''file counts will return a list of files that match one or more regular expressions.
if no patterns is defined, a default of readme is used. All patterns and files are made
case insensitive.
Parameters
==========
:param container: if provided, will use container as image. Can also provide
:param image_package: if provided, can be used instead of container
:param patterns: one or more patterns (str or list) of files to search for.
:param diff: the difference between a container and it's parent OS from get_diff
if not provided, will be generated.
'''
if file_list is None:
file_list = get_container_contents(container, split_delim='\n')['all']
if patterns == None:
patterns = 'readme'
if not isinstance(patterns,list):
patterns = [patterns]
count = 0
for pattern in patterns:
count += len([x for x in file_list if re.search(pattern.lower(),x.lower())])
bot.info("Total files matching patterns is %s" %count)
return count
def clean_up(image, existed):
'''clean up will remove an image file if existed is False (meaning it was
created as temporary for the script
'''
if existed == False:
if os.path.exists(image):
bot.info("%s created was temporary, removing" % image)
os.remove(image)
def main(args, parser, subparser):
if args.recipe is None:
subparser.print_help()
bot.newline()
print("Please specify creating a recipe with --recipe")
sys.exit(0)
# Output folder will be pwd if not specified
output_folder = os.getcwd()
if args.outfolder is not None:
output_folder = os.getcwd()
bootstrap = ''
if args.bootstrap is not None:
bootstrap = args.bootstrap
bot.debug("bootstrap: %s" % bootstrap)
bootstrap_from = ''
if args.bootstrap_from is not None:
bootstrap_from = args.bootstrap_from
bot.debug("from: %s" % bootstrap_from)
template = "Singularity"
output_file = template
app = ''
if args.app is not None:
app = args.app.lower()
template = "Singularity.app"
output_file = "Singularity.%s" % app
input_file = "%s/cli/app/templates/recipes/%s" % (get_installdir(),
template)
output_file = "%s/%s" % (output_folder, output_file)
if os.path.exists(output_file):
ext = str(uuid.uuid4())[0:4]
output_file = "%s.%s" % (output_file, ext)
# Read the file, make substitutions
contents = read_file(input_file, readlines=False)
# Replace all occurrences of app
contents = contents.replace('{{ app }}', app)
contents = contents.replace('{{ bootstrap }}', bootstrap)
contents = contents.replace('{{ from }}', bootstrap_from)
write_file(output_file, contents)
bot.info("Output file written to %s" % output_file)
def sniff_extension(file_path, verbose=True):
'''sniff_extension will attempt to determine the file type based on the extension,
and return the proper mimetype
:param file_path: the full path to the file to sniff
:param verbose: print stuff out
'''
mime_types = {
"xls": 'application/vnd.ms-excel',
"xlsx":
'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
"xml": 'text/xml',
"ods": 'application/vnd.oasis.opendocument.spreadsheet',
"csv": 'text/plain',
"tmpl": 'text/plain',
"pdf": 'application/pdf',
"php": 'application/x-httpd-php',
"jpg": 'image/jpeg',
"png": 'image/png',
"gif": 'image/gif',
"bmp": 'image/bmp',
"txt": 'text/plain',
"doc": 'application/msword',
"js": 'text/js',
"swf": 'application/x-shockwave-flash',
"mp3": 'audio/mpeg',
"zip": 'application/zip',
"simg": 'application/zip',
"rar": 'application/rar',
"tar": 'application/tar',
"arj": 'application/arj',
"cab": 'application/cab',
"html": 'text/html',
"htm": 'text/html',
"default": 'application/octet-stream',
"folder": 'application/vnd.google-apps.folder',
"img": "application/octet-stream"
}
ext = os.path.basename(file_path).split('.')[-1]
mime_type = mime_types.get(ext, None)
if mime_type == None:
mime_type = mime_types['txt']
if verbose == True:
bot.info("%s --> %s" % (file_path, mime_type))
return mime_type
def make_package_tree(matrix=None,
labels=None,
width=25,
height=10,
title=None,
font_size=None):
'''make package tree will make a dendrogram comparing a matrix of packages
:param matrix: a pandas df of packages, with names in index and columns
:param labels: a list of labels corresponding to row names, will be
pulled from rows if not defined
:param title: a title for the plot, if not defined, will be left out.
:returns a plot that can be saved with savefig
'''
from matplotlib import pyplot as plt
from scipy.cluster.hierarchy import (dendrogram, linkage)
if font_size is None:
font_size = 8.
from scipy.cluster.hierarchy import cophenet
from scipy.spatial.distance import pdist
if not isinstance(matrix, pandas.DataFrame):
bot.info(
"No pandas DataFrame (matrix) of similarities defined, will use default."
)
matrix = compare_packages()['files.txt']
title = 'Docker Library Similarity to Base OS'
Z = linkage(matrix, 'ward')
c, coph_dists = cophenet(Z, pdist(matrix))
if labels == None:
labels = matrix.index.tolist()
plt.figure(figsize=(width, height))
if title != None:
plt.title(title)
plt.xlabel('image index')
plt.ylabel('distance')
dendrogram(
Z,
leaf_rotation=90., # rotates the x axis labels
leaf_font_size=font_size, # font size for the x axis labels
labels=labels)
return plt
def pull(self, images, file_name=None, decompress=True):
bot.debug('Execution of PULL for %s images' % len(images))
for image in images:
# If we need to decompress, it's old ext3 format
if decompress is True:
ext = 'img.gz'
else:
ext = 'simg' # squashfs
q = parse_image_name(image, ext=ext)
# Verify image existence, and obtain id
url = "%s/container/%s/%s:%s" % (self.base, q['collection'],
q['image'], q['tag'])
bot.debug('Retrieving manifest at %s' % url)
manifest = self.get(url)
bot.debug(manifest)
if file_name is None:
file_name = q['storage'].replace('/', '-')
image_file = self.download(url=manifest['image'],
file_name=file_name,
show_progress=True)
bot.debug('Retrieved image file %s' % image_file)
if os.path.exists(image_file) and decompress is True:
# If compressed, decompress
try:
cli = Singularity()
sys.stdout.write('Decompressing image ')
bot.spinner.start()
image_file = cli.decompress(image_file, quiet=True)
except KeyboardInterrupt:
bot.warning('Decompression cancelled.')
except:
bot.info('Image is not compressed.')
image_name = image_file.replace('.gz', '')
image_file = shutil.move(image_file, image_name)
pass
bot.spinner.stop()
bot.custom(prefix="Success!", message=image_file)
def sniff_extension(file_path,verbose=True):
'''sniff_extension will attempt to determine the file type based on the extension,
and return the proper mimetype
:param file_path: the full path to the file to sniff
:param verbose: print stuff out
'''
mime_types = { "xls": 'application/vnd.ms-excel',
"xlsx": 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
"xml": 'text/xml',
"ods": 'application/vnd.oasis.opendocument.spreadsheet',
"csv": 'text/plain',
"tmpl": 'text/plain',
"pdf": 'application/pdf',
"php": 'application/x-httpd-php',
"jpg": 'image/jpeg',
"png": 'image/png',
"gif": 'image/gif',
"bmp": 'image/bmp',
"txt": 'text/plain',
"doc": 'application/msword',
"js": 'text/js',
"swf": 'application/x-shockwave-flash',
"mp3": 'audio/mpeg',
"zip": 'application/zip',
"simg": 'application/zip',
"rar": 'application/rar',
"tar": 'application/tar',
"arj": 'application/arj',
"cab": 'application/cab',
"html": 'text/html',
"htm": 'text/html',
"default": 'application/octet-stream',
"folder": 'application/vnd.google-apps.folder',
"img" : "application/octet-stream" }
ext = os.path.basename(file_path).split('.')[-1]
mime_type = mime_types.get(ext,None)
if mime_type == None:
mime_type = mime_types['txt']
if verbose==True:
bot.info("%s --> %s" %(file_path, mime_type))
return mime_type
def main():
parser = get_parser()
subparsers = get_subparsers(parser)
try:
args = parser.parse_args()
except:
sys.exit(0)
# Not running in Singularity Hub environment
os.environ['SINGULARITY_HUB'] = "False"
# if environment logging variable not set, make silent
if args.debug is False:
os.environ['MESSAGELEVEL'] = "CRITICAL"
# Always print the version
from singularity.logger import bot
import singularity
bot.info("Singularity Python Version: %s" % singularity.__version__)
if args.command == "create":
from .create import main
if args.command == "compare":
from .compare import main
elif args.command == "analysis":
from .analysis import main
elif args.command == "inspect":
from .inspect import main
elif args.command == "package":
from .package import main
# Pass on to the correct parser
if args.command is not None:
main(args=args,
parser=parser,
subparser=subparsers[args.command])
else:
parser.print_help()
def check_install(software=None, quiet=True):
'''check_install will attempt to run the singularity command, and
return True if installed. The command line utils will not run
without this check.
'''
if software is None:
software = "singularity"
cmd = [software, '--version']
try:
version = run_command(cmd,software)
except: # FileNotFoundError
return False
if version is not None:
if quiet is False and version['return_code'] == 0:
version = version['message']
bot.info("Found %s version %s" % (software.upper(), version))
return True
return False
def check_install(software=None, quiet=True):
'''check_install will attempt to run the singularity command, and
return True if installed. The command line utils will not run
without this check.
'''
if software is None:
software = "singularity"
cmd = [software, '--version']
try:
version = run_command(cmd, software)
except: # FileNotFoundError
return False
if version is not None:
if quiet is False and version['return_code'] == 0:
version = version['message']
bot.info("Found %s version %s" % (software.upper(), version))
return True
return False
def label_search(self, key=None, value=None):
'''search across labels'''
if key is not None:
key = key.lower()
if value is not None:
value = value.lower()
show_details = True
if key is None and value is None:
url = '%s/labels/search' % (self.base)
show_details = False
elif key is not None and value is not None:
url = '%s/labels/search/%s/key/%s/value' % (self.base, key, value)
elif key is None:
url = '%s/labels/search/%s/value' % (self.base, value)
else:
url = '%s/labels/search/%s/key' % (self.base, key)
result = self.get(url)
if len(result) == 0:
bot.info("No labels found.")
sys.exit(0)
bot.info("Labels\n")
rows = []
for l in result:
if show_details is True:
entry = [
"%s:%s" % (l['key'], l['value']),
"\n%s\n\n" % "\n".join(l['containers'])
]
else:
entry = [
"N=%s" % len(l['containers']),
"%s:%s" % (l['key'], l['value'])
]
rows.append(entry)
bot.table(rows)
def collection_search(self, query):
'''collection search will list all containers for a specific
collection. We assume query is the name of a collection'''
query = query.lower().strip('/')
url = '%s/collection/%s' % (self.base, query)
result = self.get(url)
if len(result) == 0:
bot.info("No collections found.")
sys.exit(1)
bot.custom(prefix="COLLECTION", message=query)
rows = []
for container in result['containers']:
rows.append([container['uri'], container['detail']])
bot.table(rows)
def get_tags(container=None,
image_package=None,
search_folders=None,
diff=None,
return_unique=True):
'''get tags will return a list of tags that describe the software in an image,
meaning inside of a paricular folder. If search_folder is not defined, uses lib
:param container: if provided, will use container as image. Can also provide
:param image_package: if provided, can be used instead of container
:param search_folders: specify one or more folders to look for tags
:param diff: the difference between a container and it's parent OS from get_diff
if None, will be derived.
:param return_unique: return unique files in folders. Default True.
Default is 'bin'
::notes
The algorithm works as follows:
1) first compare package to set of base OS (provided with shub)
2) subtract the most similar os from image, leaving "custom" files
3) organize custom files into dict based on folder name
4) return search_folders as tags
'''
if diff == None:
diff = get_diff(container=container, image_package=image_package)
if search_folders == None:
search_folders = 'bin'
if not isinstance(search_folders, list):
search_folders = [search_folders]
tags = []
for search_folder in search_folders:
if search_folder in diff:
bot.info("Adding tags for folder %s" % search_folder)
tags = tags + diff[search_folder]
else:
bot.info("Did not find folder %s in difference." % search_folder)
if return_unique == True:
tags = list(set(tags))
return tags
def get_build_params(metadata):
'''get_build_params uses get_build_metadata to retrieve corresponding meta data values for a build
:param metadata: a list, each item a dictionary of metadata, in format:
metadata = [{'key': 'repo_url', 'value': repo_url },
{'key': 'repo_id', 'value': repo_id },
{'key': 'credential', 'value': credential },
{'key': 'response_url', 'value': response_url },
{'key': 'token', 'value': token},
{'key': 'commit', 'value': commit }]
'''
params = dict()
for item in metadata:
if item['value'] == None:
response = get_build_metadata(key=item['key'])
item['value'] = response
params[item['key']] = item['value']
if item['key'] not in ['token', 'secret', 'credential']:
bot.info('%s is set to %s' % (item['key'], item['value']))
return params
def search(self):
'''a "show all" search that doesn't require a query'''
url = '%s/collections/' % self.base
results = self.paginate_get(url)
if len(results) == 0:
bot.info("No container collections found.")
sys.exit(1)
bot.info("Collections")
rows = []
for result in results:
if "containers" in result:
for c in result['containers']:
rows.append([c['uri'], c['detail']])
bot.table(rows)
def get_build_params(metadata):
'''get_build_params uses get_build_metadata to retrieve corresponding meta data values for a build
:param metadata: a list, each item a dictionary of metadata, in format:
metadata = [{'key': 'repo_url', 'value': repo_url },
{'key': 'repo_id', 'value': repo_id },
{'key': 'credential', 'value': credential },
{'key': 'response_url', 'value': response_url },
{'key': 'token', 'value': token},
{'key': 'commit', 'value': commit }]
'''
params = dict()
for item in metadata:
if item['value'] == None:
response = get_build_metadata(key=item['key'])
item['value'] = response
params[item['key']] = item['value']
if item['key'] not in ['token', 'secret', 'credential']:
bot.info('%s is set to %s' %(item['key'],item['value']))
return params
def get_image_name(manifest,
extension='simg',
use_commit=False,
use_hash=False):
'''get_image_name will return the image name for a manifest. The user
can name based on a hash or commit, or a name with the collection,
namespace, branch, and tag.
'''
if use_hash:
image_name = "%s.%s" % (manifest['version'], extension)
elif use_commit:
image_name = "%s.%s" % (manifest['commit'], extension)
else:
image_name = "%s-%s-%s.%s" % (manifest['name'].replace(
'/', '-'), manifest['branch'].replace(
'/', '-'), manifest['tag'].replace('/', ''), extension)
bot.info("Singularity Hub Image: %s" % image_name)
return image_name
def get_packages(family=None):
'''get packages will return a list of packages (under some family)
provided by singularity python.If no name is specified, the default (os) will
be used.
:param name: the name of the package family to load
'''
package_base = get_package_base()
package_folders = glob("%s/*" % (package_base))
package_families = [os.path.basename(x) for x in package_folders]
if family == None:
family = "docker-os"
family = family.lower()
if family in package_families:
package_folder = "%s/%s" % (package_base, family)
packages = glob("%s/*.zip" % (package_folder))
bot.info("Found %s packages in family %s" % (len(packages), family))
return packages
bot.warning("Family %s not included. Options are %s" %
(family, ", ".join(package_families)))
return None
def get_singularity_version(singularity_version=None):
'''get_singularity_version will determine the singularity version for a build
first, an environmental variable is looked at, followed by using the system
version.
'''
if singularity_version is None:
singularity_version = os.environ.get("SINGULARITY_VERSION")
if singularity_version is None:
try:
cmd = ['singularity','--version']
output = run_command(cmd)
if isinstance(output['message'],bytes):
output['message'] = output['message'].decode('utf-8')
singularity_version = output['message'].strip('\n')
bot.info("Singularity %s being used." % singularity_version)
except:
singularity_version = None
bot.warning("Singularity version not found, so it's likely not installed.")
return singularity_version
def parse_container_name(image):
'''parse_container_name will return a json structure with a repo name, tag, user.
'''
container_name = image
if not is_number(image):
image = image.replace(' ', '')
# If the user provided a number (unique id for an image), return it
if is_number(image) == True:
bot.info("Numeric image ID %s found.", image)
return int(image)
image = image.split('/')
# If there are two parts, we have username with repo (and maybe tag)
if len(image) >= 2:
user = image[0]
image = image[1]
# Otherwise, we trigger error (not supported just usernames yet)
else:
bot.error(
'You must specify a repo name and username, %s is not valid' %
container_name)
sys.exit(1)
# Now split the name by : in case there is a tag
image = image.split(':')
if len(image) == 2:
repo_name = image[0]
repo_tag = image[1]
# Otherwise, assume latest of an image
else:
repo_name = image[0]
repo_tag = "latest"
bot.info("User: %s" % user)
bot.info("Repo Name: %s" % repo_name)
bot.info("Repo Tag: %s" % repo_tag)
parsed = {'repo_name': repo_name, 'repo_tag': repo_tag, 'user': user}
return parsed
def run_build(logfile='/tmp/.shub-log'):
'''run_build will generate the Singularity build from a spec_file from a repo_url.
If no arguments are required, the metadata api is queried for the values.
:param build_dir: directory to do the build in. If not specified, will use temporary.
:param spec_file: the spec_file name to use, assumed to be in git repo
:param repo_url: the url to download the repo from
:param repo_id: the repo_id to uniquely identify the repo (in case name changes)
:param commit: the commit to checkout. If none provided, will use most recent.
:param bucket_name: the name of the bucket to send files to
:param verbose: print out extra details as we go (default True)
:param token: a token to send back to the server to authenticate the collection
:param secret: a secret to match to the correct container
:param response_url: the build url to send the response back to. Should also come
from metadata. If not specified, no response is sent
:param branch: the branch to checkout for the build.
:: note: this function is currently configured to work with Google Compute
Engine metadata api, and should (will) be customized if needed to work elsewhere
'''
# If we are building the image, this will not be set
go = get_build_metadata(key='dobuild')
if go == None:
sys.exit(0)
# If the user wants debug, this will be set
debug = True
enable_debug = get_build_metadata(key='debug')
if enable_debug == None:
debug = False
bot.info('DEBUG %s' %debug)
# Uaw /tmp for build directory
build_dir = tempfile.mkdtemp()
# Get variables from the instance metadata API
metadata = [{'key': 'repo_url', 'value': None },
{'key': 'repo_id', 'value': None },
{'key': 'response_url', 'value': None },
{'key': 'bucket_name', 'value': None },
{'key': 'tag', 'value': None },
{'key': 'container_id', 'value': None },
{'key': 'commit', 'value': None },
{'key': 'token', 'value': None},
{'key': 'branch', 'value': None },
{'key': 'spec_file', 'value': None},
{'key': 'logging_url', 'value': None },
{'key': 'logfile', 'value': logfile }]
# Obtain values from build
bot.log("BUILD PARAMETERS:")
params = get_build_params(metadata)
params['debug'] = debug
# Default spec file is Singularity
if params['spec_file'] == None:
params['spec_file'] = "Singularity"
if params['bucket_name'] == None:
params['bucket_name'] = "singularityhub"
if params['tag'] == None:
params['tag'] = "latest"
output = run_build_main(build_dir=build_dir,
params=params)
# Output includes:
finished_image = output['image']
metadata = output['metadata']
params = output['params']
# Upload image package files to Google Storage
if os.path.exists(finished_image):
bot.info("%s successfully built" %finished_image)
dest_dir = tempfile.mkdtemp(prefix='build')
# The path to the images on google drive will be the github url/commit folder
trailing_path = "%s/%s" %(params['commit'], params['version'])
image_path = get_image_path(params['repo_url'], trailing_path)
# commits are no longer unique
# storage is by commit
build_files = [finished_image]
bot.info("Sending image to storage:")
bot.info('\n'.join(build_files))
# Start the storage service, retrieve the bucket
storage_service = get_google_service() # default is "storage" "v1"
bucket = get_bucket(storage_service,params["bucket_name"])
# For each file, upload to storage
files = []
for build_file in build_files:
bot.info("Uploading %s to storage..." %build_file)
storage_file = upload_file(storage_service,
bucket=bucket,
bucket_path=image_path,
file_name=build_file)
files.append(storage_file)
# Finally, package everything to send back to shub
response = {"files": json.dumps(files),
"repo_url": params['repo_url'],
"commit": params['commit'],
"repo_id": params['repo_id'],
"branch": params['branch'],
"tag": params['tag'],
"container_id": params['container_id'],
"spec_file":params['spec_file'],
"token": params['token'],
"metadata": json.dumps(metadata)}
# Did the user specify a specific log file?
custom_logfile = get_build_metadata('logfile')
if custom_logfile is not None:
logfile = custom_logfile
response['logfile'] = logfile
# Send final build data to instance
send_build_data(build_dir=build_dir,
response_url=params['response_url'],
secret=params['token'],
data=response)
# Dump final params, for logger to retrieve
passing_params = "/tmp/params.pkl"
pickle.dump(params,open(passing_params,'wb'))
def run_build(logfile='/tmp/.shub-log'):
'''run_build will generate the Singularity build from a spec_file from a repo_url.
If no arguments are required, the metadata api is queried for the values.
:param build_dir: directory to do the build in. If not specified, will use temporary.
:param spec_file: the spec_file name to use, assumed to be in git repo
:param repo_url: the url to download the repo from
:param repo_id: the repo_id to uniquely identify the repo (in case name changes)
:param commit: the commit to checkout. If none provided, will use most recent.
:param bucket_name: the name of the bucket to send files to
:param verbose: print out extra details as we go (default True)
:param token: a token to send back to the server to authenticate the collection
:param secret: a secret to match to the correct container
:param response_url: the build url to send the response back to. Should also come
from metadata. If not specified, no response is sent
:param branch: the branch to checkout for the build.
:: note: this function is currently configured to work with Google Compute
Engine metadata api, and should (will) be customized if needed to work elsewhere
'''
# If we are building the image, this will not be set
go = get_build_metadata(key='dobuild')
if go == None:
sys.exit(0)
# If the user wants debug, this will be set
debug = True
enable_debug = get_build_metadata(key='debug')
if enable_debug == None:
debug = False
bot.info('DEBUG %s' % debug)
# Uaw /tmp for build directory
build_dir = tempfile.mkdtemp()
# Get variables from the instance metadata API
metadata = [{
'key': 'repo_url',
'value': None
}, {
'key': 'repo_id',
'value': None
}, {
'key': 'response_url',
'value': None
}, {
'key': 'bucket_name',
'value': None
}, {
'key': 'tag',
'value': None
}, {
'key': 'container_id',
'value': None
}, {
'key': 'commit',
'value': None
}, {
'key': 'token',
'value': None
}, {
'key': 'branch',
'value': None
}, {
'key': 'spec_file',
'value': None
}, {
'key': 'logging_url',
'value': None
}, {
'key': 'logfile',
'value': logfile
}]
# Obtain values from build
bot.log("BUILD PARAMETERS:")
params = get_build_params(metadata)
params['debug'] = debug
# Default spec file is Singularity
if params['spec_file'] == None:
params['spec_file'] = "Singularity"
if params['bucket_name'] == None:
params['bucket_name'] = "singularityhub"
if params['tag'] == None:
params['tag'] = "latest"
output = run_build_main(build_dir=build_dir, params=params)
# Output includes:
image_package = output['image_package']
finished_image = output['image']
metadata = output['metadata']
params = output['params']
# Upload image package files to Google Storage
if os.path.exists(image_package):
bot.info("Package %s successfully built" % image_package)
dest_dir = tempfile.mkdtemp(prefix='build')
with zipfile.ZipFile(image_package) as zf:
zf.extractall(dest_dir)
# The path to the images on google drive will be the github url/commit folder
trailing_path = "%s/%s" % (params['commit'], params['version'])
image_path = get_image_path(params['repo_url'], trailing_path)
# commits are no longer unique
# storage is by commit
build_files = glob("%s/*" % (dest_dir))
build_files.append(finished_image)
bot.info("Sending build files to storage:")
bot.info('\n'.join(build_files))
# Start the storage service, retrieve the bucket
storage_service = get_google_service() # default is "storage" "v1"
bucket = get_bucket(storage_service, params["bucket_name"])
# For each file, upload to storage
files = []
for build_file in build_files:
bot.info("Uploading %s to storage..." % build_file)
storage_file = upload_file(storage_service,
bucket=bucket,
bucket_path=image_path,
file_name=build_file)
files.append(storage_file)
# Finally, package everything to send back to shub
response = {
"files": json.dumps(files),
"repo_url": params['repo_url'],
"commit": params['commit'],
"repo_id": params['repo_id'],
"branch": params['branch'],
"tag": params['tag'],
"container_id": params['container_id'],
"spec_file": params['spec_file'],
"token": params['token'],
"metadata": json.dumps(metadata)
}
# Did the user specify a specific log file?
custom_logfile = get_build_metadata('logfile')
if custom_logfile is not None:
logfile = custom_logfile
response['logfile'] = logfile
# Send final build data to instance
send_build_data(build_dir=build_dir,
response_url=params['response_url'],
secret=params['token'],
data=response)
# Dump final params, for logger to retrieve
passing_params = "/tmp/params.pkl"
pickle.dump(params, open(passing_params, 'wb'))
def run_build(build_dir, params, verbose=True):
'''run_build takes a build directory and params dictionary, and does the following:
- downloads repo to a temporary directory
- changes branch or commit, if needed
- creates and bootstraps singularity image from Singularity file
- returns a dictionary with:
image (path), metadata (dict)
The following must be included in params:
spec_file, repo_url, branch, commit
'''
# Download the repository
download_repo(repo_url=params['repo_url'],
destination=build_dir)
os.chdir(build_dir)
if params['branch'] != None:
bot.info('Checking out branch %s' %params['branch'])
os.system('git checkout %s' %(params['branch']))
else:
params['branch'] = "master"
# Set the debug level
Client.debug = params['debug']
# Commit
if params['commit'] not in [None,'']:
bot.info('Checking out commit %s' %params['commit'])
os.system('git checkout %s .' %(params['commit']))
# From here on out commit is used as a unique id, if we don't have one, we use current
else:
params['commit'] = os.popen('git log -n 1 --pretty=format:"%H"').read()
bot.warning("commit not specified, setting to current %s" %params['commit'])
# Dump some params for the builder, in case it fails after this
passing_params = "/tmp/params.pkl"
pickle.dump(params, open(passing_params,'wb'))
# Now look for spec file
if os.path.exists(params['spec_file']):
bot.info("Found spec file %s in repository" %params['spec_file'])
# If the user has a symbolic link
if os.path.islink(params['spec_file']):
bot.info("%s is a symbolic link." %params['spec_file'])
params['spec_file'] = os.path.realpath(params['spec_file'])
# START TIMING
start_time = datetime.now()
# Secure Build
image = Client.build(recipe=params['spec_file'],
build_folder=build_dir,
isolated=True)
# Save has for metadata (also is image name)
version = get_image_file_hash(image)
params['version'] = version
pickle.dump(params, open(passing_params,'wb'))
# Rename image to be hash
finished_image = "%s/%s.simg" %(os.path.dirname(image), version)
image = shutil.move(image, finished_image)
final_time = (datetime.now() - start_time).seconds
bot.info("Final time of build %s seconds." %final_time)
# Did the container build successfully?
test_result = test_container(image)
if test_result['return_code'] != 0:
bot.error("Image failed to build, cancelling.")
sys.exit(1)
# Get singularity version
singularity_version = Client.version()
Client.debug = False
inspect = Client.inspect(image) # this is a string
Client.debug = params['debug']
# Get information on apps
Client.debug = False
app_names = Client.apps(image)
Client.debug = params['debug']
apps = extract_apps(image, app_names)
metrics = {'build_time_seconds': final_time,
'singularity_version': singularity_version,
'singularity_python_version': singularity_python_version,
'inspect': inspect,
'version': version,
'apps': apps}
output = {'image':image,
'metadata':metrics,
'params':params }
return output
else:
# Tell the user what is actually there
present_files = glob("*")
bot.error("Build file %s not found in repository" %params['spec_file'])
bot.info("Found files are %s" %"\n".join(present_files))
# Params have been exported, will be found by log
sys.exit(1)
def container_search(self,
query,
across_collections=False,
environment=False,
deffile=False,
runscript=False,
test=False):
'''search for a specific container. If across collections is False,
the query is parsed as a full container name and a specific container
is returned. If across_collections is True, the container is searched
for across collections. If across collections is True, details are
not shown'''
query = query.lower().strip('/')
q = parse_image_name(query, defaults=False)
if q['tag'] is not None:
if across_collections is True:
url = '%s/container/search/name/%s/tag/%s' % (self.base,
q['image'], q['tag'])
else:
url = '%s/container/search/collection/%s/name/%s/tag/%s' % (
self.base, q['collection'], q['image'], q['tag'])
elif q['tag'] is None:
if across_collections is True:
url = '%s/container/search/name/%s' % (self.base, q['image'])
else:
url = '%s/container/search/collection/%s/name/%s' % (
self.base, q['collection'], q['image'])
result = self.get(url)
if "containers" in result:
result = result['containers']
if len(result) == 0:
bot.info("No containers found.")
sys.exit(1)
bot.info("Containers %s" % query)
rows = []
for c in result:
# Convert date to readable thing
datetime_object = parser.parse(c['add_date'])
print_date = datetime_object.strftime('%b %d, %Y %I:%M%p')
rows.append(
['%s/%s' % (c['collection'], c['name']), c['tag'], print_date])
bot.table(rows)
# Finally, show metadata and other values
if test is True or deffile is True or environment is True or runscript is True:
bot.newline()
for c in result:
metadata = c['metadata']
if test is True:
bot.custom(prefix='%test',
message=metadata['test'],
color="CYAN")
bot.newline()
if deffile is True:
bot.custom(prefix='Singularity',
message=metadata['deffile'],
color="CYAN")
bot.newline()
if environment is True:
bot.custom(prefix='%environment',
message=metadata['environment'],
color="CYAN")
bot.newline()
if runscript is True:
bot.custom(prefix='%runscript',
message=metadata['runscript'],
color="CYAN")
bot.newline()
def run_build(build_dir,params,verbose=True, compress_image=False):
'''run_build takes a build directory and params dictionary, and does the following:
- downloads repo to a temporary directory
- changes branch or commit, if needed
- creates and bootstraps singularity image from Singularity file
- returns a dictionary with:
image (path), image_package (path), metadata (dict)
The following must be included in params:
spec_file, repo_url, branch, commit
Optional parameters
size
'''
# Download the repo and image
download_repo(repo_url=params['repo_url'],
destination=build_dir)
os.chdir(build_dir)
if params['branch'] != None:
bot.info('Checking out branch %s' %params['branch'])
os.system('git checkout %s' %(params['branch']))
else:
params['branch'] = "master"
# Commit
if params['commit'] not in [None,'']:
bot.info('Checking out commit %s' %params['commit'])
os.system('git checkout %s .' %(params['commit']))
# From here on out commit is used as a unique id, if we don't have one, we use current
else:
params['commit'] = os.popen('git log -n 1 --pretty=format:"%H"').read()
bot.warning("commit not specified, setting to current %s" %params['commit'])
# Dump some params for the builder, in case it fails after this
passing_params = "/tmp/params.pkl"
pickle.dump(params,open(passing_params,'wb'))
# Now look for spec file
if os.path.exists(params['spec_file']):
bot.info("Found spec file %s in repository" %params['spec_file'])
# If the user has a symbolic link
if os.path.islink(params['spec_file']):
bot.info("%s is a symbolic link." %params['spec_file'])
params['spec_file'] = os.path.realpath(params['spec_file'])
# START TIMING
start_time = datetime.now()
image = build_from_spec(spec_file=params['spec_file'], # default will package the image
build_dir=build_dir,
isolated=True,
sandbox=False,
debug=params['debug'])
# Save has for metadata (also is image name)
version = get_image_file_hash(image)
params['version'] = version
pickle.dump(params,open(passing_params,'wb'))
final_time = (datetime.now() - start_time).seconds
bot.info("Final time of build %s seconds." %final_time)
# Did the container build successfully?
test_result = test_container(image)
if test_result['return_code'] != 0:
bot.error("Image failed to build, cancelling.")
sys.exit(1)
# Get singularity version
singularity_version = get_singularity_version()
# Package the image metadata (files, folders, etc)
image_package = package(image_path=image,
spec_path=params['spec_file'],
output_folder=build_dir,
remove_image=True,
verbose=True)
# Derive software tags by subtracting similar OS
diff = get_diff(image_package=image_package)
# Inspect to get labels and other metadata
cli = Singularity(debug=params['debug'])
inspect = cli.inspect(image_path=image)
# Get information on apps
app_names = cli.apps(image_path=image)
apps = extract_apps(image_path=image, app_names=app_names)
# Count file types, and extensions
counts = dict()
counts['readme'] = file_counts(diff=diff)
counts['copyright'] = file_counts(diff=diff,patterns=['copyright'])
counts['authors-thanks-credit'] = file_counts(diff=diff,
patterns=['authors','thanks','credit','contributors'])
counts['todo'] = file_counts(diff=diff,patterns=['todo'])
extensions = extension_counts(diff=diff)
os_sims = estimate_os(image_package=image_package,return_top=False)
most_similar = os_sims['SCORE'].values.argmax()
most_similar = os_sims['SCORE'].index.tolist()[most_similar]
metrics = {'build_time_seconds':final_time,
'singularity_version':singularity_version,
'singularity_python_version':singularity_python_version,
'estimated_os': most_similar,
'os_sims':os_sims['SCORE'].to_dict(),
'file_counts':counts,
'file_ext':extensions,
'inspect':inspect,
'version': version,
'apps': apps}
# Compress Image
if compress_image is True:
compressed_image = "%s.gz" %image
os.system('gzip -c -9 %s > %s' %(image,compressed_image))
image = compressed_image
output = {'image':image,
'image_package':image_package,
'metadata':metrics,
'params':params }
return output
else:
# Tell the user what is actually there
present_files = glob("*")
bot.error("Build file %s not found in repository" %params['spec_file'])
bot.info("Found files are %s" %"\n".join(present_files))
# Params have been exported, will be found by log
sys.exit(1)
