def send_build_data(build_dir, data, secret,
response_url=None,clean_up=True):
'''finish build sends the build and data (response) to a response url
:param build_dir: the directory of the build
:response_url: where to send the response. If None, won't send
:param data: the data object to send as a post
:param clean_up: If true (default) removes build directory
'''
# Send with Authentication header
body = '%s|%s|%s|%s|%s' %(data['container_id'],
data['commit'],
data['branch'],
data['token'],
data['tag'])
signature = generate_header_signature(secret=secret,
payload=body,
request_type="push")
headers = {'Authorization': signature }
if response_url is not None:
finish = requests.post(response_url,data=data, headers=headers)
bot.debug("RECEIVE POST TO SINGULARITY HUB ---------------------")
bot.debug(finish.status_code)
bot.debug(finish.reason)
else:
bot.warning("response_url set to None, skipping sending of build.")
if clean_up == True:
shutil.rmtree(build_dir)
# Delay a bit, to give buffer between bringing instance down
time.sleep(20)
def make_interactive_tree(matrix=None,labels=None):
'''make interactive tree will return complete html for an interactive tree
:param title: a title for the plot, if not defined, will be left out.
'''
from scipy.cluster.hierarchy import (
dendrogram,
linkage,
to_tree
)
d3 = None
from scipy.cluster.hierarchy import cophenet
from scipy.spatial.distance import pdist
if isinstance(matrix,pandas.DataFrame):
Z = linkage(matrix, 'ward') # clusters
T = to_tree(Z, rd=False)
if labels == None:
labels = matrix.index.tolist()
lookup = dict(zip(range(len(labels)), labels))
# Create a dendrogram object without plotting
dend = dendrogram(Z,no_plot=True,
orientation="right",
leaf_rotation=90., # rotates the x axis labels
leaf_font_size=8., # font size for the x axis labels
labels=labels)
d3 = dict(children=[], name="root")
add_node(T, d3)
label_tree(d3["children"][0],lookup)
else:
bot.warning('Please provide data as pandas Data Frame.')
return d3
def get_singularity_version(singularity_version=None):
'''get_singularity_version will determine the singularity version for a build
first, an environmental variable is looked at, followed by using the system
version.
'''
if singularity_version is None:
singularity_version = os.environ.get("SINGULARITY_VERSION")
if singularity_version is None:
try:
cmd = ['singularity', '--version']
output = run_command(cmd)
if isinstance(output['message'], bytes):
output['message'] = output['message'].decode('utf-8')
singularity_version = output['message'].strip('\n')
bot.info("Singularity %s being used." % singularity_version)
except:
singularity_version = None
bot.warning(
"Singularity version not found, so it's likely not installed.")
return singularity_version
def upload_file(storage_service,bucket,bucket_path,file_name,verbose=True):
'''get_folder will return the folder with folder_name, and if create=True,
will create it if not found. If folder is found or created, the metadata is
returned, otherwise None is returned
:param storage_service: the drive_service created from get_storage_service
:param bucket: the bucket object from get_bucket
:param file_name: the name of the file to upload
:param bucket_path: the path to upload to
'''
# Set up path on bucket
upload_path = "%s/%s" %(bucket['id'],bucket_path)
if upload_path[-1] != '/':
upload_path = "%s/" %(upload_path)
upload_path = "%s%s" %(upload_path,os.path.basename(file_name))
body = {'name': upload_path }
# Create media object with correct mimetype
if os.path.exists(file_name):
mimetype = sniff_extension(file_name,verbose=verbose)
media = http.MediaFileUpload(file_name,
mimetype=mimetype,
resumable=True)
request = storage_service.objects().insert(bucket=bucket['id'],
body=body,
predefinedAcl="publicRead",
media_body=media)
result = request.execute()
return result
bot.warning('%s requested for upload does not exist, skipping' %file_name)
def package(image_path,
spec_path=None,
output_folder=None,
remove_image=False,
verbose=False,
S=None):
'''generate a zip (including the image) to a user specified output_folder.
:param image_path: full path to singularity image file
:param runscript: if True, will extract runscript to include in package as runscript
:param software: if True, will extract files.txt and folders.txt to package
:param remove_image: if True, will not include original image in package (default,False)
:param verbose: be verbose when using singularity --export (default,False)
:param S: the Singularity object (optional) will be created if not required.
'''
# Run create image and bootstrap with Singularity command line tool.
S = Singularity(debug=verbose)
file_obj, tar = get_image_tar(image_path, S=S)
members = tar.getmembers()
image_name, ext = os.path.splitext(os.path.basename(image_path))
zip_name = "%s.zip" % (image_name.replace(" ", "_"))
# Include the image in the package?
to_package = dict()
if not remove_image:
to_package["files"] = [image_path]
# If the specfile is provided, it should also be packaged
if spec_path is not None:
singularity_spec = "".join(read_file(spec_path))
to_package['Singularity'] = singularity_spec
to_package["VERSION"] = get_image_file_hash(image_path)
try:
inspection = S.inspect(image_path)
to_package["inspect.json"] = inspection
inspection = json.loads(inspection)
to_package['runscript'] = inspection['data']['attributes']['runscript']
except:
bot.warning("Trouble extracting container metadata with inspect.")
bot.info("Adding software list to package.")
files = [x.path for x in members if x.isfile()]
folders = [x.path for x in members if x.isdir()]
to_package["files.txt"] = files
to_package["folders.txt"] = folders
# Do zip up here - let's start with basic structures
zipfile = zip_up(to_package,
zip_name=zip_name,
output_folder=output_folder)
bot.debug("Package created at %s" % (zipfile))
if not delete_image_tar(file_obj, tar):
bot.warning("Could not clean up temporary tarfile.")
# return package to user
return zipfile
def send_build_data(build_dir, data, secret,
response_url=None,clean_up=True):
'''finish build sends the build and data (response) to a response url
:param build_dir: the directory of the build
:response_url: where to send the response. If None, won't send
:param data: the data object to send as a post
:param clean_up: If true (default) removes build directory
'''
# Send with Authentication header
body = '%s|%s|%s|%s|%s' %(data['container_id'],
data['commit'],
data['branch'],
data['token'],
data['tag'])
signature = generate_header_signature(secret=secret,
payload=body,
request_type="push")
headers = {'Authorization': signature }
if response_url is not None:
finish = requests.post(response_url,data=data, headers=headers)
bot.debug("RECEIVE POST TO SINGULARITY HUB ---------------------")
bot.debug(finish.status_code)
bot.debug(finish.reason)
else:
bot.warning("response_url set to None, skipping sending of build.")
if clean_up == True:
shutil.rmtree(build_dir)
# Delay a bit, to give buffer between bringing instance down
time.sleep(20)
def get_build_template(template_name,params=None,to_file=None):
'''get_build template returns a string or file for a particular build template, which is
intended to build a version of a Singularity image on a cloud resource.
:param template_name: the name of the template to retrieve in build/scripts
:param params: (if needed) a dictionary of parameters to substitute in the file
:param to_file: if defined, will write to file. Default returns string.
'''
base = get_installdir()
template_folder = "%s/build/scripts" %(base)
template_file = "%s/%s" %(template_folder,template_name)
if os.path.exists(template_file):
bot.debug("Found template %s" %template_file)
# Implement when needed - substitute params here
# Will need to read in file instead of copying below
# if params != None:
if to_file is not None:
shutil.copyfile(template_file,to_file)
bot.debug("Template file saved to %s" %to_file)
return to_file
# If the user wants a string
content = ''.join(read_file(template_file))
return content
else:
bot.warning("Template %s not found." %template_file)
def get_build_template(template_name, params=None, to_file=None):
'''get_build template returns a string or file for a particular build template, which is
intended to build a version of a Singularity image on a cloud resource.
:param template_name: the name of the template to retrieve in build/scripts
:param params: (if needed) a dictionary of parameters to substitute in the file
:param to_file: if defined, will write to file. Default returns string.
'''
base = get_installdir()
template_folder = "%s/build/scripts" % (base)
template_file = "%s/%s" % (template_folder, template_name)
if os.path.exists(template_file):
bot.debug("Found template %s" % template_file)
# Implement when needed - substitute params here
# Will need to read in file instead of copying below
# if params != None:
if to_file is not None:
shutil.copyfile(template_file, to_file)
bot.debug("Template file saved to %s" % to_file)
return to_file
# If the user wants a string
content = ''.join(read_file(template_file))
return content
else:
bot.warning("Template %s not found." % template_file)
def modify_level(level, field, values, append=True):
'''modify level is intended to add / modify a content type.
Default content type is list, meaning the entry is appended.
If you set append to False, the content will be overwritten
For any other content type, the entry is overwritten.
'''
field = field.lower()
valid_fields = ['regexp', 'skip_files', 'include_files']
if field not in valid_fields:
bot.warning("%s is not a valid field, skipping. Choices are %s" %
(field, ",".join(valid_fields)))
return level
if append:
if not isinstance(values, list):
values = [values]
if field in level:
level[field] = level[field] + values
else:
level[field] = values
else:
level[field] = values
level = make_level_set(level)
return level
def upload_file(storage_service, bucket, bucket_path, file_name, verbose=True):
'''get_folder will return the folder with folder_name, and if create=True,
will create it if not found. If folder is found or created, the metadata is
returned, otherwise None is returned
:param storage_service: the drive_service created from get_storage_service
:param bucket: the bucket object from get_bucket
:param file_name: the name of the file to upload
:param bucket_path: the path to upload to
'''
# Set up path on bucket
upload_path = "%s/%s" % (bucket['id'], bucket_path)
if upload_path[-1] != '/':
upload_path = "%s/" % (upload_path)
upload_path = "%s%s" % (upload_path, os.path.basename(file_name))
body = {'name': upload_path}
# Create media object with correct mimetype
if os.path.exists(file_name):
mimetype = sniff_extension(file_name, verbose=verbose)
media = http.MediaFileUpload(file_name,
mimetype=mimetype,
resumable=True)
request = storage_service.objects().insert(bucket=bucket['id'],
body=body,
predefinedAcl="publicRead",
media_body=media)
result = request.execute()
return result
bot.warning('%s requested for upload does not exist, skipping' % file_name)
def pull(self, images, file_name=None, decompress=True):
bot.debug('Execution of PULL for %s images' % len(images))
for image in images:
# If we need to decompress, it's old ext3 format
if decompress is True:
ext = 'img.gz'
else:
ext = 'simg' # squashfs
q = parse_image_name(image, ext=ext)
# Verify image existence, and obtain id
url = "%s/container/%s/%s:%s" % (self.base, q['collection'],
q['image'], q['tag'])
bot.debug('Retrieving manifest at %s' % url)
manifest = self.get(url)
bot.debug(manifest)
if file_name is None:
file_name = q['storage'].replace('/', '-')
image_file = self.download(url=manifest['image'],
file_name=file_name,
show_progress=True)
bot.debug('Retrieved image file %s' % image_file)
if os.path.exists(image_file) and decompress is True:
# If compressed, decompress
try:
cli = Singularity()
sys.stdout.write('Decompressing image ')
bot.spinner.start()
image_file = cli.decompress(image_file, quiet=True)
except KeyboardInterrupt:
bot.warning('Decompression cancelled.')
except:
bot.info('Image is not compressed.')
image_name = image_file.replace('.gz', '')
image_file = shutil.move(image_file, image_name)
pass
bot.spinner.stop()
bot.custom(prefix="Success!", message=image_file)
def get_packages(family=None):
'''get packages will return a list of packages (under some family)
provided by singularity python.If no name is specified, the default (os) will
be used.
:param name: the name of the package family to load
'''
package_base = get_package_base()
package_folders = glob("%s/*" % (package_base))
package_families = [os.path.basename(x) for x in package_folders]
if family == None:
family = "docker-os"
family = family.lower()
if family in package_families:
package_folder = "%s/%s" % (package_base, family)
packages = glob("%s/*.zip" % (package_folder))
bot.info("Found %s packages in family %s" % (len(packages), family))
return packages
bot.warning("Family %s not included. Options are %s" %
(family, ", ".join(package_families)))
return None
def read_client_secrets(secrets=None,required=True):
# If token file not provided, check environment
if secrets is None:
secrets = os.environ.get("SREGISTRY_CLIENT_SECRETS")
# Fall back to default
if secrets is None:
userhome = pwd.getpwuid(os.getuid())[5]
secrets = "%s/.sregistry" % (userhome)
if secrets is not None:
if os.path.exists(secrets):
return read_json(secrets)
message = 'Client secrets file not found at %s or $SREGISTRY_CLIENT_SECRETS.' %secrets
if required:
bot.error(message)
sys.exit(1)
bot.warning(message)
return None
def get_level(level, version=None, include_files=None, skip_files=None):
'''get_level returns a single level, with option to customize files
added and skipped.
'''
levels = get_levels(version=version)
level_names = list(levels.keys())
if level.upper() in level_names:
level = levels[level]
else:
bot.warning("%s is not a valid level. Options are %s" %
(level.upper(), "\n".join(levels)))
return None
# Add additional files to skip or remove, if defined
if skip_files is not None:
level = modify_level(level, 'skip_files', skip_files)
if include_files is not None:
level = modify_level(level, 'include_files', include_files)
level = make_level_set(level)
return level
def get_level(level,version=None,include_files=None,skip_files=None):
'''get_level returns a single level, with option to customize files
added and skipped.
'''
levels = get_levels(version=version)
level_names = list(levels.keys())
if level.upper() in level_names:
level = levels[level]
else:
bot.warning("%s is not a valid level. Options are %s" %(level.upper(),
"\n".join(levels)))
return None
# Add additional files to skip or remove, if defined
if skip_files is not None:
level = modify_level(level,'skip_files',skip_files)
if include_files is not None:
level = modify_level(level,'include_files',include_files)
level = make_level_set(level)
return level
def get_singularity_version(singularity_version=None):
'''get_singularity_version will determine the singularity version for a build
first, an environmental variable is looked at, followed by using the system
version.
'''
if singularity_version is None:
singularity_version = os.environ.get("SINGULARITY_VERSION")
if singularity_version is None:
try:
cmd = ['singularity','--version']
output = run_command(cmd)
if isinstance(output['message'],bytes):
output['message'] = output['message'].decode('utf-8')
singularity_version = output['message'].strip('\n')
bot.info("Singularity %s being used." % singularity_version)
except:
singularity_version = None
bot.warning("Singularity version not found, so it's likely not installed.")
return singularity_version
def modify_level(level,field,values,append=True):
'''modify level is intended to add / modify a content type.
Default content type is list, meaning the entry is appended.
If you set append to False, the content will be overwritten
For any other content type, the entry is overwritten.
'''
field = field.lower()
valid_fields = ['regexp','skip_files','include_files']
if field not in valid_fields:
bot.warning("%s is not a valid field, skipping. Choices are %s" %(field,",".join(valid_fields)))
return level
if append:
if not isinstance(values,list):
values = [values]
if field in level:
level[field] = level[field] + values
else:
level[field] = values
else:
level[field] = values
level = make_level_set(level)
return level
def push(self, path, name, tag=None, compress=False):
'''push an image to Singularity Registry'''
path = os.path.abspath(path)
image = os.path.basename(path)
bot.debug("PUSH %s" % path)
if not os.path.exists(path):
bot.error('%s does not exist.' %path)
sys.exit(1)
cli = Singularity()
metadata = cli.inspect(image_path=path, quiet=True)
metadata = json.loads(metadata)
# Try to add the size
try:
image_size = os.path.getsize(path) >> 20
if metadata['data']['attributes']['labels'] is None:
metadata['data']['attributes']['labels'] = {'SREGISTRY_SIZE_MB': image_size }
else:
metadata['data']['attributes']['labels']['SREGISTRY_SIZE_MB'] = image_size
except:
bot.warning("Cannot load metadata to add calculated size.")
pass
if "deffile" in metadata['data']['attributes']:
if metadata['data']['attributes']['deffile'] is not None:
fromimage = parse_header(metadata['data']['attributes']['deffile'],
header="from",
remove_header=True)
metadata['data']['attributes']['labels']['SREGISTRY_FROM'] = fromimage
bot.debug("%s was built from a definition file." % image)
if compress is True:
ext = 'img.gz' # ext3 format
else:
ext = 'simg' # ext3 format
metadata = json.dumps(metadata)
names = parse_image_name(name,tag=tag, ext=ext)
url = '%s/push/' % self.base
if compress is True:
try:
sys.stdout.write('Compressing image ')
bot.spinner.start()
upload_from = cli.compress(path)
bot.spinner.stop()
except KeyboardInterrupt:
print('Upload cancelled')
if os.path.exists("%s.gz" %path):
os.remove("%s.gz" %path)
sys.exit(1)
else:
upload_from = path
upload_to = os.path.basename(names['storage'])
SREGISTRY_EVENT = self.authorize(request_type="push",
names=names)
encoder = MultipartEncoder(fields={'collection': names['collection'],
'name':names['image'],
'metadata':metadata,
'tag': names['tag'],
'datafile': (upload_to, open(upload_from, 'rb'), 'text/plain')})
progress_callback = create_callback(encoder)
monitor = MultipartEncoderMonitor(encoder, progress_callback)
headers = {'Content-Type': monitor.content_type,
'Authorization': SREGISTRY_EVENT }
try:
r = requests.post(url, data=monitor, headers=headers)
message = self.read_response(r)
print('\n[Return status {0} {1}]'.format(r.status_code, message))
except KeyboardInterrupt:
print('\nUpload cancelled.')
# Clean up
if compress is True:
if os.path.exists("%s.gz" %path):
os.remove("%s.gz" %path)
def run_build(build_dir,params,verbose=True, compress_image=False):
'''run_build takes a build directory and params dictionary, and does the following:
- downloads repo to a temporary directory
- changes branch or commit, if needed
- creates and bootstraps singularity image from Singularity file
- returns a dictionary with:
image (path), image_package (path), metadata (dict)
The following must be included in params:
spec_file, repo_url, branch, commit
Optional parameters
size
'''
# Download the repo and image
download_repo(repo_url=params['repo_url'],
destination=build_dir)
os.chdir(build_dir)
if params['branch'] != None:
bot.info('Checking out branch %s' %params['branch'])
os.system('git checkout %s' %(params['branch']))
else:
params['branch'] = "master"
# Commit
if params['commit'] not in [None,'']:
bot.info('Checking out commit %s' %params['commit'])
os.system('git checkout %s .' %(params['commit']))
# From here on out commit is used as a unique id, if we don't have one, we use current
else:
params['commit'] = os.popen('git log -n 1 --pretty=format:"%H"').read()
bot.warning("commit not specified, setting to current %s" %params['commit'])
# Dump some params for the builder, in case it fails after this
passing_params = "/tmp/params.pkl"
pickle.dump(params,open(passing_params,'wb'))
# Now look for spec file
if os.path.exists(params['spec_file']):
bot.info("Found spec file %s in repository" %params['spec_file'])
# If the user has a symbolic link
if os.path.islink(params['spec_file']):
bot.info("%s is a symbolic link." %params['spec_file'])
params['spec_file'] = os.path.realpath(params['spec_file'])
# START TIMING
start_time = datetime.now()
image = build_from_spec(spec_file=params['spec_file'], # default will package the image
build_dir=build_dir,
isolated=True,
sandbox=False,
debug=params['debug'])
# Save has for metadata (also is image name)
version = get_image_file_hash(image)
params['version'] = version
pickle.dump(params,open(passing_params,'wb'))
final_time = (datetime.now() - start_time).seconds
bot.info("Final time of build %s seconds." %final_time)
# Did the container build successfully?
test_result = test_container(image)
if test_result['return_code'] != 0:
bot.error("Image failed to build, cancelling.")
sys.exit(1)
# Get singularity version
singularity_version = get_singularity_version()
# Package the image metadata (files, folders, etc)
image_package = package(image_path=image,
spec_path=params['spec_file'],
output_folder=build_dir,
remove_image=True,
verbose=True)
# Derive software tags by subtracting similar OS
diff = get_diff(image_package=image_package)
# Inspect to get labels and other metadata
cli = Singularity(debug=params['debug'])
inspect = cli.inspect(image_path=image)
# Get information on apps
app_names = cli.apps(image_path=image)
apps = extract_apps(image_path=image, app_names=app_names)
# Count file types, and extensions
counts = dict()
counts['readme'] = file_counts(diff=diff)
counts['copyright'] = file_counts(diff=diff,patterns=['copyright'])
counts['authors-thanks-credit'] = file_counts(diff=diff,
patterns=['authors','thanks','credit','contributors'])
counts['todo'] = file_counts(diff=diff,patterns=['todo'])
extensions = extension_counts(diff=diff)
os_sims = estimate_os(image_package=image_package,return_top=False)
most_similar = os_sims['SCORE'].values.argmax()
most_similar = os_sims['SCORE'].index.tolist()[most_similar]
metrics = {'build_time_seconds':final_time,
'singularity_version':singularity_version,
'singularity_python_version':singularity_python_version,
'estimated_os': most_similar,
'os_sims':os_sims['SCORE'].to_dict(),
'file_counts':counts,
'file_ext':extensions,
'inspect':inspect,
'version': version,
'apps': apps}
# Compress Image
if compress_image is True:
compressed_image = "%s.gz" %image
os.system('gzip -c -9 %s > %s' %(image,compressed_image))
image = compressed_image
output = {'image':image,
'image_package':image_package,
'metadata':metrics,
'params':params }
return output
else:
# Tell the user what is actually there
present_files = glob("*")
bot.error("Build file %s not found in repository" %params['spec_file'])
bot.info("Found files are %s" %"\n".join(present_files))
# Params have been exported, will be found by log
sys.exit(1)
def run_build(build_dir, params, verbose=True):
'''run_build takes a build directory and params dictionary, and does the following:
- downloads repo to a temporary directory
- changes branch or commit, if needed
- creates and bootstraps singularity image from Singularity file
- returns a dictionary with:
image (path), metadata (dict)
The following must be included in params:
spec_file, repo_url, branch, commit
'''
# Download the repository
download_repo(repo_url=params['repo_url'],
destination=build_dir)
os.chdir(build_dir)
if params['branch'] != None:
bot.info('Checking out branch %s' %params['branch'])
os.system('git checkout %s' %(params['branch']))
else:
params['branch'] = "master"
# Set the debug level
Client.debug = params['debug']
# Commit
if params['commit'] not in [None,'']:
bot.info('Checking out commit %s' %params['commit'])
os.system('git checkout %s .' %(params['commit']))
# From here on out commit is used as a unique id, if we don't have one, we use current
else:
params['commit'] = os.popen('git log -n 1 --pretty=format:"%H"').read()
bot.warning("commit not specified, setting to current %s" %params['commit'])
# Dump some params for the builder, in case it fails after this
passing_params = "/tmp/params.pkl"
pickle.dump(params, open(passing_params,'wb'))
# Now look for spec file
if os.path.exists(params['spec_file']):
bot.info("Found spec file %s in repository" %params['spec_file'])
# If the user has a symbolic link
if os.path.islink(params['spec_file']):
bot.info("%s is a symbolic link." %params['spec_file'])
params['spec_file'] = os.path.realpath(params['spec_file'])
# START TIMING
start_time = datetime.now()
# Secure Build
image = Client.build(recipe=params['spec_file'],
build_folder=build_dir,
isolated=True)
# Save has for metadata (also is image name)
version = get_image_file_hash(image)
params['version'] = version
pickle.dump(params, open(passing_params,'wb'))
# Rename image to be hash
finished_image = "%s/%s.simg" %(os.path.dirname(image), version)
image = shutil.move(image, finished_image)
final_time = (datetime.now() - start_time).seconds
bot.info("Final time of build %s seconds." %final_time)
# Did the container build successfully?
test_result = test_container(image)
if test_result['return_code'] != 0:
bot.error("Image failed to build, cancelling.")
sys.exit(1)
# Get singularity version
singularity_version = Client.version()
Client.debug = False
inspect = Client.inspect(image) # this is a string
Client.debug = params['debug']
# Get information on apps
Client.debug = False
app_names = Client.apps(image)
Client.debug = params['debug']
apps = extract_apps(image, app_names)
metrics = {'build_time_seconds': final_time,
'singularity_version': singularity_version,
'singularity_python_version': singularity_python_version,
'inspect': inspect,
'version': version,
'apps': apps}
output = {'image':image,
'metadata':metrics,
'params':params }
return output
else:
# Tell the user what is actually there
present_files = glob("*")
bot.error("Build file %s not found in repository" %params['spec_file'])
bot.info("Found files are %s" %"\n".join(present_files))
# Params have been exported, will be found by log
sys.exit(1)