def dockerfile_to_singularity(dockerfile_path, output_dir=None):
'''dockerfile_to_singularity will return a Singularity build file based on
a provided Dockerfile. If output directory is not specified, the string
will be returned. Otherwise, a file called Singularity will be written to
output_dir
:param dockerfile_path: the path to the Dockerfile
:param output_dir: the output directory to write the Singularity file to
'''
build_file = None
if os.path.basename(dockerfile_path) == "Dockerfile":
try:
spec = read_file(dockerfile_path)
# Use a common mapping
mapping = get_mapping()
# Put into dict of keys (section titles) and list of commands (values)
sections = organize_sections(lines=spec, mapping=mapping)
# We have to, by default, add the Docker bootstrap
sections["bootstrap"] = ["docker"]
# Put into one string based on "order" variable in mapping
build_file = print_sections(sections=sections, mapping=mapping)
if output_dir != None:
write_file("%s/Singularity" % (output_dir), build_file)
print("Singularity spec written to %s" % (output_dir))
return build_file
except:
bot.logger.error("Error generating Dockerfile from %s.",
dockerfile_path)
# If we make it here, something didn't work
bot.logger.error("Could not find %s.", dockerfile_path)
return build_file
def get_build_template(template_name, params=None, to_file=None):
'''get_build template returns a string or file for a particular build template, which is
intended to build a version of a Singularity image on a cloud resource.
:param template_name: the name of the template to retrieve in build/scripts
:param params: (if needed) a dictionary of parameters to substitute in the file
:param to_file: if defined, will write to file. Default returns string.
'''
base = get_installdir()
template_folder = "%s/build/scripts" % (base)
template_file = "%s/%s" % (template_folder, template_name)
if os.path.exists(template_file):
bot.debug("Found template %s" % template_file)
# Implement when needed - substitute params here
# Will need to read in file instead of copying below
# if params != None:
if to_file is not None:
shutil.copyfile(template_file, to_file)
bot.debug("Template file saved to %s" % to_file)
return to_file
# If the user wants a string
content = ''.join(read_file(template_file))
return content
else:
bot.warning("Template %s not found." % template_file)
def test_write_read_files(self):
'''test_write_read_files will test the functions write_file and read_file
'''
print("Testing utils.write_file...")
from singularity.utils import write_file
import json
tmpfile = tempfile.mkstemp()[1]
os.remove(tmpfile)
write_file(tmpfile,"hello!")
self.assertTrue(os.path.exists(tmpfile))
print("Testing utils.read_file...")
from singularity.utils import read_file
content = read_file(tmpfile)[0]
self.assertEqual("hello!",content)
from singularity.utils import write_json
print("Testing utils.write_json...")
print("...Case 1: Providing bad json")
bad_json = {"Wakkawakkawakka'}":[{True},"2",3]}
tmpfile = tempfile.mkstemp()[1]
os.remove(tmpfile)
with self.assertRaises(TypeError) as cm:
write_json(bad_json,tmpfile)
print("...Case 2: Providing good json")
good_json = {"Wakkawakkawakka":[True,"2",3]}
tmpfile = tempfile.mkstemp()[1]
os.remove(tmpfile)
write_json(good_json,tmpfile)
with open(tmpfile,'r') as filey:
content = json.loads(filey.read())
self.assertTrue(isinstance(content,dict))
self.assertTrue("Wakkawakkawakka" in content)
def package(image_path,
spec_path=None,
output_folder=None,
remove_image=False,
verbose=False,
S=None):
'''generate a zip (including the image) to a user specified output_folder.
:param image_path: full path to singularity image file
:param runscript: if True, will extract runscript to include in package as runscript
:param software: if True, will extract files.txt and folders.txt to package
:param remove_image: if True, will not include original image in package (default,False)
:param verbose: be verbose when using singularity --export (default,False)
:param S: the Singularity object (optional) will be created if not required.
'''
# Run create image and bootstrap with Singularity command line tool.
S = Singularity(debug=verbose)
file_obj, tar = get_image_tar(image_path, S=S)
members = tar.getmembers()
image_name, ext = os.path.splitext(os.path.basename(image_path))
zip_name = "%s.zip" % (image_name.replace(" ", "_"))
# Include the image in the package?
to_package = dict()
if not remove_image:
to_package["files"] = [image_path]
# If the specfile is provided, it should also be packaged
if spec_path is not None:
singularity_spec = "".join(read_file(spec_path))
to_package['Singularity'] = singularity_spec
to_package["VERSION"] = get_image_file_hash(image_path)
try:
inspection = S.inspect(image_path)
to_package["inspect.json"] = inspection
inspection = json.loads(inspection)
to_package['runscript'] = inspection['data']['attributes']['runscript']
except:
bot.warning("Trouble extracting container metadata with inspect.")
bot.info("Adding software list to package.")
files = [x.path for x in members if x.isfile()]
folders = [x.path for x in members if x.isdir()]
to_package["files.txt"] = files
to_package["folders.txt"] = folders
# Do zip up here - let's start with basic structures
zipfile = zip_up(to_package,
zip_name=zip_name,
output_folder=output_folder)
bot.debug("Package created at %s" % (zipfile))
if not delete_image_tar(file_obj, tar):
bot.warning("Could not clean up temporary tarfile.")
# return package to user
return zipfile
def get_build_template(template_name,params=None,to_file=None):
'''get_build template returns a string or file for a particular build template, which is
intended to build a version of a Singularity image on a cloud resource.
:param template_name: the name of the template to retrieve in build/scripts
:param params: (if needed) a dictionary of parameters to substitute in the file
:param to_file: if defined, will write to file. Default returns string.
'''
base = get_installdir()
template_folder = "%s/build/scripts" %(base)
template_file = "%s/%s" %(template_folder,template_name)
if os.path.exists(template_file):
bot.debug("Found template %s" %template_file)
# Implement when needed - substitute params here
# Will need to read in file instead of copying below
# if params != None:
if to_file is not None:
shutil.copyfile(template_file,to_file)
bot.debug("Template file saved to %s" %to_file)
return to_file
# If the user wants a string
content = ''.join(read_file(template_file))
return content
else:
bot.warning("Template %s not found." %template_file)
def main(args, parser, subparser):
if args.recipe is None:
subparser.print_help()
bot.newline()
print("Please specify creating a recipe with --recipe")
sys.exit(0)
# Output folder will be pwd if not specified
output_folder = os.getcwd()
if args.outfolder is not None:
output_folder = os.getcwd()
bootstrap = ''
if args.bootstrap is not None:
bootstrap = args.bootstrap
bot.debug("bootstrap: %s" % bootstrap)
bootstrap_from = ''
if args.bootstrap_from is not None:
bootstrap_from = args.bootstrap_from
bot.debug("from: %s" % bootstrap_from)
template = "Singularity"
output_file = template
app = ''
if args.app is not None:
app = args.app.lower()
template = "Singularity.app"
output_file = "Singularity.%s" % app
input_file = "%s/cli/app/templates/recipes/%s" % (get_installdir(),
template)
output_file = "%s/%s" % (output_folder, output_file)
if os.path.exists(output_file):
ext = str(uuid.uuid4())[0:4]
output_file = "%s.%s" % (output_file, ext)
# Read the file, make substitutions
contents = read_file(input_file, readlines=False)
# Replace all occurrences of app
contents = contents.replace('{{ app }}', app)
contents = contents.replace('{{ bootstrap }}', bootstrap)
contents = contents.replace('{{ from }}', bootstrap_from)
write_file(output_file, contents)
bot.info("Output file written to %s" % output_file)
def get_template(template_name, fields=None):
'''get_template will return a template in the template folder,
with some substitutions (eg, {'{{ graph | safe }}':"fill this in!"}
'''
template = None
if not template_name.endswith('.html'):
template_name = "%s.html" % (template_name)
here = "%s/templates" % (get_installdir())
template_path = "%s/%s" % (here, template_name)
if os.path.exists(template_path):
template = ''.join(read_file(template_path))
if fields is not None:
for tag, sub in fields.items():
template = template.replace(tag, sub)
return template
def get_template(template_name,fields=None):
'''get_template will return a template in the template folder,
with some substitutions (eg, {'{{ graph | safe }}':"fill this in!"}
'''
template = None
if not template_name.endswith('.html'):
template_name = "%s.html" %(template_name)
here = "%s/cli/app/templates" %(get_installdir())
template_path = "%s/%s" %(here,template_name)
if os.path.exists(template_path):
template = ''.join(read_file(template_path))
if fields is not None:
for tag,sub in fields.items():
template = template.replace(tag,sub)
return template
def detect_language(runscript):
'''detect_language will use pygments to detect the language of a run script
:param runscript: the full path to the run script, or string of runscript itself
'''
if os.path.exists(runscript):
content = read_file(runscript)
else:
content = runscript
try:
language = guess_lexer(''.join(content))
if language.name == "Python":
return "python"
else:
print("Language %s is not yet supported." %(language.name))
except ClassNotFound:
print("Cannot detect language.")
return None
def package(image_path,
spec_path=None,
output_folder=None,
runscript=True,
software=True,
remove_image=False,
verbose=False,
S=None,
sudopw=None,
old_version=False):
'''package will take an image and generate a zip (including the image
to a user specified output_folder.
:param image_path: full path to singularity image file
:param runscript: if True, will extract runscript to include in package as runscript
:param software: if True, will extract files.txt and folders.txt to package
:param remove_image: if True, will not include original image in package (default,False)
:param verbose: be verbose when using singularity --export (default,False)
:param S: the Singularity object (optional) will be created if not required.
'''
# Run create image and bootstrap with Singularity command line tool.
if S == None:
if sudopw != None:
S = Singularity(sudopw=sudopw, debug=verbose)
else:
S = Singularity(
debug=verbose) # This command will ask the user for sudo
# Singularity < version 2.3
if old_version == False:
file_obj, tar = get_memory_tar(image_path)
# Singularity 2.3 and up
else:
tmptar = S.export(image_path=image_path, old_version=True)
tar = tarfile.open(tmptar)
members = tar.getmembers()
image_name = os.path.basename(image_path)
zip_name = "%s.zip" % (image_name.replace(" ", "_"))
# Include the image in the package?
if remove_image:
to_package = dict()
else:
to_package = {"files": [image_path]}
# If the specfile is provided, it should also be packaged
if spec_path is not None:
singularity_spec = "".join(read_file(spec_path))
to_package['Singularity'] = singularity_spec
# Package the image with an sha1, identical standard, as VERSION
if old_version == False:
hashes = get_image_hashes(image_path)
to_package["HASHES"] = hashes
to_package["VERSION"] = hashes['IDENTICAL']
else:
to_package["VERSION"] = get_image_file_hash(image_path)
# Look for runscript
if runscript is True:
try:
to_package["runscript"] = extract_content(image_path,
'./singularity',
cli=S)
bot.logger.debug("Found runscript.")
except KeyError:
bot.logger.warning("No runscript found")
if software == True:
bot.logger.info("Adding software list to package.")
files = [x.path for x in members if x.isfile()]
folders = [x.path for x in members if x.isdir()]
to_package["files.txt"] = files
to_package["folders.txt"] = folders
# Do zip up here - let's start with basic structures
zipfile = zip_up(to_package,
zip_name=zip_name,
output_folder=output_folder)
bot.logger.debug("Package created at %s" % (zipfile))
if old_version == False:
file_obj.close()
# return package to user
return zipfile