def get_runscript_template(output_folder=None,script_name="singularity",language="py"):
'''get_runscript_template returns a template runscript "singularity" for the user
:param output_folder: output folder for template, if not specified, will use PWD
:param script_name: the name to give the template script, default is "singularity" to work
with singularity containers
:param language: corresponds to language extension, to find script. Default is python (py)
'''
base = get_installdir()
template_folder = "%s/templates" %(base)
# If no output folder provided, will be returned to PWD
if output_folder == None:
output_folder = os.getcwd()
# If the user gave an extension with dots, remove them first
while re.search("^[.]",language):
language = language[1:]
template_file = "%s/runscript.%s" %(template_folder,language)
if not os.path.exists(template_file):
print("Template for extension %s is not currently provided.\n please submit an issue to https://github.com/singularityware/singularity-python/issues.")
else:
runscript = "%s/%s" %(output_folder,script_name)
shutil.copyfile(template_file,runscript)
print("Runscript template saved to %s!\n" %(runscript))
return runscript
def get_build_template(template_name, params=None, to_file=None):
'''get_build template returns a string or file for a particular build template, which is
intended to build a version of a Singularity image on a cloud resource.
:param template_name: the name of the template to retrieve in build/scripts
:param params: (if needed) a dictionary of parameters to substitute in the file
:param to_file: if defined, will write to file. Default returns string.
'''
base = get_installdir()
template_folder = "%s/build/scripts" % (base)
template_file = "%s/%s" % (template_folder, template_name)
if os.path.exists(template_file):
bot.debug("Found template %s" % template_file)
# Implement when needed - substitute params here
# Will need to read in file instead of copying below
# if params != None:
if to_file is not None:
shutil.copyfile(template_file, to_file)
bot.debug("Template file saved to %s" % to_file)
return to_file
# If the user wants a string
content = ''.join(read_file(template_file))
return content
else:
bot.warning("Template %s not found." % template_file)
def test_get_installdir(self):
'''get install directory should return the base of where singularity
is installed
'''
from singularity.utils import get_installdir
whereami = get_installdir()
self.assertTrue(whereami.endswith('singularity'))
def get_build_template(template_name,params=None,to_file=None):
'''get_build template returns a string or file for a particular build template, which is
intended to build a version of a Singularity image on a cloud resource.
:param template_name: the name of the template to retrieve in build/scripts
:param params: (if needed) a dictionary of parameters to substitute in the file
:param to_file: if defined, will write to file. Default returns string.
'''
base = get_installdir()
template_folder = "%s/build/scripts" %(base)
template_file = "%s/%s" %(template_folder,template_name)
if os.path.exists(template_file):
bot.debug("Found template %s" %template_file)
# Implement when needed - substitute params here
# Will need to read in file instead of copying below
# if params != None:
if to_file is not None:
shutil.copyfile(template_file,to_file)
bot.debug("Template file saved to %s" %to_file)
return to_file
# If the user wants a string
content = ''.join(read_file(template_file))
return content
else:
bot.warning("Template %s not found." %template_file)
def get_levels(version=None):
'''get_levels returns a dictionary of levels (key) and values (dictionaries with
descriptions and regular expressions for files) for the user.
:param version: the version of singularity to use (default is 2.2)
:param include_files: files to add to the level, only relvant if
'''
valid_versions = ['2.3', '2.2']
if version is None:
version = "2.3"
version = str(version)
if version not in valid_versions:
bot.logger.error("Unsupported version %s, valid versions are %s",
version, ",".join(valid_versions))
levels_file = os.path.abspath(
os.path.join(get_installdir(), 'hub', 'data', 'reproduce_levels.json'))
levels = read_json(levels_file)
if version == "2.2":
# Labels not added until 2.3
del levels['LABELS']
levels = make_levels_set(levels)
return levels
def get_levels(version=None):
'''get_levels returns a dictionary of levels (key) and values (dictionaries with
descriptions and regular expressions for files) for the user.
:param version: the version of singularity to use (default is 2.2)
:param include_files: files to add to the level, only relvant if
'''
valid_versions = ['2.3','2.2']
if version is None:
version = "2.3"
version = str(version)
if version not in valid_versions:
bot.error("Unsupported version %s, valid versions are %s" %(version,
",".join(valid_versions)))
levels_file = os.path.abspath(os.path.join(get_installdir(),
'analysis',
'reproduce',
'data',
'reproduce_levels.json'))
levels = read_json(levels_file)
if version == "2.2":
# Labels not added until 2.3
del levels['LABELS']
levels = make_levels_set(levels)
return levels
def setUp(self):
self.pwd = get_installdir()
self.cli = Singularity()
self.tmpdir = tempfile.mkdtemp()
self.image1 = "%s/tests/data/busybox-2017-10-21.simg" %(self.pwd)
self.image2 = "%s/tests/data/cirros-2017-10-21.simg" %(self.pwd)
self.pkg1 = "%s/tests/data/busybox-2017-10-21.zip" %(self.pwd)
self.pkg2 = "%s/tests/data/cirros-2017-10-21.zip" %(self.pwd)
def test_get_installdir(self):
'''get install directory should return the base of where singularity
is installed
'''
print("Testing utils.get_installdir")
from singularity.utils import get_installdir
whereami = get_installdir()
self.assertTrue(whereami.endswith('singularity'))
def test_compare_packages(self):
print("Testing singularity.analysis.compare.compare_packages")
from singularity.analysis.compare import compare_packages
pwd = get_installdir()
pkg1 = "%s/tests/data/busybox-2016-02-16.img.zip" %(pwd)
pkg2 = "%s/tests/data/cirros-2016-01-04.img.zip" %(pwd)
comparison = compare_packages(pkg1,pkg2)
self.assertTrue('files.txt' in comparison)
self.assertTrue(isinstance(comparison['files.txt'],pandas.DataFrame))
def setUp(self):
self.pwd = get_installdir()
self.cli = Singularity()
self.tmpdir = tempfile.mkdtemp()
self.image1 = "%s/tests/data/busybox-2016-02-16.img" % (self.pwd)
self.image2 = "%s/tests/data/cirros-2016-01-04.img" % (self.pwd)
# We can't test creating the packages, because requires sudo :/
self.pkg1 = "%s/tests/data/busybox-2016-02-16.img.zip" % (self.pwd)
self.pkg2 = "%s/tests/data/cirros-2016-01-04.img.zip" % (self.pwd)
def get_script(script_name):
'''get_script will return a build script_name, if it is included
in singularity/build/scripts, otherwise will alert the user and return None
:param script_name: the name of the script to look for
'''
install_dir = get_installdir()
script_path = "%s/build/scripts/%s" %(install_dir,script_name)
if os.path.exists(script_path):
return script_path
else:
bot.error("Script %s is not included in singularity-python!" %script_path)
return None
def get_script(script_name):
'''get_script will return a build script_name, if it is included
in singularity/build/scripts, otherwise will alert the user and return None
:param script_name: the name of the script to look for
'''
install_dir = get_installdir()
script_path = "%s/build/scripts/%s" % (install_dir, script_name)
if os.path.exists(script_path):
return script_path
else:
bot.error("Script %s is not included in singularity-python!" %
script_path)
return None
def main(args, parser, subparser):
if args.recipe is None:
subparser.print_help()
bot.newline()
print("Please specify creating a recipe with --recipe")
sys.exit(0)
# Output folder will be pwd if not specified
output_folder = os.getcwd()
if args.outfolder is not None:
output_folder = os.getcwd()
bootstrap = ''
if args.bootstrap is not None:
bootstrap = args.bootstrap
bot.debug("bootstrap: %s" % bootstrap)
bootstrap_from = ''
if args.bootstrap_from is not None:
bootstrap_from = args.bootstrap_from
bot.debug("from: %s" % bootstrap_from)
template = "Singularity"
output_file = template
app = ''
if args.app is not None:
app = args.app.lower()
template = "Singularity.app"
output_file = "Singularity.%s" % app
input_file = "%s/cli/app/templates/recipes/%s" % (get_installdir(),
template)
output_file = "%s/%s" % (output_folder, output_file)
if os.path.exists(output_file):
ext = str(uuid.uuid4())[0:4]
output_file = "%s.%s" % (output_file, ext)
# Read the file, make substitutions
contents = read_file(input_file, readlines=False)
# Replace all occurrences of app
contents = contents.replace('{{ app }}', app)
contents = contents.replace('{{ bootstrap }}', bootstrap)
contents = contents.replace('{{ from }}', bootstrap_from)
write_file(output_file, contents)
bot.info("Output file written to %s" % output_file)
def get_template(template_name,fields=None):
'''get_template will return a template in the template folder,
with some substitutions (eg, {'{{ graph | safe }}':"fill this in!"}
'''
template = None
if not template_name.endswith('.html'):
template_name = "%s.html" %(template_name)
here = "%s/cli/app/templates" %(get_installdir())
template_path = "%s/%s" %(here,template_name)
if os.path.exists(template_path):
template = ''.join(read_file(template_path))
if fields is not None:
for tag,sub in fields.items():
template = template.replace(tag,sub)
return template
def get_template(template_name, fields=None):
'''get_template will return a template in the template folder,
with some substitutions (eg, {'{{ graph | safe }}':"fill this in!"}
'''
template = None
if not template_name.endswith('.html'):
template_name = "%s.html" % (template_name)
here = "%s/templates" % (get_installdir())
template_path = "%s/%s" % (here, template_name)
if os.path.exists(template_path):
template = ''.join(read_file(template_path))
if fields is not None:
for tag, sub in fields.items():
template = template.replace(tag, sub)
return template
from glob import glob
import os
import re
import requests
import shutil
from singularity.logger import bot
from singularity.utils import get_installdir
import sys
import subprocess
import tempfile
import zipfile
install_dir = get_installdir()
def remove_unicode_dict(input_dict):
'''remove unicode keys and values from dict, encoding in utf8
'''
if isinstance(input_dict, collections.Mapping):
return dict(map(remove_unicode_dict, input_dict.iteritems()))
elif isinstance(input_dict, collections.Iterable):
return type(input_dict)(map(remove_unicode_dict, input_dict))
else:
return input_dict
def update_dict(input_dict,key,value):
'''update_dict will update lists in a dictionary. If the key is not included,
along with this program. If not, see <https://www.gnu.org/licenses/>.
'''
from glob import glob
import os
import re
from singularity.logger import bot
from singularity.utils import get_installdir
import json
import sys
import tempfile
import zipfile
install_dir = get_installdir()
######################################################################################
# Package Data
######################################################################################
def get_packages(family=None):
'''get packages will return a list of packages (under some family)
provided by singularity python.If no name is specified, the default (os) will
be used.
:param name: the name of the package family to load
'''
package_base = get_package_base()
package_folders = glob("%s/*" % (package_base))
package_families = [os.path.basename(x) for x in package_folders]
def setUp(self):
self.pwd = get_installdir()
self.cli = Singularity()
self.tmpdir = tempfile.mkdtemp()
def setUp(self):
self.pwd = get_installdir()
self.tmpdir = tmpdir
self.image1 = image1
self.image2 = image2
def setUp(self):
self.pwd = get_installdir()
self.tmpdir = tempfile.mkdtemp()
self.spec = "%s/tests/data/Singularity" %(self.pwd)
def setUp(self):
self.pwd = get_installdir()
self.cli = Singularity()
self.tmpdir = tempfile.mkdtemp()
self.image1 = "%s/tests/data/busybox-2016-02-16.img" % (self.pwd)
self.image2 = "%s/tests/data/cirros-2016-01-04.img" % (self.pwd)
def setUp(self):
self.pwd = get_installdir()
self.tmpdir = tempfile.mkdtemp()
self.spec = "%s/tests/data/Singularity" %(self.pwd)
print("\n---START----------------------------------------")
