def get_runscript_template(output_folder=None,script_name="singularity",language="py"): '''get_runscript_template returns a template runscript "singularity" for the user :param output_folder: output folder for template, if not specified, will use PWD :param script_name: the name to give the template script, default is "singularity" to work with singularity containers :param language: corresponds to language extension, to find script. Default is python (py) ''' base = get_installdir() template_folder = "%s/templates" %(base) # If no output folder provided, will be returned to PWD if output_folder == None: output_folder = os.getcwd() # If the user gave an extension with dots, remove them first while re.search("^[.]",language): language = language[1:] template_file = "%s/runscript.%s" %(template_folder,language) if not os.path.exists(template_file): print("Template for extension %s is not currently provided.\n please submit an issue to https://github.com/singularityware/singularity-python/issues.") else: runscript = "%s/%s" %(output_folder,script_name) shutil.copyfile(template_file,runscript) print("Runscript template saved to %s!\n" %(runscript)) return runscript
def get_build_template(template_name, params=None, to_file=None): '''get_build template returns a string or file for a particular build template, which is intended to build a version of a Singularity image on a cloud resource. :param template_name: the name of the template to retrieve in build/scripts :param params: (if needed) a dictionary of parameters to substitute in the file :param to_file: if defined, will write to file. Default returns string. ''' base = get_installdir() template_folder = "%s/build/scripts" % (base) template_file = "%s/%s" % (template_folder, template_name) if os.path.exists(template_file): bot.debug("Found template %s" % template_file) # Implement when needed - substitute params here # Will need to read in file instead of copying below # if params != None: if to_file is not None: shutil.copyfile(template_file, to_file) bot.debug("Template file saved to %s" % to_file) return to_file # If the user wants a string content = ''.join(read_file(template_file)) return content else: bot.warning("Template %s not found." % template_file)
def test_get_installdir(self): '''get install directory should return the base of where singularity is installed ''' from singularity.utils import get_installdir whereami = get_installdir() self.assertTrue(whereami.endswith('singularity'))
def get_build_template(template_name,params=None,to_file=None): '''get_build template returns a string or file for a particular build template, which is intended to build a version of a Singularity image on a cloud resource. :param template_name: the name of the template to retrieve in build/scripts :param params: (if needed) a dictionary of parameters to substitute in the file :param to_file: if defined, will write to file. Default returns string. ''' base = get_installdir() template_folder = "%s/build/scripts" %(base) template_file = "%s/%s" %(template_folder,template_name) if os.path.exists(template_file): bot.debug("Found template %s" %template_file) # Implement when needed - substitute params here # Will need to read in file instead of copying below # if params != None: if to_file is not None: shutil.copyfile(template_file,to_file) bot.debug("Template file saved to %s" %to_file) return to_file # If the user wants a string content = ''.join(read_file(template_file)) return content else: bot.warning("Template %s not found." %template_file)
def get_levels(version=None): '''get_levels returns a dictionary of levels (key) and values (dictionaries with descriptions and regular expressions for files) for the user. :param version: the version of singularity to use (default is 2.2) :param include_files: files to add to the level, only relvant if ''' valid_versions = ['2.3', '2.2'] if version is None: version = "2.3" version = str(version) if version not in valid_versions: bot.logger.error("Unsupported version %s, valid versions are %s", version, ",".join(valid_versions)) levels_file = os.path.abspath( os.path.join(get_installdir(), 'hub', 'data', 'reproduce_levels.json')) levels = read_json(levels_file) if version == "2.2": # Labels not added until 2.3 del levels['LABELS'] levels = make_levels_set(levels) return levels
def get_levels(version=None): '''get_levels returns a dictionary of levels (key) and values (dictionaries with descriptions and regular expressions for files) for the user. :param version: the version of singularity to use (default is 2.2) :param include_files: files to add to the level, only relvant if ''' valid_versions = ['2.3','2.2'] if version is None: version = "2.3" version = str(version) if version not in valid_versions: bot.error("Unsupported version %s, valid versions are %s" %(version, ",".join(valid_versions))) levels_file = os.path.abspath(os.path.join(get_installdir(), 'analysis', 'reproduce', 'data', 'reproduce_levels.json')) levels = read_json(levels_file) if version == "2.2": # Labels not added until 2.3 del levels['LABELS'] levels = make_levels_set(levels) return levels
def setUp(self): self.pwd = get_installdir() self.cli = Singularity() self.tmpdir = tempfile.mkdtemp() self.image1 = "%s/tests/data/busybox-2017-10-21.simg" %(self.pwd) self.image2 = "%s/tests/data/cirros-2017-10-21.simg" %(self.pwd) self.pkg1 = "%s/tests/data/busybox-2017-10-21.zip" %(self.pwd) self.pkg2 = "%s/tests/data/cirros-2017-10-21.zip" %(self.pwd)
def test_get_installdir(self): '''get install directory should return the base of where singularity is installed ''' print("Testing utils.get_installdir") from singularity.utils import get_installdir whereami = get_installdir() self.assertTrue(whereami.endswith('singularity'))
def test_compare_packages(self): print("Testing singularity.analysis.compare.compare_packages") from singularity.analysis.compare import compare_packages pwd = get_installdir() pkg1 = "%s/tests/data/busybox-2016-02-16.img.zip" %(pwd) pkg2 = "%s/tests/data/cirros-2016-01-04.img.zip" %(pwd) comparison = compare_packages(pkg1,pkg2) self.assertTrue('files.txt' in comparison) self.assertTrue(isinstance(comparison['files.txt'],pandas.DataFrame))
def setUp(self): self.pwd = get_installdir() self.cli = Singularity() self.tmpdir = tempfile.mkdtemp() self.image1 = "%s/tests/data/busybox-2016-02-16.img" % (self.pwd) self.image2 = "%s/tests/data/cirros-2016-01-04.img" % (self.pwd) # We can't test creating the packages, because requires sudo :/ self.pkg1 = "%s/tests/data/busybox-2016-02-16.img.zip" % (self.pwd) self.pkg2 = "%s/tests/data/cirros-2016-01-04.img.zip" % (self.pwd)
def get_script(script_name): '''get_script will return a build script_name, if it is included in singularity/build/scripts, otherwise will alert the user and return None :param script_name: the name of the script to look for ''' install_dir = get_installdir() script_path = "%s/build/scripts/%s" %(install_dir,script_name) if os.path.exists(script_path): return script_path else: bot.error("Script %s is not included in singularity-python!" %script_path) return None
def get_script(script_name): '''get_script will return a build script_name, if it is included in singularity/build/scripts, otherwise will alert the user and return None :param script_name: the name of the script to look for ''' install_dir = get_installdir() script_path = "%s/build/scripts/%s" % (install_dir, script_name) if os.path.exists(script_path): return script_path else: bot.error("Script %s is not included in singularity-python!" % script_path) return None
def main(args, parser, subparser): if args.recipe is None: subparser.print_help() bot.newline() print("Please specify creating a recipe with --recipe") sys.exit(0) # Output folder will be pwd if not specified output_folder = os.getcwd() if args.outfolder is not None: output_folder = os.getcwd() bootstrap = '' if args.bootstrap is not None: bootstrap = args.bootstrap bot.debug("bootstrap: %s" % bootstrap) bootstrap_from = '' if args.bootstrap_from is not None: bootstrap_from = args.bootstrap_from bot.debug("from: %s" % bootstrap_from) template = "Singularity" output_file = template app = '' if args.app is not None: app = args.app.lower() template = "Singularity.app" output_file = "Singularity.%s" % app input_file = "%s/cli/app/templates/recipes/%s" % (get_installdir(), template) output_file = "%s/%s" % (output_folder, output_file) if os.path.exists(output_file): ext = str(uuid.uuid4())[0:4] output_file = "%s.%s" % (output_file, ext) # Read the file, make substitutions contents = read_file(input_file, readlines=False) # Replace all occurrences of app contents = contents.replace('{{ app }}', app) contents = contents.replace('{{ bootstrap }}', bootstrap) contents = contents.replace('{{ from }}', bootstrap_from) write_file(output_file, contents) bot.info("Output file written to %s" % output_file)
def get_template(template_name,fields=None): '''get_template will return a template in the template folder, with some substitutions (eg, {'{{ graph | safe }}':"fill this in!"} ''' template = None if not template_name.endswith('.html'): template_name = "%s.html" %(template_name) here = "%s/cli/app/templates" %(get_installdir()) template_path = "%s/%s" %(here,template_name) if os.path.exists(template_path): template = ''.join(read_file(template_path)) if fields is not None: for tag,sub in fields.items(): template = template.replace(tag,sub) return template
def get_template(template_name, fields=None): '''get_template will return a template in the template folder, with some substitutions (eg, {'{{ graph | safe }}':"fill this in!"} ''' template = None if not template_name.endswith('.html'): template_name = "%s.html" % (template_name) here = "%s/templates" % (get_installdir()) template_path = "%s/%s" % (here, template_name) if os.path.exists(template_path): template = ''.join(read_file(template_path)) if fields is not None: for tag, sub in fields.items(): template = template.replace(tag, sub) return template
from glob import glob import os import re import requests import shutil from singularity.logger import bot from singularity.utils import get_installdir import sys import subprocess import tempfile import zipfile install_dir = get_installdir() def remove_unicode_dict(input_dict): '''remove unicode keys and values from dict, encoding in utf8 ''' if isinstance(input_dict, collections.Mapping): return dict(map(remove_unicode_dict, input_dict.iteritems())) elif isinstance(input_dict, collections.Iterable): return type(input_dict)(map(remove_unicode_dict, input_dict)) else: return input_dict def update_dict(input_dict,key,value): '''update_dict will update lists in a dictionary. If the key is not included,
along with this program. If not, see <https://www.gnu.org/licenses/>. ''' from glob import glob import os import re from singularity.logger import bot from singularity.utils import get_installdir import json import sys import tempfile import zipfile install_dir = get_installdir() ###################################################################################### # Package Data ###################################################################################### def get_packages(family=None): '''get packages will return a list of packages (under some family) provided by singularity python.If no name is specified, the default (os) will be used. :param name: the name of the package family to load ''' package_base = get_package_base() package_folders = glob("%s/*" % (package_base)) package_families = [os.path.basename(x) for x in package_folders]
def setUp(self): self.pwd = get_installdir() self.cli = Singularity() self.tmpdir = tempfile.mkdtemp()
def setUp(self): self.pwd = get_installdir() self.tmpdir = tmpdir self.image1 = image1 self.image2 = image2
def setUp(self): self.pwd = get_installdir() self.tmpdir = tempfile.mkdtemp() self.spec = "%s/tests/data/Singularity" %(self.pwd)
def setUp(self): self.pwd = get_installdir() self.cli = Singularity() self.tmpdir = tempfile.mkdtemp() self.image1 = "%s/tests/data/busybox-2016-02-16.img" % (self.pwd) self.image2 = "%s/tests/data/cirros-2016-01-04.img" % (self.pwd)
def setUp(self): self.pwd = get_installdir() self.tmpdir = tempfile.mkdtemp() self.spec = "%s/tests/data/Singularity" %(self.pwd) print("\n---START----------------------------------------")