def avail_easyconfig_params(self):
"""
Print the available easyconfig parameters, for the given easyblock.
"""
app = get_class(self.options.easyblock)
extra = app.extra_options()
mapping = convert_to_help(extra, has_default=False)
if len(extra) > 0:
ebb_msg = " (* indicates specific for the %s EasyBlock)" % app.__name__
extra_names = [x[0] for x in extra]
else:
ebb_msg = ""
extra_names = []
txt = ["Available easyconfig parameters%s" % ebb_msg]
params = [(k, v) for (k, v) in mapping.items() if k.upper() not in ["HIDDEN"]]
for key, values in params:
txt.append("%s" % key.upper())
txt.append("-" * len(key))
for name, value in values:
tabs = "\t" * (3 - (len(name) + 1) / 8)
if name in extra_names:
starred = "(*)"
else:
starred = ""
txt.append("%s%s:%s%s" % (name, starred, tabs, value))
txt.append("")
return "\n".join(txt)
def avail_easyconfig_params(self):
"""
Print the available easyconfig parameters, for the given easyblock.
"""
app = get_class(self.options.easyblock)
extra = app.extra_options()
mapping = convert_to_help(extra, has_default=False)
if len(extra) > 0:
ebb_msg = " (* indicates specific for the %s EasyBlock)" % app.__name__
extra_names = [x[0] for x in extra]
else:
ebb_msg = ''
extra_names = []
txt = ["Available easyconfig parameters%s" % ebb_msg]
params = [(k, v) for (k, v) in mapping.items()
if k.upper() not in ['HIDDEN']]
for key, values in params:
txt.append("%s" % key.upper())
txt.append('-' * len(key))
for name, value in values:
tabs = "\t" * (3 - (len(name) + 1) / 8)
if name in extra_names:
starred = '(*)'
else:
starred = ''
txt.append("%s%s:%s%s" % (name, starred, tabs, value))
txt.append('')
return "\n".join(txt)
def build_and_install_software(module, options, origEnviron, exitOnFailure=True, silent=False):
"""
Build the software
"""
spec = module['spec']
print_msg("processing EasyBuild easyconfig %s" % spec, log=_log, silent=silent)
# restore original environment
_log.info("Resetting environment")
filetools.errorsFoundInLog = 0
modify_env(os.environ, origEnviron)
cwd = os.getcwd()
# load easyblock
easyblock = options.easyblock
if not easyblock:
# try to look in .eb file
reg = re.compile(r"^\s*easyblock\s*=(.*)$")
txt = read_file(spec)
for line in txt.split('\n'):
match = reg.search(line)
if match:
easyblock = eval(match.group(1))
break
name = module['module'][0]
try:
app_class = get_class(easyblock, name=name)
app = app_class(spec, debug=options.debug, robot_path=options.robot)
_log.info("Obtained application instance of for %s (easyblock: %s)" % (name, easyblock))
except EasyBuildError, err:
print_error("Failed to get application instance for %s (easyblock: %s): %s" % (name, easyblock, err.msg), silent=silent)
def build_and_install_software(module, options, origEnviron, exitOnFailure=True, silent=False):
"""
Build the software
"""
spec = module['spec']
print_msg("processing EasyBuild easyconfig %s" % spec, log=_log, silent=silent)
# restore original environment
_log.info("Resetting environment")
filetools.errorsFoundInLog = 0
modify_env(os.environ, origEnviron)
cwd = os.getcwd()
# load easyblock
easyblock = options.easyblock
if not easyblock:
# try to look in .eb file
reg = re.compile(r"^\s*easyblock\s*=(.*)$")
txt = read_file(spec)
for line in txt.split('\n'):
match = reg.search(line)
if match:
easyblock = eval(match.group(1))
break
name = module['module'][0]
try:
app_class = get_class(easyblock, name=name)
app = app_class(spec, debug=options.debug, robot_path=options.robot)
_log.info("Obtained application instance of for %s (easyblock: %s)" % (name, easyblock))
except EasyBuildError, err:
print_error("Failed to get application instance for %s (easyblock: %s): %s" % (name, easyblock, err.msg), silent=silent)
def template_init_test(self, easyblock):
"""Test whether all easyconfigs can be initialized."""
class_regex = re.compile("^class (.*)\(.*", re.M)
self.log.debug("easyblock: %s" % easyblock)
# obtain easyblock class name using regex
f = open(easyblock, "r")
txt = f.read()
f.close()
res = class_regex.search(txt)
if res:
ebname = res.group(1)
self.log.debug("Found class name for easyblock %s: %s" % (easyblock, ebname))
# figure out list of mandatory variables, and define with dummy values as necessary
app_class = get_class(ebname)
ec_opts = app_class.extra_options()
extra_txt = ''
for ec_opt in ec_opts:
if ec_opt[1][2] == MANDATORY:
extra_txt += '%s = "foo"\n' % ec_opt[0]
# write easyconfig file
self.writeEC(ebname, extra_txt)
# initialize easyblock
# if this doesn't fail, the test succeeds
app = app_class(self.eb_file)
else:
self.assertTrue(False, "Class found in easyblock %s" % easyblock)
def template_init_test(self, easyblock):
"""Test whether all easyconfigs can be initialized."""
class_regex = re.compile("^class (.*)\(.*", re.M)
self.log.debug("easyblock: %s" % easyblock)
# obtain easyblock class name using regex
f = open(easyblock, "r")
txt = f.read()
f.close()
res = class_regex.search(txt)
if res:
ebname = res.group(1)
self.log.debug("Found class name for easyblock %s: %s" %
(easyblock, ebname))
# figure out list of mandatory variables, and define with dummy values as necessary
app_class = get_class(ebname)
ec_opts = app_class.extra_options()
extra_txt = ''
for ec_opt in ec_opts:
if ec_opt[1][2] == MANDATORY:
extra_txt += '%s = "foo"\n' % ec_opt[0]
# write easyconfig file
self.writeEC(ebname, extra_txt)
# initialize easyblock
# if this doesn't fail, the test succeeds
app = app_class(self.eb_file)
else:
self.assertTrue(False, "Class found in easyblock %s" % easyblock)
def avail_easyconfig_params(self):
"""
Print the available easyconfig parameters, for the given easyblock.
"""
app = get_class(self.options.easyblock)
extra = app.extra_options()
mapping = convert_to_help(EasyConfig.default_config + extra)
txt = []
for key, values in mapping.items():
txt.append("%s" % key.upper())
txt.append('-' * len(key))
for name, value in values:
tabs = "\t" * (3 - (len(name) + 1) / 8)
txt.append("%s:%s%s" % (name, tabs, value))
txt.append('')
return "\n".join(txt)
def get_easyblock_instance(easyconfig, robot_path=None):
"""
Get an instance for this easyconfig
easyconfig is in the format provided by processEasyConfig
log is a logger object
returns an instance of EasyBlock (or subclass thereof)
"""
spec = easyconfig['spec']
name = easyconfig['module'][0]
# handle easyconfigs with custom easyblocks
easyblock = None
reg = re.compile(r"^\s*easyblock\s*=(.*)$")
for line in open(spec).readlines():
match = reg.search(line)
if match:
easyblock = eval(match.group(1))
break
app_class = get_class(easyblock, name=name)
return app_class(spec, debug=True, robot_path=robot_path)
def get_easyblock_instance(easyconfig, robot_path=None):
"""
Get an instance for this easyconfig
easyconfig is in the format provided by processEasyConfig
log is a logger object
returns an instance of EasyBlock (or subclass thereof)
"""
spec = easyconfig['spec']
name = easyconfig['module'][0]
# handle easyconfigs with custom easyblocks
easyblock = None
reg = re.compile(r"^\s*easyblock\s*=(.*)$")
txt = read_file(spec)
for line in txt.split('\n'):
match = reg.search(line)
if match:
easyblock = eval(match.group(1))
break
app_class = get_class(easyblock, name=name)
return app_class(spec, debug=True, robot_path=robot_path)
def template_easyconfig_test(self, spec):
"""Test whether all easyconfigs can be initialized."""
# set to False, so it's False in case of this test failing
global single_tests_ok
prev_single_tests_ok = single_tests_ok
single_tests_ok = False
f = open(spec, 'r')
spectxt = f.read()
f.close()
# determine software name directly from easyconfig file
res = self.name_regex.search(spectxt)
if res:
name = res.group(1)
else:
self.assertTrue(False, "Obtained software name directly from easyconfig file")
# parse easyconfig
ec = EasyConfig(spec, validate=False)
# sanity check for software name
self.assertTrue(ec['name'], name)
# try and fetch easyblock spec from easyconfig
easyblock = self.easyblock_regex.search(spectxt)
if easyblock:
easyblock = easyblock.group(1)
# instantiate easyblock with easyconfig file
app_class = get_class(easyblock, name=name)
app = app_class(spec, validate_ec=False)
# more sanity checks
self.assertTrue(name, app.name)
self.assertTrue(ec['version'], app.version)
# make sure all patch files are available
specdir = os.path.dirname(spec)
specfn = os.path.basename(spec)
for patch in ec['patches']:
if isinstance(patch, (tuple, list)):
patch = patch[0]
# only check actual patch files, not other files being copied via the patch functionality
if patch.endswith('.patch'):
patch_full = os.path.join(specdir, patch)
msg = "Patch file %s is available for %s" % (patch_full, specfn)
self.assertTrue(os.path.isfile(patch_full), msg)
ext_patches = []
for ext in ec['exts_list']:
if isinstance(ext, (tuple, list)) and len(ext) == 3:
self.assertTrue(isinstance(ext[2], dict), "3rd element of extension spec is a dictionary")
for ext_patch in ext[2].get('patches', []):
if isinstance(ext_patch, (tuple, list)):
ext_patch = ext_patch[0]
# only check actual patch files, not other files being copied via the patch functionality
if ext_patch.endswith('.patch'):
ext_patch_full = os.path.join(specdir, ext_patch)
msg = "Patch file %s is available for %s" % (ext_patch_full, specfn)
self.assertTrue(os.path.isfile(ext_patch_full), msg)
app.close_log()
os.remove(app.logfile)
# test passed, so set back to True
single_tests_ok = True and prev_single_tests_ok
for root, subfolders, files in walk(options.path):
if '.git' in subfolders:
log.info("found .git subfolder, ignoring it")
subfolders.remove('.git')
for file_ in files:
file_ = join(root, file_)
file_ = read(file_)
try:
ec = EasyConfig(file_, validate=False)
log.info("found valid easyconfig %s" % ec)
if not ec.name in names:
log.info("found new software package %s" % ec)
# check if an easyblock exists
module = easyblock.get_class(
None, name=ec.name).__module__.split('.')[-1]
if module != "configuremake":
ec.easyblock = module
else:
ec.easyblock = None
configs.append(ec)
names.append(ec.name)
except Exception, e:
log.warning("faulty easyconfig %s" % file)
log.debug(e)
log.info("Found easyconfigs: %s" % [x.name for x in configs])
# remove example configs
configs = [c for c in configs if not "example.com" in c['homepage']]
# sort by name
configs = sorted(configs, key=lambda config: config.name.lower())
for root, subfolders, files in walk(options.path):
if '.git' in subfolders:
log.info("found .git subfolder, ignoring it")
subfolders.remove('.git')
for file_ in files:
file_ = join(root,file_)
file_ = read(file_)
try:
ec = EasyConfig(file_, build_options={'validate': False})
log.info("found valid easyconfig %s" % ec)
if not ec.name in names:
log.info("found new software package %s" % ec)
# check if an easyblock exists
module = easyblock.get_class(None, name=ec.name).__module__.split('.')[-1]
if module != "configuremake":
ec.easyblock = module
else:
ec.easyblock = None
configs.append(ec)
names.append(ec.name)
except Exception, e:
log.warning("faulty easyconfig %s" % file)
log.debug(e)
log.info("Found easyconfigs: %s" % [x.name for x in configs])
# remove example configs
configs = [c for c in configs if not "example.com" in c['homepage']]
# sort by name
configs = sorted(configs, key=lambda config : config.name.lower())
def template_easyconfig_test(self, spec):
"""Test whether all easyconfigs can be initialized."""
# set to False, so it's False in case of this test failing
global single_tests_ok
prev_single_tests_ok = single_tests_ok
single_tests_ok = False
f = open(spec, 'r')
spectxt = f.read()
f.close()
# determine software name directly from easyconfig file
res = self.name_regex.search(spectxt)
if res:
name = res.group(1)
else:
self.assertTrue(
False, "Obtained software name directly from easyconfig file")
# parse easyconfig
ec = EasyConfig(spec, validate=False)
# sanity check for software name
self.assertTrue(ec['name'], name)
# try and fetch easyblock spec from easyconfig
easyblock = self.easyblock_regex.search(spectxt)
if easyblock:
easyblock = easyblock.group(1)
# instantiate easyblock with easyconfig file
app_class = get_class(easyblock, name=name)
app = app_class(spec, validate_ec=False)
# more sanity checks
self.assertTrue(name, app.name)
self.assertTrue(ec['version'], app.version)
# make sure all patch files are available
specdir = os.path.dirname(spec)
specfn = os.path.basename(spec)
for patch in ec['patches']:
if isinstance(patch, (tuple, list)):
patch = patch[0]
# only check actual patch files, not other files being copied via the patch functionality
if patch.endswith('.patch'):
patch_full = os.path.join(specdir, patch)
msg = "Patch file %s is available for %s" % (patch_full, specfn)
self.assertTrue(os.path.isfile(patch_full), msg)
ext_patches = []
for ext in ec['exts_list']:
if isinstance(ext, (tuple, list)) and len(ext) == 3:
self.assertTrue(isinstance(ext[2], dict),
"3rd element of extension spec is a dictionary")
for ext_patch in ext[2].get('patches', []):
if isinstance(ext_patch, (tuple, list)):
ext_patch = ext_patch[0]
# only check actual patch files, not other files being copied via the patch functionality
if ext_patch.endswith('.patch'):
ext_patch_full = os.path.join(specdir, ext_patch)
msg = "Patch file %s is available for %s" % (
ext_patch_full, specfn)
self.assertTrue(os.path.isfile(ext_patch_full), msg)
app.close_log()
os.remove(app.logfile)
# test passed, so set back to True
single_tests_ok = True and prev_single_tests_ok
