def postprocess(self):
"""Do some postprocessing, in particular print stuff"""
build_log.EXPERIMENTAL = self.options.experimental
config.SUPPORT_OLDSTYLE = self.options.oldstyleconfig
# set strictness of run module
if self.options.strict:
run.strictness = self.options.strict
if self.options.deprecated:
build_log.CURRENT_VERSION = LooseVersion(self.options.deprecated)
if self.options.unittest_file:
fancylogger.logToFile(self.options.unittest_file)
if any(
[
self.options.avail_easyconfig_params,
self.options.avail_easyconfig_templates,
self.options.list_easyblocks,
self.options.list_toolchains,
self.options.avail_easyconfig_constants,
self.options.avail_easyconfig_licenses,
self.options.avail_repositories,
self.options.show_default_moduleclasses,
self.options.avail_modules_tools,
self.options.avail_module_naming_schemes,
]
):
build_easyconfig_constants_dict() # runs the easyconfig constants sanity check
self._postprocess_list_avail()
self._postprocess_config()
def postprocess(self):
"""Do some postprocessing, in particular print stuff"""
if self.options.unittest_file:
fancylogger.logToFile(self.options.unittest_file)
if any([self.options.avail_easyconfig_params, self.options.avail_easyconfig_templates,
self.options.list_easyblocks, self.options.list_toolchains]):
self._postprocess_list_avail()
def setUp(self):
(self.fd, self.logfn) = tempfile.mkstemp()
self.handle = os.fdopen(self.fd)
# set the test log format
fancylogger.setTestLogFormat()
# make new logger
fancylogger.logToFile(self.logfn)
# disable default ones (with default format)
fancylogger.disableDefaultHandlers()
def postprocess(self):
"""Do some postprocessing, in particular print stuff"""
if self.options.unittest_file:
fancylogger.logToFile(self.options.unittest_file)
if any([self.options.avail_easyconfig_params, self.options.avail_easyconfig_templates,
self.options.list_easyblocks, self.options.list_toolchains,
self.options.avail_easyconfig_constants, self.options.avail_easyconfig_licenses,
self.options.avail_repositories, self.options.show_default_moduleclasses,
]):
build_easyconfig_constants_dict() # runs the easyconfig constants sanity check
self._postprocess_list_avail()
self._postprocess_config()
def postprocess(self):
"""Do some postprocessing, in particular print stuff"""
if self.options.unittest_file:
fancylogger.logToFile(self.options.unittest_file)
if any([self.options.avail_easyconfig_params, self.options.avail_easyconfig_templates,
self.options.list_easyblocks, self.options.list_toolchains,
self.options.avail_easyconfig_constants, self.options.avail_easyconfig_licenses,
self.options.avail_repositories, self.options.show_default_moduleclasses,
]):
build_easyconfig_constants_dict() # runs the easyconfig constants sanity check
self._postprocess_list_avail()
self._postprocess_config()
def __init__(self, hostname, port, log_dir, filename, pidfile):
"""Constructor"""
stdin = '/dev/null'
stdout = os.path.join(log_dir, 'logging_error.log')
stderr = os.path.join(log_dir, 'logging_error.log')
Daemon.__init__(self, pidfile, stdin, stdout, stderr)
self.hostname = hostname
self.port = port
#Set up logging
# get logger, we will log to file
fancylogger.logToScreen(False)
# we want to log absolutely everything that's comming at us
fancylogger.setLogLevel(0)
self.logfile = os.path.join(log_dir, filename)
fancylogger.logToFile(self.logfile)
self.logger = fancylogger.getLogger()
self.socket_ = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
def __init__(self, hostname, port, log_dir, filename, pidfile):
"""Constructor"""
stdin = '/dev/null'
stdout = os.path.join(log_dir, 'logging_error.log')
stderr = os.path.join(log_dir, 'logging_error.log')
Daemon.__init__(self, pidfile, stdin, stdout, stderr)
self.hostname = hostname
self.port = port
##Set up logging
#get logger, we will log to file
fancylogger.logToScreen(False)
#we want to log absolutely everything that's comming at us
fancylogger.setLogLevel(0)
self.logfile = os.path.join(log_dir, filename)
fancylogger.logToFile(self.logfile)
self.logger = fancylogger.getLogger()
self.socket_ = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
def make_worker_log(name,
debug=False,
logfn_name=None,
disable_defaulthandlers=False):
"""Make a basic log object"""
if logfn_name is None:
logfn_name = name
logfn = '/tmp/scoop_%s.log' % logfn_name
if debug:
setLogLevelDebug()
logToFile(logfn, name=name)
os.chmod(logfn, stat.S_IRUSR | stat.S_IWUSR)
if disable_defaulthandlers:
disableDefaultHandlers()
_log = getLogger(name=name)
return _log
def test_zzz_logtostdout(self):
"""Testing redirecting log to stdout."""
fd, dummylogfn = tempfile.mkstemp(prefix='easybuild-dummy', suffix='.log')
os.close(fd)
for stdout_arg in ['--logtostdout', '-l']:
_stdout = sys.stdout
fd, fn = tempfile.mkstemp()
fh = os.fdopen(fd, 'w')
sys.stdout = fh
args = [
'--software-name=somethingrandom',
'--robot', '.',
'--debug',
stdout_arg,
]
self.eb_main(args, logfile=dummylogfn)
# make sure we restore
sys.stdout.flush()
sys.stdout = _stdout
fancylogger.logToScreen(enable=False, stdout=True)
outtxt = read_file(fn)
self.assertTrue(len(outtxt) > 100, "Log messages are printed to stdout when %s is used (outtxt: %s)" % (stdout_arg, outtxt))
# cleanup
os.remove(fn)
modify_env(os.environ, self.orig_environ)
tempfile.tempdir = None
if os.path.exists(dummylogfn):
os.remove(dummylogfn)
fancylogger.logToFile(self.logfile)
def test_module_mismatch(self):
"""Test whether mismatch detection between modules tool and 'module' function works."""
# redefine 'module' function (deliberate mismatch with used module command in MockModulesTool)
os.environ['module'] = "() { eval `/Users/kehoste/Modules/$MODULE_VERSION/bin/modulecmd bash $*`\n}"
self.assertErrorRegex(EasyBuildError, ".*pattern .* not found in defined 'module' function", MockModulesTool)
# check whether escaping error by allowing mismatch via build options works
build_options = {
'allow_modules_tool_mismatch': True,
}
init_config(build_options=build_options)
fancylogger.logToFile(self.logfile)
mt = MockModulesTool()
f = open(self.logfile, 'r')
logtxt = f.read()
f.close()
warn_regex = re.compile("WARNING .*pattern .* not found in defined 'module' function")
self.assertTrue(warn_regex.search(logtxt), "Found pattern '%s' in: %s" % (warn_regex.pattern, logtxt))
# redefine 'module' function with correct module command
os.environ['module'] = "() { eval `/bin/echo $*`\n}"
mt = MockModulesTool()
self.assertTrue(isinstance(mt.loaded_modules(), list)) # dummy usage
# a warning should be logged if the 'module' function is undefined
del os.environ['module']
mt = MockModulesTool()
f = open(self.logfile, 'r')
logtxt = f.read()
f.close()
warn_regex = re.compile("WARNING No 'module' function defined, can't check if it matches .*")
self.assertTrue(warn_regex.search(logtxt))
fancylogger.logToFile(self.logfile, enable=False)
def main(testing_data=(None, None, None)):
"""
Main function:
@arg options: a tuple: (options, paths, logger, logfile, hn) as defined in parse_options
This function will:
- read easyconfig
- build software
"""
# disallow running EasyBuild as root
if os.getuid() == 0:
sys.stderr.write("ERROR: You seem to be running EasyBuild with root privileges.\n"
"That's not wise, so let's end this here.\n"
"Exiting.\n")
sys.exit(1)
# steer behavior when testing main
testing = testing_data[0] is not None
args, logfile, do_build = testing_data
# initialise options
eb_go = eboptions.parse_options(args=args)
options = eb_go.options
orig_paths = eb_go.args
# set umask (as early as possible)
if options.umask is not None:
new_umask = int(options.umask, 8)
old_umask = os.umask(new_umask)
# set temporary directory to use
eb_tmpdir = set_tmpdir(options.tmpdir)
# initialise logging for main
if options.logtostdout:
fancylogger.logToScreen(enable=True, stdout=True)
else:
if logfile is None:
# mkstemp returns (fd,filename), fd is from os.open, not regular open!
fd, logfile = tempfile.mkstemp(suffix='.log', prefix='easybuild-')
os.close(fd)
fancylogger.logToFile(logfile)
print_msg('temporary log file in case of crash %s' % (logfile), log=None, silent=testing)
global _log
_log = fancylogger.getLogger(fname=False)
if options.umask is not None:
_log.info("umask set to '%s' (used to be '%s')" % (oct(new_umask), oct(old_umask)))
# hello world!
_log.info(this_is_easybuild())
# how was EB called?
eb_command_line = eb_go.generate_cmd_line() + eb_go.args
_log.info("Command line: %s" % (" ".join(eb_command_line)))
_log.info("Using %s as temporary directory" % eb_tmpdir)
if not options.robot is None:
if options.robot:
_log.info("Using robot path(s): %s" % options.robot)
else:
_log.error("No robot paths specified, and unable to determine easybuild-easyconfigs install path.")
# do not pass options.robot, it's not a list instance (and it shouldn't be modified)
robot_path = None
if options.robot:
robot_path = list(options.robot)
# determine easybuild-easyconfigs package install path
easyconfigs_paths = get_paths_for("easyconfigs", robot_path=robot_path)
# keep track of paths for install easyconfigs, so we can obtain find specified easyconfigs
easyconfigs_pkg_full_paths = easyconfigs_paths[:]
if not easyconfigs_paths:
_log.warning("Failed to determine install path for easybuild-easyconfigs package.")
# specified robot paths are preferred over installed easyconfig files
if robot_path:
robot_path.extend(easyconfigs_paths)
easyconfigs_paths = robot_path[:]
_log.info("Extended list of robot paths with paths for installed easyconfigs: %s" % robot_path)
# initialise the easybuild configuration
config.init(options, eb_go.get_options_by_section('config'))
# building a dependency graph implies force, so that all dependencies are retained
# and also skips validation of easyconfigs (e.g. checking os dependencies)
retain_all_deps = False
if options.dep_graph:
_log.info("Enabling force to generate dependency graph.")
options.force = True
retain_all_deps = True
config.init_build_options({
'aggregate_regtest': options.aggregate_regtest,
'allow_modules_tool_mismatch': options.allow_modules_tool_mismatch,
'check_osdeps': not options.ignore_osdeps,
'command_line': eb_command_line,
'debug': options.debug,
'dry_run': options.dry_run,
'easyblock': options.easyblock,
'experimental': options.experimental,
'force': options.force,
'group': options.group,
'ignore_dirs': options.ignore_dirs,
'modules_footer': options.modules_footer,
'only_blocks': options.only_blocks,
'recursive_mod_unload': options.recursive_module_unload,
'regtest_online': options.regtest_online,
'regtest_output_dir': options.regtest_output_dir,
'retain_all_deps': retain_all_deps,
'robot_path': robot_path,
'sequential': options.sequential,
'silent': testing,
'set_gid_bit': options.set_gid_bit,
'skip': options.skip,
'skip_test_cases': options.skip_test_cases,
'sticky_bit': options.sticky_bit,
'stop': options.stop,
'umask': options.umask,
'valid_module_classes': module_classes(),
'valid_stops': [x[0] for x in EasyBlock.get_steps()],
'validate': not options.force,
})
# search for easyconfigs
if options.search or options.search_short:
search_path = [os.getcwd()]
if easyconfigs_paths:
search_path = easyconfigs_paths
query = options.search or options.search_short
ignore_dirs = config.build_option('ignore_dirs')
silent = config.build_option('silent')
search_file(search_path, query, short=not options.search, ignore_dirs=ignore_dirs, silent=silent)
# process software build specifications (if any), i.e.
# software name/version, toolchain name/version, extra patches, ...
(try_to_generate, build_specs) = process_software_build_specs(options)
paths = []
if len(orig_paths) == 0:
if 'name' in build_specs:
paths = [obtain_path(build_specs, easyconfigs_paths, try_to_generate=try_to_generate,
exit_on_error=not testing)]
elif not any([options.aggregate_regtest, options.search, options.search_short, options.regtest]):
print_error(("Please provide one or multiple easyconfig files, or use software build "
"options to make EasyBuild search for easyconfigs"),
log=_log, opt_parser=eb_go.parser, exit_on_error=not testing)
else:
# look for easyconfigs with relative paths in easybuild-easyconfigs package,
# unless they were found at the given relative paths
if easyconfigs_pkg_full_paths:
# determine which easyconfigs files need to be found, if any
ecs_to_find = []
for idx, orig_path in enumerate(orig_paths):
if orig_path == os.path.basename(orig_path) and not os.path.exists(orig_path):
ecs_to_find.append((idx, orig_path))
_log.debug("List of easyconfig files to find: %s" % ecs_to_find)
# find missing easyconfigs by walking paths with installed easyconfig files
for path in easyconfigs_pkg_full_paths:
_log.debug("Looking for missing easyconfig files (%d left) in %s..." % (len(ecs_to_find), path))
for (subpath, dirnames, filenames) in os.walk(path, topdown=True):
for idx, orig_path in ecs_to_find[:]:
if orig_path in filenames:
full_path = os.path.join(subpath, orig_path)
_log.info("Found %s in %s: %s" % (orig_path, path, full_path))
orig_paths[idx] = full_path
# if file was found, stop looking for it (first hit wins)
ecs_to_find.remove((idx, orig_path))
# stop os.walk insanity as soon as we have all we need (os.walk loop)
if len(ecs_to_find) == 0:
break
# ignore subdirs specified to be ignored by replacing items in dirnames list used by os.walk
dirnames[:] = [d for d in dirnames if not d in options.ignore_dirs]
# stop os.walk insanity as soon as we have all we need (paths loop)
if len(ecs_to_find) == 0:
break
# indicate that specified paths do not contain generated easyconfig files
paths = [(path, False) for path in orig_paths]
_log.debug("Paths: %s" % paths)
# run regtest
if options.regtest or options.aggregate_regtest:
_log.info("Running regression test")
if paths:
ec_paths = [path[0] for path in paths]
else: # fallback: easybuild-easyconfigs install path
ec_paths = easyconfigs_pkg_full_paths
regtest_ok = regtest(ec_paths)
if not regtest_ok:
_log.info("Regression test failed (partially)!")
sys.exit(31) # exit -> 3x1t -> 31
# read easyconfig files
easyconfigs = []
for (path, generated) in paths:
path = os.path.abspath(path)
if not os.path.exists(path):
print_error("Can't find path %s" % path)
try:
files = find_easyconfigs(path, ignore_dirs=options.ignore_dirs)
for f in files:
if not generated and try_to_generate and build_specs:
ec_file = tweak(f, None, build_specs)
else:
ec_file = f
ecs = process_easyconfig(ec_file, build_specs=build_specs)
easyconfigs.extend(ecs)
except IOError, err:
_log.error("Processing easyconfigs in path %s failed: %s" % (path, err))
for ec in ordered_ecs:
(success, _) = build_and_install_software(ec, orig_environ)
if success:
correct_built_cnt += 1
all_built_cnt += 1
print_msg("Build succeeded for %s out of %s" % (correct_built_cnt, all_built_cnt), log=_log, silent=testing)
repo = init_repository(get_repository(), get_repositorypath())
repo.cleanup()
# cleanup and spec files
for ec in easyconfigs:
if 'original_spec' in ec and os.path.isfile(ec['spec']):
os.remove(ec['spec'])
# cleanup tmp log file (all is well, all modules have their own log file)
if options.logtostdout:
fancylogger.logToScreen(enable=False, stdout=True)
else:
fancylogger.logToFile(logfile, enable=False)
cleanup(logfile, eb_tmpdir, testing)
if __name__ == "__main__":
try:
main()
except EasyBuildError, e:
sys.stderr.write('ERROR: %s\n' % e.msg)
sys.exit(1)
print_msg("Build succeeded for %s out of %s" % (correct_built_cnt, all_built_cnt), log=_log, silent=testing)
repo = init_repository(get_repository(), get_repositorypath())
repo.cleanup()
# cleanup and spec files
for ec in easyconfigs:
if 'originalSpec' in ec and os.path.isfile(ec['spec']):
os.remove(ec['spec'])
# cleanup tmp log file (all is well, all modules have their own log file)
if options.logtostdout:
fancylogger.logToScreen(enable=False, stdout=True)
else:
fancylogger.logToFile(logfile, enable=False)
cleanup_logfile_and_exit(logfile, testing, False)
logfile = None
return logfile
def cleanup_logfile_and_exit(logfile, testing, doexit):
"""Cleanup the logfile and exit"""
if not testing and logfile is not None:
os.remove(logfile)
print_msg('temporary log file %s has been removed.' % (logfile), log=None, silent=testing)
if doexit:
sys.exit(0)
# make sure we restore
sys.stdout.flush()
sys.stdout = _stdout
fancylogger.logToScreen(enable=False, stdout=True)
outtxt = read_file(fn)
self.assertTrue(len(outtxt) > 100, "Log messages are printed to stdout when %s is used (outtxt: %s)" % (stdout_arg, outtxt))
# cleanup
os.remove(fn)
if os.path.exists(dummylogfn):
os.remove(dummylogfn)
fancylogger.logToFile(self.logfile)
def test_avail_easyconfig_params(self):
"""Test listing available easyconfig parameters."""
def run_test(custom=None, extra_params=[]):
"""Inner function to run actual test in current setting."""
fd, dummylogfn = tempfile.mkstemp(prefix='easybuild-dummy', suffix='.log')
os.close(fd)
for avail_arg in [
'-a',
'--avail-easyconfig-params',
]:
def main(testing_data=(None, None)):
"""
Main function:
@arg options: a tuple: (options, paths, logger, logfile, hn) as defined in parse_options
This function will:
- read easyconfig
- build software
"""
# disallow running EasyBuild as root
if os.getuid() == 0:
sys.stderr.write(
"ERROR: You seem to be running EasyBuild with root privileges.\n"
"That's not wise, so let's end this here.\n"
"Exiting.\n"
)
sys.exit(1)
# steer behavior when testing main
testing = testing_data[0] is not None
args, logfile = testing_data
# initialise options
eb_go = eboptions.parse_options(args=args)
options = eb_go.options
orig_paths = eb_go.args
# initialise logging for main
if options.logtostdout:
fancylogger.logToScreen(enable=True, stdout=True)
else:
if logfile is None:
# mkstemp returns (fd,filename), fd is from os.open, not regular open!
fd, logfile = tempfile.mkstemp(suffix=".log", prefix="easybuild-")
os.close(fd)
fancylogger.logToFile(logfile)
print_msg("temporary log file in case of crash %s" % (logfile), log=None, silent=testing)
global _log
_log = fancylogger.getLogger(fname=False)
# hello world!
_log.info(this_is_easybuild())
# set strictness of filetools module
if options.strict:
filetools.strictness = options.strict
if not options.robot is None:
if options.robot:
_log.info("Using robot path: %s" % options.robot)
else:
_log.error("No robot path specified, and unable to determine easybuild-easyconfigs install path.")
# determine easybuild-easyconfigs package install path
easyconfigs_paths = get_paths_for("easyconfigs", robot_path=options.robot)
easyconfigs_pkg_full_path = None
search_path = os.getcwd()
if easyconfigs_paths:
easyconfigs_pkg_full_path = easyconfigs_paths[0]
if not options.robot:
search_path = easyconfigs_pkg_full_path
else:
search_path = options.robot
else:
_log.info("Failed to determine install path for easybuild-easyconfigs package.")
if options.robot:
easyconfigs_paths = [options.robot] + easyconfigs_paths
# initialise the easybuild configuration
config.init(options, eb_go.get_options_by_section("config"))
# search for easyconfigs
if options.search:
search_file(search_path, options.search, silent=testing)
# process software build specifications (if any), i.e.
# software name/version, toolchain name/version, extra patches, ...
(try_to_generate, software_build_specs) = process_software_build_specs(options)
paths = []
if len(orig_paths) == 0:
if software_build_specs.has_key("name"):
paths = [
obtain_path(
software_build_specs, easyconfigs_paths, try_to_generate=try_to_generate, exit_on_error=not testing
)
]
elif not any([options.aggregate_regtest, options.search, options.regtest]):
print_error(
(
"Please provide one or multiple easyconfig files, or use software build "
"options to make EasyBuild search for easyconfigs"
),
log=_log,
opt_parser=eb_go.parser,
exit_on_error=not testing,
)
else:
# look for easyconfigs with relative paths in easybuild-easyconfigs package,
# unless they we found at the given relative paths
if easyconfigs_pkg_full_path:
# create a mapping from filename to path in easybuild-easyconfigs package install path
easyconfigs_map = {}
for (subpath, _, filenames) in os.walk(easyconfigs_pkg_full_path):
for filename in filenames:
easyconfigs_map.update({filename: os.path.join(subpath, filename)})
# try and find non-existing non-absolute eaysconfig paths in easybuild-easyconfigs package install path
for idx, orig_path in enumerate(orig_paths):
if not os.path.isabs(orig_path) and not os.path.exists(orig_path):
if orig_path in easyconfigs_map:
_log.info(
"Found %s in %s: %s" % (orig_path, easyconfigs_pkg_full_path, easyconfigs_map[orig_path])
)
orig_paths[idx] = easyconfigs_map[orig_path]
# indicate that specified paths do not contain generated easyconfig files
paths = [(path, False) for path in orig_paths]
_log.debug("Paths: %s" % paths)
# run regtest
if options.regtest or options.aggregate_regtest:
_log.info("Running regression test")
if paths:
regtest_ok = regtest(options, [path[0] for path in paths])
else: # fallback: easybuild-easyconfigs install path
regtest_ok = regtest(options, [easyconfigs_pkg_full_path])
if not regtest_ok:
_log.info("Regression test failed (partially)!")
sys.exit(31) # exit -> 3x1t -> 31
if any([options.search, options.regtest]):
cleanup_logfile_and_exit(logfile, testing, True)
# building a dependency graph implies force, so that all dependencies are retained
# and also skips validation of easyconfigs (e.g. checking os dependencies)
validate_easyconfigs = True
retain_all_deps = False
if options.dep_graph:
_log.info("Enabling force to generate dependency graph.")
options.force = True
validate_easyconfigs = False
retain_all_deps = True
# read easyconfig files
easyconfigs = []
for (path, generated) in paths:
path = os.path.abspath(path)
if not (os.path.exists(path)):
print_error("Can't find path %s" % path)
try:
files = find_easyconfigs(path)
for f in files:
if not generated and try_to_generate and software_build_specs:
ec_file = easyconfig.tools.tweak(f, None, software_build_specs)
else:
ec_file = f
easyconfigs.extend(process_easyconfig(ec_file, options.only_blocks, validate=validate_easyconfigs))
except IOError, err:
_log.error("Processing easyconfigs in path %s failed: %s" % (path, err))
def main(testing_data=(None, None)):
"""
Main function:
@arg options: a tuple: (options, paths, logger, logfile, hn) as defined in parse_options
This function will:
- read easyconfig
- build software
"""
# disallow running EasyBuild as root
if os.getuid() == 0:
sys.stderr.write("ERROR: You seem to be running EasyBuild with root privileges.\n" \
"That's not wise, so let's end this here.\n" \
"Exiting.\n")
sys.exit(1)
# steer behavior when testing main
testing = testing_data[0] is not None
args, logfile = testing_data
# initialise options
eb_go = eboptions.parse_options(args=args)
options = eb_go.options
orig_paths = eb_go.args
# initialise logging for main
if options.logtostdout:
fancylogger.logToScreen(enable=True, stdout=True)
else:
if logfile is None:
# mkstemp returns (fd,filename), fd is from os.open, not regular open!
fd, logfile = tempfile.mkstemp(suffix='.log', prefix='easybuild-')
os.close(fd)
fancylogger.logToFile(logfile)
print_msg('temporary log file in case of crash %s' % (logfile), log=None, silent=testing)
global _log
_log = fancylogger.getLogger(fname=False)
# hello world!
_log.info(this_is_easybuild())
# set strictness of filetools module
if options.strict:
filetools.strictness = options.strict
if not options.robot is None:
if options.robot:
_log.info("Using robot path: %s" % options.robot)
else:
_log.error("No robot path specified, and unable to determine easybuild-easyconfigs install path.")
# determine easybuild-easyconfigs package install path
easyconfigs_paths = get_paths_for("easyconfigs", robot_path=options.robot)
easyconfigs_pkg_full_path = None
search_path = os.getcwd()
if easyconfigs_paths:
easyconfigs_pkg_full_path = easyconfigs_paths[0]
if not options.robot:
search_path = easyconfigs_pkg_full_path
else:
search_path = options.robot
else:
_log.info("Failed to determine install path for easybuild-easyconfigs package.")
if options.robot:
easyconfigs_paths = [options.robot] + easyconfigs_paths
# initialise the easybuild configuration
config.init(options, eb_go.get_options_by_section('config'))
# search for easyconfigs
if options.search:
search_file(search_path, options.search, silent=testing)
# process software build specifications (if any), i.e.
# software name/version, toolchain name/version, extra patches, ...
(try_to_generate, software_build_specs) = process_software_build_specs(options)
paths = []
if len(orig_paths) == 0:
if software_build_specs.has_key('name'):
paths = [obtain_path(software_build_specs, easyconfigs_paths,
try_to_generate=try_to_generate, exit_on_error=not testing)]
elif not any([options.aggregate_regtest, options.search, options.regtest]):
print_error(("Please provide one or multiple easyconfig files, or use software build "
"options to make EasyBuild search for easyconfigs"),
log=_log, opt_parser=eb_go.parser, exit_on_error=not testing)
else:
# look for easyconfigs with relative paths in easybuild-easyconfigs package,
# unless they we found at the given relative paths
if easyconfigs_pkg_full_path:
# create a mapping from filename to path in easybuild-easyconfigs package install path
easyconfigs_map = {}
for (subpath, _, filenames) in os.walk(easyconfigs_pkg_full_path):
for filename in filenames:
easyconfigs_map.update({filename: os.path.join(subpath, filename)})
# try and find non-existing non-absolute eaysconfig paths in easybuild-easyconfigs package install path
for idx, orig_path in enumerate(orig_paths):
if not os.path.isabs(orig_path) and not os.path.exists(orig_path):
if orig_path in easyconfigs_map:
_log.info("Found %s in %s: %s" % (orig_path, easyconfigs_pkg_full_path,
easyconfigs_map[orig_path]))
orig_paths[idx] = easyconfigs_map[orig_path]
# indicate that specified paths do not contain generated easyconfig files
paths = [(path, False) for path in orig_paths]
_log.debug("Paths: %s" % paths)
# run regtest
if options.regtest or options.aggregate_regtest:
_log.info("Running regression test")
if paths:
regtest_ok = regtest(options, [path[0] for path in paths])
else: # fallback: easybuild-easyconfigs install path
regtest_ok = regtest(options, [easyconfigs_pkg_full_path])
if not regtest_ok:
_log.info("Regression test failed (partially)!")
sys.exit(31) # exit -> 3x1t -> 31
if any([options.search, options.regtest]):
cleanup_logfile_and_exit(logfile, testing, True)
# building a dependency graph implies force, so that all dependencies are retained
# and also skips validation of easyconfigs (e.g. checking os dependencies)
validate_easyconfigs = True
retain_all_deps = False
if options.dep_graph:
_log.info("Enabling force to generate dependency graph.")
options.force = True
validate_easyconfigs = False
retain_all_deps = True
# read easyconfig files
easyconfigs = []
for (path, generated) in paths:
path = os.path.abspath(path)
if not (os.path.exists(path)):
print_error("Can't find path %s" % path)
try:
files = find_easyconfigs(path)
for f in files:
if not generated and try_to_generate and software_build_specs:
ec_file = easyconfig.tools.tweak(f, None, software_build_specs)
else:
ec_file = f
easyconfigs.extend(process_easyconfig(ec_file, options.only_blocks,
validate=validate_easyconfigs))
except IOError, err:
_log.error("Processing easyconfigs in path %s failed: %s" % (path, err))
def tearDown(self):
fancylogger.logToFile(self.logfn, enable=False)
self.handle.close()
os.remove(self.logfn)
sys.stdout = _stdout
fancylogger.logToScreen(enable=False, stdout=True)
outtxt = read_file(fn)
self.assertTrue(
len(outtxt) > 100,
"Log messages are printed to stdout when %s is used (outtxt: %s)"
% (stdout_arg, outtxt))
# cleanup
os.remove(fn)
if os.path.exists(dummylogfn):
os.remove(dummylogfn)
fancylogger.logToFile(self.logfile)
def test_avail_easyconfig_params(self):
"""Test listing available easyconfig parameters."""
def run_test(custom=None, extra_params=[]):
"""Inner function to run actual test in current setting."""
fd, dummylogfn = tempfile.mkstemp(prefix='easybuild-dummy',
suffix='.log')
os.close(fd)
for avail_arg in [
'-a',
'--avail-easyconfig-params',
]:
raise EasyBuildError(newMsg)
# set format for logger
LOGGING_FORMAT = EB_MSG_PREFIX + ' %(asctime)s %(name)s %(levelname)s %(message)s'
fancylogger.setLogFormat(LOGGING_FORMAT)
# set the default LoggerClass to EasyBuildLog
fancylogger.logging.setLoggerClass(EasyBuildLog)
# you can't easily set another LoggerClass before fancylogger calls getLogger on import
_init_fancylog = fancylogger.getLogger(fname=False)
del _init_fancylog.manager.loggerDict[_init_fancylog.name]
# we need to make sure there is a handler
fancylogger.logToFile(filename=os.devnull)
# EasyBuildLog
_init_easybuildlog = fancylogger.getLogger(fname=False)
def get_log(name=None):
"""
Generate logger object
"""
# fname is always get_log, useless
log = fancylogger.getLogger(name, fname=False)
log.info("Logger started for %s." % name)
log.deprecated("get_log", "2.0")
return log
import test.framework.modules as m
import test.framework.options as o
import test.framework.repository as r
import test.framework.robot as robot
import test.framework.systemtools as s
import test.framework.toolchain as tc
import test.framework.toolchainvariables as tcv
import test.framework.toy_build as t
import test.framework.variables as v
# initialize logger for all the unit tests
fd, log_fn = tempfile.mkstemp(prefix='easybuild-tests-', suffix='.log')
os.close(fd)
os.remove(log_fn)
fancylogger.logToFile(log_fn)
log = fancylogger.getLogger()
log.setLevelName('DEBUG')
# call suite() for each module and then run them all
# note: make sure the options unit tests run first, to avoid running some of them with a readily initialized config
SUITE = unittest.TestSuite([x.suite() for x in [o, r, e, mg, m, f, a, robot, b, v, g, tcv, tc, t, c, s]])
# uses XMLTestRunner if possible, so we can output an XML file that can be supplied to Jenkins
xml_msg = ""
try:
import xmlrunner # requires unittest-xml-reporting package
xml_dir = 'test-reports'
res = xmlrunner.XMLTestRunner(output=xml_dir, verbosity=1).run(SUITE)
xml_msg = ", XML output of tests available in %s directory" % xml_dir
except ImportError, err:
raise EasyBuildError(newMsg)
# set format for logger
LOGGING_FORMAT = EB_MSG_PREFIX + ' %(asctime)s %(name)s %(levelname)s %(message)s'
fancylogger.setLogFormat(LOGGING_FORMAT)
# set the default LoggerClass to EasyBuildLog
fancylogger.logging.setLoggerClass(EasyBuildLog)
# you can't easily set another LoggerClass before fancylogger calls getLogger on import
_init_fancylog = fancylogger.getLogger(fname=False)
del _init_fancylog.manager.loggerDict[_init_fancylog.name]
# we need to make sure there is a handler
fancylogger.logToFile(filename=os.devnull)
# EasyBuildLog
_init_easybuildlog = fancylogger.getLogger(fname=False)
def get_log(name=None):
"""
Generate logger object
"""
# fname is always get_log, useless
log = fancylogger.getLogger(name, fname=False)
log.info("Logger started for %s." % name)
return log
def print_msg(msg, log=None, silent=False):