def test_logtofile(self):
"""Test to see if logtofile doesn't fail when logging to a non existing file /directory"""
tempdir = tempfile.mkdtemp()
non_dir = os.path.join(tempdir, 'verytempdir')
fancylogger.logToFile(os.path.join(non_dir, 'nosuchfile'))
# clean up temp dir
shutil.rmtree(tempdir)
def tearDown(self):
fancylogger.logToFile(self.logfn, enable=False)
self.handle.close()
os.remove(self.logfn)
fancylogger.FancyLogger.RAISE_EXCEPTION_CLASS = self.orig_raise_exception_class
fancylogger.FancyLogger.RAISE_EXCEPTION_LOG_METHOD = self.orig_raise_exception_method
def test_loggedexception_specifiedlogger(self):
"""Test LoggedException custom exception class."""
fd, tmplog = tempfile.mkstemp()
os.close(fd)
# set log format, for each regex searching
setLogFormat("%(name)s :: %(message)s")
logger1 = getLogger('testlogger_one')
logger2 = getLogger('testlogger_two')
# if logger is specified, it should be used
logToFile(tmplog, enable=True)
self.assertErrorRegex(LoggedException,
'BOOM',
raise_loggedexception,
'BOOM',
logger=logger1)
logToFile(tmplog, enable=False)
rootlog = getRootLoggerName()
log_re = re.compile(
"^%s.testlogger_one :: BOOM( \(at .*:[0-9]+ in raise_loggedexception\))?$"
% rootlog, re.M)
logtxt = open(tmplog, 'r').read()
self.assertTrue(log_re.match(logtxt),
"%s matches %s" % (log_re.pattern, logtxt))
os.remove(tmplog)
def parseoptions(self, options_list=None):
"""
Handle mpirun mode:
- continue with reduced set of commandline options
- These options are the keys of opts_to_remove.
- The values of opts_to_remove are the number of arguments of these options, that also need to be removed.
"""
if options_list is None:
options_list = self.default_parseoptions()
newopts = options_list[:] # copy
if self.mpirunmode:
opts_to_remove = {
'-np': 1,
'-machinefile': 1
}
for opt in opts_to_remove.keys():
try:
pos = newopts.index(opt)
# remove 1 + args
del newopts[pos:pos + 1 + opts_to_remove[opt]]
except ValueError:
continue
GeneralOption.parseoptions(self, newopts)
# set error logging to file as soon as possible
if self.options.logtofile:
print("logtofile %s" % self.options.logtofile)
if os.path.exists(self.options.logtofile):
os.remove(self.options.logtofile)
fancylogger.logToFile(self.options.logtofile)
fancylogger.logToScreen(False)
def postprocess(self):
"""Do some postprocessing, in particular print stuff"""
build_log.EXPERIMENTAL = self.options.experimental
config.SUPPORT_OLDSTYLE = self.options.oldstyleconfig
# set strictness of run module
if self.options.strict:
run.strictness = self.options.strict
if self.options.deprecated:
build_log.CURRENT_VERSION = LooseVersion(self.options.deprecated)
if self.options.unittest_file:
fancylogger.logToFile(self.options.unittest_file)
if any([self.options.avail_easyconfig_params, self.options.avail_easyconfig_templates,
self.options.list_easyblocks, self.options.list_toolchains,
self.options.avail_easyconfig_constants, self.options.avail_easyconfig_licenses,
self.options.avail_repositories, self.options.show_default_moduleclasses,
self.options.avail_modules_tools, self.options.avail_module_naming_schemes,
]):
build_easyconfig_constants_dict() # runs the easyconfig constants sanity check
self._postprocess_list_avail()
self._postprocess_config()
def test_easybuilderror(self):
"""Tests for EasyBuildError."""
fd, tmplog = tempfile.mkstemp()
os.close(fd)
# set log format, for each regex searching
setLogFormat("%(name)s :: %(message)s")
# if no logger is available, and no logger is specified, use default 'root' fancylogger
logToFile(tmplog, enable=True)
self.assertErrorRegex(EasyBuildError, 'BOOM', raise_easybuilderror, 'BOOM')
logToFile(tmplog, enable=False)
log_re = re.compile("^%s ::.* BOOM \(at .*:[0-9]+ in [a-z_]+\)$" % getRootLoggerName(), re.M)
logtxt = open(tmplog, 'r').read()
self.assertTrue(log_re.match(logtxt), "%s matches %s" % (log_re.pattern, logtxt))
# test formatting of message
self.assertErrorRegex(EasyBuildError, 'BOOMBAF', raise_easybuilderror, 'BOOM%s', 'BAF')
# a '%s' in a value used to template the error message should not print a traceback!
self.mock_stderr(True)
self.assertErrorRegex(EasyBuildError, 'err: msg: %s', raise_easybuilderror, "err: %s", "msg: %s")
stderr = self.get_stderr()
self.mock_stderr(False)
# stderr should be *empty* (there should definitely not be a traceback)
self.assertEqual(stderr, '')
os.remove(tmplog)
def test_loggedexception_defaultlogger(self):
"""Test LoggedException custom exception class."""
fd, tmplog = tempfile.mkstemp()
os.close(fd)
# set log format, for each regex searching
setLogFormat("%(name)s :: %(message)s")
# if no logger is available, and no logger is specified, use default 'root' fancylogger
logToFile(tmplog, enable=True)
self.assertErrorRegex(LoggedException, 'BOOM', raise_loggedexception, 'BOOM')
logToFile(tmplog, enable=False)
log_re = re.compile("^%s :: BOOM( \(at .*:[0-9]+ in raise_loggedexception\))?$" % getRootLoggerName(), re.M)
with open(tmplog, 'r') as f:
logtxt = f.read()
self.assertTrue(log_re.match(logtxt), "%s matches %s" % (log_re.pattern, logtxt))
# test formatting of message
self.assertErrorRegex(LoggedException, 'BOOMBAF', raise_loggedexception, 'BOOM%s', 'BAF')
# test log message that contains '%s' without any formatting arguments being passed
# test formatting of message
self.assertErrorRegex(LoggedException, "BOOM '%s'", raise_loggedexception, "BOOM '%s'")
os.remove(tmplog)
def test_easybuilderror(self):
"""Tests for EasyBuildError."""
fd, tmplog = tempfile.mkstemp()
os.close(fd)
# set log format, for each regex searching
setLogFormat("%(name)s :: %(message)s")
# if no logger is available, and no logger is specified, use default 'root' fancylogger
logToFile(tmplog, enable=True)
self.assertErrorRegex(EasyBuildError, 'BOOM', raise_easybuilderror,
'BOOM')
logToFile(tmplog, enable=False)
log_re = re.compile(
"^%s ::.* BOOM \(at .*:[0-9]+ in [a-z_]+\)$" % getRootLoggerName(),
re.M)
logtxt = open(tmplog, 'r').read()
self.assertTrue(log_re.match(logtxt),
"%s matches %s" % (log_re.pattern, logtxt))
# test formatting of message
self.assertErrorRegex(EasyBuildError, 'BOOMBAF', raise_easybuilderror,
'BOOM%s', 'BAF')
# a '%s' in a value used to template the error message should not print a traceback!
self.mock_stderr(True)
self.assertErrorRegex(EasyBuildError, 'err: msg: %s',
raise_easybuilderror, "err: %s", "msg: %s")
stderr = self.get_stderr()
self.mock_stderr(False)
# stderr should be *empty* (there should definitely not be a traceback)
self.assertEqual(stderr, '')
os.remove(tmplog)
def test_log_levels(self):
"""Test whether log levels are respected"""
fd, tmplog = tempfile.mkstemp()
os.close(fd)
# set log format, for each regex searching
setLogFormat("%(name)s [%(levelname)s] :: %(message)s")
# test basic log methods
logToFile(tmplog, enable=True)
log = getLogger('test_easybuildlog')
for level in ['ERROR', 'WARNING', 'INFO', 'DEBUG', 'DEVEL']:
log.setLevelName(level)
log.raiseError = False
log.error('kaput')
log.deprecated('almost kaput', '10000000000000')
log.raiseError = True
log.warn('this is a warning')
log.info('fyi')
log.debug('gdb')
log.devel('tmi')
logToFile(tmplog, enable=False)
logtxt = read_file(tmplog)
root = getRootLoggerName()
devel_msg = r"%s.test_easybuildlog \[DEVEL\] :: tmi" % root
debug_msg = r"%s.test_easybuildlog \[DEBUG\] :: gdb" % root
info_msg = r"%s.test_easybuildlog \[INFO\] :: fyi" % root
warning_msg = r"%s.test_easybuildlog \[WARNING\] :: this is a warning" % root
deprecated_msg = r"%s.test_easybuildlog \[WARNING\] :: Deprecated functionality, .*: almost kaput; see .*" % root
error_msg = r"%s.test_easybuildlog \[ERROR\] :: EasyBuild crashed with an error \(at .* in .*\): kaput" % root
expected_logtxt = '\n'.join([
error_msg,
error_msg,
deprecated_msg,
warning_msg,
error_msg,
deprecated_msg,
warning_msg,
info_msg,
error_msg,
deprecated_msg,
warning_msg,
info_msg,
debug_msg,
error_msg,
deprecated_msg,
warning_msg,
info_msg,
debug_msg,
devel_msg,
])
logtxt_regex = re.compile(r'^%s' % expected_logtxt, re.M)
self.assertTrue(
logtxt_regex.search(logtxt),
"Pattern '%s' found in %s" % (logtxt_regex.pattern, logtxt))
def test_module_mismatch(self):
"""Test whether mismatch detection between modules tool and 'module' function works."""
# redefine 'module' function (deliberate mismatch with used module command in MockModulesTool)
os.environ["module"] = "() { eval `/tmp/Modules/$MODULE_VERSION/bin/modulecmd bash $*`\n}"
error_regex = ".*pattern .* not found in defined 'module' function"
self.assertErrorRegex(EasyBuildError, error_regex, MockModulesTool, testing=True)
# check whether escaping error by allowing mismatch via build options works
build_options = {"allow_modules_tool_mismatch": True}
init_config(build_options=build_options)
fancylogger.logToFile(self.logfile)
mt = MockModulesTool(testing=True)
f = open(self.logfile, "r")
logtxt = f.read()
f.close()
warn_regex = re.compile("WARNING .*pattern .* not found in defined 'module' function")
self.assertTrue(warn_regex.search(logtxt), "Found pattern '%s' in: %s" % (warn_regex.pattern, logtxt))
# redefine 'module' function with correct module command
os.environ["module"] = "() { eval `/bin/echo $*`\n}"
mt = MockModulesTool(testing=True)
self.assertTrue(isinstance(mt.loaded_modules(), list)) # dummy usage
# a warning should be logged if the 'module' function is undefined
del os.environ["module"]
mt = MockModulesTool(testing=True)
f = open(self.logfile, "r")
logtxt = f.read()
f.close()
warn_regex = re.compile("WARNING No 'module' function defined, can't check if it matches .*")
self.assertTrue(warn_regex.search(logtxt), "Pattern %s found in %s" % (warn_regex.pattern, logtxt))
fancylogger.logToFile(self.logfile, enable=False)
def test_easybuilderror(self):
"""Tests for EasyBuildError."""
fd, tmplog = tempfile.mkstemp()
os.close(fd)
# set log format, for each regex searching
setLogFormat("%(name)s :: %(message)s")
# if no logger is available, and no logger is specified, use default 'root' fancylogger
logToFile(tmplog, enable=True)
self.assertErrorRegex(EasyBuildError, 'BOOM', raise_easybuilderror,
'BOOM')
logToFile(tmplog, enable=False)
log_re = re.compile(
"^%s :: BOOM \(at .*:[0-9]+ in [a-z_]+\)$" % getRootLoggerName(),
re.M)
logtxt = open(tmplog, 'r').read()
self.assertTrue(log_re.match(logtxt),
"%s matches %s" % (log_re.pattern, logtxt))
# test formatting of message
self.assertErrorRegex(EasyBuildError, 'BOOMBAF', raise_easybuilderror,
'BOOM%s', 'BAF')
os.remove(tmplog)
def tearDown(self):
fancylogger.logToFile(self.logfn, enable=False)
self.handle.close()
os.remove(self.logfn)
fancylogger.FancyLogger.RAISE_EXCEPTION_CLASS = self.orig_raise_exception_class
fancylogger.FancyLogger.RAISE_EXCEPTION_LOG_METHOD = self.orig_raise_exception_method
def test_loggedexception_callerlogger(self):
"""Test LoggedException custom exception class."""
fd, tmplog = tempfile.mkstemp()
os.close(fd)
# set log format, for each regex searching
setLogFormat("%(name)s :: %(message)s")
logger = getLogger('testlogger_local')
# if no logger is specified, logger available in calling context should be used
logToFile(tmplog, enable=True)
self.assertErrorRegex(LoggedException, 'BOOM', raise_loggedexception,
'BOOM')
logToFile(tmplog, enable=False)
rootlog = getRootLoggerName()
log_re = re.compile(
"^%s(.testlogger_local)? :: BOOM( \(at .*:[0-9]+ in raise_loggedexception\))?$"
% rootlog)
with open(tmplog, 'r') as f:
logtxt = f.read()
self.assertTrue(log_re.match(logtxt),
"%s matches %s" % (log_re.pattern, logtxt))
os.remove(tmplog)
def test_logtofile(self):
"""Test to see if logtofile doesn't fail when logging to a non existing file /directory"""
tempdir = tempfile.mkdtemp()
non_dir = os.path.join(tempdir, 'verytempdir')
fancylogger.logToFile(os.path.join(non_dir, 'nosuchfile'))
# clean up temp dir
shutil.rmtree(tempdir)
def test_loggedexception_defaultlogger(self):
"""Test LoggedException custom exception class."""
fd, tmplog = tempfile.mkstemp()
os.close(fd)
# set log format, for each regex searching
setLogFormat("%(name)s :: %(message)s")
# if no logger is available, and no logger is specified, use default 'root' fancylogger
logToFile(tmplog, enable=True)
self.assertErrorRegex(LoggedException, 'BOOM', raise_loggedexception, 'BOOM')
logToFile(tmplog, enable=False)
log_re = re.compile("^%s :: BOOM( \(at .*:[0-9]+ in raise_loggedexception\))?$" % getRootLoggerName(), re.M)
logtxt = open(tmplog, 'r').read()
self.assertTrue(log_re.match(logtxt), "%s matches %s" % (log_re.pattern, logtxt))
# test formatting of message
self.assertErrorRegex(LoggedException, 'BOOMBAF', raise_loggedexception, 'BOOM%s', 'BAF')
# test log message that contains '%s' without any formatting arguments being passed
# test formatting of message
self.assertErrorRegex(LoggedException, "BOOM '%s'", raise_loggedexception, "BOOM '%s'")
os.remove(tmplog)
def postprocess(self):
"""Do some postprocessing, in particular print stuff"""
build_log.EXPERIMENTAL = self.options.experimental
config.SUPPORT_OLDSTYLE = self.options.oldstyleconfig
# set strictness of run module
if self.options.strict:
run.strictness = self.options.strict
# override current version of EasyBuild with version specified to --deprecated
if self.options.deprecated:
build_log.CURRENT_VERSION = LooseVersion(self.options.deprecated)
# log to specified value of --unittest-file
if self.options.unittest_file:
fancylogger.logToFile(self.options.unittest_file)
# prepare for --list/--avail
if any([
self.options.avail_easyconfig_params,
self.options.avail_easyconfig_templates,
self.options.list_easyblocks,
self.options.list_toolchains,
self.options.avail_cfgfile_constants,
self.options.avail_easyconfig_constants,
self.options.avail_easyconfig_licenses,
self.options.avail_repositories,
self.options.show_default_moduleclasses,
self.options.avail_modules_tools,
self.options.avail_module_naming_schemes,
]):
build_easyconfig_constants_dict(
) # runs the easyconfig constants sanity check
self._postprocess_list_avail()
# fail early if required dependencies for functionality requiring using GitHub API are not available:
if self.options.from_pr or self.options.upload_test_report:
if not HAVE_GITHUB_API:
self.log.error(
"Required support for using GitHub API is not available (see warnings)."
)
# make sure a GitHub token is available when it's required
if self.options.upload_test_report:
if not HAVE_KEYRING:
self.log.error(
"Python 'keyring' module required for obtaining GitHub token is not available."
)
if self.options.github_user is None:
self.log.error(
"No GitHub user name provided, required for fetching GitHub token."
)
token = fetch_github_token(self.options.github_user)
if token is None:
self.log.error(
"Failed to obtain required GitHub token for user '%s'" %
self.options.github_user)
self._postprocess_config()
def postprocess(self):
"""Do some postprocessing, in particular print stuff"""
build_log.EXPERIMENTAL = self.options.experimental
# set strictness of run module
if self.options.strict:
run.strictness = self.options.strict
# override current version of EasyBuild with version specified to --deprecated
if self.options.deprecated:
build_log.CURRENT_VERSION = LooseVersion(self.options.deprecated)
# log to specified value of --unittest-file
if self.options.unittest_file:
fancylogger.logToFile(self.options.unittest_file)
# set tmpdir
self.tmpdir = set_tmpdir(self.options.tmpdir)
# take --include options into account
self._postprocess_include()
# prepare for --list/--avail
if any([self.options.avail_easyconfig_params, self.options.avail_easyconfig_templates,
self.options.list_easyblocks, self.options.list_toolchains, self.options.avail_cfgfile_constants,
self.options.avail_easyconfig_constants, self.options.avail_easyconfig_licenses,
self.options.avail_repositories, self.options.show_default_moduleclasses,
self.options.avail_modules_tools, self.options.avail_module_naming_schemes,
self.options.show_default_configfiles,
]):
build_easyconfig_constants_dict() # runs the easyconfig constants sanity check
self._postprocess_list_avail()
# fail early if required dependencies for functionality requiring using GitHub API are not available:
if self.options.from_pr or self.options.upload_test_report:
if not HAVE_GITHUB_API:
raise EasyBuildError("Required support for using GitHub API is not available (see warnings).")
if self.options.module_syntax == ModuleGeneratorLua.SYNTAX and self.options.modules_tool != Lmod.__name__:
raise EasyBuildError("Generating Lua module files requires Lmod as modules tool.")
# make sure a GitHub token is available when it's required
if self.options.upload_test_report:
if not HAVE_KEYRING:
raise EasyBuildError("Python 'keyring' module required for obtaining GitHub token is not available.")
if self.options.github_user is None:
raise EasyBuildError("No GitHub user name provided, required for fetching GitHub token.")
token = fetch_github_token(self.options.github_user)
if token is None:
raise EasyBuildError("Failed to obtain required GitHub token for user '%s'", self.options.github_user)
# make sure autopep8 is available when it needs to be
if self.options.dump_autopep8:
if not HAVE_AUTOPEP8:
raise EasyBuildError("Python 'autopep8' module required to reformat dumped easyconfigs as requested")
self._postprocess_external_modules_metadata()
self._postprocess_config()
def setUp(self):
(self.fd, self.logfn) = tempfile.mkstemp()
self.handle = os.fdopen(self.fd)
# set the test log format
fancylogger.setTestLogFormat()
# make new logger
fancylogger.logToFile(self.logfn)
# disable default ones (with default format)
fancylogger.disableDefaultHandlers()
def test_module_mismatch(self):
"""Test whether mismatch detection between modules tool and 'module' function works."""
# redefine 'module' function (deliberate mismatch with used module command in MockModulesTool)
os.environ[
'module'] = "() { eval `/tmp/Modules/$MODULE_VERSION/bin/modulecmd bash $*`\n}"
error_regex = ".*pattern .* not found in defined 'module' function"
self.assertErrorRegex(EasyBuildError,
error_regex,
MockModulesTool,
testing=True)
# check whether escaping error by allowing mismatch via build options works
build_options = {
'allow_modules_tool_mismatch': True,
}
init_config(build_options=build_options)
fancylogger.logToFile(self.logfile)
mt = MockModulesTool(testing=True)
f = open(self.logfile, 'r')
logtxt = f.read()
f.close()
warn_regex = re.compile(
"WARNING .*pattern .* not found in defined 'module' function")
self.assertTrue(
warn_regex.search(logtxt),
"Found pattern '%s' in: %s" % (warn_regex.pattern, logtxt))
# redefine 'module' function with correct module command
os.environ['module'] = "() { eval `/bin/echo $*`\n}"
MockModulesTool._instances.pop(MockModulesTool)
mt = MockModulesTool(testing=True)
self.assertTrue(isinstance(mt.loaded_modules(), list)) # dummy usage
# a warning should be logged if the 'module' function is undefined
del os.environ['module']
MockModulesTool._instances.pop(MockModulesTool)
mt = MockModulesTool(testing=True)
f = open(self.logfile, 'r')
logtxt = f.read()
f.close()
warn_regex = re.compile(
"WARNING No 'module' function defined, can't check if it matches .*"
)
self.assertTrue(
warn_regex.search(logtxt),
"Pattern %s found in %s" % (warn_regex.pattern, logtxt))
fancylogger.logToFile(self.logfile, enable=False)
def setUp(self):
(self.fd, self.logfn) = tempfile.mkstemp()
self.handle = os.fdopen(self.fd)
# set the test log format
fancylogger.setTestLogFormat()
# make new logger
fancylogger.logToFile(self.logfn)
# disable default ones (with default format)
fancylogger.disableDefaultHandlers()
self.orig_raise_exception_class = fancylogger.FancyLogger.RAISE_EXCEPTION_CLASS
self.orig_raise_exception_method = fancylogger.FancyLogger.RAISE_EXCEPTION_LOG_METHOD
def init_logging(logfile, session, rstype):
"""Initiates the logfile"""
logfile = logfile % {
'session': session,
'rstype': rstype,
'pid': str(os.getpid())
}
logdir = os.path.dirname(logfile)
if logdir:
if not os.path.exists(logdir):
os.makedirs(logdir)
os.chmod(logdir, stat.S_IRWXU)
fancylogger.logToFile(logfile)
logger.info('Logging to file %s:' % logfile)
def setUp(self):
(self.fd, self.logfn) = tempfile.mkstemp()
self.handle = os.fdopen(self.fd)
# set the test log format
fancylogger.setTestLogFormat()
# make new logger
fancylogger.logToFile(self.logfn)
# disable default ones (with default format)
fancylogger.disableDefaultHandlers()
self.orig_raise_exception_class = fancylogger.FancyLogger.RAISE_EXCEPTION_CLASS
self.orig_raise_exception_method = fancylogger.FancyLogger.RAISE_EXCEPTION_LOG_METHOD
def init_logging(logfile, logtostdout=False, testing=False):
"""Initialize logging."""
if logtostdout:
fancylogger.logToScreen(enable=True, stdout=True)
else:
if logfile is None:
# mkstemp returns (fd,filename), fd is from os.open, not regular open!
fd, logfile = tempfile.mkstemp(suffix='.log', prefix='easybuild-')
os.close(fd)
fancylogger.logToFile(logfile)
print_msg('temporary log file in case of crash %s' % (logfile), log=None, silent=testing)
log = fancylogger.getLogger(fname=False)
return log, logfile
def init_logging(logfile, logtostdout=False, silent=False, colorize=fancylogger.Colorize.AUTO):
"""Initialize logging."""
if logtostdout:
fancylogger.logToScreen(enable=True, stdout=True, colorize=colorize)
else:
if logfile is None:
# mkstemp returns (fd,filename), fd is from os.open, not regular open!
fd, logfile = tempfile.mkstemp(suffix='.log', prefix='easybuild-')
os.close(fd)
fancylogger.logToFile(logfile)
print_msg('temporary log file in case of crash %s' % (logfile), log=None, silent=silent)
log = fancylogger.getLogger(fname=False)
return log, logfile
def __init__(self, hostname, port, log_dir, filename, pidfile):
"""Constructor"""
stdin = '/dev/null'
stdout = os.path.join(log_dir, 'logging_error.log')
stderr = os.path.join(log_dir, 'logging_error.log')
Daemon.__init__(self, pidfile, stdin, stdout, stderr)
self.hostname = hostname
self.port = port
#Set up logging
# get logger, we will log to file
fancylogger.logToScreen(False)
# we want to log absolutely everything that's comming at us
fancylogger.setLogLevel(0)
self.logfile = os.path.join(log_dir, filename)
fancylogger.logToFile(self.logfile)
self.logger = fancylogger.getLogger()
self.socket_ = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
def __init__(self, hostname, port, log_dir, filename, pidfile):
"""Constructor"""
stdin = '/dev/null'
stdout = os.path.join(log_dir, 'logging_error.log')
stderr = os.path.join(log_dir, 'logging_error.log')
Daemon.__init__(self, pidfile, stdin, stdout, stderr)
self.hostname = hostname
self.port = port
#Set up logging
# get logger, we will log to file
fancylogger.logToScreen(False)
# we want to log absolutely everything that's comming at us
fancylogger.setLogLevel(0)
self.logfile = os.path.join(log_dir, filename)
fancylogger.logToFile(self.logfile)
self.logger = fancylogger.getLogger()
self.socket_ = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
def make_worker_log(name, debug=False, logfn_name=None, disable_defaulthandlers=False):
"""Make a basic log object"""
if logfn_name is None:
logfn_name = name
logfn = '/tmp/scoop_%s.log' % logfn_name
if debug:
setLogLevelDebug()
logToFile(logfn, name=name)
os.chmod(logfn, stat.S_IRUSR | stat.S_IWUSR)
if disable_defaulthandlers:
disableDefaultHandlers()
_log = getLogger(name=name)
return _log
def test_zzz_logtostdout(self):
"""Testing redirecting log to stdout."""
fd, dummylogfn = tempfile.mkstemp(prefix='easybuild-dummy',
suffix='.log')
os.close(fd)
for stdout_arg in ['--logtostdout', '-l']:
_stdout = sys.stdout
fd, fn = tempfile.mkstemp()
fh = os.fdopen(fd, 'w')
sys.stdout = fh
args = [
'--software-name=somethingrandom',
'--robot',
'.',
'--debug',
stdout_arg,
]
self.eb_main(args, logfile=dummylogfn)
# make sure we restore
sys.stdout.flush()
sys.stdout = _stdout
fancylogger.logToScreen(enable=False, stdout=True)
outtxt = read_file(fn)
self.assertTrue(
len(outtxt) > 100,
"Log messages are printed to stdout when %s is used (outtxt: %s)"
% (stdout_arg, outtxt))
# cleanup
os.remove(fn)
modify_env(os.environ, self.orig_environ)
tempfile.tempdir = None
if os.path.exists(dummylogfn):
os.remove(dummylogfn)
fancylogger.logToFile(self.logfile)
def test_log_levels(self):
"""Test whether log levels are respected"""
fd, tmplog = tempfile.mkstemp()
os.close(fd)
# set log format, for each regex searching
setLogFormat("%(name)s [%(levelname)s] :: %(message)s")
# test basic log methods
logToFile(tmplog, enable=True)
log = getLogger('test_easybuildlog')
for level in ['ERROR', 'WARNING', 'INFO', 'DEBUG', 'DEVEL']:
log.setLevelName(level)
log.raiseError = False
log.error('kaput')
log.deprecated('almost kaput', '10000000000000')
log.raiseError = True
log.warn('this is a warning')
log.info('fyi')
log.debug('gdb')
log.devel('tmi')
logToFile(tmplog, enable=False)
logtxt = read_file(tmplog)
root = getRootLoggerName()
devel_msg = r"%s.test_easybuildlog \[DEVEL\] :: tmi" % root
debug_msg = r"%s.test_easybuildlog \[DEBUG\] :: gdb" % root
info_msg = r"%s.test_easybuildlog \[INFO\] :: fyi" % root
warning_msg = r"%s.test_easybuildlog \[WARNING\] :: this is a warning" % root
deprecated_msg = r"%s.test_easybuildlog \[WARNING\] :: Deprecated functionality, .*: almost kaput; see .*" % root
error_msg = r"%s.test_easybuildlog \[ERROR\] :: EasyBuild crashed with an error \(at .* in .*\): kaput" % root
expected_logtxt = '\n'.join([
error_msg,
error_msg, deprecated_msg, warning_msg,
error_msg, deprecated_msg, warning_msg, info_msg,
error_msg, deprecated_msg, warning_msg, info_msg, debug_msg,
error_msg, deprecated_msg, warning_msg, info_msg, debug_msg, devel_msg,
])
logtxt_regex = re.compile(r'^%s' % expected_logtxt, re.M)
self.assertTrue(logtxt_regex.search(logtxt), "Pattern '%s' found in %s" % (logtxt_regex.pattern, logtxt))
def postprocess(self):
"""Do some postprocessing, in particular print stuff"""
build_log.EXPERIMENTAL = self.options.experimental
config.SUPPORT_OLDSTYLE = self.options.oldstyleconfig
# set strictness of run module
if self.options.strict:
run.strictness = self.options.strict
# override current version of EasyBuild with version specified to --deprecated
if self.options.deprecated:
build_log.CURRENT_VERSION = LooseVersion(self.options.deprecated)
# log to specified value of --unittest-file
if self.options.unittest_file:
fancylogger.logToFile(self.options.unittest_file)
# prepare for --list/--avail
if any([self.options.avail_easyconfig_params, self.options.avail_easyconfig_templates,
self.options.list_easyblocks, self.options.list_toolchains,
self.options.avail_easyconfig_constants, self.options.avail_easyconfig_licenses,
self.options.avail_repositories, self.options.show_default_moduleclasses,
self.options.avail_modules_tools, self.options.avail_module_naming_schemes,
]):
build_easyconfig_constants_dict() # runs the easyconfig constants sanity check
self._postprocess_list_avail()
# fail early if required dependencies for functionality requiring using GitHub API are not available:
if self.options.from_pr or self.options.upload_test_report:
if not HAVE_GITHUB_API:
self.log.error("Required support for using GitHub API is not available (see warnings).")
# make sure a GitHub token is available when it's required
if self.options.upload_test_report:
if not HAVE_KEYRING:
self.log.error("Python 'keyring' module required for obtaining GitHub token is not available.")
if self.options.github_user is None:
self.log.error("No GitHub user name provided, required for fetching GitHub token.")
token = fetch_github_token(self.options.github_user)
if token is None:
self.log.error("Failed to obtain required GitHub token for user '%s'" % self.options.github_user)
self._postprocess_config()
def test_loggedexception_callerlogger(self):
"""Test LoggedException custom exception class."""
fd, tmplog = tempfile.mkstemp()
os.close(fd)
# set log format, for each regex searching
setLogFormat("%(name)s :: %(message)s")
logger = getLogger('testlogger_local')
# if no logger is specified, logger available in calling context should be used
logToFile(tmplog, enable=True)
self.assertErrorRegex(LoggedException, 'BOOM', raise_loggedexception, 'BOOM')
logToFile(tmplog, enable=False)
rootlog = getRootLoggerName()
log_re = re.compile("^%s(.testlogger_local)? :: BOOM( \(at .*:[0-9]+ in raise_loggedexception\))?$" % rootlog)
logtxt = open(tmplog, 'r').read()
self.assertTrue(log_re.match(logtxt), "%s matches %s" % (log_re.pattern, logtxt))
os.remove(tmplog)
def test_easybuilderror(self):
"""Tests for EasyBuildError."""
fd, tmplog = tempfile.mkstemp()
os.close(fd)
# set log format, for each regex searching
setLogFormat("%(name)s :: %(message)s")
# if no logger is available, and no logger is specified, use default 'root' fancylogger
logToFile(tmplog, enable=True)
self.assertErrorRegex(EasyBuildError, 'BOOM', raise_easybuilderror, 'BOOM')
logToFile(tmplog, enable=False)
log_re = re.compile("^%s :: BOOM \(at .*:[0-9]+ in [a-z_]+\)$" % getRootLoggerName(), re.M)
logtxt = open(tmplog, 'r').read()
self.assertTrue(log_re.match(logtxt), "%s matches %s" % (log_re.pattern, logtxt))
# test formatting of message
self.assertErrorRegex(EasyBuildError, 'BOOMBAF', raise_easybuilderror, 'BOOM%s', 'BAF')
os.remove(tmplog)
def test_loggedexception_specifiedlogger(self):
"""Test LoggedException custom exception class."""
fd, tmplog = tempfile.mkstemp()
os.close(fd)
# set log format, for each regex searching
setLogFormat("%(name)s :: %(message)s")
logger1 = getLogger('testlogger_one')
logger2 = getLogger('testlogger_two')
# if logger is specified, it should be used
logToFile(tmplog, enable=True)
self.assertErrorRegex(LoggedException, 'BOOM', raise_loggedexception, 'BOOM', logger=logger1)
logToFile(tmplog, enable=False)
rootlog = getRootLoggerName()
log_re = re.compile("^%s.testlogger_one :: BOOM \(at .*:[0-9]+ in raise_loggedexception\)$" % rootlog, re.M)
logtxt = open(tmplog, 'r').read()
self.assertTrue(log_re.match(logtxt), "%s matches %s" % (log_re.pattern, logtxt))
os.remove(tmplog)
def test_zzz_logtostdout(self):
"""Testing redirecting log to stdout."""
fd, dummylogfn = tempfile.mkstemp(prefix='easybuild-dummy', suffix='.log')
os.close(fd)
for stdout_arg in ['--logtostdout', '-l']:
_stdout = sys.stdout
fd, fn = tempfile.mkstemp()
fh = os.fdopen(fd, 'w')
sys.stdout = fh
args = [
'--software-name=somethingrandom',
'--robot', '.',
'--debug',
stdout_arg,
]
self.eb_main(args, logfile=dummylogfn)
# make sure we restore
sys.stdout.flush()
sys.stdout = _stdout
fancylogger.logToScreen(enable=False, stdout=True)
outtxt = read_file(fn)
self.assertTrue(len(outtxt) > 100, "Log messages are printed to stdout when %s is used (outtxt: %s)" % (stdout_arg, outtxt))
# cleanup
os.remove(fn)
modify_env(os.environ, self.orig_environ)
tempfile.tempdir = None
if os.path.exists(dummylogfn):
os.remove(dummylogfn)
fancylogger.logToFile(self.logfile)
def test_easybuildlog(self):
"""Tests for EasyBuildLog."""
fd, tmplog = tempfile.mkstemp()
os.close(fd)
# set log format, for each regex searching
setLogFormat("%(name)s [%(levelname)s] :: %(message)s")
# test basic log methods
logToFile(tmplog, enable=True)
log = getLogger('test_easybuildlog')
log.setLevelName('DEBUG')
log.debug("123 debug")
log.info("foobar info")
log.warn("justawarning")
log.raiseError = False
log.error("kaput")
log.raiseError = True
try:
log.exception("oops")
except EasyBuildError:
pass
logToFile(tmplog, enable=False)
logtxt = read_file(tmplog)
root = getRootLoggerName()
expected_logtxt = '\n'.join([
r"%s.test_easybuildlog \[DEBUG\] :: 123 debug" % root,
r"%s.test_easybuildlog \[INFO\] :: foobar info" % root,
r"%s.test_easybuildlog \[WARNING\] :: justawarning" % root,
r"%s.test_easybuildlog \[ERROR\] :: EasyBuild crashed with an error \(at .* in .*\): kaput" % root,
r"%s.test_easybuildlog \[ERROR\] :: .*EasyBuild encountered an exception \(at .* in .*\): oops" % root,
'',
])
logtxt_regex = re.compile(r'^%s' % expected_logtxt, re.M)
self.assertTrue(logtxt_regex.search(logtxt), "Pattern '%s' found in %s" % (logtxt_regex.pattern, logtxt))
# wipe log so we can reuse it
write_file(tmplog, '')
# test formatting log messages by providing extra arguments
logToFile(tmplog, enable=True)
log.warn("%s", "bleh"),
log.info("%s+%s = %d", '4', '2', 42)
args = ['this', 'is', 'just', 'a', 'test']
log.debug("%s %s %s %s %s", *args)
log.raiseError = False
log.error("foo %s baz", 'baz')
log.raiseError = True
logToFile(tmplog, enable=False)
logtxt = read_file(tmplog)
expected_logtxt = '\n'.join([
r"%s.test_easybuildlog \[WARNING\] :: bleh" % root,
r"%s.test_easybuildlog \[INFO\] :: 4\+2 = 42" % root,
r"%s.test_easybuildlog \[DEBUG\] :: this is just a test" % root,
r"%s.test_easybuildlog \[ERROR\] :: EasyBuild crashed with an error \(at .* in .*\): foo baz baz" % root,
'',
])
logtxt_regex = re.compile(r'^%s' % expected_logtxt, re.M)
self.assertTrue(logtxt_regex.search(logtxt), "Pattern '%s' found in %s" % (logtxt_regex.pattern, logtxt))
# test deprecated behaviour: raise EasyBuildError on log.error and log.exception
os.environ['EASYBUILD_DEPRECATED'] = '2.1'
init_config()
log.warning("No raise for warnings")
self.assertErrorRegex(EasyBuildError, 'EasyBuild crashed with an error', log.error, 'foo')
self.assertErrorRegex(EasyBuildError, 'EasyBuild encountered an exception', log.exception, 'bar')
else:
test_report = create_test_report(success_msg, ecs_with_res,
init_session_state)
_log.debug("Test report: %s" % test_report)
if options.dump_test_report is not None:
write_file(options.dump_test_report, test_report)
_log.info("Test report dumped to %s" % options.dump_test_report)
print_msg(success_msg, log=_log, silent=testing)
# cleanup and spec files
for ec in easyconfigs:
if 'original_spec' in ec and os.path.isfile(ec['spec']):
os.remove(ec['spec'])
# cleanup tmp log file, unless one build failed (individual logs are located in eb_tmpdir path)
if options.logtostdout:
fancylogger.logToScreen(enable=False, stdout=True)
else:
fancylogger.logToFile(logfile, enable=False)
if overall_success:
cleanup(logfile, eb_tmpdir, testing)
if __name__ == "__main__":
try:
main()
except EasyBuildError, e:
sys.stderr.write('ERROR: %s\n' % e.msg)
sys.exit(1)
def main(testing_data=(None, None, None)):
"""
Main function:
@arg options: a tuple: (options, paths, logger, logfile, hn) as defined in parse_options
This function will:
- read easyconfig
- build software
"""
# purposely session state very early, to avoid modules loaded by EasyBuild meddling in
init_session_state = session_state()
# disallow running EasyBuild as root
if os.getuid() == 0:
sys.stderr.write(
"ERROR: You seem to be running EasyBuild with root privileges.\n"
"That's not wise, so let's end this here.\n"
"Exiting.\n")
sys.exit(1)
# steer behavior when testing main
testing = testing_data[0] is not None
args, logfile, do_build = testing_data
# initialise options
eb_go = eboptions.parse_options(args=args)
options = eb_go.options
orig_paths = eb_go.args
eb_config = eb_go.generate_cmd_line(add_default=True)
init_session_state.update({'easybuild_configuration': eb_config})
# set umask (as early as possible)
if options.umask is not None:
new_umask = int(options.umask, 8)
old_umask = os.umask(new_umask)
# set temporary directory to use
eb_tmpdir = set_tmpdir(options.tmpdir)
# initialise logging for main
if options.logtostdout:
fancylogger.logToScreen(enable=True, stdout=True)
else:
if logfile is None:
# mkstemp returns (fd,filename), fd is from os.open, not regular open!
fd, logfile = tempfile.mkstemp(suffix='.log', prefix='easybuild-')
os.close(fd)
fancylogger.logToFile(logfile)
print_msg('temporary log file in case of crash %s' % (logfile),
log=None,
silent=testing)
global _log
_log = fancylogger.getLogger(fname=False)
if options.umask is not None:
_log.info("umask set to '%s' (used to be '%s')" %
(oct(new_umask), oct(old_umask)))
# hello world!
_log.info(this_is_easybuild())
# how was EB called?
eb_command_line = eb_go.generate_cmd_line() + eb_go.args
_log.info("Command line: %s" % (" ".join(eb_command_line)))
_log.info("Using %s as temporary directory" % eb_tmpdir)
if not options.robot is None:
if options.robot:
_log.info("Using robot path(s): %s" % options.robot)
else:
_log.error(
"No robot paths specified, and unable to determine easybuild-easyconfigs install path."
)
# do not pass options.robot, it's not a list instance (and it shouldn't be modified)
robot_path = None
if options.robot:
robot_path = list(options.robot)
# determine easybuild-easyconfigs package install path
easyconfigs_paths = get_paths_for("easyconfigs", robot_path=robot_path)
# keep track of paths for install easyconfigs, so we can obtain find specified easyconfigs
easyconfigs_pkg_full_paths = easyconfigs_paths[:]
if not easyconfigs_paths:
_log.warning(
"Failed to determine install path for easybuild-easyconfigs package."
)
# process software build specifications (if any), i.e.
# software name/version, toolchain name/version, extra patches, ...
(try_to_generate, build_specs) = process_software_build_specs(options)
# specified robot paths are preferred over installed easyconfig files
# --try-X and --dep-graph both require --robot, so enable it with path of installed easyconfigs
if robot_path or try_to_generate or options.dep_graph:
if robot_path is None:
robot_path = []
robot_path.extend(easyconfigs_paths)
easyconfigs_paths = robot_path[:]
_log.info(
"Extended list of robot paths with paths for installed easyconfigs: %s"
% robot_path)
# initialise the easybuild configuration
config.init(options, eb_go.get_options_by_section('config'))
# building a dependency graph implies force, so that all dependencies are retained
# and also skips validation of easyconfigs (e.g. checking os dependencies)
retain_all_deps = False
if options.dep_graph:
_log.info("Enabling force to generate dependency graph.")
options.force = True
retain_all_deps = True
config.init_build_options({
'aggregate_regtest':
options.aggregate_regtest,
'allow_modules_tool_mismatch':
options.allow_modules_tool_mismatch,
'check_osdeps':
not options.ignore_osdeps,
'cleanup_builddir':
options.cleanup_builddir,
'command_line':
eb_command_line,
'debug':
options.debug,
'dry_run':
options.dry_run,
'easyblock':
options.easyblock,
'experimental':
options.experimental,
'force':
options.force,
'github_user':
options.github_user,
'group':
options.group,
'ignore_dirs':
options.ignore_dirs,
'modules_footer':
options.modules_footer,
'only_blocks':
options.only_blocks,
'recursive_mod_unload':
options.recursive_module_unload,
'regtest_output_dir':
options.regtest_output_dir,
'retain_all_deps':
retain_all_deps,
'robot_path':
robot_path,
'sequential':
options.sequential,
'silent':
testing,
'set_gid_bit':
options.set_gid_bit,
'skip':
options.skip,
'skip_test_cases':
options.skip_test_cases,
'sticky_bit':
options.sticky_bit,
'stop':
options.stop,
'umask':
options.umask,
'valid_module_classes':
module_classes(),
'valid_stops': [x[0] for x in EasyBlock.get_steps()],
'validate':
not options.force,
})
# obtain list of loaded modules, build options must be initialized first
modlist = session_module_list()
init_session_state.update({'module_list': modlist})
_log.debug("Initial session state: %s" % init_session_state)
# search for easyconfigs
if options.search or options.search_short:
search_path = [os.getcwd()]
if easyconfigs_paths:
search_path = easyconfigs_paths
query = options.search or options.search_short
ignore_dirs = config.build_option('ignore_dirs')
silent = config.build_option('silent')
search_file(search_path,
query,
short=not options.search,
ignore_dirs=ignore_dirs,
silent=silent)
paths = []
if len(orig_paths) == 0:
if options.from_pr:
pr_path = os.path.join(eb_tmpdir, "files_pr%s" % options.from_pr)
pr_files = fetch_easyconfigs_from_pr(
options.from_pr, path=pr_path, github_user=options.github_user)
paths = [(path, False) for path in pr_files
if path.endswith('.eb')]
elif 'name' in build_specs:
paths = [
obtain_path(build_specs,
easyconfigs_paths,
try_to_generate=try_to_generate,
exit_on_error=not testing)
]
elif not any([
options.aggregate_regtest, options.search,
options.search_short, options.regtest
]):
print_error((
"Please provide one or multiple easyconfig files, or use software build "
"options to make EasyBuild search for easyconfigs"),
log=_log,
opt_parser=eb_go.parser,
exit_on_error=not testing)
else:
# look for easyconfigs with relative paths in easybuild-easyconfigs package,
# unless they were found at the given relative paths
if easyconfigs_pkg_full_paths:
# determine which easyconfigs files need to be found, if any
ecs_to_find = []
for idx, orig_path in enumerate(orig_paths):
if orig_path == os.path.basename(
orig_path) and not os.path.exists(orig_path):
ecs_to_find.append((idx, orig_path))
_log.debug("List of easyconfig files to find: %s" % ecs_to_find)
# find missing easyconfigs by walking paths with installed easyconfig files
for path in easyconfigs_pkg_full_paths:
_log.debug(
"Looking for missing easyconfig files (%d left) in %s..." %
(len(ecs_to_find), path))
for (subpath, dirnames, filenames) in os.walk(path,
topdown=True):
for idx, orig_path in ecs_to_find[:]:
if orig_path in filenames:
full_path = os.path.join(subpath, orig_path)
_log.info("Found %s in %s: %s" %
(orig_path, path, full_path))
orig_paths[idx] = full_path
# if file was found, stop looking for it (first hit wins)
ecs_to_find.remove((idx, orig_path))
# stop os.walk insanity as soon as we have all we need (os.walk loop)
if len(ecs_to_find) == 0:
break
# ignore subdirs specified to be ignored by replacing items in dirnames list used by os.walk
dirnames[:] = [
d for d in dirnames if not d in options.ignore_dirs
]
# stop os.walk insanity as soon as we have all we need (paths loop)
if len(ecs_to_find) == 0:
break
# indicate that specified paths do not contain generated easyconfig files
paths = [(path, False) for path in orig_paths]
_log.debug("Paths: %s" % paths)
# run regtest
if options.regtest or options.aggregate_regtest:
_log.info("Running regression test")
if paths:
ec_paths = [path[0] for path in paths]
else: # fallback: easybuild-easyconfigs install path
ec_paths = easyconfigs_pkg_full_paths
regtest_ok = regtest(ec_paths)
if not regtest_ok:
_log.info("Regression test failed (partially)!")
sys.exit(31) # exit -> 3x1t -> 31
# read easyconfig files
easyconfigs = []
for (path, generated) in paths:
path = os.path.abspath(path)
if not os.path.exists(path):
print_error("Can't find path %s" % path)
try:
ec_files = find_easyconfigs(path, ignore_dirs=options.ignore_dirs)
for ec_file in ec_files:
# only pass build specs when not generating easyconfig files
if try_to_generate:
ecs = process_easyconfig(ec_file)
else:
ecs = process_easyconfig(ec_file, build_specs=build_specs)
easyconfigs.extend(ecs)
except IOError, err:
_log.error("Processing easyconfigs in path %s failed: %s" %
(path, err))
def main(testing_data=(None, None, None)):
"""
Main function:
@arg options: a tuple: (options, paths, logger, logfile, hn) as defined in parse_options
This function will:
- read easyconfig
- build software
"""
# purposely session state very early, to avoid modules loaded by EasyBuild meddling in
init_session_state = session_state()
# disallow running EasyBuild as root
if os.getuid() == 0:
sys.stderr.write("ERROR: You seem to be running EasyBuild with root privileges.\n"
"That's not wise, so let's end this here.\n"
"Exiting.\n")
sys.exit(1)
# steer behavior when testing main
testing = testing_data[0] is not None
args, logfile, do_build = testing_data
# initialise options
eb_go = eboptions.parse_options(args=args)
options = eb_go.options
orig_paths = eb_go.args
eb_config = eb_go.generate_cmd_line(add_default=True)
init_session_state.update({'easybuild_configuration': eb_config})
# set umask (as early as possible)
if options.umask is not None:
new_umask = int(options.umask, 8)
old_umask = os.umask(new_umask)
# set temporary directory to use
eb_tmpdir = set_tmpdir(options.tmpdir)
# initialise logging for main
if options.logtostdout:
fancylogger.logToScreen(enable=True, stdout=True)
else:
if logfile is None:
# mkstemp returns (fd,filename), fd is from os.open, not regular open!
fd, logfile = tempfile.mkstemp(suffix='.log', prefix='easybuild-')
os.close(fd)
fancylogger.logToFile(logfile)
print_msg('temporary log file in case of crash %s' % (logfile), log=None, silent=testing)
global _log
_log = fancylogger.getLogger(fname=False)
if options.umask is not None:
_log.info("umask set to '%s' (used to be '%s')" % (oct(new_umask), oct(old_umask)))
# hello world!
_log.info(this_is_easybuild())
# how was EB called?
eb_command_line = eb_go.generate_cmd_line() + eb_go.args
_log.info("Command line: %s" % (" ".join(eb_command_line)))
_log.info("Using %s as temporary directory" % eb_tmpdir)
if not options.robot is None:
if options.robot:
_log.info("Using robot path(s): %s" % options.robot)
else:
_log.error("No robot paths specified, and unable to determine easybuild-easyconfigs install path.")
# do not pass options.robot, it's not a list instance (and it shouldn't be modified)
robot_path = []
if options.robot:
robot_path = list(options.robot)
# determine easybuild-easyconfigs package install path
easyconfigs_paths = get_paths_for("easyconfigs", robot_path=robot_path)
# keep track of paths for install easyconfigs, so we can obtain find specified easyconfigs
easyconfigs_pkg_full_paths = easyconfigs_paths[:]
if not easyconfigs_paths:
_log.warning("Failed to determine install path for easybuild-easyconfigs package.")
# process software build specifications (if any), i.e.
# software name/version, toolchain name/version, extra patches, ...
(try_to_generate, build_specs) = process_software_build_specs(options)
# specified robot paths are preferred over installed easyconfig files
# --try-X and --dep-graph both require --robot, so enable it with path of installed easyconfigs
if robot_path or try_to_generate or options.dep_graph:
robot_path.extend(easyconfigs_paths)
easyconfigs_paths = robot_path[:]
_log.info("Extended list of robot paths with paths for installed easyconfigs: %s" % robot_path)
# prepend robot path with location where tweaked easyconfigs will be placed
tweaked_ecs_path = None
if try_to_generate and build_specs:
tweaked_ecs_path = os.path.join(eb_tmpdir, 'tweaked_easyconfigs')
robot_path.insert(0, tweaked_ecs_path)
# initialise the easybuild configuration
config.init(options, eb_go.get_options_by_section('config'))
# building a dependency graph implies force, so that all dependencies are retained
# and also skips validation of easyconfigs (e.g. checking os dependencies)
retain_all_deps = False
if options.dep_graph:
_log.info("Enabling force to generate dependency graph.")
options.force = True
retain_all_deps = True
if options.dep_graph or options.dry_run or options.dry_run_short:
options.ignore_osdeps = True
pr_path = None
if options.from_pr:
# extend robot search path with location where files touch in PR will be downloaded to
pr_path = os.path.join(eb_tmpdir, "files_pr%s" % options.from_pr)
robot_path.insert(0, pr_path)
_log.info("Prepended list of robot search paths with %s: %s" % (pr_path, robot_path))
config.init_build_options({
'aggregate_regtest': options.aggregate_regtest,
'allow_modules_tool_mismatch': options.allow_modules_tool_mismatch,
'check_osdeps': not options.ignore_osdeps,
'filter_deps': options.filter_deps,
'cleanup_builddir': options.cleanup_builddir,
'command_line': eb_command_line,
'debug': options.debug,
'dry_run': options.dry_run or options.dry_run_short,
'easyblock': options.easyblock,
'experimental': options.experimental,
'force': options.force,
'github_user': options.github_user,
'group': options.group,
'hidden': options.hidden,
'ignore_dirs': options.ignore_dirs,
'modules_footer': options.modules_footer,
'only_blocks': options.only_blocks,
'optarch': options.optarch,
'recursive_mod_unload': options.recursive_module_unload,
'regtest_output_dir': options.regtest_output_dir,
'retain_all_deps': retain_all_deps,
'robot_path': robot_path,
'sequential': options.sequential,
'silent': testing,
'set_gid_bit': options.set_gid_bit,
'skip': options.skip,
'skip_test_cases': options.skip_test_cases,
'sticky_bit': options.sticky_bit,
'stop': options.stop,
'suffix_modules_path': options.suffix_modules_path,
'test_report_env_filter': options.test_report_env_filter,
'umask': options.umask,
'valid_module_classes': module_classes(),
'valid_stops': [x[0] for x in EasyBlock.get_steps()],
'validate': not options.force,
})
# obtain list of loaded modules, build options must be initialized first
modlist = session_module_list(testing=testing)
init_session_state.update({'module_list': modlist})
_log.debug("Initial session state: %s" % init_session_state)
# search for easyconfigs
if options.search or options.search_short:
search_path = [os.getcwd()]
if easyconfigs_paths:
search_path = easyconfigs_paths
query = options.search or options.search_short
ignore_dirs = config.build_option('ignore_dirs')
silent = config.build_option('silent')
search_file(search_path, query, short=not options.search, ignore_dirs=ignore_dirs, silent=silent)
paths = []
if len(orig_paths) == 0:
if options.from_pr:
pr_files = fetch_easyconfigs_from_pr(options.from_pr, path=pr_path, github_user=options.github_user)
paths = [(path, False) for path in pr_files if path.endswith('.eb')]
elif 'name' in build_specs:
paths = [obtain_path(build_specs, easyconfigs_paths, try_to_generate=try_to_generate,
exit_on_error=not testing)]
elif not any([options.aggregate_regtest, options.search, options.search_short, options.regtest]):
print_error(("Please provide one or multiple easyconfig files, or use software build "
"options to make EasyBuild search for easyconfigs"),
log=_log, opt_parser=eb_go.parser, exit_on_error=not testing)
else:
# look for easyconfigs with relative paths in easybuild-easyconfigs package,
# unless they were found at the given relative paths
if easyconfigs_pkg_full_paths:
# determine which easyconfigs files need to be found, if any
ecs_to_find = []
for idx, orig_path in enumerate(orig_paths):
if orig_path == os.path.basename(orig_path) and not os.path.exists(orig_path):
ecs_to_find.append((idx, orig_path))
_log.debug("List of easyconfig files to find: %s" % ecs_to_find)
# find missing easyconfigs by walking paths with installed easyconfig files
for path in easyconfigs_pkg_full_paths:
_log.debug("Looking for missing easyconfig files (%d left) in %s..." % (len(ecs_to_find), path))
for (subpath, dirnames, filenames) in os.walk(path, topdown=True):
for idx, orig_path in ecs_to_find[:]:
if orig_path in filenames:
full_path = os.path.join(subpath, orig_path)
_log.info("Found %s in %s: %s" % (orig_path, path, full_path))
orig_paths[idx] = full_path
# if file was found, stop looking for it (first hit wins)
ecs_to_find.remove((idx, orig_path))
# stop os.walk insanity as soon as we have all we need (os.walk loop)
if len(ecs_to_find) == 0:
break
# ignore subdirs specified to be ignored by replacing items in dirnames list used by os.walk
dirnames[:] = [d for d in dirnames if not d in options.ignore_dirs]
# stop os.walk insanity as soon as we have all we need (paths loop)
if len(ecs_to_find) == 0:
break
# indicate that specified paths do not contain generated easyconfig files
paths = [(path, False) for path in orig_paths]
_log.debug("Paths: %s" % paths)
# run regtest
if options.regtest or options.aggregate_regtest:
_log.info("Running regression test")
if paths:
ec_paths = [path[0] for path in paths]
else: # fallback: easybuild-easyconfigs install path
ec_paths = easyconfigs_pkg_full_paths
regtest_ok = regtest(ec_paths)
if not regtest_ok:
_log.info("Regression test failed (partially)!")
sys.exit(31) # exit -> 3x1t -> 31
# read easyconfig files
easyconfigs = []
generated_ecs = False
for (path, generated) in paths:
path = os.path.abspath(path)
# keep track of whether any files were generated
generated_ecs |= generated
if not os.path.exists(path):
print_error("Can't find path %s" % path)
try:
ec_files = find_easyconfigs(path, ignore_dirs=options.ignore_dirs)
for ec_file in ec_files:
# only pass build specs when not generating easyconfig files
if try_to_generate:
ecs = process_easyconfig(ec_file)
else:
ecs = process_easyconfig(ec_file, build_specs=build_specs)
easyconfigs.extend(ecs)
except IOError, err:
_log.error("Processing easyconfigs in path %s failed: %s" % (path, err))
fancylogger.FancyLogger.exception(self, ebmsg + msg, *args)
# set format for logger
LOGGING_FORMAT = EB_MSG_PREFIX + ' %(asctime)s %(filename)s:%(lineno)s %(levelname)s %(message)s'
fancylogger.setLogFormat(LOGGING_FORMAT)
# set the default LoggerClass to EasyBuildLog
fancylogger.logging.setLoggerClass(EasyBuildLog)
# you can't easily set another LoggerClass before fancylogger calls getLogger on import
_init_fancylog = fancylogger.getLogger(fname=False)
del _init_fancylog.manager.loggerDict[_init_fancylog.name]
# we need to make sure there is a handler
fancylogger.logToFile(filename=os.devnull, max_bytes=0)
# EasyBuildLog
_init_easybuildlog = fancylogger.getLogger(fname=False)
def init_logging(logfile,
logtostdout=False,
silent=False,
colorize=fancylogger.Colorize.AUTO):
"""Initialize logging."""
if logtostdout:
fancylogger.logToScreen(enable=True, stdout=True, colorize=colorize)
else:
if logfile is None:
# mkstemp returns (fd,filename), fd is from os.open, not regular open!
def test_loggedexception_location(self):
"""Test inclusion of location information in log message for LoggedException."""
class TestException(LoggedException):
LOC_INFO_TOP_PKG_NAMES = None
INCLUDE_LOCATION = True
def raise_testexception(msg, *args, **kwargs):
"""Utility function: just raise a TestException."""
raise TestException(msg, *args, **kwargs)
fd, tmplog = tempfile.mkstemp()
os.close(fd)
# set log format, for each regex searching
setLogFormat("%(name)s :: %(message)s")
# no location with default LOC_INFO_TOP_PKG_NAMES ([])
logToFile(tmplog, enable=True)
self.assertErrorRegex(LoggedException, 'BOOM', raise_testexception, 'BOOM')
logToFile(tmplog, enable=False)
rootlogname = getRootLoggerName()
log_re = re.compile("^%s :: BOOM$" % rootlogname, re.M)
logtxt = open(tmplog, 'r').read()
self.assertTrue(log_re.match(logtxt), "%s matches %s" % (log_re.pattern, logtxt))
f = open(tmplog, 'w')
f.write('')
f.close()
# location is included if LOC_INFO_TOP_PKG_NAMES is defined
TestException.LOC_INFO_TOP_PKG_NAMES = ['vsc']
logToFile(tmplog, enable=True)
self.assertErrorRegex(LoggedException, 'BOOM', raise_testexception, 'BOOM')
logToFile(tmplog, enable=False)
log_re = re.compile(r"^%s :: BOOM \(at (?:.*?/)?vsc/install/testing.py:[0-9]+ in assertErrorRegex\)$" % rootlogname)
logtxt = open(tmplog, 'r').read()
self.assertTrue(log_re.match(logtxt), "%s matches %s" % (log_re.pattern, logtxt))
f = open(tmplog, 'w')
f.write('')
f.close()
# absolute path of location is included if there's no match in LOC_INFO_TOP_PKG_NAMES
TestException.LOC_INFO_TOP_PKG_NAMES = ['foobar']
logToFile(tmplog, enable=True)
self.assertErrorRegex(LoggedException, 'BOOM', raise_testexception, 'BOOM')
logToFile(tmplog, enable=False)
log_re = re.compile(r"^%s :: BOOM \(at (?:.*?/)?vsc/install/testing.py:[0-9]+ in assertErrorRegex\)$" % rootlogname)
logtxt = open(tmplog, 'r').read()
self.assertTrue(log_re.match(logtxt), "%s matches %s" % (log_re.pattern, logtxt))
os.remove(tmplog)
def test_easybuildlog(self):
"""Tests for EasyBuildLog."""
fd, tmplog = tempfile.mkstemp()
os.close(fd)
# compose versions older/newer than current version
depr_ver = int(os.environ['EASYBUILD_DEPRECATED'])
older_ver = str(depr_ver - 1)
newer_ver = str(depr_ver + 1)
# set log format, for each regex searching
setLogFormat("%(name)s [%(levelname)s] :: %(message)s")
# test basic log methods
logToFile(tmplog, enable=True)
log = getLogger('test_easybuildlog')
self.mock_stderr(True)
log.setLevelName('DEBUG')
log.debug("123 debug")
log.info("foobar info")
log.warn("justawarning")
log.deprecated("anotherwarning", newer_ver)
log.deprecated("onemorewarning", '1.0', '2.0')
log.deprecated("lastwarning", '1.0', max_ver='2.0')
log.error("kaput")
log.error("err: %s", 'msg: %s')
stderr = self.get_stderr()
self.mock_stderr(False)
# no output to stderr (should all go to log file)
self.assertEqual(stderr, '')
try:
log.exception("oops")
except EasyBuildError:
pass
logToFile(tmplog, enable=False)
logtxt = read_file(tmplog)
root = getRootLoggerName()
expected_logtxt = '\n'.join([
r"%s.test_easybuildlog \[DEBUG\] :: 123 debug" % root,
r"%s.test_easybuildlog \[INFO\] :: foobar info" % root,
r"%s.test_easybuildlog \[WARNING\] :: justawarning" % root,
r"%s.test_easybuildlog \[WARNING\] :: Deprecated functionality.*anotherwarning.*"
% root,
r"%s.test_easybuildlog \[WARNING\] :: Deprecated functionality.*onemorewarning.*"
% root,
r"%s.test_easybuildlog \[WARNING\] :: Deprecated functionality.*lastwarning.*"
% root,
r"%s.test_easybuildlog \[ERROR\] :: EasyBuild crashed with an error \(at .* in .*\): kaput"
% root,
root +
r".test_easybuildlog \[ERROR\] :: EasyBuild crashed with an error \(at .* in .*\): err: msg: %s",
r"%s.test_easybuildlog \[ERROR\] :: .*EasyBuild encountered an exception \(at .* in .*\): oops"
% root,
'',
])
logtxt_regex = re.compile(r'^%s' % expected_logtxt, re.M)
self.assertTrue(
logtxt_regex.search(logtxt),
"Pattern '%s' found in %s" % (logtxt_regex.pattern, logtxt))
self.assertErrorRegex(EasyBuildError, "DEPRECATED \(since .*: kaput",
log.deprecated, "kaput", older_ver)
self.assertErrorRegex(EasyBuildError, "DEPRECATED \(since .*: 2>1",
log.deprecated, "2>1", '2.0', '1.0')
self.assertErrorRegex(EasyBuildError,
"DEPRECATED \(since .*: 2>1",
log.deprecated,
"2>1",
'2.0',
max_ver='1.0')
# wipe log so we can reuse it
write_file(tmplog, '')
# test formatting log messages by providing extra arguments
logToFile(tmplog, enable=True)
log.warn("%s", "bleh"),
log.info("%s+%s = %d", '4', '2', 42)
args = ['this', 'is', 'just', 'a', 'test']
log.debug("%s %s %s %s %s", *args)
log.error("foo %s baz", 'baz')
logToFile(tmplog, enable=False)
logtxt = read_file(tmplog)
expected_logtxt = '\n'.join([
r"%s.test_easybuildlog \[WARNING\] :: bleh" % root,
r"%s.test_easybuildlog \[INFO\] :: 4\+2 = 42" % root,
r"%s.test_easybuildlog \[DEBUG\] :: this is just a test" % root,
r"%s.test_easybuildlog \[ERROR\] :: EasyBuild crashed with an error \(at .* in .*\): foo baz baz"
% root,
'',
])
logtxt_regex = re.compile(r'^%s' % expected_logtxt, re.M)
self.assertTrue(
logtxt_regex.search(logtxt),
"Pattern '%s' found in %s" % (logtxt_regex.pattern, logtxt))
def main():
""" Start a new rsync client (destination or source) in a specified session """
options = {
# Zookeeper connection options:
'servers' : ('list of zk servers', 'strlist', 'store', None),
'user' : ('user with creation rights on zookeeper', None, 'store', 'root', 'u'),
'passwd' : ('password for user with creation rights', None, 'store', 'admin', 'p'),
# Role options, define exactly one of these:
'source' : ('rsync source', None, 'store_true', False, 'S'),
'destination' : ('rsync destination', None, 'store_true', False, 'D'),
'pathsonly' : ('Only do a test run of the pathlist building', None, 'store_true', False),
'state' : ('Only do the state', None, 'store_true', False),
# Session options; should be the same on all clients of the session!
'session' : ('session name', None, 'store', 'default', 'N'),
'netcat' : ('run netcat test instead of rsync', None, 'store_true', False),
'dryrun' : ('run rsync in dry run mode', None, 'store_true', False, 'n'),
'rsyncpath' : ('rsync basepath', None, 'store', None, 'r'), # May differ between sources and dests
# Pathbuilding (Source clients and pathsonly ) specific options:
'excludere' : ('Exclude from pathbuilding', None, 'regex', re.compile('/\.snapshots(/.*|$)')),
'depth' : ('queue depth', "int", 'store', 4),
# Source clients options; should be the same on all clients of the session!:
'delete' : ('run rsync with --delete', None, 'store_true', False),
# Individual client options
'domain' : ('substitute domain', None, 'store', None),
'logfile' : ('Output to logfile', None, 'store', '/tmp/zkrsync/%(session)s-%(rstype)s-%(pid)s.log'),
# Individual Destination client specific options
'rsyncport' : ('force port on which rsyncd binds', "int", 'store', None),
'startport' : ('offset to look for rsyncd ports', "int", 'store', 4444)
}
go = simple_option(options)
acreds, admin_acl, rstype = zkrsync_parse(go.options)
if go.options.logfile:
logfile = go.options.logfile % {
'session': go.options.session,
'rstype': rstype,
'pid': str(os.getpid())
}
logdir = os.path.dirname(logfile)
if logdir:
if not os.path.exists(logdir):
os.makedirs(logdir)
os.chmod(logdir, stat.S_IRWXU)
fancylogger.logToFile(logfile)
logger.debug('Logging to file %s:' % logfile)
kwargs = {
'session' : go.options.session,
'default_acl' : [admin_acl],
'auth_data' : acreds,
'rsyncpath' : go.options.rsyncpath,
'netcat' : go.options.netcat,
}
if go.options.state:
rsyncP = RsyncSource(go.options.servers, **kwargs)
logger.info('Progress: %s of %s paths remaining' % (rsyncP.len_paths(), rsyncP.paths_total))
rsyncP.exit()
sys.exit(0)
elif go.options.pathsonly:
kwargs['rsyncdepth'] = go.options.depth
kwargs['excludere'] = go.options.excludere
rsyncP = RsyncSource(go.options.servers, **kwargs)
locked = rsyncP.acq_lock()
if locked:
starttime = time.time()
rsyncP.build_pathqueue()
endtime = time.time()
timing = endtime - starttime
pathqueue = rsyncP.path_queue
logger.info('Building with depth %i took %f seconds walltime. there are %i paths in the Queue'
% (go.options.depth, timing, len(pathqueue)))
rsyncP.delete(pathqueue.path, recursive=True)
rsyncP.release_lock()
else:
logger.error('There is already a lock on the pathtree of this session')
rsyncP.exit()
sys.exit(0)
elif rstype == CL_DEST:
# Start zookeeper connection and rsync daemon
kwargs['rsyncport'] = go.options.rsyncport
kwargs['startport'] = go.options.startport
kwargs['domain'] = go.options.domain
rsyncD = RsyncDestination(go.options.servers, **kwargs)
rsyncD.run()
logger.debug('%s Ready' % rsyncD.get_whoami())
rsyncD.exit()
sys.exit(0)
elif rstype == CL_SOURCE:
# Start zookeeper connections
kwargs['rsyncdepth'] = go.options.depth
kwargs['dryrun'] = go.options.dryrun
kwargs['delete'] = go.options.delete
kwargs['excludere'] = go.options.excludere
rsyncS = RsyncSource(go.options.servers, **kwargs)
# Try to retrieve session lock
locked = rsyncS.acq_lock()
if locked:
logger.debug('lock acquired')
watchnode = rsyncS.start_ready_rwatch()
if not watchnode:
sys.exit(1)
paths_total = rsyncS.build_pathqueue()
todo_paths = paths_total
while not rsyncS.isempty_pathqueue():
if todo_paths != rsyncS.len_paths(): # Output progress state
todo_paths = rsyncS.len_paths()
logger.info('Progress: %s of %s paths remaining' % (todo_paths, paths_total))
time.sleep(SLEEP_TIME)
rsyncS.shutdown_all()
rsyncS.exit()
sys.exit(0)
else:
rsyncS.ready_with_stop_watch()
logger.debug('ready to process paths')
while not rsyncS.is_ready():
logger.debug('trying to get a path out of Queue')
rsyncS.rsync(TIME_OUT)
logger.debug('%s Ready' % rsyncS.get_whoami())
rsyncS.exit()
sys.exit(0)
def test_easybuildlog(self):
"""Tests for EasyBuildLog."""
fd, tmplog = tempfile.mkstemp()
os.close(fd)
# compose versions older/newer than current version
depr_ver = int(os.environ['EASYBUILD_DEPRECATED'])
older_ver = str(depr_ver - 1)
newer_ver = str(depr_ver + 1)
# set log format, for each regex searching
setLogFormat("%(name)s [%(levelname)s] :: %(message)s")
# test basic log methods
logToFile(tmplog, enable=True)
log = getLogger('test_easybuildlog')
self.mock_stderr(True)
log.setLevelName('DEBUG')
log.debug("123 debug")
log.info("foobar info")
log.warn("justawarning")
log.deprecated("anotherwarning", newer_ver)
log.deprecated("onemorewarning", '1.0', '2.0')
log.deprecated("lastwarning", '1.0', max_ver='2.0')
log.error("kaput")
log.error("err: %s", 'msg: %s')
stderr = self.get_stderr()
self.mock_stderr(False)
more_info = "see http://easybuild.readthedocs.org/en/latest/Deprecated-functionality.html for more information"
expected_stderr = '\n'.join([
"Deprecated functionality, will no longer work in v10000001: anotherwarning; " + more_info,
"Deprecated functionality, will no longer work in v2.0: onemorewarning",
"Deprecated functionality, will no longer work in v2.0: lastwarning",
]) + '\n'
self.assertEqual(stderr, expected_stderr)
try:
log.exception("oops")
except EasyBuildError:
pass
logToFile(tmplog, enable=False)
logtxt = read_file(tmplog)
root = getRootLoggerName()
expected_logtxt = '\n'.join([
r"%s.test_easybuildlog \[DEBUG\] :: 123 debug" % root,
r"%s.test_easybuildlog \[INFO\] :: foobar info" % root,
r"%s.test_easybuildlog \[WARNING\] :: justawarning" % root,
r"%s.test_easybuildlog \[WARNING\] :: Deprecated functionality.*anotherwarning.*" % root,
r"%s.test_easybuildlog \[WARNING\] :: Deprecated functionality.*onemorewarning.*" % root,
r"%s.test_easybuildlog \[WARNING\] :: Deprecated functionality.*lastwarning.*" % root,
r"%s.test_easybuildlog \[ERROR\] :: EasyBuild crashed with an error \(at .* in .*\): kaput" % root,
root + r".test_easybuildlog \[ERROR\] :: EasyBuild crashed with an error \(at .* in .*\): err: msg: %s",
r"%s.test_easybuildlog \[ERROR\] :: .*EasyBuild encountered an exception \(at .* in .*\): oops" % root,
'',
])
logtxt_regex = re.compile(r'^%s' % expected_logtxt, re.M)
self.assertTrue(logtxt_regex.search(logtxt), "Pattern '%s' found in %s" % (logtxt_regex.pattern, logtxt))
self.assertErrorRegex(EasyBuildError, "DEPRECATED \(since .*: kaput", log.deprecated, "kaput", older_ver)
self.assertErrorRegex(EasyBuildError, "DEPRECATED \(since .*: 2>1", log.deprecated, "2>1", '2.0', '1.0')
self.assertErrorRegex(EasyBuildError, "DEPRECATED \(since .*: 2>1", log.deprecated, "2>1", '2.0', max_ver='1.0')
# wipe log so we can reuse it
write_file(tmplog, '')
# test formatting log messages by providing extra arguments
logToFile(tmplog, enable=True)
log.warn("%s", "bleh"),
log.info("%s+%s = %d", '4', '2', 42)
args = ['this', 'is', 'just', 'a', 'test']
log.debug("%s %s %s %s %s", *args)
log.error("foo %s baz", 'baz')
logToFile(tmplog, enable=False)
logtxt = read_file(tmplog)
expected_logtxt = '\n'.join([
r"%s.test_easybuildlog \[WARNING\] :: bleh" % root,
r"%s.test_easybuildlog \[INFO\] :: 4\+2 = 42" % root,
r"%s.test_easybuildlog \[DEBUG\] :: this is just a test" % root,
r"%s.test_easybuildlog \[ERROR\] :: EasyBuild crashed with an error \(at .* in .*\): foo baz baz" % root,
'',
])
logtxt_regex = re.compile(r'^%s' % expected_logtxt, re.M)
self.assertTrue(logtxt_regex.search(logtxt), "Pattern '%s' found in %s" % (logtxt_regex.pattern, logtxt))
raise EasyBuildError(ebmsg, *args)
# set format for logger
LOGGING_FORMAT = EB_MSG_PREFIX + ' %(asctime)s %(name)s %(levelname)s %(message)s'
fancylogger.setLogFormat(LOGGING_FORMAT)
# set the default LoggerClass to EasyBuildLog
fancylogger.logging.setLoggerClass(EasyBuildLog)
# you can't easily set another LoggerClass before fancylogger calls getLogger on import
_init_fancylog = fancylogger.getLogger(fname=False)
del _init_fancylog.manager.loggerDict[_init_fancylog.name]
# we need to make sure there is a handler
fancylogger.logToFile(filename=os.devnull)
# EasyBuildLog
_init_easybuildlog = fancylogger.getLogger(fname=False)
def init_logging(logfile, logtostdout=False, testing=False):
"""Initialize logging."""
if logtostdout:
fancylogger.logToScreen(enable=True, stdout=True)
else:
if logfile is None:
# mkstemp returns (fd,filename), fd is from os.open, not regular open!
fd, logfile = tempfile.mkstemp(suffix='.log', prefix='easybuild-')
os.close(fd)
def stop_logging(logfile, logtostdout=False):
"""Stop logging."""
if logtostdout:
fancylogger.logToScreen(enable=False, stdout=True)
fancylogger.logToFile(logfile, enable=False)
def tearDown(self):
fancylogger.logToFile(self.logfn, enable=False)
self.handle.close()
os.remove(self.logfn)
raise EasyBuildError(newMsg)
# set format for logger
LOGGING_FORMAT = EB_MSG_PREFIX + ' %(asctime)s %(name)s %(levelname)s %(message)s'
fancylogger.setLogFormat(LOGGING_FORMAT)
# set the default LoggerClass to EasyBuildLog
fancylogger.logging.setLoggerClass(EasyBuildLog)
# you can't easily set another LoggerClass before fancylogger calls getLogger on import
_init_fancylog = fancylogger.getLogger(fname=False)
del _init_fancylog.manager.loggerDict[_init_fancylog.name]
# we need to make sure there is a handler
fancylogger.logToFile(filename=os.devnull)
# EasyBuildLog
_init_easybuildlog = fancylogger.getLogger(fname=False)
def init_logging(logfile, logtostdout=False, testing=False):
"""Initialize logging."""
if logtostdout:
fancylogger.logToScreen(enable=True, stdout=True)
else:
if logfile is None:
# mkstemp returns (fd,filename), fd is from os.open, not regular open!
fd, logfile = tempfile.mkstemp(suffix='.log', prefix='easybuild-')
os.close(fd)
def stop_logging(logfile, logtostdout=False):
"""Stop logging."""
if logtostdout:
fancylogger.logToScreen(enable=False, stdout=True)
fancylogger.logToFile(logfile, enable=False)
gist_url = upload_test_report_as_gist(test_report)
print_msg("Test report uploaded to %s" % gist_url)
else:
test_report = create_test_report(success_msg, ecs_with_res, init_session_state)
_log.debug("Test report: %s" % test_report)
if options.dump_test_report is not None:
write_file(options.dump_test_report, test_report)
_log.info("Test report dumped to %s" % options.dump_test_report)
print_msg(success_msg, log=_log, silent=testing)
# cleanup and spec files
for ec in easyconfigs:
if 'original_spec' in ec and os.path.isfile(ec['spec']):
os.remove(ec['spec'])
# cleanup tmp log file, unless one build failed (individual logs are located in eb_tmpdir path)
if options.logtostdout:
fancylogger.logToScreen(enable=False, stdout=True)
else:
fancylogger.logToFile(logfile, enable=False)
if overall_success:
cleanup(logfile, eb_tmpdir, testing)
if __name__ == "__main__":
try:
main()
except EasyBuildError, e:
sys.stderr.write('ERROR: %s\n' % e.msg)
sys.exit(1)
import test.framework.tweak as tw
import test.framework.variables as v
# make sure temporary files can be created/used
try:
set_tmpdir(raise_error=True)
except EasyBuildError, err:
sys.stderr.write("No execution rights on temporary files, specify another location via $TMPDIR: %s\n" % err)
sys.exit(1)
# initialize logger for all the unit tests
fd, log_fn = tempfile.mkstemp(prefix='easybuild-tests-', suffix='.log')
os.close(fd)
os.remove(log_fn)
fancylogger.logToFile(log_fn)
log = fancylogger.getLogger()
# call suite() for each module and then run them all
# note: make sure the options unit tests run first, to avoid running some of them with a readily initialized config
tests = [gen, o, r, ef, ev, ebco, ep, e, mg, m, mt, f, run, a, robot, b, v, g, tcv, tc, t, c, s, l, f_c, sc, tw, p]
SUITE = unittest.TestSuite([x.suite() for x in tests])
# uses XMLTestRunner if possible, so we can output an XML file that can be supplied to Jenkins
xml_msg = ""
try:
import xmlrunner # requires unittest-xml-reporting package
xml_dir = 'test-reports'
res = xmlrunner.XMLTestRunner(output=xml_dir, verbosity=1).run(SUITE)
xml_msg = ", XML output of tests available in %s directory" % xml_dir
def test_easybuildlog(self):
"""Tests for EasyBuildLog."""
fd, tmplog = tempfile.mkstemp()
os.close(fd)
# set log format, for each regex searching
setLogFormat("%(name)s [%(levelname)s] :: %(message)s")
# test basic log methods
logToFile(tmplog, enable=True)
log = getLogger('test_easybuildlog')
log.setLevelName('DEBUG')
log.debug("123 debug")
log.info("foobar info")
log.warn("justawarning")
log.raiseError = False
log.error("kaput")
log.raiseError = True
try:
log.exception("oops")
except EasyBuildError:
pass
logToFile(tmplog, enable=False)
logtxt = read_file(tmplog)
root = getRootLoggerName()
expected_logtxt = '\n'.join([
r"%s.test_easybuildlog \[DEBUG\] :: 123 debug" % root,
r"%s.test_easybuildlog \[INFO\] :: foobar info" % root,
r"%s.test_easybuildlog \[WARNING\] :: justawarning" % root,
r"%s.test_easybuildlog \[ERROR\] :: EasyBuild crashed with an error \(at .* in .*\): kaput"
% root,
r"%s.test_easybuildlog \[ERROR\] :: .*EasyBuild encountered an exception \(at .* in .*\): oops"
% root,
'',
])
logtxt_regex = re.compile(r'^%s' % expected_logtxt, re.M)
self.assertTrue(
logtxt_regex.search(logtxt),
"Pattern '%s' found in %s" % (logtxt_regex.pattern, logtxt))
# wipe log so we can reuse it
write_file(tmplog, '')
# test formatting log messages by providing extra arguments
logToFile(tmplog, enable=True)
log.warn("%s", "bleh"),
log.info("%s+%s = %d", '4', '2', 42)
args = ['this', 'is', 'just', 'a', 'test']
log.debug("%s %s %s %s %s", *args)
log.raiseError = False
log.error("foo %s baz", 'baz')
log.raiseError = True
logToFile(tmplog, enable=False)
logtxt = read_file(tmplog)
expected_logtxt = '\n'.join([
r"%s.test_easybuildlog \[WARNING\] :: bleh" % root,
r"%s.test_easybuildlog \[INFO\] :: 4\+2 = 42" % root,
r"%s.test_easybuildlog \[DEBUG\] :: this is just a test" % root,
r"%s.test_easybuildlog \[ERROR\] :: EasyBuild crashed with an error \(at .* in .*\): foo baz baz"
% root,
'',
])
logtxt_regex = re.compile(r'^%s' % expected_logtxt, re.M)
self.assertTrue(
logtxt_regex.search(logtxt),
"Pattern '%s' found in %s" % (logtxt_regex.pattern, logtxt))
# test deprecated behaviour: raise EasyBuildError on log.error and log.exception
os.environ['EASYBUILD_DEPRECATED'] = '2.1'
init_config()
log.warning("No raise for warnings")
self.assertErrorRegex(EasyBuildError,
'EasyBuild crashed with an error', log.error,
'foo')
self.assertErrorRegex(EasyBuildError,
'EasyBuild encountered an exception',
log.exception, 'bar')
import test.framework.variables as v
# make sure temporary files can be created/used
try:
set_tmpdir(raise_error=True)
except EasyBuildError, err:
sys.stderr.write(
"No execution rights on temporary files, specify another location via $TMPDIR: %s\n"
% err)
sys.exit(1)
# initialize logger for all the unit tests
fd, log_fn = tempfile.mkstemp(prefix='easybuild-tests-', suffix='.log')
os.close(fd)
os.remove(log_fn)
fancylogger.logToFile(log_fn)
log = fancylogger.getLogger()
# call suite() for each module and then run them all
# note: make sure the options unit tests run first, to avoid running some of them with a readily initialized config
tests = [
gen, o, r, ef, ev, ebco, ep, e, mg, m, mt, f, run, a, robot, b, v, g, tcv,
tc, t, c, s, l, f_c, sc, tw, p
]
SUITE = unittest.TestSuite([x.suite() for x in tests])
# uses XMLTestRunner if possible, so we can output an XML file that can be supplied to Jenkins
xml_msg = ""
try:
import xmlrunner # requires unittest-xml-reporting package
def test_loggedexception_location(self):
"""Test inclusion of location information in log message for LoggedException."""
class TestException(LoggedException):
LOC_INFO_TOP_PKG_NAMES = None
INCLUDE_LOCATION = True
def raise_testexception(msg, *args, **kwargs):
"""Utility function: just raise a TestException."""
raise TestException(msg, *args, **kwargs)
fd, tmplog = tempfile.mkstemp()
os.close(fd)
# set log format, for each regex searching
setLogFormat("%(name)s :: %(message)s")
# no location with default LOC_INFO_TOP_PKG_NAMES ([])
logToFile(tmplog, enable=True)
self.assertErrorRegex(LoggedException, 'BOOM', raise_testexception, 'BOOM')
logToFile(tmplog, enable=False)
rootlogname = getRootLoggerName()
log_re = re.compile("^%s :: BOOM$" % rootlogname, re.M)
with open(tmplog, 'r') as f:
logtxt = f.read()
self.assertTrue(log_re.match(logtxt), "%s matches %s" % (log_re.pattern, logtxt))
with open(tmplog, 'w') as f:
f.write('')
# location is included if LOC_INFO_TOP_PKG_NAMES is defined
TestException.LOC_INFO_TOP_PKG_NAMES = ['vsc']
logToFile(tmplog, enable=True)
self.assertErrorRegex(LoggedException, 'BOOM', raise_testexception, 'BOOM')
logToFile(tmplog, enable=False)
log_re = re.compile(r"^%s :: BOOM \(at (?:.*?/)?vsc/install/testing.py:[0-9]+ in assertErrorRegex\)$" % rootlogname)
with open(tmplog, 'r') as f:
logtxt = f.read()
self.assertTrue(log_re.match(logtxt), "%s matches %s" % (log_re.pattern, logtxt))
with open(tmplog, 'w') as f:
f.write('')
# absolute path of location is included if there's no match in LOC_INFO_TOP_PKG_NAMES
TestException.LOC_INFO_TOP_PKG_NAMES = ['foobar']
logToFile(tmplog, enable=True)
self.assertErrorRegex(LoggedException, 'BOOM', raise_testexception, 'BOOM')
logToFile(tmplog, enable=False)
log_re = re.compile(r"^%s :: BOOM \(at (?:.*?/)?vsc/install/testing.py:[0-9]+ in assertErrorRegex\)$" % rootlogname)
with open(tmplog, 'r') as f:
logtxt = f.read()
self.assertTrue(log_re.match(logtxt), "%s matches %s" % (log_re.pattern, logtxt))
os.remove(tmplog)
