def test_loggedexception_specifiedlogger(self):
"""Test LoggedException custom exception class."""
fd, tmplog = tempfile.mkstemp()
os.close(fd)
# set log format, for each regex searching
setLogFormat("%(name)s :: %(message)s")
logger1 = getLogger('testlogger_one')
logger2 = getLogger('testlogger_two')
# if logger is specified, it should be used
logToFile(tmplog, enable=True)
self.assertErrorRegex(LoggedException,
'BOOM',
raise_loggedexception,
'BOOM',
logger=logger1)
logToFile(tmplog, enable=False)
rootlog = getRootLoggerName()
log_re = re.compile(
"^%s.testlogger_one :: BOOM( \(at .*:[0-9]+ in raise_loggedexception\))?$"
% rootlog, re.M)
logtxt = open(tmplog, 'r').read()
self.assertTrue(log_re.match(logtxt),
"%s matches %s" % (log_re.pattern, logtxt))
os.remove(tmplog)
def test_loggedexception_defaultlogger(self):
"""Test LoggedException custom exception class."""
fd, tmplog = tempfile.mkstemp()
os.close(fd)
# set log format, for each regex searching
setLogFormat("%(name)s :: %(message)s")
# if no logger is available, and no logger is specified, use default 'root' fancylogger
logToFile(tmplog, enable=True)
self.assertErrorRegex(LoggedException, 'BOOM', raise_loggedexception, 'BOOM')
logToFile(tmplog, enable=False)
log_re = re.compile("^%s :: BOOM( \(at .*:[0-9]+ in raise_loggedexception\))?$" % getRootLoggerName(), re.M)
logtxt = open(tmplog, 'r').read()
self.assertTrue(log_re.match(logtxt), "%s matches %s" % (log_re.pattern, logtxt))
# test formatting of message
self.assertErrorRegex(LoggedException, 'BOOMBAF', raise_loggedexception, 'BOOM%s', 'BAF')
# test log message that contains '%s' without any formatting arguments being passed
# test formatting of message
self.assertErrorRegex(LoggedException, "BOOM '%s'", raise_loggedexception, "BOOM '%s'")
os.remove(tmplog)
def test_easybuilderror(self):
"""Tests for EasyBuildError."""
fd, tmplog = tempfile.mkstemp()
os.close(fd)
# set log format, for each regex searching
setLogFormat("%(name)s :: %(message)s")
# if no logger is available, and no logger is specified, use default 'root' fancylogger
logToFile(tmplog, enable=True)
self.assertErrorRegex(EasyBuildError, 'BOOM', raise_easybuilderror, 'BOOM')
logToFile(tmplog, enable=False)
log_re = re.compile("^%s ::.* BOOM \(at .*:[0-9]+ in [a-z_]+\)$" % getRootLoggerName(), re.M)
logtxt = open(tmplog, 'r').read()
self.assertTrue(log_re.match(logtxt), "%s matches %s" % (log_re.pattern, logtxt))
# test formatting of message
self.assertErrorRegex(EasyBuildError, 'BOOMBAF', raise_easybuilderror, 'BOOM%s', 'BAF')
# a '%s' in a value used to template the error message should not print a traceback!
self.mock_stderr(True)
self.assertErrorRegex(EasyBuildError, 'err: msg: %s', raise_easybuilderror, "err: %s", "msg: %s")
stderr = self.get_stderr()
self.mock_stderr(False)
# stderr should be *empty* (there should definitely not be a traceback)
self.assertEqual(stderr, '')
os.remove(tmplog)
def test_loggedexception_defaultlogger(self):
"""Test LoggedException custom exception class."""
fd, tmplog = tempfile.mkstemp()
os.close(fd)
# set log format, for each regex searching
setLogFormat("%(name)s :: %(message)s")
# if no logger is available, and no logger is specified, use default 'root' fancylogger
logToFile(tmplog, enable=True)
self.assertErrorRegex(LoggedException, 'BOOM', raise_loggedexception, 'BOOM')
logToFile(tmplog, enable=False)
log_re = re.compile("^%s :: BOOM( \(at .*:[0-9]+ in raise_loggedexception\))?$" % getRootLoggerName(), re.M)
with open(tmplog, 'r') as f:
logtxt = f.read()
self.assertTrue(log_re.match(logtxt), "%s matches %s" % (log_re.pattern, logtxt))
# test formatting of message
self.assertErrorRegex(LoggedException, 'BOOMBAF', raise_loggedexception, 'BOOM%s', 'BAF')
# test log message that contains '%s' without any formatting arguments being passed
# test formatting of message
self.assertErrorRegex(LoggedException, "BOOM '%s'", raise_loggedexception, "BOOM '%s'")
os.remove(tmplog)
def test_log_levels(self):
"""Test whether log levels are respected"""
fd, tmplog = tempfile.mkstemp()
os.close(fd)
# set log format, for each regex searching
setLogFormat("%(name)s [%(levelname)s] :: %(message)s")
# test basic log methods
logToFile(tmplog, enable=True)
log = getLogger('test_easybuildlog')
for level in ['ERROR', 'WARNING', 'INFO', 'DEBUG', 'DEVEL']:
log.setLevelName(level)
log.raiseError = False
log.error('kaput')
log.deprecated('almost kaput', '10000000000000')
log.raiseError = True
log.warn('this is a warning')
log.info('fyi')
log.debug('gdb')
log.devel('tmi')
logToFile(tmplog, enable=False)
logtxt = read_file(tmplog)
root = getRootLoggerName()
devel_msg = r"%s.test_easybuildlog \[DEVEL\] :: tmi" % root
debug_msg = r"%s.test_easybuildlog \[DEBUG\] :: gdb" % root
info_msg = r"%s.test_easybuildlog \[INFO\] :: fyi" % root
warning_msg = r"%s.test_easybuildlog \[WARNING\] :: this is a warning" % root
deprecated_msg = r"%s.test_easybuildlog \[WARNING\] :: Deprecated functionality, .*: almost kaput; see .*" % root
error_msg = r"%s.test_easybuildlog \[ERROR\] :: EasyBuild crashed with an error \(at .* in .*\): kaput" % root
expected_logtxt = '\n'.join([
error_msg,
error_msg,
deprecated_msg,
warning_msg,
error_msg,
deprecated_msg,
warning_msg,
info_msg,
error_msg,
deprecated_msg,
warning_msg,
info_msg,
debug_msg,
error_msg,
deprecated_msg,
warning_msg,
info_msg,
debug_msg,
devel_msg,
])
logtxt_regex = re.compile(r'^%s' % expected_logtxt, re.M)
self.assertTrue(
logtxt_regex.search(logtxt),
"Pattern '%s' found in %s" % (logtxt_regex.pattern, logtxt))
def test_loggedexception_callerlogger(self):
"""Test LoggedException custom exception class."""
fd, tmplog = tempfile.mkstemp()
os.close(fd)
# set log format, for each regex searching
setLogFormat("%(name)s :: %(message)s")
logger = getLogger('testlogger_local')
# if no logger is specified, logger available in calling context should be used
logToFile(tmplog, enable=True)
self.assertErrorRegex(LoggedException, 'BOOM', raise_loggedexception,
'BOOM')
logToFile(tmplog, enable=False)
rootlog = getRootLoggerName()
log_re = re.compile(
"^%s(.testlogger_local)? :: BOOM( \(at .*:[0-9]+ in raise_loggedexception\))?$"
% rootlog)
with open(tmplog, 'r') as f:
logtxt = f.read()
self.assertTrue(log_re.match(logtxt),
"%s matches %s" % (log_re.pattern, logtxt))
os.remove(tmplog)
def test_easybuilderror(self):
"""Tests for EasyBuildError."""
fd, tmplog = tempfile.mkstemp()
os.close(fd)
# set log format, for each regex searching
setLogFormat("%(name)s :: %(message)s")
# if no logger is available, and no logger is specified, use default 'root' fancylogger
logToFile(tmplog, enable=True)
self.assertErrorRegex(EasyBuildError, 'BOOM', raise_easybuilderror,
'BOOM')
logToFile(tmplog, enable=False)
log_re = re.compile(
"^%s ::.* BOOM \(at .*:[0-9]+ in [a-z_]+\)$" % getRootLoggerName(),
re.M)
logtxt = open(tmplog, 'r').read()
self.assertTrue(log_re.match(logtxt),
"%s matches %s" % (log_re.pattern, logtxt))
# test formatting of message
self.assertErrorRegex(EasyBuildError, 'BOOMBAF', raise_easybuilderror,
'BOOM%s', 'BAF')
# a '%s' in a value used to template the error message should not print a traceback!
self.mock_stderr(True)
self.assertErrorRegex(EasyBuildError, 'err: msg: %s',
raise_easybuilderror, "err: %s", "msg: %s")
stderr = self.get_stderr()
self.mock_stderr(False)
# stderr should be *empty* (there should definitely not be a traceback)
self.assertEqual(stderr, '')
os.remove(tmplog)
def test_easybuilderror(self):
"""Tests for EasyBuildError."""
fd, tmplog = tempfile.mkstemp()
os.close(fd)
# set log format, for each regex searching
setLogFormat("%(name)s :: %(message)s")
# if no logger is available, and no logger is specified, use default 'root' fancylogger
logToFile(tmplog, enable=True)
self.assertErrorRegex(EasyBuildError, 'BOOM', raise_easybuilderror,
'BOOM')
logToFile(tmplog, enable=False)
log_re = re.compile(
"^%s :: BOOM \(at .*:[0-9]+ in [a-z_]+\)$" % getRootLoggerName(),
re.M)
logtxt = open(tmplog, 'r').read()
self.assertTrue(log_re.match(logtxt),
"%s matches %s" % (log_re.pattern, logtxt))
# test formatting of message
self.assertErrorRegex(EasyBuildError, 'BOOMBAF', raise_easybuilderror,
'BOOM%s', 'BAF')
os.remove(tmplog)
def run(self, args):
"""Run 'create' subcommand."""
optparser = CreateOptions(go_args=args, envvar_prefix=self.envvar_prefix, usage=self.usage_txt)
options = optparser.options
if not validate_pbs_option(options):
sys.stderr.write('Missing config options. Exiting.\n')
return 1
label = options.label
if not hc.validate_label(label, hc.known_cluster_labels()):
sys.exit(1)
if not hc.validate_hodconf_or_dist(options.hodconf, options.dist):
sys.exit(1)
try:
j = PbsHodJob(optparser)
hc.report_cluster_submission(label)
j.run()
jobs = j.state()
hc.post_job_submission(label, jobs, optparser.options.workdir)
return 0
except StandardError as e:
fancylogger.setLogFormat(fancylogger.TEST_LOGGING_FORMAT)
fancylogger.logToScreen(enable=True)
_log.raiseException(e.message)
def test_loggedexception_location(self):
"""Test inclusion of location information in log message for LoggedException."""
class TestException(LoggedException):
LOC_INFO_TOP_PKG_NAMES = None
INCLUDE_LOCATION = True
def raise_testexception(msg, *args, **kwargs):
"""Utility function: just raise a TestException."""
raise TestException(msg, *args, **kwargs)
fd, tmplog = tempfile.mkstemp()
os.close(fd)
# set log format, for each regex searching
setLogFormat("%(name)s :: %(message)s")
# no location with default LOC_INFO_TOP_PKG_NAMES ([])
logToFile(tmplog, enable=True)
self.assertErrorRegex(LoggedException, 'BOOM', raise_testexception, 'BOOM')
logToFile(tmplog, enable=False)
rootlogname = getRootLoggerName()
log_re = re.compile("^%s :: BOOM$" % rootlogname, re.M)
with open(tmplog, 'r') as f:
logtxt = f.read()
self.assertTrue(log_re.match(logtxt), "%s matches %s" % (log_re.pattern, logtxt))
with open(tmplog, 'w') as f:
f.write('')
# location is included if LOC_INFO_TOP_PKG_NAMES is defined
TestException.LOC_INFO_TOP_PKG_NAMES = ['vsc']
logToFile(tmplog, enable=True)
self.assertErrorRegex(LoggedException, 'BOOM', raise_testexception, 'BOOM')
logToFile(tmplog, enable=False)
log_re = re.compile(r"^%s :: BOOM \(at (?:.*?/)?vsc/install/testing.py:[0-9]+ in assertErrorRegex\)$" % rootlogname)
with open(tmplog, 'r') as f:
logtxt = f.read()
self.assertTrue(log_re.match(logtxt), "%s matches %s" % (log_re.pattern, logtxt))
with open(tmplog, 'w') as f:
f.write('')
# absolute path of location is included if there's no match in LOC_INFO_TOP_PKG_NAMES
TestException.LOC_INFO_TOP_PKG_NAMES = ['foobar']
logToFile(tmplog, enable=True)
self.assertErrorRegex(LoggedException, 'BOOM', raise_testexception, 'BOOM')
logToFile(tmplog, enable=False)
log_re = re.compile(r"^%s :: BOOM \(at (?:.*?/)?vsc/install/testing.py:[0-9]+ in assertErrorRegex\)$" % rootlogname)
with open(tmplog, 'r') as f:
logtxt = f.read()
self.assertTrue(log_re.match(logtxt), "%s matches %s" % (log_re.pattern, logtxt))
os.remove(tmplog)
def test_loggedexception_location(self):
"""Test inclusion of location information in log message for LoggedException."""
class TestException(LoggedException):
LOC_INFO_TOP_PKG_NAMES = None
INCLUDE_LOCATION = True
def raise_testexception(msg, *args, **kwargs):
"""Utility function: just raise a TestException."""
raise TestException(msg, *args, **kwargs)
fd, tmplog = tempfile.mkstemp()
os.close(fd)
# set log format, for each regex searching
setLogFormat("%(name)s :: %(message)s")
# no location with default LOC_INFO_TOP_PKG_NAMES ([])
logToFile(tmplog, enable=True)
self.assertErrorRegex(LoggedException, "BOOM", raise_testexception, "BOOM")
logToFile(tmplog, enable=False)
rootlogname = getRootLoggerName()
log_re = re.compile("^%s :: BOOM$" % rootlogname, re.M)
logtxt = open(tmplog, "r").read()
self.assertTrue(log_re.match(logtxt), "%s matches %s" % (log_re.pattern, logtxt))
f = open(tmplog, "w")
f.write("")
f.close()
# location is included if LOC_INFO_TOP_PKG_NAMES is defined
TestException.LOC_INFO_TOP_PKG_NAMES = ["vsc"]
logToFile(tmplog, enable=True)
self.assertErrorRegex(LoggedException, "BOOM", raise_testexception, "BOOM")
logToFile(tmplog, enable=False)
log_re = re.compile(r"^%s :: BOOM \(at vsc/utils/testing.py:[0-9]+ in assertErrorRegex\)$" % rootlogname)
logtxt = open(tmplog, "r").read()
self.assertTrue(log_re.match(logtxt), "%s matches %s" % (log_re.pattern, logtxt))
f = open(tmplog, "w")
f.write("")
f.close()
# absolute path of location is included if there's no match in LOC_INFO_TOP_PKG_NAMES
TestException.LOC_INFO_TOP_PKG_NAMES = ["foobar"]
logToFile(tmplog, enable=True)
self.assertErrorRegex(LoggedException, "BOOM", raise_testexception, "BOOM")
logToFile(tmplog, enable=False)
log_re = re.compile(r"^%s :: BOOM \(at .*/vsc/utils/testing.py:[0-9]+ in assertErrorRegex\)$" % rootlogname)
logtxt = open(tmplog, "r").read()
self.assertTrue(log_re.match(logtxt), "%s matches %s" % (log_re.pattern, logtxt))
os.remove(tmplog)
def run(self, args):
"""Run 'connect' subcommand."""
optparser = ConnectOptions(go_args=args, envvar_prefix=self.envvar_prefix, usage=self.usage_txt)
try:
if len(optparser.args) > 1:
label = optparser.args[1]
else:
_log.error("No label provided.")
sys.exit(1)
print "Connecting to HOD cluster with label '%s'..." % label
try:
jobid = cluster_jobid(label)
env_script = cluster_env_file(label)
except ValueError as err:
_log.error(err)
sys.exit(1)
print "Job ID found: %s" % jobid
pbs = rm_pbs.Pbs(optparser)
jobs = pbs.state()
pbsjobs = [job for job in jobs if job.jobid == jobid]
if len(pbsjobs) == 0:
_log.error("Job with job ID '%s' not found by pbs.", jobid)
sys.exit(1)
elif len(pbsjobs) > 1:
_log.error("Multiple jobs found with job ID '%s': %s", jobid, pbsjobs)
sys.exit(1)
pbsjob = pbsjobs[0]
if pbsjob.state == ['Q', 'H']:
# This should never happen since the hod.d/<jobid>/env file is
# written on cluster startup. Maybe someone hacked the dirs.
_log.error("Cannot connect to cluster with job ID '%s' yet. It is still queued.", jobid)
sys.exit(1)
else:
print "HOD cluster '%s' @ job ID %s appears to be running..." % (label, jobid)
print "Setting up SSH connection to %s..." % pbsjob.hosts
# -i: interactive non-login shell
cmd = ['ssh', '-t', pbsjob.hosts, 'exec', 'bash', '--rcfile', env_script, '-i']
_log.info("Logging in using command: %s", ' '.join(cmd))
os.execvp('/usr/bin/ssh', cmd)
return 0 # pragma: no cover
except StandardError as err:
fancylogger.setLogFormat(fancylogger.TEST_LOGGING_FORMAT)
fancylogger.logToScreen(enable=True)
_log.raiseException(err)
def run(self, args):
"""Run 'clean' subcommand."""
optparser = CleanOptions(go_args=args, envvar_prefix=self.envvar_prefix, usage=self.usage_txt)
try:
pbs = rm_pbs.Pbs(optparser)
state = pbs.state()
labels = hc.known_cluster_labels()
rm_master = rm_pbs.master_hostname()
info = hc.mk_cluster_info_dict(labels, state, master=rm_master)
hc.clean_cluster_info(rm_master, info)
except StandardError as err:
fancylogger.setLogFormat(fancylogger.TEST_LOGGING_FORMAT)
fancylogger.logToScreen(enable=True)
_log.raiseException(err.message)
def test_classname_in_log(self):
"""Do a log and check if the classname is correctly in it"""
_stderr = sys.stderr
class Foobar:
def somefunction(self):
logger = fancylogger.getLogger(fname=True, clsname=True)
logger.warn('we are logging something here')
stringfile = StringIO()
sys.stderr = stringfile
handler = fancylogger.logToScreen()
Foobar().somefunction()
self.assertTrue('Foobar.somefunction' in stringfile.getvalue())
stringfile.close()
# restore
fancylogger.logToScreen(enable=False, handler=handler)
# and again
stringfile = StringIO()
sys.stderr = stringfile
handler = fancylogger.logToScreen()
classless_function()
self.assertTrue('unknown__getCallingClassName.classless_function' in
stringfile.getvalue())
# restore
fancylogger.logToScreen(enable=False, handler=handler)
stringfile = StringIO()
sys.stderr = stringfile
fancylogger.setLogFormat("%(className)s blabla")
handler = fancylogger.logToScreen()
logger = fancylogger.getLogger(fname=False, clsname=False)
logger.warning("blabla")
txt = stringfile.getvalue()
# this will only hold in debug mode, so also disable the test
if __debug__:
pattern = 'FancyLoggerTest'
self.assertTrue(pattern in txt,
"Pattern '%s' found in: %s" % (pattern, txt))
# restore
fancylogger.logToScreen(enable=False, handler=handler)
sys.stderr = _stderr
def run(self, args):
"""Run 'batch' subcommand."""
optparser = BatchOptions(go_args=args, envvar_prefix=self.envvar_prefix, usage=self.usage_txt)
options = optparser.options
if not validate_pbs_option(options):
sys.stderr.write('Missing config options. Exiting.\n')
return 1
if optparser.options.script is None:
sys.stderr.write('Missing script. Exiting.\n')
return 1
# resolve script path to absolute path
optparser.options.script = os.path.abspath(optparser.options.script)
if not os.path.exists(optparser.options.script):
sys.stderr.write("Specified script does not exist: %s. Exiting.\n" % optparser.options.script)
return 1
# make sure script is executable
cur_perms = os.stat(optparser.options.script)[stat.ST_MODE]
if not (cur_perms & stat.S_IXUSR):
print "Specified script %s is not executable yet, fixing that..." % optparser.options.script
os.chmod(optparser.options.script, cur_perms|stat.S_IXUSR)
label = options.label
if not hc.validate_label(label, hc.known_cluster_labels()):
sys.exit(1)
if not hc.validate_hodconf_or_dist(options.hodconf, options.dist):
sys.exit(1)
try:
j = PbsHodJob(optparser)
hc.report_cluster_submission(label)
j.run()
jobs = j.state()
hc.post_job_submission(label, jobs, optparser.options.workdir)
return 0
except StandardError as e:
fancylogger.setLogFormat(fancylogger.TEST_LOGGING_FORMAT)
fancylogger.logToScreen(enable=True)
_log.raiseException(e.message)
def test_log_levels(self):
"""Test whether log levels are respected"""
fd, tmplog = tempfile.mkstemp()
os.close(fd)
# set log format, for each regex searching
setLogFormat("%(name)s [%(levelname)s] :: %(message)s")
# test basic log methods
logToFile(tmplog, enable=True)
log = getLogger('test_easybuildlog')
for level in ['ERROR', 'WARNING', 'INFO', 'DEBUG', 'DEVEL']:
log.setLevelName(level)
log.raiseError = False
log.error('kaput')
log.deprecated('almost kaput', '10000000000000')
log.raiseError = True
log.warn('this is a warning')
log.info('fyi')
log.debug('gdb')
log.devel('tmi')
logToFile(tmplog, enable=False)
logtxt = read_file(tmplog)
root = getRootLoggerName()
devel_msg = r"%s.test_easybuildlog \[DEVEL\] :: tmi" % root
debug_msg = r"%s.test_easybuildlog \[DEBUG\] :: gdb" % root
info_msg = r"%s.test_easybuildlog \[INFO\] :: fyi" % root
warning_msg = r"%s.test_easybuildlog \[WARNING\] :: this is a warning" % root
deprecated_msg = r"%s.test_easybuildlog \[WARNING\] :: Deprecated functionality, .*: almost kaput; see .*" % root
error_msg = r"%s.test_easybuildlog \[ERROR\] :: EasyBuild crashed with an error \(at .* in .*\): kaput" % root
expected_logtxt = '\n'.join([
error_msg,
error_msg, deprecated_msg, warning_msg,
error_msg, deprecated_msg, warning_msg, info_msg,
error_msg, deprecated_msg, warning_msg, info_msg, debug_msg,
error_msg, deprecated_msg, warning_msg, info_msg, debug_msg, devel_msg,
])
logtxt_regex = re.compile(r'^%s' % expected_logtxt, re.M)
self.assertTrue(logtxt_regex.search(logtxt), "Pattern '%s' found in %s" % (logtxt_regex.pattern, logtxt))
def test_classname_in_log(self):
"""Do a log and check if the classname is correctly in it"""
_stderr = sys.stderr
class Foobar:
def somefunction(self):
logger = fancylogger.getLogger(fname=True, clsname=True)
logger.warn('we are logging something here')
stringfile = StringIO()
sys.stderr = stringfile
handler = fancylogger.logToScreen()
Foobar().somefunction()
self.assertTrue('Foobar.somefunction' in stringfile.getvalue())
stringfile.close()
# restore
fancylogger.logToScreen(enable=False, handler=handler)
# and again
stringfile = StringIO()
sys.stderr = stringfile
handler = fancylogger.logToScreen()
classless_function()
self.assertTrue('?.classless_function' in stringfile.getvalue())
# restore
fancylogger.logToScreen(enable=False, handler=handler)
stringfile = StringIO()
sys.stderr = stringfile
fancylogger.setLogFormat("%(className)s blabla")
handler = fancylogger.logToScreen()
logger = fancylogger.getLogger(fname=False, clsname=False)
logger.warn("blabla")
print stringfile.getvalue()
# this will only hold in debug mode, so also disable the test
if __debug__:
self.assertTrue('FancyLoggerTest' in stringfile.getvalue())
# restore
fancylogger.logToScreen(enable=False, handler=handler)
sys.stderr = _stderr
def run(self, args):
"""Run 'list' subcommand."""
optparser = ListOptions(go_args=args, envvar_prefix=self.envvar_prefix, usage=self.usage_txt)
try:
pbs = rm_pbs.Pbs(optparser)
state = pbs.state()
labels = hc.known_cluster_labels()
info = hc.mk_cluster_info_dict(labels, state)
if not info:
print 'No jobs found'
sys.exit(0)
headers = ['Cluster label', 'Job ID', 'State', 'Hosts']
info_rows = format_list_rows(info)
print ht.format_table(info_rows, headers)
except StandardError as err:
fancylogger.setLogFormat(fancylogger.TEST_LOGGING_FORMAT)
fancylogger.logToScreen(enable=True)
_log.raiseException(err)
return 0
def test_loggedexception_callerlogger(self):
"""Test LoggedException custom exception class."""
fd, tmplog = tempfile.mkstemp()
os.close(fd)
# set log format, for each regex searching
setLogFormat("%(name)s :: %(message)s")
logger = getLogger('testlogger_local')
# if no logger is specified, logger available in calling context should be used
logToFile(tmplog, enable=True)
self.assertErrorRegex(LoggedException, 'BOOM', raise_loggedexception, 'BOOM')
logToFile(tmplog, enable=False)
rootlog = getRootLoggerName()
log_re = re.compile("^%s(.testlogger_local)? :: BOOM( \(at .*:[0-9]+ in raise_loggedexception\))?$" % rootlog)
logtxt = open(tmplog, 'r').read()
self.assertTrue(log_re.match(logtxt), "%s matches %s" % (log_re.pattern, logtxt))
os.remove(tmplog)
def test_easybuilderror(self):
"""Tests for EasyBuildError."""
fd, tmplog = tempfile.mkstemp()
os.close(fd)
# set log format, for each regex searching
setLogFormat("%(name)s :: %(message)s")
# if no logger is available, and no logger is specified, use default 'root' fancylogger
logToFile(tmplog, enable=True)
self.assertErrorRegex(EasyBuildError, 'BOOM', raise_easybuilderror, 'BOOM')
logToFile(tmplog, enable=False)
log_re = re.compile("^%s :: BOOM \(at .*:[0-9]+ in [a-z_]+\)$" % getRootLoggerName(), re.M)
logtxt = open(tmplog, 'r').read()
self.assertTrue(log_re.match(logtxt), "%s matches %s" % (log_re.pattern, logtxt))
# test formatting of message
self.assertErrorRegex(EasyBuildError, 'BOOMBAF', raise_easybuilderror, 'BOOM%s', 'BAF')
os.remove(tmplog)
def test_loggedexception_specifiedlogger(self):
"""Test LoggedException custom exception class."""
fd, tmplog = tempfile.mkstemp()
os.close(fd)
# set log format, for each regex searching
setLogFormat("%(name)s :: %(message)s")
logger1 = getLogger('testlogger_one')
logger2 = getLogger('testlogger_two')
# if logger is specified, it should be used
logToFile(tmplog, enable=True)
self.assertErrorRegex(LoggedException, 'BOOM', raise_loggedexception, 'BOOM', logger=logger1)
logToFile(tmplog, enable=False)
rootlog = getRootLoggerName()
log_re = re.compile("^%s.testlogger_one :: BOOM \(at .*:[0-9]+ in raise_loggedexception\)$" % rootlog, re.M)
logtxt = open(tmplog, 'r').read()
self.assertTrue(log_re.match(logtxt), "%s matches %s" % (log_re.pattern, logtxt))
os.remove(tmplog)
def run(self, args):
"""Run 'relabel' subcommand."""
optparser = RelabelOptions(go_args=args, envvar_prefix=self.envvar_prefix, usage=self.usage_txt)
try:
if len(optparser.args) != 3:
_log.error(self.usage())
sys.exit(1)
labels = hc.known_cluster_labels()
if optparser.args[1] not in labels:
_log.error("Cluster with label '%s' not found", optparser.args[1])
sys.exit(1)
try:
hc.mv_cluster_info(optparser.args[1], optparser.args[2])
except (IOError, OSError) as err:
_log.error("Could not change label '%s' to '%s': %s", optparser.args[1], optparser.args[2], err)
sys.exit(1)
except StandardError as err:
fancylogger.setLogFormat(fancylogger.TEST_LOGGING_FORMAT)
fancylogger.logToScreen(enable=True)
_log.raiseException(err)
return 0
def tearDown(self):
"""Cleanup after test."""
super(BuildLogTest, self).tearDown()
# restore default logging format
setLogFormat(LOGGING_FORMAT)
def test_easybuildlog(self):
"""Tests for EasyBuildLog."""
fd, tmplog = tempfile.mkstemp()
os.close(fd)
# set log format, for each regex searching
setLogFormat("%(name)s [%(levelname)s] :: %(message)s")
# test basic log methods
logToFile(tmplog, enable=True)
log = getLogger('test_easybuildlog')
log.setLevelName('DEBUG')
log.debug("123 debug")
log.info("foobar info")
log.warn("justawarning")
log.raiseError = False
log.error("kaput")
log.raiseError = True
try:
log.exception("oops")
except EasyBuildError:
pass
logToFile(tmplog, enable=False)
logtxt = read_file(tmplog)
root = getRootLoggerName()
expected_logtxt = '\n'.join([
r"%s.test_easybuildlog \[DEBUG\] :: 123 debug" % root,
r"%s.test_easybuildlog \[INFO\] :: foobar info" % root,
r"%s.test_easybuildlog \[WARNING\] :: justawarning" % root,
r"%s.test_easybuildlog \[ERROR\] :: EasyBuild crashed with an error \(at .* in .*\): kaput"
% root,
r"%s.test_easybuildlog \[ERROR\] :: .*EasyBuild encountered an exception \(at .* in .*\): oops"
% root,
'',
])
logtxt_regex = re.compile(r'^%s' % expected_logtxt, re.M)
self.assertTrue(
logtxt_regex.search(logtxt),
"Pattern '%s' found in %s" % (logtxt_regex.pattern, logtxt))
# wipe log so we can reuse it
write_file(tmplog, '')
# test formatting log messages by providing extra arguments
logToFile(tmplog, enable=True)
log.warn("%s", "bleh"),
log.info("%s+%s = %d", '4', '2', 42)
args = ['this', 'is', 'just', 'a', 'test']
log.debug("%s %s %s %s %s", *args)
log.raiseError = False
log.error("foo %s baz", 'baz')
log.raiseError = True
logToFile(tmplog, enable=False)
logtxt = read_file(tmplog)
expected_logtxt = '\n'.join([
r"%s.test_easybuildlog \[WARNING\] :: bleh" % root,
r"%s.test_easybuildlog \[INFO\] :: 4\+2 = 42" % root,
r"%s.test_easybuildlog \[DEBUG\] :: this is just a test" % root,
r"%s.test_easybuildlog \[ERROR\] :: EasyBuild crashed with an error \(at .* in .*\): foo baz baz"
% root,
'',
])
logtxt_regex = re.compile(r'^%s' % expected_logtxt, re.M)
self.assertTrue(
logtxt_regex.search(logtxt),
"Pattern '%s' found in %s" % (logtxt_regex.pattern, logtxt))
# test deprecated behaviour: raise EasyBuildError on log.error and log.exception
os.environ['EASYBUILD_DEPRECATED'] = '2.1'
init_config()
log.warning("No raise for warnings")
self.assertErrorRegex(EasyBuildError,
'EasyBuild crashed with an error', log.error,
'foo')
self.assertErrorRegex(EasyBuildError,
'EasyBuild encountered an exception',
log.exception, 'bar')
def tearDown(self):
"""Cleanup after test."""
# restore default logging format
setLogFormat(LOGGING_FORMAT)
def tearDown(self):
"""Cleanup after test."""
super(BuildLogTest, self).tearDown()
# restore default logging format
setLogFormat(LOGGING_FORMAT)
def test_easybuildlog(self):
"""Tests for EasyBuildLog."""
fd, tmplog = tempfile.mkstemp()
os.close(fd)
# compose versions older/newer than current version
depr_ver = int(os.environ['EASYBUILD_DEPRECATED'])
older_ver = str(depr_ver - 1)
newer_ver = str(depr_ver + 1)
# set log format, for each regex searching
setLogFormat("%(name)s [%(levelname)s] :: %(message)s")
# test basic log methods
logToFile(tmplog, enable=True)
log = getLogger('test_easybuildlog')
self.mock_stderr(True)
log.setLevelName('DEBUG')
log.debug("123 debug")
log.info("foobar info")
log.warn("justawarning")
log.deprecated("anotherwarning", newer_ver)
log.deprecated("onemorewarning", '1.0', '2.0')
log.deprecated("lastwarning", '1.0', max_ver='2.0')
log.error("kaput")
log.error("err: %s", 'msg: %s')
stderr = self.get_stderr()
self.mock_stderr(False)
# no output to stderr (should all go to log file)
self.assertEqual(stderr, '')
try:
log.exception("oops")
except EasyBuildError:
pass
logToFile(tmplog, enable=False)
logtxt = read_file(tmplog)
root = getRootLoggerName()
expected_logtxt = '\n'.join([
r"%s.test_easybuildlog \[DEBUG\] :: 123 debug" % root,
r"%s.test_easybuildlog \[INFO\] :: foobar info" % root,
r"%s.test_easybuildlog \[WARNING\] :: justawarning" % root,
r"%s.test_easybuildlog \[WARNING\] :: Deprecated functionality.*anotherwarning.*"
% root,
r"%s.test_easybuildlog \[WARNING\] :: Deprecated functionality.*onemorewarning.*"
% root,
r"%s.test_easybuildlog \[WARNING\] :: Deprecated functionality.*lastwarning.*"
% root,
r"%s.test_easybuildlog \[ERROR\] :: EasyBuild crashed with an error \(at .* in .*\): kaput"
% root,
root +
r".test_easybuildlog \[ERROR\] :: EasyBuild crashed with an error \(at .* in .*\): err: msg: %s",
r"%s.test_easybuildlog \[ERROR\] :: .*EasyBuild encountered an exception \(at .* in .*\): oops"
% root,
'',
])
logtxt_regex = re.compile(r'^%s' % expected_logtxt, re.M)
self.assertTrue(
logtxt_regex.search(logtxt),
"Pattern '%s' found in %s" % (logtxt_regex.pattern, logtxt))
self.assertErrorRegex(EasyBuildError, "DEPRECATED \(since .*: kaput",
log.deprecated, "kaput", older_ver)
self.assertErrorRegex(EasyBuildError, "DEPRECATED \(since .*: 2>1",
log.deprecated, "2>1", '2.0', '1.0')
self.assertErrorRegex(EasyBuildError,
"DEPRECATED \(since .*: 2>1",
log.deprecated,
"2>1",
'2.0',
max_ver='1.0')
# wipe log so we can reuse it
write_file(tmplog, '')
# test formatting log messages by providing extra arguments
logToFile(tmplog, enable=True)
log.warn("%s", "bleh"),
log.info("%s+%s = %d", '4', '2', 42)
args = ['this', 'is', 'just', 'a', 'test']
log.debug("%s %s %s %s %s", *args)
log.error("foo %s baz", 'baz')
logToFile(tmplog, enable=False)
logtxt = read_file(tmplog)
expected_logtxt = '\n'.join([
r"%s.test_easybuildlog \[WARNING\] :: bleh" % root,
r"%s.test_easybuildlog \[INFO\] :: 4\+2 = 42" % root,
r"%s.test_easybuildlog \[DEBUG\] :: this is just a test" % root,
r"%s.test_easybuildlog \[ERROR\] :: EasyBuild crashed with an error \(at .* in .*\): foo baz baz"
% root,
'',
])
logtxt_regex = re.compile(r'^%s' % expected_logtxt, re.M)
self.assertTrue(
logtxt_regex.search(logtxt),
"Pattern '%s' found in %s" % (logtxt_regex.pattern, logtxt))
def _log_and_raise(self, err):
"""Report error that occured, and raise Exception"""
fancylogger.setLogFormat(fancylogger.TEST_LOGGING_FORMAT)
fancylogger.logToScreen(enable=True)
self.log.raiseException(str(err))
def test_easybuildlog(self):
"""Tests for EasyBuildLog."""
fd, tmplog = tempfile.mkstemp()
os.close(fd)
# set log format, for each regex searching
setLogFormat("%(name)s [%(levelname)s] :: %(message)s")
# test basic log methods
logToFile(tmplog, enable=True)
log = getLogger('test_easybuildlog')
log.setLevelName('DEBUG')
log.debug("123 debug")
log.info("foobar info")
log.warn("justawarning")
log.raiseError = False
log.error("kaput")
log.raiseError = True
try:
log.exception("oops")
except EasyBuildError:
pass
logToFile(tmplog, enable=False)
logtxt = read_file(tmplog)
root = getRootLoggerName()
expected_logtxt = '\n'.join([
r"%s.test_easybuildlog \[DEBUG\] :: 123 debug" % root,
r"%s.test_easybuildlog \[INFO\] :: foobar info" % root,
r"%s.test_easybuildlog \[WARNING\] :: justawarning" % root,
r"%s.test_easybuildlog \[ERROR\] :: EasyBuild crashed with an error \(at .* in .*\): kaput" % root,
r"%s.test_easybuildlog \[ERROR\] :: .*EasyBuild encountered an exception \(at .* in .*\): oops" % root,
'',
])
logtxt_regex = re.compile(r'^%s' % expected_logtxt, re.M)
self.assertTrue(logtxt_regex.search(logtxt), "Pattern '%s' found in %s" % (logtxt_regex.pattern, logtxt))
# wipe log so we can reuse it
write_file(tmplog, '')
# test formatting log messages by providing extra arguments
logToFile(tmplog, enable=True)
log.warn("%s", "bleh"),
log.info("%s+%s = %d", '4', '2', 42)
args = ['this', 'is', 'just', 'a', 'test']
log.debug("%s %s %s %s %s", *args)
log.raiseError = False
log.error("foo %s baz", 'baz')
log.raiseError = True
logToFile(tmplog, enable=False)
logtxt = read_file(tmplog)
expected_logtxt = '\n'.join([
r"%s.test_easybuildlog \[WARNING\] :: bleh" % root,
r"%s.test_easybuildlog \[INFO\] :: 4\+2 = 42" % root,
r"%s.test_easybuildlog \[DEBUG\] :: this is just a test" % root,
r"%s.test_easybuildlog \[ERROR\] :: EasyBuild crashed with an error \(at .* in .*\): foo baz baz" % root,
'',
])
logtxt_regex = re.compile(r'^%s' % expected_logtxt, re.M)
self.assertTrue(logtxt_regex.search(logtxt), "Pattern '%s' found in %s" % (logtxt_regex.pattern, logtxt))
# test deprecated behaviour: raise EasyBuildError on log.error and log.exception
os.environ['EASYBUILD_DEPRECATED'] = '2.1'
init_config()
log.warning("No raise for warnings")
self.assertErrorRegex(EasyBuildError, 'EasyBuild crashed with an error', log.error, 'foo')
self.assertErrorRegex(EasyBuildError, 'EasyBuild encountered an exception', log.exception, 'bar')
def test_easybuildlog(self):
"""Tests for EasyBuildLog."""
fd, tmplog = tempfile.mkstemp()
os.close(fd)
# compose versions older/newer than current version
depr_ver = int(os.environ['EASYBUILD_DEPRECATED'])
older_ver = str(depr_ver - 1)
newer_ver = str(depr_ver + 1)
# set log format, for each regex searching
setLogFormat("%(name)s [%(levelname)s] :: %(message)s")
# test basic log methods
logToFile(tmplog, enable=True)
log = getLogger('test_easybuildlog')
self.mock_stderr(True)
log.setLevelName('DEBUG')
log.debug("123 debug")
log.info("foobar info")
log.warn("justawarning")
log.deprecated("anotherwarning", newer_ver)
log.deprecated("onemorewarning", '1.0', '2.0')
log.deprecated("lastwarning", '1.0', max_ver='2.0')
log.error("kaput")
log.error("err: %s", 'msg: %s')
stderr = self.get_stderr()
self.mock_stderr(False)
more_info = "see http://easybuild.readthedocs.org/en/latest/Deprecated-functionality.html for more information"
expected_stderr = '\n'.join([
"Deprecated functionality, will no longer work in v10000001: anotherwarning; " + more_info,
"Deprecated functionality, will no longer work in v2.0: onemorewarning",
"Deprecated functionality, will no longer work in v2.0: lastwarning",
]) + '\n'
self.assertEqual(stderr, expected_stderr)
try:
log.exception("oops")
except EasyBuildError:
pass
logToFile(tmplog, enable=False)
logtxt = read_file(tmplog)
root = getRootLoggerName()
expected_logtxt = '\n'.join([
r"%s.test_easybuildlog \[DEBUG\] :: 123 debug" % root,
r"%s.test_easybuildlog \[INFO\] :: foobar info" % root,
r"%s.test_easybuildlog \[WARNING\] :: justawarning" % root,
r"%s.test_easybuildlog \[WARNING\] :: Deprecated functionality.*anotherwarning.*" % root,
r"%s.test_easybuildlog \[WARNING\] :: Deprecated functionality.*onemorewarning.*" % root,
r"%s.test_easybuildlog \[WARNING\] :: Deprecated functionality.*lastwarning.*" % root,
r"%s.test_easybuildlog \[ERROR\] :: EasyBuild crashed with an error \(at .* in .*\): kaput" % root,
root + r".test_easybuildlog \[ERROR\] :: EasyBuild crashed with an error \(at .* in .*\): err: msg: %s",
r"%s.test_easybuildlog \[ERROR\] :: .*EasyBuild encountered an exception \(at .* in .*\): oops" % root,
'',
])
logtxt_regex = re.compile(r'^%s' % expected_logtxt, re.M)
self.assertTrue(logtxt_regex.search(logtxt), "Pattern '%s' found in %s" % (logtxt_regex.pattern, logtxt))
self.assertErrorRegex(EasyBuildError, "DEPRECATED \(since .*: kaput", log.deprecated, "kaput", older_ver)
self.assertErrorRegex(EasyBuildError, "DEPRECATED \(since .*: 2>1", log.deprecated, "2>1", '2.0', '1.0')
self.assertErrorRegex(EasyBuildError, "DEPRECATED \(since .*: 2>1", log.deprecated, "2>1", '2.0', max_ver='1.0')
# wipe log so we can reuse it
write_file(tmplog, '')
# test formatting log messages by providing extra arguments
logToFile(tmplog, enable=True)
log.warn("%s", "bleh"),
log.info("%s+%s = %d", '4', '2', 42)
args = ['this', 'is', 'just', 'a', 'test']
log.debug("%s %s %s %s %s", *args)
log.error("foo %s baz", 'baz')
logToFile(tmplog, enable=False)
logtxt = read_file(tmplog)
expected_logtxt = '\n'.join([
r"%s.test_easybuildlog \[WARNING\] :: bleh" % root,
r"%s.test_easybuildlog \[INFO\] :: 4\+2 = 42" % root,
r"%s.test_easybuildlog \[DEBUG\] :: this is just a test" % root,
r"%s.test_easybuildlog \[ERROR\] :: EasyBuild crashed with an error \(at .* in .*\): foo baz baz" % root,
'',
])
logtxt_regex = re.compile(r'^%s' % expected_logtxt, re.M)
self.assertTrue(logtxt_regex.search(logtxt), "Pattern '%s' found in %s" % (logtxt_regex.pattern, logtxt))
def tearDown(self):
"""Cleanup after test."""
# restore default logging format
setLogFormat(LOGGING_FORMAT)
def exception(self, msg, *args):
"""Print exception message and raise EasyBuildError."""
# don't raise the exception from within error
newMsg = "EasyBuild encountered an exception %s: %s" % (
self.caller_info(), msg)
self.raiseError = False
fancylogger.FancyLogger.exception(self, newMsg, *args)
self.raiseError = True
raise EasyBuildError(newMsg)
# set format for logger
LOGGING_FORMAT = EB_MSG_PREFIX + ' %(asctime)s %(name)s %(levelname)s %(message)s'
fancylogger.setLogFormat(LOGGING_FORMAT)
# set the default LoggerClass to EasyBuildLog
fancylogger.logging.setLoggerClass(EasyBuildLog)
# you can't easily set another LoggerClass before fancylogger calls getLogger on import
_init_fancylog = fancylogger.getLogger(fname=False)
del _init_fancylog.manager.loggerDict[_init_fancylog.name]
# we need to make sure there is a handler
fancylogger.logToFile(filename=os.devnull)
# EasyBuildLog
_init_easybuildlog = fancylogger.getLogger(fname=False)
def run(self, args):
"""Run 'destroy' subcommand."""
optparser = DestroyOptions(go_args=args, envvar_prefix=self.envvar_prefix, usage=self.usage_txt)
try:
label, jobid = None, None
if len(optparser.args) > 1:
label = optparser.args[1]
print "Destroying HOD cluster with label '%s'..." % label
else:
_log.error("No label provided.")
sys.exit(1)
try:
jobid = cluster_jobid(label)
print "Job ID: %s" % jobid
except ValueError as err:
_log.error(err)
sys.exit(1)
# try to figure out job state
job_state = None
pbs = rm_pbs.Pbs(optparser)
jobs = pbs.state()
pbsjobs = [job for job in jobs if job.jobid == jobid]
_log.debug("Matching jobs for job ID '%s': %s", jobid, pbsjobs)
if len(pbsjobs) == 1:
job_state = pbsjobs[0].state
print "Job status: %s" % job_state
elif len(pbsjobs) == 0:
print "(job no longer found)"
else:
_log.error("Multiple jobs found with job ID '%s': %s", jobid, pbsjobs)
sys.exit(1)
# request confirmation is case the job is currently running
if job_state == 'R':
resp = raw_input("Confirm destroying the *running* HOD cluster with label '%s'? [y/n]: " % label)
if resp != 'y':
print "(destruction aborted)"
return
elif job_state in ['C', 'E']:
print "(job has already ended/completed)"
job_state = None
print "\nStarting actual destruction of HOD cluster with label '%s'...\n" % label
# actually destroy HOD cluster by deleting job and removing cluster info dir and local work dir
if job_state is not None:
# if job was not successfully deleted, pbs.remove will print an error message
if pbs.remove(jobid):
print "Job with ID %s deleted." % jobid
rm_cluster_localworkdir(label)
if cluster_info_exists(label):
rm_cluster_info(label)
print "\nHOD cluster with label '%s' (job ID: %s) destroyed." % (label, jobid)
except StandardError as err:
fancylogger.setLogFormat(fancylogger.TEST_LOGGING_FORMAT)
fancylogger.logToScreen(enable=True)
_log.raiseException(err)
def _log_and_raise(self, err):
"""Report error that occured, and raise Exception"""
fancylogger.setLogFormat(fancylogger.TEST_LOGGING_FORMAT)
fancylogger.logToScreen(enable=True)
self.log.raiseException(str(err))
"""Print exception message and raise EasyBuildError."""
# don't raise the exception from within error
ebmsg = "EasyBuild encountered an exception %s: " + msg
args = (self.caller_info(),) + args
self.raiseError = False
fancylogger.FancyLogger.exception(self, ebmsg, *args)
self.raiseError = True
self.deprecated("Use 'raise EasyBuildError' rather than exception() logging method that raises", '3.0')
raise EasyBuildError(ebmsg, *args)
# set format for logger
LOGGING_FORMAT = EB_MSG_PREFIX + ' %(asctime)s %(name)s %(levelname)s %(message)s'
fancylogger.setLogFormat(LOGGING_FORMAT)
# set the default LoggerClass to EasyBuildLog
fancylogger.logging.setLoggerClass(EasyBuildLog)
# you can't easily set another LoggerClass before fancylogger calls getLogger on import
_init_fancylog = fancylogger.getLogger(fname=False)
del _init_fancylog.manager.loggerDict[_init_fancylog.name]
# we need to make sure there is a handler
fancylogger.logToFile(filename=os.devnull)
# EasyBuildLog
_init_easybuildlog = fancylogger.getLogger(fname=False)
