def test_parentinfo(self):
"""Test the collection of parentinfo"""
log_fr = fancylogger.getLogger(fname=False) # rootfancylogger
pi_fr = log_fr._get_parent_info()
self.assertEqual(len(pi_fr), 2)
log_l1 = fancylogger.getLogger('level1', fname=False)
# fname=False is required to have the naming similar for child relations
pi_l1 = log_l1._get_parent_info()
self.assertEqual(len(pi_l1), 3)
py_v_27 = sys.version_info >= (2, 7, 0)
if py_v_27:
log_l2a = log_l1.getChild('level2a')
pi_l2a = log_l2a._get_parent_info()
self.assertEqual(len(pi_l2a), 4)
# this should be identical to getChild
log_l2b = fancylogger.getLogger('level1.level2b', fname=False)
# fname=False is required to have the name similar
# cutoff last letter (a vs b)
if py_v_27:
self.assertEqual(log_l2a.name[:-1], log_l2b.name[:-1])
pi_l2b = log_l2b._get_parent_info()
# yes, this broken on several levels (incl in logging itself)
# adding '.' in the name does not automatically create the parent/child relations
# if the parent with the name exists, this works
self.assertEqual(len(pi_l2b), 4)
log_l2c = fancylogger.getLogger('level1a.level2c', fname=False)
pi_l2c = log_l2c._get_parent_info()
self.assertEqual(len(pi_l2c), 3) # level1a as parent does not exist
def test_raiseException(self):
"""Test raiseException log method."""
# truncate the logfile
open(self.logfn, 'w')
def test123(exception, msg):
"""Utility function for testing raiseException."""
try:
raise exception(msg)
except:
logger.raiseException('HIT')
logger = fancylogger.getLogger('fail_test')
self.assertErrorRegex(Exception, 'failtest', test123, Exception, 'failtest')
self.assertTrue(re.match("^WARNING.*HIT.*failtest\n.*in test123.*$", open(self.logfn, 'r').read(), re.M))
open(self.logfn, 'w')
fancylogger.FancyLogger.RAISE_EXCEPTION_CLASS = KeyError
logger = fancylogger.getLogger('fail_test')
self.assertErrorRegex(KeyError, 'failkeytest', test123, KeyError, 'failkeytest')
self.assertTrue(re.match("^WARNING.*HIT.*'failkeytest'\n.*in test123.*$", open(self.logfn, 'r').read(), re.M))
open(self.logfn, 'w')
fancylogger.FancyLogger.RAISE_EXCEPTION_LOG_METHOD = lambda c, msg: c.warning(msg)
logger = fancylogger.getLogger('fail_test')
self.assertErrorRegex(AttributeError, 'attrtest', test123, AttributeError, 'attrtest')
self.assertTrue(re.match("^WARNING.*HIT.*attrtest\n.*in test123.*$", open(self.logfn, 'r').read(), re.M))
def request(self, method, url, body, headers, content_type=None):
"""Low-level networking. All HTTP-method methods call this"""
if headers is None:
headers = {}
if content_type is not None:
headers['Content-Type'] = content_type
if self.auth_header is not None:
headers['Authorization'] = self.auth_header
headers['User-Agent'] = self.user_agent
fancylogger.getLogger().debug('cli request: %s, %s, %s, %s', method, url, body, headers)
# TODO: in recent python: Context manager
conn = self.get_connection(method, url, body, headers)
status = conn.code
if method == self.HEAD:
pybody = conn.headers
else:
body = conn.read()
try:
pybody = json.loads(body)
except ValueError:
pybody = body
fancylogger.getLogger().debug('reponse len: %s ', len(pybody))
conn.close()
return status, pybody
def test_get_callers_logger(self):
"""Test get_callers_logger function."""
# returns None if no logger is available
self.assertEqual(get_callers_logger(), None)
# find defined logger in caller's context
logger = getLogger("foo")
callers_logger = get_callers_logger()
# result depends on whether tests were run under 'python' or 'python -O'
self.assertTrue(callers_logger in [logger, None])
# also works when logger is 'higher up'
class Test(object):
"""Dummy test class"""
def foo(self, logger=None):
"""Dummy test method, returns logger from calling context."""
return get_callers_logger()
test = Test()
self.assertTrue(logger, [test.foo(), None])
# closest logger to caller is preferred
logger2 = getLogger(test.__class__.__name__)
self.assertTrue(logger2 in [test.foo(logger=logger2), None])
def test_fancylogger_as_rootlogger_logging(self):
"""
Test if just using import logging, logging with logging uses fancylogger
after setting the root logger
"""
# test logging.root is loggin root logger
# this is an assumption made to make the fancyrootlogger code work
orig_root = logging.getLogger()
self.assertEqual(logging.root, orig_root,
msg='logging.root is the root logger')
self.assertFalse(isinstance(logging.root, fancylogger.FancyLogger),
msg='logging.root is not a FancyLogger')
stringfile = StringIO()
sys.stderr = stringfile
handler = fancylogger.logToScreen()
fancylogger.setLogLevelDebug()
logger = fancylogger.getLogger()
self.assertEqual(logger.handlers, [self.handler, handler],
msg='active handler for root fancylogger')
self.assertEqual(logger.level, fancylogger.getLevelInt('DEBUG'), msg='debug level set')
msg = 'this is my string'
logging.debug(msg)
self.assertEqual(stringfile.getvalue(), '',
msg="logging.debug reports nothing when fancylogger loglevel is debug")
fancylogger.setroot()
self.assertTrue(isinstance(logging.root, fancylogger.FancyLogger),
msg='logging.root is a FancyLogger after setRootLogger')
self.assertEqual(logging.root.level, fancylogger.getLevelInt('DEBUG'), msg='debug level set for root')
self.assertEqual(logger.level, logging.NOTSET, msg='original root fancylogger level set to NOTSET')
self.assertEqual(logging.root.handlers, [self.handler, handler],
msg='active handler for root logger from previous root fancylogger')
self.assertEqual(logger.handlers, [], msg='no active handlers on previous root fancylogger')
root_logger = logging.getLogger('')
self.assertEqual(root_logger, logging.root,
msg='logging.getLogger() returns logging.root FancyLogger')
frl = fancylogger.getLogger()
self.assertEqual(frl, logging.root,
msg='fancylogger.getLogger() returns logging.root FancyLogger')
logging.debug(msg)
self.assertTrue(msg in stringfile.getvalue(),
msg="logging.debug reports when fancylogger loglevel is debug")
fancylogger.resetroot()
self.assertEqual(logging.root, orig_root,
msg='logging.root is the original root logger after resetroot')
# restore
fancylogger.logToScreen(enable=False, handler=handler)
def getCluster(name):
"""
static factory method, should be in Cluster class
returns a cluster object of the given name
"""
for cls in Cluster.__subclasses__():
if cls._is_cluster_for(name):
return cls()
fancylogger.getLogger("clusters.getCluster").raiseException("No such cluster %s" % name, NoSuchClusterException)
def get_job(classname, options):
"""
This is a job factory.
Returns an instance of classname initialized with options
"""
for cls in get_subclasses(Job):
if cls._is_job_for(classname):
return cls(options)
getLogger().error("No job class found for %s", classname)
def main():
"""Main function"""
try:
instance_options = get_mpi_and_sched_and_options()
if instance_options:
instance = getinstance(*instance_options)
instance.main()
except Exception:
fancylogger.getLogger().exception("Main failed")
sys.exit(1)
def get_connection(self, method, url, body, headers):
if not self.url.endswith('/') and not url.startswith('/'):
sep = '/'
else:
sep = ''
request = urllib2.Request(self.url + sep + url, data=body)
for header, value in headers.iteritems():
request.add_header(header, value)
request.get_method = lambda: method
fancylogger.getLogger().debug('opening request: %s%s%s', self.url, sep, url)
connection = self.opener.open(request)
return connection
def det_pylibdir(plat_specific=False, python_cmd=None):
"""Determine Python library directory."""
log = fancylogger.getLogger('det_pylibdir', fname=False)
if python_cmd is None:
# use 'python' that is listed first in $PATH if none was specified
python_cmd = 'python'
# determine Python lib dir via distutils
# use run_cmd, we can to talk to the active Python, not the system Python running EasyBuild
prefix = '/tmp/'
args = 'plat_specific=%s, prefix="%s"' % (plat_specific, prefix)
pycode = "import distutils.sysconfig; print(distutils.sysconfig.get_python_lib(%s))" % args
cmd = "%s -c '%s'" % (python_cmd, pycode)
log.debug("Determining Python library directory using command '%s'", cmd)
out, ec = run_cmd(cmd, simple=False, force_in_dry_run=True)
txt = out.strip().split('\n')[-1]
# value obtained should start with specified prefix, otherwise something is very wrong
if not txt.startswith(prefix):
raise EasyBuildError("Last line of output of %s does not start with specified prefix %s: %s (exit code %s)",
cmd, prefix, out, ec)
pylibdir = txt[len(prefix):]
log.debug("Determined pylibdir using '%s': %s", cmd, pylibdir)
return pylibdir
def test_deprecated(self):
"""Test deprecated log function."""
# truncate the logfile
open(self.logfn, 'w')
# log message
logger = fancylogger.getLogger('deprecated_test')
max_ver = "1.0"
# test whether deprecation works
msgre_tpl_error = r"DEPRECATED\s*\(since v%s\).*%s" % (max_ver, MSG)
self.assertErrorRegex(Exception, msgre_tpl_error, logger.deprecated, MSG, "1.1", max_ver)
self.assertErrorRegex(Exception, msgre_tpl_error, logger.deprecated, MSG, "1.0", max_ver)
# test whether deprecated warning works
# no deprecation if current version is lower than max version
logger.deprecated(MSG, "0.9", max_ver)
msgre_tpl_warning = r"WARNING.*DEPRECATED\s*\(since v%s\).*%s" % (max_ver, MSG)
msgre_warning = re.compile(msgre_tpl_warning)
txt = open(self.logfn, 'r').read()
self.assertTrue(msgre_warning.search(txt))
# test handling of non-UTF8 chars
msg = MSG + u"\x81"
msgre_tpl_error = r"DEPRECATED\s*\(since v%s\).*\xc2\x81" % max_ver
self.assertErrorRegex(Exception, msgre_tpl_error, logger.deprecated, msg, "1.1", max_ver)
logger.deprecated(msg, "0.9", max_ver)
txt = open(self.logfn, 'r').read()
self.assertTrue(msgre_warning.search(txt))
def __init__(self, options):
self.log = fancylogger.getLogger(self.__class__.__name__, fname=False)
self.vars = {"cwd": None, "jobid": None}
self.jobid = None
self.job_filter = None
def tearDown(self):
"""Clean up after running testcase."""
super(EnhancedTestCase, self).tearDown()
self.log.info("Cleaning up for test %s", self.id())
# go back to where we were before
os.chdir(self.cwd)
# restore original environment
modify_env(os.environ, self.orig_environ, verbose=False)
# restore original Python search path
sys.path = self.orig_sys_path
# remove any log handlers that were added (so that log files can be effectively removed)
log = fancylogger.getLogger(fname=False)
new_log_handlers = [h for h in log.handlers if h not in self.orig_log_handlers]
for log_handler in new_log_handlers:
log_handler.close()
log.removeHandler(log_handler)
# cleanup test tmp dir
try:
shutil.rmtree(self.test_prefix)
except (OSError, IOError):
pass
# restore original 'parent' tmpdir
for var in ['TMPDIR', 'TEMP', 'TMP']:
os.environ[var] = self.orig_tmpdir
# reset to make sure tempfile picks up new temporary directory to use
tempfile.tempdir = None
def __init__(self, script, name, env_vars=None, resources={}, conn=None, ppn=None):
"""
create a new Job to be submitted to PBS
env_vars is a dictionary with key-value pairs of environment variables that should be passed on to the job
resources is a dictionary with optional keys: ['hours', 'cores'] both of these should be integer values.
hours can be 1 - MAX_WALLTIME, cores depends on which cluster it is being run.
"""
self.clean_conn = True
self.log = fancylogger.getLogger(self.__class__.__name__, fname=False)
self.script = script
if env_vars:
self.env_vars = env_vars.copy()
else:
self.env_vars = {}
self.name = name
if pbs_import_failed:
self.log.error(pbs_import_failed)
try:
self.pbs_server = pbs.pbs_default()
if conn:
self.pbsconn = conn
self.clean_conn = False
else:
self.pbsconn = pbs.pbs_connect(self.pbs_server)
except Exception, err:
self.log.error("Failed to connect to the default pbs server: %s" % err)
def __init__(self, options=None, **kwargs):
if not hasattr(self, 'log'):
self.log = getLogger(self.__class__.__name__)
if not hasattr(self, 'options'):
self.options = options
self.nodes = None
self.nrnodes = None
self.uniquenodes = None
self.nruniquenodes = None
self.mpinodes = None
self.mpinrnodes = None
self.mpitotalppn = None
self.id = None
self.foundppn = None
self.ppn = None
self.totalppn = None
self.cpus = []
# collect data
self.get_id()
self._cores_on_this_node()
self.which_cpus()
self.get_node_list()
self.get_unique_nodes()
self.set_ppn()
super(Sched, self).__init__(**kwargs)
def __init__(self):
self.log = fancylogger.getLogger(self.__class__.__name__, fname=False)
self.hadoopversion = {
'major':-1,
'minor':-1,
'small':-1,
'suffix': None,
}
self.hadoophome = None # hadoop home (could be $EBROOTHADOOP)
self.hadoop = None
self.javaversion = {
'major':-1,
'minor':-1,
'suffix': None,
}
self.java = None
self.javahome = None
self.name = 'all' # default task start-all.sh, stop-all.sh
self.daemonname = 'hadoop'
self.start_script = None
self.stop_script = None
self.daemon_script = None
self.extrasearchpaths = []
def test_uft8_decoding(self):
"""Test UTF8 decoding."""
# truncate the logfile
open(self.logfn, 'w')
logger = fancylogger.getLogger('utf8_test')
logger.setLevel('DEBUG')
msgs = [
# bytestrings
"This is a pure ASCII text.", # pure ASCII
"Here are some UTF-8 characters: ß, ©, Ω, £.", # only UTF8 characters
"This non-UTF-8 character '\x80' should be handled properly.", # contains non UTF-8 character
# unicode strings
u"This is a pure ASCII text.", # pure ASCII
u"Here are some UTF8 characters: ß, ©, Ω, £.", # only UTF8 characters
u"This non-UTF8 character '\x80' should be handled properly.", # contains non UTF-8 character
]
for msg in msgs:
logger.critical(msg)
logger.debug(msg)
logger.error(msg)
logger.exception(msg)
logger.fatal(msg)
logger.info(msg)
logger.warning(msg)
logger.warn(msg)
if isinstance(msg, unicode):
regex = msg.encode('utf8', 'replace')
else:
regex = str(msg)
self.assertErrorRegex(Exception, regex, logger.raiseException, msg)
def main():
"""
main method
parses arguments and constructs Manager object and performs actions
"""
logger = fancylogger.getLogger()
options = get_options()
logger.debug("arguments parsed, starting manager")
manager = Manager(options)
#print status
if manager.status:
print "Status:\n %s" % manager.status
#actually do requested commands
out = manager.doit()
#parse and display the output
errors = []
if out:
for i in out: # this is an array of nodes and their output,err
if len(i[1]) > 0: # only print if something to show
print "%s:" % i[0] # first element is the node
for j in i[1]: # second element is an array of [outputs of commands,errors]
print " output: %s" % str(j[1][0])
if j[1][1]:
print " error: %s" % str(j[1][1])
errors.append(i[0])
# print all nodes with errors out on the end
if len(errors) > 0:
print "ERRORS occured in: \n ", ",".join([str(x) for x in errors])
def test_error_env_options(self):
"""Test log error on unknown environment option"""
self.reset_logcache()
mylogger = fancylogger.getLogger('ExtOptionParser')
mylogger.error = self.mock_logmethod(mylogger.error)
mylogger.debug = self.mock_logmethod(mylogger.debug)
self.assertEqual(self.count_logcache('error'), 0)
os.environ['GENERALOPTIONTEST_XYZ'] = '1'
topt1 = TestOption1(go_args=['--level-level'], envvar_prefix='GENERALOPTIONTEST')
self.assertEqual(self.count_logcache('error'), 0,
msg='no errors logged, got %s' % self.count_logcache('error'))
topt1 = TestOption1(go_args=['--level-level'], envvar_prefix='GENERALOPTIONTEST', error_env_options=True)
print self.LOGCACHE['error']
self.assertEqual(self.count_logcache('error'), 1,
msg='one error should be logged, got %s' % self.count_logcache('error'))
# using a custom error method
def raise_error(msg, *args):
"""Raise error with given message and string arguments to format it."""
raise Exception(msg % args)
self.assertErrorRegex(Exception, "Found 1 environment variable.* prefixed with GENERALOPTIONTEST", TestOption1,
go_args=['--level-level'], envvar_prefix='GENERALOPTIONTEST', error_env_options=True,
error_env_option_method=raise_error)
def __init__(self, cmd=None, **kwargs):
"""
Handle initiliastion
@param cmd: command to run
@param input: set "simple" input
@param startpath: directory to change to before executing command
@param disable_log: use fake logger (won't log anything)
"""
self.input = kwargs.pop('input', None)
self.startpath = kwargs.pop('startpath', None)
if kwargs.pop('disable_log', None):
self.log = DummyFunction() # No logging
if not hasattr(self, 'log'):
self.log = getLogger(self._get_log_name())
self.cmd = cmd # actual command
self._cwd_before_startpath = None
self._process_module = None
self._process = None
self.readsize = 1024 # number of bytes to read blocking
self._shellcmd = None
self._popen_named_args = None
self._process_exitcode = None
self._process_output = None
self._post_exitcode_log_failure = self.log.error
super(Run, self).__init__(**kwargs)
def __init__(self, options=None, **kwargs):
if not hasattr(self, 'log'):
self.log = getLogger(self.__class__.__name__)
if not hasattr(self, 'options'):
self.options = options
self.cores_per_node = None
self.set_cores_per_node()
self.sched_id = None
self.set_sched_id()
self.cpus = []
self.set_cpus()
self.nodes, self.nodes_uniq, self.nodes_tot_cnt = None, None, None
self.set_nodes()
self.multiplier = None
self.set_multiplier()
ppn = os.environ.get('PBS_NUM_PPN')
if ppn is not None:
self.ppn = int(ppn)
self.log.debug("Determined # cores per node via $PBS_NUM_PPN: %s" % self.ppn)
else:
self.ppn = len(self.cpus)
self.log.debug("Failed to determine # cores per node via $PBS_NUM_PPN, using affinity: found %s" % self.ppn)
self.set_ppn()
self.mpinodes = None
self.set_mpinodes()
super(Sched, self).__init__(**kwargs)
def __init__(self, script, name, env_vars=None, hours=None, cores=None):
"""Create a new Job to be submitted to SLURM."""
self.log = fancylogger.getLogger(self.__class__.__name__, fname=False)
self.jobid = None
self.script = script
self.name = name
self.job_specs = {
'job-name': self.name,
# pattern for output file for submitted job;
# SLURM replaces %x with job name, %j with job ID (see https://slurm.schedmd.com/sbatch.html#lbAF)
'output': '%x-%j.out',
'wrap': self.script,
}
if env_vars:
self.job_specs['export'] = ','.join(sorted(env_vars.keys()))
max_walltime = build_option('job_max_walltime')
if hours is None:
hours = max_walltime
if hours > max_walltime:
self.log.warn("Specified %s hours, but this is impossible. (resetting to %s hours)" % (hours, max_walltime))
hours = max_walltime
self.job_specs['time'] = hours * 60
if cores:
self.job_specs['nodes'] = 1
self.job_specs['ntasks'] = cores
self.job_specs['ntasks-per-node'] = cores
else:
self.log.warn("Number of cores to request not specified, falling back to whatever Slurm does by default")
def request(self, method, url, body, headers):
if self.auth_header is not None:
headers['Authorization'] = self.auth_header
headers['User-Agent'] = self.user_agent
fancylogger.getLogger().debug('cli request: %s, %s, %s, %s', method, url, body, headers)
#TODO: in recent python: Context manager
conn = self.get_connection(method, url, body, headers)
status = conn.code
body = conn.read()
try:
pybody = json.loads(body)
except ValueError:
pybody = body
fancylogger.getLogger().debug('reponse len: %s ', len(pybody))
conn.close()
return status, pybody
def __init__(self):
self.log = getLogger(fname=False)
self.svc = MpiService(log=self.log)
self.work_max_age = 3600 * 71
self.work_start_time = time.time()
self.controldir = None
def __init__(self, url=None, password=None, connection_dn=None, validation_method=None, check_certificate=False):
self.url = url
self.password = password
self.connection_dn = connection_dn
self.validation_method = validation_method
self.check_server_certificate = lambda : check_certificate
self.log = getLogger(self.__class__.__name__)
def setUp(self):
"""Set up testcase."""
super(EnhancedTestCase, self).setUp()
self.log = fancylogger.getLogger(self.__class__.__name__, fname=False)
fd, self.logfile = tempfile.mkstemp(suffix='.log', prefix='eb-test-')
os.close(fd)
self.cwd = os.getcwd()
self.test_prefix = tempfile.mkdtemp()
# keep track of original environment to restore
self.orig_environ = copy.deepcopy(os.environ)
# keep track of original environment/Python search path to restore
self.orig_sys_path = sys.path[:]
self.orig_paths = {}
for path in ['buildpath', 'installpath', 'sourcepath']:
self.orig_paths[path] = os.environ.get('EASYBUILD_%s' % path.upper(), None)
testdir = os.path.dirname(os.path.abspath(__file__))
self.test_sourcepath = os.path.join(testdir, 'sandbox', 'sources')
os.environ['EASYBUILD_SOURCEPATH'] = self.test_sourcepath
os.environ['EASYBUILD_PREFIX'] = self.test_prefix
self.test_buildpath = tempfile.mkdtemp()
os.environ['EASYBUILD_BUILDPATH'] = self.test_buildpath
self.test_installpath = tempfile.mkdtemp()
os.environ['EASYBUILD_INSTALLPATH'] = self.test_installpath
# make sure that the tests only pick up easyconfigs provided with the tests
os.environ['EASYBUILD_ROBOT_PATHS'] = os.path.join(testdir, 'easyconfigs')
# make sure no deprecated behaviour is being triggered (unless intended by the test)
# trip *all* log.deprecated statements by setting deprecation version ridiculously high
self.orig_current_version = eb_build_log.CURRENT_VERSION
os.environ['EASYBUILD_DEPRECATED'] = '10000000'
init_config()
# remove any entries in Python search path that seem to provide easyblocks
for path in sys.path[:]:
if os.path.exists(os.path.join(path, 'easybuild', 'easyblocks', '__init__.py')):
sys.path.remove(path)
# add test easyblocks to Python search path and (re)import and reload easybuild modules
import easybuild
sys.path.append(os.path.join(testdir, 'sandbox'))
reload(easybuild)
import easybuild.easyblocks
reload(easybuild.easyblocks)
import easybuild.easyblocks.generic
reload(easybuild.easyblocks.generic)
reload(easybuild.tools.module_naming_scheme) # required to run options unit tests stand-alone
modtool = modules_tool()
self.reset_modulepath([os.path.join(testdir, 'modules')])
# purge out any loaded modules with original $MODULEPATH before running each test
modtool.purge()
def __init__(self):
"""Initialise Squashed instance"""
self.log = fancylogger.getLogger(self.__class__.__name__, fname=False)
# OrderedVersionOperators instances to keep track of the data of the matching
# version and toolchain version sections
self.versions = OrderedVersionOperators()
self.tcversions = OrderedVersionOperators()
self.result = {}
def get_log(name=None):
"""
Generate logger object
"""
# fname is always get_log, useless
log = fancylogger.getLogger(name, fname=False)
log.info("Logger started for %s." % name)
log.deprecated("get_log", "2.0")
return log
def __init__(self, obj):
"""Support the conversion of obj to something"""
self.__dict__['log'] = fancylogger.getLogger(self.__class__.__name__, fname=False)
self.__dict__['data'] = None
if isinstance(obj, basestring):
self.data = self._from_string(obj)
else:
raise EasyBuildError("unsupported type %s for %s: %s", type(obj), self.__class__.__name__, obj)
super(Convert, self).__init__(self.data)
def __init__(self, command=None, timeout=get_config("COMMAND_TIMEOUT"), host='localhost'):
'''
Constructor
command is a string representing the command to be run
'''
self.log = fancylogger.getLogger(self.__class__.__name__)
self.command = command
self.host = host
self.timeout = int(timeout)
"""
Easyconfig templates module that provides templating that can
be used within an Easyconfig file.
@author: Stijn De Weirdt (Ghent University)
@author: Fotis Georgatos (Uni.Lu, NTUA)
"""
import re
from vsc.utils import fancylogger
from distutils.version import LooseVersion
from easybuild.tools.build_log import EasyBuildError
from easybuild.tools.systemtools import get_shared_lib_ext
_log = fancylogger.getLogger('easyconfig.templates', fname=False)
# derived from easyconfig, but not from ._config directly
TEMPLATE_NAMES_EASYCONFIG = [
('nameletter', "First letter of software name"),
('toolchain_name', "Toolchain name"),
('toolchain_version', "Toolchain version"),
('version_major_minor', "Major.Minor version"),
('version_major', "Major version"),
('version_minor', "Minor version"),
]
# derived from EasyConfig._config
TEMPLATE_NAMES_CONFIG = [
'name',
'version',
'versionsuffix',
import platform
import pwd
import re
import struct
import sys
import termios
from socket import gethostname
from vsc.utils import fancylogger
from vsc.utils.affinity import sched_getaffinity
from easybuild.tools.build_log import EasyBuildError
from easybuild.tools.filetools import read_file, which
from easybuild.tools.run import run_cmd
_log = fancylogger.getLogger('systemtools', fname=False)
# Architecture constants
AARCH32 = 'AArch32'
AARCH64 = 'AArch64'
POWER = 'POWER'
X86_64 = 'x86_64'
# Vendor constants
AMD = 'AMD'
APM = 'Applied Micro'
ARM = 'ARM'
BROADCOM = 'Broadcom'
CAVIUM = 'Cavium'
DEC = 'DEC'
IBM = 'IBM'
from pygraph.classes.digraph import digraph
# https://pypi.python.org/pypi/python-graph-dot
import pygraph.readwrite.dot as dot
# graphviz (used for creating dependency graph images)
sys.path.append('..')
sys.path.append('/usr/lib/graphviz/python/')
sys.path.append('/usr/lib64/graphviz/python/')
# Python bindings to Graphviz (http://www.graphviz.org/),
# see https://pypi.python.org/pypi/graphviz-python
# graphviz-python (yum) or python-pygraphviz (apt-get)
# or brew install graphviz --with-bindings (OS X)
import gv
except ImportError:
pass
_log = fancylogger.getLogger('easyconfig.tools', fname=False)
def skip_available(easyconfigs, modtool):
"""Skip building easyconfigs for existing modules."""
module_names = [ec['full_mod_name'] for ec in easyconfigs]
modules_exist = modtool.exist(module_names)
retained_easyconfigs = []
for ec, mod_name, mod_exists in zip(easyconfigs, module_names, modules_exist):
if mod_exists:
_log.info("%s is already installed (module found), skipping" % mod_name)
else:
_log.debug("%s is not installed yet, so retaining it" % mod_name)
retained_easyconfigs.append(ec)
return retained_easyconfigs
import os
import re
import shutil
import tempfile
import glob
from distutils.version import LooseVersion
import easybuild.tools.environment as env
from easybuild.framework.easyblock import EasyBlock
from easybuild.framework.easyconfig import CUSTOM
from easybuild.tools.build_log import EasyBuildError
from easybuild.tools.filetools import find_flexlm_license, read_file
from easybuild.tools.run import run_cmd
from vsc.utils import fancylogger
_log = fancylogger.getLogger('generic.intelbase')
# different supported activation types (cfr. Intel documentation)
ACTIVATION_EXIST_LIC = 'exist_lic' # use a license which exists on the system
ACTIVATION_LIC_FILE = 'license_file' # use a license file
ACTIVATION_LIC_SERVER = 'license_server' # use a license server
ACTIVATION_SERIAL = 'serial_number' # use a serial number
ACTIVATION_TRIAL = 'trial_lic' # use trial activation
ACTIVATION_TYPES = [
ACTIVATION_EXIST_LIC,
ACTIVATION_LIC_FILE,
ACTIVATION_LIC_SERVER,
ACTIVATION_SERIAL,
ACTIVATION_TRIAL,
]
-1] == recvbuf[next_idx]['affinity'][0] - 1:
log.error(
"No nn on same node for rank %s (aff %s) and next rank %s (aff %s)"
% (idx, recvbuf[idx]['affinity'], next_idx,
recvbuf[next_idx]['affinity']))
else:
if not recvbuf[next_idx]['affinity'][0] == 0:
log.error(
"No nn on different nodes for rank %s (hn %s aff %s) and next rank %s (hn %s aff %s)"
% (idx, recvbuf[idx]['hostname'], recvbuf[idx]['affinity'],
next_idx, recvbuf[next_idx]['hostname'],
recvbuf[next_idx]['affinity']))
if __name__ == '__main__':
log = getLogger('mympisanity')
setLogLevelInfo()
if MPI4PY_EXCEPTION:
log.error("No mpi4py found: %s", MPI4PY_EXCEPTION)
sys.exit(1)
log.info("mympisanity started")
comm = MPI.COMM_WORLD
# gather the info from all processes
recvbuf = comm.gather(Report(), 0)
log.info("mympisanity gather report finished")
if comm.rank == 0:
# #
"""
Module with various utility functions
:author: Kenneth Hoste (Ghent University)
"""
import glob
import os
import string
import sys
from vsc.utils import fancylogger
from easybuild.tools.build_log import EasyBuildError, print_msg
from easybuild.tools.config import build_option
_log = fancylogger.getLogger('tools.utilities')
# a list of all ascii characters
ASCII_CHARS = string.maketrans('', '')
# a list of all unwanted ascii characters (we only want to keep digits, letters and _)
UNWANTED_CHARS = ASCII_CHARS.translate(
ASCII_CHARS, string.digits + string.ascii_letters + "_")
def read_environment(env_vars, strict=False):
"""NO LONGER SUPPORTED: use read_environment from easybuild.tools.environment instead"""
_log.nosupport(
"read_environment has been moved to easybuild.tools.environment",
'2.0')
:author: Kenneth Hoste (Ghent University)
"""
from distutils.version import LooseVersion
from time import gmtime, strftime
import re
import time
from vsc.utils import fancylogger
from easybuild.tools.build_log import EasyBuildError, print_msg
from easybuild.tools.config import build_option
from easybuild.tools.job.backend import JobBackend
from easybuild.tools.utilities import only_if_module_is_available
_log = fancylogger.getLogger('gc3pie', fname=False)
try:
import gc3libs
import gc3libs.exceptions
from gc3libs import Application, Run, create_engine
from gc3libs.core import Engine
from gc3libs.quantity import hours as hr
from gc3libs.workflow import DependentTaskCollection
# inject EasyBuild logger into GC3Pie
gc3libs.log = fancylogger.getLogger('gc3pie', fname=False)
# make handling of log.error compatible with stdlib logging
gc3libs.log.raiseError = False
def __init__(self, application, fake=False):
"""ModuleGenerator constructor."""
self.app = application
self.log = fancylogger.getLogger(self.__class__.__name__, fname=False)
self.fake_mod_path = tempfile.mkdtemp()
def classless_function():
logger = fancylogger.getLogger(fname=True, clsname=True)
logger.warn("from classless_function")
:author: Stijn De Weirdt (Ghent University)
:author: Dries Verdegem (Ghent University)
:author: Kenneth Hoste (Ghent University)
:author: Pieter De Baets (Ghent University)
:author: Jens Timmerman (Ghent University)
:author: Toon Willems (Ghent University)
:author: Ward Poelmans (Ghent University)
:author: Fotis Georgatos (Uni.Lu, NTUA)
"""
from vsc.utils import fancylogger
from vsc.utils.missing import get_subclasses
from easybuild.tools.build_log import EasyBuildError
from easybuild.tools.utilities import import_available_modules
_log = fancylogger.getLogger('repository', fname=False)
class Repository(object):
"""
Interface for repositories
"""
DESCRIPTION = None
USABLE = True # can the Repository be used?
def __init__(self, repo_path, subdir=''):
"""
Initialize a repository. self.repo and self.subdir will be set.
self.wc will be set to None.
def __init__(self, requested_users):
"""
Generate data base with user account data
requested_users: (iterable) list of usernames to include in the data base
"""
self.log = fancylogger.getLogger(name=self.__class__.__name__)
# Set number of procs for parallel processing from configuration file
try:
self.max_procs = MainConf.get_digit('nodegroups',
'max_procs',
fallback=None,
mandatory=False)
except (KeyError, ValueError) as err:
error_exit(self.log, err)
else:
self.log.debug("Maximum number of processor set to %s",
self.max_procs)
# Check requested list of users
try:
self.users = list(requested_users)
except TypeError as err:
errmsg = f"Cannot generate user data base from non-iterable user list: {requested_users}"
error_exit(self.log, errmsg)
# Get token from configuration file to access VSC account page
TokenConfig = ConfigFile()
try:
vsc_token_file = MainConf.get('userdb',
'vsc_token_file',
fallback='vsc-access.ini',
mandatory=False)
self.vsc_token = TokenConfig.load(vsc_token_file).get(
'MAIN', 'access_token')
except KeyError as err:
error_exit(self.log, err)
# Load user data base from local cache
self.cache = self.load_db_cache()
# Update list of users with their records in the cache
for n, user in enumerate(self.users):
if user in self.cache.contents['db']:
self.users[n] = (user, self.cache.contents['db'][user])
else:
self.users[n] = (user, None)
# Retrieve account data of requested users
self.log.info(f"Retrieving {len(self.users)} user account records...")
requested_records = parallel_exec(
get_updated_record, # worker function
f"User account retrieval", # label prefixing log messages
self.users, # stack of items to process
self.cache.contents[
'valid_days'], # record_validity: forwarded to worker function
self.vsc_token, # vsc_token: forwarded to worker function
procs=self.max_procs,
logger=self.log,
)
# Generate dict of user accounts and update cache
self.records = dict()
for user_record in requested_records:
self.records.update(user_record)
self.cache.contents['db'].update(user_record)
# Save local cache
self.cache.save_data()
def get_vsc_record(username, vsc_token, logger=None):
"""
Retrieve and update list of VSC users with data from VSC account page
- username: (string) VSC ID or institute user of the VSC account
- vsc_token: (string) access token to VSC account page
- logger: (object) fancylogger object of the caller
"""
if logger is None:
logger = fancylogger.getLogger()
vsc_api_client = AccountpageClient(token=vsc_token)
# Get institute login of the VSC account attached to this username
if username[0:3] == 'vsc' and username[3].isdigit():
# VSC ID: query institute login to VSC account page
logger.debug(f"[{username}] user treated as VSC ID")
try:
vsc_account = vsc_api_client.account[username].person.get()[1]
except HTTPError as err:
if err.code == 404:
error_exit(
logger,
f"[{username}] VSC ID not found in VSC account page")
else:
error_exit(logger, f"[{username}] {err}")
except (TimeoutError, URLError) as err:
error_exit(
logger,
f"[{username}] connection to VSC account page timed out")
else:
vsc_login = {
'username': vsc_account['institute_login'],
'site': vsc_account['institute']['name']
}
logger.debug(
f"[{username}] VSC ID belongs to VSC account '{vsc_login['username']}'"
)
else:
# Others: assume NetID from Brussels
logger.debug(f"[{username}] user treated as NetID")
vsc_login = {'username': username, 'site': BRUSSEL}
# Retrieve user data from VSC account page
try:
vsc_account = vsc_api_client.account.institute[vsc_login['site']].id[
vsc_login['username']].get()[1]
except HTTPError as err:
if err.code == 404:
logger.debug(
f"[{username}] with VSC account '{vsc_login['username']}' not found"
)
return None
else:
error_exit(logger, f"[{username}] {err}")
except (TimeoutError, URLError) as err:
error_exit(logger,
f"[{username}] connection to VSC account page timed out")
else:
logger.debug(
f"[{username}] user account record retrieved from VSC account '{vsc_login['username']}'"
)
user_record = {
# only use first entry of research field
'field': vsc_account['research_field'][0],
'site':
INSTITUTE_LONGNAME[vsc_account['person']['institute']['name']],
'updated': date.today().isoformat(),
}
return user_record
import re
import sys
from vsc.utils import fancylogger
from vsc.utils.missing import get_subclasses, nub
import easybuild.tools.toolchain
from easybuild.tools.build_log import EasyBuildError
from easybuild.tools.toolchain.toolchain import Toolchain
from easybuild.tools.utilities import import_available_modules
TC_CONST_PREFIX = 'TC_CONSTANT_'
_initial_toolchain_instances = {}
_log = fancylogger.getLogger("toolchain.utilities")
def search_toolchain(name):
"""
Obtain a Toolchain instance for the toolchain with specified name, next to a list of available toolchains.
:param name: toolchain name
:return: Toolchain instance (or None), found_toolchains
"""
package = easybuild.tools.toolchain
check_attr_name = '%s_PROCESSED' % TC_CONST_PREFIX
if not hasattr(package, check_attr_name) or not getattr(package, check_attr_name):
# import all available toolchains, so we know about them
tc_modules = import_available_modules('easybuild.toolchains')
- MonoidDict: dictionary that combines values upon insertiong
according to the given monoid
- RUDict: dictionary that allows recursively updating its values (if they are dicts too) with a new RUDict
- shell_quote / shell_unquote : convenience functions to quote / unquote strings in shell context
@author: Andy Georges (Ghent University)
@author: Stijn De Weirdt (Ghent University)
"""
import shlex
import subprocess
import time
from vsc.utils import fancylogger
from vsc.utils.frozendict import FrozenDict
_log = fancylogger.getLogger('vsc.utils.missing')
def partial(func, *args, **keywords):
"""
Return a new partial object which when called will behave like func called with the positional arguments args
and keyword arguments keywords. If more arguments are supplied to the call, they are appended to args. If additional
keyword arguments are supplied, they extend and override keywords.
new in python 2.5, from https://docs.python.org/2/library/functools.html#functools.partial
"""
def newfunc(*fargs, **fkeywords):
newkeywords = keywords.copy()
newkeywords.update(fkeywords)
return func(*(args + fargs), **newkeywords)
newfunc.func = func
def test_fancyrecord(self):
"""
Test fancyrecord usage
"""
logger = fancylogger.getLogger()
self.assertEqual(logger.fancyrecord, True)
logger = fancylogger.getLogger(fancyrecord=True)
self.assertEqual(logger.fancyrecord, True)
logger = fancylogger.getLogger(fancyrecord=False)
self.assertEqual(logger.fancyrecord, False)
logger = fancylogger.getLogger('myname')
self.assertEqual(logger.fancyrecord, False)
logger = fancylogger.getLogger('myname', fancyrecord=True)
self.assertEqual(logger.fancyrecord, True)
orig = fancylogger.FANCYLOG_FANCYRECORD
fancylogger.FANCYLOG_FANCYRECORD = False
logger = fancylogger.getLogger()
self.assertEqual(logger.fancyrecord, False)
logger = fancylogger.getLogger('myname', fancyrecord=True)
self.assertEqual(logger.fancyrecord, True)
fancylogger.FANCYLOG_FANCYRECORD = True
logger = fancylogger.getLogger()
self.assertEqual(logger.fancyrecord, True)
logger = fancylogger.getLogger('myname')
self.assertEqual(logger.fancyrecord, True)
logger = fancylogger.getLogger('myname', fancyrecord=False)
self.assertEqual(logger.fancyrecord, False)
logger = fancylogger.getLogger('myname', fancyrecord='yes')
self.assertEqual(logger.fancyrecord, True)
logger = fancylogger.getLogger('myname', fancyrecord=0)
self.assertEqual(logger.fancyrecord, False)
fancylogger.FANCYLOG_FANCYRECORD = orig
"""Module to handle rst operations."""
import re
from vsc.utils import fancylogger
from vsc.utils.run import asyncloop
from panhandler import rst_from_pan
logger = fancylogger.getLogger()
MAILREGEX = re.compile(
("([a-zA-Z0-9!#$%&'*+\/=?^_`{|}~-]+(?:\.[a-zA-Z0-9!#$%&'*+\/=?^_`"
"{|}~-]+)*(@|\sat\s)(?:[a-zA-Z0-9](?:[a-zA-Z0-9-]*[a-zA-Z0-9])?(\.|"
"\sdot\s))+[a-zA-Z0-9](?:[a-zA-Z0-9-]*[a-zA-Z0-9])?)"))
PATHREGEX = re.compile(r'(\s+)((?:/[\w{}]+)+\.?\w*)(\s*)')
EXAMPLEMAILS = ["example", "username", "system.admin"]
def generate_rst(sources):
"""Generate rst."""
logger.info("Generating rst files.")
rstlist = {}
for title, source in sources.iteritems():
logger.debug("Parsing %s." % source)
rst = None
if source.endswith(".pan"):
rst = rst_from_pan(source, title)
else:
rst = rst_from_perl(source, title)
import os
import re
import sys
import tempfile
from distutils.version import LooseVersion
from textwrap import wrap
from vsc.utils import fancylogger
from vsc.utils.missing import get_subclasses
from easybuild.tools.build_log import EasyBuildError
from easybuild.tools.config import build_option, get_module_syntax, install_path
from easybuild.tools.filetools import convert_name, mkdir, read_file, remove_file, resolve_path, symlink, write_file
from easybuild.tools.modules import ROOT_ENV_VAR_NAME_PREFIX, modules_tool
from easybuild.tools.utilities import quote_str
_log = fancylogger.getLogger('module_generator', fname=False)
def avail_module_generators():
"""
Return all known module syntaxes.
"""
return dict([(k.SYNTAX, k) for k in get_subclasses(ModuleGenerator)])
def module_generator(app, fake=False):
"""
Return ModuleGenerator instance that matches the selected module file syntax to be used
"""
module_syntax = get_module_syntax()
available_mod_gens = avail_module_generators()
def somefunction(self):
logger = fancylogger.getLogger(fname=True, clsname=True)
logger.warn('we are logging something here')
"""
import copy
import glob
import os
import random
import string
import tempfile
import time
from vsc.utils import fancylogger
from vsc.utils.missing import FrozenDictKnownKeys
from vsc.utils.patterns import Singleton
from easybuild.tools.build_log import EasyBuildError
from easybuild.tools.module_naming_scheme import GENERAL_CLASS
_log = fancylogger.getLogger('config', fname=False)
ERROR = 'error'
IGNORE = 'ignore'
PURGE = 'purge'
UNLOAD = 'unload'
UNSET = 'unset'
WARN = 'warn'
PKG_TOOL_FPM = 'fpm'
PKG_TYPE_RPM = 'rpm'
DEFAULT_JOB_BACKEND = 'GC3Pie'
DEFAULT_LOGFILE_FORMAT = (
"easybuild", "easybuild-%(name)s-%(version)s-%(date)s.%(time)s.log")
DEFAULT_MAX_FAIL_RATIO_PERMS = 0.5
def __init__(self, *args, **kwargs):
super(Variables, self).__init__(*args, **kwargs)
self.log = fancylogger.getLogger(self.__class__.__name__, fname=False)
""" from __future__ import print_function import operator import os import pwd import re import stat import sys import time from vsc.utils.cache import FileCache from vsc.utils.fancylogger import getLogger from vsc.utils.py2vs3 import is_string, FileNotFoundErrorExc log = getLogger(__name__) NAGIOS_CACHE_DIR = '/var/cache' NAGIOS_CACHE_FILENAME_TEMPLATE = '%s.nagios.json.gz' NAGIOS_OK = 'OK' NAGIOS_WARNING = 'WARNING' NAGIOS_CRITICAL = 'CRITICAL' NAGIOS_UNKNOWN = 'UNKNOWN' NAGIOS_EXIT_OK = (0, NAGIOS_OK) NAGIOS_EXIT_WARNING = (1, NAGIOS_WARNING) NAGIOS_EXIT_CRITICAL = (2, NAGIOS_CRITICAL) NAGIOS_EXIT_UNKNOWN = (3, NAGIOS_UNKNOWN) NAGIOS_MAX_MESSAGE_LENGTH = 8192
# along with EasyBuild. If not, see <http://www.gnu.org/licenses/>.
# #
"""
Module that contains a set of classes and function to generate variables to be used
e.g., in compiling or linking
:author: Stijn De Weirdt (Ghent University)
:author: Kenneth Hoste (Ghent University)
"""
import copy
import os
from vsc.utils import fancylogger
from easybuild.tools.build_log import EasyBuildError
_log = fancylogger.getLogger('variables', fname=False)
def get_class(name, default_class, map_class=None):
"""Return class based on default
map_class
if key == str -> value = class
else: key = class -> list of strings
"""
if map_class is None:
map_class = {}
klass = default_class
if name is not None:
try:
klass = map_class[name]
:author: Kenneth Hoste (Ghent University)
:author: Pieter De Baets (Ghent University)
:author: Jens Timmerman (Ghent University)
:author: Fotis Georgatos (Uni.Lu, NTUA)
"""
import os
import string
from vsc.utils import fancylogger
from vsc.utils.missing import get_subclasses
from easybuild.tools import module_naming_scheme
from easybuild.tools.module_naming_scheme import ModuleNamingScheme
from easybuild.tools.toolchain import DUMMY_TOOLCHAIN_NAME
from easybuild.tools.utilities import import_available_modules
_log = fancylogger.getLogger('module_naming_scheme.utilities', fname=False)
def det_full_ec_version(ec):
"""
Determine exact install version, based on supplied easyconfig.
e.g. 1.2.3-goalf-1.1.0-no-OFED or 1.2.3 (for dummy toolchains)
"""
ecver = None
toolchain = ec.get('toolchain', {'name': DUMMY_TOOLCHAIN_NAME})
# determine main install version based on toolchain
if toolchain['name'] == DUMMY_TOOLCHAIN_NAME:
ecver = ec['version']
else:
# #
"""
Set of fucntions to help with jenkins setup
@author: Kenneth Hoste (Ghent University)
"""
import glob
import os
import xml.dom.minidom as xml
from datetime import datetime
from vsc.utils import fancylogger
from easybuild.tools.version import FRAMEWORK_VERSION, EASYBLOCKS_VERSION
_log = fancylogger.getLogger('jenkins', fname=False)
def write_to_xml(succes, failed, filename):
"""
Create xml output, using minimal output required according to
http://stackoverflow.com/questions/4922867/junit-xml-format-specification-that-hudson-supports
"""
dom = xml.getDOMImplementation()
root = dom.createDocument(None, "testsuite", None)
def create_testcase(name):
el = root.createElement("testcase")
el.setAttribute("name", name)
return el
# along with EasyBuild. If not, see <http://www.gnu.org/licenses/>.
#
"""
Easyconfig constants module that provides all constants that can
be used within an Easyconfig file.
@author: Stijn De Weirdt (Ghent University)
@author: Kenneth Hoste (Ghent University)
"""
import platform
from vsc.utils import fancylogger
from easybuild.tools.systemtools import get_shared_lib_ext, get_os_name, get_os_type, get_os_version
_log = fancylogger.getLogger('easyconfig.constants', fname=False)
# constants that can be used in easyconfig
EASYCONFIG_CONSTANTS = {
'SYS_PYTHON_VERSION': (platform.python_version(), "System Python version (platform.python_version())"),
'OS_TYPE': (get_os_type(), "System type (e.g. 'Linux' or 'Darwin')"),
'OS_NAME': (get_os_name(), "System name (e.g. 'fedora' or 'RHEL')"),
'OS_VERSION': (get_os_version(), "System version"),
}
def constant_documentation():
"""Generate the easyconfig constant documentation"""
indent_l0 = " " * 2
indent_l1 = indent_l0 + " " * 2
doc = []
def setUp(self):
"""Set up testcase."""
super(EnhancedTestCase, self).setUp()
# make sure option parser doesn't pick up any cmdline arguments/options
while len(sys.argv) > 1:
sys.argv.pop()
# keep track of log handlers
log = fancylogger.getLogger(fname=False)
self.orig_log_handlers = log.handlers[:]
log.info("setting up test %s" % self.id())
self.orig_tmpdir = tempfile.gettempdir()
# use a subdirectory for this test (which we can clean up easily after the test completes)
self.test_prefix = set_tmpdir()
self.log = fancylogger.getLogger(self.__class__.__name__, fname=False)
fd, self.logfile = tempfile.mkstemp(suffix='.log', prefix='eb-test-')
os.close(fd)
self.cwd = os.getcwd()
# keep track of original environment to restore
self.orig_environ = copy.deepcopy(os.environ)
# keep track of original environment/Python search path to restore
self.orig_sys_path = sys.path[:]
testdir = os.path.dirname(os.path.abspath(__file__))
self.test_sourcepath = os.path.join(testdir, 'sandbox', 'sources')
os.environ['EASYBUILD_SOURCEPATH'] = self.test_sourcepath
os.environ['EASYBUILD_PREFIX'] = self.test_prefix
self.test_buildpath = tempfile.mkdtemp()
os.environ['EASYBUILD_BUILDPATH'] = self.test_buildpath
self.test_installpath = tempfile.mkdtemp()
os.environ['EASYBUILD_INSTALLPATH'] = self.test_installpath
# make sure that the tests only pick up easyconfigs provided with the tests
os.environ['EASYBUILD_ROBOT_PATHS'] = os.path.join(
testdir, 'easyconfigs', 'test_ecs')
# make sure no deprecated behaviour is being triggered (unless intended by the test)
# trip *all* log.deprecated statements by setting deprecation version ridiculously high
self.orig_current_version = eb_build_log.CURRENT_VERSION
os.environ['EASYBUILD_DEPRECATED'] = '10000000'
init_config()
import easybuild
# try to import easybuild.easyblocks(.generic) packages
# it's OK if it fails here, but important to import first before fiddling with sys.path
try:
import easybuild.easyblocks
import easybuild.easyblocks.generic
except ImportError:
pass
# add sandbox to Python search path, update namespace packages
sys.path.append(os.path.join(testdir, 'sandbox'))
# workaround for bug in recent setuptools version (19.4 and newer, atleast until 20.3.1)
# injecting <prefix>/easybuild is required to avoid a ValueError being thrown by fixup_namespace_packages
# cfr. https://bitbucket.org/pypa/setuptools/issues/520/fixup_namespace_packages-may-trigger
for path in sys.path[:]:
if os.path.exists(
os.path.join(path, 'easybuild', 'easyblocks',
'__init__.py')):
# keep track of 'easybuild' paths to inject into sys.path later
sys.path.append(os.path.join(path, 'easybuild'))
# required to make sure the 'easybuild' dir in the sandbox is picked up;
# this relates to the other 'reload' statements below
reload(easybuild)
# this is strictly required to make the test modules in the sandbox available, due to declare_namespace
fixup_namespace_packages(os.path.join(testdir, 'sandbox'))
# remove any entries in Python search path that seem to provide easyblocks (except the sandbox)
for path in sys.path[:]:
if os.path.exists(
os.path.join(path, 'easybuild', 'easyblocks',
'__init__.py')):
if not os.path.samefile(path, os.path.join(testdir,
'sandbox')):
sys.path.remove(path)
# hard inject location to (generic) test easyblocks into Python search path
# only prepending to sys.path is not enough due to 'declare_namespace' in easybuild/easyblocks/__init__.py
import easybuild.easyblocks
reload(easybuild.easyblocks)
test_easyblocks_path = os.path.join(testdir, 'sandbox', 'easybuild',
'easyblocks')
easybuild.easyblocks.__path__.insert(0, test_easyblocks_path)
import easybuild.easyblocks.generic
reload(easybuild.easyblocks.generic)
test_easyblocks_path = os.path.join(test_easyblocks_path, 'generic')
easybuild.easyblocks.generic.__path__.insert(0, test_easyblocks_path)
# save values of $PATH & $PYTHONPATH, so they can be restored later
# this is important in case EasyBuild was installed as a module, since that module may be unloaded,
# for example due to changes to $MODULEPATH in case EasyBuild was installed in a module hierarchy
# cfr. https://github.com/easybuilders/easybuild-framework/issues/1685
self.env_path = os.environ.get('PATH')
self.env_pythonpath = os.environ.get('PYTHONPATH')
self.modtool = modules_tool()
self.reset_modulepath([os.path.join(testdir, 'modules')])
reset_module_caches()
def __init__(self):
"""initialize logger."""
self.log = fancylogger.getLogger(self.__class__.__name__, fname=False)
implemented as an easyblock
@author: Kenneth Hoste (Ghent University)
"""
import copy
import os
from vsc.utils import fancylogger
from easybuild.framework.easyblock import EasyBlock
from easybuild.framework.easyconfig import CUSTOM
from easybuild.framework.extension import Extension
from easybuild.tools.build_log import EasyBuildError
from easybuild.tools.filetools import apply_patch, extract_file
from easybuild.tools.utilities import remove_unwanted_chars
_log = fancylogger.getLogger('extensioneasyblock', fname=False)
class ExtensionEasyBlock(EasyBlock, Extension):
"""
Install an extension as a separate module, or as an extension.
Deriving classes should implement the following functions:
* required EasyBlock functions:
- configure_step
- build_step
- install_step
* required Extension functions
- run
"""
@staticmethod
# #
"""
Easyconfig module that contains the default EasyConfig configuration parameters.
:author: Stijn De Weirdt (Ghent University)
:author: Dries Verdegem (Ghent University)
:author: Kenneth Hoste (Ghent University)
:author: Pieter De Baets (Ghent University)
:author: Jens Timmerman (Ghent University)
:author: Toon Willems (Ghent University)
"""
from vsc.utils import fancylogger
from easybuild.tools.build_log import EasyBuildError
_log = fancylogger.getLogger('easyconfig.default', fname=False)
# we use a tuple here so we can sort them based on the numbers
ALL_CATEGORIES = {
'HIDDEN': (-1, 'hidden'),
'MANDATORY': (0, 'mandatory'),
'CUSTOM': (1, 'easyblock-specific'),
'TOOLCHAIN': (2, 'toolchain'),
'BUILD': (3, 'build'),
'FILEMANAGEMENT': (4, 'file-management'),
'DEPENDENCIES': (5, 'dependencies'),
'LICENSE': (6, 'license'),
'EXTENSIONS': (7, 'extensions'),
'MODULES': (8, 'modules'),
'OTHER': (9, 'other'),
}
@author: Toon Willems (Ghent University)
@author: Ward Poelmans (Ghent University)
"""
import os
import re
import signal
import subprocess
import tempfile
import time
from vsc.utils import fancylogger
from easybuild.tools.asyncprocess import PIPE, STDOUT, Popen, recv_some, send_all
import easybuild.tools.build_log # this import is required to obtain a correct (EasyBuild) logger!
_log = fancylogger.getLogger('run', fname=False)
errors_found_in_log = 0
# constants for strictness levels
IGNORE = 'ignore'
WARN = 'warn'
ERROR = 'error'
# default strictness level
strictness = WARN
def adjust_cmd(func):
"""Make adjustments to given command, if required."""
def inner(cmd, *args, **kwargs):
