def test_parentinfo(self): """Test the collection of parentinfo""" log_fr = fancylogger.getLogger(fname=False) # rootfancylogger pi_fr = log_fr._get_parent_info() self.assertEqual(len(pi_fr), 2) log_l1 = fancylogger.getLogger('level1', fname=False) # fname=False is required to have the naming similar for child relations pi_l1 = log_l1._get_parent_info() self.assertEqual(len(pi_l1), 3) py_v_27 = sys.version_info >= (2, 7, 0) if py_v_27: log_l2a = log_l1.getChild('level2a') pi_l2a = log_l2a._get_parent_info() self.assertEqual(len(pi_l2a), 4) # this should be identical to getChild log_l2b = fancylogger.getLogger('level1.level2b', fname=False) # fname=False is required to have the name similar # cutoff last letter (a vs b) if py_v_27: self.assertEqual(log_l2a.name[:-1], log_l2b.name[:-1]) pi_l2b = log_l2b._get_parent_info() # yes, this broken on several levels (incl in logging itself) # adding '.' in the name does not automatically create the parent/child relations # if the parent with the name exists, this works self.assertEqual(len(pi_l2b), 4) log_l2c = fancylogger.getLogger('level1a.level2c', fname=False) pi_l2c = log_l2c._get_parent_info() self.assertEqual(len(pi_l2c), 3) # level1a as parent does not exist
def test_raiseException(self): """Test raiseException log method.""" # truncate the logfile open(self.logfn, 'w') def test123(exception, msg): """Utility function for testing raiseException.""" try: raise exception(msg) except: logger.raiseException('HIT') logger = fancylogger.getLogger('fail_test') self.assertErrorRegex(Exception, 'failtest', test123, Exception, 'failtest') self.assertTrue(re.match("^WARNING.*HIT.*failtest\n.*in test123.*$", open(self.logfn, 'r').read(), re.M)) open(self.logfn, 'w') fancylogger.FancyLogger.RAISE_EXCEPTION_CLASS = KeyError logger = fancylogger.getLogger('fail_test') self.assertErrorRegex(KeyError, 'failkeytest', test123, KeyError, 'failkeytest') self.assertTrue(re.match("^WARNING.*HIT.*'failkeytest'\n.*in test123.*$", open(self.logfn, 'r').read(), re.M)) open(self.logfn, 'w') fancylogger.FancyLogger.RAISE_EXCEPTION_LOG_METHOD = lambda c, msg: c.warning(msg) logger = fancylogger.getLogger('fail_test') self.assertErrorRegex(AttributeError, 'attrtest', test123, AttributeError, 'attrtest') self.assertTrue(re.match("^WARNING.*HIT.*attrtest\n.*in test123.*$", open(self.logfn, 'r').read(), re.M))
def request(self, method, url, body, headers, content_type=None): """Low-level networking. All HTTP-method methods call this""" if headers is None: headers = {} if content_type is not None: headers['Content-Type'] = content_type if self.auth_header is not None: headers['Authorization'] = self.auth_header headers['User-Agent'] = self.user_agent fancylogger.getLogger().debug('cli request: %s, %s, %s, %s', method, url, body, headers) # TODO: in recent python: Context manager conn = self.get_connection(method, url, body, headers) status = conn.code if method == self.HEAD: pybody = conn.headers else: body = conn.read() try: pybody = json.loads(body) except ValueError: pybody = body fancylogger.getLogger().debug('reponse len: %s ', len(pybody)) conn.close() return status, pybody
def test_get_callers_logger(self): """Test get_callers_logger function.""" # returns None if no logger is available self.assertEqual(get_callers_logger(), None) # find defined logger in caller's context logger = getLogger("foo") callers_logger = get_callers_logger() # result depends on whether tests were run under 'python' or 'python -O' self.assertTrue(callers_logger in [logger, None]) # also works when logger is 'higher up' class Test(object): """Dummy test class""" def foo(self, logger=None): """Dummy test method, returns logger from calling context.""" return get_callers_logger() test = Test() self.assertTrue(logger, [test.foo(), None]) # closest logger to caller is preferred logger2 = getLogger(test.__class__.__name__) self.assertTrue(logger2 in [test.foo(logger=logger2), None])
def test_fancylogger_as_rootlogger_logging(self): """ Test if just using import logging, logging with logging uses fancylogger after setting the root logger """ # test logging.root is loggin root logger # this is an assumption made to make the fancyrootlogger code work orig_root = logging.getLogger() self.assertEqual(logging.root, orig_root, msg='logging.root is the root logger') self.assertFalse(isinstance(logging.root, fancylogger.FancyLogger), msg='logging.root is not a FancyLogger') stringfile = StringIO() sys.stderr = stringfile handler = fancylogger.logToScreen() fancylogger.setLogLevelDebug() logger = fancylogger.getLogger() self.assertEqual(logger.handlers, [self.handler, handler], msg='active handler for root fancylogger') self.assertEqual(logger.level, fancylogger.getLevelInt('DEBUG'), msg='debug level set') msg = 'this is my string' logging.debug(msg) self.assertEqual(stringfile.getvalue(), '', msg="logging.debug reports nothing when fancylogger loglevel is debug") fancylogger.setroot() self.assertTrue(isinstance(logging.root, fancylogger.FancyLogger), msg='logging.root is a FancyLogger after setRootLogger') self.assertEqual(logging.root.level, fancylogger.getLevelInt('DEBUG'), msg='debug level set for root') self.assertEqual(logger.level, logging.NOTSET, msg='original root fancylogger level set to NOTSET') self.assertEqual(logging.root.handlers, [self.handler, handler], msg='active handler for root logger from previous root fancylogger') self.assertEqual(logger.handlers, [], msg='no active handlers on previous root fancylogger') root_logger = logging.getLogger('') self.assertEqual(root_logger, logging.root, msg='logging.getLogger() returns logging.root FancyLogger') frl = fancylogger.getLogger() self.assertEqual(frl, logging.root, msg='fancylogger.getLogger() returns logging.root FancyLogger') logging.debug(msg) self.assertTrue(msg in stringfile.getvalue(), msg="logging.debug reports when fancylogger loglevel is debug") fancylogger.resetroot() self.assertEqual(logging.root, orig_root, msg='logging.root is the original root logger after resetroot') # restore fancylogger.logToScreen(enable=False, handler=handler)
def getCluster(name): """ static factory method, should be in Cluster class returns a cluster object of the given name """ for cls in Cluster.__subclasses__(): if cls._is_cluster_for(name): return cls() fancylogger.getLogger("clusters.getCluster").raiseException("No such cluster %s" % name, NoSuchClusterException)
def get_job(classname, options): """ This is a job factory. Returns an instance of classname initialized with options """ for cls in get_subclasses(Job): if cls._is_job_for(classname): return cls(options) getLogger().error("No job class found for %s", classname)
def main(): """Main function""" try: instance_options = get_mpi_and_sched_and_options() if instance_options: instance = getinstance(*instance_options) instance.main() except Exception: fancylogger.getLogger().exception("Main failed") sys.exit(1)
def get_connection(self, method, url, body, headers): if not self.url.endswith('/') and not url.startswith('/'): sep = '/' else: sep = '' request = urllib2.Request(self.url + sep + url, data=body) for header, value in headers.iteritems(): request.add_header(header, value) request.get_method = lambda: method fancylogger.getLogger().debug('opening request: %s%s%s', self.url, sep, url) connection = self.opener.open(request) return connection
def det_pylibdir(plat_specific=False, python_cmd=None): """Determine Python library directory.""" log = fancylogger.getLogger('det_pylibdir', fname=False) if python_cmd is None: # use 'python' that is listed first in $PATH if none was specified python_cmd = 'python' # determine Python lib dir via distutils # use run_cmd, we can to talk to the active Python, not the system Python running EasyBuild prefix = '/tmp/' args = 'plat_specific=%s, prefix="%s"' % (plat_specific, prefix) pycode = "import distutils.sysconfig; print(distutils.sysconfig.get_python_lib(%s))" % args cmd = "%s -c '%s'" % (python_cmd, pycode) log.debug("Determining Python library directory using command '%s'", cmd) out, ec = run_cmd(cmd, simple=False, force_in_dry_run=True) txt = out.strip().split('\n')[-1] # value obtained should start with specified prefix, otherwise something is very wrong if not txt.startswith(prefix): raise EasyBuildError("Last line of output of %s does not start with specified prefix %s: %s (exit code %s)", cmd, prefix, out, ec) pylibdir = txt[len(prefix):] log.debug("Determined pylibdir using '%s': %s", cmd, pylibdir) return pylibdir
def test_deprecated(self): """Test deprecated log function.""" # truncate the logfile open(self.logfn, 'w') # log message logger = fancylogger.getLogger('deprecated_test') max_ver = "1.0" # test whether deprecation works msgre_tpl_error = r"DEPRECATED\s*\(since v%s\).*%s" % (max_ver, MSG) self.assertErrorRegex(Exception, msgre_tpl_error, logger.deprecated, MSG, "1.1", max_ver) self.assertErrorRegex(Exception, msgre_tpl_error, logger.deprecated, MSG, "1.0", max_ver) # test whether deprecated warning works # no deprecation if current version is lower than max version logger.deprecated(MSG, "0.9", max_ver) msgre_tpl_warning = r"WARNING.*DEPRECATED\s*\(since v%s\).*%s" % (max_ver, MSG) msgre_warning = re.compile(msgre_tpl_warning) txt = open(self.logfn, 'r').read() self.assertTrue(msgre_warning.search(txt)) # test handling of non-UTF8 chars msg = MSG + u"\x81" msgre_tpl_error = r"DEPRECATED\s*\(since v%s\).*\xc2\x81" % max_ver self.assertErrorRegex(Exception, msgre_tpl_error, logger.deprecated, msg, "1.1", max_ver) logger.deprecated(msg, "0.9", max_ver) txt = open(self.logfn, 'r').read() self.assertTrue(msgre_warning.search(txt))
def __init__(self, options): self.log = fancylogger.getLogger(self.__class__.__name__, fname=False) self.vars = {"cwd": None, "jobid": None} self.jobid = None self.job_filter = None
def tearDown(self): """Clean up after running testcase.""" super(EnhancedTestCase, self).tearDown() self.log.info("Cleaning up for test %s", self.id()) # go back to where we were before os.chdir(self.cwd) # restore original environment modify_env(os.environ, self.orig_environ, verbose=False) # restore original Python search path sys.path = self.orig_sys_path # remove any log handlers that were added (so that log files can be effectively removed) log = fancylogger.getLogger(fname=False) new_log_handlers = [h for h in log.handlers if h not in self.orig_log_handlers] for log_handler in new_log_handlers: log_handler.close() log.removeHandler(log_handler) # cleanup test tmp dir try: shutil.rmtree(self.test_prefix) except (OSError, IOError): pass # restore original 'parent' tmpdir for var in ['TMPDIR', 'TEMP', 'TMP']: os.environ[var] = self.orig_tmpdir # reset to make sure tempfile picks up new temporary directory to use tempfile.tempdir = None
def __init__(self, script, name, env_vars=None, resources={}, conn=None, ppn=None): """ create a new Job to be submitted to PBS env_vars is a dictionary with key-value pairs of environment variables that should be passed on to the job resources is a dictionary with optional keys: ['hours', 'cores'] both of these should be integer values. hours can be 1 - MAX_WALLTIME, cores depends on which cluster it is being run. """ self.clean_conn = True self.log = fancylogger.getLogger(self.__class__.__name__, fname=False) self.script = script if env_vars: self.env_vars = env_vars.copy() else: self.env_vars = {} self.name = name if pbs_import_failed: self.log.error(pbs_import_failed) try: self.pbs_server = pbs.pbs_default() if conn: self.pbsconn = conn self.clean_conn = False else: self.pbsconn = pbs.pbs_connect(self.pbs_server) except Exception, err: self.log.error("Failed to connect to the default pbs server: %s" % err)
def __init__(self, options=None, **kwargs): if not hasattr(self, 'log'): self.log = getLogger(self.__class__.__name__) if not hasattr(self, 'options'): self.options = options self.nodes = None self.nrnodes = None self.uniquenodes = None self.nruniquenodes = None self.mpinodes = None self.mpinrnodes = None self.mpitotalppn = None self.id = None self.foundppn = None self.ppn = None self.totalppn = None self.cpus = [] # collect data self.get_id() self._cores_on_this_node() self.which_cpus() self.get_node_list() self.get_unique_nodes() self.set_ppn() super(Sched, self).__init__(**kwargs)
def __init__(self): self.log = fancylogger.getLogger(self.__class__.__name__, fname=False) self.hadoopversion = { 'major':-1, 'minor':-1, 'small':-1, 'suffix': None, } self.hadoophome = None # hadoop home (could be $EBROOTHADOOP) self.hadoop = None self.javaversion = { 'major':-1, 'minor':-1, 'suffix': None, } self.java = None self.javahome = None self.name = 'all' # default task start-all.sh, stop-all.sh self.daemonname = 'hadoop' self.start_script = None self.stop_script = None self.daemon_script = None self.extrasearchpaths = []
def test_uft8_decoding(self): """Test UTF8 decoding.""" # truncate the logfile open(self.logfn, 'w') logger = fancylogger.getLogger('utf8_test') logger.setLevel('DEBUG') msgs = [ # bytestrings "This is a pure ASCII text.", # pure ASCII "Here are some UTF-8 characters: ß, ©, Ω, £.", # only UTF8 characters "This non-UTF-8 character '\x80' should be handled properly.", # contains non UTF-8 character # unicode strings u"This is a pure ASCII text.", # pure ASCII u"Here are some UTF8 characters: ß, ©, Ω, £.", # only UTF8 characters u"This non-UTF8 character '\x80' should be handled properly.", # contains non UTF-8 character ] for msg in msgs: logger.critical(msg) logger.debug(msg) logger.error(msg) logger.exception(msg) logger.fatal(msg) logger.info(msg) logger.warning(msg) logger.warn(msg) if isinstance(msg, unicode): regex = msg.encode('utf8', 'replace') else: regex = str(msg) self.assertErrorRegex(Exception, regex, logger.raiseException, msg)
def main(): """ main method parses arguments and constructs Manager object and performs actions """ logger = fancylogger.getLogger() options = get_options() logger.debug("arguments parsed, starting manager") manager = Manager(options) #print status if manager.status: print "Status:\n %s" % manager.status #actually do requested commands out = manager.doit() #parse and display the output errors = [] if out: for i in out: # this is an array of nodes and their output,err if len(i[1]) > 0: # only print if something to show print "%s:" % i[0] # first element is the node for j in i[1]: # second element is an array of [outputs of commands,errors] print " output: %s" % str(j[1][0]) if j[1][1]: print " error: %s" % str(j[1][1]) errors.append(i[0]) # print all nodes with errors out on the end if len(errors) > 0: print "ERRORS occured in: \n ", ",".join([str(x) for x in errors])
def test_error_env_options(self): """Test log error on unknown environment option""" self.reset_logcache() mylogger = fancylogger.getLogger('ExtOptionParser') mylogger.error = self.mock_logmethod(mylogger.error) mylogger.debug = self.mock_logmethod(mylogger.debug) self.assertEqual(self.count_logcache('error'), 0) os.environ['GENERALOPTIONTEST_XYZ'] = '1' topt1 = TestOption1(go_args=['--level-level'], envvar_prefix='GENERALOPTIONTEST') self.assertEqual(self.count_logcache('error'), 0, msg='no errors logged, got %s' % self.count_logcache('error')) topt1 = TestOption1(go_args=['--level-level'], envvar_prefix='GENERALOPTIONTEST', error_env_options=True) print self.LOGCACHE['error'] self.assertEqual(self.count_logcache('error'), 1, msg='one error should be logged, got %s' % self.count_logcache('error')) # using a custom error method def raise_error(msg, *args): """Raise error with given message and string arguments to format it.""" raise Exception(msg % args) self.assertErrorRegex(Exception, "Found 1 environment variable.* prefixed with GENERALOPTIONTEST", TestOption1, go_args=['--level-level'], envvar_prefix='GENERALOPTIONTEST', error_env_options=True, error_env_option_method=raise_error)
def __init__(self, cmd=None, **kwargs): """ Handle initiliastion @param cmd: command to run @param input: set "simple" input @param startpath: directory to change to before executing command @param disable_log: use fake logger (won't log anything) """ self.input = kwargs.pop('input', None) self.startpath = kwargs.pop('startpath', None) if kwargs.pop('disable_log', None): self.log = DummyFunction() # No logging if not hasattr(self, 'log'): self.log = getLogger(self._get_log_name()) self.cmd = cmd # actual command self._cwd_before_startpath = None self._process_module = None self._process = None self.readsize = 1024 # number of bytes to read blocking self._shellcmd = None self._popen_named_args = None self._process_exitcode = None self._process_output = None self._post_exitcode_log_failure = self.log.error super(Run, self).__init__(**kwargs)
def __init__(self, options=None, **kwargs): if not hasattr(self, 'log'): self.log = getLogger(self.__class__.__name__) if not hasattr(self, 'options'): self.options = options self.cores_per_node = None self.set_cores_per_node() self.sched_id = None self.set_sched_id() self.cpus = [] self.set_cpus() self.nodes, self.nodes_uniq, self.nodes_tot_cnt = None, None, None self.set_nodes() self.multiplier = None self.set_multiplier() ppn = os.environ.get('PBS_NUM_PPN') if ppn is not None: self.ppn = int(ppn) self.log.debug("Determined # cores per node via $PBS_NUM_PPN: %s" % self.ppn) else: self.ppn = len(self.cpus) self.log.debug("Failed to determine # cores per node via $PBS_NUM_PPN, using affinity: found %s" % self.ppn) self.set_ppn() self.mpinodes = None self.set_mpinodes() super(Sched, self).__init__(**kwargs)
def __init__(self, script, name, env_vars=None, hours=None, cores=None): """Create a new Job to be submitted to SLURM.""" self.log = fancylogger.getLogger(self.__class__.__name__, fname=False) self.jobid = None self.script = script self.name = name self.job_specs = { 'job-name': self.name, # pattern for output file for submitted job; # SLURM replaces %x with job name, %j with job ID (see https://slurm.schedmd.com/sbatch.html#lbAF) 'output': '%x-%j.out', 'wrap': self.script, } if env_vars: self.job_specs['export'] = ','.join(sorted(env_vars.keys())) max_walltime = build_option('job_max_walltime') if hours is None: hours = max_walltime if hours > max_walltime: self.log.warn("Specified %s hours, but this is impossible. (resetting to %s hours)" % (hours, max_walltime)) hours = max_walltime self.job_specs['time'] = hours * 60 if cores: self.job_specs['nodes'] = 1 self.job_specs['ntasks'] = cores self.job_specs['ntasks-per-node'] = cores else: self.log.warn("Number of cores to request not specified, falling back to whatever Slurm does by default")
def request(self, method, url, body, headers): if self.auth_header is not None: headers['Authorization'] = self.auth_header headers['User-Agent'] = self.user_agent fancylogger.getLogger().debug('cli request: %s, %s, %s, %s', method, url, body, headers) #TODO: in recent python: Context manager conn = self.get_connection(method, url, body, headers) status = conn.code body = conn.read() try: pybody = json.loads(body) except ValueError: pybody = body fancylogger.getLogger().debug('reponse len: %s ', len(pybody)) conn.close() return status, pybody
def __init__(self): self.log = getLogger(fname=False) self.svc = MpiService(log=self.log) self.work_max_age = 3600 * 71 self.work_start_time = time.time() self.controldir = None
def __init__(self, url=None, password=None, connection_dn=None, validation_method=None, check_certificate=False): self.url = url self.password = password self.connection_dn = connection_dn self.validation_method = validation_method self.check_server_certificate = lambda : check_certificate self.log = getLogger(self.__class__.__name__)
def setUp(self): """Set up testcase.""" super(EnhancedTestCase, self).setUp() self.log = fancylogger.getLogger(self.__class__.__name__, fname=False) fd, self.logfile = tempfile.mkstemp(suffix='.log', prefix='eb-test-') os.close(fd) self.cwd = os.getcwd() self.test_prefix = tempfile.mkdtemp() # keep track of original environment to restore self.orig_environ = copy.deepcopy(os.environ) # keep track of original environment/Python search path to restore self.orig_sys_path = sys.path[:] self.orig_paths = {} for path in ['buildpath', 'installpath', 'sourcepath']: self.orig_paths[path] = os.environ.get('EASYBUILD_%s' % path.upper(), None) testdir = os.path.dirname(os.path.abspath(__file__)) self.test_sourcepath = os.path.join(testdir, 'sandbox', 'sources') os.environ['EASYBUILD_SOURCEPATH'] = self.test_sourcepath os.environ['EASYBUILD_PREFIX'] = self.test_prefix self.test_buildpath = tempfile.mkdtemp() os.environ['EASYBUILD_BUILDPATH'] = self.test_buildpath self.test_installpath = tempfile.mkdtemp() os.environ['EASYBUILD_INSTALLPATH'] = self.test_installpath # make sure that the tests only pick up easyconfigs provided with the tests os.environ['EASYBUILD_ROBOT_PATHS'] = os.path.join(testdir, 'easyconfigs') # make sure no deprecated behaviour is being triggered (unless intended by the test) # trip *all* log.deprecated statements by setting deprecation version ridiculously high self.orig_current_version = eb_build_log.CURRENT_VERSION os.environ['EASYBUILD_DEPRECATED'] = '10000000' init_config() # remove any entries in Python search path that seem to provide easyblocks for path in sys.path[:]: if os.path.exists(os.path.join(path, 'easybuild', 'easyblocks', '__init__.py')): sys.path.remove(path) # add test easyblocks to Python search path and (re)import and reload easybuild modules import easybuild sys.path.append(os.path.join(testdir, 'sandbox')) reload(easybuild) import easybuild.easyblocks reload(easybuild.easyblocks) import easybuild.easyblocks.generic reload(easybuild.easyblocks.generic) reload(easybuild.tools.module_naming_scheme) # required to run options unit tests stand-alone modtool = modules_tool() self.reset_modulepath([os.path.join(testdir, 'modules')]) # purge out any loaded modules with original $MODULEPATH before running each test modtool.purge()
def __init__(self): """Initialise Squashed instance""" self.log = fancylogger.getLogger(self.__class__.__name__, fname=False) # OrderedVersionOperators instances to keep track of the data of the matching # version and toolchain version sections self.versions = OrderedVersionOperators() self.tcversions = OrderedVersionOperators() self.result = {}
def get_log(name=None): """ Generate logger object """ # fname is always get_log, useless log = fancylogger.getLogger(name, fname=False) log.info("Logger started for %s." % name) log.deprecated("get_log", "2.0") return log
def __init__(self, obj): """Support the conversion of obj to something""" self.__dict__['log'] = fancylogger.getLogger(self.__class__.__name__, fname=False) self.__dict__['data'] = None if isinstance(obj, basestring): self.data = self._from_string(obj) else: raise EasyBuildError("unsupported type %s for %s: %s", type(obj), self.__class__.__name__, obj) super(Convert, self).__init__(self.data)
def __init__(self, command=None, timeout=get_config("COMMAND_TIMEOUT"), host='localhost'): ''' Constructor command is a string representing the command to be run ''' self.log = fancylogger.getLogger(self.__class__.__name__) self.command = command self.host = host self.timeout = int(timeout)
""" Easyconfig templates module that provides templating that can be used within an Easyconfig file. @author: Stijn De Weirdt (Ghent University) @author: Fotis Georgatos (Uni.Lu, NTUA) """ import re from vsc.utils import fancylogger from distutils.version import LooseVersion from easybuild.tools.build_log import EasyBuildError from easybuild.tools.systemtools import get_shared_lib_ext _log = fancylogger.getLogger('easyconfig.templates', fname=False) # derived from easyconfig, but not from ._config directly TEMPLATE_NAMES_EASYCONFIG = [ ('nameletter', "First letter of software name"), ('toolchain_name', "Toolchain name"), ('toolchain_version', "Toolchain version"), ('version_major_minor', "Major.Minor version"), ('version_major', "Major version"), ('version_minor', "Minor version"), ] # derived from EasyConfig._config TEMPLATE_NAMES_CONFIG = [ 'name', 'version', 'versionsuffix',
import platform import pwd import re import struct import sys import termios from socket import gethostname from vsc.utils import fancylogger from vsc.utils.affinity import sched_getaffinity from easybuild.tools.build_log import EasyBuildError from easybuild.tools.filetools import read_file, which from easybuild.tools.run import run_cmd _log = fancylogger.getLogger('systemtools', fname=False) # Architecture constants AARCH32 = 'AArch32' AARCH64 = 'AArch64' POWER = 'POWER' X86_64 = 'x86_64' # Vendor constants AMD = 'AMD' APM = 'Applied Micro' ARM = 'ARM' BROADCOM = 'Broadcom' CAVIUM = 'Cavium' DEC = 'DEC' IBM = 'IBM'
from pygraph.classes.digraph import digraph # https://pypi.python.org/pypi/python-graph-dot import pygraph.readwrite.dot as dot # graphviz (used for creating dependency graph images) sys.path.append('..') sys.path.append('/usr/lib/graphviz/python/') sys.path.append('/usr/lib64/graphviz/python/') # Python bindings to Graphviz (http://www.graphviz.org/), # see https://pypi.python.org/pypi/graphviz-python # graphviz-python (yum) or python-pygraphviz (apt-get) # or brew install graphviz --with-bindings (OS X) import gv except ImportError: pass _log = fancylogger.getLogger('easyconfig.tools', fname=False) def skip_available(easyconfigs, modtool): """Skip building easyconfigs for existing modules.""" module_names = [ec['full_mod_name'] for ec in easyconfigs] modules_exist = modtool.exist(module_names) retained_easyconfigs = [] for ec, mod_name, mod_exists in zip(easyconfigs, module_names, modules_exist): if mod_exists: _log.info("%s is already installed (module found), skipping" % mod_name) else: _log.debug("%s is not installed yet, so retaining it" % mod_name) retained_easyconfigs.append(ec) return retained_easyconfigs
import os import re import shutil import tempfile import glob from distutils.version import LooseVersion import easybuild.tools.environment as env from easybuild.framework.easyblock import EasyBlock from easybuild.framework.easyconfig import CUSTOM from easybuild.tools.build_log import EasyBuildError from easybuild.tools.filetools import find_flexlm_license, read_file from easybuild.tools.run import run_cmd from vsc.utils import fancylogger _log = fancylogger.getLogger('generic.intelbase') # different supported activation types (cfr. Intel documentation) ACTIVATION_EXIST_LIC = 'exist_lic' # use a license which exists on the system ACTIVATION_LIC_FILE = 'license_file' # use a license file ACTIVATION_LIC_SERVER = 'license_server' # use a license server ACTIVATION_SERIAL = 'serial_number' # use a serial number ACTIVATION_TRIAL = 'trial_lic' # use trial activation ACTIVATION_TYPES = [ ACTIVATION_EXIST_LIC, ACTIVATION_LIC_FILE, ACTIVATION_LIC_SERVER, ACTIVATION_SERIAL, ACTIVATION_TRIAL, ]
-1] == recvbuf[next_idx]['affinity'][0] - 1: log.error( "No nn on same node for rank %s (aff %s) and next rank %s (aff %s)" % (idx, recvbuf[idx]['affinity'], next_idx, recvbuf[next_idx]['affinity'])) else: if not recvbuf[next_idx]['affinity'][0] == 0: log.error( "No nn on different nodes for rank %s (hn %s aff %s) and next rank %s (hn %s aff %s)" % (idx, recvbuf[idx]['hostname'], recvbuf[idx]['affinity'], next_idx, recvbuf[next_idx]['hostname'], recvbuf[next_idx]['affinity'])) if __name__ == '__main__': log = getLogger('mympisanity') setLogLevelInfo() if MPI4PY_EXCEPTION: log.error("No mpi4py found: %s", MPI4PY_EXCEPTION) sys.exit(1) log.info("mympisanity started") comm = MPI.COMM_WORLD # gather the info from all processes recvbuf = comm.gather(Report(), 0) log.info("mympisanity gather report finished") if comm.rank == 0:
# # """ Module with various utility functions :author: Kenneth Hoste (Ghent University) """ import glob import os import string import sys from vsc.utils import fancylogger from easybuild.tools.build_log import EasyBuildError, print_msg from easybuild.tools.config import build_option _log = fancylogger.getLogger('tools.utilities') # a list of all ascii characters ASCII_CHARS = string.maketrans('', '') # a list of all unwanted ascii characters (we only want to keep digits, letters and _) UNWANTED_CHARS = ASCII_CHARS.translate( ASCII_CHARS, string.digits + string.ascii_letters + "_") def read_environment(env_vars, strict=False): """NO LONGER SUPPORTED: use read_environment from easybuild.tools.environment instead""" _log.nosupport( "read_environment has been moved to easybuild.tools.environment", '2.0')
:author: Kenneth Hoste (Ghent University) """ from distutils.version import LooseVersion from time import gmtime, strftime import re import time from vsc.utils import fancylogger from easybuild.tools.build_log import EasyBuildError, print_msg from easybuild.tools.config import build_option from easybuild.tools.job.backend import JobBackend from easybuild.tools.utilities import only_if_module_is_available _log = fancylogger.getLogger('gc3pie', fname=False) try: import gc3libs import gc3libs.exceptions from gc3libs import Application, Run, create_engine from gc3libs.core import Engine from gc3libs.quantity import hours as hr from gc3libs.workflow import DependentTaskCollection # inject EasyBuild logger into GC3Pie gc3libs.log = fancylogger.getLogger('gc3pie', fname=False) # make handling of log.error compatible with stdlib logging gc3libs.log.raiseError = False
def __init__(self, application, fake=False): """ModuleGenerator constructor.""" self.app = application self.log = fancylogger.getLogger(self.__class__.__name__, fname=False) self.fake_mod_path = tempfile.mkdtemp()
def classless_function(): logger = fancylogger.getLogger(fname=True, clsname=True) logger.warn("from classless_function")
:author: Stijn De Weirdt (Ghent University) :author: Dries Verdegem (Ghent University) :author: Kenneth Hoste (Ghent University) :author: Pieter De Baets (Ghent University) :author: Jens Timmerman (Ghent University) :author: Toon Willems (Ghent University) :author: Ward Poelmans (Ghent University) :author: Fotis Georgatos (Uni.Lu, NTUA) """ from vsc.utils import fancylogger from vsc.utils.missing import get_subclasses from easybuild.tools.build_log import EasyBuildError from easybuild.tools.utilities import import_available_modules _log = fancylogger.getLogger('repository', fname=False) class Repository(object): """ Interface for repositories """ DESCRIPTION = None USABLE = True # can the Repository be used? def __init__(self, repo_path, subdir=''): """ Initialize a repository. self.repo and self.subdir will be set. self.wc will be set to None.
def __init__(self, requested_users): """ Generate data base with user account data requested_users: (iterable) list of usernames to include in the data base """ self.log = fancylogger.getLogger(name=self.__class__.__name__) # Set number of procs for parallel processing from configuration file try: self.max_procs = MainConf.get_digit('nodegroups', 'max_procs', fallback=None, mandatory=False) except (KeyError, ValueError) as err: error_exit(self.log, err) else: self.log.debug("Maximum number of processor set to %s", self.max_procs) # Check requested list of users try: self.users = list(requested_users) except TypeError as err: errmsg = f"Cannot generate user data base from non-iterable user list: {requested_users}" error_exit(self.log, errmsg) # Get token from configuration file to access VSC account page TokenConfig = ConfigFile() try: vsc_token_file = MainConf.get('userdb', 'vsc_token_file', fallback='vsc-access.ini', mandatory=False) self.vsc_token = TokenConfig.load(vsc_token_file).get( 'MAIN', 'access_token') except KeyError as err: error_exit(self.log, err) # Load user data base from local cache self.cache = self.load_db_cache() # Update list of users with their records in the cache for n, user in enumerate(self.users): if user in self.cache.contents['db']: self.users[n] = (user, self.cache.contents['db'][user]) else: self.users[n] = (user, None) # Retrieve account data of requested users self.log.info(f"Retrieving {len(self.users)} user account records...") requested_records = parallel_exec( get_updated_record, # worker function f"User account retrieval", # label prefixing log messages self.users, # stack of items to process self.cache.contents[ 'valid_days'], # record_validity: forwarded to worker function self.vsc_token, # vsc_token: forwarded to worker function procs=self.max_procs, logger=self.log, ) # Generate dict of user accounts and update cache self.records = dict() for user_record in requested_records: self.records.update(user_record) self.cache.contents['db'].update(user_record) # Save local cache self.cache.save_data()
def get_vsc_record(username, vsc_token, logger=None): """ Retrieve and update list of VSC users with data from VSC account page - username: (string) VSC ID or institute user of the VSC account - vsc_token: (string) access token to VSC account page - logger: (object) fancylogger object of the caller """ if logger is None: logger = fancylogger.getLogger() vsc_api_client = AccountpageClient(token=vsc_token) # Get institute login of the VSC account attached to this username if username[0:3] == 'vsc' and username[3].isdigit(): # VSC ID: query institute login to VSC account page logger.debug(f"[{username}] user treated as VSC ID") try: vsc_account = vsc_api_client.account[username].person.get()[1] except HTTPError as err: if err.code == 404: error_exit( logger, f"[{username}] VSC ID not found in VSC account page") else: error_exit(logger, f"[{username}] {err}") except (TimeoutError, URLError) as err: error_exit( logger, f"[{username}] connection to VSC account page timed out") else: vsc_login = { 'username': vsc_account['institute_login'], 'site': vsc_account['institute']['name'] } logger.debug( f"[{username}] VSC ID belongs to VSC account '{vsc_login['username']}'" ) else: # Others: assume NetID from Brussels logger.debug(f"[{username}] user treated as NetID") vsc_login = {'username': username, 'site': BRUSSEL} # Retrieve user data from VSC account page try: vsc_account = vsc_api_client.account.institute[vsc_login['site']].id[ vsc_login['username']].get()[1] except HTTPError as err: if err.code == 404: logger.debug( f"[{username}] with VSC account '{vsc_login['username']}' not found" ) return None else: error_exit(logger, f"[{username}] {err}") except (TimeoutError, URLError) as err: error_exit(logger, f"[{username}] connection to VSC account page timed out") else: logger.debug( f"[{username}] user account record retrieved from VSC account '{vsc_login['username']}'" ) user_record = { # only use first entry of research field 'field': vsc_account['research_field'][0], 'site': INSTITUTE_LONGNAME[vsc_account['person']['institute']['name']], 'updated': date.today().isoformat(), } return user_record
import re import sys from vsc.utils import fancylogger from vsc.utils.missing import get_subclasses, nub import easybuild.tools.toolchain from easybuild.tools.build_log import EasyBuildError from easybuild.tools.toolchain.toolchain import Toolchain from easybuild.tools.utilities import import_available_modules TC_CONST_PREFIX = 'TC_CONSTANT_' _initial_toolchain_instances = {} _log = fancylogger.getLogger("toolchain.utilities") def search_toolchain(name): """ Obtain a Toolchain instance for the toolchain with specified name, next to a list of available toolchains. :param name: toolchain name :return: Toolchain instance (or None), found_toolchains """ package = easybuild.tools.toolchain check_attr_name = '%s_PROCESSED' % TC_CONST_PREFIX if not hasattr(package, check_attr_name) or not getattr(package, check_attr_name): # import all available toolchains, so we know about them tc_modules = import_available_modules('easybuild.toolchains')
- MonoidDict: dictionary that combines values upon insertiong according to the given monoid - RUDict: dictionary that allows recursively updating its values (if they are dicts too) with a new RUDict - shell_quote / shell_unquote : convenience functions to quote / unquote strings in shell context @author: Andy Georges (Ghent University) @author: Stijn De Weirdt (Ghent University) """ import shlex import subprocess import time from vsc.utils import fancylogger from vsc.utils.frozendict import FrozenDict _log = fancylogger.getLogger('vsc.utils.missing') def partial(func, *args, **keywords): """ Return a new partial object which when called will behave like func called with the positional arguments args and keyword arguments keywords. If more arguments are supplied to the call, they are appended to args. If additional keyword arguments are supplied, they extend and override keywords. new in python 2.5, from https://docs.python.org/2/library/functools.html#functools.partial """ def newfunc(*fargs, **fkeywords): newkeywords = keywords.copy() newkeywords.update(fkeywords) return func(*(args + fargs), **newkeywords) newfunc.func = func
def test_fancyrecord(self): """ Test fancyrecord usage """ logger = fancylogger.getLogger() self.assertEqual(logger.fancyrecord, True) logger = fancylogger.getLogger(fancyrecord=True) self.assertEqual(logger.fancyrecord, True) logger = fancylogger.getLogger(fancyrecord=False) self.assertEqual(logger.fancyrecord, False) logger = fancylogger.getLogger('myname') self.assertEqual(logger.fancyrecord, False) logger = fancylogger.getLogger('myname', fancyrecord=True) self.assertEqual(logger.fancyrecord, True) orig = fancylogger.FANCYLOG_FANCYRECORD fancylogger.FANCYLOG_FANCYRECORD = False logger = fancylogger.getLogger() self.assertEqual(logger.fancyrecord, False) logger = fancylogger.getLogger('myname', fancyrecord=True) self.assertEqual(logger.fancyrecord, True) fancylogger.FANCYLOG_FANCYRECORD = True logger = fancylogger.getLogger() self.assertEqual(logger.fancyrecord, True) logger = fancylogger.getLogger('myname') self.assertEqual(logger.fancyrecord, True) logger = fancylogger.getLogger('myname', fancyrecord=False) self.assertEqual(logger.fancyrecord, False) logger = fancylogger.getLogger('myname', fancyrecord='yes') self.assertEqual(logger.fancyrecord, True) logger = fancylogger.getLogger('myname', fancyrecord=0) self.assertEqual(logger.fancyrecord, False) fancylogger.FANCYLOG_FANCYRECORD = orig
"""Module to handle rst operations.""" import re from vsc.utils import fancylogger from vsc.utils.run import asyncloop from panhandler import rst_from_pan logger = fancylogger.getLogger() MAILREGEX = re.compile( ("([a-zA-Z0-9!#$%&'*+\/=?^_`{|}~-]+(?:\.[a-zA-Z0-9!#$%&'*+\/=?^_`" "{|}~-]+)*(@|\sat\s)(?:[a-zA-Z0-9](?:[a-zA-Z0-9-]*[a-zA-Z0-9])?(\.|" "\sdot\s))+[a-zA-Z0-9](?:[a-zA-Z0-9-]*[a-zA-Z0-9])?)")) PATHREGEX = re.compile(r'(\s+)((?:/[\w{}]+)+\.?\w*)(\s*)') EXAMPLEMAILS = ["example", "username", "system.admin"] def generate_rst(sources): """Generate rst.""" logger.info("Generating rst files.") rstlist = {} for title, source in sources.iteritems(): logger.debug("Parsing %s." % source) rst = None if source.endswith(".pan"): rst = rst_from_pan(source, title) else: rst = rst_from_perl(source, title)
import os import re import sys import tempfile from distutils.version import LooseVersion from textwrap import wrap from vsc.utils import fancylogger from vsc.utils.missing import get_subclasses from easybuild.tools.build_log import EasyBuildError from easybuild.tools.config import build_option, get_module_syntax, install_path from easybuild.tools.filetools import convert_name, mkdir, read_file, remove_file, resolve_path, symlink, write_file from easybuild.tools.modules import ROOT_ENV_VAR_NAME_PREFIX, modules_tool from easybuild.tools.utilities import quote_str _log = fancylogger.getLogger('module_generator', fname=False) def avail_module_generators(): """ Return all known module syntaxes. """ return dict([(k.SYNTAX, k) for k in get_subclasses(ModuleGenerator)]) def module_generator(app, fake=False): """ Return ModuleGenerator instance that matches the selected module file syntax to be used """ module_syntax = get_module_syntax() available_mod_gens = avail_module_generators()
def somefunction(self): logger = fancylogger.getLogger(fname=True, clsname=True) logger.warn('we are logging something here')
""" import copy import glob import os import random import string import tempfile import time from vsc.utils import fancylogger from vsc.utils.missing import FrozenDictKnownKeys from vsc.utils.patterns import Singleton from easybuild.tools.build_log import EasyBuildError from easybuild.tools.module_naming_scheme import GENERAL_CLASS _log = fancylogger.getLogger('config', fname=False) ERROR = 'error' IGNORE = 'ignore' PURGE = 'purge' UNLOAD = 'unload' UNSET = 'unset' WARN = 'warn' PKG_TOOL_FPM = 'fpm' PKG_TYPE_RPM = 'rpm' DEFAULT_JOB_BACKEND = 'GC3Pie' DEFAULT_LOGFILE_FORMAT = ( "easybuild", "easybuild-%(name)s-%(version)s-%(date)s.%(time)s.log") DEFAULT_MAX_FAIL_RATIO_PERMS = 0.5
def __init__(self, *args, **kwargs): super(Variables, self).__init__(*args, **kwargs) self.log = fancylogger.getLogger(self.__class__.__name__, fname=False)
""" from __future__ import print_function import operator import os import pwd import re import stat import sys import time from vsc.utils.cache import FileCache from vsc.utils.fancylogger import getLogger from vsc.utils.py2vs3 import is_string, FileNotFoundErrorExc log = getLogger(__name__) NAGIOS_CACHE_DIR = '/var/cache' NAGIOS_CACHE_FILENAME_TEMPLATE = '%s.nagios.json.gz' NAGIOS_OK = 'OK' NAGIOS_WARNING = 'WARNING' NAGIOS_CRITICAL = 'CRITICAL' NAGIOS_UNKNOWN = 'UNKNOWN' NAGIOS_EXIT_OK = (0, NAGIOS_OK) NAGIOS_EXIT_WARNING = (1, NAGIOS_WARNING) NAGIOS_EXIT_CRITICAL = (2, NAGIOS_CRITICAL) NAGIOS_EXIT_UNKNOWN = (3, NAGIOS_UNKNOWN) NAGIOS_MAX_MESSAGE_LENGTH = 8192
# along with EasyBuild. If not, see <http://www.gnu.org/licenses/>. # # """ Module that contains a set of classes and function to generate variables to be used e.g., in compiling or linking :author: Stijn De Weirdt (Ghent University) :author: Kenneth Hoste (Ghent University) """ import copy import os from vsc.utils import fancylogger from easybuild.tools.build_log import EasyBuildError _log = fancylogger.getLogger('variables', fname=False) def get_class(name, default_class, map_class=None): """Return class based on default map_class if key == str -> value = class else: key = class -> list of strings """ if map_class is None: map_class = {} klass = default_class if name is not None: try: klass = map_class[name]
:author: Kenneth Hoste (Ghent University) :author: Pieter De Baets (Ghent University) :author: Jens Timmerman (Ghent University) :author: Fotis Georgatos (Uni.Lu, NTUA) """ import os import string from vsc.utils import fancylogger from vsc.utils.missing import get_subclasses from easybuild.tools import module_naming_scheme from easybuild.tools.module_naming_scheme import ModuleNamingScheme from easybuild.tools.toolchain import DUMMY_TOOLCHAIN_NAME from easybuild.tools.utilities import import_available_modules _log = fancylogger.getLogger('module_naming_scheme.utilities', fname=False) def det_full_ec_version(ec): """ Determine exact install version, based on supplied easyconfig. e.g. 1.2.3-goalf-1.1.0-no-OFED or 1.2.3 (for dummy toolchains) """ ecver = None toolchain = ec.get('toolchain', {'name': DUMMY_TOOLCHAIN_NAME}) # determine main install version based on toolchain if toolchain['name'] == DUMMY_TOOLCHAIN_NAME: ecver = ec['version'] else:
# # """ Set of fucntions to help with jenkins setup @author: Kenneth Hoste (Ghent University) """ import glob import os import xml.dom.minidom as xml from datetime import datetime from vsc.utils import fancylogger from easybuild.tools.version import FRAMEWORK_VERSION, EASYBLOCKS_VERSION _log = fancylogger.getLogger('jenkins', fname=False) def write_to_xml(succes, failed, filename): """ Create xml output, using minimal output required according to http://stackoverflow.com/questions/4922867/junit-xml-format-specification-that-hudson-supports """ dom = xml.getDOMImplementation() root = dom.createDocument(None, "testsuite", None) def create_testcase(name): el = root.createElement("testcase") el.setAttribute("name", name) return el
# along with EasyBuild. If not, see <http://www.gnu.org/licenses/>. # """ Easyconfig constants module that provides all constants that can be used within an Easyconfig file. @author: Stijn De Weirdt (Ghent University) @author: Kenneth Hoste (Ghent University) """ import platform from vsc.utils import fancylogger from easybuild.tools.systemtools import get_shared_lib_ext, get_os_name, get_os_type, get_os_version _log = fancylogger.getLogger('easyconfig.constants', fname=False) # constants that can be used in easyconfig EASYCONFIG_CONSTANTS = { 'SYS_PYTHON_VERSION': (platform.python_version(), "System Python version (platform.python_version())"), 'OS_TYPE': (get_os_type(), "System type (e.g. 'Linux' or 'Darwin')"), 'OS_NAME': (get_os_name(), "System name (e.g. 'fedora' or 'RHEL')"), 'OS_VERSION': (get_os_version(), "System version"), } def constant_documentation(): """Generate the easyconfig constant documentation""" indent_l0 = " " * 2 indent_l1 = indent_l0 + " " * 2 doc = []
def setUp(self): """Set up testcase.""" super(EnhancedTestCase, self).setUp() # make sure option parser doesn't pick up any cmdline arguments/options while len(sys.argv) > 1: sys.argv.pop() # keep track of log handlers log = fancylogger.getLogger(fname=False) self.orig_log_handlers = log.handlers[:] log.info("setting up test %s" % self.id()) self.orig_tmpdir = tempfile.gettempdir() # use a subdirectory for this test (which we can clean up easily after the test completes) self.test_prefix = set_tmpdir() self.log = fancylogger.getLogger(self.__class__.__name__, fname=False) fd, self.logfile = tempfile.mkstemp(suffix='.log', prefix='eb-test-') os.close(fd) self.cwd = os.getcwd() # keep track of original environment to restore self.orig_environ = copy.deepcopy(os.environ) # keep track of original environment/Python search path to restore self.orig_sys_path = sys.path[:] testdir = os.path.dirname(os.path.abspath(__file__)) self.test_sourcepath = os.path.join(testdir, 'sandbox', 'sources') os.environ['EASYBUILD_SOURCEPATH'] = self.test_sourcepath os.environ['EASYBUILD_PREFIX'] = self.test_prefix self.test_buildpath = tempfile.mkdtemp() os.environ['EASYBUILD_BUILDPATH'] = self.test_buildpath self.test_installpath = tempfile.mkdtemp() os.environ['EASYBUILD_INSTALLPATH'] = self.test_installpath # make sure that the tests only pick up easyconfigs provided with the tests os.environ['EASYBUILD_ROBOT_PATHS'] = os.path.join( testdir, 'easyconfigs', 'test_ecs') # make sure no deprecated behaviour is being triggered (unless intended by the test) # trip *all* log.deprecated statements by setting deprecation version ridiculously high self.orig_current_version = eb_build_log.CURRENT_VERSION os.environ['EASYBUILD_DEPRECATED'] = '10000000' init_config() import easybuild # try to import easybuild.easyblocks(.generic) packages # it's OK if it fails here, but important to import first before fiddling with sys.path try: import easybuild.easyblocks import easybuild.easyblocks.generic except ImportError: pass # add sandbox to Python search path, update namespace packages sys.path.append(os.path.join(testdir, 'sandbox')) # workaround for bug in recent setuptools version (19.4 and newer, atleast until 20.3.1) # injecting <prefix>/easybuild is required to avoid a ValueError being thrown by fixup_namespace_packages # cfr. https://bitbucket.org/pypa/setuptools/issues/520/fixup_namespace_packages-may-trigger for path in sys.path[:]: if os.path.exists( os.path.join(path, 'easybuild', 'easyblocks', '__init__.py')): # keep track of 'easybuild' paths to inject into sys.path later sys.path.append(os.path.join(path, 'easybuild')) # required to make sure the 'easybuild' dir in the sandbox is picked up; # this relates to the other 'reload' statements below reload(easybuild) # this is strictly required to make the test modules in the sandbox available, due to declare_namespace fixup_namespace_packages(os.path.join(testdir, 'sandbox')) # remove any entries in Python search path that seem to provide easyblocks (except the sandbox) for path in sys.path[:]: if os.path.exists( os.path.join(path, 'easybuild', 'easyblocks', '__init__.py')): if not os.path.samefile(path, os.path.join(testdir, 'sandbox')): sys.path.remove(path) # hard inject location to (generic) test easyblocks into Python search path # only prepending to sys.path is not enough due to 'declare_namespace' in easybuild/easyblocks/__init__.py import easybuild.easyblocks reload(easybuild.easyblocks) test_easyblocks_path = os.path.join(testdir, 'sandbox', 'easybuild', 'easyblocks') easybuild.easyblocks.__path__.insert(0, test_easyblocks_path) import easybuild.easyblocks.generic reload(easybuild.easyblocks.generic) test_easyblocks_path = os.path.join(test_easyblocks_path, 'generic') easybuild.easyblocks.generic.__path__.insert(0, test_easyblocks_path) # save values of $PATH & $PYTHONPATH, so they can be restored later # this is important in case EasyBuild was installed as a module, since that module may be unloaded, # for example due to changes to $MODULEPATH in case EasyBuild was installed in a module hierarchy # cfr. https://github.com/easybuilders/easybuild-framework/issues/1685 self.env_path = os.environ.get('PATH') self.env_pythonpath = os.environ.get('PYTHONPATH') self.modtool = modules_tool() self.reset_modulepath([os.path.join(testdir, 'modules')]) reset_module_caches()
def __init__(self): """initialize logger.""" self.log = fancylogger.getLogger(self.__class__.__name__, fname=False)
implemented as an easyblock @author: Kenneth Hoste (Ghent University) """ import copy import os from vsc.utils import fancylogger from easybuild.framework.easyblock import EasyBlock from easybuild.framework.easyconfig import CUSTOM from easybuild.framework.extension import Extension from easybuild.tools.build_log import EasyBuildError from easybuild.tools.filetools import apply_patch, extract_file from easybuild.tools.utilities import remove_unwanted_chars _log = fancylogger.getLogger('extensioneasyblock', fname=False) class ExtensionEasyBlock(EasyBlock, Extension): """ Install an extension as a separate module, or as an extension. Deriving classes should implement the following functions: * required EasyBlock functions: - configure_step - build_step - install_step * required Extension functions - run """ @staticmethod
# # """ Easyconfig module that contains the default EasyConfig configuration parameters. :author: Stijn De Weirdt (Ghent University) :author: Dries Verdegem (Ghent University) :author: Kenneth Hoste (Ghent University) :author: Pieter De Baets (Ghent University) :author: Jens Timmerman (Ghent University) :author: Toon Willems (Ghent University) """ from vsc.utils import fancylogger from easybuild.tools.build_log import EasyBuildError _log = fancylogger.getLogger('easyconfig.default', fname=False) # we use a tuple here so we can sort them based on the numbers ALL_CATEGORIES = { 'HIDDEN': (-1, 'hidden'), 'MANDATORY': (0, 'mandatory'), 'CUSTOM': (1, 'easyblock-specific'), 'TOOLCHAIN': (2, 'toolchain'), 'BUILD': (3, 'build'), 'FILEMANAGEMENT': (4, 'file-management'), 'DEPENDENCIES': (5, 'dependencies'), 'LICENSE': (6, 'license'), 'EXTENSIONS': (7, 'extensions'), 'MODULES': (8, 'modules'), 'OTHER': (9, 'other'), }
@author: Toon Willems (Ghent University) @author: Ward Poelmans (Ghent University) """ import os import re import signal import subprocess import tempfile import time from vsc.utils import fancylogger from easybuild.tools.asyncprocess import PIPE, STDOUT, Popen, recv_some, send_all import easybuild.tools.build_log # this import is required to obtain a correct (EasyBuild) logger! _log = fancylogger.getLogger('run', fname=False) errors_found_in_log = 0 # constants for strictness levels IGNORE = 'ignore' WARN = 'warn' ERROR = 'error' # default strictness level strictness = WARN def adjust_cmd(func): """Make adjustments to given command, if required.""" def inner(cmd, *args, **kwargs):