def test_parentinfo(self):
"""Test the collection of parentinfo"""
log_fr = fancylogger.getLogger(fname=False) # rootfancylogger
pi_fr = log_fr._get_parent_info()
self.assertEqual(len(pi_fr), 2)
log_l1 = fancylogger.getLogger("level1", fname=False)
# fname=False is required to have the naming similar for child relations
pi_l1 = log_l1._get_parent_info()
self.assertEqual(len(pi_l1), 3)
py_v_27 = sys.version_info >= (2, 7, 0)
if py_v_27:
log_l2a = log_l1.getChild("level2a")
pi_l2a = log_l2a._get_parent_info()
self.assertEqual(len(pi_l2a), 4)
# this should be identical to getChild
log_l2b = fancylogger.getLogger("level1.level2b", fname=False)
# fname=False is required to have the name similar
# cutoff last letter (a vs b)
if py_v_27:
self.assertEqual(log_l2a.name[:-1], log_l2b.name[:-1])
pi_l2b = log_l2b._get_parent_info()
# yes, this broken on several levels (incl in logging itself)
# adding '.' in the name does not automatically create the parent/child relations
# if the parent with the name exists, this works
self.assertEqual(len(pi_l2b), 4)
log_l2c = fancylogger.getLogger("level1a.level2c", fname=False)
pi_l2c = log_l2c._get_parent_info()
self.assertEqual(len(pi_l2c), 3) # level1a as parent does not exist
def setUp(self):
""" dynamically replace Modules class with MockModule """
# replace Modules class with something we have control over
config.modules_tool = mock_module
main.modules_tool = mock_module
self.log = fancylogger.getLogger("RobotTest", fname=False)
# redefine the main log when calling the main functions directly
main._log = fancylogger.getLogger("main", fname=False)
self.cwd = os.getcwd()
self.base_easyconfig_dir = find_full_path(os.path.join("test", "framework", "easyconfigs"))
self.assertTrue(self.base_easyconfig_dir)
def test_deprecated(self):
"""Test deprecated log function."""
# truncate the logfile
open(self.logfn, "w")
# log message
logger = fancylogger.getLogger("deprecated_test")
max_ver = "1.0"
# test whether deprecation works
msgre_tpl_error = r"DEPRECATED\s*\(since v%s\).*%s" % (max_ver, MSG)
self.assertErrorRegex(Exception, msgre_tpl_error, logger.deprecated, MSG, "1.1", max_ver)
# test whether deprecated warning works
logger.deprecated(MSG, "0.9", max_ver)
msgre_tpl_warning = r"WARNING.*DEPRECATED\s*\(since v%s\).*%s" % (max_ver, MSG)
msgre_warning = re.compile(msgre_tpl_warning)
txt = open(self.logfn, "r").read()
self.assertTrue(msgre_warning.search(txt))
# test handling of non-UTF8 chars
msg = MSG + "\x81"
msgre_tpl_error = r"DEPRECATED\s*\(since v%s\).*%s" % (max_ver, msg)
self.assertErrorRegex(Exception, msgre_tpl_error, logger.deprecated, msg, "1.1", max_ver)
logger.deprecated(msg, "0.9", max_ver)
txt = open(self.logfn, "r").read()
self.assertTrue(msgre_warning.search(txt))
def __init__(self, modulePath=None):
"""
Create a Modules object
@param modulePath: A list of paths where the modules can be located
@type modulePath: list
"""
self.log = fancylogger.getLogger(self.__class__.__name__, fname=False)
# make sure we don't have the same path twice
if modulePath:
self.modulePath = set(modulePath)
else:
self.modulePath = None
self.modules = []
self.check_module_path()
# make sure lmod is available somewhere
eclm = subprocess.call(["which", "lmod"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
# make sure environment-modules is installed
ecem = subprocess.call(["which", "modulecmd"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
if ecem and eclm:
msg = "Could not find the modulecmd command, environment-modules is not installed?\n"
msg += "Exit code of 'which modulecmd': %d" % ecem
self.log.error(msg)
raise EasyBuildError(msg)
elif ecem:
self.modulecmd = "lmod"
os.environ['LMOD_EXPERT'] = '1'
elif eclm:
self.modulecmd = "modulecmd"
def __init__(self, options=None, **kwargs):
if not hasattr(self, 'log'):
self.log = getLogger(self.__class__.__name__)
if not hasattr(self, 'options'):
self.options = options
self.nodes = None
self.nrnodes = None
self.uniquenodes = None
self.nruniquenodes = None
self.mpinodes = None
self.mpinrnodes = None
self.mpitotalppn = None
self.id = None
self.foundppn = None
self.ppn = None
self.totalppn = None
self.cpus = []
# # collect data
self.get_id()
self.cores_on_this_node()
self.which_cpus()
self.get_node_list()
self.get_unique_nodes()
self.set_ppn()
super(Sched, self).__init__(**kwargs)
def __init__(self, cmd=None, **kwargs):
if kwargs.pop('disable_log', None):
self.log = DummyFunction() ## No logging
if not hasattr(self, 'log'):
self.log = getLogger(self._get_log_name())
self.cmd = cmd ## actual command
self.input = None
self.startpath = None
self._cwd_before_startpath = None
self._process_module = None
self._process = None
self.readsize = 1024 ## number of bytes to read blocking
self._shellcmd = None
self._popen_named_args = None
self._process_exitcode = None
self._process_output = None
self._post_exitcode_log_failure = self.log.error
super(Run, self).__init__(**kwargs)
def __init__(self, filename, retain_old=False):
"""Initializer.
Checks if the file can be accessed and load the data therein if any. If the file does not yet exist, start
with an empty shelf. This ensures that old data is readily available when the FileCache instance is created.
The file is closed after reading the data.
@type filename: string
@param filename: (absolute) path to the cache file.
"""
self.log = fancylogger.getLogger(self.__class__.__name__)
self.filename = filename
self.retain_old = retain_old
try:
f = open(self.filename, 'rb')
try:
self.shelf = pickle.load(f)
except:
self.log.raiseException("Could not load pickle data from %s" % (self.filename))
finally:
f.close()
except (OSError, IOError), err:
self.log.error("Could not access the file cache at %s [%s]" % (self.filename, err))
self.shelf = {}
def __init__(self, initcomm=True, log=None):
self.log = log
if self.log is None:
self.log = fancylogger.getLogger(name=self.__class__.__name__, fname=False)
self.comm = None
self.size = -1
self.rank = -1
self.masterrank = MASTERRANK
self.barriercounter = 0
self.stopwithbarrier = True
self.wait_iter_sleep = 60 # run through all active work, then wait wait_iter_sleep seconds
self.allnodes = None # Node info per rank
self.topocomm = None
self.tempcomm = []
self.active_work = []
self.dists = None
self.thisnode = None
if initcomm:
self.log.debug(
"Going to initialise the __init__ default communicators")
self.init_comm()
else:
self.log.debug("No communicators initialised in __init__")
def __init__(self, exe, args):
"""Initialisation arguments determine executable to run and arguments of said executable"""
self.log = fancylogger.getLogger(self.__class__.__name__)
self.exe = exe
self.args = args
self.maxprocs = 2
self.commself = None
self.selfsize = None
self.selfrank = None
self.commclone = None
self.clonesize = None
self.clonerank = None
self.comm = None
self.size = None
self.rank = None
self.group = None
self.groupsize = None
self.grouprank = None
self.stopwithbarrier = True
self.workinitbarrier = True
self.work = None
self.dowork = None ## no work to be done is default
def __init__(self, ranks):
self.log = fancylogger.getLogger(self.__class__.__name__)
MpiService.__init__(self, initcomm=False, log=self.log)
self.allranks = ranks
self.commands = {} ## dict with command : list of ranks
def __init__(self, script, name, env_vars=None, resources={}, conn=None, ppn=None):
"""
create a new Job to be submitted to PBS
env_vars is a dictionary with key-value pairs of environment variables that should be passed on to the job
resources is a dictionary with optional keys: ['hours', 'cores'] both of these should be integer values.
hours can be 1 - MAX_WALLTIME, cores depends on which cluster it is being run.
"""
self.clean_conn = True
self.log = fancylogger.getLogger(self.__class__.__name__, fname=False)
self.script = script
if env_vars:
self.env_vars = env_vars.copy()
else:
self.env_vars = {}
self.name = name
if pbs_import_failed:
self.log.error(pbs_import_failed)
try:
self.pbs_server = pbs.pbs_default()
if conn:
self.pbsconn = conn
self.clean_conn = False
else:
self.pbsconn = pbs.pbs_connect(self.pbs_server)
except Exception, err:
self.log.error("Failed to connect to the default pbs server: %s" % err)
def __init__(self, mod_paths=None):
"""
Create a ModulesTool object
@param mod_paths: A list of paths where the modules can be located
@type mod_paths: list
"""
self.log = fancylogger.getLogger(self.__class__.__name__, fname=False)
# make sure we don't have the same path twice
if mod_paths:
self.mod_paths = nub(mod_paths)
else:
self.mod_paths = None
# DEPRECATED!
self._modules = []
self.check_module_path()
# actual module command (i.e., not the 'module' wrapper function, but the binary)
self.cmd = None
# shell that should be used to run module command (specified above) in (if any)
self.shell = None
# version of modules tool
self.version = None
# terse command line option
self.add_terse_opt_fn = lambda x: x.insert(0, '--terse')
def __init__(self, filename, retain_old=False):
"""Initializer.
Checks if the file can be accessed and load the data therein if any. If the file does not yet exist, start
with an empty shelf. This ensures that old data is readily available when the FileCache instance is created.
The file is closed after reading the data.
@type filename: string
@param filename: (absolute) path to the cache file.
"""
self.log = fancylogger.getLogger(self.__class__.__name__)
self.filename = filename
self.retain_old = retain_old
try:
f = open(self.filename, 'rb')
try:
# FIXME: This double block is due to a workaround for Python 2.4
# see http://stackoverflow.com/questions/820778/syntaxerror-in-finally-django
try:
self.shelf = pickle.load(f)
except:
self.log.raiseException("Could not load pickle data from %s" % (self.filename))
finally:
f.close()
except (OSError, IOError), err:
self.log.warning("Could not access the file cache at %s [%s]" % (self.filename, err))
self.shelf = {}
def setUp(self):
""" create temporary easyconfig file """
self.log = fancylogger.getLogger("EasyConfigTest", fname=False)
self.cwd = os.getcwd()
self.all_stops = [x[0] for x in EasyBlock.get_steps()]
if os.path.exists(self.eb_file):
os.remove(self.eb_file)
config.variables['source_path'] = os.path.join(os.path.dirname(__file__), 'easyconfigs')
def __init__(self):
self.url = None
self.password = None
self.connection_dn = None
self.validation_method = None
self.check_server_certificate = False
self.log = fancylogger.getLogger(self.__class__.__name__)
def get_log(name=None):
"""
Generate logger object
"""
# fname is always get_log, useless
log = fancylogger.getLogger(name, fname=False)
log.info("Logger started for %s." % name)
return log
def request(self, method, url, body, headers):
if self.username:
headers['Authorization'] = self.auth_header
fancylogger.getLogger().debug('cli request: %s, %s, %s %s', method, url, body, headers)
#TODO: Context manager
conn = self.get_connection()
conn.request(method, url, body, headers)
response = conn.getresponse()
status = response.status
body = response.read()
try:
pybody = json.loads(body)
except ValueError:
pybody = body
fancylogger.getLogger().debug('reponse len: %s ', len(pybody))
conn.close()
return status, pybody
def __init__(self, configuration):
"""Initialisation. Not done lazily.
@type configuration: vsc.ldap.utils.Configuration subclass instance, implementing the actual functions to request
information.
"""
self.log = fancylogger.getLogger(name=self.__class__.__name__)
self.configuration = configuration
self.ldap_connection = None
def __init__(self, obj):
"""Support the conversion of obj to something"""
self.__dict__['log'] = fancylogger.getLogger(self.__class__.__name__, fname=False)
self.__dict__['data'] = None
if isinstance(obj, basestring):
self.data = self._from_string(obj)
else:
self.log.error('unsupported type %s for %s: %s' % (type(obj), self.__class__.__name__, obj))
super(Convert, self).__init__(self.data)
def setUp(self):
"""Set up everything for a unit test."""
# initialize configuration so config.get_modules_tool function works
eb_go = eboptions.parse_options()
config.init(eb_go.options, eb_go.get_options_by_section('config'))
# replace Modules class with something we have control over
config.modules_tool = mock_module
main.modules_tool = mock_module
self.log = fancylogger.getLogger("RobotTest", fname=False)
# redefine the main log when calling the main functions directly
main._log = fancylogger.getLogger("main", fname=False)
self.cwd = os.getcwd()
self.base_easyconfig_dir = find_full_path(os.path.join("test", "framework", "easyconfigs"))
self.assertTrue(self.base_easyconfig_dir)
def __init__(self, commands=None):
"""
constructor
"""
self.log = fancylogger.getLogger(self.__class__.__name__)
if commands:
self.commands = commands # queue commands here, these will be run when doIt is called
else:
self.commands = []
def __init__(self, command=None, timeout=get_config("COMMAND_TIMEOUT"), host='localhost'):
'''
Constructor
command is a string representing the command to be run
'''
self.log = fancylogger.getLogger(self.__class__.__name__)
self.command = command
self.host = host
self.timeout = float(timeout)
def __init__(self, command=None, timeout=COMMAND_TIMEOUT):
'''
Constructor
command is a string representing the command to be run
'''
self.log = fancylogger.getLogger(self.__class__.__name__, fname=False)
self.command = command
self.timeout = timeout
self.fake_pty = False
def __init__(self, options):
self.log = fancylogger.getLogger(self.__class__.__name__, fname=False)
self.vars = {
'cwd': None,
'jobid': None,
}
self.jobid = None
self.job_filter = None
def __init__(self):
"""Initialise the EasyConfigFormat class"""
self.log = fancylogger.getLogger(self.__class__.__name__, fname=False)
if not len(self.VERSION) == len(FORMAT_VERSION_TEMPLATE.split('.')):
self.log.error('Invalid version number %s (incorrect length)' % self.VERSION)
self.rawtext = None # text version of the easyconfig
self.header = None # easyconfig header (e.g., format version, license, ...)
self.docstring = None # easyconfig docstring (e.g., author, maintainer, ...)
def __init__(self, nodelist, imms=False):
'''
Constructor
nodelist is a list of hostnames for nodes to perform actions on
if imms=True it will also do the same action on the imm nodes of the given nodelist nodes
'''
Worker.__init__(self)
self.nodelist = nodelist
self.log = fancylogger.getLogger(self.__class__.__name__)
self.imms = imms
def setUp(self):
""" create temporary easyconfig file """
self.log = fancylogger.getLogger("EasyConfigTest", fname=False)
if self.contents is not None:
fd, self.eb_file = tempfile.mkstemp(prefix='easyconfig_test_file_', suffix='.eb')
os.close(fd)
write_file(self.eb_file, self.contents)
self.cwd = os.getcwd()
self.all_stops = [x[0] for x in EasyBlock.get_steps()]
def __init__(self, *args, **kwargs):
"""
- disable_log : boolean use dummy logger
"""
disable_log = kwargs.pop('disable_log', False)
if disable_log:
log = DummyFunction()
else:
log = getLogger(self.__class__.__name__)
self.log = log
def __init__(self, repo_path, subdir=''):
"""
Initialize a repository. self.repo and self.subdir will be set.
self.wc will be set to None.
Then, setupRepo and createWorkingCopy will be called (in that order)
"""
self.log = fancylogger.getLogger(self.__class__.__name__, fname=False)
self.subdir = subdir
self.repo = repo_path
self.wc = None
self.initialized = False
def __init__(self, object_classes=[]):
"""Initialisation.
Note that the LdapQuery singleton should have been instantiated elsewhere.
"""
self.ldap_query = LdapQuery(None)
self.ldap_info = None
self.object_classes = object_classes # LDAP object class name for which the schema will be checked
self.log = fancylogger.getLogger(self.__class__.__name__)
@author: Toon Willems (Ghent University)
"""
import getpass
import os
import socket
import tempfile
import time
from vsc import fancylogger
from vsc.utils.missing import get_subclasses
from easybuild.framework.easyconfig.easyconfig import EasyConfig
from easybuild.framework.easyconfig.tools import stats_to_str
from easybuild.tools.filetools import rmtree2, read_file, write_file
from easybuild.tools.version import VERBOSE_VERSION
_log = fancylogger.getLogger('repository', fname=False)
# optional Python packages, these might be missing
# failing imports are just ignored
# a NameError should be catched where these are used
# GitPython
try:
import git
from git import GitCommandError
HAVE_GIT = True
except ImportError:
_log.debug('Failed to import git module')
HAVE_GIT = False
# PySVN
def __init__(self, *args, **kwargs):
super(StrList, self).__init__(*args, **kwargs)
self.log = fancylogger.getLogger(self.__class__.__name__, fname=False)
def __init__(self, *args, **kwargs):
super(cpu_set_t, self).__init__(*args, **kwargs)
self.log = getLogger(self.__class__.__name__)
self.cpus = None
def __init__(self):
self.log = fancylogger.getLogger(self.__class__.__name__, fname=False)
self.options_map = {} # map between options name and value
self.description = {} # short description of the options
def __init__(self):
"""Initialise the list of version operators as an empty list."""
self.log = fancylogger.getLogger(self.__class__.__name__, fname=False)
self.versops = []
self.datamap = {}
from vsc import fancylogger
from vsc.utils.missing import get_subclasses
from easybuild.framework.easyconfig.format.version import EasyVersion
# format is mandatory major.minor
FORMAT_VERSION_KEYWORD = "EASYCONFIGFORMAT"
FORMAT_VERSION_TEMPLATE = "%(major)s.%(minor)s"
FORMAT_VERSION_HEADER_TEMPLATE = "# %s %s\n" % (
FORMAT_VERSION_KEYWORD, FORMAT_VERSION_TEMPLATE) # must end in newline
FORMAT_VERSION_REGEXP = re.compile(
r'^#\s+%s\s*(?P<major>\d+)\.(?P<minor>\d+)\s*$' % FORMAT_VERSION_KEYWORD,
re.M)
FORMAT_DEFAULT_VERSION = EasyVersion('1.0')
_log = fancylogger.getLogger('easyconfig.format.format', fname=False)
def get_format_version(txt):
"""Get the easyconfig format version as EasyVersion instance."""
res = FORMAT_VERSION_REGEXP.search(txt)
format_version = None
if res is not None:
try:
maj_min = res.groupdict()
format_version = EasyVersion(FORMAT_VERSION_TEMPLATE % maj_min)
except (KeyError, TypeError), err:
_log.error("Failed to get version from match %s: %s" %
(res.groups(), err))
return format_version
# You should have received a copy of the GNU General Public License
# along with EasyBuild. If not, see <http://www.gnu.org/licenses/>.
##
"""
Module with useful functions for getting system information
@author: Jens Timmerman (Ghent University)
"""
import os
import platform
import re
from vsc import fancylogger
from easybuild.tools.filetools import read_file, run_cmd
_log = fancylogger.getLogger('systemtools', fname=False)
INTEL = 'Intel'
AMD = 'AMD'
class SystemToolsException(Exception):
"""raised when systemtools fails"""
def get_core_count():
"""Try to detect the number of virtual or physical CPUs on this system.
inspired by http://stackoverflow.com/questions/1006289/how-to-find-out-the-number-of-cpus-in-python/1006301#1006301
"""
def __init__(self, *args, **kwargs):
"""Initialize logger."""
self.log = fancylogger.getLogger(self.__class__.__name__, fname=False)
import glob
import os
import re
import sys
import tempfile
from distutils.version import LooseVersion
from vsc import fancylogger
from vsc.utils.missing import nub
from easybuild.tools.build_log import EasyBuildError
from easybuild.tools.filetools import run_cmd, read_file, write_file
from easybuild.tools.ordereddict import OrderedDict
from easybuild.tools.utilities import quote_str
from easybuild.framework.easyconfig.easyconfig import EasyConfig, det_installversion
_log = fancylogger.getLogger('easyconfig.tools', fname=False)
def ec_filename_for(path):
"""
Return a suiting file name for the easyconfig file at <path>,
as determined by its contents.
"""
ec = EasyConfig(path, validate=False)
fn = "%s-%s.eb" % (ec['name'],
det_installversion(
ec['version'], ec['toolchain']['name'],
ec['toolchain']['version'], ec['versionprefix'],
ec['versionsuffix']))
class EasyConfigTest(TestCase):
"""Baseclass for easyconfig testcases."""
# initialize configuration (required for e.g. default modules_tool setting)
eb_go = eboptions.parse_options()
config.init(eb_go.options, eb_go.get_options_by_section('config'))
build_options = {
'check_osdeps': False,
'external_modules_metadata': {},
'force': True,
'optarch': 'test',
'robot_path': get_paths_for("easyconfigs")[0],
'suffix_modules_path': GENERAL_CLASS,
'valid_module_classes': config.module_classes(),
'valid_stops': [x[0] for x in EasyBlock.get_steps()],
}
config.init_build_options(build_options=build_options)
config.set_tmpdir()
del eb_go
# mock 'exist' and 'load' methods of modules tool, which are used for 'craype' external module
modtool = modules_tool()
modtool.exist = lambda m: [True] * len(m)
modtool.load = lambda m: True
log = fancylogger.getLogger("EasyConfigTest", fname=False)
# make sure a logger is present for main
main._log = log
ordered_specs = None
parsed_easyconfigs = []
def process_all_easyconfigs(self):
"""Process all easyconfigs and resolve inter-easyconfig dependencies."""
# all available easyconfig files
easyconfigs_path = get_paths_for("easyconfigs")[0]
specs = glob.glob('%s/*/*/*.eb' % easyconfigs_path)
# parse all easyconfigs if they haven't been already
if not self.parsed_easyconfigs:
for spec in specs:
self.parsed_easyconfigs.extend(process_easyconfig(spec))
# filter out external modules
for ec in self.parsed_easyconfigs:
for dep in ec['dependencies'][:]:
if dep.get('external_module', False):
ec['dependencies'].remove(dep)
for dep in ec['unresolved_deps'][:]:
if dep.get('external_module', False):
ec['unresolved_deps'].remove(dep)
self.ordered_specs = resolve_dependencies(self.parsed_easyconfigs,
retain_all_deps=True)
def test_dep_graph(self):
"""Unit test that builds a full dependency graph."""
# pygraph dependencies required for constructing dependency graph are not available prior to Python 2.6
if LooseVersion(
sys.version) >= LooseVersion('2.6') and single_tests_ok:
# temporary file for dep graph
(hn, fn) = tempfile.mkstemp(suffix='.dot')
os.close(hn)
if self.ordered_specs is None:
self.process_all_easyconfigs()
dep_graph(fn, self.ordered_specs, silent=True)
try:
os.remove(fn)
except OSError, err:
log.error("Failed to remove %s: %s" % (fn, err))
else:
# You should have received a copy of the GNU General Public License
# along with EasyBuild. If not, see <http://www.gnu.org/licenses/>.
##
"""
Interface module to TORQUE (PBS).
@author: Stijn De Weirdt (Ghent University)
@author: Toon Willems (Ghent University)
@author: Kenneth Hoste (Ghent University)
"""
import os
import time
from vsc import fancylogger
_log = fancylogger.getLogger('pbs_job', fname=False)
pbs_import_failed = None
try:
from PBSQuery import PBSQuery
import pbs
except ImportError:
_log.debug("Failed to import pbs from pbs_python. Silently ignoring, is only a real issue with --job")
pbs_import_failed = ("PBSQuery or pbs modules not available. "
"Please make sure pbs_python is installed and usable.")
MAX_WALLTIME = 72
def connect_to_server(pbs_server=None):
"""Connect to PBS server and return connection."""
if pbs_import_failed:
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with EasyBuild. If not, see <http://www.gnu.org/licenses/>.
##
"""
Utility module for modifying os.environ
@author: Toon Willems (Ghent University)
"""
import os
from vsc import fancylogger
_log = fancylogger.getLogger('environment', fname=False)
changes = {}
def write_changes(filename):
"""
Write current changes to filename and reset environment afterwards
"""
script = None
try:
script = open(filename, 'w')
for key in changes:
script.write('export %s="%s"\n' % (key, changes[key]))
@author: Jens Timmerman (Ghent University)
"""
import os
import shutil
import tempfile
import easybuild.tools.environment as env
from easybuild.framework.easyblock import EasyBlock
from easybuild.framework.easyconfig import CUSTOM
from easybuild.tools.filetools import rmtree2, run_cmd
# required for deprecated log in static function (ie no self)
from easybuild.framework.easyconfig.licenses import License
from vsc import fancylogger
_log = fancylogger.getLogger('generic.intelbase')
class IntelBase(EasyBlock):
"""
Base class for Intel software
- no configure/make : binary release
- add license_file variable
"""
def __init__(self, *args, **kwargs):
"""Constructor, adds extra config options"""
self.license_file = None
self.home_subdir = os.path.join(os.getenv('HOME'), 'intel')
self.home_subdir_local = os.path.join(tempfile.gettempdir(),
os.getenv('USER'),
def search_toolchain(name):
"""
Find a toolchain with matching name
returns toolchain (or None), found_toolchains
"""
log = fancylogger.getLogger("search_toolchain")
# import all available toolchains, so we know about them
tc_modules = []
for path in sys.path:
for module in glob.glob(os.path.join(path, 'easybuild', 'toolchains', '*.py')):
if not module.endswith('__init__.py'):
modpath = "easybuild.toolchains.%s" % module.split(os.path.sep)[-1].split('.')[0]
log.debug("importing toolchain module %s" % modpath)
tc_modules.append(__import__(modpath, globals(), locals(), ['']))
# make sure all defined toolchain constants are available in toolchain module
package = easybuild.tools.toolchain
tc_const_prefix = 'TC_CONSTANT_'
tc_const_re = re.compile('^%s(.*)$' % tc_const_prefix)
for tc_mod in tc_modules:
# determine classes imported in this module
mod_classes = []
for elem in [getattr(tc_mod, x) for x in dir(tc_mod)]:
if hasattr(elem, '__module__'):
# exclude the toolchain class defined in that module
if not tc_mod.__file__ == sys.modules[elem.__module__].__file__:
log.debug("Adding %s to list of imported classes used for looking for constants" % elem.__name__)
mod_classes.append(elem)
# look for constants in modules of imported classes, and make them available
for mod_class_mod in [sys.modules[mod_class.__module__] for mod_class in mod_classes]:
for elem in dir(mod_class_mod):
res = tc_const_re.match(elem)
if res:
tc_const_name = res.group(1)
tc_const_value = getattr(mod_class_mod, elem)
log.debug("Found constant %s ('%s') in module %s, adding it to %s" % (tc_const_name,
tc_const_value,
mod_class_mod.__name__,
package.__name__))
if hasattr(package, tc_const_name):
cur_value = getattr(package, tc_const_name)
if not tc_const_value == cur_value:
log.error("Constant %s.%s defined as '%s', can't set it to '%s'." % (package.__name__,
tc_const_name,
cur_value,
tc_const_value
))
else:
setattr(package, tc_const_name, tc_const_value)
# obtain all subclasses of toolchain
found_tcs = nub(get_subclasses(Toolchain))
# filter found toolchain subclasses based on whether they can be used a toolchains
found_tcs = [tc for tc in found_tcs if tc._is_toolchain_for(None)]
for tc in found_tcs:
if tc._is_toolchain_for(name):
return tc, found_tcs
return None, found_tcs
@author: Kenneth Hoste (Ghent University)
@author: Stijn De Weirdt (Ghent University)
"""
import math
import os
import re
import easybuild.tools.config as config
from easybuild.framework.easyblock import get_class
from easybuild.tools.pbs_job import PbsJob, connect_to_server, disconnect_from_server, get_ppn
from easybuild.tools.config import get_repository, get_repositorypath
from easybuild.tools.filetools import read_file
from easybuild.tools.repository import init_repository
from vsc import fancylogger
_log = fancylogger.getLogger('parallelbuild', fname=False)
def build_easyconfigs_in_parallel(build_command,
easyconfigs,
output_dir,
robot_path=None):
"""
easyconfigs is a list of easyconfigs which can be built (e.g. they have no unresolved dependencies)
this function will build them in parallel by submitting jobs
returns the jobs
"""
_log.info("going to build these easyconfigs in parallel: %s", easyconfigs)
job_module_dict = {}
# dependencies have already been resolved,
def setUp(self):
"""Setup test."""
self.log = fancylogger.getLogger("EasyblocksInitTest", fname=False)
fd, self.eb_file = tempfile.mkstemp(prefix='easyblocks_init_test_',
suffix='.eb')
os.close(fd)
#
"""
SCOOP support
http://code.google.com/p/scoop/
based on 0.6.0 code
"""
import itertools
import os
import sys
from collections import namedtuple
from distutils.version import LooseVersion
from vsc.fancylogger import getLogger
from vsc.mympirun.mpi.mpi import MPI
from vsc.mympirun.exceptions import WrongPythonVersionExcpetion, InitImportException
_logger = getLogger("MYSCOOP")
# requires Python 2.6 at least (str.format)
if LooseVersion(".".join(["%s" % x
for x in sys.version_info])) < LooseVersion('2.6'):
_logger.raiseException("MYSCOOP / scoop requires python 2.6 or later",
WrongPythonVersionExcpetion)
try:
import scoop
except:
_logger.raiseException(
"MYSCOOP requires the scoop module and scoop requires (amongst others) pyzmq",
InitImportException)
from scoop.__main__ import ScoopApp
def setUp(self):
self.log = fancylogger.getLogger(self.__class__.__name__)
self.legacySetUp()
self.raiseError = False
fancylogger.FancyLogger.exception(self, newMsg, *args)
self.raiseError = True
raise EasyBuildError(newMsg)
# set format for logger
LOGGING_FORMAT = EB_MSG_PREFIX + ' %(asctime)s %(name)s %(levelname)s %(message)s'
fancylogger.setLogFormat(LOGGING_FORMAT)
# set the default LoggerClass to EasyBuildLog
fancylogger.logging.setLoggerClass(EasyBuildLog)
# you can't easily set another LoggerClass before fancylogger calls getLogger on import
_init_fancylog = fancylogger.getLogger(fname=False)
del _init_fancylog.manager.loggerDict[_init_fancylog.name]
# we need to make sure there is a handler
fancylogger.logToFile(filename=os.devnull)
# EasyBuildLog
_init_easybuildlog = fancylogger.getLogger(fname=False)
def get_log(name=None):
"""
Generate logger object
"""
# fname is always get_log, useless
log = fancylogger.getLogger(name, fname=False)
@author: Jens Timmerman (Ghent University)
@author: Toon Willems (Ghent University)
"""
import os
import random
import string
import tempfile
import time
from vsc import fancylogger
from vsc.utils.missing import nub
import easybuild.tools.build_log # this import is required to obtain a correct (EasyBuild) logger!
from easybuild.tools.environment import read_environment as _read_environment
_log = fancylogger.getLogger('config', fname=False)
# class constant to prepare migration to generaloption as only way of configuration (maybe for v2.X)
SUPPORT_OLDSTYLE = True
DEFAULT_LOGFILE_FORMAT = (
"easybuild", "easybuild-%(name)s-%(version)s-%(date)s.%(time)s.log")
DEFAULT_PATH_SUBDIRS = {
'buildpath': 'build',
'installpath': '',
'repositorypath': 'ebfiles_repo',
'sourcepath': 'sources',
'subdir_modules': 'modules',
'subdir_software': 'software',
}
def __init__(self, path, extra_options=None, validate=True, valid_module_classes=None, valid_stops=None):
"""
initialize an easyconfig.
path should be a path to a file that can be parsed
extra_options is a dict of extra variables that can be set in this specific instance
validate specifies whether validations should happen
"""
self.template_values = None
self.enable_templating = True # a boolean to control templating
self.log = fancylogger.getLogger(self.__class__.__name__, fname=False)
self.valid_module_classes = None
if valid_module_classes:
self.valid_module_classes = valid_module_classes
self.log.info("Obtained list of valid module classes: %s" % self.valid_module_classes)
else:
self.valid_module_classes = ['base', 'compiler', 'lib'] # legacy module classes
# replace the category name with the category
self._config = {}
for k, [def_val, descr, cat] in copy.deepcopy(DEFAULT_CONFIG).items():
self._config[k] = [def_val, descr, ALL_CATEGORIES[cat]]
if extra_options is None:
extra_options = {}
elif isinstance(extra_options, (list, tuple,)):
# TODO legacy behaviour. should be more strictly enforced. do we log here?
extra_options = dict(extra_options)
self._legacy_license(extra_options)
self._config.update(extra_options)
self.path = path
self.mandatory = MANDATORY_PARAMS[:]
# extend mandatory keys
for key, value in extra_options.items():
if value[2] == ALL_CATEGORIES['MANDATORY']:
self.mandatory.append(key)
# set valid stops
self.valid_stops = []
if valid_stops:
self.valid_stops = valid_stops
self.log.debug("List of valid stops obtained: %s" % self.valid_stops)
# store toolchain
self._toolchain = None
if not os.path.isfile(path):
self.log.error("EasyConfig __init__ expected a valid path")
self.validations = {
'moduleclass': self.valid_module_classes,
'stop': self.valid_stops,
}
# parse easyconfig file
self.parse(path)
# handle allowed system dependencies
self.handle_allowed_system_deps()
# perform validations
self.validation = validate
if self.validation:
self.validate()
import tempfile
import unittest
from vsc import fancylogger
from easybuild.tools.build_log import EasyBuildError
from easybuild.tools.config import set_tmpdir
import test.easyblocks.init_easyblocks as i
import test.easyblocks.module_only as m
# initialize logger for all the unit tests
fd, log_fn = tempfile.mkstemp(prefix='easybuild-easyblocks-tests-', suffix='.log')
os.close(fd)
os.remove(log_fn)
fancylogger.logToFile(log_fn)
log = fancylogger.getLogger()
log.setLevelName('DEBUG')
try:
tmpdir = set_tmpdir(raise_error=True)
except EasyBuildError, err:
sys.stderr.write("No execution rights on temporary files, specify another location via $TMPDIR: %s\n" % err)
sys.exit(1)
os.environ['EASYBUILD_TMP_LOGDIR'] = tempfile.mkdtemp(prefix='easyblocks_test_')
# call suite() for each module and then run them all
SUITE = unittest.TestSuite([x.suite() for x in [i, m]])
# uses XMLTestRunner if possible, so we can output an XML file that can be supplied to Jenkins
xml_msg = ""
def main(testing_data=(None, None)):
"""
Main function:
@arg options: a tuple: (options, paths, logger, logfile, hn) as defined in parse_options
This function will:
- read easyconfig
- build software
"""
# disallow running EasyBuild as root
if os.getuid() == 0:
sys.stderr.write("ERROR: You seem to be running EasyBuild with root privileges.\n" \
"That's not wise, so let's end this here.\n" \
"Exiting.\n")
sys.exit(1)
# steer behavior when testing main
testing = testing_data[0] is not None
args, logfile = testing_data
# initialise options
eb_go = eboptions.parse_options(args=args)
options = eb_go.options
orig_paths = eb_go.args
# initialise logging for main
if options.logtostdout:
fancylogger.logToScreen(enable=True, stdout=True)
else:
if logfile is None:
# mkstemp returns (fd,filename), fd is from os.open, not regular open!
fd, logfile = tempfile.mkstemp(suffix='.log', prefix='easybuild-')
os.close(fd)
fancylogger.logToFile(logfile)
print_msg('temporary log file in case of crash %s' % (logfile), log=None, silent=testing)
global _log
_log = fancylogger.getLogger(fname=False)
# hello world!
_log.info(this_is_easybuild())
# set strictness of filetools module
if options.strict:
filetools.strictness = options.strict
if not options.robot is None:
if options.robot:
_log.info("Using robot path: %s" % options.robot)
else:
_log.error("No robot path specified, and unable to determine easybuild-easyconfigs install path.")
# determine easybuild-easyconfigs package install path
easyconfigs_paths = get_paths_for("easyconfigs", robot_path=options.robot)
easyconfigs_pkg_full_path = None
search_path = os.getcwd()
if easyconfigs_paths:
easyconfigs_pkg_full_path = easyconfigs_paths[0]
if not options.robot:
search_path = easyconfigs_pkg_full_path
else:
search_path = options.robot
else:
_log.info("Failed to determine install path for easybuild-easyconfigs package.")
if options.robot:
easyconfigs_paths = [options.robot] + easyconfigs_paths
# initialise the easybuild configuration
config.init(options, eb_go.get_options_by_section('config'))
# search for easyconfigs
if options.search:
search_file(search_path, options.search, silent=testing)
# process software build specifications (if any), i.e.
# software name/version, toolchain name/version, extra patches, ...
(try_to_generate, software_build_specs) = process_software_build_specs(options)
paths = []
if len(orig_paths) == 0:
if software_build_specs.has_key('name'):
paths = [obtain_path(software_build_specs, easyconfigs_paths,
try_to_generate=try_to_generate, exit_on_error=not testing)]
elif not any([options.aggregate_regtest, options.search, options.regtest]):
print_error(("Please provide one or multiple easyconfig files, or use software build "
"options to make EasyBuild search for easyconfigs"),
log=_log, opt_parser=eb_go.parser, exit_on_error=not testing)
else:
# look for easyconfigs with relative paths in easybuild-easyconfigs package,
# unless they we found at the given relative paths
if easyconfigs_pkg_full_path:
# create a mapping from filename to path in easybuild-easyconfigs package install path
easyconfigs_map = {}
for (subpath, _, filenames) in os.walk(easyconfigs_pkg_full_path):
for filename in filenames:
easyconfigs_map.update({filename: os.path.join(subpath, filename)})
# try and find non-existing non-absolute eaysconfig paths in easybuild-easyconfigs package install path
for idx, orig_path in enumerate(orig_paths):
if not os.path.isabs(orig_path) and not os.path.exists(orig_path):
if orig_path in easyconfigs_map:
_log.info("Found %s in %s: %s" % (orig_path, easyconfigs_pkg_full_path,
easyconfigs_map[orig_path]))
orig_paths[idx] = easyconfigs_map[orig_path]
# indicate that specified paths do not contain generated easyconfig files
paths = [(path, False) for path in orig_paths]
_log.debug("Paths: %s" % paths)
# run regtest
if options.regtest or options.aggregate_regtest:
_log.info("Running regression test")
if paths:
regtest_ok = regtest(options, [path[0] for path in paths])
else: # fallback: easybuild-easyconfigs install path
regtest_ok = regtest(options, [easyconfigs_pkg_full_path])
if not regtest_ok:
_log.info("Regression test failed (partially)!")
sys.exit(31) # exit -> 3x1t -> 31
if any([options.search, options.regtest]):
cleanup_logfile_and_exit(logfile, testing, True)
# building a dependency graph implies force, so that all dependencies are retained
# and also skips validation of easyconfigs (e.g. checking os dependencies)
validate_easyconfigs = True
retain_all_deps = False
if options.dep_graph:
_log.info("Enabling force to generate dependency graph.")
options.force = True
validate_easyconfigs = False
retain_all_deps = True
# read easyconfig files
easyconfigs = []
for (path, generated) in paths:
path = os.path.abspath(path)
if not (os.path.exists(path)):
print_error("Can't find path %s" % path)
try:
files = find_easyconfigs(path)
for f in files:
if not generated and try_to_generate and software_build_specs:
ec_file = easyconfig.tools.tweak(f, None, software_build_specs)
else:
ec_file = f
easyconfigs.extend(process_easyconfig(ec_file, options.only_blocks,
validate=validate_easyconfigs))
except IOError, err:
_log.error("Processing easyconfigs in path %s failed: %s" % (path, err))
#
# Any modules with a name that does not match the regex constructed below, will be HIDDEN from EasyBuild
#
'available':
re.compile(
r"""
^(?!-*\s) # disallow lines starting with (empty) list of '-'s followed by a space
(?P<mod_name> # start named group for module name
[^\s\(]*[^:/] # module name must not have '(' or whitespace in it, must not end with ':' or '/'
) # end named group for module name
(?P<default>\(default\))? # optional '(default)' that's not part of module name
\s*$ # ignore whitespace at the end of the line
""", re.VERBOSE),
}
_log = fancylogger.getLogger('modules', fname=False)
class ModulesTool(object):
"""An abstract interface to a tool that deals with modules."""
def __init__(self, mod_paths=None):
"""
Create a ModulesTool object
@param mod_paths: A list of paths where the modules can be located
@type mod_paths: list
"""
self.log = fancylogger.getLogger(self.__class__.__name__, fname=False)
# make sure we don't have the same path twice
if mod_paths:
self.mod_paths = nub(mod_paths)
else:
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with EasyBuild. If not, see <http://www.gnu.org/licenses/>.
#
"""
Easyconfig licenses module that provides all licenses that can
be used within an Easyconfig file.
@author: Stijn De Weirdt (Ghent University)
"""
from vsc import fancylogger
from vsc.utils.missing import get_subclasses
_log = fancylogger.getLogger('easyconfig.licenses', fname=False)
class License(object):
"""EasyBuild easyconfig license class"""
HIDDEN = False # disable subclasses from being seen/used
NAME = None
VERSION = None
DESCRIPTION = None
DISTRIBUTE_SOURCE = False # does the license allows to (re)distribute the code
GROUP_SOURCE = True # does the license require to keep the source under dedicated group
GROUP_BINARY = True # does the license require to install the binaries under dedicated group
CLASSNAME_PREFIX = 'License_'
import os
import re
import shutil
import signal
import stat
import subprocess
import tempfile
import time
import urllib
from vsc import fancylogger
import easybuild.tools.environment as env
from easybuild.tools.asyncprocess import Popen, PIPE, STDOUT
from easybuild.tools.asyncprocess import send_all, recv_some
_log = fancylogger.getLogger('filetools', fname=False)
errorsFoundInLog = 0
# constants for strictness levels
IGNORE = 'ignore'
WARN = 'warn'
ERROR = 'error'
# default strictness level
strictness = WARN
# easyblock class prefix
EASYBLOCK_CLASS_PREFIX = 'EB_'
# character map for encoding strings
STRING_ENCODING_CHARMAP = {
#
# You should have received a copy of the GNU General Public License
# along with EasyBuild. If not, see <http://www.gnu.org/licenses/>.
# #
"""
Module that contains a set of classes and function to generate variables to be used
e.g., in compiling or linking
@author: Stijn De Weirdt (Ghent University)
@author: Kenneth Hoste (Ghent University)
"""
from vsc import fancylogger
import copy
import os
_log = fancylogger.getLogger('variables', fname=False)
def get_class(name, default_class, map_class=None):
"""Return class based on default
map_class
if key == str -> value = class
else: key = class -> list of strings
"""
if map_class is None:
map_class = {}
klass = default_class
if name is not None:
try:
klass = map_class[name]
@author: Stijn De Weirdt (Ghent University)
@author: Dries Verdegem (Ghent University)
@author: Kenneth Hoste (Ghent University)
@author: Pieter De Baets (Ghent University)
@author: Jens Timmerman (Ghent University)
@author: Fotis Georgatos (Uni.Lu)
"""
import os
import tempfile
from vsc import fancylogger
from easybuild.tools.config import install_path
from easybuild.tools.utilities import quote_str
_log = fancylogger.getLogger('moduleGenerator', fname=False)
# general module class
GENERAL_CLASS = 'all'
class ModuleGenerator(object):
"""
Class for generating module files.
"""
def __init__(self, application, fake=False):
self.app = application
self.fake = fake
self.filename = None
self.tmpdir = None
"""
Linux cpu affinity.
- Based on C{sched.h} and C{bits/sched.h},
- see man pages for C{sched_getaffinity} and C{sched_setaffinity}
- also provides a C{cpuset} class to convert between human readable cpusets and the bit version
Linux priority
- Based on sys/resources.h and bits/resources.h see man pages for
C{getpriority} and C{setpriority}
"""
import ctypes
import os
from ctypes.util import find_library
from vsc.fancylogger import getLogger, setLogLevelDebug
_logger = getLogger("affinity")
_libc_lib = find_library('c')
_libc = ctypes.cdll.LoadLibrary(_libc_lib)
#/* Type for array elements in 'cpu_set_t'. */
#typedef unsigned long int __cpu_mask;
cpu_mask_t = ctypes.c_ulong
##define __CPU_SETSIZE 1024
##define __NCPUBITS (8 * sizeof(__cpu_mask))
CPU_SETSIZE = 1024
NCPUBITS = 8 * ctypes.sizeof(cpu_mask_t)
NMASKBITS = CPU_SETSIZE / NCPUBITS
#/* Priority limits. */
# You should have received a copy of the GNU General Public License
# along with EasyBuild. If not, see <http://www.gnu.org/licenses/>.
#
"""
Easyconfig constants module that provides all constants that can
be used within an Easyconfig file.
@author: Stijn De Weirdt (Ghent University)
@author: Kenneth Hoste (Ghent University)
"""
import platform
from vsc import fancylogger
from easybuild.tools.systemtools import get_shared_lib_ext, get_os_name, get_os_type, get_os_version
_log = fancylogger.getLogger('easyconfig.constants', fname=False)
# constants that can be used in easyconfig
EASYCONFIG_CONSTANTS = [
('SYS_PYTHON_VERSION', platform.python_version(),
"System Python version (platform.python_version())"),
('OS_TYPE', get_os_type(), "System type (e.g. 'Linux' or 'Darwin')"),
('OS_NAME', get_os_name(), "System name (e.g. 'fedora' or 'RHEL')"),
('OS_VERSION', get_os_version(), "System version"),
]
def constant_documentation():
"""Generate the easyconfig constant documentation"""
indent_l0 = " " * 2
indent_l1 = indent_l0 + " " * 2
