def obtain_config_guess(download_source_path=None, search_source_paths=None):
"""
Locate or download an up-to-date config.guess
:param download_source_path: Path to download config.guess to
:param search_source_paths: Paths to search for config.guess
:return: Path to config.guess or None
"""
log = fancylogger.getLogger('obtain_config_guess')
eb_source_paths = source_paths()
if download_source_path is None:
download_source_path = eb_source_paths[0]
else:
log.deprecated(
"Specifying custom source path to download config.guess via 'download_source_path'",
'5.0')
if search_source_paths is None:
search_source_paths = eb_source_paths
else:
log.deprecated(
"Specifying custom location to search for updated config.guess via 'search_source_paths'",
'5.0')
config_guess = 'config.guess'
sourcepath_subdir = os.path.join('generic', 'eb_v%s' % EASYBLOCKS_VERSION,
'ConfigureMake')
config_guess_path = None
# check if config.guess has already been downloaded to source path
for path in search_source_paths:
cand_config_guess_path = os.path.join(path, sourcepath_subdir,
config_guess)
if os.path.isfile(cand_config_guess_path) and check_config_guess(
cand_config_guess_path):
force_download = build_option('force_download')
if force_download:
print_warning(
"Found file %s at %s, but re-downloading it anyway..." %
(config_guess, cand_config_guess_path))
else:
config_guess_path = cand_config_guess_path
log.info("Found %s at %s", config_guess, config_guess_path)
break
if not config_guess_path:
cand_config_guess_path = os.path.join(download_source_path,
sourcepath_subdir, config_guess)
config_guess_url = CONFIG_GUESS_URL_STUB + CONFIG_GUESS_COMMIT_ID
if not download_file(config_guess, config_guess_url,
cand_config_guess_path):
print_warning("Failed to download recent %s to %s",
config_guess,
cand_config_guess_path,
log=log)
elif not check_config_guess(cand_config_guess_path):
print_warning("Verification failed for file %s, not using it!",
cand_config_guess_path,
log=log)
remove_file(cand_config_guess_path)
else:
config_guess_path = cand_config_guess_path
adjust_permissions(config_guess_path,
stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH,
add=True)
log.info("Verified %s at %s, using it if required", config_guess,
config_guess_path)
return config_guess_path
import tempfile
import time
from abc import ABCMeta
from easybuild.base import fancylogger
from easybuild.base.frozendict import FrozenDictKnownKeys
from easybuild.tools.build_log import EasyBuildError
from easybuild.tools.py2vs3 import ascii_letters, create_base_metaclass, string_type
try:
import rich # noqa
HAVE_RICH = True
except ImportError:
HAVE_RICH = False
_log = fancylogger.getLogger('config', fname=False)
ERROR = 'error'
IGNORE = 'ignore'
PURGE = 'purge'
UNLOAD = 'unload'
UNSET = 'unset'
WARN = 'warn'
PKG_TOOL_FPM = 'fpm'
PKG_TYPE_RPM = 'rpm'
CONT_IMAGE_FORMAT_EXT3 = 'ext3'
CONT_IMAGE_FORMAT_SANDBOX = 'sandbox'
CONT_IMAGE_FORMAT_SIF = 'sif'
CONT_IMAGE_FORMAT_SQUASHFS = 'squashfs'
"""
import math
import os
import re
from easybuild.base import fancylogger
from easybuild.framework.easyblock import get_easyblock_instance
from easybuild.framework.easyconfig.easyconfig import ActiveMNS
from easybuild.tools.build_log import EasyBuildError
from easybuild.tools.config import build_option, get_repository, get_repositorypath
from easybuild.tools.module_naming_scheme.utilities import det_full_ec_version
from easybuild.tools.job.backend import job_backend
from easybuild.tools.repository.repository import init_repository
_log = fancylogger.getLogger('parallelbuild', fname=False)
def _to_key(dep):
"""Determine key for specified dependency."""
return ActiveMNS().det_full_module_name(dep)
def build_easyconfigs_in_parallel(build_command, easyconfigs, output_dir='easybuild-build', prepare_first=True):
"""
Build easyconfigs in parallel by submitting jobs to a batch-queuing system.
Return list of jobs submitted.
Argument `easyconfigs` is a list of easyconfigs which can be
built: e.g. they have no unresolved dependencies. This function
will build them in parallel by submitting jobs.
"""
Easyconfig module that contains the default EasyConfig configuration parameters.
:author: Stijn De Weirdt (Ghent University)
:author: Dries Verdegem (Ghent University)
:author: Kenneth Hoste (Ghent University)
:author: Pieter De Baets (Ghent University)
:author: Jens Timmerman (Ghent University)
:author: Toon Willems (Ghent University)
"""
from easybuild.base import fancylogger
from easybuild.tools.build_log import EasyBuildError
from easybuild.tools.config import MODULECLASS_BASE
_log = fancylogger.getLogger('easyconfig.default', fname=False)
# constants for different categories of easyconfig parameters
# use tuples so we can sort them based on the numbers
HIDDEN = (-1, 'hidden')
MANDATORY = (0, 'mandatory')
CUSTOM = (1, 'easyblock-specific')
TOOLCHAIN = (2, 'toolchain')
BUILD = (3, 'build')
FILEMANAGEMENT = (4, 'file-management')
DEPENDENCIES = (5, 'dependencies')
LICENSE = (6, 'license')
EXTENSIONS = (7, 'extensions')
MODULES = (8, 'modules')
OTHER = (9, 'other')
# #
"""
Module that contains a set of classes and function to generate variables to be used
e.g., in compiling or linking
:author: Stijn De Weirdt (Ghent University)
:author: Kenneth Hoste (Ghent University)
"""
import copy
import os
from easybuild.base import fancylogger
from easybuild.tools.build_log import EasyBuildError
_log = fancylogger.getLogger('variables', fname=False)
def get_class(name, default_class, map_class=None):
"""Return class based on default
map_class
if key == str -> value = class
else: key = class -> list of strings
"""
if map_class is None:
map_class = {}
klass = default_class
if name is not None:
try:
klass = map_class[name]
information obtained from provided file (easystack) with build specifications.
:author: Denis Kristak (Inuits)
:author: Pavel Grochal (Inuits)
"""
from easybuild.base import fancylogger
from easybuild.tools.build_log import EasyBuildError
from easybuild.tools.filetools import read_file
from easybuild.tools.module_naming_scheme.utilities import det_full_ec_version
from easybuild.tools.utilities import only_if_module_is_available
try:
import yaml
except ImportError:
pass
_log = fancylogger.getLogger('easystack', fname=False)
class EasyStack(object):
"""One class instance per easystack. General options + list of all SoftwareSpecs instances"""
def __init__(self):
self.easybuild_version = None
self.robot = False
self.software_list = []
def compose_ec_filenames(self):
"""Returns a list of all easyconfig names"""
ec_filenames = []
for sw in self.software_list:
full_ec_version = det_full_ec_version({
'toolchain': {
# #
"""
Support for checking types of easyconfig parameter values.
:author: Caroline De Brouwer (Ghent University)
:author: Kenneth Hoste (Ghent University)
"""
from distutils.util import strtobool
from easybuild.base import fancylogger
from easybuild.framework.easyconfig.format.format import DEPENDENCY_PARAMETERS
from easybuild.framework.easyconfig.format.format import SANITY_CHECK_PATHS_DIRS, SANITY_CHECK_PATHS_FILES
from easybuild.tools.build_log import EasyBuildError
from easybuild.tools.py2vs3 import string_type
_log = fancylogger.getLogger('easyconfig.types', fname=False)
def as_hashable(dict_value):
"""Helper function, convert dict value to hashable equivalent via tuples."""
res = []
for key, val in sorted(dict_value.items()):
if isinstance(val, list):
val = tuple(val)
elif isinstance(val, dict):
val = as_hashable(val)
res.append((key, val))
return tuple(res)
def check_element_types(elems, allowed_types):
import copy
import os
import sys
from easybuild.base import fancylogger
from easybuild.framework.easyconfig.easyconfig import EASYCONFIGS_ARCHIVE_DIR, ActiveMNS, process_easyconfig
from easybuild.framework.easyconfig.easyconfig import robot_find_easyconfig, verify_easyconfig_filename
from easybuild.framework.easyconfig.tools import find_resolved_modules, skip_available
from easybuild.tools.build_log import EasyBuildError
from easybuild.tools.config import build_option
from easybuild.tools.filetools import det_common_path_prefix, search_file
from easybuild.tools.module_naming_scheme.easybuild_mns import EasyBuildMNS
from easybuild.tools.module_naming_scheme.utilities import det_full_ec_version
from easybuild.tools.utilities import flatten, nub
_log = fancylogger.getLogger('tools.robot', fname=False)
def det_robot_path(robot_paths_option,
tweaked_ecs_paths,
pr_paths,
auto_robot=False):
"""Determine robot path."""
robot_path = robot_paths_option[:]
_log.info("Using robot path(s): %s", robot_path)
tweaked_ecs_path, tweaked_ecs_deps_path = None, None
# paths to tweaked easyconfigs or easyconfigs downloaded from a PR have priority
if tweaked_ecs_paths is not None:
tweaked_ecs_path, tweaked_ecs_deps_path = tweaked_ecs_paths
# easyconfigs listed on the command line (and tweaked) should be found first
from easybuild.framework.easyconfig.constants import EASYCONFIG_CONSTANTS
from easybuild.framework.easyconfig.default import get_easyconfig_parameter_default
from easybuild.framework.easyconfig.easyconfig import EasyConfig, create_paths, process_easyconfig
from easybuild.framework.easyconfig.easyconfig import get_toolchain_hierarchy, ActiveMNS
from easybuild.framework.easyconfig.format.format import DEPENDENCY_PARAMETERS
from easybuild.toolchains.gcccore import GCCcore
from easybuild.tools.build_log import EasyBuildError, print_warning
from easybuild.tools.config import build_option
from easybuild.tools.filetools import read_file, write_file
from easybuild.tools.module_naming_scheme.utilities import det_full_ec_version
from easybuild.tools.robot import resolve_dependencies, robot_find_easyconfig
from easybuild.tools.toolchain.toolchain import SYSTEM_TOOLCHAIN_NAME
from easybuild.tools.toolchain.toolchain import TOOLCHAIN_CAPABILITIES
from easybuild.tools.utilities import flatten, nub, quote_str
_log = fancylogger.getLogger('easyconfig.tweak', fname=False)
EASYCONFIG_TEMPLATE = "TEMPLATE"
def ec_filename_for(path):
"""
Return a suiting file name for the easyconfig file at <path>,
as determined by its contents.
"""
ec = EasyConfig(path, validate=False)
fn = "%s-%s.eb" % (ec['name'], det_full_ec_version(ec))
return fn
"""
import os
import tempfile
import pprint
from easybuild.base import fancylogger
from easybuild.tools.config import PKG_TOOL_FPM, PKG_TYPE_RPM, Singleton
from easybuild.tools.config import build_option, get_package_naming_scheme, log_path
from easybuild.tools.build_log import EasyBuildError
from easybuild.tools.filetools import change_dir, which
from easybuild.tools.package.package_naming_scheme.pns import PackageNamingScheme
from easybuild.tools.py2vs3 import create_base_metaclass
from easybuild.tools.run import run_cmd
from easybuild.tools.utilities import get_subclasses, import_available_modules
_log = fancylogger.getLogger('tools.package') # pylint: disable=C0103
def avail_package_naming_schemes():
"""
Returns the list of valed naming schemes
They are loaded from the easybuild.package.package_naming_scheme namespace
"""
import_available_modules('easybuild.tools.package.package_naming_scheme')
class_dict = dict([(x.__name__, x)
for x in get_subclasses(PackageNamingScheme)])
return class_dict
def package(easyblock):
"""
# You should have received a copy of the GNU General Public License
# along with EasyBuild. If not, see <http://www.gnu.org/licenses/>.
# #
"""
This module implements all supported formats and their converters
:author: Stijn De Weirdt (Ghent University)
"""
import re
from easybuild.base import fancylogger
from easybuild.base.wrapper import Wrapper
from easybuild.tools.build_log import EasyBuildError
from easybuild.tools.py2vs3 import string_type
_log = fancylogger.getLogger('tools.convert', fname=False)
class Convert(Wrapper):
"""
Convert casts a string passed via the initialisation to a Convert (sub)class instance,
mainly for typechecking and printing purposes.
"""
SEPARATOR = None
def __init__(self, obj):
"""Support the conversion of obj to something"""
self.__dict__['log'] = fancylogger.getLogger(self.__class__.__name__,
fname=False)
self.__dict__['data'] = None
if isinstance(obj, string_type):
# along with EasyBuild. If not, see <http://www.gnu.org/licenses/>.
#
"""
Easyconfig licenses module that provides all licenses that can
be used within an Easyconfig file.
:author: Stijn De Weirdt (Ghent University)
:author: Kenneth Hoste (Ghent University)
"""
from easybuild.base import fancylogger
from easybuild.tools.utilities import get_subclasses
_log = fancylogger.getLogger('easyconfig.licenses', fname=False)
class License(object):
"""EasyBuild easyconfig license class
This is also the default restrictive license
"""
HIDDEN = False # disable subclasses from being seen/used
NAME = None
VERSION = None
DESCRIPTION = None
DISTRIBUTE_SOURCE = False # does the license allows to (re)distribute the code
GROUP_SOURCE = True # does the license require to keep the source under dedicated group
GROUP_BINARY = True # does the license require to install the binaries under dedicated group
class EasyConfigTest(TestCase):
"""Baseclass for easyconfig testcases."""
# initialize configuration (required for e.g. default modules_tool setting)
eb_go = eboptions.parse_options()
config.init(eb_go.options, eb_go.get_options_by_section('config'))
build_options = {
'check_osdeps': False,
'external_modules_metadata': {},
'force': True,
'local_var_naming_check': 'error',
'optarch': 'test',
'robot_path': get_paths_for("easyconfigs")[0],
'silent': True,
'suffix_modules_path': GENERAL_CLASS,
'valid_module_classes': config.module_classes(),
'valid_stops': [x[0] for x in EasyBlock.get_steps()],
}
config.init_build_options(build_options=build_options)
set_tmpdir()
del eb_go
# put dummy 'craype-test' module in place, which is required for parsing easyconfigs using Cray* toolchains
TMPDIR = tempfile.mkdtemp()
os.environ['MODULEPATH'] = TMPDIR
write_file(os.path.join(TMPDIR, 'craype-test'), '#%Module\n')
log = fancylogger.getLogger("EasyConfigTest", fname=False)
# make sure a logger is present for main
eb_main._log = log
ordered_specs = None
parsed_easyconfigs = []
def process_all_easyconfigs(self):
"""Process all easyconfigs and resolve inter-easyconfig dependencies."""
# all available easyconfig files
easyconfigs_path = get_paths_for("easyconfigs")[0]
specs = glob.glob('%s/*/*/*.eb' % easyconfigs_path)
# parse all easyconfigs if they haven't been already
if not EasyConfigTest.parsed_easyconfigs:
for spec in specs:
EasyConfigTest.parsed_easyconfigs.extend(
process_easyconfig(spec))
# filter out external modules
for ec in EasyConfigTest.parsed_easyconfigs:
for dep in ec['dependencies'][:]:
if dep.get('external_module', False):
ec['dependencies'].remove(dep)
EasyConfigTest.ordered_specs = resolve_dependencies(
EasyConfigTest.parsed_easyconfigs,
modules_tool(),
retain_all_deps=True)
def test_dep_graph(self):
"""Unit test that builds a full dependency graph."""
# pygraph dependencies required for constructing dependency graph are not available prior to Python 2.6
if LooseVersion(
sys.version) >= LooseVersion('2.6') and single_tests_ok:
# temporary file for dep graph
(hn, fn) = tempfile.mkstemp(suffix='.dot')
os.close(hn)
if EasyConfigTest.ordered_specs is None:
self.process_all_easyconfigs()
dep_graph(fn, EasyConfigTest.ordered_specs)
remove_file(fn)
else:
print("(skipped dep graph test)")
def test_conflicts(self):
"""Check whether any conflicts occur in software dependency graphs."""
if not single_tests_ok:
print("(skipped conflicts test)")
return
if EasyConfigTest.ordered_specs is None:
self.process_all_easyconfigs()
self.assertFalse(
check_conflicts(EasyConfigTest.ordered_specs,
modules_tool(),
check_inter_ec_conflicts=False),
"No conflicts detected")
def check_dep_vars(self, dep, dep_vars):
"""Check whether available variants of a particular dependency are acceptable or not."""
# 'guilty' until proven 'innocent'
res = False
# filter out wrapped Java versions
# i.e. if the version of one is a prefix of the version of the other one (e.g. 1.8 & 1.8.0_181)
if dep == 'Java':
dep_vars_to_check = sorted(dep_vars.keys())
retained_dep_vars = []
while dep_vars_to_check:
dep_var = dep_vars_to_check.pop()
dep_var_version = dep_var.split(';')[0]
# remove dep vars wrapped by current dep var
dep_vars_to_check = [
x for x in dep_vars_to_check
if not x.startswith(dep_var_version + '.')
]
retained_dep_vars = [
x for x in retained_dep_vars
if not x.startswith(dep_var_version + '.')
]
retained_dep_vars.append(dep_var)
for key in list(dep_vars.keys()):
if key not in retained_dep_vars:
del dep_vars[key]
# filter out binutils with empty versionsuffix which is used to build toolchain compiler
if dep == 'binutils' and len(dep_vars) > 1:
empty_vsuff_vars = [
v for v in dep_vars.keys() if v.endswith('versionsuffix: ')
]
if len(empty_vsuff_vars) == 1:
dep_vars = dict((k, v) for (k, v) in dep_vars.items()
if k != empty_vsuff_vars[0])
# multiple variants of HTSlib is OK as long as they are deps for a matching version of BCFtools
if dep == 'HTSlib' and len(dep_vars) > 1:
for key in list(dep_vars):
ecs = dep_vars[key]
# filter out HTSlib variants that are only used as dependency for BCFtools with same version
htslib_ver = re.search('^version: (?P<ver>[^;]+);',
key).group('ver')
if all(
ec.startswith('BCFtools-%s-' % htslib_ver)
for ec in ecs):
dep_vars.pop(key)
# multiple versions of Boost is OK as long as they are deps for a matching Boost.Python
if dep == 'Boost' and len(dep_vars) > 1:
for key in list(dep_vars):
ecs = dep_vars[key]
# filter out Boost variants that are only used as dependency for Boost.Python with same version
boost_ver = re.search('^version: (?P<ver>[^;]+);',
key).group('ver')
if all(
ec.startswith('Boost.Python-%s-' % boost_ver)
for ec in ecs):
dep_vars.pop(key)
# filter out FFTW and imkl with -serial versionsuffix which are used in non-MPI subtoolchains
if dep in ['FFTW', 'imkl']:
serial_vsuff_vars = [
v for v in dep_vars.keys()
if v.endswith('versionsuffix: -serial')
]
if len(serial_vsuff_vars) == 1:
dep_vars = dict((k, v) for (k, v) in dep_vars.items()
if k != serial_vsuff_vars[0])
# for some dependencies, we allow exceptions for software that depends on a particular version,
# as long as that's indicated by the versionsuffix
if dep in ['ASE', 'Boost', 'Java', 'Lua', 'PLUMED', 'R'
] and len(dep_vars) > 1:
for key in list(dep_vars):
dep_ver = re.search('^version: (?P<ver>[^;]+);',
key).group('ver')
# use version of Java wrapper rather than full Java version
if dep == 'Java':
dep_ver = '.'.join(dep_ver.split('.')[:2])
# filter out dep version if all easyconfig filenames using it include specific dep version
if all(
re.search('-%s-%s' % (dep, dep_ver), v)
for v in dep_vars[key]):
dep_vars.pop(key)
# always retain at least one dep variant
if len(dep_vars) == 1:
break
# filter R dep for a specific version of Python 2.x
if dep == 'R' and len(dep_vars) > 1:
for key in list(dep_vars):
if '; versionsuffix: -Python-2' in key:
dep_vars.pop(key)
# always retain at least one variant
if len(dep_vars) == 1:
break
# filter out variants that are specific to a particular version of CUDA
cuda_dep_vars = [v for v in dep_vars.keys() if '-CUDA' in v]
if len(dep_vars) > len(cuda_dep_vars):
for key in list(dep_vars):
if re.search('; versionsuffix: .*-CUDA-[0-9.]+', key):
dep_vars.pop(key)
# some software packages require an old version of a particular dependency
old_dep_versions = {
# libxc 2.x or 3.x is required by ABINIT, AtomPAW, CP2K, GPAW, PySCF, WIEN2k
# (Qiskit depends on PySCF)
'libxc': (r'[23]\.', [
'ABINIT-', 'AtomPAW-', 'CP2K-', 'GPAW-', 'PySCF-', 'Qiskit-',
'WIEN2k-'
]),
# OPERA requires SAMtools 0.x
'SAMtools':
(r'0\.', ['ChimPipe-0.9.5', 'Cufflinks-2.2.1', 'OPERA-2.0.6']),
# Kraken 1.x requires Jellyfish 1.x (Roary & metaWRAP depend on Kraken 1.x)
'Jellyfish':
(r'1\.', ['Kraken-1.', 'Roary-3.12.0', 'metaWRAP-1.2']),
# EMAN2 2.3 requires Boost(.Python) 1.64.0
'Boost': ('1.64.0;', ['Boost.Python-1.64.0-', 'EMAN2-2.3-']),
'Boost.Python': ('1.64.0;', ['EMAN2-2.3-']),
}
if dep in old_dep_versions and len(dep_vars) > 1:
for key in list(dep_vars):
version_pattern, parents = old_dep_versions[dep]
# filter out known old dependency versions
if re.search('^version: %s' % version_pattern, key):
# only filter if the easyconfig using this dep variants is known
if all(
any(x.startswith(p) for p in parents)
for x in dep_vars[key]):
dep_vars.pop(key)
# only single variant is always OK
if len(dep_vars) == 1:
res = True
elif len(dep_vars) == 2 and dep in ['Python', 'Tkinter']:
# for Python & Tkinter, it's OK to have on 2.x and one 3.x version
v2_dep_vars = [
x for x in dep_vars.keys() if x.startswith('version: 2.')
]
v3_dep_vars = [
x for x in dep_vars.keys() if x.startswith('version: 3.')
]
if len(v2_dep_vars) == 1 and len(v3_dep_vars) == 1:
res = True
# two variants is OK if one is for Python 2.x and the other is for Python 3.x (based on versionsuffix)
elif len(dep_vars) == 2:
py2_dep_vars = [
x for x in dep_vars.keys()
if '; versionsuffix: -Python-2.' in x
]
py3_dep_vars = [
x for x in dep_vars.keys()
if '; versionsuffix: -Python-3.' in x
]
if len(py2_dep_vars) == 1 and len(py3_dep_vars) == 1:
res = True
return res
def test_check_dep_vars(self):
"""Test check_dep_vars utility method."""
# one single dep version: OK
self.assertTrue(
self.check_dep_vars(
'testdep', {
'version: 1.2.3; versionsuffix:':
['foo-1.2.3.eb', 'bar-4.5.6.eb'],
}))
self.assertTrue(
self.check_dep_vars(
'testdep', {
'version: 1.2.3; versionsuffix: -test':
['foo-1.2.3.eb', 'bar-4.5.6.eb'],
}))
# two or more dep versions (no special case: not OK)
self.assertFalse(
self.check_dep_vars(
'testdep', {
'version: 1.2.3; versionsuffix:': ['foo-1.2.3.eb'],
'version: 4.5.6; versionsuffix:': ['bar-4.5.6.eb'],
}))
self.assertFalse(
self.check_dep_vars(
'testdep', {
'version: 0.0; versionsuffix:': ['foobar-0.0.eb'],
'version: 1.2.3; versionsuffix:': ['foo-1.2.3.eb'],
'version: 4.5.6; versionsuffix:': ['bar-4.5.6.eb'],
}))
# Java is a special case, with wrapped Java versions
self.assertTrue(
self.check_dep_vars(
'Java', {
'version: 1.8.0_221; versionsuffix:': ['foo-1.2.3.eb'],
'version: 1.8; versionsuffix:': ['foo-1.2.3.eb'],
}))
# two Java wrappers is not OK
self.assertFalse(
self.check_dep_vars(
'Java', {
'version: 1.8.0_221; versionsuffix:': ['foo-1.2.3.eb'],
'version: 1.8; versionsuffix:': ['foo-1.2.3.eb'],
'version: 11.0.2; versionsuffix:': ['bar-4.5.6.eb'],
'version: 11; versionsuffix:': ['bar-4.5.6.eb'],
}))
# OK to have two or more wrappers if versionsuffix is used to indicate exception
self.assertTrue(
self.check_dep_vars(
'Java', {
'version: 1.8.0_221; versionsuffix:': ['foo-1.2.3.eb'],
'version: 1.8; versionsuffix:': ['foo-1.2.3.eb'],
'version: 11.0.2; versionsuffix:':
['bar-4.5.6-Java-11.eb'],
'version: 11; versionsuffix:': ['bar-4.5.6-Java-11.eb'],
}))
# versionsuffix must be there for all easyconfigs to indicate exception
self.assertFalse(
self.check_dep_vars(
'Java', {
'version: 1.8.0_221; versionsuffix:': ['foo-1.2.3.eb'],
'version: 1.8; versionsuffix:': ['foo-1.2.3.eb'],
'version: 11.0.2; versionsuffix:':
['bar-4.5.6-Java-11.eb', 'bar-4.5.6.eb'],
'version: 11; versionsuffix:':
['bar-4.5.6-Java-11.eb', 'bar-4.5.6.eb'],
}))
self.assertTrue(
self.check_dep_vars(
'Java', {
'version: 1.8.0_221; versionsuffix:': ['foo-1.2.3.eb'],
'version: 1.8; versionsuffix:': ['foo-1.2.3.eb'],
'version: 11.0.2; versionsuffix:':
['bar-4.5.6-Java-11.eb'],
'version: 11; versionsuffix:': ['bar-4.5.6-Java-11.eb'],
'version: 12.1.6; versionsuffix:':
['foobar-0.0-Java-12.eb'],
'version: 12; versionsuffix:': ['foobar-0.0-Java-12.eb'],
}))
# strange situation: odd number of Java versions
# not OK: two Java wrappers (and no versionsuffix to indicate exception)
self.assertFalse(
self.check_dep_vars(
'Java', {
'version: 1.8.0_221; versionsuffix:': ['foo-1.2.3.eb'],
'version: 1.8; versionsuffix:': ['foo-1.2.3.eb'],
'version: 11; versionsuffix:': ['bar-4.5.6.eb'],
}))
# OK because of -Java-11 versionsuffix
self.assertTrue(
self.check_dep_vars(
'Java', {
'version: 1.8.0_221; versionsuffix:': ['foo-1.2.3.eb'],
'version: 1.8; versionsuffix:': ['foo-1.2.3.eb'],
'version: 11; versionsuffix:': ['bar-4.5.6-Java-11.eb'],
}))
# not OK: two Java wrappers (and no versionsuffix to indicate exception)
self.assertFalse(
self.check_dep_vars(
'Java', {
'version: 1.8; versionsuffix:': ['foo-1.2.3.eb'],
'version: 11.0.2; versionsuffix:': ['bar-4.5.6.eb'],
'version: 11; versionsuffix:': ['bar-4.5.6.eb'],
}))
# OK because of -Java-11 versionsuffix
self.assertTrue(
self.check_dep_vars(
'Java', {
'version: 1.8; versionsuffix:': ['foo-1.2.3.eb'],
'version: 11.0.2; versionsuffix:':
['bar-4.5.6-Java-11.eb'],
'version: 11; versionsuffix:': ['bar-4.5.6-Java-11.eb'],
}))
# two different versions of Boost is not OK
self.assertFalse(
self.check_dep_vars(
'Boost', {
'version: 1.64.0; versionsuffix:': ['foo-1.2.3.eb'],
'version: 1.70.0; versionsuffix:': ['foo-2.3.4.eb'],
}))
# a different Boost version that is only used as dependency for a matching Boost.Python is fine
self.assertTrue(
self.check_dep_vars(
'Boost', {
'version: 1.64.0; versionsuffix:':
['Boost.Python-1.64.0-gompi-2019a.eb'],
'version: 1.70.0; versionsuffix:': ['foo-2.3.4.eb'],
}))
self.assertTrue(
self.check_dep_vars(
'Boost', {
'version: 1.64.0; versionsuffix:':
['Boost.Python-1.64.0-gompi-2018b.eb'],
'version: 1.66.0; versionsuffix:':
['Boost.Python-1.66.0-gompi-2019a.eb'],
'version: 1.70.0; versionsuffix:': ['foo-2.3.4.eb'],
}))
self.assertFalse(
self.check_dep_vars(
'Boost', {
'version: 1.64.0; versionsuffix:':
['Boost.Python-1.64.0-gompi-2019a.eb'],
'version: 1.66.0; versionsuffix:': ['foo-1.2.3.eb'],
'version: 1.70.0; versionsuffix:': ['foo-2.3.4.eb'],
}))
self.assertTrue(
self.check_dep_vars(
'Boost', {
'version: 1.63.0; versionsuffix: -Python-2.7.14':
['EMAN2-2.21a-foss-2018a-Python-2.7.14-Boost-1.63.0.eb'],
'version: 1.64.0; versionsuffix:':
['Boost.Python-1.64.0-gompi-2018a.eb'],
'version: 1.66.0; versionsuffix:':
['BLAST+-2.7.1-foss-2018a.eb'],
}))
self.assertTrue(
self.check_dep_vars(
'Boost', {
'version: 1.64.0; versionsuffix:': [
'Boost.Python-1.64.0-gompi-2019a.eb',
'EMAN2-2.3-foss-2019a-Python-2.7.15.eb',
],
'version: 1.70.0; versionsuffix:': [
'BLAST+-2.9.0-gompi-2019a.eb',
'Boost.Python-1.70.0-gompi-2019a.eb',
],
}))
def test_dep_versions_per_toolchain_generation(self):
"""
Check whether there's only one dependency version per toolchain generation actively used.
This is enforced to try and limit the chance of running into conflicts when multiple modules built with
the same toolchain are loaded together.
"""
if EasyConfigTest.ordered_specs is None:
self.process_all_easyconfigs()
def get_deps_for(ec):
"""Get list of (direct) dependencies for specified easyconfig."""
deps = []
for dep in ec['ec']['dependencies']:
dep_mod_name = dep['full_mod_name']
deps.append((dep['name'], dep['version'], dep['versionsuffix'],
dep_mod_name))
res = [
x for x in EasyConfigTest.ordered_specs
if x['full_mod_name'] == dep_mod_name
]
if len(res) == 1:
deps.extend(get_deps_for(res[0]))
else:
raise EasyBuildError(
"Failed to find %s in ordered list of easyconfigs",
dep_mod_name)
return deps
# some software also follows <year>{a,b} versioning scheme,
# which throws off the pattern matching done below for toolchain versions
false_positives_regex = re.compile('^MATLAB-Engine-20[0-9][0-9][ab]')
# restrict to checking dependencies of easyconfigs using common toolchains (start with 2018a)
# and GCCcore subtoolchain for common toolchains, starting with GCCcore 7.x
for pattern in [
'201[89][ab]', '20[2-9][0-9][ab]', 'GCCcore-[7-9]\.[0-9]'
]:
all_deps = {}
regex = re.compile('^.*-(?P<tc_gen>%s).*\.eb$' % pattern)
# collect variants for all dependencies of easyconfigs that use a toolchain that matches
for ec in EasyConfigTest.ordered_specs:
ec_file = os.path.basename(ec['spec'])
# take into account software which also follows a <year>{a,b} versioning scheme
ec_file = false_positives_regex.sub('', ec_file)
res = regex.match(ec_file)
if res:
tc_gen = res.group('tc_gen')
all_deps_tc_gen = all_deps.setdefault(tc_gen, {})
for dep_name, dep_ver, dep_versuff, dep_mod_name in get_deps_for(
ec):
dep_variants = all_deps_tc_gen.setdefault(dep_name, {})
# a variant is defined by version + versionsuffix
variant = "version: %s; versionsuffix: %s" % (
dep_ver, dep_versuff)
# keep track of which easyconfig this is a dependency
dep_variants.setdefault(variant, set()).add(ec_file)
# check which dependencies have more than 1 variant
multi_dep_vars, multi_dep_vars_msg = [], ''
for tc_gen in sorted(all_deps.keys()):
for dep in sorted(all_deps[tc_gen].keys()):
dep_vars = all_deps[tc_gen][dep]
if not self.check_dep_vars(dep, dep_vars):
multi_dep_vars.append(dep)
multi_dep_vars_msg += "\nfound %s variants of '%s' dependency " % (
len(dep_vars), dep)
multi_dep_vars_msg += "in easyconfigs using '%s' toolchain generation\n* " % tc_gen
multi_dep_vars_msg += '\n* '.join(
"%s as dep for %s" % v
for v in sorted(dep_vars.items()))
multi_dep_vars_msg += '\n'
error_msg = "No multi-variant deps found for '%s' easyconfigs:\n%s" % (
regex.pattern, multi_dep_vars_msg)
self.assertFalse(multi_dep_vars, error_msg)
def test_sanity_check_paths(self):
"""Make sure specified sanity check paths adher to the requirements."""
if EasyConfigTest.ordered_specs is None:
self.process_all_easyconfigs()
for ec in EasyConfigTest.parsed_easyconfigs:
ec_scp = ec['ec']['sanity_check_paths']
if ec_scp != {}:
# if sanity_check_paths is specified (i.e., non-default), it must adher to the requirements
# both 'files' and 'dirs' keys, both with list values and with at least one a non-empty list
error_msg = "sanity_check_paths for %s does not meet requirements: %s" % (
ec['spec'], ec_scp)
self.assertEqual(sorted(ec_scp.keys()), ['dirs', 'files'],
error_msg)
self.assertTrue(isinstance(ec_scp['dirs'], list), error_msg)
self.assertTrue(isinstance(ec_scp['files'], list), error_msg)
self.assertTrue(ec_scp['dirs'] or ec_scp['files'], error_msg)
def test_easyconfig_locations(self):
"""Make sure all easyconfigs files are in the right location."""
easyconfig_dirs_regex = re.compile(
r'/easybuild/easyconfigs/[0a-z]/[^/]+$')
topdir = os.path.dirname(os.path.dirname(os.path.dirname(__file__)))
for (dirpath, _, filenames) in os.walk(topdir):
# ignore git/svn dirs & archived easyconfigs
if '/.git/' in dirpath or '/.svn/' in dirpath or '__archive__' in dirpath:
continue
# check whether list of .eb files is non-empty
easyconfig_files = [fn for fn in filenames if fn.endswith('eb')]
if easyconfig_files:
# check whether path matches required pattern
if not easyconfig_dirs_regex.search(dirpath):
# only exception: TEMPLATE.eb
if not (dirpath.endswith('/easybuild/easyconfigs')
and filenames == ['TEMPLATE.eb']):
self.assertTrue(
False,
"List of easyconfig files in %s is empty: %s" %
(dirpath, filenames))
def check_sha256_checksums(self, changed_ecs):
"""Make sure changed easyconfigs have SHA256 checksums in place."""
# list of software for which checksums can not be required,
# e.g. because 'source' files need to be constructed manually
whitelist = ['Kent_tools-*', 'MATLAB-*', 'OCaml-*']
# the check_sha256_checksums function (again) creates an EasyBlock instance
# for easyconfigs using the Bundle easyblock, this is a problem because the 'sources' easyconfig parameter
# is updated in place (sources for components are added the 'parent' sources) in Bundle's __init__;
# therefore, we need to reset 'sources' to an empty list here if Bundle is used...
# likewise for 'checksums'
for ec in changed_ecs:
if ec['easyblock'] == 'Bundle':
ec['sources'] = []
ec['checksums'] = []
# filter out deprecated easyconfigs
retained_changed_ecs = []
for ec in changed_ecs:
if not ec['deprecated']:
retained_changed_ecs.append(ec)
checksum_issues = check_sha256_checksums(retained_changed_ecs,
whitelist=whitelist)
self.assertTrue(
len(checksum_issues) == 0,
"No checksum issues:\n%s" % '\n'.join(checksum_issues))
def check_python_packages(self, changed_ecs):
"""Several checks for easyconfigs that install (bundles of) Python packages."""
# MATLAB-Engine, PyTorch do not support installation with 'pip'
whitelist_pip = ['MATLAB-Engine-*', 'PyTorch-*']
failing_checks = []
for ec in changed_ecs:
ec_fn = os.path.basename(ec.path)
easyblock = ec.get('easyblock')
exts_defaultclass = ec.get('exts_defaultclass')
download_dep_fail = ec.get('download_dep_fail')
exts_download_dep_fail = ec.get('exts_download_dep_fail')
use_pip = ec.get('use_pip')
# download_dep_fail should be set when using PythonPackage
if easyblock == 'PythonPackage':
if download_dep_fail is None:
failing_checks.append("'download_dep_fail' set in %s" %
ec_fn)
# use_pip should be set when using PythonPackage or PythonBundle (except for whitelisted easyconfigs)
if easyblock in ['PythonBundle', 'PythonPackage']:
if use_pip is None and not any(
re.match(regex, ec_fn) for regex in whitelist_pip):
failing_checks.append("'use_pip' set in %s" % ec_fn)
# download_dep_fail is enabled automatically in PythonBundle easyblock, so shouldn't be set
if easyblock == 'PythonBundle':
if download_dep_fail or exts_download_dep_fail:
fail = "'*download_dep_fail' set in %s (shouldn't, since PythonBundle easyblock is used)" % ec_fn
failing_checks.append(fail)
elif exts_defaultclass == 'PythonPackage':
# bundle of Python packages should use PythonBundle
if easyblock == 'Bundle':
fail = "'PythonBundle' easyblock is used for bundle of Python packages in %s" % ec_fn
failing_checks.append(fail)
else:
# both download_dep_fail and use_pip should be set via exts_default_options
# when installing Python packages as extensions
exts_default_options = ec.get('exts_default_options', {})
for key in ['download_dep_fail', 'use_pip']:
if exts_default_options.get(key) is None:
failing_checks.append(
"'%s' set in exts_default_options in %s" %
(key, ec_fn))
# if Python is a dependency, that should be reflected in the versionsuffix
# Tkinter is an exception, since its version always matches the Python version anyway
if any(dep['name'] == 'Python'
for dep in ec['dependencies']) and ec.name != 'Tkinter':
if not re.search(r'-Python-[23]\.[0-9]+\.[0-9]+',
ec['versionsuffix']):
failing_checks.append(
"'-Python-%%(pyver)s' included in versionsuffix in %s"
% ec_fn)
self.assertFalse(failing_checks, '\n'.join(failing_checks))
def check_sanity_check_paths(self, changed_ecs):
"""Make sure a custom sanity_check_paths value is specified for easyconfigs that use a generic easyblock."""
# PythonBundle & PythonPackage already have a decent customised sanity_check_paths
# BuildEnv, ModuleRC and Toolchain easyblocks doesn't install anything so there is nothing to check.
whitelist = [
'CrayToolchain', 'ModuleRC', 'PythonBundle', 'PythonPackage',
'Toolchain', 'BuildEnv'
]
# GCC is just a bundle of GCCcore+binutils
bundles_whitelist = ['GCC']
failing_checks = []
for ec in changed_ecs:
easyblock = ec.get('easyblock')
if is_generic_easyblock(
easyblock) and not ec.get('sanity_check_paths'):
if easyblock in whitelist or (easyblock == 'Bundle' and
ec['name'] in bundles_whitelist):
pass
else:
ec_fn = os.path.basename(ec.path)
failing_checks.append(
"No custom sanity_check_paths found in %s" % ec_fn)
self.assertFalse(failing_checks, '\n'.join(failing_checks))
def check_https(self, changed_ecs):
"""Make sure https:// URL is used (if it exists) for homepage/source_urls (rather than http://)."""
whitelist = [
'Kaiju', # invalid certificate at https://kaiju.binf.ku.dk
'libxml2', # https://xmlsoft.org works, but invalid certificate
'p4vasp', # https://www.p4vasp.at doesn't work
'ITSTool', # https://itstool.org/ doesn't work
'UCX-', # bad certificate for https://www.openucx.org
]
http_regex = re.compile('http://[^"\'\n]+', re.M)
failing_checks = []
for ec in changed_ecs:
ec_fn = os.path.basename(ec.path)
# skip whitelisted easyconfigs
if any(ec_fn.startswith(x) for x in whitelist):
continue
# ignore commented out lines in easyconfig files when checking for http:// URLs
ec_txt = '\n'.join(l for l in ec.rawtxt.split('\n')
if not l.startswith('#'))
for http_url in http_regex.findall(ec_txt):
https_url = http_url.replace('http://', 'https://')
try:
https_url_works = bool(urlopen(https_url))
except Exception:
https_url_works = False
if https_url_works:
failing_checks.append(
"Found http:// URL in %s, should be https:// : %s" %
(ec_fn, http_url))
self.assertFalse(failing_checks, '\n'.join(failing_checks))
def test_changed_files_pull_request(self):
"""Specific checks only done for the (easyconfig) files that were changed in a pull request."""
# $TRAVIS_PULL_REQUEST should be a PR number, otherwise we're not running tests for a PR
travis_pr_test = re.match(
'^[0-9]+$', os.environ.get('TRAVIS_PULL_REQUEST', '(none)'))
# when testing a PR in GitHub Actions, $GITHUB_EVENT_NAME will be set to 'pull_request'
github_pr_test = os.environ.get('GITHUB_EVENT_NAME') == 'pull_request'
if travis_pr_test or github_pr_test:
# target branch should be anything other than 'master';
# usually is 'develop', but could also be a release branch like '3.7.x'
if travis_pr_test:
target_branch = os.environ.get('TRAVIS_BRANCH', None)
else:
target_branch = os.environ.get('GITHUB_BASE_REF', None)
if target_branch is None:
self.assertTrue(
False,
"Failed to determine target branch for current pull request."
)
if target_branch != 'master':
if not EasyConfigTest.parsed_easyconfigs:
self.process_all_easyconfigs()
# relocate to top-level directory of repository to run 'git diff' command
top_dir = os.path.dirname(
os.path.dirname(get_paths_for('easyconfigs')[0]))
cwd = change_dir(top_dir)
# get list of changed easyconfigs
cmd = "git diff --name-only --diff-filter=AM %s...HEAD" % target_branch
out, ec = run_cmd(cmd, simple=False)
changed_ecs_filenames = [
os.path.basename(f) for f in out.strip().split('\n')
if f.endswith('.eb')
]
print("\nList of changed easyconfig files in this PR: %s" %
'\n'.join(changed_ecs_filenames))
change_dir(cwd)
# grab parsed easyconfigs for changed easyconfig files
changed_ecs = []
for ec_fn in changed_ecs_filenames:
match = None
for ec in EasyConfigTest.parsed_easyconfigs:
if os.path.basename(ec['spec']) == ec_fn:
match = ec['ec']
break
if match:
changed_ecs.append(match)
else:
# if no easyconfig is found, it's possible some archived easyconfigs were touched in the PR...
# so as a last resort, try to find the easyconfig file in __archive__
easyconfigs_path = get_paths_for("easyconfigs")[0]
specs = glob.glob('%s/__archive__/*/*/%s' %
(easyconfigs_path, ec_fn))
if len(specs) == 1:
ec = process_easyconfig(specs[0])[0]
changed_ecs.append(ec['ec'])
else:
error_msg = "Failed to find parsed easyconfig for %s" % ec_fn
error_msg += " (and could not isolate it in easyconfigs archive either)"
self.assertTrue(False, error_msg)
# run checks on changed easyconfigs
self.check_sha256_checksums(changed_ecs)
self.check_python_packages(changed_ecs)
self.check_sanity_check_paths(changed_ecs)
self.check_https(changed_ecs)
def test_zzz_cleanup(self):
"""Dummy test to clean up global temporary directory."""
shutil.rmtree(self.TMPDIR)
def setUp(self):
"""Setup test."""
self.log = fancylogger.getLogger("EasyblocksInitTest", fname=False)
fd, self.eb_file = tempfile.mkstemp(prefix='easyblocks_init_test_',
suffix='.eb')
os.close(fd)
:author: Riccardo Murri (University of Zurich)
:author: Kenneth Hoste (Ghent University)
"""
from distutils.version import LooseVersion
import os
from time import gmtime, strftime
import time
from easybuild.base import fancylogger
from easybuild.tools.build_log import EasyBuildError, print_msg, print_warning
from easybuild.tools.config import JOB_DEPS_TYPE_ABORT_ON_ERROR, JOB_DEPS_TYPE_ALWAYS_RUN, build_option
from easybuild.tools.job.backend import JobBackend
from easybuild.tools.utilities import only_if_module_is_available
_log = fancylogger.getLogger('gc3pie', fname=False)
try:
import gc3libs
import gc3libs.exceptions
from gc3libs import Application, Run, create_engine
from gc3libs.quantity import hours as hr
from gc3libs.workflow import AbortOnError, DependentTaskCollection
# inject EasyBuild logger into GC3Pie
gc3libs.log = fancylogger.getLogger('gc3pie', fname=False)
# make handling of log.error compatible with stdlib logging
gc3libs.log.raiseError = False
# instruct GC3Pie to not ignore errors, but raise exceptions instead
gc3libs.UNIGNORE_ALL_ERRORS = True
# #
"""
Hook support.
:author: Kenneth Hoste (Ghent University)
"""
import difflib
import imp
import os
from easybuild.base import fancylogger
from easybuild.tools.build_log import EasyBuildError, print_msg
from easybuild.tools.config import build_option
_log = fancylogger.getLogger('hooks', fname=False)
BUILD_STEP = 'build'
CLEANUP_STEP = 'cleanup'
CONFIGURE_STEP = 'configure'
EXTENSIONS_STEP = 'extensions'
FETCH_STEP = 'fetch'
INSTALL_STEP = 'install'
MODULE_STEP = 'module'
PACKAGE_STEP = 'package'
PATCH_STEP = 'patch'
PERMISSIONS_STEP = 'permissions'
POSTITER_STEP = 'postiter'
POSTPROC_STEP = 'postproc'
PREPARE_STEP = 'prepare'
READY_STEP = 'ready'
"""
import copy
import os
import tempfile
from distutils.version import LooseVersion
from easybuild.base import fancylogger
import easybuild.tools.environment as env
import easybuild.tools.toolchain as toolchain
from easybuild.tools.build_log import EasyBuildError
from easybuild.tools.config import build_option
from easybuild.tools.filetools import write_file
from easybuild.tools.toolchain.constants import COMPILER_VARIABLES, MPI_COMPILER_TEMPLATE, SEQ_COMPILER_TEMPLATE
from easybuild.tools.toolchain.toolchain import Toolchain
_log = fancylogger.getLogger('tools.toolchain.mpi', fname=False)
def get_mpi_cmd_template(mpi_family, params, mpi_version=None):
"""
Return template for MPI command, for specified MPI family.
:param mpi_family: MPI family to use to determine MPI command template
"""
params = copy.deepcopy(params)
mpi_cmd_template = build_option('mpi_cmd_template')
if mpi_cmd_template:
_log.info("Using specified template for MPI commands: %s",
mpi_cmd_template)
from easybuild.base import fancylogger
from easybuild.tools.build_log import EasyBuildError
from easybuild.tools.options import set_tmpdir
import test.easyblocks.easyblock_specific as e
import test.easyblocks.general as g
import test.easyblocks.init_easyblocks as i
import test.easyblocks.module as m
# initialize logger for all the unit tests
fd, log_fn = tempfile.mkstemp(prefix='easybuild-easyblocks-tests-', suffix='.log')
os.close(fd)
os.remove(log_fn)
fancylogger.logToFile(log_fn)
log = fancylogger.getLogger()
log.setLevelName('DEBUG')
try:
tmpdir = set_tmpdir(raise_error=True)
except EasyBuildError as err:
sys.stderr.write("No execution rights on temporary files, specify another location via $TMPDIR: %s\n" % err)
sys.exit(1)
os.environ['EASYBUILD_TMP_LOGDIR'] = tempfile.mkdtemp(prefix='easyblocks_test_')
# call suite() for each module and then run them all
SUITE = unittest.TestSuite([x.suite() for x in [g, i, m, e]])
res = unittest.TextTestRunner().run(SUITE)
fancylogger.logToFile(log_fn, enable=False)
:author: Caroline De Brouwer (Ghent University)
:author: Kenneth Hoste (Ghent University)
"""
import os
import platform
from distutils.version import LooseVersion
from easybuild.base import fancylogger
from easybuild.framework.easyconfig.format.format import EasyConfigFormat
from easybuild.framework.easyconfig.format.pyheaderconfigobj import build_easyconfig_constants_dict
from easybuild.tools.py2vs3 import string_type
from easybuild.tools.utilities import INDENT_4SPACES, only_if_module_is_available, quote_str
_log = fancylogger.getLogger('easyconfig.format.yeb', fname=False)
YAML_DIR = r'%YAML'
YAML_SEP = '---'
YEB_FORMAT_EXTENSION = '.yeb'
YAML_SPECIAL_CHARS = set(":{}[],&*#?|-<>=!%@\\")
def yaml_join(loader, node):
"""
defines custom YAML join function.
see http://stackoverflow.com/questions/5484016/
how-can-i-do-string-concatenation-or-string-replacement-in-yaml/23212524#23212524
:param loader: the YAML Loader
:param node: the YAML (sequence) node
"""
:author: Fotis Georgatos (Uni.Lu, NTUA)
"""
import getpass
import os
import socket
import tempfile
import time
from easybuild.base import fancylogger
from easybuild.tools.build_log import EasyBuildError
from easybuild.tools.filetools import rmtree2
from easybuild.tools.repository.filerepo import FileRepository
from easybuild.tools.utilities import only_if_module_is_available
from easybuild.tools.version import VERSION
_log = fancylogger.getLogger('gitrepo', fname=False)
# optional Python packages, these might be missing
# failing imports are just ignored
# a NameError should be catched where these are used
# GitPython (http://gitorious.org/git-python)
try:
import git
from git import GitCommandError
HAVE_GIT = True
except ImportError:
_log.debug('Failed to import git module')
HAVE_GIT = False
def setUp(self):
"""Set up testcase."""
super(EnhancedTestCase, self).setUp()
# make sure option parser doesn't pick up any cmdline arguments/options
while len(sys.argv) > 1:
sys.argv.pop()
# keep track of log handlers
log = fancylogger.getLogger(fname=False)
self.orig_log_handlers = log.handlers[:]
log.info("setting up test %s" % self.id())
self.orig_tmpdir = tempfile.gettempdir()
# use a subdirectory for this test (which we can clean up easily after the test completes)
self.test_prefix = set_tmpdir()
self.log = fancylogger.getLogger(self.__class__.__name__, fname=False)
fd, self.logfile = tempfile.mkstemp(suffix='.log', prefix='eb-test-')
os.close(fd)
self.cwd = os.getcwd()
# keep track of original environment to restore
self.orig_environ = copy.deepcopy(os.environ)
# keep track of original environment/Python search path to restore
self.orig_sys_path = sys.path[:]
testdir = os.path.dirname(os.path.abspath(__file__))
self.test_sourcepath = os.path.join(testdir, 'sandbox', 'sources')
os.environ['EASYBUILD_SOURCEPATH'] = self.test_sourcepath
os.environ['EASYBUILD_PREFIX'] = self.test_prefix
self.test_buildpath = tempfile.mkdtemp()
os.environ['EASYBUILD_BUILDPATH'] = self.test_buildpath
self.test_installpath = tempfile.mkdtemp()
os.environ['EASYBUILD_INSTALLPATH'] = self.test_installpath
# make sure that the tests only pick up easyconfigs provided with the tests
os.environ['EASYBUILD_ROBOT_PATHS'] = os.path.join(testdir, 'easyconfigs', 'test_ecs')
# make sure no deprecated behaviour is being triggered (unless intended by the test)
# trip *all* log.deprecated statements by setting deprecation version ridiculously high
self.orig_current_version = eb_build_log.CURRENT_VERSION
os.environ['EASYBUILD_DEPRECATED'] = '10000000'
init_config()
import easybuild
# try to import easybuild.easyblocks(.generic) packages
# it's OK if it fails here, but important to import first before fiddling with sys.path
try:
import easybuild.easyblocks
import easybuild.easyblocks.generic
except ImportError:
pass
# add sandbox to Python search path, update namespace packages
sys.path.append(os.path.join(testdir, 'sandbox'))
# required to make sure the 'easybuild' dir in the sandbox is picked up;
# this relates to the other 'reload' statements below
reload(easybuild)
# required to 'reset' easybuild.tools.module_naming_scheme namespace
reload(easybuild.tools)
reload(easybuild.tools.module_naming_scheme)
# remove any entries in Python search path that seem to provide easyblocks (except the sandbox)
for path in sys.path[:]:
if os.path.exists(os.path.join(path, 'easybuild', 'easyblocks', '__init__.py')):
if not os.path.samefile(path, os.path.join(testdir, 'sandbox')):
sys.path.remove(path)
# hard inject location to (generic) test easyblocks into Python search path
# only prepending to sys.path is not enough due to 'pkgutil.extend_path' in easybuild/easyblocks/__init__.py
easybuild.__path__.insert(0, os.path.join(testdir, 'sandbox', 'easybuild'))
import easybuild.easyblocks
test_easyblocks_path = os.path.join(testdir, 'sandbox', 'easybuild', 'easyblocks')
easybuild.easyblocks.__path__.insert(0, test_easyblocks_path)
reload(easybuild.easyblocks)
import easybuild.easyblocks.generic
test_easyblocks_path = os.path.join(test_easyblocks_path, 'generic')
easybuild.easyblocks.generic.__path__.insert(0, test_easyblocks_path)
reload(easybuild.easyblocks.generic)
# save values of $PATH & $PYTHONPATH, so they can be restored later
# this is important in case EasyBuild was installed as a module, since that module may be unloaded,
# for example due to changes to $MODULEPATH in case EasyBuild was installed in a module hierarchy
# cfr. https://github.com/easybuilders/easybuild-framework/issues/1685
self.env_path = os.environ.get('PATH')
self.env_pythonpath = os.environ.get('PYTHONPATH')
self.modtool = modules_tool()
self.reset_modulepath([os.path.join(testdir, 'modules')])
reset_module_caches()
"""
import copy
import re
import sys
import easybuild.tools.toolchain
from easybuild.base import fancylogger
from easybuild.tools.build_log import EasyBuildError
from easybuild.tools.toolchain.toolchain import Toolchain
from easybuild.tools.utilities import get_subclasses, import_available_modules, nub
TC_CONST_PREFIX = 'TC_CONSTANT_'
_initial_toolchain_instances = {}
_log = fancylogger.getLogger("toolchain.utilities")
def search_toolchain(name):
"""
Obtain a Toolchain instance for the toolchain with specified name, next to a list of available toolchains.
:param name: toolchain name
:return: Toolchain instance (or None), found_toolchains
"""
package = easybuild.tools.toolchain
check_attr_name = '%s_PROCESSED' % TC_CONST_PREFIX
if not hasattr(package, check_attr_name) or not getattr(
package, check_attr_name):
# import all available toolchains, so we know about them
# these are imported just to we can reload them later
import easybuild.tools.module_naming_scheme
import easybuild.toolchains
import easybuild.toolchains.compiler
import easybuild.toolchains.fft
import easybuild.toolchains.linalg
import easybuild.toolchains.mpi
# importing easyblocks namespace may fail if easybuild-easyblocks is not available
# for now, we don't really care
try:
import easybuild.easyblocks
import easybuild.easyblocks.generic
except ImportError:
pass
_log = fancylogger.getLogger('tools.include', fname=False)
# body for __init__.py file in package directory, which takes care of making sure the package can be distributed
# across multiple directories
PKG_INIT_BODY = """
__path__ = __import__('pkgutil').extend_path(__path__, __name__)
"""
# more extensive __init__.py specific to easybuild.easyblocks package;
# this is required because of the way in which the easyblock Python modules are organised in the easybuild-easyblocks
# repository, i.e. in first-letter subdirectories
EASYBLOCKS_PKG_INIT_BODY = """
import pkgutil
# extend path so Python finds our easyblocks in the subdirectories where they are located
subdirs = [chr(char) for char in range(ord('a'), ord('z') + 1)] + ['0']
:author: Kenneth Hoste (Ghent University)
"""
import copy
import re
import sys
from easybuild.base import fancylogger
from easybuild.framework.easyconfig.constants import EASYCONFIG_CONSTANTS
from easybuild.framework.easyconfig.format.format import get_format_version, EasyConfigFormat
from easybuild.framework.easyconfig.licenses import EASYCONFIG_LICENSES_DICT
from easybuild.framework.easyconfig.templates import TEMPLATE_CONSTANTS
from easybuild.tools.build_log import EasyBuildError
from easybuild.tools.configobj import ConfigObj
from easybuild.tools.systemtools import get_shared_lib_ext
_log = fancylogger.getLogger('easyconfig.format.pyheaderconfigobj',
fname=False)
def build_easyconfig_constants_dict():
"""Make a dictionary with all constants that can be used"""
all_consts = [
('TEMPLATE_CONSTANTS', dict([(x[0], x[1])
for x in TEMPLATE_CONSTANTS])),
('EASYCONFIG_CONSTANTS',
dict([(key, val[0]) for key, val in EASYCONFIG_CONSTANTS.items()])),
('EASYCONFIG_LICENSES',
dict([(klass().name, name)
for name, klass in EASYCONFIG_LICENSES_DICT.items()])),
]
err = []
const_dict = {}
def __init__(self, *args, **kwargs):
super(Variables, self).__init__(*args, **kwargs)
self.log = fancylogger.getLogger(self.__class__.__name__, fname=False)
def __init__(self):
"""initialize logger."""
self.log = fancylogger.getLogger(self.__class__.__name__, fname=False)
:author: Kenneth Hoste (Ghent University)
"""
import copy
import os
from easybuild.base import fancylogger
from easybuild.framework.easyblock import EasyBlock
from easybuild.framework.easyconfig import CUSTOM
from easybuild.framework.extension import Extension
from easybuild.tools.build_log import EasyBuildError
from easybuild.tools.filetools import change_dir, extract_file
from easybuild.tools.utilities import remove_unwanted_chars, trace_msg
_log = fancylogger.getLogger('extensioneasyblock', fname=False)
class ExtensionEasyBlock(EasyBlock, Extension):
"""
Install an extension as a separate module, or as an extension.
Deriving classes should implement the following functions:
* required EasyBlock functions:
- configure_step
- build_step
- install_step
* required Extension functions
- run
"""
"""
Easyconfig constants module that provides all constants that can
be used within an Easyconfig file.
:author: Stijn De Weirdt (Ghent University)
:author: Kenneth Hoste (Ghent University)
"""
import os
import platform
from easybuild.base import fancylogger
from easybuild.tools.systemtools import get_os_name, get_os_type, get_os_version
_log = fancylogger.getLogger('easyconfig.constants', fname=False)
EXTERNAL_MODULE_MARKER = 'EXTERNAL_MODULE'
# constants that can be used in easyconfig
EASYCONFIG_CONSTANTS = {
'ARCH': (platform.uname()[4], "CPU architecture of current system (aarch64, x86_64, ppc64le, ...)"),
'EXTERNAL_MODULE': (EXTERNAL_MODULE_MARKER, "External module marker"),
'HOME': (os.path.expanduser('~'), "Home directory ($HOME)"),
'OS_TYPE': (get_os_type(), "System type (e.g. 'Linux' or 'Darwin')"),
'OS_NAME': (get_os_name(), "System name (e.g. 'fedora' or 'RHEL')"),
'OS_VERSION': (get_os_version(), "System version"),
'SYS_PYTHON_VERSION': (platform.python_version(), "System Python version (platform.python_version())"),
'SYSTEM': ({'name': 'system', 'version': 'system'}, "System toolchain"),
def pick_python_cmd(req_maj_ver=None, req_min_ver=None):
"""
Pick 'python' command to use, based on specified version requirements.
If the major version is specified, it must be an exact match (==).
If the minor version is specified, it is considered a minimal minor version (>=).
List of considered 'python' commands (in order)
* 'python' available through $PATH
* 'python<major_ver>' available through $PATH
* 'python<major_ver>.<minor_ver>' available through $PATH
* Python executable used in current session (sys.executable)
"""
log = fancylogger.getLogger('pick_python_cmd', fname=False)
def check_python_cmd(python_cmd):
"""Check whether specified Python command satisfies requirements."""
# check whether specified Python command is available
if os.path.isabs(python_cmd):
if not os.path.isfile(python_cmd):
log.debug("Python command '%s' does not exist", python_cmd)
return False
else:
python_cmd_path = which(python_cmd)
if python_cmd_path is None:
log.debug("Python command '%s' not available through $PATH",
python_cmd)
return False
if req_maj_ver is not None:
if req_min_ver is None:
req_majmin_ver = '%s.0' % req_maj_ver
else:
req_majmin_ver = '%s.%s' % (req_maj_ver, req_min_ver)
pyver = det_python_version(python_cmd)
# (strict) check for major version
maj_ver = pyver.split('.')[0]
if maj_ver != str(req_maj_ver):
log.debug("Major Python version does not match: %s vs %s",
maj_ver, req_maj_ver)
return False
# check for minimal minor version
if LooseVersion(pyver) < LooseVersion(req_majmin_ver):
log.debug(
"Minimal requirement for minor Python version not satisfied: %s vs %s",
pyver, req_majmin_ver)
return False
# all check passed
log.debug("All check passed for Python command '%s'!", python_cmd)
return True
# compose list of 'python' commands to consider
python_cmds = ['python']
if req_maj_ver:
python_cmds.append('python%s' % req_maj_ver)
if req_min_ver:
python_cmds.append('python%s.%s' % (req_maj_ver, req_min_ver))
python_cmds.append(sys.executable)
log.debug("Considering Python commands: %s", ', '.join(python_cmds))
# try and find a 'python' command that satisfies the requirements
res = None
for python_cmd in python_cmds:
if check_python_cmd(python_cmd):
log.debug("Python command '%s' satisfies version requirements!",
python_cmd)
if os.path.isabs(python_cmd):
res = python_cmd
else:
res = which(python_cmd)
log.debug("Absolute path to retained Python command: %s", res)
break
else:
log.debug(
"Python command '%s' does not satisfy version requirements (maj: %s, min: %s), moving on",
python_cmd, req_maj_ver, req_min_ver)
return res
from easybuild.base import fancylogger
from easybuild.framework.easyblock import build_easyconfigs
from easybuild.framework.easyconfig.tools import process_easyconfig
from easybuild.framework.easyconfig.tools import skip_available
from easybuild.tools.build_log import EasyBuildError
from easybuild.tools.config import build_option
from easybuild.tools.filetools import find_easyconfigs, mkdir, read_file, write_file
from easybuild.tools.github import GITHUB_EASYBLOCKS_REPO, GITHUB_EASYCONFIGS_REPO, create_gist, post_comment_in_issue
from easybuild.tools.jenkins import aggregate_xml_in_dirs
from easybuild.tools.parallelbuild import build_easyconfigs_in_parallel
from easybuild.tools.robot import resolve_dependencies
from easybuild.tools.systemtools import UNKNOWN, get_system_info
from easybuild.tools.version import FRAMEWORK_VERSION, EASYBLOCKS_VERSION
_log = fancylogger.getLogger('testing', fname=False)
def regtest(easyconfig_paths, modtool, build_specs=None):
"""
Run regression test, using easyconfigs available in given path
:param easyconfig_paths: path of easyconfigs to run regtest on
:param modtool: ModulesTool instance to use
:param build_specs: dictionary specifying build specifications (e.g. version, toolchain, ...)
"""
cur_dir = os.getcwd()
aggregate_regtest = build_option('aggregate_regtest')
if aggregate_regtest is not None:
output_file = os.path.join(
