def add_module(self, modules):
"""
Check if module exist, if so add to list.
"""
self.log.deprecated("Use of add_module function should be replaced by load([<list of modules>])", '2.0')
for mod in modules:
if isinstance(mod, (list, tuple)):
mod_dict = {
'name': mod[0],
'version': mod[1],
'versionsuffix': '',
'toolchain': {
'name': DUMMY_TOOLCHAIN_NAME,
'version': DUMMY_TOOLCHAIN_VERSION,
},
}
mod_name = det_full_module_name(mod_dict)
elif isinstance(mod, basestring):
mod_name = mod
elif isinstance(mod, dict):
mod_name = det_full_module_name(mod)
else:
self.log.error("Can't add module %s: unknown type" % str(mod))
mods = self.available(mod_name)
if mod_name in mods:
# ok
self._modules.append(mod_name)
else:
if len(mods) == 0:
self.log.warning('No module %s available' % str(mod))
else:
self.log.warning('More than one module found for %s: %s' % (mod, mods))
continue
def test_buildininstalldir(self):
"""Test specifying build in install dir."""
config.variables['buildpath'] = tempfile.mkdtemp()
config.variables['installpath'] = tempfile.mkdtemp()
self.contents = '\n'.join([
'name = "pi"',
'version = "3.14"',
'homepage = "http://example.com"',
'description = "test easyconfig"',
'toolchain = {"name": "dummy", "version": "dummy"}',
'buildininstalldir = True',
])
self.prep()
eb = EasyBlock(self.eb_file)
eb.gen_builddir()
eb.mod_name = det_full_module_name(eb.cfg) # required by gen_installdir()
eb.gen_installdir()
eb.make_builddir()
eb.make_installdir()
self.assertEqual(eb.builddir, eb.installdir)
self.assertTrue(os.path.isdir(eb.builddir))
# cleanup
shutil.rmtree(config.variables['buildpath'])
shutil.rmtree(config.variables['installpath'])
def process_easyconfig(path, build_options=None, build_specs=None):
"""
Process easyconfig, returning some information for each block
@param path: path to easyconfig file
@param build_options: dictionary specifying build options (e.g. robot_path, check_osdeps, ...)
@param build_specs: dictionary specifying build specifications (e.g. version, toolchain, ...)
"""
blocks = retrieve_blocks_in_spec(path, build_options.get("only_blocks", None))
easyconfigs = []
for spec in blocks:
# process for dependencies and real installversionname
_log.debug("Processing easyconfig %s" % spec)
# create easyconfig
try:
ec = EasyConfig(spec, build_options=build_options, build_specs=build_specs)
except EasyBuildError, err:
msg = "Failed to process easyconfig %s:\n%s" % (spec, err.msg)
_log.exception(msg)
name = ec["name"]
# this app will appear as following module in the list
easyconfig = {
"ec": ec,
"spec": spec,
"module": det_full_module_name(ec),
"dependencies": [],
"builddependencies": [],
}
if len(blocks) > 1:
easyconfig["originalSpec"] = path
# add build dependencies
for dep in ec.builddependencies():
_log.debug("Adding build dependency %s for app %s." % (dep, name))
easyconfig["builddependencies"].append(dep)
# add dependencies (including build dependencies)
for dep in ec.dependencies():
_log.debug("Adding dependency %s for app %s." % (dep, name))
easyconfig["dependencies"].append(dep)
# add toolchain as dependency too
if ec.toolchain.name != DUMMY_TOOLCHAIN_NAME:
dep = ec.toolchain.as_dict()
_log.debug("Adding toolchain %s as dependency for app %s." % (dep, name))
easyconfig["dependencies"].append(dep)
del ec
# this is used by the parallel builder
easyconfig["unresolved_deps"] = copy.deepcopy(easyconfig["dependencies"])
easyconfigs.append(easyconfig)
def add_dependencies(self, dependencies):
""" Verify if the given dependencies exist and add them """
self.log.debug("add_dependencies: adding toolchain dependencies %s" % dependencies)
for dep in dependencies:
mod_name = det_full_module_name(dep)
if not self.modules_tool.exists(mod_name):
self.log.error("add_dependencies: no module found for dependency %s" % str(dep))
else:
self.dependencies.append(dep)
self.log.debug("add_dependencies: added toolchain dependency %s" % dep)
def process_easyconfig(path, onlyBlocks=None, regtest_online=False, validate=True):
"""
Process easyconfig, returning some information for each block
"""
blocks = retrieve_blocks_in_spec(path, onlyBlocks)
easyconfigs = []
for spec in blocks:
# process for dependencies and real installversionname
# - use mod? __init__ and importCfg are ignored.
_log.debug("Processing easyconfig %s" % spec)
# create easyconfig
try:
all_stops = [x[0] for x in EasyBlock.get_steps()]
ec = EasyConfig(spec, validate=validate, valid_module_classes=module_classes(), valid_stops=all_stops)
except EasyBuildError, err:
msg = "Failed to process easyconfig %s:\n%s" % (spec, err.msg)
_log.exception(msg)
name = ec['name']
# this app will appear as following module in the list
easyconfig = {
'ec': ec,
'spec': spec,
'module': det_full_module_name(ec),
'dependencies': [],
'builddependencies': [],
}
if len(blocks) > 1:
easyconfig['originalSpec'] = path
# add build dependencies
for dep in ec.builddependencies():
_log.debug("Adding build dependency %s for app %s." % (dep, name))
easyconfig['builddependencies'].append(dep)
# add dependencies (including build dependencies)
for dep in ec.dependencies():
_log.debug("Adding dependency %s for app %s." % (dep, name))
easyconfig['dependencies'].append(dep)
# add toolchain as dependency too
if ec.toolchain.name != DUMMY_TOOLCHAIN_NAME:
dep = ec.toolchain.as_dict()
_log.debug("Adding toolchain %s as dependency for app %s." % (dep, name))
easyconfig['dependencies'].append(dep)
del ec
# this is used by the parallel builder
easyconfig['unresolved_deps'] = copy.deepcopy(easyconfig['dependencies'])
easyconfigs.append(easyconfig)
def create_job(build_command, easyconfig, output_dir="", conn=None, ppn=None):
"""
Creates a job, to build a *single* easyconfig
build_command is a format string in which a full path to an eb file will be substituted
easyconfig should be in the format as processEasyConfig returns them
output_dir is an optional path. EASYBUILDTESTOUTPUT will be set inside the job with this variable
returns the job
"""
# create command based on build_command template
command = build_command % {'spec': easyconfig['spec']}
# capture PYTHONPATH, MODULEPATH and all variables starting with EASYBUILD
easybuild_vars = {}
for name in os.environ:
if name.startswith("EASYBUILD"):
easybuild_vars[name] = os.environ[name]
others = ["PYTHONPATH", "MODULEPATH"]
for env_var in others:
if env_var in os.environ:
easybuild_vars[env_var] = os.environ[env_var]
_log.info("Dictionary of environment variables passed to job: %s" %
easybuild_vars)
# obtain unique name based on name/easyconfig version tuple
ec_tuple = (easyconfig['ec']['name'],
det_full_ec_version(easyconfig['ec']))
name = '-'.join(ec_tuple)
var = config.oldstyle_environment_variables['test_output_path']
easybuild_vars[var] = os.path.join(os.path.abspath(output_dir), name)
# just use latest build stats
repo = init_repository(get_repository(), get_repositorypath())
buildstats = repo.get_buildstats(*ec_tuple)
resources = {}
if buildstats:
previous_time = buildstats[-1]['build_time']
resources['hours'] = int(math.ceil(previous_time * 2 / 60))
job = PbsJob(command,
name,
easybuild_vars,
resources=resources,
conn=conn,
ppn=ppn)
job.module = det_full_module_name(easyconfig['ec'])
return job
def add_dependencies(self, dependencies):
""" Verify if the given dependencies exist and add them """
self.log.debug("add_dependencies: adding toolchain dependencies %s" %
dependencies)
for dep in dependencies:
mod_name = det_full_module_name(dep)
if not self.modules_tool.exists(mod_name):
self.log.error(
'add_dependencies: no module found for dependency %s' %
str(dep))
else:
self.dependencies.append(dep)
self.log.debug(
'add_dependencies: added toolchain dependency %s' % dep)
def add_module(self, modules):
"""
Check if module exist, if so add to list.
"""
self.log.deprecated(
"Use of add_module function should be replaced by load([<list of modules>])",
'2.0')
for mod in modules:
if isinstance(mod, (list, tuple)):
mod_dict = {
'name': mod[0],
'version': mod[1],
'versionsuffix': '',
'toolchain': {
'name': DUMMY_TOOLCHAIN_NAME,
'version': DUMMY_TOOLCHAIN_VERSION,
},
}
mod_name = det_full_module_name(mod_dict)
elif isinstance(mod, basestring):
mod_name = mod
elif isinstance(mod, dict):
mod_name = det_full_module_name(mod)
else:
self.log.error("Can't add module %s: unknown type" % str(mod))
mods = self.available(mod_name)
if mod_name in mods:
# ok
self._modules.append(mod_name)
else:
if len(mods) == 0:
self.log.warning('No module %s available' % str(mod))
else:
self.log.warning('More then one module found for %s: %s' %
(mod, mods))
continue
def create_job(build_command, easyconfig, output_dir=None, conn=None, ppn=None):
"""
Creates a job, to build a *single* easyconfig
@param build_command: format string for command, full path to an easyconfig file will be substituted in it
@param easyconfig: easyconfig as processed by process_easyconfig
@param output_dir: optional output path; $EASYBUILDTESTOUTPUT will be set inside the job with this variable
@param conn: open connection to PBS server
@param ppn: ppn setting to use (# 'processors' (cores) per node to use)
returns the job
"""
if output_dir is None:
output_dir = 'easybuild-build'
# create command based on build_command template
command = build_command % {'spec': easyconfig['spec']}
# capture PYTHONPATH, MODULEPATH and all variables starting with EASYBUILD
easybuild_vars = {}
for name in os.environ:
if name.startswith("EASYBUILD"):
easybuild_vars[name] = os.environ[name]
others = ["PYTHONPATH", "MODULEPATH"]
for env_var in others:
if env_var in os.environ:
easybuild_vars[env_var] = os.environ[env_var]
_log.info("Dictionary of environment variables passed to job: %s" % easybuild_vars)
# obtain unique name based on name/easyconfig version tuple
ec_tuple = (easyconfig['ec']['name'], det_full_ec_version(easyconfig['ec']))
name = '-'.join(ec_tuple)
var = config.oldstyle_environment_variables['test_output_path']
easybuild_vars[var] = os.path.join(os.path.abspath(output_dir), name)
# just use latest build stats
repo = init_repository(get_repository(), get_repositorypath())
buildstats = repo.get_buildstats(*ec_tuple)
resources = {}
if buildstats:
previous_time = buildstats[-1]['build_time']
resources['hours'] = int(math.ceil(previous_time * 2 / 60))
job = PbsJob(command, name, easybuild_vars, resources=resources, conn=conn, ppn=ppn)
job.module = det_full_module_name(easyconfig['ec'])
return job
def test_gen_dirs(self):
"""Test methods that generate/set build/install directory names."""
self.contents = '\n'.join([
"name = 'pi'",
"version = '3.14'",
"homepage = 'http://example.com'",
"description = 'test easyconfig'",
"toolchain = {'name': 'dummy', 'version': 'dummy'}",
])
self.writeEC()
stdoutorig = sys.stdout
sys.stdout = open("/dev/null", 'w')
eb = EasyBlock(EasyConfig(self.eb_file))
resb = eb.gen_builddir()
eb.mod_name = det_full_module_name(
eb.cfg) # required by gen_installdir()
resi = eb.gen_installdir()
eb.make_builddir()
eb.make_installdir()
# doesn't return anything
self.assertEqual(resb, None)
self.assertEqual(resi, None)
# directories are set, and exist
self.assertTrue(os.path.isdir(eb.builddir))
self.assertTrue(os.path.isdir(eb.installdir))
# make sure cleaning up old build dir is default
self.assertTrue(eb.cfg['cleanupoldbuild']
or eb.cfg.get('cleanupoldbuild', True))
builddir = eb.builddir
eb.gen_builddir()
self.assertEqual(builddir, eb.builddir)
eb.cfg['cleanupoldbuild'] = True
eb.gen_builddir()
self.assertEqual(builddir, eb.builddir)
# make sure build dir is unique
eb.cfg['cleanupoldbuild'] = False
builddir = eb.builddir
for i in range(3):
eb.gen_builddir()
self.assertEqual(eb.builddir, "%s.%d" % (builddir, i))
eb.make_builddir()
# cleanup
sys.stdout.close()
sys.stdout = stdoutorig
eb.close_log()
def print_dry_run(easyconfigs, robot=None):
if robot is None:
print_msg("Dry run: printing build status of easyconfigs")
all_specs = easyconfigs
else:
print_msg("Dry run: printing build status of easyconfigs and dependencies")
all_specs = resolve_dependencies(easyconfigs, robot, True)
unbuilt_specs = skip_available(all_specs, True)
dry_run_fmt = "%3s %s (module: %s)"
for spec in all_specs:
if spec in unbuilt_specs:
ans = '[ ]'
else:
ans = '[x]'
mod = det_full_module_name(spec['ec'])
print dry_run_fmt % (ans, spec['spec'], mod)
def find_resolved_modules(unprocessed, processed):
"""
Find easyconfigs in unprocessed which can be fully resolved using easyconfigs in processed
"""
ordered_ecs = []
for ec in unprocessed:
ec['dependencies'] = [d for d in ec['dependencies'] if not det_full_module_name(d) in processed]
if len(ec['dependencies']) == 0:
_log.debug("Adding easyconfig %s to final list" % ec['spec'])
ordered_ecs.append(ec)
processed.append(ec['module'])
unprocessed[:] = [m for m in unprocessed if len(m['dependencies']) > 0]
return ordered_ecs
def test_gen_dirs(self):
"""Test methods that generate/set build/install directory names."""
self.contents = '\n'.join([
"name = 'pi'",
"version = '3.14'",
"homepage = 'http://example.com'",
"description = 'test easyconfig'",
"toolchain = {'name': 'dummy', 'version': 'dummy'}",
])
self.writeEC()
stdoutorig = sys.stdout
sys.stdout = open("/dev/null", 'w')
eb = EasyBlock(EasyConfig(self.eb_file))
resb = eb.gen_builddir()
eb.mod_name = det_full_module_name(eb.cfg) # required by gen_installdir()
resi = eb.gen_installdir()
eb.make_builddir()
eb.make_installdir()
# doesn't return anything
self.assertEqual(resb, None)
self.assertEqual(resi, None)
# directories are set, and exist
self.assertTrue(os.path.isdir(eb.builddir))
self.assertTrue(os.path.isdir(eb.installdir))
# make sure cleaning up old build dir is default
self.assertTrue(eb.cfg['cleanupoldbuild'] or eb.cfg.get('cleanupoldbuild', True))
builddir = eb.builddir
eb.gen_builddir()
self.assertEqual(builddir, eb.builddir)
eb.cfg['cleanupoldbuild'] = True
eb.gen_builddir()
self.assertEqual(builddir, eb.builddir)
# make sure build dir is unique
eb.cfg['cleanupoldbuild'] = False
builddir = eb.builddir
for i in range(3):
eb.gen_builddir()
self.assertEqual(eb.builddir, "%s.%d" % (builddir, i))
eb.make_builddir()
# cleanup
sys.stdout.close()
sys.stdout = stdoutorig
eb.close_log()
def create_job(build_command, easyconfig, output_dir="", conn=None, ppn=None):
"""
Creates a job, to build a *single* easyconfig
build_command is a format string in which a full path to an eb file will be substituted
easyconfig should be in the format as processEasyConfig returns them
output_dir is an optional path. EASYBUILDTESTOUTPUT will be set inside the job with this variable
returns the job
"""
# create command based on build_command template
command = build_command % {'spec': easyconfig['spec']}
# capture PYTHONPATH, MODULEPATH and all variables starting with EASYBUILD
easybuild_vars = {}
for name in os.environ:
if name.startswith("EASYBUILD"):
easybuild_vars[name] = os.environ[name]
others = ["PYTHONPATH", "MODULEPATH"]
for env_var in others:
if env_var in os.environ:
easybuild_vars[env_var] = os.environ[env_var]
_log.info("Dictionary of environment variables passed to job: %s" % easybuild_vars)
# obtain unique name based on name/easyconfig version tuple
ec_tuple = (easyconfig['ec']['name'], det_full_ec_version(easyconfig['ec']))
name = '-'.join(ec_tuple)
var = config.oldstyle_environment_variables['test_output_path']
easybuild_vars[var] = os.path.join(os.path.abspath(output_dir), name)
# just use latest build stats
repo = init_repository(get_repository(), get_repositorypath())
buildstats = repo.get_buildstats(*ec_tuple)
resources = {}
if buildstats:
previous_time = buildstats[-1]['build_time']
resources['hours'] = int(math.ceil(previous_time * 2 / 60))
job = PbsJob(command, name, easybuild_vars, resources=resources, conn=conn, ppn=ppn)
job.module = det_full_module_name(easyconfig['ec'])
return job
def print_dry_run(easyconfigs, short=False, build_options=None, build_specs=None):
"""
Print dry run information
@param easyconfigs: list of easyconfig files
@param short: print short output (use a variable for the common prefix)
@param build_options: dictionary specifying build options (e.g. robot_path, check_osdeps, ...)
@param build_specs: dictionary specifying build specifications (e.g. version, toolchain, ...)
"""
lines = []
if build_options.get("robot_path", None) is None:
lines.append("Dry run: printing build status of easyconfigs")
all_specs = easyconfigs
else:
lines.append("Dry run: printing build status of easyconfigs and dependencies")
build_options = copy.deepcopy(build_options)
build_options.update({"retain_all_deps": True, "check_osdeps": False})
all_specs = resolve_dependencies(easyconfigs, build_options=build_options, build_specs=build_specs)
unbuilt_specs = skip_available(all_specs, testing=True)
dry_run_fmt = " * [%1s] %s (module: %s)" # markdown compatible (list of items with checkboxes in front)
var_name = "CFGS"
common_prefix = det_common_path_prefix([spec["spec"] for spec in all_specs])
# only allow short if common prefix is long enough
short = short and common_prefix is not None and len(common_prefix) > len(var_name) * 2
for spec in all_specs:
if spec in unbuilt_specs:
ans = " "
else:
ans = "x"
mod = det_full_module_name(spec["ec"])
if short:
item = os.path.join("$%s" % var_name, spec["spec"][len(common_prefix) + 1 :])
else:
item = spec["spec"]
lines.append(dry_run_fmt % (ans, item, mod))
if short:
# insert after 'Dry run:' message
lines.insert(1, "%s=%s" % (var_name, common_prefix))
silent = build_options.get("silent", False)
print_msg("\n".join(lines), log=_log, silent=silent, prefix=False)
def test_buildininstalldir(self):
"""Test specifying build in install dir."""
self.contents = '\n'.join([
'name = "pi"',
'version = "3.14"',
'homepage = "http://example.com"',
'description = "test easyconfig"',
'toolchain = {"name": "dummy", "version": "dummy"}',
'buildininstalldir = True',
])
self.prep()
ec = EasyConfig(self.eb_file)
eb = EasyBlock(ec)
eb.gen_builddir()
eb.mod_name = det_full_module_name(eb.cfg) # required by gen_installdir()
eb.gen_installdir()
eb.make_builddir()
eb.make_installdir()
self.assertEqual(eb.builddir, eb.installdir)
self.assertTrue(os.path.isdir(eb.builddir))
def find_resolved_modules(unprocessed, avail_modules):
"""
Find easyconfigs in 1st argument which can be fully resolved using modules specified in 2nd argument
"""
ordered_ecs = []
new_avail_modules = avail_modules[:]
new_unprocessed = []
for ec in unprocessed:
new_ec = ec.copy()
new_ec["dependencies"] = [d for d in new_ec["dependencies"] if not det_full_module_name(d) in new_avail_modules]
if len(new_ec["dependencies"]) == 0:
_log.debug("Adding easyconfig %s to final list" % new_ec["spec"])
ordered_ecs.append(new_ec)
new_avail_modules.append(ec["module"])
else:
new_unprocessed.append(new_ec)
return ordered_ecs, new_unprocessed, new_avail_modules
def resolve_dependencies(unprocessed, robot, force=False):
"""
Work through the list of easyconfigs to determine an optimal order
enabling force results in retaining all dependencies and skipping validation of easyconfigs
"""
if force:
# assume that no modules are available when forced
available_modules = []
_log.info("Forcing all dependencies to be retained.")
else:
# Get a list of all available modules (format: [(name, installversion), ...])
available_modules = modules_tool().available()
if len(available_modules) == 0:
_log.warning("No installed modules. Your MODULEPATH is probably incomplete: %s" % os.getenv('MODULEPATH'))
ordered_ecs = []
# All available modules can be used for resolving dependencies except
# those that will be installed
being_installed = [p['module'] for p in unprocessed]
processed = [m for m in available_modules if not m in being_installed]
_log.debug('unprocessed before resolving deps: %s' % unprocessed)
# as long as there is progress in processing the modules, keep on trying
loopcnt = 0
maxloopcnt = 10000
robot_add_dep = True
while robot_add_dep:
robot_add_dep = False
# make sure this stops, we really don't want to get stuck in an infinite loop
loopcnt += 1
if loopcnt > maxloopcnt:
msg = "Maximum loop cnt %s reached, so quitting." % maxloopcnt
_log.error(msg)
# first try resolving dependencies without using external dependencies
last_processed_count = -1
while len(processed) > last_processed_count:
last_processed_count = len(processed)
ordered_ecs.extend(find_resolved_modules(unprocessed, processed))
# robot: look for an existing dependency, add one
if robot and len(unprocessed) > 0:
being_installed = [det_full_module_name(p['ec'], eb_ns=True) for p in unprocessed]
for entry in unprocessed:
# do not choose an entry that is being installed in the current run
# if they depend, you probably want to rebuild them using the new dependency
deps = entry['dependencies']
candidates = [d for d in deps if not det_full_module_name(d, eb_ns=True) in being_installed]
if len(candidates) > 0:
cand_dep = candidates[0]
# find easyconfig, might not find any
_log.debug("Looking for easyconfig for %s" % str(cand_dep))
path = robot_find_easyconfig(robot, cand_dep['name'], det_full_ec_version(cand_dep))
else:
path = None
mod_name = det_full_module_name(entry['ec'], eb_ns=True)
_log.debug("No more candidate dependencies to resolve for %s" % mod_name)
if path is not None:
cand_dep = candidates[0]
_log.info("Robot: resolving dependency %s with %s" % (cand_dep, path))
processed_ecs = process_easyconfig(path, validate=(not force))
# ensure that selected easyconfig provides required dependency
mods = [det_full_module_name(spec['ec']) for spec in processed_ecs]
dep_mod_name = det_full_module_name(cand_dep)
if not dep_mod_name in mods:
_log.error("easyconfig file %s does not contain module %s (mods: %s)" % (path, dep_mod_name, mods))
unprocessed.extend(processed_ecs)
robot_add_dep = True
break
_log.debug('unprocessed after resolving deps: %s' % unprocessed)
# there are dependencies that cannot be resolved
if len(unprocessed) > 0:
_log.debug("List of unresolved dependencies: %s" % unprocessed)
missing_dependencies = []
for ec in unprocessed:
for dep in ec['dependencies']:
missing_dependencies.append('%s for %s' % (det_full_module_name(dep, eb_ns=True), dep))
msg = "Dependencies not met. Cannot resolve %s" % missing_dependencies
_log.error(msg)
_log.info("Dependency resolution complete, building as follows:\n%s" % ordered_ecs)
return ordered_ecs
def det_module_name(self, name=None, version=None):
"""Determine module name for this toolchain."""
return det_full_module_name(self.as_dict(name, version))
def mk_dep_mod_name(spec):
return tuple(det_full_module_name(spec).split(os.path.sep))
def tokey(dep):
"""Determine key for specified dependency."""
return det_full_module_name(dep)
def tokey(dep):
"""Determine key for specified dependency."""
return det_full_module_name(dep)
def prepare(self, onlymod=None):
"""
Prepare a set of environment parameters based on name/version of toolchain
- load modules for toolchain and dependencies
- generate extra variables and set them in the environment
onlymod: Boolean/string to indicate if the toolchain should only load the environment
with module (True) or also set all other variables (False) like compiler CC etc
(If string: comma separated list of variables that will be ignored).
"""
if self.modules_tool is None:
self.log.raiseException("No modules tool defined.")
if not self._toolchain_exists():
self.log.raiseException(
"No module found for toolchain name '%s' (%s)" %
(self.name, self.version))
if self.name == DUMMY_TOOLCHAIN_NAME:
if self.version == DUMMY_TOOLCHAIN_VERSION:
self.log.info(
'prepare: toolchain dummy mode, dummy version; not loading dependencies'
)
else:
self.log.info(
'prepare: toolchain dummy mode and loading dependencies')
self.modules_tool.load(
[det_full_module_name(dep) for dep in self.dependencies])
return
# Load the toolchain and dependencies modules
self.log.debug("Loading toolchain module and dependencies...")
self.modules_tool.load([self.det_module_name()])
self.modules_tool.load(
[det_full_module_name(dep) for dep in self.dependencies])
# determine direct toolchain dependencies
mod_name = self.det_module_name()
self.toolchain_dependencies = self.modules_tool.dependencies_for(
mod_name, depth=0)
self.log.debug('prepare: list of direct toolchain dependencies: %s' %
self.toolchain_dependencies)
# verify whether elements in toolchain definition match toolchain deps specified by loaded toolchain module
toolchain_module_deps = set([
self.modules_tool.module_software_name(d)
for d in self.toolchain_dependencies
])
toolchain_elements_mod_names = set([
y for x in dir(self) if x.endswith('_MODULE_NAME')
for y in eval("self.%s" % x)
])
# filter out toolchain name (e.g. 'GCC') from list of toolchain elements
toolchain_elements_mod_names = set(
[x for x in toolchain_elements_mod_names if not x == self.name])
# filter out optional toolchain elements if they're not used in the module
for mod_name in toolchain_elements_mod_names.copy():
if not self.is_required(mod_name):
if not mod_name in toolchain_module_deps:
self.log.debug(
"Removing optional module %s from list of toolchain elements."
% mod_name)
toolchain_elements_mod_names.remove(mod_name)
self.log.debug(
"List of toolchain dependency modules from loaded toolchain module: %s"
% toolchain_module_deps)
self.log.debug(
"List of toolchain elements from toolchain definition: %s" %
toolchain_elements_mod_names)
if toolchain_module_deps == toolchain_elements_mod_names:
self.log.info(
"List of toolchain dependency modules and toolchain definition match!"
)
else:
self.log.error("List of toolchain dependency modules and toolchain definition do not match " \
"(%s vs %s)" % (toolchain_module_deps, toolchain_elements_mod_names))
# Generate the variables to be set
self.set_variables()
# set the variables
# onlymod can be comma-separated string of variables not to be set
if onlymod == True:
self.log.debug(
"prepare: do not set additional variables onlymod=%s" %
onlymod)
self.generate_vars()
else:
self.log.debug("prepare: set additional variables onlymod=%s" %
onlymod)
# add LDFLAGS and CPPFLAGS from dependencies to self.vars
self._add_dependency_variables()
self.generate_vars()
self._setenv_variables(onlymod)
def resolve_dependencies(unprocessed, build_options=None, build_specs=None):
"""
Work through the list of easyconfigs to determine an optimal order
@param unprocessed: list of easyconfigs
@param build_options: dictionary specifying build options (e.g. robot_path, check_osdeps, ...)
@param build_specs: dictionary specifying build specifications (e.g. version, toolchain, ...)
"""
robot = build_options.get("robot_path", None)
if build_options.get("retain_all_deps", False):
# assume that no modules are available when forced, to retain all dependencies
avail_modules = []
_log.info("Forcing all dependencies to be retained.")
else:
# Get a list of all available modules (format: [(name, installversion), ...])
avail_modules = modules_tool().available()
if len(avail_modules) == 0:
_log.warning("No installed modules. Your MODULEPATH is probably incomplete: %s" % os.getenv("MODULEPATH"))
ordered_ecs = []
# all available modules can be used for resolving dependencies except those that will be installed
being_installed = [p["module"] for p in unprocessed]
avail_modules = [m for m in avail_modules if not m in being_installed]
_log.debug("unprocessed before resolving deps: %s" % unprocessed)
# resolve all dependencies, put a safeguard in place to avoid an infinite loop (shouldn't occur though)
irresolvable = []
loopcnt = 0
maxloopcnt = 10000
while unprocessed:
# make sure this stops, we really don't want to get stuck in an infinite loop
loopcnt += 1
if loopcnt > maxloopcnt:
tup = (maxloopcnt, unprocessed, irresolvable)
msg = "Maximum loop cnt %s reached, so quitting (unprocessed: %s, irresolvable: %s)" % tup
_log.error(msg)
# first try resolving dependencies without using external dependencies
last_processed_count = -1
while len(avail_modules) > last_processed_count:
last_processed_count = len(avail_modules)
more_ecs, unprocessed, avail_modules = find_resolved_modules(unprocessed, avail_modules)
for ec in more_ecs:
if not ec["module"] in [x["module"] for x in ordered_ecs]:
ordered_ecs.append(ec)
# robot: look for existing dependencies, add them
if robot and unprocessed:
being_installed = [det_full_module_name(p["ec"], eb_ns=True) for p in unprocessed]
additional = []
for i, entry in enumerate(unprocessed):
# do not choose an entry that is being installed in the current run
# if they depend, you probably want to rebuild them using the new dependency
deps = entry["dependencies"]
candidates = [d for d in deps if not det_full_module_name(d, eb_ns=True) in being_installed]
if len(candidates) > 0:
cand_dep = candidates[0]
# find easyconfig, might not find any
_log.debug("Looking for easyconfig for %s" % str(cand_dep))
# note: robot_find_easyconfig may return None
path = robot_find_easyconfig(robot, cand_dep["name"], det_full_ec_version(cand_dep))
if path is None:
# no easyconfig found for dependency, add to list of irresolvable dependencies
if cand_dep not in irresolvable:
irresolvable.append(cand_dep)
# remove irresolvable dependency from list of dependencies so we can continue
entry["dependencies"].remove(cand_dep)
else:
_log.info("Robot: resolving dependency %s with %s" % (cand_dep, path))
processed_ecs = process_easyconfig(path, build_options=build_options, build_specs=build_specs)
# ensure that selected easyconfig provides required dependency
mods = [det_full_module_name(spec["ec"]) for spec in processed_ecs]
dep_mod_name = det_full_module_name(cand_dep)
if not dep_mod_name in mods:
tup = (path, dep_mod_name, mods)
_log.error("easyconfig file %s does not contain module %s (mods: %s)" % tup)
for ec in processed_ecs:
if not ec in unprocessed + additional:
additional.append(ec)
_log.debug("Added %s as dependency of %s" % (ec, entry))
else:
mod_name = det_full_module_name(entry["ec"], eb_ns=True)
_log.debug("No more candidate dependencies to resolve for %s" % mod_name)
# add additional (new) easyconfigs to list of stuff to process
unprocessed.extend(additional)
if irresolvable:
irresolvable_mod_deps = [(det_full_module_name(dep, eb_ns=True), dep) for dep in irresolvable]
_log.error("Irresolvable dependencies encountered: %s" % irresolvable_mod_deps)
_log.info("Dependency resolution complete, building as follows:\n%s" % ordered_ecs)
return ordered_ecs
_log.debug("Build stats: %s" % buildstats)
if app.cfg['stop']:
ended = "STOPPED"
newLogDir = os.path.join(app.builddir, config.log_path())
else:
newLogDir = os.path.join(app.installdir, config.log_path())
try:
# upload spec to central repository
repo = init_repository(get_repository(), get_repositorypath())
if 'originalSpec' in module:
block = det_full_ec_version(app.cfg) + ".block"
repo.add_easyconfig(module['originalSpec'], app.name, block, buildstats, currentbuildstats)
repo.add_easyconfig(spec, app.name, det_full_ec_version(app.cfg), buildstats, currentbuildstats)
repo.commit("Built %s" % det_full_module_name(app.cfg))
del repo
except EasyBuildError, err:
_log.warn("Unable to commit easyconfig to repository: %s", err)
exitCode = 0
succ = "successfully"
summary = "COMPLETED"
# cleanup logs
app.close_log()
try:
if not os.path.isdir(newLogDir):
os.makedirs(newLogDir)
applicationLog = os.path.join(newLogDir, get_log_filename(app.name, app.version))
shutil.move(app.logfile, applicationLog)
def det_module_name(self, name=None, version=None):
"""Determine module name for this toolchain."""
return det_full_module_name(self.as_dict(name, version))
def mk_node_name(spec):
if omit_versions:
return spec["name"]
else:
return det_full_module_name(spec)
def mk_dep_mod_name(spec):
return tuple(det_full_module_name(spec).split(os.path.sep))
def prepare(self, onlymod=None):
"""
Prepare a set of environment parameters based on name/version of toolchain
- load modules for toolchain and dependencies
- generate extra variables and set them in the environment
onlymod: Boolean/string to indicate if the toolchain should only load the environment
with module (True) or also set all other variables (False) like compiler CC etc
(If string: comma separated list of variables that will be ignored).
"""
if self.modules_tool is None:
self.log.raiseException("No modules tool defined.")
if not self._toolchain_exists():
self.log.raiseException("No module found for toolchain name '%s' (%s)" % (self.name, self.version))
if self.name == DUMMY_TOOLCHAIN_NAME:
if self.version == DUMMY_TOOLCHAIN_VERSION:
self.log.info("prepare: toolchain dummy mode, dummy version; not loading dependencies")
else:
self.log.info("prepare: toolchain dummy mode and loading dependencies")
self.modules_tool.load([det_full_module_name(dep) for dep in self.dependencies])
return
# Load the toolchain and dependencies modules
self.log.debug("Loading toolchain module and dependencies...")
self.modules_tool.load([self.det_module_name()])
self.modules_tool.load([det_full_module_name(dep) for dep in self.dependencies])
# determine direct toolchain dependencies
mod_name = self.det_module_name()
self.toolchain_dependencies = self.modules_tool.dependencies_for(mod_name, depth=0)
self.log.debug("prepare: list of direct toolchain dependencies: %s" % self.toolchain_dependencies)
# verify whether elements in toolchain definition match toolchain deps specified by loaded toolchain module
toolchain_module_deps = set([self.modules_tool.module_software_name(d) for d in self.toolchain_dependencies])
toolchain_elements_mod_names = set(
[y for x in dir(self) if x.endswith("_MODULE_NAME") for y in eval("self.%s" % x)]
)
# filter out toolchain name (e.g. 'GCC') from list of toolchain elements
toolchain_elements_mod_names = set([x for x in toolchain_elements_mod_names if not x == self.name])
# filter out optional toolchain elements if they're not used in the module
for mod_name in toolchain_elements_mod_names.copy():
if not self.is_required(mod_name):
if not mod_name in toolchain_module_deps:
self.log.debug("Removing optional module %s from list of toolchain elements." % mod_name)
toolchain_elements_mod_names.remove(mod_name)
self.log.debug("List of toolchain dependency modules from loaded toolchain module: %s" % toolchain_module_deps)
self.log.debug("List of toolchain elements from toolchain definition: %s" % toolchain_elements_mod_names)
if toolchain_module_deps == toolchain_elements_mod_names:
self.log.info("List of toolchain dependency modules and toolchain definition match!")
else:
self.log.error(
"List of toolchain dependency modules and toolchain definition do not match "
"(%s vs %s)" % (toolchain_module_deps, toolchain_elements_mod_names)
)
# Generate the variables to be set
self.set_variables()
# set the variables
# onlymod can be comma-separated string of variables not to be set
if onlymod == True:
self.log.debug("prepare: do not set additional variables onlymod=%s" % onlymod)
self.generate_vars()
else:
self.log.debug("prepare: set additional variables onlymod=%s" % onlymod)
# add LDFLAGS and CPPFLAGS from dependencies to self.vars
self._add_dependency_variables()
self.generate_vars()
self._setenv_variables(onlymod)