def setUp(self):
""" dynamically replace Modules class with MockModule """
# replace Modules class with something we have control over
modules.Modules = MockModule
main.Modules = MockModule
self.log = get_log("RobotTest")
main.log = get_log("main") # redefine the main log when calling the main functions directly
self.cwd = os.getcwd()
self.base_easyconfig_dir = find_full_path(os.path.join("easybuild", "test", "easyconfigs"))
self.assertTrue(self.base_easyconfig_dir)
def __init__(self, script, name, env_vars=None, resources={}, conn=None, ppn=None):
"""
create a new Job to be submitted to PBS
env_vars is a dictionary with key-value pairs of environment variables that should be passed on to the job
resources is a dictionary with optional keys: ['hours', 'cores'] both of these should be integer values.
hours can be 1 - MAX_WALLTIME, cores depends on which cluster it is being run.
"""
self.clean_conn = True
self.log = get_log(self.__class__.__name__)
self.script = script
if env_vars:
self.env_vars = env_vars.copy()
else:
self.env_vars = {}
self.name = name
if pbs_import_failed:
self.log.error(pbs_import_failed)
try:
self.pbs_server = pbs.pbs_default()
if conn:
self.pbsconn = conn
self.clean_conn = False
else:
self.pbsconn = pbs.pbs_connect(self.pbs_server)
except Exception, err:
self.log.error("Failed to connect to the default pbs server: %s" % err)
def setUp(self):
""" create temporary easyconfig file """
self.log = get_log("EasyConfigTest")
if self.contents is not None:
fd, self.eb_file = tempfile.mkstemp(prefix='easyconfig_test_file_', suffix='.eb')
os.close(fd)
f = open(self.eb_file, "w")
f.write(self.contents)
f.close()
self.cwd = os.getcwd()
self.all_stops = [x[0] for x in EasyBlock.get_steps()]
def get_ppn():
"""Guess the ppn for full node"""
log = get_log('pbs_job.get_ppn')
pq = PBSQuery()
node_vals = pq.getnodes().values() # only the values, not the names
interesting_nodes = ('free', 'job-exclusive',)
res = {}
for np in [int(x['np'][0]) for x in node_vals if x['state'][0] in interesting_nodes]:
res.setdefault(np, 0)
res[np] += 1
# return most frequent
freq_count, freq_np = max([(j, i) for i, j in res.items()])
log.debug("Found most frequent np %s (%s times) in interesting nodes %s" % (freq_np, freq_count, interesting_nodes))
return freq_np
def __init__(self, modulePath=None):
"""
Create a Modules object
@param modulePath: A list of paths where the modules can be located
@type modulePath: list
"""
self.log = get_log(self.__class__.__name__)
# make sure we don't have the same path twice
if modulePath:
self.modulePath = set(modulePath)
else:
self.modulePath = None
self.modules = []
self.check_module_path()
# make sure environment-modules is installed
ec = subprocess.call(["which", "modulecmd"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
if ec:
msg = "Could not find the modulecmd command, environment-modules is not installed?\n"
msg += "Exit code of 'which modulecmd': %d" % ec
self.log.error(msg)
raise EasyBuildError(msg)
@author: Pieter De Baets (Ghent University)
@author: Jens Timmerman (Ghent University)
@author: Fotis Georgatos (Uni.Lu)
"""
import os
import shutil
import tempfile
from easybuild.tools.build_log import get_log
from easybuild.tools.config import install_path
from easybuild.tools.filetools import rmtree2
from easybuild.tools.modules import Modules
from easybuild.tools.utilities import quote_str
log = get_log('moduleGenerator')
# general module class
GENERAL_CLASS = 'all'
class ModuleGenerator(object):
"""
Class for generating module files.
"""
def __init__(self, application, fake=False):
self.app = application
self.fake = fake
self.filename = None
self.tmpdir = None
import re
import shutil
import signal
import stat
import subprocess
import tempfile
import time
import urllib
import easybuild.tools.environment as env
from easybuild.tools.asyncprocess import Popen, PIPE, STDOUT
from easybuild.tools.asyncprocess import send_all, recv_some
from easybuild.tools.build_log import get_log
log = get_log('fileTools')
errorsFoundInLog = 0
# constants for strictness levels
IGNORE = 'ignore'
WARN = 'warn'
ERROR = 'error'
# default strictness level
strictness = WARN
def extract_file(fn, dest, cmd=None, extra_options=None, overwrite=False):
"""
Given filename fn, try to extract in directory dest
- returns the directory name in case of success
except ImportError:
pass
# PySVN
try:
import pysvn #@UnusedImport
from pysvn import ClientError #IGNORE:E0611 pysvn fails to recognize ClientError is available
except ImportError:
pass
from easybuild.framework.easyconfig import EasyConfig, stats_to_str
from easybuild.tools.build_log import get_log
from easybuild.tools.version import VERBOSE_VERSION
log = get_log('repo')
class Repository(object):
"""
Interface for repositories
"""
def __init__(self, repo_path, subdir=''):
"""
Initialize a repository. self.repo and self.subdir will be set.
self.wc will be set to None.
Then, setupRepo and createWorkingCopy will be called (in that order)
"""
self.subdir = subdir
self.repo = repo_path
self.wc = None
self.setup_repo()
@author: Stijn De Weirdt (Ghent University)
@author: Dries Verdegem (Ghent University)
@author: Kenneth Hoste (Ghent University)
@author: Pieter De Baets (Ghent University)
@author: Jens Timmerman (Ghent University)
@author: Toon Willems (Ghent University)
"""
import os
import tempfile
import time
from easybuild.tools.build_log import get_log
import easybuild.tools.repository as repo
_log = get_log('config')
variables = {}
requiredVariables = ['build_path', 'install_path', 'source_path', 'log_format', 'repository']
environmentVariables = {
'build_path': 'EASYBUILDBUILDPATH', # temporary build path
'install_path': 'EASYBUILDINSTALLPATH', # final install path
'log_dir': 'EASYBUILDLOGDIR', # log directory where temporary log files are stored
'config_file': 'EASYBUILDCONFIG', # path to the config file
'test_output_path': 'EASYBUILDTESTOUTPUT', # path to where jobs should place test output
'source_path': 'EASYBUILDSOURCEPATH', # path to where sources should be downloaded
'log_format': 'EASYBUILDLOGFORMAT', # format of the log file
}
def get_user_easybuild_dir():
"""Return the per-user easybuild dir (e.g. to store config files)"""
@author: Stijn De Weirdt (Ghent University)
@author: Dries Verdegem (Ghent University)
@author: Kenneth Hoste (Ghent University)
@author: Pieter De Baets (Ghent University)
@author: Jens Timmerman (Ghent University)
@author: Toon Willems (Ghent University)
"""
import os
import tempfile
from easybuild.tools.build_log import get_log
import easybuild.tools.config as config
log = get_log('easybuild_config')
# this should result in a MODULEPATH=($HOME/.local/easybuild|$EASYBUILDPREFIX)/<modules install suffix>/all
if os.getenv('EASYBUILDPREFIX'):
prefix = os.getenv('EASYBUILDPREFIX')
else:
prefix = os.path.join(os.getenv('HOME'), ".local", "easybuild")
# build/install/source paths configuration for EasyBuild
# build_path possibly overridden by EASYBUILDBUILDPATH
# install_path possibly overridden by EASYBUILDINSTALLPATH
build_path = os.path.join(prefix, 'build')
install_path = prefix
source_path = os.path.join(prefix, 'sources')
# repository for eb files
#
# You should have received a copy of the GNU General Public License
# along with EasyBuild. If not, see <http://www.gnu.org/licenses/>.
##
"""
Interface module to TORQUE (PBS).
@author: Stijn De Weirdt (Ghent University)
@author: Toon Willems (Ghent University)
@author: Kenneth Hoste (Ghent University)
"""
import os
from easybuild.tools.build_log import get_log
_log = get_log('pbs_job')
pbs_import_failed = None
try:
from PBSQuery import PBSQuery
import pbs
except ImportError:
_log.debug("Failed to import pbs from pbs_python. Silently ignoring, is only a real issue with --job")
pbs_import_failed = ("PBSQuery or pbs modules not available. "
"Please make sure pbs_python is installed and usable.")
MAX_WALLTIME = 72
def connect_to_server(pbs_server=None):
"""Connect to PBS server and return connection."""
if pbs_import_failed:
def search_toolchain(name):
"""
Find a toolchain with matching name
returns toolchain (or None), found_toolchains
"""
log = get_log("search_toolchain")
# import all available toolchains, so we know about them
tc_modules = []
for path in sys.path:
for module in glob.glob(os.path.join(path, 'easybuild', 'toolchains', '*.py')):
if not module.endswith('__init__.py'):
modpath = "easybuild.toolchains.%s" % module.split(os.path.sep)[-1].split('.')[0]
log.debug("importing toolchain module %s" % modpath)
tc_modules.append(__import__(modpath, globals(), locals(), ['']))
# make sure all defined toolchain constants are available in toolchain module
package = easybuild.tools.toolchain
tc_const_prefix = 'TC_CONSTANT_'
tc_const_re = re.compile('^%s(.*)$' % tc_const_prefix)
for tc_mod in tc_modules:
# determine classes imported in this module
mod_classes = []
for elem in [getattr(tc_mod, x) for x in dir(tc_mod)]:
if hasattr(elem, '__module__'):
# exclude the toolchain class defined in that module
if not tc_mod.__file__ == sys.modules[elem.__module__].__file__:
log.debug("Adding %s to list of imported classes used for looking for constants" % elem.__name__)
mod_classes.append(elem)
# look for constants in modules of imported classes, and make them available
for mod_class_mod in [sys.modules[mod_class.__module__] for mod_class in mod_classes]:
for elem in dir(mod_class_mod):
res = tc_const_re.match(elem)
if res:
tc_const_name = res.group(1)
tc_const_value = getattr(mod_class_mod, elem)
log.debug("Found constant %s ('%s') in module %s, adding it to %s" % (tc_const_name,
tc_const_value,
mod_class_mod.__name__,
package.__name__))
if hasattr(package, tc_const_name):
cur_value = getattr(package, tc_const_name)
if not tc_const_value == cur_value:
log.error("Constant %s.%s defined as '%s', can't set it to '%s'." % (package.__name__,
tc_const_name,
cur_value,
tc_const_value
))
else:
setattr(package, tc_const_name, tc_const_value)
# obtain all subclasses of toolchain
found_tcs = get_subclasses(Toolchain)
# filter found toolchain subclasses based on whether they can be used a toolchains
found_tcs = [tc for tc in found_tcs if tc._is_toolchain_for(None)]
for tc in found_tcs:
if tc._is_toolchain_for(name):
return tc, found_tcs
return None, found_tcs