def setUp(self): """ dynamically replace Modules class with MockModule """ # replace Modules class with something we have control over modules.Modules = MockModule main.Modules = MockModule self.log = get_log("RobotTest") main.log = get_log("main") # redefine the main log when calling the main functions directly self.cwd = os.getcwd() self.base_easyconfig_dir = find_full_path(os.path.join("easybuild", "test", "easyconfigs")) self.assertTrue(self.base_easyconfig_dir)
def __init__(self, script, name, env_vars=None, resources={}, conn=None, ppn=None): """ create a new Job to be submitted to PBS env_vars is a dictionary with key-value pairs of environment variables that should be passed on to the job resources is a dictionary with optional keys: ['hours', 'cores'] both of these should be integer values. hours can be 1 - MAX_WALLTIME, cores depends on which cluster it is being run. """ self.clean_conn = True self.log = get_log(self.__class__.__name__) self.script = script if env_vars: self.env_vars = env_vars.copy() else: self.env_vars = {} self.name = name if pbs_import_failed: self.log.error(pbs_import_failed) try: self.pbs_server = pbs.pbs_default() if conn: self.pbsconn = conn self.clean_conn = False else: self.pbsconn = pbs.pbs_connect(self.pbs_server) except Exception, err: self.log.error("Failed to connect to the default pbs server: %s" % err)
def setUp(self): """ create temporary easyconfig file """ self.log = get_log("EasyConfigTest") if self.contents is not None: fd, self.eb_file = tempfile.mkstemp(prefix='easyconfig_test_file_', suffix='.eb') os.close(fd) f = open(self.eb_file, "w") f.write(self.contents) f.close() self.cwd = os.getcwd() self.all_stops = [x[0] for x in EasyBlock.get_steps()]
def get_ppn(): """Guess the ppn for full node""" log = get_log('pbs_job.get_ppn') pq = PBSQuery() node_vals = pq.getnodes().values() # only the values, not the names interesting_nodes = ('free', 'job-exclusive',) res = {} for np in [int(x['np'][0]) for x in node_vals if x['state'][0] in interesting_nodes]: res.setdefault(np, 0) res[np] += 1 # return most frequent freq_count, freq_np = max([(j, i) for i, j in res.items()]) log.debug("Found most frequent np %s (%s times) in interesting nodes %s" % (freq_np, freq_count, interesting_nodes)) return freq_np
def __init__(self, modulePath=None): """ Create a Modules object @param modulePath: A list of paths where the modules can be located @type modulePath: list """ self.log = get_log(self.__class__.__name__) # make sure we don't have the same path twice if modulePath: self.modulePath = set(modulePath) else: self.modulePath = None self.modules = [] self.check_module_path() # make sure environment-modules is installed ec = subprocess.call(["which", "modulecmd"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT) if ec: msg = "Could not find the modulecmd command, environment-modules is not installed?\n" msg += "Exit code of 'which modulecmd': %d" % ec self.log.error(msg) raise EasyBuildError(msg)
@author: Pieter De Baets (Ghent University) @author: Jens Timmerman (Ghent University) @author: Fotis Georgatos (Uni.Lu) """ import os import shutil import tempfile from easybuild.tools.build_log import get_log from easybuild.tools.config import install_path from easybuild.tools.filetools import rmtree2 from easybuild.tools.modules import Modules from easybuild.tools.utilities import quote_str log = get_log('moduleGenerator') # general module class GENERAL_CLASS = 'all' class ModuleGenerator(object): """ Class for generating module files. """ def __init__(self, application, fake=False): self.app = application self.fake = fake self.filename = None self.tmpdir = None
import re import shutil import signal import stat import subprocess import tempfile import time import urllib import easybuild.tools.environment as env from easybuild.tools.asyncprocess import Popen, PIPE, STDOUT from easybuild.tools.asyncprocess import send_all, recv_some from easybuild.tools.build_log import get_log log = get_log('fileTools') errorsFoundInLog = 0 # constants for strictness levels IGNORE = 'ignore' WARN = 'warn' ERROR = 'error' # default strictness level strictness = WARN def extract_file(fn, dest, cmd=None, extra_options=None, overwrite=False): """ Given filename fn, try to extract in directory dest - returns the directory name in case of success
except ImportError: pass # PySVN try: import pysvn #@UnusedImport from pysvn import ClientError #IGNORE:E0611 pysvn fails to recognize ClientError is available except ImportError: pass from easybuild.framework.easyconfig import EasyConfig, stats_to_str from easybuild.tools.build_log import get_log from easybuild.tools.version import VERBOSE_VERSION log = get_log('repo') class Repository(object): """ Interface for repositories """ def __init__(self, repo_path, subdir=''): """ Initialize a repository. self.repo and self.subdir will be set. self.wc will be set to None. Then, setupRepo and createWorkingCopy will be called (in that order) """ self.subdir = subdir self.repo = repo_path self.wc = None self.setup_repo()
@author: Stijn De Weirdt (Ghent University) @author: Dries Verdegem (Ghent University) @author: Kenneth Hoste (Ghent University) @author: Pieter De Baets (Ghent University) @author: Jens Timmerman (Ghent University) @author: Toon Willems (Ghent University) """ import os import tempfile import time from easybuild.tools.build_log import get_log import easybuild.tools.repository as repo _log = get_log('config') variables = {} requiredVariables = ['build_path', 'install_path', 'source_path', 'log_format', 'repository'] environmentVariables = { 'build_path': 'EASYBUILDBUILDPATH', # temporary build path 'install_path': 'EASYBUILDINSTALLPATH', # final install path 'log_dir': 'EASYBUILDLOGDIR', # log directory where temporary log files are stored 'config_file': 'EASYBUILDCONFIG', # path to the config file 'test_output_path': 'EASYBUILDTESTOUTPUT', # path to where jobs should place test output 'source_path': 'EASYBUILDSOURCEPATH', # path to where sources should be downloaded 'log_format': 'EASYBUILDLOGFORMAT', # format of the log file } def get_user_easybuild_dir(): """Return the per-user easybuild dir (e.g. to store config files)"""
@author: Stijn De Weirdt (Ghent University) @author: Dries Verdegem (Ghent University) @author: Kenneth Hoste (Ghent University) @author: Pieter De Baets (Ghent University) @author: Jens Timmerman (Ghent University) @author: Toon Willems (Ghent University) """ import os import tempfile from easybuild.tools.build_log import get_log import easybuild.tools.config as config log = get_log('easybuild_config') # this should result in a MODULEPATH=($HOME/.local/easybuild|$EASYBUILDPREFIX)/<modules install suffix>/all if os.getenv('EASYBUILDPREFIX'): prefix = os.getenv('EASYBUILDPREFIX') else: prefix = os.path.join(os.getenv('HOME'), ".local", "easybuild") # build/install/source paths configuration for EasyBuild # build_path possibly overridden by EASYBUILDBUILDPATH # install_path possibly overridden by EASYBUILDINSTALLPATH build_path = os.path.join(prefix, 'build') install_path = prefix source_path = os.path.join(prefix, 'sources') # repository for eb files
# # You should have received a copy of the GNU General Public License # along with EasyBuild. If not, see <http://www.gnu.org/licenses/>. ## """ Interface module to TORQUE (PBS). @author: Stijn De Weirdt (Ghent University) @author: Toon Willems (Ghent University) @author: Kenneth Hoste (Ghent University) """ import os from easybuild.tools.build_log import get_log _log = get_log('pbs_job') pbs_import_failed = None try: from PBSQuery import PBSQuery import pbs except ImportError: _log.debug("Failed to import pbs from pbs_python. Silently ignoring, is only a real issue with --job") pbs_import_failed = ("PBSQuery or pbs modules not available. " "Please make sure pbs_python is installed and usable.") MAX_WALLTIME = 72 def connect_to_server(pbs_server=None): """Connect to PBS server and return connection.""" if pbs_import_failed:
def search_toolchain(name): """ Find a toolchain with matching name returns toolchain (or None), found_toolchains """ log = get_log("search_toolchain") # import all available toolchains, so we know about them tc_modules = [] for path in sys.path: for module in glob.glob(os.path.join(path, 'easybuild', 'toolchains', '*.py')): if not module.endswith('__init__.py'): modpath = "easybuild.toolchains.%s" % module.split(os.path.sep)[-1].split('.')[0] log.debug("importing toolchain module %s" % modpath) tc_modules.append(__import__(modpath, globals(), locals(), [''])) # make sure all defined toolchain constants are available in toolchain module package = easybuild.tools.toolchain tc_const_prefix = 'TC_CONSTANT_' tc_const_re = re.compile('^%s(.*)$' % tc_const_prefix) for tc_mod in tc_modules: # determine classes imported in this module mod_classes = [] for elem in [getattr(tc_mod, x) for x in dir(tc_mod)]: if hasattr(elem, '__module__'): # exclude the toolchain class defined in that module if not tc_mod.__file__ == sys.modules[elem.__module__].__file__: log.debug("Adding %s to list of imported classes used for looking for constants" % elem.__name__) mod_classes.append(elem) # look for constants in modules of imported classes, and make them available for mod_class_mod in [sys.modules[mod_class.__module__] for mod_class in mod_classes]: for elem in dir(mod_class_mod): res = tc_const_re.match(elem) if res: tc_const_name = res.group(1) tc_const_value = getattr(mod_class_mod, elem) log.debug("Found constant %s ('%s') in module %s, adding it to %s" % (tc_const_name, tc_const_value, mod_class_mod.__name__, package.__name__)) if hasattr(package, tc_const_name): cur_value = getattr(package, tc_const_name) if not tc_const_value == cur_value: log.error("Constant %s.%s defined as '%s', can't set it to '%s'." % (package.__name__, tc_const_name, cur_value, tc_const_value )) else: setattr(package, tc_const_name, tc_const_value) # obtain all subclasses of toolchain found_tcs = get_subclasses(Toolchain) # filter found toolchain subclasses based on whether they can be used a toolchains found_tcs = [tc for tc in found_tcs if tc._is_toolchain_for(None)] for tc in found_tcs: if tc._is_toolchain_for(name): return tc, found_tcs return None, found_tcs