コード例 #1
0
ファイル: robot.py プロジェクト: MGaafar/easybuild-framework
    def setUp(self):
        """ dynamically replace Modules class with MockModule """
        # replace Modules class with something we have control over
        modules.Modules = MockModule
        main.Modules = MockModule

        self.log = get_log("RobotTest")
        main.log = get_log("main")  # redefine the main log when calling the main functions directly

        self.cwd = os.getcwd()

        self.base_easyconfig_dir = find_full_path(os.path.join("easybuild", "test", "easyconfigs"))
        self.assertTrue(self.base_easyconfig_dir)
コード例 #2
0
    def __init__(self, script, name, env_vars=None, resources={}, conn=None, ppn=None):
        """
        create a new Job to be submitted to PBS
        env_vars is a dictionary with key-value pairs of environment variables that should be passed on to the job
        resources is a dictionary with optional keys: ['hours', 'cores'] both of these should be integer values.
        hours can be 1 - MAX_WALLTIME, cores depends on which cluster it is being run.
        """
        self.clean_conn = True
        self.log = get_log(self.__class__.__name__)
        self.script = script
        if env_vars:
            self.env_vars = env_vars.copy()
        else:
            self.env_vars = {}
        self.name = name

        if pbs_import_failed:
            self.log.error(pbs_import_failed)

        try:
            self.pbs_server = pbs.pbs_default()
            if conn:
                self.pbsconn = conn
                self.clean_conn = False
            else:
                self.pbsconn = pbs.pbs_connect(self.pbs_server)
        except Exception, err:
            self.log.error("Failed to connect to the default pbs server: %s" % err)
コード例 #3
0
    def setUp(self):
        """ create temporary easyconfig file """
        self.log = get_log("EasyConfigTest")
        if self.contents is not None:
            fd, self.eb_file = tempfile.mkstemp(prefix='easyconfig_test_file_', suffix='.eb')
            os.close(fd)
            f = open(self.eb_file, "w")
            f.write(self.contents)
            f.close()
        self.cwd = os.getcwd()

        self.all_stops = [x[0] for x in EasyBlock.get_steps()]
コード例 #4
0
def get_ppn():
    """Guess the ppn for full node"""

    log = get_log('pbs_job.get_ppn')

    pq = PBSQuery()
    node_vals = pq.getnodes().values()  # only the values, not the names
    interesting_nodes = ('free', 'job-exclusive',)
    res = {}
    for np in [int(x['np'][0]) for x in node_vals if x['state'][0] in interesting_nodes]:
        res.setdefault(np, 0)
        res[np] += 1

    # return most frequent
    freq_count, freq_np = max([(j, i) for i, j in res.items()])
    log.debug("Found most frequent np %s (%s times) in interesting nodes %s" % (freq_np, freq_count, interesting_nodes))

    return freq_np
コード例 #5
0
    def __init__(self, modulePath=None):
        """
        Create a Modules object
        @param modulePath: A list of paths where the modules can be located
        @type modulePath: list
        """
        self.log = get_log(self.__class__.__name__)
        # make sure we don't have the same path twice
        if modulePath:
            self.modulePath = set(modulePath)
        else:
            self.modulePath = None
        self.modules = []

        self.check_module_path()

        # make sure environment-modules is installed
        ec = subprocess.call(["which", "modulecmd"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
        if ec:
            msg = "Could not find the modulecmd command, environment-modules is not installed?\n"
            msg += "Exit code of 'which modulecmd': %d" % ec
            self.log.error(msg)
            raise EasyBuildError(msg)
コード例 #6
0
@author: Pieter De Baets (Ghent University)
@author: Jens Timmerman (Ghent University)
@author: Fotis Georgatos (Uni.Lu)
"""
import os
import shutil
import tempfile

from easybuild.tools.build_log import get_log
from easybuild.tools.config import install_path
from easybuild.tools.filetools import rmtree2
from easybuild.tools.modules import Modules
from easybuild.tools.utilities import quote_str


log = get_log('moduleGenerator')

# general module class
GENERAL_CLASS = 'all'


class ModuleGenerator(object):
    """
    Class for generating module files.
    """
    def __init__(self, application, fake=False):
        self.app = application
        self.fake = fake
        self.filename = None
        self.tmpdir = None
コード例 #7
0
import re
import shutil
import signal
import stat
import subprocess
import tempfile
import time
import urllib

import easybuild.tools.environment as env
from easybuild.tools.asyncprocess import Popen, PIPE, STDOUT
from easybuild.tools.asyncprocess import send_all, recv_some
from easybuild.tools.build_log import get_log


log = get_log('fileTools')
errorsFoundInLog = 0

# constants for strictness levels
IGNORE = 'ignore'
WARN = 'warn'
ERROR = 'error'

# default strictness level
strictness = WARN


def extract_file(fn, dest, cmd=None, extra_options=None, overwrite=False):
    """
    Given filename fn, try to extract in directory dest
    - returns the directory name in case of success
コード例 #8
0
except ImportError:
    pass

# PySVN
try:
    import pysvn  #@UnusedImport
    from pysvn import ClientError #IGNORE:E0611 pysvn fails to recognize ClientError is available
except ImportError:
    pass

from easybuild.framework.easyconfig import EasyConfig, stats_to_str
from easybuild.tools.build_log import get_log
from easybuild.tools.version import VERBOSE_VERSION


log = get_log('repo')

class Repository(object):
    """
    Interface for repositories
    """
    def __init__(self, repo_path, subdir=''):
        """
        Initialize a repository. self.repo and self.subdir will be set.
        self.wc will be set to None.
        Then, setupRepo and createWorkingCopy will be called (in that order)
        """
        self.subdir = subdir
        self.repo = repo_path
        self.wc = None
        self.setup_repo()
コード例 #9
0
ファイル: config.py プロジェクト: MGaafar/easybuild-framework
@author: Stijn De Weirdt (Ghent University)
@author: Dries Verdegem (Ghent University)
@author: Kenneth Hoste (Ghent University)
@author: Pieter De Baets (Ghent University)
@author: Jens Timmerman (Ghent University)
@author: Toon Willems (Ghent University)
"""

import os
import tempfile
import time

from easybuild.tools.build_log import get_log
import easybuild.tools.repository as repo

_log = get_log('config')

variables = {}
requiredVariables = ['build_path', 'install_path', 'source_path', 'log_format', 'repository']
environmentVariables = {
    'build_path': 'EASYBUILDBUILDPATH',  # temporary build path
    'install_path': 'EASYBUILDINSTALLPATH',  # final install path
    'log_dir': 'EASYBUILDLOGDIR',  # log directory where temporary log files are stored
    'config_file': 'EASYBUILDCONFIG',  # path to the config file
    'test_output_path': 'EASYBUILDTESTOUTPUT',  # path to where jobs should place test output
    'source_path': 'EASYBUILDSOURCEPATH',  # path to where sources should be downloaded
    'log_format': 'EASYBUILDLOGFORMAT',  # format of the log file
}

def get_user_easybuild_dir():
    """Return the per-user easybuild dir (e.g. to store config files)"""
コード例 #10
0
@author: Stijn De Weirdt (Ghent University)
@author: Dries Verdegem (Ghent University)
@author: Kenneth Hoste (Ghent University)
@author: Pieter De Baets (Ghent University)
@author: Jens Timmerman (Ghent University)
@author: Toon Willems (Ghent University)
"""

import os
import tempfile

from easybuild.tools.build_log import get_log
import easybuild.tools.config as config

log = get_log('easybuild_config')

# this should result in a MODULEPATH=($HOME/.local/easybuild|$EASYBUILDPREFIX)/<modules install suffix>/all
if os.getenv('EASYBUILDPREFIX'):
    prefix = os.getenv('EASYBUILDPREFIX')
else:
    prefix = os.path.join(os.getenv('HOME'), ".local", "easybuild")

# build/install/source paths configuration for EasyBuild
# build_path possibly overridden by EASYBUILDBUILDPATH
# install_path possibly overridden by EASYBUILDINSTALLPATH
build_path = os.path.join(prefix, 'build')
install_path = prefix
source_path = os.path.join(prefix, 'sources')

# repository for eb files
コード例 #11
0
#
# You should have received a copy of the GNU General Public License
# along with EasyBuild.  If not, see <http://www.gnu.org/licenses/>.
##
"""
Interface module to TORQUE (PBS).

@author: Stijn De Weirdt (Ghent University)
@author: Toon Willems (Ghent University)
@author: Kenneth Hoste (Ghent University)
"""

import os
from easybuild.tools.build_log import get_log

_log = get_log('pbs_job')

pbs_import_failed = None
try:
    from PBSQuery import PBSQuery
    import pbs
except ImportError:
    _log.debug("Failed to import pbs from pbs_python. Silently ignoring, is only a real issue with --job")
    pbs_import_failed = ("PBSQuery or pbs modules not available. "
                         "Please make sure pbs_python is installed and usable.")

MAX_WALLTIME = 72

def connect_to_server(pbs_server=None):
    """Connect to PBS server and return connection."""
    if pbs_import_failed:
コード例 #12
0
def search_toolchain(name):
    """
    Find a toolchain with matching name
    returns toolchain (or None), found_toolchains
    """

    log = get_log("search_toolchain")

    # import all available toolchains, so we know about them
    tc_modules = []
    for path in sys.path:
        for module in glob.glob(os.path.join(path, 'easybuild', 'toolchains', '*.py')):
            if not module.endswith('__init__.py'):
                modpath = "easybuild.toolchains.%s" % module.split(os.path.sep)[-1].split('.')[0]
                log.debug("importing toolchain module %s" % modpath)
                tc_modules.append(__import__(modpath, globals(), locals(), ['']))

    # make sure all defined toolchain constants are available in toolchain module
    package = easybuild.tools.toolchain
    tc_const_prefix = 'TC_CONSTANT_'
    tc_const_re = re.compile('^%s(.*)$' % tc_const_prefix)
    for tc_mod in tc_modules:
        # determine classes imported in this module
        mod_classes = []
        for elem in [getattr(tc_mod, x) for x in dir(tc_mod)]:
            if hasattr(elem, '__module__'):
                # exclude the toolchain class defined in that module
                if not tc_mod.__file__ == sys.modules[elem.__module__].__file__:
                    log.debug("Adding %s to list of imported classes used for looking for constants" % elem.__name__)
                    mod_classes.append(elem)

        # look for constants in modules of imported classes, and make them available
        for mod_class_mod in [sys.modules[mod_class.__module__] for mod_class in mod_classes]:
            for elem in dir(mod_class_mod):
                res = tc_const_re.match(elem)
                if res:
                    tc_const_name = res.group(1)
                    tc_const_value = getattr(mod_class_mod, elem)
                    log.debug("Found constant %s ('%s') in module %s, adding it to %s" % (tc_const_name,
                                                                                          tc_const_value,
                                                                                          mod_class_mod.__name__,
                                                                                          package.__name__))
                    if hasattr(package, tc_const_name):
                        cur_value = getattr(package, tc_const_name)
                        if not tc_const_value == cur_value:
                            log.error("Constant %s.%s defined as '%s', can't set it to '%s'." % (package.__name__,
                                                                                                 tc_const_name,
                                                                                                 cur_value,
                                                                                                 tc_const_value
                                                                                                ))
                    else:
                        setattr(package, tc_const_name, tc_const_value)

    # obtain all subclasses of toolchain
    found_tcs = get_subclasses(Toolchain)

    # filter found toolchain subclasses based on whether they can be used a toolchains
    found_tcs = [tc for tc in found_tcs if tc._is_toolchain_for(None)]

    for tc in found_tcs:
        if tc._is_toolchain_for(name):
            return tc, found_tcs

    return None, found_tcs