Esempio n. 1
0
    def postprocess(self):
        """Do some postprocessing, in particular print stuff"""
        build_log.EXPERIMENTAL = self.options.experimental
        config.SUPPORT_OLDSTYLE = self.options.oldstyleconfig

        # set strictness of run module
        if self.options.strict:
            run.strictness = self.options.strict

        if self.options.deprecated:
            build_log.CURRENT_VERSION = LooseVersion(self.options.deprecated)

        if self.options.unittest_file:
            fancylogger.logToFile(self.options.unittest_file)

        if any(
            [
                self.options.avail_easyconfig_params,
                self.options.avail_easyconfig_templates,
                self.options.list_easyblocks,
                self.options.list_toolchains,
                self.options.avail_easyconfig_constants,
                self.options.avail_easyconfig_licenses,
                self.options.avail_repositories,
                self.options.show_default_moduleclasses,
                self.options.avail_modules_tools,
                self.options.avail_module_naming_schemes,
            ]
        ):
            build_easyconfig_constants_dict()  # runs the easyconfig constants sanity check
            self._postprocess_list_avail()

        self._postprocess_config()
Esempio n. 2
0
    def postprocess(self):
        """Do some postprocessing, in particular print stuff"""
        if self.options.unittest_file:
            fancylogger.logToFile(self.options.unittest_file)

        if any([self.options.avail_easyconfig_params, self.options.avail_easyconfig_templates,
                self.options.list_easyblocks, self.options.list_toolchains]):
            self._postprocess_list_avail()
Esempio n. 3
0
    def setUp(self):
        (self.fd, self.logfn) = tempfile.mkstemp()
        self.handle = os.fdopen(self.fd)

        # set the test log format
        fancylogger.setTestLogFormat()

        # make new logger
        fancylogger.logToFile(self.logfn)

        # disable default ones (with default format)
        fancylogger.disableDefaultHandlers()
Esempio n. 4
0
    def postprocess(self):
        """Do some postprocessing, in particular print stuff"""
        if self.options.unittest_file:
            fancylogger.logToFile(self.options.unittest_file)

        if any([self.options.avail_easyconfig_params, self.options.avail_easyconfig_templates,
                self.options.list_easyblocks, self.options.list_toolchains,
                self.options.avail_easyconfig_constants, self.options.avail_easyconfig_licenses,
                self.options.avail_repositories, self.options.show_default_moduleclasses,
                ]):
            build_easyconfig_constants_dict()  # runs the easyconfig constants sanity check
            self._postprocess_list_avail()

        self._postprocess_config()
    def postprocess(self):
        """Do some postprocessing, in particular print stuff"""
        if self.options.unittest_file:
            fancylogger.logToFile(self.options.unittest_file)

        if any([self.options.avail_easyconfig_params, self.options.avail_easyconfig_templates,
                self.options.list_easyblocks, self.options.list_toolchains,
                self.options.avail_easyconfig_constants, self.options.avail_easyconfig_licenses,
                self.options.avail_repositories, self.options.show_default_moduleclasses,
                ]):
            build_easyconfig_constants_dict()  # runs the easyconfig constants sanity check
            self._postprocess_list_avail()

        self._postprocess_config()
Esempio n. 6
0
 def __init__(self, hostname, port, log_dir, filename, pidfile):
     """Constructor"""
     stdin = '/dev/null'
     stdout = os.path.join(log_dir, 'logging_error.log')
     stderr = os.path.join(log_dir, 'logging_error.log')
     Daemon.__init__(self, pidfile, stdin, stdout, stderr)
     self.hostname = hostname
     self.port = port
     #Set up logging
     # get logger, we will log to file
     fancylogger.logToScreen(False)
     # we want to log absolutely everything that's comming at us
     fancylogger.setLogLevel(0)
     self.logfile = os.path.join(log_dir, filename)
     fancylogger.logToFile(self.logfile)
     self.logger = fancylogger.getLogger()
     self.socket_ = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
Esempio n. 7
0
 def __init__(self, hostname, port, log_dir, filename, pidfile):
     """Constructor"""
     stdin = '/dev/null'
     stdout = os.path.join(log_dir, 'logging_error.log')
     stderr = os.path.join(log_dir, 'logging_error.log')
     Daemon.__init__(self, pidfile, stdin, stdout, stderr)
     self.hostname = hostname
     self.port = port
     ##Set up logging
     #get logger, we will log to file
     fancylogger.logToScreen(False)
     #we want to log absolutely everything that's comming at us
     fancylogger.setLogLevel(0)
     self.logfile = os.path.join(log_dir, filename)
     fancylogger.logToFile(self.logfile)
     self.logger = fancylogger.getLogger()
     self.socket_ = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
Esempio n. 8
0
def make_worker_log(name,
                    debug=False,
                    logfn_name=None,
                    disable_defaulthandlers=False):
    """Make a basic log object"""
    if logfn_name is None:
        logfn_name = name
    logfn = '/tmp/scoop_%s.log' % logfn_name

    if debug:
        setLogLevelDebug()

    logToFile(logfn, name=name)
    os.chmod(logfn, stat.S_IRUSR | stat.S_IWUSR)

    if disable_defaulthandlers:
        disableDefaultHandlers()

    _log = getLogger(name=name)

    return _log
Esempio n. 9
0
    def test_zzz_logtostdout(self):
        """Testing redirecting log to stdout."""

        fd, dummylogfn = tempfile.mkstemp(prefix='easybuild-dummy', suffix='.log')
        os.close(fd)

        for stdout_arg in ['--logtostdout', '-l']:

            _stdout = sys.stdout

            fd, fn = tempfile.mkstemp()
            fh = os.fdopen(fd, 'w')
            sys.stdout = fh

            args = [
                    '--software-name=somethingrandom',
                    '--robot', '.',
                    '--debug',
                    stdout_arg,
                   ]
            self.eb_main(args, logfile=dummylogfn)

            # make sure we restore
            sys.stdout.flush()
            sys.stdout = _stdout
            fancylogger.logToScreen(enable=False, stdout=True)

            outtxt = read_file(fn)

            self.assertTrue(len(outtxt) > 100, "Log messages are printed to stdout when %s is used (outtxt: %s)" % (stdout_arg, outtxt))

            # cleanup
            os.remove(fn)
            modify_env(os.environ, self.orig_environ)
            tempfile.tempdir = None

        if os.path.exists(dummylogfn):
            os.remove(dummylogfn)
        fancylogger.logToFile(self.logfile)
Esempio n. 10
0
    def test_module_mismatch(self):
        """Test whether mismatch detection between modules tool and 'module' function works."""
        # redefine 'module' function (deliberate mismatch with used module command in MockModulesTool)
        os.environ['module'] = "() {  eval `/Users/kehoste/Modules/$MODULE_VERSION/bin/modulecmd bash $*`\n}"
        self.assertErrorRegex(EasyBuildError, ".*pattern .* not found in defined 'module' function", MockModulesTool)

        # check whether escaping error by allowing mismatch via build options works
        build_options = {
            'allow_modules_tool_mismatch': True,
        }
        init_config(build_options=build_options)

        fancylogger.logToFile(self.logfile)

        mt = MockModulesTool()
        f = open(self.logfile, 'r')
        logtxt = f.read()
        f.close()
        warn_regex = re.compile("WARNING .*pattern .* not found in defined 'module' function")
        self.assertTrue(warn_regex.search(logtxt), "Found pattern '%s' in: %s" % (warn_regex.pattern, logtxt))

        # redefine 'module' function with correct module command
        os.environ['module'] = "() {  eval `/bin/echo $*`\n}"
        mt = MockModulesTool()
        self.assertTrue(isinstance(mt.loaded_modules(), list))  # dummy usage

        # a warning should be logged if the 'module' function is undefined
        del os.environ['module']
        mt = MockModulesTool()
        f = open(self.logfile, 'r')
        logtxt = f.read()
        f.close()
        warn_regex = re.compile("WARNING No 'module' function defined, can't check if it matches .*")
        self.assertTrue(warn_regex.search(logtxt))

        fancylogger.logToFile(self.logfile, enable=False)
Esempio n. 11
0
def main(testing_data=(None, None, None)):
    """
    Main function:
    @arg options: a tuple: (options, paths, logger, logfile, hn) as defined in parse_options
    This function will:
    - read easyconfig
    - build software
    """
    # disallow running EasyBuild as root
    if os.getuid() == 0:
        sys.stderr.write("ERROR: You seem to be running EasyBuild with root privileges.\n"
                         "That's not wise, so let's end this here.\n"
                         "Exiting.\n")
        sys.exit(1)

    # steer behavior when testing main
    testing = testing_data[0] is not None
    args, logfile, do_build = testing_data

    # initialise options
    eb_go = eboptions.parse_options(args=args)
    options = eb_go.options
    orig_paths = eb_go.args

    # set umask (as early as possible)
    if options.umask is not None:
        new_umask = int(options.umask, 8)
        old_umask = os.umask(new_umask)

    # set temporary directory to use
    eb_tmpdir = set_tmpdir(options.tmpdir)

    # initialise logging for main
    if options.logtostdout:
        fancylogger.logToScreen(enable=True, stdout=True)
    else:
        if logfile is None:
            # mkstemp returns (fd,filename), fd is from os.open, not regular open!
            fd, logfile = tempfile.mkstemp(suffix='.log', prefix='easybuild-')
            os.close(fd)

        fancylogger.logToFile(logfile)
        print_msg('temporary log file in case of crash %s' % (logfile), log=None, silent=testing)

    global _log
    _log = fancylogger.getLogger(fname=False)

    if options.umask is not None:
        _log.info("umask set to '%s' (used to be '%s')" % (oct(new_umask), oct(old_umask)))

    # hello world!
    _log.info(this_is_easybuild())

    # how was EB called?
    eb_command_line = eb_go.generate_cmd_line() + eb_go.args
    _log.info("Command line: %s" % (" ".join(eb_command_line)))

    _log.info("Using %s as temporary directory" % eb_tmpdir)

    if not options.robot is None:
        if options.robot:
            _log.info("Using robot path(s): %s" % options.robot)
        else:
            _log.error("No robot paths specified, and unable to determine easybuild-easyconfigs install path.")

    # do not pass options.robot, it's not a list instance (and it shouldn't be modified)
    robot_path = None
    if options.robot:
        robot_path = list(options.robot)

    # determine easybuild-easyconfigs package install path
    easyconfigs_paths = get_paths_for("easyconfigs", robot_path=robot_path)
    # keep track of paths for install easyconfigs, so we can obtain find specified easyconfigs
    easyconfigs_pkg_full_paths = easyconfigs_paths[:]
    if not easyconfigs_paths:
        _log.warning("Failed to determine install path for easybuild-easyconfigs package.")

    # specified robot paths are preferred over installed easyconfig files
    if robot_path:
        robot_path.extend(easyconfigs_paths)
        easyconfigs_paths = robot_path[:]
        _log.info("Extended list of robot paths with paths for installed easyconfigs: %s" % robot_path)

    # initialise the easybuild configuration
    config.init(options, eb_go.get_options_by_section('config'))

    # building a dependency graph implies force, so that all dependencies are retained
    # and also skips validation of easyconfigs (e.g. checking os dependencies)
    retain_all_deps = False
    if options.dep_graph:
        _log.info("Enabling force to generate dependency graph.")
        options.force = True
        retain_all_deps = True

    config.init_build_options({
        'aggregate_regtest': options.aggregate_regtest,
        'allow_modules_tool_mismatch': options.allow_modules_tool_mismatch,
        'check_osdeps': not options.ignore_osdeps,
        'command_line': eb_command_line,
        'debug': options.debug,
        'dry_run': options.dry_run,
        'easyblock': options.easyblock,
        'experimental': options.experimental,
        'force': options.force,
        'group': options.group,
        'ignore_dirs': options.ignore_dirs,
        'modules_footer': options.modules_footer,
        'only_blocks': options.only_blocks,
        'recursive_mod_unload': options.recursive_module_unload,
        'regtest_online': options.regtest_online,
        'regtest_output_dir': options.regtest_output_dir,
        'retain_all_deps': retain_all_deps,
        'robot_path': robot_path,
        'sequential': options.sequential,
        'silent': testing,
        'set_gid_bit': options.set_gid_bit,
        'skip': options.skip,
        'skip_test_cases': options.skip_test_cases,
        'sticky_bit': options.sticky_bit,
        'stop': options.stop,
        'umask': options.umask,
        'valid_module_classes': module_classes(),
        'valid_stops': [x[0] for x in EasyBlock.get_steps()],
        'validate': not options.force,
    })

    # search for easyconfigs
    if options.search or options.search_short:
        search_path = [os.getcwd()]
        if easyconfigs_paths:
            search_path = easyconfigs_paths
        query = options.search or options.search_short
        ignore_dirs = config.build_option('ignore_dirs')
        silent = config.build_option('silent')
        search_file(search_path, query, short=not options.search, ignore_dirs=ignore_dirs, silent=silent)

    # process software build specifications (if any), i.e.
    # software name/version, toolchain name/version, extra patches, ...
    (try_to_generate, build_specs) = process_software_build_specs(options)

    paths = []
    if len(orig_paths) == 0:
        if 'name' in build_specs:
            paths = [obtain_path(build_specs, easyconfigs_paths, try_to_generate=try_to_generate,
                                 exit_on_error=not testing)]
        elif not any([options.aggregate_regtest, options.search, options.search_short, options.regtest]):
            print_error(("Please provide one or multiple easyconfig files, or use software build "
                         "options to make EasyBuild search for easyconfigs"),
                        log=_log, opt_parser=eb_go.parser, exit_on_error=not testing)
    else:
        # look for easyconfigs with relative paths in easybuild-easyconfigs package,
        # unless they were found at the given relative paths
        if easyconfigs_pkg_full_paths:
            # determine which easyconfigs files need to be found, if any
            ecs_to_find = []
            for idx, orig_path in enumerate(orig_paths):
                if orig_path == os.path.basename(orig_path) and not os.path.exists(orig_path):
                    ecs_to_find.append((idx, orig_path))
            _log.debug("List of easyconfig files to find: %s" % ecs_to_find)

            # find missing easyconfigs by walking paths with installed easyconfig files
            for path in easyconfigs_pkg_full_paths:
                _log.debug("Looking for missing easyconfig files (%d left) in %s..." % (len(ecs_to_find), path))
                for (subpath, dirnames, filenames) in os.walk(path, topdown=True):
                    for idx, orig_path in ecs_to_find[:]:
                        if orig_path in filenames:
                            full_path = os.path.join(subpath, orig_path)
                            _log.info("Found %s in %s: %s" % (orig_path, path, full_path))
                            orig_paths[idx] = full_path
                            # if file was found, stop looking for it (first hit wins)
                            ecs_to_find.remove((idx, orig_path))

                    # stop os.walk insanity as soon as we have all we need (os.walk loop)
                    if len(ecs_to_find) == 0:
                        break

                    # ignore subdirs specified to be ignored by replacing items in dirnames list used by os.walk
                    dirnames[:] = [d for d in dirnames if not d in options.ignore_dirs]

                # stop os.walk insanity as soon as we have all we need (paths loop)
                if len(ecs_to_find) == 0:
                    break

        # indicate that specified paths do not contain generated easyconfig files
        paths = [(path, False) for path in orig_paths]

    _log.debug("Paths: %s" % paths)

    # run regtest
    if options.regtest or options.aggregate_regtest:
        _log.info("Running regression test")
        if paths:
            ec_paths = [path[0] for path in paths]
        else:  # fallback: easybuild-easyconfigs install path
            ec_paths = easyconfigs_pkg_full_paths
        regtest_ok = regtest(ec_paths)

        if not regtest_ok:
            _log.info("Regression test failed (partially)!")
            sys.exit(31)  # exit -> 3x1t -> 31

    # read easyconfig files
    easyconfigs = []
    for (path, generated) in paths:
        path = os.path.abspath(path)
        if not os.path.exists(path):
            print_error("Can't find path %s" % path)

        try:
            files = find_easyconfigs(path, ignore_dirs=options.ignore_dirs)
            for f in files:
                if not generated and try_to_generate and build_specs:
                    ec_file = tweak(f, None, build_specs)
                else:
                    ec_file = f
                ecs = process_easyconfig(ec_file, build_specs=build_specs)
                easyconfigs.extend(ecs)
        except IOError, err:
            _log.error("Processing easyconfigs in path %s failed: %s" % (path, err))
Esempio n. 12
0
        for ec in ordered_ecs:
            (success, _) = build_and_install_software(ec, orig_environ)
            if success:
                correct_built_cnt += 1
            all_built_cnt += 1

    print_msg("Build succeeded for %s out of %s" % (correct_built_cnt, all_built_cnt), log=_log, silent=testing)

    repo = init_repository(get_repository(), get_repositorypath())
    repo.cleanup()

    # cleanup and spec files
    for ec in easyconfigs:
        if 'original_spec' in ec and os.path.isfile(ec['spec']):
            os.remove(ec['spec'])

    # cleanup tmp log file (all is well, all modules have their own log file)
    if options.logtostdout:
        fancylogger.logToScreen(enable=False, stdout=True)
    else:
        fancylogger.logToFile(logfile, enable=False)
    cleanup(logfile, eb_tmpdir, testing)


if __name__ == "__main__":
    try:
        main()
    except EasyBuildError, e:
        sys.stderr.write('ERROR: %s\n' % e.msg)
        sys.exit(1)
Esempio n. 13
0
    print_msg("Build succeeded for %s out of %s" % (correct_built_cnt, all_built_cnt), log=_log, silent=testing)

    repo = init_repository(get_repository(), get_repositorypath())
    repo.cleanup()

    # cleanup and spec files
    for ec in easyconfigs:
        if 'originalSpec' in ec and os.path.isfile(ec['spec']):
            os.remove(ec['spec'])

    # cleanup tmp log file (all is well, all modules have their own log file)
    if options.logtostdout:
        fancylogger.logToScreen(enable=False, stdout=True)
    else:
        fancylogger.logToFile(logfile, enable=False)
        cleanup_logfile_and_exit(logfile, testing, False)
        logfile = None

    return logfile


def cleanup_logfile_and_exit(logfile, testing, doexit):
    """Cleanup the logfile and exit"""
    if not testing and logfile is not None:
        os.remove(logfile)
        print_msg('temporary log file %s has been removed.' % (logfile), log=None, silent=testing)
    if doexit:
        sys.exit(0)

            # make sure we restore
            sys.stdout.flush()
            sys.stdout = _stdout
            fancylogger.logToScreen(enable=False, stdout=True)

            outtxt = read_file(fn)

            self.assertTrue(len(outtxt) > 100, "Log messages are printed to stdout when %s is used (outtxt: %s)" % (stdout_arg, outtxt))

            # cleanup
            os.remove(fn)

        if os.path.exists(dummylogfn):
            os.remove(dummylogfn)
        fancylogger.logToFile(self.logfile)

    def test_avail_easyconfig_params(self):
        """Test listing available easyconfig parameters."""

        def run_test(custom=None, extra_params=[]):
            """Inner function to run actual test in current setting."""

            fd, dummylogfn = tempfile.mkstemp(prefix='easybuild-dummy', suffix='.log')
            os.close(fd)

            for avail_arg in [
                              '-a',
                              '--avail-easyconfig-params',
                             ]:
Esempio n. 15
0
def main(testing_data=(None, None)):
    """
    Main function:
    @arg options: a tuple: (options, paths, logger, logfile, hn) as defined in parse_options
    This function will:
    - read easyconfig
    - build software
    """
    # disallow running EasyBuild as root
    if os.getuid() == 0:
        sys.stderr.write(
            "ERROR: You seem to be running EasyBuild with root privileges.\n"
            "That's not wise, so let's end this here.\n"
            "Exiting.\n"
        )
        sys.exit(1)

    # steer behavior when testing main
    testing = testing_data[0] is not None
    args, logfile = testing_data

    # initialise options
    eb_go = eboptions.parse_options(args=args)
    options = eb_go.options
    orig_paths = eb_go.args

    # initialise logging for main
    if options.logtostdout:
        fancylogger.logToScreen(enable=True, stdout=True)
    else:
        if logfile is None:
            # mkstemp returns (fd,filename), fd is from os.open, not regular open!
            fd, logfile = tempfile.mkstemp(suffix=".log", prefix="easybuild-")
            os.close(fd)

        fancylogger.logToFile(logfile)
        print_msg("temporary log file in case of crash %s" % (logfile), log=None, silent=testing)

    global _log
    _log = fancylogger.getLogger(fname=False)

    # hello world!
    _log.info(this_is_easybuild())

    # set strictness of filetools module
    if options.strict:
        filetools.strictness = options.strict

    if not options.robot is None:
        if options.robot:
            _log.info("Using robot path: %s" % options.robot)
        else:
            _log.error("No robot path specified, and unable to determine easybuild-easyconfigs install path.")

    # determine easybuild-easyconfigs package install path
    easyconfigs_paths = get_paths_for("easyconfigs", robot_path=options.robot)
    easyconfigs_pkg_full_path = None

    search_path = os.getcwd()
    if easyconfigs_paths:
        easyconfigs_pkg_full_path = easyconfigs_paths[0]
        if not options.robot:
            search_path = easyconfigs_pkg_full_path
        else:
            search_path = options.robot
    else:
        _log.info("Failed to determine install path for easybuild-easyconfigs package.")

    if options.robot:
        easyconfigs_paths = [options.robot] + easyconfigs_paths

    # initialise the easybuild configuration
    config.init(options, eb_go.get_options_by_section("config"))

    # search for easyconfigs
    if options.search:
        search_file(search_path, options.search, silent=testing)

    # process software build specifications (if any), i.e.
    # software name/version, toolchain name/version, extra patches, ...
    (try_to_generate, software_build_specs) = process_software_build_specs(options)

    paths = []
    if len(orig_paths) == 0:
        if software_build_specs.has_key("name"):
            paths = [
                obtain_path(
                    software_build_specs, easyconfigs_paths, try_to_generate=try_to_generate, exit_on_error=not testing
                )
            ]
        elif not any([options.aggregate_regtest, options.search, options.regtest]):
            print_error(
                (
                    "Please provide one or multiple easyconfig files, or use software build "
                    "options to make EasyBuild search for easyconfigs"
                ),
                log=_log,
                opt_parser=eb_go.parser,
                exit_on_error=not testing,
            )
    else:
        # look for easyconfigs with relative paths in easybuild-easyconfigs package,
        # unless they we found at the given relative paths

        if easyconfigs_pkg_full_path:
            # create a mapping from filename to path in easybuild-easyconfigs package install path
            easyconfigs_map = {}
            for (subpath, _, filenames) in os.walk(easyconfigs_pkg_full_path):
                for filename in filenames:
                    easyconfigs_map.update({filename: os.path.join(subpath, filename)})

            # try and find non-existing non-absolute eaysconfig paths in easybuild-easyconfigs package install path
            for idx, orig_path in enumerate(orig_paths):
                if not os.path.isabs(orig_path) and not os.path.exists(orig_path):
                    if orig_path in easyconfigs_map:
                        _log.info(
                            "Found %s in %s: %s" % (orig_path, easyconfigs_pkg_full_path, easyconfigs_map[orig_path])
                        )
                        orig_paths[idx] = easyconfigs_map[orig_path]

        # indicate that specified paths do not contain generated easyconfig files
        paths = [(path, False) for path in orig_paths]

    _log.debug("Paths: %s" % paths)

    # run regtest
    if options.regtest or options.aggregate_regtest:
        _log.info("Running regression test")
        if paths:
            regtest_ok = regtest(options, [path[0] for path in paths])
        else:  # fallback: easybuild-easyconfigs install path
            regtest_ok = regtest(options, [easyconfigs_pkg_full_path])

        if not regtest_ok:
            _log.info("Regression test failed (partially)!")
            sys.exit(31)  # exit -> 3x1t -> 31

    if any([options.search, options.regtest]):
        cleanup_logfile_and_exit(logfile, testing, True)

    # building a dependency graph implies force, so that all dependencies are retained
    # and also skips validation of easyconfigs (e.g. checking os dependencies)
    validate_easyconfigs = True
    retain_all_deps = False
    if options.dep_graph:
        _log.info("Enabling force to generate dependency graph.")
        options.force = True
        validate_easyconfigs = False
        retain_all_deps = True

    # read easyconfig files
    easyconfigs = []
    for (path, generated) in paths:
        path = os.path.abspath(path)
        if not (os.path.exists(path)):
            print_error("Can't find path %s" % path)

        try:
            files = find_easyconfigs(path)
            for f in files:
                if not generated and try_to_generate and software_build_specs:
                    ec_file = easyconfig.tools.tweak(f, None, software_build_specs)
                else:
                    ec_file = f
                easyconfigs.extend(process_easyconfig(ec_file, options.only_blocks, validate=validate_easyconfigs))
        except IOError, err:
            _log.error("Processing easyconfigs in path %s failed: %s" % (path, err))
Esempio n. 16
0
def main(testing_data=(None, None)):
    """
    Main function:
    @arg options: a tuple: (options, paths, logger, logfile, hn) as defined in parse_options
    This function will:
    - read easyconfig
    - build software
    """
    # disallow running EasyBuild as root
    if os.getuid() == 0:
        sys.stderr.write("ERROR: You seem to be running EasyBuild with root privileges.\n" \
                        "That's not wise, so let's end this here.\n" \
                        "Exiting.\n")
        sys.exit(1)

    # steer behavior when testing main
    testing = testing_data[0] is not None
    args, logfile = testing_data

    # initialise options
    eb_go = eboptions.parse_options(args=args)
    options = eb_go.options
    orig_paths = eb_go.args

    # initialise logging for main
    if options.logtostdout:
        fancylogger.logToScreen(enable=True, stdout=True)
    else:
        if logfile is None:
            # mkstemp returns (fd,filename), fd is from os.open, not regular open!
            fd, logfile = tempfile.mkstemp(suffix='.log', prefix='easybuild-')
            os.close(fd)

        fancylogger.logToFile(logfile)
        print_msg('temporary log file in case of crash %s' % (logfile), log=None, silent=testing)

    global _log
    _log = fancylogger.getLogger(fname=False)

    # hello world!
    _log.info(this_is_easybuild())

    # set strictness of filetools module
    if options.strict:
        filetools.strictness = options.strict

    if not options.robot is None:
        if options.robot:
            _log.info("Using robot path: %s" % options.robot)
        else:
            _log.error("No robot path specified, and unable to determine easybuild-easyconfigs install path.")

    # determine easybuild-easyconfigs package install path
    easyconfigs_paths = get_paths_for("easyconfigs", robot_path=options.robot)
    easyconfigs_pkg_full_path = None

    search_path = os.getcwd()
    if easyconfigs_paths:
        easyconfigs_pkg_full_path = easyconfigs_paths[0]
        if not options.robot:
            search_path = easyconfigs_pkg_full_path
        else:
            search_path = options.robot
    else:
        _log.info("Failed to determine install path for easybuild-easyconfigs package.")

    if options.robot:
        easyconfigs_paths = [options.robot] + easyconfigs_paths

    # initialise the easybuild configuration
    config.init(options, eb_go.get_options_by_section('config'))

    # search for easyconfigs
    if options.search:
        search_file(search_path, options.search, silent=testing)

    # process software build specifications (if any), i.e.
    # software name/version, toolchain name/version, extra patches, ...
    (try_to_generate, software_build_specs) = process_software_build_specs(options)

    paths = []
    if len(orig_paths) == 0:
        if software_build_specs.has_key('name'):
            paths = [obtain_path(software_build_specs, easyconfigs_paths,
                                 try_to_generate=try_to_generate, exit_on_error=not testing)]
        elif not any([options.aggregate_regtest, options.search, options.regtest]):
            print_error(("Please provide one or multiple easyconfig files, or use software build "
                  "options to make EasyBuild search for easyconfigs"),
                  log=_log, opt_parser=eb_go.parser, exit_on_error=not testing)
    else:
        # look for easyconfigs with relative paths in easybuild-easyconfigs package,
        # unless they we found at the given relative paths

        if easyconfigs_pkg_full_path:
            # create a mapping from filename to path in easybuild-easyconfigs package install path
            easyconfigs_map = {}
            for (subpath, _, filenames) in os.walk(easyconfigs_pkg_full_path):
                for filename in filenames:
                    easyconfigs_map.update({filename: os.path.join(subpath, filename)})

            # try and find non-existing non-absolute eaysconfig paths in easybuild-easyconfigs package install path
            for idx, orig_path in enumerate(orig_paths):
                if not os.path.isabs(orig_path) and not os.path.exists(orig_path):
                    if orig_path in easyconfigs_map:
                        _log.info("Found %s in %s: %s" % (orig_path, easyconfigs_pkg_full_path,
                                                         easyconfigs_map[orig_path]))
                        orig_paths[idx] = easyconfigs_map[orig_path]

        # indicate that specified paths do not contain generated easyconfig files
        paths = [(path, False) for path in orig_paths]

    _log.debug("Paths: %s" % paths)

    # run regtest
    if options.regtest or options.aggregate_regtest:
        _log.info("Running regression test")
        if paths:
            regtest_ok = regtest(options, [path[0] for path in paths])
        else:  # fallback: easybuild-easyconfigs install path
            regtest_ok = regtest(options, [easyconfigs_pkg_full_path])

        if not regtest_ok:
            _log.info("Regression test failed (partially)!")
            sys.exit(31)  # exit -> 3x1t -> 31

    if any([options.search, options.regtest]):
        cleanup_logfile_and_exit(logfile, testing, True)

    # building a dependency graph implies force, so that all dependencies are retained
    # and also skips validation of easyconfigs (e.g. checking os dependencies)
    validate_easyconfigs = True
    retain_all_deps = False
    if options.dep_graph:
        _log.info("Enabling force to generate dependency graph.")
        options.force = True
        validate_easyconfigs = False
        retain_all_deps = True

    # read easyconfig files
    easyconfigs = []
    for (path, generated) in paths:
        path = os.path.abspath(path)
        if not (os.path.exists(path)):
            print_error("Can't find path %s" % path)

        try:
            files = find_easyconfigs(path)
            for f in files:
                if not generated and try_to_generate and software_build_specs:
                    ec_file = easyconfig.tools.tweak(f, None, software_build_specs)
                else:
                    ec_file = f
                easyconfigs.extend(process_easyconfig(ec_file, options.only_blocks,
                                                      validate=validate_easyconfigs))
        except IOError, err:
            _log.error("Processing easyconfigs in path %s failed: %s" % (path, err))
Esempio n. 17
0
 def tearDown(self):
     fancylogger.logToFile(self.logfn, enable=False)
     self.handle.close()
     os.remove(self.logfn)
Esempio n. 18
0
            sys.stdout = _stdout
            fancylogger.logToScreen(enable=False, stdout=True)

            outtxt = read_file(fn)

            self.assertTrue(
                len(outtxt) > 100,
                "Log messages are printed to stdout when %s is used (outtxt: %s)"
                % (stdout_arg, outtxt))

            # cleanup
            os.remove(fn)

        if os.path.exists(dummylogfn):
            os.remove(dummylogfn)
        fancylogger.logToFile(self.logfile)

    def test_avail_easyconfig_params(self):
        """Test listing available easyconfig parameters."""
        def run_test(custom=None, extra_params=[]):
            """Inner function to run actual test in current setting."""

            fd, dummylogfn = tempfile.mkstemp(prefix='easybuild-dummy',
                                              suffix='.log')
            os.close(fd)

            for avail_arg in [
                    '-a',
                    '--avail-easyconfig-params',
            ]:
Esempio n. 19
0
        raise EasyBuildError(newMsg)


# set format for logger
LOGGING_FORMAT = EB_MSG_PREFIX + ' %(asctime)s %(name)s %(levelname)s %(message)s'
fancylogger.setLogFormat(LOGGING_FORMAT)

# set the default LoggerClass to EasyBuildLog
fancylogger.logging.setLoggerClass(EasyBuildLog)

# you can't easily set another LoggerClass before fancylogger calls getLogger on import
_init_fancylog = fancylogger.getLogger(fname=False)
del _init_fancylog.manager.loggerDict[_init_fancylog.name]

# we need to make sure there is a handler
fancylogger.logToFile(filename=os.devnull)

# EasyBuildLog
_init_easybuildlog = fancylogger.getLogger(fname=False)


def get_log(name=None):
    """
    Generate logger object
    """
    # fname is always get_log, useless
    log = fancylogger.getLogger(name, fname=False)
    log.info("Logger started for %s." % name)
    log.deprecated("get_log", "2.0")
    return log
Esempio n. 20
0
import test.framework.modules as m
import test.framework.options as o
import test.framework.repository as r
import test.framework.robot as robot
import test.framework.systemtools as s
import test.framework.toolchain as tc
import test.framework.toolchainvariables as tcv
import test.framework.toy_build as t
import test.framework.variables as v


# initialize logger for all the unit tests
fd, log_fn = tempfile.mkstemp(prefix='easybuild-tests-', suffix='.log')
os.close(fd)
os.remove(log_fn)
fancylogger.logToFile(log_fn)
log = fancylogger.getLogger()
log.setLevelName('DEBUG')

# call suite() for each module and then run them all
# note: make sure the options unit tests run first, to avoid running some of them with a readily initialized config
SUITE = unittest.TestSuite([x.suite() for x in [o, r, e, mg, m, f, a, robot, b, v, g, tcv, tc, t, c, s]])

# uses XMLTestRunner if possible, so we can output an XML file that can be supplied to Jenkins
xml_msg = ""
try:
    import xmlrunner  # requires unittest-xml-reporting package
    xml_dir = 'test-reports'
    res = xmlrunner.XMLTestRunner(output=xml_dir, verbosity=1).run(SUITE)
    xml_msg = ", XML output of tests available in %s directory" % xml_dir
except ImportError, err:
Esempio n. 21
0
        raise EasyBuildError(newMsg)


# set format for logger
LOGGING_FORMAT = EB_MSG_PREFIX + ' %(asctime)s %(name)s %(levelname)s %(message)s'
fancylogger.setLogFormat(LOGGING_FORMAT)

# set the default LoggerClass to EasyBuildLog
fancylogger.logging.setLoggerClass(EasyBuildLog)

# you can't easily set another LoggerClass before fancylogger calls getLogger on import
_init_fancylog = fancylogger.getLogger(fname=False)
del _init_fancylog.manager.loggerDict[_init_fancylog.name]

# we need to make sure there is a handler
fancylogger.logToFile(filename=os.devnull)

# EasyBuildLog
_init_easybuildlog = fancylogger.getLogger(fname=False)

def get_log(name=None):
    """
    Generate logger object
    """
    # fname is always get_log, useless
    log = fancylogger.getLogger(name, fname=False)
    log.info("Logger started for %s." % name)
    return log


def print_msg(msg, log=None, silent=False):