def test_zzz_logtostdout(self): """Testing redirecting log to stdout.""" for stdout_arg in ['--logtostdout', '-l']: _stdout = sys.stdout myerr = None fd, fn = tempfile.mkstemp() fh = os.fdopen(fd, 'w') sys.stdout = fh args = [ '--software-name=somethingrandom', '--robot=.', '--debug', stdout_arg, ] try: main((args, None)) except Exception, err: myerr = err # make sure we restore sys.stdout.flush() sys.stdout = _stdout fancylogger.logToScreen(enable=False, stdout=True) outtxt = open(fn, 'r').read() self.assertTrue(len(outtxt) > 100, "Log messages are printed to stdout when %s is used (outtxt: %s)" % (stdout_arg, outtxt)) # cleanup os.remove(fn)
def parse_options(args=None): """wrapper function for option parsing""" if os.environ.get('DEBUG_EASYBUILD_OPTIONS', '0').lower() in ('1', 'true', 'yes', 'y'): # very early debug, to debug the generaloption itself fancylogger.logToScreen(enable=True) fancylogger.setLogLevel('DEBUG') usage = "%prog [options] easyconfig [...]" description = ("Builds software based on easyconfig (or parse a directory).\n" "Provide one or more easyconfigs or directories, use -H or --help more information.") eb_go = EasyBuildOptions(usage=usage, description=description, prog='eb', envvar_prefix='EASYBUILD', go_args=args) return eb_go
def __init__(self, hostname, port, log_dir, filename, pidfile): """Constructor""" stdin = '/dev/null' stdout = os.path.join(log_dir, 'logging_error.log') stderr = os.path.join(log_dir, 'logging_error.log') Daemon.__init__(self, pidfile, stdin, stdout, stderr) self.hostname = hostname self.port = port ##Set up logging #get logger, we will log to file fancylogger.logToScreen(False) #we want to log absolutely everything that's comming at us fancylogger.setLogLevel(0) self.logfile = os.path.join(log_dir, filename) fancylogger.logToFile(self.logfile) self.logger = fancylogger.getLogger() self.socket_ = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
def __init__(self, hostname, port, log_dir, filename, pidfile): """Constructor""" stdin = '/dev/null' stdout = os.path.join(log_dir, 'logging_error.log') stderr = os.path.join(log_dir, 'logging_error.log') Daemon.__init__(self, pidfile, stdin, stdout, stderr) self.hostname = hostname self.port = port #Set up logging # get logger, we will log to file fancylogger.logToScreen(False) # we want to log absolutely everything that's comming at us fancylogger.setLogLevel(0) self.logfile = os.path.join(log_dir, filename) fancylogger.logToFile(self.logfile) self.logger = fancylogger.getLogger() self.socket_ = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
def parse_options(args=None): """wrapper function for option parsing""" if os.environ.get('DEBUG_EASYBUILD_OPTIONS', '0').lower() in ('1', 'true', 'yes', 'y'): # very early debug, to debug the generaloption itself fancylogger.logToScreen(enable=True) fancylogger.setLogLevel('DEBUG') usage = "%prog [options] easyconfig [...]" description = ("Builds software based on easyconfig (or parse a directory).\n" "Provide one or more easyconfigs or directories, use -H or --help more information.") eb_go = EasyBuildOptions( usage=usage, description=description, prog='eb', envvar_prefix='EASYBUILD', go_args=args, ) return eb_go
def test_zzz_logtostdout(self): """Testing redirecting log to stdout.""" fd, dummylogfn = tempfile.mkstemp(prefix='easybuild-dummy', suffix='.log') os.close(fd) for stdout_arg in ['--logtostdout', '-l']: _stdout = sys.stdout myerr = None fd, fn = tempfile.mkstemp() fh = os.fdopen(fd, 'w') sys.stdout = fh args = [ '--software-name=somethingrandom', '--robot', '.', '--debug', stdout_arg, ] try: main((args, dummylogfn, False)) except (SystemExit, Exception), err: myerr = err # make sure we restore sys.stdout.flush() sys.stdout = _stdout fancylogger.logToScreen(enable=False, stdout=True) outtxt = read_file(fn) self.assertTrue( len(outtxt) > 100, "Log messages are printed to stdout when %s is used (outtxt: %s)" % (stdout_arg, outtxt)) # cleanup os.remove(fn)
def _stream_stdouterr(self, isstdout=True, expect_match=True): fd, logfn = tempfile.mkstemp() # fh will be checked fh = os.fdopen(fd, "w") _stdout = sys.stdout _stderr = sys.stderr if isstdout == expect_match: sys.stdout = fh sys.stderr = open(os.devnull, "w") else: sys.stdout = open(os.devnull, "w") sys.stderr = fh fancylogger.setLogLevelInfo() name = "test_stream_stdout" lh = fancylogger.logToScreen(stdout=isstdout) logger = fancylogger.getLogger(name) # logfn makes it unique msg = "TEST isstdout %s expect_match %s logfn %s" % (isstdout, expect_match, logfn) logger.info(msg) # restore fancylogger.logToScreen(enable=False, handler=lh) sys.stdout = _stdout sys.stderr = _stderr fh2 = open(logfn) txt = fh2.read().strip() fh2.close() reg_exp = re.compile(r"INFO\s+\S+.%s.%s\s+\S+\s+%s" % (name, "_stream_stdouterr", msg)) match = reg_exp.search(txt) is not None self.assertEqual(match, expect_match) try: fh.close() os.remove(logfn) except: pass
def test_zzz_logtostdout(self): """Testing redirecting log to stdout.""" fd, dummylogfn = tempfile.mkstemp(prefix='easybuild-dummy', suffix='.log') os.close(fd) for stdout_arg in ['--logtostdout', '-l']: _stdout = sys.stdout fd, fn = tempfile.mkstemp() fh = os.fdopen(fd, 'w') sys.stdout = fh args = [ '--software-name=somethingrandom', '--robot', '.', '--debug', stdout_arg, ] self.eb_main(args, logfile=dummylogfn) # make sure we restore sys.stdout.flush() sys.stdout = _stdout fancylogger.logToScreen(enable=False, stdout=True) outtxt = read_file(fn) self.assertTrue(len(outtxt) > 100, "Log messages are printed to stdout when %s is used (outtxt: %s)" % (stdout_arg, outtxt)) # cleanup os.remove(fn) modify_env(os.environ, self.orig_environ) tempfile.tempdir = None if os.path.exists(dummylogfn): os.remove(dummylogfn) fancylogger.logToFile(self.logfile)
def test_zzz_logtostdout(self): """Testing redirecting log to stdout.""" fd, dummylogfn = tempfile.mkstemp(prefix='easybuild-dummy', suffix='.log') os.close(fd) for stdout_arg in ['--logtostdout', '-l']: _stdout = sys.stdout myerr = None fd, fn = tempfile.mkstemp() fh = os.fdopen(fd, 'w') sys.stdout = fh args = [ '--software-name=somethingrandom', '--robot', '.', '--debug', stdout_arg, ] try: main((args, dummylogfn)) except (SystemExit, Exception), err: myerr = err # make sure we restore sys.stdout.flush() sys.stdout = _stdout fancylogger.logToScreen(enable=False, stdout=True) outtxt = read_file(fn) self.assertTrue(len(outtxt) > 100, "Log messages are printed to stdout when %s is used (outtxt: %s)" % (stdout_arg, outtxt)) # cleanup os.remove(fn)
def test_zzz_logtostdout(self): """Testing redirecting log to stdout.""" fd, dummylogfn = tempfile.mkstemp(prefix="easybuild-dummy", suffix=".log") os.close(fd) for stdout_arg in ["--logtostdout", "-l"]: _stdout = sys.stdout myerr = None fd, fn = tempfile.mkstemp() fh = os.fdopen(fd, "w") sys.stdout = fh args = ["--software-name=somethingrandom", "--robot", ".", "--debug", stdout_arg] try: main((args, dummylogfn, False)) except (SystemExit, Exception), err: myerr = err # make sure we restore sys.stdout.flush() sys.stdout = _stdout fancylogger.logToScreen(enable=False, stdout=True) outtxt = read_file(fn) self.assertTrue( len(outtxt) > 100, "Log messages are printed to stdout when %s is used (outtxt: %s)" % (stdout_arg, outtxt), ) # cleanup os.remove(fn) modify_env(os.environ, self.orig_environ) tempfile.tempdir = None
def main(testing_data=(None, None, None)): """ Main function: @arg options: a tuple: (options, paths, logger, logfile, hn) as defined in parse_options This function will: - read easyconfig - build software """ # disallow running EasyBuild as root if os.getuid() == 0: sys.stderr.write("ERROR: You seem to be running EasyBuild with root privileges.\n" "That's not wise, so let's end this here.\n" "Exiting.\n") sys.exit(1) # steer behavior when testing main testing = testing_data[0] is not None args, logfile, do_build = testing_data # initialise options eb_go = eboptions.parse_options(args=args) options = eb_go.options orig_paths = eb_go.args # set umask (as early as possible) if options.umask is not None: new_umask = int(options.umask, 8) old_umask = os.umask(new_umask) # set temporary directory to use eb_tmpdir = set_tmpdir(options.tmpdir) # initialise logging for main if options.logtostdout: fancylogger.logToScreen(enable=True, stdout=True) else: if logfile is None: # mkstemp returns (fd,filename), fd is from os.open, not regular open! fd, logfile = tempfile.mkstemp(suffix='.log', prefix='easybuild-') os.close(fd) fancylogger.logToFile(logfile) print_msg('temporary log file in case of crash %s' % (logfile), log=None, silent=testing) global _log _log = fancylogger.getLogger(fname=False) if options.umask is not None: _log.info("umask set to '%s' (used to be '%s')" % (oct(new_umask), oct(old_umask))) # hello world! _log.info(this_is_easybuild()) # how was EB called? eb_command_line = eb_go.generate_cmd_line() + eb_go.args _log.info("Command line: %s" % (" ".join(eb_command_line))) _log.info("Using %s as temporary directory" % eb_tmpdir) if not options.robot is None: if options.robot: _log.info("Using robot path(s): %s" % options.robot) else: _log.error("No robot paths specified, and unable to determine easybuild-easyconfigs install path.") # do not pass options.robot, it's not a list instance (and it shouldn't be modified) robot_path = None if options.robot: robot_path = list(options.robot) # determine easybuild-easyconfigs package install path easyconfigs_paths = get_paths_for("easyconfigs", robot_path=robot_path) # keep track of paths for install easyconfigs, so we can obtain find specified easyconfigs easyconfigs_pkg_full_paths = easyconfigs_paths[:] if not easyconfigs_paths: _log.warning("Failed to determine install path for easybuild-easyconfigs package.") # specified robot paths are preferred over installed easyconfig files if robot_path: robot_path.extend(easyconfigs_paths) easyconfigs_paths = robot_path[:] _log.info("Extended list of robot paths with paths for installed easyconfigs: %s" % robot_path) # initialise the easybuild configuration config.init(options, eb_go.get_options_by_section('config')) # building a dependency graph implies force, so that all dependencies are retained # and also skips validation of easyconfigs (e.g. checking os dependencies) retain_all_deps = False if options.dep_graph: _log.info("Enabling force to generate dependency graph.") options.force = True retain_all_deps = True config.init_build_options({ 'aggregate_regtest': options.aggregate_regtest, 'allow_modules_tool_mismatch': options.allow_modules_tool_mismatch, 'check_osdeps': not options.ignore_osdeps, 'command_line': eb_command_line, 'debug': options.debug, 'dry_run': options.dry_run, 'easyblock': options.easyblock, 'experimental': options.experimental, 'force': options.force, 'group': options.group, 'ignore_dirs': options.ignore_dirs, 'modules_footer': options.modules_footer, 'only_blocks': options.only_blocks, 'recursive_mod_unload': options.recursive_module_unload, 'regtest_online': options.regtest_online, 'regtest_output_dir': options.regtest_output_dir, 'retain_all_deps': retain_all_deps, 'robot_path': robot_path, 'sequential': options.sequential, 'silent': testing, 'set_gid_bit': options.set_gid_bit, 'skip': options.skip, 'skip_test_cases': options.skip_test_cases, 'sticky_bit': options.sticky_bit, 'stop': options.stop, 'umask': options.umask, 'valid_module_classes': module_classes(), 'valid_stops': [x[0] for x in EasyBlock.get_steps()], 'validate': not options.force, }) # search for easyconfigs if options.search or options.search_short: search_path = [os.getcwd()] if easyconfigs_paths: search_path = easyconfigs_paths query = options.search or options.search_short ignore_dirs = config.build_option('ignore_dirs') silent = config.build_option('silent') search_file(search_path, query, short=not options.search, ignore_dirs=ignore_dirs, silent=silent) # process software build specifications (if any), i.e. # software name/version, toolchain name/version, extra patches, ... (try_to_generate, build_specs) = process_software_build_specs(options) paths = [] if len(orig_paths) == 0: if 'name' in build_specs: paths = [obtain_path(build_specs, easyconfigs_paths, try_to_generate=try_to_generate, exit_on_error=not testing)] elif not any([options.aggregate_regtest, options.search, options.search_short, options.regtest]): print_error(("Please provide one or multiple easyconfig files, or use software build " "options to make EasyBuild search for easyconfigs"), log=_log, opt_parser=eb_go.parser, exit_on_error=not testing) else: # look for easyconfigs with relative paths in easybuild-easyconfigs package, # unless they were found at the given relative paths if easyconfigs_pkg_full_paths: # determine which easyconfigs files need to be found, if any ecs_to_find = [] for idx, orig_path in enumerate(orig_paths): if orig_path == os.path.basename(orig_path) and not os.path.exists(orig_path): ecs_to_find.append((idx, orig_path)) _log.debug("List of easyconfig files to find: %s" % ecs_to_find) # find missing easyconfigs by walking paths with installed easyconfig files for path in easyconfigs_pkg_full_paths: _log.debug("Looking for missing easyconfig files (%d left) in %s..." % (len(ecs_to_find), path)) for (subpath, dirnames, filenames) in os.walk(path, topdown=True): for idx, orig_path in ecs_to_find[:]: if orig_path in filenames: full_path = os.path.join(subpath, orig_path) _log.info("Found %s in %s: %s" % (orig_path, path, full_path)) orig_paths[idx] = full_path # if file was found, stop looking for it (first hit wins) ecs_to_find.remove((idx, orig_path)) # stop os.walk insanity as soon as we have all we need (os.walk loop) if len(ecs_to_find) == 0: break # ignore subdirs specified to be ignored by replacing items in dirnames list used by os.walk dirnames[:] = [d for d in dirnames if not d in options.ignore_dirs] # stop os.walk insanity as soon as we have all we need (paths loop) if len(ecs_to_find) == 0: break # indicate that specified paths do not contain generated easyconfig files paths = [(path, False) for path in orig_paths] _log.debug("Paths: %s" % paths) # run regtest if options.regtest or options.aggregate_regtest: _log.info("Running regression test") if paths: ec_paths = [path[0] for path in paths] else: # fallback: easybuild-easyconfigs install path ec_paths = easyconfigs_pkg_full_paths regtest_ok = regtest(ec_paths) if not regtest_ok: _log.info("Regression test failed (partially)!") sys.exit(31) # exit -> 3x1t -> 31 # read easyconfig files easyconfigs = [] for (path, generated) in paths: path = os.path.abspath(path) if not os.path.exists(path): print_error("Can't find path %s" % path) try: files = find_easyconfigs(path, ignore_dirs=options.ignore_dirs) for f in files: if not generated and try_to_generate and build_specs: ec_file = tweak(f, None, build_specs) else: ec_file = f ecs = process_easyconfig(ec_file, build_specs=build_specs) easyconfigs.extend(ecs) except IOError, err: _log.error("Processing easyconfigs in path %s failed: %s" % (path, err))
for ec in ordered_ecs: (success, _) = build_and_install_software(ec, orig_environ) if success: correct_built_cnt += 1 all_built_cnt += 1 print_msg("Build succeeded for %s out of %s" % (correct_built_cnt, all_built_cnt), log=_log, silent=testing) repo = init_repository(get_repository(), get_repositorypath()) repo.cleanup() # cleanup and spec files for ec in easyconfigs: if 'original_spec' in ec and os.path.isfile(ec['spec']): os.remove(ec['spec']) # cleanup tmp log file (all is well, all modules have their own log file) if options.logtostdout: fancylogger.logToScreen(enable=False, stdout=True) else: fancylogger.logToFile(logfile, enable=False) cleanup(logfile, eb_tmpdir, testing) if __name__ == "__main__": try: main() except EasyBuildError, e: sys.stderr.write('ERROR: %s\n' % e.msg) sys.exit(1)
def main(testing_data=(None, None)): """ Main function: @arg options: a tuple: (options, paths, logger, logfile, hn) as defined in parse_options This function will: - read easyconfig - build software """ # disallow running EasyBuild as root if os.getuid() == 0: sys.stderr.write( "ERROR: You seem to be running EasyBuild with root privileges.\n" "That's not wise, so let's end this here.\n" "Exiting.\n" ) sys.exit(1) # steer behavior when testing main testing = testing_data[0] is not None args, logfile = testing_data # initialise options eb_go = eboptions.parse_options(args=args) options = eb_go.options orig_paths = eb_go.args # initialise logging for main if options.logtostdout: fancylogger.logToScreen(enable=True, stdout=True) else: if logfile is None: # mkstemp returns (fd,filename), fd is from os.open, not regular open! fd, logfile = tempfile.mkstemp(suffix=".log", prefix="easybuild-") os.close(fd) fancylogger.logToFile(logfile) print_msg("temporary log file in case of crash %s" % (logfile), log=None, silent=testing) global _log _log = fancylogger.getLogger(fname=False) # hello world! _log.info(this_is_easybuild()) # set strictness of filetools module if options.strict: filetools.strictness = options.strict if not options.robot is None: if options.robot: _log.info("Using robot path: %s" % options.robot) else: _log.error("No robot path specified, and unable to determine easybuild-easyconfigs install path.") # determine easybuild-easyconfigs package install path easyconfigs_paths = get_paths_for("easyconfigs", robot_path=options.robot) easyconfigs_pkg_full_path = None search_path = os.getcwd() if easyconfigs_paths: easyconfigs_pkg_full_path = easyconfigs_paths[0] if not options.robot: search_path = easyconfigs_pkg_full_path else: search_path = options.robot else: _log.info("Failed to determine install path for easybuild-easyconfigs package.") if options.robot: easyconfigs_paths = [options.robot] + easyconfigs_paths # initialise the easybuild configuration config.init(options, eb_go.get_options_by_section("config")) # search for easyconfigs if options.search: search_file(search_path, options.search, silent=testing) # process software build specifications (if any), i.e. # software name/version, toolchain name/version, extra patches, ... (try_to_generate, software_build_specs) = process_software_build_specs(options) paths = [] if len(orig_paths) == 0: if software_build_specs.has_key("name"): paths = [ obtain_path( software_build_specs, easyconfigs_paths, try_to_generate=try_to_generate, exit_on_error=not testing ) ] elif not any([options.aggregate_regtest, options.search, options.regtest]): print_error( ( "Please provide one or multiple easyconfig files, or use software build " "options to make EasyBuild search for easyconfigs" ), log=_log, opt_parser=eb_go.parser, exit_on_error=not testing, ) else: # look for easyconfigs with relative paths in easybuild-easyconfigs package, # unless they we found at the given relative paths if easyconfigs_pkg_full_path: # create a mapping from filename to path in easybuild-easyconfigs package install path easyconfigs_map = {} for (subpath, _, filenames) in os.walk(easyconfigs_pkg_full_path): for filename in filenames: easyconfigs_map.update({filename: os.path.join(subpath, filename)}) # try and find non-existing non-absolute eaysconfig paths in easybuild-easyconfigs package install path for idx, orig_path in enumerate(orig_paths): if not os.path.isabs(orig_path) and not os.path.exists(orig_path): if orig_path in easyconfigs_map: _log.info( "Found %s in %s: %s" % (orig_path, easyconfigs_pkg_full_path, easyconfigs_map[orig_path]) ) orig_paths[idx] = easyconfigs_map[orig_path] # indicate that specified paths do not contain generated easyconfig files paths = [(path, False) for path in orig_paths] _log.debug("Paths: %s" % paths) # run regtest if options.regtest or options.aggregate_regtest: _log.info("Running regression test") if paths: regtest_ok = regtest(options, [path[0] for path in paths]) else: # fallback: easybuild-easyconfigs install path regtest_ok = regtest(options, [easyconfigs_pkg_full_path]) if not regtest_ok: _log.info("Regression test failed (partially)!") sys.exit(31) # exit -> 3x1t -> 31 if any([options.search, options.regtest]): cleanup_logfile_and_exit(logfile, testing, True) # building a dependency graph implies force, so that all dependencies are retained # and also skips validation of easyconfigs (e.g. checking os dependencies) validate_easyconfigs = True retain_all_deps = False if options.dep_graph: _log.info("Enabling force to generate dependency graph.") options.force = True validate_easyconfigs = False retain_all_deps = True # read easyconfig files easyconfigs = [] for (path, generated) in paths: path = os.path.abspath(path) if not (os.path.exists(path)): print_error("Can't find path %s" % path) try: files = find_easyconfigs(path) for f in files: if not generated and try_to_generate and software_build_specs: ec_file = easyconfig.tools.tweak(f, None, software_build_specs) else: ec_file = f easyconfigs.extend(process_easyconfig(ec_file, options.only_blocks, validate=validate_easyconfigs)) except IOError, err: _log.error("Processing easyconfigs in path %s failed: %s" % (path, err))
parser.add_option("-l", "--local", action="store_true", dest="local", help="Use a local path, not on github.com (Default false)") options, args = parser.parse_args() # get and configure logger log = fancylogger.getLogger(__name__) if options.verbose == 1: fancylogger.setLogLevelWarning() elif options.verbose == 2: fancylogger.setLogLevelInfo() elif options.verbose >= 3: fancylogger.setLogLevelDebug() if options.quiet: fancylogger.logToScreen(False) else: fancylogger.logToScreen(True) # other options if not options.branch: options.branch = "develop" if not options.username: options.username = "******" if not options.repo: options.repo = "easybuild-easyconfigs" if not options.path: options.path = "easybuild/easyconfigs" if options.local: import os walk = os.walk
action="store_true", dest="local", help="Use a local path, not on github.com (Default false)") options, args = parser.parse_args() # get and configure logger log = fancylogger.getLogger(__name__) if options.verbose == 1: fancylogger.setLogLevelWarning() elif options.verbose == 2: fancylogger.setLogLevelInfo() elif options.verbose >= 3: fancylogger.setLogLevelDebug() if options.quiet: fancylogger.logToScreen(False) else: fancylogger.logToScreen(True) # other options if not options.branch: options.branch = "develop" if not options.username: options.username = "******" if not options.repo: options.repo = "easybuild-easyconfigs" if not options.path: options.path = "easybuild/easyconfigs" if options.local: import os walk = os.walk
correct_built_cnt += 1 all_built_cnt += 1 print_msg("Build succeeded for %s out of %s" % (correct_built_cnt, all_built_cnt), log=_log, silent=testing) repo = init_repository(get_repository(), get_repositorypath()) repo.cleanup() # cleanup and spec files for ec in easyconfigs: if 'originalSpec' in ec and os.path.isfile(ec['spec']): os.remove(ec['spec']) # cleanup tmp log file (all is well, all modules have their own log file) if options.logtostdout: fancylogger.logToScreen(enable=False, stdout=True) else: fancylogger.logToFile(logfile, enable=False) cleanup_logfile_and_exit(logfile, testing, False) logfile = None return logfile def cleanup_logfile_and_exit(logfile, testing, doexit): """Cleanup the logfile and exit""" if not testing and logfile is not None: os.remove(logfile) print_msg('temporary log file %s has been removed.' % (logfile), log=None, silent=testing) if doexit: sys.exit(0)
def main(testing_data=(None, None)): """ Main function: @arg options: a tuple: (options, paths, logger, logfile, hn) as defined in parse_options This function will: - read easyconfig - build software """ # disallow running EasyBuild as root if os.getuid() == 0: sys.stderr.write("ERROR: You seem to be running EasyBuild with root privileges.\n" \ "That's not wise, so let's end this here.\n" \ "Exiting.\n") sys.exit(1) # steer behavior when testing main testing = testing_data[0] is not None args, logfile = testing_data # initialise options eb_go = eboptions.parse_options(args=args) options = eb_go.options orig_paths = eb_go.args # initialise logging for main if options.logtostdout: fancylogger.logToScreen(enable=True, stdout=True) else: if logfile is None: # mkstemp returns (fd,filename), fd is from os.open, not regular open! fd, logfile = tempfile.mkstemp(suffix='.log', prefix='easybuild-') os.close(fd) fancylogger.logToFile(logfile) print_msg('temporary log file in case of crash %s' % (logfile), log=None, silent=testing) global _log _log = fancylogger.getLogger(fname=False) # hello world! _log.info(this_is_easybuild()) # set strictness of filetools module if options.strict: filetools.strictness = options.strict if not options.robot is None: if options.robot: _log.info("Using robot path: %s" % options.robot) else: _log.error("No robot path specified, and unable to determine easybuild-easyconfigs install path.") # determine easybuild-easyconfigs package install path easyconfigs_paths = get_paths_for("easyconfigs", robot_path=options.robot) easyconfigs_pkg_full_path = None search_path = os.getcwd() if easyconfigs_paths: easyconfigs_pkg_full_path = easyconfigs_paths[0] if not options.robot: search_path = easyconfigs_pkg_full_path else: search_path = options.robot else: _log.info("Failed to determine install path for easybuild-easyconfigs package.") if options.robot: easyconfigs_paths = [options.robot] + easyconfigs_paths # initialise the easybuild configuration config.init(options, eb_go.get_options_by_section('config')) # search for easyconfigs if options.search: search_file(search_path, options.search, silent=testing) # process software build specifications (if any), i.e. # software name/version, toolchain name/version, extra patches, ... (try_to_generate, software_build_specs) = process_software_build_specs(options) paths = [] if len(orig_paths) == 0: if software_build_specs.has_key('name'): paths = [obtain_path(software_build_specs, easyconfigs_paths, try_to_generate=try_to_generate, exit_on_error=not testing)] elif not any([options.aggregate_regtest, options.search, options.regtest]): print_error(("Please provide one or multiple easyconfig files, or use software build " "options to make EasyBuild search for easyconfigs"), log=_log, opt_parser=eb_go.parser, exit_on_error=not testing) else: # look for easyconfigs with relative paths in easybuild-easyconfigs package, # unless they we found at the given relative paths if easyconfigs_pkg_full_path: # create a mapping from filename to path in easybuild-easyconfigs package install path easyconfigs_map = {} for (subpath, _, filenames) in os.walk(easyconfigs_pkg_full_path): for filename in filenames: easyconfigs_map.update({filename: os.path.join(subpath, filename)}) # try and find non-existing non-absolute eaysconfig paths in easybuild-easyconfigs package install path for idx, orig_path in enumerate(orig_paths): if not os.path.isabs(orig_path) and not os.path.exists(orig_path): if orig_path in easyconfigs_map: _log.info("Found %s in %s: %s" % (orig_path, easyconfigs_pkg_full_path, easyconfigs_map[orig_path])) orig_paths[idx] = easyconfigs_map[orig_path] # indicate that specified paths do not contain generated easyconfig files paths = [(path, False) for path in orig_paths] _log.debug("Paths: %s" % paths) # run regtest if options.regtest or options.aggregate_regtest: _log.info("Running regression test") if paths: regtest_ok = regtest(options, [path[0] for path in paths]) else: # fallback: easybuild-easyconfigs install path regtest_ok = regtest(options, [easyconfigs_pkg_full_path]) if not regtest_ok: _log.info("Regression test failed (partially)!") sys.exit(31) # exit -> 3x1t -> 31 if any([options.search, options.regtest]): cleanup_logfile_and_exit(logfile, testing, True) # building a dependency graph implies force, so that all dependencies are retained # and also skips validation of easyconfigs (e.g. checking os dependencies) validate_easyconfigs = True retain_all_deps = False if options.dep_graph: _log.info("Enabling force to generate dependency graph.") options.force = True validate_easyconfigs = False retain_all_deps = True # read easyconfig files easyconfigs = [] for (path, generated) in paths: path = os.path.abspath(path) if not (os.path.exists(path)): print_error("Can't find path %s" % path) try: files = find_easyconfigs(path) for f in files: if not generated and try_to_generate and software_build_specs: ec_file = easyconfig.tools.tweak(f, None, software_build_specs) else: ec_file = f easyconfigs.extend(process_easyconfig(ec_file, options.only_blocks, validate=validate_easyconfigs)) except IOError, err: _log.error("Processing easyconfigs in path %s failed: %s" % (path, err))