def test_zzz_logtostdout(self):
"""Testing redirecting log to stdout."""
for stdout_arg in ['--logtostdout', '-l']:
_stdout = sys.stdout
myerr = None
fd, fn = tempfile.mkstemp()
fh = os.fdopen(fd, 'w')
sys.stdout = fh
args = [
'--software-name=somethingrandom',
'--robot=.',
'--debug',
stdout_arg,
]
try:
main((args, None))
except Exception, err:
myerr = err
# make sure we restore
sys.stdout.flush()
sys.stdout = _stdout
fancylogger.logToScreen(enable=False, stdout=True)
outtxt = open(fn, 'r').read()
self.assertTrue(len(outtxt) > 100, "Log messages are printed to stdout when %s is used (outtxt: %s)" % (stdout_arg, outtxt))
# cleanup
os.remove(fn)
def parse_options(args=None):
"""wrapper function for option parsing"""
if os.environ.get('DEBUG_EASYBUILD_OPTIONS', '0').lower() in ('1', 'true', 'yes', 'y'):
# very early debug, to debug the generaloption itself
fancylogger.logToScreen(enable=True)
fancylogger.setLogLevel('DEBUG')
usage = "%prog [options] easyconfig [...]"
description = ("Builds software based on easyconfig (or parse a directory).\n"
"Provide one or more easyconfigs or directories, use -H or --help more information.")
eb_go = EasyBuildOptions(usage=usage, description=description, prog='eb', envvar_prefix='EASYBUILD', go_args=args)
return eb_go
def __init__(self, hostname, port, log_dir, filename, pidfile):
"""Constructor"""
stdin = '/dev/null'
stdout = os.path.join(log_dir, 'logging_error.log')
stderr = os.path.join(log_dir, 'logging_error.log')
Daemon.__init__(self, pidfile, stdin, stdout, stderr)
self.hostname = hostname
self.port = port
##Set up logging
#get logger, we will log to file
fancylogger.logToScreen(False)
#we want to log absolutely everything that's comming at us
fancylogger.setLogLevel(0)
self.logfile = os.path.join(log_dir, filename)
fancylogger.logToFile(self.logfile)
self.logger = fancylogger.getLogger()
self.socket_ = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
def __init__(self, hostname, port, log_dir, filename, pidfile):
"""Constructor"""
stdin = '/dev/null'
stdout = os.path.join(log_dir, 'logging_error.log')
stderr = os.path.join(log_dir, 'logging_error.log')
Daemon.__init__(self, pidfile, stdin, stdout, stderr)
self.hostname = hostname
self.port = port
#Set up logging
# get logger, we will log to file
fancylogger.logToScreen(False)
# we want to log absolutely everything that's comming at us
fancylogger.setLogLevel(0)
self.logfile = os.path.join(log_dir, filename)
fancylogger.logToFile(self.logfile)
self.logger = fancylogger.getLogger()
self.socket_ = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
def parse_options(args=None):
"""wrapper function for option parsing"""
if os.environ.get('DEBUG_EASYBUILD_OPTIONS', '0').lower() in ('1', 'true', 'yes', 'y'):
# very early debug, to debug the generaloption itself
fancylogger.logToScreen(enable=True)
fancylogger.setLogLevel('DEBUG')
usage = "%prog [options] easyconfig [...]"
description = ("Builds software based on easyconfig (or parse a directory).\n"
"Provide one or more easyconfigs or directories, use -H or --help more information.")
eb_go = EasyBuildOptions(
usage=usage,
description=description,
prog='eb',
envvar_prefix='EASYBUILD',
go_args=args,
)
return eb_go
def test_zzz_logtostdout(self):
"""Testing redirecting log to stdout."""
fd, dummylogfn = tempfile.mkstemp(prefix='easybuild-dummy',
suffix='.log')
os.close(fd)
for stdout_arg in ['--logtostdout', '-l']:
_stdout = sys.stdout
myerr = None
fd, fn = tempfile.mkstemp()
fh = os.fdopen(fd, 'w')
sys.stdout = fh
args = [
'--software-name=somethingrandom',
'--robot',
'.',
'--debug',
stdout_arg,
]
try:
main((args, dummylogfn, False))
except (SystemExit, Exception), err:
myerr = err
# make sure we restore
sys.stdout.flush()
sys.stdout = _stdout
fancylogger.logToScreen(enable=False, stdout=True)
outtxt = read_file(fn)
self.assertTrue(
len(outtxt) > 100,
"Log messages are printed to stdout when %s is used (outtxt: %s)"
% (stdout_arg, outtxt))
# cleanup
os.remove(fn)
def _stream_stdouterr(self, isstdout=True, expect_match=True):
fd, logfn = tempfile.mkstemp()
# fh will be checked
fh = os.fdopen(fd, "w")
_stdout = sys.stdout
_stderr = sys.stderr
if isstdout == expect_match:
sys.stdout = fh
sys.stderr = open(os.devnull, "w")
else:
sys.stdout = open(os.devnull, "w")
sys.stderr = fh
fancylogger.setLogLevelInfo()
name = "test_stream_stdout"
lh = fancylogger.logToScreen(stdout=isstdout)
logger = fancylogger.getLogger(name)
# logfn makes it unique
msg = "TEST isstdout %s expect_match %s logfn %s" % (isstdout, expect_match, logfn)
logger.info(msg)
# restore
fancylogger.logToScreen(enable=False, handler=lh)
sys.stdout = _stdout
sys.stderr = _stderr
fh2 = open(logfn)
txt = fh2.read().strip()
fh2.close()
reg_exp = re.compile(r"INFO\s+\S+.%s.%s\s+\S+\s+%s" % (name, "_stream_stdouterr", msg))
match = reg_exp.search(txt) is not None
self.assertEqual(match, expect_match)
try:
fh.close()
os.remove(logfn)
except:
pass
def test_zzz_logtostdout(self):
"""Testing redirecting log to stdout."""
fd, dummylogfn = tempfile.mkstemp(prefix='easybuild-dummy', suffix='.log')
os.close(fd)
for stdout_arg in ['--logtostdout', '-l']:
_stdout = sys.stdout
fd, fn = tempfile.mkstemp()
fh = os.fdopen(fd, 'w')
sys.stdout = fh
args = [
'--software-name=somethingrandom',
'--robot', '.',
'--debug',
stdout_arg,
]
self.eb_main(args, logfile=dummylogfn)
# make sure we restore
sys.stdout.flush()
sys.stdout = _stdout
fancylogger.logToScreen(enable=False, stdout=True)
outtxt = read_file(fn)
self.assertTrue(len(outtxt) > 100, "Log messages are printed to stdout when %s is used (outtxt: %s)" % (stdout_arg, outtxt))
# cleanup
os.remove(fn)
modify_env(os.environ, self.orig_environ)
tempfile.tempdir = None
if os.path.exists(dummylogfn):
os.remove(dummylogfn)
fancylogger.logToFile(self.logfile)
def test_zzz_logtostdout(self):
"""Testing redirecting log to stdout."""
fd, dummylogfn = tempfile.mkstemp(prefix='easybuild-dummy', suffix='.log')
os.close(fd)
for stdout_arg in ['--logtostdout', '-l']:
_stdout = sys.stdout
myerr = None
fd, fn = tempfile.mkstemp()
fh = os.fdopen(fd, 'w')
sys.stdout = fh
args = [
'--software-name=somethingrandom',
'--robot', '.',
'--debug',
stdout_arg,
]
try:
main((args, dummylogfn))
except (SystemExit, Exception), err:
myerr = err
# make sure we restore
sys.stdout.flush()
sys.stdout = _stdout
fancylogger.logToScreen(enable=False, stdout=True)
outtxt = read_file(fn)
self.assertTrue(len(outtxt) > 100, "Log messages are printed to stdout when %s is used (outtxt: %s)" % (stdout_arg, outtxt))
# cleanup
os.remove(fn)
def test_zzz_logtostdout(self):
"""Testing redirecting log to stdout."""
fd, dummylogfn = tempfile.mkstemp(prefix="easybuild-dummy", suffix=".log")
os.close(fd)
for stdout_arg in ["--logtostdout", "-l"]:
_stdout = sys.stdout
myerr = None
fd, fn = tempfile.mkstemp()
fh = os.fdopen(fd, "w")
sys.stdout = fh
args = ["--software-name=somethingrandom", "--robot", ".", "--debug", stdout_arg]
try:
main((args, dummylogfn, False))
except (SystemExit, Exception), err:
myerr = err
# make sure we restore
sys.stdout.flush()
sys.stdout = _stdout
fancylogger.logToScreen(enable=False, stdout=True)
outtxt = read_file(fn)
self.assertTrue(
len(outtxt) > 100,
"Log messages are printed to stdout when %s is used (outtxt: %s)" % (stdout_arg, outtxt),
)
# cleanup
os.remove(fn)
modify_env(os.environ, self.orig_environ)
tempfile.tempdir = None
def main(testing_data=(None, None, None)):
"""
Main function:
@arg options: a tuple: (options, paths, logger, logfile, hn) as defined in parse_options
This function will:
- read easyconfig
- build software
"""
# disallow running EasyBuild as root
if os.getuid() == 0:
sys.stderr.write("ERROR: You seem to be running EasyBuild with root privileges.\n"
"That's not wise, so let's end this here.\n"
"Exiting.\n")
sys.exit(1)
# steer behavior when testing main
testing = testing_data[0] is not None
args, logfile, do_build = testing_data
# initialise options
eb_go = eboptions.parse_options(args=args)
options = eb_go.options
orig_paths = eb_go.args
# set umask (as early as possible)
if options.umask is not None:
new_umask = int(options.umask, 8)
old_umask = os.umask(new_umask)
# set temporary directory to use
eb_tmpdir = set_tmpdir(options.tmpdir)
# initialise logging for main
if options.logtostdout:
fancylogger.logToScreen(enable=True, stdout=True)
else:
if logfile is None:
# mkstemp returns (fd,filename), fd is from os.open, not regular open!
fd, logfile = tempfile.mkstemp(suffix='.log', prefix='easybuild-')
os.close(fd)
fancylogger.logToFile(logfile)
print_msg('temporary log file in case of crash %s' % (logfile), log=None, silent=testing)
global _log
_log = fancylogger.getLogger(fname=False)
if options.umask is not None:
_log.info("umask set to '%s' (used to be '%s')" % (oct(new_umask), oct(old_umask)))
# hello world!
_log.info(this_is_easybuild())
# how was EB called?
eb_command_line = eb_go.generate_cmd_line() + eb_go.args
_log.info("Command line: %s" % (" ".join(eb_command_line)))
_log.info("Using %s as temporary directory" % eb_tmpdir)
if not options.robot is None:
if options.robot:
_log.info("Using robot path(s): %s" % options.robot)
else:
_log.error("No robot paths specified, and unable to determine easybuild-easyconfigs install path.")
# do not pass options.robot, it's not a list instance (and it shouldn't be modified)
robot_path = None
if options.robot:
robot_path = list(options.robot)
# determine easybuild-easyconfigs package install path
easyconfigs_paths = get_paths_for("easyconfigs", robot_path=robot_path)
# keep track of paths for install easyconfigs, so we can obtain find specified easyconfigs
easyconfigs_pkg_full_paths = easyconfigs_paths[:]
if not easyconfigs_paths:
_log.warning("Failed to determine install path for easybuild-easyconfigs package.")
# specified robot paths are preferred over installed easyconfig files
if robot_path:
robot_path.extend(easyconfigs_paths)
easyconfigs_paths = robot_path[:]
_log.info("Extended list of robot paths with paths for installed easyconfigs: %s" % robot_path)
# initialise the easybuild configuration
config.init(options, eb_go.get_options_by_section('config'))
# building a dependency graph implies force, so that all dependencies are retained
# and also skips validation of easyconfigs (e.g. checking os dependencies)
retain_all_deps = False
if options.dep_graph:
_log.info("Enabling force to generate dependency graph.")
options.force = True
retain_all_deps = True
config.init_build_options({
'aggregate_regtest': options.aggregate_regtest,
'allow_modules_tool_mismatch': options.allow_modules_tool_mismatch,
'check_osdeps': not options.ignore_osdeps,
'command_line': eb_command_line,
'debug': options.debug,
'dry_run': options.dry_run,
'easyblock': options.easyblock,
'experimental': options.experimental,
'force': options.force,
'group': options.group,
'ignore_dirs': options.ignore_dirs,
'modules_footer': options.modules_footer,
'only_blocks': options.only_blocks,
'recursive_mod_unload': options.recursive_module_unload,
'regtest_online': options.regtest_online,
'regtest_output_dir': options.regtest_output_dir,
'retain_all_deps': retain_all_deps,
'robot_path': robot_path,
'sequential': options.sequential,
'silent': testing,
'set_gid_bit': options.set_gid_bit,
'skip': options.skip,
'skip_test_cases': options.skip_test_cases,
'sticky_bit': options.sticky_bit,
'stop': options.stop,
'umask': options.umask,
'valid_module_classes': module_classes(),
'valid_stops': [x[0] for x in EasyBlock.get_steps()],
'validate': not options.force,
})
# search for easyconfigs
if options.search or options.search_short:
search_path = [os.getcwd()]
if easyconfigs_paths:
search_path = easyconfigs_paths
query = options.search or options.search_short
ignore_dirs = config.build_option('ignore_dirs')
silent = config.build_option('silent')
search_file(search_path, query, short=not options.search, ignore_dirs=ignore_dirs, silent=silent)
# process software build specifications (if any), i.e.
# software name/version, toolchain name/version, extra patches, ...
(try_to_generate, build_specs) = process_software_build_specs(options)
paths = []
if len(orig_paths) == 0:
if 'name' in build_specs:
paths = [obtain_path(build_specs, easyconfigs_paths, try_to_generate=try_to_generate,
exit_on_error=not testing)]
elif not any([options.aggregate_regtest, options.search, options.search_short, options.regtest]):
print_error(("Please provide one or multiple easyconfig files, or use software build "
"options to make EasyBuild search for easyconfigs"),
log=_log, opt_parser=eb_go.parser, exit_on_error=not testing)
else:
# look for easyconfigs with relative paths in easybuild-easyconfigs package,
# unless they were found at the given relative paths
if easyconfigs_pkg_full_paths:
# determine which easyconfigs files need to be found, if any
ecs_to_find = []
for idx, orig_path in enumerate(orig_paths):
if orig_path == os.path.basename(orig_path) and not os.path.exists(orig_path):
ecs_to_find.append((idx, orig_path))
_log.debug("List of easyconfig files to find: %s" % ecs_to_find)
# find missing easyconfigs by walking paths with installed easyconfig files
for path in easyconfigs_pkg_full_paths:
_log.debug("Looking for missing easyconfig files (%d left) in %s..." % (len(ecs_to_find), path))
for (subpath, dirnames, filenames) in os.walk(path, topdown=True):
for idx, orig_path in ecs_to_find[:]:
if orig_path in filenames:
full_path = os.path.join(subpath, orig_path)
_log.info("Found %s in %s: %s" % (orig_path, path, full_path))
orig_paths[idx] = full_path
# if file was found, stop looking for it (first hit wins)
ecs_to_find.remove((idx, orig_path))
# stop os.walk insanity as soon as we have all we need (os.walk loop)
if len(ecs_to_find) == 0:
break
# ignore subdirs specified to be ignored by replacing items in dirnames list used by os.walk
dirnames[:] = [d for d in dirnames if not d in options.ignore_dirs]
# stop os.walk insanity as soon as we have all we need (paths loop)
if len(ecs_to_find) == 0:
break
# indicate that specified paths do not contain generated easyconfig files
paths = [(path, False) for path in orig_paths]
_log.debug("Paths: %s" % paths)
# run regtest
if options.regtest or options.aggregate_regtest:
_log.info("Running regression test")
if paths:
ec_paths = [path[0] for path in paths]
else: # fallback: easybuild-easyconfigs install path
ec_paths = easyconfigs_pkg_full_paths
regtest_ok = regtest(ec_paths)
if not regtest_ok:
_log.info("Regression test failed (partially)!")
sys.exit(31) # exit -> 3x1t -> 31
# read easyconfig files
easyconfigs = []
for (path, generated) in paths:
path = os.path.abspath(path)
if not os.path.exists(path):
print_error("Can't find path %s" % path)
try:
files = find_easyconfigs(path, ignore_dirs=options.ignore_dirs)
for f in files:
if not generated and try_to_generate and build_specs:
ec_file = tweak(f, None, build_specs)
else:
ec_file = f
ecs = process_easyconfig(ec_file, build_specs=build_specs)
easyconfigs.extend(ecs)
except IOError, err:
_log.error("Processing easyconfigs in path %s failed: %s" % (path, err))
for ec in ordered_ecs:
(success, _) = build_and_install_software(ec, orig_environ)
if success:
correct_built_cnt += 1
all_built_cnt += 1
print_msg("Build succeeded for %s out of %s" % (correct_built_cnt, all_built_cnt), log=_log, silent=testing)
repo = init_repository(get_repository(), get_repositorypath())
repo.cleanup()
# cleanup and spec files
for ec in easyconfigs:
if 'original_spec' in ec and os.path.isfile(ec['spec']):
os.remove(ec['spec'])
# cleanup tmp log file (all is well, all modules have their own log file)
if options.logtostdout:
fancylogger.logToScreen(enable=False, stdout=True)
else:
fancylogger.logToFile(logfile, enable=False)
cleanup(logfile, eb_tmpdir, testing)
if __name__ == "__main__":
try:
main()
except EasyBuildError, e:
sys.stderr.write('ERROR: %s\n' % e.msg)
sys.exit(1)
def main(testing_data=(None, None)):
"""
Main function:
@arg options: a tuple: (options, paths, logger, logfile, hn) as defined in parse_options
This function will:
- read easyconfig
- build software
"""
# disallow running EasyBuild as root
if os.getuid() == 0:
sys.stderr.write(
"ERROR: You seem to be running EasyBuild with root privileges.\n"
"That's not wise, so let's end this here.\n"
"Exiting.\n"
)
sys.exit(1)
# steer behavior when testing main
testing = testing_data[0] is not None
args, logfile = testing_data
# initialise options
eb_go = eboptions.parse_options(args=args)
options = eb_go.options
orig_paths = eb_go.args
# initialise logging for main
if options.logtostdout:
fancylogger.logToScreen(enable=True, stdout=True)
else:
if logfile is None:
# mkstemp returns (fd,filename), fd is from os.open, not regular open!
fd, logfile = tempfile.mkstemp(suffix=".log", prefix="easybuild-")
os.close(fd)
fancylogger.logToFile(logfile)
print_msg("temporary log file in case of crash %s" % (logfile), log=None, silent=testing)
global _log
_log = fancylogger.getLogger(fname=False)
# hello world!
_log.info(this_is_easybuild())
# set strictness of filetools module
if options.strict:
filetools.strictness = options.strict
if not options.robot is None:
if options.robot:
_log.info("Using robot path: %s" % options.robot)
else:
_log.error("No robot path specified, and unable to determine easybuild-easyconfigs install path.")
# determine easybuild-easyconfigs package install path
easyconfigs_paths = get_paths_for("easyconfigs", robot_path=options.robot)
easyconfigs_pkg_full_path = None
search_path = os.getcwd()
if easyconfigs_paths:
easyconfigs_pkg_full_path = easyconfigs_paths[0]
if not options.robot:
search_path = easyconfigs_pkg_full_path
else:
search_path = options.robot
else:
_log.info("Failed to determine install path for easybuild-easyconfigs package.")
if options.robot:
easyconfigs_paths = [options.robot] + easyconfigs_paths
# initialise the easybuild configuration
config.init(options, eb_go.get_options_by_section("config"))
# search for easyconfigs
if options.search:
search_file(search_path, options.search, silent=testing)
# process software build specifications (if any), i.e.
# software name/version, toolchain name/version, extra patches, ...
(try_to_generate, software_build_specs) = process_software_build_specs(options)
paths = []
if len(orig_paths) == 0:
if software_build_specs.has_key("name"):
paths = [
obtain_path(
software_build_specs, easyconfigs_paths, try_to_generate=try_to_generate, exit_on_error=not testing
)
]
elif not any([options.aggregate_regtest, options.search, options.regtest]):
print_error(
(
"Please provide one or multiple easyconfig files, or use software build "
"options to make EasyBuild search for easyconfigs"
),
log=_log,
opt_parser=eb_go.parser,
exit_on_error=not testing,
)
else:
# look for easyconfigs with relative paths in easybuild-easyconfigs package,
# unless they we found at the given relative paths
if easyconfigs_pkg_full_path:
# create a mapping from filename to path in easybuild-easyconfigs package install path
easyconfigs_map = {}
for (subpath, _, filenames) in os.walk(easyconfigs_pkg_full_path):
for filename in filenames:
easyconfigs_map.update({filename: os.path.join(subpath, filename)})
# try and find non-existing non-absolute eaysconfig paths in easybuild-easyconfigs package install path
for idx, orig_path in enumerate(orig_paths):
if not os.path.isabs(orig_path) and not os.path.exists(orig_path):
if orig_path in easyconfigs_map:
_log.info(
"Found %s in %s: %s" % (orig_path, easyconfigs_pkg_full_path, easyconfigs_map[orig_path])
)
orig_paths[idx] = easyconfigs_map[orig_path]
# indicate that specified paths do not contain generated easyconfig files
paths = [(path, False) for path in orig_paths]
_log.debug("Paths: %s" % paths)
# run regtest
if options.regtest or options.aggregate_regtest:
_log.info("Running regression test")
if paths:
regtest_ok = regtest(options, [path[0] for path in paths])
else: # fallback: easybuild-easyconfigs install path
regtest_ok = regtest(options, [easyconfigs_pkg_full_path])
if not regtest_ok:
_log.info("Regression test failed (partially)!")
sys.exit(31) # exit -> 3x1t -> 31
if any([options.search, options.regtest]):
cleanup_logfile_and_exit(logfile, testing, True)
# building a dependency graph implies force, so that all dependencies are retained
# and also skips validation of easyconfigs (e.g. checking os dependencies)
validate_easyconfigs = True
retain_all_deps = False
if options.dep_graph:
_log.info("Enabling force to generate dependency graph.")
options.force = True
validate_easyconfigs = False
retain_all_deps = True
# read easyconfig files
easyconfigs = []
for (path, generated) in paths:
path = os.path.abspath(path)
if not (os.path.exists(path)):
print_error("Can't find path %s" % path)
try:
files = find_easyconfigs(path)
for f in files:
if not generated and try_to_generate and software_build_specs:
ec_file = easyconfig.tools.tweak(f, None, software_build_specs)
else:
ec_file = f
easyconfigs.extend(process_easyconfig(ec_file, options.only_blocks, validate=validate_easyconfigs))
except IOError, err:
_log.error("Processing easyconfigs in path %s failed: %s" % (path, err))
parser.add_option("-l", "--local", action="store_true", dest="local",
help="Use a local path, not on github.com (Default false)")
options, args = parser.parse_args()
# get and configure logger
log = fancylogger.getLogger(__name__)
if options.verbose == 1:
fancylogger.setLogLevelWarning()
elif options.verbose == 2:
fancylogger.setLogLevelInfo()
elif options.verbose >= 3:
fancylogger.setLogLevelDebug()
if options.quiet:
fancylogger.logToScreen(False)
else:
fancylogger.logToScreen(True)
# other options
if not options.branch:
options.branch = "develop"
if not options.username:
options.username = "******"
if not options.repo:
options.repo = "easybuild-easyconfigs"
if not options.path:
options.path = "easybuild/easyconfigs"
if options.local:
import os
walk = os.walk
action="store_true",
dest="local",
help="Use a local path, not on github.com (Default false)")
options, args = parser.parse_args()
# get and configure logger
log = fancylogger.getLogger(__name__)
if options.verbose == 1:
fancylogger.setLogLevelWarning()
elif options.verbose == 2:
fancylogger.setLogLevelInfo()
elif options.verbose >= 3:
fancylogger.setLogLevelDebug()
if options.quiet:
fancylogger.logToScreen(False)
else:
fancylogger.logToScreen(True)
# other options
if not options.branch:
options.branch = "develop"
if not options.username:
options.username = "******"
if not options.repo:
options.repo = "easybuild-easyconfigs"
if not options.path:
options.path = "easybuild/easyconfigs"
if options.local:
import os
walk = os.walk
correct_built_cnt += 1
all_built_cnt += 1
print_msg("Build succeeded for %s out of %s" % (correct_built_cnt, all_built_cnt), log=_log, silent=testing)
repo = init_repository(get_repository(), get_repositorypath())
repo.cleanup()
# cleanup and spec files
for ec in easyconfigs:
if 'originalSpec' in ec and os.path.isfile(ec['spec']):
os.remove(ec['spec'])
# cleanup tmp log file (all is well, all modules have their own log file)
if options.logtostdout:
fancylogger.logToScreen(enable=False, stdout=True)
else:
fancylogger.logToFile(logfile, enable=False)
cleanup_logfile_and_exit(logfile, testing, False)
logfile = None
return logfile
def cleanup_logfile_and_exit(logfile, testing, doexit):
"""Cleanup the logfile and exit"""
if not testing and logfile is not None:
os.remove(logfile)
print_msg('temporary log file %s has been removed.' % (logfile), log=None, silent=testing)
if doexit:
sys.exit(0)
def main(testing_data=(None, None)):
"""
Main function:
@arg options: a tuple: (options, paths, logger, logfile, hn) as defined in parse_options
This function will:
- read easyconfig
- build software
"""
# disallow running EasyBuild as root
if os.getuid() == 0:
sys.stderr.write("ERROR: You seem to be running EasyBuild with root privileges.\n" \
"That's not wise, so let's end this here.\n" \
"Exiting.\n")
sys.exit(1)
# steer behavior when testing main
testing = testing_data[0] is not None
args, logfile = testing_data
# initialise options
eb_go = eboptions.parse_options(args=args)
options = eb_go.options
orig_paths = eb_go.args
# initialise logging for main
if options.logtostdout:
fancylogger.logToScreen(enable=True, stdout=True)
else:
if logfile is None:
# mkstemp returns (fd,filename), fd is from os.open, not regular open!
fd, logfile = tempfile.mkstemp(suffix='.log', prefix='easybuild-')
os.close(fd)
fancylogger.logToFile(logfile)
print_msg('temporary log file in case of crash %s' % (logfile), log=None, silent=testing)
global _log
_log = fancylogger.getLogger(fname=False)
# hello world!
_log.info(this_is_easybuild())
# set strictness of filetools module
if options.strict:
filetools.strictness = options.strict
if not options.robot is None:
if options.robot:
_log.info("Using robot path: %s" % options.robot)
else:
_log.error("No robot path specified, and unable to determine easybuild-easyconfigs install path.")
# determine easybuild-easyconfigs package install path
easyconfigs_paths = get_paths_for("easyconfigs", robot_path=options.robot)
easyconfigs_pkg_full_path = None
search_path = os.getcwd()
if easyconfigs_paths:
easyconfigs_pkg_full_path = easyconfigs_paths[0]
if not options.robot:
search_path = easyconfigs_pkg_full_path
else:
search_path = options.robot
else:
_log.info("Failed to determine install path for easybuild-easyconfigs package.")
if options.robot:
easyconfigs_paths = [options.robot] + easyconfigs_paths
# initialise the easybuild configuration
config.init(options, eb_go.get_options_by_section('config'))
# search for easyconfigs
if options.search:
search_file(search_path, options.search, silent=testing)
# process software build specifications (if any), i.e.
# software name/version, toolchain name/version, extra patches, ...
(try_to_generate, software_build_specs) = process_software_build_specs(options)
paths = []
if len(orig_paths) == 0:
if software_build_specs.has_key('name'):
paths = [obtain_path(software_build_specs, easyconfigs_paths,
try_to_generate=try_to_generate, exit_on_error=not testing)]
elif not any([options.aggregate_regtest, options.search, options.regtest]):
print_error(("Please provide one or multiple easyconfig files, or use software build "
"options to make EasyBuild search for easyconfigs"),
log=_log, opt_parser=eb_go.parser, exit_on_error=not testing)
else:
# look for easyconfigs with relative paths in easybuild-easyconfigs package,
# unless they we found at the given relative paths
if easyconfigs_pkg_full_path:
# create a mapping from filename to path in easybuild-easyconfigs package install path
easyconfigs_map = {}
for (subpath, _, filenames) in os.walk(easyconfigs_pkg_full_path):
for filename in filenames:
easyconfigs_map.update({filename: os.path.join(subpath, filename)})
# try and find non-existing non-absolute eaysconfig paths in easybuild-easyconfigs package install path
for idx, orig_path in enumerate(orig_paths):
if not os.path.isabs(orig_path) and not os.path.exists(orig_path):
if orig_path in easyconfigs_map:
_log.info("Found %s in %s: %s" % (orig_path, easyconfigs_pkg_full_path,
easyconfigs_map[orig_path]))
orig_paths[idx] = easyconfigs_map[orig_path]
# indicate that specified paths do not contain generated easyconfig files
paths = [(path, False) for path in orig_paths]
_log.debug("Paths: %s" % paths)
# run regtest
if options.regtest or options.aggregate_regtest:
_log.info("Running regression test")
if paths:
regtest_ok = regtest(options, [path[0] for path in paths])
else: # fallback: easybuild-easyconfigs install path
regtest_ok = regtest(options, [easyconfigs_pkg_full_path])
if not regtest_ok:
_log.info("Regression test failed (partially)!")
sys.exit(31) # exit -> 3x1t -> 31
if any([options.search, options.regtest]):
cleanup_logfile_and_exit(logfile, testing, True)
# building a dependency graph implies force, so that all dependencies are retained
# and also skips validation of easyconfigs (e.g. checking os dependencies)
validate_easyconfigs = True
retain_all_deps = False
if options.dep_graph:
_log.info("Enabling force to generate dependency graph.")
options.force = True
validate_easyconfigs = False
retain_all_deps = True
# read easyconfig files
easyconfigs = []
for (path, generated) in paths:
path = os.path.abspath(path)
if not (os.path.exists(path)):
print_error("Can't find path %s" % path)
try:
files = find_easyconfigs(path)
for f in files:
if not generated and try_to_generate and software_build_specs:
ec_file = easyconfig.tools.tweak(f, None, software_build_specs)
else:
ec_file = f
easyconfigs.extend(process_easyconfig(ec_file, options.only_blocks,
validate=validate_easyconfigs))
except IOError, err:
_log.error("Processing easyconfigs in path %s failed: %s" % (path, err))
