def is_easybuild_app():
""" returns True if an easyconfig file found in installation directory """
app_name = get_appname()
app_ver = get_appversion()
modulefiles = []
modtrees = get_module_ebroot()
for tree in modtrees:
for root, dirs, files in os.walk(tree):
for file in files:
# skipping files that are symbolic links
if os.path.islink(os.path.join(root, file)):
continue
# only add module file to list specified by -s <app>/<version>. The file name will be the version and directory will be the application name
if os.path.splitext(file)[0] == app_ver and os.path.basename(
root) == app_name:
modulefiles.append(os.path.join(root, file))
ec_list, no_ec_list = find_easyconfigs_from_modulelist(modulefiles)
# if no easyconfigs found then ec_list will be empty so we should stop and report this application is not built by easybuild. This feature can be changed in future
if len(ec_list) == 0:
print "Application:", os.path.join(
app_name, app_ver
), " is not built from Easybuild, cannot find easyconfig file in installation directory"
sys.exit(1)
return
def verify_Ruby_gem(Ruby_gem):
""" check if ruby gem exist for request ruby module, if it exists create test otherwise skip test creation"""
logger = logging.getLogger(logID)
appname = get_appname()
appver = get_appversion()
BUILDTEST_MODULE_NAMING_SCHEME = config_opts[
'BUILDTEST_MODULE_NAMING_SCHEME']
cmd = "module purge; module load " + os.path.join(
appname, appver) + "; gem list -i " + Ruby_gem
if BUILDTEST_MODULE_NAMING_SCHEME == "HMNS":
tcname = get_toolchain_name()
tcver = get_toolchain_version()
if len(tcname) > 0:
cmd = "module purge; module load " + os.path.join(
tcname, tcver) + "; module load " + os.path.join(
appname, appver) + "; gem list - i " + Ruby_gem
logger.debug("Check Ruby gem:" + R_lib)
logger.debug("Running command - " + cmd)
ret = subprocess.Popen(cmd,
shell=True,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
ret.communicate()
return ret.returncode
def verify_perl_module(perl_module):
""" check if perl module exist for request perl module, if it exists create test otherwise skip test creation"""
logger = logging.getLogger(logID)
appname = get_appname()
appver = get_appversion()
BUILDTEST_MODULE_NAMING_SCHEME = config_opts[
'BUILDTEST_MODULE_NAMING_SCHEME']
cmd = "module purge; module load " + os.path.join(
appname, appver) + "; perl -e \' use " + perl_module + ";\'"
if BUILDTEST_MODULE_NAMING_SCHEME == "HMNS":
tcname = get_toolchain_name()
tcver = get_toolchain_version()
if len(tcname) > 0:
cmd = "module purge; module load " + os.path.join(
tcname, tcver) + "; module load " + os.path.join(
appname,
appver) + "; perl -e \' use " + perl_module + ";\'"
logger.debug("Checking Perl Module " + perl_module)
logger.debug("Running command - " + cmd)
ret = subprocess.Popen(cmd,
shell=True,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
ret.communicate()
return ret.returncode
def verify_R_library(R_lib):
""" check if R library exist for request R module, if it exists create test otherwise skip test creation"""
logger = logging.getLogger(logID)
appname = get_appname()
appver = get_appversion()
BUILDTEST_MODULE_NAMING_SCHEME = config_opts[
'BUILDTEST_MODULE_NAMING_SCHEME']
cmd = ""
cmd = "module purge; module load " + os.path.join(
appname,
appver) + "; echo \"library(" + R_lib + ")\" | R -q --no-save "
if BUILDTEST_MODULE_NAMING_SCHEME == "HMNS":
tcname = get_toolchain_name()
tcver = get_toolchain_version()
if len(tcname) > 0:
cmd = "module purge; module load " + os.path.join(
tcname, tcver) + "; module load " + os.path.join(
appname, appver
) + "; echo \"library(" + R_lib + ")\" | R -q --no-save "
logger.debug("Check R Package:" + R_lib)
logger.debug("Running command - " + cmd)
ret = subprocess.Popen(cmd,
shell=True,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
ret.communicate()
return ret.returncode
def recursive_gen_test(configdir, codedir):
""" if config directory exists then process all .yaml files to build source test """
# only process yaml files if configdir directory is found
if os.path.isdir(configdir):
logger = logging.getLogger(logID)
logger.debug("Processing all YAML files in %s", configdir)
print "--------------------------------------------"
print "[STAGE 2]: Building Source Tests"
print "--------------------------------------------"
print "Processing all YAML files in directory:", configdir
count = 0
for root, subdirs, files in os.walk(configdir):
# skip to next element if subdirectory has no files
if len(files) == 0:
continue
#filepath=configdir+filename
for file in files:
filepath = os.path.join(root, file)
subdir = os.path.basename(root)
# if there is no subdirectory in configdir that means subdir would be set to "config" so it can
# be set to empty string in order to concat codedir and subdir. This way both subdirectory and
# and no subdirectory structure for yaml will work
if subdir == "config":
subdir = ""
code_destdir = os.path.join(codedir, subdir)
logger.debug("Parsing YAML file: %s", filepath)
configmap = parse_config(filepath, code_destdir)
# error processing config file, then parse_config will return an empty dictionary
if len(configmap) == 0:
continue
count = count + 1
generate_source_test(configmap, code_destdir, subdir)
appname = get_appname()
appver = get_appversion()
tcname = get_toolchain_name()
tcver = get_toolchain_version()
BUILDTEST_TESTDIR = config_opts['BUILDTEST_TESTDIR']
destdir = os.path.join(BUILDTEST_TESTDIR, "ebapp", appname, appver,
tcname, tcver)
print "Generating " + str(
count) + " Source Tests and writing at ", destdir
else:
return
def run_testset(arg_dict):
""" checks the testset parameter to determine which set of scripts to use to create tests """
print "--------------------------------------------"
print "[STAGE 3]: Building Testset"
print "--------------------------------------------"
BUILDTEST_CONFIGS_REPO = config_opts['BUILDTEST_CONFIGS_REPO']
BUILDTEST_PYTHON_REPO = config_opts['BUILDTEST_PYTHON_REPO']
BUILDTEST_TCL_REPO = config_opts['BUILDTEST_TCL_REPO']
BUILDTEST_R_REPO = config_opts['BUILDTEST_R_REPO']
BUILDTEST_PERL_REPO = config_opts['BUILDTEST_PERL_REPO']
BUILDTEST_RUBY_REPO = config_opts['BUILDTEST_RUBY_REPO']
appname = get_appname()
source_app_dir = ""
codedir = ""
logcontent = ""
runtest = False
if appname.lower() in PYTHON_APPS and arg_dict.testset == "Python":
source_app_dir = os.path.join(BUILDTEST_PYTHON_REPO, "python")
runtest = True
if appname.lower() in ["perl"] and arg_dict.testset == "Perl":
source_app_dir = os.path.join(BUILDTEST_PERL_REPO, "perl")
runtest = True
# condition to run R testset
if appname.lower() in ["r"] and arg_dict.testset == "R":
source_app_dir = os.path.join(BUILDTEST_R_REPO, "R")
runtest = True
# condition to run R testset
if appname.lower() in ["ruby"] and arg_dict.testset == "Ruby":
source_app_dir = os.path.join(BUILDTEST_RUBY_REPO, "ruby")
runtest = True
# condition to run R testset
if appname.lower() in ["tcl"] and arg_dict.testset == "Tcl":
source_app_dir = os.path.join(BUILDTEST_TCL_REPO, "Tcl")
runtest = True
# for MPI we run recursive_gen_test since it processes YAML files
if appname in MPI_APPS and arg_dict.testset == "MPI":
source_app_dir = os.path.join(BUILDTEST_CONFIGS_REPO, "mpi")
configdir = os.path.join(source_app_dir, "config")
codedir = os.path.join(source_app_dir, "code")
recursive_gen_test(configdir, codedir)
return
if runtest == True:
codedir = os.path.join(source_app_dir, "code")
testset_generator(arg_dict, codedir)
def add_test_to_CMakeLists(app_destdir,subdir,cmakelist,testname):
""" update CMakeLists.txt with add_test command to allow ctest to run test """
fd=open(cmakelist,'a')
add_test_str=""
logger = logging.getLogger(logID)
BUILDTEST_TESTDIR = config_opts['BUILDTEST_TESTDIR']
args_dict = buildtest_menu().parse_options()
shell_type = config_opts['BUILDTEST_SHELL']
appname = get_appname()
appversion = get_appversion()
tcname = get_toolchain_name()
tcversion = get_toolchain_version()
# if YAML files are in subdirectory of config directory then update CMakeList
# in app_destdir to add tag "add_subdirectory" for subdirectory so CMakeList
# can find tests in subdirectory
if subdir != "":
# only update the app_destdir/CMakeLists.txt if subdirectory doesn't exist. This avoids
# writing duplicate values when there are multiple tests in subdirectory
parent_cmakelist = os.path.join(app_destdir,"CMakeLists.txt")
cmake_content="add_subdirectory("+subdir+") \n"
ret = string_in_file(cmake_content,parent_cmakelist)
if ret == False:
fd1=open(parent_cmakelist,'a')
fd1.write(cmake_content)
fd1.close()
# the string add_test in CMakeLists allows you to test script with ctest. The NAME tag is
# <name>-<version>-<toolchain-name>-<toolchain-version>-<subdir>-<testname>. This
# naming scheme should allow buildtest to reuse same YAML configs for multiple version
# built with any toolchains. Subdirectories come in handy when you need to organize tests
# effectively to avoid naming conflict
# condition to check of toolchain exists, if so then add it to add_test()
if tcname == "":
add_test_str="add_test(NAME " + appname + "-" + appversion + "-" + subdir + "-" + testname + "\t COMMAND "+ shell_type + " " + testname + "\t WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) \n"
else:
add_test_str="add_test(NAME " + appname + "-" + appversion + "-" + tcname + "-" + tcversion + "-" + subdir + "-" + testname + "\t COMMAND " + shell_type + " " + testname + "\t WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) \n"
else:
if tcname == "":
add_test_str="add_test(NAME " + appname + "-" + appversion + "-" + testname + "\t COMMAND " + shell_type + " " + testname + "\t WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) \n"
else:
add_test_str="add_test(NAME " + appname + "-" + appversion + "-" + tcname + "-" + tcversion + "-" + testname + "\t COMMAND " + shell_type + " " + testname + "\t WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) \n"
fd.write(add_test_str)
fd.close()
logger.debug("Updating File " + cmakelist + " with: " + add_test_str + "\n")
def setup_software_cmake():
logger = logging.getLogger(logID)
BUILDTEST_TESTDIR = config_opts['BUILDTEST_TESTDIR']
name = get_appname()
version = get_appversion()
toolchain_name = get_toolchain_name()
toolchain_version = get_toolchain_version()
# if top level software directory is not present, create it
test_ebapp_dir=os.path.join(BUILDTEST_TESTDIR,"ebapp")
# variables to reference each subdirectory in <software>/<version>/<toolchain-name>/<toolchain-version>
test_name_dir=os.path.join(test_ebapp_dir,name)
test_version_dir=os.path.join(test_name_dir,version)
test_toolchain_name_dir=os.path.join(test_version_dir,toolchain_name)
test_toolchain_version_dir=os.path.join(test_toolchain_name_dir,toolchain_version)
# BUILDTEST_TESTDIR/CMakeLists.txt
test_cmakelist = os.path.join(BUILDTEST_TESTDIR,"CMakeLists.txt")
# BUILDTEST_TESTDIR/ebapps/CMakeLists.txt
test_ebapp_cmakelist = os.path.join(test_ebapp_dir,"CMakeLists.txt")
# CMakeLists.txt files in <software>/<version>/<toolchain-name>/<toolchain-version>
test_name_cmakelist = os.path.join(test_name_dir,"CMakeLists.txt")
test_version_cmakelist = os.path.join(test_version_dir,"CMakeLists.txt")
test_toolchain_name_cmakelist = os.path.join(test_toolchain_name_dir,"CMakeLists.txt")
test_toolchain_version_cmakelist = os.path.join(test_toolchain_version_dir,"CMakeLists.txt")
test_destdir=test_toolchain_version_dir
logger.debug("----------------------------------------")
logger.debug("EB Directory Assignments")
logger.debug("----------------------------------------")
logger.debug("EB Directory in $BUILDTEST_TESTDIR: %s ", test_ebapp_dir)
logger.debug("EB Application Directory: %s", test_name_dir)
logger.debug("EB Application Version Directory: %s", test_version_dir)
logger.debug("EB Toolchain Name Directory: %s", test_toolchain_name_dir)
logger.debug("EB Toolchain Version Directory: %s", test_toolchain_version_dir)
logger.debug("\n")
logger.debug("Test Directory for EB Application: %s ", test_destdir)
logger.debug("\n\n")
logger.debug("----------------------------------------")
logger.debug("EB CMakeList Assignments")
logger.debug("----------------------------------------")
logger.debug("$BUILDTEST_TESTDIR CMakeList Path: %s ", test_cmakelist)
logger.debug("EB Application CMakeList Path: %s ", test_name_cmakelist)
logger.debug("EB Application Version CMakeList Path: %s", test_version_cmakelist)
logger.debug("EB Toolchain Name CMakeList Path: %s", test_toolchain_name_cmakelist)
logger.debug("EB Toolchain Version CMakeList Path: %s", test_toolchain_version_cmakelist)
# if test directory exist, delete and recreate it inorder for reproducible test builds
if os.path.isdir(test_destdir):
shutil.rmtree(test_destdir)
logger.debug("Removing test directory before creating test: %s", test_destdir)
# create directories if they don't exist
# Directory Format: <software>/<version>/toolchain-name>/<toolchain-version>
create_dir(test_ebapp_dir)
create_dir(test_name_dir)
create_dir(test_version_dir)
if len(toolchain_name) != 0:
create_dir(test_toolchain_name_dir)
create_dir(test_toolchain_version_dir)
# create CMakeList.txt file in each directory of <software>/<version>/<toolchain-name>/<toolchain-version> if it doesn't exist
create_file(test_ebapp_cmakelist)
create_file(test_name_cmakelist)
create_file(test_version_cmakelist)
if len(toolchain_name) != 0:
create_file(test_toolchain_name_cmakelist)
create_file(test_toolchain_version_cmakelist)
# update CMakeLists.txt with tags add_subdirectory(ebapp)
update_CMakeLists(test_cmakelist,"ebapp")
# update CMakeLists.txt with tags add_subdirectory(X) where X=name|version|toolchain-name|toolchain-version
update_CMakeLists(test_ebapp_cmakelist,name)
update_CMakeLists(test_name_cmakelist,version)
if len(toolchain_name) != 0:
update_CMakeLists(test_version_cmakelist,toolchain_name)
update_CMakeLists(test_toolchain_name_cmakelist,toolchain_version)
return test_destdir,test_toolchain_version_cmakelist
def main():
""" entry point to buildtest """
IGNORE_EASYBUILD = False
override_configuration()
check_configuration()
BUILDTEST_CONFIGS_REPO = config_opts['BUILDTEST_CONFIGS_REPO']
print BUILDTEST_CONFIGS_REPO
parser = buildtest_menu()
bt_opts = parser.parse_options()
if config_opts.get('BUILDTEST_IGNORE_EASYBUILD'):
IGNORE_EASYBUILD = config_opts['BUILDTEST_IGNORE_EASYBUILD']
if bt_opts.version:
buildtest_version()
sys.exit(0)
if bt_opts.show:
show_configuration()
if bt_opts.logdir:
config_opts['BUILDTEST_LOGDIR'] = bt_opts.logdir
if bt_opts.testdir:
config_opts['BUILDTEST_TESTDIR'] = bt_opts.testdir
if bt_opts.ignore_easybuild:
IGNORE_EASYBUILD = True
if bt_opts.enable_job:
config_opts['BUILDTEST_ENABLE_JOB'] = True
if bt_opts.clean_logs:
clean_logs()
if bt_opts.clean_tests:
clean_tests()
if bt_opts.module_naming_scheme:
config_opts[
'BUILDTEST_MODULE_NAMING_SCHEME'] = bt_opts.module_naming_scheme
if bt_opts.runtest:
runtest_menu()
if bt_opts.scantest:
scantest()
if bt_opts.list_toolchain is True:
toolchain_set = list_toolchain()
print_toolchain(toolchain_set)
sys.exit(0)
if bt_opts.list_unique_software:
software_set = get_unique_software()
print_software(software_set)
sys.exit(0)
if bt_opts.software_version_relation:
software_dict = software_version_relation()
print_software_version_relation(software_dict)
sys.exit(0)
if bt_opts.easyconfigs_in_moduletrees:
find_easyconfigs()
sys.exit(0)
if bt_opts.diff_trees:
args_trees = bt_opts.diff_trees
diff_trees(args_trees)
sys.exit(0)
# when no argument is specified to -fc then output all yaml files
if bt_opts.findconfig == "all":
find_all_yaml_configs()
sys.exit(0)
# find yaml configs by argument instead of reporting all yaml files
elif bt_opts.findconfig is not None:
find_yaml_configs_by_arg(bt_opts.findconfig)
sys.exit(0)
# report all buildtest generated test scripts
if bt_opts.findtest == "all":
find_all_tests()
sys.exit(0)
# find test by argument instead of all tests
elif bt_opts.findtest is not None:
find_tests_by_arg(bt_opts.findtest)
sys.exit(0)
if bt_opts.shell:
config_opts['BUILDTEST_SHELL'] = bt_opts.shell
if bt_opts.job_template is not None:
update_job_template(bt_opts.job_template)
if bt_opts.submitjob is not None:
submit_job_to_scheduler(bt_opts.submitjob)
sys.exit(0)
if bt_opts.sysyaml is not None:
create_system_yaml(bt_opts.sysyaml)
if bt_opts.ebyaml != None:
raise NotImplementedError
logger, logpath, logfile = init_log()
BUILDTEST_LOGDIR = config_opts['BUILDTEST_LOGDIR']
BUILDTEST_TESTDIR = config_opts['BUILDTEST_TESTDIR']
create_dir(BUILDTEST_LOGDIR)
create_dir(BUILDTEST_TESTDIR)
cmd = "env | grep BUILDTEST"
ret = subprocess.Popen(cmd,
shell=True,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
output = ret.communicate()[0]
output = output.split("\n")
for line in output:
logger.debug(line)
# generate system pkg test
if bt_opts.system is not None:
if bt_opts.system == "all":
systempkg = bt_opts.system
logger.info(
"Generating all system package tests from YAML files in %s",
os.path.join(BUILDTEST_CONFIGS_REPO, "system"))
BUILDTEST_LOGDIR = os.path.join(BUILDTEST_LOGDIR, "system", "all")
systempkg_list = os.listdir(
os.path.join(BUILDTEST_CONFIGS_REPO, "system"))
logger.info("List of system packages to test: %s ", systempkg_list)
for pkg in systempkg_list:
generate_binary_test(bt_opts, pkg)
else:
systempkg = bt_opts.system
BUILDTEST_LOGDIR = os.path.join(BUILDTEST_LOGDIR, "system",
systempkg)
generate_binary_test(bt_opts, systempkg)
create_dir(BUILDTEST_LOGDIR)
logger.warning("Creating directory %s, to write log file",
BUILDTEST_LOGDIR)
destpath = os.path.join(BUILDTEST_LOGDIR, logfile)
os.rename(logpath, destpath)
logger.info("Moving log file from %s to %s", logpath, destpath)
print "Writing Log file to:", destpath
sys.exit(0)
# when -s is specified
if bt_opts.software is not None:
software = bt_opts.software.split("/")
if bt_opts.toolchain is None:
toolchain = "dummy/dummy".split("/")
else:
toolchain = bt_opts.toolchain.split("/")
appname = get_appname()
appversion = get_appversion()
tcname = get_toolchain_name()
tcversion = get_toolchain_version()
print "Detecting Software: ", os.path.join(appname, appversion)
logger.debug("Generating Test from EB Application")
logger.debug("Software: %s", appname)
logger.debug("Software Version: %s", appversion)
logger.debug("Toolchain: %s", tcname)
logger.debug("Toolchain Version: %s", tcversion)
logger.debug("Checking if software: %s/%s exists", appname, appversion)
# check if software is an easybuild applicationa
if IGNORE_EASYBUILD == False:
is_easybuild_app()
source_app_dir = os.path.join(BUILDTEST_CONFIGS_REPO, "ebapps",
appname.lower())
configdir = os.path.join(source_app_dir, "config")
codedir = os.path.join(source_app_dir, "code")
BUILDTEST_LOGDIR = os.path.join(BUILDTEST_LOGDIR, appname, appversion,
tcname, tcversion)
# if directory tree for software log is not present, create the directory
create_dir(BUILDTEST_LOGDIR)
logger.debug("Source App Directory: %s", source_app_dir)
logger.debug("Config Directory: %s ", configdir)
logger.debug("Code Directory: %s", codedir)
generate_binary_test(bt_opts, None)
# this generates all the compilation tests found in application directory ($BUILDTEST_CONFIGS_REPO/ebapps/<software>)
recursive_gen_test(configdir, codedir)
# if flag --testset is set, then
if bt_opts.testset is not None:
run_testset(bt_opts)
# moving log file from $BUILDTEST_LOGDIR/buildtest_%H_%M_%d_%m_%Y.log to $BUILDTEST_LOGDIR/app/appver/tcname/tcver/buildtest_%H_%M_%d_%m_%Y.log
os.rename(logpath, os.path.join(BUILDTEST_LOGDIR, logfile))
logger.debug("Writing Log file to %s",
os.path.join(BUILDTEST_LOGDIR, logfile))
print "Writing Log file: ", os.path.join(BUILDTEST_LOGDIR, logfile)
def generate_source_test(configmap, codedir, subdir):
"""
This function generates the tests that requires compilation for EB apps. The
tests are written <software>/<version>/<toolchain-name>/<toolchain-version>.
The test script is named according to "name" key tag with the shell extension
CMakeLists.txt has an entry for each test that executes the shell-script. Most
test requires a compilation step, while every test requires a execution stage.
This is done via buildcmd and runcmd tags in YAML for explicit builds. buildtest
will try to generate this automatically if nothing is provided.
"""
appname = get_appname()
appver = get_appversion()
tcname = get_toolchain_name()
tcver = get_toolchain_version()
logger = logging.getLogger(logID)
BUILDTEST_TESTDIR = config_opts['BUILDTEST_TESTDIR']
testname = ""
testpath = ""
sourcefilepath = ""
executable = ""
buildopts = ""
envvars = ""
env_msg = []
# app_destdir is root of test directory
app_destdir = os.path.join(BUILDTEST_TESTDIR, "ebapp", appname, appver,
tcname, tcver)
# destdir is where test script and CMakeLists.txt will be generated.
# If there is a subdirectory then testscript will reside in subdir
destdir = os.path.join(app_destdir, subdir)
cmakelist = os.path.join(destdir, "CMakeLists.txt")
# if subdirectory exists, create subdirectory in destdir so we can write test script
#if subdir != "":
# if sub directory does not exist, then create all directories and its parents directories
create_dir(destdir)
shell_type = config_opts['BUILDTEST_SHELL']
BUILDTEST_JOB_TEMPLATE = config_opts['BUILDTEST_JOB_TEMPLATE']
BUILDTEST_ENABLE_JOB = config_opts['BUILDTEST_ENABLE_JOB']
# testname is key value "name" with .sh extension
testname = configmap["name"] + "." + shell_type
testpath = os.path.join(destdir, testname)
sourcefilepath = os.path.join(codedir, configmap["source"])
# name of the executable is the value of source tag with ".exe" extension
executable = configmap["source"] + ".exe"
flags = ""
# if buildopts key exists in dictionary, then add flags to compilation step (buildcmd)
if "buildopts" in configmap:
flags = configmap["buildopts"]
if "envvars" in configmap:
envvars = configmap["envvars"]
for key in envvars.keys():
# append value from function openmp_env_string into env_msg that sets OpenMP environment variable according to shell
env_msg.append(openmp_env_string(shell_type, envvars[key]))
if "procrange" in configmap:
procrange = configmap["procrange"]
startproc = int(procrange.split(",")[0])
endproc = int(procrange.split(",")[1])
procinterval = int(procrange.split(",")[2])
testname = configmap["name"] + "_nproc_" + str(
startproc) + "." + shell_type
testpath = os.path.join(destdir, testname)
mpi_proc_list = range(startproc, endproc + 1, procinterval)
if "threadrange" in configmap:
threadrange = configmap["threadrange"]
startthread = int(threadrange.split(",")[0])
endthread = int(threadrange.split(",")[1])
threadinterval = int(threadrange.split(",")[2])
testname = configmap["name"] + "_nthread_" + str(
startthread) + "." + shell_type
testpath = os.path.join(destdir, testname)
thread_list = range(startthread, endthread + 1, threadinterval)
openmp_env_str = openmp_env_string(shell_type, startthread)
logger.debug("Test Name: %s", testname)
logger.debug("Test Path: %s", testpath)
logger.debug("Source File: %s", sourcefilepath)
logger.debug("Executable Name: %s", executable)
logger.debug("Build Flags: %s", flags)
# write the preamble to test-script to initialize app environment using module cmds
fd = open(testpath, 'w')
header = load_modules(shell_type)
fd.write(header)
# string used for generating the compilation step
buildcmd = ""
# string used for running the executable and used for injecting commands at pre/post test
runcmd = ""
# string to decide the compiler wrapper to use based on application and file extension
compiler = ""
# used for parallel processing to specify # of procs with mpirun -np
nproc = ""
logger.debug("""Checking for YAML key "buildcmd" and "runcmd" """)
# if there is a buildcmd & runcmd in yaml file, place this directly in test script
if "buildcmd" in configmap and "runcmd" in configmap:
logger.debug("YAML file found buildcmd and runcmd.")
logger.debug(
"buildtest will generate explicit build/run commands from buildcmd and runcmd field"
)
# only process buildcmd if there is a value specified for buildcmd key
if configmap["buildcmd"] != None:
# for each element from dictionary configmap["buildcmd"], write
# each instruction in buildcmd separated by newline
for cmd in configmap["buildcmd"]:
buildcmd += cmd + "\n"
else:
msg = "buildcmd is declared but value is not specified \n"
logger.debug("%s", msg)
if configmap["runcmd"] != None:
# process the runcmd tag similar same as buildcmd and store in variable
# runcmd except if no value is specified for runcmd in YAML then throw
# an error
for cmd in configmap["runcmd"]:
runcmd += cmd + "\n"
else:
msg = "runcmd is declared but value is not specified. Need runcmd to run executable \n"
print msg
logger.debug.append("%s", msg)
logging.warning(
"Unable to create test from YAML config, skipping test generation"
)
return
fd.write(buildcmd)
fd.write(runcmd)
# otherwise generate the buildcmd and runcmd automatically
else:
# checking if either buildcmd or runcmd specified but not both, then report an error.
if "buildcmd" in configmap and "runcmd" not in configmap or "buildcmd" not in configmap and "runcmd" in configmap:
print "Need to specify both key: buildcmd and runcmd"
logger.warning(
"Need to declare both key: buildcmd and runcmd. Skipping to next YAML config \n"
)
return
# get the compiler tag and type based on application and toolchain
compiler, compiler_type = get_compiler(configmap, appname, tcname)
logger.debug("buildtest will auto-generate buildcmd & runcmd")
logger.debug("Compiler: %s", compiler)
logger.debug("Compiler Type: %s", compiler_type)
# set buildcmd based on compiler_type. compiler is either nvcc,gcc,icc,mpicc, or mpiicc for intel
if compiler_type == "gnu" or compiler_type == "intel" or compiler_type == "cuda":
buildcmd = compiler + " -o " + executable + " " + sourcefilepath + " " + flags + "\n"
#print buildcmd
# set runcmd for mpi tags using mpirun otherwise just run executable
if compiler in [
"mpicc", "mpic++", "mpifort", "mpiicc", "mpiic++",
"mpiifort"
]:
# if nproc is defined in yaml, store value in nproc which will use it in mpirun command
if "nproc" in configmap:
nproc = str(configmap["nproc"])
logger.debug("nproc key found in YAML config file")
logger.debug("nproc: %s", nproc)
# check for procrange in case nproc is not defined
elif "procrange" in configmap:
nproc = str(startproc)
# if nproc is not specified set it to 1 when building mpi apps
else:
nproc = "1"
logger.debug(
"nproc key not found in YAML config file, will set nproc = 1"
)
# add argument to runcmd in MPI jobs
if "args" in configmap:
arglist = configmap["args"]
runcmd = "mpirun -np " + nproc + " ./" + executable
runcmd = add_arg_to_runcmd(runcmd, arglist)
else:
runcmd = "mpirun -np " + nproc + " ./" + executable
else:
# add argument to runcmd in general jobs
if "args" in configmap:
arglist = configmap["args"]
runcmd = "./" + executable
runcmd = add_arg_to_runcmd(runcmd, arglist)
else:
runcmd = "./" + executable
# Scripting languages like Python, R, Perl no compilation stage, just run script. So we just need to update runcmd string
elif compiler_type == "python" or compiler_type == "perl" or compiler_type == "R":
if "args" in configmap:
arglist = configmap["args"]
runcmd = compiler + " " + sourcefilepath
runcmd = add_arg_to_runcmd(runcmd, arglist)
else:
runcmd = compiler + " " + sourcefilepath
# java programs need "javac" to compile and "java" to run program. This works best if you are
# in current directory where source file exists.
elif compiler_type == "java":
buildcmd = compiler + " " + sourcefilepath + "\n"
java_codedir = os.path.dirname(sourcefilepath)
filename = os.path.basename(os.path.splitext(sourcefilepath)[0])
# need to be directory where java files reside
runcmd = "cd " + java_codedir + "\n"
runcmd += "java " + filename + "\n"
# would like to remove .class files that are generated due to javac
runcmd += "rm -v " + filename + ".class"
# if inputfile key is found, add this as part of runcmd
if "inputfile" in configmap:
runcmd += " < " + os.path.join(codedir, configmap["inputfile"])
# if output of program needs to be written to file instead of STDOUT
if "outputfile" in configmap:
runcmd += " > " + configmap["outputfile"]
if "envvars" in configmap:
for key in env_msg:
fd.write(key)
if "threadrange" in configmap:
fd.write(openmp_env_str)
fd.write(buildcmd)
fd.write(runcmd)
# runextracmd is used when you want buildtest to generate compilation and execution step
# automatically but need to add extra commands after execution.
if "runextracmd" in configmap:
for cmd in configmap["runextracmd"]:
fd.write(cmd + "\n")
logger.debug("runextracmd found in YAML config file")
logger.debug("runextracmd: %s", str(configmap["runextracmd"]))
fd.close()
if BUILDTEST_ENABLE_JOB:
if "scheduler" in configmap:
generate_job_by_config(testpath, shell_type, configmap)
else:
# generate job script based on template, if "scheduler" found in
# then module below will do nothing and taken care off by
# generate_job_by_config
generate_job(testpath, shell_type, BUILDTEST_JOB_TEMPLATE,
configmap)
# by default run the commands below which will add the test to CMakeLists.txt and update the logfile
if "iter" not in configmap:
add_test_to_CMakeLists(app_destdir, subdir, cmakelist, testname)
# print "Creating Test: " + testpath
logger.debug("Creating Test: %s ", testpath)
logger.debug("[TEST START-BLOCK]")
fd = open(testpath, 'r')
content = fd.read().splitlines()
for line in content:
logger.debug(line)
fd.close()
logger.debug("[TEST END-BLOCK]")
if "procrange" in configmap:
create_procrange_test(testpath, startproc, mpi_proc_list, subdir)
if "threadrange" in configmap:
create_threadrange_test(testpath, startthread, thread_list, subdir)
# if keyword iter is found in YAML, lets try to recreate N tests by renaming test such as
# hello.sh to hello_1.sh and create N-1 copies with file names hello_2.sh, hello_3.sh, ...
if "iter" in configmap:
filename = os.path.basename(os.path.splitext(sourcefilepath)[0])
testname = filename + "_1." + shell_type
testpath_testname = os.path.join(destdir, testname).replace("\n", '')
os.rename(testpath, testpath_testname)
out = "Rename Iteration Test: " + testpath + " -> " + testpath_testname
logger.debug("%s", out)
# writing test to CMakeLists.txt
add_test_to_CMakeLists(app_destdir, subdir, cmakelist, testname)
logger.debug("Content of Test file: %s", testpath_testname)
logger.debug("[START TEST-BLOCK]")
fd = open(testpath_testname, 'r')
content = fd.read()
logger.debug("%s", content)
fd.close()
logger.debug("[END TEST-BLOCK]")
numiter = int(configmap["iter"])
logger.debug("Making %s copy of test: %s", numiter, testpath_testname)
# creating N-1 copies of tests
for index in range(1, numiter):
testname = filename + "_" + str(index + 1) + "." + shell_type
src_testpath = testpath_testname
dest_testpathname = os.path.join(destdir,
testname).replace('\n', '')
copyfile(src_testpath, dest_testpathname)
out = "Iteration Test: " + dest_testpathname
logger.info("%s", out)
logger.debug("Adding test: %s to CMakeList", testname)
add_test_to_CMakeLists(app_destdir, subdir, cmakelist, testname)
def testset_generator(arg_dict, codedir):
logger = logging.getLogger(logID)
BUILDTEST_TESTDIR = config_opts['BUILDTEST_TESTDIR']
wrapper = ""
appname = get_appname()
appver = get_appversion()
tcname = get_toolchain_name()
tcver = get_toolchain_version()
BUILDTEST_SHELL = config_opts['BUILDTEST_SHELL']
BUILDTEST_ENABLE_JOB = config_opts['BUILDTEST_ENABLE_JOB']
BUILDTEST_JOB_TEMPLATE = config_opts['BUILDTEST_JOB_TEMPLATE']
app_destdir = os.path.join(BUILDTEST_TESTDIR, "ebapp", appname, appver,
tcname, tcver)
cmakelist = os.path.join(app_destdir, "CMakeLists.txt")
# setup CMakeList in all subdirectories for the app if CMakeList.txt was not generated from
# binary test
if not os.path.exists(cmakelist):
setup_software_cmake()
emptylist = []
testset_name = os.path.basename(os.path.dirname(codedir))
print codedir, testset_name
if os.path.isdir(codedir):
totalcount = 0
for root, subdirs, files in os.walk(codedir):
package_name = os.path.basename(root)
if testset_name == "python":
ret = verify_python_library(package_name)
# if import package fails then skip test generation
if ret > 0:
continue
#print testset_name,package_name
if testset_name == "R":
ret = verify_R_library(package_name)
# if import package fails then skip test generation
if ret > 0:
continue
#print testset_name,package_name
if testset_name == "Ruby":
ret = verify_Ruby_gem(package_name)
# if import package fails then skip test generation
if ret > 0:
continue
# skip to next item in loop when a sub-directory has no files
if len(files) == 0:
continue
count = 0
for file in files:
# get file name without extension
fname = os.path.splitext(file)[0]
# get file extension
ext = os.path.splitext(file)[1]
if testset_name == "perl":
#print files, file
#print root,files, file, fname
perl_module = os.path.basename(root) + "::" + fname
print perl_module
ret = verify_perl_module(perl_module)
# if import package fails then skip test generation
if ret > 0:
continue
if ext == ".py":
wrapper = "python"
elif ext == ".R":
wrapper = "Rscript"
elif ext == ".pl":
wrapper = "perl"
elif ext == ".rb":
wrapper = "ruby"
elif ext == ".tcl":
wrapper = "tclsh"
else:
continue
# command to execute the script
cmd = wrapper + " " + os.path.join(root, file)
# getting subdirectory path to write test to correct path
subdir = os.path.basename(root)
subdirpath = os.path.join(app_destdir, subdir)
if not os.path.exists(subdirpath):
os.makedirs(subdirpath)
testname = fname + "." + BUILDTEST_SHELL
testpath = os.path.join(subdirpath, testname)
fd = open(testpath, 'w')
header = load_modules(BUILDTEST_SHELL)
fd.write(header)
fd.write(cmd)
fd.close()
cmakelist = os.path.join(subdirpath, "CMakeLists.txt")
add_test_to_CMakeLists(app_destdir, subdir, cmakelist,
testname)
msg = "Creating Test: " + testpath
logger.info(msg)
count = count + 1
if BUILDTEST_ENABLE_JOB:
generate_job(testpath, BUILDTEST_SHELL,
BUILDTEST_JOB_TEMPLATE, emptylist)
print "Generating ", count, "tests for ", os.path.basename(root)
totalcount += count
print "Total Tests created in stage 3:", totalcount
print "Writing tests to ", app_destdir
def process_binary_file(filename,args_dict,test_type,pkg):
"""
Module responsible for actually creating the test scripts for binary tests along
with CMakeLists.txt in subdirectories under $BUILDTEST_TESTDIR. This module
is used for generating binary tests for both system and ebapps tests.
"""
logger = logging.getLogger(logID)
BUILDTEST_SHELL = config_opts['BUILDTEST_SHELL']
BUILDTEST_JOB_TEMPLATE = config_opts['BUILDTEST_JOB_TEMPLATE']
BUILDTEST_ENABLE_JOB = config_opts['BUILDTEST_ENABLE_JOB']
print "--------------------------------------------"
print "[STAGE 1]: Building Binary Tests"
print "--------------------------------------------"
if test_type == "software":
name = get_appname()
version = get_appversion()
toolchain_name = get_toolchain_name()
toolchain_version = get_toolchain_version()
test_destdir,test_destdir_cmakelist = setup_software_cmake()
print "Detecting Test Type: Software"
#print "[BINARYTEST]: Processing YAML file for ", os.path.join(name,version), os.path.join(toolchain_name,toolchain_version), " at ", filename
# load preamble for test-script that initializes environment.
header=load_modules(BUILDTEST_SHELL)
else:
system=args_dict.system
print "Detecting Test Type: System Package"
#print "[BINARYTEST]: Processing YAML file for ", pkg , " at ", filename
test_destdir,test_destdir_cmakelist = setup_system_cmake(pkg)
print "Processing Binary YAML configuration: ", filename.rstrip()
logger.info("Reading File: %s", filename)
fd=open(filename,'r')
content=yaml.load(fd)
logger.debug("Loading YAML content")
# if key binaries is not in yaml file, exit program
if "binaries" not in content:
logger.error("Can't find key: binaries in file %s", filename)
print "Can't find key: binaries in file %s", filename
sys.exit(1)
# create a binary test script for each key,value item in dictionary
binarydict=content["binaries"]
# keep track of number of binary test
count = 0
for key in binarydict:
count = count + 1
name_str=key.replace(" ","_")
# replace / with _ when creating testname for yaml configuration that have path name
name_str = name_str.replace("/","_")
testname=name_str+"."+BUILDTEST_SHELL
testpath=os.path.join(test_destdir,testname)
logger.debug("Creating and Opening test file: %s for writing ", testpath)
fd=open(testpath,'w')
if test_type == "software":
fd.write(header)
else:
shell_magic = "#!/" + os.path.join("bin",BUILDTEST_SHELL)
fd.write(shell_magic + "\n")
fd.write("module purge \n")
fd.write(key)
fd.close()
# reading test script for writing content of test in logcontent
fd=open(testpath,'r')
content=fd.read().splitlines()
fd.close()
logger.info("Content of test file: %s ", testpath)
logger.info("[START TEST-BLOCK]")
for line in content:
logger.info("%s", line)
logger.info("[END TEST-BLOCK]")
logger.debug("Updating CMakeList file: %s", test_destdir_cmakelist)
fd=open(test_destdir_cmakelist,'a')
if test_type == "software":
# modify add_test string when toolchain is not defined
if len(toolchain_name) == 0:
add_test_str="add_test(NAME " + name + "-" + version + "-" + testname + "\t COMMAND " + BUILDTEST_SHELL + " " + testname + "\t WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) \n"
else:
add_test_str="add_test(NAME " + name + "-" + version + "-" + toolchain_name + "-" + toolchain_version + "-" + testname + "\t COMMAND " + BUILDTEST_SHELL + " " + testname + "\t WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) \n"
else:
add_test_str="add_test(NAME system-" + pkg + "-" + testname + "\t COMMAND " + BUILDTEST_SHELL + " " + testname + "\t WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) \n"
logger.debug("Adding content: %s ", add_test_str)
fd.write(add_test_str)
if BUILDTEST_ENABLE_JOB:
generate_job(testpath,BUILDTEST_SHELL,BUILDTEST_JOB_TEMPLATE, content)
fd.close()
print
if test_type == "system":
print "Generating " + str(count) + " binary tests"
else:
print "Generating " + str(count) + " binary tests"
print "Binary Tests are written in " + test_destdir
