def git_update(self):
""" Do a git update of the repository. If githash is not empty, then
we will check out that version instead of git-pulling. """
os.chdir(self.dir)
# find out current branch so that we can go back later if we need.
stdout0, stderr0, rc = test_util.run("git rev-parse --abbrev-ref HEAD")
self.branch_orig = stdout0.rstrip('\n')
if self.branch_orig != self.branch_wanted:
self.suite.log.log("git checkout {} in {}".format(self.branch_wanted, self.dir))
stdout, stderr, rc = test_util.run("git checkout {}".format(self.branch_wanted),
stdin=True)
else:
self.branch_wanted = self.branch_orig
if self.hash_wanted == "" or self.hash_wanted is None:
self.suite.log.log("'git pull' in {}".format(self.dir))
# we need to be tricky here to make sure that the stdin is
# presented to the user to get the password.
stdout, stderr, rc = test_util.run("git pull", stdin=True,
outfile="git.{}.out".format(self.name))
else:
stdout, stderr, rc = test_util.run("git checkout {}".format(self.hash_wanted),
outfile="git.{}.out".format(self.name))
if not rc == 0:
self.suite.log.fail("ERROR: git update was unsuccessful")
shutil.copy("git.{}.out".format(self.name), self.suite.full_web_dir)
def assert_equal_openssl(nss_ciphers, ossl_ciphers):
(nss, err, rc) = run([exe, "--o", nss_ciphers])
assert rc == 0
(ossl, err, rc) = run([openssl, "ciphers", ossl_ciphers])
assert rc == 0
nss_list = nss.strip().split(':')
nss_list.sort()
ossl_list = ossl.strip().split(':')
ossl_list = list(set(ossl_list))
ossl_list.sort()
# NSS doesn't support the SHA-384 ciphers, remove them from the OpenSSL
# output.
t = list()
for o in ossl_list:
if 'SHA384' in o:
continue
if o in CIPHERS_NOT_IN_NSS:
continue
t.append(o)
ossl_list = t
if len(nss_list) > len(ossl_list):
diff = set(nss_list) - set(ossl_list)
elif len(ossl_list) > len(nss_list):
diff = set(ossl_list) - set(nss_list)
else:
diff = ''
assert nss_list == ossl_list, '%r != %r. Difference %r' % (':'.join(nss_list), ':'.join(ossl_list), diff)
def make_changelog(self):
""" generate a ChangeLog git repository, and copy it to the
web directory"""
os.chdir(self.dir)
self.suite.log.log("generating ChangeLog for {}/".format(self.name))
test_util.run("git log --name-only",
outfile="ChangeLog.{}".format(self.name), outfile_mode="w")
shutil.copy("ChangeLog.{}".format(self.name), self.suite.full_web_dir)
def slack_post_it(self, message):
payload = {}
# make sure there are no quotes in the strings
payload["channel"] = self.slack_channel.replace('"', '')
payload["username"] = self.slack_username.replace('"', '')
payload["text"] = message.replace("'", "") # apostrophes
s = json.dumps(payload)
cmd = "curl -X POST --data-urlencode 'payload={}' {}".format(s, self.slack_webhook_url)
test_util.run(cmd)
def make_realclean(self, repo="source"):
build_comp_string = ""
if self.repos[repo].build == 1:
if not self.repos[repo].comp_string is None:
build_comp_string = self.repos[repo].comp_string
extra_src_comp_string = ""
if not self.extra_src_comp_string is None:
extra_src_comp_string = self.extra_src_comp_string
cmd = "{} BOXLIB_HOME={} {} {} realclean".format(
self.MAKE, self.boxlib_dir,
extra_src_comp_string, build_comp_string)
test_util.run(cmd)
def build_f(self, test=None, opts="", target="", outfile=None):
""" build an executable with the Fortran BoxLib build system """
build_opts = ""
if test is not None:
build_opts += "NDEBUG={} ".format(f_flag(test.debug, test_not=True))
build_opts += "ACC={} ".format(f_flag(test.acc))
build_opts += "MPI={} ".format(f_flag(test.useMPI))
build_opts += "OMP={} ".format(f_flag(test.useOMP))
if not test.extra_build_dir == "":
build_opts += self.repos[test.extra_build_dir].comp_string + " "
if not test.addToCompileString == "":
build_opts += test.addToCompileString + " "
all_opts = "{} {} {}".format(self.extra_src_comp_string, build_opts, opts)
comp_string = "{} -j{} BOXLIB_HOME={} COMP={} {} {} {}".format(
self.MAKE, self.numMakeJobs, self.boxlib_dir,
self.FCOMP, self.add_to_f_make_command, all_opts, target)
self.log.log(comp_string)
stdout, stderr, rc = test_util.run(comp_string, outfile=outfile)
# make returns 0 if everything was good
if not rc == 0:
self.log.warn("build failed")
return comp_string, rc
def build_c(self, test=None, opts="", outfile=None):
build_opts = ""
if test is not None:
build_opts += "DEBUG={} ".format(c_flag(test.debug))
build_opts += "USE_ACC={} ".format(c_flag(test.acc))
build_opts += "USE_MPI={} ".format(c_flag(test.useMPI))
build_opts += "USE_OMP={} ".format(c_flag(test.useOMP))
build_opts += "DIM={} ".format(test.dim)
if not test.extra_build_dir == "":
build_opts += self.repos[test.extra_build_dir].comp_string + " "
if not test.addToCompileString == "":
build_opts += test.addToCompileString + " "
all_opts = "{} {} {}".format(self.extra_src_comp_string, build_opts, opts)
comp_string = "{} -j{} BOXLIB_HOME={} {} COMP={} FCOMP={} {}".format(
self.MAKE, self.numMakeJobs, self.boxlib_dir,
all_opts, self.COMP, self.FCOMP, self.add_to_c_make_command)
self.log.log(comp_string)
stdout, stderr, rc = test_util.run(comp_string, outfile=outfile)
# make returns 0 if everything was good
if not rc == 0:
self.log.warn("build failed")
return comp_string, rc
def save_head(self):
os.chdir(self.dir)
self.suite.log.log("saving git HEAD for {}/".format(self.name))
stdout, stderr, rc = test_util.run("git rev-parse HEAD",
outfile="git.{}.HEAD".format(self.name) )
self.hash_current = stdout
shutil.copy("git.{}.HEAD".format(self.name), self.suite.full_web_dir)
def cmake_config(self, name, path, configOpts="", install=0, env=None):
"Generate Cmake configuration"
self.log.outdent()
self.log.skip()
self.log.bold("configuring " + name + " build...")
self.log.indent()
# Setup dir names
builddir = path + 'builddir'
if install:
installdir = path + 'installdir'
else:
installdir = None
# Define enviroment
ENV = {}
ENV = dict(os.environ) # Copy of current enviroment
ENV['FC'] = self.FCOMP
ENV['CXX'] = self.COMP
if env is not None: ENV.update(env)
# remove build and installation directories if present and re-make them
if os.path.isdir(builddir):
shutil.rmtree(builddir)
self.log.log("mkdir " + builddir)
os.mkdir(builddir)
if install:
if os.path.isdir(installdir):
shutil.rmtree(installdir)
self.log.log("mkdir " + installdir)
os.mkdir(installdir)
# Logfile
coutfile = '{}{}.cmake.log'.format(self.full_test_dir, name)
# Run cmake
cmd = 'cmake {} -H{} -B{} '.format(configOpts, path, builddir)
if install:
cmd += '-DCMAKE_INSTALL_PREFIX:PATH=' + installdir
self.log.log(cmd)
stdout, stderr, rc = test_util.run(cmd, outfile=coutfile, env=ENV)
# Check exit condition
if not rc == 0:
errstr = "\n \nERROR! Cmake configuration failed for " + name + " \n"
errstr += "Check " + coutfile + " for more information."
sys.exit(errstr)
return builddir, installdir
def test_holes():
stdin = """
a,b,
1,,3
x,y,z
"""
stdout = """
b,
,3
y,z
"""
assert rm_whitespace(stdout) + '\n' == run(rm_whitespace(stdin), 'bsv | bcut 2,3 | csv')
def test_repeats():
stdin = """
x,y,z
1,2,3
a,b,c,d
"""
stdout = """
x,z,x,x
1,3,1,1
a,c,a,a
"""
assert rm_whitespace(stdout) + '\n' == run(rm_whitespace(stdin), 'bsv | bcut 1,3,1,1 | csv')
def git_back(self):
""" switch the repo back to its original branch """
os.chdir(self.dir)
self.suite.log.log("git checkout {} in {}".format(self.branch_orig, self.dir))
stdout, stderr, rc = test_util.run("git checkout {}".format(self.branch_orig),
stdin=True,
outfile="git.{}.out".format(self.name))
if not rc == 0:
self.suite.log.fail("ERROR: git checkout was unsuccessful")
def test_compatability():
stdin = """
b
c
a
"""
stdout = """
c
b
a
"""
assert rm_whitespace(stdout) + '\n' == run(rm_whitespace(stdin), 'bsv | bsort --reversed | csv')
def test_compatability():
stdin = """
b
c
a
"""
stdout = """
c
b
a
"""
assert rm_whitespace(stdout) + '\n' == run(rm_whitespace(stdin), 'bsv | brsort | bin/csv')
def build_f(self, test=None, opts="", target="", outfile=None):
""" build an executable with the Fortran AMReX build system """
comp_string = self.get_comp_string_f(test, opts, target, outfile)
self.log.log(comp_string)
stdout, stderr, rc = test_util.run(comp_string, outfile=outfile)
# make returns 0 if everything was good
if not rc == 0:
self.log.warn("build failed")
return comp_string, rc
def git_back(self):
""" switch the repo back to its original branch """
os.chdir(self.dir)
self.suite.log.log(f"git checkout {self.branch_orig} in {self.dir}")
_, _, rc = test_util.run(f"git checkout {self.branch_orig}",
stdin=True,
outfile=f"git.{self.name}.out")
if rc != 0:
self.suite.log.fail("ERROR: git checkout was unsuccessful")
# if we were working on a PR, delete the temporary branch, since we can't pull on it
if self.pr_wanted is not None:
self.suite.log.log(f"removing pr-{self.pr_wanted}")
_, _, rc = test_util.run(f"git branch -D pr-{self.pr_wanted}",
stdin=True)
if rc != 0:
self.suite.log.fail("ERROR: git branch deletion was unsuccessful")
def save_head(self):
"""Save the current head of the repo"""
os.chdir(self.dir)
self.suite.log.log(f"saving git HEAD for {self.name}/")
stdout, _, _ = test_util.run("git rev-parse HEAD",
outfile=f"git.{self.name}.HEAD")
self.hash_current = stdout
shutil.copy(f"git.{self.name}.HEAD", self.suite.full_web_dir)
def save_head(self):
os.chdir(self.dir)
self.suite.log.log("saving git HEAD for {}/".format(self.name))
stdout, stderr, rc = test_util.run("git rev-parse HEAD",
outfile="git.{}.HEAD".format(
self.name))
self.hash_current = stdout
shutil.copy("git.{}.HEAD".format(self.name), self.suite.full_web_dir)
def test_basic():
stdin = """
a
a
a
b
b
a
"""
stdout = """
6
"""
assert rm_whitespace(stdout) + '\n' == run(rm_whitespace(stdin), 'bsv | bcountrows | bschema i64:a | csv')
def test_basic():
stdin = """
a,b,c,d
e,f,g
x,y
"""
stdout = """
3,a,b,c,d
3,e,f,g
2,x,y
"""
assert rm_whitespace(stdout) + '\n' == run(rm_whitespace(stdin),
'bsv | bbucket 4 | bin/csv')
def test_single_column():
stdin = """
a
y
x
"""
stdout = """
3,a
3,y
2,x
"""
assert rm_whitespace(stdout) + '\n' == run(rm_whitespace(stdin),
'bsv | bbucket 4 | bin/csv')
def git_back(self):
""" switch the repo back to its original branch """
os.chdir(self.dir)
self.suite.log.log("git checkout {} in {}".format(
self.branch_orig, self.dir))
_, _, rc = test_util.run("git checkout {}".format(self.branch_orig),
stdin=True,
outfile="git.{}.out".format(self.name))
if rc != 0:
self.suite.log.fail("ERROR: git checkout was unsuccessful")
def test_compatability2():
stdin = """
c,c
b,b
a,a
"""
stdout = """
a,a
b,b
c,c
"""
assert rm_whitespace(stdout) + '\n' == run(rm_whitespace(stdin),
'bsv | bsort | csv')
def test_basic():
stdin = """
a,b
cd,e
"""
stdout = """
a
b
cd
e
"""
assert typed(rm_whitespace(stdout)) + '\n' == run(rm_whitespace(stdin),
'bin/_csv.8')
def test_basic1():
stdin = """
a,b,c,d
1,2,3
x,y
"""
stdout = """
a:b,a,b,c,d
1:2,1,2,3
x:y,x,y
"""
assert rm_whitespace(stdout) + '\n' == run(rm_whitespace(stdin),
'bsv | bcombine 1,2 | csv')
def test_basic2():
stdin = """
a,b,c,d
1,2,3
x,y
"""
stdout = """
b:a,a,b,c,d
2:1,1,2,3
y:x,x,y
"""
assert rm_whitespace(stdout) + '\n' == run(rm_whitespace(stdin),
'bsv | bcombine 2,1 | csv')
def git_update(self):
""" Do a git update of the repository. If githash is not empty, then
we will check out that version instead of git-pulling. """
os.chdir(self.dir)
# find out current branch so that we can go back later if we need.
stdout0, stderr0, rc = test_util.run("git rev-parse --abbrev-ref HEAD")
self.branch_orig = stdout0.rstrip('\n')
if self.branch_orig != self.branch_wanted:
self.suite.log.log("git checkout {} in {}".format(
self.branch_wanted, self.dir))
stdout, stderr, rc = test_util.run("git checkout {}".format(
self.branch_wanted),
stdin=True)
else:
self.branch_wanted = self.branch_orig
if self.hash_wanted == "" or self.hash_wanted is None:
self.suite.log.log("'git pull' in {}".format(self.dir))
# we need to be tricky here to make sure that the stdin is
# presented to the user to get the password.
stdout, stderr, rc = test_util.run("git pull",
stdin=True,
outfile="git.{}.out".format(
self.name))
else:
stdout, stderr, rc = test_util.run(
"git checkout {}".format(self.hash_wanted),
outfile="git.{}.out".format(self.name))
if not rc == 0:
self.suite.log.fail("ERROR: git update was unsuccessful")
shutil.copy("git.{}.out".format(self.name), self.suite.full_web_dir)
def test_example1():
csv = ',\n'
val = runb(csv, 'bsv')
bsv = b''.join([
struct.pack(
'i', 8
), # uint32 num bytes in this chunk, chunks contain 1 or more rows
struct.pack('H', 1), # uint16 max, see load.h
struct.pack('H', 0), # uint16 sizes, see load.h
struct.pack('H', 0), # uint16 sizes, see load.h
b'\0\0',
])
assert bsv == val
assert csv == run(csv, 'bsv | csv')
def test_reverse():
stdin = """
a,b,c,d
1,2,3
x,y
"""
stdout = """
b,a
2,1
y,x
"""
assert rm_whitespace(stdout) + '\n' == run(rm_whitespace(stdin),
'bsv | bcut 2,1 | bin/csv')
stdin = """
a,b,c,d
1,2,3
x,y,z
"""
stdout = """
c,a
3,1
z,x
"""
assert rm_whitespace(stdout) + '\n' == run(rm_whitespace(stdin),
'bsv | bcut 3,1 | bin/csv')
stdin = """
x,y,z
1,2,3
a,b,c,d
"""
stdout = """
z,x
3,1
c,a
"""
assert rm_whitespace(stdout) + '\n' == run(rm_whitespace(stdin),
'bsv | bcut 3,1 | bin/csv')
def test_forward():
stdin = """
a,b,c,d
1,2,3
x,y
"""
stdout = """
a,b
1,2
x,y
"""
assert rm_whitespace(stdout) + '\n' == run(rm_whitespace(stdin),
'bsv | bcut 1,2 | bin/csv')
stdin = """
a,b,c,d
1,2,3
x,y,z
"""
stdout = """
a,c
1,3
x,z
"""
assert rm_whitespace(stdout) + '\n' == run(rm_whitespace(stdin),
'bsv | bcut 1,3 | bin/csv')
stdin = """
x,y,z
1,2,3
a,b,c,d
"""
stdout = """
x,z
1,3
a,c
"""
assert rm_whitespace(stdout) + '\n' == run(rm_whitespace(stdin),
'bsv | bcut 1,3 | bin/csv')
def test_single_column():
stdin = """
x,y
1,2,3
a,b,c,d
"""
stdout = """
x
1
a
"""
assert rm_whitespace(stdout) + '\n' == run(rm_whitespace(stdin),
'bsv | bcut 1 | bin/csv')
stdin = """
a,b,c,d
1,2,3
x,y
"""
stdout = """
a
1
x
"""
assert rm_whitespace(stdout) + '\n' == run(rm_whitespace(stdin),
'bsv | bcut 1 | bin/csv')
stdin = """
a,b,c,d
1,2,3
x,y
"""
stdout = """
b
2
y
"""
assert rm_whitespace(stdout) + '\n' == run(rm_whitespace(stdin),
'bsv | bcut 2 | bin/csv')
def test_basic():
stdin = """
a
a
a
b
b
a
"""
stdout = """
a,3
b,2
a,1
"""
assert rm_whitespace(stdout) + '\n' == run(rm_whitespace(stdin), 'bsv | bcounteach | bschema *,i64:a | csv')
def build_f(self, test=None, opts="", target="", outfile=None):
""" build an executable with the Fortran BoxLib build system """
build_opts = ""
if test is not None:
build_opts += "NDEBUG={} ".format(f_flag(test.debug, test_not=True))
build_opts += "MPI={} ".format(f_flag(test.useMPI))
build_opts += "OMP={} ".format(f_flag(test.useOMP))
if not test.extra_build_dir == "":
build_opts += self.repos[test.extra_build_dir].comp_string + " "
if not test.addToCompileString == "":
build_opts += test.addToCompileString + " "
all_opts = "{} {} {}".format(self.extra_src_comp_string, build_opts, opts)
comp_string = "{} -j{} BOXLIB_HOME={} COMP={} {} {} {}".format(
self.MAKE, self.numMakeJobs, self.boxlib_dir,
self.FCOMP, self.add_to_f_make_command, all_opts, target)
self.log.log(comp_string)
test_util.run(comp_string, outfile=outfile)
return comp_string
def get_variable_names(suite, plotfile):
""" uses fvarnames to extract the names of variables
stored in a plotfile """
# Run fvarnames
command = "{} {}".format(suite.tools["fvarnames"], plotfile)
sout, serr, ierr = test_util.run(command)
if ierr != 0:
return serr
# Split on whitespace
vars = re.split("\s+", sout)[2:-1:2]
return set(vars)
def test_basic():
stdin = """
a,1
a,2
a,3
b,4
b,5
a,6
a,7
"""
stdout = """
a,1
b,4
"""
assert rm_whitespace(stdout) + '\n' == run(rm_whitespace(stdin),
'bsv | bdedupe-hash | csv')
def build_c(self,
test=None,
opts="",
target="",
outfile=None,
c_make_additions=None):
comp_string = self.get_comp_string_c(test, opts, target, outfile,
c_make_additions)
self.log.log(comp_string)
stdout, stderr, rc = test_util.run(comp_string, outfile=outfile)
# make returns 0 if everything was good
if not rc == 0:
self.log.warn("build failed")
return comp_string, rc
def build_c(self,
test=None,
opts="",
target="",
outfile=None,
c_make_additions=None):
build_opts = ""
if c_make_additions is None:
c_make_additions = self.add_to_c_make_command
if test is not None:
build_opts += "DEBUG={} ".format(c_flag(test.debug))
build_opts += "USE_ACC={} ".format(c_flag(test.acc))
build_opts += "USE_MPI={} ".format(c_flag(test.useMPI))
build_opts += "USE_OMP={} ".format(c_flag(test.useOMP))
build_opts += "DIM={} ".format(test.dim)
if not test.extra_build_dir == "":
build_opts += self.repos[
test.extra_build_dir].comp_string + " "
if "source" in self.repos:
if not self.repos["source"].comp_string is None:
build_opts += self.repos["source"].comp_string + " "
if not test.addToCompileString == "":
build_opts += test.addToCompileString + " "
if test.ignoreGlobalMakeAdditions:
c_make_additions = ""
all_opts = "{} {} {}".format(self.extra_src_comp_string, build_opts,
opts)
comp_string = "{} -j{} AMREX_HOME={} {} COMP={} {} {}".format(
self.MAKE, self.numMakeJobs, self.amrex_dir, all_opts, self.COMP,
c_make_additions, target)
self.log.log(comp_string)
stdout, stderr, rc = test_util.run(comp_string, outfile=outfile)
# make returns 0 if everything was good
if not rc == 0:
self.log.warn("build failed")
return comp_string, rc
def run_test(self, test, base_command):
test_env = None
if test.useOMP:
test_env = dict(os.environ, OMP_NUM_THREADS="{}".format(test.numthreads))
if test.useMPI:
test_run_command = self.MPIcommand
test_run_command = test_run_command.replace("@host@", self.MPIhost)
test_run_command = test_run_command.replace("@nprocs@", "{}".format(test.numprocs))
test_run_command = test_run_command.replace("@command@", base_command)
else:
test_run_command = base_command
self.log.log(test_run_command)
sout, serr, ierr = test_util.run(test_run_command, stdin=True,
outfile="{}.run.out".format(test.name), env=test_env)
test.run_command = test_run_command
def test_basic():
stdin = """
a,1
a,2
a,3
b,4
b,5
a,6
"""
stdout = """
a,12
b,9
"""
assert rm_whitespace(stdout) + '\n' == run(
rm_whitespace(stdin),
'bsv | bschema *,a:i64 | bsumeach-hash i64 | bschema *,i64:a | bsort | csv'
)
def run_test(self, test, base_command):
test_env = None
if test.useOMP:
test_env = dict(os.environ, OMP_NUM_THREADS="{}".format(test.numthreads))
if test.useMPI:
test_run_command = self.MPIcommand
test_run_command = test_run_command.replace("@host@", self.MPIhost)
test_run_command = test_run_command.replace("@nprocs@", "{}".format(test.numprocs))
test_run_command = test_run_command.replace("@command@", base_command)
else:
test_run_command = base_command
self.log.log(test_run_command)
sout, serr, ierr = test_util.run(test_run_command, stdin=True,
outfile="{}.run.out".format(test.name),
errfile="{}.err.out".format(test.name),
env=test_env)
test.run_command = test_run_command
def test_SSLv3_equals_TLSv1(self):
(nss, err, rc) = run([exe, "--o", "SSLv3"])
(nss2, err, rc2) = run([exe, "--o", "TLSv1"])
assert rc == 0
assert rc2 == 0
assert_equal(nss, nss2)
def test_suite(argv):
"""
the main test suite driver
"""
# parse the commandline arguments
args = test_util.get_args(arg_string=argv)
# read in the test information
suite, test_list = params.load_params(args)
active_test_list = [t.name for t in test_list]
test_list = suite.get_tests_to_run(test_list)
suite.log.skip()
suite.log.bold("running tests: ")
suite.log.indent()
for obj in test_list:
suite.log.log(obj.name)
suite.log.outdent()
if not args.complete_report_from_crash == "":
# make sure the web directory from the crash run exists
suite.full_web_dir = "{}/{}/".format(
suite.webTopDir, args.complete_report_from_crash)
if not os.path.isdir(suite.full_web_dir):
suite.log.fail("Crash directory does not exist")
suite.test_dir = args.complete_report_from_crash
# find all the tests that completed in that web directory
tests = []
test_file = ""
was_benchmark_run = 0
for sfile in os.listdir(suite.full_web_dir):
if os.path.isfile(sfile) and sfile.endswith(".status"):
index = string.rfind(sfile, ".status")
tests.append(sfile[:index])
with open(suite.full_web_dir + sfile, "r") as f:
for line in f:
if line.find("benchmarks updated") > 0:
was_benchmark_run = 1
if os.path.isfile(sfile) and sfile.endswith(".ini"):
test_file = sfile
# create the report for this test run
num_failed = report.report_this_test_run(suite, was_benchmark_run,
"recreated report after crash of suite",
"", tests, test_file)
# create the suite report
suite.log.bold("creating suite report...")
report.report_all_runs(suite, active_test_list)
suite.log.close_log()
sys.exit("done")
#--------------------------------------------------------------------------
# check bench dir and create output directories
#--------------------------------------------------------------------------
all_compile = all([t.compileTest == 1 for t in test_list])
if not all_compile:
bench_dir = suite.get_bench_dir()
if not args.copy_benchmarks is None:
last_run = suite.get_last_run()
suite.make_test_dirs()
if suite.slack_post:
msg = "{} ({}) test suite started, id: {}\n{}".format(
suite.suiteName, suite.sub_title, suite.test_dir, args.note)
suite.slack_post_it(msg)
if not args.copy_benchmarks is None:
old_full_test_dir = suite.testTopDir + suite.suiteName + "-tests/" + last_run
copy_benchmarks(old_full_test_dir, suite.full_web_dir,
test_list, bench_dir, suite.log)
# here, args.copy_benchmarks plays the role of make_benchmarks
num_failed = report.report_this_test_run(suite, args.copy_benchmarks,
"copy_benchmarks used -- no new tests run",
"",
test_list, args.input_file[0])
report.report_all_runs(suite, active_test_list)
if suite.slack_post:
msg = "copied benchmarks\n{}".format(args.copy_benchmarks)
suite.slack_post_it(msg)
sys.exit("done")
#--------------------------------------------------------------------------
# figure out what needs updating and do the git updates, save the
# current hash / HEAD, and make a ChangeLog
# --------------------------------------------------------------------------
now = time.localtime(time.time())
update_time = time.strftime("%Y-%m-%d %H:%M:%S %Z", now)
no_update = args.no_update.lower()
if not args.copy_benchmarks is None:
no_update = "all"
# the default is to update everything, unless we specified a hash
# when constructing the Repo object
if no_update == "none":
pass
elif no_update == "all":
for k in suite.repos:
suite.repos[k].update = False
else:
nouplist = [k.strip() for k in no_update.split(",")]
for repo in suite.repos.keys():
if repo.lower() in nouplist:
suite.repos[repo].update = False
os.chdir(suite.testTopDir)
for k in suite.repos:
suite.log.skip()
suite.log.bold("repo: {}".format(suite.repos[k].name))
suite.log.indent()
if suite.repos[k].update or suite.repos[k].hash_wanted:
suite.repos[k].git_update()
suite.repos[k].save_head()
if suite.repos[k].update:
suite.repos[k].make_changelog()
suite.log.outdent()
# keep track if we are running on any branch that is not the suite
# default
branches = [suite.repos[r].branch_wanted for r in suite.repos]
if not all(suite.default_branch == b for b in branches):
suite.log.warn("some git repos are not on the default branch")
bf = open("{}/branch.status".format(suite.full_web_dir), "w")
bf.write("branch different than suite default")
bf.close()
#--------------------------------------------------------------------------
# build the tools and do a make clean, only once per build directory
#--------------------------------------------------------------------------
suite.build_tools(test_list)
all_build_dirs = find_build_dirs(test_list)
suite.log.skip()
suite.log.bold("make clean in...")
for d, source_tree in all_build_dirs:
if not source_tree == "":
suite.log.log("{} in {}".format(d, source_tree))
os.chdir(suite.repos[source_tree].dir + d)
suite.make_realclean(repo=source_tree)
else:
suite.log.log("{}".format(d))
os.chdir(suite.source_dir + d)
if suite.sourceTree == "BoxLib":
suite.make_realclean(repo="BoxLib")
else:
suite.make_realclean()
os.chdir(suite.testTopDir)
#--------------------------------------------------------------------------
# main loop over tests
#--------------------------------------------------------------------------
for test in test_list:
suite.log.outdent() # just to make sure we have no indentation
suite.log.skip()
suite.log.bold("working on test: {}".format(test.name))
suite.log.indent()
if not args.make_benchmarks is None and (test.restartTest or test.compileTest or
test.selfTest):
suite.log.warn("benchmarks not needed for test {}".format(test.name))
continue
output_dir = suite.full_test_dir + test.name + '/'
os.mkdir(output_dir)
test.output_dir = output_dir
#----------------------------------------------------------------------
# compile the code
#----------------------------------------------------------------------
if not test.extra_build_dir == "":
bdir = suite.repos[test.extra_build_dir].dir + test.buildDir
else:
bdir = suite.source_dir + test.buildDir
os.chdir(bdir)
if test.reClean == 1:
# for one reason or another, multiple tests use different
# build options, make clean again to be safe
suite.log.log("re-making clean...")
if not test.extra_build_dir == "":
suite.make_realclean(repo=test.extra_build_dir)
else:
suite.make_realclean()
suite.log.log("building...")
coutfile="{}/{}.make.out".format(output_dir, test.name)
if suite.sourceTree == "C_Src" or test.testSrcTree == "C_Src":
comp_string, rc = suite.build_c(test=test, outfile=coutfile)
executable = test_util.get_recent_filename(bdir, "", ".ex")
elif suite.sourceTree == "F_Src" or test.testSrcTree == "F_Src":
comp_string, rc = suite.build_f(test=test, outfile=coutfile)
executable = test_util.get_recent_filename(bdir, "main", ".exe")
test.comp_string = comp_string
# make return code is 0 if build was successful
if rc == 0: test.compile_successful = True
# copy the make.out into the web directory
shutil.copy("{}/{}.make.out".format(output_dir, test.name), suite.full_web_dir)
if not test.compile_successful:
error_msg = "ERROR: compilation failed"
report.report_single_test(suite, test, test_list, failure_msg=error_msg)
continue
if test.compileTest:
suite.log.log("creating problem test report ...")
report.report_single_test(suite, test, test_list)
continue
#----------------------------------------------------------------------
# copy the necessary files over to the run directory
#----------------------------------------------------------------------
suite.log.log("copying files to run directory...")
needed_files = []
needed_files.append((executable, "move"))
needed_files.append((test.inputFile, "copy"))
# strip out any sub-directory from the build dir
test.inputFile = os.path.basename(test.inputFile)
if test.probinFile != "":
needed_files.append((test.probinFile, "copy"))
# strip out any sub-directory from the build dir
test.probinFile = os.path.basename(test.probinFile)
for auxf in test.auxFiles:
needed_files.append((auxf, "copy"))
# if any copy/move fail, we move onto the next test
skip_to_next_test = 0
for nfile, action in needed_files:
if action == "copy":
act = shutil.copy
elif action == "move":
act = shutil.move
else:
suite.log.fail("invalid action")
try: act(nfile, output_dir)
except IOError:
error_msg = "ERROR: unable to {} file {}".format(action, nfile)
report.report_single_test(suite, test, test_list, failure_msg=error_msg)
skip_to_next_test = 1
break
if skip_to_next_test: continue
skip_to_next_test = 0
for lfile in test.linkFiles:
if not os.path.exists(lfile):
error_msg = "ERROR: link file {} does not exist".format(lfile)
report.report_single_test(suite, test, test_list, failure_msg=error_msg)
skip_to_next_test = 1
break
else:
link_source = os.path.abspath(lfile)
link_name = os.path.join(output_dir, os.path.basename(lfile))
try: os.symlink(link_source, link_name)
except IOError:
error_msg = "ERROR: unable to symlink link file: {}".format(lfile)
report.report_single_test(suite, test, test_list, failure_msg=error_msg)
skip_to_next_test = 1
break
if skip_to_next_test: continue
#----------------------------------------------------------------------
# run the test
#----------------------------------------------------------------------
suite.log.log("running the test...")
os.chdir(output_dir)
test.wall_time = time.time()
if suite.sourceTree == "C_Src" or test.testSrcTree == "C_Src":
base_cmd = "./{} {} amr.plot_file={}_plt amr.check_file={}_chk".format(
executable, test.inputFile, test.name, test.name)
# keep around the checkpoint files only for the restart runs
if test.restartTest:
base_cmd += " amr.checkpoint_files_output=1 amr.check_int=%d" % \
(test.restartFileNum)
else:
base_cmd += " amr.checkpoint_files_output=0"
base_cmd += "{} {}".format(suite.globalAddToExecString, test.runtime_params)
elif suite.sourceTree == "F_Src" or test.testSrcTree == "F_Src":
base_cmd = "./{} {} --plot_base_name {}_plt --check_base_name {}_chk ".format(
executable, test.inputFile, test.name, test.name)
# keep around the checkpoint files only for the restart runs
if not test.restartTest: base_cmd += " --chk_int 0 "
base_cmd += "{} {}".format(suite.globalAddToExecString, test.runtime_params)
if args.with_valgrind:
base_cmd = "valgrind " + args.valgrind_options + " " + base_cmd
suite.run_test(test, base_cmd)
# if it is a restart test, then rename the final output file and
# restart the test
if test.restartTest:
skip_restart = False
last_file = test.get_last_plotfile(output_dir=output_dir)
if last_file == "":
error_msg = "ERROR: test did not produce output. Restart test not possible"
skip_restart = True
if len(test.find_backtrace()) > 0:
error_msg = "ERROR: test produced backtraces. Restart test not possible"
skip_restart = True
if skip_restart:
# copy what we can
test.wall_time = time.time() - test.wall_time
shutil.copy("{}.run.out".format(test.name), suite.full_web_dir)
if os.path.isfile("{}.err.out".format(test.name)):
shutil.copy("{}.err.out".format(test.name), suite.full_web_dir)
test.has_stderr = True
suite.copy_backtrace(test)
report.report_single_test(suite, test, test_list, failure_msg=error_msg)
continue
orig_last_file = "orig_{}".format(last_file)
shutil.move(last_file, orig_last_file)
if test.diffDir:
orig_diff_dir = "orig_{}".format(test.diffDir)
shutil.move(test.diffDir, orig_diff_dir)
# get the file number to restart from
restart_file = "%s_chk%5.5d" % (test.name, test.restartFileNum)
suite.log.log("restarting from {} ... ".format(restart_file))
if suite.sourceTree == "C_Src" or test.testSrcTree == "C_Src":
base_cmd = "./{} {} amr.plot_file={}_plt amr.check_file={}_chk amr.checkpoint_files_output=0 amr.restart={}".format(
executable, test.inputFile, test.name, test.name, restart_file)
elif suite.sourceTree == "F_Src" or test.testSrcTree == "F_Src":
base_cmd = "./{} {} --plot_base_name {}_plt --check_base_name {}_chk --chk_int 0 --restart {} {}".format(
executable, test.inputFile, test.name, test.name, test.restartFileNum, suite.globalAddToExecString)
suite.run_test(test, base_cmd)
test.wall_time = time.time() - test.wall_time
#----------------------------------------------------------------------
# do the comparison
#----------------------------------------------------------------------
if not test.selfTest:
if test.outputFile == "":
if test.compareFile == "":
compare_file = test.get_last_plotfile(output_dir=output_dir)
else:
# we specified the name of the file we want to
# compare to -- make sure it exists
compare_file = test.compareFile
if not os.path.isdir(compare_file):
compare_file = ""
output_file = compare_file
else:
output_file = test.outputFile
compare_file = test.name+'_'+output_file
# get the number of levels for reporting
prog = "{} -l {}".format(suite.tools["fboxinfo"], output_file)
stdout0, stderr0, rc = test_util.run(prog)
test.nlevels = stdout0.rstrip('\n')
if not type(params.convert_type(test.nlevels)) is int:
test.nlevels = ""
if args.make_benchmarks is None:
suite.log.log("doing the comparison...")
suite.log.indent()
suite.log.log("comparison file: {}".format(output_file))
test.compare_file_used = output_file
if not test.restartTest:
bench_file = bench_dir + compare_file
else:
bench_file = orig_last_file
# see if it exists
# note, with BoxLib, the plotfiles are actually directories
if not os.path.isdir(bench_file):
suite.log.warn("no corresponding benchmark found")
bench_file = ""
with open("{}.compare.out".format(test.name), 'w') as cf:
cf.write("WARNING: no corresponding benchmark found\n")
cf.write(" unable to do a comparison\n")
else:
if not compare_file == "":
suite.log.log("benchmark file: {}".format(bench_file))
command = "{} -n 0 {} {}".format(
suite.tools["fcompare"], bench_file, output_file)
sout, serr, ierr = test_util.run(command,
outfile="{}.compare.out".format(test.name), store_command=True)
if ierr == 0:
test.compare_successful = True
else:
suite.log.warn("unable to do a comparison")
with open("{}.compare.out".format(test.name), 'w') as cf:
cf.write("WARNING: run did not produce any output\n")
cf.write(" unable to do a comparison\n")
suite.log.outdent()
if not test.diffDir == "":
if not test.restartTest:
diff_dir_bench = bench_dir + '/' + test.name + '_' + test.diffDir
else:
diff_dir_bench = orig_diff_dir
suite.log.log("doing the diff...")
suite.log.log("diff dir: {}".format(test.diffDir))
command = "diff {} -r {} {}".format(
test.diffOpts, diff_dir_bench, test.diffDir)
outfile = "{}.compare.out".format(test.name)
sout, serr, diff_status = test_util.run(command, outfile=outfile, store_command=True)
if diff_status == 0:
diff_successful = True
with open("{}.compare.out".format(test.name), 'a') as cf:
cf.write("\ndiff was SUCCESSFUL\n")
else:
diff_successful = False
test.compare_successful = test.compare_successful and diff_successful
else: # make_benchmarks
suite.log.log("storing output of {} as the new benchmark...".format(test.name))
suite.log.indent()
suite.log.warn("new benchmark file: {}".format(compare_file))
suite.log.outdent()
if not compare_file == "":
if not output_file == compare_file:
source_file = output_file
else:
source_file = compare_file
try: shutil.rmtree("{}/{}".format(bench_dir, compare_file))
except: pass
shutil.copytree(source_file, "{}/{}".format(bench_dir, compare_file))
with open("{}.status".format(test.name), 'w') as cf:
cf.write("benchmarks updated. New file: {}\n".format(compare_file) )
else:
with open("{}.status".format(test.name), 'w') as cf:
cf.write("benchmarks failed")
# copy what we can
shutil.copy("{}.run.out".format(test.name), suite.full_web_dir)
if os.path.isfile("{}.err.out".format(test.name)):
shutil.copy("{}.err.out".format(test.name), suite.full_web_dir)
test.has_stderr = True
suite.copy_backtrace(test)
error_msg = "ERROR: runtime failure during benchmark creation"
report.report_single_test(suite, test, test_list, failure_msg=error_msg)
if not test.diffDir == "":
diff_dir_bench = "{}/{}_{}".format(bench_dir, test.name, test.diffDir)
if os.path.isdir(diff_dir_bench):
shutil.rmtree(diff_dir_bench)
shutil.copytree(test.diffDir, diff_dir_bench)
else:
shutil.copy(test.diffDir, diff_dir_bench)
suite.log.log("new diffDir: {}_{}".format(test.name, test.diffDir))
else: # selfTest
if args.make_benchmarks is None:
suite.log.log("looking for selfTest success string: {} ...".format(test.stSuccessString))
try: of = open("{}.run.out".format(test.name), 'r')
except IOError:
suite.log.warn("no output file found")
out_lines = ['']
else:
out_lines = of.readlines()
# successful comparison is indicated by presence
# of success string
for line in out_lines:
if line.find(test.stSuccessString) >= 0:
test.compare_successful = True
break
of.close()
with open("{}.compare.out".format(test.name), 'w') as cf:
if test.compare_successful:
cf.write("SELF TEST SUCCESSFUL\n")
else:
cf.write("SELF TEST FAILED\n")
#----------------------------------------------------------------------
# do any requested visualization (2- and 3-d only) and analysis
#----------------------------------------------------------------------
if not test.selfTest:
if output_file != "":
if args.make_benchmarks is None:
# get any parameters for the summary table
job_info_file = "{}/job_info".format(output_file)
if os.path.isfile(job_info_file):
test.has_jobinfo = 1
try: jif = open(job_info_file, "r")
except:
suite.log.warn("unable to open the job_info file")
else:
job_file_lines = jif.readlines()
if suite.summary_job_info_field1 is not "":
for l in job_file_lines:
if l.find(suite.summary_job_info_field1) >= 0 and l.find(":") >= 0:
_tmp = l.split(":")[1]
idx = _tmp.rfind("/") + 1
test.job_info_field1 = _tmp[idx:]
break
if suite.summary_job_info_field2 is not "":
for l in job_file_lines:
if l.find(suite.summary_job_info_field2) >= 0 and l.find(":") >= 0:
_tmp = l.split(":")[1]
idx = _tmp.rfind("/") + 1
test.job_info_field2 = _tmp[idx:]
break
if suite.summary_job_info_field3 is not "":
for l in job_file_lines:
if l.find(suite.summary_job_info_field3) >= 0 and l.find(":") >= 0:
_tmp = l.split(":")[1]
idx = _tmp.rfind("/") + 1
test.job_info_field3 = _tmp[idx:]
break
# visualization
if test.doVis:
if test.dim == 1:
suite.log.log("Visualization not supported for dim = {}".format(test.dim))
else:
suite.log.log("doing the visualization...")
tool = suite.tools["fsnapshot{}d".format(test.dim)]
test_util.run('{} --palette {}/Palette -cname "{}" -p "{}"'.format(
tool, suite.compare_tool_dir, test.visVar, output_file))
# convert the .ppm files into .png files
ppm_file = test_util.get_recent_filename(output_dir, "", ".ppm")
if not ppm_file is None:
png_file = ppm_file.replace(".ppm", ".png")
test_util.run("convert {} {}".format(ppm_file, png_file))
test.png_file = png_file
# analysis
if not test.analysisRoutine == "":
suite.log.log("doing the analysis...")
if not test.extra_build_dir == "":
tool = "{}/{}".format(suite.repos[test.extra_build_dir].dir, test.analysisRoutine)
else:
tool = "{}/{}".format(suite.source_dir, test.analysisRoutine)
shutil.copy(tool, os.getcwd())
option = eval("suite.{}".format(test.analysisMainArgs))
test_util.run("{} {} {}".format(os.path.basename(test.analysisRoutine),
option, output_file))
else:
if test.doVis or test.analysisRoutine != "":
suite.log.warn("no output file. Skipping visualization")
#----------------------------------------------------------------------
# move the output files into the web directory
#----------------------------------------------------------------------
if args.make_benchmarks is None:
shutil.copy("{}.run.out".format(test.name), suite.full_web_dir)
if os.path.isfile("{}.err.out".format(test.name)):
shutil.copy("{}.err.out".format(test.name), suite.full_web_dir)
test.has_stderr = True
shutil.copy("{}.compare.out".format(test.name), suite.full_web_dir)
shutil.copy(test.inputFile, "{}/{}.{}".format(
suite.full_web_dir, test.name, test.inputFile) )
if test.has_jobinfo:
shutil.copy(job_info_file, "{}/{}.job_info".format(
suite.full_web_dir, test.name))
if suite.sourceTree == "C_Src" and test.probinFile != "":
shutil.copy(test.probinFile, "{}/{}.{}".format(
suite.full_web_dir, test.name, test.probinFile) )
for af in test.auxFiles:
# strip out any sub-directory under build dir for the aux file
# when copying
shutil.copy(os.path.basename(af),
"{}/{}.{}".format(suite.full_web_dir,
test.name, os.path.basename(af)) )
if not test.png_file is None:
try: shutil.copy(test.png_file, suite.full_web_dir)
except IOError:
# visualization was not successful. Reset image
test.png_file = None
if not test.analysisRoutine == "":
try: shutil.copy(test.analysisOutputImage, suite.full_web_dir)
except IOError:
# analysis was not successful. Reset the output image
test.analysisOutputImage = ""
# were any Backtrace files output (indicating a crash)
suite.copy_backtrace(test)
else:
shutil.copy("{}.status".format(test.name), suite.full_web_dir)
#----------------------------------------------------------------------
# archive (or delete) the output
#----------------------------------------------------------------------
suite.log.log("archiving the output...")
for pfile in os.listdir(output_dir):
if (os.path.isdir(pfile) and
(pfile.startswith("{}_plt".format(test.name)) or
pfile.startswith("{}_chk".format(test.name)) ) ):
if suite.purge_output == 1 and not pfile == output_file:
# delete the plt/chk file
if os.path.isdir(pfile):
try: shutil.rmtree(pfile)
except:
suite.log.warn("unable to remove {}".format(pfile))
else:
# tar it up
try:
tar = tarfile.open("{}.tgz".format(pfile), "w:gz")
tar.add("{}".format(pfile))
tar.close()
except:
suite.log.warn("unable to tar output file {}".format(pfile))
else:
try: shutil.rmtree(pfile)
except OSError:
suite.log.warn("unable to remove {}".format(pfile))
#----------------------------------------------------------------------
# write the report for this test
#----------------------------------------------------------------------
if args.make_benchmarks is None:
suite.log.log("creating problem test report ...")
report.report_single_test(suite, test, test_list)
#--------------------------------------------------------------------------
# write the report for this instance of the test suite
#--------------------------------------------------------------------------
suite.log.outdent()
suite.log.skip()
suite.log.bold("creating new test report...")
num_failed = report.report_this_test_run(suite, args.make_benchmarks, args.note,
update_time,
test_list, args.input_file[0])
# make sure that all of the files in the web directory are world readable
for file in os.listdir(suite.full_web_dir):
current_file = suite.full_web_dir + file
if os.path.isfile(current_file):
os.chmod(current_file, 0o644)
# reset the branch to what it was originally
suite.log.skip()
suite.log.bold("reverting git branches/hashes")
suite.log.indent()
for k in suite.repos:
if suite.repos[k].update or suite.repos[k].hash_wanted:
suite.repos[k].git_back()
suite.log.outdent()
# For temporary run, return now without creating suote report.
if args.do_temp_run:
return num_failed
# store an output file in the web directory that can be parsed easily by
# external program
name = "source"
if suite.sourceTree == "BoxLib": name = "BoxLib"
branch = suite.repos[name].branch_wanted.strip("\"")
with open("{}/suite.{}.status".format(suite.webTopDir, branch), "w") as f:
f.write("{}; num failed: {}; source hash: {}".format(
suite.repos[name].name, num_failed, suite.repos[name].hash_current))
#--------------------------------------------------------------------------
# generate the master report for all test instances
#--------------------------------------------------------------------------
suite.log.skip()
suite.log.bold("creating suite report...")
report.report_all_runs(suite, active_test_list)
def email_developers():
msg = email.message_from_string(suite.emailBody)
msg['From'] = suite.emailFrom
msg['To'] = ",".join(suite.emailTo)
msg['Subject'] = suite.emailSubject
server = smtplib.SMTP('localhost')
server.sendmail(suite.emailFrom, suite.emailTo, msg.as_string())
server.quit()
if num_failed > 0 and suite.sendEmailWhenFail and not args.send_no_email:
suite.log.skip()
suite.log.bold("sending email...")
email_developers()
if suite.slack_post:
suite.slack_post_it("test complete, num failed = {}\n{}".format(num_failed, suite.emailBody))
return num_failed
def test_nss_rsa_rc4_128(self):
# Test NSS cipher parsing
(out, err, rc) = run([exe, "+rsa_rc4_128_md5,+rsa_rc4_128_sha"])
assert rc == 0
assert_equal(out, 'rsa_rc4_128_md5, rsa_rc4_128_sha')
def test_nss_subtraction(self):
(out, err, rc) = run([exe, "+rsa_rc4_128_md5,+rsa_rc4_128_sha,-rsa_rc4_128_md5"])
assert rc == 0
assert_equal(out, 'rsa_rc4_128_sha')
def test_openssl_cipher(self):
(out, err, rc) = run([exe, "DES-CBC3-SHA"])
assert rc == 0
assert_equal(out, 'rsa_3des_sha')
def test_openssl_cipherlist(self):
(out, err, rc) = run([exe, "DES-CBC3-SHA:RC4-SHA"])
assert rc == 0
assert_equal(out, 'rsa_rc4_128_sha, rsa_3des_sha')
def test_nss_unknown(self):
(out, err, rc) = run([exe, "+rsa_rc4_128_md5,+unknown"])
assert rc == 0
assert_equal(out, 'rsa_rc4_128_md5')
def test_invalid_format(self):
(out, err, rc) = run([exe, "none"])
assert rc == 1
def setUpClass(cls):
(out, err, rc) = run([exe, "--count"])
assert rc == 0
cls.ciphernum = int(out)
def test_EDH(self):
# No DH ciphers supported yet
(out, err, rc) = run([exe, "EDH"])
assert rc == 1
def test_nss_single(self):
(out, err, rc) = run([exe, "+aes_128_sha_256"])
assert rc == 0
assert_equal(out, 'aes_128_sha_256')
