def test_subdirs(compare_dir=None,examples_dir='.',\
verbose=True,relocatable=False):
from make_all import list_examples
if compare_dir is None:
print "Attempting to find gallery for comparison..."
try:
CLAW = os.environ['CLAW']
except:
raise Exception("Need to set CLAW environment variable")
examples_dir = os.path.abspath(examples_dir)
iclaw = examples_dir.find(CLAW)
if iclaw==-1:
print "*** examples_dir is not under $CLAW, failed to find gallery"
return False
gallery_home = os.path.join(CLAW,'clawpack.github.com','doc','_static')
package = os.path.split(os.path.split(examples_dir)[0])[1]
compare_dir = os.path.join(gallery_home,package,'examples')
if not os.path.isdir(compare_dir):
print "*** Failed to find ",compare_dir
return False
all_ok = True
dir_list = list_examples(examples_dir)
print "Found the following example subdirectories:"
for d in dir_list:
print " ", d
print "Will run imagediff on _plots in the above subdirectories of "
print " ", examples_dir
print "Will compare to _plots in corresponding subdirectories of: \n %s" \
% compare_dir
if 0:
ans = raw_input("Ok? ")
if ans.lower() not in ['y','yes']:
print "Aborting."
all_ok = False
return all_ok
top_dir = os.getcwd()
for test_subdir in dir_list:
print "\n============================================================="
print test_subdir
print "============================================================="
#subdir = os.path.split(test_subdir)[1]
subdir = test_subdir[len(examples_dir)+1:]
compare_subdir = os.path.join(compare_dir, subdir)
test_plots = os.path.join(test_subdir,'_plots')
compare_plots = os.path.join(compare_subdir, '_plots')
if not os.path.isdir(test_plots):
print "*** Cannot find _plots directory "
print "*** Looking for ",test_plots
all_ok = False
continue
if not os.path.isdir(compare_plots):
print "*** Cannot find _plots directory to compare against"
print "*** Looking for ",compare_plots
all_ok = False
continue
os.chdir(subdir)
try:
regression_ok = imagediff.imagediff_dir(test_plots,compare_plots, \
relocatable=relocatable,overwrite=True, \
verbose=verbose)
if not regression_ok:
print "*** Regression files are not all identical in "
print " ",test_subdir
except:
print "*** Error running imagediff with directories \n %s\n %s\n" \
% (test_plots,compare_plots)
regression_ok = False
all_ok = all_ok and regression_ok
os.chdir(top_dir)
return all_ok
def compare_regression_tests(regression_dir="_regression_tests", \
regression_output_files = 'all', \
regression_plot_files = 'all', \
relocatable=False, verbose=False):
try:
archived_dir, tar_file = fetch_regression_data(regression_dir)
archived_dir = os.path.abspath(archived_dir)
os.system("rm %s" % tar_file)
except:
print "*** Error retrieving regression results"
sys.exit()
try:
os.chdir(regression_dir)
except:
print "*** Cannot cd to ", regression_dir
sys.exit()
index_file = 'index.html'
hfile = open(index_file, 'w')
index_file = os.path.abspath(index_file)
hfile.write("""<html>
<h1>Regression test results</h1>
<h2>Comparison of output files:</h2>
Pass or Fail based only on files %s
<ul>
""" % regression_output_files)
outdirs = glob.glob('_output*')
if outdirs == []:
print "*** No output directories found"
else:
if glob.glob('_chardiffs*') != []:
ans = raw_input("Ok to clobber all _chardiffs directories? ")
if ans == 'y':
os.system("rm -rf _chardiffs*")
output_ok_dirs = []
for outdir in outdirs:
outdir_archived = archived_dir + "/" + outdir
if not os.path.exists(outdir_archived):
print "*** Cannot find directory ", outdir_archived
print "*** will not compare to ", outdir
else:
diffdir = "_chardiffs" + outdir
regression_ok = chardiff_dir(outdir_archived, outdir, dir3=diffdir, \
regression_test_files=regression_output_files, \
verbose=verbose)
output_ok_dirs.append(regression_ok)
hfile.write('<li> <a href="%s/_DiffIndex.html">%s</a>' %
(diffdir, outdir))
if regression_ok:
hfile.write(' <font color="green">Pass</font>\n')
else:
hfile.write(' <font color="red">Fail</font>\n')
hfile.write("""</ul>
<h2>Comparison of plot files: </h2>
Pass or Fail based only on files %s
<ul>
""" % regression_plot_files)
plot_ok_dirs = []
plotdirs = glob.glob('_plots*')
if plotdirs == []:
print "*** No plots directories found"
else:
if glob.glob('_imagediffs*') != []:
ans = raw_input("Ok to clobber all _imagediffs directories? ")
if ans == 'y':
os.system("rm -rf _imagediffs*")
for plotdir in plotdirs:
plotdir_archived = archived_dir + "/" + plotdir
if not os.path.exists(plotdir_archived):
print "*** Cannot find directory ", plotdir_archived
print "*** will not compare to ", plotdir
else:
diffdir = "_imagediffs" + plotdir
regression_ok = imagediff_dir(plotdir_archived, plotdir, dir3=diffdir, \
regression_test_files=regression_plot_files, \
relocatable=relocatable, verbose=verbose)
plot_ok_dirs.append(regression_ok)
hfile.write('<li> <a href="%s/_ImageDiffIndex.html">%s</a>' %
(diffdir, plotdir))
if regression_ok:
hfile.write(' <font color="green">Pass</font>\n')
else:
hfile.write(' <font color="red">Fail</font>\n')
hfile.write("""</ul>
</html>
""")
hfile.close()
print "\nTo see resulting diffs, open \n %s\n" % index_file
fail_dirs = \
[outdirs[i] for i in range(len(outdirs)) if not output_ok_dirs[i]] + \
[plotdirs[i] for i in range(len(plotdirs)) if not plot_ok_dirs[i]]
regression_ok = (fail_dirs == [])
if not regression_ok:
print "*** Regression files do not match in directories: ", fail_dirs
return regression_ok
def test_subdirs(compare_dir=None,examples_dir='.',\
verbose=True,relocatable=False):
from make_all import list_examples
if compare_dir is None:
print "Attempting to find gallery for comparison..."
try:
CLAW = os.environ['CLAW']
except:
raise Exception("Need to set CLAW environment variable")
examples_dir = os.path.abspath(examples_dir)
iclaw = examples_dir.find(CLAW)
if iclaw == -1:
print "*** examples_dir is not under $CLAW, failed to find gallery"
return False
gallery_home = os.path.join(CLAW, 'clawpack.github.com', 'doc',
'_static')
package = os.path.split(os.path.split(examples_dir)[0])[1]
compare_dir = os.path.join(gallery_home, package, 'examples')
if not os.path.isdir(compare_dir):
print "*** Failed to find ", compare_dir
return False
all_ok = True
dir_list = list_examples(examples_dir)
print "Found the following example subdirectories:"
for d in dir_list:
print " ", d
print "Will run imagediff on _plots in the above subdirectories of "
print " ", examples_dir
print "Will compare to _plots in corresponding subdirectories of: \n %s" \
% compare_dir
if 0:
ans = raw_input("Ok? ")
if ans.lower() not in ['y', 'yes']:
print "Aborting."
all_ok = False
return all_ok
top_dir = os.getcwd()
for test_subdir in dir_list:
print "\n============================================================="
print test_subdir
print "============================================================="
#subdir = os.path.split(test_subdir)[1]
subdir = test_subdir[len(examples_dir) + 1:]
compare_subdir = os.path.join(compare_dir, subdir)
test_plots = os.path.join(test_subdir, '_plots')
compare_plots = os.path.join(compare_subdir, '_plots')
if not os.path.isdir(test_plots):
print "*** Cannot find _plots directory "
print "*** Looking for ", test_plots
all_ok = False
continue
if not os.path.isdir(compare_plots):
print "*** Cannot find _plots directory to compare against"
print "*** Looking for ", compare_plots
all_ok = False
continue
os.chdir(subdir)
try:
regression_ok = imagediff.imagediff_dir(test_plots,compare_plots, \
relocatable=relocatable,overwrite=True, \
verbose=verbose)
if not regression_ok:
print "*** Regression files are not all identical in "
print " ", test_subdir
except:
print "*** Error running imagediff with directories \n %s\n %s\n" \
% (test_plots,compare_plots)
regression_ok = False
all_ok = all_ok and regression_ok
os.chdir(top_dir)
return all_ok
def test(stdout=None, stderr=None):
"""
Redirect output and errors if stdout or stderr passed in.
"""
import sys
if stdout is not None:
sys.stdout = stdout
if stderr is not None:
sys.stderr = stderr
try:
CLAW = os.environ['CLAW']
except:
raise Exception("Environment variable CLAW not set")
# Compare plots to what is in the Clawpack gallery:
gallery_dir = CLAW + "/clawpack.github.com/doc/_static/"
# For testing with gallery built locally, instead use:
# gallery_dir = CLAW + "/doc/doc/gallery/"
this_example = os.path.split(os.getcwd())[-1]
gallery_plots = gallery_dir + "amrclaw/examples/" + this_example + "/_plots"
if not os.path.isdir(gallery_plots):
error_msg = "Missing directory %s\n Need to clone clawpack.github.com\n"\
% gallery_plots
#raise Exception(error_msg)
sys.stderr.write(error_msg)
gallery_found = False
else:
gallery_found = True
# Run the code and create _plots directory:
cmd = "make clean; make .output"
status = subprocess.Popen(cmd, shell=True).wait()
if status != 0:
#raise Exception("Problem running the code, status = %s" % status)
error_msg = "Problem running the code, status = %s" % status
sys.stderr.write(error_msg)
else:
cmd = "make .plots"
status = subprocess.Popen(cmd, shell=True).wait()
if status != 0:
error_msg = "Problem running the code, status = %s" % status
sys.stderr.write(error_msg)
# Compare the resulting plots to the gallery version:
if (status == 0) and gallery_found:
try:
regression_ok = imagediff.imagediff_dir('_plots',gallery_plots, \
relocatable=relocatable,overwrite=True, \
verbose=verbose)
if not regression_ok:
sys.stderr.write("***Regression files are not all identical")
except:
error_msg = "Error running imagediff with directories \n %s\n %s\n" \
% ('_plots',gallery_plots)
#raise Exception(error_msg)
sys.stderr.write(error_msg)
regression_ok = False
else:
regression_ok = False
return regression_ok
def compare_regression_tests(regression_dir="_regression_tests", \
regression_output_files = 'all', \
regression_plot_files = 'all', \
relocatable=False, verbose=False):
try:
archived_dir, tar_file = fetch_regression_data(regression_dir)
archived_dir = os.path.abspath(archived_dir)
os.system("rm %s" % tar_file)
except:
print "*** Error retrieving regression results"
sys.exit()
try:
os.chdir(regression_dir)
except:
print "*** Cannot cd to ",regression_dir
sys.exit()
index_file = 'index.html'
hfile = open(index_file,'w')
index_file = os.path.abspath(index_file)
hfile.write("""<html>
<h1>Regression test results</h1>
<h2>Comparison of output files:</h2>
Pass or Fail based only on files %s
<ul>
""" % regression_output_files)
outdirs = glob.glob('_output*')
if outdirs == []:
print "*** No output directories found"
else:
if glob.glob('_chardiffs*') != []:
ans = raw_input("Ok to clobber all _chardiffs directories? ")
if ans=='y':
os.system("rm -rf _chardiffs*")
output_ok_dirs = []
for outdir in outdirs:
outdir_archived = archived_dir + "/" + outdir
if not os.path.exists(outdir_archived):
print "*** Cannot find directory ",outdir_archived
print "*** will not compare to ",outdir
else:
diffdir = "_chardiffs" + outdir
regression_ok = chardiff_dir(outdir_archived, outdir, dir3=diffdir, \
regression_test_files=regression_output_files, \
verbose=verbose)
output_ok_dirs.append(regression_ok)
hfile.write('<li> <a href="%s/_DiffIndex.html">%s</a>' % (diffdir,outdir))
if regression_ok:
hfile.write(' <font color="green">Pass</font>\n')
else:
hfile.write(' <font color="red">Fail</font>\n')
hfile.write("""</ul>
<h2>Comparison of plot files: </h2>
Pass or Fail based only on files %s
<ul>
""" % regression_plot_files)
plot_ok_dirs = []
plotdirs = glob.glob('_plots*')
if plotdirs == []:
print "*** No plots directories found"
else:
if glob.glob('_imagediffs*') != []:
ans = raw_input("Ok to clobber all _imagediffs directories? ")
if ans=='y':
os.system("rm -rf _imagediffs*")
for plotdir in plotdirs:
plotdir_archived = archived_dir + "/" + plotdir
if not os.path.exists(plotdir_archived):
print "*** Cannot find directory ",plotdir_archived
print "*** will not compare to ",plotdir
else:
diffdir = "_imagediffs" + plotdir
regression_ok = imagediff_dir(plotdir_archived, plotdir, dir3=diffdir, \
regression_test_files=regression_plot_files, \
relocatable=relocatable, verbose=verbose)
plot_ok_dirs.append(regression_ok)
hfile.write('<li> <a href="%s/_ImageDiffIndex.html">%s</a>' % (diffdir,plotdir))
if regression_ok:
hfile.write(' <font color="green">Pass</font>\n')
else:
hfile.write(' <font color="red">Fail</font>\n')
hfile.write("""</ul>
</html>
""")
hfile.close()
print "\nTo see resulting diffs, open \n %s\n" % index_file
fail_dirs = \
[outdirs[i] for i in range(len(outdirs)) if not output_ok_dirs[i]] + \
[plotdirs[i] for i in range(len(plotdirs)) if not plot_ok_dirs[i]]
regression_ok = (fail_dirs == [])
if not regression_ok:
print "*** Regression files do not match in directories: ", fail_dirs
return regression_ok
import os
from clawpack.clawutil.chardiff import chardiff_dir
from clawpack.clawutil.imagediff import imagediff_dir
outdir = '_output_new'
plotdir = '_plots_new'
regression_dir= "/Users/rjl/git/rjleveque/amrclaw_regression_13feb2013/acoustics-example1"
make_commands = """
make new
make .plots OUTDIR=%s PLOTDIR=%s
""" % (outdir,plotdir)
if True:
os.system(make_commands)
regression_outdir = regression_dir + '/_output'
regression_plotdir = regression_dir + '/_plots'
print "\n======= output comparison ======="
chardiff_dir(regression_outdir, outdir, dir3='_regression_output_cdiff')
print "\n======= plots comparison ======="
imagediff_dir(regression_plotdir, plotdir, dir3='_regression_plots_idiff')
