def check_gauges(self, save=False, gauge_id=1, indices=[0],
rtol=1e-14, atol=1e-8, tolerance=None):
r"""Basic test to assert gauge equality
:Input:
- *save* (bool) - If *True* will save the output from this test to
the file *regresion_data.txt*. Default is *False*.
- *indices* (tuple) - Contains indices to compare in the gague
comparison. Defaults to *(0)*.
- *rtol* (float) - Relative tolerance used in the comparison, default
is *1e-14*. Note that the old *tolerance* input is now synonymous
with this parameter.
- *atol* (float) - Absolute tolerance used in the comparison, default
is *1e-08*.
"""
if isinstance(tolerance, float):
rtol = tolerance
if not(isinstance(indices, tuple) or isinstance(indices, list)):
indices = tuple(indices)
# Get gauge data
gauge = gauges.GaugeSolution(gauge_id, path=self.temp_path)
# Get regression comparison data
regression_data_path = os.path.join(self.test_path, "regression_data")
if save:
gauge_file_name = "gauge%s.txt" % str(gauge_id).zfill(5)
shutil.copy(os.path.join(self.temp_path, gauge_file_name),
regression_data_path)
claw_git_status.make_git_status_file(outdir=regression_data_path)
regression_gauge = gauges.GaugeSolution(gauge_id,
path=regression_data_path)
# Compare data
try:
for n in indices:
numpy.testing.assert_allclose(gauge.q[n, :],
regression_gauge.q[n, :],
rtol=rtol, atol=atol,
verbose=False)
except AssertionError as e:
err_msg = "\n".join((e.args[0],
"Gauge Match Failed for gauge = %s" % gauge_id))
err_msg = "\n".join((err_msg, " failures in fields:"))
failure_indices = []
for n in indices:
if ~numpy.allclose(gauge.q[n, :], regression_gauge.q[n, :],
rtol=rtol, atol=atol):
failure_indices.append(str(n))
index_str = ", ".join(failure_indices)
raise AssertionError(" ".join((err_msg, index_str)))
def check_gauges(self, save=False, gauge_id=1, indices=[0],
rtol=1e-14, atol=1e-8, tolerance=None):
r"""Basic test to assert gauge equality
:Input:
- *save* (bool) - If *True* will save the output from this test to
the file *regresion_data.txt*. Default is *False*.
- *indices* (tuple) - Contains indices to compare in the gague
comparison. Defaults to *(0)*.
- *rtol* (float) - Relative tolerance used in the comparison, default
is *1e-14*. Note that the old *tolerance* input is now synonymous
with this parameter.
- *atol* (float) - Absolute tolerance used in the comparison, default
is *1e-08*.
"""
if isinstance(tolerance, float):
rtol = tolerance
if not(isinstance(indices, tuple) or isinstance(indices, list)):
indices = tuple(indices)
# Get gauge data
gauge = gauges.GaugeSolution(gauge_id, path=self.temp_path)
# Get regression comparison data
regression_data_path = os.path.join(self.test_path, "regression_data")
if save:
gauge_file_name = "gauge%s.txt" % str(gauge_id).zfill(5)
shutil.copy(os.path.join(self.temp_path, gauge_file_name),
regression_data_path)
claw_git_status.make_git_status_file(outdir=regression_data_path)
regression_gauge = gauges.GaugeSolution(gauge_id,
path=regression_data_path)
# Compare data
try:
for n in indices:
numpy.testing.assert_allclose(gauge.q[n, :],
regression_gauge.q[n, :],
rtol=rtol, atol=atol,
verbose=False)
except AssertionError as e:
err_msg = "\n".join((e.args[0],
"Gauge Match Failed for gauge = %s" % gauge_id))
err_msg = "\n".join((err_msg, " failures in fields:"))
failure_indices = []
for n in indices:
if ~numpy.allclose(gauge.q[n, :], regression_gauge.q[n, :],
rtol=rtol, atol=atol):
failure_indices.append(str(n))
index_str = ", ".join(failure_indices)
raise AssertionError(" ".join((err_msg, index_str)))
def check_frame(
self,
save=False,
indices=[0],
frame_num=1,
file_name="regression_data.txt",
rtol=1e-14,
atol=1e-08,
tolerance=None,
):
r"""Compare choosen frame to the comparison data
:Input:
- *save* (bool) - If *True* will save the output from this test to
the file *regresion_data.txt*. Default is *False*.
- *indices* (tuple) - Contains indices to compare in the gague
comparison. Defaults to *(0)*.
- *frame_num* (int) - Frame number to load from the run data. Defaults
to *1*.
- *regression_data_path* (path) - Path to the regression test data.
Defaults to 'regression_data.txt'.
- *rtol* (float) - Relative tolerance used in the comparison, default
is *1e-14*. Note that the old *tolerance* input is now synonymous
with this parameter.
- *atol* (float) - Absolute tolerance used in the comparison, default
is *1e-08*.
"""
if isinstance(tolerance, float):
rtol = tolerance
# Load test data
data = solution.Solution(frame_num, path=self.temp_path)
data_sum = []
for index in indices:
data_sum.append(data.q[index, ...].sum())
# Get (and save) regression comparison data
regression_data_file = os.path.join(self.test_path, "regression_data", file_name)
if save:
numpy.savetxt(regression_data_file, data_sum)
claw_git_status.make_git_status_file(outdir=os.path.join(self.test_path, "regression_data"))
regression_sum = numpy.loadtxt(regression_data_file)
assert numpy.allclose(data_sum, regression_sum, rtol=rtol, atol=atol), "\n new_data: %s, \n expected: %s" % (
data_sum,
regression_sum,
)
def check_frame(self, save=False, indices=[0], frame_num=1,
file_name="regression_data.txt",
rtol=1e-14, atol=1e-08, tolerance=None):
r"""Compare choosen frame to the comparison data
:Input:
- *save* (bool) - If *True* will save the output from this test to
the file *regresion_data.txt*. Default is *False*.
- *indices* (tuple) - Contains indices to compare in the gague
comparison. Defaults to *(0)*.
- *frame_num* (int) - Frame number to load from the run data. Defaults
to *1*.
- *regression_data_path* (path) - Path to the regression test data.
Defaults to 'regression_data.txt'.
- *rtol* (float) - Relative tolerance used in the comparison, default
is *1e-14*. Note that the old *tolerance* input is now synonymous
with this parameter.
- *atol* (float) - Absolute tolerance used in the comparison, default
is *1e-08*.
"""
if isinstance(tolerance, float):
rtol = tolerance
# Load test data
data = solution.Solution(frame_num, path=self.temp_path)
data_sum = []
for index in indices:
data_sum.append(data.q[index, ...].sum())
# Get (and save) regression comparison data
regression_data_file = os.path.join(self.test_path, "regression_data",
file_name)
if save:
numpy.savetxt(regression_data_file, data_sum)
claw_git_status.make_git_status_file(
outdir=os.path.join(self.test_path, "regression_data"))
regression_sum = numpy.loadtxt(regression_data_file)
assert numpy.allclose(data_sum, regression_sum, rtol=rtol, atol=atol), \
"\n new_data: %s, \n expected: %s" % (data_sum, regression_sum)
def runclaw(xclawcmd=None, outdir=None, overwrite=True, restart=None,
rundir=None, print_git_status=False, nohup=False, nice=None,
xclawout=None, xclawerr=None, verbose=True):
"""
Run the Fortran version of Clawpack using executable xclawcmd, which is
typically set to 'xclaw', 'xamr', etc.
If it is not set by the call, get it from the environment variable
CLAW_EXE. Default to 'xclaw' if that's not set.
If overwrite is True, it's ok to zap current contents of the outdir.
If overwrite is False, move the outdir (or copy in the case of a restart)
to a backup directory with a unique name based on time executed.
If restart is None, determine whether this is a restart from claw.data
(as set in setrun.py). Can remove setting RESTART in Makefiles.
If rundir is None, all *.data is copied from current directory, if a path
is given, data files are copied from there instead.
If print_git_status is True, print a summary of the git status of all
clawpack repositories in the file claw_git_status.txt in outdir.
If nohup is True, run the xclawcmd in nohup mode: runs in background and
output goes into nohup.out file in the directory specified by outdir,
rather than all output going to the screen. The job keeps running if user
logs off. Useful for scripts starting long jobs in batch mode.
If type(nice) is int, runs the code using "nice -n "
with this nice value so it doesn't hog computer resources.
xclawout and xclawerr define the locations of stdout and stderr for the
execution of CLAW_EXE. They should be strings to filepaths or open file
objects or ``subprocess.PIPE`` or ``subprocess.STDOUT``. To pass both
to the same file, specify ``xclawout`` as the filepath and
``xclawerr=subprocess.STDOUT``.
"""
if nice is not None:
try:
nice = int(nice)
except:
nice = None
# convert strings passed in from Makefile to boolean:
if type(overwrite) is str:
overwrite = (overwrite.lower() in ['true','t'])
if type(restart) is str:
if restart == 'None':
restart = None
else:
restart = (restart.lower() in ['true','t'])
if type(print_git_status) is str:
print_git_status = (print_git_status.lower() in ['true','t'])
if type(nohup) is str:
nohup = (nohup.lower() in ['true','t'])
if xclawcmd is None:
# Determine what executable to use from environment variable CLAW_EXE
# Default to 'xclaw' if it's not set:
xclawcmd = os.environ.get('CLAW_EXE', 'xclaw')
if outdir is None:
outdir = '.'
if rundir is None:
rundir = os.getcwd()
rundir = os.path.abspath(rundir)
print("==> runclaw: Will take data from ", rundir)
# directory for fort.* files:
outdir = os.path.abspath(outdir)
print('==> runclaw: Will write output to ',outdir)
if restart is None:
# Added option to determine restart from claw.data (i.e. setrun.py)
clawdata = ClawData()
clawdata.read(os.path.join(rundir,'claw.data'), force=True)
restart = clawdata.restart
xclawcmd = os.path.abspath(xclawcmd)
if os.path.isfile(outdir):
print("==> runclaw: Error: outdir specified is a file")
return
if (os.path.isdir(outdir) & (not overwrite)):
# copy the old outdir before possibly overwriting
tm = time.localtime(os.path.getmtime(outdir))
year = str(tm[0]).zfill(4)
month = str(tm[1]).zfill(2)
day = str(tm[2]).zfill(2)
hour = str(tm[3]).zfill(2)
minute = str(tm[4]).zfill(2)
second = str(tm[5]).zfill(2)
outdir_backup = outdir + '_%s-%s-%s-%s%s%s' \
% (year,month,day,hour,minute,second)
if verbose:
print("==> runclaw: Directory already exists: ",os.path.split(outdir)[1])
if restart:
print("==> runclaw: Copying directory to: ",os.path.split(outdir_backup)[1])
else:
print("==> runclaw: Moving directory to: ",os.path.split(outdir_backup)[1])
time.sleep(1)
try:
shutil.move(outdir,outdir_backup)
if restart:
shutil.copytree(outdir_backup,outdir)
except:
print("==> runclaw: Could not move directory... copy already exists?")
if six.PY2:
if (not os.path.isdir(outdir)):
try:
os.mkdir(outdir)
except:
print("Cannot make directory ",outdir)
return
else:
os.makedirs(outdir, exist_ok=True)
if print_git_status not in [False,'False']:
# create files claw_git_status.txt and claw_git_diffs.txt in
# outdir:
make_git_status_file(outdir=outdir)
# old fort.* files to be removed for new run?
fortfiles = glob.glob(os.path.join(outdir,'fort.*'))
# also need to remove gauge*.txt output files now that the gauge
# output is no longer in fort.gauge (but don't remove new gauges.data)
gaugefiles = glob.glob(os.path.join(outdir,'gauge*.txt'))
if (overwrite and (not restart)):
# remove any old versions:
if verbose:
print("==> runclaw: Removing all old fort/gauge files in ", outdir)
for file in fortfiles + gaugefiles:
os.remove(file)
elif restart:
if verbose:
print("==> runclaw: Restart: leaving original fort/gauge files in ", outdir)
else:
# this should never be reached:
# if overwrite==False then outdir has already been moved
if len(fortfiles+gaugefiles) > 1:
print("==> runclaw: *** Remove fort.* and gauge*.txt")
print(" from output directory %s and try again," % outdir)
print(" or use overwrite=True in call to runclaw")
print(" e.g., by setting OVERWRITE = True in Makefile")
return
datafiles = glob.glob(os.path.join(rundir,'*.data'))
if datafiles == ():
print("==> runclaw: Warning: no data files found in directory ",rundir)
else:
if rundir != outdir:
for file in datafiles:
shutil.copy(file, os.path.join(outdir,os.path.basename(file)))
b4run = None
if os.path.isfile('b4run.py'):
b4run_file = os.path.abspath('b4run.py')
else:
b4run_file = os.environ.get('B4RUN', '')
if os.path.isfile(b4run_file):
try:
file_globals = runpy.run_path(b4run_file)
b4run = file_globals.get('b4run', None)
except:
w = r"*** WARNING: problem running b4run_file = %s" % b4run_file
warnings.warn(w, UserWarning)
b4run = None
if b4run is not None:
try:
b4run(rundir, outdir)
print('Executed b4run function from %s' % b4run_file)
except:
w = r"*** WARNING: problem executing b4run from %s" % b4run_file
warnings.warn(w, UserWarning)
# execute command to run fortran program:
if nohup:
# run in nohup mode:
print("\n==> Running in nohup mode, output will be sent to:")
print(" %s/nohup.out" % outdir)
if type(nice) is int:
cmd = "nohup time nice -n %s %s " % (nice,xclawcmd)
else:
cmd = "nohup time %s " % xclawcmd
print("\n==> Running with command:\n ", cmd)
else:
if type(nice) is int:
cmd = "nice -n %s %s " % (nice,xclawcmd)
else:
cmd = xclawcmd
print("\n==> Running with command:\n ", cmd)
cmd_split = shlex.split(cmd)
if isinstance(xclawout, str):
xclawout = open(xclawout,'w', encoding='utf-8',
buffering=1)
if isinstance(xclawerr, str):
xclawerr = open(xclawerr,'w', encoding='utf-8',
buffering=1)
try:
proc = subprocess.check_call(cmd_split,
cwd=outdir,
stdout=xclawout,
stderr=xclawerr)
except subprocess.CalledProcessError as cpe:
exe_error_str = "\n\n*** FORTRAN EXE FAILED ***\n"
raise ClawExeError(exe_error_str, cpe.returncode, cpe.cmd,
output=cpe.output,
stderr=cpe.stderr)
print('==> runclaw: Done executing %s via clawutil.runclaw.py' %\
xclawcmd)
print('==> runclaw: Output is in ', outdir)
return proc
def runclaw(xclawcmd=None, outdir=None, overwrite=True, restart=None,
rundir=None, print_git_status=False, nohup=False, nice=None):
"""
Run the Fortran version of Clawpack using executable xclawcmd, which is
typically set to 'xclaw', 'xamr', etc.
If it is not set by the call, get it from the environment variable
CLAW_EXE. Default to 'xclaw' if that's not set.
If overwrite is True, it's ok to zap current contents of the outdir.
If overwrite is False, move the outdir (or copy in the case of a restart)
to a backup directory with a unique name based on time executed.
If restart is None, determine whether this is a restart from claw.data
(as set in setrun.py). Can remove setting RESTART in Makefiles.
If rundir is None, all *.data is copied from current directory, if a path
is given, data files are copied from there instead.
If print_git_status is True, print a summary of the git status of all
clawpack repositories in the file claw_git_status.txt in outdir.
If nohup is True, run the xclawcmd in nohup mode: runs in background and
output goes into nohup.out file in the directory specified by outdir,
rather than all output going to the screen. The job keeps running if user
logs off. Useful for scripts starting long jobs in batch mode.
If type(nice) is int, runs the code using "nice -n "
with this nice value so it doesn't hog computer resources.
"""
from clawpack.clawutil.data import ClawData
import os,glob,shutil,time
verbose = True
xclawout = None
xclawerr = None
try:
nice = int(nice)
except:
nice = None
# convert strings passed in from Makefile to boolean:
if type(overwrite) is str:
overwrite = (overwrite.lower() in ['true','t'])
if type(restart) is str:
if restart == 'None':
restart = None
else:
restart = (restart.lower() in ['true','t'])
if type(print_git_status) is str:
print_git_status = (print_git_status.lower() in ['true','t'])
if type(nohup) is str:
nohup = (nohup.lower() in ['true','t'])
if xclawcmd is None:
# Determine what executable to use from environment variable CLAW_EXE
# Default to 'xclaw' if it's not set:
xclawcmd = os.environ.get('CLAW_EXE', 'xclaw')
if outdir is None:
outdir = '.'
if rundir is None:
rundir = os.getcwd()
rundir = os.path.abspath(rundir)
print("==> runclaw: Will take data from ", rundir)
# directory for fort.* files:
outdir = os.path.abspath(outdir)
print('==> runclaw: Will write output to ',outdir)
if restart is None:
# Added option to determine restart from claw.data (i.e. setrun.py)
clawdata = ClawData()
clawdata.read(os.path.join(rundir,'claw.data'), force=True)
restart = clawdata.restart
#print '+++ From claw.data determined restart = %s' % restart
#returncode = clawjob.runxclaw()
if 1:
startdir = os.getcwd()
xdir = os.path.abspath(startdir)
outdir = os.path.abspath(outdir)
rundir = os.path.abspath(rundir)
xclawcmd = os.path.join(xdir,xclawcmd)
try:
os.chdir(xdir)
except:
raise Exception( "==> runclaw: Cannot change to directory xdir = %s" %xdir)
return
if os.path.isfile(outdir):
print("==> runclaw: Error: outdir specified is a file")
return
if (os.path.isdir(outdir) & (not overwrite)):
# copy the old outdir before possibly overwriting
tm = time.localtime(os.path.getmtime(outdir))
year = str(tm[0]).zfill(4)
month = str(tm[1]).zfill(2)
day = str(tm[2]).zfill(2)
hour = str(tm[3]).zfill(2)
minute = str(tm[4]).zfill(2)
second = str(tm[5]).zfill(2)
outdir_backup = outdir + '_%s-%s-%s-%s%s%s' \
% (year,month,day,hour,minute,second)
if verbose:
print("==> runclaw: Directory already exists: ",os.path.split(outdir)[1])
if restart:
print("==> runclaw: Copying directory to: ",os.path.split(outdir_backup)[1])
else:
print("==> runclaw: Moving directory to: ",os.path.split(outdir_backup)[1])
time.sleep(1)
try:
shutil.move(outdir,outdir_backup)
if restart:
shutil.copytree(outdir_backup,outdir)
except:
print("==> runclaw: Could not move directory... copy already exists?")
if (not os.path.isdir(outdir)):
try:
os.mkdir(outdir)
except:
print("Cannot make directory ",outdir)
return
try:
os.chdir(outdir)
except:
print('==> runclaw: *** Error in runxclaw: cannot move to outdir = ',\
outdir)
raise
return
if print_git_status not in [False,'False']:
make_git_status_file()
fortfiles = glob.glob(os.path.join(outdir,'fort.*'))
if (overwrite and (not restart)):
# remove any old versions:
if verbose:
print("==> runclaw: Removing all old fort files in ", outdir)
for file in fortfiles:
os.remove(file)
elif restart:
if verbose:
print("==> runclaw: Restart: leaving original fort files in ", outdir)
else:
if len(fortfiles) > 1:
print("==> runclaw: *** Remove fort.* and try again,")
print(" or use overwrite=True in call to runxclaw")
print(" e.g., by setting CLAW_OVERWRITE = True in Makefile")
return
try:
os.chdir(rundir)
except:
raise Exception("Cannot change to directory %s" % rundir)
return
datafiles = glob.glob('*.data')
if datafiles == ():
print("==> runclaw: Warning: no data files found in directory ",rundir)
else:
if rundir != outdir:
for file in datafiles:
shutil.copy(file,os.path.join(outdir,file))
if xclawout:
xclawout = open(xclawout,'wb')
if xclawerr:
xclawerr = open(xclawerr,'wb')
os.chdir(outdir)
#print "\nIn directory outdir = ",outdir,"\n"
# execute command to run fortran program:
try:
#print "\nExecuting ",xclawcmd, " ... "
#pclaw = subprocess.Popen(xclawcmd,stdout=xclawout,stderr=xclawerr)
#print '+++ pclaw started'
#pclaw.wait() # wait for code to run
#returncode = pclaw.returncode
#print '+++ pclaw done'
if nohup:
# run in nohup mode:
print("\n==> Running in nohup mode, output will be sent to:")
print(" %s/nohup.out" % outdir)
if type(nice) is int:
cmd = "nohup time nice -n %s %s " % (nice,xclawcmd)
else:
cmd = "nohup time %s " % xclawcmd
print("\n==> Running with command:\n ", cmd)
returncode = os.system(cmd)
else:
if type(nice) is int:
cmd = "nice -n %s %s " % (nice,xclawcmd)
else:
cmd = xclawcmd
print("\n==> Running with command:\n ", cmd)
returncode = os.system(cmd)
if returncode == 0:
print("\n==> runclaw: Finished executing\n")
else:
print("\n ==> runclaw: *** Runtime error: return code = %s\n "\
% returncode)
print('==> runclaw: Done executing %s via clawutil.runclaw.py' %\
xclawcmd)
print('==> runclaw: Output is in ', outdir)
except:
raise Exception("Could not execute command %s" % xclawcmd)
os.chdir(startdir)
if returncode != 0:
print('==> runclaw: *** fortran returncode = ', returncode, ' aborting')