def check_gauges(self, save=False, gauge_id=1, indices=[0], rtol=1e-14, atol=1e-8, tolerance=None): r"""Basic test to assert gauge equality :Input: - *save* (bool) - If *True* will save the output from this test to the file *regresion_data.txt*. Default is *False*. - *indices* (tuple) - Contains indices to compare in the gague comparison. Defaults to *(0)*. - *rtol* (float) - Relative tolerance used in the comparison, default is *1e-14*. Note that the old *tolerance* input is now synonymous with this parameter. - *atol* (float) - Absolute tolerance used in the comparison, default is *1e-08*. """ if isinstance(tolerance, float): rtol = tolerance if not(isinstance(indices, tuple) or isinstance(indices, list)): indices = tuple(indices) # Get gauge data gauge = gauges.GaugeSolution(gauge_id, path=self.temp_path) # Get regression comparison data regression_data_path = os.path.join(self.test_path, "regression_data") if save: gauge_file_name = "gauge%s.txt" % str(gauge_id).zfill(5) shutil.copy(os.path.join(self.temp_path, gauge_file_name), regression_data_path) claw_git_status.make_git_status_file(outdir=regression_data_path) regression_gauge = gauges.GaugeSolution(gauge_id, path=regression_data_path) # Compare data try: for n in indices: numpy.testing.assert_allclose(gauge.q[n, :], regression_gauge.q[n, :], rtol=rtol, atol=atol, verbose=False) except AssertionError as e: err_msg = "\n".join((e.args[0], "Gauge Match Failed for gauge = %s" % gauge_id)) err_msg = "\n".join((err_msg, " failures in fields:")) failure_indices = [] for n in indices: if ~numpy.allclose(gauge.q[n, :], regression_gauge.q[n, :], rtol=rtol, atol=atol): failure_indices.append(str(n)) index_str = ", ".join(failure_indices) raise AssertionError(" ".join((err_msg, index_str)))
def check_gauges(self, save=False, gauge_id=1, indices=[0], rtol=1e-14, atol=1e-8, tolerance=None): r"""Basic test to assert gauge equality :Input: - *save* (bool) - If *True* will save the output from this test to the file *regresion_data.txt*. Default is *False*. - *indices* (tuple) - Contains indices to compare in the gague comparison. Defaults to *(0)*. - *rtol* (float) - Relative tolerance used in the comparison, default is *1e-14*. Note that the old *tolerance* input is now synonymous with this parameter. - *atol* (float) - Absolute tolerance used in the comparison, default is *1e-08*. """ if isinstance(tolerance, float): rtol = tolerance if not(isinstance(indices, tuple) or isinstance(indices, list)): indices = tuple(indices) # Get gauge data gauge = gauges.GaugeSolution(gauge_id, path=self.temp_path) # Get regression comparison data regression_data_path = os.path.join(self.test_path, "regression_data") if save: gauge_file_name = "gauge%s.txt" % str(gauge_id).zfill(5) shutil.copy(os.path.join(self.temp_path, gauge_file_name), regression_data_path) claw_git_status.make_git_status_file(outdir=regression_data_path) regression_gauge = gauges.GaugeSolution(gauge_id, path=regression_data_path) # Compare data try: for n in indices: numpy.testing.assert_allclose(gauge.q[n, :], regression_gauge.q[n, :], rtol=rtol, atol=atol, verbose=False) except AssertionError as e: err_msg = "\n".join((e.args[0], "Gauge Match Failed for gauge = %s" % gauge_id)) err_msg = "\n".join((err_msg, " failures in fields:")) failure_indices = [] for n in indices: if ~numpy.allclose(gauge.q[n, :], regression_gauge.q[n, :], rtol=rtol, atol=atol): failure_indices.append(str(n)) index_str = ", ".join(failure_indices) raise AssertionError(" ".join((err_msg, index_str)))
def check_frame( self, save=False, indices=[0], frame_num=1, file_name="regression_data.txt", rtol=1e-14, atol=1e-08, tolerance=None, ): r"""Compare choosen frame to the comparison data :Input: - *save* (bool) - If *True* will save the output from this test to the file *regresion_data.txt*. Default is *False*. - *indices* (tuple) - Contains indices to compare in the gague comparison. Defaults to *(0)*. - *frame_num* (int) - Frame number to load from the run data. Defaults to *1*. - *regression_data_path* (path) - Path to the regression test data. Defaults to 'regression_data.txt'. - *rtol* (float) - Relative tolerance used in the comparison, default is *1e-14*. Note that the old *tolerance* input is now synonymous with this parameter. - *atol* (float) - Absolute tolerance used in the comparison, default is *1e-08*. """ if isinstance(tolerance, float): rtol = tolerance # Load test data data = solution.Solution(frame_num, path=self.temp_path) data_sum = [] for index in indices: data_sum.append(data.q[index, ...].sum()) # Get (and save) regression comparison data regression_data_file = os.path.join(self.test_path, "regression_data", file_name) if save: numpy.savetxt(regression_data_file, data_sum) claw_git_status.make_git_status_file(outdir=os.path.join(self.test_path, "regression_data")) regression_sum = numpy.loadtxt(regression_data_file) assert numpy.allclose(data_sum, regression_sum, rtol=rtol, atol=atol), "\n new_data: %s, \n expected: %s" % ( data_sum, regression_sum, )
def check_frame(self, save=False, indices=[0], frame_num=1, file_name="regression_data.txt", rtol=1e-14, atol=1e-08, tolerance=None): r"""Compare choosen frame to the comparison data :Input: - *save* (bool) - If *True* will save the output from this test to the file *regresion_data.txt*. Default is *False*. - *indices* (tuple) - Contains indices to compare in the gague comparison. Defaults to *(0)*. - *frame_num* (int) - Frame number to load from the run data. Defaults to *1*. - *regression_data_path* (path) - Path to the regression test data. Defaults to 'regression_data.txt'. - *rtol* (float) - Relative tolerance used in the comparison, default is *1e-14*. Note that the old *tolerance* input is now synonymous with this parameter. - *atol* (float) - Absolute tolerance used in the comparison, default is *1e-08*. """ if isinstance(tolerance, float): rtol = tolerance # Load test data data = solution.Solution(frame_num, path=self.temp_path) data_sum = [] for index in indices: data_sum.append(data.q[index, ...].sum()) # Get (and save) regression comparison data regression_data_file = os.path.join(self.test_path, "regression_data", file_name) if save: numpy.savetxt(regression_data_file, data_sum) claw_git_status.make_git_status_file( outdir=os.path.join(self.test_path, "regression_data")) regression_sum = numpy.loadtxt(regression_data_file) assert numpy.allclose(data_sum, regression_sum, rtol=rtol, atol=atol), \ "\n new_data: %s, \n expected: %s" % (data_sum, regression_sum)
def runclaw(xclawcmd=None, outdir=None, overwrite=True, restart=None, rundir=None, print_git_status=False, nohup=False, nice=None, xclawout=None, xclawerr=None, verbose=True): """ Run the Fortran version of Clawpack using executable xclawcmd, which is typically set to 'xclaw', 'xamr', etc. If it is not set by the call, get it from the environment variable CLAW_EXE. Default to 'xclaw' if that's not set. If overwrite is True, it's ok to zap current contents of the outdir. If overwrite is False, move the outdir (or copy in the case of a restart) to a backup directory with a unique name based on time executed. If restart is None, determine whether this is a restart from claw.data (as set in setrun.py). Can remove setting RESTART in Makefiles. If rundir is None, all *.data is copied from current directory, if a path is given, data files are copied from there instead. If print_git_status is True, print a summary of the git status of all clawpack repositories in the file claw_git_status.txt in outdir. If nohup is True, run the xclawcmd in nohup mode: runs in background and output goes into nohup.out file in the directory specified by outdir, rather than all output going to the screen. The job keeps running if user logs off. Useful for scripts starting long jobs in batch mode. If type(nice) is int, runs the code using "nice -n " with this nice value so it doesn't hog computer resources. xclawout and xclawerr define the locations of stdout and stderr for the execution of CLAW_EXE. They should be strings to filepaths or open file objects or ``subprocess.PIPE`` or ``subprocess.STDOUT``. To pass both to the same file, specify ``xclawout`` as the filepath and ``xclawerr=subprocess.STDOUT``. """ if nice is not None: try: nice = int(nice) except: nice = None # convert strings passed in from Makefile to boolean: if type(overwrite) is str: overwrite = (overwrite.lower() in ['true','t']) if type(restart) is str: if restart == 'None': restart = None else: restart = (restart.lower() in ['true','t']) if type(print_git_status) is str: print_git_status = (print_git_status.lower() in ['true','t']) if type(nohup) is str: nohup = (nohup.lower() in ['true','t']) if xclawcmd is None: # Determine what executable to use from environment variable CLAW_EXE # Default to 'xclaw' if it's not set: xclawcmd = os.environ.get('CLAW_EXE', 'xclaw') if outdir is None: outdir = '.' if rundir is None: rundir = os.getcwd() rundir = os.path.abspath(rundir) print("==> runclaw: Will take data from ", rundir) # directory for fort.* files: outdir = os.path.abspath(outdir) print('==> runclaw: Will write output to ',outdir) if restart is None: # Added option to determine restart from claw.data (i.e. setrun.py) clawdata = ClawData() clawdata.read(os.path.join(rundir,'claw.data'), force=True) restart = clawdata.restart xclawcmd = os.path.abspath(xclawcmd) if os.path.isfile(outdir): print("==> runclaw: Error: outdir specified is a file") return if (os.path.isdir(outdir) & (not overwrite)): # copy the old outdir before possibly overwriting tm = time.localtime(os.path.getmtime(outdir)) year = str(tm[0]).zfill(4) month = str(tm[1]).zfill(2) day = str(tm[2]).zfill(2) hour = str(tm[3]).zfill(2) minute = str(tm[4]).zfill(2) second = str(tm[5]).zfill(2) outdir_backup = outdir + '_%s-%s-%s-%s%s%s' \ % (year,month,day,hour,minute,second) if verbose: print("==> runclaw: Directory already exists: ",os.path.split(outdir)[1]) if restart: print("==> runclaw: Copying directory to: ",os.path.split(outdir_backup)[1]) else: print("==> runclaw: Moving directory to: ",os.path.split(outdir_backup)[1]) time.sleep(1) try: shutil.move(outdir,outdir_backup) if restart: shutil.copytree(outdir_backup,outdir) except: print("==> runclaw: Could not move directory... copy already exists?") if six.PY2: if (not os.path.isdir(outdir)): try: os.mkdir(outdir) except: print("Cannot make directory ",outdir) return else: os.makedirs(outdir, exist_ok=True) if print_git_status not in [False,'False']: # create files claw_git_status.txt and claw_git_diffs.txt in # outdir: make_git_status_file(outdir=outdir) # old fort.* files to be removed for new run? fortfiles = glob.glob(os.path.join(outdir,'fort.*')) # also need to remove gauge*.txt output files now that the gauge # output is no longer in fort.gauge (but don't remove new gauges.data) gaugefiles = glob.glob(os.path.join(outdir,'gauge*.txt')) if (overwrite and (not restart)): # remove any old versions: if verbose: print("==> runclaw: Removing all old fort/gauge files in ", outdir) for file in fortfiles + gaugefiles: os.remove(file) elif restart: if verbose: print("==> runclaw: Restart: leaving original fort/gauge files in ", outdir) else: # this should never be reached: # if overwrite==False then outdir has already been moved if len(fortfiles+gaugefiles) > 1: print("==> runclaw: *** Remove fort.* and gauge*.txt") print(" from output directory %s and try again," % outdir) print(" or use overwrite=True in call to runclaw") print(" e.g., by setting OVERWRITE = True in Makefile") return datafiles = glob.glob(os.path.join(rundir,'*.data')) if datafiles == (): print("==> runclaw: Warning: no data files found in directory ",rundir) else: if rundir != outdir: for file in datafiles: shutil.copy(file, os.path.join(outdir,os.path.basename(file))) b4run = None if os.path.isfile('b4run.py'): b4run_file = os.path.abspath('b4run.py') else: b4run_file = os.environ.get('B4RUN', '') if os.path.isfile(b4run_file): try: file_globals = runpy.run_path(b4run_file) b4run = file_globals.get('b4run', None) except: w = r"*** WARNING: problem running b4run_file = %s" % b4run_file warnings.warn(w, UserWarning) b4run = None if b4run is not None: try: b4run(rundir, outdir) print('Executed b4run function from %s' % b4run_file) except: w = r"*** WARNING: problem executing b4run from %s" % b4run_file warnings.warn(w, UserWarning) # execute command to run fortran program: if nohup: # run in nohup mode: print("\n==> Running in nohup mode, output will be sent to:") print(" %s/nohup.out" % outdir) if type(nice) is int: cmd = "nohup time nice -n %s %s " % (nice,xclawcmd) else: cmd = "nohup time %s " % xclawcmd print("\n==> Running with command:\n ", cmd) else: if type(nice) is int: cmd = "nice -n %s %s " % (nice,xclawcmd) else: cmd = xclawcmd print("\n==> Running with command:\n ", cmd) cmd_split = shlex.split(cmd) if isinstance(xclawout, str): xclawout = open(xclawout,'w', encoding='utf-8', buffering=1) if isinstance(xclawerr, str): xclawerr = open(xclawerr,'w', encoding='utf-8', buffering=1) try: proc = subprocess.check_call(cmd_split, cwd=outdir, stdout=xclawout, stderr=xclawerr) except subprocess.CalledProcessError as cpe: exe_error_str = "\n\n*** FORTRAN EXE FAILED ***\n" raise ClawExeError(exe_error_str, cpe.returncode, cpe.cmd, output=cpe.output, stderr=cpe.stderr) print('==> runclaw: Done executing %s via clawutil.runclaw.py' %\ xclawcmd) print('==> runclaw: Output is in ', outdir) return proc
def runclaw(xclawcmd=None, outdir=None, overwrite=True, restart=None, rundir=None, print_git_status=False, nohup=False, nice=None): """ Run the Fortran version of Clawpack using executable xclawcmd, which is typically set to 'xclaw', 'xamr', etc. If it is not set by the call, get it from the environment variable CLAW_EXE. Default to 'xclaw' if that's not set. If overwrite is True, it's ok to zap current contents of the outdir. If overwrite is False, move the outdir (or copy in the case of a restart) to a backup directory with a unique name based on time executed. If restart is None, determine whether this is a restart from claw.data (as set in setrun.py). Can remove setting RESTART in Makefiles. If rundir is None, all *.data is copied from current directory, if a path is given, data files are copied from there instead. If print_git_status is True, print a summary of the git status of all clawpack repositories in the file claw_git_status.txt in outdir. If nohup is True, run the xclawcmd in nohup mode: runs in background and output goes into nohup.out file in the directory specified by outdir, rather than all output going to the screen. The job keeps running if user logs off. Useful for scripts starting long jobs in batch mode. If type(nice) is int, runs the code using "nice -n " with this nice value so it doesn't hog computer resources. """ from clawpack.clawutil.data import ClawData import os,glob,shutil,time verbose = True xclawout = None xclawerr = None try: nice = int(nice) except: nice = None # convert strings passed in from Makefile to boolean: if type(overwrite) is str: overwrite = (overwrite.lower() in ['true','t']) if type(restart) is str: if restart == 'None': restart = None else: restart = (restart.lower() in ['true','t']) if type(print_git_status) is str: print_git_status = (print_git_status.lower() in ['true','t']) if type(nohup) is str: nohup = (nohup.lower() in ['true','t']) if xclawcmd is None: # Determine what executable to use from environment variable CLAW_EXE # Default to 'xclaw' if it's not set: xclawcmd = os.environ.get('CLAW_EXE', 'xclaw') if outdir is None: outdir = '.' if rundir is None: rundir = os.getcwd() rundir = os.path.abspath(rundir) print("==> runclaw: Will take data from ", rundir) # directory for fort.* files: outdir = os.path.abspath(outdir) print('==> runclaw: Will write output to ',outdir) if restart is None: # Added option to determine restart from claw.data (i.e. setrun.py) clawdata = ClawData() clawdata.read(os.path.join(rundir,'claw.data'), force=True) restart = clawdata.restart #print '+++ From claw.data determined restart = %s' % restart #returncode = clawjob.runxclaw() if 1: startdir = os.getcwd() xdir = os.path.abspath(startdir) outdir = os.path.abspath(outdir) rundir = os.path.abspath(rundir) xclawcmd = os.path.join(xdir,xclawcmd) try: os.chdir(xdir) except: raise Exception( "==> runclaw: Cannot change to directory xdir = %s" %xdir) return if os.path.isfile(outdir): print("==> runclaw: Error: outdir specified is a file") return if (os.path.isdir(outdir) & (not overwrite)): # copy the old outdir before possibly overwriting tm = time.localtime(os.path.getmtime(outdir)) year = str(tm[0]).zfill(4) month = str(tm[1]).zfill(2) day = str(tm[2]).zfill(2) hour = str(tm[3]).zfill(2) minute = str(tm[4]).zfill(2) second = str(tm[5]).zfill(2) outdir_backup = outdir + '_%s-%s-%s-%s%s%s' \ % (year,month,day,hour,minute,second) if verbose: print("==> runclaw: Directory already exists: ",os.path.split(outdir)[1]) if restart: print("==> runclaw: Copying directory to: ",os.path.split(outdir_backup)[1]) else: print("==> runclaw: Moving directory to: ",os.path.split(outdir_backup)[1]) time.sleep(1) try: shutil.move(outdir,outdir_backup) if restart: shutil.copytree(outdir_backup,outdir) except: print("==> runclaw: Could not move directory... copy already exists?") if (not os.path.isdir(outdir)): try: os.mkdir(outdir) except: print("Cannot make directory ",outdir) return try: os.chdir(outdir) except: print('==> runclaw: *** Error in runxclaw: cannot move to outdir = ',\ outdir) raise return if print_git_status not in [False,'False']: make_git_status_file() fortfiles = glob.glob(os.path.join(outdir,'fort.*')) if (overwrite and (not restart)): # remove any old versions: if verbose: print("==> runclaw: Removing all old fort files in ", outdir) for file in fortfiles: os.remove(file) elif restart: if verbose: print("==> runclaw: Restart: leaving original fort files in ", outdir) else: if len(fortfiles) > 1: print("==> runclaw: *** Remove fort.* and try again,") print(" or use overwrite=True in call to runxclaw") print(" e.g., by setting CLAW_OVERWRITE = True in Makefile") return try: os.chdir(rundir) except: raise Exception("Cannot change to directory %s" % rundir) return datafiles = glob.glob('*.data') if datafiles == (): print("==> runclaw: Warning: no data files found in directory ",rundir) else: if rundir != outdir: for file in datafiles: shutil.copy(file,os.path.join(outdir,file)) if xclawout: xclawout = open(xclawout,'wb') if xclawerr: xclawerr = open(xclawerr,'wb') os.chdir(outdir) #print "\nIn directory outdir = ",outdir,"\n" # execute command to run fortran program: try: #print "\nExecuting ",xclawcmd, " ... " #pclaw = subprocess.Popen(xclawcmd,stdout=xclawout,stderr=xclawerr) #print '+++ pclaw started' #pclaw.wait() # wait for code to run #returncode = pclaw.returncode #print '+++ pclaw done' if nohup: # run in nohup mode: print("\n==> Running in nohup mode, output will be sent to:") print(" %s/nohup.out" % outdir) if type(nice) is int: cmd = "nohup time nice -n %s %s " % (nice,xclawcmd) else: cmd = "nohup time %s " % xclawcmd print("\n==> Running with command:\n ", cmd) returncode = os.system(cmd) else: if type(nice) is int: cmd = "nice -n %s %s " % (nice,xclawcmd) else: cmd = xclawcmd print("\n==> Running with command:\n ", cmd) returncode = os.system(cmd) if returncode == 0: print("\n==> runclaw: Finished executing\n") else: print("\n ==> runclaw: *** Runtime error: return code = %s\n "\ % returncode) print('==> runclaw: Done executing %s via clawutil.runclaw.py' %\ xclawcmd) print('==> runclaw: Output is in ', outdir) except: raise Exception("Could not execute command %s" % xclawcmd) os.chdir(startdir) if returncode != 0: print('==> runclaw: *** fortran returncode = ', returncode, ' aborting')