Exemple #1
0
    def check_gauges(self, save=False, gauge_id=1, indices=[0],
                           rtol=1e-14, atol=1e-8, tolerance=None):
        r"""Basic test to assert gauge equality

        :Input:
         - *save* (bool) - If *True* will save the output from this test to 
           the file *regresion_data.txt*.  Default is *False*.
         - *indices* (tuple) - Contains indices to compare in the gague 
           comparison.  Defaults to *(0)*.
         - *rtol* (float) - Relative tolerance used in the comparison, default 
           is *1e-14*.  Note that the old *tolerance* input is now synonymous 
           with this parameter.
         - *atol* (float) - Absolute tolerance used in the comparison, default
           is *1e-08*.
        """

        if isinstance(tolerance, float):
            rtol = tolerance

        if not(isinstance(indices, tuple) or isinstance(indices, list)):
            indices = tuple(indices)

        # Get gauge data
        gauge = gauges.GaugeSolution(gauge_id, path=self.temp_path)

        # Get regression comparison data
        regression_data_path = os.path.join(self.test_path, "regression_data")
        if save:
            gauge_file_name = "gauge%s.txt" % str(gauge_id).zfill(5)
            shutil.copy(os.path.join(self.temp_path, gauge_file_name), 
                                                           regression_data_path)
            claw_git_status.make_git_status_file(outdir=regression_data_path)

        regression_gauge = gauges.GaugeSolution(gauge_id,
                                                path=regression_data_path)

        # Compare data
        try:
            for n in indices:
                numpy.testing.assert_allclose(gauge.q[n, :],
                                              regression_gauge.q[n, :], 
                                              rtol=rtol, atol=atol, 
                                              verbose=False)
        except AssertionError as e:
            err_msg = "\n".join((e.args[0], 
                                "Gauge Match Failed for gauge = %s" % gauge_id))
            err_msg = "\n".join((err_msg, "  failures in fields:"))
            failure_indices = []
            for n in indices:
                if ~numpy.allclose(gauge.q[n, :], regression_gauge.q[n, :], 
                                                          rtol=rtol, atol=atol):
                    failure_indices.append(str(n))
            index_str = ", ".join(failure_indices)
            raise AssertionError(" ".join((err_msg, index_str)))
Exemple #2
0
    def check_gauges(self, save=False, gauge_id=1, indices=[0],
                           rtol=1e-14, atol=1e-8, tolerance=None):
        r"""Basic test to assert gauge equality

        :Input:
         - *save* (bool) - If *True* will save the output from this test to 
           the file *regresion_data.txt*.  Default is *False*.
         - *indices* (tuple) - Contains indices to compare in the gague 
           comparison.  Defaults to *(0)*.
         - *rtol* (float) - Relative tolerance used in the comparison, default 
           is *1e-14*.  Note that the old *tolerance* input is now synonymous 
           with this parameter.
         - *atol* (float) - Absolute tolerance used in the comparison, default
           is *1e-08*.
        """

        if isinstance(tolerance, float):
            rtol = tolerance

        if not(isinstance(indices, tuple) or isinstance(indices, list)):
            indices = tuple(indices)

        # Get gauge data
        gauge = gauges.GaugeSolution(gauge_id, path=self.temp_path)

        # Get regression comparison data
        regression_data_path = os.path.join(self.test_path, "regression_data")
        if save:
            gauge_file_name = "gauge%s.txt" % str(gauge_id).zfill(5)
            shutil.copy(os.path.join(self.temp_path, gauge_file_name), 
                                                           regression_data_path)
            claw_git_status.make_git_status_file(outdir=regression_data_path)

        regression_gauge = gauges.GaugeSolution(gauge_id,
                                                path=regression_data_path)

        # Compare data
        try:
            for n in indices:
                numpy.testing.assert_allclose(gauge.q[n, :],
                                              regression_gauge.q[n, :], 
                                              rtol=rtol, atol=atol, 
                                              verbose=False)
        except AssertionError as e:
            err_msg = "\n".join((e.args[0], 
                                "Gauge Match Failed for gauge = %s" % gauge_id))
            err_msg = "\n".join((err_msg, "  failures in fields:"))
            failure_indices = []
            for n in indices:
                if ~numpy.allclose(gauge.q[n, :], regression_gauge.q[n, :], 
                                                          rtol=rtol, atol=atol):
                    failure_indices.append(str(n))
            index_str = ", ".join(failure_indices)
            raise AssertionError(" ".join((err_msg, index_str)))
Exemple #3
0
    def check_frame(
        self,
        save=False,
        indices=[0],
        frame_num=1,
        file_name="regression_data.txt",
        rtol=1e-14,
        atol=1e-08,
        tolerance=None,
    ):
        r"""Compare choosen frame to the comparison data

        :Input:
         - *save* (bool) - If *True* will save the output from this test to 
           the file *regresion_data.txt*.  Default is *False*.
         - *indices* (tuple) - Contains indices to compare in the gague 
           comparison.  Defaults to *(0)*.
         - *frame_num* (int) - Frame number to load from the run data.  Defaults
           to *1*. 
         - *regression_data_path* (path) - Path to the regression test data.
           Defaults to 'regression_data.txt'.
         - *rtol* (float) - Relative tolerance used in the comparison, default 
           is *1e-14*.  Note that the old *tolerance* input is now synonymous 
           with this parameter.
         - *atol* (float) - Absolute tolerance used in the comparison, default
           is *1e-08*.
        """

        if isinstance(tolerance, float):
            rtol = tolerance

        # Load test data
        data = solution.Solution(frame_num, path=self.temp_path)
        data_sum = []
        for index in indices:
            data_sum.append(data.q[index, ...].sum())

        # Get (and save) regression comparison data
        regression_data_file = os.path.join(self.test_path, "regression_data", file_name)
        if save:
            numpy.savetxt(regression_data_file, data_sum)
            claw_git_status.make_git_status_file(outdir=os.path.join(self.test_path, "regression_data"))

        regression_sum = numpy.loadtxt(regression_data_file)

        assert numpy.allclose(data_sum, regression_sum, rtol=rtol, atol=atol), "\n  new_data: %s, \n  expected: %s" % (
            data_sum,
            regression_sum,
        )
Exemple #4
0
    def check_frame(self, save=False, indices=[0], frame_num=1,
                          file_name="regression_data.txt",
                          rtol=1e-14, atol=1e-08, tolerance=None):
        r"""Compare choosen frame to the comparison data

        :Input:
         - *save* (bool) - If *True* will save the output from this test to 
           the file *regresion_data.txt*.  Default is *False*.
         - *indices* (tuple) - Contains indices to compare in the gague 
           comparison.  Defaults to *(0)*.
         - *frame_num* (int) - Frame number to load from the run data.  Defaults
           to *1*. 
         - *regression_data_path* (path) - Path to the regression test data.
           Defaults to 'regression_data.txt'.
         - *rtol* (float) - Relative tolerance used in the comparison, default 
           is *1e-14*.  Note that the old *tolerance* input is now synonymous 
           with this parameter.
         - *atol* (float) - Absolute tolerance used in the comparison, default
           is *1e-08*.
        """

        if isinstance(tolerance, float):
            rtol = tolerance

        # Load test data
        data = solution.Solution(frame_num, path=self.temp_path)
        data_sum = []
        for index in indices:
            data_sum.append(data.q[index, ...].sum())
        
        # Get (and save) regression comparison data
        regression_data_file = os.path.join(self.test_path, "regression_data",
                                            file_name)
        if save:
            numpy.savetxt(regression_data_file, data_sum)
            claw_git_status.make_git_status_file(
                         outdir=os.path.join(self.test_path, "regression_data"))

        regression_sum = numpy.loadtxt(regression_data_file)

        assert numpy.allclose(data_sum, regression_sum, rtol=rtol, atol=atol), \
            "\n  new_data: %s, \n  expected: %s"  % (data_sum, regression_sum)
Exemple #5
0
def runclaw(xclawcmd=None, outdir=None, overwrite=True, restart=None, 
            rundir=None, print_git_status=False, nohup=False, nice=None,
            xclawout=None, xclawerr=None, verbose=True):
    """
    Run the Fortran version of Clawpack using executable xclawcmd, which is
    typically set to 'xclaw', 'xamr', etc.

    If it is not set by the call, get it from the environment variable
    CLAW_EXE.  Default to 'xclaw' if that's not set.

    If overwrite is True, it's ok to zap current contents of the outdir.
    If overwrite is False, move the outdir (or copy in the case of a restart)
    to a backup directory with a unique name based on time executed.

    If restart is None, determine whether this is a restart from claw.data
    (as set in setrun.py).  Can remove setting RESTART in Makefiles.
    
    If rundir is None, all *.data is copied from current directory, if a path 
    is given, data files are copied from there instead.

    If print_git_status is True, print a summary of the git status of all
    clawpack repositories in the file claw_git_status.txt in outdir.

    If nohup is True, run the xclawcmd in nohup mode:  runs in background and
    output goes into nohup.out file in the directory specified by outdir,
    rather than all output going to the screen.  The job keeps running if user 
    logs off.  Useful for scripts starting long jobs in batch mode.

    If type(nice) is int, runs the code using "nice -n "
    with this nice value so it doesn't hog computer resources.
    
    xclawout and xclawerr define the locations of stdout and stderr for the 
    execution of CLAW_EXE. They should be strings to filepaths or open file
    objects or ``subprocess.PIPE`` or ``subprocess.STDOUT``. To pass both
    to the same file, specify ``xclawout`` as the filepath and 
    ``xclawerr=subprocess.STDOUT``.

    """
    
    if nice is not None:
        try:
            nice = int(nice)
        except:
            nice = None

    # convert strings passed in from Makefile to boolean:
    if type(overwrite) is str:
        overwrite = (overwrite.lower() in ['true','t'])
    if type(restart) is str:
        if restart == 'None':
            restart = None
        else:
            restart = (restart.lower() in ['true','t'])
    if type(print_git_status) is str:
        print_git_status = (print_git_status.lower() in ['true','t'])
    if type(nohup) is str:
        nohup = (nohup.lower() in ['true','t'])
    

    if xclawcmd is None:
        # Determine what executable to use from environment variable CLAW_EXE
        # Default to 'xclaw' if it's not set:
        xclawcmd = os.environ.get('CLAW_EXE', 'xclaw')
    

    if outdir is None:
        outdir = '.'

    if rundir is None:
        rundir = os.getcwd()
    rundir = os.path.abspath(rundir)
    print("==> runclaw: Will take data from ", rundir)

    # directory for fort.* files:
    outdir = os.path.abspath(outdir)
    print('==> runclaw: Will write output to ',outdir)
    
    if restart is None:
        # Added option to determine restart from claw.data (i.e. setrun.py)
        clawdata = ClawData()
        clawdata.read(os.path.join(rundir,'claw.data'), force=True) 
        restart = clawdata.restart
        
    xclawcmd = os.path.abspath(xclawcmd)

    if os.path.isfile(outdir):
        print("==> runclaw: Error: outdir specified is a file")
        return

    if (os.path.isdir(outdir) & (not overwrite)):
        # copy the old outdir before possibly overwriting
        tm = time.localtime(os.path.getmtime(outdir))
        year = str(tm[0]).zfill(4)
        month = str(tm[1]).zfill(2)
        day = str(tm[2]).zfill(2)
        hour = str(tm[3]).zfill(2)
        minute = str(tm[4]).zfill(2)
        second = str(tm[5]).zfill(2)
        outdir_backup = outdir + '_%s-%s-%s-%s%s%s' \
              % (year,month,day,hour,minute,second)
        if verbose:
            print("==> runclaw: Directory already exists: ",os.path.split(outdir)[1])
            if restart:
                print("==> runclaw: Copying directory to:      ",os.path.split(outdir_backup)[1])
            else:
                print("==> runclaw: Moving directory to:      ",os.path.split(outdir_backup)[1])
            time.sleep(1)

        try:
            shutil.move(outdir,outdir_backup)
            if restart:
                shutil.copytree(outdir_backup,outdir)
        except:
            print("==> runclaw: Could not move directory... copy already exists?")

    if six.PY2:
        if (not os.path.isdir(outdir)):
            try:
                os.mkdir(outdir)
            except:
                print("Cannot make directory ",outdir)
                return
    else:
        os.makedirs(outdir, exist_ok=True)

    if print_git_status not in [False,'False']:
        # create files claw_git_status.txt and claw_git_diffs.txt in
        # outdir:
        make_git_status_file(outdir=outdir)

    # old fort.* files to be removed for new run?
    fortfiles = glob.glob(os.path.join(outdir,'fort.*'))
    # also need to remove gauge*.txt output files now that the gauge
    # output is no longer in fort.gauge  (but don't remove new gauges.data)
    gaugefiles = glob.glob(os.path.join(outdir,'gauge*.txt'))

    if (overwrite and (not restart)):
        # remove any old versions:
        if verbose:
            print("==> runclaw: Removing all old fort/gauge files in ", outdir)
        for file in fortfiles + gaugefiles:
            os.remove(file)
    elif restart:
        if verbose:
            print("==> runclaw: Restart: leaving original fort/gauge files in ", outdir)
    else:
        # this should never be reached: 
        # if overwrite==False then outdir has already been moved
        if len(fortfiles+gaugefiles) > 1:
            print("==> runclaw: *** Remove fort.* and gauge*.txt")
            print("  from output directory %s and try again," % outdir)
            print("  or use overwrite=True in call to runclaw")
            print("  e.g., by setting OVERWRITE = True in Makefile")
            return

    datafiles = glob.glob(os.path.join(rundir,'*.data'))
    if datafiles == ():
        print("==> runclaw: Warning: no data files found in directory ",rundir)
    else:
        if rundir != outdir:
            for file in datafiles:
                shutil.copy(file, os.path.join(outdir,os.path.basename(file)))

    b4run = None
    if os.path.isfile('b4run.py'):
        b4run_file = os.path.abspath('b4run.py')
    else:
        b4run_file = os.environ.get('B4RUN', '')

    if os.path.isfile(b4run_file):
        try:
            file_globals = runpy.run_path(b4run_file)
            b4run = file_globals.get('b4run', None)
        except:
            w = r"*** WARNING: problem running b4run_file = %s" % b4run_file
            warnings.warn(w, UserWarning)
            b4run = None
        
    if b4run is not None:
        try:
            b4run(rundir, outdir)
            print('Executed b4run function from %s' % b4run_file)
        except:
            w = r"*** WARNING: problem executing b4run from %s" % b4run_file
            warnings.warn(w, UserWarning)

    # execute command to run fortran program:
    if nohup:
        # run in nohup mode:
        print("\n==> Running in nohup mode, output will be sent to:")
        print("      %s/nohup.out" % outdir)
        if type(nice) is int:
            cmd = "nohup time nice -n %s %s " % (nice,xclawcmd)
        else:
            cmd = "nohup time %s " % xclawcmd
        print("\n==> Running with command:\n   ", cmd)
    else:
        if type(nice) is int:
            cmd = "nice -n %s %s " % (nice,xclawcmd)
        else:
            cmd = xclawcmd
        print("\n==> Running with command:\n   ", cmd)

    cmd_split = shlex.split(cmd)
    if isinstance(xclawout, str):
        xclawout = open(xclawout,'w', encoding='utf-8',
                        buffering=1)
    if isinstance(xclawerr, str):
        xclawerr = open(xclawerr,'w', encoding='utf-8',
                        buffering=1)
    try:
        proc = subprocess.check_call(cmd_split,
                                     cwd=outdir,
                                     stdout=xclawout,
                                     stderr=xclawerr)

    except subprocess.CalledProcessError as cpe:
        exe_error_str = "\n\n*** FORTRAN EXE FAILED ***\n"
        raise ClawExeError(exe_error_str, cpe.returncode, cpe.cmd,
                           output=cpe.output, 
                           stderr=cpe.stderr)
    
    print('==> runclaw: Done executing %s via clawutil.runclaw.py' %\
                xclawcmd)
    print('==> runclaw: Output is in ', outdir)

    return proc
Exemple #6
0
def runclaw(xclawcmd=None, outdir=None, overwrite=True, restart=None, 
            rundir=None, print_git_status=False, nohup=False, nice=None):
    """
    Run the Fortran version of Clawpack using executable xclawcmd, which is
    typically set to 'xclaw', 'xamr', etc.

    If it is not set by the call, get it from the environment variable
    CLAW_EXE.  Default to 'xclaw' if that's not set.

    If overwrite is True, it's ok to zap current contents of the outdir.
    If overwrite is False, move the outdir (or copy in the case of a restart)
    to a backup directory with a unique name based on time executed.

    If restart is None, determine whether this is a restart from claw.data
    (as set in setrun.py).  Can remove setting RESTART in Makefiles.
    
    If rundir is None, all *.data is copied from current directory, if a path 
    is given, data files are copied from there instead.

    If print_git_status is True, print a summary of the git status of all
    clawpack repositories in the file claw_git_status.txt in outdir.

    If nohup is True, run the xclawcmd in nohup mode:  runs in background and
    output goes into nohup.out file in the directory specified by outdir,
    rather than all output going to the screen.  The job keeps running if user 
    logs off.  Useful for scripts starting long jobs in batch mode.

    If type(nice) is int, runs the code using "nice -n "
    with this nice value so it doesn't hog computer resources.

    """
    
    from clawpack.clawutil.data import ClawData
    import os,glob,shutil,time
    verbose = True
    xclawout = None
    xclawerr = None

    try:
        nice = int(nice)
    except:
        nice = None

    # convert strings passed in from Makefile to boolean:
    if type(overwrite) is str:
        overwrite = (overwrite.lower() in ['true','t'])
    if type(restart) is str:
        if restart == 'None':
            restart = None
        else:
            restart = (restart.lower() in ['true','t'])
    if type(print_git_status) is str:
        print_git_status = (print_git_status.lower() in ['true','t'])
    if type(nohup) is str:
        nohup = (nohup.lower() in ['true','t'])
    

    if xclawcmd is None:
        # Determine what executable to use from environment variable CLAW_EXE
        # Default to 'xclaw' if it's not set:
        xclawcmd = os.environ.get('CLAW_EXE', 'xclaw')
    

    if outdir is None:
        outdir = '.'
        
    if rundir is None:
        rundir = os.getcwd()
    rundir = os.path.abspath(rundir)
    print("==> runclaw: Will take data from ", rundir)

    # directory for fort.* files:
    outdir = os.path.abspath(outdir)
    print('==> runclaw: Will write output to ',outdir)
    
    if restart is None:
        # Added option to determine restart from claw.data (i.e. setrun.py)
        clawdata = ClawData()
        clawdata.read(os.path.join(rundir,'claw.data'), force=True) 
        restart = clawdata.restart
        #print '+++ From claw.data determined restart = %s' % restart

    
    #returncode = clawjob.runxclaw()

    if 1:
        startdir = os.getcwd()
        xdir = os.path.abspath(startdir)
        outdir = os.path.abspath(outdir)
        rundir = os.path.abspath(rundir)
        xclawcmd = os.path.join(xdir,xclawcmd)
        
        try:
            os.chdir(xdir)
        except:
            raise Exception( "==> runclaw: Cannot change to directory xdir = %s" %xdir)
            return 
    
    
        if os.path.isfile(outdir):
            print("==> runclaw: Error: outdir specified is a file")
            return
        
        if (os.path.isdir(outdir) & (not overwrite)):
            # copy the old outdir before possibly overwriting
            tm = time.localtime(os.path.getmtime(outdir))
            year = str(tm[0]).zfill(4)
            month = str(tm[1]).zfill(2)
            day = str(tm[2]).zfill(2)
            hour = str(tm[3]).zfill(2)
            minute = str(tm[4]).zfill(2)
            second = str(tm[5]).zfill(2)
            outdir_backup = outdir + '_%s-%s-%s-%s%s%s' \
                  % (year,month,day,hour,minute,second)
            if verbose:
                print("==> runclaw: Directory already exists: ",os.path.split(outdir)[1])
                if restart:
                    print("==> runclaw: Copying directory to:      ",os.path.split(outdir_backup)[1])
                else:
                    print("==> runclaw: Moving directory to:      ",os.path.split(outdir_backup)[1])
                time.sleep(1)
            
            try:
                shutil.move(outdir,outdir_backup)
                if restart:
                    shutil.copytree(outdir_backup,outdir)
            except:
                print("==> runclaw: Could not move directory... copy already exists?")
            
            
        if (not os.path.isdir(outdir)):
            try:
                os.mkdir(outdir)
            except:
                print("Cannot make directory ",outdir)
                return
    
        try:
            os.chdir(outdir)
        except:
            print('==> runclaw: *** Error in runxclaw: cannot move to outdir = ',\
                  outdir)
            raise
            return
    
        if print_git_status not in [False,'False']:
            make_git_status_file()

        fortfiles = glob.glob(os.path.join(outdir,'fort.*'))
        if (overwrite and (not restart)):
            # remove any old versions:
            if verbose:
                print("==> runclaw: Removing all old fort files in ", outdir)
            for file in fortfiles:
                os.remove(file)
        elif restart:
            if verbose:
                print("==> runclaw: Restart: leaving original fort files in ", outdir)
        else:
            if len(fortfiles) > 1:
                print("==> runclaw: *** Remove fort.* and try again,")
                print("  or use overwrite=True in call to runxclaw")
                print("  e.g., by setting CLAW_OVERWRITE = True in Makefile")
                return
            
        
        try:
            os.chdir(rundir)
        except:
            raise Exception("Cannot change to directory %s" % rundir)
            return 
    
        datafiles = glob.glob('*.data')
        if datafiles == ():
            print("==> runclaw: Warning: no data files found in directory ",rundir)
        else:
            if rundir != outdir:
                for file in datafiles:
                    shutil.copy(file,os.path.join(outdir,file))
    
        if xclawout:
            xclawout = open(xclawout,'wb')
        if xclawerr:
            xclawerr = open(xclawerr,'wb')
    
        os.chdir(outdir)
    
        #print "\nIn directory outdir = ",outdir,"\n"
    
        # execute command to run fortran program:
    
        try:
            #print "\nExecuting ",xclawcmd, "  ...  "
            #pclaw = subprocess.Popen(xclawcmd,stdout=xclawout,stderr=xclawerr)
            #print '+++ pclaw started'
                #pclaw.wait()   # wait for code to run
            #returncode = pclaw.returncode
            #print '+++ pclaw done'
            
            if nohup:
                # run in nohup mode:
                print("\n==> Running in nohup mode, output will be sent to:")
                print("      %s/nohup.out" % outdir)
                if type(nice) is int:
                    cmd = "nohup time nice -n %s %s " % (nice,xclawcmd)
                else:
                    cmd = "nohup time %s " % xclawcmd
                print("\n==> Running with command:\n   ", cmd)
                returncode = os.system(cmd)
            else:
                if type(nice) is int:
                    cmd = "nice -n %s %s " % (nice,xclawcmd)
                else:
                    cmd = xclawcmd
                print("\n==> Running with command:\n   ", cmd)
                returncode = os.system(cmd)
    
            if returncode == 0:
                print("\n==> runclaw: Finished executing\n")
            else:
                print("\n ==> runclaw: *** Runtime error: return code = %s\n "\
                        % returncode)
            print('==> runclaw: Done executing %s via clawutil.runclaw.py' %\
                        xclawcmd)
            print('==> runclaw: Output is in ', outdir)
            
        except:
            raise Exception("Could not execute command %s" % xclawcmd)
    
        os.chdir(startdir)

    if returncode != 0:
        print('==> runclaw: *** fortran returncode = ', returncode, '   aborting')