def __setup_directory(self):
"""generate test directory including all required links and sub-directories"""
self.logger.info('Creating directory for ' + self.name)
node = self.node
# create run directory and move there
status = system_command('/bin/mkdir -p ' + self.rundir, self.logger)
status = change_dir(self.rundir, self.logger)
# removal of all the possible pre-existing files
status = system_command('/bin/rm -r -f *', self.logger)
# explicit copy of the namelists (copy is required since we will apply the change_par)
status = system_command('/bin/cp -f ' + self.namelistdir + 'INPUT* .',
self.logger)
# copy of the auxiliary input parameters if exists
if not glob.glob(os.path.join(
dir_path(self.inputdir) + 'in_aux/', '*')) == []:
status = system_command(
'/bin/cp -f -r ' + dir_path(self.inputdir) + 'in_aux/* ./',
self.logger)
# linking input binary fields
status = system_command(
'/bin/ln -s ' + dir_path(self.inputdir) + 'input .', self.logger)
# generation of the output folder
status = system_command('/bin/mkdir -p output', self.logger)
def check():
# init values
err_count_identical=0
err_count=0
# get name of myself
myname = os.path.basename(__file__)
header = myname+': '
# get environment variables
verbose = int(os.environ['TS_VERBOSE'])
namelistdir = dir_path(os.environ['TS_NAMELISTDIR'])
rundir = dir_path(os.environ['TS_RUNDIR'])
refoutdir = dir_path(os.environ['TS_REFOUTDIR'])
tolerance = os.environ['TS_TOLERANCE']
#get tolerance and minval
tolerance_file=namelistdir+tolerance
if not os.path.exists(tolerance_file):
print('Missing tolerance file %s' %(tolerance_file) )
return 20 #FAIL
thresholds = ts_thresholds.Thresholds(tolerance_file)
tol = thresholds.get_threshold("CHKDAT", 0) # tolerence for t
minval = thresholds.minval
for it in range(len(yufiles)):
# defines the 2 file that belongs logically to the checker
yufile1 = rundir + yufiles[it]
yufile2 = refoutdir + yufiles[it]
try:
# check for bit identical results
err = cmp_table(yufile1, yufile2, \
colpattern[it],minval,0,0,ncomplines[it])
if err !=0 : err_count_identical=err
# check for error within tolerance
err = cmp_table(yufile1, yufile2, \
colpattern[it],minval,tol,1,ncomplines[it])
if err !=0 : err_count=err
except Exception as e:
if verbose:
print e
return 20 # FAIL
if err_count_identical == 0:
return 0 # MATCH
elif err_count == 0:
return 10 # OK
else:
return 20 # FAIL
def check():
# initialize status
status = 0 # MATCH
# get name of myself
myname = os.path.basename(__file__)
header = myname + ': '
# get environment variables
env = read_environ()
verbose = int(env['VERBOSE'])
rundir = env['RUNDIR']
logfile = dir_path(rundir) + env['LOGFILE']
if verbose > 2:
print header + 'checking presence of CLEAN UP cleanup line in ' + logfile
try:
file = open(logfile, 'r')
pattern = '(.*)^(.*)CLEAN(\s)UP(.*)'
result = 30 # CRASH
for text in file:
if re.match(pattern, text):
result = 0 # MATCH
file.close()
except:
if verbose:
print header + 'failed to open ' + logfile
result = 30 # CRASH
return result
def run_checker():
# get environment variables
env = read_environ()
verbose = int(env['VERBOSE'])
rundir = env['RUNDIR']
log_output = env['LOGFILE']
icon = str_to_bool(env['ICON'])
# construct stdout filename
working_dir = dir_path(rundir).replace("./", "", 1)
logfile = os.path.join(working_dir, log_output)
if icon:
patterns = [
# Class/Type Name RegularExpression
OccurrenceCrashPattern( "Cleanup pattern", "0")
]
else:
patterns = [
# Class/Type Name RegularExpression
WarningPattern( "CFL pattern", "CFL" ),
OccurrenceCrashPattern( "Cleanup pattern", "(.*)^(.*)CLEAN(\s*)UP(.*)" )
]
cosmo_filechecker = FileChecker(verbose)
cosmo_filechecker.add_pattern_list(patterns)
return cosmo_filechecker.check(logfile, verbose)
def get_reference_timings():
import ConfigParser
config = ConfigParser.RawConfigParser()
env = read_environ()
rundir = env['RUNDIR']
timings_file = env['TIMINGS']
file_path = dir_path(rundir) + timings_file
config.read(file_path)
threshold = float(config.get('timings', 'threshold'))
timings = {}
for name, value in config.items("timings"):
if name == "threshold":
continue
timings[name] = float(value)
return (timings, threshold)
def check():
# get name of myself
myname = os.path.basename(__file__)
header = myname+': '
# get environment variables
env = read_environ()
verbose = int(env['VERBOSE'])
rundir = dir_path(env['RUNDIR'])
refoutdir = dir_path(env['REFOUTDIR'])
namelistdir = dir_path(env['NAMELISTDIR'])
tolerance = env['TOLERANCE']
# defines the 2 file that belongs logically to the checker
yufile1 = rundir + yufile
yufile2 = refoutdir + yufile
# extract timestep
try:
dt = float(get_param(rundir+nlfile2, 'dt'))
except:
if verbose:
print header+'failed to extract dt from '+rundir+nlfile
return 20 # FAIL
# check for output lists
try:
nout_list = get_param(rundir+nlfile, lnout)
except:
if verbose:
print header+'no output lists in '+rundir+nlfile
return 20 # FAIL
# check if special testsuite output was activated in every output list
if get_param(rundir+nlfile, yuswitch, occurrence=nout_list) in ['.FALSE.', '.false.']:
if verbose:
print yuswitch+' is set to .false. in '+rundir+nlfile+' for this simulation'
return 20 # FAIL
#check if tolerance file exists in namelistdir or type dir
tolerance_path_1=namelistdir + tolerance
tolerance_path_2=namelistdir + '../'+ tolerance
if os.path.exists(tolerance_path_1):
tolerance_file=tolerance_path_1 #in namelist dir
ltol_file=True
elif os.path.exists(tolerance_path_2):
tolerance_file=tolerance_path_2 #in type dir
ltol_file=True
else:
tolerance_file=''
ltol_file=False
# define tolerances for comparing YUCHKDAT files
# Use tolerance file if exists
if ltol_file:
if verbose==2:
print header + 'Using tolerance values from file'
elif verbose>2:
print header + 'Using tolerance values from file '+tolerance_file
try:
tol_out = get_param(tolerance_file,'tol_out')
tol_out = [float(x) for x in tol_out.split(',')]
tol_times = get_param(tolerance_file,'tol_times')
tol_times = [float(x) for x in tol_times.split(',')]
minval = float(get_param(tolerance_file,'minval'))
except:
if verbose:
print header+'Error while reading '+tolerance_file
print header+'Cannot read one of the follwoing parameter: tol_times,tol_out,minval'
return 20 # FAIL
else:
if verbose>1:
print header + 'Using default tolerance values'
tol_times = [1200,2400,4000] # time limit for tolerance setting in seconds
tol_out = [1.0e-7,1.0e-4,1.0e-0] # tolerance threshold for all other prognostic fields
minval = 1.0e-9 # values below min val are not considered
#set tolerance time step index
nts = [int(x/dt) for x in tol_times]
try:
# check for bit identical results
if verbose>1:
print header + 'Checking first if results are bit identical'
if verbose>2:
print header + 'comp_yuprtest()'
print header + ' file1 = '+yufile1
print header + ' file2 = '+yufile2
print header + ' minval = '+str(-1)
print header + ' nts = [%s]' % ','.join(map(str, [10000]))
print header + ' tol_temp = [%s]' % ','.join(map(str, [0.0]))
print header + ' tol_out = [%s]' % ','.join(map(str, [0.0]))
err_count_identical = comp_yuchkdat.cmp_(yufile1, yufile2, -1, -1, [0.0], [0.0])
if verbose>1:
if err_count_identical==0:
print header + 'Results are bit identical'
else:
print header + 'Results are not bit identical'
# check if results are within tolerance values
if verbose>1:
print header + 'Checking if results are within tolerance'
print header + ' minimal value : %s' %(minval)
print header + ' tolerance time : %s' % ','.join(map(str, tol_times))
print header + ' corresponding time step : %s' % ','.join(map(str, nts))
print header + ' thresholds for all output fields: %s' % ','.join(map(str, tol_out))
if verbose>2:
print header + 'comp_yuprtest()'
print header + ' file1 = '+yufile1
print header + ' file2 = '+yufile2
print header + ' minval = '+str(minval)
print header + ' nts = [%s]' % ','.join(map(str, nts))
print header + ' tol_out = [%s]' % ','.join(map(str, tol_out))
error_count = comp_yuchkdat.cmp_(yufile1, yufile2, 0, minval, nts, tol_out)
if verbose>1:
if error_count==0:
print header + 'Results are within thresholds'
else:
print header + 'Results are not within thresholds'
except Exception as e:
if verbose:
print e
return 20 # FAIL
if err_count_identical == 0:
return 0 # MATCH
elif error_count == 0:
return 10 # OK
else:
return 20 # FAIL
def __init__(self, node, options, conf, logger):
# store private information for this test
self.node = node # xml node
self.name = node.attrib['name'] # assign the name to the test
self.type = node.attrib['type'] # set test type
self.description = node.findtext(
'description') # storage of the description string
self.options = copy.copy(options) # test options
self.conf = copy.copy(conf) # storage of the auxiliary parameters
self.logger = logger # store logger
self.result = 30 # default to CRASH
# define prerun actions
if node.findtext('prerun'):
self.prerun_actions = node.findtext('prerun').split(',')
else:
self.prerun_actions = []
# setup of directory paths
self.basedir = dir_path(conf.basedir)
self.inputdir = dir_path(self.basedir + 'data/' +
self.type) # set path for input directory
self.rundir = dir_path(self.basedir) + dir_path(
options.workdir) + dir_path(self.type) + dir_path(self.name)
if node.findtext('namelistdir'):
self.namelistdir = dir_path(self.basedir + 'data/' +
node.findtext('namelistdir'))
else:
#default is self.type/self.name
self.namelistdir = dir_path(self.basedir + 'data/' + self.type +
'/' + self.name)
if node.findtext('refoutdir'):
# set reference output directory, two possibility
# 1. relative to current path, if name starts with ../
# 2. relative to the main directory is starts with a string
refoutdir = dir_path(node.findtext('refoutdir'))
pattern = '[.][.][/](.*)'
matchobj = re.match(pattern, refoutdir)
if matchobj:
self.refoutdir = self.rundir + refoutdir
else:
self.refoutdir = self.basedir + 'data/' + refoutdir
else:
#default is self.namelistdir
self.refoutdir = self.namelistdir
# set dependecy directory
depend = self.node.findtext("depend")
if depend != None:
depend = depend.strip(' ')
pattern = '[.][.][/].*'
matchobj = re.match(pattern, depend)
# relative to rundir
if matchobj:
self.dependdir = depend
else:
self.dependdir = self.basedir + '/' + depend
else:
self.dependdir = None
# set executable
if self.options.exe is not None:
self.executable = self.options.exe
elif node.findtext("executable"):
self.executable = self.node.findtext("executable")
else:
raise SkipError(
'An executable must be defined in the command line or in testlist.xml'
)
# overide nprocs if define in xml
if node.findtext("nprocs"):
self.nprocs = int(self.node.findtext("nprocs"))
else:
self.nprocs = self.options.nprocs
# set tolerance folder name (used by tolerance checker)
self.tolerance = self.options.tolerance
def check():
# get name of myself
myname = os.path.basename(__file__)
header = myname + ': '
# get environment variables
env = read_environ()
verbose = int(env['VERBOSE'])
rundir = dir_path(env['RUNDIR'])
refoutdir = dir_path(env['REFOUTDIR'])
switch = env['DT_FILE']
tune_thresholds = env['TUNE_THRESHOLDS']
if (str_to_bool(tune_thresholds)):
print(
"WARNING: Skipped as identical checker as it isn't usable while tuning thresholds"
)
return 15 #SKIP
# defines the 2 file that belongs logically to the checker
yufile1 = rundir + yufile
yufile2 = refoutdir + yufile
if switch: # extract timestep
nlfile = switch # namelist file containing dt
try:
dt = get_param(rundir + nlfile, 'dt')
assert (dt != '')
dt = float(dt)
except:
if verbose:
print(header + 'failed to extract dt from ' + rundir + nlfile)
return 20 # FAIL
# define tolerances for comparing YUPRTEST files
nts = [maxsize]
tols = [0.0]
tas = [0.0]
try:
# check for bit identical results
if verbose > 1:
print(header + 'Checking if results are bit identical')
if verbose > 2:
print(header + 'comp_yuprtest()')
print(header + ' file1 = ' + yufile1)
print(header + ' file2 = ' + yufile2)
print(header + ' minval = ' + str(-1))
print(header + ' nts = [%s]' % ','.join(map(str, nts)))
print(header + ' tols = [%s]' % ','.join(map(str, tols)))
print(header + ' tas = [%s]' % ','.join(map(str, tas)))
error_count = comp_yuprtest.cmp_(yufile1, yufile2, 0, -1, nts, tols,
tas)
if verbose > 1:
if error_count == 0:
print(header + 'Results are bit identical')
else:
print(header + 'Results are not bit identical')
except RuntimeError as e:
if verbose:
print(e)
return 20 # FAIL
if error_count:
return 20 # FAIL
else:
return 0 # MATCH
def check():
# get name of myself
myname = os.path.basename(__file__)
header = myname + ': '
# get environment variables
env = read_environ()
verbose = int(env['VERBOSE'])
rundir = dir_path(env['RUNDIR'])
refoutdir = dir_path(env['REFOUTDIR'])
namelistdir = dir_path(env['NAMELISTDIR'])
switch = env['DT_FILE']
tolerance = env['TOLERANCE']
forcematch = int(env['FORCEMATCH']) == 1
tune_thresholds = str_to_bool(env['TUNE_THRESHOLDS'])
# defines the 2 file that belongs logically to the checker
yufile1 = rundir + yufile
yufile2 = refoutdir + yufile
if not switch: # no need to extract timestep and to check output lists
nlfile = env['NL_TS_SWITCH']
else:
nlfile = 'INPUT_IO' # namelist file containing yuswitch
nlfile2 = 'INPUT_ORG' # namelist file containing dt
# extract timestep
try:
dt = get_param(rundir + nlfile2, 'dt')
assert (dt != '')
dt = float(dt)
except:
if verbose:
print(header + 'failed to extract dt from ' + rundir + nlfile2)
return 20 # FAIL
# check for output lists
try:
nout_list = get_param(rundir + nlfile, lnout)
assert (nout_list != '')
nout_list = int(nout_list)
except:
if verbose:
print(header + 'no output lists in ' + rundir + nlfile)
return 20 # FAIL
# check if special testsuite output was activated in every output list
yuswitch_value = get_param(rundir + nlfile,
yuswitch,
occurrence=nout_list)
assert (yuswitch_value != '')
if yuswitch_value in ['.FALSE.', '.false.']:
if verbose:
print(yuswitch + ' is set to .false. in ' + rundir + nlfile +
' for this simulation')
return 20 # FAIL
#check if tolerance file exists in namelistdir
tolerance_path = namelistdir + tolerance
if os.path.exists(tolerance_path):
tolerance_file = tolerance_path #in namelist dir
ltol_file = True
else:
tolerance_file = ''
ltol_file = False
# Create Thresholds object
thres = """
minval = 1e-12
steps = 1 20 60 100
* = 1.00e-7 1.00e-4 1.00e-06 1.00e-02
"""
threshold = ts_thresholds.Thresholds(thres)
threshold_var = '*'
# define tolerances for comparing YUCHKDAT files
# Use tolerance file if exists
if ltol_file:
if verbose == 2:
print(header + 'Using tolerance values from file')
elif verbose > 2:
print(header + 'Using tolerance values from file ' +
tolerance_file)
try:
threshold = ts_thresholds.Thresholds(tolerance_file)
if "CHKDAT" in threshold.variables:
threshold_var = "CHKDAT"
except:
if verbose:
print(header + 'Error while reading ' + tolerance_file)
print(
header +
'Cannot read one of the following parameter: tol_times,tol_out,minval'
)
return 20 # FAIL
# Identical thresholds
thres = """
minval = 0.0
steps = 0
* = 0
"""
threshold_identical = ts_thresholds.Thresholds(thres)
# Override the tolerances in case FORCEMATCH is enabled
if forcematch:
threshold = threshold_identical
try:
# check for bit identical results
if verbose > 1:
print(header + 'Checking first if results are bit identical')
err_count_identical = ts_yuchdat.compare(yufile1,
yufile2,
threshold_identical,
threshold_var,
v_level=-1)
if verbose > 1:
if err_count_identical == 0:
print(header + 'Results are bit identical')
else:
print(header + 'Results are not bit identical')
# check if results are within tolerance values
if verbose > 1:
print(header + 'Checking if results are within tolerance')
error_count = ts_yuchdat.compare(yufile1,
yufile2,
threshold,
threshold_var,
tune_thresholds,
v_level=0)
if (tune_thresholds):
threshold.to_file(tolerance_path)
if verbose > 1:
if error_count == 0:
print(header + 'Results are within thresholds')
else:
print(header + 'Results are not within thresholds')
except Exception as e:
if verbose:
print(e)
return 20 # FAIL
if err_count_identical == 0:
return 0 # MATCH
elif error_count == 0:
return 10 # OK
else:
return 20 # FAIL
def check():
# get name of myself
myname = os.path.basename(__file__)
header = myname+': '
# get environment variables
env = read_environ()
verbosity = int(env['VERBOSE'])
rundir = dir_path(env['RUNDIR'])
refoutdir = dir_path(env['REFOUTDIR'])
namelistdir = dir_path(env['NAMELISTDIR'])
tolerance = env['TOLERANCE']
switch = env['NL_TS_SWITCH']
forcematch = int(env['FORCEMATCH']) == 1
tune_thresholds = str_to_bool(env['TUNE_THRESHOLDS'])
tune_times = int(env['TUNING_ITERATIONS'])
reset_thresholds = str_to_bool(env['RESET_THRESHOLDS'])
#check if namelist file with switch exists in namelistdir
switch_path = namelistdir + switch
if not os.path.exists(switch_path):
if verbosity>0:
print header + "unable to find namelist file with switch in " + switch_path
return 20 # FAIL
# defines the 1 file that belongs logically to the checker
yufile1 = rundir + yufile
yufile2 = refoutdir + yufile
# check if special testsuite output was activated
if get_param(rundir+switch, yuswitch) in ['.FALSE.', '.false.']:
if verbosity:
print yuswitch +' is set to .false. in '+ rundir + switch +' for this simulation'
return 20 # FAIL
#check if tolerance file exists in namelistdir or type dir
tolerance_path = namelistdir + tolerance
# The forcematch option gives precedence over the tolerance file
if forcematch:
thresh = """
minval = 0.0
steps = {maxsteps}
* = 0.0""".format(maxsteps=maxsize)
elif os.path.exists(tolerance_path):
thresh=tolerance_path #in namelist dir
else:
if verbosity>0:
print header + "unable to find tolerance file at " + tolerance
return 20 # FAIL
try:
c = Compare(yufile1, yufile2, thresh)
if reset_thresholds:
c.reset_thresholds()
if tune_thresholds:
c.update_thresholds()
c.write_threshold_to_file(tolerance_path)
result = c.compare_data()
if (verbosity > 1) or (result >= 2):
print(header)
c.print_results()
elif verbosity > 1:
print(header)
except Exception as e :
traceback.print_exc(file=sys.stdout)
print(header + str(e))
return 30 # CRASH
if (result == 0):
if verbosity>1:
print header + "Results are within the thresholds and bit identical"
return 0 # MATCH
if (result == 1):
if verbosity>1:
print header + "Results are within thresholds, but are not bit identical"
return 10 # OK
if (result == 2):
if verbosity>1:
print header + "Some or all Results are not within thresholds"
return 20 # FAIL