def __set_parallelization(self):
self.logger.info('Set domain decomposition and number of I/O PEs')
### extract number of I/O processors
if self.options.nprocio is not None:
nprocio = self.options.nprocio
else:
if get_param(self.conf.par_file, 'num_iope_percomm') != '':
num_iope_percomm = int(
get_param(self.conf.par_file, 'num_iope_percomm'))
num_asynio_comm = int(
get_param(self.conf.par_file, 'num_asynio_comm'))
nprocio = num_asynio_comm * num_iope_percomm
else:
nprocio = int(get_param(self.conf.par_file, 'nprocio'))
# sets the number of I/O processors
if get_param(self.conf.par_file, 'num_iope_percomm') != '':
if nprocio == 0:
replace_param(self.conf.par_file, 'num_iope_percomm',
' num_iope_percomm= 0')
replace_param(self.conf.io_file, 'lasync_io',
' lasync_io=.FALSE.')
else:
replace_param(self.conf.par_file, 'num_iope_percomm',
' num_iope_percomm= 1')
replace_param(self.conf.io_file, 'lasync_io',
' lasync_io=.TRUE.')
replace_param(self.conf.par_file, 'num_asynio_comm',
' num_asynio_comm= %i' % nprocio)
else:
replace_param(self.conf.par_file, 'nprocio',
' nprocio= %i' % nprocio)
# generates the parallelist
parlist = []
parlist = self.set_parallelization(self.nprocs, nprocio)
# select the parallelization
ap = int(self.node.findtext("autoparallel"))
if ap > len(parlist):
raise SkipError(
'The selected autoparallel number is too large for the given number of processor (not enough decompositions available)'
)
# writes the new MPI decomposition
nprocx = parlist[ap - 1][0]
nprocy = parlist[ap - 1][1]
replace_param(self.conf.par_file, 'nprocx', ' nprocx= %i' % nprocx)
replace_param(self.conf.par_file, 'nprocy', ' nprocy= %i' % nprocy)
# echo to log
self.logger.info('Processors distribution set to ' +
'(nprocx,nprocy,nprocio)=(%i,%i,%i)' %
(nprocx, nprocy, nprocio))
def __adapt_namelists(self):
self.logger.info('Modify namelists (according to XML specification)')
# change_par from the xml file
changepar_list = self.node.findall("changepar")
# iterate over all changepars
for chpar in changepar_list:
filename = chpar.attrib['file']
if filename is None:
self.logger.error(
'changepar encountered without file attribute')
continue
filename = str(filename)
newparname = str(chpar.attrib['name'])
# look if optional attribute occurrence exists
try:
occurrence = int(chpar.attrib['occurrence'])
except:
occurrence = 1
# check if parameter is one of the dual parameters
parname = newparname
for (param1, param2) in self.conf.dual_params:
if param1 == parname:
if get_param(filename, parname,
occurrence=occurrence) == '':
parname = param2
if param2 == parname:
if get_param(filename, parname,
occurrence=occurrence) == '':
parname = param1
value = chpar.text
modstring = newparname + '=' + str(value)
replace_param(filename, parname, modstring, occurrence=occurrence)
# if nprocio has been overwritten, remove configuration (XML should have precedence)
if parname == 'nprocio':
self.options.nprocio = None
def __set_timesteps(self):
if self.options.steps is not None:
self.logger.info('Number of timesteps (nstop=%i)' %
(self.options.steps))
modstring = 'nstop=' + str(self.options.steps)
parname = 'nstop'
filename = self.conf.par_file
if get_param(filename, 'nstop') == '':
parname = 'hstop'
replace_param(filename, parname, modstring)
def check():
# get name of myself
myname = os.path.basename(__file__)
header = myname+': '
# get environment variables
env = read_environ()
verbose = int(env['VERBOSE'])
rundir = dir_path(env['RUNDIR'])
refoutdir = dir_path(env['REFOUTDIR'])
namelistdir = dir_path(env['NAMELISTDIR'])
tolerance = env['TOLERANCE']
# defines the 2 file that belongs logically to the checker
yufile1 = rundir + yufile
yufile2 = refoutdir + yufile
# extract timestep
try:
dt = float(get_param(rundir+nlfile2, 'dt'))
except:
if verbose:
print header+'failed to extract dt from '+rundir+nlfile
return 20 # FAIL
# check for output lists
try:
nout_list = get_param(rundir+nlfile, lnout)
except:
if verbose:
print header+'no output lists in '+rundir+nlfile
return 20 # FAIL
# check if special testsuite output was activated in every output list
if get_param(rundir+nlfile, yuswitch, occurrence=nout_list) in ['.FALSE.', '.false.']:
if verbose:
print yuswitch+' is set to .false. in '+rundir+nlfile+' for this simulation'
return 20 # FAIL
#check if tolerance file exists in namelistdir or type dir
tolerance_path_1=namelistdir + tolerance
tolerance_path_2=namelistdir + '../'+ tolerance
if os.path.exists(tolerance_path_1):
tolerance_file=tolerance_path_1 #in namelist dir
ltol_file=True
elif os.path.exists(tolerance_path_2):
tolerance_file=tolerance_path_2 #in type dir
ltol_file=True
else:
tolerance_file=''
ltol_file=False
# define tolerances for comparing YUCHKDAT files
# Use tolerance file if exists
if ltol_file:
if verbose==2:
print header + 'Using tolerance values from file'
elif verbose>2:
print header + 'Using tolerance values from file '+tolerance_file
try:
tol_out = get_param(tolerance_file,'tol_out')
tol_out = [float(x) for x in tol_out.split(',')]
tol_times = get_param(tolerance_file,'tol_times')
tol_times = [float(x) for x in tol_times.split(',')]
minval = float(get_param(tolerance_file,'minval'))
except:
if verbose:
print header+'Error while reading '+tolerance_file
print header+'Cannot read one of the follwoing parameter: tol_times,tol_out,minval'
return 20 # FAIL
else:
if verbose>1:
print header + 'Using default tolerance values'
tol_times = [1200,2400,4000] # time limit for tolerance setting in seconds
tol_out = [1.0e-7,1.0e-4,1.0e-0] # tolerance threshold for all other prognostic fields
minval = 1.0e-9 # values below min val are not considered
#set tolerance time step index
nts = [int(x/dt) for x in tol_times]
try:
# check for bit identical results
if verbose>1:
print header + 'Checking first if results are bit identical'
if verbose>2:
print header + 'comp_yuprtest()'
print header + ' file1 = '+yufile1
print header + ' file2 = '+yufile2
print header + ' minval = '+str(-1)
print header + ' nts = [%s]' % ','.join(map(str, [10000]))
print header + ' tol_temp = [%s]' % ','.join(map(str, [0.0]))
print header + ' tol_out = [%s]' % ','.join(map(str, [0.0]))
err_count_identical = comp_yuchkdat.cmp_(yufile1, yufile2, -1, -1, [0.0], [0.0])
if verbose>1:
if err_count_identical==0:
print header + 'Results are bit identical'
else:
print header + 'Results are not bit identical'
# check if results are within tolerance values
if verbose>1:
print header + 'Checking if results are within tolerance'
print header + ' minimal value : %s' %(minval)
print header + ' tolerance time : %s' % ','.join(map(str, tol_times))
print header + ' corresponding time step : %s' % ','.join(map(str, nts))
print header + ' thresholds for all output fields: %s' % ','.join(map(str, tol_out))
if verbose>2:
print header + 'comp_yuprtest()'
print header + ' file1 = '+yufile1
print header + ' file2 = '+yufile2
print header + ' minval = '+str(minval)
print header + ' nts = [%s]' % ','.join(map(str, nts))
print header + ' tol_out = [%s]' % ','.join(map(str, tol_out))
error_count = comp_yuchkdat.cmp_(yufile1, yufile2, 0, minval, nts, tol_out)
if verbose>1:
if error_count==0:
print header + 'Results are within thresholds'
else:
print header + 'Results are not within thresholds'
except Exception as e:
if verbose:
print e
return 20 # FAIL
if err_count_identical == 0:
return 0 # MATCH
elif error_count == 0:
return 10 # OK
else:
return 20 # FAIL
def check():
# get name of myself
myname = os.path.basename(__file__)
header = myname + ': '
# get environment variables
env = read_environ()
verbose = int(env['VERBOSE'])
rundir = dir_path(env['RUNDIR'])
refoutdir = dir_path(env['REFOUTDIR'])
switch = env['DT_FILE']
tune_thresholds = env['TUNE_THRESHOLDS']
if (str_to_bool(tune_thresholds)):
print(
"WARNING: Skipped as identical checker as it isn't usable while tuning thresholds"
)
return 15 #SKIP
# defines the 2 file that belongs logically to the checker
yufile1 = rundir + yufile
yufile2 = refoutdir + yufile
if switch: # extract timestep
nlfile = switch # namelist file containing dt
try:
dt = get_param(rundir + nlfile, 'dt')
assert (dt != '')
dt = float(dt)
except:
if verbose:
print(header + 'failed to extract dt from ' + rundir + nlfile)
return 20 # FAIL
# define tolerances for comparing YUPRTEST files
nts = [maxsize]
tols = [0.0]
tas = [0.0]
try:
# check for bit identical results
if verbose > 1:
print(header + 'Checking if results are bit identical')
if verbose > 2:
print(header + 'comp_yuprtest()')
print(header + ' file1 = ' + yufile1)
print(header + ' file2 = ' + yufile2)
print(header + ' minval = ' + str(-1))
print(header + ' nts = [%s]' % ','.join(map(str, nts)))
print(header + ' tols = [%s]' % ','.join(map(str, tols)))
print(header + ' tas = [%s]' % ','.join(map(str, tas)))
error_count = comp_yuprtest.cmp_(yufile1, yufile2, 0, -1, nts, tols,
tas)
if verbose > 1:
if error_count == 0:
print(header + 'Results are bit identical')
else:
print(header + 'Results are not bit identical')
except RuntimeError as e:
if verbose:
print(e)
return 20 # FAIL
if error_count:
return 20 # FAIL
else:
return 0 # MATCH
def check():
# get name of myself
myname = os.path.basename(__file__)
header = myname + ': '
# get environment variables
env = read_environ()
verbose = int(env['VERBOSE'])
rundir = dir_path(env['RUNDIR'])
refoutdir = dir_path(env['REFOUTDIR'])
namelistdir = dir_path(env['NAMELISTDIR'])
switch = env['DT_FILE']
tolerance = env['TOLERANCE']
forcematch = int(env['FORCEMATCH']) == 1
tune_thresholds = str_to_bool(env['TUNE_THRESHOLDS'])
# defines the 2 file that belongs logically to the checker
yufile1 = rundir + yufile
yufile2 = refoutdir + yufile
if not switch: # no need to extract timestep and to check output lists
nlfile = env['NL_TS_SWITCH']
else:
nlfile = 'INPUT_IO' # namelist file containing yuswitch
nlfile2 = 'INPUT_ORG' # namelist file containing dt
# extract timestep
try:
dt = get_param(rundir + nlfile2, 'dt')
assert (dt != '')
dt = float(dt)
except:
if verbose:
print(header + 'failed to extract dt from ' + rundir + nlfile2)
return 20 # FAIL
# check for output lists
try:
nout_list = get_param(rundir + nlfile, lnout)
assert (nout_list != '')
nout_list = int(nout_list)
except:
if verbose:
print(header + 'no output lists in ' + rundir + nlfile)
return 20 # FAIL
# check if special testsuite output was activated in every output list
yuswitch_value = get_param(rundir + nlfile,
yuswitch,
occurrence=nout_list)
assert (yuswitch_value != '')
if yuswitch_value in ['.FALSE.', '.false.']:
if verbose:
print(yuswitch + ' is set to .false. in ' + rundir + nlfile +
' for this simulation')
return 20 # FAIL
#check if tolerance file exists in namelistdir
tolerance_path = namelistdir + tolerance
if os.path.exists(tolerance_path):
tolerance_file = tolerance_path #in namelist dir
ltol_file = True
else:
tolerance_file = ''
ltol_file = False
# Create Thresholds object
thres = """
minval = 1e-12
steps = 1 20 60 100
* = 1.00e-7 1.00e-4 1.00e-06 1.00e-02
"""
threshold = ts_thresholds.Thresholds(thres)
threshold_var = '*'
# define tolerances for comparing YUCHKDAT files
# Use tolerance file if exists
if ltol_file:
if verbose == 2:
print(header + 'Using tolerance values from file')
elif verbose > 2:
print(header + 'Using tolerance values from file ' +
tolerance_file)
try:
threshold = ts_thresholds.Thresholds(tolerance_file)
if "CHKDAT" in threshold.variables:
threshold_var = "CHKDAT"
except:
if verbose:
print(header + 'Error while reading ' + tolerance_file)
print(
header +
'Cannot read one of the following parameter: tol_times,tol_out,minval'
)
return 20 # FAIL
# Identical thresholds
thres = """
minval = 0.0
steps = 0
* = 0
"""
threshold_identical = ts_thresholds.Thresholds(thres)
# Override the tolerances in case FORCEMATCH is enabled
if forcematch:
threshold = threshold_identical
try:
# check for bit identical results
if verbose > 1:
print(header + 'Checking first if results are bit identical')
err_count_identical = ts_yuchdat.compare(yufile1,
yufile2,
threshold_identical,
threshold_var,
v_level=-1)
if verbose > 1:
if err_count_identical == 0:
print(header + 'Results are bit identical')
else:
print(header + 'Results are not bit identical')
# check if results are within tolerance values
if verbose > 1:
print(header + 'Checking if results are within tolerance')
error_count = ts_yuchdat.compare(yufile1,
yufile2,
threshold,
threshold_var,
tune_thresholds,
v_level=0)
if (tune_thresholds):
threshold.to_file(tolerance_path)
if verbose > 1:
if error_count == 0:
print(header + 'Results are within thresholds')
else:
print(header + 'Results are not within thresholds')
except Exception as e:
if verbose:
print(e)
return 20 # FAIL
if err_count_identical == 0:
return 0 # MATCH
elif error_count == 0:
return 10 # OK
else:
return 20 # FAIL
def check():
# get name of myself
myname = os.path.basename(__file__)
header = myname+': '
# get environment variables
env = read_environ()
verbosity = int(env['VERBOSE'])
rundir = dir_path(env['RUNDIR'])
refoutdir = dir_path(env['REFOUTDIR'])
namelistdir = dir_path(env['NAMELISTDIR'])
tolerance = env['TOLERANCE']
switch = env['NL_TS_SWITCH']
forcematch = int(env['FORCEMATCH']) == 1
tune_thresholds = str_to_bool(env['TUNE_THRESHOLDS'])
tune_times = int(env['TUNING_ITERATIONS'])
reset_thresholds = str_to_bool(env['RESET_THRESHOLDS'])
#check if namelist file with switch exists in namelistdir
switch_path = namelistdir + switch
if not os.path.exists(switch_path):
if verbosity>0:
print header + "unable to find namelist file with switch in " + switch_path
return 20 # FAIL
# defines the 1 file that belongs logically to the checker
yufile1 = rundir + yufile
yufile2 = refoutdir + yufile
# check if special testsuite output was activated
if get_param(rundir+switch, yuswitch) in ['.FALSE.', '.false.']:
if verbosity:
print yuswitch +' is set to .false. in '+ rundir + switch +' for this simulation'
return 20 # FAIL
#check if tolerance file exists in namelistdir or type dir
tolerance_path = namelistdir + tolerance
# The forcematch option gives precedence over the tolerance file
if forcematch:
thresh = """
minval = 0.0
steps = {maxsteps}
* = 0.0""".format(maxsteps=maxsize)
elif os.path.exists(tolerance_path):
thresh=tolerance_path #in namelist dir
else:
if verbosity>0:
print header + "unable to find tolerance file at " + tolerance
return 20 # FAIL
try:
c = Compare(yufile1, yufile2, thresh)
if reset_thresholds:
c.reset_thresholds()
if tune_thresholds:
c.update_thresholds()
c.write_threshold_to_file(tolerance_path)
result = c.compare_data()
if (verbosity > 1) or (result >= 2):
print(header)
c.print_results()
elif verbosity > 1:
print(header)
except Exception as e :
traceback.print_exc(file=sys.stdout)
print(header + str(e))
return 30 # CRASH
if (result == 0):
if verbosity>1:
print header + "Results are within the thresholds and bit identical"
return 0 # MATCH
if (result == 1):
if verbosity>1:
print header + "Results are within thresholds, but are not bit identical"
return 10 # OK
if (result == 2):
if verbosity>1:
print header + "Some or all Results are not within thresholds"
return 20 # FAIL