def get_mushy_layer_exec(self):
exec_name = 'mushyLayer%dd' % self.properties['dim']
exec_dir = os.path.join(mushyLayerRunUtils.get_mushy_layer_dir(), 'execSubcycle')
mushy_layer_exec_path = mushyLayerRunUtils.get_executable_name(exec_dir, exec_name=exec_name,
return_full_path=True)
return mushy_layer_exec_path
def run_compare(next_folder, this_folder, err_type):
print('Running compare between this folder %s \n and next folder %s' %
(this_folder, next_folder))
# check this folder contains some valid files
valid_files = [x for x in os.listdir(this_folder) if '.hdf5' in x]
final_computed_plt_file = get_final_plot_file(this_folder)
if not final_computed_plt_file:
print('No plot files found in this folder')
return
final_exact_plt_file = get_final_plot_file(next_folder)
if not final_exact_plt_file:
print('No plot files found in higher resolution folder')
return
chombo_dir = os.environ['CHOMBO_HOME']
compare_dir = os.path.join(chombo_dir, 'lib', 'util', 'ChomboCompare')
compare_exec = get_executable_name(compare_dir, 'compare2d')
compare_exec = os.path.join(compare_dir, compare_exec)
# print('Found executable %s ' % compare_exec)
this_err_folder = os.path.join(this_folder, 'error-' + err_type)
if not os.path.exists(this_err_folder):
os.makedirs(this_err_folder)
compare_params_file = os.path.join(this_err_folder,
'compare-' + err_type + '.inputs')
error_file = os.path.join(this_err_folder, 'err-' + err_type + '.2d.hdf5')
computed_file = os.path.join(this_folder, final_computed_plt_file)
exact_file = os.path.join(next_folder, final_exact_plt_file)
compare_params = {
'compare.sameSize': 0,
'compare.exactRoot': exact_file,
'compare.computedRoot': computed_file,
'compare.errorRoot': error_file,
'compare.doPlots': 1,
'compare.HOaverage': 0,
'compare.no_average_var': 'T err'
}
write_inputs(compare_params_file, compare_params)
cmd = 'cd %s \n %s %s \n \n' % (this_err_folder, compare_exec,
compare_params_file)
print(cmd)
os.system(cmd)
inputs['bc.aHi'] = [0, opt['a']]
inputs['bc.bHi'] = [0, opt['b']]
inputs['bc.TRefHi'] = [0, opt['Tref']]
inputs['bc.enthalpyLoVal'] = [0.0, T_max + st]
inputs['bc.enthalpyHiVal'] = [0.0, T_max + st]
inputs['parameters.nonDimVel'] = opt['V']
inputs['parameters.compositionRatio'] = cr
inputs['bc.HC_noflux'] = True
inputs['main.output_folder'] = full_output_folder
new_inputs_loc = os.path.join(full_output_folder, 'inputs')
mushyLayerRunUtils.write_inputs(new_inputs_loc, inputs)
exec_loc = mushyLayerRunUtils.get_executable_name(return_full_path=True)
cmd = 'cd %s; %s inputs' % (full_output_folder, exec_loc)
print(cmd)
print('Running... (usually takes ~ 30 seconds) ')
process = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE)
process.wait()
print(process.returncode)
print('Finished')
# Run comparison
plt_files = [
f for f in os.listdir(full_output_folder)
def amr_convergence_test(params,
full_output_dir,
physicalProblem,
nzs,
num_procs=[1],
num_restarts=0,
analysis_command='',
restart_from_low_res=False):
os.environ["CH_TIMER"] = "1"
dependencies = []
prev_job_id = -1
prev_dir = ''
prev_num_cells = 0
while len(num_procs) < len(nzs):
num_procs.append(num_procs[-1])
param_i = -1
for p in params:
param_i = param_i + 1
num_proc = p['num_proc']
# Don't repeat runs, unless we're restarting
#cwd = os.getcwd()
run_name = p['main.plot_prefix']
test_name = run_name + get_suffix(0)
full_path = os.path.join(full_output_dir, test_name)
allow_restarts = False
print(full_path)
if os.path.exists(full_path):
print(Fore.YELLOW + ' Run already done' + Fore.RESET)
if num_restarts > 0:
print(' ' + str(num_restarts) + ' restart(s) allowed \n ')
# Find most recent folder and restart from there
i = 0
while os.path.exists(full_path):
i = i + 1
test_name = run_name + get_suffix(i)
full_path = os.path.join(full_output_dir, test_name)
if i > num_restarts:
continue
most_recent_path = os.path.join(full_output_dir,
run_name + get_suffix(i - 1))
# Now get the restart file
restart_file = get_restart_file(most_recent_path)
if restart_file:
p['main.restart_file'] = os.path.join(
most_recent_path, restart_file)
allow_restarts = True
print(' Set restart file: ' + p['main.restart_file'])
else:
continue
else:
print(Fore.YELLOW + ' No previous runs' + Fore.RESET)
print(Fore.GREEN + ' **Do run**' + Fore.RESET)
# Don't need output dir or exec dir here, MushyLayerRun will fill these in
s = BatchJob('', p['concise_run_name'], '', num_proc)
if restart_from_low_res:
python_file = os.path.join(get_mushy_layer_dir(), 'test',
'create_refined_restart.py')
new_dir = full_path
if prev_job_id > -1:
refinement = int(
float(p['main.num_cells'][0]) / float(prev_num_cells[0]))
preprocess_cmd = 'python %s -p %s -n %s -r %d \n' % (
python_file, prev_dir, new_dir, refinement)
s.set_dependency(prev_job_id)
s.set_preprocess(preprocess_cmd)
p['main.restart_file'] = os.path.join(new_dir,
'restart.2d.hdf5')
else:
# Search for an old file we can restart from, just in case
refinement = 2
this_nx = int(p['main.num_cells'][0])
coarse_nx = int(float(this_nx) / 2.0)
coarser_dir = new_dir.replace(str(this_nx), str(coarse_nx))
if os.path.exists(coarser_dir) and get_final_chk_file(
coarser_dir):
preprocess_cmd = 'python %s -p %s -n %s -r %d \n' % (
python_file, coarser_dir, new_dir, refinement)
s.set_preprocess(preprocess_cmd)
p['main.restart_file'] = os.path.join(
new_dir, 'restart.2d.hdf5')
ml_run = MushyLayerRunSimple(full_output_dir, num_proc, p, s,
allow_restarts, get_executable_name())
ml_run.single_run(run_name)
prev_job_id = s.job_id
prev_dir = full_path
prev_num_cells = p['main.num_cells']
dependencies.append(s.job_id)
print('==================')
return dependencies
def test_hele_shaw_fixed_chill(argv):
mushy_layer_base_dir = os.path.abspath(os.pardir)
# params_file = mushyLayerBaseDir + '/params/convergenceTest/FixedChill.parameters'
params_file = mushy_layer_base_dir + '/params/convergenceTest/FixedChill.parameters'
# Defaults
default_params = read_inputs(params_file)
extra_params = {
'main.max_time':
float(default_params['main.max_time']),
'parameters.rayleighComp':
float(default_params['parameters.rayleighComp']),
'parameters.darcy':
float(default_params['parameters.darcy']),
'parameters.compositionRatio':
float(default_params['parameters.compositionRatio']),
'parameters.nonDimReluctance':
float(default_params['parameters.nonDimReluctance']),
'parameters.prandtl':
float(default_params['parameters.prandtl'])
}
do_amr = True
# Pr = 10.0 # fix this for now
periodic = False
try:
opts, _ = getopt.getopt(argv, "t:R:D:C:P:N:A")
except getopt.GetoptError as err:
print(str(err))
print(
'testHeleShawFixedChill.py -t<max time> -R<Compositional Rayleigh Number> -D<Darcy number> '
'-C<Composition Ratio> -A<do amr> -P<true(1)/false(0) do plot files> -N<non dimensional reluctance>'
)
sys.exit(2)
for opt, arg in opts:
if opt in "-t":
extra_params['main.max_time'] = float(arg)
elif opt in "-R":
extra_params['parameters.rayleighComp'] = float(arg)
elif opt in "-D":
extra_params['parameters.darcy'] = float(arg)
elif opt in "-C":
extra_params['parameters.compositionRatio'] = float(arg)
elif opt in "-N":
extra_params['parameters.nonDimReluctance'] = float(arg)
elif opt in "-P":
do_plot_files = bool(int(arg))
print('Do plot files: ' + str(do_plot_files) + ', arg = ' +
str(arg))
if not do_plot_files:
extra_params['main.plot_interval'] = -1
elif opt in "-A":
do_amr = True
base_output_dir = get_base_output_dir()
print(Fore.GREEN + 'Setup tests for fixed chill in a Hele-Shaw cell' +
Style.RESET_ALL)
physical_problem = 'FixedChill'
folder_name = "FixedChill-t%1.1e-Ra%.0e-Da%1.1e-C%1.2f-Rel%1.1e" % (
extra_params['main.max_time'], extra_params['parameters.rayleighComp'],
extra_params['parameters.darcy'],
extra_params['parameters.compositionRatio'],
extra_params['parameters.nonDimReluctance'])
if periodic:
folder_name = folder_name + '-periodic'
data_folder = os.path.join(base_output_dir, folder_name)
analysis_command = get_matlab_base_command(
) + ' "%s; exit;"' % fixed_chill_command(folder_name + '-0')
single_run = True
if single_run:
num_proc = 1
default_params['concise_run_name'] = folder_name
default_params = add_params(default_params, extra_params)
# Overwrite default params with any extra params we've specified
default_params = add_params(default_params, extra_params)
allow_restarts = False
s = BatchJob('', default_params['concise_run_name'], '', num_proc)
s.set_post_process(analysis_command)
ml_run = MushyLayerRunSimple(base_output_dir, num_proc, default_params,
s, allow_restarts, get_executable_name())
ml_run.single_run(folder_name)
else:
# Nz_uniform = 256
# Nz_amr_2 = int(float(Nz_uniform) / 2)
# Nz_amr_4 = int(float(Nz_uniform) / 4)
# amr_setup = [{'max_level': 0, 'ref_rat': 1, 'run_types': ['uniform'], 'Nzs': [Nz_uniform]},
# {'max_level': 1, 'ref_rat': 2, 'run_types': ['amr'], 'Nzs': [Nz_amr_2]},
# {'max_level': 2, 'ref_rat': 2, 'run_types': ['amr'], 'Nzs': [Nz_amr_4]},
# {'max_level': 1, 'ref_rat': 4, 'run_types': ['amr'], 'Nzs': [Nz_amr_4]}]
# While testing:
nz_uniform = [64]
# amr_setup = [{'max_level': 0, 'ref_rat': 1, 'run_types': ['uniform'], 'Nzs': Nz_uniform},
# {'max_level': 1, 'ref_rat': 2, 'run_types': ['amr'], 'Nzs': [64]}]
amr_setup = [{
'max_level': 0,
'ref_rat': 1,
'run_types': ['uniform'],
'Nzs': nz_uniform
}]
if do_amr:
amr_setup.append({
'max_level': 1,
'ref_rat': 2,
'run_types': ['amr'],
'Nzs': nz_uniform
})
amr_setup.append({
'max_level': 2,
'ref_rat': 2,
'run_types': ['amr'],
'Nzs': nz_uniform
})
num_procs = [1] # Needs to be as long as the longest Nzs
run_test(data_folder, physical_problem,
hele_shaw_resolution_specific_params, amr_setup, num_procs,
analysis_command, extra_params, 0)
def test_folder(test_directory, verbose_output=False):
"""
Run the regression test contained within a folder. Expects the folder o have the following files:
- inputs
- properties.json
- at least one file ending with .expected to compare with the computed ouptut,
e.g. pout.0.expected, or plt000010.2d.hdf5.expected
:param test_directory:
:return: success - if test was successful or not
"""
logger.log('Processing folder "%s"' % test_directory)
remove_existing_diffs_cmd = 'rm %s/diff-*' % test_directory
logger.log('Remove existing diffs: ' + remove_existing_diffs_cmd)
os.system(remove_existing_diffs_cmd)
test_files = os.listdir(test_directory)
logger.log('Initial files in folder ' + str(test_files))
# Check the required files exist
# if not, skip this test
if not all(elem in test_files for elem in REQUIRED):
test_name = test_directory.split('/')[-1]
logger.logl('%-25s ' % test_name, console_display=True)
logger.log(
' Required files for conducting a test (%s) not found in %s' %
(REQUIRED, test_directory))
logger.log(' Skipping test \n')
logger.log_void()
return False, 'Void'
# Load properties
with open(os.path.join(test_directory, PROPERTIES_FILE)) as json_file:
properties = json.load(json_file)
# logger.log('==Running test: %s==' % properties['name'])
logger.logl('%-25s ' % properties['name'], console_display=True)
try:
mpi = subprocess.check_output(['which', 'mpiruna'])
mpi_path = str(mpi.decode()).strip()
except subprocess.CalledProcessError:
mpi = None
mpi_path = None
# Skip if parallel test and no mpirun
if properties['proc'] > 1 and mpi is None:
logger.log_void()
return False, 'Void'
# Get correct executable
# mushy_layer_exec = mushyLayerRunUtils.get_executable(dim=properties['dim'])
# mushy_layer_exec_path = os.path.join(mushyLayerRunUtils.get_mushy_layer_dir(), 'execSubcycle', mushy_layer_exec)
exec_name = 'mushyLayer%dd' % properties['dim']
exec_dir = os.path.join(mushyLayerRunUtils.get_mushy_layer_dir(),
'execSubcycle')
mushy_layer_exec_path = mushyLayerRunUtils.get_executable_name(
exec_dir, exec_name=exec_name, return_full_path=True)
if not os.path.exists(mushy_layer_exec_path):
logger.log('\n**Could not find mushy layer executable: %s' %
mushy_layer_exec_path)
logger.log(
'**Have you compiled the code for the right number of dimensions?')
logger.logl('**Use \'make all DIM=3\' to compile in 3D ')
logger.log_failed()
return False, 'Failed'
# Run test
# try:
if mpi is not None:
cmd = 'cd %s; %s -np %d %s inputs' % (test_directory, mpi_path,
properties['proc'],
mushy_layer_exec_path)
os.system(cmd)
# mpi_path = 'mpirun'
# res = subprocess.run([mpi_path, '-n ', str(properties['proc']), ' inputs'], cwd=test_directory)
else:
cmd = 'cd %s; %s inputs > pout.0' % (test_directory,
mushy_layer_exec_path)
logger.log(cmd)
res = subprocess.check_output(cmd, shell=True)
logger.logl('Response: "%s"' % str(res.decode()))
# os.system(cmd)
# except subprocess.CalledProcessError:
# logger.logl('[Exception]')
# logger.log_failed()
# return False, 'Failed'
# Compare output against the expected output
# logger.log('Test files: %s' % test_files)
expected_files = [f for f in test_files if EXPECTED in f and DIFF not in f]
if not expected_files:
logger.log('No expected files to compared against')
logger.log_void()
return False, 'Void'
failed_test = False
for expected_file in expected_files:
logger.log('Expected file: %s' % expected_file)
# Check an output file to compare against exists
test_output_filename = expected_file.replace(EXPECTED, '')
test_output_file_path = os.path.join(test_directory,
test_output_filename)
if not os.path.exists(test_output_file_path):
logger.log('No output file generated to compare against: %s' %
test_output_file_path)
logger.log_failed()
return False, 'Failed'
if '.hdf5' in expected_file:
# Do chombo compare
# logger.log('Using chombo compare')
# Make diffs folder if it doesn't exist
diffs_folder = os.path.join(test_directory, DIFF_FOLDER)
if not os.path.exists(diffs_folder):
os.makedirs(diffs_folder)
logger.log('Making folder %s' % diffs_folder)
chombo_dir = os.environ['CHOMBO_HOME']
compare_dir = os.path.join(chombo_dir, 'util', 'ChomboCompare')
compare_exec = mushyLayerRunUtils.get_executable_name(
compare_dir, 'compare%dd' % properties['dim'])
compare_exec = os.path.join(compare_dir, compare_exec)
# logger.log('Found executable %s ' % compare_exec)
if not os.path.exists(compare_exec):
logger.log('Could not find Chombo compare executable %s' %
compare_exec)
logger.log('So cannot compare hdf5 output files')
compare_params_file = os.path.join(test_directory,
'compare.inputs')
computed_file = os.path.join(test_directory, test_output_file_path)
exact_file = os.path.join(test_directory, expected_file)
error_file = os.path.join(diffs_folder,
DIFF + test_output_filename)
compare_params = {
'compare.sameSize': 1,
'compare.exactRoot': exact_file,
'compare.computedRoot': computed_file,
'compare.errorRoot': error_file,
'compare.doPlots': 1,
'compare.HOaverage': 0,
'compare.no_average_var': 'T err'
}
mushyLayerRunUtils.write_inputs(compare_params_file,
compare_params)
cmd = 'cd %s ; %s %s > pout.0' % (diffs_folder, compare_exec,
compare_params_file)
logger.log('Executing: %s' % cmd)
res = subprocess.check_call(cmd,
shell=True,
stderr=subprocess.STDOUT)
logger.log('Compare command run, response: %s' % res)
# Rename pout.0 in case we make lots
old_pout_name = os.path.join(diffs_folder, 'pout.0')
new_pout_name = os.path.join(diffs_folder,
'pout-%s.0' % test_output_filename)
# logger.log('Rename %s to %s' % (old_pout_name, new_pout_name))
if not os.path.exists(old_pout_name):
logger.log('Cannot find %s' % old_pout_name,
console_display=True)
os.rename(old_pout_name, new_pout_name)
# Now check the output diff
errs = chcompare.load_error_file(new_pout_name)
# logger.log(errs)
for field in errs.keys():
field_errs = errs[field]
# logger.log(field_errs)
for err_type in field_errs.keys():
this_field_err = field_errs[err_type]
if abs(this_field_err) > err_tolerance:
logger.log('Error in field %s is non-zero' % field)
logger.log('See %s and %s for more info' %
(new_pout_name, error_file))
# logger.log_failed()
# return False, 'Failed'
failed_test = True
elif '.csv' in expected_file:
data = pd.read_csv(
os.path.join(test_directory, test_output_file_path))
data_expected = pd.read_csv(
os.path.join(test_directory, expected_file))
data_keys = data.keys()
diff = data.copy()
for key in data_expected.keys():
if key not in data_keys:
logger.log('Key not found: %s' % key, verbose_output)
failed_test = True
break
tolerance = 1e-5
expected_value = data_expected[key].iloc[0]
value_diff = float(data[key].iloc[0] - data_expected[key])
# Compute relative different for large values
if abs(expected_value) > 1.0:
value_diff = value_diff / expected_value
diff[key] = value_diff
if abs(value_diff) > tolerance:
logger.log('Error in field %s = %.2g' % (key, value_diff),
verbose_output)
failed_test = True
diff.to_csv(os.path.join(test_directory,
'diff-%s' % expected_file))
else:
# Assume text file - do a diff
text1 = open(os.path.join(test_directory,
expected_file)).readlines()
text2 = open(os.path.join(test_directory,
test_output_file_path)).readlines()
text1 = filter_pout(text1)
text2 = filter_pout(text2)
diff = difflib.unified_diff(text1, text2)
# diff = difflib.ndiff(text1, text2)
differences = [line for line in diff]
diff_out_file = os.path.join(test_directory,
DIFF + test_output_filename)
with open(diff_out_file, 'w') as diff_file:
diff_file.writelines(differences)
logger.log('Diff: %s' % differences)
if differences:
logger.log('Differences found in %s' % test_output_file_path)
logger.log('For details, see %s' % diff_out_file)
# logger.log('** Test failed \n')
# logger.log_failed()
# return False, 'Failed'
# failed_test = True
if failed_test:
logger.log_failed()
return False, 'Failed'
else:
logger.log_ok()
return True, 'OK'
import mushyLayerRunUtils as util
import os
from valgrind_parser import ValgrindLogParser
import shutil
mem_leak_path = os.path.join(util.get_mushy_layer_dir(), 'test', 'memory_leak')
VALGRIND_LOG = os.path.join(mem_leak_path, 'valgrind_log.txt')
shutil.copy('valgrind_regexes.json', '../../env/lib/python3.6/site-packages/valgrind_parser/data/')
# Create inputs file and directory
inputs_base = os.path.join(util.get_mushy_layer_dir(), 'test', 'regression', 'AMRDarcyBrinkman', 'inputs')
inputs = util.read_inputs(inputs_base)
inputs['main.max_step'] = 5
util.write_inputs('inputs', inputs)
exec_file = util.get_executable_name(exec_name='mushyLayer2d', return_full_path=True)
# , os.path.join(mem_leak_path, 'inputs')
# --log-file="%s"
cmd = 'valgrind --child-silent-after-fork=yes --xml=yes --xml-file=valgrind.xml --leak-check=full ' \
'--show-reachable=yes %s inputs > %s ' % (exec_file, VALGRIND_LOG)
# print(cmd)
os.system(cmd)
vlp = ValgrindLogParser(VALGRIND_LOG, html_report_location='valgrind.html')
vlp.generate_html_report()
inputs['bc.TRefHi'] = [0, opt['Tref']]
inputs['bc.enthalpyLoVal'] = [0.0, T_max + st]
inputs['bc.enthalpyHiVal'] = [0.0, T_max + st]
inputs['parameters.nonDimVel'] = opt['V']
inputs['parameters.compositionRatio'] = cr
inputs['bc.HC_noflux'] = True
if include_solution_diffusion:
inputs['parameters.lewis'] = 10.0
inputs['main.output_folder'] = full_output_folder
new_inputs_loc = os.path.join(full_output_folder, 'inputs')
write_inputs(new_inputs_loc, inputs)
exec_loc = get_executable_name(return_full_path=True)
cmd = 'cd %s; %s inputs' % (full_output_folder, exec_loc)
print(cmd)
print('Running... (usually takes ~ 30 seconds) ')
process = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE)
process.wait()
print(process.returncode)
print('Finished')
# Run comparison
plt_files = [
f for f in os.listdir(full_output_folder)