from mule_local.JobMule import *
from mule.plotting.Plotting import *
from mule.postprocessing.JobsData import *
from mule.postprocessing.JobsDataConsolidate import *
import matplotlib.pyplot as plt
from matplotlib.lines import Line2D
groups = ['runtime.timestepping_method', 'runtime.timestepping_order']
tagnames_y = [
'plane_data_diff_prog_u.res_norm_linf',
]
j = JobsData(verbosity=0)
c = JobsDataConsolidate(j)
print("")
print("Groups:")
job_groups = c.create_groups(groups)
for key, g in job_groups.items():
print(" + " + key)
# Filter out errors beyond this value!
def data_filter(x, y, jobdata):
if y == None:
return True
def _setup(
self,
ref_file_ending=None,
jobdir_pattern=None,
params=[],
):
if jobdir_pattern == None:
jobdir_pattern = './job_bench*'
j = JobsData(jobdir_pattern, verbosity=0)
jobs = j.get_flattened_data()
no_reference_job_unique_id_found = True
if len(jobs) == 0:
raise Exception("No jobs found!")
for key, job in jobs.items():
print("Processing " + key)
# job directory path
job_dirpath = job['jobgeneration.job_dirpath']
# files
data_files = glob.glob(job_dirpath + '/output*prog*.csv')
if len(data_files) == 0:
self.info("WARNING")
self.info("WARNING: No files found")
self.info(
"WARNING: However, there should be at least one file (the one at t=0)"
)
self.info("WARNING")
# Iterate over all files
for data_file in data_files:
# Skip files stored in spectral space
if '_spec' in data_file:
continue
try:
s = PlaneData_Spectrum(data_file, params)
except FileNotFoundError as e:
# Ignoring missing files should be configured via "ignore_missing_file" parameter, see above
if "ignore_missing_file" in params:
self.info("Ignoring Error:")
self.info(str(e))
continue
raise Exception(e)
except IOError as e:
# Ignoring missing files should be configured via "ignore_missing_file" parameter, see above
if "ignore_missing_file" in params:
self.info("Ignoring Error:")
self.info(str(e))
continue
raise Exception(e)
# determine (time-depending) ending of reference file
pickle_filename = data_file.replace('.csv', '_spectrum.pickle')
print("Writing file " + pickle_filename)
s.write_file(pickle_filename)
muletag = sys.argv[1]
output_filename = sys.argv[2]
else:
print("")
print("Usage:")
print("")
print(
" " + sys.argv[0] +
" [jobdata mule tag for y axis] [output_filename.pdf] [jobdir1] [jobdir2] ... [jobdirN]"
)
print("")
sys.exit(1)
if len(sys.argv) > 3:
# Load Jobs specified via program parameters
jd = JobsData(job_dirs=sys.argv[3:])
else:
# Load all Jobs
jd = JobsData()
# Consolidate data...
jdc = JobsDataConsolidate(jd)
# ... which belongs to the same time integration method
jdc_groups = jdc.create_groups(['runtime.timestepping_method'])
#
# Filter to exclude data which indicates instabilities
#
def data_filter(x, y, jd):
sys.path.append('../')
import pretty_plotting as pp
sys.path.pop()
mule_plotting_usetex(False)
groups = ['runtime.timestepping_method']
tagnames_y = [
'sphere_data_diff_prog_phi_pert.res_norm_l1',
'sphere_data_diff_prog_phi_pert.res_norm_l2',
'sphere_data_diff_prog_phi_pert.res_norm_linf',
]
j = JobsData('./job_bench_*', verbosity=0)
c = JobsDataConsolidate(j)
print("")
print("Groups:")
job_groups = c.create_groups(groups)
for key, g in job_groups.items():
print(key)
for tagname_y in tagnames_y:
params = []
params += [
{
'tagname_x': 'runtime.timestep_size',
'xlabel': "Timestep size (seconds)",
def _setup(
self,
ref_file_ending = None,
jobdir_pattern = None,
job_dirs = None,
params = [],
):
if job_dirs != None:
j = JobsData(job_dirs=job_dirs, verbosity=0)
else:
if jobdir_pattern == None:
jobdir_pattern = './job_bench*'
j = JobsData(jobdir_pattern, verbosity=0)
jobs = j.get_flattened_data()
no_reference_job_unique_id_found = True
if len(jobs) == 0:
raise Exception("No jobs found!")
for key, job in jobs.items():
print("Processing "+key)
# job directory path
job_dirpath = job['jobgeneration.job_dirpath']
# u-velocity
u_vel_files = glob.glob(job_dirpath+'/output_*_u_*.csv')
if len(u_vel_files) == 0:
self.info("WARNING")
self.info("WARNING: No velocity files found")
self.info("WARNING: However, there should be at least one velocity file (the one at t=0)")
self.info("WARNING")
# only process the very last file
if 'only_last_file' in params:
u_vel_files.sort()
u_vel_files = [u_vel_files[-1]]
# determine (time-depending) ending of reference file
pos = u_vel_files[0].rfind('_')
if pos < 0:
raise Exception("File ending not found for reference file '"+u_vel_files[0]+"'")
# Iterate over all velocity files for different time stamps
for u_vel_file in u_vel_files:
v_vel_file = u_vel_file.replace('_u_', '_v_')
try:
s = PlaneData_KineticEnergy(
u_vel_file,
v_vel_file,
params
)
except FileNotFoundError as e:
# Ignoring missing files should be configured via "ignore_missing_file" parameter, see above
if "ignore_missing_file" in params:
self.info("Ignoring Error:")
self.info(str(e))
continue
raise Exception(e)
except IOError as e:
# Ignoring missing files should be configured via "ignore_missing_file" parameter, see above
if "ignore_missing_file" in params:
self.info("Ignoring Error:")
self.info(str(e))
continue
raise Exception(e)
#s.print()
if 'only_last_file' in params:
pickle_filename = 'plane_data_kinetic_energy.pickle'
else:
# determine (time-depending) ending of reference file
pos = u_vel_files[0].rfind('_')
if pos < 0:
raise Exception("File ending not found for reference file '"+u_vel_files[0]+"'")
pickle_file_ending = u_vel_file[pos:]
pickle_file_ending = pickle_file_ending.replace('.csv', '')
print("pickle_file_ending: "+pickle_file_ending)
pickle_filename = 'plane_data_kinetic_energy'+pickle_file_ending+'.pickle'
print("Writing file "+pickle_filename)
s.write_file(job['jobgeneration.job_dirpath']+'/'+pickle_filename)
if len(sys.argv) >= 3:
#return d['plane_data_kinetic_energy.spectrum']
muletag = sys.argv[1]
output_filename = sys.argv[2]
else:
print("")
print("Usage:")
print("")
print(" " + sys.argv[0] +
" [output_filename.pdf] [jobdir1] [jobdir2] ... [jobdirN]")
print("")
sys.exit(1)
if len(sys.argv) >= 4:
# Load Jobs specified via program parameters
jd = JobsData(job_dirs=sys.argv[3:])
else:
# Load all Jobs
jd = JobsData()
data = jd.get_flattened_data()
def label(d):
val = d['runtime.timestepping_method'].replace(
'_', '\\_') + ', $\Delta t = ' + str(d['runtime.timestep_size']) + '$'
return val
def x_values(d):
return d[muletag + '.spectrum_wavelength']
def _setup(
self,
ref_file_ending = None,
jobdir_pattern = None,
job_dirs = None,
params = [],
):
if job_dirs != None:
j = JobsData(job_dirs=job_dirs, verbosity=0)
else:
if jobdir_pattern == None:
jobdir_pattern = './job_bench*'
j = JobsData(jobdir_pattern, verbosity=0)
jobs = j.get_flattened_data()
no_reference_job_unique_id_found = True
if len(jobs) == 0:
raise Exception("No jobs found!")
for key, job in jobs.items():
print("Processing "+key)
# Sort out jobs which don't have a reference job id
# These jobs are likely the reference jobs themselves
if 'jobgeneration.reference_job_unique_id' not in job:
continue
no_reference_job_unique_id_found = False
reference_job_unique_id = job['jobgeneration.reference_job_unique_id']
print(" + ref job id: "+reference_job_unique_id)
ref_key = None
for skey, sjob in jobs.items():
if sjob['jobgeneration.job_unique_id'] == reference_job_unique_id:
if ref_key != None:
raise Exception("FATAL: Reference job already found and now there's another one? This is probably not what you wanted, there might be 2 reference jobs")
ref_key = skey
if ref_key == None:
print("")
print("FATAL: missing reference job with id "+reference_job_unique_id)
print("")
print("FATAL: reference job was intended for job with dirpath: "+job['jobgeneration.job_dirpath'])
print("")
print("FATAL: Hint: If specifying job directories manually, reference job *MUST* be included in the provided job directories!")
print("")
raise Exception("Reference job not found!")
# Load reference job
ref_job = jobs[ref_key]
#
# Load
# ref_files: list of reference files
# use_ref_file_ending: file ending for pickle output file
#
# Were the reference filenames provided?
# ...
if 'output.reference_filenames' in sjob:
# ... then we use the reference files
# They are available in 'output.reference_filenames' and separated by ';'
ref_files = sjob['output.reference_filenames'].split(";")
#
# Now we have to find the file ending without the time stamp
# We do this to generate a unique pickle file which is independent of the time
#
# We guess that this starts at the last '_' character in the filename
# E.g. '_t00000864000.00000000.csv'
pos = ref_files[0].rfind('_')
if pos < 0:
raise Exception("File ending not found for reference file '"+ref_files[0]+"'")
use_ref_file_ending = ref_files[0][pos:]
print("use_ref_file_ending: "+use_ref_file_ending)
else:
if ref_file_ending != None:
use_ref_file_ending = ref_file_ending
else:
print("*"*80)
# "output_%s_t%020.8f.csv"
use_ref_file_ending = "_t{:020.8f}.csv".format(float(ref_job['runtime.max_simulation_time'])/(60*60))
if use_ref_file_ending == "":
raise Exception("No reference file ending provided / found")
# Load reference files
ref_files = []
files = os.listdir(ref_job['jobgeneration.job_dirpath'])
for f in files:
if use_ref_file_ending in f:
ref_files.append(f)
if len(ref_files) == 0:
print("No reference files found!")
print("*"*80)
print("Reference directory: "+ref_job['jobgeneration.job_dirpath'])
print("Reference file endings: "+use_ref_file_ending)
print("*"*80)
raise Exception("Reference files not found!")
for ref_file in ref_files:
print("")
print("Reference file: "+ref_file)
if '_spec_' in ref_file:
self.info("WARNING: Skipping '"+ref_file+"', since this is spectral data")
else:
s = None
try:
s = PlaneDataPhysicalDiff(
ref_job['jobgeneration.job_dirpath']+'/'+ref_file,
job['jobgeneration.job_dirpath']+'/'+ref_file,
params
)
except FileNotFoundError as e:
# Ignoring missing files should be configured via "ignore_missing_file" parameter, see above
if "ignore_missing_file" in params:
self.info("Ignoring Error:")
self.info(str(e))
continue
raise Exception(e)
except IOError as e:
# Ignoring missing files should be configured via "ignore_missing_file" parameter, see above
if "ignore_missing_file" in params:
self.info("Ignoring Error:")
self.info(str(e))
continue
raise Exception(e)
s.print()
pickle_filename = 'plane_data_diff_'+ref_file.replace('output_', '').replace(use_ref_file_ending, '')+'.pickle'
print("Writing file "+pickle_filename)
s.write_file(job['jobgeneration.job_dirpath']+'/'+pickle_filename)
print(ref_key)
print("")
if no_reference_job_unique_id_found:
print("*"*80)
print("Warning: No data generated")
print("No job with a reference_job_unique_id found!")
print("*"*80)
def _setup(
self,
ref_file_ending=None,
jobdir_pattern=None,
):
if jobdir_pattern == None:
jobdir_pattern = './job_bench*'
j = JobsData(jobdir_pattern, verbosity=0)
jobs = j.get_flattened_data()
no_reference_job_unique_id_found = True
if len(jobs) == 0:
raise Exception("No jobs found!")
for key, job in jobs.items():
print("Processing " + key)
# Sort out jobs which don't have a reference job id
# These jobs are likely the reference jobs themselves
if 'jobgeneration.reference_job_unique_id' not in job:
continue
no_reference_job_unique_id_found = False
reference_job_unique_id = job[
'jobgeneration.reference_job_unique_id']
print(" + ref job id: " + reference_job_unique_id)
ref_key = None
for skey, sjob in jobs.items():
if sjob['jobgeneration.job_unique_id'] == reference_job_unique_id:
ref_key = skey
if ref_key == None:
print("Fatal: missing reference job with id " +
reference_job_unique_id)
print(
"Fatal: reference job was intended for job with dirpath: "
+ job['jobgeneration.job_dirpath'])
raise Exception("Reference job not found!")
# Load reference job
ref_job = jobs[ref_key]
if ref_file_ending != None:
use_ref_file_ending = ref_file_ending
else:
# "output_%s_t%020.8f.csv"
use_ref_file_ending = "_t{:020.8f}.csv".format(
float(ref_job['runtime.max_simulation_time']) / (60 * 60))
if use_ref_file_ending == "":
raise Exception("No reference file ending provided / found")
# Load reference files
ref_files = []
files = os.listdir(ref_job['jobgeneration.job_dirpath'])
for f in files:
if use_ref_file_ending in f:
ref_files.append(f)
if len(ref_files) == 0:
print("No reference files found!")
print("*" * 80)
print("Reference directory: " +
ref_job['jobgeneration.job_dirpath'])
print("Reference file endings: " + use_ref_file_ending)
print("*" * 80)
raise Exception("Reference files not found!")
for ref_file in ref_files:
s = None
try:
s = SphereDataPhysicalDiff(
ref_job['jobgeneration.job_dirpath'] + '/' + ref_file,
job['jobgeneration.job_dirpath'] + '/' + ref_file)
except Exception as e:
print("Error occured which is ignored")
print(str(e))
# Ignore missing files
continue
s.print()
pickle_filename = 'sphere_data_diff_' + ref_file.replace(
'output_', '').replace(use_ref_file_ending, '') + '.pickle'
print("Writing file " + pickle_filename)
s.write_file(job['jobgeneration.job_dirpath'] + '/' +
pickle_filename)
print(ref_key)
print("")
if no_reference_job_unique_id_found:
print("*" * 80)
print("Warning: No data generated")
print("No job with a reference_job_unique_id found!")
print("*" * 80)
def arraylist_remove_last_col(data):
data_new = [[None for i in range(len(data[0]) - 1)]
for j in range(len(data))]
for j in range(len(data)):
for i in range(len(data[0]) - 1):
data_new[j][i] = data[j][i]
return data_new
num_ensembles = 10
ensemble_data = []
for ensemble_id in range(num_ensembles):
j = JobsData('job_bench_*ensemble' + str(ensemble_id).zfill(2) + '*',
verbosity=100)
c = JobsDataConsolidate(j)
d = JobsData_DataTable(j, 'parallelization.num_ranks', bar_data)
data = d.get_data_float()
if True:
#if False:
"""
Add last column 'nl_timestepping'
"""
data_new = [[None for i in range(len(data[0]) + 1)]
for j in range(len(data))]
for j in range(len(data)):
for i in range(len(data[0])):
for i in range(n):
title_in = title_in + " (" + str(obj.nmodes[i]) + ";" + str(
obj.mmodes[i]) + ")"
else:
for i in range(3):
title_in = title_in + " (" + str(obj.nmodes[i]) + ";" + str(
obj.mmodes[i]) + ")"
title_in = title_in + " + small energy n=[2,7] "
print(title_in)
#------------------------------
# Get jobs data (sweet things)
#---------------------------
print("Extracting jobs")
j = JobsData(basedir + '/job_bench_*', verbosity=1)
jobs = j.get_jobs_data()
print("...done")
#print(jobs)
main_tag = 'runtime.benchmark_name'
alpha_tag = 'output.benchmark_barotropic_vort_modes.0.ampl' #mode 0 is used as reference
jobs_flat = []
jobs_raw = []
alphas = []
umax = []
vmax = []
job_dirs = []
#print("jobs extracted")
#print(jobs)
print("-----------------------------------")
def _setup(
self,
job_directories = None,
cmp_file_ending = None,
jobdir_pattern = None,
):
if job_directories != None:
j = JobsData(job_dirs = job_directories, verbosity=0)
else:
if jobdir_pattern == None:
jobdir_pattern = './job_bench*'
j = JobsData(jobdir_pattern, verbosity=0)
jobs = j.get_flattened_data()
no_reference_job_unique_id_found = True
if len(jobs) == 0:
raise Exception("No jobs found!")
def proc_job(key, job):
print("Processing "+key)
# Sort out jobs which don't have a reference job id
# These jobs are likely the reference jobs themselves
if 'jobgeneration.reference_job_unique_id' not in job:
return
no_reference_job_unique_id_found = False
reference_job_unique_id = job['jobgeneration.reference_job_unique_id']
print(" + ref job id: "+reference_job_unique_id)
ref_key = None
for skey, sjob in jobs.items():
if sjob['jobgeneration.job_unique_id'] == reference_job_unique_id:
ref_key = skey
if ref_key == None:
print("Fatal: missing reference job with id "+reference_job_unique_id)
print("Fatal: reference job was intended for job with dirpath: "+job['jobgeneration.job_dirpath'])
raise Exception("Reference job not found!")
# Load reference job
ref_job = jobs[ref_key]
if cmp_file_ending != None:
use_cmp_file_ending = cmp_file_ending
else:
# "output_%s_t%020.8f.csv"
use_cmp_file_ending = "_t{:020.8f}.sweet".format(float(ref_job['runtime.max_simulation_time'])/(60*60))
if use_cmp_file_ending == "":
raise Exception("No comparison file ending provided / found")
# Load comparison files
cmp_files = []
files = os.listdir(ref_job['jobgeneration.job_dirpath'])
for f in files:
if use_cmp_file_ending in f:
cmp_files.append(f)
if len(cmp_files) == 0:
print("No reference files found!")
print("*"*80)
print("Reference directory: "+ref_job['jobgeneration.job_dirpath'])
print("Comparison file endings: "+use_cmp_file_ending)
print("*"*80)
raise Exception("Reference files not found!")
for cmp_file in cmp_files:
s = None
try:
s = SphereDataSpectralDiff(
ref_job['jobgeneration.job_dirpath']+'/'+cmp_file,
job['jobgeneration.job_dirpath']+'/'+cmp_file
)
except Exception as e:
#raise e
#raise Exception("Error occured which is ignored")
print(str(e))
# Ignore missing files
return
s.print()
pickle_filename = 'sphere_data_diff_'+cmp_file.replace('output_', '').replace(use_cmp_file_ending, '')+'.pickle'
print("Writing file "+pickle_filename)
s.write_file(job['jobgeneration.job_dirpath']+'/'+pickle_filename)
print(ref_key)
print("")
k = list(jobs.keys())
v = list(jobs.values())
n = len(jobs)
def fun(i):
try:
proc_job(k[i], v[i])
except:
raise
if 0:
# Doesn't really provide any speedup :-(
parhelper(n, fun, None, use_tqdm=True, verbose=1)
else:
for i in range(len(k)):
fun(i)
if no_reference_job_unique_id_found:
print("*"*80)
print("Warning: No data generated")
print("No job with a reference_job_unique_id found!")
print("*"*80)
