def prepare_failed(compress=False,
wine_arch='win32',
wine_prefix='~/.wine32',
prelude='',
zipchunks=False):
cc = sim.Cases(POST_DIR, sim_id)
df_tags = cc.cases2df()
# -------------------------------------------------------------------------
# find failed cases and create pbs_in_failed dir
cc.find_failed(df_cases=df_tags)
sim.copy_pbs_in_failedcases(cc.cases_fail, path=opt.pbs_failed_path)
if zipchunks:
# and for chunks as well
sorts_on = ['[DLC]', '[Windspeed]']
create_chunks_htc_pbs(cc.cases_fail,
sort_by_values=sorts_on,
ppn=20,
nr_procs_series=3,
walltime='20:00:00',
chunks_dir='zip-chunks-jess-fail',
compress=compress,
wine_arch=wine_arch,
wine_prefix=wine_prefix,
prelude=prelude,
queue='windq',
i0=1000)
def create_inputs(self, iter_dict, opt_tags, runmethod='pbs'):
self.runmethod = runmethod
sim.prepare_launch(iter_dict, opt_tags, self.master, self._var_tag_func,
write_htc=True, runmethod=runmethod, verbose=False,
copyback_turb=False, msg='', update_cases=False,
ignore_non_unique=False, run_only_new=False,
pbs_fname_appendix=False, short_job_names=False,
windows_nr_cpus=2)
def plot_dlc00(sim_ids,
post_dirs,
run_dirs,
fig_dir_base=None,
labels=None,
cnames=[
'dlc00_stair_wsp04_25_noturb.htc',
'dlc00_ramp_wsp04_25_04_noturb.htc'
],
figsize=(14, 11)):
"""
This version is an update over plot_staircase.
"""
stairs = []
# if sim_id is a list, combine the two dataframes into one
if type(sim_ids).__name__ == 'list':
for ii, sim_id in enumerate(sim_ids):
if isinstance(post_dirs, list):
post_dir = post_dirs[ii]
else:
post_dir = post_dirs
stairs.append(sim.Cases(post_dir, sim_id, rem_failed=True))
else:
post_dir = post_dirs
stairs.append(sim.Cases(post_dir, sim_id, rem_failed=True))
for cname in cnames:
fp = PlotPerf(figsize=figsize)
for i, cc in enumerate(stairs):
if isinstance(cname, list):
_cname = cname[i]
else:
_cname = cname
if _cname in cc.cases_fail:
print('no result for %s' % cc.sim_id)
continue
cc.change_results_dir(run_dirs[i])
try:
res = cc.load_result_file(cc.cases[_cname])
except KeyError:
for k in sorted(cc.cases.keys()):
print(k)
print('-' * 79)
print(cc.sim_id, _cname)
print('-' * 79)
raise KeyError
if labels is not None:
label = labels[i]
else:
label = cc.sim_id
fp.plot(res, label)
dlcf = 'dlc' + cc.cases[_cname]['[DLC]']
fig_path = os.path.join(fig_dir_base, dlcf)
fp.final(fig_path, _cname.replace('.htc', '.png'))
def solve_nonlinear(self, params, unknowns, resids):
for i in range(self.n_cases):
p_name = 'outputs_statistics_%d' % i
out = params[p_name]
for j, var in enumerate(self.stat_var):
unknowns[var][i, :] = out[j, :]
for i in range(self.n_cases):
p_name = 'outputs_fatigue_%d' % i
out = params[p_name]
for j, var in enumerate(self.fatigue_var):
unknowns[var][i, :] = out[j, :]
if self.ch_envelope != []:
for j, var in enumerate(self.envelope_var):
p_list = []
envelope_cloud = {}
for i in range(self.n_cases):
p_name = 'outputs_envelope_%d' % i
p_list.append(p_name)
out = params[p_name]
cloud = out[:,6*self.Nrblades*j:6*self.Nrblades*(j+1)]
envelope_cloud[p_name] = \
cloud.reshape([(self.Nx_envelope+1)*self.Nrblades,6])
vec = \
Simulations.compute_env_of_env(envelope_cloud, p_list,
Nx=(self.Nx_envelope + 1) *
self.Nrblades - 1,
Nsectors=self.Nsectors_env)
# swap envelope columns to get vectors with
# Fx Fy Fz Mx My Mz
unknowns[var] = vec[:, [3, 4, 5, 0, 1, 2]] * 1.e3
def solve_nonlinear(self, params, unknowns, resids):
for i in range(self.n_cases):
p_name = 'outputs_statistics_%d' % i
out = params[p_name]
for j, var in enumerate(self.stat_var):
unknowns[var][i, :] = out[j, :]
for i in range(self.n_cases):
p_name = 'outputs_fatigue_%d' % i
out = params[p_name]
for j, var in enumerate(self.fatigue_var):
unknowns[var][i, :] = out[j, :]
if self.ch_envelope != []:
for j, var in enumerate(self.envelope_var):
p_list = []
envelope_cloud = {}
for i in range(self.n_cases):
p_name = 'outputs_envelope_%d' % i
p_list.append(p_name)
out = params[p_name]
cloud = out[:, 6 * self.Nrblades * j:6 * self.Nrblades *
(j + 1)]
envelope_cloud[p_name] = \
cloud.reshape([(self.Nx_envelope+1)*self.Nrblades,6])
vec = \
Simulations.compute_env_of_env(envelope_cloud, p_list,
Nx=(self.Nx_envelope + 1) *
self.Nrblades - 1,
Nsectors=self.Nsectors_env)
# swap envelope columns to get vectors with
# Fx Fy Fz Mx My Mz
unknowns[var] = vec[:, [3, 4, 5, 0, 1, 2]] * 1.e3
def post_processing(self, statistics=True, resdir=None, complib='blosc',
calc_mech_power=False):
"""
Parameters
----------
resdir : str, default=None
Defaults to reading the results from the [run_dir] tag.
Force to any other directory using this variable. You can also use
the presets as defined for runmethod in _set_path_config.
"""
post_dir = self.POST_DIR
# =========================================================================
# check logfiles, results files, pbs output files
# logfile analysis is written to a csv file in logfiles directory
# =========================================================================
# load the file saved in post_dir
cc = sim.Cases(post_dir, self.sim_id, rem_failed=False, complib=complib)
if isinstance(resdir, str):
forcedir = os.path.join(resdir, self.PROJECT, self.sim_id)
cc.change_results_dir(forcedir)
cc.post_launch()
cc.remove_failed()
if statistics:
tags=['[windspeed]']
stats_df = cc.statistics(calc_mech_power=calc_mech_power,
ch_fatigue=[], tags=tags, update=False)
ftarget = os.path.join(self.POST_DIR, '%s_statistics.xlsx')
stats_df.to_excel(ftarget % self.sim_id)
def __init__(self, sim_id, P_MASTERFILE, MASTERFILE, P_SOURCE, P_RUN,
PROJECT, POST_DIR, master_tags_default):
"""
Create HtcMaster() object
=========================
the HtcMaster contains all the settings to start creating htc files.
It holds the master file, server paths and more.
The master.tags dictionary holds those tags who do not vary for different
cases. Variable tags, i.e. tags who are a function of other variables
or other tags, are defined in the function variable_tag_func().
It is considered as good practice to define the default values for all
the variable tags in the master_tags
Parameters
----------
sim_id : str
P_MASTERFILE : str
MASTERFILE : str
P_SOURCE : str
P_RUN : str
PROJECT : str
POST_DIR : str
master_tags_default : dict
Dictionary with the default master tag values. Should be created
by the turbine specific class Configurations.set_master_defaults()
Members
-------
Returns
-------
"""
self.sim_id = sim_id
self.P_MASTERFILE = P_MASTERFILE
self.MASTERFILE = MASTERFILE
self.P_SOURCE = P_SOURCE
self.P_RUN = P_RUN
self.PROJECT = PROJECT
self.POST_DIR = POST_DIR
# TODO: write a lot of logical tests for the tags!!
# TODO: tests to check if the dirs are setup properly (ending slahses)
# FIXME: some tags are still variable! Only static tags here that do
# not depent on any other variable that can change
self.master = sim.HtcMaster()
self.master.tags.update(master_tags_default)
def postpro_node_merge(tqdm=False, zipchunks=False, m=[3, 4, 6, 8, 9, 10, 12]):
"""With postpro_node each individual case has a .csv file for the log file
analysis and a .csv file for the statistics tables. Merge all these single
files into one table/DataFrame.
When using the zipchunk approach, all log file analysis and statistics
are grouped into tar archives in the prepost-data directory.
Parameters
----------
tqdm : boolean, default=False
Set to True for displaying a progress bar (provided by the tqdm module)
when merging all csv files into a single table/pd.DataFrame.
zipchunks : boolean, default=False
Set to True if merging post-processing files grouped into tar archives
as generated by the zipchunks approach.
"""
# -------------------------------------------------------------------------
# MERGE POSTPRO ON NODE APPROACH INTO ONE DataFrame
# -------------------------------------------------------------------------
lf = windIO.LogFile()
path_pattern = os.path.join(P_RUN, 'logfiles', '*', '*.csv')
if zipchunks:
path_pattern = os.path.join(POST_DIR, 'loganalysis_chnk*.tar.xz')
csv_fname = '%s_ErrorLogs.csv' % sim_id
fcsv = os.path.join(POST_DIR, csv_fname)
mdf = AppendDataFrames(tqdm=tqdm)
# individual log file analysis does not have header, make sure to include
# a line for the header
mdf.txt2txt(fcsv,
path_pattern,
tarmode='r:xz',
header=None,
header_fjoined=lf._header(),
recursive=True)
# FIXME: this is due to bug in log file analysis. What is going on here??
# fix that some cases do not have enough columns
with open(fcsv.replace('.csv', '2.csv'), 'w') as f1:
with open(fcsv) as f2:
for line in f2.readlines():
if len(line.split(';')) == 96:
line = line.replace(';0.00000000000;nan;-0.0000;',
'0.00000000000;nan;-0.0000;')
f1.write(line)
# convert from CSV to DataFrame
df = lf.csv2df(fcsv.replace('.csv', '2.csv'))
df.to_hdf(fcsv.replace('.csv', '.h5'), 'table')
# -------------------------------------------------------------------------
path_pattern = os.path.join(P_RUN, 'res', '*', '*.csv')
csv_fname = '%s_statistics.csv' % sim_id
if zipchunks:
path_pattern = os.path.join(POST_DIR, 'statsdel_chnk*.tar.xz')
fcsv = os.path.join(POST_DIR, csv_fname)
mdf = AppendDataFrames(tqdm=tqdm)
# individual log file analysis does not have header, make sure to include
# a line for the header
mdf.txt2txt(fcsv,
path_pattern,
tarmode='r:xz',
header=0,
sep=',',
header_fjoined=None,
recursive=True,
fname_col='[case_id]')
# and convert to df: takes 2 minutes
fdf = fcsv.replace('.csv', '.h5')
store = pd.HDFStore(fdf,
mode='w',
format='table',
complevel=9,
complib='zlib')
colnames = [
'channel', 'max', 'min', 'mean', 'std', 'range', 'absmax', 'rms',
'int', 'intabs'
]
colnames.extend(['m=%1.0f' % k for k in m])
colnames.extend(['[case_id]'])
dtypes = {col: np.float64 for col in colnames}
dtypes['channel'] = str
dtypes['[case_id]'] = str
# when using min_itemsize the column names should be valid variable names
# mitemsize = {'channel':60, '[case_id]':60}
mdf.csv2df_chunks(store,
fcsv,
chunksize=1000000,
min_itemsize={},
sep=',',
colnames=colnames,
dtypes=dtypes,
header=0)
store.close()
# -------------------------------------------------------------------------
# merge missing cols onto stats
# FIXME: HAS TO BE THE SAME AS tags IN post_launch
required = [
'[DLC]', '[run_dir]', '[wdir]', '[Windspeed]', '[res_dir]',
'[case_id]', '[Case folder]'
]
df = pd.read_hdf(fdf, 'table')
# FIXME: why do some cases have a leading ./ (but most do not)?
sel = df['[case_id]'].str.startswith('./')
df.loc[sel, '[case_id]'] = df.loc[sel,
'[case_id]'].str.replace('./', '', 1)
# df now has case_id as the path to the statistics file: res/dlc12_xxx/yyy
# while df_tags will have just yyy as case_id
tmp = df['[case_id]'].str.split('/', expand=True)
df['[case_id]'] = tmp[tmp.columns[-1]]
cc = sim.Cases(POST_DIR, sim_id)
df_tags = cc.cases2df()
df_stats = pd.merge(df, df_tags[required], on=['[case_id]'])
# if the merge didn't work due to other misaligned case_id tags, do not
# overwrite our otherwise ok tables!
if len(df_stats) != len(df):
print('failed to merge required tags, something is wrong!')
# find out which cases we lost and why
print('number of entries lost:', len(df) - len(df_stats))
s_df = set(df['[case_id]'].unique())
s_stats = set(df_stats['[case_id]'].unique())
print('nr of channels:', len(df['channel'].unique()))
msg = 'nr of case_ids lost:'
print(msg, (len(df) - len(df_stats)) / len(df['channel'].unique()))
print('following case_ids have mysteriously disappeared:')
print(s_df - s_stats)
return
df_stats.to_hdf(fdf, 'table', mode='w')
df_stats.to_csv(fdf.replace('.h5', '.csv'))
# -------------------------------------------------------------------------
# save channel list
chans = df_stats['channel'].unique()
chans.sort()
fname = os.path.join(POST_DIR, '%s_unique-channel-names.csv' % sim_id)
pd.DataFrame(chans).to_csv(fname)
def master_tags(sim_id, runmethod='local', silent=False, verbose=False):
"""
Create HtcMaster() object
=========================
the HtcMaster contains all the settings to start creating htc files.
It holds the master file, server paths and more.
The master.tags dictionary holds those tags who do not vary for different
cases. Variable tags, i.e. tags who are a function of other variables
or other tags, are defined in the function variable_tag_func().
It is considered as good practice to define the default values for all
the variable tags in the master_tags
Members
-------
Returns
-------
"""
# TODO: write a lot of logical tests for the tags!!
# TODO: tests to check if the dirs are setup properly (ending slahses ...)
# FIXME: some tags are still variable! Only static tags here that do
# not depent on any other variable that can change
master = sim.HtcMaster(verbose=verbose, silent=silent)
# set the default tags
master = dlcdefs.tags_defaults(master)
# =========================================================================
# SOURCE FILES
# =========================================================================
# # TODO: move to variable_tag
# rpl = (p_root, project, sim_id)
# if runmethod in ['local', 'local-script', 'none', 'local-ram']:
# master.tags['[run_dir]'] = '%s/%s/%s/' % rpl
# elif runmethod == 'windows-script':
# master.tags['[run_dir]'] = '%s/%s/%s/' % rpl
# elif runmethod == 'gorm':
# master.tags['[run_dir]'] = '%s/%s/%s/' % rpl
# elif runmethod == 'jess':
# master.tags['[run_dir]'] = '%s/%s/%s/' % rpl
# else:
# msg='unsupported runmethod, options: none, local, gorm or opt'
# raise ValueError, msg
master.tags['[master_htc_file]'] = MASTERFILE
master.tags['[master_htc_dir]'] = P_MASTERFILE
# directory to data, htc, SOURCE DIR
if P_SOURCE[-1] == os.sep:
master.tags['[model_dir_local]'] = P_SOURCE
else:
master.tags['[model_dir_local]'] = P_SOURCE + os.sep
if P_RUN[-1] == os.sep:
master.tags['[run_dir]'] = P_RUN
else:
master.tags['[run_dir]'] = P_RUN + os.sep
master.tags['[post_dir]'] = POST_DIR
master.tags['[sim_id]'] = sim_id
# set the model_zip tag to include the sim_id
master.tags['[model_zip]'] = PROJECT
master.tags['[model_zip]'] += '_' + master.tags['[sim_id]'] + '.zip'
# -------------------------------------------------------------------------
# FIXME: this is very ugly. We should read default values set in the htc
# master file with the HAWC2Wrapper !!
# default tags turbulence generator (required for 64-bit Mann generator)
# alfaeps, L, gamma, seed, nr_u, nr_v, nr_w, du, dv, dw high_freq_comp
master.tags['[MannAlfaEpsilon]'] = 1.0
master.tags['[MannL]'] = 29.4
master.tags['[MannGamma]'] = 3.0
master.tags['[seed]'] = None
master.tags['[turb_nr_u]'] = 8192
master.tags['[turb_nr_v]'] = 32
master.tags['[turb_nr_w]'] = 32
master.tags['[turb_dx]'] = 1
master.tags['[turb_dy]'] = 6.5
master.tags['[turb_dz]'] = 6.5
master.tags['[high_freq_comp]'] = 1
# -------------------------------------------------------------------------
return master
def post_launch(sim_id,
statistics=True,
rem_failed=True,
check_logs=True,
force_dir=False,
update=False,
saveinterval=2000,
csv=False,
m=[3, 4, 6, 8, 9, 10, 12],
neq=1e7,
no_bins=46,
int_env=False,
years=20.0,
fatigue=True,
A=None,
AEP=False,
nx=300,
save_new_sigs=False,
envelopeturbine=False,
envelopeblade=False,
save_iter=False,
pbs_failed_path=False):
# =========================================================================
# check logfiles, results files, pbs output files
# logfile analysis is written to a csv file in logfiles directory
# =========================================================================
# load the file saved in post_dir
config = {}
config['Weibull'] = {}
config['Weibull']['Vr'] = 11.
config['Weibull']['Vref'] = 50
config['nn_shaft'] = 4
cc = sim.Cases(POST_DIR, sim_id, rem_failed=rem_failed, config=config)
if force_dir:
for case in cc.cases:
cc.cases[case]['[post_dir]'] = POST_DIR
cc.cases[case]['[run_dir]'] = force_dir
if check_logs:
cc.post_launch(save_iter=save_iter, pbs_failed_path=pbs_failed_path)
elif rem_failed:
cc.remove_failed()
# using suffix is only relevant if we have more cases then the save interval
if len(cc.cases) > saveinterval:
suffix = True
else:
suffix = False
df_stats, df_AEP, df_Leq = None, None, None
if statistics:
i0, i1 = 0, -1
# example for combination of signals
# name = 'stress1'
# expr = '[p1-p1-node-002-forcevec-z]*3 + [p1-p1-node-002-forcevec-y]'
# add_sigs = {name:expr}
# in addition, sim_id and case_id are always added by default
# FIXME: HAS TO BE THE SAME AS required IN postpro_node_merge
tags = [
'[Case folder]', '[run_dir]', '[res_dir]', '[DLC]', '[wsp]',
'[Windspeed]', '[wdir]'
]
add = None
# general statistics for all channels channel
# set neq=None here to calculate 1Hz equivalent loads
df_stats = cc.statistics(calc_mech_power=False,
i0=i0,
i1=i1,
tags=tags,
add_sensor=add,
ch_fatigue=None,
update=update,
saveinterval=saveinterval,
suffix=suffix,
save_new_sigs=save_new_sigs,
csv=csv,
m=m,
neq=None,
no_bins=no_bins,
chs_resultant=[],
A=A,
add_sigs={})
# save channel list
chans = df_stats['channel'].unique()
chans.sort()
fname = os.path.join(POST_DIR, '%s_unique-channel-names.csv' % sim_id)
pd.DataFrame(chans).to_csv(fname)
# annual energy production
if AEP:
# load the statistics in case they are missing
if not statistics:
df_stats, Leq_df, AEP_df = cc.load_stats()
# CAUTION: depending on the type output, electrical power can be two
# different things with the DTU Wind Energy Controller.
# Either manually set ch_powe to the correct value or use simple
# mechanism to figure out which one of two expected values it is.
if 'DLL-2-inpvec-2' in df_stats['channel'].unique():
ch_powe = 'DLL-2-inpvec-2'
elif 'DLL-dtu_we_controller-inpvec-2' in df_stats['channel'].unique():
ch_powe = 'DLL-dtu_we_controller-inpvec-2'
df_AEP = cc.AEP(df_stats,
csv=csv,
update=update,
save=True,
ch_powe=ch_powe)
if envelopeblade:
ch_list = []
for iblade in range(1, 4):
for i in range(1, 18):
rpl = (iblade, iblade, i)
ch_list.append([
'blade%i-blade%i-node-%3.3i-momentvec-x' % rpl,
'blade%i-blade%i-node-%3.3i-momentvec-y' % rpl,
'blade%i-blade%i-node-%3.3i-momentvec-z' % rpl,
'blade%i-blade%i-node-%3.3i-forcevec-x' % rpl,
'blade%i-blade%i-node-%3.3i-forcevec-y' % rpl,
'blade%i-blade%i-node-%3.3i-forcevec-z' % rpl
])
cc.envelopes(ch_list=ch_list, append='_blade', int_env=int_env, Nx=nx)
if envelopeturbine:
ch_list = [[
'tower-tower-node-001-momentvec-x',
'tower-tower-node-001-momentvec-y',
'tower-tower-node-001-momentvec-z'
],
[
'tower-tower-node-022-momentvec-x',
'tower-tower-node-022-momentvec-y',
'tower-tower-node-022-momentvec-z',
'tower-tower-node-022-forcevec-x',
'tower-tower-node-022-forcevec-y',
'tower-tower-node-022-forcevec-z'
],
[
'hub1-hub1-node-001-momentvec-x',
'hub1-hub1-node-001-momentvec-y',
'hub1-hub1-node-001-momentvec-z'
]]
cc.envelopes(ch_list=ch_list,
append='_turbine',
int_env=int_env,
Nx=nx)
if fatigue:
# load the statistics in case they are missing
if not statistics:
df_stats, Leq_df, AEP_df = cc.load_stats()
# life time equivalent load for all channels
df_Leq = cc.fatigue_lifetime(df_stats,
neq,
csv=csv,
update=update,
years=years,
save=True)
return df_stats, df_AEP, df_Leq
def launch_dlcs_excel(sim_id,
silent=False,
verbose=False,
pbs_turb=False,
runmethod=None,
write_htc=True,
zipchunks=False,
walltime='04:00:00',
postpro_node=False,
compress=False,
dlcs_dir='htc/DLCs',
postpro_node_zipchunks=True,
wine_arch='win32',
wine_prefix='~/.wine32',
m=[3, 4, 6, 8, 9, 10, 12],
prelude='',
linux=False):
"""
Launch load cases defined in Excel files
"""
iter_dict = dict()
iter_dict['[empty]'] = [False]
if postpro_node or postpro_node_zipchunks:
# pyenv = 'py36-wetb'
pyenv = 'wetb_py3'
else:
pyenv = None
# FIXME: THIS IS VERY MESSY, we have wine_prefix/arch and exesingle/chunks
if linux:
wine_arch = None
wine_prefix = None
prelude = 'module load mpi/openmpi_1.6.5_intelv14.0.0\n'
# if linux:
# pyenv = 'wetb_py3'
# pyenv_cmd = 'source /home/python/miniconda3/bin/activate'
# exesingle = "{hawc2_exe:} {fname_htc:}"
# exechunks = "({winenumactl:} {hawc2_exe:} {fname_htc:}) "
# exechunks += "2>&1 | tee {fname_pbs_out:}"
# else:
# pyenv = ''
# pyenv_cmd = 'source /home/ozgo/bin/activate_hawc2cfd.sh'
# exesingle = "time {hawc2_exe:} {fname_htc:}"
# exechunks = "(time numactl --physcpubind=$CPU_NR {hawc2_exe:} {fname_htc:}) "
# exechunks += "2>&1 | tee {fname_pbs_out:}"
# see if a htc/DLCs dir exists
# Load all DLC definitions and make some assumptions on tags that are not
# defined
if os.path.exists(dlcs_dir):
opt_tags = dlcdefs.excel_stabcon(dlcs_dir,
silent=silent,
p_source=P_SOURCE)
else:
opt_tags = dlcdefs.excel_stabcon(os.path.join(P_SOURCE, 'htc'),
silent=silent,
p_source=P_SOURCE)
if len(opt_tags) < 1:
raise ValueError('There are is not a single case defined. Make sure '
'the DLC spreadsheets are configured properly.')
# add all the root files, except anything with *.zip
f_ziproot = []
for (dirpath, dirnames, fnames) in os.walk(P_SOURCE):
# remove all zip files
for i, fname in enumerate(fnames):
if not fname.endswith('.zip'):
f_ziproot.append(fname)
break
# and add those files
for opt in opt_tags:
opt['[zip_root_files]'] = f_ziproot
if runmethod == None:
runmethod = RUNMETHOD
master = master_tags(sim_id,
runmethod=runmethod,
silent=silent,
verbose=verbose)
master.tags['[sim_id]'] = sim_id
master.tags['[walltime]'] = walltime
master.output_dirs.append('[Case folder]')
master.output_dirs.append('[Case id.]')
# TODO: copy master and DLC exchange files to p_root too!!
# all tags set in master_tags will be overwritten by the values set in
# variable_tag_func(), iter_dict and opt_tags
# values set in iter_dict have precedence over opt_tags vartag_func()
# has precedense over iter_dict, which has precedence over opt_tags.
# dlcdefs.vartag_excel_stabcon adds support for creating hydro files
vartag_func = dlcdefs.vartag_excel_stabcon
cases = sim.prepare_launch(iter_dict,
opt_tags,
master,
vartag_func,
write_htc=write_htc,
runmethod=runmethod,
copyback_turb=True,
update_cases=False,
msg='',
ignore_non_unique=False,
run_only_new=False,
pbs_fname_appendix=False,
short_job_names=False,
silent=silent,
verbose=verbose,
pyenv=pyenv,
m=[3, 4, 6, 8, 9, 10, 12],
postpro_node=postpro_node,
exechunks=None,
exesingle=None,
prelude=prelude,
postpro_node_zipchunks=postpro_node_zipchunks,
wine_arch=wine_arch,
wine_prefix=wine_prefix)
if pbs_turb:
# to avoid confusing HAWC2 simulations and Mann64 generator PBS files,
# MannTurb64 places PBS launch scripts in a "pbs_in_turb" folder
mann64 = sim.MannTurb64(silent=silent)
mann64.walltime = '00:59:59'
mann64.queue = 'workq'
mann64.gen_pbs(cases)
if zipchunks:
# create chunks
# sort so we have minimal copying turb files from mimer to node/scratch
# note that walltime here is for running all cases assigned to the
# respective nodes. It is not walltime per case.
sorts_on = ['[DLC]', '[Windspeed]']
create_chunks_htc_pbs(cases,
sort_by_values=sorts_on,
queue='workq',
ppn=20,
nr_procs_series=3,
walltime='20:00:00',
chunks_dir='zip-chunks-jess',
compress=compress,
wine_arch=wine_arch,
wine_prefix=wine_prefix,
prelude=prelude)
# create_chunks_htc_pbs(cases, sort_by_values=sorts_on, queue='workq',
# ppn=12, nr_procs_series=3, walltime='20:00:00',
# chunks_dir='zip-chunks-gorm', compress=compress,
# wine_arch=wine_arch, wine_prefix=wine_prefix)
df = sim.Cases(cases).cases2df()
df.to_excel(os.path.join(POST_DIR, sim_id + '.xls'))
def plot_staircase(sim_ids,
post_dirs,
run_dirs,
fig_dir_base=None,
cname='dlc00_stair_wsp04_25_noturb.htc'):
"""
Default stair and ramp names:
dlc00_stair_wsp04_25_noturb
dlc00_ramp_wsp04_25_04_noturb
"""
stairs = []
col = ['r', 'k']
alf = [1.0, 0.7]
# if sim_id is a list, combine the two dataframes into one
if type(sim_ids).__name__ == 'list':
for ii, sim_id in enumerate(sim_ids):
if isinstance(post_dirs, list):
post_dir = post_dirs[ii]
else:
post_dir = post_dirs
stairs.append(sim.Cases(post_dir, sim_id, rem_failed=True))
else:
sim_id = sim_ids
sim_ids = [sim_id]
post_dir = post_dirs
stairs.append(sim.Cases(post_dir, sim_id, rem_failed=True))
fig, axes = mplutils.make_fig(nrows=3, ncols=1, figsize=(14, 10))
ax = axes.ravel()
for i, cc in enumerate(stairs):
if cname in cc.cases_fail:
print('no result for %s' % cc.sim_id)
continue
cc.change_results_dir(run_dirs[i])
res = cc.load_result_file(cc.cases[cname])
respath = cc.cases[cname]['[run_dir]']
fname = os.path.join(respath, cname)
df_respost = pd.read_hdf(fname + '_postres.h5', 'table')
sim_id = cc.sim_id
time = res.sig[:, 0]
t0, t1 = time[0], time[-1]
# find the wind speed
for channame, chan in res.ch_dict.items():
if channame.startswith('windspeed-global-Vy-0.00-0.00'):
break
wind = res.sig[:, chan['chi']]
chi = res.ch_dict['bearing-pitch1-angle-deg']['chi']
pitch = res.sig[:, chi]
chi = res.ch_dict['bearing-shaft_rot-angle_speed-rpm']['chi']
rpm = res.sig[:, chi]
chi = res.ch_dict['bearing-pitch1-angle-deg']['chi']
pitch = res.sig[:, chi]
chi = res.ch_dict['tower-tower-node-001-momentvec-x']['chi']
tx = res.sig[:, chi]
chi = res.ch_dict['tower-tower-node-001-momentvec-y']['chi']
ty = res.sig[:, chi]
chi = res.ch_dict['DLL-2-inpvec-2']['chi']
power = res.sig[:, chi]
chi = res.ch_dict['DLL-2-inpvec-2']['chi']
power_mech = df_respost['stats-shaft-power']
ax[0].plot(time,
wind,
col[i] + '--',
label='%s wind speed' % sim_id,
alpha=alf[i])
ax[0].plot(time,
pitch,
col[i] + '-.',
label='%s pitch' % sim_id,
alpha=alf[i])
ax[0].plot(time,
rpm,
col[i] + '-',
label='%s RPM' % sim_id,
alpha=alf[i])
ax[1].plot(time,
tx,
col[i] + '--',
label='%s Tower FA' % sim_id,
alpha=alf[i])
ax[1].plot(time,
ty,
col[i] + '-',
label='%s Tower SS' % sim_id,
alpha=alf[i])
ax[2].plot(time,
power / 1e6,
col[i] + '-',
label='%s El Power' % sim_id,
alpha=alf[i])
ax[2].plot(time,
power_mech / 1e3,
col[i] + '-',
alpha=alf[i],
label='%s Mech Power' % sim_id)
ax[0].set_xlim([t0, t1])
ax[0].grid()
ax[0].legend(loc='best')
ax[0].set_xticklabels([])
# ax[0].set_xlabel('time [s]')
ax[1].set_xlim([t0, t1])
ax[1].grid()
ax[1].legend(loc='best')
ax[1].set_xticklabels([])
# ax[1].set_xlabel('time [s]')
ax[2].set_xlim([t0, t1])
ax[2].grid()
ax[2].legend(loc='best')
ax[2].set_xlabel('time [s]')
fig.tight_layout()
fig.subplots_adjust(hspace=0.06)
fig.subplots_adjust(top=0.92)
if not os.path.exists(fig_dir_base):
os.makedirs(fig_dir_base)
fig_path = os.path.join(fig_dir_base, '-'.join(sim_ids) + '_stair.png')
print('saving: %s ...' % fig_path, end='')
fig.savefig(fig_path) #.encode('latin-1')
print('done')
fig.clear()
def load_stats(self, sim_ids, post_dirs, post_dir_save=False):
self.sim_ids = sim_ids
self.post_dirs = post_dirs
# reduce required memory, only use following columns
cols = [
'[run_dir]', '[DLC]', 'channel', '[res_dir]', '[Windspeed]',
'mean', 'max', 'min', 'std', '[wdir]'
]
# if sim_id is a list, combine the two dataframes into one
df_stats = pd.DataFrame()
if type(sim_ids).__name__ == 'list':
for ii, sim_id in enumerate(sim_ids):
if isinstance(post_dirs, list):
post_dir = post_dirs[ii]
else:
post_dir = post_dirs
cc = sim.Cases(post_dir, sim_id, rem_failed=True)
df_stats, _, _ = cc.load_stats(columns=cols, leq=False)
print('%s Cases loaded.' % sim_id)
# if specified, save the merged sims elsewhere
if post_dir_save:
fpath = os.path.join(post_dir_save,
'-'.join(sim_ids) + '.h5')
try:
os.makedirs(post_dir_save)
except OSError:
pass
else:
fpath = os.path.join(post_dir, '-'.join(sim_ids) + '.h5')
if ii == 0:
# and save somewhere so we can add the second data frame on
# disc
df_stats.to_hdf(fpath,
'table',
mode='w',
format='table',
complevel=9,
complib='blosc')
print('%s merged stats written to: %s' % (sim_id, fpath))
else:
# instead of doing a concat in memory, add to the hdf store
df_stats.to_hdf(fpath,
'table',
mode='r+',
format='table',
complevel=9,
complib='blosc',
append=True)
print('%s merging stats into: %s' % (sim_id, fpath))
# we might run into memory issues
del df_stats, _, cc
gc.collect()
# and load the reduced combined set
print('loading merged stats: %s' % fpath)
df_stats = pd.read_hdf(fpath, 'table')
else:
sim_id = sim_ids
sim_ids = [sim_id]
post_dir = post_dirs
cc = sim.Cases(post_dir, sim_id, rem_failed=True)
df_stats, _, _ = cc.load_stats(leq=False)
return df_stats
def merge_sim_ids(sim_ids, post_dirs, post_dir_save=False, columns=None):
"""
"""
cols_extra = ['[run_dir]', '[res_dir]', '[wdir]', '[DLC]', '[Case folder]']
min_itemsize = {
'channel': 100,
'[run_dir]': 100,
'[res_dir]': 100,
'[DLC]': 10,
'[Case folder]': 100
}
# map the run_dir to the same order as the post_dirs, labels
run_dirs = []
# avoid saving merged cases if there is only one!
if type(sim_ids).__name__ == 'list' and len(sim_ids) == 1:
sim_ids = sim_ids[0]
# if sim_id is a list, combine the two dataframes into one
df_stats = pd.DataFrame()
if type(sim_ids).__name__ == 'list':
for ii, sim_id in enumerate(sim_ids):
if isinstance(post_dirs, list):
post_dir = post_dirs[ii]
else:
post_dir = post_dirs
cc = sim.Cases(post_dir, sim_id, rem_failed=True)
df_stats, _, _ = cc.load_stats(leq=False)
if columns is not None:
df_stats = df_stats[columns]
# stats has only a few columns identifying the different cases
# add some more for selecting them
dfc = cc.cases2df()
if '[wsp]' in dfc.columns:
wsp = '[wsp]'
else:
wsp = '[Windspeed]'
# columns we want to add from cc.cases (cases dict) to stats
cols_cc = set(cols_extra + [wsp])
# do not add column twice, some might already be in df stats
add_cols = list(cols_cc - set(df_stats.columns))
add_cols.append('[case_id]')
dfc = dfc[add_cols]
df_stats = pd.merge(df_stats, dfc, on='[case_id]')
# FIXME: this is very messy, we can end up with both [wsp] and
# [Windspeed] columns
if '[Windspeed]' in df_stats.columns and '[wsp]' in df_stats.columns:
df_stats.drop('[wsp]', axis=1, inplace=True)
if wsp != '[Windspeed]':
df_stats.rename(columns={wsp: '[Windspeed]'}, inplace=True)
# map the run_dir to the same order as the post_dirs, labels
run_dirs.append(df_stats['[run_dir]'].unique()[0])
print('%s Cases loaded.' % sim_id)
# if specified, save the merged sims elsewhere
if post_dir_save:
fpath = os.path.join(post_dir_save, '-'.join(sim_ids) + '.h5')
try:
os.makedirs(post_dir_save)
except OSError:
pass
else:
fpath = os.path.join(post_dir, '-'.join(sim_ids) + '.h5')
fmerged = fpath.replace('.h5', '_statistics.h5')
if ii == 0:
# and save somewhere so we can add the second data frame on
# disc
store = pd.HDFStore(fmerged,
mode='w',
complevel=9,
complib='zlib')
store.append('table', df_stats, min_itemsize=min_itemsize)
print(store.get_storer('table').table.description)
# df_stats.to_hdf(fmerged, 'table', mode='w', format='table',
# complevel=9, complib='blosc')
print('%s merged stats written to: %s' % (sim_id, fpath))
else:
# instead of doing a concat in memory, add to the hdf store
store.append('table', df_stats)
# will fail if there are longer string columns compared to ii=0
# df_stats.to_hdf(fmerged, 'table', mode='r+', format='table',
# complevel=9, complib='blosc', append=True)
print('%s merging stats into: %s' % (sim_id, fpath))
# df_stats = pd.concat([df_stats, df_stats2], ignore_index=True)
# df_stats2 = None
# we might run into memory issues
del df_stats, _, cc
gc.collect()
store.close()
# and load the reduced combined set
print('loading merged stats: %s' % fmerged)
df_stats = pd.read_hdf(fmerged, 'table')
else:
sim_id = sim_ids
sim_ids = [sim_id]
post_dir = post_dirs
if isinstance(post_dirs, list):
post_dir = post_dirs[0]
cc = sim.Cases(post_dir, sim_id, rem_failed=True)
df_stats, _, _ = cc.load_stats(columns=columns, leq=False)
if columns is not None:
df_stats = df_stats[columns]
run_dirs = [df_stats['[run_dir]'].unique()[0]]
# stats has only a few columns identifying the different cases
# add some more for selecting them
dfc = cc.cases2df()
if '[wsp]' in dfc.columns:
wsp = '[wsp]'
else:
wsp = '[Windspeed]'
# columns we want to add from cc.cases (cases dict) to stats
cols_cc = set(cols_extra + [wsp])
# do not add column twice, some might already be in df stats
add_cols = list(cols_cc - set(df_stats.columns))
add_cols.append('[case_id]')
dfc = dfc[add_cols]
df_stats = pd.merge(df_stats, dfc, on='[case_id]')
if '[Windspeed]' in df_stats.columns and '[wsp]' in df_stats.columns:
df_stats.drop('[wsp]', axis=1, inplace=True)
if wsp != '[Windspeed]':
df_stats.rename(columns={wsp: '[Windspeed]'}, inplace=True)
return run_dirs, df_stats
def test_leq_life(self):
"""Verify if prepost.Simulation.Cases.fatigue_lifetime() returns
the expected life time equivalent load.
"""
# ---------------------------------------------------------------------
# very simple case
cases = {'case1':{'[post_dir]':'no-path', '[sim_id]':'A0'},
'case2':{'[post_dir]':'no-path', '[sim_id]':'A0'}}
cc = sim.Cases(cases)
fh_list = [('case1', 10/3600), ('case2', 20/3600)]
dfs = pd.DataFrame({'m=1.0' : [2, 3],
'channel' : ['channel1', 'channel1'],
'[case_id]' : ['case1', 'case2']})
neq_life = 1.0
df_Leq = cc.fatigue_lifetime(dfs, neq_life, fh_lst=fh_list,
save=False, update=False, csv=False,
xlsx=False, silent=False)
np.testing.assert_allclose(df_Leq['m=1.0'].values, 2*10 + 3*20)
self.assertTrue(df_Leq['channel'].values[0]=='channel1')
# ---------------------------------------------------------------------
# slightly more complicated
neq_life = 3.0
df_Leq = cc.fatigue_lifetime(dfs, neq_life, fh_lst=fh_list,
save=False, update=False, csv=False,
xlsx=False, silent=False)
np.testing.assert_allclose(df_Leq['m=1.0'].values,
(2*10 + 3*20)/neq_life)
# ---------------------------------------------------------------------
# a bit more complex and also test the sorting of fh_lst and dfs
cases = {'case1':{'[post_dir]':'no-path', '[sim_id]':'A0'},
'case2':{'[post_dir]':'no-path', '[sim_id]':'A0'},
'case3':{'[post_dir]':'no-path', '[sim_id]':'A0'},
'case4':{'[post_dir]':'no-path', '[sim_id]':'A0'}}
cc = sim.Cases(cases)
fh_list = [('case3', 10/3600), ('case2', 20/3600),
('case1', 50/3600), ('case4', 40/3600)]
dfs = pd.DataFrame({'m=3.0' : [2, 3, 4, 5],
'channel' : ['channel1']*4,
'[case_id]' : ['case4', 'case2', 'case3', 'case1']})
neq_life = 5.0
df_Leq = cc.fatigue_lifetime(dfs, neq_life, fh_lst=fh_list,
save=False, update=False, csv=False,
xlsx=False, silent=False)
expected = ((2*2*2*40 + 3*3*3*20 + 4*4*4*10 + 5*5*5*50)/5)**(1/3)
np.testing.assert_allclose(df_Leq['m=3.0'].values, expected)
# ---------------------------------------------------------------------
# more cases and with sorting
base = {'[post_dir]':'no-path', '[sim_id]':'A0'}
cases = {'case%i' % k : base for k in range(50)}
cc = sim.Cases(cases)
# reverse the order of how they appear in dfs and fh_lst
fh_list = [('case%i' % k, k*10/3600) for k in range(49,-1,-1)]
dfs = pd.DataFrame({'m=5.2' : np.arange(1,51,1),
'channel' : ['channel1']*50,
'[case_id]' : ['case%i' % k for k in range(50)]})
df_Leq = cc.fatigue_lifetime(dfs, neq_life, fh_lst=fh_list,
save=False, update=False, csv=False,
xlsx=False, silent=False)
expected = np.sum(np.power(np.arange(1,51,1), 5.2)*np.arange(0,50,1)*10)
expected = np.power(expected/neq_life, 1/5.2)
np.testing.assert_allclose(df_Leq['m=5.2'].values, expected)
