def create_inputs(self, iter_dict, opt_tags, runmethod='pbs'):
self.runmethod = runmethod
sim.prepare_launch(iter_dict, opt_tags, self.master, self._var_tag_func,
write_htc=True, runmethod=runmethod, verbose=False,
copyback_turb=False, msg='', update_cases=False,
ignore_non_unique=False, run_only_new=False,
pbs_fname_appendix=False, short_job_names=False,
windows_nr_cpus=2)
def launch_dlcs_excel(sim_id,
silent=False,
verbose=False,
pbs_turb=False,
runmethod=None,
write_htc=True,
zipchunks=False,
walltime='04:00:00',
postpro_node=False,
compress=False,
dlcs_dir='htc/DLCs',
postpro_node_zipchunks=True,
wine_arch='win32',
wine_prefix='~/.wine32',
m=[3, 4, 6, 8, 9, 10, 12],
prelude='',
linux=False):
"""
Launch load cases defined in Excel files
"""
iter_dict = dict()
iter_dict['[empty]'] = [False]
if postpro_node or postpro_node_zipchunks:
# pyenv = 'py36-wetb'
pyenv = 'wetb_py3'
else:
pyenv = None
# FIXME: THIS IS VERY MESSY, we have wine_prefix/arch and exesingle/chunks
if linux:
wine_arch = None
wine_prefix = None
prelude = 'module load mpi/openmpi_1.6.5_intelv14.0.0\n'
# if linux:
# pyenv = 'wetb_py3'
# pyenv_cmd = 'source /home/python/miniconda3/bin/activate'
# exesingle = "{hawc2_exe:} {fname_htc:}"
# exechunks = "({winenumactl:} {hawc2_exe:} {fname_htc:}) "
# exechunks += "2>&1 | tee {fname_pbs_out:}"
# else:
# pyenv = ''
# pyenv_cmd = 'source /home/ozgo/bin/activate_hawc2cfd.sh'
# exesingle = "time {hawc2_exe:} {fname_htc:}"
# exechunks = "(time numactl --physcpubind=$CPU_NR {hawc2_exe:} {fname_htc:}) "
# exechunks += "2>&1 | tee {fname_pbs_out:}"
# see if a htc/DLCs dir exists
# Load all DLC definitions and make some assumptions on tags that are not
# defined
if os.path.exists(dlcs_dir):
opt_tags = dlcdefs.excel_stabcon(dlcs_dir,
silent=silent,
p_source=P_SOURCE)
else:
opt_tags = dlcdefs.excel_stabcon(os.path.join(P_SOURCE, 'htc'),
silent=silent,
p_source=P_SOURCE)
if len(opt_tags) < 1:
raise ValueError('There are is not a single case defined. Make sure '
'the DLC spreadsheets are configured properly.')
# add all the root files, except anything with *.zip
f_ziproot = []
for (dirpath, dirnames, fnames) in os.walk(P_SOURCE):
# remove all zip files
for i, fname in enumerate(fnames):
if not fname.endswith('.zip'):
f_ziproot.append(fname)
break
# and add those files
for opt in opt_tags:
opt['[zip_root_files]'] = f_ziproot
if runmethod == None:
runmethod = RUNMETHOD
master = master_tags(sim_id,
runmethod=runmethod,
silent=silent,
verbose=verbose)
master.tags['[sim_id]'] = sim_id
master.tags['[walltime]'] = walltime
master.output_dirs.append('[Case folder]')
master.output_dirs.append('[Case id.]')
# TODO: copy master and DLC exchange files to p_root too!!
# all tags set in master_tags will be overwritten by the values set in
# variable_tag_func(), iter_dict and opt_tags
# values set in iter_dict have precedence over opt_tags vartag_func()
# has precedense over iter_dict, which has precedence over opt_tags.
# dlcdefs.vartag_excel_stabcon adds support for creating hydro files
vartag_func = dlcdefs.vartag_excel_stabcon
cases = sim.prepare_launch(iter_dict,
opt_tags,
master,
vartag_func,
write_htc=write_htc,
runmethod=runmethod,
copyback_turb=True,
update_cases=False,
msg='',
ignore_non_unique=False,
run_only_new=False,
pbs_fname_appendix=False,
short_job_names=False,
silent=silent,
verbose=verbose,
pyenv=pyenv,
m=[3, 4, 6, 8, 9, 10, 12],
postpro_node=postpro_node,
exechunks=None,
exesingle=None,
prelude=prelude,
postpro_node_zipchunks=postpro_node_zipchunks,
wine_arch=wine_arch,
wine_prefix=wine_prefix)
if pbs_turb:
# to avoid confusing HAWC2 simulations and Mann64 generator PBS files,
# MannTurb64 places PBS launch scripts in a "pbs_in_turb" folder
mann64 = sim.MannTurb64(silent=silent)
mann64.walltime = '00:59:59'
mann64.queue = 'workq'
mann64.gen_pbs(cases)
if zipchunks:
# create chunks
# sort so we have minimal copying turb files from mimer to node/scratch
# note that walltime here is for running all cases assigned to the
# respective nodes. It is not walltime per case.
sorts_on = ['[DLC]', '[Windspeed]']
create_chunks_htc_pbs(cases,
sort_by_values=sorts_on,
queue='workq',
ppn=20,
nr_procs_series=3,
walltime='20:00:00',
chunks_dir='zip-chunks-jess',
compress=compress,
wine_arch=wine_arch,
wine_prefix=wine_prefix,
prelude=prelude)
# create_chunks_htc_pbs(cases, sort_by_values=sorts_on, queue='workq',
# ppn=12, nr_procs_series=3, walltime='20:00:00',
# chunks_dir='zip-chunks-gorm', compress=compress,
# wine_arch=wine_arch, wine_prefix=wine_prefix)
df = sim.Cases(cases).cases2df()
df.to_excel(os.path.join(POST_DIR, sim_id + '.xls'))