def confirm_add_additional_file(folder, file):
if os.path.isfile(os.path.join(self.modelpath, folder, file)):
filename = fmt_path(os.path.join(folder, file))
if self.get_confirmation("File missing", "'%s' seems to be missing in the temporary working directory. \n\nDo you want to add it to additional_files.txt" % filename):
self.add_additional_input_file(filename)
self.show_message(
"'%s' is now added to additional_files.txt.\n\nPlease restart the simulation" % filename)
def fmt(dst):
if os.path.isabs(dst):
dst = os.path.relpath(os.path.abspath(dst), self.exepath)
else:
dst = os.path.relpath(dst)
dst = fmt_path(dst)
assert not os.path.relpath(os.path.join(self.exepath, dst), self.modelpath).startswith(
".."), "%s referes to a file outside the model path\nAll input files be inside model path" % dst
return dst
def finish_simulation(self):
if self.status == ABORTED:
return
if self.status != ERROR:
self.status = FINISHING
def fmt(dst):
if os.path.isabs(dst):
dst = os.path.relpath(os.path.abspath(dst), self.exepath)
else:
dst = os.path.relpath(dst)
dst = fmt_path(dst)
assert not os.path.relpath(
os.path.join(self.exepath, dst), self.modelpath
).startswith(
".."
), "%s referes to a file outside the model path\nAll input files be inside model path" % dst
return dst
turb_files = [
f for f in self.htcFile.turbulence_files() if self.copy_turbulence
and not os.path.isfile(os.path.join(self.exepath, f))
]
if self.ios:
output_patterns = ["../output/*", "../output/"] + turb_files + \
[os.path.join(self.exepath, self.stdout_filename)]
else:
output_patterns = self.htcFile.output_files() + turb_files + \
[os.path.join(self.exepath, self.stdout_filename)]
output_files = set(
self.host.glob([
fmt_path(os.path.join(self.tmp_exepath, fmt(p)))
for p in output_patterns
],
recursive=self.ios))
if not os.path.isdir(
os.path.dirname(self.exepath + self.stdout_filename)):
os.makedirs(os.path.dirname(self.exepath + self.stdout_filename))
try:
self.host._finish_simulation(output_files)
if self.status != ERROR:
self.status = CLEANED
except Warning as e:
self.errors.append(str(e))
if self.status != ERROR:
self.status = CLEANED
except Exception as e:
self.errors.append(str(e))
raise
finally:
self.set_id(self.filename)
self.logFile.reset()
self.htcFile.reset()
def __init__(self,
modelpath,
htcfilename,
hawc2exe="HAWC2MB.exe",
copy_turbulence=True):
self.modelpath = os.path.abspath(modelpath) + "/"
if os.path.isabs(htcfilename):
htcfilename = os.path.relpath(htcfilename, modelpath)
if htcfilename.startswith("input/"):
htcfilename = htcfilename[6:]
exists = [
os.path.isfile(os.path.join(modelpath, htcfilename)),
os.path.isfile(os.path.join(modelpath, "input/", htcfilename))
]
if all(exists):
raise Exception(
"Both standard and input/output file structure available for %s in %s. Delete one of the options"
% (htcfilename, modelpath))
if not any(exists):
raise Exception("%s not found in %s" % (htcfilename, modelpath))
else:
self.ios = exists[1] # input/output file structure
if self.ios:
self.exepath = self.modelpath + "input/"
else:
self.exepath = self.modelpath
# model_path: top level path containing all resources
# exepath: parent path for relative paths
htcfilename = fmt_path(htcfilename)
self.tmp_modelpath = self.exepath
self.folder = os.path.dirname(htcfilename)
self.filename = os.path.basename(htcfilename)
self.htcFile = HTCFile(os.path.join(self.exepath, htcfilename),
self.exepath)
self.time_stop = self.htcFile.simulation.time_stop[0]
self.hawc2exe = hawc2exe
self.copy_turbulence = copy_turbulence
self.simulation_id = (
htcfilename.replace("\\", "/").replace("/", "_")[:50] +
"_%d" % id(self))
if self.simulation_id.startswith("input_"):
self.simulation_id = self.simulation_id[6:]
self.stdout_filename = fmt_path(
os.path.join(os.path.relpath(self.exepath, self.modelpath),
(os.path.splitext(htcfilename)[0] + ".out").replace(
'htc', 'stdout', 1)))
if self.ios:
assert self.stdout_filename.startswith("input/")
self.stdout_filename = self.stdout_filename.replace(
"input/", "../output/")
# self.stdout_filename = "stdout/%s.out" % self.simulation_id
if 'logfile' in self.htcFile.simulation:
self.log_filename = self.htcFile.simulation.logfile[0]
else:
self.log_filename = self.stdout_filename
if os.path.isabs(self.log_filename):
self.log_filename = os.path.relpath(self.log_filename,
self.modelpath)
else:
self.log_filename = os.path.relpath(self.log_filename)
self.log_filename = fmt_path(self.log_filename)
self.logFile = LogFile(os.path.join(self.exepath, self.log_filename),
self.time_stop)
self.logFile.clear()
self.last_status = self.status
self.errors = []
self.non_blocking_simulation_thread = Thread(
target=lambda: self.simulate_distributed(raise_simulation_errors=
False))
self.updateSimStatusThread = UpdateSimStatusThread(self)
from wetb.hawc2.simulation_resources import LocalSimulationHost
self.host = LocalSimulationHost(self)
self.stdout = ""
self.returncode = 0
def pbsjobfile(self, ios=False):
cp_back = ""
for folder in set([
fmt_path(os.path.dirname(os.path.relpath(f)))
for f in self.htcFile.output_files() +
self.htcFile.turbulence_files()
]):
if folder == '':
continue
cp_back += "mkdir -p $PBS_O_WORKDIR/%s/. \n" % folder
cp_back += "cp -R -f %s/. $PBS_O_WORKDIR/%s/.\n" % (folder, folder)
rel_htcfilename = fmt_path(
os.path.relpath(self.htcFile.filename, self.exepath))
try:
steps = self.htcFile.simulation.time_stop[
0] / self.htcFile.simulation.newmark.deltat[0]
walltime = "%02d:00:00" % np.ceil(steps / 500 / 60)
except Exception:
walltime = "04:00:00"
init = """
### Standard Output
#PBS -N h2l_%s
### merge stderr into stdout
#PBS -j oe
#PBS -o %s
### Maximum wallclock time format HOURS:MINUTES:SECONDS
#PBS -l walltime=%s
###PBS -a 201547.53
#PBS -lnodes=1:ppn=1
### Queue name
#PBS -q %s
### Create scratch directory and copy data to it
cd $PBS_O_WORKDIR
pwd""" % (self.simulation_id, self.stdout_filename, walltime,
self.resource.queue)
copy_to = """
cp -R %s /scratch/$USER/$PBS_JOBID
### Execute commands on scratch nodes
cd /scratch/$USER/$PBS_JOBID%s
pwd""" % ((".", "../")[ios], ("", "/input")[ios])
run = '''
%s
### modelpath: %s
### htc: %s
echo "---------------------"
%s -c "from wetb.hawc2.cluster_simulation import ClusterSimulation;ClusterSimulation('.','%s', ('%s','%s'))"
echo "---------------------"
echo $?
echo "---------------------"''' % (self.resource.init_cmd, self.modelpath,
self.htcFile.filename,
self.resource.python_cmd, rel_htcfilename,
self.resource.wine_cmd, self.hawc2exe)
copy_back = """
### Copy back from scratch directory
cd /scratch/$USER/$PBS_JOBID%s
%s
echo $PBS_JOBID
cd /scratch/
### rm -r $PBS_JOBID
exit""" % (("", "/input")[ios], cp_back)
return init + copy_to + run + copy_back
