def read_model_config(self):
model_cf = self.model_config()
print("\nReading NEMS model config file %s" %model_cf)
try:
with open(model_cf,'r') as fptr:
lines = fptr.readlines()
for line in lines:
if line.startswith("#") or len(line) < 2:
continue
key, value = line.split(":")
if 'start_year' in key:
self.start_year = value.strip()
elif 'start_month' in key:
self.start_month = value.strip()
elif 'start_day' in key:
self.start_day = value.strip()
elif 'start_hour' in key:
self.start_hour = value.strip()
elif 'start_minute' in key:
self.start_minute = value.strip()
elif 'start_second' in key:
self.start_second = value.strip()
elif 'nhours_fcst' in key:
self.nhours_fcst = value.strip()
except Exception as e:
print(nus.colory("red", "{}\n".format(str(e))))
sys.exit(-1)
msg = "Runtime duration since: %s/%s/%s %s:%s:%s for %s hours\n" %(self.start_year,self.start_month,self.start_day,
self.start_hour, self.start_minute, self.start_second, self.nhours_fcst)
print(nus.colory("green", msg))
def setup_model_config(self, mc):
'''creates a new model_config from initial values'''
model_cf = self.model_config()
print("\nWriting NEMS model config file %s" %model_cf)
try:
lines="""# NEMS model_configure - autogenerated by NSEM at {}
print_esmf: {}
# The RUN_CONTINUE flag tells us if the RUN step of the
# NEMS component must be called multiple times for ensembles.
RUN_CONTINUE: {}
ENS_SPS: {}
total_member: {}
PE_MEMBER01: {}
start_year: {}
start_month: {}
start_day: {}
start_hour: {}
start_minute: {}
start_second: {}
nhours_fcst: {}
""".format(nus.now(frmt=3),mc['print_esmf'], mc['run_continue'], mc['ens_sps'],
mc['total_member'], mc['pe_member01'], mc['start_year'],mc['start_month'],
mc['start_day'], mc['start_hour'],mc['start_minute'],mc['start_second'],
mc['nhours_fcst'])
except OSError as err:
print(nus.colory("red", "Error on 'model_configure' values in initialization file - {}\n".format(err)))
sys.exit(0)
try:
with open(model_cf, 'w+') as fptr:
fptr.write(lines)
except Exception as e:
print(nus.colory("red", "Error writing to {}\n".format(model_cf)))
def main():
msg = "\n%s: Setting up %s for storm %s .........." % (__file__, env.run,
env.storm)
print(util.colory("blue", msg))
tide_spin_sdate, tide_spin_edate, _, _, nems_sdate, nems_edate, = prep.spinup_time(
)
# To prepare a clod start ADCIRC-Only run for spining-up the tide
dic = setvars(env.storm, tide_spin_sdate, tide_spin_edate, nems_sdate,
nems_edate)
# prep atm data
msg = "\nPreparing ATM data files ....."
print(util.colory("red", msg))
adc_atm_data(env.storm, "hwrf")
msg = "\nPreparing ADCIRC configuration files ....."
print(util.colory("red", msg))
prep.prep_adc(dic)
msg = "\nPreparing NEMS configuration files ....."
print(util.colory("red", msg))
prep.prep_nems(dic)
msg = "\nFinished preparing data for NSEM %s\n" % (env.run)
print(util.colory("blue", msg))
def prep_adc(dic):
"""
adcirc preparation per storm and per run name requires:
1) update model parameter and periodic boundary condition file (fort.15)
a) copy adcprep executable
2) copy adc-grd-input (i.e. ocn mesh)
3) copy wave-grd-input (i.e. wav mesh)
4) copy restart files, fort.67.nc, fort.68.nc
5) copy tidefac - for which case??
6) copy adc-input - what and where are these?
7) pet nums - reason?
8) copy atm files - which one?, do you need processing?
9) dependencies: nems model_configure and nems.configure
spinup period
/scratch2/COASTAL/coastal/save/NAMED_STORMS/NSEM_app_run_workflow
application_dir = /scratch2/COASTAL/coastal/save/NAMED_STORMS/NSEM_app_run_workflow/ADC-WW3-NWM-NEMS
app_inp_dir = /scratch2/COASTAL/coastal/save/NAMED_STORMS/NSEM_app_run_workflow/nsemodel_inps
"""
msg = "\tProcessing boundary condition, fort15 ....."
print(util.colory("green", msg))
update_fort15(dic)
msg = "\tProcessing ATMesh data ..... "
print(util.colory("green", msg))
msg = "\tProcessing adc-wave data ...."
print(util.colory("green", msg))
# copy files that are needed
adcprep = os.path.join(env.EXECnsem, 'adcprep')
adc_grd_inp = os.path.join(env.FIXnsem, 'meshes', env.storm,
'ocn') # copy HSOFS grid similar for all cases
""" TODO go through these
tidefac = os.path.join(EXECnsem, 'tidefac')
# adc_inp = os.path.join(COMINadc) # this must be copied there manually by HYDRO_STREAM, see CONOPS7
adc_grd_inp = os.path.join(FIXnsem,'meshes',storm,'ocn') # copy HSOFS grid similar for all cases
ww3_grd_inp = os.path.join(FIXnsem,'meshes',storm,'wav')
hot_file = os.path.join(GESIN,'hotfiles','fort.67.nc') # TODO once manual next after running tide_spinup??
os.system('cp -f ' + hot_file +' ' + run_dir)
hot_file = os.path.join(GESIN,'hotfiles','fort.68.nc')
os.system('cp -f ' + hot_file +' ' + run_dir)
"""
# $COMROOT/$NET/$envir/$RUN.$PDY
target = env.COMINadc
try:
msg = "\tCopying files to %s ...." % (target)
print(util.colory("green", msg))
shutil.copy(adcprep, target)
except IOError as e:
print("\tUnable to copy file. %s" % e)
except:
print("\tUnexpected error:", sys.exc_info())
def check_files(self, ini):
msg = "\nFollowing files not found - can not continue"
mlen = len(msg)
for f in self.domain_files.keys():
domain = os.path.join(self.data_path, "domain", self.domain,
self.domain_files[f])
if not os.path.isfile(domain):
msg += "\n" + self.domain_files[f]
# 2016100822.LDASIN_DOMAIN1
forcing = [
os.path.join(self.data_path, "forcing", self.storm, f)
for f in self.forcing_files
]
for f in forcing:
if not os.path.isfile(f):
msg += "\n" + f
slices = [
os.path.join(self.data_path, "nudgingTimeSliceObs", self.storm, f)
for f in self.discharge_obs_files
]
for f in slices:
if not os.path.isfile(f):
msg += "\n" + f
for f in self.restart_files.keys():
restart = os.path.join(self.data_path, "restart", self.storm,
self.restart_files[f])
if not os.path.isfile(restart):
msg += "\n" + self.restart_files[f]
for f in self.config_files.keys():
if not os.path.isfile(
os.path.join(self.data_path, self.config_files[f])):
msg += "\n" + self.config_files[f]
tables = [os.path.join(self.data_path, f) for f in self.table_files]
for f in tables:
if not os.path.isfile(f):
msg += "\n" + f
if len(msg) > mlen:
print(nus.colory("red", msg))
sys.exit(0)
else:
print(nus.colory("green", "input files exist - good to go"))
def read_nems_config(self):
nems_cf = self.nems_config()
print("\nReading NEMS config file %s" %nems_cf)
try:
with open(nems_cf,'r') as fptr:
cnt = 0; sentinel = '::'
# saves the line # of the "::" in file nems.configure
indx = []
lines = fptr.readlines()
for line in lines:
cnt += 1
if line.startswith(sentinel):
indx.append(cnt)
# cut each section of the file based on indx
earth_list = []; model_list = []; seq_list = []
earth_list = lines[:indx[0]]
model_list = lines[indx[0]:indx[len(indx)-2]]
seq_list = lines[indx[len(indx)-2]:]
# persists newely writen config files into __dic__
self.process_earth_section(earth_list)
self.process_model_section(model_list)
self.process_runseq_section(seq_list)
except Exception as e:
print(nus.colory("red", "{}\n".format(str(e))))
sys.exit(-1)
def build_nems_app(self):
try:
print("\nStart compiling ............\n")
subprocess.check_call(['./build.sh'], cwd=self.source_dir, stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as err:
print(nus.colory("red", 'Error in executing build.sh:\n'.format(err)))
def process_runseq_section(self, runseq_lines):
sentinel = '::'; i = 0
lines = [line.strip() for line in runseq_lines]
line = lines[i]
while True :
# for now put this as common property -TODO
if re.search('runSeq', line, re.IGNORECASE) :
self.__dict__["runSeq"] = []
vals = self.__dict__['runSeq']
i += 1
line = lines[i]
while line.find(sentinel) == -1:
vals.append(line.strip())
i += 1
line = lines[i]
self.__dict__["runSeq"] = vals
break
else:
i += 1
line = lines[i]
print("Processed runSeq")
print(nus.colory("green", ", \n".join(self.__dict__['runSeq'])))
def main():
# these are being set before prep_adc is called
spin_time()
adc_atm_data(env.storm,"hwrf")
adc_wave_data(env.storm,"hwrf")
msg = "\nFinished preparing data for NSEM\n"
print(util.colory("blue", msg))
def print_model(self):
for model in self.nems_models():
name = model.name
petlist = model.get_petlist()
attrs = model.get_attributes()
alias = model.get_alias()
msg = "Model Name : %s\n" %name
msg+= "Model Alias : %s\n" %alias
msg+= "Model petlist : %s\n" %petlist
msg+= "Model attributes: %s\n" %attrs
print(nus.colory("green", msg))
def git_nsem(repo, nco_sorc_dir):
print("\nCloning {} to {}".format(repo, nco_sorc_dir))
# dest_dir is the name part of the repo and so it becomes
# the location of NEMS and all the models. This is one layer
# below ROOTDIR, as mentioned in NEMS HOWTO.
dest_dir = os.path.join(nco_sorc_dir, Path(repo).name)
if nus.exist(dest_dir):
print("\nRemoving directory: {}".format(dest_dir))
try:
subprocess.run(['rm', '-rf', dest_dir ], check=True)
except subprocess.CalledProcessError as err:
print(nus.colory("red", 'Error deleting directory {}\n'.format(err)))
cmd = ['git', 'clone', '--recursive', repo] # TO DO - add the user/pass
try:
# must wait to finishe, so check_call or run
return subprocess.check_call(cmd, cwd=nco_sorc_dir, stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as e:
print(nus.colory("red", "Exception on process, rc=".format(e.returncode)))
def adc_wave_data(storm, provider):
msg = "\nCopping wave ata ....."
print(util.colory("red",msg))
return
wav_netcdf_file_names = np.array([
'01_ww3.test1.2008_sxy_OC.nc',
'02_ww3.test2.2008_sxy_OC_SM.nc',
'03_ww3.test3.2008_sxy_WRF.nc',
'04_ww3.test4.2008_sxy_OC_WRF.nc',
'05_ww3.test5.2008_sxy_OC_DA_HSOFS_orig.nc',
'06_ww3.test6.2008_sxy_OC_DA_HSOFS_Smoothing.nc',
'07_ww3.test7.2008_sxy_OC_DA_WRF_SM.nc',
])
def prep_adc(dic):
"""
adcirc preparation per storm and per run name requires:
1) update model parameter and periodic boundary condition file (fort.15)
a) copy adcprep executable
2) copy adc-grd-input (i.e. ocn mesh)
3) copy wave-grd-input (i.e. wav mesh)
4) copy restart files, fort.67.nc, fort.68.nc
5) copy tidefac - for which case??
6) copy adc-input - what and where are these?
7) pet nums - reason?
8) copy atm files - which one?, do you need processing?
9) dependencies: nems model_configure and nems.configure
spinup period
"""
msg = "\tUpdating boundary condition, fort15 ....."
print(util.colory("green",msg))
update_fort15(dic)
# copy files that are needed
adcprep = os.path.join(env.EXECnsem, 'adcprep')
"""
fort15 = os.path.join(run_dir,'fort.15') #copy after update_fort15
adcprep = os.path.join(EXECnsem, 'adcprep')
tidefac = os.path.join(EXECnsem, 'tidefac')
# adc_inp = os.path.join(COMINadc) # this must be copied there manually by HYDRO_STREAM, see CONOPS7
adc_grd_inp = os.path.join(FIXnsem,'meshes',storm,'ocn') # copy HSOFS grid similar for all cases
ww3_grd_inp = os.path.join(FIXnsem,'meshes',storm,'wav')
hot_file = os.path.join(GESIN,'hotfiles','fort.67.nc') # TODO once manual next after running tide_spinup??
os.system('cp -f ' + hot_file +' ' + run_dir)
hot_file = os.path.join(GESIN,'hotfiles','fort.68.nc')
os.system('cp -f ' + hot_file +' ' + run_dir)
"""
# $COMROOT/$NET/$envir/$RUN.$PDY
target = env.COMINadc
try:
shutil.copy(adcprep, target)
except IOError as e:
print("\nUnable to copy file. %s" % e)
except:
print("\nUnexpected error:", sys.exc_info())
def spinup_time(ts=12.5,ws=27):
"""calculates spinup timing for both tide and wave
relative to forecast start time and end time """
msg = "\nCalculating spinup time ....."
print(util.colory("red",msg))
duration = env.frcst_hrs
start_date_str = env.start_date_str
# string date to date object
start_date = datetime.strptime(start_date_str, '%m/%d/%Y %H:%M:%S')
hours_added = timedelta(hours=duration)
end_date = start_date + hours_added
delta = end_date - start_date
num_days = delta.days
# print(start_date, end_date, num_days)
# tide spinup start time must be 12 days prior to forecast start date
tide_spin_start_date = start_date - timedelta(days=ts)
# wave spinup star time must be 27 prior to start date
wave_spin_start_date = start_date - timedelta(days=ws)
tide_spin_end_date = tide_spin_start_date + timedelta(days=ts)
wave_spin_end_date = wave_spin_start_date + timedelta(days=ws)
# 2008-08-23 00:00:00
# 2008-09-04 12:00:00
# 2008-09-19 00:00:00
# print("Start wave: %s\nStart tide: %s\nEnd tide : %s\nEnd wave : %s\nStart date: %s\nEnd date : %s\n"
# %(wave_spin_start_date, tide_spin_start_date, tide_spin_end_date,
# wave_spin_end_date, start_date, end_date))
msg = tide_spin_start_date, tide_spin_end_date, wave_spin_start_date, wave_spin_end_date, start_date, end_date
logger.info(msg)
return msg
"""
# standard libs
import os, sys
import argparse
# local libs
import func_nsem_build as fbn
import func_nsem_prep as fnp
import func_nsem_workflow as fnw
import nsem_utils as nus
if __name__ == '__main__':
usage = "%s --help | -h \n" % sys.argv[0]
print(nus.colory("red", usage))
parser = argparse.ArgumentParser()
subp = parser.add_subparsers()
nco = subp.add_parser(
"workflow",
help="reads an initialization file and construct the NSEM workflow")
nco.add_argument("--ini",
help="an init python file with prepopulated values",
default="nsem_ini.py")
nco.set_defaults(func=fnw.nsem_workflow)
build_install = subp.add_parser(
"build", help="buils and installs NSEM models with NEMS")
build_install.add_argument(
def main():
msg = "\n%s: Reading initialization file %s .........." % (__file__,
"nsem_ini.py")
print(util.colory("blue", msg))
msg = "\n%s: Setting up run type \"%s\" for storm \"%s\" .........." % (
__file__, ini.RUN_TYPE, ini.STORM)
print(util.colory("blue", msg))
logger = logging.getLogger(ini.jlogfile)
msg = "\n%s: Setting up logfile \"%s\" .........." % (__file__,
ini.jlogfile)
print(util.colory("blue", msg))
msg = "\nCalculating spinup time ....."
print(util.colory("red", msg))
tide_spin_sdate, tide_spin_edate, _, _, nems_sdate, nems_edate, = spinup_time(
)
# To prepare a clod start ADCIRC-Only run for spining-up the tide
msg = "\nSetting model variables ....."
print(util.colory("red", msg))
dic = setvars(tide_spin_sdate, tide_spin_edate, nems_sdate, nems_edate)
msg = "\nPreprocessing ATM input files ....."
print(util.colory("red", msg))
# prep_atm()
msg = "\nPreprocessing ATMESH input files ....."
print(util.colory("red", msg))
# prep_atmesh()
msg = "\nPreprocessing ADCIRC input files ....."
print(util.colory("red", msg))
# prep_adc()
msg = "\nPreprocessing WW3 input files ....."
print(util.colory("red", msg))
# prep_ww3()
msg = "\nPreprocessing NWM input files ....."
print(util.colory("red", msg))
prep_nwm()
msg = "\nPreparing NEMS configuration files ....."
print(util.colory("red", msg))
def setup_nems_config_old(self, nc):
'''creates a new nems.configure from initial values'''
nems_cf = self.nems_config()
print("Writing NEMS config file {}".format(nems_cf))
earth_comp = nc['EARTH_component_list']
earth_str = " ".join([i.strip() for i in earth_comp])
atm_model = nc['ATM_model']
atm_str, atm_pet = atm_model
atm_start, atm_end = atm_pet
atm_pet_str = str(atm_start) + " " + str(atm_end)
ocn_model = nc['OCN_model']
ocn_str, ocn_pet = ocn_model
ocn_start, ocn_end = ocn_pet
ocn_pet_str = str(ocn_start) + " " + str(ocn_end)
wav_model = nc['WAV_model']
wav_str, wav_pet = wav_model
wav_start, wav_end = wav_pet
wav_pet_str = str(wav_start) + " " + str(wav_end)
nwm_model = nc['NWM_model']
nwm_str, nwm_pet = nwm_model
nwm_start, nwm_end = nwm_pet
nwm_pet_str = str(nwm_start) + " " + str(nwm_end)
run_seq = nc['runSeq']['coupling_interval_sec']
try:
lines="""# nems.configure - autogenerated by NSEM at {}
#############################################
#### NEMS Run-Time Configuration File #####
#############################################
# EARTH #
EARTH_component_list: {}
EARTH_attributes::
Verbosity = max
::
# ATM #
ATM_model: {}
ATM_petlist_bounds: {}
ATM_attributes::
Verbosity = max
::
# OCN #
OCN_model: {}
OCN_petlist_bounds: {}
OCN_attributes::
Verbosity = max
::
# WAV #
WAV_model: {}
WAV_petlist_bounds: {}
WAV_attributes::
Verbosity = max
::
# HYD #
NWM_model: {}
NWM_petlist_bounds: {}
NWM_attributes::
Verbosity = max
::
# Run Sequence #
runSeq::
{}
ATM -> OCN :remapMethod=redist
WAV -> OCN :remapMethod=redist
ATM -> NWM :remapMethod=redist
WAV -> NWM :remapMethod=redist
OCN -> NWM :remapMethod=bilinear
NWM -> OCN :remapMethod=redist
ATM
WAV
OCN
NWM
@
::
""".format(nus.now(frmt=3), earth_str, atm_str, atm_pet_str,
ocn_str, ocn_pet_str, wav_str, wav_pet_str,
nwm_str, nwm_pet_str,
run_seq
)
except OSError as err:
print(nus.olory("red", "Error on 'model_configure' values in initialization file - {0}\n".format(err)))
sys.exit(0)
try:
with open(nems_cf, 'w+') as fptr:
fptr.write(lines)
except Exception as e:
print(nus.colory("red", "Error writing to {}\n".format(nems_cf)))
def prep_nems(dic=None):
"""Prepare nems run files and copy them into $COMIN
if manual and live :
check and copy model_configure
check and copy nems.configure
check and copy build.sh
check and copy slurm.job
compile the model components
copy the model installs file
submit the slurm.job
if archive :
create model_configure and nems.configure templates per storm/run_name
not sure to go this way!!
"""
# build.sh
# slurm.job
# NEMS.x
# nems model_configure
ocn_pet_num = 0
atm_pet_num = 0
wav_pet_num = 0
nwm_pet_num = 0
# prepare nems.configure
dc2 = {}
dc2.update({'c': '#'})
try:
if 'ocn_name' in dic:
dc2.update({'_ocn_model_': dic['ocn_name']})
if 'ocn_petlist' in dic:
dc2.update({'_ocn_petlist_bounds_': dic['ocn_petlist']})
pet_nums_arr = dic['ocn_petlist'].split()
ocn_pet_num = int(pet_nums_arr[-1]) - int(pet_nums_arr[0]) + 1
if 'coupling_interval_sec' in dic:
dc2.update(
{'_coupling_interval_sec_': dic['coupling_interval_sec']})
if 'nems_configure' in dic:
template = os.path.join(env.FIXnsem, 'templates',
dic['nems_configure'])
dest = os.path.join(env.COMIN, 'nems.configure')
nems_conf = util.tmp2scr(filename=dest, tmpname=template, d=dc2)
except Exception as e:
print("Error: ", str(e))
sys.exit(0)
msg = "\tFinished nems.configure file ...."
print(util.colory("green", msg))
# prepare model_configure
total_pets = ocn_pet_num + atm_pet_num + wav_pet_num
sdate = env.start_date_str
month = sdate[0:1]
day = sdate[3:4]
year = sdate[6:9]
hour = sdate[11:12]
minute = sdate[4:5]
second = sdate[17:18]
dc = {}
dc.update({'start_year': year})
dc.update({'start_month': month})
dc.update({'start_day': day})
dc.update({'start_hour': hour})
dc.update({'start_minute': minute})
dc.update({'start_second': second})
dc.update({'nhours': str(env.frcst_hrs)})
dc.update({'total_pets': str(total_pets)})
#
template = os.path.join(env.FIXnsem, 'templates',
'atm_namelist.rc.template'
) #TODO check on this file, use model_configure
dest = os.path.join(env.COMIN, 'model_configure')
#model_configure = os.path.join(run_dir,'atm_namelist.rc')
model_conf = util.tmp2scr(filename=dest, tmpname=template, d=dc)
#os.system('ln -svf ' + model_configure + ' ' + os.path.join(run_dir,'model_configure' ) )
#os.system('cp -fr ' + tmpname +' ' +run_dir+'/scr/')
msg = "\tFinished model_configure file ...."
print(util.colory("green", msg))
# cp NEMS.x
msg = "\tCopying NEMS.x file into %s...." % (env.COMIN)
print(util.colory("green", msg))