def setup_model_config(self, mc): '''creates a new model_config from initial values''' model_cf = self.model_config() print("\nWriting NEMS model config file %s" %model_cf) try: lines="""# NEMS model_configure - autogenerated by NSEM at {} print_esmf: {} # The RUN_CONTINUE flag tells us if the RUN step of the # NEMS component must be called multiple times for ensembles. RUN_CONTINUE: {} ENS_SPS: {} total_member: {} PE_MEMBER01: {} start_year: {} start_month: {} start_day: {} start_hour: {} start_minute: {} start_second: {} nhours_fcst: {} """.format(nus.now(frmt=3),mc['print_esmf'], mc['run_continue'], mc['ens_sps'], mc['total_member'], mc['pe_member01'], mc['start_year'],mc['start_month'], mc['start_day'], mc['start_hour'],mc['start_minute'],mc['start_second'], mc['nhours_fcst']) except OSError as err: print(nus.colory("red", "Error on 'model_configure' values in initialization file - {}\n".format(err))) sys.exit(0) try: with open(model_cf, 'w+') as fptr: fptr.write(lines) except Exception as e: print(nus.colory("red", "Error writing to {}\n".format(model_cf)))
def write_build(self): user_module = self.nems_cfg.user_module nems_models = self.nems_cfg.nems_models p = os.path.join(self.source_dir,'build.sh') junk, modulefile = user_module.split(self.source_dir) lines = """#!/bin/bash\n # Description : Script to compile NSEModel NEMS application # Usage : ./build.sh # Date : Autogenerated by NSEM at {}\n # Contacts : [email protected] # [email protected] # [email protected] # [email protected] # load modules source {} cd NEMS\n""".format(nus.now(2), modulefile[1:]) # remove the '/' from string lines += '\n#clean up\n' for model in nems_models(): alias = model.get_alias().upper() if not nus.exist(os.path.join(self.source_dir, alias)): sys.exit(0) lines += 'make -f GNUmakefile distclean_' + \ alias + ' COMPONENTS=' + \ '"' + alias + '"\n' lines += 'make -f GNUmakefile distclean_NEMS COMPONENTS="NEMS"\n' lines += '\n#make\n' lines += 'make -f GNUmakefile build COMPONENTS="' for model in nems_models(): alias = model.get_alias().upper() lines += alias+ ' ' lines = lines[:-1] + '"\n' with open(p, 'w') as fptr: fptr.write(lines) # change mode subprocess.call(["chmod", "a+x", p]) # save self.build_script = p print("Processed build script %s" %p)
def setup_nems_config_old(self, nc): '''creates a new nems.configure from initial values''' nems_cf = self.nems_config() print("Writing NEMS config file {}".format(nems_cf)) earth_comp = nc['EARTH_component_list'] earth_str = " ".join([i.strip() for i in earth_comp]) atm_model = nc['ATM_model'] atm_str, atm_pet = atm_model atm_start, atm_end = atm_pet atm_pet_str = str(atm_start) + " " + str(atm_end) ocn_model = nc['OCN_model'] ocn_str, ocn_pet = ocn_model ocn_start, ocn_end = ocn_pet ocn_pet_str = str(ocn_start) + " " + str(ocn_end) wav_model = nc['WAV_model'] wav_str, wav_pet = wav_model wav_start, wav_end = wav_pet wav_pet_str = str(wav_start) + " " + str(wav_end) nwm_model = nc['NWM_model'] nwm_str, nwm_pet = nwm_model nwm_start, nwm_end = nwm_pet nwm_pet_str = str(nwm_start) + " " + str(nwm_end) run_seq = nc['runSeq']['coupling_interval_sec'] try: lines="""# nems.configure - autogenerated by NSEM at {} ############################################# #### NEMS Run-Time Configuration File ##### ############################################# # EARTH # EARTH_component_list: {} EARTH_attributes:: Verbosity = max :: # ATM # ATM_model: {} ATM_petlist_bounds: {} ATM_attributes:: Verbosity = max :: # OCN # OCN_model: {} OCN_petlist_bounds: {} OCN_attributes:: Verbosity = max :: # WAV # WAV_model: {} WAV_petlist_bounds: {} WAV_attributes:: Verbosity = max :: # HYD # NWM_model: {} NWM_petlist_bounds: {} NWM_attributes:: Verbosity = max :: # Run Sequence # runSeq:: {} ATM -> OCN :remapMethod=redist WAV -> OCN :remapMethod=redist ATM -> NWM :remapMethod=redist WAV -> NWM :remapMethod=redist OCN -> NWM :remapMethod=bilinear NWM -> OCN :remapMethod=redist ATM WAV OCN NWM @ :: """.format(nus.now(frmt=3), earth_str, atm_str, atm_pet_str, ocn_str, ocn_pet_str, wav_str, wav_pet_str, nwm_str, nwm_pet_str, run_seq ) except OSError as err: print(nus.olory("red", "Error on 'model_configure' values in initialization file - {0}\n".format(err))) sys.exit(0) try: with open(nems_cf, 'w+') as fptr: fptr.write(lines) except Exception as e: print(nus.colory("red", "Error writing to {}\n".format(nems_cf)))
def write_jjob(self): lines="""#!/bin/sh date export PS4=' $SECONDS + ' set -x export pgm="NWPS" export pgmout=OUTPUT.$$ export KEEPDATA="YES" # TODO - to see PDY={} # temp location of the job working directory, typically $DATAROOT/$jobid export DATA=${{DATA:-${{DATAROOT:?}}/$jobid}} mkdir -p $DATA cd $DATA #################################### # Specify NET and RUN Name and model #################################### export NET={} #export RUN=$(echo j{}|awk -F"_" '{{print $2}}') #################################### # SENDECF - Flag Events on ECFLOW # SENDCOM - Copy Files From TMPDIR to $COMOUT # SENDDBN - Issue DBNet Client Calls #################################### export SENDCOM=${{SENDCOM:-YES}} export SENDECF=${{SENDECF:-YES}} export SENDDBN=${{SENDDBN:-NO}} # Specify Execution Areas #################################### export HOMEnsem={} export FIXnsem={} export EXECnsem={} export SORCnsem={} export PARMnsem={} export USHnsem={} # Set processing DIRs here export INPUTdir=${{INPUTdir:-${{DATA}}/input}} export COMdir=${{COMdir:-${{DATA}}/run}} export TMPdir=${{TMPdir:-${{DATA}}/tmp}} export LOGdir=${{LOGdir:-${{DATA}}/logs}} # Set NWPS run conditions export DEBUGGING=${{DEBUGGING:-TRUE}} export DEBUG_LEVEL=${{DEBUG_LEVEL:-1}} export ISPRODUCTION=${{ISPRODUCTION:-TRUE}} export SITETYPE=${{SITETYPE:-EMC}} ############################################## # Define COM directory ############################################## # com directory for current model's output data, typically $COMROOT/$NET/$envir/$RUN.$PDY export COMOUT={} # nwges directory for output guess fields; typically $GESROOT/$envir export GESOUT={} # nwges directory for input guess fields; typically $GESROOT/$envir export GESIN={} # loop over nems.configure per model(alias), also must match run_name # COMIN com directory for current model's input data, typically $COMROOT/$NET/$envir/$RUN.$PDY # COMINmodel com directory for incoming data from model model export COMINatm={} export COMINwave={} export COMINwavedata={} export COMINadc={} export COMINmeshdata={} export COMINnwm={} # check if we need to create COMOUT and GESOUT here or in main? mkdir -p $INPUTdir $COMdir $TMPdir $LOGdir $COMOUT $GESOUT ############################################## # Execute the script {} ############################################## #startmsg msg="JOB j{} HAS COMPLETED NORMALLY." # postmsg $jlogfile "$msg" if [ -e $pgmout ]; then cat $pgmout fi cd {} if [ "$KEEPDATA" != YES ]; then rm -rf $DATA fi date """.format(nus.now(4), self.NET, self.job, self.NWROOT, self.fix_dir(), self.exec_dir(), self.source_dir(), self.parm_dir(), self.ush_dir(), self.COMOUT, self.GESOUT, self.GESIN, os.path.join(self.COMIN,"atm"), os.path.join(self.COMIN,"ww3"), os.path.join(self.COMIN,"ww3data"), os.path.join(self.COMIN,"adcirc"), os.path.join(self.COMIN,"atmesh"), os.path.join(self.COMIN,"nwm"), os.path.join(self.script_dir(),"ex"+self.RUN+".sh"), self.job, self.DATAROOT) outfile = os.path.join(self.jjob_dir(), "J"+self.RUN.upper()) # running two storms at once, even for testing, overrides ecf files # p = self.storm_dir("job") # outfile = os.path.join(p, "J"+self.RUN.upper()) with open(outfile, 'w') as fptr: fptr.write(lines) # change mode subprocess.call(["chmod", "a+x", outfile]) print("\nProcessed ecflow jjob script %s" %outfile) # create the script at the same time as jjob out = os.path.join(self.script_dir(),"ex"+self.RUN+".sh") # this script must exists that does the real work p = os.path.join(self.ush_dir(), self.RUN+".py") if not nus.found(p): print("Warrning - make sure to prepare the script!") lines=""" echo "=============================================================" echo "= " echo "= RUNNING NSEM FOR STORM: {} echo "= RUN NAME: {} echo "= " echo "=============================================================" # python ${{USHnsem}}/{} echo "{} completed" exit 0 """.format(self.NET, self.RUN, self.RUN+".py", self.RUN) with open(out, 'w') as fptr: fptr.write(lines) # change mode subprocess.call(["chmod", "a+x", out]) print("Processed ecflow script %s" %out)
def write_ecf(self, slurm): sbatch_part = slurm.get_sbatch() lines = """\n\n#%include <head.h> #%include <envir.h> # base name of model directory (i.e. storm) export model={} # set to test during the initial testing phase, para when running # in parallel (on a schedule), and prod in production export envir={} export DEVWCOSS_USER=$(whoami) # models input/output location export COMROOT={} # models restart/spinup files location export GESROOT={} # root directory for application, typically /nw$envir export NWROOT={} # unique job name (unique per day and environment), run_name export job=j{} export jobid=$job.$$ # temp directory containing the working directory, often /tmpnwprd1 in production export DATAROOT={} # log directory and files mkdir -p {} export jlogfile={} # call the jjob script {} #%manual ###################################################################### # Task script: j{} # Last modifier: Andre van der Westhuysen # Organization: NOAA/NCEP/EMC/NOS/OWP # Date: {} # Purpose: To execute the job for ADC-WW3-NWM model ###################################################################### ###################################################################### # Job specific troubleshooting instructions: # see generic troubleshoot manual page # ###################################################################### #%end""".format(self.NET, self.envir, self.COMROOT, self.GESROOT, self.NWROOT, self.RUN, self.DATAROOT, os.path.join(self.COMROOT, self.envir, "logs"), self.jlogfile, os.path.join(self.jjob_dir(),"J"+self.RUN.upper()), self.RUN, nus.now(2)) outfile = os.path.join(self.ecf_dir(), "j"+self.RUN + ".ecf") # running two storms at once, even for testing, overrides ecf files # p = self.storm_dir("ecf") # outfile = os.path.join(p, "j"+self.RUN + ".ecf") with open(outfile, 'w') as fptr: fptr.write(sbatch_part+lines) # change mode subprocess.call(["chmod", "a+x", outfile]) print("\nProcessed ecflow script %s" %outfile)