def check_prerequisites(self, env):
""" check prerequisites
"""
print(" Checking prerequisites for : {0}".format(
self.__class__.__name__))
super(modelVsObs, self).check_prerequisites(env)
# clean out the old working plot files from the workdir
if env['CLEANUP_FILES'].upper() in ['T', 'TRUE']:
cesmEnvLib.purge(env['WORKDIR'], '.*\.pro')
cesmEnvLib.purge(env['WORKDIR'], '.*\.gif')
cesmEnvLib.purge(env['WORKDIR'], '.*\.dat')
cesmEnvLib.purge(env['WORKDIR'], '.*\.ps')
cesmEnvLib.purge(env['WORKDIR'], '.*\.png')
cesmEnvLib.purge(env['WORKDIR'], '.*\.html')
# create the plot.dat file in the workdir used by all NCL plotting routines
diagUtilsLib.create_plot_dat(env['WORKDIR'], env['XYRANGE'],
env['DEPTHS'])
# setup the gridfile based on the resolution and vertical levels
os.environ['gridfile'] = '{0}/omwg/za_grids/{1}_grid_info.nc'.format(
env['DIAGOBSROOT'], env['RESOLUTION'])
if env['VERTICAL'] == '42':
os.environ[
'gridfile'] = '{0}/omwg/za_grids/{1}_42lev_grid_info.nc'.format(
env['DIAGOBSROOT'], env['RESOLUTION'])
if env['VERTICAL'] == '62':
os.environ[
'gridfile'] = '{0}/omwg/za_grids/{1}_62lev_grid_info.nc'.format(
env['DIAGOBSROOT'], env['RESOLUTION'])
# check if gridfile exists and is readable
rc, err_msg = cesmEnvLib.checkFile(os.environ['gridfile'], 'read')
if not rc:
print(
'model_vs_obs: check_prerequisites could not find gridfile = {0}'
.format(os.environ['gridfile']))
raise ocn_diags_bc.PrerequisitesError
env['GRIDFILE'] = os.environ['gridfile']
# check the resolution and decide if some plot modules should be turned off
if (env['RESOLUTION'] == 'tx0.1v2' or env['RESOLUTION'] == 'tx0.1v3'):
env['MVO_PM_VELISOPZ'] = os.environ['MVO_PM_VELISOPZ'] = 'FALSE'
env['MVO_PM_KAPPAZ'] = os.environ['MVO_PM_KAPPAZ'] = 'FALSE'
# create the global zonal average file used by most of the plotting classes
print(' model vs. obs - calling create_za')
diagUtilsLib.create_za(env['WORKDIR'], env['TAVGFILE'],
env['GRIDFILE'], env['TOOLPATH'], env)
return env
def check_prerequisites(self, env):
""" check prerequisites
"""
print(" Checking prerequisites for : {0}".format(self.__class__.__name__))
super(modelVsObsEcosys, self).check_prerequisites(env)
# clean out the old working plot files from the workdir
if env['CLEANUP_FILES'].upper() in ['T','TRUE']:
cesmEnvLib.purge(env['WORKDIR'], '.*\.pro')
cesmEnvLib.purge(env['WORKDIR'], '.*\.gif')
cesmEnvLib.purge(env['WORKDIR'], '.*\.dat')
cesmEnvLib.purge(env['WORKDIR'], '.*\.ps')
cesmEnvLib.purge(env['WORKDIR'], '.*\.png')
cesmEnvLib.purge(env['WORKDIR'], '.*\.html')
# create the plot.dat file in the workdir used by all NCL plotting routines
diagUtilsLib.create_plot_dat(env['WORKDIR'], env['XYRANGE'], env['DEPTHS'])
# read in the ecosys_vars.txt file and create the corresponding link files to the climatology files
try:
ecosys_vars_file = env['ECOSYSVARSFILE']
f = open(ecosys_vars_file)
ecosys_vars = f.read().split()
f.close()
except IOError as e:
print ('ERROR: unable to open {0} error {1} : {2}'.format(ecosys_vars_file, e.errno, e.strerror))
except ValueError:
print ('ERROR: unable to split {0} into separate variable names'.format(ecosys_vars_file))
except:
print ('ERROR: unexpected error in {0}'.format(self._name))
raise
# loop over the ecosys_vars list and create the links to the mavg file
sourceFile = os.path.join(env['WORKDIR'],'mavg.{0:04d}.{1:04d}.nc'.format(int(env['YEAR0']),int(env['YEAR1'])))
for var in ecosys_vars:
linkFile = os.path.join(env['WORKDIR'],'{0}.{1}.clim.{2:04d}-{3:04d}.nc'.format(env['CASE'],var,int(env['YEAR0']),int(env['YEAR1'])))
try:
os.symlink(sourceFile, linkFile)
except OSError as e:
print ('ERROR: unable to create symbolic link {0} to {1} error {2} : {3}'.format(linkFile, sourceFile, e.errno, e.strerror))
# create the POPDIAG and PME environment variables
env['POPDIAGPY'] = env['POPDIAGPY2'] = env['POPDIAG'] = os.environ['POPDIAG'] = 'TRUE'
env['PME'] = os.environ['PME'] = '1'
env['mappdir'] = env['ECODATADIR']+'/mapping'
# create the plot_depths.dat file
fh = open('{0}/plot_depths.dat'.format(env['WORKDIR']),'w')
fh.write('{0}\n'.format(env['DEPTHS']))
fh.close()
return env
def check_prerequisites(self, env):
""" check prerequisites
"""
print(" Checking prerequisites for : {0}".format(self.__class__.__name__))
super(modelVsObsEcosys, self).check_prerequisites(env)
# clean out the old working plot files from the workdir
if env['CLEANUP_FILES'].upper() in ['T','TRUE']:
cesmEnvLib.purge(env['WORKDIR'], '.*\.pro')
cesmEnvLib.purge(env['WORKDIR'], '.*\.gif')
cesmEnvLib.purge(env['WORKDIR'], '.*\.dat')
cesmEnvLib.purge(env['WORKDIR'], '.*\.ps')
cesmEnvLib.purge(env['WORKDIR'], '.*\.png')
cesmEnvLib.purge(env['WORKDIR'], '.*\.html')
# create the plot.dat file in the workdir used by all NCL plotting routines
diagUtilsLib.create_plot_dat(env['WORKDIR'], env['XYRANGE'], env['DEPTHS'])
# read in the ecosys_vars.txt file and create the corresponding link files to the climatology files
try:
ecosys_vars_file = env['ECOSYSVARSFILE']
f = open(ecosys_vars_file)
ecosys_vars = f.read().split()
f.close()
except IOError as e:
print ('ERROR: unable to open {0} error {1} : {2}'.format(ecosys_vars_file, e.errno, e.strerror))
except ValueError:
print ('ERROR: unable to split {0} into separate variable names'.format(ecosys_vars_file))
except:
print ('ERROR: unexpected error in {0}'.format(self._name))
raise
# loop over the ecosys_vars list and create the links to the mavg file
sourceFile = os.path.join(env['WORKDIR'],'mavg.{0:04d}.{1:04d}.nc'.format(int(env['YEAR0']),int(env['YEAR1'])))
for var in ecosys_vars:
linkFile = os.path.join(env['WORKDIR'],'{0}.{1}.clim.{2:04d}-{3:04d}.nc'.format(env['CASE'],var,int(env['YEAR0']),int(env['YEAR1'])))
try:
os.symlink(sourceFile, linkFile)
except OSError as e:
print ('ERROR: unable to create symbolic link {0} to {1} error {2} : {3}'.format(linkFile, sourceFile, e.errno, e.strerror))
# create the POPDIAG and PME environment variables
env['POPDIAGPY'] = env['POPDIAGPY2'] = env['POPDIAG'] = os.environ['POPDIAG'] = 'TRUE'
env['PME'] = os.environ['PME'] = '1'
env['mappdir'] = env['ECODATADIR']+'/mapping'
# create the plot_depths.dat file
fh = open('{0}/plot_depths.dat'.format(env['WORKDIR']),'w')
fh.write('{0}\n'.format(env['DEPTHS']))
fh.close()
return env
def check_prerequisites(self, env):
""" check prerequisites
"""
print(" Checking prerequisites for : {0}".format(self.__class__.__name__))
super(modelVsObs, self).check_prerequisites(env)
# clean out the old working plot files from the workdir
if env['CLEANUP_FILES'].upper() in ['T','TRUE']:
cesmEnvLib.purge(env['WORKDIR'], '.*\.pro')
cesmEnvLib.purge(env['WORKDIR'], '.*\.gif')
cesmEnvLib.purge(env['WORKDIR'], '.*\.dat')
cesmEnvLib.purge(env['WORKDIR'], '.*\.ps')
cesmEnvLib.purge(env['WORKDIR'], '.*\.png')
cesmEnvLib.purge(env['WORKDIR'], '.*\.html')
# create the plot.dat file in the workdir used by all NCL plotting routines
diagUtilsLib.create_plot_dat(env['WORKDIR'], env['XYRANGE'], env['DEPTHS'])
# setup the gridfile based on the resolution and vertical levels
os.environ['gridfile'] = '{0}/omwg/za_grids/{1}_grid_info.nc'.format(env['DIAGOBSROOT'],env['RESOLUTION'])
if env['VERTICAL'] == '42':
os.environ['gridfile'] = '{0}/omwg/za_grids/{1}_42lev_grid_info.nc'.format(env['DIAGOBSROOT'],env['RESOLUTION'])
if env['VERTICAL'] == '62':
os.environ['gridfile'] = '{0}/omwg/za_grids/{1}_62lev_grid_info.nc'.format(env['DIAGOBSROOT'],env['RESOLUTION'])
# check if gridfile exists and is readable
rc, err_msg = cesmEnvLib.checkFile(os.environ['gridfile'], 'read')
if not rc:
print('model_vs_obs: check_prerequisites could not find gridfile = {0}'.format(os.environ['gridfile']))
raise ocn_diags_bc.PrerequisitesError
env['GRIDFILE'] = os.environ['gridfile']
# check the resolution and decide if some plot modules should be turned off
if (env['RESOLUTION'] == 'tx0.1v2' or env['RESOLUTION'] == 'tx0.1v3'):
env['MVO_PM_VELISOPZ'] = os.environ['MVO_PM_VELISOPZ'] = 'FALSE'
env['MVO_PM_KAPPAZ'] = os.environ['MVO_PM_KAPPAZ'] = 'FALSE'
# create the global zonal average file used by most of the plotting classes
print(' model vs. obs - calling create_za')
diagUtilsLib.create_za( env['WORKDIR'], env['TAVGFILE'], env['GRIDFILE'], env['TOOLPATH'], env)
return env
def check_prerequisites(self, env):
""" check prerequisites
"""
print(" Checking prerequisites for : {0}".format(self.__class__.__name__))
super(modelTimeseries, self).check_prerequisites(env)
# chdir into the working directory
os.chdir(env['WORKDIR'])
# clean out the old working plot files from the workdir
if env['CLEANUP_FILES'].upper() in ['T','TRUE']:
cesmEnvLib.purge(env['WORKDIR'], '.*\.pro')
cesmEnvLib.purge(env['WORKDIR'], '.*\.gif')
cesmEnvLib.purge(env['WORKDIR'], '.*\.dat')
cesmEnvLib.purge(env['WORKDIR'], '.*\.ps')
cesmEnvLib.purge(env['WORKDIR'], '.*\.png')
cesmEnvLib.purge(env['WORKDIR'], '.*\.html')
cesmEnvLib.purge(env['WORKDIR'], '.*\.log\.*')
cesmEnvLib.purge(env['WORKDIR'], '.*\.pop\.d.\.*')
# create the plot.dat file in the workdir used by all NCL plotting routines
diagUtilsLib.create_plot_dat(env['WORKDIR'], env['XYRANGE'], env['DEPTHS'])
# set the OBSROOT
env['OBSROOT'] = env['OBSROOTPATH']
# check the resolution and decide if some plot modules should be turned off
if (env['RESOLUTION'] == 'tx0.1v2' or env['RESOLUTION'] == 'tx0.1v3') :
env['MTS_PM_MOCANN'] = os.environ['PM_MOCANN'] = 'FALSE'
env['MTS_PM_MOCMON'] = os.environ['PM_MOCMON'] = 'FALSE'
# check if cpl log file path is defined
if len(env['CPLLOGFILEPATH']) == 0:
# print a message that the cpl log path isn't defined and turn off CPLLOG plot module
print('model timeseries - CPLLOGFILEPATH is undefined. Disabling MTS_PM_CPLLOG module')
env['MTS_PM_CPLLOG'] = os.environ['PM_CPLLOG'] = 'FALSE'
else:
# check that cpl log files exist and gunzip them if necessary
initcplLogs = cplLogs = list()
initCplLogs = glob.glob('{0}/cpl.log.*'.format(env['CPLLOGFILEPATH']))
if len(initCplLogs) > 0:
for cplLog in initCplLogs:
logFileList = cplLog.split('/')
cplLogFile = logFileList[-1]
shutil.copy2(cplLog, '{0}/{1}'.format(env['WORKDIR'],cplLogFile))
# gunzip the cplLog in the workdir
if cplLogFile.lower().find('.gz') != -1:
cplLog_gunzip = cplLogFile[:-3]
inFile = gzip.open('{0}/{1}'.format(env['WORKDIR'],cplLogFile), 'rb')
outFile = open('{0}/{1}'.format(env['WORKDIR'],cplLog_gunzip), 'wb')
outFile.write( inFile.read() )
inFile.close()
outFile.close()
# append the gunzipped cpl log file to the cplLogs list
cplLogs.append('{0}/{1}'.format(env['WORKDIR'],cplLog_gunzip))
# remove the original .gz file in the workdir
os.remove('{0}/{1}'.format(env['WORKDIR'],cplLogFile))
else:
# append the original gunzipped cpl log file to the cplLogs list
cplLogs.append('{0}/{1}'.format(env['WORKDIR'],cplLogFile))
# parse the cpllog depending on the coupler version - default to 7b
print('model_timeseries: setting up heat and freshwater awk calls with cplLogs = {0}'.format(cplLogs))
heatFile = 'cplheatbudget'
freshWaterFile = 'cplfwbudget'
cplVersion = 'cpl7b'
env['ntailht'] = os.environ['ntailht'] = '22'
env['ntailfw'] = os.environ['ntailfw'] = '16'
if '7' == env['TS_CPL'] or '6' == env['TS_CPL']:
cplVersion = 'cpl{0}'.format(env['TS_CPL'])
env['ntailht'] = os.environ['ntailht'] = '21'
env['ntailfw'] = os.environ['ntailfw'] = '16'
# expand the cpl.log* into a list
cplLogs.sort()
cplLogsString = ' '.join(cplLogs)
# define the awk scripts to parse the cpllog file
heatPath = '{0}/process_{1}_logfiles_heat.awk'.format(env['TOOLPATH'], cplVersion)
heatPath = os.path.abspath(heatPath)
fwPath = '{0}/process_{1}_logfiles_fw.awk'.format(env['TOOLPATH'], cplVersion)
fwPath = os.path.abspath(fwPath)
heatCmd = '{0} y0={1} y1={2} {3}'.format(heatPath, env['TSERIES_YEAR0'], env['TSERIES_YEAR1'], cplLogsString).split(' ')
freshWaterCmd = '{0} y0={1} y1={2} {3}'.format(fwPath, env['TSERIES_YEAR0'], env['TSERIES_YEAR1'], cplLogsString).split(' ')
# run the awk scripts to generate the .txt files from the cpllogs
cmdList = [ (heatCmd, heatFile, env['ntailht']), (freshWaterCmd, freshWaterFile, env['ntailfw']) ]
for cmd in cmdList:
outFile = '{0}.txt'.format(cmd[1])
with open (outFile, 'w') as results:
try:
subprocess.check_call(cmd[0], stdout=results, env=env)
except subprocess.CalledProcessError as e:
print('WARNING: {0} time series error executing command:'.format(self._name))
print(' {0}'.format(e.cmd))
print(' rc = {0}'.format(e.returncode))
rc, err_msg = cesmEnvLib.checkFile(outFile, 'read')
if rc:
# get the tail of the .txt file and redirect to a .asc file for the web
ascFile = '{0}.asc'.format(cmd[1])
with open (ascFile, 'w') as results:
try:
# TODO - read the .txt in and write just the lines needed to avoid subprocess call
tailCmd = 'tail -{0} {1}.txt'.format(cmd[2], cmd[1]).split(' ')
subprocess.check_call(tailCmd, stdout=results, env=env)
except subprocess.CalledProcessError as e:
print('WARNING: {0} time series error executing command:'.format(self._name))
print(' {0}'.format(e.cmd))
print(' rc = {0}'.format(e.returncode))
else:
print('model timeseries - Coupler logs do not exist. Disabling MTS_PM_CPLLOG module')
env['MTS_PM_CPLLOG'] = os.environ['PM_CPLLOG'] = 'FALSE'
# check if ocn log files exist
if len(env['OCNLOGFILEPATH']) == 0:
# print a message that the ocn log path isn't defined and turn off POPLOG plot module
print('model timeseries - OCNLOGFILEPATH is undefined. Disabling MTS_PM_YPOPLOG module')
env['MTS_PM_YPOPLOG'] = os.environ['PM_YPOPLOG'] = 'FALSE'
else:
# check that ocn log files exist and gunzip them if necessary
initOcnLogs = ocnLogs = list()
initOcnLogs = glob.glob('{0}/ocn.log.*'.format(env['OCNLOGFILEPATH']))
if len(initOcnLogs) > 0:
for ocnLog in initOcnLogs:
logFileList = ocnLog.split('/')
ocnLogFile = logFileList[-1]
shutil.copy2(ocnLog, '{0}/{1}'.format(env['WORKDIR'],ocnLogFile))
# gunzip the ocnLog in the workdir
if ocnLogFile.lower().find('.gz') != -1:
ocnLog_gunzip = ocnLogFile[:-3]
inFile = gzip.open('{0}/{1}'.format(env['WORKDIR'],ocnLogFile), 'rb')
outFile = open('{0}/{1}'.format(env['WORKDIR'],ocnLog_gunzip), 'wb')
outFile.write( inFile.read() )
inFile.close()
outFile.close()
# append the gunzipped ocn log file to the ocnLogs list
ocnLogs.append('{0}/{1}'.format(env['WORKDIR'],ocnLog_gunzip))
# remove the original .gz file in the workdir
os.remove('{0}/{1}'.format(env['WORKDIR'],ocnLogFile))
else:
# append the original gunzipped ocn log file to the ocnLogs list
ocnLogs.append('{0}/{1}'.format(env['WORKDIR'],ocnLogFile))
# expand the ocn.log* into a list
ocnLogs.sort()
ocnLogsString = ' '.join(ocnLogs)
# define the awk script to parse the ocn log files
globalDiagAwkPath = '{0}/process_pop2_logfiles.globaldiag.awk'.format(env['TOOLPATH'])
globalDiagAwkCmd = '{0} {1}'.format(globalDiagAwkPath, ocnLogsString).split(' ')
print('model_timeseries: globalDiagAwkCmd = {0}'.format(globalDiagAwkCmd))
# run the awk scripts to generate the .txt files from the ocn logs
try:
subprocess.check_call(globalDiagAwkCmd)
except subprocess.CalledProcessError as e:
print('WARNING: {0} time series error executing command:'.format(self._name))
print(' {0}'.format(e.cmd))
print(' rc = {0}'.format(e.returncode))
else:
print('model timeseries - Ocean logs do not exist. Disabling MTS_PM_YPOPLOG and MTS_PM_ENSOWVLT modules')
env['MTS_PM_YPOPLOG'] = os.environ['PM_YPOPLOG'] = 'FALSE'
env['MTS_PM_ENSOWVLT'] = os.environ['PM_ENSOWVLT'] = 'FALSE'
# check if dt files exist
if len(env['DTFILEPATH']) == 0:
# print a message that the dt file path isn't defined and turn off POPLOG plot module
print('model timeseries - DTFILEPATH is undefined. Disabling MTS_PM_YPOPLOG and MTS_PM_ENSOWVLT modules')
env['MTS_PM_YPOPLOG'] = os.environ['PM_YPOPLOG'] = 'FALSE'
env['MTS_PM_ENSOWVLT'] = os.environ['PM_ENSOWVLT'] = 'FALSE'
else:
# check that dt files exist
dtFiles = list()
dtFiles = glob.glob('{0}/{1}.pop.dt.*'.format(env['DTFILEPATH'], env['CASE']))
print('dtFiles = {0}'.format(dtFiles))
if len(dtFiles) > 0:
for dtFile in dtFiles:
logFileList = dtFile.split('/')
dtLogFile = logFileList[-1]
shutil.copy2(dtFile, '{0}/{1}'.format(env['WORKDIR'],dtLogFile))
# expand the *.dt.* into a list
dtFiles.sort()
dtFilesString = ' '.join(dtFiles)
# define the awk script to parse the dt log files
dtFilesAwkPath = '{0}/process_pop2_dtfiles.awk'.format(env['TOOLPATH'])
dtFilesAwkCmd = '{0} {1}'.format(dtFilesAwkPath, dtFilesString).split(' ')
print('model_timeseries: dtFilesAwkCmd = {0}'.format(dtFilesAwkCmd))
# run the awk scripts to generate the .txt files from the dt log files
try:
subprocess.check_call(dtFilesAwkCmd)
except subprocess.CalledProcessError as e:
print('WARNING: {0} time series error executing command:'.format(self._name))
print(' {0}'.format(e.cmd))
print(' rc = {0}'.format(e.returncode))
else:
print('model_timeseries - ocean dt files do not exist. Disabling MTS_PM_YPOPLOG and MTS_PM_ENSOWVLT modules')
env['MTS_PM_YPOPLOG'] = os.environ['PM_YPOPLOG'] = 'FALSE'
env['MTS_PM_ENSOWVLT'] = os.environ['PM_ENSOWVLT'] = 'FALSE'
return env
def check_prerequisites(self, env):
""" check prerequisites
"""
print(' Checking prerequisites for : {0}'.format(
self.__class__.__name__))
self._name = '{0}_{1}'.format(self._name, env['CNTRLCASE'])
super(modelVsControl, self).check_prerequisites(env)
# clean out the old working plot files from the workdir
if env['CLEANUP_FILES'].upper() in ['T', 'TRUE']:
cesmEnvLib.purge(env['WORKDIR'], '.*\.pro')
cesmEnvLib.purge(env['WORKDIR'], '.*\.gif')
cesmEnvLib.purge(env['WORKDIR'], '.*\.dat')
cesmEnvLib.purge(env['WORKDIR'], '.*\.ps')
cesmEnvLib.purge(env['WORKDIR'], '.*\.png')
cesmEnvLib.purge(env['WORKDIR'], '.*\.html')
# create the plot.dat file in the workdir used by all NCL plotting routines
diagUtilsLib.create_plot_dat(env['WORKDIR'], env['XYRANGE'],
env['DEPTHS'])
# setup prerequisites for the model
# setup the gridfile based on the resolution and levels
os.environ['gridfile'] = '{0}/omwg/za_grids/{1}_grid_info.nc'.format(
env['DIAGOBSROOT'], env['RESOLUTION'])
if env['VERTICAL'] == '42':
os.environ[
'gridfile'] = '{0}/omwg/za_grids/{1}_42lev_grid_info.nc'.format(
env['DIAGOBSROOT'], env['RESOLUTION'])
if env['VERTICAL'] == '62':
os.environ[
'gridfile'] = '{0}/omwg/za_grids/{1}_62lev_grid_info.nc'.format(
env['DIAGOBSROOT'], env['RESOLUTION'])
# check if gridfile exists and is readable
rc, err_msg = cesmEnvLib.checkFile(os.environ['gridfile'], 'read')
if not rc:
raise OSError(err_msg)
env['GRIDFILE'] = os.environ['gridfile']
# check the resolution and decide if some plot modules should be turned off
if env['RESOLUTION'] == 'tx0.1v2':
env['MVC_PM_VELISOPZ'] = os.environ['MVC_PM_VELISOPZ'] = 'FALSE'
env['MVC_PM_KAPPAZ'] = os.environ['MVC_PM_KAPPAZ'] = 'FALSE'
# create the global zonal average file used by most of the plotting classes
print(' model vs. control - calling create_za')
diagUtilsLib.create_za(env['WORKDIR'], env['TAVGFILE'],
env['GRIDFILE'], env['TOOLPATH'], env)
# setup prerequisites for the model control
control = True
env['CNTRL_MAVGFILE'], env[
'CNTRL_TAVGFILE'] = diagUtilsLib.createLinks(
env['CNTRLYEAR0'], env['CNTRLYEAR1'], env['CNTRLTAVGDIR'],
env['WORKDIR'], env['CNTRLCASE'], control)
env['CNTRLFILE'] = env['CNTRL_TAVGFILE']
# setup the gridfile based on the resolution and vertical levels
os.environ[
'gridfilecntrl'] = '{0}/omwg/za_grids/{1}_grid_info.nc'.format(
env['DIAGOBSROOT'], env['CNTRLRESOLUTION'])
if env['VERTICAL'] == '42':
os.environ[
'gridfilecntrl'] = '{0}/omwg/za_grids/{1}_42lev_grid_info.nc'.format(
env['DIAGOBSROOT'], env['CNTRLRESOLUTION'])
if env['VERTICAL'] == '62':
os.environ[
'gridfilecntrl'] = '{0}/omwg/za_grids/{1}_62lev_grid_info.nc'.format(
env['DIAGOBSROOT'], env['CNTRLRESOLUTION'])
# check if gridfile exists and is readable
rc, err_msg = cesmEnvLib.checkFile(os.environ['gridfilecntrl'], 'read')
if not rc:
raise OSError(err_msg)
env['GRIDFILECNTRL'] = os.environ['gridfilecntrl']
# check the resolution and decide if some plot modules should be turned off
if env['CNTRLRESOLUTION'] == 'tx0.1v2':
env['MVC_PM_VELISOPZ'] = os.environ['MVC_PM_VELISOPZ'] = 'FALSE'
env['MVC_PM_KAPPAZ'] = os.environ['MVC_PM_KAPPAZ'] = 'FALSE'
# create the control global zonal average file used by most of the plotting classes
print(' model vs. control - calling create_za for control run')
diagUtilsLib.create_za(env['WORKDIR'], env['CNTRL_TAVGFILE'],
env['GRIDFILECNTRL'], env['TOOLPATH'], env)
return env
def check_prerequisites(self, env):
""" check prerequisites
"""
print(" Checking prerequisites for : {0}".format(self.__class__.__name__))
super(modelTimeseries, self).check_prerequisites(env)
# chdir into the working directory
os.chdir(env['WORKDIR'])
# clean out the old working plot files from the workdir
if env['CLEANUP_FILES'].upper() in ['T','TRUE']:
cesmEnvLib.purge(env['WORKDIR'], '.*\.pro')
cesmEnvLib.purge(env['WORKDIR'], '.*\.gif')
cesmEnvLib.purge(env['WORKDIR'], '.*\.dat')
cesmEnvLib.purge(env['WORKDIR'], '.*\.ps')
cesmEnvLib.purge(env['WORKDIR'], '.*\.png')
cesmEnvLib.purge(env['WORKDIR'], '.*\.html')
cesmEnvLib.purge(env['WORKDIR'], '.*\.log\.*')
cesmEnvLib.purge(env['WORKDIR'], '.*\.pop\.d.\.*')
# create the plot.dat file in the workdir used by all NCL plotting routines
diagUtilsLib.create_plot_dat(env['WORKDIR'], env['XYRANGE'], env['DEPTHS'])
# set the OBSROOT
env['OBSROOT'] = env['OBSROOTPATH']
# check the resolution and decide if some plot modules should be turned off
if env['RESOLUTION'] == 'tx0.1v2' :
env['MTS_PM_MOCANN'] = os.environ['PM_MOCANN'] = 'FALSE'
env['MTS_PM_MOCMON'] = os.environ['PM_MOCMON'] = 'FALSE'
# check if cpl log file path is defined
if len(env['CPLLOGFILEPATH']) == 0:
# print a message that the cpl log path isn't defined and turn off CPLLOG plot module
print('model timeseries - CPLLOGFILEPATH is undefined. Disabling MTS_PM_CPLLOG module')
env['MTS_PM_CPLLOG'] = os.environ['PM_CPLLOG'] = 'FALSE'
else:
# check that cpl log files exist and gunzip them if necessary
initcplLogs = cplLogs = list()
initCplLogs = glob.glob('{0}/cpl.log.*'.format(env['CPLLOGFILEPATH']))
if len(initCplLogs) > 0:
for cplLog in initCplLogs:
logFileList = cplLog.split('/')
cplLogFile = logFileList[-1]
shutil.copy2(cplLog, '{0}/{1}'.format(env['WORKDIR'],cplLogFile))
# gunzip the cplLog in the workdir
if cplLogFile.lower().find('.gz') != -1:
cplLog_gunzip = cplLogFile[:-3]
inFile = gzip.open('{0}/{1}'.format(env['WORKDIR'],cplLogFile), 'rb')
outFile = open('{0}/{1}'.format(env['WORKDIR'],cplLog_gunzip), 'wb')
outFile.write( inFile.read() )
inFile.close()
outFile.close()
# append the gunzipped cpl log file to the cplLogs list
cplLogs.append('{0}/{1}'.format(env['WORKDIR'],cplLog_gunzip))
# remove the original .gz file in the workdir
os.remove('{0}/{1}'.format(env['WORKDIR'],cplLogFile))
else:
# append the original gunzipped cpl log file to the cplLogs list
cplLogs.append('{0}/{1}'.format(env['WORKDIR'],cplLogFile))
# parse the cpllog depending on the coupler version - default to 7b
print('model_timeseries: setting up heat and freshwater awk calls with cplLogs = {0}'.format(cplLogs))
heatFile = 'cplheatbudget'
freshWaterFile = 'cplfwbudget'
cplVersion = 'cpl7b'
env['ntailht'] = os.environ['ntailht'] = '22'
env['ntailfw'] = os.environ['ntailfw'] = '16'
if '7' == env['TS_CPL'] or '6' == env['TS_CPL']:
cplVersion = 'cpl{0}'.format(env['TS_CPL'])
env['ntailht'] = os.environ['ntailht'] = '21'
env['ntailfw'] = os.environ['ntailfw'] = '16'
# expand the cpl.log* into a list
cplLogsString = ' '.join(cplLogs)
# define the awk scripts to parse the cpllog file
heatPath = '{0}/process_{1}_logfiles_heat.awk'.format(env['TOOLPATH'], cplVersion)
heatPath = os.path.abspath(heatPath)
fwPath = '{0}/process_{1}_logfiles_fw.awk'.format(env['TOOLPATH'], cplVersion)
fwPath = os.path.abspath(fwPath)
heatCmd = '{0} y0={1} y1={2} {3}'.format(heatPath, env['TSERIES_YEAR0'], env['TSERIES_YEAR1'], cplLogsString).split(' ')
freshWaterCmd = '{0} y0={1} y1={2} {3}'.format(fwPath, env['TSERIES_YEAR0'], env['TSERIES_YEAR1'], cplLogsString).split(' ')
# run the awk scripts to generate the .txt files from the cpllogs
cmdList = [ (heatCmd, heatFile, env['ntailht']), (freshWaterCmd, freshWaterFile, env['ntailfw']) ]
for cmd in cmdList:
outFile = '{0}.txt'.format(cmd[1])
with open (outFile, 'w') as results:
try:
subprocess.check_call(cmd[0], stdout=results, env=env)
except subprocess.CalledProcessError as e:
print('WARNING: {0} time series error executing command:'.format(self._name))
print(' {0}'.format(e.cmd))
print(' rc = {0}'.format(e.returncode))
rc, err_msg = cesmEnvLib.checkFile(outFile, 'read')
if rc:
# get the tail of the .txt file and redirect to a .asc file for the web
ascFile = '{0}.asc'.format(cmd[1])
with open (ascFile, 'w') as results:
try:
# TODO - read the .txt in and write just the lines needed to avoid subprocess call
tailCmd = 'tail -{0} {1}.txt'.format(cmd[2], cmd[1]).split(' ')
subprocess.check_call(tailCmd, stdout=results, env=env)
except subprocess.CalledProcessError as e:
print('WARNING: {0} time series error executing command:'.format(self._name))
print(' {0}'.format(e.cmd))
print(' rc = {0}'.format(e.returncode))
else:
print('model timeseries - Coupler logs do not exist. Disabling MTS_PM_CPLLOG module')
env['MTS_PM_CPLLOG'] = os.environ['PM_CPLLOG'] = 'FALSE'
# check if ocn log files exist
if len(env['OCNLOGFILEPATH']) == 0:
# print a message that the ocn log path isn't defined and turn off POPLOG plot module
print('model timeseries - OCNLOGFILEPATH is undefined. Disabling MTS_PM_YPOPLOG module')
env['MTS_PM_YPOPLOG'] = os.environ['PM_YPOPLOG'] = 'FALSE'
else:
# check that ocn log files exist and gunzip them if necessary
initOcnLogs = ocnLogs = list()
initOcnLogs = glob.glob('{0}/ocn.log.*'.format(env['OCNLOGFILEPATH']))
if len(initOcnLogs) > 0:
for ocnLog in initOcnLogs:
logFileList = ocnLog.split('/')
ocnLogFile = logFileList[-1]
shutil.copy2(ocnLog, '{0}/{1}'.format(env['WORKDIR'],ocnLogFile))
# gunzip the ocnLog in the workdir
if ocnLogFile.lower().find('.gz') != -1:
ocnLog_gunzip = ocnLogFile[:-3]
inFile = gzip.open('{0}/{1}'.format(env['WORKDIR'],ocnLogFile), 'rb')
outFile = open('{0}/{1}'.format(env['WORKDIR'],ocnLog_gunzip), 'wb')
outFile.write( inFile.read() )
inFile.close()
outFile.close()
# append the gunzipped ocn log file to the ocnLogs list
ocnLogs.append('{0}/{1}'.format(env['WORKDIR'],ocnLog_gunzip))
# remove the original .gz file in the workdir
os.remove('{0}/{1}'.format(env['WORKDIR'],ocnLogFile))
else:
# append the original gunzipped ocn log file to the ocnLogs list
ocnLogs.append('{0}/{1}'.format(env['WORKDIR'],ocnLogFile))
# expand the ocn.log* into a list
ocnLogs.sort()
ocnLogsString = ' '.join(ocnLogs)
# define the awk script to parse the ocn log files
globalDiagAwkPath = '{0}/process_pop2_logfiles.globaldiag.awk'.format(env['TOOLPATH'])
globalDiagAwkCmd = '{0} {1}'.format(globalDiagAwkPath, ocnLogsString).split(' ')
print('model_timeseries: globalDiagAwkCmd = {0}'.format(globalDiagAwkCmd))
# run the awk scripts to generate the .txt files from the ocn logs
try:
subprocess.check_call(globalDiagAwkCmd)
except subprocess.CalledProcessError as e:
print('WARNING: {0} time series error executing command:'.format(self._name))
print(' {0}'.format(e.cmd))
print(' rc = {0}'.format(e.returncode))
else:
print('model timeseries - Ocean logs do not exist. Disabling MTS_PM_YPOPLOG and MTS_PM_ENSOWVLTmodules')
env['MTS_PM_YPOPLOG'] = os.environ['PM_YPOPLOG'] = 'FALSE'
env['MTS_PM_ENSOWVLT'] = os.environ['PM_ENSOWVLT'] = 'FALSE'
# check if dt files exist
if len(env['DTFILEPATH']) == 0:
# print a message that the dt file path isn't defined and turn off POPLOG plot module
print('model timeseries - DTFILEPATH is undefined. Disabling MTS_PM_YPOPLOG and MTS_PM_ENSOWVLTmodules')
env['MTS_PM_YPOPLOG'] = os.environ['PM_YPOPLOG'] = 'FALSE'
env['MTS_PM_ENSOWVLT'] = os.environ['PM_ENSOWVLT'] = 'FALSE'
else:
# check that dt files exist
dtFiles = list()
dtFiles = glob.glob('{0}/{1}.pop.dt.*'.format(env['DTFILEPATH'], env['CASE']))
print('dtFiles = {0}'.format(dtFiles))
if len(dtFiles) > 0:
for dtFile in dtFiles:
logFileList = dtFile.split('/')
dtLogFile = logFileList[-1]
shutil.copy2(dtFile, '{0}/{1}'.format(env['WORKDIR'],dtLogFile))
# expand the *.dt.* into a list
dtFiles.sort()
dtFilesString = ' '.join(dtFiles)
# define the awk script to parse the dt log files
dtFilesAwkPath = '{0}/process_pop2_dtfiles.awk'.format(env['TOOLPATH'])
dtFilesAwkCmd = '{0} {1}'.format(dtFilesAwkPath, dtFilesString).split(' ')
print('model_timeseries: dtFilesAwkCmd = {0}'.format(dtFilesAwkCmd))
# run the awk scripts to generate the .txt files from the dt log files
try:
subprocess.check_call(dtFilesAwkCmd)
except subprocess.CalledProcessError as e:
print('WARNING: {0} time series error executing command:'.format(self._name))
print(' {0}'.format(e.cmd))
print(' rc = {0}'.format(e.returncode))
else:
print('model_timeseries - ocean dt files do not exist. Disabling MTS_PM_YPOPLOG and MTS_PM_ENSOWVLT modules')
env['MTS_PM_YPOPLOG'] = os.environ['PM_YPOPLOG'] = 'FALSE'
env['MTS_PM_ENSOWVLT'] = os.environ['PM_ENSOWVLT'] = 'FALSE'
return env
def check_prerequisites(self, env):
""" check prerequisites
"""
print(' Checking prerequisites for : {0}'.format(self.__class__.__name__))
self._name = '{0}_{1}'.format(self._name, env['CNTRLCASE'])
super(modelVsControl, self).check_prerequisites(env)
# clean out the old working plot files from the workdir
if env['CLEANUP_FILES'].upper() in ['T','TRUE']:
cesmEnvLib.purge(env['WORKDIR'], '.*\.pro')
cesmEnvLib.purge(env['WORKDIR'], '.*\.gif')
cesmEnvLib.purge(env['WORKDIR'], '.*\.dat')
cesmEnvLib.purge(env['WORKDIR'], '.*\.ps')
cesmEnvLib.purge(env['WORKDIR'], '.*\.png')
cesmEnvLib.purge(env['WORKDIR'], '.*\.html')
# create the plot.dat file in the workdir used by all NCL plotting routines
diagUtilsLib.create_plot_dat(env['WORKDIR'], env['XYRANGE'], env['DEPTHS'])
# setup prerequisites for the model
# setup the gridfile based on the resolution and levels
os.environ['gridfile'] = '{0}/omwg/za_grids/{1}_grid_info.nc'.format(env['DIAGOBSROOT'],env['RESOLUTION'])
if env['VERTICAL'] == '42':
os.environ['gridfile'] = '{0}/omwg/za_grids/{1}_42lev_grid_info.nc'.format(env['DIAGOBSROOT'],env['RESOLUTION'])
if env['VERTICAL'] == '62':
os.environ['gridfile'] = '{0}/omwg/za_grids/{1}_62lev_grid_info.nc'.format(env['DIAGOBSROOT'],env['RESOLUTION'])
# check if gridfile exists and is readable
rc, err_msg = cesmEnvLib.checkFile(os.environ['gridfile'], 'read')
if not rc:
raise OSError(err_msg)
env['GRIDFILE'] = os.environ['gridfile']
# check the resolution and decide if some plot modules should be turned off
if (env['RESOLUTION'] == 'tx0.1v2' or env['RESOLUTION'] == 'tx0.1v3') :
env['MVC_PM_VELISOPZ'] = os.environ['MVC_PM_VELISOPZ'] = 'FALSE'
env['MVC_PM_KAPPAZ'] = os.environ['MVC_PM_KAPPAZ'] = 'FALSE'
# create the global zonal average file used by most of the plotting classes
print(' model vs. control - calling create_za')
diagUtilsLib.create_za( env['WORKDIR'], env['TAVGFILE'], env['GRIDFILE'], env['TOOLPATH'], env)
# setup prerequisites for the model control
control = True
env['CNTRL_MAVGFILE'], env['CNTRL_TAVGFILE'] = diagUtilsLib.createLinks(env['CNTRLYEAR0'], env['CNTRLYEAR1'], env['CNTRLTAVGDIR'], env['WORKDIR'], env['CNTRLCASE'], control)
env['CNTRLFILE'] = env['CNTRL_TAVGFILE']
# setup the gridfile based on the resolution and vertical levels
os.environ['gridfilecntrl'] = '{0}/omwg/za_grids/{1}_grid_info.nc'.format(env['DIAGOBSROOT'],env['CNTRLRESOLUTION'])
if env['VERTICAL'] == '42':
os.environ['gridfilecntrl'] = '{0}/omwg/za_grids/{1}_42lev_grid_info.nc'.format(env['DIAGOBSROOT'],env['CNTRLRESOLUTION'])
if env['VERTICAL'] == '62':
os.environ['gridfilecntrl'] = '{0}/omwg/za_grids/{1}_62lev_grid_info.nc'.format(env['DIAGOBSROOT'],env['CNTRLRESOLUTION'])
# check if gridfile exists and is readable
rc, err_msg = cesmEnvLib.checkFile(os.environ['gridfilecntrl'], 'read')
if not rc:
raise OSError(err_msg)
env['GRIDFILECNTRL'] = os.environ['gridfilecntrl']
# check the resolution and decide if some plot modules should be turned off
if (env['CNTRLRESOLUTION'] == 'tx0.1v2' or env['CNTRLRESOLUTION'] == 'tx0.1v3') :
env['MVC_PM_VELISOPZ'] = os.environ['MVC_PM_VELISOPZ'] = 'FALSE'
env['MVC_PM_KAPPAZ'] = os.environ['MVC_PM_KAPPAZ'] = 'FALSE'
# create the control global zonal average file used by most of the plotting classes
print(' model vs. control - calling create_za for control run')
diagUtilsLib.create_za( env['WORKDIR'], env['CNTRL_TAVGFILE'], env['GRIDFILECNTRL'], env['TOOLPATH'], env)
return env